3pandas Ipynb
3pandas Ipynb
3pandas Ipynb
import pandas as pd
from pandas import Series, DataFrame
import numpy as np
Series
obj = pd.Series([4, 7, -5, 3])
obj
0 4
1 7
2 -5
3 3
dtype: int64
print(obj.values)
obj.index # like range(4)
[ 4 7 -5 3]
obj2 = pd.Series([4, 7, -5, 3], index=['d', 'b', 'a', 'c'])
print(obj2)
obj2.index
d 4
b 7
a -5
c 3
dtype: int64
obj2['a']
obj2['d'] = 6
obj2[['c', 'a', 'd']]
c 3
a -5
d 6
dtype: int64
obj2[obj2 > 0]
obj2 * 2
np.exp(obj2)
d 403.428793
b 1096.633158
a 0.006738
c 20.085537
dtype: float64
print('b' in obj2)
print('e' in obj2)
True
False
sdata = {'Ohio': 35000, 'Texas': 71000, 'Oregon': 16000, 'Utah': 5000}
obj3 = pd.Series(sdata)
obj3
Ohio 35000
Texas 71000
Oregon 16000
Utah 5000
dtype: int64
states = ['California', 'Ohio', 'Oregon', 'Texas']
obj4 = pd.Series(sdata, index=states)
obj4
California NaN
Ohio 35000.0
Oregon 16000.0
Texas 71000.0
dtype: float64
pd.isnull(obj4)
pd.notnull(obj4)
California False
Ohio True
Oregon True
Texas True
dtype: bool
obj4.isnull()
California True
Ohio False
Oregon False
Texas False
dtype: bool
print(obj3)
print(obj4)
print(obj3 + obj4)
Ohio 35000
Texas 71000
Oregon 16000
Utah 5000
dtype: int64
California NaN
Ohio 35000.0
Oregon 16000.0
Texas 71000.0
dtype: float64
California NaN
Ohio 70000.0
Oregon 32000.0
Texas 142000.0
Utah NaN
dtype: float64
obj4.name = 'population'
obj4.index.name = 'state'
obj4
state
California NaN
Ohio 35000.0
Oregon 16000.0
Texas 71000.0
print(obj)
obj.index = ['Bob', 'Steve', 'Jeff', 'Ryan']
print(obj)
0 4
1 7
2 -5
3 3
dtype: int64
Bob 4
Steve 7
Jeff -5
Ryan 3
dtype: int64
DataFrame
data = {'state': ['Ohio', 'Ohio', 'Ohio', 'Nevada', 'Nevada', 'Nevada'],
'year': [2000, 2001, 2002, 2001, 2002, 2003],
'pop': [1.5, 1.7, 3.6, 2.4, 2.9, 3.2]}
frame = pd.DataFrame(data)
frame
frame.head()
pd.DataFrame(data, columns=['year', 'state', 'pop'])
frame2 = pd.DataFrame(data, columns=['year', 'state', 'pop', 'debt'],
index=['one', 'two', 'three', 'four',
'five', 'six'])
print(frame2)
print(frame2.columns)
frame2['state']
frame2.year
one 2000
two 2001
three 2002
four 2001
five 2002
six 2003
frame2.loc['three']
year 2002
state Ohio
pop 3.6
debt NaN
frame2['debt'] = 16.5
print(frame2)
frame2['debt'] = np.arange(6.)
print(frame2)
val = pd.Series([-1.2, -1.5, -1.7], index=['two', 'four', 'five'])
frame2['debt'] = val
frame2
frame2['eastern'] = frame2.state == 'Ohio'
six year
2003 state
Nevada pop
3.2 debt
NaN eastern
del frame2['eastern']
frame2.columns
pop = {'Nevada': {2001: 2.4, 2002: 2.9},
'Ohio': {2000: 1.5, 2001: 1.7, 2002: 3.6}}
frame3 = pd.DataFrame(pop)
frame3
Nevada Ohio
frame3.T
pd.DataFrame(pop, index=[2001, 2002, 2003])
Nevada Ohio
pdata = {'Ohio': frame3['Ohio'][:-1],
'Nevada': frame3['Nevada'][:2]}
pd.DataFrame(pdata)
Ohio Nevada
frame3.index.name = 'year'; frame3.columns.name = 'state'
frame3
year
frame3.values
array([[2.4, 1.7],
[2.9, 3.6],
[nan, 1.5]])
frame2.values
Index Objects
obj = pd.Series(range(3), index=['a', 'b', 'c'])
index = obj.index
print(index)
print(index[1:])
labels = pd.Index(np.arange(3))
print(labels)
obj2 = pd.Series([1.5, -2.5, 0], index=labels)
print(obj2)
obj2.index is labels
0 1.5
1 -2.5
2 0.0
dtype: float64
True
frame3
frame3.columns
'Ohio' in frame3.columns
2003 in frame3.index
False
dup_labels = pd.Index(['foo', 'foo', 'bar', 'bar'])
dup_labels
Essential Functionality
Reindexing
obj = pd.Series([4.5, 7.2, -5.3, 3.6], index=['d', 'b', 'a', 'c'])
obj
d 4.5
b 7.2
a -5.3
c 3.6
dtype: float64
obj2 = obj.reindex(['a', 'b', 'c', 'd', 'e'])
obj2
a -5.3
b 7.2
c 3.6
d 4.5
e NaN
dtype: float64
obj3 = pd.Series(['blue', 'purple', 'yellow'], index=[0, 2, 4])
print(obj3)
obj3.reindex(range(6), method='ffill')
0 blue
2 purple
4 yellow
dtype: object
0 blue
1 blue
2 purple
3 purple
4 yellow
5 yellow
dtype: object
frame = pd.DataFrame(np.arange(9).reshape((3, 3)),
index=['a', 'c', 'd'],
columns=['Ohio', 'Texas', 'California'])
print(frame)
frame2 = frame.reindex(['a', 'b', 'c', 'd'])
print(frame2)
a 0 1 2
c 3 4 5
d 6 7 8
states = ['Texas', 'Utah', 'California']
frame.reindex(columns=states)
a 1 NaN 2
c 4 NaN 5
d 7 NaN 8
print(obj)
new_obj = obj.drop('c')
print(new_obj)
print(obj.drop(['d', 'c']))
a 0.0
b 1.0
c 2.0
d 3.0
e 4.0
dtype: float64
a 0.0
b 1.0
d 3.0
e 4.0
dtype: float64
a 0.0
b 1.0
e 4.0
dtype: float64
data = pd.DataFrame(np.arange(16).reshape((4, 4)),
index=['Ohio', 'Colorado', 'Utah', 'New York'],
columns=['one', 'two', 'three', 'four'])
data
Ohio 0 1 2 3
Colorado 4 5 6 7
Utah 8 9 10 11
New York 12 13 14 15
data.drop(['Colorado', 'Ohio'])
Utah 8 9 10 11
New York 12 13 14 15
print(data.drop('two', axis=1))
data.drop(['two', 'four'], axis='columns')
Ohio 0 2 3
Colorado 4 6 7
Utah 8 10 11
New York 12 14 15
one three
Ohio 0 2
Colorado 4 6
obj.drop('c', inplace=True)
obj
Utah 8 10
aNew 0.0
York 12 14
b 1.0
d 3.0
e 4.0
dtype: float64
obj = pd.Series(np.arange(4.), index=['a', 'b', 'c', 'd'])
print(obj)
print(obj['b'])
print(obj[1])
print(obj[2:4])
print(obj[['b', 'a', 'd']])
print(obj[[1, 3]])
print(obj[obj < 2])
a 0.0
b 1.0
c 2.0
d 3.0
dtype: float64
1.0
1.0
c 2.0
d 3.0
dtype: float64
b 1.0
a 0.0
d 3.0
dtype: float64
b 1.0
d 3.0
dtype: float64
a 0.0
b 1.0
dtype: float64
print(obj['b':'c'])
b 1.0
c 2.0
dtype: float64
obj['b':'c'] = 5
obj
a 0.0
b 5.0
c 5.0
d 3.0
dtype: float64
data = pd.DataFrame(np.arange(16).reshape((4, 4)),
index=['Ohio', 'Colorado', 'Utah', 'New York'],
columns=['one', 'two', 'three', 'four'])
print(data)
print(data['two'])
print(data[['three', 'one']])
Ohio 0 1 2 3
Colorado 4 5 6 7
Utah 8 9 10 11
New York 12 13 14 15
Ohio 1
Colorado 5
Utah 9
New York 13
three one
Ohio 2 0
Colorado 6 4
Utah 10 8
New York 14 12
data[:2]
data[data['three'] > 5]
Colorado 4 5 6 7
Utah 8 9 10 11
New York 12 13 14 15
data < 5
data[data < 5] = 0
data
one two three four
Ohio 0 0 0 0
Utah 8 9 10 11
data.loc['Colorado', ['two', 'three']]
New York 12 13 14 15
two 5
three 6
print(data.iloc[2, [3, 0, 1]])
print(data.iloc[2])
print(data.iloc[[1, 2], [3, 0, 1]])
four 11
one 8
two 9
one 8
two 9
three 10
four 11
Colorado 7 0 5
Utah 11 8 9
print(data.loc[:'Utah', 'two'])
print(data.iloc[:, :3][data.three > 5])
Ohio 0
Colorado 5
Utah 9
Colorado 0 5 6
Utah 8 9 10
New York 12 13 14
Integer Indexes
ser = pd.Series(np.arange(3.))
ser
ser[-1]
ser = pd.Series(np.arange(3.))
ser
0 0.0
1 1.0
2 2.0
dtype: float64
ser2 = pd.Series(np.arange(3.), index=['a', 'b', 'c'])
ser2[-1]
2.0
print(ser[:1])
print(ser.loc[:1])
print(ser.iloc[:1])
0 0.0
dtype: float64
0 0.0
1 1.0
dtype: float64
0 0.0
dtype: float64
s1 = pd.Series([7.3, -2.5, 3.4, 1.5], index=['a', 'c', 'd', 'e'])
s2 = pd.Series([-2.1, 3.6, -1.5, 4, 3.1],
index=['a', 'c', 'e', 'f', 'g'])
print(s1)
print(s2)
a 7.3
c -2.5
d 3.4
e 1.5
dtype: float64
a -2.1
c 3.6
e -1.5
f 4.0
g 3.1
dtype: float64
s1 + s2
a 5.2
c 1.1
d NaN
e 0.0
f NaN
g NaN
dtype: float64
df1 = pd.DataFrame(np.arange(9.).reshape((3, 3)), columns=list('bcd'),
index=['Ohio', 'Texas', 'Colorado'])
df2 = pd.DataFrame(np.arange(12.).reshape((4, 3)), columns=list('bde'),
index=['Utah', 'Ohio', 'Texas', 'Oregon'])
print(df1)
print(df2)
b c d
b d e
df1 + df2
b c d e
df1 = pd.DataFrame({'A': [1, 2]})
df2 = pd.DataFrame({'B': [3, 4]})
print(df1)
print(df2)
print(df1 - df2)
0 1
1 2
0 3
1 4
A B
0 NaN NaN
1 NaN NaN
df1 = pd.DataFrame(np.arange(12.).reshape((3, 4)),
columns=list('abcd'))
df2 = pd.DataFrame(np.arange(20.).reshape((4, 5)),
columns=list('abcde'))
df2.loc[1, 'b'] = np.nan
print(df1)
print(df2)
a b c d
a b c d e
df1 + df2
a b c d e
df1.add(df2, fill_value=0)
a b c d e
print(1 / df1)
print(df1.rdiv(1))
a b c d
a b c d
df1.reindex(columns=df2.columns, fill_value=0)
a b c d e
Operations
0 0.0 between
1.0 2.0 DataFrame
3.0 0 and Series
1 4.0 5.0 6.0 7.0 0
arr = np.arange(12.).reshape((3, 4))
2 8.0 9.0 10.0 11.0 0
print(arr)
print(arr[0])
print(arr - arr[0])
[[ 0. 1. 2. 3.]
[ 4. 5. 6. 7.]
[ 8. 9. 10. 11.]]
[0. 1. 2. 3.]
[[0. 0. 0. 0.]
[4. 4. 4. 4.]
[8. 8. 8. 8.]]
frame = pd.DataFrame(np.arange(12.).reshape((4, 3)),
columns=list('bde'),
index=['Utah', 'Ohio', 'Texas', 'Oregon'])
series = frame.iloc[0]
print(frame)
print(series)
b d e
b 0.0
d 1.0
e 2.0
frame - series
b d e
series2 = pd.Series(range(3), index=['b', 'e', 'f'])
frame + series2
b d e f
b d e
Utah 1.0
Ohio 4.0
Texas 7.0
Oregon 10.0
b d e
frame = pd.DataFrame(np.random.randn(4, 3), columns=list('bde'),
index=['Utah', 'Ohio', 'Texas', 'Oregon'])
frame
np.abs(frame)
b d e
f = lambda x: x.max() - x.min()
frame.apply(f)
b 1.911192
d 0.861766
e 6.320462
dtype: float64
frame.apply(f, axis='columns')
Utah 1.328548
Ohio 1.100655
Texas 2.413484
Oregon 3.930118
dtype: float64
def f(x):
return pd.Series([x.min(), x.max()], index=['min', 'max'])
frame.apply(f)
b d e
format = lambda x: '%.2f' % x
frame.applymap(format)
b d e
frame['e'].map(format)
Utah 0.61
Ohio -0.26
Texas 3.78
Oregon -2.54
obj = pd.Series(range(4), index=['d', 'a', 'b', 'c'])
obj.sort_index()
a 1
b 2
c 3
d 0
dtype: int64
frame = pd.DataFrame(np.arange(8).reshape((2, 4)),
index=['three', 'one'],
columns=['d', 'a', 'b', 'c'])
print(frame.sort_index())
print(frame.sort_index(axis=1))
d a b c
one 4 5 6 7
three 0 1 2 3
a b c d
three 1 2 3 0
one 5 6 7 4
frame.sort_index(axis=1, ascending=False)
d c b a
three 0 3 2 1
one 4 7 6 5
obj = pd.Series([4, 7, -3, 2])
obj.sort_values()
2 -3
3 2
0 4
1 7
dtype: int64
obj = pd.Series([4, np.nan, 7, np.nan, -3, 2])
obj.sort_values()
4 -3.0
5 2.0
0 4.0
2 7.0
1 NaN
3 NaN
dtype: float64
frame = pd.DataFrame({'b': [4, 7, -3, 2], 'a': [0, 1, 0, 1]})
print(frame)
print(frame.sort_values(by='b'))
b a
0 4 0
1 7 1
2 -3 0
3 2 1
b a
2 -3 0
3 2 1
0 4 0
1 7 1
frame.sort_values(by=['a', 'b'])
b a
2 -3 0
0 4 0
3 2 1
1 7 1
obj = pd.Series([7, -5, 7, 4, 2, 0, 4])
obj.rank()
0 6.5
1 1.0
2 6.5
3 4.5
4 3.0
5 2.0
6 4.5
dtype: float64
obj.rank(method='first')
0 6.0
1 1.0
2 7.0
3 4.0
4 3.0
5 2.0
6 5.0
dtype: float64
# Assign tie values the maximum rank in the group
obj.rank(ascending=False, method='max')
0 2.0
1 7.0
2 2.0
3 4.0
4 5.0
5 6.0
6 4.0
dtype: float64
frame = pd.DataFrame({'b': [4.3, 7, -3, 2], 'a': [0, 1, 0, 1],
'c': [-2, 5, 8, -2.5]})
print(frame)
print(frame.rank(axis='columns'))
b a c
0 4.3 0 -2.0
1 7.0 1 5.0
2 -3.0 0 8.0
3 2.0 1 -2.5
b a c
obj = pd.Series(range(5), index=['a', 'a', 'b', 'b', 'c'])
obj
a 0
a 1
b 2
b 3
c 4
dtype: int64
obj.index.is_unique
False
print(obj['a'])
print(obj['c'])
a 0
a 1
dtype: int64
df = pd.DataFrame(np.random.randn(4, 3), index=['a', 'a', 'b', 'b'])
print(df)
print(df.loc['b'])
0 1 2
0 1 2
df = pd.DataFrame([[1.4, np.nan], [7.1, -4.5],
[np.nan, np.nan], [0.75, -1.3]],
index=['a', 'b', 'c', 'd'],
columns=['one', 'two'])
df
one two
a 1.40 NaN
b 7.10 -4.5
c NaN NaN
d 0.75 -1.3
df.sum()
one 9.25
two -5.80
dtype: float64
df.sum(axis='columns')
a 1.40
b 2.60
c 0.00
d -0.55
dtype: float64
df.mean(axis='columns', skipna=False)
a NaN
b 1.300
c NaN
d -0.275
dtype: float64
df.idxmax()
one b
two d
dtype: object
df.cumsum()
one two
a 1.40 NaN
b 8.50 -4.5
c NaN NaN
d 9.25 -5.8
df.describe()
one two
count 16
unique 3
top a
freq 8
dtype: object
obj = pd.Series(['c', 'a', 'd', 'a', 'a', 'b', 'b', 'c', 'c'])
uniques = obj.unique()
uniques
obj.value_counts()
c 3
a 3
b 2
d 1
dtype: int64
pd.value_counts(obj.values, sort=False)
c 3
a 3
d 1
b 2
dtype: int64
print(obj)
mask = obj.isin(['b', 'c'])
print(mask)
print(obj[mask])
0 c
1 a
2 d
3 a
4 a
5 b
6 b
7 c
8 c
dtype: object
0 True
1 False
2 False
3 False
4 False
5 True
6 True
7 True
8 True
dtype: bool
0 c
5 b
6 b
7 c
8 c
dtype: object
to_match = pd.Series(['c', 'a', 'b', 'b', 'c', 'a'])
unique_vals = pd.Series(['c', 'b', 'a'])
pd.Index(unique_vals).get_indexer(to_match)
array([0, 2, 1, 1, 0, 2])
data = pd.DataFrame({'Qu1': [1, 3, 4, 3, 4],
'Qu2': [2, 3, 1, 2, 3],
'Qu3': [1, 5, 2, 4, 4]})
data
0 1 2 1
1 3 3 5
2 4 1 2
3 3 2 4
4 4 3 4
result = data.apply(pd.value_counts).fillna(0)
result