Pandas - 《深度学习》

创建
选择
设置值
处理丢失数据
导入导出
- 读取
- 存储
合并文件
- concatenating
  - join [‘inner’,’outer]
  - append
- merge
绘图
处理成离散数据

创建

pd.Series([1,3,np.nan,44,1])创建数列
pd.date_range('20160101',periods=6)创建时间数列
pd.DataFrame(np.random.randn(6,4),index=dates,columns=['a','b','c','d']) # 其中也可以输入字典
df.dtypes df.index df.columns df.values df.describe() df.T
df.sort_index(axis=1,ascending=False) # 排序
df.sort_values(by=1) # 排序

选择

                   a         b         c         d
2016-01-01 -1.463828 -0.888810  0.628694 -0.931318
2016-01-02 -0.132622 -0.146939  1.156180  0.503257
2016-01-03  0.958399  0.036183 -1.231498  0.340976
2016-01-04  0.397120 -1.811356  1.217574  0.132399
2016-01-05  0.360953  1.589439  0.184544  1.631463
2016-01-06  0.207713  0.333487 -0.516895  1.131951

df['A'] df.A # 列的选择
df[0:3] df['20160101':'20160103] # 行的选择

select by label: loc

df.loc['20160102'] df.loc[:,['A','B]]

select by position: iloc

df.iloc[3] # 第三行
df.iloc[3,1] # 第三行第一列
df.iloc[[1,3,5],1:3]

Boolean indexing

df[df.a>8]

设置值

df.iloc[2,2]=111
df[df.A>4]=0
df.A[df.A>4]=0
df['F']=np.nan # F是不存在的列标签
df['E']=pd.Series([1,2,3,4,5,6],index=pd.date_range('20130101',periods=6)) # 设置新列的值时，需要把index对齐

处理丢失数据

df.dropna(axis=0,how='any') how={‘any’,’all’} 有一个丢一行，全部为NaN丢一行
df.fillna(value=0) 填上数据
df.isnull()返回是否缺失数据
df.any(df.isnull())==True 如果有丢失就返回True

导入导出

读取

read_csv
read_excel
read_hdf
read_sql
read_json
read_msgpack
read_html
read_gbq
read_stata
read_sas
read_clipboard
read_pickle

存储

to_csv
to_excel
to_hdf
to_sql
to_json
to_msgpack
to_html
to_gbq
to_stata
to_sas
to_clipboard
to_pickle

合并文件

concatenating

df1 = pd.DataFrame(np.ones((3,4))*0,columns=['a','b','c','d'])
df2 = pd.DataFrame(np.ones((3,4))*1,columns=['a','b','c','d'])
df3 = pd.DataFrame(np.ones((3,4))*2,columns=['a','b','c','d'])
res = pd.concat([df1,df2,df3],axis=0,ignore_index=True)

join [‘inner’,’outer]

df1 = pd.DataFrame(np.ones((3,4))*0,columns=['a','b','c','d'],index=[1,2,3])
df2 = pd.DataFrame(np.ones((3,4))*1,columns=['b','c','d','e'],index=[2,3,4])
res = pd.concat([df1,df2],join='outer') # 默认outer
Out[8]: 
     a    b    c    d    e
1  0.0  0.0  0.0  0.0  NaN
2  0.0  0.0  0.0  0.0  NaN
3  0.0  0.0  0.0  0.0  NaN
2  NaN  1.0  1.0  1.0  1.0
3  NaN  1.0  1.0  1.0  1.0
4  NaN  1.0  1.0  1.0  1.0
res = pd.concat([df1,df2],join='inner')
Out[10]: 
     b    c    d
1  0.0  0.0  0.0
2  0.0  0.0  0.0
3  0.0  0.0  0.0
2  1.0  1.0  1.0
3  1.0  1.0  1.0
4  1.0  1.0  1.0

append

df1 = pd.DataFrame(np.ones((3,4))*0,columns=['a','b','c','d'])
df2 = pd.DataFrame(np.ones((3,4))*1,columns=['a','b','c','d'])
df3 = pd.DataFrame(np.ones((3,4))*2,columns=['a','b','c','d'])
s1=pd.Series([1,2,3,4],index=['a','b','c','d'])
res = df1.append(s1,ignore_index=True)
res = df1.append([df2,df3],ignore_index=True)

merge

依据一组key合并

left = pd.DataFrame({'key': ['K0', 'K1', 'K2', 'K3'],
                             'A': ['A0', 'A1', 'A2', 'A3'],
                             'B': ['B0', 'B1', 'B2', 'B3']})
right = pd.DataFrame({'key': ['K0', 'K1', 'K2', 'K3'],
                              'C': ['C0', 'C1', 'C2', 'C3'],
                              'D': ['D0', 'D1', 'D2', 'D3']})
res = pd.merge(left, right, on='key')
print(res)
     A   B key   C   D
# 0  A0  B0  K0  C0  D0
# 1  A1  B1  K1  C1  D1
# 2  A2  B2  K2  C2  D2
# 3  A3  B3  K3  C3  D3

依据两组key合并

合并时有4种方法how = ['left', 'right', 'outer', 'inner']，预设值how='inner'。

#定义资料集并打印出
left = pd.DataFrame({'key1': ['K0', 'K0', 'K1', 'K2'],
                      'key2': ['K0', 'K1', 'K0', 'K1'],
                      'A': ['A0', 'A1', 'A2', 'A3'],
                      'B': ['B0', 'B1', 'B2', 'B3']})
right = pd.DataFrame({'key1': ['K0', 'K1', 'K1', 'K2'],
                       'key2': ['K0', 'K0', 'K0', 'K0'],
                       'C': ['C0', 'C1', 'C2', 'C3'],
                       'D': ['D0', 'D1', 'D2', 'D3']})
#依据key1与key2 columns进行合并，并打印出四种结果['left', 'right', 'outer', 'inner']
res = pd.merge(left, right, on=['key1', 'key2'], how='inner')
print(res)
#    A   B key1 key2   C   D
# 0  A0  B0   K0   K0  C0  D0
# 1  A2  B2   K1   K0  C1  D1
# 2  A2  B2   K1   K0  C2  D2
res = pd.merge(left, right, on=['key1', 'key2'], how='outer')
print(res)
#     A    B key1 key2    C    D
# 0   A0   B0   K0   K0   C0   D0
# 1   A1   B1   K0   K1  NaN  NaN
# 2   A2   B2   K1   K0   C1   D1
# 3   A2   B2   K1   K0   C2   D2
# 4   A3   B3   K2   K1  NaN  NaN
# 5  NaN  NaN   K2   K0   C3   D3
res = pd.merge(left, right, on=['key1', 'key2'], how='left')
print(res)
#    A   B key1 key2    C    D
# 0  A0  B0   K0   K0   C0   D0
# 1  A1  B1   K0   K1  NaN  NaN
# 2  A2  B2   K1   K0   C1   D1
# 3  A2  B2   K1   K0   C2   D2
# 4  A3  B3   K2   K1  NaN  NaN
res = pd.merge(left, right, on=['key1', 'key2'], how='right')
print(res)
#     A    B key1 key2   C   D
# 0   A0   B0   K0   K0  C0  D0
# 1   A2   B2   K1   K0  C1  D1
# 2   A2   B2   K1   K0  C2  D2
# 3  NaN  NaN   K2   K0  C3  D3

Indicator

indicator=True会将合并的记录放在新的一列。

res = pd.merge(df1, df2, on='col1', how='outer', indicator='indicator_column')

依据index合并

res = pd.merge(left, right, left_index=True, right_index=True, how='inner')

解决overlapping的问题

#定义资料集
boys = pd.DataFrame({'k': ['K0', 'K1', 'K2'], 'age': [1, 2, 3]})
girls = pd.DataFrame({'k': ['K0', 'K0', 'K3'], 'age': [4, 5, 6]})

#使用suffixes解决overlapping的问题
res = pd.merge(boys, girls, on='k', suffixes=['_boy', '_girl'], how='inner')
print(res)
#    age_boy   k  age_girl
# 0        1  K0         4
# 1        1  K0         5

绘图

data.plot() # pandas的data
plt.show()

plot 可以指定很多参数，具体的用法大家可以自己查一下这里

除了plot，我经常会用到还有scatter，这个会显示散点图，首先给大家说一下在 pandas 中有多少种方法

bar
hist
box
kde
area
scatter
hexbin

处理成离散数据

pd.get_dummies() 将object转化为one-hot形式