1 读取文件

read_csv

  1. import pandas as pd
  3. # 读取普通tab分隔的文本
  4. df = pd.read_csv('test.tsv', sep='\t')
  6. # 跳过注释行，指定列名
  7. df = pd.read_csv('test.tsv', sep='\t', comment='#', names=['a', 'b', 'c'])
  9. # 读取指定列(按列的下标)
  10. df = pd.read_csv('test.tsv', sep='\t', usecols=[0, 1, 3])
  12. # 读取指定列(按列名)
  13. df = pd.read_csv('test.tsv', sep='\t', usecols=['a', 'b', 'd'])
  15. # 读取压缩文件 {'gzip', 'bz2', 'zip', 'xz'}
  16. df = pd.read_csv('test.tsv.gz', sep='\t', compression='gzip')
  18. # 指定engine {'c', 'python'}  c更快，python更完善
  19. df = pd.read_csv('test.tsv.gz', sep='\t', compression='gzip', engine='c')

大文件分块读取

  1. bigfile = 'bigtest.tsv.gz'
  3. chunksize = 10000  # 每次读取的行数
  4. df = pd.DataFrame()
  6. # =================================
  7. # 方式1：直接设定chunksize    (推荐)
  8. # =================================
  9. reader = pd.read_csv(bigfile, sep='\t', chunksize=chunksize, compression='gzip')
  11. for chunk in reader:
  12.     # you can do some filtration or something else here
  13.     df = pd.concat([df, chunk])
  15. # ===========================
  16. # 方式2：设置iterator=True 
  17. # ===========================
  18. reader = pd.read_csv(bigfile, sep='\t', iterator=True, compression='gzip')
  20. while True:
  21.     try:
  22.         chunk = reader.get_chunk(chunksize)
  23.         df = pd.concat([df, chunk])
  24.     except StopIteration:
  25.         break

其他文件读取

  1. pd.read_table  # 将废弃，用read_csv(sep='\t')代替
  3. pd.read_excel
  5. pd.read_pickle
  7. pd.read_json
  9. pd.read_html
  11. pd.read_sql
  13. ...

2 写入文件

  1. # 写入文本文件
  2. df.to_csv('out.tsv', sep='\t', index=False)
  4. # 压缩文件
  5. df.to_csv('out.tsv.gz', sep='\t', compression='gzip', index=False)
  7. # 写指定列
  8. df.to_csv('out.tsv', sep='\t', index=False, columns=['a', 'c'])
  11. # 写入pickle
  12. df.to_pickle('out.pkl')
  15. # 写入SQLite3
  16. ## with sqlite3
  17. import sqlite3
  18. con = sqlite3.Connection()
  19. df.to_sql('tablename', con, index=False)
  21. # if_exists: {'fail', 'replace', 'append'}
  22. df.to_sql('tablename', con, if_exists='replcae')
  24. ## with sqlalchemy
  25. from sqlalchemy import create_engine
  27. # in-memory
  28. engine = create_engine('sqlite://', echo=False)
  29. # to a file
  30. engine = create_engine('sqlite:///test.sqlite', echo=False)
  32. df.to_sql('users', con=engine, if_exists='replace', index_label='id'

3 读取网络文件

  1. url = 'http://rest.kegg.jp/list/hsa:100750326+hsa:100422907'
  3. df = pd.read_csv(url, sep='\t', names=['geneid', 'symbol'])