以下程序使用python2.7

requests模块例子

  1. #coding=utf-8
  3. import requests;
  5. url = "http://www.baidu.com";
  6. r = requests.get(url);
  8. # print r.content;
  9. # print r.text;
  10. # print r.status_code;
  11. # print r.headers;
  12. print r.cookies;

hackhttp模块例子

testHackHttp1

如果使用的是python2.7.8版本会出现’module’ object has no attribute ‘_create_unverified_context’报错信息
image.png
解决：升级版本
image.png

  1. # coding=utf-8
  3. import hackhttp
  4. # 创建一个hackhttp对象
  5. hh = hackhttp.hackhttp()
  6. # 定义url
  7. url = "http://www.baidu.com"
  8. # 发出一个简单的请求
  9. try:
  10.     code,head,html,redirect_url,log = hh.http(url)
  11. except Exception as e:
  12.     print(e)
  13. # 状态码
  14. print code
  15. # http头
  16. print head
  17. # html内容
  18. print html

testHackHttp2

  2. # coding=utf-8
  3. import hackhttp
  5. hh = hackhttp.hackhttp()
  7. # 可以指定第二个参数，post提交的数据
  8. code, head, body, redirect, log = hh.http('http://httpbin.org/post', post="key1=val1&key2=val2")
  10. print body

testHackHttp2

  2. # coding=utf-8
  3. import hackhttp
  5. hh = hackhttp.hackhttp()
  7. # 可以指定第二个参数，post提交的数据
  8. code, head, body, redirect, log = hh.http('http://httpbin.org/post', post="key1=val1&key2=val2")
  10. print body

testHackHttp3

  1. # coding=utf-8
  2. import hackhttp
  4. raw_content  = '''
  5. POST /post HTTP/1.1
  6. Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8
  7. Accept-Language: zh-CN
  8. Upgrade-Insecure-Requests: 1
  9. User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.140 Safari/537.36 Edge/18.17763
  10. Accept-Encoding: gzip, deflate
  11. Host: httpbin.org
  12. Connection: close
  14. '''
  16. raw='''POST /post HTTP/1.1
  17. Host: httpbin.org
  18. User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10.11; rv:45.0) Gecko/20100101 Firefox/45.0
  19. Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8
  20. Accept-Language: zh-CN,zh;q=0.8,en-US;q=0.5,en;q=0.3
  21. Accept-Encoding: gzip, deflate
  22. Connection: close
  23. Content-Type: application/x-www-form-urlencoded
  24. Content-Length: 19
  26. key1=val1&key2=val2'''
  28. hh = hackhttp.hackhttp()
  30. code, head, html, redirect, log = hh.http('http://httpbin.org/post', raw=raw)
  32. print html;

testHackHttp4

  1. # coding=utf-8
  3. import hackhttp
  4. # 自定义请求头
  5. hh = hackhttp.hackhttp()
  6. headers_dict = {
  7.     'X-Forwarder-For': 'https://q.bugscan.net',
  8.     'Hack-Http': 'Header Dict Val'
  9. }
  11. header_str='HH_HEADER_1: hh h1 val\r\nHH_HEADER_2:hh h2 val'
  13. code, head, body, redirect, log = hh.http('https://www.baidu.com', headers=header_str)
  15. print log['request']

testHackHHttp5

  2. import hackhttp
  3. hh = hackhttp.hackhttp()
  4. proxy_str = ('127.0.0.1', 8080)
  6. code, head, body, redirect, log = hh.http('http://httpbin.org/post', post="key1=val1&key2=val2", proxy=proxy_str)
  8. # code, head, body, redirect, log = hh.http('http://www.baidu.com', proxy=proxy_str)
  10. print code;

BeautifulSoup模块例子

testBeautifulSoup1

  1. #coding=utf-8
  2. from bs4 import BeautifulSoup
  3. # https://www.crummy.com/software/BeautifulSoup/bs4/doc.zh/
  4. html_doc = """
  5. <html><head><title>The Dormouse's story</title></head>
  6. <body>
  7. <p class="title"><b>The Dormouse's story</b></p>
  9. <p class="story">Once upon a time there were three little sisters; and their names were
  10. <a href="http://example.com/elsie" class="sister" id="link1">Elsie</a>,
  11. <a href="http://example.com/lacie" class="sister" id="link2">Lacie</a> and
  12. <a href="http://example.com/tillie" class="sister" id="link3">Tillie</a>;
  13. and they lived at the bottom of a well.</p>
  15. <p class="story">...</p>
  16. """
  17. # 使用BS库解析html文档，返回soup对象
  18. #soup = BeautifulSoup(html_doc)
  19. #soup = BeautifulSoup(html_doc,"html.parser")
  20. soup = BeautifulSoup(html_doc,"lxml")
  21. #print(soup)
  22. # 格式化 html文档
  23. # print(soup.prettify())
  25. # 几个简单的浏览结构化数据的方法
  26. # 打印title标签
  27. # print(soup.title)
  28. # <title>The Dormouse's story</title>
  30. # 打印title标签名
  31. # print(soup.title.name)
  32. # u'title'
  34. # 打印title标签中的内容
  35. #print(soup.title.string)
  36. # u'The Dormouse's story'
  38. # 打印title标签的父标签的名字
  39. #print(soup.title.parent.name)
  40. # u'head'
  42. # 打印p标签里的所有内容
  43. # print(soup.p)
  44. # <p class="title"><b>The Dormouse's story</b></p>
  46. # 打印p标签的class属性值
  47. # print(soup.p['class']) # 显示第一个p标签一个值
  48. # u'title'
  50. # 打印a标签的内容
  51. #print(soup.a)
  52. # <a class="sister" href="http://example.com/elsie" id="link1">Elsie</a>
  54. # 获取所有的a标签
  55. #print(soup.find_all('a'))
  56. # [<a class="sister" href="http://example.com/elsie" id="link1">Elsie</a>,
  57. #  <a class="sister" href="http://example.com/lacie" id="link2">Lacie</a>,
  58. #  <a class="sister" href="http://example.com/tillie" id="link3">Tillie</a>]
  60. # 获取id为link3的标签
  61. print(soup.find(id="link3"))
  62. # <a class="sister" href="http://example.com/tillie" id="link3">Tillie</a>

testBeautifulSoup2

  1. #coding=utf-8
  2. from bs4 import BeautifulSoup
  4. html = """
  5. <html>
  6.     <head>
  7.         <title>The Dormouse's story</title>
  8.     </head>
  9.     <body>
  10.         <p class="story">
  11.             Once upon a time there were three little sisters; and their names were
  12.             <a href="http://example.com/elsie" class="sister" id="link1">
  13.                 <span>Elsie</span>
  14.             </a>
  15.             <a href="http://example.com/lacie" class="sister" id="link2">Lacie</a>
  16.             and
  17.             <a href="http://example.com/tillie" class="sister" id="link3">Tillie</a>
  18.             and they lived at the bottom of a well.
  19.         </p>
  20.         <p class="story">...</p>
  21. """
  23. soup = BeautifulSoup(html,'lxml')
  25. # 获取一个标签下的子标签
  26. #将p标签下的所有子标签存入到了一个列表中
  27. #print(soup.p.contents)
  29. # 遍历输出
  30. for c in range(len(soup.p.contents)):
  31.     print soup.p.contents[c];

testBeautifulSoup3

  1. #coding=utf-8
  2. from bs4 import BeautifulSoup
  4. html='''
  5. <div class="panel">
  6.     <div class="panel-heading">
  7.         <h4>Hello</h4>
  8.     </div>
  9.     <div class="panel-body">
  10.         <ul class="list" id="list-1">
  11.             <li class="element">Foo</li>
  12.             <li class="element">Bar</li>
  13.             <li class="element">Jay</li>
  14.         </ul>
  15.         <ul class="list list-small" id="list-2">
  16.             <li class="element">Foo</li>
  17.             <li class="element">Bar</li>
  18.         </ul>
  19.     </div>
  20. </div>
  21. '''
  23. soup = BeautifulSoup(html, 'lxml')
  25. #查找所有的ul标签内容
  26. #print(soup.find_all('ul'))
  27. #print(type(soup.find_all('ul')[0]))  #查找ul标签的类型
  28. for ul in soup.find_all('ul'):
  29.   print(ul.find_all('li'))  #针对结果再次find_all,从而获取所有的li标签信息
  30. '''
  31. find_all(name,attrs,recursive,text,**kwargs)
  32. 可以根据标签名，属性，内容查找文档
  33. '''

testBeautifulSoup4

  1. #coding=utf-8
  2. from bs4 import BeautifulSoup
  4. html='''
  5. <div class="panel">
  6.     <div class="panel-heading">
  7.         <h4>Hello</h4>
  8.     </div>
  9.     <div class="panel-body">
  10.         <ul class="list" id="list-1" name="elements">
  11.             <li class="element">Foo</li>
  12.             <li class="element">Bar</li>
  13.             <li class="element">Jay</li>
  14.         </ul>
  15.         <ul class="list list-small" id="list-2">
  16.             <li class="element">Foo</li>
  17.             <li class="element">Bar</li>
  18.         </ul>
  19.     </div>
  20. </div>
  21. '''
  23. soup = BeautifulSoup(html, 'lxml')
  24. print(soup.find_all(attrs={'id': 'list-1'}))      # 查找id为list-1的内容
  25. #print(soup.find_all(attrs={'name': 'elements'}))  # 查找name为elements的内容
  26. #print(soup.find_all(attrs={'class':'list'}));

testBeautifulSoup5

  1. #coding=utf-8
  2. from bs4 import BeautifulSoup
  4. html='''
  5. <div class="panel">
  6.     <div class="panel-heading">
  7.         <h4>Hello</h4>
  8.     </div>
  9.     <div class="panel-body">
  10.         <ul class="list" id="list-1">
  11.             <li class="element">Foo</li>
  12.             <li class="element">Bar</li>
  13.             <li class="element">Jay</li>
  14.         </ul>
  15.         <ul class="list list-small" id="list-2">
  16.             <li class="element">Foo</li>
  17.             <li class="element">Bar</li>
  18.         </ul>
  19.     </div>
  20. </div>
  21. '''
  23. soup = BeautifulSoup(html, 'lxml')
  24. print(soup.find_all(text='Foo'))  #查找所有的text='Foo'的文本

testBeautifulSoup6

  1. #coding=utf-8
  2. from bs4 import BeautifulSoup
  4. html='''
  5. <div class="panel">
  6.     <div class="panel-heading">
  7.         <h4>Hello</h4>
  8.     </div>
  9.     <div class="panel-body">
  10.         <ul class="list" id="list-1">
  11.             <li class="element">Foo</li>
  12.             <li class="element">Bar</li>
  13.             <li class="element">Jay</li>
  14.         </ul>
  15.         <ul class="list list-small" id="list-2">
  16.             <li class="element">Foo</li>
  17.             <li class="element">Bar</li>
  18.         </ul>
  19.     </div>
  20. </div>
  21. '''
  23. soup = BeautifulSoup(html, 'lxml')
  24. #print(soup.select('.panel .panel-heading'))  #获取class名为panel和panel-heading的内容
  25. #print(soup.select('ul li'))  #获取标签名为ul和li的内容
  26. #print(soup.select('#list-2 .element'))  #获取class名为element，id为list-2的内容
  27. print(type(soup.select('ul')[0]))  #获取class名为ul的类型

testBeautifulSoup7

  1. #coding=utf-8
  2. from bs4 import BeautifulSoup
  4. html='''
  5. <div class="panel">
  6.     <div class="panel-heading">
  7.         <h4>Hello</h4>
  8.     </div>
  9.     <div class="panel-body">
  10.         <ul class="list" id="list-1">
  11.             <li class="element">Foo</li>
  12.             <li class="element">Bar</li>
  13.             <li class="element">Jay</li>
  14.         </ul>
  15.         <ul class="list list-small" id="list-2">
  16.             <li class="element">Foo</li>
  17.             <li class="element">Bar</li>
  18.         </ul>
  19.     </div>
  20. </div>
  21. '''
  23. soup = BeautifulSoup(html, 'lxml')
  24. # 获取li标签
  25. for li in soup.select('li'):
  26.     print(li.get_text())
  27. #使用get_text()获取文本内容

testBeautifulSoup8

  1. #coding=utf-8
  2. from bs4 import BeautifulSoup
  4. html='''
  5. <div class="panel">
  6.     <div class="panel-heading">
  7.         <h4>Hello</h4>
  8.     </div>
  9.     <div class="panel-body">
  10.         <ul class="list" id="list-1">
  11.             <li class="element">Foo</li>
  12.             <li class="element">Bar</li>
  13.             <li class="element">Jay</li>
  14.         </ul>
  15.         <ul class="list list-small" id="list-2">
  16.             <li class="element">Foo</li>
  17.             <li class="element">Bar</li>
  18.         </ul>
  19.     </div>
  20. </div>
  21. '''
  23. soup = BeautifulSoup(html, 'lxml')
  24. for ul in soup.select('ul'):
  25.     print(ul['id'])
  26.     print(ul.attrs['id'])
  27. #获取属性的时候可以通过[属性名]或者attrs[属性名]

re模块例子

  1. #coding:utf-8;
  3. # 1. 正则表达式
  4. # 正则表达式是一个特殊的字符序列，它能帮助你方便的检查一个字符串是否与某种模式匹配
  6. # 2. re模块
  7. # compile(模式)  创建模式对象
  8. # search(模式，字符串)  在字符串中寻找模式
  9. # match(模式，字符串) 在字符串开始处匹配模式
  10. # findall(模式，字符串) 列表形式返回匹配项
  11. # sub(模式，替换字符，字符串) 替换字符串中的匹配项
  12. # split(模式) 根据模式分割字符串
  14. import re
  15. # 6个函数的使用
  16. #模式1 普通字符串， 匹配字符串a
  17. a = re.compile('abc');
  18. print("模式对象：" + str(a))
  19. b = a.search('abdabcbabab');
  20. print('匹配到的字符串信息：'+ str(b))
  21. c = a.match('abdabcbabab');
  22. print("第一个匹配的字符串：" + str(c))
  23. d = re.findall('abc', 'abdabcbabcab');
  24. print("匹配到的字符列表：" + str(d))
  25. e = re.sub('abc','aaa','abdabcbabcab');
  26. print("匹配到的字符列表：" + str(e))
  27. f = re.split(',', 'ab,bc,cd,de');
  28. print("分割字符：" + str(f))
  30. # 正则表达式匹配修饰符
  31. # re.I  使大小写不敏感
  32. # re.L  做本地化失败匹配
  33. # re.M  多行匹配，影响^和$
  34. # re.S  使.匹配包括换行在内的所有字符
  35. # re.U  根据Unicode字符集解析字符，影响\w,\W,\b,\B
  36. # re.X  该标志通过给予你更灵活的格式以便你将正则表达式写得更易于理解。
  37. # 对大小写不敏感
  38. aa = re.search('abc', 'ABCc1B3ABC2',re.I);
  39. print("大小写不敏感：" + str(aa))
  41. # ^ 匹配字符串的开头
  42. # $ 匹配字符串的结尾
  43. # .  匹配任意字符，除了换行
  44. # [...]   用来表示一组字符，单独列出：[amk] 匹配 'a','m','k'
  45. # [^...]  不在[]中的字符，[^abc] 匹配除了a、b、c之外的字符
  46. # re*      匹配0个或多个表达式
  47. # re+      匹配1个或多个表达式
  48. # re?      匹配0个或1个由前面的正则表达式定义的片段，非贪婪模式
  49. # re{n}    精确匹配n个前面的表达式，例如o{2},不能匹配“Bob”中的'o'，但能匹配'food'中的两个o
  50. # re{n,}   匹配n个前面表达式。例如 o{2,} 不能匹配“Bob”中的'o'，但能匹配'foooood'中的所有o，o{1,}等价于o+, o{0,}等价于o*
  51. # re{n,m}  匹配n到m此由前面的正则表达式定义的片段，贪婪模式
  53. # ^ 匹配字符串的开头
  54. bb = re.findall('^abc',  'ABCc1B3abc2');
  55. print("匹配开头:" + str(bb))
  56. # $ 匹配字符串的结尾
  57. cc = re.findall('abc$',  'ABCc1B3abc');
  58. print("匹配开头:" + str(cc))
  59. # .  匹配任意字符，除了换行
  60. dd = re.findall('.',  'ABCc1B3abc2');
  61. print("匹配任意字符: " +str(dd))
  62. # 匹配一组单个字符,匹配a,b,c
  63. ee = re.findall('[abc]',  'ABCc1B3abc');
  64. print("匹配一组单个字符:" + str(ee))
  65. # 匹配不在[]的单个字符
  66. ee = re.findall('[^abc]',  'ABCc1B3abc');
  67. print("匹配不在[]的单个字符:" + str(ee))
  68. # 前面的字符匹配0个或多个字符
  69. ff = re.findall('1*',  'ABCc1B3abc');
  70. print("匹配0个或多个字符:" + str(ff))
  71. # 前面的字符匹配1个或多个字符
  72. ff = re.findall('1+',  'ABCc1B3abc');
  73. print("匹配1个或多个字符:" + str(ff))
  74. # 前面的字符匹配0个或1个字符,非贪婪模式
  75. ff = re.findall('1B3?',  'ABCc1B3abc');
  76. print("匹配0个或1个字符:" + str(ff))
  77. # re{n}    精确匹配n个前面的表达式
  78. ff = re.findall('c{2}',  'ABCcc1B3abc');
  79. print("精确匹配n个前面的表达式:" + str(ff))
  80. # re{n,}   匹配n个前面表达式
  81. ff = re.findall('c{2,}',  'ABcCcc1Bccccc3abc');
  82. print("匹配n个前面表达式:" + str(ff))
  83. # re{n,m}  匹配n到m此由前面的正则表达式定义的片段，贪婪模式
  84. ff = re.findall('c{2,4}',  'ABcCcc1Bccc3abc');
  85. print("匹配n到m此由前面的正则表达式定义的片段:" + str(ff))
  87. #  \d  匹配一个数字字符  等价于[0-9]
  88. #  \D  匹配一个非数字字符  等价于 [^0-9]
  89. #  \s  匹配任何空白字符，包括空格、制表符、换页符等等，等价于[\f\n\r\t\v]
  90. #  \S  匹配任何非空白哦字符，等价于[^\f\n\r\t\v]
  91. #  \w  匹配包括下划线的任何单词字符，等价于[A-Za-z9-0_]
  92. #  \W  匹配任何非单词字符。等价于[^A-Za-z9-0_]
  94. #  \d  匹配一个数字字符  等价于[0-9]
  95. ff = re.findall('\d',  'ABcCcc1Bccc3abc');
  96. print("匹配一个数字字符 :" + str(ff))
  97. #  \D  匹配一个非数字字符  等价于 [^0-9]
  98. ff = re.findall('\D',  '123c1Bc3abc');
  99. print("匹配一个非数字字符 :" + str(ff))
  100. #  \s  匹配任何空白字符，包括空格、制表符、换页符等等，等价于[\f\n\r\t\v]
  101. ff = re.findall('\s',  '123 c1Bc 3abc');
  102. print("匹配任何空白字符 :" + str(ff))
  103. #  \S  匹配任何非空白哦字符，等价于[^\f\n\r\t\v]
  104. ff = re.findall('\S',  '1 c2c 3abc');
  105. print("匹配任何非空白字符 :" + str(ff))
  106. #  \w  匹配包括下划线的任何单词字符，等价于[A-Za-z9-0_]
  107. ff = re.findall('\w',  '1 c_A2c 3abc');
  108. print("匹配包括下划线的任何单词字符 :" + str(ff))
  109. #  \W  匹配任何非单词字符。等价于[^A-Za-z9-0_]
  110. ff = re.findall('\W',  '1 c_2c 3abc');
  111. print("匹配任何非单词字符 :" + str(ff))
  113. # 贪婪模式和非贪婪模式
  114. # 正则表达式通畅常用于在文本中查找匹配的字符串。python里的数量词默认是贪婪的，
  115. # 意思就是总是尝试匹配尽可能多的字符
  116. # 非贪婪模式则相反，总是尝试匹配尽可能少的字符
  118. # 前面的字符匹配0个或1个字符,非贪婪模式,也是可以匹配到的
  119. ff = re.findall('1B3?',  'ABCc1Bx3abc');
  120. print("匹配0个或1个字符:" + str(ff))