关键词：python 爬虫

get示例

  1. log_in_url = "http://www.uimaker.com/member/ubi.php?action=qd&_=1526202830017"
  2. # headers
  3. headers = {
  4.     "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36",
  5.     "Referer": "http://www.uimaker.com/member/index.php"
  6. }
  7. # 参数使用params参数
  8. cookies = {}
  9. print(os.path.join(os.path.abspath("."), "cookies"))
  10. with open(os.path.join(os.path.abspath("."), "cookies"), 'r', encoding='utf-8') as f:
  11.     for item in f.read().split(';'):
  12.         name, value = item.split('=', 1)
  13.         cookies[name] = value
  15. # 请求时保留cookies
  16. session = requests.session()
  17. response = session.get(log_in_url, headers=headers, cookies=cookies)
  18. print(response.json())
  19. with open('./log.txt', 'a+', encoding='utf-8') as f:
  20.     f.write(str(datetime.datetime.now()) + "\t")
  21.     f.write(str(response.status_code) + os.linesep)

post示例

jq22签到示例，其实cookies直接放入到header里面即可

  1. import requests
  2. from bs4 import BeautifulSoup as bs
  3. import os
  4. import datetime
  6. session = requests.session()
  8. log_in_url = "http://www.jq22.com/signIn.aspx"
  9. # headers
  10. headers = {
  11.     "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36",
  12.     "Referer":"http://www.jq22.com/signIn.aspx"
  13. }
  14. # 参数
  15. data = {
  16.     "Button1": "签 到",
  17.     "__VIEWSTATE": "OQz0gcxiE2uemW6coqrtvy4sKAViSr6o8Xxn4QjA2WoOiQFGhlAYIOEl9am94jQqkvf4+19RlxfIFvNagglpiyBGz4smPulqXEeQ7q4kU0VsdhfifzVlfGrj+DTYHKTR",
  18.     "__VIEWSTATEGENERATOR": "ECDA716A",  # 不变
  19.     "__EVENTVALIDATION": "/Up6CRbE1Igwin/Bz2lx2qJo9+keEisPxD70frbfphQPhghu+ViAae9BGgf/NgO1peWGTpB1g0dHGJUyuCP9R53m201mCW3WyGvL6fjdFH58Ds7ZxC7HP3GlDan4Jl1C"
  20. }
  21. cookies = {}
  22. print(os.path.join(os.path.abspath("."),"cookies"))
  23. with open(os.path.join(os.path.abspath("."),"cookies"), 'r', encoding='utf-8') as f:
  24.     for item in f.read().split(';'):
  25.         name, value = item.split('=', 1)
  26.         cookies[name] = value
  27. cookie
  29. response = session.post(log_in_url, headers=headers, data=data, cookies=cookies)
  30. print(response.status_code)
  31. with open('./log.txt','a+',encoding='utf-8') as f:
  32.     f.write(str(datetime.datetime.now()) + "\t")
  33.     f.write(str(response.status_code) + os.linesep)

json请求示例

  1. import requests
  2. import json
  3. headers = {
  4.     "Content-Type": "application/json; charset=UTF-8",
  5.     "Referer": "http://jinbao.pinduoduo.com/index?page=5",
  6.     "User-Agent": "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.87 Safari/537.36",
  7.     }
  8. url = "http://jinbao.pinduoduo.com/network/api/common/goodsList"
  9. pyload = {"keyword": "", "sortType": 0, "withCoupon": 0, "categoryId": 16, "pageNumber": 1, "pageSize": 60}
  10. response = requests.post(url, data=json.dumps(pyload), headers=headers).text
  11. print(response）

获取代理

  1. import requests
  2. import re
  4. """
  5. 可能是歧梦谷系统不完善，每天访问推广链接添加的梦感不超过10，这里面以不同的ip访问一百次
  6. """
  7. proxies_list = []
  9. # 获取proxies
  10. url = "http://www.xicidaili.com/wn"
  11. headers = {
  12.     "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36"
  13. }
  14. response = requests.get(url, headers=headers)
  16. re_list = re.findall(
  17.     r'<td>\d+\.\d+\.\d+\.\d+</td>[\s\S]+?<td>\d+</td>', response.text, re.S)
  18. for x in re_list:
  19.     ip = re.search('<td>([\s\S]+)</td>[\s\S]+?<td>', x).group(1)
  20.     port = re.search('<td>[\s\S]+</td>[\s\S]+?<td>([\s\S]+)</td>', x).group(1)
  21.     proxies_list.append("http://" + ip + ":" + port)
  23. url = "http://www.68m.com/?fromuid=566043"
  24. headers = {
  25.     "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36"
  26. }
  27. proxies = {}
  28. i = 0
  29. for x in proxies_list:
  30.     proxies["https"] = x
  31.     response = requests.get(url, proxies=proxies, headers=headers)
  32.     print(response)
  33.     i += 1
  34.     if i > 10:
  35.         break

其他问题

cookie问题参考