爬虫模板

  1. # pip install requests
  2. # pip install bs4
  3. # pip install pycryptodome
  4. # pip install lxml
  6. import requests,re,time,os
  7. from bs4 import BeautifulSoup
  8. from Crypto.Cipher import AES
  9. from lxml import etree
  10. from utils import func
  14. url = "https://passport.17k.com/ck/user/login"
  15. data = "loginName=13271473920&password=qqqqqqq1"
  16. session = requests.session()
  17. head={
  18.     "User-Agent": func.random_ua()
  19. }
  20. proxies={
  21.     "https": "https://218.45.56.8:3125"
  22. }
  24. with session.post(url,data=data,headers=head,proxies="") as res:    # 拿到身份cookie
  25.     url = "https://user.17k.com/ck/author/shelf?page=1&appKey=2406394919"
  26.     with session.get(url,data=data,headers=head) as res2:    # 获取想要的内容
  27.         print(res.json())
  28.         print(res.cookies)
  29.         # # --------------------------BeautifulSoup解析 html拿值
  30.         # soup = BeautifulSoup(res.text, "html.parser")  # html.parser指定文件类型为html，防止警告信息
  31.         # table = soup.find("div", class_="tbl-body") # find(标签,属性=值)
  32.         # table = table.find_all("th")
  33.         # for i in range(0, len(table) - 1):
  34.         #     print(table[i].text)
  36.         # # --------------------------re正则拿值
  37.         # # ?P<guang> 起一个别名 re.S 让.具有换行的功能
  38.         # obj = re.compile(r'<span style="font-size:\d+px;">(?P<guang>.*?)</span>', re.S)
  39.         # res = obj.finditer(res.json())
  40.         # for i in res:
  41.         #     print(i.group('guang'))
  43.         # # --------------------------xpath解析 XHR拿值
  44.         # tree = etree.parse(res.text)
  45.         # result = tree.xpath("/html/body/ul/li[1]/a/text()")  # 指定li下标取a标签文本
  46.         # result = tree.xpath('/html/body/ul/ol/li/a/@href')  # 取指定a标签href的内容
  47.         # result = tree.xpath('/html/body/ul/ol/li/a[@href="feiji"]/text()')  # 取指定a标签href属性为feiji的文本
  50. # # --------------------------AES解密
  51. # def dec_ts(name, key):
  52. #     aes = AES.new(key=key, IV=b"0000000000000000", mode=AES.MODE_CBC)
  53. #     with open(f"noval/{name}", mode="rb") as f1,\
  54. #         open(f"noval/temp_{name}", mode="wb") as f2:
  55. #         bs = f1.read()  # 从源文件读取内容
  56. #         f2.write(aes.decrypt(bs))  # 把解密好的内容写入文件
  57. #     print(f"{name}处理完毕")
  59. # # --------------------------os合并文件
  60. # def merge_ts():
  61. #     # mac: cat 1.ts 2.ts 3.ts > xxx.mp4
  62. #     # windows: copy /b 1.ts+2.ts+3.ts xxx.mp4
  63. #     lsts = []
  64. #     with open("noval/越狱第一季第一集_second_m3u8.txt", mode="r", encoding="utf-8") as f:
  65. #         for line in f:
  66. #             if line.startswith("#"):
  67. #                 continue
  68. #             line = line.strip()
  69. #             lsts.append(f"noval/temp_{line}")
  70. #
  71. #     s = "+".join(lsts)  # 1.ts+2.ts+3.ts
  72. #     os.system(f"copy /b {s} movie.mp4")
  73. #     print("搞定!")

单线程异步协程模板

  1. import aiohttp # pip install aiohttp
  2. import aiofiles # pip install aiofiles
  3. import time
  4. import asyncio
  6. urls = [
  7.         "http://kr.shanghai-jiuxin.com/file/2022/0511/f92746daee951aea129ca7b8bafbdb97.jpg",
  8.         "http://kr.shanghai-jiuxin.com/file/2022/0511/d93db47964abd557794ccd68d8a8ab16.jpg",
  9.         "http://kr.shanghai-jiuxin.com/file/2022/0511/8d647c98d9f59ea4ae38a0ba1e568bc3.jpg",
  10.         "http://kr.shanghai-jiuxin.com/file/2022/0511/0d69db8b2f5ad317c39bfd4ab1033064.jpg"
  11. ]
  13. async def download(url):
  14.     name = url.rsplit("/",1)[1]     # 按/分割符 从右边开始切 切1次 取第二个值
  15.     async with aiohttp.ClientSession() as session:  # session 相当于request
  16.         async with session.get(url) as resp:    # resp 相当于request.get()
  17.             async with aiofiles.open(name,mode="wb") as f:     # 创建文件写入
  18.                 await f.write(await resp.content.read())  # 读图片是异步操作，需要await挂起
  20.     print("开始准备下载")
  21.     await asyncio.sleep(2)
  22.     print("下载完成")
  24. async def main():
  25.     tasks = []
  26.     for url in urls:
  27.         t = download(url)
  28.         tasks.append(t) # 在py3.8以上以列表方式加函数格式为 asyncio.create_task(函数名())
  30.     await asyncio.wait(tasks) # 异步投递任务 异步中用await可以让程序在等待的时候挂起继续执行下一个任务
  32. if __name__ == '__main__':
  33.     t1 = time.time()
  34.     asyncio.get_event_loop().run_until_complete(main()) # 异步执行入口 py3.7用asyncio.run()
  35.     t2 = time.time()
  36.     print(t2-t1)

线程池

  1. # 1. 如何提取单个页面的数据
  2. # 2. 上线程池,多个页面同时抓取
  3. import requests
  4. from lxml import etree
  5. import csv
  6. from concurrent.futures import ThreadPoolExecutor
  8. f = open("data.csv", mode="w", encoding="utf-8")
  9. csvwriter = csv.writer(f)
  12. def download_one_page(url):
  13.     # 拿到页面源代码
  14.     resp = requests.get(url)
  15.     html = etree.HTML(resp.text)
  16.     table = html.xpath("/html/body/div[2]/div[4]/div[1]/table")[0]
  17.     # trs = table.xpath("./tr")[1:]
  18.     trs = table.xpath("./tr[position()>1]")
  19.     # 拿到每个tr
  20.     for tr in trs:
  21.         txt = tr.xpath("./td/text()")
  22.         # 对数据做简单的处理: \\  / 去掉
  23.         txt = (item.replace("\\", "").replace("/", "") for item in txt)
  24.         # 把数据存放在文件中
  25.         csvwriter.writerow(txt)
  26.     print(url, "提取完毕!")
  29. if __name__ == '__main__':
  30.     # 创建线程池
  31.     with ThreadPoolExecutor(50) as t:
  32.         for i in range(1, 200):  # 199 * 20 = 3980
  33.             # 把下载任务提交给线程池
  34.             t.submit(download_one_page, f"http://www.xinfadi.com.cn/marketanalysis/0/list/{i}.shtml")
  36.     print("全部下载完毕!")

selenium模拟

  1. # pip install selenium
  2. from selenium.webdriver import Chrome   # 操作chrom浏览器
  3. from selenium.webdriver.common.action_chains import ActionChains    # 事件链 基于页面中的其他框架操作
  4. from selenium.webdriver.chrome.options import Options   # 一些启动参数配置
  5. from selenium.webdriver.common.keys import Keys     # 按键
  6. from selenium.webdriver.support.select import Select    # 处理select下拉列表
  8. from chaojiying import Chaojiying_Client
  9. import time
  11. # 初始化超级鹰
  12. chaojiying = Chaojiying_Client('13271473920', 'qqqqqqq1', '933391')
  15. # # -------------------------------------如果你的程序被识别到了,修改navigator的返回值为false
  16. # # -------------------------------------1.chrome的版本号如果小于88  启动浏览器的时候,向页面嵌入js代码
  17. # web = Chrome()
  18. # web.execute_cdp_cmd("Page.addScriptToEvaluateOnNewDocument", {
  19. #   "source": """
  20. #    navigator.webdriver = undefined
  21. #     Object.defineProperty(navigator, 'webdriver', {
  22. #       get: () => undefined
  23. #     })
  24. #   """
  25. # })
  26. # web.get("https://www.baidu.com")
  27. # # -------------------------------------2.chrome的版本大于等于88
  28. option = Options()
  29. # # option.add_experimental_option('excludeSwitches', ['enable-automation'])
  30. option.add_argument('--disable-blink-features=AutomationControlled')
  32. # # -------------------------------------设置后台无头模式
  33. # option.add_argument("--headless")
  34. # option.add_argument("--disbale-gpu")
  36. web = Chrome(options=option)
  38. web.get("https://kyfw.12306.cn/otn/resources/login.html")
  40. time.sleep(2)
  41. web.find_element_by_xpath('/html/body/div[2]/div[2]/ul/li[2]/a').click()
  42. time.sleep(3)
  44. # -------------------------------------拿页面中的图片
  45. verify_img_element = web.find_element_by_xpath('//*[@id="J-loginImg"]')
  46. verify_img = verify_img_element.screenshot_as_png
  48. # -------------------------------------用超级鹰去识别验证码
  49. dic = chaojiying.PostPic(verify_img, 9004)
  50. result = dic['pic_str']  # x1,y1|x2,y2|x3,y3
  51. rs_list = result.split("|")
  52. for rs in rs_list:  # x1,y1
  53.     p_temp = rs.split(",")
  54.     x = int(p_temp[0])
  55.     y = int(p_temp[1])
  56.     # 要让鼠标移动到某一个位置. 然后进行点击
  57.     # 醒了 -> 掀开被子 -> 坐起来 -> 穿鞋子  -> 穿衣服  -> 开始执行动作
  58.     ActionChains(web).move_to_element_with_offset(verify_img_element, x, y).click().perform()
  60. time.sleep(1)
  61. # -------------------------------------输入用户名和密码及按键
  62. web.find_element_by_xpath('//*[@id="J-userName"]').send_keys("123456789")
  63. web.find_element_by_xpath('//*[@id="J-password"]').send_keys("12346789")
  64. # web.find_element_by_xpath('//*[@id="search_input"]').send_keys("python", Keys.ENTER)
  66. # -------------------------------------点击登录
  67. web.find_element_by_xpath('//*[@id="J-login"]').click()
  69. time.sleep(5)
  71. # -------------------------------------拖拽
  72. btn = web.find_element_by_xpath('//*[@id="nc_1_n1z"]')
  73. ActionChains(web).drag_and_drop_by_offset(btn, 300, 0).perform()
  76. # # -------------------------------------切换窗口
  77. # web.switch_to.window(web.window_handles[-1])
  78. # # 在新窗口中提取内容
  79. # job_detail = web.find_element_by_xpath('//*[@id="job_detail"]/dd[2]/div').text
  80. # print(job_detail)
  81. # # 关掉子窗口
  82. # web.close()
  83. # # 变更selenium的窗口视角. 回到原来的窗口中
  84. # web.switch_to.window(web.window_handles[0])
  86. # # -------------------------------------页面中遇到了 iframe
  87. # web.get("https://www.91kanju.com/vod-play/541-2-1.html")
  88. # # 处理iframe的话. 必须先拿到iframe. 然后切换视角到iframe . 再然后才可以拿数据
  89. # iframe = web.find_element_by_xpath('//*[@id="player_iframe"]')
  90. # web.switch_to.frame(iframe)  # 切换到iframe
  91. # # web.switch_to.default_content()  # 切换回原页面
  92. # tx = web.find_element_by_xpath('//*[@id="main"]/h3[1]').text
  93. # print(tx)
  95. # # -------------------------------------修改 select下拉框选中项
  96. # sel_el = web.find_element_by_xpath('//*[@id="OptionDate"]')
  97. # # 对元素进行包装, 包装成下拉菜单
  98. # sel = Select(sel_el)
  99. # sel.select_by_index(i)  # 按照索引进行切换

猜年龄 循环判断

  1. import random
  3. age = random.randint(10,20)
  4. print("随机年龄是：%d"%age)
  5. i = 1
  6. while i<4:
  7.     age_i = int(input("请输入要猜的年龄："))
  8.     if age_i==age:
  9.         print("恭喜你猜对了")
  10.         break
  11.     else:
  12.         i += 1
  13.         if i == 4:
  14.             print("猜3次上限了")
  15.             res = input("请输入Y继续或Q退出：")
  16.             if res == "Y":
  17.                 i = 1
  18.                 continue
  19.             elif res == "Q":
  20.                 print("我要退出了")
  21.                 break
  22.             else:
  23.                 print("请输入正确的命令")
  24.                 res = input("请输入Y继续或Q退出：")
  25.         else:
  26.             print("猜错了，请重新猜一猜")

BIM判断 加减乘除

  1. # BIM计算公式 体重除以身高的平方
  2. h = float(input("请输入你的身高："))
  3. t = float(input("请输入你的体重："))
  4. s = t/(h**2)
  6. if s < 18.5:
  7.     print("体重过轻")
  8. elif s>18.5 and s<25:
  9.     print("体重正常")
  10. elif s>15 and s<28:
  11.     print("体重过重")
  12. elif s>28 and s<32:
  13.     print("体重肥胖")
  14. elif s>32:
  15.     print("严重肥胖")

求出一个集合中不重复的值 循环

  1. def l():
  2.     list1 = [1, 1, 2, 2, 3, 4, 4, 5, 5,5,3,3,4,4,7,8,9,9,0]
  3.     set1 = set(list1) #set1为list1去重后的唯一值
  4.     for s in set1:
  5.         a = 0
  6.         for i in list1:
  7.             if i == s:
  8.                 a += 1
  9.         if a==1:
  10.             print("出现不重复的值：{}".format(s))
  11. l()

双人对战 类和对象的魔术方法调用

  1. import random
  2. import time
  4. class Person:
  5.     def __init__(self,name,blood): #定义初始化方法
  6.         '''
  7.         双人对战
  8.         :param name: 姓名
  9.         :param blood: 血量
  10.         '''
  11.         self.name=name #类中的全局实例属性
  12.         self.blood=blood
  13.     def tong(self,dr):
  14.         dr.blood-=10
  15.         print("%s捅了%s一刀，%s掉10点血，%s剩余%s点血"%(self.name,dr.name,dr.name,dr.name,dr.blood))
  16.     def kan(self,dr):
  17.         dr.blood -= 20
  18.         print("%s砍了%s一刀，%s掉20点血，%s剩余%s点血"%(self.name,dr.name,dr.name,dr.name,dr.blood))
  19.     def chi(self,dr):
  20.         dr.blood += 10
  21.         print("%s吃了一颗药，加10点血，%s剩余%s点血"%(dr.name,dr.name,dr.blood))
  22.     def __str__(self):
  23.         return("%s还剩下%s血"%(self.name,self.blood))
  25. xm=Person('西门',100)
  26. ygc=Person('叶孤城',100)
  28. while True:
  29.     sj = random.randint(1,6)
  30.     if xm.blood <= 0 or ygc.blood <= 0:
  31.         break
  32.     elif sj == 1:
  33.         xm.tong(ygc)
  34.     elif sj == 2:
  35.         xm.kan(ygc)
  36.     elif sj == 3:
  37.         xm.chi(ygc)
  38.     elif sj == 4:
  39.         ygc.tong(xm)
  40.     elif sj == 5:
  41.         ygc.kan(xm)
  42.     elif sj == 6:
  43.         ygc.chi(xm)
  44.     print(ygc,xm)
  45.     print("*"*40)
  46.     time.sleep(1)
  48. print("游戏结束",ygc,xm)