代码实现

说明todo

  1. import multiprocessing as mp
  2. import os
  3. import re
  4. import time
  6. import pandas as pd
  7. import requests
  8. from bs4 import BeautifulSoup
  10. headers = {
  11.     'cookie': 'ip_ck=58aI7/31j7QuMDE5ODcyLjE2MaTA5NTU2NTI%3D; z_pro_city=s_provice%3Dzhejiang%26s_city%3Dningbo; userProvinceId=26; userCityId=154; userCountyId=929; userLocationId=12374; realLocationId=12374; userFidLocationId=12374; dwhis=%22s%22%3A%2228%22%2C%22m%22%3A%22125%22%2C%22p%22%3A%221296179%22; lv=1614734211; vn=11; visited_subcateId=6|0|443|529|28; visited_subcateProId=6-0|0-0|443-0|529-0|28-0,1289592|57-0; visited_serachKw=RTX%20%203090%7CRTX%20%203090/robots.txt%7CRTX%203090%7CGT%20210%7C210%7CRTX%203080; listSubcateId=6; Adshow=4; questionnaire_pv=1614729617',
  12.     'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.182 Safari/537.36 Edg/88.0.705.81'
  13. }
  14. base_url = "https://detail.zol.com.cn"
  15. pattern = re.compile(u'.+[0-9]+.+')
  16. products_A = {
  17.     "hdd": "https://detail.zol.com.cn/hard_drives/s5994/",  # 机械硬盘
  18.     "ssd": "https://detail.zol.com.cn/solid_state_drive/s8155/",  # 固态硬盘
  19.     "motherboard": "https://detail.zol.com.cn/motherboard/",  # 主板
  20.     "power": "https://detail.zol.com.cn/power/",  # 电源
  21.     "memory": "https://detail.zol.com.cn/memory/s5974/zhejiang/",  # 内存
  22.     "case": "https://detail.zol.com.cn/case/",  # 机箱
  23.     "lcd": "https://detail.zol.com.cn/lcd/",  # 液晶显示器
  24. }
  26. products_B = {
  27.     "vga": "https://detail.zol.com.cn/vga/",  # 显卡
  28.     "cooling_product": "https://detail.zol.com.cn/cooling_product/",  # 散热器
  29. }
  32. def get_all_urls(base_page: str) -> list:
  33.     url_list = []
  34.     url_pattern = base_page + "{0}.html"
  35.     index = 0
  36.     while True:
  37.         index += 1
  38.         cur_page = url_pattern.format(index)
  39.         response = requests.get(cur_page, headers=headers)
  40.         soup = BeautifulSoup(response.text, features="lxml")
  41.         items = soup.find("div", attrs={"class": "content"}).find("div", attrs={"class": "pic-mode-box"}). \
  42.             find("ul").find_all("li")
  44.         for item in items:
  45.             try:
  46.                 url_list.append(base_url + item.find("a").get("href"))
  47.             except:
  48.                 continue
  49.         end_flag = soup.find("div", attrs={"class": "history-tips"}).get_text()
  50.         if pattern.match(end_flag) is not None:
  51.             break
  52.     return url_list
  55. class MyCrawler:
  56.     def __init__(self, url_list: list, type: str):
  57.         self.type = type
  58.         self.workerNum = mp.cpu_count()
  59.         self.urlQueue = mp.Manager().Queue()
  60.         self.resultList = mp.Manager().list()
  61.         for url in url_list:
  62.             self.urlQueue.put(url)
  64.     def job(self, index: int):
  66.         Process_id = 'Process-' + str(index)
  67.         while not self.urlQueue.empty():
  68.             url = self.urlQueue.get()
  69.             try:
  70.                 start = time.time()
  71.                 map = {"link": url}
  72.                 response = requests.get(map["link"], headers=headers)
  73.                 soup = BeautifulSoup(response.text, features="lxml")
  74.                 map["name"] = soup.find("h1").get_text()
  75.                 map["price"] = soup.find("b", attrs={"class": "price-type"}).get_text()
  77.                 if self.type == "A":
  78.                     section = soup.find("div", attrs={"class": "section"}).find("div",
  79.                                                                                 attrs={"class": "section-content"})
  80.                     params = section.find_all("p")
  81.                     for param in params:
  82.                         key = param.find("span").get_text()[:-1].replace("\n", " ").replace("\r", " ")
  83.                         value = str(param.contents[1]).replace("\n", " ").replace("\r", " ")
  84.                         map[key] = value
  85.                 else:
  86.                     section = soup.find("div", attrs={"class": "section", "id": "proParamSection"}).find("div", attrs={
  87.                         "class": "section-content"})
  88.                     params = section.find_all("tr")
  89.                     for param in params:
  90.                         tds = param.find_all("td")
  91.                         key = str(tds[0].get_text()).replace("\n", " ").replace("\r", " ")
  92.                         value = str(tds[1].get_text()).replace("\n", " ").replace("\r", " ")
  93.                         map[key] = value
  94.                 print("单次耗时：{0}".format(time.time() - start))
  95.                 print(map)
  96.                 self.resultList.append(map)
  98.                 # self.resultList.append(map)
  99.             except Exception as e:
  100.                 print(Process_id, self.urlQueue.qsize(), url, "Error", e)
  102.     def run(self):
  103.         start = time.time()
  104.         N = mp.cpu_count()
  105.         pool = mp.Pool(processes=N)
  106.         for i in range(N):
  107.             pool.apply_async(self.job, args=(i,))
  109.         print('Started process')
  110.         pool.close()
  111.         pool.join()
  113.         end = time.time()
  114.         print('Pool + Queue :', end - start)
  115.         print('Main process Ended!')
  118. if __name__ == "__main__":
  119.     for key, value in products_A.items():
  120.         # 判断是否已经爬过了
  121.         file_path = "../load/{0}.csv".format(key)
  122.         is_exist = os.path.exists(file_path)
  123.         if is_exist:
  124.             print("{0}爬过了".format(key))
  125.             continue
  126.         # 获取所有产品的url
  127.         url_list = get_all_urls(value)
  128.         print("产品数量:{0}".format(len(url_list)))
  130.         crawler = MyCrawler(url_list, "A")
  131.         crawler.run()
  132.         # print("list:\n", crawler.resultList)
  133.         native_list = crawler.resultList[:]
  134.         df = pd.DataFrame(native_list)
  135.         df.to_csv(file_path)
  136.     for key, value in products_B.items():
  137.         # 判断是否已经爬过了
  138.         file_path = "../load/{0}.csv".format(key)
  139.         is_exist = os.path.exists(file_path)
  140.         if is_exist:
  141.             print("{0}爬过了".format(key))
  142.             continue
  143.         # 获取所有产品的url
  144.         url_list = get_all_urls(value)
  145.         print("产品数量:{0}".format(len(url_list)))
  147.         crawler = MyCrawler(url_list, "B")
  148.         crawler.run()
  149.         # print("list:\n", crawler.resultList)
  150.         native_list = crawler.resultList[:]
  151.         df = pd.DataFrame(native_list)
  152.         df.to_csv(file_path)

参考链接