模板一

  1. # -*- coding: utf-8 -*-
  2. import time
  3. from queue import Queue
  4. from threading import Thread
  7. # 适合一层网站解析
  8. class Demo(Thread):
  9.     def __init__(self, queue):
  10.         Thread.__init__(self)
  11.         self.queue = queue
  13.     def run(self):
  14.         while True:
  15.             q_url = self.queue.get()
  16.             try:
  17.                 self.parse(q_url)
  18.             finally:
  19.                 self.queue.task_done()
  21.     def parse(self, q_url):
  22.         print("开始解析链接:", q_url)
  25. if __name__ == '__main__':
  26.     start = time.time()
  27.     # 构造链接
  28.     base_url = 'https://www.baidu.com/{}.html'
  29.     url_list = [base_url.format(i) for i in range(1, 201)]
  31.     queue = Queue()
  33.     # 创建线程
  34.     for x in range(20):
  35.         worker = Demo(queue)
  36.         worker.daemon = True
  37.         worker.start()
  39.     for url in url_list:
  40.         queue.put(url)
  42.     queue.join()
  44.     print('下载完毕耗时:{}s'.format(round(time.time() - start, 2)))

模板二

  1. import queue
  2. import threading
  5. # 适合两层网站访问
  6. # 解析线程类
  7. class Parse(threading.Thread):
  8.     def __init__(self, number, data_list, req_thread):
  9.         super(Parse, self).__init__()
  10.         self.number = number
  11.         self.data_list = data_list
  12.         self.req_thread = req_thread
  13.         self.is_parse = True  # 判断是否从数据队列里提取数据
  15.     def run(self):
  16.         print('启动%d号解析线程' % self.number)
  17.         while True:
  18.             # 如何判断解析线程的结束条件
  19.             for t in self.req_thread:
  20.                 if t.is_alive():
  21.                     break
  22.             else:
  23.                 if self.data_list.qsize() == 0:
  24.                     self.is_parse = False
  26.             if self.is_parse:  # 解析
  27.                 try:
  28.                     data = self.data_list.get(timeout=3)
  29.                 except Exception as e:
  30.                     data = None
  31.                 if data is not None:
  32.                     self.parse(data)
  33.             else:
  34.                 break
  35.         print('退出%d号解析线程' % self.number)
  37.     # 页面解析函数
  38.     def parse(self, data):
  39.         # 下载文件
  40.         pass
  43. # 采集线程类
  44. class Crawl(threading.Thread):
  45.     def __init__(self, number, req_list, data_list):
  46.         super(Crawl, self).__init__()
  47.         self.number = number
  48.         self.req_list = req_list
  49.         self.data_list = data_list
  50.         self.headers = {
  51.             'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.89 Safari/537.36'
  52.         }
  54.     def run(self):
  55.         print('启动采集线程%d号' % self.number)
  56.         while self.req_list.qsize() > 0:
  57.             url = self.req_list.get()
  58.             print('%d号线程采集：%s' % (self.number, url))
  59.             # time.sleep(random.randint(1, 3))
  61.             self.data_list.put("填充详细页面链接")  # 向数据队列里追加
  64. def main():
  65.     concurrent = 3
  66.     conparse = 3
  68.     # 生成请求队列
  69.     req_list = queue.Queue()
  70.     # 生成数据队列
  71.     data_list = queue.Queue()
  73.     # 填充请求数据
  74.     for i in range(1, 13 + 1):
  75.         base_url = 'https://www.qiushibaike.com/8hr/page/%d/' % i
  76.         req_list.put(base_url)
  78.     # 生成N个采集线程
  79.     req_thread = []
  80.     for i in range(concurrent):
  81.         t = Crawl(i + 1, req_list, data_list)  # 创造线程
  82.         t.start()
  83.         req_thread.append(t)
  85.     # 生成N个解析线程
  86.     parse_thread = []
  87.     for i in range(conparse):
  88.         t = Parse(i + 1, data_list, req_thread)  # 创造解析线程
  89.         t.start()
  90.         parse_thread.append(t)
  92.     for t in req_thread:
  93.         t.join()
  94.     for t in parse_thread:
  95.         t.join()
  98. if __name__ == '__main__':
  99.     main()