一、说明

生产者消费者模式链接

二、代码

  1. # -*- coding:utf-8 -*-
  2. import requests
  3. from lxml import etree
  4. import threading
  5. from queue import Queue
  6. import os
  9. class Producer_mulu(threading.Thread):
  11.     def __init__(self, mulu_queue, zhangjie_queue, *args, **kwargs):
  12.         super(Producer_mulu, self).__init__(*args, **kwargs)
  13.         self.mulu_queue = mulu_queue
  14.         self.zhangjie_queue = zhangjie_queue
  16.     def run(self):
  17.         while True:
  18.             if self.mulu_queue.empty():
  19.                 print("所有目录path都已生产完成，生产者停止")
  20.                 break
  21.             xuanhuan_url = self.mulu_queue.get()
  22.             self.get_mulu(xuanhuan_url)
  24.     def get_mulu(self, xuanhuan_url):
  25.         req_text = requests.get(xuanhuan_url).content.decode('utf-8')
  26.         text_html = etree.HTML(req_text)
  27.         file_name = text_html.xpath('//div[@id="newscontent"]/div[1]/ul/li/span[1]/a/text()')
  28.         mulu_url = text_html.xpath('//div[@id="newscontent"]/div[1]/ul/li/span[1]/a/@href')
  29.         mulu_path = list(zip(file_name, mulu_url))
  30.         for xiaoshuo_name, url in mulu_path:
  31.             # self.zhangjie_queue.put((xiaoshuo_name,url))
  32.             # print(self.zhangjie_queue.get())
  33.             self.zhangjie_queue.put(url)
  36. class Producer_zhangjie(threading.Thread):
  38.     def __init__(self, mulu_queue, zhangjie_queue, content_queue, *args, **kwargs):
  39.         super(Producer_zhangjie, self).__init__(*args, **kwargs)
  40.         self.mulu_queue = mulu_queue
  41.         self.zhangjie_queue = zhangjie_queue
  42.         self.content_queue = content_queue
  44.     def run(self):
  45.         while True:
  46.             if self.mulu_queue.empty() and self.zhangjie_queue.empty():
  47.                 print("所有章节目录都已生产完成，生产者停止")
  48.                 break
  49.             mulu_url = self.zhangjie_queue.get()
  50.             self.get_zj_path(mulu_url)
  52.     # 获取章节path
  53.     def get_zj_path(self, mulu_url):
  54.         base_url = 'http://www.xbiquge.la/'
  55.         req_mulu = requests.get(mulu_url).content.decode('utf-8')
  56.         mulu_html = etree.HTML(req_mulu)
  57.         base_path_list = mulu_html.xpath("//div[@class='box_con'][2]/div[@id='list']/dl/dd/a/@href")
  58.         zhang_name_list = mulu_html.xpath("//div[@class='box_con'][2]/div[@id='list']/dl/dd/a/text()")
  59.         mulu_path_list = [base_url + x for x in base_path_list]
  60.         zj_path = list(zip(zhang_name_list, mulu_path_list))
  61.         for zhangjie_name, content_url in zj_path:
  62.             self.content_queue.put((zhangjie_name, content_url))
  65. class Consumers_content(threading.Thread):
  67.     def __init__(self, mulu_queue, zhangjie_queue, content_queue, *args, **kwargs):
  68.         super(Consumers_content, self).__init__(*args, **kwargs)
  69.         self.mulu_queue = mulu_queue
  70.         self.zhangjie_queue = zhangjie_queue
  71.         self.content_queue = content_queue
  73.     def run(self):
  74.         while True:
  75.             if self.mulu_queue.empty() and self.zhangjie_queue.empty() and self.content_queue.empty():
  76.                 break
  77.             zhangjie_name, content_url = self.content_queue.get()
  78.             self.get_text(content_url, zhangjie_name)
  80.     # 获取内容
  81.     def get_text(self, text_url, text_name):
  82.         req_text = requests.get(text_url).content.decode('utf-8')
  83.         text_html = etree.HTML(req_text)
  84.         xiaoshuo_name = text_html.xpath("//div[@class='box_con']/div[@class='con_top']/a[3]/text()")[0]
  85.         data = text_html.xpath("//div[@class='box_con']/div[@id='content']/text()")
  86.         data = (str(data).replace('\\xa0', '')).replace("\\r',", '').replace('\'', '').replace('”', '').replace('“', '')
  87.         dirs = '.\\xiaoshuo\\{}'.format(xiaoshuo_name)
  88.         if not os.path.exists(dirs):
  89.             os.makedirs(dirs)
  90.         with open(r'.\xiaoshuo\{0}\{1}'.format(xiaoshuo_name, text_name) + '.txt', 'w', encoding='utf-8') as f:
  91.             f.write(text_name)
  92.             f.write('\n')
  93.             f.write(data[1:-1])
  94.             f.write('\n')
  95.             print("《" + xiaoshuo_name + "》" + text_name + "下载成功")
  98. if __name__ == '__main__':
  99.     mulu_Queue = Queue(1000000)
  100.     zhangjie_Queue = Queue(1000000)
  101.     content_Queue = Queue(1000000)
  102.     for i in range(2, 100):
  103.         xh_url = "http://www.xbiquge.la/fenlei/1_{}.html".format(i)
  104.         mulu_Queue.put(xh_url)
  105.     for i in range(10):
  106.         mulu = Producer_mulu(mulu_Queue, zhangjie_Queue)
  107.         mulu.start()
  108.     for i in range(10):
  109.         zhangjie = Producer_zhangjie(mulu_Queue, zhangjie_Queue, content_Queue)
  110.         zhangjie.start()
  111.     for i in range(100):
  112.         content = Consumers_content(mulu_Queue, zhangjie_Queue, content_Queue)
  113.         content.start()

三、总结

流程说明

使用threading.Thread创建两个生产者，一个消费者；
Producer_mulu生产玄幻小说分类下的所有小说“href”，
Consumers_content生产每个小说的章节名称和章节URL，
Consumers_content使用Consumers_content生产的内容生成小说内容并分类保存

zip的使用

  1. mulu_path = list(zip(file_name, mulu_url))
  2. for zhangjie_name, content_url in zj_path:
  1. a = [1,2,3]
  2. b = [4,5,6]
  3. c = [4,5,6,7,8]
  4. >>> zipped = zip(a,b)     # 打包为元组的列表
  5. [(1, 4), (2, 5), (3, 6)]
  6. >>> zip(a,c)              # 元素个数与最短的列表一致
  7. [(1, 4), (2, 5), (3, 6)]
  8. >>> zip(*zipped)          # 与 zip 相反，*zipped 可理解为解压，返回二维矩阵式
  9. [(1, 2, 3), (4, 5, 6)]

相关操作

  1. a_list = ["a1", "a2", "a3"]
  2. b_list = ["b1", "b2", "b3"]
  3. c_list = ["c1", "c2", "c3"] # 元素个数与最短的列表一致
  4. lis = list(map(list, zip(a_list, b_list, c_list)))
  5. lisb = list(zip(a_list, b_list, c_list)) # 打包为元组的列表
  6. new_list = []
  7. for i in lis:
  8.     new_dict = {'A': i[0], 'B': i[1], 'C': i[2]}
  9.     new_list.append(new_dict)
  11. print(lis)
  12. print(lisb)
  13. print(new_list)
  14. for a, b, c in lis:
  15.     print(a, b, c)
  17. lis  [['a1', 'b1', 'c1'], ['a2', 'b2', 'c2'], ['a3', 'b3', 'c3']]
  18. lisb [('a1', 'b1', 'c1'), ('a2', 'b2', 'c2'), ('a3', 'b3', 'c3')]
  19. new_list [{'A': 'a1', 'B': 'b1', 'C': 'c1'}, {'A': 'a2', 'B': 'b2', 'C': 'c2'}, {'A': 'a3', 'B': 'b3', 'C': 'c3'}]
  20. a1 b1 c1
  21. a2 b2 c2
  22. a3 b3 c3

lxml

  1. req_mulu = requests.get(mulu_url).content.decode('utf-8')
  2. mulu_html = etree.HTML(req_mulu)
  3. base_path_list = mulu_html.xpath("//div[@class='box_con'][2]/div[@id='list']/dl/dd/a/@href")

Queue

  1. for i in range(2, 100):
  2.         xh_url = "http://www.xbiquge.la/fenlei/1_{}.html".format(i)
  3.         mulu_Queue.put(xh_url)
  4.     for i in range(10):
  5.         mulu = Producer_mulu(mulu_Queue, zhangjie_Queue)
  6.         mulu.start()
  7.     for i in range(10):
  8.         zhangjie = Producer_zhangjie(mulu_Queue, zhangjie_Queue, content_Queue)
  9.         zhangjie.start()
  10.     for i in range(100):
  11.         content = Consumers_content(mulu_Queue, zhangjie_Queue, content_Queue)
  12.         content.start()

将xh_url即翻页url，循环后存入mulu_Queue队列中
mulu_Queue中的url在Producer_mulu中取出，并生成zhangjie_Queue队列；Producer_zhangjie生产content_Queue队列
Consumers_content中mulu_Queue，zhangjie_Queue，content_Queue全为空后，循环停止