1、继承scrapy_redis类并修改

  1. import scrapy
  2. from daomu.items import DaomuItem
  3. import os
  4. import re
  5. from copy import deepcopy
  6. from scrapy_redis.spiders import RedisSpider  # 导入类包
  9. class DmSpider(RedisSpider): # 这里改成继承RedisSpider
  10.     name = 'dm'
  11.     allowed_domains = ['daomubiji.com']
  12.     # start_urls = ['http://daomubiji.com/']  # 删除start_urls
  13.     redis_key = 'dmbj'    # 增加redis_key
  15.     def parse(self, response):
  16.         # 解析一级页面
  17.         a_list = response.xpath('//li[contains(@id, "menu-item-")]/a')
  18.         for a in a_list:
  19.             item = DaomuItem()
  20.             # 大标题
  21.             item['first_title'] = a.xpath('./text()').get()
  22.             # 跳转到二级页面的url
  23.             first_url = a.xpath('./@href').get()
  24.             # print(item, first_url)
  26.             # 创建文件夹
  27.             new_second_title = re.sub(r'[\\\/\:\*\?\"\<\>\|]', '_', item['first_title'])
  28.             dir_path = './novel/{}/'.format(new_second_title)
  29.             # 先判断有没有文件 没有就创建
  30.             if not os.path.exists(dir_path):
  31.                 os.makedirs(dir_path)
  33.             yield scrapy.Request(
  34.                 url=first_url,
  35.                 meta={'item': deepcopy(item)},
  36.                 callback=self.parse_second
  37.             )
  39.     # 二级页面解析
  40.     def parse_second(self, response):
  41.         item = response.meta.get('item')
  42.         a_list = response.xpath('//article/a')
  43.         for a in a_list:
  44.             # 小标题
  45.             item['second_title'] = a.xpath('./text()').get()
  46.             # 跳转到三级页面的url
  47.             second_url = a.xpath('./@href').get()
  48.             # print(item, second_url)
  49.             yield scrapy.Request(
  50.                 url=second_url,
  51.                 meta={'item': deepcopy(item)},
  52.                 callback=self.parse_content
  53.             )
  55.     # 解析三级页面
  56.     def parse_content(self, response):
  57.         item = response.meta.get('item')
  58.         content_list = response.xpath('//article[@class="article-content"]/p/text()').getall()
  59.         item['content'] = '\n'.join(content_list)
  60.         yield item

2、items.py文件

  1. class DaomuItem(scrapy.Item):
  2.     # define the fields for your item here like:
  3.     # name = scrapy.Field()
  4.     first_title = scrapy.Field()
  5.     second_title = scrapy.Field()
  6.     content = scrapy.Field()
  7.     pass

3、管道文件 pipeline.py

  2. class DaomuPipeline:
  3.     def process_item(self, item, spider):
  5.         filename = './novel/{}/{}.txt'.format(
  6.             re.sub(r'[\\\/\:\*\?\"\<\>\|]', '_', item['first_title']),
  7.             re.sub(r'[\\\/\:\*\?\"\<\>\|]', '_', item['second_title'])
  8.         )
  9.         print('正在写入：', filename)
  10.         with open(filename, 'w', encoding='utf-8') as file_obj:
  11.             file_obj.write(item['content'])
  12.         return item

4、修改配置文件settings.py

  1. # 需要改
  2. USER_AGENT = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.74 Safari/537.36'
  3. # 指定去重方式 给请求对象去重
  4. DUPEFILTER_CLASS = "scrapy_redis.dupefilter.RFPDupeFilter"
  5. # 设置调度器
  6. SCHEDULER = "scrapy_redis.scheduler.Scheduler"
  7. # 队列中的内容是否进行持久保留 True redis关闭的时候数据会保留
  8. # False 不会保留
  9. SCHEDULER_PERSIST = True
  10. ITEM_PIPELINES = {
  11.    'daomu.pipelines.DaomuPipeline': 300,
  12.     # 将数据保存到redis中
  13.     'scrapy_redis.pipelines.RedisPipeline': 400,
  14. }
  16. DEFAULT_REQUEST_HEADERS = {
  17.     'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
  18.     'Accept-Language': 'en',
  19.     'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.74 Safari/537.36'
  20. }