一、页面分析：

1、需要在一级页面获取：章节大标题和大章节链接

  1. a_list = // li[contains(@id,''menu-item-")]/a
  2. for a in a_list:
  3.       a.xpath('./text()')
  4.       a.xpath('./@href')

2、在二级页面获取 小标题以及跳转到小说内容链接。

  1. a_list = //article/a
  2. for a in a_list:
  3.    a.xpath('./text()')
  4.    a.xpath('./@href')

3、在三级页面获取到小说具体内容。

  1. //article[@class="article-content"]/p/text()
  2. '\n'.join(['段落1','段落2'])拼接段落内容

二、编写代码

1、编写scrapy

  1. import scrapy
  2. from daomubiji.items import DaomubijiItem
  3. import os
  4. import re
  5. from copy import deepcopy
  6. class DaomubijispiderSpider(scrapy.Spider):
  7.     name = 'daomubijiSpider'
  8.     allowed_domains = ['daomubiji.com']
  9.     start_urls = ['http://daomubiji.com/']
  11.     def parse(self, response):
  12.         # 解析一级页面
  13.         a_list = response.xpath('//li[contains(@id,"menu-item-")]/a')
  14.         for a in a_list:
  15.             item = DaomubijiItem()
  16.             # 大标题
  17.             item['first_title'] = a.xpath('./text()').get()
  18.             first_url = a.xpath('./@href').get()
  19.             new_second_title = re.sub(r'[\\\/\:\*\?\"\<\>\|]', '_', item['first_title'])
  20.             dir_path = './novel/{}/'.format(new_second_title)
  21.             # 先判断有没有文件 没有就创建
  22.             if not os.path.exists(dir_path):
  23.                 os.makedirs(dir_path)
  24.             yield scrapy.Request(url = first_url,meta={'item':deepcopy(item)},callback=self.parse_second)
  26.     # 二级页面的解析
  27.     def parse_second(self,response):
  28.         item = response.meta.get('item')
  29.         a_list = response.xpath('//article/a')
  30.         for a in a_list:
  31.             # 小标题
  32.             item['second_title'] = a.xpath('./text()').get()
  33.             # # 跳转到三级页面的url
  34.             second_url = a.xpath('./@href').get()
  35.             yield scrapy.Request(url =second_url,meta={'item':deepcopy(item)},callback=self.parse_content)
  37.     # 解析三级页面
  38.     def parse_content(self,response):
  39.         item = response.meta.get('item')
  40.         content_list = response.xpath('//article[@class="article-content"]/p/text()').getall()
  41.         item['content'] = '\n'.join(content_list)
  42.         yield item

2、编写items.py

  1. class DaomubijiItem(scrapy.Item):
  2.     first_title = scrapy.Field()
  3.     second_title = scrapy.Field()
  4.     content = scrapy.Field()

3、编写管道文件

  1. import re
  3. class DaomubijiPipeline:
  4.     def process_item(self, item, spider):
  5.         filename = './novel/{}/{}.txt'.format(
  6.             re.sub(r'[\\\/\:\*\?\"\<\>\|]', '_', item['first_title']),
  7.             re.sub(r'[\\\/\:\*\?\"\<\>\|]', '_', item['second_title'])
  8.         )
  9.         print('正在写入：', filename)
  10.         with open(filename, 'w', encoding='utf-8') as file_obj:
  11.             file_obj.write(item['content'])
  12.         return item

4、编写settings.py文件

  1. DEFAULT_REQUEST_HEADERS = {
  2.   'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
  3.   'Accept-Language': 'en',
  4.   'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.74 Safari/537.36'
  5. }
  6. ITEM_PIPELINES = {
  7.    'daomubiji.pipelines.DaomubijiPipeline': 300,
  8. }
  9. LOG_LEVEL = 'WARNING'
  10. ROBOTSTXT_OBEY = False  # 是否遵守此协议