1. # 需求：爬取三国演义小说所有的章节标题和章节内容http://www.shicimingju.com/book/sanguoyanyi.html
    2. if __name__ == "__main__":
    3.     # 对首页的页面数据进行爬取
    4.     headers = {
    5.         'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.121 Safari/537.36'
    6.     }
    7.     url = 'http://www.shicimingju.com/book/sanguoyanyi.html'
    8.     page_text = requests.get(url=url, headers=headers).text.encode('ISO-8859-1')
    10.     # 在首页中解析出章节的标题和详情页的url
    11.     # 1.实例化BeautifulSoup对象，需要将页面源码数据加载到该对象中
    12.     soup = BeautifulSoup(page_text, 'lxml')
    13.     # 解析章节标题和详情页的url
    14.     li_list = soup.select('.book-mulu > ul > li')
    15.     fp = open('./sanguo.txt', 'w', encoding='utf-8')
    16.     for li in li_list:
    17.         title = li.a.string
    18.         detail_url = 'http://www.shicimingju.com' + li.a['href']
    19.         # 对详情页发起请求，解析出章节内容
    20.         detail_page_text = requests.get(url=detail_url, headers=headers).text.encode('ISO-8859-1')
    21.         # 解析出详情页中相关的章节内容
    22.         detail_soup = BeautifulSoup(detail_page_text, 'lxml')
    23.         div_tag = detail_soup.find('div', class_='chapter_content')
    24.         # 解析到了章节的内容
    25.         content = div_tag.text
    26.         fp.write(title + ':' + content + '\n')
    27.         print(title, '爬取成功！！！')