1. # http://www.17989.com/xiaohua/5.htm
    2. # 标题和正文
    3. from lxml import etree
    4. import requests
    5. from fake_useragent import UserAgent
    6. # 获取网页源代码
    7. def download(url):
    8.     headers = {
    9.         'User-Agent':UserAgent().random
    10.     }
    11.     html = requests.get(url=url,headers=headers)
    12.     html.encoding='utf-=8'
    13.     return html.text
    15. # 获取内容
    16. def get_data(html):
    17.     xml = etree.HTML(html)
    18.     li_list = xml.xpath('//div[@class="module articlelist"]/ul/li')
    19.     j = 1
    20.     for li in li_list:
    21.         title = li.xpath('.//div[@class="hd"]/text()')[0]
    22.         content = li.xpath('.//pre/text()')
    23.         print(f'第{i}页、第{j}条——标题【{title}】 内容为:{content}')
    24.         data_list.append({'title':title,'content':content})
    25.         j += 1
    26.     return data_list
    29. # 获取翻页
    30. def get_next_page(html):
    31.     xml = etree.HTML(html)
    32.     href = xml.xpath('//div[@class="page"]/a[text()="下一页"]/@href')[0]
    33.     return "http://www.17989.com" + href
    35. if __name__ == '__main__':
    36.     url = 'http://www.17989.com/xiaohua/1.htm'
    37.     i = 1
    38.     data_list = []
    39.     while i < 10:
    40.         html = download(url)
    41.         get_data(html)
    42.         url = get_next_page(html)
    43.         i += 1
    44.         if not url:
    45.             break
    46.     print(data_list)