1. # http://www.17989.com/xiaohua/5.htm
    2. # 标题和正文
    3. from lxml import etree
    4. import requests
    5. from fake_useragent import UserAgent
    6. import csv
    8. # 获取网页源代码
    9. def download(url):
    10.     headers = {
    11.         'User-Agent':UserAgent().random
    12.     }
    13.     html = requests.get(url=url,headers=headers)
    14.     html.encoding='utf-=8'
    15.     return html.text
    17. # 获取内容
    18. def get_data(html):
    19.     xml = etree.HTML(html)
    20.     li_list = xml.xpath('//div[@class="module articlelist"]/ul/li')
    21.     j = 1
    22.     for li in li_list:
    23.         item = {}
    24.         title = li.xpath('.//div[@class="hd"]/text()')[0]
    25.         content = li.xpath('.//pre/text()')
    26.         contentData = ' '.join([i.strip() for i in content])
    27.         item['title'] = title
    28.         item['content'] = contentData
    29.         data_list.append(item)
    30.         j += 1
    31.     return data_list
    33. def saveData():
    34.     headers = ['title','content']
    35.     with open('xiaohua.csv', 'w', encoding='utf-8-sig', newline='') as file:
    36.         dwriter = csv.DictWriter(file,headers)
    37.         dwriter.writeheader()
    38.         dwriter.writerows(data_list)
    40. # 获取翻页
    41. def get_next_page(html):
    42.     xml = etree.HTML(html)
    43.     href = xml.xpath('//div[@class="page"]/a[text()="下一页"]/@href')[0]
    44.     return "http://www.17989.com" + href
    46. if __name__ == '__main__':
    47.     url = 'http://www.17989.com/xiaohua/1.htm'
    48.     i = 1
    49.     data_list = []
    50.     while i < 100:
    51.         html = download(url)
    52.         get_data(html)
    53.         url = get_next_page(html)
    54.         i += 1
    55.         if not url:
    56.             break
    57.     saveData()