1. #coding:utf-8
    2. import requests
    3. from lxml import etree
    6. class Tieba(object):
    8.     def __init__(self, name):
    9.         self.name = name
    10.         self.url = 'http://tieba.baidu.com/f?kw={}'.format(self.name)
    11.         self.headers = {
    12.             'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.86 Safari/537.36'
    13.             # 'User-Agent': 'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1) '
    14.         }
    16.     def get_data(self, url):
    17.         response = requests.get(url,headers=self.headers)
    18.         return response.content
    20.     def parse_list_page(self, data):
    21.         """
    22.             解析贴吧帖子列表页面的响应
    23.         :param data: 帖子列表页面的响应
    24.         :return: 帖子标题和链接列表 与 下一页url
    25.         """
    26.         data = data.decode().replace("<!--","").replace("-->","")
    27.         html = etree.HTML(data)
    29.         el_list = html.xpath('//li[@class=" j_thread_list clearfix"]/div/div[2]/div[1]/div[1]/a')
    30.         # print(len(el_list))
    31.         data_list = []
    32.         for el in el_list:
    33.             temp = {}
    35.             temp['title'] = el.xpath('./text()')[0]
    36.             temp['link'] = 'http://tieba.baidu.com' + el.xpath('./@href')[0]
    37.             data_list.append(temp)
    38.         try:
    39.             next_url = 'http:' + html.xpath('//*[contains(text(),"下一页>")]/@href')[0]
    40.         except:
    41.             next_url = None
    43.         return data_list, next_url
    45.     def parse_detail_page(self, data):
    47.         html = etree.HTML(data)
    49.         return html.xpath('//*[contains(@id,"post_content_")]/img/@src')
    51.     def run(self):
    52.         # url
    53.         # headers
    54.         next_url = self.url
    55.         while True:
    56.             # 发送列表请求，获取响应
    57.             list_page_data = self.get_data(next_url)
    59.             # 解析列表页面的响应，提取帖子列表数据和下一页url
    60.             data_list, next_url = self.parse_list_page(list_page_data)
    62.             # 遍历帖子列表，获取每一个详细url
    63.             for data in data_list:
    64.                 # 发起请求，获取到详情页面的响应
    65.                 data = self.get_data(data['link'])
    66.                 # 从响应中提取图片地址
    67.                 image_list = self.parse_detail_page(data)
    68.                 # 保存
    69.                 print(image_list)
    70.             # 翻页&循环终止条件
    71.             if next_url == None:
    72.                 break
    75. if __name__ == '__main__':
    76.     tieba = Tieba("传智播客")
    77.     tieba.run()