1. # !/bin/env python
    2. # encoding=utf-8
    4. import re
    5. import requests
    6. from bs4 import BeautifulSoup as bs
    7. from time import sleep
    8. import lxml.etree
    9. import pandas as pd
    11. mn = []  # 电影名称
    12. mr = []  # 电影评分
    13. mp = []  # 电影评价人数
    14. mh = []  # 电影链接
    15. mtp = []  # 电影短评
    16. title_1 = []  # 评论1
    17. title_2 = []  # 评论2
    18. title_3 = []  # 评论3
    19. title_4 = []  # 评论4
    20. title_5 = []  # 评论5
    23. def get_url_name(url):
    24.     user_agent = 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.86 Safari/537.36'
    25.     header = {}
    26.     header['user-agent'] = user_agent
    27.     cookie = '''bid=ZNJjsDB1mu4; douban-fav-remind=1; __yadk_uid=EH2JTTZhVzZmeLKBWnLIsMyMuUKyCmlr; ll="108288"; push_noty_num=0; push_doumail_num=0; dbcl2="170609308:/CPcn6wSV8s"; ck=gCQ_; _pk_ref.100001.8cb4=%5B%22%22%2C%22%22%2C1583642177%2C%22https%3A%2F%2Faccounts.douban.com%2Fpassport%2Flogin%22%5D; _pk_ses.100001.8cb4=*; __utma=30149280.1482876592.1570801355.1583329616.1583642177.4; __utmc=30149280; __utmz=30149280.1583642177.4.3.utmcsr=accounts.douban.com|utmccn=(referral)|utmcmd=referral|utmcct=/passport/login; __utmt=1; __utmv=30149280.17060; douban-profile-remind=1; _pk_id.100001.8cb4=0032b1c289fde0dd.1570801350.3.1583642452.1570956509.; __gads=ID=356f8d35ed66b5f6:T=1583642453:S=ALNI_MbmKR7vLQRINc1rm5RCeVkIvGC60w; __utmb=30149280.8.10.1583642177'''
    28.     cookie_dic = {i.split("=")[0]: i.split("=")[-1] for i in cookie.split("; ")}
    29.     response = requests.get(url, headers=header,cookies=cookie_dic)
    30.     selector = lxml.etree.HTML(response.text)
    31.     movie_tag = selector.xpath('//ol[@class="grid_view"]')
    32.     # bs_info = bs(response.text,'html.parser')     #语法分析器,以html的方式进行分析
    33.     for data in movie_tag:
    34.         movie_name = data.xpath('//div[@class="hd"]//span[@class="title"][1]/text()')
    35.         for mn_info in movie_name:
    36.             mn.append(mn_info)
    37.         movie_rating = data.xpath('//div[@class="bd"]//div[@class="star"]//span[@property="v:average"]/text()')
    38.         # print(movie_rating)
    39.         for mr_info in movie_rating:
    40.             mr.append(mr_info)
    41.         movie_pingjia = data.xpath('//div[@class="star"]//span[4]/text()')
    42.         # print(movie_pingjia)
    43.         for mp_info in movie_pingjia:
    44.             mp.append(mp_info)
    45.         movie_hrefs = data.xpath('li//div[@class="item"]/div[@class="pic"]/a/@href')
    46.         for movie_href in movie_hrefs:
    47.             mh.append(movie_href.strip().replace('\n',''))
    48.             # print(mh)
    50.             response_2 = requests.get(movie_href, headers=header,cookies=cookie_dic)
    51.             selector_2 = lxml.etree.HTML(response_2.text)
    52.             comments = selector_2.xpath('//*[@id="hot-comments"]')
    53.             for topics in comments:
    54.                 # mtp.clear()
    55.                 topic = topics.xpath('div//div[@class="comment"]/p/span[@class="short"]/text()')
    57.                 # print(topic)
    58.                 # for aa in topic:
    59.                 mtp.append(tuple(topic))
    62. urls = tuple(f'https://movie.douban.com/top250?start={page * 25}&filter=' for page in range(10))
    64. if __name__ == '__main__':
    65.     for page in urls:
    66.         print(page)
    67.         get_url_name(page)
    68.         sleep(10)
    70.     dict_info ={}
    71.     for i in range(len(mn)):
    73.         title_1.append(mtp[i][0])
    74.         title_2.append(mtp[i][1])
    75.         title_3.append(mtp[i][2])
    76.         title_4.append(mtp[i][3])
    77.         title_5.append(mtp[i][4])
    79.         dict_info = {
    80.             '电影名称': mn,
    81.             '电影评分': mr,
    82.             '评价人数': mp,
    83.             '电影链接': mh,
    84.             '电影评价1': title_1,
    85.             '电影评价2': title_2,
    86.             '电影评价3': title_3,
    87.             '电影评价4': title_4,
    88.             '电影评价5': title_5
    89.         }
    90.     if dict_info:
    91.         print(dict_info)
    92.         book1 = pd.DataFrame(dict_info,
    93.                              columns=['电影名称',
    94.                                       '电影评分',
    95.                                       '评价人数',
    96.                                       '电影链接',
    97.                                       '电影评价1',
    98.                                       '电影评价2',
    99.                                       '电影评价3',
    100.                                       '电影评价4',
    101.                                       '电影评价5'])
    102.         book1.to_csv('./movie_info.csv', encoding='utf-8')