3. import requests
    4. from bs4 import BeautifulSoup
    5. import re
    6. import time
    7. import random
    8. import get_auto_id
    11. def crawler_program(url):
    12.     res = {}
    13.     results = []
    14.     headers = [{'User-Agent': "Mozilla/5.0 (X11; CrOS x86_64 10066.0.0) AppleWebKit/537.36 (KHTML, like Gecko)"
    15.                               " Chrome/69.0.3497.100 Safari/537.36",
    16.                 'Accept': 'text/html;q=0.9,*/*;q=0.8',
    17.                 'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.3',
    18.                 'Connection': 'close'},
    19.                {'User-Agent': "Mozilla/5.0 (iPhone; CPU iPhone OS 9_1 like Mac OS X) AppleWebKit/601.1 (KHTML,"
    20.                               " like Gecko) CriOS/69.0.3497.100 Mobile/13B143 Safari/601.1.4",
    21.                 'Accept': 'text/html;q=0.9,*/*;q=0.8',
    22.                 'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.3',
    23.                 'Connection': 'close'},
    24.                {'User-Agent': "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_3) AppleWebKit/537.75.14 (KHTML, "
    25.                               "like Gecko) Version/7.0.3 Safari/7046A194A",
    26.                 'Accept': 'text/html;q=0.9,*/*;q=0,8',
    27.                 'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.3',
    28.                 'Connection': 'close'},
    29.                {'User-Agent': "Mozilla/5.0 (Windows NT 10.0; WOW64; rv:46.0) Gecko/20100101 Firefox/46.0",
    30.                 'Accept': 'application/json, text/plain, */*',
    31.                 'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.3',
    32.                 'Connection': 'close'},
    33.                {'User-Agent': "Mozilla/5.0 (Windows NT 10.0; WOW64; Trident/7.0; rv:11.0) like Gecko",
    34.                 'Accept': 'application/json, text/plain, */*',
    35.                 'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.3',
    36.                 'Connection': 'close'}]
    38.     s = requests.Session()
    39.     s.keep_alive = False
    40.     html = s.get(url, headers=random.choice(headers), allow_redirects=False).text
    41.     if html != '':
    42.         bs = BeautifulSoup(html, "html.parser")
    43.         time.sleep(0.2)
    45.         try:
    46.             obj = bs.find("div", id="maodian").find_all("div", class_="mouthcon")  ###
    47.             if not obj:
    48.                 raise AttributeError
    49.             else:
    50.                 for i in obj:
    51.                     date = i.find("div", class_="mouthcon-cont-right commentParentBox").find("div", class_="title-name name-width-01").find("a").get_text()
    52.                     try:
    53.                         name = re.findall('[.a-zA-Z0-9_\u4e00-\u9fa5]+', i.find("div", class_="mouthcon-cont-left").find("div", class_="name-text").get_text())[0]
    54.                     except IndexError:
    55.                         name = "用户名含有特殊字符"
    57.                     # print(name)
    58.                     res["用户名"] = [name]
    59.                     res["发帖日期"] = [date]
    61.                     object_text = i.find("div", class_="choose-con mt-10").find_all("dl", class_="choose-dl")
    63.                     for ii in object_text:
    64.                         try:
    65.                             title = re.findall('[.a-zA-Z0-9_\u4e00-\u9fa5]+', ii.find("dt").get_text())[0]
    66.                         except IndexError:
    67.                             title = ""
    68.                         # print(title)
    69.                         content = re.findall('[.a-zA-Z0-9_\u4e00-\u9fa5]+', ii.find("dd").get_text())
    70.                         res[title] = content
    72.                     # print(res)
    73.                     results.append(res.copy())
    74.                     res.clear()
    75.                 s.close()
    76.                 return results
    78.         except AttributeError:
    79.             try:
    80.                 obj = bs.find("div", id="maodian").find("div", class_="mouth")
    81.                 text = re.findall('[.a-zA-Z0-9_\u4e00-\u9fa5]+', obj.find("div", class_="text-normal").get_text())[0]
    82.                 if text == "暂无符合该列表的口碑":
    83.                     results.append(text)
    84.                     print(text)
    85.                     s.close()
    86.                     return results
    87.                 else:
    88.                     results.append("未知错误")
    89.                     print("未知错误")
    90.                     return results
    91.             except AttributeError:
    92.                 print("AttributeError")
    93.                 time.sleep(5)
    94.                 s.close()
    95.                 return crawler_program(url)
    96.     elif html == '':
    97.         results.append("该车型不存在在售状态")
    98.         print("该车型不存在在售状态")
    99.         s.close()
    100.         return results
    103. def gen_url(auto_id, sell_state):
    104.     url_lists = []
    105.     s = requests.Session()
    106.     try:
    107.         if sell_state == "在售":
    108.             urls = "https://k.autohome.com.cn/" + auto_id + "/index_2.html#dataList"
    109.             html = s.get(urls, allow_redirects=False).text
    110.             if html == '':
    111.                 raise AttributeError
    112.             else:
    113.                 bs = BeautifulSoup(html, "html.parser")
    114.                 page_num = int(bs.find("span", class_="page-item-info").get_text()[1:-1])
    115.                 for ai in range(2, page_num + 1):
    116.                     url_lists.append("https://k.autohome.com.cn/" + auto_id + "/index_" + str(ai) + ".html#dataList")
    117.                 url_lists.insert(0, "https://k.autohome.com.cn/" + auto_id + "/#pvareaid=102519")
    119.         elif sell_state == "停售":
    120.             urls = "https://k.autohome.com.cn/" + auto_id + "/StopSelling/index_2.html#dataList"
    121.             html = s.get(urls, allow_redirects=False).text
    122.             if html == '':
    123.                 raise AttributeError
    124.             else:
    125.                 bs = BeautifulSoup(html, "html.parser")
    126.                 page_num = int(bs.find("span", class_="page-item-info").get_text()[1:-1])
    127.                 for ai in range(2, page_num + 1):
    128.                     url_lists.append("https://k.autohome.com.cn/" + auto_id + "/StopSelling/index_" + str(ai) + ".html#dataList")
    129.                 url_lists.insert(0, "https://k.autohome.com.cn/" + auto_id + "/StopSelling/#pvareaid=102519")
    130.         s.close()
    131.         return url_lists
    133.     except AttributeError:
    134.         if sell_state == "在售":
    135.             url_lists.append("https://k.autohome.com.cn/" + auto_id + "/#pvareaid=102519")
    136.         elif sell_state == "停售":
    137.             url_lists.append("https://k.autohome.com.cn/" + auto_id + "/StopSelling/#pvareaid=102519")
    138.         return url_lists
    141. def data_clear(res):
    142.     result = []
    143.     contain = []
    145.     for xi in res:
    146.         for xii in ["车型代码", "销售状态", "用户名", "发帖日期", "购买车型", "购买地点", "购车经销商", "购买时间", "裸车购买价", "油耗目前行驶", "耗电量目前行驶", "空间", "动力", "操控", "油耗", "舒适性", "外观", "内饰", "性价比", "购车目的"]:
    147.             if xii == "车型代码":
    148.                 contain.append(xi[xii][0])
    150.             if xii == "销售状态":
    151.                 contain.append(xi[xii][0])
    153.             if xii in ["用户名", "发帖日期"]:
    154.                 contain.append(xi[xii][0])
    156.             if xii == "购买车型":
    157.                 chexing = xi[xii][0]
    158.                 xinghao = " ".join(xi[xii][1:])
    159.                 contain.append(chexing)
    160.                 contain.append(xinghao)
    162.             try:
    163.                 if xii == "购买地点":
    164.                     contain.append(xi[xii][0])
    165.             except IndexError:
    166.                 contain.append("")
    168.             try:
    169.                 if xii == "购车经销商":
    170.                     contain.append("")
    171.             except KeyError or IndexError:
    172.                 contain.append("")
    174.             try:
    175.                 if xii == "购买时间":
    176.                     contain.append(xi[xii][0])
    177.             except KeyError or IndexError:
    178.                 contain.append("")
    180.             if xii == "裸车购买价":
    181.                 contain.append("".join(xi[xii]))
    183.             try:
    184.                 if xii == "油耗目前行驶":
    185.                     contain.append("".join(xi[xii][:2]))
    186.                     contain.append("".join(xi[xii][2:])[3:])
    187.             except KeyError or IndexError:
    188.                 contain.append("")
    189.                 contain.append("")
    191.             try:
    192.                 if xii == "耗电量目前行驶":
    193.                     contain.append("".join(xi[xii][:2]))
    194.                     contain.append("".join(xi[xii][2:])[3:])
    195.             except KeyError or IndexError:
    196.                 contain.append("")
    197.                 contain.append("")
    199.             try:
    200.                 if xii in ["空间", "动力", "操控", "油耗", "舒适性", "外观", "内饰", "性价比"]:
    201.                     contain.append(xi[xii][0])
    202.             except KeyError:
    203.                 if xii in ["空间", "动力", "操控", "耗电量", "舒适性", "外观", "内饰", "性价比"]:
    204.                     contain.append(xi[xii][0])
    206.             if xii == "购车目的":
    207.                 contain.append(" ".join(xi[xii]))
    209.         print(contain)
    210.         result.append(contain.copy())
    211.         contain.clear()
    212.     return result
    215. def get_auto_series_data(series_id, sell_state):
    216.     url_list = gen_url(series_id, sell_state)
    217.     print(url_list)
    218.     rr = []
    219.     for i in url_list:
    220.         r = crawler_program(i)
    221.         if isinstance(r[0], dict):
    222.             for ii in r:
    223.                 ii["车型代码"] = [series_id]
    224.                 ii["销售状态"] = [sell_state]
    225.             r_clear = data_clear(r)
    226.             rr.extend(r_clear)
    227.         else:
    228.             rr.append(series_id)
    229.             rr.append(r[0])
    230.     return rr
    233. auto_id_list = get_auto_id.get_auto_id_data()  # 获取auto的Factory id，总计308个
    234. auto_series_list = get_auto_id.get_specific_model_auto(auto_id_list)  # 获取auto不同series的id,总计2226个
    237. final = []
    239. for s in auto_series_list[500:700]:
    240.     print(s)
    241.     tt = get_auto_series_data(str(s[1]), "在售")
    242.     final.append(tt)
    2. import requests
    3. import requests.exceptions
    4. import time
    5. import random
    6. import json
    9. def read_csv(name):
    10.     '''该函数读取CSV文件数据'''
    11.     import csv
    12.     csv_file = csv.reader(open("C:\\Users\\mengxin\\Desktop\\" + name + ".csv", "r", encoding="utf8"))
    13.     object_website = []
    14.     for i in csv_file:
    15.         object_website.append(i[0])
    16.         # print(i)
    17.     return object_website
    20. def get_auto_id(brand_id):
    21.     url = "https://k.autohome.com.cn/ajax/factorybybrand?&brandid=" + brand_id + "&state=0X001C&typeid=0"
    23.     headers = [{'User-Agent': "Mozilla/5.0 (X11; CrOS x86_64 10066.0.0) AppleWebKit/537.36 (KHTML, like Gecko)"
    24.                               " Chrome/69.0.3497.100 Safari/537.36",
    25.                 'Accept': 'text/html;q=0.9,*/*;q=0.8',
    26.                 'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.3',
    27.                 'Connection': 'close'},
    28.                {'User-Agent': "Mozilla/5.0 (iPhone; CPU iPhone OS 9_1 like Mac OS X) AppleWebKit/601.1 (KHTML,"
    29.                               " like Gecko) CriOS/69.0.3497.100 Mobile/13B143 Safari/601.1.4",
    30.                 'Accept': 'text/html;q=0.9,*/*;q=0.8',
    31.                 'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.3',
    32.                 'Connection': 'close'},
    33.                {'User-Agent': "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_3) AppleWebKit/537.75.14 (KHTML, "
    34.                               "like Gecko) Version/7.0.3 Safari/7046A194A",
    35.                 'Accept': 'text/html;q=0.9,*/*;q=0,8',
    36.                 'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.3',
    37.                 'Connection': 'close'},
    38.                {'User-Agent': "Mozilla/5.0 (Windows NT 10.0; WOW64; rv:46.0) Gecko/20100101 Firefox/46.0",
    39.                 'Accept': 'application/json, text/plain, */*',
    40.                 'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.3',
    41.                 'Connection': 'close'},
    42.                {'User-Agent': "Mozilla/5.0 (Windows NT 10.0; WOW64; Trident/7.0; rv:11.0) like Gecko",
    43.                 'Accept': 'application/json, text/plain, */*',
    44.                 'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.3',
    45.                 'Connection': 'close'}]
    47.     # 代理服务器
    48.     proxyHost = "http-dyn.abuyun.com"
    49.     proxyPort = "9020"
    51.     # 代理隧道验证信息
    52.     proxyUser = "H21HT1P7C30F1P5D"
    53.     proxyPass = "8F451A3BD7C939D0"
    55.     proxyMeta = "http://%(user)s:%(pass)s@%(host)s:%(port)s" % {
    56.       "host": proxyHost,
    57.       "port": proxyPort,
    58.       "user": proxyUser,
    59.       "pass": proxyPass,
    60.     }
    62.     proxies = {
    63.         "http": proxyMeta,
    64.         "https": proxyMeta,
    65.     }
    67.     try:
    68.         s = requests.Session()
    69.         js = s.get(url, headers=random.choice(headers), allow_redirects=False).json()
    70.         js = json.loads(js)
    72.         time.sleep(0.2)
    74.         result = []
    76.         if js["message"] == "成功":
    77.             res = js["result"]["factoryitems"]
    79.             for i in res:
    80.                 name = i["name"]
    81.                 idn = i["id"]
    82.                 first_letter = i["ffirstletter"]
    84.                 result.append([name, idn, first_letter])
    86.         s.close()
    87.         return result
    89.     except requests.exceptions.ProxyError:
    90.         print("ProxyError")
    91.         time.sleep(2)
    92.         return get_auto_id(brand_id)
    93.     except json.decoder.JSONDecodeError:
    94.         print("JSONDecodeError")
    95.         time.sleep(2)
    96.         return get_auto_id(brand_id)
    100. def get_auto_id_data():
    102.     brand_id_list = read_csv("brand_id_list")
    104.     obj_dict = {}
    105.     brand_id_list[0] = "117"
    106.     for k in brand_id_list:
    107.         get_auto_id_list = get_auto_id(k)
    108.         print(get_auto_id_list)
    109.         obj_dict[k] = get_auto_id_list
    111.     auto_id_list = []
    112.     for m in obj_dict:
    113.         brand = obj_dict[m]
    114.         for n in brand:
    115.             n.insert(0, m)
    116.             auto_id_list.append(n)
    118.     return auto_id_list
    121. def get_specific_model_auto(auto_id_list):
    122.     ress = []
    123.     results = []
    124.     for d in auto_id_list:
    125.         brand_id = d[0]
    126.         factory_id = d[2]
    128.         try:
    129.             url = "https://k.autohome.com.cn/ajax/seriesbyfactory?&brandid=" + brand_id + "&factoryid=" + str(factory_id) + "&state=0X001C&typeid=0"
    130.             s = requests.Session()
    131.             js = s.get(url, allow_redirects=False).json()
    132.             js = json.loads(js)
    133.             if js["message"] == "成功":
    134.                 res = js["result"]["seriesitems"]
    136.                 for ii in res[:]:
    137.                     # print(ii)
    138.                     name = ii["name"]
    139.                     idn = ii["id"]
    140.                     sfirst_letter = ii["sfirstletter"]
    141.                     state = ii["seriesstate"]
    142.                     order = ii["seriesorder"]
    144.                     results.append([name, idn, sfirst_letter, state, order])
    145.                     print(results)
    147.                     ress.extend(results.copy())
    148.                     results.clear()
    150.         except requests.exceptions.ProxyError:
    151.             print("ProxyError")
    152.             time.sleep(3)
    153.             return get_specific_model_auto(auto_id_list)
    154.         except json.decoder.JSONDecodeError:
    155.             print("JSONDecodeError")
    156.             time.sleep(3)
    157.             return get_specific_model_auto(auto_id_list)
    158.         s.close()
    159.     return ress