3. import requests
    4. from bs4 import BeautifulSoup
    5. import random
    6. import time
    9. def get_data(school_num, l, z):
    10.     location = {1: "北京", 2: "天津", 3: "辽宁", 4: "吉林", 5: "黑龙江", 6: "上海", 7: "江苏", 8: "浙江", 9: "安徽",
    11.                 10: "福建", 11: "山东", 12: "湖北", 13: "湖南", 14: "广东", 15: "重庆", 16: "四川", 17: "陕西", 18: "甘肃",
    12.                 19: "河北", 20: "山西", 21: "内蒙古", 22: "河南", 23: "海南", 24: "广西", 25: "贵州", 26: "云南", 27: "西藏", 28: "青海",
    13.                 29: "宁夏", 30: "新疆", 31: "江西", 33: "香港", 38: "澳门", 39: "台湾"}
    15.     cat = {1: "理科", 2: "文科", 3: "综合", 4: "艺术理", 5: "艺术文"}
    17.     headers = {
    18.         "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.113 Safari/537.36",
    19.         "Accept": "text/javascript, application/javascript, application/ecmascript, application/x-ecmascript, */*; q=0.01",
    20.         "Accept-Encoding": "gzip,deflate",
    21.         "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
    22.         "Content-Encoding": "gzip",
    23.         "Server": "nginx",
    24.         "Vary": "Accept-Encoding",
    25.         "Connection": "close",
    26.     }
    28.     proxyHost = "http-dyn.abuyun.com"
    29.     proxyPort = "9020"
    31.     # 代理隧道验证信息
    32.     proxyUser = "HX7YIG4D7IR9907D"
    33.     proxyPass = "D9A0153CCBC8081F"
    35.     proxyMeta = "http://%(user)s:%(pass)s@%(host)s:%(port)s" % {
    36.       "host": proxyHost,
    37.       "port": proxyPort,
    38.       "user": proxyUser,
    39.       "pass": proxyPass,
    40.     }
    42.     proxies = {
    43.         "http": proxyMeta,
    44.         "https": proxyMeta,
    45.     }
    47.     result = []
    48.     contain = []
    49.     s = requests.Session()
    50.     s.keep_alive = False
    51.     # for l in [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 33, 38, 39]:
    52.     #     for z in [1, 2, 3, 4, 5]:
    53.     url = "http://college.gaokao.com/school/tinfo/" + str(school_num) + "/result/" + str(l) + "/" + str(z) + "/"
    54.     try:
    55.         html = s.get(url, headers=headers, proxies=proxies, allow_redirects=False).text
    56.         time.sleep(0.1)
    57.         bs = BeautifulSoup(html, 'lxml')
    58.         try:
    59.             bs_school_name = bs.find("div", class_="bg_sez").find('h2').get_text()
    60.             print("收集 { %d } %d %s %s %s 数据" % (school_num, l, bs_school_name, location[l], cat[z]))
    61.             try:
    62.                 bs_parser = bs.find("div", id="pointbyarea").find_all("tr", class_=True)
    63.                 for i in bs_parser:
    64.                     res = i.find_all("td")
    65.                     contain.append(bs_school_name)
    66.                     contain.append(location[l])
    67.                     contain.append(cat[z])
    68.                     for m in res:
    69.                         data = m.get_text()
    70.                         contain.append(data)
    71.                     # print(contain)
    72.                     lis = contain.copy()
    73.                     result.append(lis)
    74.                     contain.clear()
    75.                     # s.close()
    76.                     # print(result)
    77.             except AttributeError:
    78.                 try:
    79.                     bs_parser = bs.find("div", class_="ts").find("h3").get_text()
    80.                     contain.append(bs_school_name)
    81.                     contain.append(location[l])
    82.                     contain.append(cat[z])
    83.                     contain.extend([bs_parser, bs_parser, bs_parser, bs_parser, bs_parser, bs_parser])
    84.                     lis_1 = contain.copy()
    85.                     result.append(lis_1)
    86.                     # print(contain)
    87.                     contain.clear()
    88.                     # return result
    89.                 except AttributeError:
    90.                     bs_parser = bs.find("div", class_="ts lq").find("h3").get_text()
    91.                     contain.append(bs_school_name)
    92.                     contain.append(location[l])
    93.                     contain.append(cat[z])
    94.                     contain.extend([bs_parser, bs_parser, bs_parser, bs_parser, bs_parser, bs_parser])
    95.                     lis_1 = contain.copy()
    96.                     result.append(lis_1)
    97.                     # s.close()
    98.                     # print(contain)
    99.                     contain.clear()
    100.         except AttributeError:
    101.             print("'NoneType' object has no attribute 'find'")
    102.             return get_data(school_num, l, z)
    103.     except requests.exceptions.ProxyError:
    104.         print("ProxyError!")
    105.         return get_data(school_num, l, z)
    106.         # contain.append(school_num)
    107.         # contain.append(location[l])
    108.         # contain.append(cat[z])
    109.         # contain.extend(["代理异常！", "代理异常！", "代理异常！", "代理异常！", "代理异常！", "代理异常！"])
    110.         # lis_1 = contain.copy()
    111.         # result.append(lis_1)
    112.         # contain.clear()
    113.     except requests.exceptions.ChunkedEncodingError:
    114.         print("ChunkedEncodingError!")
    115.         return get_data(school_num, l, z)
    116.     s.close()
    117.     return result
    120. # start = time.time()
    121. # test_2 = get_data(school_num=1)
    122. # end = time.time()
    123. # print("用时：%s" % (end - start))
    126. # location = {1: "北京", 2: "天津", 3: "辽宁", 4: "吉林", 5: "黑龙江", 6: "上海", 7: "江苏", 8: "浙江", 9: "安徽",
    127. #             10: "福建", 11: "山东", 12: "湖北", 13: "湖南", 14: "广东", 15: "重庆", 16: "四川", 17: "陕西", 18: "甘肃",
    128. #             19: "河北", 20: "山西", 21: "内蒙古", 22: "河南", 23: "海南", 24: "广西", 25: "贵州", 26: "云南", 27: "西藏", 28: "青海",
    129. #             29: "宁夏", 30: "新疆", 31: "江西", 33: "香港", 38: "澳门", 39: "台湾"}
    130. #
    131. # cat = {1: "理科", 2: "文科", 3: "综合", 4: "艺术理", 5: "艺术文"}
    134. final = []
    135. #
    136. start = time.time()
    137. for q in range(1021, 1031):    # 2668
    138.     for w in [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 33, 38, 39]:
    139.         for ze in [1, 2, 3, 4, 5]:
    140.             res_uid = get_data(q, w, ze)
    141.             final.extend(res_uid)
    142.             # time.sleep(0.5 + random.random())
    143. end = time.time()
    144. print("总共耗时：%f" % (end - start))