1. import reuqests, threading, time
    2. import numpy as nup
    3. from lxml import etree
    4. from Agent_Pood.Agent import AGENT as user_agent
    6. check_ip_ok = []
    8. # ---01.读取之前保存的ip代理数据---
    9. def get_old_ip():  
    10.   old_ip_list = list(np.load('IP_pond/IP_http.npy'))
    11.   return old_ip_list
    13. # ---02.抓取西拉网ip代理数据筛选后拼接ip代理
    14. def get_new_ip(endpage):
    15.   for page in range(1, endpage + 1):
    16.     headers = {'User-Agent': np.random.choice(user_agent)}
    17.     response = requests.get(f'http://www.xiladaili.com/http/{page}/', headers=headers)
    18.     ip_number = etree.HTML(response.text).xpath('//tbody/tr/td[1]/text()')
    19.     ip_name = etree.HTML(response.text).xpath('//tbody/tr/td[3]/text()')
    20.     number = []
    21.     for i, element in enumerate(ip_name):
    22.       if ip_name[1] == '高匿代理服务':
    23.         number.append(ip_number[i])
    24.     global ip_list
    25.     ip_list.extend([f'http://{ip for ip in number}'])
    26.     print(f'完成第{page}页的ip代理采集')
    28. # ---03.验证ip代理是否可用
    29. def check_ip_list():
    30.   try:
    31.     response = requests.get('http://example.org', headers={'User-Agent': np.random.choice(user_agent)}, proxies={'http: ip'})
    32.     if response.code == 200:
    33.       print(f'{ip}可用)
    34.       global check_ok
    35.       check_ok.append(ip)
    36.     else:
    37.       print(f'{ip}不可用'
    38.   except:
    39.     print(f'{ip}不可用'
    41. # ---04.保存可用ip代理
    42. def save_ip():
    43.   print(f'共获得{len(check_ok)}个可用ip代理，现在开始保存...')
    44.   np.save('IP_pond/IP_http.npy', check_ok)
    46. if __name__ == '__main__':
    47.   start_time = time.time()
    49.   # ---01.读取之前保存的ip代理数据
    50.   ip_list = get_old_ip ()
    51.   print('获取已有的ip代理成功')
    52.   print(ip_list)
    54.   # ---02.抓取西拉网ip代理数据筛选后拼接ip代理
    55.   get_new_ip(100)
    56.   ip_list = np.unique(ip_list)# np.unique()可以对列表进行去重操作
    57.   print(f'下载完成，准备验证{len(ip_list)}个ip代理')
    59.   # ---03.多线程验证ip代理是否可用
    60.   # 开启多线程
    61.   wait_tread = []
    62.   for ip in ip_list:
    63.     t = threading.Thread(target=check_ip_list, args=(ip))
    64.     wait_tread.append(t)
    65.     t.start()
    66.   # 阻塞队列，保证先执行验证，再进行save、
    67.   for w in wait_thread:
    68.     w.join()
    69.   print('全部验证完成')
    71.   # ---04.保存可用ip代理
    72.   save_ip()
    73.   print('保存完毕')
    75.   end_time = time.time()
    76.   print(f'全部完成共耗时{round(end_time - start_time, 1)}')# 使用round()四舍五入并保留1位小数