1. import csv
    3. from lxml import etree
    4. from fake_useragent import UserAgent
    5. import random
    6. import time
    7. import requests
    9. class IP(object):
    10.     def __init__(self):
    11.         self.url = 'http://www.xiladaili.com/gaoni/{}/'
    13.     def get_html(self, url):
    14.         headers = {'User-Agent': UserAgent().random}
    15.         html = requests.get(url=url, headers=headers).text
    16.         return html
    18.     def xpath_func(self, html, xpath_bds):
    19.         obj = etree.HTML(html)
    20.         ip_lists = obj.xpath(xpath_bds)
    21.         return ip_lists
    23.     def parse_html(self, url):
    24.         html = self.get_html(url)
    25.         xpath_bds = '//tr/td/text()'
    26.         ip_lists = self.xpath_func(html, xpath_bds)
    27.         print(ip_lists)
    29.         for i in range(0, len(ip_lists), 8):
    30.             # 检查
    31.             try:
    32.                 self.fangwen(ip_lists[i])
    33.                 self.save_ip(ip_lists[i])
    34.                 print("可用")
    35.             except Exception:
    36.                 print("此ip不可用")
    37.                 pass
    38.             # if self.fangwen(ip_lists[i]) == 200:
    39.             #     self.save_ip(ip_lists[i])
    40.             # else:
    41.             #     print('此ip不可用')
    43.     def save_ip(self, L):
    44.         with open('ip.txt', 'a', encoding='utf-8', newline='') as f:
    45.             f.write(L)
    46.             f.write('\r\n')
    47.             print('save success')
    49.     def fangwen(self, ip):
    50.         proxies = {
    51.             'http': 'http://{}'.format(ip),
    52.             'https': 'https://{}'.format(ip)
    53.         }
    54.         headers = {'User-Agent': UserAgent().random}
    55.         res = requests.get('http://www.baidu.com', proxies=proxies, headers=headers, timeout=5)
    56.         code = res.status_code
    57.         print(code)
    58.         return code
    61.     def run(self):
    62.         for i in range(1, 500):
    63.             url = self.url.format(i)
    64.             self.parse_html(url)
    65.             time.sleep(random.randint(1, 10))
    68. if __name__ == '__main__':   # 1, 8, 15
    69.     spider = IP()
    70.     spider.run()