一 代理池服务器搭建

https://github.com/jhao104/proxy_pool

搭建好后，如下:
image.png
image.png

二 为scrapy项目添加代理中间件

1 编写中间件类

示例目录
image.png

随机UA中间件
  1. # proxies/user_agent.py
  2. import random
  5. class UA(object):
  6.     @classmethod
  7.     def random_ua(cls):
  8.         ua_list = [
  9.             'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.132 Safari/537.36',
  10.             'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/30.0.1599.101',
  11.             'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/38.0.2125.122',
  12.             'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.71',
  13.             'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95',
  14.             'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/21.0.1180.71',
  15.             'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; QQDownload 732; .NET4.0C; .NET4.0E)',
  16.             'Mozilla/5.0 (Windows NT 5.1; U; en; rv:1.8.1) Gecko/20061208 Firefox/2.0.0 Opera 9.50',
  17.             'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:34.0) Gecko/20100101 Firefox/34.0',
  18.         ]
  19.         return random.choice(ua_list)
  22. class RandomUserAgentMiddleware(object):
  23.     def __init__(self, crawler):
  24.         super(RandomUserAgentMiddleware, self).__init__()
  25.         self.ua = UA.random_ua()
  27.     @classmethod
  28.     def  from_crawler(cls, crawler):
  29.         return cls(crawler)
  31.     def process_requests(self, request, spider):
  32.         request.headers.setdefault('User-Agent', self.ua)

获取可用的代理IP
  1. # proxies/pool.py
  2. import time
  3. import requests
  5. from omim import settings
  6. from user_agent import UA
  9. class Pool(object):
  10.     proxy_url = settings.PROXY_URL
  12.     @classmethod
  13.     def get(cls):
  14.         url = cls.proxy_url + '/get/'
  15.         res = requests.get(url).json()
  16.         if not res:
  17.             print('proxy pool is empty, wait a minute ...')
  18.             time.sleep(60)
  19.             return cls.get()
  21.         ip = res['proxy']
  22.         return ip
  24.     @classmethod
  25.     def delete(cls, ip):
  26.         url = '{}/delete/?proxy={}'.format(cls.proxy_url, ip)
  27.         requests.get(url)
  29.     @classmethod
  30.     def check(cls, ip):
  31.         print('checking ip: {} ...'.format(ip))
  32.         url = 'https://www.omim.org/statistics/entry'
  33.         proxies = {'https': 'https://{}'.format(ip)}
  34.         headers = {'User-Agent':  UA.random_ua()}
  35.         try:
  36.             resp = requests.get(url, proxies=proxies, headers=headers, timeout=15)
  37.             if '<title>Error 403</title>' in resp.text:
  38.                 return False
  39.             return True
  40.         except Exception as e:
  41.             print e
  42.             return False
  44.     @classmethod
  45.     def get_good_proxy(cls):
  46.         ip = cls.get()
  47.         if cls.check(ip):
  48.             return 'https://{}'.format(ip)
  49.         else:
  50.             print('delete ip: {}'.format(ip))
  51.             cls.delete(ip)
  52.             return cls.get_good_proxy()

代理池中间件编写
  1. from pool import Pool
  2. from user_agent import UA
  4. class ProxyMiddleware(object):
  5.     def __init__(self):
  6.         pass
  8.     def process_request(self, request, spider):
  9.         proxy = Pool.get_good_proxy()
  10.         print '>>> crawling with proxy: {}'.format(proxy)
  11.         request.meta['proxy'] = proxy

2 开启中间件

  1. # setttings.py
  3. DOWNLOADER_MIDDLEWARES = {
  4. #    'omim.middlewares.OmimDownloaderMiddleware': 543,
  5.    'omim.proxies.ProxyMiddleware': 543,
  6.    'omim.proxies.user_agent.RandomUserAgentMiddleware': 542,
  7.    'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware': None,
  8. }

https://blog.csdn.net/jjjndk1314/article/details/80250146