1. import logging
    3. import time
    7. from scrapy.dupefilters import BaseDupeFilter
    9. from scrapy.utils.request import request_fingerprint
    13. from .connection import get_redis_from_settings
    19. DEFAULT_DUPEFILTER_KEY = "dupefilter:%(timestamp)s"
    23. logger = logging.getLogger(__name__)
    29. # TODO: Rename class to RedisDupeFilter.
    31. class RFPDupeFilter(BaseDupeFilter):
    33.     """Redis-based request duplicates filter.
    35.     This class can also be used with default Scrapy's scheduler.
    37.     """
    41.     logger = logger
    45.     def __init__(self, server, key, debug=False):
    47.         """Initialize the duplicates filter.
    49.         Parameters
    51.         ----------
    53.         server : redis.StrictRedis
    55.             The redis server instance.
    57.         key : str
    59.             Redis key Where to store fingerprints.
    61.         debug : bool, optional
    63.             Whether to log filtered requests.
    65.         """
    67.         self.server = server
    69.         self.key = key
    71.         self.debug = debug
    73.         self.logdupes = True
    77.     @classmethod
    79.     def from_settings(cls, settings):
    81.         """Returns an instance from given settings.
    83.         This uses by default the key ``dupefilter:<timestamp>``. When using the
    85.         ``scrapy_redis.scheduler.Scheduler`` class, this method is not used as
    87.         it needs to pass the spider name in the key.
    89.         Parameters
    91.         ----------
    93.         settings : scrapy.settings.Settings
    95.         Returns
    97.         -------
    99.         RFPDupeFilter
    101.             A RFPDupeFilter instance.
    103.         """
    105.         server = get_redis_from_settings(settings)
    107.         # XXX: This creates one-time key. needed to support to use this
    109.         # class as standalone dupefilter with scrapy's default scheduler
    111.         # if scrapy passes spider on open() method this wouldn't be needed
    113.         # TODO: Use SCRAPY_JOB env as default and fallback to timestamp.
    115.         key = DEFAULT_DUPEFILTER_KEY % {'timestamp': int(time.time())}
    117.         debug = settings.getbool('DUPEFILTER_DEBUG')
    119.         return cls(server, key=key, debug=debug)
    123.     @classmethod
    125.     def from_crawler(cls, crawler):
    127.         """Returns instance from crawler.
    129.         Parameters
    131.         ----------
    133.         crawler : scrapy.crawler.Crawler
    135.         Returns
    137.         -------
    139.         RFPDupeFilter
    141.             Instance of RFPDupeFilter.
    143.         """
    145.         return cls.from_settings(crawler.settings)
    149.     def request_seen(self, request):
    151.         """Returns True if request was already seen.
    153.         Parameters
    155.         ----------
    157.         request : scrapy.http.Request
    159.         Returns
    161.         -------
    163.         bool
    165.         """
    167.         fp = self.request_fingerprint(request)
    169.         # This returns the number of values added, zero if already exists.
    171.         added = self.server.sadd(self.key, fp)
    173.         return added == 0
    177.     def request_fingerprint(self, request):
    179.         """Returns a fingerprint for a given request.
    181.         Parameters
    183.         ----------
    185.         request : scrapy.http.Request
    187.         Returns
    189.         -------
    191.         str
    193.         """
    195.         return request_fingerprint(request)
    199.     def close(self, reason=''):
    201.         """Delete data on close. Called by Scrapy's scheduler.
    203.         Parameters
    205.         ----------
    207.         reason : str, optional
    209.         """
    211.         self.clear()
    215.     def clear(self):
    217.         """Clears fingerprints data."""
    219.         self.server.delete(self.key)
    223.     def log(self, request, spider):
    225.         """Logs given request.
    227.         Parameters
    229.         ----------
    231.         request : scrapy.http.Request
    233.         spider : scrapy.spiders.Spider
    235.         """
    237.         if self.debug:
    239.             msg = "Filtered duplicate request: %(request)s"
    241.             self.logger.debug(msg, {'request': request}, extra={'spider': spider})
    243.         elif self.logdupes:
    245.             msg = ("Filtered duplicate request %(request)s"
    247.                    " - no more duplicates will be shown"
    249.                    " (see DUPEFILTER_DEBUG to show all duplicates)")
    251.             msg = "Filtered duplicate request: %(request)s"
    253.             self.logger.debug(msg, {'request': request}, extra={'spider': spider})
    255.             self.logdupes = False