1. class Spider(object_ref):
    3.     #定义spider名字的字符串(string)。spider的名字定义了Scrapy如何定位(并初始化)spider，
    4.     #所以其必须是唯一的。
    5.     #name是spider最重要的属性，而且是必须的。
    6.     #一般做法是以该网站(domain)(加或不加 后缀 )来命名spider。 
    7.     #例如，如果spider爬取 mywebsite.com ，该spider通常会被命名为 mywebsite
    10.     name = None
    11.     custom_settings = None
    13.     # 初始化爬虫名字，及请求连接
    14.     def __init__(self, name=None, **kwargs):
    15.         # 判断是否有爬虫名字，如果爬虫没有名字，中断后续操作则报错
    16.         if name is not None:
    17.             self.name = name
    18.         elif not getattr(self, 'name', None):
    19.             raise ValueError("%s must have a name" % type(self).__name__)
    21.         # python对象或类型通过内置成员__dict__来存储成员信息
    22.         self.__dict__.update(kwargs)
    24.         # URL列表。当没有指定的URL时，spider将从该列表中开始进行爬取。 
    25.         # 因此，第一个被获取到的页面的URL将是该列表之一。
    26.         # 后续的URL将会从获取到的数据中提取。
    27.         if not hasattr(self, 'start_urls'):
    28.             self.start_urls = []
    31.     @property
    32.     def logger(self):
    33.         logger = logging.getLogger(self.name)
    34.         return logging.LoggerAdapter(logger, {'spider': self})
    36.      # 打印Scrapy执行后的log信息
    37.     def log(self, message, level=logging.DEBUG, **kw):
    38.         """Log the given message at the given log level
    40.         This helper wraps a log call to the logger within the spider, but you
    41.         can use it directly (e.g. Spider.logger.info('msg')) or use any other
    42.         Python logger too.
    43.         """
    44.         self.logger.log(level, message, **kw)
    46.     @classmethod
    47.     def from_crawler(cls, crawler, *args, **kwargs):
    48.         spider = cls(*args, **kwargs)
    49.         spider._set_crawler(crawler)
    50.         return spider
    52.     def _set_crawler(self, crawler):
    53.         self.crawler = crawler
    54.         self.settings = crawler.settings
    55.         crawler.signals.connect(self.close, signals.spider_closed)
    57.     # 该方法将读取start_urls内的地址，
    58.     # 并为每一个地址生成一个Request对象，交给Scrapy下载并返回Response
    59.     # 该方法仅调用一次
    60.     def start_requests(self):
    61.         cls = self.__class__
    62.         if method_is_overridden(cls, Spider, 'make_requests_from_url'):
    63.             warnings.warn(
    64.                 "Spider.make_requests_from_url method is deprecated; it "
    65.                 "won't be called in future Scrapy releases. Please "
    66.                 "override Spider.start_requests method instead (see %s.%s)." % (
    67.                     cls.__module__, cls.__name__
    68.                 ),
    69.             )
    70.             for url in self.start_urls:
    71.                 yield self.make_requests_from_url(url)
    72.         else:
    73.             for url in self.start_urls:
    74.                 yield Request(url, dont_filter=True)
    76.     def make_requests_from_url(self, url):
    77.         """ This method is deprecated. """
    78.         return Request(url, dont_filter=True)
    80.     # 默认的Request对象回调函数，处理返回的response。
    81.     # 生成Item或者Request对象。用户必须实现这个类
    82.     def parse(self, response):
    83.         raise NotImplementedError('{}.parse callback is not defined'.format(self.__class__.__name__))
    85.     @classmethod
    86.     def update_settings(cls, settings):
    87.         settings.setdict(cls.custom_settings or {}, priority='spider')
    89.     @classmethod
    90.     def handles_request(cls, request):
    91.         return url_is_from_spider(request.url, cls)
    93.     @staticmethod
    94.     def close(spider, reason):
    95.         closed = getattr(spider, 'closed', None)
    96.         if callable(closed):
    97.             return closed(reason)
    99.     def __str__(self):
    100.         return "<%s %r at 0x%0x>" % (type(self).__name__, self.name, id(self))
    102.     __repr__ = __str__
    105. # Top-level imports
    106. from scrapy.spiders.crawl import CrawlSpider, Rule
    107. from scrapy.spiders.feed import XMLFeedSpider, CSVFeedSpider
    108. from scrapy.spiders.sitemap import SitemapSpider