# -*- coding: utf-8 -*-
# 在这里定义蜘蛛中间件的模型
# Define here the models for your spider middleware
#
# See documentation in:
# https://doc.scrapy.org/en/latest/topics/spider-middleware.html
from scrapy import signals
# ===========================Spider Middleware============================
# 定义：介于Scrapy引擎和爬虫之间的框架，主要工作是处理蜘蛛的响应输入和请求输出。
# Spider Middleware功能：处理爬虫的请求输入和响应输出
# scrapy已经提供了一些直接使用的中间件，他被SPIDER_MIDDLEWARES_BASE定义：
# {
# 'scrapy.spidermiddlewares.httperror.HttpErrorMiddleware': 50,
# 'scrapy.spidermiddlewares.offsite.OffsiteMiddleware': 500,
# 'scrapy.spidermiddlewares.referer.RefererMiddleware': 700,
# 'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware': 800,
# 'scrapy.spidermiddlewares.depth.DepthMiddleware': 900,
# }
# =================SpiderMiddleware类==================
class MaoyanSpiderMiddleware(object):
@classmethod
# 类方法，参数crawler，可以通过crawler调用settings里的全局参数
def from_crawler(cls, crawler):
"""
:param crawler: 获取settings里的全局参数，如crawler.settings.get(参数)
"""
s = cls()
# 调用spider_opened函数进行爬取数据并对该函数发送该信号。该信号一般用来分配spider的资源
crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
# 调用spider_closed函数进行关闭爬虫并对该函数发送该信号。该信号用来释放spider在spider_opened时占用的资源。
# crawler.signals.connect(s.spider_closed, signal=signals.spider_closed)
return s
# 当返回来的response被Spider Middleware处理时，该方法被调用
def process_spider_input(self, response, spider):
"""
:param response: 被Spider Middleware处理的response对象
:param spider: 返回response对应的spider对象
"""
return None
# 当spider处理response对象的结果后，该方法被调用
def process_spider_output(self, response, result, spider):
"""
:param response: 被spider处理后得到结果的response对象
:param result: result包含Item或request对象的可迭代对象，即spider返回的response结果
:param spider: 返回response对象的spider对象
"""
# 遍历返回的可迭代对象
for i in result:
yield i
# 当spider的process_spider_input和process_spider_output发生异常时调用该方法
def process_spider_exception(self, response, exception, spider):
"""
:param response: 异常被抛出时被处理的response对象
:param exception: 抛出的异常
:param spider: 抛出该异常的spider对象
"""
pass
# 以spider启动的request为参数调用该方法,返回一个request可迭代对象
def process_start_requests(self, start_requests, spider):
"""
:param start_requests: 开始请求的可迭代对象
:param spider: 开始请求所对应的spider对象
"""
# 遍历可迭代对象
for r in start_requests:
yield r
# 当spider开启时调用该函数，说明开始爬取数据并分配spider的资源
def spider_opened(self, spider):
"""
:param spider: 开始爬取的spider对象
"""
spider.logger.info('Spider opened: %s' % spider.name)
# # 当某个spider被关闭时，说明关闭该爬虫并释放spider在spider_opened时占用的资源。
# def spider_closed(self, spider):
# """
# :param spider: 开始爬取的spider对象
# """
# spider.logger.info('Spider opened:%s'%spider.name)
# ======================Downloader Middleware========================
# 定义：位于Scrapy引擎和下载器之间的框架，主要是处理Scrapy引擎与下载器之间的请求及响应。见scrapy框架图
# Downloader Middleware功能：可以修改User-Agent、处理重定向、设置代理、失败重试、设置Cookies等
# scrapy已经提供了一些直接使用的中间件，他被DOWNLOADER_MIDDLEWARES_BASE定义：
# {
# 'scrapy.contrib.downloadermiddleware.robotstxt.RobotsTxtMiddleware': 100,
# 'scrapy.contrib.downloadermiddleware.httpauth.HttpAuthMiddleware': 300,
# 'scrapy.contrib.downloadermiddleware.downloadtimeout.DownloadTimeoutMiddleware': 350,
# 'scrapy.contrib.downloadermiddleware.useragent.UserAgentMiddleware': 400,
# 'scrapy.contrib.downloadermiddleware.retry.RetryMiddleware': 500,
# 'scrapy.contrib.downloadermiddleware.defaultheaders.DefaultHeadersMiddleware': 550,
# 'scrapy.contrib.downloadermiddleware.redirect.MetaRefreshMiddleware': 580,
# 'scrapy.contrib.downloadermiddleware.httpcompression.HttpCompressionMiddleware': 590,
# 'scrapy.contrib.downloadermiddleware.redirect.RedirectMiddleware': 600,
# 'scrapy.contrib.downloadermiddleware.cookies.CookiesMiddleware': 700,
# 'scrapy.contrib.downloadermiddleware.httpproxy.HttpProxyMiddleware': 750,
# 'scrapy.contrib.downloadermiddleware.chunked.ChunkedTransferMiddleware': 830,
# 'scrapy.contrib.downloadermiddleware.stats.DownloaderStats': 850,
# 'scrapy.contrib.downloadermiddleware.httpcache.HttpCacheMiddleware': 900,
# }
# ===============DownloaderMiddleware类=================
class MaoyanDownloaderMiddleware(object):
@classmethod
# 类方法，参数crawler，可以通过crawler调用settings里的全局参数
def from_crawler(cls, crawler):
"""
:param crawler: 获取settings里的全局参数，如crawler.settings.get(参数)
"""
s = cls()
# 调用spider_opened函数进行爬取数据并对该函数发送该信号。该信号一般用来分配spider的资源
crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
# 调用spider_closed函数进行关闭爬虫并对该函数发送该信号。该信号用来释放spider在spider_opened时占用的资源。
# crawler.signals.connect(s.spider_closed, signal=signals.spider_closed)
return s
# request被scrapy从调度器调度给Downloader Middleware之前调用该方法对request对象进行处理
def process_request(self, request, spider):
"""
:param request: 就是scrapy从调度器调度出来的request对象
:param spider: 就是scrapy调度出来的request对象的spider对象
"""
return None
# request对象被Downloader Middleware执行后返回response是才调用该方法对response对象进行处理
def process_response(self, request, response, spider):
"""
:param request: 调度出来被Downloader Middleware处理的request对象
:param response: Downloader Middleware处理request对象返回后的response对象
:param spider: response返回来的spider对象
"""
return response
# 当process_request和process_response发生异常时调用
def process_exception(self, request, exception, spider):
"""
:param request: 产生异常的request对象
:param exception: 抛出的异常对象
:param spider: 产生异常的request对象的spider对象
"""
pass
# 当spider开启时调用该函数，说明开始爬取数据并分配spider的资源
def spider_opened(self, spider):
"""
:param spider: 开始爬取的spider对象
"""
spider.logger.info('Spider opened: %s' % spider.name)
# # 当某个spider被关闭时，说明关闭该爬虫并释放spider在spider_opened时占用的资源。
# def spider_closed(self, spider):
# """
# :param spider: 开始爬取的spider对象
# """
# spider.logger.info('Spider opened: %s' % spider.name)