Python 爬虫系统学习:
随机IP,随机UA,在下载中间件:DownloadMiddleware 里面定义 一下,然后在settings里面开启一下中间件就OK
class RandomUserAgentMiddleware(object): def process_request(self,request,spider): ua = random.choice(spider.settings.get("USER_AGENT_LIST")) request.headers["User-Agent"] = ua class ProxyMiddleware(object): def process_request(self,request,spider): ip = random.choice(spider.settings.get("IP_LIST")) request.meta["proxy"] = ip