【爬虫】随机获取UA

使用模块  fake-useragent

https://github.com/hellysmile/fake-useragent

1.安装模块

2.配置

# settings.py

'''下载器中间件'''
DOWNLOADER_MIDDLEWARES = {
   'Lagou.middlewares.RandomUserAgentMiddleware': 543,
    'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware': None,
}

'''UA的类型'''
RANDOM_UA_TYPE = "random"
# middlewares.py
'''
模仿middlewares的UserAgentMiddleware写的类''' class RandomUserAgentMiddleware(object): """This middleware allows spiders to override the user_agent""" def __init__(self, crawler): # 实例化UserAgent(),从配置文件读取ua的类型 super().__init__() self.ua = UserAgent() self.ua_type = crawler.settings.get("RANDOM_UA_TYPE","random") @classmethod def from_crawler(cls, crawler): return cls(crawler) # def spider_opened(self, spider): # self.user_agent = getattr(spider, 'user_agent', self.user_agent) def process_request(self, request, spider): def get_ua(): # 通过反射获取随机UA random_ua = getattr(self.ua,self.ua_type) return random_ua request.headers.setdefault("User-Agent", get_ua())
原文地址:https://www.cnblogs.com/st-st/p/10444764.html