企查查网站中汇聚了有关注册企业的详细信息，为了更好的查询企业相关信息，本人对网站中安徽省境内的企业进行了爬取，其中遇到的问题和使用的技术如下：
1、遇到的问题：
1>企查查PC版数据只显示前500页，为了尽可能最大化爬取网站数据，本次爬取按照市级分别爬取，共计爬取安徽省境内16个市区共计80000条企业信息；
2>在爬取网站数据时，若爬取速度过快，会出现手动验证功能，为了解决手动验证，同时为了避免封号，直接采用随机更换IP代理，IP代理可以在《89免费代理》网站获取免费代理账号，网址为：http://www.89ip.cn/，可以一次性获取30个代理IP，如果不够用，
可以多次提取，然后构建代理池，本人试了，该网站的免费代理比西次代理和快代理网站的免费代理要好很多，如下图：
2、使用的技术：
1>请求模块：requests请求，为了避免反爬，采用随机代理，同时使用fake_useragent随机产生user-agent；
2>解析库：使用xpath和正则表达式
3>提速优化：采用多线程，同时对爬取的数据进行一次性保存，避免磁盘频繁IO；
3、核心代码如下：
import requests from lxml import etree from queue import Queue from threading import Thread from fake_useragent import UserAgent import csv import os import re import random import time from ippools import ProxySpider from proxy_ip import IP_LIST class QichachaSpider: def __init__(self): self.url = 'https://www.qichacha.com/gongsi_area.html?prov={}&city={}&p={}' self.q = Queue() self.company_info = [] self.headers = { 'Host': 'www.qichacha.com', 'Referer': 'https: // www.qichacha.com /', 'X-Requested-With': 'XMLHttpRequest' } # 随机User-Agent def random_ua(self): ua = UserAgent() return ua.random # 随机IP def random_proxy(self): proxy_list = ProxySpider().get_training_ip('https://www.qichacha.com/') return proxy_list # 爬取目标入队列 def put_url(self): self.headers['User-Agent'] = self.random_ua() url = 'https://www.qichacha.com/' html = requests.get(url, headers=self.headers).content.decode('utf-8', 'ignore') parse_html = etree.HTML(html) r_list = parse_html.xpath('//div[@class="areacom"]/div[2]/div[2]/a/@href') for r in r_list: link = r.split('_')[1:] for i in range(1, 501): url = self.url.format(link[0], link[1], i) print(url) self.q.put(url) # 获取一级页面数据 def get_data(self): while True: if not self.q.empty(): url = self.q.get() self.headers['User-Agent'] = self.random_ua() # proxies = self.random_proxy() proxies = random.choice(IP_LIST) try: html = requests.get(url, headers=self.headers, proxies=proxies, timeout=3).content.decode('utf-8', 'ignore') # html = requests.get(url, headers=self.headers).content.decode('utf-8', 'ignore') # time.sleep(random.uniform(0.5, 1.5)) parse_html = etree.HTML(html) company_list = parse_html.xpath('//table[@class="m_srchList"]/tbody/tr') if company_list is not None: for company in company_list: try: company_name = company.xpath('./td[2]/a/text()')[0].strip() company_link = 'https://www.qichacha.com' + company.xpath('./td[2]/a/@href')[0].strip() company_type, company_industry, company_business_scope = self.get_company_info( company_link) company_person = company.xpath('./td[2]/p[1]/a/text()')[0].strip() company_money = company.xpath('./td[2]/p[1]/span[1]/text()')[0].split('：')[-1].strip() company_time = company.xpath('./td[2]/p[1]/span[2]/text()')[0].split('：')[-1].strip() company_email = company.xpath('./td[2]/p[2]/text()')[0].split('：')[-1].strip() company_phone = company.xpath('td[2]/p[2]/span/text()')[0].split('：')[-1].strip() company_address = company.xpath('td[2]/p[3]/text()')[0].split('：')[-1].strip() company_status = company.xpath('td[3]/span/text()')[0].strip() company_dict = { '公司名称': company_name, '公司链接': company_link, '公司类型': company_type, '所属行业': company_industry, '经营范围': company_business_scope, '公司法人': company_person, '注册资本': company_money, '注册时间': company_time, '邮箱': company_email, '电话': company_phone, '地址': company_address, '是否存续': company_status, } print(company_dict) # self.company_info.append( # (company_name, company_link, company_type, company_industry, company_business_scope, # company_person, company_money, company_time, company_email, company_phone, # company_address, company_status)) info_list = [company_name, company_link, company_type, company_industry, company_business_scope, company_person, company_money, company_time, company_email, company_phone, company_address, company_status] self.save_data(info_list) except: with open('./bad.csv', 'a', encoding='utf-8', newline='') as f: writer = csv.writer(f) writer.writerow(url) continue except: self.q.put(url) else: break # 获取二级页面数据 def get_company_info(self, company_link): headers = {'User-Agent': UserAgent().random} html = requests.get(company_link, headers=headers, proxies=random.choice(IP_LIST), timeout=3).content.decode( 'utf-8', 'ignore') while True: if '企业类型' not in html: html = requests.get(company_link, headers=headers, proxies=random.choice(IP_LIST), timeout=3).content.decode( 'utf-8', 'ignore') else: break try: company_type = re.findall(r'企业类型</td> <td class="">(.*?)</td>', html, re.S)[0].strip() company_industry = re.findall(r'所属行业</td> <td class="">(.*?)</td>', html, re.S)[0].strip() company_business_scope = re.findall(r'经营范围.*?"3">(.*?)</td>', html, re.S)[0].strip() return company_type, company_industry, company_business_scope except: return '无', '无', '无' # 保存数据 def save_data(self, info): with open('./1111.csv', 'a', encoding='utf-8', newline='') as f: writer = csv.writer(f) writer.writerow(info) def main(self): if os.path.exists('./1111.csv'): os.remove('./1111.csv') with open('./1111.csv', 'a', encoding='utf-8', newline='') as f: writer = csv.writer(f) writer.writerow( ['公司名称', '公司链接', '公司类型', '所属行业', '经营范围', '公司法人', '注册资本', '注册时间', '邮箱', '电话', '地址', '是否存续']) self.put_url() t_list = [] for i in range(0, 10): t = Thread(target=self.get_data) t_list.append(t) t.start() for j in t_list: j.join() if __name__ == "__main__": spider = QichachaSpider() spider.main()
未了提高爬取速率，同时为了显示高大上，下面使用scrapy框架进行爬取，代码如下：
1、items.py
# -*- coding: utf-8 -*- # Define here the models for your scraped items # # See documentation in: # https://docs.scrapy.org/en/latest/topics/items.html import scrapy class QichachaItem(scrapy.Item): company_name = scrapy.Field() company_person = scrapy.Field() company_money = scrapy.Field() company_establish = scrapy.Field() company_email = scrapy.Field() company_phone = scrapy.Field() company_address = scrapy.Field() company_risk = scrapy.Field() company_status = scrapy.Field() company_type = scrapy.Field() company_trade = scrapy.Field() company_business_scope = scrapy.Field() company_link = scrapy.Field() company_city = scrapy.Field()
2、qichacha.py
# -*- coding: utf-8 -*- import scrapy import time import random import re from ..items import QichachaItem class QichachaSpider(scrapy.Spider): name = 'qichacha' allowed_domains = ['www.qichacha.com'] base_url = 'https://www.qichacha.com/gongsi_area.html?prov=AH&city={}&p={}' city_code_list = [340100, 340200, 340300, 340400, 340500, 340600, 340700, 340800, 341000, 341100, 341200, 341300, 341500, 341600, 341700, 341800] city_name_list = ['合肥市', '芜湖市', '蚌埠市', '淮南市', '马鞍山市', '淮北市', '铜陵市', '安庆市', '黄山市', '滁州市', '阜阳市', '宿州市', '六安市', '亳州市', '池州市', '宣城市'] base_company_url = 'https://www.qichacha.com{}' def start_requests(self): for i in range(len(self.city_code_list)): for j in range(1, 501): item = QichachaItem() item['company_city'] = self.city_name_list[i] url = self.base_url.format(self.city_code_list[i], j) yield scrapy.Request( url=url, meta={'item': item}, callback=self.parse_page ) time.sleep(random.randint(30, 60)) def parse_page(self, response): item = response.meta['item'] company_list = response.xpath('//*[@id="searchlist"]/table/tbody/tr') for company in company_list: item['company_name'] = company.xpath('td[2]/a/text()').extract_first() item['company_link'] = self.base_company_url.format(company.xpath('td[2]/a/@href').extract_first()) item['company_person'] = company.xpath('td[2]/p[1]/a/text()').extract_first() item['company_money'] = company.xpath('td[2]/p[1]/span[1]/text()').extract_first().split('：')[-1] item['company_establish'] = company.xpath('td[2]/p[1]/span[2]/text()').extract_first().split('：')[-1] item['company_email'] = company.xpath('td[2]/p[2]/text()').extract_first().split('：')[-1].strip() item['company_phone'] = company.xpath('td[2]/p[2]/span/text()').extract_first().split('：')[-1] item['company_address'] = company.xpath('td[2]/p[3]/text()').extract_first().split('：')[-1].strip() item['company_status'] = company.xpath('td[3]/span/text()').extract_first().split('：')[-1] yield scrapy.Request( url=item['company_link'], meta={'item': item}, callback=self.parse_company ) time.sleep(random.randint(10, 20)) def parse_company(self, response): item = response.meta['item'] html = response.text if re.findall(r'<h2>经营风险.*?<span>(.*?)</span>', html, re.S): item['company_risk'] = re.findall(r'<h2>经营风险.*?<span>(.*?)</span>', html, re.S)[0].strip() else: item['company_risk'] = '-' if re.findall(r'企业类型</td> <td class="">(.*?)</td>', html, re.S): item['company_type'] = re.findall(r'企业类型</td> <td class="">(.*?)</td>', html, re.S)[0].strip() else: item['company_type'] = '-' if re.findall(r'所属行业</td> <td class="">(.*?)</td>', html, re.S): item['company_trade'] = re.findall(r'所属行业</td> <td class="">(.*?)</td>', html, re.S)[0].strip() else: item['company_trade'] = '-' if re.findall(r'经营范围</td> <td class="" colspan="3">(.*?)</td>', html, re.S): item['company_business_scope'] = re.findall(r'经营范围</td> <td class="" colspan="3">(.*?)</td>', html, re.S)[ 0].strip() else: item['company_business_scope'] = '-' yield item time.sleep(random.uniform(0.5, 1))
3、pipelines.py
# -*- coding: utf-8 -*- # Define your item pipelines here # # Don't forget to add your pipeline to the ITEM_PIPELINES setting # See: https://docs.scrapy.org/en/latest/topics/item-pipeline.html import pymysql from .settings import * class QichachaPipeline(object): def process_item(self, item, spider): print([item['company_name'], item['company_person'], item['company_money'], item['company_establish'], item['company_email'], item['company_phone'], item['company_address'], item['company_risk'], item['company_status'], item['company_type'], item['company_trade'], item['company_link'], item['company_city'], item['company_business_scope']]) return item class MysqlPipeline(object): def open_spider(self, spider): self.db = pymysql.connect(host=MYSQL_HOST, port=MYSQL_PORT, user=MYSQL_USER, password=MYSQL_PWD, database=MYSQL_DB, charset='utf8') self.cursor = self.db.cursor() def process_item(self, item, spider): ins = 'insert into qichachatab values(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)' info_list = [item['company_name'], item['company_city'], item['company_person'], item['company_money'], item['company_establish'], item['company_email'], item['company_phone'], item['company_address'], item['company_risk'], item['company_status'], item['company_type'], item['company_trade'], item['company_link'], item['company_business_scope']] self.cursor.execute(ins, info_list) self.db.commit() return item def close_spider(self, spider): self.cursor.close() self.db.close()
4、middlewares.py
# -*- coding: utf-8 -*- # Define here the models for your spider middleware # # See documentation in: # https://docs.scrapy.org/en/latest/topics/spider-middleware.html from scrapy import signals class QichachaSpiderMiddleware(object): # Not all methods need to be defined. If a method is not defined, # scrapy acts as if the spider middleware does not modify the # passed objects. @classmethod def from_crawler(cls, crawler): # This method is used by Scrapy to create your spiders. s = cls() crawler.signals.connect(s.spider_opened, signal=signals.spider_opened) return s def process_spider_input(self, response, spider): # Called for each response that goes through the spider # middleware and into the spider. # Should return None or raise an exception. return None def process_spider_output(self, response, result, spider): # Called with the results returned from the Spider, after # it has processed the response. # Must return an iterable of Request, dict or Item objects. for i in result: yield i def process_spider_exception(self, response, exception, spider): # Called when a spider or process_spider_input() method # (from other spider middleware) raises an exception. # Should return either None or an iterable of Request, dict # or Item objects. pass def process_start_requests(self, start_requests, spider): # Called with the start requests of the spider, and works # similarly to the process_spider_output() method, except # that it doesn’t have a response associated. # Must return only requests (not items). for r in start_requests: yield r def spider_opened(self, spider): spider.logger.info('Spider opened: %s' % spider.name) class QichachaDownloaderMiddleware(object): # Not all methods need to be defined. If a method is not defined, # scrapy acts as if the downloader middleware does not modify the # passed objects. @classmethod def from_crawler(cls, crawler): # This method is used by Scrapy to create your spiders. s = cls() crawler.signals.connect(s.spider_opened, signal=signals.spider_opened) return s def process_request(self, request, spider): # Called for each request that goes through the downloader # middleware. # Must either: # - return None: continue processing this request # - or return a Response object # - or return a Request object # - or raise IgnoreRequest: process_exception() methods of # installed downloader middleware will be called return None def process_response(self, request, response, spider): # Called with the response returned from the downloader. # Must either; # - return a Response object # - return a Request object # - or raise IgnoreRequest return response def process_exception(self, request, exception, spider): # Called when a download handler or a process_request() # (from other downloader middleware) raises an exception. # Must either: # - return None: continue processing this exception # - return a Response object: stops process_exception() chain # - return a Request object: stops process_exception() chain pass def spider_opened(self, spider): spider.logger.info('Spider opened: %s' % spider.name) from fake_useragent import UserAgent class RandomUserAgentDownloaderMiddleware(object): def process_request(self, requset, spider): ua = UserAgent().random requset.headers['User-Agent'] = ua import redis from .settings import * from .proxies import ProxypoolSpider class RandomProxyDownloaderMiddleware(object): def __init__(self): self.db = redis.Redis(host=REDIS_HOST, port=REDIS_PORT, db=REDIS_DB, password=REDIS_PWD) def process_request(self, request, spider): proxy_list = self.db.zrangebyscore(REDIS_PROXY_KEY, 90, 100, withscores=True) if len(proxy_list) == 0: proxy_spider = ProxypoolSpider() proxy_spider.get_proxy() proxy = random.choice(proxy_list)[0].decode('utf-8') request.meta['proxy'] = proxy def process_response(self, request, response, spider): print(response.status, ": ", request.url) return request def process_exception(self, request, exception, spider): cur_proxy = request.meta['proxy'] print('异常来了') self.db.zincrby(REDIS_PROXY_KEY, -1, cur_proxy) del request.meta['proxy'] return request
5、settings.py
# -*- coding: utf-8 -*- # Scrapy settings for Qichacha project # # For simplicity, this file contains only settings considered important or # commonly used. You can find more settings consulting the documentation: # # https://docs.scrapy.org/en/latest/topics/settings.html # https://docs.scrapy.org/en/latest/topics/downloader-middleware.html # https://docs.scrapy.org/en/latest/topics/spider-middleware.html import time import random BOT_NAME = 'Qichacha' SPIDER_MODULES = ['Qichacha.spiders'] NEWSPIDER_MODULE = 'Qichacha.spiders' # Crawl responsibly by identifying yourself (and your website) on the user-agent USER_AGENT = 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36' # Obey robots.txt rules ROBOTSTXT_OBEY = False # Configure maximum concurrent requests performed by Scrapy (default: 16) CONCURRENT_REQUESTS = 10 # Configure a delay for requests for the same website (default: 0) # See https://docs.scrapy.org/en/latest/topics/settings.html#download-delay # See also autothrottle settings and docs DOWNLOAD_DELAY = time.sleep(random.uniform(10, 20)) # The download delay setting will honor only one of: # CONCURRENT_REQUESTS_PER_DOMAIN = 16 # CONCURRENT_REQUESTS_PER_IP = 16 # Disable cookies (enabled by default) COOKIES_ENABLED = False # Disable Telnet Console (enabled by default) # TELNETCONSOLE_ENABLED = False # Override the default request headers: DEFAULT_REQUEST_HEADERS = { 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', 'Accept-Language': 'en', 'Cookie': 'UM_distinctid=16bb7adb9252d8-09b3389bed6ae2-3a65420e-1fa400-16bb7adb92636e; zg_did=%7B%22did%22%3A%20%2216bb7adbb84740-04a7e287a3fa12-3a65420e-1fa400-16bb7adbb85669%22%7D; _uab_collina=156215474498922246746771; zg_63e87cf22c3e4816a30bfbae9ded4af2=%7B%22sid%22%3A%201562193465906%2C%22updated%22%3A%201562193465917%2C%22info%22%3A%201562193465914%2C%22superProperty%22%3A%20%22%7B%7D%22%2C%22platform%22%3A%20%22%7B%7D%22%2C%22utm%22%3A%20%22%7B%7D%22%2C%22referrerDomain%22%3A%20%22%22%7D; QCCSESSID=lnr0huo5t5s058h9tmlso56nu1; acw_tc=65e21c2915648350628141700eeaf85114e84964375db3a9f1b718d751; CNZZDATA1254842228=845561946-1562153840-https%253A%252F%252Fwww.baidu.com%252F%7C1565064428; hasShow=1; Hm_lvt_3456bee468c83cc63fb5147f119f1075=1565048078,1565048449,1565048590,1565067806; zg_de1d1a35bfa24ce29bbf2c7eb17e6c4f=%7B%22sid%22%3A%201565067805845%2C%22updated%22%3A%201565069298085%2C%22info%22%3A%201564658796236%2C%22superProperty%22%3A%20%22%7B%7D%22%2C%22platform%22%3A%20%22%7B%7D%22%2C%22utm%22%3A%20%22%7B%7D%22%2C%22referrerDomain%22%3A%20%22sp0.baidu.com%22%2C%22cuid%22%3A%20%22d48f6830513b318400fcc23636a23a7f%22%7D; Hm_lpvt_3456bee468c83cc63fb5147f119f1075=1565069298', 'Referer': 'https://www.qichacha.com/', } # Enable or disable spider middlewares # See https://docs.scrapy.org/en/latest/topics/spider-middleware.html # SPIDER_MIDDLEWARES = { # 'Qichacha.middlewares.QichachaSpiderMiddleware': 543, # } # Enable or disable downloader middlewares # See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html DOWNLOADER_MIDDLEWARES = { 'Qichacha.middlewares.QichachaDownloaderMiddleware': 543, # 'Qichacha.middlewares.RandomUserAgentDownloaderMiddleware': 200, # 'Qichacha.middlewares.RandomProxyDownloaderMiddleware': 250, } # Enable or disable extensions # See https://docs.scrapy.org/en/latest/topics/extensions.html # EXTENSIONS = { # 'scrapy.extensions.telnet.TelnetConsole': None, # } # Configure item pipelines # See https://docs.scrapy.org/en/latest/topics/item-pipeline.html ITEM_PIPELINES = { 'Qichacha.pipelines.QichachaPipeline': 300, 'Qichacha.pipelines.MysqlPipeline': 100, } # Enable and configure the AutoThrottle extension (disabled by default) # See https://docs.scrapy.org/en/latest/topics/autothrottle.html # AUTOTHROTTLE_ENABLED = True # The initial download delay # AUTOTHROTTLE_START_DELAY = 5 # The maximum download delay to be set in case of high latencies # AUTOTHROTTLE_MAX_DELAY = 60 # The average number of requests Scrapy should be sending in parallel to # each remote server # AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0 # Enable showing throttling stats for every response received: # AUTOTHROTTLE_DEBUG = False # Enable and configure HTTP caching (disabled by default) # See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings # HTTPCACHE_ENABLED = True # HTTPCACHE_EXPIRATION_SECS = 0 # HTTPCACHE_DIR = 'httpcache' # HTTPCACHE_IGNORE_HTTP_CODES = [] # HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage' MYSQL_HOST = '127.0.0.1' MYSQL_PORT = 3306 MYSQL_DB = 'qichachadb' MYSQL_USER = 'root' MYSQL_PWD = '123456' # Redis数据库定义 REDIS_HOST = '127.0.0.1' REDIS_PORT = 6379 REDIS_PWD = '123456' REDIS_DB = 0 REDIS_PROXY_KEY = 'proxy' # 日志管理 LOG_LEVEL = 'WARNING' # LOG_FILE = 'qichacha.log' # 编码设置 FEED_EXPORT_ENCODING = 'utf-8'
6、run.py
from scrapy import cmdline cmdline.execute('scrapy crawl qichacha'.split())