Scrapy 框架
### 安装 流程
- 环境的安装：
- mac/linux：pip install scrapy
- window:
- pip install wheel
- 下载twisted http://www.lfd.uci.edu/~gohlke/pythonlibs/#twisted
- 进入下载目录，执行 pip install Twisted‑17.1.0‑cp35‑cp35m‑win_amd64.whl
- pip install pywin32
- pip install scrapy
# 新建一个 scrapy 工程
scrapy startproject ProName
# 进入爬虫工程
cd ProName
# 创建一个爬虫文件
# python 解释器 scrapy genspider 文件名 url地址
python.exe -m scrapy genspider first www.baidu.com
# 执行工程
python.exe -m scrapy crawl spiderName
first.py 文件内容
# encoding:utf-8
import scrapy
# 爬虫 类 , 父类是 Spider,还有其他形式的父类
class FirstSpider(scrapy.Spider):
# 爬虫文件名称, 爬虫源文件的唯一标识
name = 'first'
# 允许的域名，一般不用
# allowed_domains = ['www.baidu.com']
# 起始的URL请求地址 列表:
# 1. 列表中存放的 URL 都可以被 scrapy 都进行 异步网络请求
# 2.
start_urls = ['https://www.baidu.com/','https://www.sogou.com/']
# 用于数据解析 , 相当于 异步回调的处理 结果的函数
def parse(self, response):
'''
:param response: 响应对象
:return:
'''
print(response)
执行 工程 获取数据
### 执行 scrapy 工程中的爬虫脚本
## robots.txt 问题
# 在setting 中
ROBOTSTXT_OBEY = False
## 日志问题
# 输入日志(全日志)
- python.exe -m scrapy crawl first
# 不输入日志
- python.exe -m scrapy crawl first --nolog
# setting 修改日志等级
LOG_LEVEL='ERROR'
LOG_FILE='./LOG.log'
## UA 伪装,settings.py配置
USER_AGENT = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.121 Safari/537.36'
持久化存储
# 两种方式
- 基于终端指令的持久化存储
- 只可以将 parse方法返回的结果存储到本地磁盘中
# 指令 , 只能是符合 scrapy标准的文件格式
- scrapy crawl spiderName -o filepath
- 基于管道的持久化存储
# 编码流程:
- 1. 在爬虫文件中进行解析
- 2. 要在 item.py文件中 创建对应的字段/属性,来存储解析后的结果值
- 3. 在parse将解析的数据封装到 item类型的对象中
- 4. 在parse.py文件中将 item 类型的对象 提交到管道
yield item
- 5. 管道类的 process_item 方法接收到 item,可以执行任意持久化
- 6. 在配置setting文件中开启管道设置
# 300 表示的优先级 , 数值越小优先级越高
ITEM_PIPELINES = {
'choutiPro.pipelines.ChoutiproPipeline': 300,
}
# 一个管道 定义一种存储 数据的方式 : mysql,本地,psql等需要多个管道类,并将管道类添加到settings配置文件中
实例代码
piplines.py
# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://docs.scrapy.org/en/latest/topics/item-pipeline.html
# useful for handling different item types with a single interface
from itemadapter import ItemAdapter
class ChoutiproPipeline:
fp = None
def open_spider(self, spider):
'''
# 重写父类的方法
# 此方法就执行一次.
:param spider:
:return:
'''
print('开始爬虫')
self.fp = open('chouti.txt', 'w', encoding='utf-8')
def close_spider(self, spider):
'''
# 重写父类的方法
# 此方法就执行一次.
:param spider:
:return:
'''
print('结束爬虫')
self.fp.close()
# 该方法 调用后. 就能接收到 item的对象
def process_item(self, item, spider):
'''
:param item: item对象
:param spider:
:return:
'''
author = item['author']
title = item['title']
print(author, title)
self.fp.write(author + ":" + title + "
")
return item
import pymysql
class ChoutiproPipelineMySQL():
conn = None
cursor = None
def open_spider(self, spider):
self.conn = pymysql.Connection(
host='127.0.0.1',
port=3306,
user='root',
password='123',
db='chouti_data',
charset='utf8'
)
self.cursor = self.conn.cursor()
def process_item(self, item, spider):
title = item['title']
author = item['author']
sql = """
insert into chouti values("%s","%s")
""" % (author, title)
try:
self.cursor.execute(sql)
self.conn.commit()
except Exception as e:
print("错误:", e)
self.conn.rollback()
### 非常重要, 将item 传入到下一个 管道类
return item
def close_spider(self, spider):
self.cursor.close()
self.conn.close()
settings.py
# Scrapy settings for choutiPro project
#
# For simplicity, this file contains only settings considered important or
# commonly used. You can find more settings consulting the documentation:
#
# https://docs.scrapy.org/en/latest/topics/settings.html
# https://docs.scrapy.org/en/latest/topics/downloader-middleware.html
# https://docs.scrapy.org/en/latest/topics/spider-middleware.html
BOT_NAME = 'choutiPro'
SPIDER_MODULES = ['choutiPro.spiders']
NEWSPIDER_MODULE = 'choutiPro.spiders'
# Crawl responsibly by identifying yourself (and your website) on the user-agent
USER_AGENT = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.121 Safari/537.36'
# Obey robots.txt rules
ROBOTSTXT_OBEY = False
LOG_LEVEL='ERROR'
LOG_FILE='./LOG.log'
# Configure maximum concurrent requests performed by Scrapy (default: 16)
#CONCURRENT_REQUESTS = 32
# Configure a delay for requests for the same website (default: 0)
# See https://docs.scrapy.org/en/latest/topics/settings.html#download-delay
# See also autothrottle settings and docs
#DOWNLOAD_DELAY = 3
# The download delay setting will honor only one of:
#CONCURRENT_REQUESTS_PER_DOMAIN = 16
#CONCURRENT_REQUESTS_PER_IP = 16
# Disable cookies (enabled by default)
#COOKIES_ENABLED = False
# Disable Telnet Console (enabled by default)
#TELNETCONSOLE_ENABLED = False
# Override the default request headers:
#DEFAULT_REQUEST_HEADERS = {
# 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
# 'Accept-Language': 'en',
#}
# Enable or disable spider middlewares
# See https://docs.scrapy.org/en/latest/topics/spider-middleware.html
#SPIDER_MIDDLEWARES = {
# 'choutiPro.middlewares.ChoutiproSpiderMiddleware': 543,
#}
# Enable or disable downloader middlewares
# See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html
#DOWNLOADER_MIDDLEWARES = {
# 'choutiPro.middlewares.ChoutiproDownloaderMiddleware': 543,
#}
# Enable or disable extensions
# See https://docs.scrapy.org/en/latest/topics/extensions.html
#EXTENSIONS = {
# 'scrapy.extensions.telnet.TelnetConsole': None,
#}
# Configure item pipelines
# See https://docs.scrapy.org/en/latest/topics/item-pipeline.html
ITEM_PIPELINES = {
'choutiPro.pipelines.ChoutiproPipelineMySQL': 100,
'choutiPro.pipelines.ChoutiproPipeline': 300,
}
# Enable and configure the AutoThrottle extension (disabled by default)
# See https://docs.scrapy.org/en/latest/topics/autothrottle.html
#AUTOTHROTTLE_ENABLED = True
# The initial download delay
#AUTOTHROTTLE_START_DELAY = 5
# The maximum download delay to be set in case of high latencies
#AUTOTHROTTLE_MAX_DELAY = 60
# The average number of requests Scrapy should be sending in parallel to
# each remote server
#AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0
# Enable showing throttling stats for every response received:
#AUTOTHROTTLE_DEBUG = False
# Enable and configure HTTP caching (disabled by default)
# See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings
#HTTPCACHE_ENABLED = True
#HTTPCACHE_EXPIRATION_SECS = 0
#HTTPCACHE_DIR = 'httpcache'
#HTTPCACHE_IGNORE_HTTP_CODES = []
#HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage'
items.py
# Define here the models for your scraped items
#
# See documentation in:
# https://docs.scrapy.org/en/latest/topics/items.html
import scrapy
class ChoutiproItem(scrapy.Item):
# define the fields for your item here like:
# name = scrapy.Field()
# scrapy.Field() 可作为一个万能的数据类型
title = scrapy.Field()
author = scrapy.Field()
chouti.py
import scrapy
from choutiPro.items import ChoutiproItem
class ChoutiSpider(scrapy.Spider):
name = 'chouti'
# allowed_domains = ['www.baidu.com']
start_urls = ['https://dig.chouti.com/']
def parse(self, response):
# scapy 封装了xpath
# 解析内容 和 发布平台
div_list = response.xpath('//div[@class="link-con"]/div')
all_data = []
for div in div_list:
title_text = div.xpath('.//div[@class="link-detail"]/a/text()').extract_first()
author = div.xpath('.//div[@class="operate-author-con clearfix"]//span[@class="left author-name"]/text()').extract_first()
"""
Data:
[<Selector xpath='.//div[@class="link-detail"]/a/text()' data='河南博物院上新文物修复盲盒！竟然和做模型有这么多相
似之处！'>] []
# 需要对结果进行 进行提取字符串
extract() : 提取多个 , 相当于列表
extract_first() : 提取第0个元素 , 具体值
"""
# 1. 方式一 基于 命令行存储
# all_data.append({"title_text": title_text, "author": author or 'DD'})
# return all_data
# 2. 基于本地存储
# 实例化 一个 item对象
item = ChoutiproItem()
# 给 item 创建属性值 ,item['title'] 调用的 __setitem__
item['title'] = title_text
item['author'] = author
# 提交管道 , 将item 提交到管道
yield item