Scrapy 学习笔记爬豆瓣 250

Scrapy 是比较上层的库，基于中间层开发，它基于高层，所以它依赖许多其它库。事件驱动的异步技术。

Scrapy 爬取网页，以豆瓣电影 Top 250 为例子。
首先打开命令提示符，输入。scrapy startproject douban

使用 Scrapy 提供的 cmd 命令

from scrapy import cmdline
cmdline.execute("scrapy crawl doubanmovie".split())

设置 settings.py

DOWNLOAD_DELAY = 2
RANDOMIZE_DOWNLOAD_DELAY = True
USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_3) AppleWebKit/536.5 (KHTML, like Gecko) Chrome/19.0.1084.54 Safari/536.5'
COOKIES_ENABLED = True


FEED_URI = u'file:douban.csv'
FEED_FORMAT = 'csv'

class DoubanMovieItem(Item):
    title = Field()
    movieInfo = Field()
    star = Field()
    quote = Field()

主要的爬虫程序：

from scrapy import Request
from scrapy import Selector
from scrapy.spiders import Spider

from douban.items import DoubanMovieItem


class Douban(Spider):
    name = "douban"
    start_urls = ["https://movie.douban.com/top250"]

    url = 'https://movie.douban.com/top250'

    def parse(self, response):
        print("--- 爬取的内容 ---")
        print(response.url)

        item = DoubanMovieItem()
        selector = Selector(response)
        Movies = selector.xpath("//div[@class='info']")
        for eachMovice in Movies:
            title = eachMovice.xpath("div[@class='hd']/a/span/text()").extract()

            fullTitle = ''
            for each in title:
                fullTitle += each

            movieInfo = eachMovice.xpath("div[@class='bd']/p/text()").extract()
            # 评分，xpath 从的数组下标从 1 开始
            star = eachMovice.xpath("div[@class='bd']/div[@class='star']/span[2]/text()").extract()
            print(star)
            # 一句脍炙人口的话
            quote = eachMovice.xpath("div[@class='bd']/div[@class='star']/span[4]/text()").extract()
            if (quote):
                quote = quote[0]
            else:
                quote = ''
            item['title'] = fullTitle
            item['movieInfo'] = ";".join(movieInfo)
            item['star'] = star
            item['quote'] = quote
            yield item
        nextLink = selector.xpath("//div[@class='paginator']/span[@class='next']/a/@href").extract()

        if (nextLink):
            nextLink = nextLink[0]
            print("下一页", nextLink)
            yield Request(self.url + nextLink, callback=self.parse)