scrapy 多个爬虫运行

from scrapy import cmdline
import datetime
import time
import os
import scrapy
from scrapy.crawler import CrawlerProcess
from scrapy.utils.project import get_project_settings
os.environ.setdefault('SCRAPY_SETTINGS_MODULE', 'doubanMovie.settings')
global process
process= CrawlerProcess(get_project_settings())
def dosth():
    print("执行第一个爬虫")
    #cmdline.execute("scrapy crawlall".split())
    process.crawl("doubanMovieSpider1")

    #print(type(cmdline.execute("scrapy crawl doubanMovie".split())))

def doanother():
    print("执行第二个爬虫")
    process.crawl("doubanMovieSpider2")


def main():

    while True:
        now=datetime.datetime.now()

        if now.second==30:
            dosth()
            time.sleep(1)
            doanother()
            process.start()

main()

  

原文地址:https://www.cnblogs.com/duole/p/11597374.html