爬取赶集网二手物品下所有物品的信息。
大致思路：
1、爬取频道页url；
2、爬取商品详情页url，写入mongodb，url_list表；
3、从url_list表读取url，爬取商品信息，写入mongodb，p_info表
分成3个py文件：
1、channel_url.py，获取频道页url；
2、page_parse.py，主要是2个爬虫函数，分别完成2个数据表的写入；
3、main.py，主程序文件，也就是本文件，开启多进程，完成整个工作
最后顺利完成任务，感觉现在赶集真不行了，没多少数据。
channel_url.py文件：
import requests
from bs4 import BeautifulSoup
start_url = 'http://bj.ganji.com/wu/'
url_host = 'http://bj.ganji.com'
def get_channel_url(url):
channel_urls = []
raw_data = requests.get(url).text
soup = BeautifulSoup(raw_data,'lxml')
eles = soup.select('div.content dt>a')
for e in eles:
channel_url = url_host + e.get('href')
print(channel_url)
channel_urls.append(channel_url)
return channel_urls
# channel_urls = get_channel_url(start_url)
# print('len(channel_urls):',len(channel_urls))
# 这是程序运行的结果，直接保存下来了，就不用再运行get_channel_url()了
channel_urls = '''
http://bj.ganji.com/jiaju/
http://bj.ganji.com/rirongbaihuo/
http://bj.ganji.com/shouji/
http://bj.ganji.com/bangong/
http://bj.ganji.com/nongyongpin/
http://bj.ganji.com/jiadian/
http://bj.ganji.com/ershoubijibendiannao/
http://bj.ganji.com/ruanjiantushu/
http://bj.ganji.com/yingyouyunfu/
http://bj.ganji.com/diannao/
http://bj.ganji.com/xianzhilipin/
http://bj.ganji.com/fushixiaobaxuemao/
http://bj.ganji.com/meironghuazhuang/
http://bj.ganji.com/shuma/
http://bj.ganji.com/laonianyongpin/
http://bj.ganji.com/xuniwupin/
http://bj.ganji.com/qitawupin/
http://bj.ganji.com/ershoufree/
http://bj.ganji.com/wupinjiaohuan/
'''
page_parse.py文件：
import requests
from bs4 import BeautifulSoup
from time import sleep
from pymongo import MongoClient
client = MongoClient('localhost',27017)
ganji = client['ganji']
url_list = ganji['url_list']
p_info = ganji['p_info']
# 给定频道url，爬取此频道下所有商品的url，打印并写入mongo数据库
def get_product_url(url):
channel_url = url
page_num = 1
while True:
raw_page = requests.get(url).text
print('正在get网页:',url)
sleep(2)
soup = BeautifulSoup(raw_page,'lxml')
eles = soup.select('a.ft-tit')
print('len(eles):',len(eles))
for e in eles:
p_url = e.get('href')
url_list.insert_one({'p_url':p_url})
print(p_url)
if soup.select('a.next'):
page_num += 1
url = channel_url + 'o' + str(page_num) + '/'
else:
break
# 给定商品详情页url,爬取商品具体信息，打印并写入mongo数据库
def get_product_info(url):
raw_page = requests.get(url).text
sleep(2)
soup = BeautifulSoup(raw_page,'lxml')
if soup.select("p:contains('信息刚被删除~')"): # 判断商品信息是否已经删除
print('信息刚被删除~')
pass
else:
title = soup.select('h1.title-name')[0].get_text() if soup.select('h1.title-name') else None
category = list(soup.select('div.crumbs.routes.clearfix')[0].stripped_strings) if soup.select('div.crumbs.routes.clearfix') else None
date = soup.select('i.pr-5')[0].get_text().split('\')[0].strip() if soup.select('i.pr-5') else None
price = soup.select('i.f22.fc-orange.f-type')[0].get_text() if soup.select('i.f22.fc-orange.f-type') else None
address = soup.select('ul.det-infor>li:nth-child(2)>a')[0].get_text() if soup.select('ul.det-infor>li:nth-child(2)>a') else None
p_dict = {'title':title,'category':category,'date':date,'price':price,'address':address,'url':url}
p_info.insert_one(p_dict)
print(p_dict)
main.py文件：
from channel_url import channel_urls # 从channel_url.py导入某变量，会把channel_url.py都执行一遍，但变量只在模块内部保留
from page_parse import get_product_url, get_product_info, url_list # 需要导入url_list
from multiprocessing import Pool
from datetime import datetime
# 从mongodb中读取商品url，返回所有商品的url
def read_all_p_urls():
all_p_urls = []
for item in url_list.find():
all_p_urls.append(item['p_url'])
return all_p_urls
if __name__ == '__main__':
start_time = datetime.now()
# 不用多进程的方式，耗时会多好几倍
# for channel in channel_urls.split():
# get_product_url(channel)
pool = Pool()
#用多进程的方式，4进程和自动分配进程耗时差不多
#pool = Pool(processes=4)
# 根据channel url，获取商品url，写入mongodb
pool.map(get_product_url,channel_urls.split())
# 根据商品url，获取商品信息，写入mongodb；这一句可以跟上面那句分开执行
pool.map(get_product_info,read_all_p_urls())
end_time = datetime.now()
during = end_time - start_time
print('总共耗时：',during)