抓取活动树网站中会议活动数据（http://www.huodongshu.com/html/index.html）
具体的思路是[Python爬虫] 之十一中抓取活动行网站的类似，都是用多线程来抓取，但是由于活动树网站 ，每个关键字搜索页的ur是固定，比如搜索“数字”结果有470个结果，没页10条记录，第二页的url和第一页的 url是一样的。
因此针对每个关键字用一个线程进行搜索。
具体代码如下：
# coding=utf-8
import os
import re
from selenium import webdriver
import selenium.webdriver.support.ui as ui
from selenium.webdriver.common.keys import Keys
import time
from selenium.webdriver.common.action_chains import ActionChains
from selenium.webdriver.support.select import Select
import IniFile
from selenium.webdriver.common.keys import Keys
from threading import Thread
import thread
import LogFile
import urllib
import mongoDbBase
#抓取数据线程类
class ScrapyData_Thread(Thread):
#抓取数据线程类
def __init__(self,webSearchUrl,pageCountLable,htmlLable,originalUrlLabel,nextUrlLabel,keyword,db):
'''
构造函数
:param webSearchUrl: 搜索页url
:param pageCountLable: 搜索页数标签
:param htmlLable: 要搜索的标签
:param OriginalUrlLabel: 每个记录对应的url标签
:param nextUrlLabel: 下一页标签
:param keywords: 要搜索的关键字，多个关键字中间用分号(;)隔开
:param db: 保存数据库引擎
'''
Thread.__init__(self)
self.webSearchUrl = webSearchUrl
self.pageCountLable = pageCountLable
self.htmlLable = htmlLable
self.originalUrlLabel = originalUrlLabel
self.nextUrlLabel = nextUrlLabel
self.keyword = keyword
self.db = db
# IEDriverServer = self.cf.GetValue("section", "IEDriverServer")
# os.environ["webdriver.ie.driver"] = IEDriverServer
# self.urldriver = webdriver.Ie(IEDriverServer)
self.driver = webdriver.PhantomJS()
self.wait = ui.WebDriverWait(self.driver, 20)
self.driver.maximize_window()
def compareDate(self, dateLeft, dateRight):
'''
比较俩个日期的大小
:param dateLeft: 日期 格式2017-03-04
:param dateRight:日期 格式2017-03-04
:return: 1：左大于右，0：相等，-1：左小于右
'''
dls = dateLeft.split('-')
drs = dateRight.split('-')
if len(dls) > len(drs):
return 1
if int(dls[0]) == int(drs[0]) and int(dls[1]) == int(drs[1]) and int(dls[2]) == int(drs[2]):
return 0
if int(dls[0]) > int(drs[0]):
return 1
elif int(dls[0]) == int(drs[0]) and int(dls[1]) > int(drs[1]):
return 1
elif int(dls[0]) == int(drs[0]) and int(dls[1]) == int(drs[1]) and int(dls[2]) > int(drs[2]):
return 1
return -1
def date_isValid(self, strDateText):
'''
判断日期时间字符串是否合法：如果给定时间大于当前时间是合法，或者说当前时间给定的范围内
:param strDateText: 三种格式 '017.04.27 ~ 04.28'; '2017.04.20 08:30 ~ 12:30' ; '2015.12.29 ~ 2016.01.03'
:return: True:合法；False:不合法
'''
datePattern = re.compile(r'd{4}-d{2}-d{2}')
date = strDateText.replace('.', '-')
strDate = re.findall(datePattern, date)
currentDate = time.strftime('%Y-%m-%d')
if len(strDate) == 2:
if self.compareDate(strDate[1], currentDate) > 0:
return True
elif len(strDate) == 1:
if self.compareDate(strDate[0], currentDate) >= 0:
return True
else:
datePattern = re.compile(r'd{4}-d{2}-d{2}s~sd{2}-d{2}')
#2015-06-04 13:30 ~ 17:30
strDate = re.findall(datePattern, date)
if len(strDate) >0:
if self.compareDate(strDate[0][0:5] + strDate[0][13:], currentDate) >= 0:
return True
else:
return False
return False
def run(self):
print ''
print '关键字：%s ' % self.keyword
self.driver.get(self.webSearchUrl)
time.sleep(5)
# 记录数
pageCount_elements = self.driver.find_elements_by_xpath(self.pageCountLable)
if len(pageCount_elements) > 0:
strCount = pageCount_elements[0].text.encode('utf8')
pageCount = int(strCount) / 10
if int(strCount) % 10 > 0:
pageCount = pageCount + 1
page_Count = pageCount
pageIndex = 0
kword = self.keyword
recordCount = 0
while pageCount > 0:
pageCount = pageCount - 1
if pageIndex > 0:
next_element = self.driver.find_elements_by_xpath(self.nextUrlLabel)
if len(next_element) > 0:
next_element[0].click()
time.sleep(3)
self.wait.until(lambda driver: self.driver.find_elements_by_xpath(self.htmlLable))
Elements = self.driver.find_elements_by_xpath(self.htmlLable)
# 查找微博对应的原始url
urlList = []
self.wait.until(lambda driver: self.driver.find_elements_by_xpath(self.originalUrlLabel))
hrefElements = self.driver.find_elements_by_xpath(self.originalUrlLabel)
for hrefe in hrefElements:
urlList.append(hrefe.get_attribute('href').encode('utf8'))
index = 0
strMessage = ' '
strsplit = ' ------------------------------------------------------------------------------------ '
index = 0
# 每页中有用记录
usefulCount = 0
meetingList = []
for element in Elements:
txt = element.text.encode('utf8')
txts = txt.split(' ')
# strDate = re.findall(self.datePattern, txt)
# 日期大于今天并且搜索的关键字在标题中才认为是复合要求的数据
if self.date_isValid(txts[1]) and txts[0].find(kword) > -1:
dictM = {'title': txts[0], 'date': txts[1],
'url': urlList[index], 'keyword': kword, 'info': txt}
meetingList.append(dictM)
# print ' '
# print txt
# print '活动链接：' + urlList[index]
# print strsplit
#
# strMessage = txt + " "
# strMessage += '活动链接：' + urlList[index] + " "
# strMessage += strsplit
# strMessage = unicode(strMessage, 'utf8')
# # log.WriteLog(strMessage)
usefulCount = usefulCount + 1
recordCount = recordCount + 1
index = index + 1
pageIndex = pageIndex + 1
if usefulCount == 0:
break
else:
self.db.SaveMeetings(meetingList) #保存数据库中
print "共浏览了: %d 页数据" % page_Count
print "共抓取了: %d 个符合条件的活动记录" % recordCount
self.driver.close()
self.driver.quit()
if __name__ == '__main__':
configfile = os.path.join(os.getcwd(), 'activity.conf')
cf = IniFile.ConfigFile(configfile)
webSearchUrl = cf.GetValue("section", "webSearchUrl")
pageCountLable = cf.GetValue("section", "pageCountLable")
htmlLable = cf.GetValue("section", "htmlLable")
originalUrlLabel = cf.GetValue("section", "originalUrlLabel")
nextUrlLabel = cf.GetValue("section", "nextUrlLabel")
keywords= cf.GetValue("section", "keywords")
keywordlist = keywords.split(';')
start = time.clock()
db = mongoDbBase.mongoDbBase()
for keyword in keywordlist:
if len(keyword) > 0:
url = webSearchUrl + urllib.quote(keyword)
t = ScrapyData_Thread(url, pageCountLable, htmlLable,originalUrlLabel,nextUrlLabel,keyword,db)
t.setDaemon(True)
t.start()
t.join()
end = time.clock()
print "整个过程用时间: %f 秒" % (end - start)
配置文件内容：
[section]
#IE驱动的路径
iedriverserver = C:Program FilesInternet ExplorerIEDriverServer.exe
#要搜索的标签，如果有多个，中间用分号隔开
htmlLable = //div[@id ='eventList']/div[@class ='list']
#要获取爬虫也是的标签
pageCountLable = //span[@id='eventNumber']
#给定网址的搜索首页Url
webSearchUrl = http://www.huodongshu.com/html/find_search.html?search_keyword=
#查找对应的原始url
originalUrlLabel = //div[@class='listR']/h2/a
#下一页链接对应的标签
nextUrlLabel = //dt[@class='next']/a
#文本输入框要搜索的关键字
keywords = 互联网电视;智能电视;数字;影音;家庭娱乐;节目;视听;版权;数据