2018年1月26日 16:04:06 更新代码。
excel结果：
#!/usr/bin/env python3 # -*- coding: utf-8 -*- # by 默不知然 # 2017年12月18日 10:01:30 import requests from urllib import parse from bs4 import BeautifulSoup import xlwt import zlib import re import time import xlsxwriter import sys import datetime import random import threadpool ''' 运行方法： python spider_cnnvd.py 2017-10-01 2017-10-31 178 第一个为开始时间，第二个为结束时间，第三个为总页数。 ''' #获取代理,并写入列表agent_lists def agent_list(url): header = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:57.0) Gecko/20100101 Firefox/57.0'} r = requests.get(url,headers=header) agent_info = BeautifulSoup(r.content,'lxml').find(id="ip_list").find_all('tr')[1:] for i in range(len(agent_info)): info = agent_info[i].find_all('td') agents = {info[5].string : 'http://' + info[1].string} agent_lists.append(agents) #获得漏洞详情链接列表 def vulnerabilities_url_list(url,start_time,end_time): header = { 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:58.0) Gecko/20100101 Firefox/58.0', 'Content-Type': 'application/x-www-form-urlencoded' } data ={ 'qstartdate':'2018-02-01', 'qenddate':'2018-02-01' } data['qstartdate'] = start_time data['qenddate'] = end_time proxy = random.sample(agent_lists,1)[0] vulnerabilities_url_html = requests.post(url,headers=header,proxies=proxy,data=data) vulnerabilities_url_html = vulnerabilities_url_html.content.decode() #提取漏洞详情链接 response = r'href="(.+?)" target="_blank" class="a_title2"' vulnerabilities_link_list = re.compile(response).findall(vulnerabilities_url_html) #添加http前序 i = 0 for link in vulnerabilities_link_list: vulnerabilities_lists.append('http://cnnvd.org.cn'+vulnerabilities_link_list[i]) i+=1 print("已完成爬行第%d个漏洞链接"%i) #漏洞信息爬取函数 def vulnerabilities_data(url): header = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.221 Safari/537.36 SE 2.X MetaSr 1.0'} #proxy = random.sample(agent_lists,1)[0] vulnerabilities_data_html = requests.get(url,headers=header) #,proxies=proxy) vulnerabilities_data_html = vulnerabilities_data_html.content.decode() #global vulnerabilities_result_list vulnerabilities_result_list_eve=[] #抓取信息列表命名 #添加漏洞信息详情 vulnerabilities_detainled_soup1 = BeautifulSoup(vulnerabilities_data_html,'html.parser') vulnerabilities_detainled_data = vulnerabilities_detainled_soup1.find('div',attrs={'class':'detail_xq w770'}) ##定义 漏洞信息详情 块的soup vulnerabilities_detainled_data = vulnerabilities_detainled_data.encode().decode() vulnerabilities_detainled_soup = BeautifulSoup(vulnerabilities_detainled_data,'html.parser') #二次匹配 vulnerabilities_detainled_data_list = vulnerabilities_detainled_soup.find_all('li') #标签a信息汇总 try: vulnerabilities_name = vulnerabilities_detainled_soup.h2.string #漏洞名称 except: vulnerabilities_name = '' vulnerabilities_result_list_eve.append(vulnerabilities_name) try: vulnerabilities_cnnvd_num = vulnerabilities_detainled_soup.span.string #cnnvd编号 vulnerabilities_cnnvd_num = re.findall(r"：([sS]*)",vulnerabilities_cnnvd_num)[0] except: vulnerabilities_cnnvd_num = '' vulnerabilities_result_list_eve.append(vulnerabilities_cnnvd_num) try: #漏洞等级 vulnerabilities_rank = vulnerabilities_detainled_soup.a.decode() vulnerabilities_rank = re.search(u'([u4e00-u9fa5]+)',vulnerabilities_rank).group(0) except: vulnerabilities_rank = '' vulnerabilities_result_list_eve.append(vulnerabilities_rank) vulnerabilities_cve_html = vulnerabilities_detainled_data_list[2].encode().decode() #漏洞cve编号 vulnerabilities_cve_soup = BeautifulSoup(vulnerabilities_cve_html,'html.parser') try: vulnerabilities_cve = vulnerabilities_cve_soup.a.string vulnerabilities_cve = vulnerabilities_cve.replace(" ","").replace(" ","").replace(" ","").replace(" ","") except: vulnerabilities_cve = '' vulnerabilities_result_list_eve.append(vulnerabilities_cve) vulnerabilities_type_html = vulnerabilities_detainled_data_list[3].encode().decode() #漏洞类型 vulnerabilities_type_soup = BeautifulSoup(vulnerabilities_type_html,'html.parser') try: vulnerabilities_type = vulnerabilities_type_soup.a.string vulnerabilities_type = vulnerabilities_type.replace(" ","").replace(" ","").replace(" ","").replace(" ","") except: vulnerabilities_type = '' vulnerabilities_result_list_eve.append(vulnerabilities_type) vulnerabilities_time_html = vulnerabilities_detainled_data_list[4].encode().decode() #发布时间 vulnerabilities_time_soup = BeautifulSoup(vulnerabilities_time_html,'html.parser') try: vulnerabilities_time = vulnerabilities_time_soup.a.string vulnerabilities_time = vulnerabilities_time.replace(" ","").replace(" ","").replace(" ","") except: vulnerabilities_time = '' vulnerabilities_result_list_eve.append(vulnerabilities_time) vulnerabilities_attack_html = vulnerabilities_detainled_data_list[5].encode().decode() #威胁类型 vulnerabilities_attack_soup = BeautifulSoup(vulnerabilities_attack_html,'html.parser') try: vulnerabilities_attack = vulnerabilities_attack_soup.a.string vulnerabilities_attack = vulnerabilities_attack.replace(" ","").replace(" ","").replace(" ","") except: vulnerabilities_attack = '' vulnerabilities_result_list_eve.append(vulnerabilities_attack) vulnerabilities_update_html = vulnerabilities_detainled_data_list[6].encode().decode() #更新时间 vulnerabilities_update_soup = BeautifulSoup(vulnerabilities_update_html,'html.parser') try: vulnerabilities_update = vulnerabilities_update_soup.a.string vulnerabilities_update = vulnerabilities_update.replace(" ","").replace(" ","").replace(" ","") except: vulnerabilities_update = '' vulnerabilities_result_list_eve.append(vulnerabilities_update) vulnerabilities_firm_html = vulnerabilities_detainled_data_list[7].encode().decode() #厂商 vulnerabilities_firm_soup = BeautifulSoup(vulnerabilities_firm_html,'html.parser') try: vulnerabilities_firm = vulnerabilities_firm_soup.a.string vulnerabilities_firm = vulnerabilities_firm.replace(" ","").replace(" ","").replace(" ","") except: vulnerabilities_firm = '' vulnerabilities_result_list_eve.append(vulnerabilities_firm) vulnerabilities_source_html = vulnerabilities_detainled_data_list[8].encode().decode() #漏洞来源 vulnerabilities_source_soup = BeautifulSoup(vulnerabilities_source_html,'html.parser') try: vulnerabilities_source = vulnerabilities_source_soup.a.string vulnerabilities_source = vulnerabilities_source.replace(" ","").replace(" ","").replace(" ","") except: vulnerabilities_source = '' vulnerabilities_result_list_eve.append(vulnerabilities_source) #添加漏洞简介详情 vulnerabilities_title_html = vulnerabilities_detainled_soup1.find('div',attrs={'class':'d_ldjj'}) #定义 漏洞简介 块的soup vulnerabilities_title_html = vulnerabilities_title_html.encode().decode() vulnerabilities_title_soup2 = BeautifulSoup(vulnerabilities_title_html,'html.parser') try: vulnerabilities_titles1 = vulnerabilities_title_soup2.find_all(name='p')[0].string vulnerabilities_titles2 = vulnerabilities_title_soup2.find_all(name='p')[1].string vulnerabilities_titles = vulnerabilities_titles1 + vulnerabilities_titles2 vulnerabilities_titles = vulnerabilities_titles.replace(' ','').replace(' ','').replace(' ','').replace(' ','') except: vulnerabilities_titles = '' vulnerabilities_result_list_eve.append(vulnerabilities_titles) #漏洞公告 vulnerabilities_notice_html = vulnerabilities_detainled_soup1.find('div',attrs={'class':'d_ldjj m_t_20'}) #定义 漏洞公告 块的soup vulnerabilities_notice_html = vulnerabilities_notice_html.encode().decode() vulnerabilities_notice_soup2 = BeautifulSoup(vulnerabilities_notice_html,'html.parser') try: vulnerabilities_notice1 = vulnerabilities_notice_soup2.find_all(name='p')[0].string vulnerabilities_notice2 = vulnerabilities_notice_soup2.find_all(name='p')[1].string vulnerabilities_notice = vulnerabilities_notice1+vulnerabilities_notice2 vulnerabilities_notice = vulnerabilities_notice.replace(' ','').replace(' ','').replace(' ','') except: vulnerabilities_notice = '' vulnerabilities_result_list_eve.append(vulnerabilities_notice) #参考网址 vulnerabilities_reference_html = vulnerabilities_detainled_soup1.find_all('div',attrs={'class':'d_ldjj m_t_20'})[1] #定义 参考网址 块的soup vulnerabilities_reference_html = vulnerabilities_reference_html.encode().decode() vulnerabilities_reference_soup2 = BeautifulSoup(vulnerabilities_reference_html,'html.parser') try: vulnerabilities_reference = vulnerabilities_reference_soup2.find_all(name='p')[1].string vulnerabilities_reference = vulnerabilities_reference.replace(' ','').replace(' ','').replace(' ','').replace('链接:','') except: vulnerabilities_reference = '' vulnerabilities_result_list_eve.append(vulnerabilities_reference) #受影响实体 vulnerabilities_effect_html = vulnerabilities_detainled_soup1.find_all('div',attrs={'class':'d_ldjj m_t_20'})[2] #定义 受影响实体 块的soup vulnerabilities_effect_html = vulnerabilities_effect_html.encode().decode() vulnerabilities_effect_soup2 = BeautifulSoup(vulnerabilities_effect_html,'html.parser') try: vulnerabilities_effect = vulnerabilities_effect_soup2.find_all(name='p')[0].string vulnerabilities_effect = vulnerabilities_effect.replace(' ','').replace(' ','').replace(' ','').replace(' ','') except: try: vulnerabilities_effect = vulnerabilities_effect_soup2.find_all(name='a')[0].string vulnerabilities_effect = vulnerabilities_effect.replace(' ','').replace(' ','').replace(' ','').replace(' ','') except: vulnerabilities_effect = '' vulnerabilities_result_list_eve.append(vulnerabilities_effect) #补丁 vulnerabilities_patch_html = vulnerabilities_detainled_soup1.find_all('div',attrs={'class':'d_ldjj m_t_20'})[3] #定义 补丁 块的soup vulnerabilities_patch_html = vulnerabilities_patch_html.encode().decode() vulnerabilities_patch_soup2 = BeautifulSoup(vulnerabilities_patch_html,'html.parser') try: vulnerabilities_patch = vulnerabilities_patch_soup2.find_all(name='p')[0].string vulnerabilities_patch = vulnerabilities_patch.replace(' ','').replace(' ','').replace(' ','').replace(' ','') except: vulnerabilities_patch = '' vulnerabilities_result_list_eve.append(vulnerabilities_patch) for i in vulnerabilities_result_list_eve: vulnerabilities_result_list.append(i) print (re.findall(r'CNNVD-[s+S+]+',url)[0]) #漏洞信息写入excel def vulnerabilities_excel(excel): workbook = xlsxwriter.Workbook('spider_cnnvd.xlsx') worksheet = workbook.add_worksheet() row = 0 col = 0 worksheet.write(row,0,'漏洞名称') worksheet.write(row,1,'CNNVD编号') worksheet.write(row,2,'危害等级') worksheet.write(row,3,'CVE编号') worksheet.write(row,4,'漏洞类型') worksheet.write(row,5,'发布时间') worksheet.write(row,6,'攻击途径') worksheet.write(row,7,'更新时间') worksheet.write(row,8,'厂商') worksheet.write(row,9,'漏洞来源') worksheet.write(row,10,'漏洞描述') worksheet.write(row,11,'解决方案') worksheet.write(row,12,'参考链接') worksheet.write(row,13,'受影响实体') worksheet.write(row,14,'补丁') row = 1 n = 0 while n < len(excel): worksheet.write(row,col,excel[n]) worksheet.write(row,col+1,excel[n+1]) worksheet.write(row,col+2,excel[n+2]) worksheet.write(row,col+3,excel[n+3]) worksheet.write(row,col+4,excel[n+4]) worksheet.write(row,col+5,excel[n+5]) worksheet.write(row,col+6,excel[n+6]) worksheet.write(row,col+7,excel[n+7]) worksheet.write(row,col+8,excel[n+8]) worksheet.write(row,col+9,excel[n+9]) worksheet.write(row,col+10,excel[n+10]) worksheet.write(row,col+11,excel[n+11]) worksheet.write(row,col+12,excel[n+12]) worksheet.write(row,col+13,excel[n+13]) worksheet.write(row,col+14,excel[n+14]) row += 1 n += 15 workbook.close()def main():
agent_lists=[]
vulnerabilities_lists=[]
vulnerabilities_result_list = [] #获取代理 for i in range(1,2): url='http://www.xicidaili.com/nn/'+str(i) agent_list(url) #调用漏洞列表函数并获得漏洞链接列表 begin = datetime.datetime.now() page_count = sys.argv[3] j = 1 page_count = int(page_count) start_time = sys.argv[1] end_time = sys.argv[2] while j<=page_count: try: holes_url = 'http://cnnvd.org.cn/web/vulnerability/queryLds.tag?pageno=%d&repairLd='%j vulnerabilities_url_list(holes_url,start_time,end_time) print("已完成爬行第%d页"%j) print(' ') j+=1 except: print('爬取失败，更换代理重新爬取。') time.sleep(10) global bi bi = 0 pool = threadpool.ThreadPool(5) requests = threadpool.makeRequests(vulnerabilities_data,vulnerabilities_lists) [pool.putRequest(req) for req in requests] pool.wait() #漏洞信息写入excel vulnerabilities_excel(vulnerabilities_result_list) #爬行结束 end = datetime.datetime.now() total_time = end - begin print ('漏洞信息爬取结束') print ('爬行漏洞数量： ',len(vulnerabilities_lists)) print ('爬行时间： ',total_time) if __name__ == '__main__': main()