import re
import json
from odps import ODPS
from threading import Thread
import threading
from urllib import parse
import datetime
from lxml import etree
import random
import requests
import time
from models import *
store_urls_data = {"https://fh.dujia.qunar.com/",'https://aaag9.package.qunar.com'}
def write_txt(html_data):
f = open("a.txt", 'a+')
f.write(html_data)
f.write("
")
f.close()
# 获取到的商户信息
def process_nodes_list(url):
res_text = requests.get(url)
html = etree.HTML(res_text.text)
try:
score = html.xpath("//var[@class='score']/text()")[0]
except:
# print(url)
score = 0.0
#print(score)
try:
try:
up_down = html.xpath('//div[@class="rankline"]//i[contains(@class,"up")]')[0]
except:
up_down = html.xpath('//div[@class="rankline"]//i[contains(@class,"down")]')[0]
up_down = up_down.attrib
except:
try:
up_down = html.xpath('//div[@class="rankline"]//i[contains(@class,"equal")]')[0]
up_down = up_down.attrib
except:
up_down = "NULL" # 此处为空值
print(url)
print(up_down)
try:
store_name = html.xpath("//div[@class='shop-rank']/strong/text()")[0]
except:
store_name = "NULL"
print(url)
res_li = html.xpath("//div[@class='business']//li/text()")
if len(res_li) > 9:
store = qunar_Store()
store.store_name = store_name # 店铺名称
store.store_score = score # 店铺评分
if 'up' in str(up_down):
store.store_rankline = 1
store.store_percent = int(re.search(r'd+',str(up_down)).group(0)) / 100
if 'down' in str(up_down):
store.store_rankline = -1
store.store_percent = int(re.search(r'd+',str(up_down)).group(0)) / 100
if 'equal' in str(up_down):
store.store_rankline = 0
store.store_percent = 0.0
if str(up_down) == "NULL":
store.store_rankline = 2 # 出现2表示商户数据为空
store.store_percent = 0.0
store.company_name = res_li[1] # 公司的名字
store.company_legal_person = res_li[3] # 公司的法人
store.licence_num = res_li[5] #
store.trading_certificate = res_li[7]
store.business_scope = res_li[9]
store.create_time = datetime.datetime.now()
store.save(force_insert=True)
# except:
# write_txt(url)
# if goods_list:
# data_0.up_product_to_odps(goods_list)
def get_nodes_json():
url = r.lpop('test.com:store_url')
#url = 'https://zqlr1.package.qunar.com/'
if url and url not in store_urls_data:
process_nodes_list(url)
class parse_qunar_url_Thread(Thread):
def run(self):
while(1):
get_nodes_json()
#保存最终的数据
if __name__ == "__main__":
for i in range(50):
parse_qunar_url_thread = parse_qunar_url_Thread()
parse_qunar_url_thread.start()