import re
import json
from odps import ODPS
from threading import Thread
import threading
from urllib import parse
import datetime
from lxml import etree
import random
import requests
import time
from models import *
def write_txt(html_data):
f = open("vacation_error_url.txt", 'a+')
f.write(html_data)
f.write("
")
f.close()
def get_comment_url(url):
response = requests.request("post", url)
html = etree.HTML(response.text)
data = html.xpath('//a[contains(@class,"pack js-taocan-item")]')
num_list = []
for data_0 in data:
product_id = data_0.attrib
product_id = product_id['data-subid']
num_list.append(product_id)
# if len(num_list) > 0:
# print(url)
# print(num_list)
url_id = url.split("id=")[1]
#print(url_id)
url_ = url.split("/user/")[0]
url_temp = url_ + '/user/comment/product/queryComments.json'
payload = "type=all&pageNo=1&pageSize=10&productIds=4068686556&rateStatus=ALL"
if url_id not in num_list:
payload = payload.replace('4068686556',str(url_id))
else :
temp_id = ""
for item in range(len(num_list)):
if item != len(num_list) -1:
temp_id = temp_id + str(num_list[item]) + '%2C'
else:
temp_id = temp_id + str(num_list[item])
payload = payload.replace('4068686556',str(temp_id))
#print(payload)
#url = "https://kmgl2.package.qunar.com/user/comment/product/queryComments.json"
headers = {
'content-type': 'application/x-www-form-urlencoded; charset=UTF-8',
'Cookie': 'QN1=00001480319827120b981f99; QN300=hotel.qunar.com'
}
response_id = requests.request("POST", url_temp, headers=headers, data = payload)
try:
json_data = json.loads(response_id.text)
return json_data['data']['ratingExcellent'],json_data['data']['ratingAverage'],json_data['data']['ratingAwful']
except:
write_txt(url_temp + "评论数错误")
return 0,0,0
def get_nodes_json():
url = r.lpop('test.com:vacation_url')
vacation_url = "https://dujia.qunar.com/golfz/routeList/adaptors/pcTop?isTouch=0&t=all&o=pop-desc&lm=60%2C60&fhLimit=0%2C60&q=%E4%B8%8A%E6%B5%B7&d=%E5%8D%97%E9%80%9A&s=all&qs_ts=1597025404801&tf=Ihot_02&ti=3&tm=l02&sourcepage=list&userResident=%E5%8D%97%E9%80%9A&random=749376&aroundWeight=1&qssrc=eyJ0cyI6IjE1OTcwMjU2OTU3NTIiLCJzcmMiOiJhbGwuZW52YyIsImFjdCI6InNjcm9sbCIsInJhbmRvbSI6IjM5NTUyIn0%3D&m=l%2CbookingInfo%2CbrowsingInfo%2Clm&displayStatus=pc&ddf=true&userId=00001b802e1827051e5021c6&hlFields=title&gpscity=%E5%8D%97%E9%80%9A&lines6To10=0"
if url:
city_name = re.search(r"%E5%8D%97%E9%80%9A_(.*)_all?",url).group(1) # 此处获取城市的名字
vacation_url = vacation_url.replace("%E4%B8%8A%E6%B5%B7",city_name) # 替换
product_city = url.split(" ")[1] # 度假产品管辖城市
url = vacation_url
vacation_number = 0
while(1):
response = requests.request("GET", url)
false = False
true = True
null = None
json_data = eval(response.text)
if response.status_code == 200:
if json_data['status'] == 0:
vacation_number = json_data['data']['list']['numFound']
break
#print(vacation_number)
start_num = 0
while(1):
if vacation_number > 0:
vacation_url = vacation_url.replace("%E4%B8%8A%E6%B5%B7",city_name) # 替换
page_start = re.search(r"&lm=(d+)%2C60",vacation_url).group(1)
vacation_url = vacation_url.replace("&lm=" + str(page_start) + "%2C60","&lm=" + str(start_num) + "%2C60")
#print(vacation_url)
process_response_data(vacation_url,product_city)
start_num = start_num + 60
vacation_number = vacation_number - 60
else:
break
def process_response_data(vacation_url,product_city): # 处理response的相应信息
flag_num = 0 # 设置请求标志位
while(1):
flag_num = flag_num + 1
response = requests.request("GET", vacation_url)
if response.status_code == 200:
json_data = json.loads(response.text)
# false = False
# true = True
# null = None
# json_data = eval(response.text)
if json_data['status'] == 0:
try:
#product_city = ''.join(json_data['data']['qdata']['destinations']) # list数据类型转换成str类型数据
product_district = json_data['data']['qdata']['realQuery'] # 目的地城市
except:
product_district = "NULL"
break
#write_txt(json_data['data']['qdata']['destinations'])
for data in json_data['data']['list']['results']:
temp_url = 'https:' + data['summary']['supplier']['url']
temp_id = data['ttsEnid']
temp_url = temp_url + '/user/detail.jsp?id=' + str(temp_id) # 拼接获取评论数据的url
product_negativeCount = 0 # 度假产品差评数
product_neutralCount = 0 # 度假产品中评数
product_positiveCount = 0 # 度假产品好评数
product_positiveCount,product_neutralCount,product_negativeCount = get_comment_url(temp_url)
# print(product_positiveCount,product_neutralCount,product_negativeCount)
vacation_data = qunar_Vacation_Product()
vacation_data.product_title = data['title'].encode('utf-16','surrogatepass').decode('utf-16') # 度假产品名称
vacation_data.product_title = vacation_data.product_title.replace("<em>","")
vacation_data.product_title = str(vacation_data.product_title.replace("</em>",""))
vacation_data.product_city = product_city # 度假产品管辖城市
vacation_data.product_district = product_district # 度假产品地级县市
vacation_data.product_price = data['accuratePrice'] # 度假产品价格
vacation_data.product_score = data['productScore'] # 度假产品评分
try:
vacation_data.product_commentCount = data['reviews'] # 度假产品评论数
except:
vacation_data.product_commentCount = 0 # 度假产品评论数
write_txt(vacation_url+ "缺少reviews")
vacation_data.product_positiveCount = product_positiveCount # 度假产品好评数
vacation_data.product_neutralCount = product_neutralCount # 度假产品中评数
vacation_data.product_negativeCount = product_negativeCount # 度假产品差评数
try:
vacation_data.product_soldCount = data['soldCount'] # 度假产品销量
except:
vacation_data.product_soldCount = 0 # 度假产品销量
write_txt(vacation_url + "缺少soldCount")
vacation_data.tripTime = data['details']['tripTime'] # 度假时长
vacation_data.hotel_night = data['details']['hotelNight'] # 住宿时长
try:
vacation_data.traffic_tool = data['details']['traffic'] # 出行工具
except:
vacation_data.traffic_tool = "无"
write_txt(vacation_url + "缺少['details']['traffic']")
vacation_data.supplier_name = data['summary']['supplier']['name'] # 商家名称
#vacation_data.supplier_url = "https:" + data['summary']['supplier']['url'] # 商家url链接
vacation_data.supplier_url = data['summary']['supplier']['url'].split('.')[0] # 商家url链接
vacation_data.create_time = datetime.datetime.now() # 抓取时间
r.lpush('test.com:store_url',"https:" + data['summary']['supplier']['url'])#度假商品商户的url
#vacation_data.save(force_insert=True)
break
else:
pass
else:
if flag_num > 5:
break
class parse_qunar_url_Thread(Thread):
def run(self):
while(1):
get_nodes_json()
#保存最终的数据
if __name__ == "__main__":
#get_nodes_json()
for i in range(150):
parse_qunar_url_thread = parse_qunar_url_Thread()
parse_qunar_url_thread.start()