import re
import json
from odps import ODPS
from threading import Thread
import threading
from urllib import parse
import datetime
from lxml import etree
import random
import requests
import time
from models import *
# def write_txt(html_data):
# f = open("a.txt", 'a+')
# f.write(html_data)
# f.write("
")
# f.close()
def get_ticket_id(url): # 获取门票景点的id
response = requests.request("GET", url)
while(response.status_code == 404):
time.sleep(2)
response = requests.request("GET", url)
html = etree.HTML(response.text)
data = html.xpath('//link[@rel="canonical"]')
try:
data_0 = data[0].attrib
num = re.search(r'd+',str(data_0)).group(0)
return num
except:
print(url)
print(response.status_code)
return '0'
def get_nodes_json():
url = r.lpop('test.com:ticket_url')
#print(url)
if url:
district = re.search(r"list_(.*).html",url).group(1) # 获取管辖区域名称
district = parse.unquote(district) # 对管理辖区进行url转ascii
response = requests.request("POST", url)
while(response.status_code == 404):
response = requests.request("POST", url)
#print(response.status_code,"status_code")
html = etree.HTML(response.text)
data = html.xpath('//div[@class="result_list"]/div[contains(@class,"sight_item")]')
ticket_num = html.xpath('//div[contains(@id,"pager-container")]')[0]
ticket_num = re.search(r'd+',str(ticket_num.attrib))[0]
ticket_num = int(ticket_num)
page_num = int(ticket_num / 15 + 1)
#print(page_num)
url_temp = url
while(page_num > 0):
url_temp = url.replace("page=1","page=" + str(page_num))
#print(url_temp,page_num)
process_response_data(url_temp)
page_num = page_num - 1
def process_response_data(url): # 处理response的相应信息
#url = 'https://piao.qunar.com/ticket/list_%E6%96%97%E9%97%A8%E5%8C%BA.html?from=mps_search_suggest_c&keyword=%E6%96%97%E9%97%A8%E5%8C%BA&page=1'
district = re.search(r"list_(.*).html",url).group(1) # 获取管辖区域名称
district = parse.unquote(district) # 对管理辖区进行url转ascii
response = requests.request("POST", url)
while(response.status_code == 404):
response = requests.request("POST", url)
#print(response.status_code,"status_code")
html = etree.HTML(response.text)
data = html.xpath('//div[@class="result_list"]/div[contains(@class,"sight_item")]')
ticket_num = html.xpath('//div[contains(@id,"pager-container")]')[0]
ticket_num = re.search(r'd+',str(ticket_num.attrib))[0]
ticket_num = int(ticket_num)
# print(ticket_num)
for item in range(len(data)):
data_0 = data[item].attrib
ticket = qunar_Tickets()
ticket.ticket_name = data_0['data-sight-name']
ticket.ticket_pic = data_0['data-sight-img-u-r-l']
url_ticket = data[item].xpath('./div[@class="sight_item_detail clrfix"]/div[@class="sight_item_about"]/h3[@class="sight_item_caption"]//@href') # 此处用来拼接单独门票旅游景点url
url_ticket = 'https://piao.qunar.com' + "".join(url_ticket)
num = 0
try:
num = get_ticket_id(url_ticket)
except:
continue
#print(num,"num")
json_url = 'https://piao.qunar.com/ticket/detailLight/sightCommentList.json?sightId=13893&index=1&page=1' # url请求获取json数据，处理评论数
json_url = json_url.replace('13893',num)
# print(response_json.status_code,"response_json.status_code")
# print(response_json.text)
response_json = requests.request("GET", json_url)
#print(response_json.status_code)
while(len(response_json.text) < 48):
response_json = requests.request("GET", json_url)
json_data = json.loads(response_json.text)
#print(json_data['data']['tagList'])
for data_com in json_data['data']['tagList']:
if data_com['tagType'] == 3:
ticket.ticket_negativeCount = data_com['tagNum']
if data_com['tagType'] == 2:
ticket.ticket_neutralCount = data_com['tagNum']
if data_com['tagType'] == 1:
ticket.ticket_positiveCount = data_com['tagNum']
if data_com['tagType'] == 0:
ticket.ticket_commentCount = data_com['tagNum']
try:
ticket_level = data[item].xpath('./div[@class="sight_item_detail clrfix"]/div[@class="sight_item_about"]/div[@class="sight_item_info"]/div[@class="clrfix"]/span[@class="level"]/text()') # 景区等级
ticket_level = "".join(ticket_level)
ticket.ticket_level = ticket_level
except:
ticket.ticket_level = " "
try:
ticket_price = data[item].xpath('./div[@class="sight_item_detail clrfix"]/div[@class="sight_item_pop"]/table/tr/td/span[@class="sight_item_price"]/em/text()') # 获取票面价
ticket_price = "".join(ticket_price)
ticket.ticket_price = float(ticket_price)
except:
ticket.ticket_price = 0
try:
ticket_city = data_0['data-districts']
ticket_city = ticket_city.split("·")[0] + " " + ticket_city.split("·")[1]
ticket.ticket_city = ticket_city
ticket.ticket_district = district
ticket.ticket_location = data_0['data-address']
ticket.ticket_sales = data_0['data-sale-count']
ticket.create_time = datetime.datetime.now()
ticket.save(force_insert=True)
except:
pass
class parse_qunar_url_Thread(Thread):
def run(self):
while(1):
get_nodes_json()
#保存最终的数据
if __name__ == "__main__":
#get_nodes_json()
for i in range(30):
parse_qunar_url_thread = parse_qunar_url_Thread()
parse_qunar_url_thread.start()