今天完成内容：
学习web
学习python爬取：爬取航班信息并存储数据库
#coding:utf-8
import requests
from lxml import etree
import pymysql
import random
def create():
db = pymysql.connect(host="localhost", user="root", password="123456", db="jichang") # 连接数据库
cursor = db.cursor()
cursor.execute("DROP TABLE IF EXISTS LINE")
sql = """CREATE TABLE LINE (
ID INT PRIMARY KEY AUTO_INCREMENT,
dp CHAR(255), # 出发地
ap CHAR(255), # 目的地
ndate CHAR(255), #出发日期
dtime CHAR(255), # 出发时间
atime CHAR(255), # 到达时间
dname CHAR(255), # 出发机场
aname CHAR(255), # 到达机场
flightname CHAR(255), # 航班信息
flightspace CHAR(255), # 舱位
rate CHAR(255), # 折扣率
price CHAR(255) # 价格
)"""
cursor.execute(sql)
db.close()
def insert(value):
db = pymysql.connect(host="localhost", user="root", password="123456", db="jichang")
cursor = db.cursor()
sql = "INSERT INTO LINE(dp,ap,ndate,dtime,atime,dname,aname,flightname,flightspace,rate,price) VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)"
try:
cursor.execute(sql, value)
db.commit()
print('插入数据成功')
except:
db.rollback()
print("插入数据失败")
db.close()
def chaxun(start,end,date):
cookies = {
'arrCityPy': start,
'depCityPy': end,
}
headers = {
'Connection': 'keep-alive',
'Pragma': 'no-cache',
'Cache-Control': 'no-cache',
'sec-ch-ua': '^\^',
'sec-ch-ua-mobile': '?0',
'Upgrade-Insecure-Requests': '1',
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.93 Safari/537.36',
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9',
'Sec-Fetch-Site': 'none',
'Sec-Fetch-Mode': 'navigate',
'Sec-Fetch-User': '?1',
'Sec-Fetch-Dest': 'document',
'Accept-Language': 'zh-CN,zh;q=0.9,en-US;q=0.8,en;q=0.7',
}
params = (
('unionId', '427'),
('godate', date),
('searchType', '0'),
)
response = requests.get(f'https://jipiao.114piaowu.com/{start}-{end}.html', headers=headers, params=params, cookies=cookies)
# print(response.text)
return response.content.decode('utf-8')
def jianxi(html,start,end,time):
xp = etree.HTML(html)
hangban_data = []
hangban_list = xp.xpath('//*[@class="jp_list"]//*[@class="mainDiv66"]')
for i in range(1,len(hangban_list)+1):
start_time = xp.xpath(f'//*[@class="jp_list"]/div[{i}]//*[@class="aboutfj"]/li[1]/b/text()')[0] #起飞时间
start_time = ''.join(start_time.split())
end_time = xp.xpath(f'//*[@class="jp_list"]/div[{i}]//*[@class="aboutfj"]/li[1]/p/text()')[0] #到达时间
end_time = ''.join(end_time.split())
start_address = xp.xpath(f'//*[@class="jp_list"]/div[{i}]//*[@class="aboutfj"]/li[2]/p[1]/text()')[0] #出发地点
start_address = ''.join(start_address.split())
end_address = xp.xpath(f'//*[@class="jp_list"]/div[{i}]//*[@class="aboutfj"]/li[2]/p[2]/text()')[0] #抵达地点
end_address = ''.join(end_address.split())
hangban = xp.xpath(f'//*[@class="jp_list"]/div[{i}]//*[@class="aboutfj"]/li[3]/p[1]//text()') #航班
hangban = ''.join([''.join(hangban[0].split()),''.join(hangban[1].split())])
# jijian_ranyou = xp.xpath(f'//*[@class="jp_list"]/div[{i}]//*[@class="aboutfj"]/li[4]/p[2]/text()')[0] #机建加燃油
# jijian_ranyou = ''.join(jijian_ranyou.split())
jipiao = [] #机票
jipiao_list = xp.xpath(f'//*[@class="jp_list"]/div[{i}]//*[@class="yd_list"]')
for j in range(1,len(jipiao_list)+1):
cangwei = xp.xpath(f'//*[@class="jp_list"]/div[{i}]//*[@class="yd_list"][{j}]/li[2]/b/text()')[0] #舱位
cangwei = ''.join(cangwei.split())
price = xp.xpath(f'//*[@class="jp_list"]/div[{i}]//*[@class="yd_list"][{j}]/li[3]/b//text()') #价格
price = ''.join([''.join(price[0].split()), ''.join(price[1].split())])
zhekou = xp.xpath(f'//*[@class="jp_list"]/div[{i}]//*[@class="yd_list"][{j}]/li[3]/em/text()')[0] #折扣
zhekou = ''.join(zhekou.split())
jipiao.append({'舱位':cangwei,'价格':price,'折扣':zhekou})
value = [start,end,time,start_time,end_time,start_address,end_address,hangban,cangwei,zhekou,price]
insert(value)
# hangban_data.append({'起飞时间':start_time,'到达时间':end_time,'出发地点':start_address,'抵达地点':end_address,'航班':hangban,'机建加燃油':jijian_ranyou,'机票':jipiao})
hangban_data.append({'起飞时间': start_time, '到达时间': end_time, '出发日期': time, '出发地点': start_address, '抵达地点': end_address, '航班': hangban,'机票': jipiao})
#hangban_data= [start_time,end_time,start_address,end_address,hangban,jipiao[cangwei],jipiao[zhekou],jipiao[price]]
return hangban_data
if __name__ == '__main__':
'''
start_time = time.time()
start1 = input('请输入出发地的拼音：')
end1 = input('请输入目的地的拼音：')
date = input('请输入出发日期：')
while len(re.findall('d{4}-d{2}-d{2}',date)) ==0:
date = input('请输入出发日期：')
print('正在查询')
'''
'''
for fcity in fcitys:
start1 = fcity
for tcity in tcitys:
end1 = tcity
html = chaxun(start1,end1,date)
datas = jianxi(html,start1,end1)
print(len(datas))
if len(datas) != 0:
for data in datas:
print(data)
else:
print('未查询到相关数据')
'''
create()
date = '2021-06-17'
start1 = 'shanghai'
end1 = 'beijing'
html = chaxun(start1, end1, date)
datas = jianxi(html, start1, end1,date)
print(len(datas))
if len(datas) != 0:
for data in datas:
print(data)
else:
print('未查询到相关数据')
看视频
遇到问题：
无
明日目标：
学习web端的开发
学习css