苏宁selenium代码
import requests
import re
import time
import mysql
import json
def getitem(keyword, n):
itemurl = "https://search.suning.com/emall/searchV1Product.do?keyword=" + keyword + "&pg=01&cp=" + "0" + "&paging=" + str(n)
print("itemurl:"+itemurl)
response = requests.get(itemurl, headers=headers).text.replace("|||||", ",")
# print(response)
# 商品的prdid 和 shopid
getID = re.compile(r'<span class="def-price" datasku="(.*?),(.*?)" brand_id=".*?" mdmGroupId=".*?">')
IDlist = re.findall(getID, response)
# 商品的图片和描述
getDetail = re.compile(r'<img alt="(.*?)" src="(.*?)" picPriority=".*?">')
dList = re.findall(getDetail, response)
# 商品的评价数
# getComment = re.compile(r'<i>(.*?)</i>评价</a>')
# cList = re.findall(getComment, response)
i = 0
print("长度:"+str(len(IDlist)))
item = []
for key in IDlist:
print("——————————————————————第"+str(i+1)+"件商品——————————————————————")
print("描述:"+dList[i][0]+" 图片:"+dList[i][1])
name = dList[i][0]
image = dList[i][1]
# if cList:
# print("评价:"+cList[i])
# else:
# print("评价:暂无")
i = i+1
# getprice(key[0], key[1])
durl = "https://product.suning.com/" + key[1] + "/" + key[0] + ".html"
print("商品网址:" + durl)
link = durl
shopid2 = int(key[1])
pjurl = "https://product.suning.com/pds-web/ajax/getApiRemoteMap_" + str(shopid2) + "_shopScoreCallback.html?"
# print(pjurl)
response2 = requests.get(pjurl, headers=headers).text.replace("\", "")
getpj = re.compile(
r'{"parentIndexName":"评价","parentIndexScore":"(.*?)","serParentIndexCateCon":".*?","parentIndexCode":".*?"},{"parentIndexName":"物流","parentIndexScore":"(.*?)","serParentIndexCateCon":".*?","parentIndexCode":".*?"},{"parentIndexName":"售后","parentIndexScore":"(.*?)"')
pjlist = re.findall(getpj, response2)
if pjlist:
print("店铺评分:" + str(pjlist[0][0]))
print("物流评分:" + str(pjlist[0][1]))
print("售后评分:" + str(pjlist[0][2]))
else:
print("评分:暂无店铺各项评分")
response3 = requests.get(durl, headers=headers).text
getNum = re.compile(r'<span>货号</span> </div> </td> <td class="val">(.*?)</td>')
numlist = re.findall(getNum, response3)
id = ""
if numlist:
id = str(numlist[0])
print("货号:" + str(numlist[0]))
else:
id = "暂无"
print("货号:暂无此商品货号")
getShop = re.compile(r'<a id="chead_indexUrl" href="(.*?)" title="(.*?)">')
shopList = re.findall(getShop, response3)
shopI = ""
shopH = ""
if shopList:
shopI = shopList[0][1]
shopH = shopList[0][0]
print("店铺:"+shopI+" 店铺链接:"+shopH)
else:
shopI = "暂无"
shopH = "暂无"
print("店铺:" + "暂无" + " 店铺链接:" + "暂无")
pjurl = "https://review.suning.com/ajax/getClusterReview_labels/style--0000000"+key[0]+"-"+key[1]+"-----commodityrLabels.htm?"
# print(pjurl)
reponse4 = requests.get(pjurl, headers=headers).text.replace('commodityrLabels(', '').replace('})', '}')
# print(reponse4)
if reponse4:
d = json.loads(reponse4)
print("评论关键字个数:"+str(len(d["commodityLabelCountList"])))
for q in d["commodityLabelCountList"]:
print("label:"+q["labelName"]+" num:"+str(q["labelCnt"]))
else:
print("无")
priceurl = " https://pas.suning.com/nspcsale_0_0000000" + key[0] + "_0000000" + key[0] + "_" + key[1] + "_60_311_3110199_20089_1000095_9095_10638_Z001___R1901001_0.36_0___000060021____0___448.224_2_01_20002_20006__.html?"
# print(priceurl)
res = requests.get(priceurl, headers=headers).text
# print(res)
getK = re.compile(r'"gbPrice":"(.*?)"')
keyL = re.findall(getK, res)
price = 0.0
if keyL:
price = str(keyL[0])
print("价格:" + str(keyL[0]))
else :
getK = re.compile(r'"netPrice":"(.*?)"')
keyL = re.findall(getK, res)
if keyL:
price = str(keyL[0])
print("价格:" + str(keyL[0]))
else:
price = "无"
print("无价格:")
orgin = "苏宁"
item.append([time.strftime("%Y-%m-%d"), id, price, name, link, image, orgin])
print("++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++")
mysql.insert_item(item)
# if not mysql.get_conn()[1]:
# mysql.insert_item(item)
# else:
# mysql.update_item(item)
# def getprice(prdid, shopid):
# durl = "https://product.suning.com/"+shopid+"/"+prdid+".html"
# print("商品网址:"+durl)
#
# shopid2 = int(shopid)
# pjurl = "https://product.suning.com/pds-web/ajax/getApiRemoteMap_"+str(shopid2)+"_shopScoreCallback.html?"
# # print(pjurl)
# response2 = requests.get(pjurl).text.replace("\", "")
# getpj = re.compile(r'{"parentIndexName":"评价","parentIndexScore":"(.*?)","serParentIndexCateCon":".*?","parentIndexCode":".*?"},{"parentIndexName":"物流","parentIndexScore":"(.*?)","serParentIndexCateCon":".*?","parentIndexCode":".*?"},{"parentIndexName":"售后","parentIndexScore":"(.*?)"')
# pjlist = re.findall(getpj, response2)
# if pjlist:
# print("店铺评分:"+str(pjlist[0][0]))
# print("物流评分:"+str(pjlist[0][1]))
# print("售后评分:"+str(pjlist[0][2]))
# else:
# print("评分:暂无店铺各项评分")
#
# response3 = requests.get(durl).text
# getNum = re.compile(r'<span>货号</span> </div> </td> <td class="val">(.*?)</td>')
# numlist = re.findall(getNum, response3)
# if numlist:
# print("货号:"+str(numlist[0]))
# else:
# print("货号:暂无此商品货号")
#
# priceurl = " https://pas.suning.com/nspcsale_0_0000000"+prdid+"_0000000"+prdid+"_"+shopid+"_60_311_3110199_20089_1000095_9095_10638_Z001___R1901001_0.36_0___000060021____0___448.224_2_01_20002_20006__.html?"
# res = requests.get(priceurl).text
# # print(res)
# getK = re.compile(r'"gbPrice":"(.*?)"')
# keyL = re.findall(getK, res)
# if keyL:
# print("价格:"+str(keyL[0]))
# else:
# getK = re.compile(r'"netPrice":"(.*?)"')
# keyL = re.findall(getK, res)
# print("价格:"+str(keyL[0]))
if __name__ == '__main__':
headers = {
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36(KHTML, like Gecko) Chrome/83.0.4103.106 Safari/537.36"
}
key = {"手机", "运营商", "智能数码", "家用电器", "帮客", "房产", "厨卫", "生活家电", "厨具", "电脑办公", "相机", "电竞", "家具", "家装", "家纺", "灯具",
"食品", "酒水", "生鲜", "特产", "美妆", "个护", "清洁", "宠物", "母婴", "玩具", "车床", "童装", "运动", "户外", "国米", "骑行", "女装", "男装",
"内衣", "鞋靴", "箱包", "钟表", "珠宝", "艺术", "汽车", "电摩", "汽车用品", "图书", "艺术", "原版", "文学", "医药健康", "计生情趣", "理财", "分期",
"便民"}
for i in range(200):
getitem("外套", i)