#!/usr/bin/env python
# -*- coding:utf-8 -*-
# Author : zhibo.wang
# E-mail : d_1206@qq.com
# Date : 18/03/23 14:22:58
# Desc : qq登陆 , 滑动验证暂没处理
import os
import time
from selenium import webdriver
from yichuxing.settings import qq_list
from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
class Login(object):
# 登陆qq，获取cookie
LoginURL = "http://c.easygo.qq.com/eg_toc/map.html?origin=csfw&cityid=110000"
def __init__(self, **kwargs):
self.qq_num = kwargs.get("qq_num")
self.qq_passwd = kwargs.get("qq_passwd")
def after_smoothly_login(self, driver):
cookie = {}
for elem in driver.get_cookies():
cookie[elem["name"]] = elem["value"]
#driver.quit()
return cookie
def get_cookie_by_Chrome(self):
try:
chromedriver = "C:Program Files (x86)GoogleChromeApplicationchromedriver.exe"
os.environ["webdriver.chrme.driver"] = chromedriver
driver = webdriver.Chrome(chromedriver)
#driver = webdriver.Chrome()
driver.set_page_load_timeout(10)
driver.get(self.LoginURL)
driver.find_element_by_id("u").send_keys(self.qq_num)
driver.find_element_by_id("p").send_keys(self.qq_passwd)
driver.maximize_window()
driver.find_element_by_id("go").click()
time.sleep(6)
if "宜出行" in driver.title:
return self.after_smoothly_login(driver)
elif "手机统一登录" in driver.title:
return None
except Exception:
# driver.close()
return None
def get_cookie_by_PhantomJS(self):
try:
dcap = dict(DesiredCapabilities.PHANTOMJS)
dcap["phantomjs.page.settings.userAgent"] = (
"Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.162 Safari/537.36"
)
driver = webdriver.PhantomJS(desired_capabilities=dcap)
driver.set_page_load_timeout(10)
driver.get(self.LoginURL)
driver.find_element_by_id("u").clear()
driver.find_element_by_id("u").send_keys(self.qq_num)
driver.find_element_by_id("p").clear()
driver.find_element_by_id("p").send_keys(self.qq_passwd)
driver.find_element_by_id("go").click()
time.sleep(6)
if "宜出行" in driver.title:
return self.after_smoothly_login(driver)
elif "手机统一登录" in driver.title:
return None
except:
# driver.close()
return None
class CookieException(Exception):
# 创建一个异常类，用于在cookie失效时抛出异常
def __init__(self):
Exception.__init__(self)
"""
if __name__ == "__main__":
#L = Login(qq_num="xxxx", qq_passwd="xxxx")
#L.get_cookie_by_Chrome()
"""
#!/usr/bin/env python
# -*- coding:utf-8 -*-
# Author : zhibo.wang
# E-mail : d_1206@qq.com
# Date : 18/03/23 14:22:58
# Desc : 宜出行热力图
import hashlib
import socket
import os
import json
import time
import random
import datetime
import requests
from yichuxing.settings import qq_list, s_fre, proxyMeta, is_proxy
from requests.exceptions import RequestException
#from utils.user_angents import agents
from data_utils.ali_oss import OSS2
from data_utils.time_convert import get_time_stamp
from yichuxing.yichuxing_utils.qqlogin import CookieException, Login
from data_utils.conmongodb import mongo_con_keepalive
from yichuxing.yichuxing_utils.create_grid import create_grid_by_center, get_gd_data
class Crawl():
db = mongo_con_keepalive()
header = {
"Host": "c.easygo.qq.com",
"Connection": "keep-alive",
"Accept": "application/json",
"Accept-Encoding": "gzip, deflate",
"Accept-Language": "zh-CN,zh;q=0.9",
"X-Requested-With": "XMLHttpRequest",
"Referer": "http://c.easygo.qq.com/eg_toc/map.html?origin=csfw",
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36",
}
start_url = "http://c.easygo.qq.com/api/egc/heatmapdata"
cookie_data = None
if is_proxy:
wait_time = [0.16, 0.17]
else:
wait_time = [3, 3.1, 3.2, 3.3, 3.4]
time_stamp = get_time_stamp()
time_local = time.localtime(int(time_stamp))
date = time.strftime("%Y-%m-%d", time_local)
proxies = {
"http": proxyMeta,
"https": proxyMeta,
}
fre_data = {"qq": None,"pwd": None}
fre = 0
pid = os.getpid()
oss = OSS2()
path_dir = None
website = "population_yichuxing"
qq_status = "yichuxing_qq_status"
def __init__(self):
self.path_dir = "population/yichuxing/{0}/".format(self.time_stamp)
self.db.get_collection('pathdir_dict').insert_one(
{'pathdir': self.path_dir, 'website': self.website, 'flag': False}
)
if self.db.get_collection(self.qq_status).find_one({"date": self.date}) is None:
self.db.get_collection(self.qq_status).remove({})
print("新的一天,新的开始 初始化所有账号")
self.db.get_collection(self.qq_status).insert_many(
[{"qq": i["qq"], "pwd": i["pwd"], "n": 0,
'status': False, "date": self.date} for i in qq_list]
)
super(Crawl, self).__init__()
def kill(self):
try:
os.system("kill {0}".format(self.pid))
except OSError as e:
print("kill pid error: ", e)
def get_cookie(self):
all_qq = self.db.get_collection(self.qq_status).find(
{"status": False}, {"_id": 0}
)
qq_list = [i for i in all_qq]
if len(qq_list) > 0:
self.fre = 0
self.fre_data = random.choice(qq_list)
qq_num = self.fre_data.get("qq")
qq_passwd = self.fre_data.get("pwd")
L = Login(qq_num=qq_num, qq_passwd=qq_passwd)
cookie_data = L.get_cookie_by_PhantomJS()
#cookie_data = L.get_cookie_by_Chrome()
if cookie_data:
self.cookie_data = cookie_data
elif len(qq_list) == 0:
print("没有账号了, 杀死自己")
self.kill()
def spyder_params(self, item):
# 生成 请求参数
params = {"lng_min": item.get("lng_min"),
"lat_max": item.get("lat_max"),
"lng_max": item.get("lng_max"),
"lat_min": item.get("lat_min"),
"level": 16,
"city": "",
"lat": "undefined",
"lng": "undefined",
"_token": ""
}
return params
def spyder(self, params):
time.sleep(random.choice(self.wait_time))
try:
if self.fre >= s_fre:
print("账号: {0}, 抓取次数达到上限, 更换qq账号".format(self.fre_data.get("qq")))
qq = self.fre_data.get("qq")
self.db.get_collection(self.qq_status).update_one(
{"qq": qq}, {"$set": {"status": True}}
)
self.get_cookie()
if is_proxy:
r = requests.get(self.start_url, headers=self.header,
cookies=self.cookie_data, params=params, proxies=self.proxies)
else:
r = requests.get(self.start_url, headers=self.header,
cookies=self.cookie_data, params=params)
if r.status_code == 200:
self.fre = self.fre + 1
try:
return r.json()
except:
raise CookieException
else:
raise CookieException
except RequestException :
self.spyder(params)
def get(self, params):
data_json = None
try:
data_json = self.spyder(params)
except CookieException:
print("账号: {0}, cookie 失效，获取新账号登陆, 并抓取".format(
self.fre_data.get("qq")))
qq = self.fre_data.get("qq")
self.db.get_collection(self.qq_status).update_one(
{"qq": qq}, {"$set": {"status": True}}
)
self.get_cookie()
data_json = self.spyder(params)
return data_json
def create_filename(self, url):
# 生成文件名
fname = '%s_%s_%s_%s.json' % (socket.gethostname(),
url.split('//')[-1].split('/')[0].replace('.', '-'),
hashlib.md5(url.encode()).hexdigest(),
str(time.time()).split('.')[0])
return fname
def start(self):
self.get_cookie()
for i in get_gd_data():
print("begin: ", i)
latlng_dict = create_grid_by_center(i)
print("将要抓取的次数: ", len(latlng_dict))
for o in latlng_dict:
print("抓取范围: ", o)
params = self.spyder_params(o)
data_json = self.get(params)
file_ = "{0}{1}".format(self.path_dir, self.create_filename("{0}{1}".format(self.start_url, params)))
if data_json.get("code") != 0:
print("code: {0}, 获取新的账号,再一次抓取".format(data_json.get("code")))
qq = self.fre_data.get("qq")
self.db.get_collection(self.qq_status).update_one(
{"qq": qq}, {"$set": {"status": True}}
)
self.get_cookie()
data_json = self.get(params)
if data_json.get("code") == 0 and len(data_json.get("data")) > 0:
data_json["cityname"] = o["cityname"]
#print(data_json)
self.oss.uploadfiledata(file_, json.dumps(data_json))
co = self.db.get_collection(self.qq_status).find({"status": False}).count()
print("剩余可用qq count: ", co)
if __name__ == "__main__":
c = Crawl()
c.start()
# 每个账号抓取次数
s_fre = 70
# 每次爬取方格的边长（0.04 > 4公里） 平移量
lat_offset = 0.04
lng_offset = 0.04
# 是否开始代理 True:开启, False:不开启
is_proxy = True
grade = {0:6, 1: 6, 2: 5, 3: 4, 4: 4, 5: 4} # 城市对应 抓取圈数
# 代理ip地址
proxyMeta = "http://xxx:xxx@proxy.abuyun.com:9020"
# qq 账号
qq_list = [
{"qq": "xxx", "pwd": "xxx"},
]
#!/usr/bin/env python
# -*- coding:utf-8 -*-
# Author : zhibo.wang
# E-mail : d_1206@qq.com
# Date : 18/03/23 16:28:43
# Desc :
import json
import numpy as np
from yichuxing.settings import lat_offset, lng_offset, grade
from data_utils.conmongodb import mongo_con_keepalive
from data_utils.location_convert import bd09togcj02
db = mongo_con_keepalive()
def get_gd_data():
city_list = db.get_collection("params_citys").find({"exists_city": True}, {"_id": 0}).sort(
"class")
location = []
for city in city_list:
if city.get("province") != "广东省":
if city.get("class") == 3:
center_lng = city.get("center_lng")
center_lat = city.get("center_lat")
del city["center_lng"]
del city["center_lat"]
lng, lat = bd09togcj02(center_lng, center_lat) # 转腾讯坐标系
city["lng"] = lng
city["lat"] = lat
location.append(city)
return location
def create_grid_by_center(location, n=None):
# 以城市中心点辐射n圈 即 4*4*(5*2)`2 1600平方公里
# 4*4 是每个区域的大小 区域大小可在setting里设置， (5*2)`2 5是圈数
lng, lat, city_class, cityname = location["lng"], location["lat"], location["class"], location["cityname"]
if n is None:
n = grade.get(city_class)
n = float(n)
bottom_lat, top_lat = lat - lat_offset*n, lat + lat_offset*n
left_lng, right_lng = lng - lng_offset*n, lng + lng_offset*n
lat_range = np.arange(bottom_lat, top_lat, lat_offset)
end_data = []
for lat_ in lat_range:
lng_range = np.arange(left_lng, right_lng, lng_offset)
for lng_ in lng_range:
end_data.append({"lng_min": lng_,
"lat_max":lat_ + lat_offset ,
"lng_max": lng_ + lng_offset,
"lat_min": lat_,
"cityname": cityname})
return end_data
"cityname" : "北京市", "province" : "北京市", "citycode" : "131", "center_lat" : 39.904211, # 百度坐标 "center_lng" : 116.407394, "class" : 0, "ftx_code" : "bj", "meituan_code" : "beijing", "meituan_id" : 1, "dianping_id" : 2, "dianping_code" : "beijing", "gd_adcode" : "110000", "gd_citycode" : "010", "shunqi_code" : "beijing", "xiecheng_code" : "BJS", "xiecheng_status" : true, "zhilian_code" : "beijing", "baidu_id" : 131, "exists_city" : true
{ "scale" : "20,50,100,200", "lng_a" : 116.550125, "lat_a" : 39.843624999999996, "lng_b" : 116.55662935278988, "lat_b" : 39.84962393215385, "lng_g" : 116.54429316621265, "lat_g" : 39.842540318493164, "gps_s" : "a", "count" : 800, "grid_y" : 159374, "grid_x" : 466200, "max_data" : 32000, "crawl_time" : "2018-05-29 10:03:37", "city" : "北京市", }
经纬度解密代码
http://c.easygo.qq.com/eg_toc/js/map-d76c21c16d.bundle.js
lng = 1e-6 * (250.0 * d['grid_x'] + 125.0)
lat = 1e-6 * (250.0 * d['grid_y'] + 125.0)
教程仅供技术研究学习使用，若有侵权，联系本人删除