# -*- coding: UTF-8 -*- import math import random import sys import threading import time from time import ctime, sleep import requests import xlrd target_citycode_list = ['010', '021', '020', '0755'] # target_citycode_list = ['0755'] adcode_dic = {} FEXCEL = '高德地图API_城市编码对照表.xlsx' data = xlrd.open_workbook(FEXCEL) table = data.sheets()[1] nrows = table.nrows ncols = table.ncols for i in range(0, nrows): l = table.row_values(i) name_ = l[0] adcode = l[1] citycode = l[2] if citycode in target_citycode_list: if citycode not in adcode_dic: adcode_dic[citycode] = {} adcode_dic[citycode][adcode] = {} adcode_dic[citycode][adcode]['name'] = name_ adcode_dic[citycode][adcode]['adcode'] = adcode REQUEST_LIST = [] for i in adcode_dic: for ii in adcode_dic[i]: REQUEST_LIST.append(adcode_dic[i][ii]['adcode']) REQUEST_LEN = len(REQUEST_LIST) EACH_THREAD_REQUEST_NUM = 1 MAX_PAGINATION = 100 QPS = 50 QPS_TIME_UNIT = 1 # http://lbs.amap.com/api/webservice/guide/tools/info INFOCODE_OK = '10000' KEY_POOL_LIST = [] touse_key = '' def dynamic_write_pool_file(): global KEY_POOL_LIST file_name_key_pool = 'key_pool.pool' keypoollist_old = KEY_POOL_LIST KEY_POOL_LIST = [] f = open(file_name_key_pool, 'r', encoding='utf-8') KEY_POOL_LIST = [] for i in f: try: key = i.split(' ')[1].split() KEY_POOL_LIST.append(key[0]) except Exception: print(Exception) f.closed d1 = keypoollist_old.reverse() d2 = KEY_POOL_LIST.reverse() print(63, d1) print(64, d2) if d1 == d2: print(time.time(), '-old') else: print(time.time(), '66POOL-new') # if (d1>d2)-(d1<d2) == 0: # print('64POOL-new') # else: # print('66POOL-old') # # if cmp(KEY_POOL_LIST_old.reverse(),KEY_POOL_LIST.reverse())==0: # print('64POOL-new') # else: # print('66POOL-new') # if KEY_POOL_LIST == KEY_POOL_LIST_old: # pass # else: # print('POOL-new') dynamic_write_pool_file() URL_TYPE = 'http://restapi.amap.com/v3/place/text' touse_key = '' # keywords = '&keywords=' OFFSET_NUM = 24 OFFSET = '&offset=%s' % (OFFSET_NUM) CITYLIMIT = '&citylimit=true' EXTENTION = '&extention=all' # 120000 商务住宅 商务住宅相关 商务住宅相关 # 120100 商务住宅 产业园区 产业园区 # 120200 商务住宅 楼宇 楼宇相关 # 120201 商务住宅 楼宇 商务写字楼 # 120202 商务住宅 楼宇 工业大厦建筑物 # 120203 商务住宅 楼宇 商住两用楼宇 # 120300 商务住宅 住宅区 住宅区 # 120301 商务住宅 住宅区 别墅 # 120302 商务住宅 住宅区 住宅小区 # 120303 商务住宅 住宅区 宿舍 # 120304 商务住宅 住宅区 社区中心 # 由于高德至多返回1000条，所以type值以最小粒度请求，逐个请求 POI_TYPES_LIST = ['120000', '120100', '120200', '120201', '120202', '120203', '120300', '120301', '120302', '120303', '120304'] # POI_TYPES = '&types=120000|120100|120200|120201|120202120203|120300|120301|120302|120303|120304' URL_FOR_CHANGE_KEY = 'http://restapi.amap.com/v3/place/text?key=%s&types=060100&city=010&OFFSET=1' change_key_qps = 0 def change_key(): global touse_key, change_key_qps, KEY_POOL_LIST dynamic_write_pool_file() # 高德没有遵守自己的QPS/日限策略；所不能通过其返回码，来控制key的使用； pool_num = len(KEY_POOL_LIST) mean_use_key = random.randint(0, pool_num) for i in range(mean_use_key, pool_num, 1): key = KEY_POOL_LIST[i] if key == touse_key: if i == pool_num: change_key() return else: continue touse_key = key url = URL_FOR_CHANGE_KEY % (touse_key) try: change_key_qps += 1 if change_key_qps % QPS == 0: sleep(QPS_TIME_UNIT) r = requests.get(url) json_ = r.json() except Exception: print('requests.get(url)', Exception) change_key() return infocode = json_['infocode'] if not infocode == INFOCODE_OK: if i == pool_num: sys.exit('NOInvalidKEY') change_key() return return requests_counter = 0 todo_list = REQUEST_LIST # {adcode:[[],[]]} tosupply_dic = {} def supply_dic(request): global tosupply_dic, requests_counter, todo_list, touse_key, POI_TYPES_LIST, OFFSET_NUM if requests_counter == 0: change_key() for type in POI_TYPES_LIST: url = '%s?key=%s&city=%s&type=%s%s%s' % (URL_TYPE, touse_key, request, type, OFFSET, CITYLIMIT) if requests_counter % QPS == 0: sleep(QPS_TIME_UNIT) try: requests_counter += 1 r = requests.get(url) r_json = r.json() except Exception: # 冗余 if request not in todo_list: todo_list.append(request) infocode = r_json['infocode'] if infocode == '10000': count = r_json['count'] page_count = math.ceil(int(count) / OFFSET_NUM) if page_count > 0: for page in range(1, page_count, 1): url_ = '%s&page=%s' % (url, page) print(url_) try: requests_counter += 1 r_ = requests.get(url_) r_json_ = r_.json() except Exception: # 冗余 if request not in todo_list: todo_list.append(request) infocode_ = r_json_['infocode'] if infocode_ == '10000': pois_list = r_json['pois'] if request not in tosupply_dic: tosupply_dic[request] = [] tosupply_dic[request].append(pois_list) if request in todo_list: list_index = todo_list.index(request) del todo_list[list_index] else: if request not in todo_list: todo_list.append(request) change_key() else: if request not in todo_list: todo_list.append(request) change_key() MAX_EXCEPTION_URL_NUM = 0 def deal_exception_list(): global todo_list print(todo_list) len_ = len(todo_list) if len_ > MAX_EXCEPTION_URL_NUM: for nloop in range(0, len_, 1): adcode = REQUEST_LIST[nloop] supply_dic(adcode) else: return deal_exception_list() class MyThread(threading.Thread): def __init__(self, func, args, name=''): threading.Thread.__init__(self) self.name = name self.func = func self.args = args def run(self): self.func(self.args) def main(): print('starting at:', ctime()) threads_list = [] thread_sum = math.ceil(REQUEST_LEN / EACH_THREAD_REQUEST_NUM) for nloop in range(0, thread_sum, 1): adcode = REQUEST_LIST[nloop] print(184, adcode) thread_instance = MyThread(supply_dic, (adcode), supply_dic.__name__) threads_list.append(thread_instance) # 主进程将在所有非守护进程退出后，退出 for t in threads_list: t.setDaemon = False t.start() # wait for all thrades to finish for t in threads_list: t.join() deal_exception_list() FGEN = 'GEN_GD_business_building.csv' fo = open(FGEN, 'w', encoding='utf-8-sig') fo.write( 'id,name,type,typecode,biz_type,address,location,tel,distance,biz_ext,pname,cityname,adname,shopid,shopinfo,poiweight ') fo.closed fo = open(FGEN, 'a', encoding='utf-8-sig') for request in tosupply_dic: l = tosupply_dic[request] for ll in l: for dic_ in ll: str = '%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s ' % ( dic_['id'], dic_['name'], dic_['type'], dic_['typecode'], dic_['biz_type'], dic_['address'], dic_['location'].replace(',', ' '), dic_['tel'], dic_['distance'], dic_['biz_ext'], dic_['pname'], dic_['cityname'], dic_['adname'], dic_['shopid'], dic_['shopinfo'], dic_['poiweight']) fo.write(str) fo.closed if __name__ == '__main__': main()