某宝使用Chrome 自动化会被检测，所以采用Firefox浏览器做自动化，运行此脚本请按照Firefox和将驱动文件放在运行目录中的driversgeckodriver.exe路径
# -*- coding: utf-8 -*- # @Time : 2020-4-22 22:39 # @Author : hlikex # @File : main.py from selenium import webdriver from urllib import parse from selenium.webdriver.chrome.options import Options import logging import time import random import pandas as pd import os import re class Taobao: def __init__(self): self.root_url = "https://ad.alimama.com/zhaoshang/cpevent/index.htm?srcCode=1&pageNo=1&onlyCanJoin=0&pageSize=800" if os.path.exists("data.xls"): self.df = pd.read_excel("data.xls",index_col=0) else: self.df = pd.DataFrame(columns=("团长",'旺旺','V标团长','总成交金额（星级）',"总成交笔数（星级）",'打爆能力（星级）','服务时间','钉钉','团队介绍')) self.driver = webdriver.Firefox(executable_path=os.getcwd() + "driversgeckodriver.exe") self.driver.implicitly_wait(10) logging.basicConfig(format='%(asctime)s - %(pathname)s[line:%(lineno)d] - %(levelname)s: %(message)s', level=logging.WARNING) def isElementExist(self,id, element): try: self.driver.find_element(id,element) return True except: return False def login(self): self.driver.get(self.root_url) try: self.driver.switch_to.frame('taobaoLoginIfr') self.driver.find_element_by_id('fm-login-id').send_keys("username") # 你的账号 self.driver.find_element_by_id('fm-login-password').send_keys('password') # 你的密码 self.driver.find_element_by_css_selector('[class="fm-button fm-submit password-login"]').click() time.sleep(3) print(self.driver.current_url) self.driver.switch_to.default_content() self.driver.refresh()
time.sleep(2.5)
except Exception as e: print(e) def getInfo(self): elements = self.driver.find_elements_by_tag_name('tr') for element in elements: try: html = element.get_attribute("outerHTML") Name = re.findall(r"{cpName:'(.*?)',cpPubId:'.*?'}",html)[0] list = self.df["团长"].values.tolist() if Name in list: continue WUid = re.findall(r'uid=(.*?)&',html)[1] star = re.findall(r'app/gallery/mx-effects/star?num=(d)',html) if str(html).find('V标团长')!=-1: V = "是" else: V = "否" element.find_element_by_css_selector('[class="card pointer"]').click() time.sleep(2) html2 = self.driver.find_element_by_css_selector('[class="adv-threebq adv-threebu"]').get_attribute("outerHTML") ServiceDate = re.findall(r"服务时间：</div><div>(.*?)</div>",html2)[0] dings = re.findall(r'dingtalk_id=(.*?)"',html2) ding = "、".join(str(parse.unquote(i)) for i in dings) # self.driver.find_element_by_class_name('word-break').get_attribute("outerHTML") about = self.driver.find_element_by_xpath('/html/body/div[4]/div/div/div[2]/div/div[1]/div/div[2]/div[3]/p[2]').text if str(about).find('...更多')!=-1: self.driver.find_element_by_xpath('/html/body/div[4]/div/div/div[2]/div/div[1]/div/div[2]/div[3]/p[2]/span').click() about = self.driver.find_element_by_xpath('/html/body/div[5]/div/div/div/div[1]/div[2]').get_attribute('outerHTML') about = re.sub(r'<.*?>', "", about) self.driver.find_element_by_link_text('确定').click() print({"团长":Name,'旺旺':WUid,'V标团长':V,'总成交金额（星级）': star[0],"总成交笔数（星级）": star[1],'打爆能力（星级）': star[2],'服务时间':ServiceDate,'钉钉':ding,'团队介绍':about}) num = len(self.df) df2 = pd.DataFrame( {"团长":Name,'旺旺':WUid,'V标团长':V,'总成交金额（星级）': star[0],"总成交笔数（星级）": star[1],'打爆能力（星级）': star[2],'服务时间':ServiceDate,'钉钉':ding,'团队介绍':about}, index=[num + 1]) self.driver.switch_to.default_content() time.sleep(1.5) self.driver.find_element_by_link_text('关闭').click() self.df = self.df.append(df2) self.df.to_excel("data.xls") except IndexError as e: pass except Exception as e : if self.isElementExist('link text',"确定"): self.driver.find_element_by_link_text('关闭').click() if self.isElementExist('link text',"关闭"): self.driver.find_element_by_link_text('关闭').click() self.df.to_excel("data.xls") print(e) def nextPage(self): try: self.driver.switch_to.default_content() self.driver.find_element_by_css_selector('[class="mc-iconfont adv-threefY rotate180 "]').click() except Exception as e: print(e) def run(self): self.login() while True: try: page = re.findall(r"pageNo=(.*?)&",str(self.driver.current_url))[0] except IndexError: print("用户被注销") print("当前页码：{}".format(page)) self.getInfo() time.sleep(60) self.driver.refresh() if __name__ == '__main__': T = Taobao() T.run()