from selenium import webdriver
import os
import time
import pymysql
from bs4 import BeautifulSoup
import requests
import threading
from selenium.webdriver.common.keys import Keys
h, pt, u, p, db = 'localhost', 3306, 'root', '', 'qqzone'
def mysql_fetch(sql, res_type='tuple'):
global h, pt, u, p, db
try:
conn = pymysql.connect(host=h, port=pt, user=u, passwd=p, db=db, charset='utf8mb4')
except Exception as e:
print(e)
return ()
if res_type == 'dic':
cursor = conn.cursor(pymysql.cursors.DictCursor)
else:
cursor = conn.cursor()
cursor.execute(sql)
conn.commit()
r = cursor.fetchall()
cursor.close()
conn.close()
return r
def mysql_write(sql):
global h, pt, u, p, db
try:
conn = pymysql.connect(host=h, port=pt, user=u, passwd=p, db=db, charset='utf8mb4')
except Exception as e:
print(e)
return 1
cursor = conn.cursor()
cursor.execute(sql)
conn.commit()
cursor.close()
conn.close()
return 0
# D:pyaction outiao_team_win
img_dir = 'C:/Users/Administrator/Desktop/1/toutiao_team/dl_img/'
img_dir = 'D:/pyaction/toutiao_team_win/dl_img/'
import random
def spider_webimg_dl_return_local_img_path(img_dir, img_url, media_type='img',
local_default='default.DONOT_REMOVE.png'):
r = '%s%s' % (img_dir, local_default)
if media_type == 'img':
try:
req = requests.get(img_url)
time.sleep(3)
if req.status_code != 200:
print('-!=200')
return r
time.sleep(30)
print(img_url)
bytes = req._content
# r = '%s%s%s%s%s' % (
# img_dir, time.strftime('%Y%m%d%H%M%S', time.localtime(time.time())), str(threading.get_ident()),
# img_url.split('!/c')[0].split('/')[-1].replace('*', '_'), '.png')
# print(r)
r = '%s%s%s%s%s' % (
img_dir, time.strftime('%Y%m%d%H%M%S', time.localtime(time.time())), str(threading.get_ident()),
str(random.randrange(1000, 9999)), '.png')
print(r)
filter_l = ['&', '=', '?', '-']
for fi in filter_l:
r.replace(fi, '')
if bytes != 0:
with open(r, 'wb')as f:
f.write(bytes)
except Exception as e:
print(e)
elif media_type == 'mp4':
try:
time.sleep(30)
print(img_url)
r = '%s%s%s%s%s' % (
img_dir, time.strftime('%Y%m%d%H%M%S', time.localtime(time.time())), str(threading.get_ident()),
img_url.split('.mp4?')[0].split('/')[-1].replace('*', '_'), '.mp4')
print(r)
req = requests.get(img_url)
time.sleep(3)
if req.status_code != 200:
print('-!=200')
return ''
bytes = req._content
time.sleep(210)
if bytes != 0:
with open(r, 'wb')as f:
f.write(bytes)
except Exception as e:
r = ''
print(e)
return r
driver = webdriver.Chrome()
myurl = 'https://weibo.com/u/1779073702'
myurl = 'https://weibo.com/u/1779073702?is_all=1'
myurl = 'https://weibo.com/login.php'
driver.get(myurl)
# 此处需要内存和cpu空余，能够支持dom解析和处理重js页面
time.sleep(10)
driver.refresh()
time.sleep(10)
# xp = '//*[@id="pl_common_top"]/div/div/div[3]/div[2]/ul/li[3]/a'
# try:
# # 此处解决了不能点击该元素报错，第三次尝试ok
# driver.find_element_by_xpath(xp).click()
# time.sleep(40)
#
# except Exception as e:
# print(e)
# os._exit(1024)
js = 'document.getElementsByClassName("username")[1].childNodes[0].value="welcome.cn";'
'document.getElementsByClassName("password")[0].childNodes[0].value="welcome";'
'document.getElementsByClassName("form_login_register")[0].childNodes[5].childNodes[0].click();'
js = 'document.getElementById("loginname").value="welcome.cn";'
'document.getElementsByName("password")[0].value="welcome";'
'document.getElementsByClassName("W_btn_a btn_32px")[0].click();'
js = 'document.getElementById("loginname").value="welcome.cn";'
'document.getElementsByName("password")[0].value="yidingok0422";'
'document.getElementsByClassName("W_btn_a btn_32px")[0].click();'
try:
# 需要键盘事件 - response
driver.find_element_by_id("loginname").send_keys(Keys.SPACE)
driver.find_element_by_id("loginname").send_keys(Keys.BACK_SPACE)
except Exception as e:
print(e)
try:
driver.execute_script(js)
time.sleep(30)
except Exception as e:
print(e)
os._exit(1024)
while True:
sql = 'SELECT id, words,imgurls,time_site FROM qqzoneshuoshuo WHERE lefttimes_weibo>0 AND INSTR(imgurls,".mp4")=0 AND id IN ( SELECT MAX(id) FROM qqzoneshuoshuo GROUP BY id_site) ORDER BY time_script DESC,id ASC ;'
res_content = mysql_fetch(sql, 'dic')
print(res_content)
if len(res_content) == 0:
continue
comment_l_sq = 0
for i in res_content[0:]:
# id, words,imgurls,time_site
dbid, content, img_list, time_site = i['id'], i['words'], i['imgurls'], i['time_site']
if 1 > 13:
if '天' in time_site or '月' in time_site:
continue
lh = int(time.strftime("%H", time.localtime()))
if lh - int(time_site.split(':')[0]) >= 24:
continue
if '早安' in content and lh >= 11:
continue
elif '晚安' in content and lh <= 20:
continue
# because another element <div> obscures it
content = content.split('展开全文')[0].split('上传')[0].split('浏览')[0].replace('"', ' ').replace("'", ' ')
content = content.replace('"', ' ').replace("'", ' ').replace('
', ' ')
filter_l = ['密龄素材空间', '评论']
for fi in filter_l:
content = content.replace(fi, ' ')
ad_url_l = ['567557180229', '565875313425', '545159271159', '546048319163', '567693004121'] # 补水喷雾 手链 面膜 防晒喷雾
ad_url_l = [554545061378, 567877981760, 567693004121, 567557180229, 545159271159, 546048319163, 565875313425]
ad_this = ad_url_l[int(time.time()) % len(ad_url_l)]
ad_url = 'https://item.taobao.com/item.htm?id={}'.format(ad_this)
print(ad_url)
hot_topic_list_url = 'https://weibo.com/u/1779073702/home'
js = 'window.location.href="{}"'.format(hot_topic_list_url)
driver.execute_script(js)
time.sleep(20)
try:
hot_url_l = [i.get_attribute('href') for i in driver.find_elements_by_css_selector('li>p>a')]
except Exception as e:
print(e)
continue
try:
# hot_url_l_index = random.choice([int(time.time()) % len(hot_url_l), 0, 1, 2])
hot_url_l_index = int(time.time()) % len(hot_url_l)
except Exception as e:
print(e)
continue
js = 'window.location.href="{}"'.format(hot_url_l[hot_url_l_index])
driver.execute_script(js)
time.sleep(random.randrange(13, 16))
comment_l = driver.find_elements_by_css_selector('.WB_row_line>li:nth-child(3)>a>span>span>span')
ele_clickable = False
comment_l_sq = random.choice([-1, -1, 1, 1, 1, 2, 2, 3])
# comment_l_sq = int(time.time()) % len(comment_l)
# '.WB_publish>div>textarea'
mytopic, myname = ' #doaez朵韵诗磁石娃娃燕窝润颜面膜# ', '南京同仁堂密龄白藜芦醇-燕窝美妆-DOAEZ朵韵诗-阿静@ '
mystr = '{}{}{}{}'.format(mytopic, myname, content, ad_url)
isc_log, rep_times, textarea_i, isc = 0, 3, 1, 0
for rep in range(rep_times):
comment_l_sq += random.choice([1, 1, 2, 2, 3])
comment_l_sq = min(comment_l_sq, len(comment_l))
isc_log = isc
for isc in range(isc_log, 100):
# time.sleep(1)
js = 'window.scrollTo(0,{})'.format(isc * 30)
driver.execute_script(js)
not_pull_up_index, pull_up_index = 1, 1
try:
comment_l[comment_l_sq].click()
time.sleep(2)
# 不需要键盘事件 - response
# driver.find_elements_by_tag_name("textarea")[not_pull_up_index].send_keys(Keys.SPACE)
# time.sleep(0.5)
# driver.find_elements_by_tag_name("textarea")[not_pull_up_index].send_keys(Keys.BACK_SPACE)
driver.find_elements_by_tag_name("textarea")[pull_up_index].send_keys(Keys.SPACE)
time.sleep(0.5)
driver.find_elements_by_tag_name("textarea")[pull_up_index].send_keys(Keys.BACK_SPACE)
#mystr = '{}{}'.format(mystr, random.randint(0, 9))
js = 'document.getElementsByTagName("textarea")[{}].value="{}"'.format(pull_up_index, mystr)
# js = 'document.getElementsByTagName("textarea")[{}].value="{}"'.format(not_pull_up_index, mystr)
print(js)
driver.execute_script(js)
time.sleep(2)
js = "document.getElementsByName('forward')[0].click();" # pull_up_index
# js = "document.getElementsByName('forward')[{}].click();".format(not_pull_up_index)
driver.execute_script(js)
time.sleep(2)
js = "document.getElementsByClassName('btn W_fr')[0].childNodes[0].click()" # pull_up_index
# js = "document.getElementsByClassName('btn W_fr')[{}].childNodes[0].click()".format(
# not_pull_up_index)
driver.execute_script(js)
time.sleep(2)
ele_clickable = True
not_pull_up_index += 1
# 不收起，输入框+1 comment_l[comment_l_sq].click()
# 收起，输入框
print('=================>')
comment_l[comment_l_sq].click()
print('<=================')
time.sleep(2)
break
except Exception as e:
print(e)
if e == 'list index out of range':
break
continue
if not ele_clickable:
continue
sql = 'UPDATE qqzoneshuoshuo SET lefttimes_weibo=lefttimes_weibo-1 WHERE id={}'.format(dbid)
print(sql)
try:
mysql_write(sql)
except:
pass
# 15min后刷新，循环存入数据,期间定时刷新,维持页面
for si in range(15):
try:
driver.refresh()
time.sleep(60)
time.sleep(random.randint(0, 10))
print(si)
except Exception as e:
print(145, e)