selenium + 无头浏览器

phantomJS是一款无头浏览器, 之前我们通过selenium操作PhantomJS来完成动态加载数据的加载,

现在PhantomJS已经停止更新,  不过可以使用谷歌浏览器的无头浏览器来代替PhantomJS来完成上述操作

使用谷歌无头浏览器的实例代码如下:

from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from time import sleep
chrome_options = Options()
chrome_options.add_argument('--headless')
chrome_options.add_argument('--disable-gpu')

bro = webdriver.Chrome(chrome_options=chrome_options)

bro.get('https://www.baidu.com')
sleep(3)

print(bro.page_source)
bro.save_screenshot('1.png')

bro.quit()

执行下拉滚动条操作

from selenium import webdriver
from selenium.webdriver.chrome.options import Options
import time

chrome_options = Options()
chrome_options.add_argument('--headless')
chrome_options.add_argument('--disable-gpu')

bro = webdriver.Chrome(chrome_options=chrome_options)

bro.get(url='https://movie.douban.com/typerank?type_name=%E7%88%B1%E6%83%85&type=13&interval_id=100:90&action=')

time.sleep(3)

bro.save_screenshot('baidu/aiqing.png')

#让bro直行简单的js代码,模拟滚到到底部
js = 'window.scrollBy(500,100000)'
bro.execute_script(js)
time.sleep(3)

bro.save_screenshot('baidu/aiqing2.png')

#获取网页代码,保存到文件中
html = bro.page_source
with open('douban.html','w',encoding='utf8') as f:
    f.write(html)

bro.quit()

通过selenium加上下拉滚动条抓取懒加载图片

from selenium import webdriver
from selenium.webdriver.chrome.options import Options
import time

chrome_options = Options()
chrome_options.add_argument('--headless')
chrome_options.add_argument('--disable-gpu')

bro = webdriver.Chrome(chrome_options=chrome_options)

bro.get(url='http://sc.chinaz.com/tupian/ribenmeinv.html')
time.sleep(2)
with open('lanjiazai.html', 'w', encoding='utf8') as f:
    f.write(bro.page_source)
# bro.save_screenshot('lanjiazai.png')

bro.execute_script('window.scrollBy(0,10000)')
time.sleep(3)
with open('lanjiazai2.html', 'w', encoding='utf8') as f:
    f.write(bro.page_source)
# bro.save_screenshot('lanjiazai2.png')
time.sleep(1)


bro.close()

 qq空间登录

from selenium import webdriver
from time import sleep
bro = webdriver.Chrome(executable_path='chromedriver.exe')

url = 'https://qzone.qq.com'
bro.get(url)

bro.switch_to.frame('login_frame')
a_tag = bro.find_element_by_id('switcher_plogin')
a_tag.click()

bro.find_element_by_id('u').send_keys('328410948')
bro.find_element_by_id('p').send_keys('xxxxxx')
bro.find_element_by_id('login_button').click()

sleep(2)

#登陆成功后对应的主页
page_text = bro.page_source

 规避监测

from selenium import webdriver
from selenium.webdriver import ChromeOptions
option = ChromeOptions()
option.add_experimental_option('excludeSwitches', ['enable-automation'])
#实现了规避监测
bro = webdriver.Chrome(executable_path='chromedriver.exe',options=option)
bro.get('https://www.taobao.com/')