Python爬虫入门-3

---恢复内容开始---

云打码平台

• 注册:普通用户和开发者用户
• 登录:
• 登录普通用户(查看余额)
• 登录开发者用户:
• 创建一个软件:我的软件->创建软件
• 下载示例代码:开发者中心->下载最新的DLL->pythonHttp示例代码下载

import http.client, mimetypes, urllib, json, time, requests

######################################################################

class YDMHttp:

    apiurl = 'http://api.yundama.com/api.php'
    username = ''
    password = ''
    appid = ''
    appkey = ''

    def __init__(self, username, password, appid, appkey):
        self.username = username  
        self.password = password
        self.appid = str(appid)
        self.appkey = appkey

    def request(self, fields, files=[]):
        response = self.post_url(self.apiurl, fields, files)
        response = json.loads(response)
        return response
    
    def balance(self):
        data = {'method': 'balance', 'username': self.username, 'password': self.password, 'appid': self.appid, 'appkey': self.appkey}
        response = self.request(data)
        if (response):
            if (response['ret'] and response['ret'] < 0):
                return response['ret']
            else:
                return response['balance']
        else:
            return -9001
    
    def login(self):
        data = {'method': 'login', 'username': self.username, 'password': self.password, 'appid': self.appid, 'appkey': self.appkey}
        response = self.request(data)
        if (response):
            if (response['ret'] and response['ret'] < 0):
                return response['ret']
            else:
                return response['uid']
        else:
            return -9001

    def upload(self, filename, codetype, timeout):
        data = {'method': 'upload', 'username': self.username, 'password': self.password, 'appid': self.appid, 'appkey': self.appkey, 'codetype': str(codetype), 'timeout': str(timeout)}
        file = {'file': filename}
        response = self.request(data, file)
        if (response):
            if (response['ret'] and response['ret'] < 0):
                return response['ret']
            else:
                return response['cid']
        else:
            return -9001

    def result(self, cid):
        data = {'method': 'result', 'username': self.username, 'password': self.password, 'appid': self.appid, 'appkey': self.appkey, 'cid': str(cid)}
        response = self.request(data)
        return response and response['text'] or ''

    def decode(self, filename, codetype, timeout):
        cid = self.upload(filename, codetype, timeout)
        if (cid > 0):
            for i in range(0, timeout):
                result = self.result(cid)
                if (result != ''):
                    return cid, result
                else:
                    time.sleep(1)
            return -3003, ''
        else:
            return cid, ''

    def report(self, cid):
        data = {'method': 'report', 'username': self.username, 'password': self.password, 'appid': self.appid, 'appkey': self.appkey, 'cid': str(cid), 'flag': '0'}
        response = self.request(data)
        if (response):
            return response['ret']
        else:
            return -9001

    def post_url(self, url, fields, files=[]):
        for key in files:
            files[key] = open(files[key], 'rb');
        res = requests.post(url, files=files, data=fields)
        return res.text

######################################################################

# 用户名(普通用户)
username    = 'bobo328410948'

# 密码
password    = 'bobo328410948'                            

# 软件ＩＤ，开发者分成必要参数。登录开发者后台【我的软件】获得！
appid       = 6003                                    

# 软件密钥，开发者分成必要参数。登录开发者后台【我的软件】获得！
appkey      = '1f4b564483ae5c907a1d34f8e2f2776c'    

# 图片文件
filename    = 'getimage.jpg'                        

# 验证码类型，# 例：1004表示4位字母数字，不同类型收费不同。请准确填写，否则影响识别率。在此查询所有类型 http://www.yundama.com/price.html
codetype    = 1004

# 超时时间，秒
timeout     = 10                                    

# 检查
if (username == 'username'):
    print('请设置好相关参数再测试')
else:
    # 初始化
    yundama = YDMHttp(username, password, appid, appkey)

    # 登陆云打码
    uid = yundama.login();
    print('uid: %s' % uid)

    # 查询余额
    balance = yundama.balance();
    print('balance: %s' % balance)

    # 开始识别，图片路径，验证码类型ID，超时时间（秒），识别结果
    cid, result = yundama.decode(filename, codetype, timeout);
    print('cid: %s, result: %s' % (cid, result))

######################################################################

def getCodeDate(userName,pwd,codePath,codeType):
    # 用户名(普通用户)
    username    = userName

    # 密码
    password    = pwd                            

    # 软件ＩＤ，开发者分成必要参数。登录开发者后台【我的软件】获得！
    appid       = 6003                                    

    # 软件密钥，开发者分成必要参数。登录开发者后台【我的软件】获得！
    appkey      = '1f4b564483ae5c907a1d34f8e2f2776c'    

    # 图片文件
    filename    = codePath                       

    # 验证码类型，# 例：1004表示4位字母数字，不同类型收费不同。请准确填写，否则影响识别率。在此查询所有类型 http://www.yundama.com/price.html
    codetype    = codeType

    # 超时时间，秒
    timeout     = 2                                   
    result = None
    # 检查
    if (username == 'username'):
        print('请设置好相关参数再测试')
    else:
        # 初始化
        yundama = YDMHttp(username, password, appid, appkey)

        # 登陆云打码
        uid = yundama.login();
        #print('uid: %s' % uid)

        # 查询余额
        balance = yundama.balance();
        #print('balance: %s' % balance)

        # 开始识别，图片路径，验证码类型ID，超时时间（秒），识别结果
        cid, result = yundama.decode(filename, codetype, timeout);
        #print('cid: %s, result: %s' % (cid, result))
    return result

#人人网的模拟登录
import requests
import urllib
from lxml import etree
#获取session对象
session = requests.Session()
#将验证码图片进行下载
headers = {
    'User-Agent':'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.119 Safari/537.36'
}
url = 'http://www.renren.com/'
page_text = requests.get(url=url,headers=headers).text

tree = etree.HTML(page_text)
code_img_url = tree.xpath('//*[@id="verifyPic_login"]/@src')[0]
urllib.request.urlretrieve(url=code_img_url,filename='code.jpg')

#识别验证码图片中的数据值
code_data = getCodeDate('bobo328410948','bobo328410948','./code.jpg',2004)

#模拟登录
login_url = 'http://www.renren.com/ajaxLogin/login?1=1&uniqueTimestamp=201914927558'
data = {
    "email":"www.zhangbowudi@qq.com",
    "icode":code_data,
    "origURL":"http://www.renren.com/home",
    "domain":"renren.com",
    "key_id":"1",
    "captcha_type":"web_login",
    "password":"4f0350f09aeffeef86307747218b214b0960bdf35e30811c0d611fe39db96ec1",
    "rkey":"9e75e8dc3457b14c55a74627fa64fb43",
    "f":"http%3A%2F%2Fwww.renren.com%2F289676607",
}
#该次请求产生的cookie会被自动存储到session对象中
session.post(url=login_url,data=data,headers=headers)

url = 'http://www.renren.com/289676607/profile'
page_text = session.get(url=url,headers=headers).text

with open('renren.html','w',encoding='utf-8') as fp:
    fp.write(page_text)

from fake_useragent import UserAgent
ua = UserAgent(verify_ssl=False,use_cache_server=False).random
print(ua)

import requests
import urllib
from lxml import etree
headers = {
    'User-Agent':'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.119 Safari/537.36'
}

#模拟登录古诗文网
s = requests.Session()
login_url = 'https://so.gushiwen.org/user/login.aspx?from=http://so.gushiwen.org/user/collect.aspx'
page_text = requests.get(url=login_url,headers=headers).text
tree = etree.HTML(page_text)
img_src = 'https://so.gushiwen.org'+tree.xpath('//*[@id="imgCode"]/@src')[0]
img_data = s.get(url=img_src,headers=headers).content
with open('./img.jpg','wb') as fp:
    fp.write(img_data)
img_text = getCodeDate('bobo328410948','bobo328410948','./img.jpg',1004)

#模拟登录
url = 'https://so.gushiwen.org/user/login.aspx?from=http%3a%2f%2fso.gushiwen.org%2fuser%2fcollect.aspx'
data = {
    "__VIEWSTATE":"9AsGvh3Je/0pfxId7DYRUi258ayuEG4rrQ1Z3abBgLoDSOeAUatOZOrAIxudqiOauXpR9Zq+dmKJ28+AGjXYHaCZJTTtGgrEemBWI1ed7oS7kpB7Rm/4yma/+9Q=",
    "__VIEWSTATEGENERATOR":"C93BE1AE",
    "from":"http://so.gushiwen.org/user/collect.aspx",
    "email":"www.zhangbowudi@qq.com",
    "pwd":"bobo328410948",
    "code":img_text,
    "denglu":"登录",
}
page_text = s.post(url=url,headers=headers,data=data).text
with open('./gushiwen.html','w',encoding='utf-8') as fp:
    fp.write(page_text)

selenium 的详细用法

1 按键操作

from selenium.webdriver.common.keys import Keys
from selenium import webdriver
import time

driver = webdriver.Chrome()
driver.get('http://www.baidu.com')

time.sleep(3)
driver.find_element_by_id('kw').send_keys(u'我爱你')

time.sleep(3)
# 表示ctrl+a : 全选
driver.find_element_by_id('kw').send_keys(Keys.CONTROL, 'a')
time.sleep(3)
# 表示ctrl+x: 剪贴
driver.find_element_by_id('kw').send_keys(Keys.CONTROL, 'x')

time.sleep(3)
driver.find_element_by_id('kw').send_keys(u'我爱爱你')
driver.find_element_by_id('su').click()

time.sleep(3)
# 退出浏览器
driver.quit()

2 时间等待
selenium 由网页驱动，驱使浏览器进行操作，速度慢是一大特点，经常会出现代码执行完了，但是网页内容还没有加载完毕， 里面的标签没有显示出来，如果这时候操作里面的标签，就会爆出异常， NoSuchElementException
解决办法： 时间休眠 ， 不管页面的内容有没有加载完毕， 一定要休眠够指定的秒数

from selenium.webdriver.support.ui import WebDriverWait
from selenium import webdriver

driver = webdriver.Chrome()
driver.get('http://www.baidu.com')
driver.find_element_by_id('kw').send_keys(u'我爱你')
button = driver.find_element_by_id('su')

# WebDriverWait: 网页等待
#   值1： 等待的对象
#   值2： 等待的时间
# WebDriverWait 经常和 until 以及 until not 一起使用         until:直到...
# 等待直到目标标签出现
is_visible = WebDriverWait(button, 10).until(lambda x: button.is_displayed())
print(is_visible)
# 返回True
button.click()

WebDriverWait 和time.sleep() 的异同：
1 都是让程序等待指定的时间，
2 time 的时间是固定的，时间长短不会随着标签的加载速度而改变，
WebDriverWait 时间是不固定的，等待多少时间要看标签的加载时间和指定的固定时间
3 如果在指定的时间内，标签仍然没有加载出来，那么time 和WebDriverWait都会爆出异常

3 点击事件

from selenium import webdriver
from selenium.webdriver.common.action_chains import ActionChains
from selenium.webdriver.support.ui import WebDriverWait

driver = webdriver.Chrome()
driver.get('http://www.baidu.com')

# 三者输出的内容是一样的
logo = driver.find_element_by_xpath('//div[@id="lg"]/img')
logo2 = driver.find_element_by_class_name('index-logo-src')
logo3 = driver.find_element_by_css_selector('#lg > img')

WebDriverWait(driver, 10).until(lambda x: logo.is_displayed())
ActionChains.double_click(logo).perform()


# context.click 表示右击
action = ActionChains.context_click(logo)
# 操作事件会跑到perform 队列里面
action.perform()

# 鼠标移动
more = driver.find_element_by_class_name('bri')
WebDriverWait(driver, 10).until(lambda x: more.is_displayed())
ActionChains.move_to_element(more).perform()

4, 标签选择

这个是要解析的网页

<!DOCTYPE html>
<html lang="en">
<head>
    <meta charset="UTF-8">
    <title>Title</title>
</head>
<body>
    <button id="red" class="red" onclick="fun1()">按钮1</button>
    <button type="button" name="username" onclick="fun2()">按钮2</button>
    <button type="button" onclick="fun3()">按钮3</button>
    <button id="yellow" onclick="fun4()">按钮4</button>
    <script>
        function fun1(){
            document.body.style.backgroundColor = 'black'
        }
        function fun2(){
            document.body.style.backgroundColor = 'purple'
        }
        function fun3(){
            document.body.style.backgroundColor = 'pink'
        }
        function fun4(){
            document.body.style.backgroundColor = 'yellow'
        }
    </script>
</body>
</html>

标签选择的具体用法：

from selenium import webdriver
from selenium.webdriver.common.by import By
import os

driver = webdriver.Chrome()
driver.get('file:///' + os.path.abspath('4 index.html'))

# 通过标签名字来找到指定的标签
# 注意这里是elements
btns = driver.find_elements_by_tag_name('button')
btns[1].click()


# 1 通过索引来找到指定的标签
for btn in btns:
    btn.click()
    # 2 通过属性来找到特定的标签
    if btns.get_attribute('name') == 'username':
        btns.click()


# find_element_by_XXX  通过XXX来找到所有标签当中的第一个标签
# find_elements_by_XXX 通过XXX 来找到所有符合的标签
# 下面这个找到的是第一个标签名字是button的
btn = driver.find_element_by_tag_name('button').click()

# 弹出指定的元素，如果不写索引，默认是最后一个
driver.find_elements_by_css_selector('button').pop(1).click()

# [type=button]  [] 里面为限制条件，限制选择的内容

# 它是一个列表，[0] 代表的就是符合type=button的所有标签，里面的第一个元素
driver.find_elements_by_css_selector('button[type=button]')[0].click()

# 通过...来找到指定标签，相当于一个总和
driver.find_element(by=By.id, value='yellow').click()

5 window 切换

from selenium import webdriver
from selenium.webdriver.support.ui import WebDriverWait
import time

driver = webdriver.Chrome()
driver.get('http://www.baidu.com')

# 获取当前window对象
current_window = driver.current_window_handle
print(current_window, driver.title)
# CDwindow-E5F3FC897FF4B4F7EA29CE1D42CCF738 百度一下，你就知道

time.sleep(3)
driver.find_element_by_name('tj_trnews').click()

news = WebDriverWait(driver, 10).until(lambda x: driver.find_element_by_css_selector('.hdline0 .a3'))
news.click()

all_windows = driver.window_handles

for window in all_windows:
    if window != current_window:
        time.sleep(4)
        driver.switch_to_window(window)
        # 获取百度新闻 h1 标题
        title = driver.find_element_by_xpath('//div[@class="cnt_bd"]/h1')
        WebDriverWait(title, 10).until(lambda x: title.is_displayed())
        print(title.text)

# 关闭浏览器
# driver.quit()

driver.switch_to_window(current_window)
print(driver.find_element_by_css_selector('#footer span').text)

6使用selenium简单的对淘宝信息进行爬取

from selenium import webdriver
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.action_chains import ActionChains
import time

driver = webdriver.Chrome()
driver.get('http://www.taobao.com')

driver.find_element_by_id('q').send_keys(u'辣条')
time.sleep(3)

# 下面是寻找四种‘点击搜索‘按钮的方式：

# 标签name之间有空格，只取最后一个
# driver.find_element_by_css_selector('.J_SearchPanel .tb-bg').click()

# 这个是只取， 空格前面的第一个 。也可以加载
# driver.find_element_by_css_selector('.search-panel .btn-search').click()

# 这个是根据标签的属性值来寻找，也可以执行
# driver.find_element_by_css_selector('button[data-spm-click="gostr=/tbindex;locaid=d13"]').click()

# 根据xpath也可以执行 点击按钮
driver.find_element_by_xpath('//div[@class="search-button"]/button').click()

for page in range(1, 3):
    print('正在爬取第{}页数据'.format(page))
    # 这里爬取的电脑页面，一页总共显示13行，一个屏幕最多显示两行
    for row in range(1, 13, 2):
        # x 表示把页面平均分成12份
        x = float(row) / 12
        # document : 网页
        # documentElement: 网页标签
        # scroll: 滑动
        # scrollTop: 从屏幕顶部往下滑动多少
        # 计算每次移动的js 代码片段，
        # 在Python里不能直接执行Js代码，所以将代码写成字符串形式
        # scrollTop 整个网页内从上到下可以滑动的长度
        js = 'document.documentElement.scrollTop=document.
        documentElement.scrollHeight * {}'.format(x)
        # js = 'document.documentElement.scrollTop=document
        .documentElement.scrollHeight * %f' % x
        driver.execute_script(js)
        time.sleep(3)
    item_list = driver.find_elements_by_class_name('J_MouserOnverReq')
    for item in item_list:
        with open('lianxi.txt', 'a', encoding='utf8') as f:
            f.write(item.text)
            f.write('
')
    # 这个表示跳转到下一页
    driver.find_element_by_xpath('//li[@class="item next"]/a').click()

• 环境安装:pip install selenium
• 编码流程:
• 导报:from selenium import webdriver
• 实例化某一款浏览器对象
• 自指定自动化操作代码

from selenium import webdriver
from time import sleep

bro = webdriver.Chrome(executable_path=r'C:UsersAdministratorDesktop爬虫+数据day_03_爬虫chromedriver.exe')
bro.get(url='https://www.baidu.com/')
sleep(2)
text_input = bro.find_element_by_id('kw')

text_input.send_keys('人民币')
sleep(2)
bro.find_element_by_id('su').click()

sleep(3)

#获取当前的页面源码数据
print(bro.page_source)
bro.quit()

#获取豆瓣电影中更多电影详情数据
url = 'https://movie.douban.com/typerank?type_name=%E6%83%8A%E6%82%9A&type=19&interval_id=100:90&action='
bro = webdriver.Chrome(executable_path=r'C:UsersAdministratorDesktop爬虫+数据day_03_爬虫chromedriver.exe')
bro.get(url)
sleep(3)
bro.execute_script('window.scrollTo(0,document.body.scrollHeight)')
sleep(3)
bro.execute_script('window.scrollTo(0,document.body.scrollHeight)')
sleep(3)
bro.execute_script('window.scrollTo(0,document.body.scrollHeight)')
sleep(2)
page_text = bro.page_source

with open('./douban.html','w',encoding='utf-8') as fp:
    fp.write(page_text)

sleep(1)
bro.quit()

#谷歌无头浏览器
from selenium.webdriver.chrome.options import Options
chrome_options = Options()
chrome_options.add_argument('--headless')
chrome_options.add_argument('--disable-gpu')

#获取豆瓣电影中更多电影详情数据
url = 'https://movie.douban.com/typerank?type_name=%E6%83%8A%E6%82%9A&type=19&interval_id=100:90&action='
bro = webdriver.Chrome(executable_path=r'C:UsersAdministratorDesktop爬虫+数据day_03_爬虫chromedriver.exe',chrome_options=chrome_options)
bro.get(url)
sleep(3)
bro.execute_script('window.scrollTo(0,document.body.scrollHeight)')
sleep(3)
bro.execute_script('window.scrollTo(0,document.body.scrollHeight)')
sleep(3)
bro.execute_script('window.scrollTo(0,document.body.scrollHeight)')
sleep(2)
page_text = bro.page_source

with open('./douban.html','w',encoding='utf-8') as fp:
    fp.write(page_text)
print(page_text)
sleep(1)
bro.quit()

#phantomJs
#获取豆瓣电影中更多电影详情数据
url = 'https://movie.douban.com/typerank?type_name=%E6%83%8A%E6%82%9A&type=19&interval_id=100:90&action='
bro = webdriver.Chrome(executable_path=r'C:UsersAdministratorDesktop爬虫+数据day_03_爬虫phantomjs-2.1.1-windowsinphantomjs.exe')
bro.get(url)
sleep(3)
bro.execute_script('window.scrollTo(0,document.body.scrollHeight)')
sleep(3)
bro.execute_script('window.scrollTo(0,document.body.scrollHeight)')
sleep(3)
bro.execute_script('window.scrollTo(0,document.body.scrollHeight)')
sleep(2)
page_text = bro.page_source

with open('./douban.html','w',encoding='utf-8') as fp:
    fp.write(page_text)

sleep(1)
bro.quit()

#qq空间
bro = webdriver.Chrome(executable_path=r'C:UsersAdministratorDesktop爬虫+数据day_03_爬虫chromedriver.exe')
url = 'https://qzone.qq.com/'
bro.get(url=url)
sleep(2)
#定位到一个具体的iframe
bro.switch_to.frame('login_frame')
bro.find_element_by_id('switcher_plogin').click()
sleep(2)

bro.find_element_by_id('u').send_keys('460086804')
bro.find_element_by_id('p').send_keys('shuo0127')

bro.find_element_by_id('login_button').click()

sleep(5)

page_text = bro.page_source
with open('qq.html','w',encoding='utf-8') as fp:
    fp.write(page_text)
bro.quit()

#爬取梨视频数据
import requests
import re
from lxml import etree
from multiprocessing.dummy import Pool
import random

#实例化一个线程池对象
pool = Pool(5)
url = 'https://www.pearvideo.com/category_1'
headers = {
    'User-Agent':'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.119 Safari/537.36'
}
page_text = requests.get(url=url,headers=headers).text
tree = etree.HTML(page_text)
li_list = tree.xpath('//div[@id="listvideoList"]/ul/li')

video_url_list = []
for li in li_list:
    detail_url = 'https://www.pearvideo.com/'+li.xpath('./div/a/@href')[0]
    detail_page = requests.get(url=detail_url,headers=headers).text
    video_url = re.findall('srcUrl="(.*?)",vdoUrl',detail_page,re.S)[0]
    video_url_list.append(video_url)
    
video_data_list = pool.map(getVideoData,video_url_list)

pool.map(saveVideo,video_data_list)

def getVideoData(url):
    return requests.get(url=url,headers=headers).content

def saveVideo(data):
    fileName = str(random.randint(0,5000))+'.mp4'
    with open(fileName,'wb') as fp:
        fp.write(data)

---恢复内容结束---