微信公众号爬取

driver = webdriver.Chrome()
driver.get("https://mp.weixin.qq.com/cgi-bin/loginpage?url=%2Fcgi-bin%2Fhome%3Ft%3Dhome%2Findex%26lang%3Dzh_CN%26token%3D260217419")
wait = WebDriverWait(driver,1000)

#用户名
input1 = wait.until(EC.element_to_be_clickable((By.XPATH,'//*[@id="header"]/div[2]/div/div/form/div[1]/div[1]/div/span/input')))
input1.send_keys("用户名")
#密码
input2 = wait.until(EC.element_to_be_clickable((By.XPATH,'//*[@id="header"]/div[2]/div/div/form/div[1]/div[2]/div/span/input')))
input2.send_keys("密码")

#点击
btn = driver.find_element_by_xpath('//*[@id="header"]/div[2]/div/div/form/div[4]/a')
btn.click()
#等待页面
wait.until(EC.element_to_be_clickable((By.XPATH,'//*[@id="m_open"]')))
# wait.untilil(EC.(r"https://mp.weixin.qq.com/cgi-bin/home?t=home/index&lang=zh_CN&token=d+"))
#全屏
#获取cookies
cookie_items = driver.get_cookies()
post = {}
import json
for cookie_item in cookie_items:
    post[cookie_item['name']] = cookie_item['value']
# cookie_str = json.dumps(post)

driver.maximize_window()
#获取token
re_url = driver.current_url
token = re.findall(r'token=(d+)',str(re_url))[0]
import random
search_url = 'https://mp.weixin.qq.com/cgi-bin/searchbiz?'
query_id = {
    'action': 'search_biz',
    'token' : token,
    'lang': 'zh_CN',
    'f': 'json',
    'ajax': '1',
    'random': random.random(),
    'query': '科技美学',
    'begin': '0',
    'count': '5'
    }
header = {
        "HOST": "mp.weixin.qq.com",
        "User-Agent": "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:53.0) Gecko/20100101 Firefox/53.0"
        }

search_response = requests.get(search_url, cookies=post, headers=header, params=query_id)
# 取搜索结果中的第一个公众号
lists = search_response.json().get('list')[0]
print(lists)
 #获取这个公众号的fakeid，后面爬取公众号文章需要此字段
fakeid = lists.get('fakeid')
print(search_response.url)
#微信公众号文章接口地址
appmsg_url = 'https://mp.weixin.qq.com/cgi-bin/appmsg?'
#搜索文章需要传入几个参数：登录的公众号token、要爬取文章的公众号fakeid、随机数random
query_id_data = {
    'token': token,
    'lang': 'zh_CN',
    'f': 'json',
    'ajax': '1',
    'random': random.random(),
    'action': 'list_ex',
    'begin': '0',#不同页，此参数变化，变化规则为每页加5
    'count': '5',
    'query': '',
    'fakeid': fakeid,
    'type': '9'
    }
#打开搜索的微信公众号文章列表页
appmsg_response = requests.get(appmsg_url, cookies=post, headers=header, params=query_id_data)
#获取文章总数
max_num = appmsg_response.json().get('app_msg_cnt')
print(appmsg_response.url)

#每页至少有5条，获取文章总的页数，爬取时需要分页爬
num = int(int(max_num) / 5)
#起始页begin参数，往后每页加5
begin = 0
while num + 1 > 0 :
    query_id_data = {
        'token': token,
        'lang': 'zh_CN',
        'f': 'json',
        'ajax': '1',
        'random': random.random(),
        'action': 'list_ex',
        'begin': '{}'.format(str(begin)),
        'count': '5',
        'query': '',
        'fakeid': fakeid,
        'type': '9'
    }
    print('正在翻页：--------------', begin)

    # 获取每一页文章的标题和链接地址，并写入本地文本中
    query_fakeid_response = requests.get(appmsg_url, cookies=post , headers=header, params=query_id_data)
    fakeid_list = query_fakeid_response.json().get('app_msg_list')
    for item in fakeid_list:
        content_link = item.get('link')
        content_title = item.get('title')
        fileName =  '科技美学.txt'
        with open(fileName, 'a', encoding='utf-8') as fh:
            fh.write(content_title + ":
" + content_link + "
")
    num -= 1
    begin = int(begin)
    begin += 5
    time.sleep(2)