#更新记录20180630----------
1。添加存在文件跳过逻辑
2。非法文件路径名字去除逻辑的添加
# -*- coding: utf-8 -*-
# @Time : 2018/11/16 10:02 AM
# @Author : cxa
# @File : cosmic.py
# @Software: PyCharm
# !/usr/bin/env python
# encoding: utf-8
# !/usr/bin/env python
# encoding: utf-8
from requests_html import HTMLSession
import aiohttp
import asyncio
import hashlib
import os
from traceback import format_exc
import base64
from cryptography.fernet import Fernet
# 文件下载也要是异步
import aiofiles
import multiprocessing
from tomorrow import threads
from retrying import retry
workers = multiprocessing.cpu_count() * 2 + 1
# 开始索引数
strat_num = 227002
# 结束索引数
end_num = 250606
key = "X0JxSkg4NFVBQVBPODlUM0VzT1liNnloeWtLcndkSldRT2xURzQ4MEM5RT0="
page_num_xpath = "//p[@class='selectpage']/select[@id='page_number']/option[last()]/@file"
page_id_xpath = "//img[@id='thumbnail']/@src"
def aes_cbc_decrypt(message):
decrypted_text = Fernet(base64.b64decode(key).decode("utf8")).decrypt(bytes("{}".format(message), encoding="utf8"))
return decrypted_text.decode("utf8")
# 漫画题目
cosmic_name = "//head//title/text()"
# 漫画id
cosmic_id = "//img[@id='curPic']/@src"
main_url = aes_cbc_decrypt(
"gAAAAABbNdhqCnxkaJwZ2VL7HUXne_IOic-NsHtE30W-J68oecVmgm0dzO_lLXgTlI7a5_NbUWlkGm7FqLwY81XIBddNWbac4rCgBA9NFAECsNISkhTvdRl4uDSaS6bHY8sbcJJwO13Z")
cosmic_urllist = [main_url.format(i) for i in range(strat_num, end_num + 1)]
pagenum_xpath = "//font[@id='TotalPage']/text()"
full_url = aes_cbc_decrypt(
"gAAAAABbNdk5FLeX55hOiDAXxgCwwYmGrokYvU3Nd1AOYuOE7OdIEcBdAmSG_Q3kOltealBKMOgUBKDuPUJtzFFPwqoxL-FUip"
"VNQU-JmBW_K5qxgzTQ3IOla_F61Rscy0fJOaN-mEXKPqrakctyDRN7OVm1LARTMhylQELLuBnJgIT4WXilchg=") # 漫画的总id，序号的id和格式使用(jpg)
session = HTMLSession()
sema = asyncio.Semaphore(5)
session = HTMLSession()
async def getbuff(url, c_name):
async with aiohttp.ClientSession() as session2:
async with session2.get(url, timeout=60) as r:
buff = await r.read()
if not len(buff):
url = url.replace(".jpg", ".png")
async with session2.get(url, timeout=60) as r2:
buff = await r2.read()
await getimg(url, buff, c_name)
async def run(url, c_name):
with (await sema):
await getbuff(url, c_name)
#
@threads(30)
@retry(stop_max_attempt_number=5)
def asyc_get_req(url):
req = session.get(url, timeout=15)
if req.status_code == 200:
return req
else:
raise ValueError("访问出错")
def spider(req):
try:
if req.status_code == 200:
root = req.html
name = root.xpath(cosmic_name)[0]
print(name)
with open("1.html", 'a', encoding='utf-8') as fs:
fs.write(req.text)
id = root.xpath(page_id_xpath)[0].split('/')[-2]
max_page = root.xpath(page_num_xpath)[0].split('.')[0]
full_urllist = [full_url.format(id, i, "jpg") for i in range(1, int(max_page) + 1)]
event_loop = asyncio.get_event_loop()
tasks = [run(url, name) for url in full_urllist]
results = event_loop.run_until_complete(asyncio.wait(tasks))
except:
print(format_exc())
async def getimg(url, buff, c_name):
# 题目那层目录
filepath = os.path.join(os.getcwd(), "comics_images", c_name)
# 如果标题太长就转md5，然后单独启动一个text写入内容为标题
md5name = hashlib.md5(c_name.encode("utf-8")).hexdigest()
filepath2 = os.path.join(os.getcwd(), "comics_images", md5name)
id = url.split('/')[-1]
image_id = os.path.join(filepath, id)
image_id2 = os.path.join(filepath2, md5name)
# 题目层目录是否存在
if not os.path.exists(filepath) and not os.path.exists(filepath2):
try:
os.makedirs(filepath)
except:
os.makedirs(filepath2)
image_id = image_id2
fs = await aiofiles.open(os.path.join(filepath2, "title.txt"), 'w')
await fs.write(c_name)
# 文件是否存在
if not os.path.exists(image_id) and not os.path.exists(image_id2):
f = await aiofiles.open(image_id, 'wb')
await f.write(buff)
if __name__ == '__main__':
# with ThreadPool(workers) as pool:
# pool.map(spider, cosmic_urllist)
req_list = []
for url in cosmic_urllist:
req = asyc_get_req(url)
req_list.append(req)
for req in req_list:
spider(req)