python 爬恶魔法则(单线程卡成狗)

from bs4 import BeautifulSoup
import requests
import sys
class down(object):
def __init__(self):
self.name = []
self.urls = []
self.nums = 0

def get_down_url(self):
html = "http://www.biquge5200.com/63_63238/"
req =requests.get(html)
bea = BeautifulSoup(req.text)
dd =bea.findAll('dd')
a = BeautifulSoup(str(dd)) # 把 找到 dd 再次 str 话 寻找a"
a_d = a.findAll("a")
self.nums=len(a_d[18:])
for eatch in a_d[18:] :
self.name.append(eatch.string)
self.urls.append(eatch.get('href'))
print(self.urls)
print(self.name)



def get_conent(self,target):
req = requests.get(url=target)
html =req.text
cont = BeautifulSoup(html)
texts =cont.find("div",id="content")
print(texts.text.replace('xa0'*8,' '))
ds =down()
ds.get_down_url()
##for i in range(ds.nums):
## print(ds.get_conent(ds.urls[i]))
原文地址:https://www.cnblogs.com/baili-luoyun/p/8577357.html