今天学习了python网络爬虫的简单知识
首先是一个爬取百度的按行读取和一次性爬取
逐行爬取
for line in urllib.request.urlopen("http://www.baidu.com"):
print(line.decode("utf-8"))
全部爬取
mystr = urllib.request.urlopen("http://www.baidu.com").read()
print(mystr.decode("utf-8"))
分别用栈和队列实现了DFS和BFS的邮箱爬取
用队列deque实现BFS
import re
import urllib
import urllib.request
from collections import deque
def getallemail(data): #邮箱的正则表达式获取所有的邮箱
try:
mailregex = re.compile(r"([A-Z0-9._%+-]+@[A-Z0-9.-]+.[A-Z]{2,4})", re.IGNORECASE)
mylist = mailregex.findall(data)
return mylist
except:
return []
def getdata(url): #用utf-8编码读取url返回网页源代码
try:
data = urllib.request.urlopen(url).read().decode("utf-8")
return data
except:
return ""
def geteveryurl(data): #获得网页所有的url
alllist = []
mylist1 = getallhttp(data)
mylist2 = []
if len(mylist1)>0:
mylist2 = getabsurl(mylist1[0],data) #mylist[0]作用是提取元素
alllist.extend(mylist1)
alllist.extend(mylist2)
return alllist
def gethostname(httpstr):
try:
mailregex = re.compile(r"(http://S*?)/",re.IGNORECASE) #预编译提取主机名的regex
mylist = mailregex.findall(httpstr)
if len(mylist)==0:
return None
else:
return mylist[0]
except:
return None
def getabsurl(url,data):
try:
regex = re.compile("href="(.*?)"",re.IGNORECASE) #预编译提取href正则表达式
httplist = regex.findall(data)
newhttplist = httplist.copy() #进行一次深拷贝，以进行后面的删除行为
for data in newhttplist:
if data.find("http://")!=-1: #如果其中包含http
httplist.remove(data) #在原list中remove此data
if data.find("javascript")!=-1:
httplist.remove(data) #同理
hostname = gethostname(url)
if hostname!=None:
for i in range(len(httplist)):
httplist[i] = hostname + httplist[i]
return httplist
except:
return []
def getallhttp(data):#找到所有的http
try:
mailregex = re.compile(r"(http://S*?)["|>|)]",re.IGNORECASE)
mylist = mailregex.findall(data)
return mylist
except:
return[]
def BFS(urlstr):
urlqueue = deque([]) #新建一个队列
urlqueue.append(urlstr) #队列中加入最初的url
while len(urlqueue)!=0: #判断队列是否为空
url = urlqueue.popleft() #队列弹出的数据（url）
print(url) #打印url连接
pagedata = getdata(url) #获取网页源代码
emaillist = getallemail(pagedata) #提取邮箱到列表
if len(emaillist)!=0: #若邮箱列表不为空
for email in emaillist:
print(email) #打印所有的邮箱
newurllist = geteveryurl(pagedata) #抓取该网页的所有的url
if len(newurllist)!=0: #若列表不为空
for urlstr in newurllist:
if urlstr not in urlqueue:
urlqueue.append(urlstr) #若url不在该队列中，则将该url加入队列中
BFS(input("请输入你想爬取的最初页面"))
用栈stack实现DFS
import re
import urllib
import urllib.request
def getallemail(data): #邮箱的正则表达式获取所有的邮箱
try:
mailregex = re.compile(r"([A-Z0-9._%+-]+@[A-Z0-9.-]+.[A-Z]{2,4})", re.IGNORECASE)
mylist = mailregex.findall(data)
return mylist
except:
return []
def getdata(url): #用utf-8编码读取url返回网页源代码
try:
data = urllib.request.urlopen(url).read().decode("utf-8")
return data
except:
return ""
def geteveryurl(data): #获得网页所有的url
alllist = []
mylist1 = getallhttp(data)
mylist2 = []
if len(mylist1)>0:
mylist2 = getabsurl(mylist1[0],data) #mylist[0]作用是提取元素
alllist.extend(mylist1)
alllist.extend(mylist2)
return alllist
def gethostname(httpstr):
try:
mailregex = re.compile(r"(http://S*?)/",re.IGNORECASE) #预编译提取主机名的regex
mylist = mailregex.findall(httpstr)
if len(mylist)==0:
return None
else:
return mylist[0]
except:
return None
def getabsurl(url,data):
try:
regex = re.compile("href="(.*?)"",re.IGNORECASE) #预编译提取href正则表达式
httplist = regex.findall(data)
newhttplist = httplist.copy() #进行一次深拷贝，以进行后面的删除行为
for data in newhttplist:
if data.find("http://")!=-1: #如果其中包含http
httplist.remove(data) #在原list中remove此data
if data.find("javascript")!=-1:
httplist.remove(data) #同理
hostname = gethostname(url)
if hostname!=None:
for i in range(len(httplist)):
httplist[i] = hostname + httplist[i]
return httplist
except:
return []
def getallhttp(data):#找到所有的http
try:
mailregex = re.compile(r"(http://S*?)["|>|)]",re.IGNORECASE)
mylist = mailregex.findall(data)
return mylist
except:
return[]
def DFS(urlstr):
visitlist = [] #代表已经访问过的url，防止深度遍历出现死循环
urlstack=[] #栈
urlstack.append(urlstr)
while len(urlstack)!=0:
url = urlstack.pop()
print(url) #打印url链接
if url not in visitlist:
pagedata = getdata(url)
emaillist = getallemail(pagedata)
if len(emaillist)!=0:
for email in emaillist:
print(email)
newurllist = geteveryurl(pagedata)
if len(newurllist)!=0:
for urlstr in newurllist :
if urlstr not in urlstack:
urlstack.append(urlstr)
visitlist.append(url)
DFS(input("请输入你想爬取的最初页面"))
#提取数据容易出现广度遍历
#深度遍历容易出现死循环
- 其中需要注意的是，DFS容易出现死循环现象，故使用visitlist来避免，数据提取适合使用广度遍历实现，因为深度遍历是一撸到底，适合挖掘网站的层数。
代码来自尹成python教学