python爬虫

原创博文，转载请注明出处。

单线程爬取：

所用模块urllib2,re

 1 # -*- coding: cp936 -*-
 2 import urllib2
 3 import re
 4 
 5 def main():
 6     url="http://www.baidu.com/"
 7     req = urllib2.Request(url)
 8     resp = urllib2.urlopen(req)
 9     respHtml = resp.read()
10 #<a href="/duty/" name="tj_duty">使用百度前必读</a> ，匹配内容
11     ahn='<as+?href="/duty/"s+?name="tj_duty">(?P<content>.+)</a>'
12     found=re.search(ahn,respHtml)
13     print 'found=',found
14     if(found):
15         a1=found.group("content")
16         print 'content',a1
17 if __name__=='__main__':
18     main()

(?P<name>...)通过定义一个group name,在后续的匹配对象中可以把name参数传入group()而得到(?P<name>...)所匹配的内容。

Beautiful soup 是一个可以从HTML或XML文件中提取数据的Python库。官当文档我们同样可以使用beautiful soup来爬取数据。解释器是lxml库。

 1 # -*- coding: cp936 -*-
 2 from bs4 import BeautifulSoup
 3 import urllib2
 4 import re
 5 
 6 def main():
 7     url="http://www.baidu.com/"
 8     req = urllib2.Request(url)
 9     resp = urllib2.urlopen(req)
10     respHtml = resp.read()
11     soup = BeautifulSoup(respHtml)
12     found = soup.find(href='/duty/')
13     #found = soup.find(attrs={'name':'tj_duty'})两种匹配方法  
14     print 'found:',found
15     if(found):
16         content = found.string
17         print 'content:',content
18 if __name__=='__main__':
19     main()

想要提高爬取数据的效率，多线程的使用是必须的，下面是简单的多线程爬取

 1 # -*- coding: cp936 -*-
 2 from Queue import Queue
 3 from threading import Thread
 4 import time
 5 import urllib2
 6 import urlparse
 7 
 8 num_threads =2
 9 q = Queue()
10 urls=['http://www.baidu.com',
11       'http://www.sina.com',
12       'http://www.qq.com',
13       ]
14 for url in urls:
15     q.put(url)
16     
17 def download(i,q):
18     while True:
19         print 'start download %s'%i
20         url = q.get()#在get()遇到了阻塞并等待 
21         parsed_url = urlparse.urlparse(url)
22         print 'Downloading: %s'%url
23         req = urllib2.Request(url)
24         resp = urllib2.urlopen(req)
25         data = resp.read()
26         filename = url.rpartition('/')[-1]
27         with open(filename+'.html','wb') as outfile:
28             outfile.write(data)
29         print 'complete download %s:%s'%(url,i)
30         q.task_done()
31 
32 for i in range(num_threads):
33     worker = Thread(target=download,args=(i,q,))
34     worker.setDaemon(True)
35     worker.start()
36 q.join()

线程池可以提高并发执行任务的效率，这里有关于线程池的介绍http://www.cnblogs.com/tracylining/p/3471594.html 现在我们将其进行应用。

 1 from Queue import Queue
 2 from threading import Thread
 3 import datetime
 4 import urllib2
 5 import urlparse
 6 import threadpool 
 7 
 8 url_list=['http://www.baidu.com',
 9           'http://www.qq.com',
10           'http://www.sina.com',
11           ]
12 
13 def download(url):
14     try:
15         parsed_url = urlparse.urlparse(url)
16         req = urllib2.Request(url)
17         resp = urllib2.urlopen(req)
18         data = resp.read()
19         with open("download/"+str(hash(url)),'wb') as f:
20             f.write(data)
21             f.flush()
22             f.close()
23         return url,'success'
24     except Exception:
25         return url,'falied'
26     
27 def callback(url,result):
28     print '%s download is %s'%(url,result)
29     
30 def threadPoolDownload(poolsize,args):
31     start = datetime.datetime.now()
32     pool = threadpool.ThreadPool(poolsize)
33     requests = threadpool.makeRequests(download,args,callback)
34     [pool.putRequest(req) for req in requests]
35     pool.wait()
36     end = datetime.datetime.now()
37     print "Start download : ",start
38     print "End download : ",end

实现多线程并发还可以使用stackless 和 twisted ，关于stackless微线程，我还尚未进行学习。关于twisted的内容可以查看我的随笔。最近比较忙，代码留着以后再编吧。届时我打算将这几种并发方法进行一个比较，希望继续关注。