python写简单爬虫的五种方法 （转）

原文地址链接：  http://blog.sina.com.cn/s/blog_5e32cc130100fszx.html

2, 【用Python写爬虫】获取html的方法【二】：使用pycurl

# Pycurl参考地址：http://pycurl.sourceforge.net/
# Pycurl下载地址：http://pycurl.sourceforge.net/download/pycurl-7.18.1.tar.gz

#!/usr/bin/python
#coding:utf8

import pycurl
import StringIO

def getURLContent_pycurl(url):
    c=pycurl.Curl()
    c.setopt(pycurl.URL,url)
    b=StringIO.StringIO()
    c.setopt(pycurl.WRITEFUNCTION,b.write)
    c.setopt(pycurl.FOLLOWLOCATION,1)
    c.setopt(pycurl.MAXREDIRS,5)

    c.perform()
    return b.getvalue()

url='http://www.baidu.com'

content = getURLContent_pycurl(url)

print content

3,【用Python写爬虫】获取html的方法【三】：使用cPAMIE
# cPAMIE下载：http://sourceforge.net/project/showfiles.php?group_id=103662

     1    #!/usr/bin/python
     2    #coding:utf8
     3    
     4    import cPAMIE
     5    
     6    def getURLContent_cPAMIE(url):
     7        g_ie = cPAMIE.PAMIE()
     8        g_ie = showDebugging = False
     9        g_ie.frameName = None
    10        g_ie.navigate(url)
    11        
    12        content = g_ie.pageGetText()
    13        g_ie.quit()
    14        return content
    15    
    16    url = 'http://www.baidu.com'
    17    
    18    content = getURLContent_cPAMIE(url)
    19    
    20    print content