python之中国大学爬虫

 1 #!/usr/bin/env python3
 2 #-*- coding:utf-8 -*-
 3 ############################
 4 #File Name: zuihaodaxuepaiming.py
 5 #Author: frank
 6 #Mail: frank0903@aliyun.com
 7 #Created Time:2018-05-15 21:45:55
 8 ############################
 9 
10 import requests
11 from bs4 import BeautifulSoup
12 
13 
14 #1. 获取html页面的内容
15 def getHTMLText(url):
16     try:
17         r = requests.get(url)
18         r.raise_for_status()
19         #r.encoding = r.apparent_encoding
20         r.encoding = 'utf-8'
21         return r.text
22 
23     except:
24         print("爬取异常")
25         return ""
26     
27 #2. 从html页面内容中爬取大学排名信息, (排名，学校名称，总分)
28 def getUnivInfo(uLst, html):
29     soup = BeautifulSoup(html, "lxml")
30     #获取排名中的大学的数量
31     print(len(soup.tbody.find_all("tr", recursive=False)))
32     #列出tbody的子节点
33     #for tag in soup.tbody.find_all('tr', recursive=False):
34     #    #print("name:{},type:{}".format(tag.name, type(tag)))
35     #    print(tag.td.string)
36     for tr in soup.tbody.find_all('tr', recursive=False,limit=1):
37         tds = tr('td')
38         #print(tr.prettify())
39         #print(tds)
40         #print(tds[0].string,tds[1].string, tds[3].string)
41         uLst.append([tds[0].string,tds[1].string, tds[3].string])
42 
43 #3. 显示排名信息
44 #def printUnivInfo(uLst, num):
45 #    print("{0:^10}	{1:{3}^10}	{2:^10}".format("排名","学校名称","总分",chr(12288)))
46 #    for uni in uLst[0:num]:
47 #        print("{0:^10}	{1:{3}^10}	{2:^10}".format(uni[0],uni[1],uni[2],chr(12288)))
48 
49 def printUnivInfo(uLst, num):
50     ptstr = "{0:^10}	{1:{3}^10}	{2:^10}"
51     print(ptstr.format("排名","学校名称","总分",chr(12288)))
52     for uni in uLst[0:num]:
53         print(ptstr.format(uni[0],uni[1],uni[2],chr(12288))) #中文字符的空格填充 chr(12288)
54 
55 def main():
56     url = "http://www.zuihaodaxue.cn/zuihaodaxuepaiming2016.html"
57     uLst = []
58     html = getHTMLText(url)
59     #print(html)
60     print("getHTMLText")
61     getUnivInfo(uLst, html)
62     print("getUnivInfo")
63     printUnivInfo(uLst, 10)
64 
65 main()
该实例主要是针对 requests 和 BeautifulSoup的应用。