爬取代理IP，并判断是否可用。

 1 # -*- coding:utf-8 -*-
 2 from gevent import monkey
 3 monkey.patch_all()
 4 
 5 import urllib2
 6 from gevent.pool import Pool
 7 
 8 import requests
 9 import re
10 
11 class SpiderProxy: 
12     def __init__(self):
13         self.headers = {
14             "Host": "www.xicidaili.com",
15             "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.11; rv:47.0) Gecko/20100101 Firefox/47.0",
16             "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
17             "Accept-Language": "en-US,en;q=0.5",
18             "Accept-Encoding": "gzip, deflate",
19             "Referer": "http://www.xicidaili.com/nn/",
20             }
21         self.url = 'http://www.xicidaili.com/nn/'
22         self.proxy_list = []
23         self.re_ip = re.compile(r'(?<![.d])(?:d{1,3}.){3}d{1,3}(?![.d])')
24         self.re_port = re.compile(r'<td>(d+)</td>')
25 
26     def get_pagesource(self):
27         ''' 取得所有1-n页上的代理IP'''
28         try:
29             num = int(raw_input('please input 1-'))
30             for i in range(1, num + 1):
31                 pageurl = self.url + str(i)
32                 req = requests.session()
33                 html = req.get(pageurl, headers=self.headers)
34                 ip_list = self.re_ip.findall(html.text)
35                 port_list = self.re_port.findall(html.text)
36                 proxy_zip = zip(ip_list, port_list)
37                 for i in proxy_zip:
38                     self.proxy_list.append({'http':i[0] + ':' + i[1]})
39         except ValueError:
40             print 'please input a num!'
41         return self.proxy_list
42 
43 class IsActiveProxyIP:
44     def __init__(self):
45         self.is_active_proxy_ip = []
46 
47     def probe_proxy_ip(self, proxy_ip):
48         proxy = urllib2.ProxyHandler(proxy_ip)
49         opener = urllib2.build_opener(proxy)
50         urllib2.install_opener(opener)
51         try:
52             html = urllib2.urlopen('http://1212.ip138.com/ic.asp')
53             if html:
54                 self.is_active_proxy_ip.append(proxy_ip)
55                 return True
56             else:
57                 return False
58         except Exception as e:
59             return False
60 
61 if __name__ == '__main__':
62     Proxy = SpiderProxy()
63     proxy_list = Proxy.get_pagesource()
64     proxy_isactive = IsActiveProxyIP()
65     pool = Pool(20)
66     pool.map(proxy_isactive.probe_proxy_ip, proxy_list)
67     with open(r'E:python_demoproxy_ip.txt', 'wb') as f:
68         for ip in proxy_isactive.is_active_proxy_ip:
69             ip = str(ip)
70             f.write(ip[11:-2] + '
')
71     print 'file successed written'