1 #!/usr/bin/env python 2 3 import cStringIO # 4 import formatter # 5 from htmllib import HTMLParser # We use various classes in these modules for parsing HTML. 6 import httplib # We only need an exception from this module 7 import os # This provides various file system functions 8 import sys # We are just using argv for command-line arguments 9 import urllib # We only need the urlretrieve()function for downloading Web pages 10 import urlparse # We use the urlparse()and urljoin()functions for URL manipulation 11 12 class Retriever(object): 13 __slots__ = ('url','file') 14 15 def __init__(self,url): 16 self.url, self.file = self.get_file(url) 17 18 def get_file(self, url, default='index.html'): 19 'Create usable local filename from URL' 20 parsed = urlparse.urlparse(url) # ParseResult(scheme='http', netloc='www.baidu.com', path='', params='', query='', fragment='') 21 host = parsed.netloc.split('@')[-1].split(':')[0] # 'www.baidu.com' 22 filepath = '%s%s' % (host,parsed.path) # 'www.baidu.com' 23 if not os.path.splitext(parsed.path)[1]: # '' 24 filepath = os.path.join(filepath, default) # 'www.baidu.com\index.html' 25 linkdir = os.path.dirname(filepath) # 'www.baidu.com' 26 if not os.path.isdir(linkdir): # False 27 if os.path.exists(linkdir): # False 28 os.unlink(linkdir) 29 os.makedirs(linkdir) # make a directory named by link directory on the hard disc 30 return url, filepath 31 32 def download(self): 33 'Download URL to specific name file' 34 try: 35 retval = urllib.urlretrieve(self.url, self.file) 36 except (IOError, httplib.InvalidURL) as e: 37 retval = (('*** ERROR:bad URL "%s": %s' % (self.url,e)),) 38 return retval 39 40 def parse_links(self): 41 'Parse out the links found in downloaded HTML file' 42 f = open(self.file, 'r') 43 data = f.read() 44 f.close() 45 parser = HTMLParser(formatter.AbstractFormatter(formatter.DumbWriter(cStringIO.StringIO()))) 46 parser.feed(data) 47 parser.close() 48 return parser.anchorlist 49 50 class Crawler(object): 51 count = 0 # the number of objects downloaded from the internet 52 53 def __init__(self, url): 54 self.q = [url] # a queue of links to download 55 self.seen = set() # a set containing all the links that we have seen(downloaded) already 56 parsed = urlparse.urlparse(url) 57 host = parsed.netloc.split('@')[-1].split(':')[0] 58 self.dom = '.'.join(host.split('.')[-2:]) # 'b.a.i.d.u' 59 60 def get_page(self, url, media=False): 61 'Download page & parse links, add to queue if nec' 62 r = Retriever(url) 63 fname = r.download()[0] # 'www.baidu.com\index.html' 64 if fname[0] == '*': # 'w' 65 print fname, '... skipping parse' 66 return 67 Crawler.count += 1 # 1 68 print ' (', Crawler.count, ')' # (1) 69 print 'URL:', url # URL: http://www.baidu.com 70 print 'FILE:', fname # FILE: www.baidu.com\index.html 71 self.seen.add(url) # set(['http://www.baidu.com']) 72 ftype = os.path.splitext(fname)[1] # '.html' 73 if ftype not in ('.htm', '.html'): # False 74 return 75 76 for link in r.parse_links(): 77 if link.startswith('mailto:'): # False 78 print '... discarded, mailto link' 79 continue 80 if not media: # False 81 ftype = os.path.splitext(link)[1] 82 if ftype in ('.mp3','.mp4','.m4v','.wav'): 83 print '... discarded, media file' 84 continue 85 if not link.startswith('http://'): # False 86 link = urlparse.urljoin(url, link) 87 print '*', link, 88 if link not in self.seen: # True 89 if self.dom not in link: # False 90 print '... discarded, not in domain' 91 else: 92 if link not in self.q: 93 self.q.append(link) 94 print '... new, added to Q' 95 else: 96 print '... discarded, already in Q' 97 else: 98 print '... discarded, already processed' 99 100 def go(self, media=False): 101 'Process next page in queue (if any)' 102 while self.q: 103 url = self.q.pop() 104 self.get_page(url, media) 105 106 def main(): 107 if len(sys.argv) > 1: 108 url = sys.argv[1] 109 else: 110 try: 111 url = raw_input('Enter starting URL:') 112 except(KeyboardInterrupt, EOFError): 113 url = '' 114 if not url: 115 return 116 if not url.startswith('http://') and not url.startswith('ftp://'): 117 url = 'http://%s/' % url 118 robot = Crawler(url) 119 robot.go() 120 121 if __name__ == '__main__': 122 main()