发起一个开源项目http://www.abelkhan.com/
目前而言，已经用python编写了一个网络爬虫抓取页面，和一个简单的前端
网络爬虫，已经有很多高手写过，我基本上奉行了拿来主义，
得益于python完善的lib，这个网络爬虫实现起来非常的简单:
使用urllib2从对应的url地址抓取html
def get_page(url): try: headers = {'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebkit/537.36 (KHTML, like Gecko) Chrome/42.0.2311.135 Safari/537.36 Edge/12.10240', 'Connection':'Keep-Alive', 'Accept':'text/html, application/xhtml+xml, image/jxr, */*', 'Accept-Language':'zh-Hans-CN,zh-Hans;q=0.8,en-US;q=0.5,en;q=0.3', } cookie_jar = cookielib.CookieJar() opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cookie_jar)) req = urllib2.Request(url = url, headers = headers) response = opener.open(req, timeout = 5) the_page = response.read() headers = response.info() return the_page, headers except: import traceback traceback.print_exc()
一个需要注意的地方是，有部分网站会限制爬虫访问，所以我加入了headers用于模拟浏览器访问。
这个方法差强人意，但是我也没有找到一个更完善的办法。
抓取到页面后，基于HTMLParser做了html的解析:
class htmlprocess(HTMLParser.HTMLParser): def __init__(self, urlinfo): HTMLParser.HTMLParser.__init__(self) self.urllist = {} self.sub_url = "" self.urlinfo = urlinfo self.current_url = urlinfo['url'] keywords = doclex.simplesplit(self.current_url) for key in keywords: if key != "com" and key != "www" and key != "cn": self.urlinfo['keys']['1'].append(key) self.current_tag = "" self.style = "" def handle_starttag(self, tag, attrs): self.current_tag = tag self.style = 'None' self.sub_url = "" if tag == 'meta': for name,value in attrs: if name == 'name': if value == 'keywords' or value == 'metaKeywords': self.style = 'keywords' elif value == 'description' or value == 'metaDescription': self.style = 'profile' for name,value in attrs: if name == 'content': if self.style == 'keywords': keywords = doclex.simplesplit(value) if isinstance(keywords, list): for key in keywords: self.urlinfo['keys']['1'].append(key) elif self.style == 'profile': self.urlinfo['profile']['0'] = value encodingdate = chardet.detect(value) if encodingdate['encoding']: udata = unicode(value, encodingdate['encoding']) tlen = 16 if len(udata) < 16: tlen = len(udata) self.urlinfo['titlegen'].append(udata[0:tlen].encode('utf-8')) else: self.urlinfo['titlegen'].append(value) if tag == 'a' or tag == 'A' or tag == 'link': self.sub_url = "" for name,value in attrs: if name == 'href': if len(value) == 0: return if not judged_url(value): if self.current_url[len(self.current_url) - 1] != '/' and value[0] != '/': value = self.current_url + '/' + value else: value = self.current_url + value if value.find('javascript') != -1: return if value.find('javaScript') != -1: return if self.current_url.find("apple") != -1: if value.find("http://www.apple.com/cn/mac#ac-gn-menustate") !=-1: return if self.current_url.find("cnblogs") != -1: if value.find("http://msg.cnblogs.com/send?recipient=itwriter") != -1: return elif value.find("http://i.cnblogs.com/EditPosts.aspx?opt=1") != -1: return elif value.find("http://i.cnblogs.com/EditPosts.aspx?postid=1935371") != -1: return elif value.find("http://msg.cnblogs.com/send?recipient=itwriter/") != -1: return elif value.find("http://msg.cnblogs.com/send?recipient=itwriter/GetUsername.aspx") != -1: return elif value.find("/EnterMyBlog.aspx?NewArticle=1") != -1: return elif value.find("GetUsername") != -1: return elif value.find("GetMyPassword") != -1: return elif value.find("http://i.cnblogs.com/EditPosts.aspx?postid=") != -1: return elif value[len(value) - 1] == '#': value = value[0:-1] if self.current_url.find(value) != -1: return if value[len(value) - 1] == '#': value = value[0:-1] if value != self.current_url and len(value) < 64 and not ingoreurl(value): self.urllist[value] = {'url':value, 'keys':{'1':[], '2':[], '3':[]}, 'title':'', 'titlegen':[], 'profile':{'0':'', '1':'', '2':[]}} self.sub_url = value print value def handle_data(self, data): if self.current_tag == 'title': try: data = doclex.delspace(data) keys = doclex.lex(data) if isinstance(keys, list) and len(keys) > 0: for key in keys: self.urlinfo['keys']['2'].append(key) if len(data) > 0: self.urlinfo['title'] = data except: import traceback traceback.print_exc() elif self.current_tag == 'a': try: if self.sub_url != "": keys = doclex.simplesplit(data) if isinstance(keys, list) and len(keys) > 0: for key in keys: if key in self.urllist[self.sub_url]['keys']['3']: self.urllist[self.sub_url]['keys']['3'].remove(key) if key not in self.urllist[self.sub_url]['keys']['1'] and key not in self.urllist[self.sub_url]['keys']['2']: self.urllist[self.sub_url]['keys']['2'].append(key) encodingdate = chardet.detect(data) if encodingdate['encoding']: udata = unicode(data, encodingdate['encoding']) tlen = 16 if len(udata) < 16: tlen = len(udata) self.urllist[self.sub_url]['titlegen'].append(udata[0:tlen].encode('utf-8')) if len(udata) > 16: self.urllist[self.sub_url]['profile']['1'] = udata[0:32].encode('utf-8') except: import traceback traceback.print_exc() else: try: if not doclex.invialddata(data): data = doclex.delspace(data) encodingdate = chardet.detect(data) udata = unicode(data, encodingdate['encoding']) tlen = 16 if len(udata) < 16: tlen = len(udata) self.urlinfo['titlegen'].append(udata[0:tlen].encode('utf-8')) if len(udata) > 32: self.urlinfo['profile']['2'].append((udata[0:32] + u"...").encode('utf-8')) keys1 = doclex.lex(data) for key in keys1: self.urlinfo['keys']['3'].append(key) except: import traceback traceback.print_exc()
基本上，要说的就是HTMLParser使用方法见文档，HTMLParser预先了定义了一组虚接口handle_starttag，handle_data和handle_endtag，使用者通过重载这三个接口，来实现对html中的tag进行处理，进而完整的解析抓取到的html。
然后从搜索结果来看，搜索的质量还很不尽如人意，欢迎大家的参与和提出意见
向我们提出意见:http://www.abelkhan.com/guestbook/
对项目进行捐助:http://www.abelkhan.com/collection/
代码托管地址如下:https://github.com/qianqians/websearch欢迎大家参与