网页正文提取,降噪的实现(readability/Document)

安装: pip install readability-lxml

使用:

# encoding:utf-8
import html2text
import requests
import re
from readability.readability import Document


res = requests.get('http://finance.sina.com.cn/roll/2019-02-12/doc-ihrfqzka5034116.shtml')

# 获取新闻标题
readable_title = Document(res.content).short_title()
# 获取内容并清洗
readable_article = Document(res.content).summary()
text_p = re.sub(r'</?div.*?>', '', readable_article)
text_p = re.sub(r'((</p>)?<a href=.*?>|</a>(<p>)?)', '', text_p)
text_p = re.sub(r'<select>.*?</select>', '', text_p)
print text_p

html2text

html2text的使用:

安装: pip install html2text

使用:

def test_func2(html):
    """获取指定URL的html，对html进行处理"""
    h = html2text.HTML2Text()
    h.ignore_links = True  # (True剔除超链接,False保留)
    print h.handle(html)
res = requests.get('http://finance.sina.com.cn/roll/2019-02-12/doc-ihrfqzka5034116.shtml')

test_func2(res.content.decode('utf-8'))