爬取新闻网站的财经信息

要爬取的新闻信息
可以通过Ctrl+U快捷键查看页面的html源码，便于数据结构分析
数据结构：类news_li下的h2标签下的a链接指定每个新闻的详情
完整代码如下:

#encoding:utf-8
import requests
from pyquery import PyQuery as pq
import os
import datetime

header = {
        "Referer": "https://www.thepaper.cn/channel_25951",
        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.100 Safari/537.36"
}
base_url = "https://www.thepaper.cn/"
def index():
    url = base_url + "channel_25951"
    response = requests.get(url, header).text
    # 数据初始化
    doc = pq(response)
    # 使用css选择器的方式进行数据提取(类（news_li）———》h2标签--》a标签内容)
    a = doc(".news_li h2 a").items()
    for x in a:
        # 获取新闻详情链接
        href = base_url + x.attr("href")
        # 提取文本数据
        title = x.text()
        content(href, title)

def content(href, title):
    response = requests.get(href, header).text
    doc = pq(response)
    news = doc(".news_txt").items()
    for x in news:
        new = x.text()
        date = datetime.datetime.now().strftime("%Y-%m-%d") + "//"
        if not os.path.exists(date):
            os.mkdir(date)
        with open(date + "{}.txt".format(title),"a", encoding="utf-8") as f:
            f.write(new)

index()