# -*- coding: utf-8 -*-
import requests, re
import time
import os
import csv
import sys
import importlib
from fake_useragent import UserAgent
importlib.reload(sys)
class WeiBoSpider():
def __init__(self, page):
self.path = os.getcwd() + "/weibo.csv"
self.csvfile = open(self.path, "a", newline="", encoding="utf-8-sig")
self.writer = csv.writer(self.csvfile)
# csv头部
self.writer.writerow(('话题链接', '话题内容', '楼主ID', '楼主昵称', '楼主性别', '发布日期',
'发布时间', '转发量', '评论量', '点赞量', '评论者ID', '评论者昵称',
'评论者性别', '评论日期', '评论时间', '评论内容'))
self.headers = {
'Cookie': '_T_WM=22822641575; H5_wentry=H5; backURL=https%3A%2F%2Fm.weibo.cn%2F; ALF=1584226439; MLOGIN=1; SUBP=0033WrSXqPxfM725Ws9jqgMF55529P9D9W5RJaVYrb.BEuOvUQ8Ca2OO5JpX5K-hUgL.FoqESh-7eKzpShM2dJLoIp7LxKML1KBLBKnLxKqL1hnLBoMceoBfeh2EeKBN; SCF=AnRSOFp6QbWzfH1BqL4HB8my8eWNC5C33KhDq4Ko43RUIzs6rjJC49kIvz5_RcOJV2pVAQKvK2UbAd1Uh6j0pyo.; SUB=_2A25zQaQBDeRhGeBM71cR8SzNzzuIHXVQzcxJrDV6PUJbktAKLXD-kW1NRPYJXhsrLRnku_WvhsXi81eY0FM2oTtt; SUHB=0mxU9Kb_Ce6s6S; SSOLoginState=1581634641; WEIBOCN_FROM=1110106030; XSRF-TOKEN=dc7c27; M_WEIBOCN_PARAMS=oid%3D4471980021481431%26luicode%3D20000061%26lfid%3D4471980021481431%26uicode%3D20000061%26fid%3D4471980021481431',
'Referer': 'https://m.weibo.cn/detail/4312409864846621',
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.100 Safari/537.36',
'X-Requested-With': 'XMLHttpRequest'
}
self.comments_ID = []
self.page = page
def get_title_id(self):
# 获取内容中的id列表
for page in range(1, self.page):
self.headers = {
"User-Agent": UserAgent().chrome
}
time.sleep(1)
api_url = 'https://m.weibo.cn/api/feed/trendtop?containerid=102803_ctg1_600059_-_ctg1_600059&page=' + str(page)
rep = requests.get(url=api_url, headers=self.headers)
# 获取ID值并写入列表comment_ID中
for json in rep.json()['data']['statuses']:
comment_ID = json['id']
self.comments_ID.append(comment_ID)
def spider_title(self, id):
"""爬取战役情每个主题的详情页面"""
try:
title_url = 'https://m.weibo.cn/detail/' + str(id)
html_text = requests.get(url=title_url, headers=self.headers).text
# 内容
title = re.findall('.*?"text": "(.*?)",.*?', html_text)[0]
# 去掉title中的html标签
text = re.sub('<(S*?)[^>]*>.*?|<.*? />', '', title)
# 用户id
user_id = re.findall('.*?"id": "(.*?)",.*?', html_text)[0]
# 用户昵称
user_nicname = re.findall('.*?"screen_name": "(.*?)",.*?', html_text)[0]
# 性别
user_gender = re.findall('.*?"gender": "(.*?)",.*?', html_text)[0]
# 发布时间
created_title_time = re.findall('.*?"created_at": "(.*?)",.*?', html_text)[0].split(" ")
# 日期
if 'Mar' in created_title_time:
title_created_YMD = "{}/{}/{}".format(created_title_time[-1], '03', created_title_time[2])
elif 'Feb' in created_title_time:
title_created_YMD = "{}/{}/{}".format(created_title_time[-1], '02', created_title_time[2])
elif 'Jan' in created_title_time:
title_created_YMD = "{}/{}/{}".format(created_title_time[-1], '01', created_title_time[2])
else:
pass
# 发布时间
add_title_time = created_title_time[3]
# 转发量
reposts_count = re.findall('.*?"reposts_count": (.*?),.*?', html_text)[0]
# 评论量
comments_count = re.findall('.*?"comments_count": (.*?),.*?', html_text)[0]
# 点赞量
attitudes_count = re.findall('.*?"attitudes_count": (.*?),.*?', html_text)[0]
comment_count = int(int(comments_count) / 20) # 每个ajax一次加载20条数据
position1 = (title_url, text, user_id, user_nicname, user_gender, title_created_YMD,
add_title_time, reposts_count, comments_count, attitudes_count, " ", " ", " ", " ", " ", " ")
# 写入数据
print(title_url, text, user_id, user_nicname, user_gender, title_created_YMD,
add_title_time, reposts_count, comments_count, attitudes_count)
self.writer.writerow((position1))
return comment_count
except:
pass
def get_page(self, id, max_id, id_type):
# 抓取评论信息
params = {
'max_id': max_id,
'max_id_type': id_type
}
url = 'https://m.weibo.cn/comments/hotflow?id={}&mid={}&max_id'.format(id, id )
try:
r = requests.get(url, params=params, headers=self.headers)
if r.status_code == 200:
return r.json()
except requests.ConnectionError as e:
print('error', e.args)
pass
def parse_page(self, jsondata):
if jsondata:
items = jsondata.get('data')
item_max_id = {}
item_max_id['max_id'] = items['max_id']
item_max_id['max_id_type'] = items['max_id_type']
return item_max_id
def write_csv(self, jsondata):
for json in jsondata['data']['data']:
# 用户ID
user_id = json['user']['id']
# 用户昵称
user_name = json['user']['screen_name']
# 用户性别,m表示男性，表示女性
user_gender = json['user']['gender']
# 获取评论
comments_text = json['text']
comment_text = re.sub('<(S*?)[^>]*>.*?|<.*? />', '', comments_text) # 正则匹配掉html标签
# 评论时间
created_times = json['created_at'].split(' ')
if 'Feb' in created_times:
created_YMD = "{}/{}/{}".format(created_times[-1], '02', created_times[2])
elif 'Jan' in created_times:
created_YMD = "{}/{}/{}".format(created_times[-1], '01', created_times[2])
else:
print('该时间不在疫情范围内，估计数据有误！')
pass
created_time = created_times[3] # 评论时间时分秒
position2 = (
" ", " ", " ", " ", " ", " ", " ", " ", " ", " ", user_id, user_name, user_gender, created_YMD,
created_time,
comment_text)
self.writer.writerow((position2)) # 写入数据
def main(self):
self.get_title_id()
count_title = len(self.comments_ID)
for count, comment_ID in enumerate(self.comments_ID):
print("正在爬取第%s个话题，一共找到个%s话题需要爬取" % (count + 1, count_title))
# maxPage获取返回的最大评论数量
maxPage = self.spider_title(comment_ID)
m_id = 0
id_type = 0
if maxPage != 0: # 小于20条评论的不需要循环
try:
# 用评论数量控制循环
for page in range(0, maxPage):
# 自定义函数-抓取网页评论信息
jsondata = self.get_page(comment_ID, m_id, id_type)
# 自定义函数-写入CSV文件
self.write_csv(jsondata)
# 自定义函数-获取评论item最大值
results = self.parse_page(jsondata)
time.sleep(1)
m_id = results['max_id']
id_type = results['max_id_type']
except:
pass
print("--------------------------分隔符---------------------------")
self.csvfile.close()
if __name__ == '__main__':
startTime = time.time()
spider = WeiBoSpider(15)
spider.main()
endTime = time.time()
useTime = (endTime - startTime) / 60
print("该次所获的信息一共使用%s分钟" % useTime)