import requests import csv import time import numpy as np from bs4 import BeautifulSoup import json import pandas as pd from snownlp import SnowNLP from snownlp import sentiment import matplotlib.pyplot as plt import jieba #分词库 import wordcloud #词云库 # 必要的库 print("开始爬取评论") name='评论' def get_html(url): headers = { 'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8', 'user-agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) appleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36', } # 模拟访问信息 r = requests.get(url, timeout=30, headers=headers) r.raise_for_status() r.endcodding = 'utf-8' return r.text def get_content(url): comments = [] html = get_html(url) try: s = json.loads(html) except: print("jsonload error") num = len(s['data']['replies']) # 获取每页评论栏的数量 i = 0 while i < num: comment = s['data']['replies'][i] # 获取每栏信息 InfoDict = {} # 存储每组信息字典 InfoDict['用户名'] = comment['member']['uname'] InfoDict['uid号'] = comment['member']['mid'] InfoDict['评论内容'] = comment['content']['message'] InfoDict['性别'] = comment['member']['sex'] InfoDict['点赞'] = comment['like'] InfoDict['回复数量'] = comment['rcount'] InfoDict['用户等级'] = comment['member']['level_info']['current_level'] InfoDict['用户类别'] = comment['member']['vip']['vipType'] comments.append(InfoDict) i+=1 return comments def Out2File(dict): with open('data\\txt\\'+name+'.txt', 'a+', encoding='utf-8') as f: for user in dict: try: f.write('{},{},{},{},{},{},{},{}\n'.format( user['用户名'], user['uid号'], user['性别'], user['点赞'], user['回复数量'], user['用户等级'], user['用户类别'], user['评论内容'])) except: print("out2File error") print('当前页面保存完成') e = 0 page = 1 while e == 0: url = "https://api.bilibili.com/x/v2/reply/main?&jsonp=jsonp&next=" + str(page) + "&type=1&oid=379315227&mode=3&plat=1&_=1641373150736" #https: // api.bilibili.com / x / v2 / reply / main?87 & jsonp = jsonp & next = 2 & type = 1 & oid = 892600469 & mode = 3 & plat = 1 & _ = 1641279217907 try: print() content = get_content(url) print("page:", page) Out2File(content) page = page + 1 # 为了降低被封ip的风险，每爬10页便歇5秒。 if page % 10 == 0: # 求余数 time.sleep(5) except: e = 1 print("爬取评论完成") print("开始转化文件格式") txt = pd.read_csv('data\\txt\\'+name+'.txt',names=['用户名','uid号','性别','点赞数量','回复数量','用户等级','vip类别','评论内容'],error_bad_lines=False,delimiter=',',encoding='utf-8') txt.to_csv('data\\beforecsv\\'+name+'.csv',index=False,encoding='utf-8',float_format=int) print("转化文件格式完成") print("开始情感分析") df=pd.read_csv('data\\beforecsv\\'+name+'.csv',header=1,usecols=[7]) contents=df.values.tolist() print(len(contents)) score=[] for contents in contents: try: s=SnowNLP(contents[0]) score.append(s.sentiments) except: print("something is wrong") score.append(0.5) print(len(score)) data2=pd.DataFrame(score) data2.to_csv('data\\emotion\\'+name+'.csv',index=False,mode='a+') print("情感分析完成") print("开始得出结果数据") f1 = pd.read_csv('data\\beforecsv\\'+name+'.csv',header=1) f2 = pd.read_csv('data\\emotion\\'+name+'.csv') file = [f1,f2] train = pd.concat(file,axis=1) train.to_csv('data\\finalcsv\\'+name+'.csv',sep=',',index=False) print("成功得出结果数据") print("产生词云图") #1.读取文件 f=open('data\\txt\\'+name+'.txt',encoding='utf-8') #f=open('..\\paqushuju\\评论文件\\魔王勇者.txt',encoding='utf-8') text=f.read() #2.分词，把一句话分割成一个个词语 text_list=jieba.lcut(text) #list转换为字符串 text_str=''.join(text_list) print(text_list) wc = wordcloud.WordCloud( width=500,#宽度 height=500,#高度 background_color='white',#背景颜色 stopwords={'姓名','uid','性别','点赞数量','回复数量','用户等级','用户vip等级','评论内容', '男','女','保密','doge','藏狐'}, font_path='msyh.ttc'#字体 ) wc.generate(text_str) wc.to_file('data\\wordcloud\\'+name+'.png') print("成功得出词云图") #df=pd.read_csv('d:/python/out/cut.csv',encoding='utf8')bins=[0,20,40,60,80,100,120]#设置自定义标签，定义的标签需要和区间个数及顺序都一一对应customLabels=['0到20','20到40','40到60','60到80','80到100','100到120']df['cut']=pd.cut(df.cost,bins,right=False,labels=customLabels)df.to_csv('d:/python/out/cut628c_out.csv')df
实现将爬虫，text格式转换csv，再进行情感分析，再将结果整合到一起，最后得出目标表和词云图
后续应该就是数据的重复爬取了