1 ''' 2 主页: 3 图标地址、下载次数、大小、详情页地址 4 5 详情页: 6 游戏名、好评率、评论数、小编点评、下载地址、简介、网友评论、1-5张截图链接地址、 7 https://www.wandoujia.com/wdjweb/api/category/more?catId=6001&subCatId=0&page=1&ctoken=FRsWKgWBqMBZLdxLaK4iem9B 8 9 https://www.wandoujia.com/wdjweb/api/category/more?catId=6001&subCatId=0&page=2&ctoken=FRsWKgWBqMBZLdxLaK4iem9B 10 11 https://www.wandoujia.com/wdjweb/api/category/more?catId=6001&subCatId=0&page=3&ctoken=FRsWKgWBqMBZLdxLaK4iem9B 12 13 32 14 ''' 15 import requests 16 from bs4 import BeautifulSoup 17 from pymongo import MongoClient 18 import re 19 20 #连接mongoDB数据库 21 client=MongoClient('localhost',27017) 22 #主页信息 23 index_col=client['wandoujia']['index'] 24 #详情页信息 25 detail_col=client['wandoujia']['detail'] 26 27 # 1、发送请求 28 def get_page(url): 29 response = requests.get(url) 30 return response 31 32 # 2、开始解析 33 # 解析详情页 34 def parse_detail(text): 35 soup = BeautifulSoup(text, 'lxml') 36 # print(soup) 37 38 # app名称 39 try: 40 name = soup.find(name="span", attrs={"class": "title"}).text 41 except Exception: 42 name=None 43 # print(name) 44 45 # 好评率 46 try: 47 love = soup.find(name='span', attrs={"class": "love"}).text 48 except Exception: 49 love = None 50 # print(love) 51 52 # 评论数 53 try: 54 commit_num = soup.find(name='a', attrs={"class": "comment-open"}).text 55 except Exception: 56 commit_num = None 57 # print(commit_num) 58 59 # 小编点评 60 try: 61 commit_content = soup.find(name='div', attrs={"class": "con"}).text 62 except Exception: 63 commit_content = None 64 # print(commit_content) 65 66 # app下载链接 67 try: 68 download_url = soup.find(name='a', attrs={"class": "normal-dl-btn"}).attrs['href'] 69 except Exception: 70 download_url = None 71 # print(download_url) 72 73 print(''' 74 ============= tank ============== 75 app名称:{name} 76 好评率: {love} 77 评论数: {commit_num} 78 小编点评: {commit_content} 79 app下载链接: {download_url} 80 ============= end ============== 81 '''.format(name='name',love='love',commit_num='commit_num',commit_content='commit_content',download_url='download_url') 82 ) 83 84 #判断所有数据都存在，正常赋值 85 if name and love and commit_num and commit_content and download_url: 86 detail_data={ 87 'name':name, 88 'love':love, 89 'commit_num':commit_num, 90 'commit_content':commit_content, 91 'download_url':download_url, 92 } 93 94 #若love没有值，则设置为 没人点赞，很惨 95 if not love: 96 detail_data = { 97 'name': name, 98 'love': "没人点赞，很惨", 99 'commit_num':commit_num, 100 'commit_content':commit_content, 101 'download_url':download_url 102 } 103 104 # 若download_url没有值，则设置为 没有安装包 105 if not love: 106 detail_data = { 107 'name':name, 108 'love':love, 109 'commit_num': commit_num, 110 'commit_content': commit_content, 111 'download_url': "没有安装包", 112 } 113 114 #插入详情页数据 115 detail_col.insert(detail_data) 116 print('{name}app数据插入成功！') 117 118 119 120 121 # 解析主页 122 def parse_index(data): 123 soup = BeautifulSoup(data, 'lxml') 124 125 # 获取所有app的li标签 126 app_list = soup.find_all(name='li', attrs={"class": "card"}) 127 for app in app_list: 128 # print(app) 129 # print('tank' * 1000) 130 # print('tank *' * 1000) 131 # print(app) 132 # 图标地址 133 # 获取第一个img标签中的data-original属性 134 img = app.find(name='img').attrs['data-original'] 135 print(img) 136 137 # 下载次数 138 # 获取class为install-count的span标签中的文本 139 down_num = app.find(name='span', attrs={"class": "install-count"}).text 140 print(down_num) 141 142 143 # 大小 144 # 根据文本正则获取到文本中包含 数字 + MB（d+代表数字）的span标签中的文本 145 size = soup.find(name='span', text=re.compile("d+MB")).text 146 print(size) 147 148 # 详情页地址 149 # 获取class为detail-check-btn的a标签中的href属性 150 # detail_url = soup.find(name='a', attrs={"class": "name"}).attrs['href'] 151 # print(detail_url) 152 153 # 详情页地址 154 detail_url = app.find(name='a').attrs['href'] 155 print(detail_url) 156 157 # 拼接数据 158 index_data = { 159 'img': img, 160 'down_num': down_num, 161 'size': size, 162 'detail_url': detail_url, 163 } 164 165 # 插入数据 166 index_col.insert(index_data) 167 print('主页数据插入成功！') 168 169 # 3、往app详情页发送请求 170 response = get_page(detail_url) 171 172 # 4、解析app详情页 173 parse_detail(response.text) 174 175 176 def main(): 177 for line in range(1, 33): 178 url = "https://www.wandoujia.com/wdjweb/api/category/more?catId=6001&subCatId=0&page={line}&ctoken=FRsWKgWBqMBZLdxLaK4iem9B" 179 180 # 1、往app接口发送请求 181 response = get_page(url) 182 # print(response.text) 183 print('*' * 1000) 184 # 反序列化为字典 185 data = response.json() 186 187 # 获取接口中app标签数据 188 app_li = data['data']['content'] 189 # print(app_li) 190 # 2、解析app标签数据 191 parse_index(app_li) 192 193 #执行完所有函数关闭mongoDB客户端 194 client.close() 195 196 if __name__ == '__main__': 197 main()