python爬虫之json数据处理

 1 # -*- coding: utf-8 -*-
 2 # @Time : 2019/11/5 23:18
 3 # @Author : AForever
 4 # @Site : 
 5 # @File : Spider_05.py
 6 # @Software: PyCharm
 7 
 8 # 处理json数据
 9 
10 from urllib import request
11 import json
12 
13 
14 def get_data():
15     url = 'https://movie.douban.com/j/search_subjects?type=movie&tag=%E7%83%AD%E9%97%A8&sort=recommend&page_limit=400&page_start=0'
16     headers = {
17         'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36'
18     }
19     req = request.Request(url, headers=headers)
20     response = request.urlopen(req)
21     if response.getcode() == 200:
22         result = response.read()
23         # print(type(result))  # bytes类型
24         # print(result)
25         result = str(result, encoding='utf8')
26         print(result)
27         return result
28 
29 
30 def parse_data(html):
31     # 将字符串形式的json转换为dict字典
32     data = json.loads(html)
33     movies = data['subjects']
34     for movie in movies:
35         print(movie['title'], movie['rate'])
36 
37 
38 if __name__ == '__main__':
39     # get_data()
40     parse_data(get_data())