''' 作业2： 判断user-agent,判断是否是正常浏览器访问 ''' from urllib import request base_url = "http://www.langlang2017.com" headers = { "connnction":"keep-alive", "USer_Agent":"mozilla/5.0 (Windows nt 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.84 Safari/537.36" } req = request.Request(base_url,headers=headers) user_agent = req.get_header("User_agent") print(req.headers,user_agent) # if user_agent: # print("是浏览器访问") # else: # print("不是浏览请求！") # response = request.urlopen(req) # # html = response.read() # # html = html.decode("utf-8") # # # # print(html) ''' 作业1： 假装浏览访问，假装不同的浏览器访问。 从user_agent_list.txt文件中，读取user_agent数据，用来封装成一个带headers的request对象，进行网站页面的爬取。 ''' #1.读取文件内容 # text = "" # with open("user_agent_list.txt","r",encoding="utf-8") as f: # text = f.read() # print(text) # f.close() #2.使用文件内容 # import random # user_agent_list = text.split(" ") # print(len(user_agent_list)) # headers = { # "user_agent":random.choice(user_agent_list) # } # # # req = request.Request(base_url,headers=headers) # # response = request.urlopen(req) # # html = response.read() # # html = html.decode("utf-8") # # print(html,req.get_header("User_agent")) # # #小结：遇到的问题，一次读全部的user_agent，太占内存，而且大多数用不着。 # # 解决方法：使用迭代读取文件内容，确保节省内存。 # # for line in open("user_agent_list.txt"): # headers = { # "user_agent":line.strip() # } # # req = request.Request(base_url,headers=headers) # response = request.urlopen(req) # html = response.read() # html = html.decode("utf-8") # print(html)