总结常用的功能小实例,快速学习并掌握python技能
1.墨迹天气
import requests from lxml.html import etree import json import time # 导入模块 class MoJiWeather(): def city_name(self): # 定义一个输入城市名称的函数 cityname = str(input("输入城市名称：")) return cityname def search_city(city_name):# 搜索这个城市 index_url = "http://tianqi.moji.com/api/citysearch/%s"%city_name # 构造查询相应城市天气的url response = requests.get(index_url) response.encoding = "utf-8" try:# 异常捕获 city_id = json.loads(response.text).get('city_list')[0].get('cityId')# 通过上面的url获取城市的id city_url = "http://tianqi.moji.com/api/redirect/%s"%str(city_id) # 通过城市id获取城市天气 return city_url except: print('城市名输入错误') exit() def parse(city_url):# 解析函数 response = requests.get(city_url) response.encoding = 'utf-8' html = etree.HTML(response.text) current_city = html.xpath("//div[@class='search_default']/em/text()")[0]# 下面都是利用xpath解析的 print('当前城市：'+current_city) current_kongqi = html.xpath("//div[@class='left']/div[@class='wea_alert clearfix']/ul/li/a/em/text()")[0] print('空气质量：'+current_kongqi) current_wendu = html.xpath("//div[@class='left']/div[@class='wea_weather clearfix']/em/text()")[0] print('当前温度：'+current_wendu+'℃') current_weather = html.xpath("//div[@class='wea_weather clearfix']/b/text()")[0] print('天气状况：' + current_weather) current_shidu = html.xpath("//div[@class='left']/div[@class='wea_about clearfix']/span/text()")[0] print('当前湿度：'+current_shidu) current_fengji = html.xpath("//div[@class='left']/div[@class='wea_about clearfix']/em/text()")[0] print('当前风速：'+current_fengji) jingdian = html.xpath("//div[@class='right']/div[@class='near'][2]/div[@class='item clearfix']/ul/li/a/text()") print('附近景点：') for j in jingdian: print(' '+j) if __name__ == '__main__': print("欢迎使用墨迹天气查询系统") city_name = MoJiWeather.city_name(1) city_url = MoJiWeather.search_city(city_name) MoJiWeather.parse(city_url) print("谢谢使用本查询系统") input("按任意键退出...")
2.Tiobe排行榜
import json from lxml import etree from lxml.etree import ParseError import requests from requests.exceptions import RequestException ''' lxml实例应用 ''' ''' 获取页面数据 ''' def one_to_page(url): headers = { 'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.62 Safari/537.36' } try: res = requests.get(url,headers=headers) body = res.text #获取网页内容 except RequestException as e: print('request is error',e) try: html = etree.HTML(body,etree.HTMLParser()) # tr 下的所有子孙节点(只获取文本数,图片资源不获取) result = html.xpath('//table[contains(@class,"table-top20")]/tbody/tr//text()') pos = 0 for i in range(20): if i == 0: yield result[i:5] else: yield result[pos:pos+5] pos += 5 except ParseError as e: print(e.position) ''' 写入文件 ''' def write_file(data): for item in data: sul = { '2018年6月排行': item[0], '2017年6排行': item[1], '开发语言': item[2], '评级': item[3], '变化率': item[4] } # with 更好处理异常情况,进行文件的关闭后续工作 with open('test.txt','a',encoding='utf-8') as f: f.write(json.dumps(sul,ensure_ascii=False)+' ') f.close() print(sul) return None ''' 主程序 ''' def main(): url = 'https://www.tiobe.com/tiobe-index/' data = one_to_page(url) ret = write_file(data) if ret == None: print('ok') if __name__ == '__main__': main()
3.新闻列表
''' 墨迹天气文章爬虫 ''' import requests import json from lxml.html import etree from lxml.etree import ParseError ''' 解析页面内容 ''' def parseHtml(content): try: html = etree.HTML(content,etree.HTMLParser()) # one = html.xpath('//ul[@class="advisory_list_item"]//text()') one = html.xpath('//ul[@class="advisory_list_item"]//li/a/@href') print(one) exit(0) LOOP = 8 pos = 0 for i in range(20): if i == 0: yield one[0:LOOP] else: yield one[pos:pos+LOOP] pos += LOOP except ParseError as e: print(e.position) ''' 写入文件 ''' def write_log(data): for item in data: msg = { '发文时间':item[3], '文章标题':item[5] } with open('moji.log','a',encoding='utf-8') as f: f.write(json.dumps(msg,ensure_ascii=False)+' ') f.close() print(msg) return None ''' 主程序 ''' def main(): for page in range(1,73): url = 'https://tianqi.moji.com/news/list/moji/{}'.format(page) res = requests.get(url) res.encoding = 'utf-8' content = parseHtml(res.text) ret = write_log(content) if ret is None: print('ok') if __name__ == '__main__': main()
4.爬取IP
import requests import re import random from bs4 import BeautifulSoup ua_list = [ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.109 Safari/537.36", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.75 Safari/537.36", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.119 Safari/537.36", "Mozilla / 5.0(Windows NT 6.1;WOW64) AppleWebKit / 537.36(KHTML, likeGecko) Chrome / 45.0.2454.101Safari / 537.36" ] def ip_parse_xici(page): """ :param page: 采集的页数 :return: """ ip_list = [] for pg in range(1, int(page)): url = 'http://www.xicidaili.com/nn/' + str(pg) user_agent = random.choice(ua_list) my_headers = { 'Accept': 'text/html, application/xhtml+xml, application/xml;', 'Accept-Encoding': 'gzip, deflate, sdch', 'Accept-Language': 'zh-CN,zh;q=0.8', 'Referer': 'http: // www.xicidaili.com/nn', 'User-Agent': user_agent } try: r = requests.get(url, headers=my_headers) soup = BeautifulSoup(r.text, 'html.parser') except requests.exceptions.ConnectionError: print('ConnectionError') else: data = soup.find_all('td') # 定义IP和端口Pattern规则 ip_compile = re.compile(r'<td>(d+.d+.d+.d+)</td>') # 匹配IP port_compile = re.compile(r'<td>(d+)</td>') # 匹配端口 ips = re.findall(ip_compile, str(data)) # 获取所有IP ports = re.findall(port_compile, str(data)) # 获取所有端口 check_api = "http://ip.taobao.com/service/getIpInfo2.php?ip=" for i in range(len(ips)): if i < len(ips): ip = ips[i] api = check_api + ip api_headers = { 'User-Agent': user_agent } try: response = requests.get(url=api, headers=api_headers, timeout=2) print("ip：%s 可用" % ip) except Exception as e: print("此ip %s 已失效：%s" % (ip, e)) del ips[i] del ports[i] ips_usable = ips ip_list += [':'.join(n) for n in zip(ips_usable, ports)] # 列表生成式 print('第{}页ip采集完成'.format(pg)) print(ip_list) if __name__ == '__main__': xici_pg = input("请输入需要采集的页数：") ip_parse_xici(page=xici_pg)