python爬虫获取QQ音乐和豆瓣的最新电影音乐名字
先上代码开源大家一起学习，代码如下：
#!python2 #coding:utf-8 __author__ = 'OldHarry' import urllib2 import os import re import json import xlsxwriter import sys defaultencoding = 'utf-8' if sys.getdefaultencoding() != defaultencoding: reload(sys) sys.setdefaultencoding(defaultencoding) def getHtml(url): send_headers = { 'User-Agent':'Mozilla/5.0 (Windows NT 6.2; rv:16.0) Gecko/20100101 Firefox/16.0', 'Accept':'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', 'Connection':'keep-alive' } urls = urllib2.Request(url,headers=send_headers) html = urllib2.urlopen(urls) if html.getcode() == 200: pass #print ("已捕获"),url,"目标站数据..." else: print ("访问出现错误...错误代码："),html.getcode() return html.read() def kugoumusic(url): xx=getHtml(url) rr=re.compile(r'<span class="songName">(.*?) - (.*?)</span>') x=rr.findall(xx) nk=[] for xxx in x: if xxx not in nk: nk.append(xxx[1].decode('utf8')) print json.dumps(nk, encoding="UTF-8", ensure_ascii=False) return nk def qqmusic(url): xx=getHtml(url) rr=re.compile(r'{"action":{"alert":[0-9]+,"icons":[0-9]+,"msgdown":[0-9]+,"msgfav":[0-9]+,"msgid":[0-9]+,"msgpay":[0-9]+,"msgshare":[0-9]+,"switch":[0-9]+},"album":{"id":[0-9]+,"mid":"[a-zA-Z0-9]+","name":"(.*?)"') x=rr.findall(xx) nq=[] for xxx in x: xxx.strip() if xxx not in nq: nq.append(xxx) print json.dumps(nq, encoding="UTF-8", ensure_ascii=False) return nq def dbmovie(url): ssd = getHtml(url) tt=re.compile(r'alt="(.*?)" rel="[a-z]+" class="" />') shu=tt.findall(ssd) print json.dumps(shu, encoding="UTF-8", ensure_ascii=False) return shu def rmmovie(url): ssd = getHtml(url) tt=re.compile(r'"title":"(.*?)"') shu=tt.findall(ssd) print json.dumps(shu, encoding="UTF-8", ensure_ascii=False) return shu def rmdsj(): ssd = rmmovie('https://movie.douban.com/j/search_subjects?type=tv&tag=%E7%83%AD%E9%97%A8&sort=recommend&page_limit=20&page_start=0')+rmmovie('https://movie.douban.com/j/search_subjects?type=tv&tag=%E7%83%AD%E9%97%A8&sort=recommend&page_limit=20&page_start=20')+rmmovie('https://movie.douban.com/j/search_subjects?type=tv&tag=%E7%83%AD%E9%97%A8&sort=recommend&page_limit=20&page_start=40') return ssd def runtest(): IP_PATH = os.path.abspath('.') + 'TXT.xls' print "酷狗音乐--新歌榜" a=kugoumusic("http://www.kugou.com/") print "腾讯音乐--内地新歌榜" b=qqmusic("https://u.y.qq.com/cgi-bin/musicu.fcg?-=recom2388477980207393&g_tk=5381&loginUin=0&hostUin=0&format=json&inCharset=utf8&outCharset=utf-8¬ice=0&platform=yqq.json&needNewCode=0&data=%7B%22comm%22%3A%7B%22ct%22%3A24%7D%2C%22new_song%22%3A%7B%22module%22%3A%22QQMusic.MusichallServer%22%2C%22method%22%3A%22GetNewSong%22%2C%22param%22%3A%7B%22type%22%3A1%7D%7D%7D") print "腾讯音乐--港台新歌榜" c=qqmusic("https://u.y.qq.com/cgi-bin/musicu.fcg?-=recom6698628102261504&g_tk=5381&loginUin=0&hostUin=0&format=json&inCharset=utf8&outCharset=utf-8¬ice=0&platform=yqq.json&needNewCode=0&data=%7B%22comm%22%3A%7B%22ct%22%3A24%7D%2C%22new_song%22%3A%7B%22module%22%3A%22QQMusic.MusichallServer%22%2C%22method%22%3A%22GetNewSong%22%2C%22param%22%3A%7B%22type%22%3A2%7D%7D%7D") print "腾讯音乐--欧美新歌榜" d=qqmusic("https://u.y.qq.com/cgi-bin/musicu.fcg?-=recom08419989487702839&g_tk=5381&loginUin=0&hostUin=0&format=json&inCharset=utf8&outCharset=utf-8¬ice=0&platform=yqq.json&needNewCode=0&data=%7B%22comm%22%3A%7B%22ct%22%3A24%7D%2C%22new_song%22%3A%7B%22module%22%3A%22QQMusic.MusichallServer%22%2C%22method%22%3A%22GetNewSong%22%2C%22param%22%3A%7B%22type%22%3A3%7D%7D%7D") print "腾讯音乐--日本新歌榜" e=qqmusic("https://u.y.qq.com/cgi-bin/musicu.fcg?-=recom24411354608866187&g_tk=5381&loginUin=0&hostUin=0&format=json&inCharset=utf8&outCharset=utf-8¬ice=0&platform=yqq.json&needNewCode=0&data=%7B%22comm%22%3A%7B%22ct%22%3A24%7D%2C%22new_song%22%3A%7B%22module%22%3A%22QQMusic.MusichallServer%22%2C%22method%22%3A%22GetNewSong%22%2C%22param%22%3A%7B%22type%22%3A4%7D%7D%7D") print "腾讯音乐--韩国新歌榜" f=qqmusic("https://u.y.qq.com/cgi-bin/musicu.fcg?-=recom909302436024819&g_tk=5381&loginUin=0&hostUin=0&format=json&inCharset=utf8&outCharset=utf-8¬ice=0&platform=yqq.json&needNewCode=0&data=%7B%22comm%22%3A%7B%22ct%22%3A24%7D%2C%22new_song%22%3A%7B%22module%22%3A%22QQMusic.MusichallServer%22%2C%22method%22%3A%22GetNewSong%22%2C%22param%22%3A%7B%22type%22%3A5%7D%7D%7D") print "豆瓣电影--正在热映" g=dbmovie("https://movie.douban.com/") print "豆瓣电影--热门电影" h=rmmovie("https://movie.douban.com/j/search_subjects?type=movie&tag=%E7%83%AD%E9%97%A8&sort=recommend&page_limit=20&page_start=0") print "豆瓣电影--最新电影" i=rmmovie("https://movie.douban.com/j/search_subjects?type=movie&tag=%E6%9C%80%E6%96%B0&page_limit=20&page_start=0") print "豆瓣电影--经典电影" j=rmmovie("https://movie.douban.com/j/search_subjects?type=movie&tag=%E7%BB%8F%E5%85%B8&sort=time&page_limit=20&page_start=0") print "豆瓣电影--可播放电影" k=rmmovie("https://movie.douban.com/j/search_subjects?type=movie&tag=%E5%8F%AF%E6%92%AD%E6%94%BE&sort=time&page_limit=20&page_start=0") print "豆瓣电影--高分电影" l=rmmovie("https://movie.douban.com/j/search_subjects?type=movie&tag=%E8%B1%86%E7%93%A3%E9%AB%98%E5%88%86&sort=time&page_limit=20&page_start=0") print "豆瓣电影--热门电视剧" m=rmdsj() workbook = xlsxwriter.Workbook(IP_PATH) worksheet = workbook.add_worksheet() bold = workbook.add_format({'bold': 1, 'align': 'center', 'border': 1}) bold2 = workbook.add_format({'align': 'center', 'border': 1}) headings = ['酷狗音乐--新歌榜', '腾讯音乐--内地新歌榜','腾讯音乐--港台新歌榜','腾讯音乐--欧美新歌榜','腾讯音乐--日本新歌榜','腾讯音乐--韩国新歌榜','豆瓣电影--正在热映','豆瓣电影--热门电影','豆瓣电影--最新电影','豆瓣电影--经典电影','豆瓣电影--可播放电影','豆瓣电影--高分电影','豆瓣电影--热门电视剧'] worksheet.write_row('A1', headings, bold) SS=30 worksheet.set_column('A:A', SS) worksheet.set_column('B:B', SS) worksheet.set_column('C:C', SS) worksheet.set_column('D:D', SS) worksheet.set_column('E:E', SS) worksheet.set_column('F:F', SS) worksheet.set_column('G:G', SS) worksheet.set_column('H:H', SS) worksheet.set_column('I:I', SS) worksheet.set_column('J:J', SS) worksheet.set_column('K:K', SS) worksheet.set_column('L:L', SS) worksheet.set_column('M:M', SS) worksheet.write_column('A2', a, bold2) worksheet.write_column('B2', b, bold2) worksheet.write_column('C2', c, bold2) worksheet.write_column('D2', d, bold2) worksheet.write_column('E2', e, bold2) worksheet.write_column('F2', f, bold2) worksheet.write_column('G2', g, bold2) worksheet.write_column('H2', h, bold2) worksheet.write_column('I2', i, bold2) worksheet.write_column('J2', j, bold2) worksheet.write_column('K2', k, bold2) worksheet.write_column('L2', l, bold2) worksheet.write_column('M2', m, bold2) workbook.close() if __name__ == '__main__': runtest()
主要思路是：第一步解析网站，第二步选择自己想要的数据，第三步在当前文件夹生成一个文件夹写入excl
第一次写博客，各路大神不喜勿喷，python萌新一枚。
开发环境：Pycharm python2.7
2019-04-0411:33:23
Study hard and make progress every day！