python获取数据网页数据并创建文件夹保存(基于python3.6)

from urllib.parse import urljoin
import urllib.request

from bs4 import BeautifulSoup


import os
import datetime
import re
import errno


def mkdir_p(path): #递归创建多级目录
try:
os.makedirs(path)
except OSError as exc: # Python >2.5 (except OSError, exc: for Python <2.5)
if exc.errno == errno.EEXIST and os.path.isdir(path):
pass
else: raise

def get_link(page): # 寻找链接的href
linkData = []
for page in page.find_all('td'):
links = page.select("a")
for each in links:
# if str(each.get('href'))[:1] == '/': 过滤if代码
data=each.get('href')
linkData.append(data)
return(linkData)


def gain(url): #获取网页指定内容
page = urllib.request.urlopen(url).read()
soup = BeautifulSoup(page, 'lxml') #利用soup获取网页内容
links = get_link(soup) #获取<a href= ? 内容
return links


def main():
url = 'https://www.tide-forecast.com/countries/China'
Web_Link=gain(url)

for Link in range(len(Web_Link)):
Link_Add = Web_Link[Link]
Link_One = re.split("/", Link_Add) #去除'/',将Link_Add变成数组
Link_Address = Link_One[2] #获取数组第3位值
Link_Address = (Link_Address + '.js')
url_Tide = 'https://www.tide-forecast.com/tides/'
connet = urljoin(url_Tide, Link_Address) # 拼接网址路径
file = os.path.join('D:\TideData\China' + "/" ) # 拼接绝对路径
mkdir_p(file)
print(connet)
if os.path.isfile(file):
print('文件已存在')
else:
start = datetime.datetime.now().replace(microsecond=0) #计时工具

url = connet
wp = urllib.request.urlopen(url) #打开数据网页数据
content = wp.read()
fp = open(file + Link_Address, "wb") #写入指定文件夹
fp.write(content) #写入数据
fp.close() #关闭文件

end = datetime.datetime.now().replace(microsecond=0)
print("用时: ", end='')
print(end - start)





if __name__ == '__main__':
main()
原文地址:https://www.cnblogs.com/setname/p/7453778.html