关键知识:
paramiko 模块, stat模块,os模块
threading 模块 --> 实现同时与多个远程上传和下载单个文件或者整个目录
paramiko 模块 --> 实现链接远程,传输数据
stat中的 stat.S_ISDIR(remote_path_mode) --> 判断远程是目录还是文件
os中的 os.walk(path) --> 提取本地目录中的所有文件和子目录
定义一个基础类Connection,继承 threading.Thread
1 import paramiko, datetime, os, threading, re, stat 2 3 4 class Connection(threading.Thread): 5 6 def __init__(self, hostname=None, port=None, password=None, username=None): 7 super(Connection, self).__init__() 8 self.hostname = hostname 9 self.port = port 10 self.password = password 11 self.username = username 12 self.str_host = '%s:%s' % (self.hostname, self.port) 13 14 def run(self): 15 pass
定义一个cmd执行类CMDThread,继承Connection. CMDThread类实例化后会处理cmd相关命令
1 class CMDThread(Connection): 2 3 def __init__(self, hostname=None, port=None, password=None, username=None, echo_cmd=None): 4 super(CMDThread, self).__init__(hostname, port, password, username,) 5 self.echo_cmd = echo_cmd 6 7 def run(self): 8 paramiko.util.log_to_file('MyFabric.log') 9 ssh = paramiko.SSHClient() 10 ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) 11 try: 12 ssh.connect(hostname=self.hostname, port=self.port, username=self.username, password=self.password) 13 stdin, stdout, stderr = ssh.exec_command(self.echo_cmd) 14 res, err = stdout.read(), stderr.read() 15 result = res if res else err 16 print(self.str_host.center(60, '-')) 17 print(result.decode()) 18 ssh.close() 19 except Exception as e: 20 print(e)
定义一个上传文件的类PutThread,继承Connection,PutThread实例化, 先判断本地要上传的是单个文件还是整个目录,然后链接远程上传至远程指定位置.
1 class PutThread(Connection): 2 3 def __init__(self, hostname=None, password=None, username=None, port=None, local_path=None, remote_path=None): 4 super(PutThread, self).__init__(hostname, password, username, port) 5 self.local_path = local_path 6 self.remote_path = remote_path 7 8 def run(self): 9 paramiko.util.log_to_file('MyFabric.log') 10 t = paramiko.Transport((self.hostname, self.port)) 11 t.connect(username=self.username, password=self.password) 12 sftp = paramiko.SFTPClient.from_transport(t) 13 if os.path.isfile(self.local_path): 14 filename = os.path.split(self.local_path)[-1] 15 remote_file = os.path.join(self.remote_path, filename).replace('\', '/') 16 try: 17 sftp.put(self.local_path, remote_file) 18 print('[*] Put dir %s success %s --%s' % (filename, datetime.datetime.now(), self.str_host)) 19 t.close() 20 except Exception as e: 21 print('[*] Put file %s failed %s Err:%s --%s' % (filename, datetime.datetime.now(), e, self.str_host)) 22 elif os.path.isdir(self.local_path): 23 local_dir_root = os.path.split(self.local_path)[0].replace('\', '/') 24 dir_name = os.path.split(self.local_path)[-1] 25 remote_dir_root = self.remote_path 26 remote_files_path = os.path.join(remote_dir_root, dir_name).replace('\', '/') 27 try: 28 try: 29 sftp.stat(remote_files_path) 30 except IOError: 31 print('mkdir remote dir [%s]' % remote_files_path) 32 sftp.mkdir(remote_files_path) 33 for root, dirs, files in os.walk(self.local_path): 34 if local_dir_root: # 这里判断输入的self.local_path是不是abspath, 然后做相应的路径字符串拼接 35 remote_root = re.sub(local_dir_root, remote_dir_root, root.replace('\', '/')) 36 else: 37 remote_root = os.path.join(remote_dir_root, root).replace('\', '/') 38 if dirs: 39 for name in dirs: 40 mk_remote_path = os.path.join(remote_root, name).replace('\', '/') 41 sftp.mkdir(mk_remote_path) 42 if files: 43 for filename in files: 44 local_file = os.path.join(root, filename) 45 remote_file = os.path.join(remote_root, filename).replace('\', '/') 46 sftp.put(local_file, remote_file) 47 print('[*] Put dir %s success %s --%s' % (dir_name, datetime.datetime.now(), self.str_host)) 48 t.close() 49 except Exception as e: 50 print('[*] Put dir %s failed %s Err:%s --%s' % (dir_name, datetime.datetime.now(), e, self.str_host))
定义一个下载文件的类GetThread, 继承Connection.GetThread实例化后,会先链接远程,判断远程下载的是单个文件还是整个目录, 然后在本地指定位置建立一个以远程hostname命名的文件夹,来存放从该远程下载的文件或整个目录
1 class GetThread(Connection): 2 """ 3 'local_path' must be abspath 4 """ 5 def __init__(self, hostname=None, password=None, username=None, port=None, local_path=None, remote_path=None): 6 super(GetThread, self).__init__(hostname, password, username, port) 7 self.local_path = local_path 8 self.remote_path = remote_path 9 10 def run(self): 11 paramiko.util.log_to_file('MyFabric.log') 12 t = paramiko.Transport((self.hostname, self.port)) 13 t.connect(username=self.username, password=self.password) 14 sftp = paramiko.SFTPClient.from_transport(t) 15 remote_path_mode = sftp.stat(self.remote_path).st_mode # 获取远程文件的类型st_mode 16 if stat.S_ISDIR(remote_path_mode): # 使用stat模块的S_ISDIR方法判断远程文件的st_mode是否是目录类型 17 all_files = get_all_files(sftp, self.remote_path) # get_all_files函数用来从远程目录提取该目录下所有文件的路径 18 try: 19 for remote_file in all_files: 20 local_root_path = self.local_path + '/' + self.hostname # 这里做一个本地路径的字符串拼接,把远程hostname拼进去,为后面建立相应的文件夹做准备 21 remote_root_path = os.path.split(self.remote_path)[0] 22 local_file = re.sub(remote_root_path, local_root_path, remote_file) # 拼接好文件的完整路径 23 root = os.path.split(local_file)[0] # 这个是该文件的根目录 24 check_dirs(root) # check_dirs函数用来检查该文件的根目录root是否在本地存在,如果不存在就建立该目录 25 sftp.get(remote_file, local_file) 26 print('[*] Get dir %s success %s --%s' % (self.remote_path, 27 datetime.datetime.now(), 28 self.str_host)) 29 except Exception as e: 30 print('[*] Get dir %s failed %s Err:%s --%s' % (self.remote_path, 31 datetime.datetime.now(), 32 e, 33 self.str_host)) 34 else: 35 filename = os.path.split(self.remote_path)[-1] 36 local_root_path = self.local_path + '/' + self.hostname 37 local_file = local_root_path + '/' + filename 38 check_dirs(local_root_path) # 同上检查目录是否存在,不存在就建立目录 39 try: 40 sftp.get(self.remote_path, local_file) 41 print('[*] Get file %s success %s --%s' % (filename, datetime.datetime.now(), self.str_host)) 42 except Exception as e: 43 print('[*] Get file %s failed %s Err:%s --%s' % (filename, datetime.datetime.now(), e, self.str_host)) 44 t.close()
其中get_all_files用来从远程目录提取该目录下所有文件的路径.
1 def get_all_files(sftp, remote_dir): 2 """ 3 get all abspath of files in remote dir 4 :param sftp: 5 :param remote_dir: 6 :return: 7 """ 8 all_files = [] 9 if remote_dir[-1] == '/': 10 remote_dir = remote_dir[0:-1] 11 files = sftp.listdir_attr(remote_dir) # 获取目录下的所有文件和子目录stat信息 12 for i in files: 13 file_path = remote_dir + '/' + i.filename # 拼接完整的文件路径 14 if stat.S_ISDIR(i.st_mode): # 用stat模块的S_IDDIR方法判断文件是否是目录 15 all_files.extend(get_all_files(sftp, file_path)) # 如果是子目录就递归处理此目录 16 else: 17 all_files.append(file_path) # 如果不是子目录,就把文件路径加到all_files这个列表中 18 return all_files
注意: 其中列表操作extend和append的区别,extend()的参数只能是列表,append()的参数什么都可以,具体可自行百度.
其中check_dirs 函数是在本地windows上检查目录是否存在,不存在就创建目录
1 def check_dirs(path): 2 """ 3 Check the root of 'path' and create the root without existing 4 :param path: 5 :return: 6 """ 7 dirs = get_dirs_path(path)[::-1] # get_dirs_path函数用来提取一个目录的所有层级根目录路径包括自己本身的路径 8 for i in dirs: 9 if not os.path.exists(i): 10 os.mkdir(i)
get_dirs_path(path)获取到的是类似这样的列表['c:/Desktop/test' , 'c:/Desktop', 'c:'], [::-1]这个是把列表顺序反转, --->['c:', 'c:/Desktop', 'c:/Desktop/test']
其中 get_dirs_path函数用来提取一个目录的所有层级根目录路径包括自己本身的路径, 类似上面的列表
1 def get_dirs_path(path): # 这里的path必须是abspath 2 """ 3 gets the root directory at all levels of path(including 'path') 4 example: 'c:/Desktop/test/' --> ['c:/Desktop/test' , 'c:/Desktop', 'c:'] 5 'path' must be abspath on windows 6 """ 7 dirs = [] 8 if path[-1] == '/': 9 path = path[0:-1] 10 dirs.append(path) 11 lis = os.path.split(path) # 把根目录路径和目录名字分开,然后一层一层扒开处理直到最后出现类似这样的列表['c:', '']这个就是最终扒开的不能再扒了,再扒就进入死循环了,所以下面要在出现这个情况的时候终止递归 12 path = lis[0] 13 if lis[-1]: 14 dirs.extend(get_dirs_path(path)) # 这里也是用递归方法处理一个路径 15 return dirs
下面就是主逻辑了
1 def cmd_handler(): 2 hosts = get_hosts() 3 echo_cmd = input('Enter echo cmd:').strip() 4 obj = [] 5 for host in hosts: 6 t = CMDThread(host['hostname'], host['port'], host['password'], host['username'], echo_cmd) 7 t.start() 8 obj.append(t) 9 for i in obj: 10 i.join() 11 print('end'.center(60, '-')) 12 13 14 def put_handler(): 15 transport(tag='put') 16 17 18 def get_handler(): 19 transport(tag='get') 20 21 22 def transport(tag=None): 23 hosts = get_hosts() 24 while True: 25 remote_path = input('remote path >>').strip().replace('\', '/') 26 local_path = input('local path >>').strip().replace('\', '/') 27 if local_path and remote_path: 28 break 29 if not local_path or not remote_path: 30 continue 31 obj = [] 32 for host in hosts: 33 if tag == 'get': 34 t = GetThread(host['hostname'], host['port'], host['password'], host['username'], local_path, remote_path) 35 elif tag == 'put': 36 t = PutThread(host['hostname'], host['port'], host['password'], host['username'], local_path, remote_path) 37 t.start() 38 obj.append(t) 39 for i in obj: 40 i.join() 41 print('end'.center(60, '-')) 42 43 44 def get_hosts(): 45 hosts = [] 46 with open('hosts_setting', 'r') as f: 47 dic = {} 48 for line in f: 49 date = line.strip().split() 50 dic['hostname'] = date[0].split(':')[0] 51 dic['port'] = int(date[0].split(':')[-1]) 52 dic['username'] = date[1] 53 dic['password'] = date[-1] 54 hosts.append(dic) 55 dic = {} 56 return hosts 57 58 if __name__ == '__main__': 59 while True: 60 print(''' 61 menu: 62 1.cmd 63 2.put 64 3.get 65 4.quit 66 ''') 67 menu = { 68 '1': cmd_handler, 69 '2': put_handler, 70 '3': get_handler, 71 '4': exit 72 } 73 choice = input('Choice handler:').strip() 74 if choice in menu: 75 menu[choice]()
有个小问题就是, 如果遇到目录里面有空的子目录, 上传可以把空子目录结构一起复制到远程, 但下载不会把空的子目录结构也复制到本地, 有需要的亲,自己可以动手写,这里就不多说了.