import os import argparse import yaml import smtplib import csv from email.mime.multipart import MIMEMultipart from email.mime.text import MIMEText from urllib import unquote import logging import logging.handlers logger = logging.getLogger() error_list = [] all_error_file_list=[] def extension(filename): """ get file extension :param filename: :return: """ ext = os.path.splitext(filename)[1] if ext.startswith('.'): # os.path.splitext retains . separator ext = ext[1:] return ext def list_file(file_name, list): """ get all failed file list ,contains sub folder :param file_name: file name or folder name :param list: :return: """ if os.path.isdir(file_name): for s in os.listdir(file_name): list_file(os.path.join(file_name, s), list) else: ext = extension(file_name) if ext == "failed": list.append(file_name) def read_email_record(faild_path): """ read last failed file record :param faild_path: :return: """ record = os.path.join(faild_path, "email_record.txt") if not os.path.isfile(record): return {} try: return yaml.load(open(record)) except Exception as err: return {} def rewrite_record(args): """ save record file :param args: :return: """ if not error_list: return record = os.path.join(args.failed_path, "email_record.txt") map = {} for s in all_error_file_list: map[s] = "emailed" f = open(record, "w") yaml.dump(map, f, default_flow_style=False) f.close() def process(args): """ process : * list all failed files * read record list * remove duplicate data :param args: :return: """ faild_path = args.failed_path if not os.path.isdir(faild_path): logger.error("failed path:%s is not exist, exist", faild_path) return list_file(faild_path, all_error_file_list) record_map = read_email_record(faild_path) if not record_map: error_list.extend(all_error_file_list) else: for file in all_error_file_list: if not record_map.has_key(file): error_list.append(file) def generate_body(s3path): """ generate email body :return: """ body = """<h4>The following billing csv files are format failed,please check</h4> """ logger.info("failed file list as below:") # list all failed list which need send email for s in error_list: # define line number from 0 line_num = 0 # to get error file name and path , just support {orgid}/{failed_file} format ps = os.path.split(s) # to regenerate new web path, such as s3://{bucket}/xxx/{orgid}/{fail_file} s3 = os.path.join(s3path, os.path.split(ps[0])[1], ps[1]) # generate html body body += "<hr>" # add file name with link body += """<div>File: <a href="%s">%s<a></div>""" % (s3, ps[1]) # add URL which can copy body += "<div>URL: %s</div>" % (s3) # add logger info logger.info("-"*50) logger.info("file:[%s]",s) # lines which read each time total_rows_each_read=10000000 # open failed file with open(s, 'r') as f: # read file in loop while 1: # get lines array list lines = f.readlines(total_rows_each_read) if not lines: break # check each line with csv , when failed , # generate new array from failed index and re-check until finished while len(lines)>0: # check csv format with strict = True reader=csv.reader(lines, strict=True) # define array index i = 0 try: # get rows of each line for row in reader: # when row is csv format , array index will ++ i += 1 line_num += 1 for item in row: # check each column whether with , if yes ,will add line number if str(item).find(' ')>0: i += 1 line_num += 1 # when all lines is correct , array will set to empty to break loop lines=[] # get Exceptions except Exception as err: # when error occur , line number will increase 1 line_num += 1 #line_num += reader.line_num line = lines[i] # generate each error line information body += "<div></div><br><br>" body += "<li><div>Error line number: [#%s] </div></li>" "<div>#Exceptions: [%s]</div>" % (line_num, err) body += """<div>#Error row data as below:</div> <div>%s</div>""" % line if (i+1) == len(lines): lines=[] else: tmp=lines[i+1:] lines = tmp body += "<hr>" return body def email_sender(args): """ send email ,iterator each receiver then send email :param args: :return: """ subject = "Error Pipeline CSV File List" #if not error_list: # return False #body = generate_body(args.s3url) body = "test" emails = args.email smtp_host = args.smtpHost smtp_port = args.smtpPort username = args.username password = args.password sent_from = args.sentFrom if emails: mail_server = smtplib.SMTP() try: msg = MIMEMultipart() msg['From'] = sent_from msg['Subject'] = subject msg.attach(MIMEText(body, 'html')) mail_server.connect(smtp_host, smtp_port) mail_server.ehlo() mail_server.starttls() mail_server.ehlo() if password and smtp_host != 'eng-smtp.calix.local': mail_server.login(username, unquote(password)) for recipient in emails: logger.info("send email to %s", recipient) msg['To'] = recipient mail_server.sendmail(sent_from, recipient, msg.as_string()) except Exception as err: logger.error("send email failed:%s", err) return False finally: if mail_server: mail_server.close() return True return False def init_log(log_file): """ init logger :param log_file: :return: """ log_path = os.path.split(log_file)[0] if not os.path.isdir(log_path): os.makedirs(log_path) logger.setLevel(logging.DEBUG) handler = logging.handlers.RotatingFileHandler(log_file, maxBytes=20 * 1024 * 1024, backupCount=5, ) fmt = '%(asctime)s-[%(filename)s:%(lineno)s]-[%(levelname)s]- %(message)s' formatter = logging.Formatter(fmt) handler.setFormatter(formatter) logger.addHandler(handler) def main(): parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) parser.add_argument('--failed_path', default='/var/tmp/billing/', help='store failed files') parser.add_argument('-e', '--email', action='append', help="To send multiple emails --email <email-address-1> --email <email-address-2> ...") parser.add_argument('--smtpHost', type=str, help="Host of SMTP server", required=False, default="outlook.office365.com") parser.add_argument('--smtpPort', type=int, help="Port of SMTP server", required=False, default=587) parser.add_argument('--username', type=str, help="outlook username", required=False) parser.add_argument('--password', type=str, help="outlook password", required=False) parser.add_argument('--sentFrom', type=str, help="outlook email", required=False, default="noreply-compass-fa@calix.com") parser.add_argument('--logfile', help='logger file name', required=False, default='/var/log/sxadp-api-server/scan_failed_email.log') parser.add_argument('--s3url',type=str,help="billing s3 URL ",required=False) args = parser.parse_args() init_log(args.logfile) #process(args) if email_sender(args): print "done" #rewrite_record(args) if __name__ == '__main__': main()