python 抓取日志

# !/usr/bin/env python
# -*- coding: utf-8 -*-
import MySQLdb
from datetime import datetime
import  cx_Oracle
import os
import xlwt
import sys
import commands
import re
import time
reload(sys)
sys.setdefaultencoding('utf-8')
os.environ['NLS_LANG'] = 'SIMPLIFIED CHINESE_CHINA.UTF8'
if len(sys.argv) != 3:
    print sys.argv[0] + ' ' + 'in/out' + ' ' + 'buss_seq_num'
    exit()

def get_data(sql):
    # 创建数据库连接.
    conn = cx_Oracle.connect('xx/xx@1.1.0.141/esbdb')
    # 创建游标
    cur = conn.cursor()
    # 执行查询，
    cur.execute(sql)
    # 由于查询语句仅会返回受影响的记录条数并不会返回数据库中实际的值，所以此处需要fetchall()来获取所有内容。
    result = cur.fetchall()
    # 关闭游标
    cur.close()
    # 关闭数据库连接
    conn.close
    # 返给结果给函数调用者。
    #print result
    return result
esbtag = sys.argv[1]
esbserviceflowno = sys.argv[2]
try :
  os.unlink("%s.txt" % esbserviceflowno)
except:
 pass
vdate=esbserviceflowno[6:14]
#print vdate
mysql="select t.esbflowno from esb2_trans_log t 
 where/* t.trans_date >= 
       to_date('2018-08-28 00:00:00', 'yyyy-mm-dd hh24:mi:ss') 
   and t.trans_date <= 
       to_date('2018-08-28 23:59:59', 'yyyy-mm-dd hh24:mi:ss')*/ 
    t.esbserviceflowno='%s'" % (esbserviceflowno)
#print mysql
aa=get_data(mysql)
#print aa
#print  type(aa)
for x in aa:
  print x[0]
  aa=x[0][0:7]
  bb=aa+'|'
  #print aa
  #./ansible -i esbhosts  esbgroup -m shell -a 'grep 10106020180821010031479887 $HOME/esblog/in/`date +'%Y%m%d'`/common*'
  cmd="ansible -i /etc/ansible/hosts  %s -u esb -m shell -a 'echo -n '%s';grep -l  '%s'  /app/esb/esblog/%s/%s/[0-9]{1,}*.log*' >>%s.txt" %(aa,aa,x[0],esbtag,vdate,esbserviceflowno)
  print cmd
  response=commands.getoutput(cmd)
  print response
d={}
f = open("%s.txt" % esbserviceflowno)
for x in f:
  y= x.strip()
  if re.match("^esb",y):
   if not y in d:
     d[y]=1
   else:
     d[y]=d[y]+1
print d
filearray=[]
for key in d:
  print key
  host=key[0:7]
  print host
  path=key[7:] 
  print path 
  filename=key[35:]
  filename=host+filename
  print filename
  try :
   os.unlink("/esb/logdir/%s.txt" % filename)
  except:
   pass
  cmd="ansible -i /etc/ansible/hosts  %s -u esb -m shell -a ' cat %s' >>/esb/logdir/%s.txt" %(host,path,filename)
  print cmd
  response=commands.getoutput(cmd)
  print response
  try :
   os.unlink("/esb/logdir/%s.txt.gz" % filename)
  except:
   pass
  commands.getoutput("gzip /esb/logdir/%s.txt" % filename)
  filearray.append("%s.txt.gz" % filename)
print filearray