python统计日志小脚本

日志格式如下：

[ 2016-06-28T00:10:33-03:00 ] xxx.xx.xx.xxx /api/index/xxx/
ERR: code:400
message:
params:
       country:us
       token:uq6euz9dou6aqtk1

Python（3）脚本如下：

import urllib.request
import ntpath
import os, sys
import time


def dirList(path):
	filelist = os.listdir(path)
	fpath = os.getcwd()
	allfile = []
	for filename in filelist:
		filepath = os.path.abspath(os.path.join(path, filename))
		if os.path.isdir(filepath):
			allfile.extend(dirList(filepath))
		else:
			if filepath.endswith(".log"):
				allfile.append(filepath)
	return allfile


def readlog(log):
	ret = []
	loginfo = {}
	f = open(log)
	line = f.readline()
	while line != "":
		if line.startswith("[ 201"):
			if loginfo.get("time", "") != "":
				if loginfo.get("params", "") != "":
					loginfo["params"] = loginfo.get("params").rstrip(' ,')
				if loginfo.get("code", "") != "":
					ret.append(loginfo)
				loginfo = {}
			loginfo["time"] = line[2:21]
			loginfo["ip"] = line[30:45].strip()
			loginfo["url"] = line[45:].rstrip('
')
		elif line.startswith("ERR: code"):
			loginfo["code"] = line[10:].rstrip('
')
		elif line.startswith("message"):
			loginfo["message"] = line[8:].rstrip('
')
		elif line.startswith("params"):
			loginfo["params"] = ""
		elif line.startswith("ERR: fcmResult"):
			loginfo["fcm"] = ""
		elif line.startswith("ERR: Illegal"):
			loginfo["code"] = '800'
			loginfo["message"] = line[20:].rstrip('
')
		elif line.startswith("ERR: "):
			loginfo["code"] = '900'
			loginfo["message"] = line[4:].rstrip('
')
		elif line.startswith(" [ SQL"):
			loginfo["sql"] = line.rstrip('
')
		else:
			#print(line)
			params = line.strip().rstrip('
')
			#print(log, params)
			if params != "" and loginfo.get("params", "-1") != "-1":
				loginfo["params"] += params + ", "
		line = f.readline()
	return ret


def ana_log(logdir):
	logs = dirList(logdir)
	logInfos = {}
	for log in logs:
		logInfos[log]=readlog(log)
	return logInfos


def logSummary():
	logdir = "/var/www/Runtime/Logs/Api"
	logInfos = ana_log(logdir)
	logCount = {}
	for day in logInfos.keys():
		#print(day)
		logDay = {}
		for logitem in logInfos.get(day):
			#print(logitem)
			#print("-------------------")
			#break
			if logDay.get(logitem["code"], -1) == -1:
				logDay[logitem["code"]] = 1
			else:
				logDay[logitem["code"]] += 1
			#print(logDay)
		logCount[day] = logDay
	return logCount


logInfo = logSummary()
for l in logInfo.keys():
	print(l)
	oneLog = logInfo.get(l)
	oneLog = sorted(oneLog.items(), key=lambda d:d[1], reverse = True)
	for k in oneLog:
		print(str(k[0]) + ": " + str(k[1]))

输出结果如下：

/var/www/Runtime/Logs/Api/16_06_25.log
400: 22
110: 12
101: 10
119: 10
404: 2
600: 1
/var/www/Runtime/Logs/Api/16_06_26.log
110: 5
119: 4
600: 2
400: 1
/var/www/Runtime/Logs/Api/16_06_27.log
110: 42
400: 32
600: 14
119: 8
404: 1
/var/www/Runtime/Logs/Api/16_06_28.log
400: 5
110: 2
404: 2
119: 1