注意：复制该脚本到linux系统中使用时，需要先安装dos2unix工具进行格式转换
yum install dos2unix -y
安装成功后，使用该命令转换windows的换行符
dos2unix a.sh
正在写入的文件不能用tar进行压缩
--------压缩日志----------------------
94 access.log
95 tar: access.log: file changed as we read it
96 #### 压缩日志失败 ####
#!/bin/sh
#分析nginx日志
DATE=`date '+%Y%m%d-%H%M'`
ARCHIVE=/usr/log_bak/nginx_$DATE.tar.gz
MESSAGE=/usr/log_bak/"Nginx_Analysis""$DATE"
FILENAME=/data/nginx/logs/access.log
BACKNAME=/usr/log_bak/"nginx_bak"
function Mail(){
mail -s "***Nginx Report***" 666@qq.com < $MESSAGE
}
function Bowser(){
key[0]='" 200 [0-9]{3}';word[0]='http 200'
key[1]='" 206 [0-9]{3}';word[1]='http 206'
key[2]='" 404 [0-9]{3}';word[2]='http 404'
key[3]='" 503 [0-9]{3}';word[3]='http 503'
##########
# seo/seo.html" target="_blank">
key[4]='Googlebot.*google.com/bot.html';word[4]='Google Browser'
key[5]='Baiduspider.*baidu.com/search/spider.html';word[5]='Baidu Browser'
key[6]='bingbot.*bing.com/bingbot.htm';word[6]='Bing Browser'
#Soso 'Sosospider.*soso.com/webspider.htm'
#ÓеÀ 'YoudaoBot.*youdao.com/help/webmaster/spider/'
#YahooÖйú 'Yahoo! Slurp China'
##########
#
key[7]='MSIE';word[7]='MSIE'
key[8]='Gecko/.*Firefox';word[8]='Firefox'
key[9]='AppleWebKit.*like Gecko';word[9]='Webkit'
key[10]='Opera.*Presto';word[10]='Opera'
key[11]='Windows NT 6.1';word[11]='Windows 7 访问'
key[12]='Macintosh; Intel Mac OS X';word[12]='Mac OS X 访问'
key[13]='X11.*Linux';word[13]='Linux with X11'
key[14]='Android;';word[14]='Android'
#WindowsϵÁÐ win2000'Windows NT 5.0' winxp'Windows NT 5.1' winvasta'Windows NT 6.0' win7'Windows NT 6.1
#SymbianOS 'SymbianOS'
##########
#
key[15]='iPad.*like Mac OS X';word[15]='iPad 访问'
key[16]='Nokia';word[16]='Nokia'
key[17]='Nokia5800';word[17]='Nokia5800 XpressMusic'
#iPhone 'iPhone.*like Mac OS X'
##########
#
key[18]='GET /.*.mp3 HTTP';word[18]="访问 mp3 file"
key[19]='GET /.*.jpg HTTP';word[19]="访问 jpg file"
#echo $filename
#echo "nginx日志: ${FILENAME},一共${totle}行,需要处理 ${#key[@]}条" >> $MESSAGE
#echo "来源IP$(cat $FILENAME | awk '{print $1}' |sort|uniq|wc -l)" >> $MESSAGE
i=4
echo "----浏览器来源----" >> $MESSAGE
echo "--浏览器-----总计------占比--" >> $MESSAGE
while [ $i -lt ${#key[@]} ]
do
s1=${word[$i]}
s2=$(cat $BACKNAME | grep ''"${key[$i]}"'' | wc -l)
s3=$(awk 'BEGIN{printf "%.2f%",('$s2'/'$totle')*100}')
echo "${s1} ${s2} ${s3}" >> $MESSAGE
((i++))
done
if [[ $? == 0 ]]; then
echo "分析浏览器标示成功" >> $MESSAGE
else
echo "分析浏览器标示失败" >> $MESSAGE
fi
echo "--------------------" >> $MESSAGE
}
Check_http_status()
{
#grep -ioE "HTTP/1.[1|0]"[[:blank:]][0-9]{3}" access.log
#拿到日志中所有的包含HTTP状态码的部分，拿出第二段来判断，并将结果分配到数组中
codes=(`grep -ioE "HTTP/1.[1|0]"[[:blank:]][0-9]{3}" $BACKNAME | awk -F"[ ]+" 'BEGIN{i=0;j=0;k=0;n=0;p=0;}{ if($2>=100&&$2<200)
{i++}
else if($2>=200&&$2<300)
{j++}
else if($2>=300&&$2<400)
{k++}
else if($2>=400&&$2<500)
{n++}
else if($2>=500)
{p++}
}END{
print i?i:0,j?j:0,k?k:0,n?n:0,p?p:0,i+j+k+n+p
}'`)
echo "--HTTP状态码---COUNT---PERCENT------" >> $MESSAGE
echo "status[100+]:--${codes[0]}--$(awk 'BEGIN{printf "%.2f%",('${codes[0]}'/'${codes[5]}')*100}')" >> $MESSAGE
echo "status[200+]:--${codes[1]}--$(awk 'BEGIN{printf "%.2f%",('${codes[1]}'/'${codes[5]}')*100}')" >> $MESSAGE
echo "status[300+]:--${codes[2]}--$(awk 'BEGIN{printf "%.2f%",('${codes[2]}'/'${codes[5]}')*100}')" >> $MESSAGE
echo "status[400+]:--${codes[3]}--$(awk 'BEGIN{printf "%.2f%",('${codes[3]}'/'${codes[5]}')*100}')" >> $MESSAGE
echo "status[500+]:--${codes[4]}--$(awk 'BEGIN{printf "%.2f%",('${codes[4]}'/'${codes[5]}')*100}')" >> $MESSAGE
echo "----所有的状态码: ${codes[5]}----" >> $MESSAGE
}
function IpUrlTime(){
echo "来源IP共--$(cat $BACKNAME | awk '{print $1}' |sort|uniq|wc -l)--个" >> $MESSAGE
ip=$(cat $BACKNAME | awk '{print $1}'|sort | uniq -c | sort -nr | head -n 20)
echo "----访问前20个IP统计----" >> $MESSAGE
echo "$ip" >> $MESSAGE
#通过日志查看当天访问页面的url:
url=$(cat $BACKNAME | awk '{print $7}'|sort | uniq -c | sort -nr |head -n 20)
echo "----访问前20个URL统计----" >> $MESSAGE
echo "$url" >> $MESSAGE
#通过日志查看当天访问次数最多的时间段
time=$(awk '{print $4}' $BACKNAME |cut -c 14-18 | sort | uniq -c | sort -nr | head | head -n 20)
echo "----访问前20个时间点统计----" >> $MESSAGE
echo "$time" >> $MESSAGE
}
#----------start---------------
ip=`ifconfig | grep 'inet addr:'|grep -v '127.0.0.1'|awk -F '[ :]+' '{print $4}'`
echo "--------Server $ip---------------" >> $MESSAGE
echo "--------$(df -h)---------------" >> $MESSAGE
cd /usr/log_bak
if [ $? == 0 ]
then
echo "进入目录/usr/log_bak" >> $MESSAGE
else
echo "####进入目录失败,退出####" >> $MESSAGE
exit 0
fi
echo "---------------------" >> $MESSAGE
echo "备份日志:" $(date +"%y-%m-%d %H:%M:%S") >> $MESSAGE
echo "---------------------" >> $MESSAGE
#bak access.log
cp $FILENAME $BACKNAME
#check bak
if [[ $? == 0 ]]
then
echo "日志复制成功" >> $MESSAGE
else
echo "####日志复制失败,退出####" >> $MESSAGE
exit 0
fi
echo "-------------------------------" >> $MESSAGE
echo "分析时间:" $(date +"%y-%m-%d %H:%M:%S") >> $MESSAGE
echo "-------------------------------" >> $MESSAGE
totle=$(cat $BACKNAME | wc -l)
size=$(ls -sh $BACKNAME | awk '{print $1}')
echo "nginx日志,${size},一共${totle}行 " >> $MESSAGE
IpUrlTime
Check_http_status
Bowser
echo "--------压缩日志----------------------" >> $MESSAGE
#直接备份复制的文件，否者access.log1正在写入，无法压缩
tar czvf $ARCHIVE nginx_bak >> $MESSAGE 2>&1
#判断catalina.out备份是否成功
if [[ $? == 0 ]]
then
#创建备份文件的压缩包
# tar czvf $ARCHIVE $LOG >> log.txt 2>&1
echo "[$ARCHIVE] 日志压缩成功!" >> $MESSAGE
# clear access.log
> $FILENAME
if [[ $? == 0 ]]
then
echo "清空日志清空日志成功" >> $MESSAGE
rm -f $BACKNAME
else
echo "###清空日志失败 Failed #####" >> $MESSAGE
fi
#只需保留备份文件的压缩包即可
else
echo "#### 压缩日志失败 ####" >> $MESSAGE
exit 0
fi
echo "---------------------" >> $MESSAGE
echo "结束时间:" $(date +"%y-%m-%d %H:%M:%S") >> $MESSAGE
echo "---------------------" >> $MESSAGE
Mail