logstash grok nginx log

#cat logstash.conf

input {
        file {
                path => "/alidata/logs/nginx/appapi.dayutang.cn.access*.log"
                type => "nginx-access"
                start_position => "beginning"
                #sincedb_path => "/alidata/server/logstash/sincedb"
        }
}
filter {
        if [type] == "nginx-access" {
                grok {
                        patterns_dir => "/alidata/server/logstash/patterns"
                        match => {
                                "message" => "%{NGINXACCESS}"
                        }
                }
                date {
                        match => ["log_timestamp", "dd/MMM/yyyy:HH:mm:ss Z"]
                }
        }
}

output {
        if [type] == "nginx-access" {
                elasticsearch {
                        hosts => ["172.17.149.148:9200"]
                        manage_template => true
                        index => "logstash-nginx-access-%{+YYYY-MM}"
                }
        }

}

#cat /data/server/logstash/patterns/nginx URIPATH1 (?:/[A-Za-z0-9$.+!*'(){},~:;=@#%&_- ]*)+ URIPARM1 [A-Za-z0-9$.+!*'|(){},~@#%&/=:;_?-[]]* URI1 (%{URIPROTO}://)?(?:%{USER}(?::[^@]*)?@)?(?:%{URIHOST})?(?:%{URIPATHPARAM})? STATUS ([0-9.]{0,3}[, ]{0,2})+ HOSTPORT1 (%{IPV4}:%{POSINT}[, ]{0,2})+ FORWORD (?:%{IPV4}[,]?[ ]?)+|%{WORD} NGINXACCESS %{IPORHOST:remote_addr} - (%{USERNAME:user}|-) [%{HTTPDATE:log_timestamp}] %{HOSTNAME:http_host} %{WORD:request_method} "%{URIPATH1:uri}" "%{URIPARM1:param}" %{BASE10NUM:http_status} (?:%{BASE10NUM:body_bytes_sent}|-) "(?:%{URI1:http_referrer}|-)" (%{BASE10NUM:upstream_status}|-) (?:%{HOSTPORT1:upstream_addr}) (%{BASE16FLOAT:upstream_response_time}|-) (%{BASE16FLOAT:request_time}|-) (?:%{QUOTEDSTRING:user_agent}|-) "(%{WORD:x_forword_for}|-)"
#cat /usr/local/nginx/conf/nginx.conf


log_format  main  '$remote_addr - $remote_user [$time_local] $http_host $request_method "$uri" "$query_string" '
                  '$status $body_bytes_sent "$http_referer" $upstream_status $upstream_addr $request_time $upstream_response_time '
                  '"$http_user_agent" "$http_x_forwarded_for"' ;

  

原文地址:https://www.cnblogs.com/ipyanthony/p/11266340.html