[elk@zjtest7-frontend config]$ cat logstash_agent.conf
input {
file {
type => "zj_nginx_access"
path => ["/rsyslog/data/nginx/zjzc/nginx_access0*_log.*"]
ignore_older => 87400
}
file {
type => "uat_nginx_access"
path => ["/rsyslog/data/nginx/uat/nginx_access0*_log.*"]
ignore_older => 87400
}
}
filter {
grok {
match => {
"message" => "%{IPORHOST:clientip} [%{HTTPDATE:time}] "%{WORD:verb} %{URIPATHPARAM:request} HTTP/%{NUMBER:httpversion}" %{NUMBER:http_status_code} %{NUMBER:bytes} "(?<http_referer>S+)" "(?<http_user_agent>S+)" "(?<http_x_forwarded_for>S+)""
}
}
}
output {
if [type] == "zj_nginx_access" {
redis {
host => "192.168.32.67"
data_type => "list"
key => "zj_nginx:redis"
port=>"6379"
password => "1234567"
}
}
else if [type] == "uat_nginx_access"{
redis {
host => "192.168.32.67"
data_type => "list"
key => "uat_nginx:redis"
port=>"6379"
password => "1234567"
}
}
}
indexer.conf:
input {
redis {
host => "192.168.32.67"
data_type => "list"
key => "zj_nginx:redis"
password => "1234567"
port =>"6379"
}
redis {
host => "192.168.32.67"
data_type => "list"
key => "uat_nginx:redis"
password => "1234567"
port =>"6379"
}
}
output {
if [type] == "zj_nginx_access"{
elasticsearch {
hosts => "192.168.32.80:9200"
index => "logstash-zjzc-nginx-%{+YYYY.MM.dd}"
}
stdout {
codec => rubydebug
}
}
else if [type] == "uat_nginx_access"{
elasticsearch {
hosts => "192.168.32.81:9200"
index => "logstash-uat-nginx-%{+YYYY.MM.dd}"
}
stdout {
codec => rubydebug
}
}
}
redis消息里有type字段；
127.0.0.1:6379> LPOP "zj_nginx:redis"
"{"message":" 120.26.44.206:8001 120.26.44.206 120.26.44.206 [22/Aug/2016:22:12:58 +0800] \"GET / HTTP/1.1\" - 200 30626 \"-\" \"curl/7.19.7 (x86_64-redhat-linux-gnu) libcurl/7.19.7 NSS/3.15.3 zlib/1.2.3 libidn/1.18 libssh2/1.4.2\" 0.000 -","@version":"1","@timestamp":"2016-08-22T14:10:55.846Z","path":"/rsyslog/data/nginx/zjzc/nginx_access01_log.2016-08-22","host":"0.0.0.0",
"type":"zj_nginx_access","tags":["_grokparsefailure"]}"
{
"message" => " 120.26.44.206:8001 120.26.44.206 120.26.44.206 [22/Aug/2016:22:18:58 +0800] "GET / HTTP/1.1" - 200 30626 "-" "curl/7.19.7 (x86_64-redhat-linux-gnu) libcurl/7.19.7 NSS/3.15.3 zlib/1.2.3 libidn/1.18 libssh2/1.4.2" 0.000 -",
"@version" => "1",
"@timestamp" => "2016-08-22T14:16:55.738Z",
"path" => "/rsyslog/data/nginx/zjzc/nginx_access01_log.2016-08-22",
"host" => "0.0.0.0",
"type" => "zj_nginx_access",
"tags" => [
[0] "_grokparsefailure"
]
}
{
"message" => " 121.40.189.90:8001 121.40.189.90 120.26.44.206 [22/Aug/2016:22:14:13 +0800] "GET / HTTP/1.1" - 200 30338 "-" "curl/7.19.7 (x86_64-redhat-linux-gnu) libcurl/7.19.7 NSS/3.15.3 zlib/1.2.3 libidn/1.18 libssh2/1.4.2" 0.001 -",
"@version" => "1",
"@timestamp" => "2016-08-22T14:17:04.110Z",
"path" => "/rsyslog/data/nginx/uat/nginx_access01_log.2016-08-22",
"host" => "0.0.0.0",
"type" => "uat_nginx_access",
"tags" => [
[0] "_grokparsefailure"
]
}