字段引用:
10.168.255.134 [09/Oct/2016:15:28:52 +0800] "GET / HTTP/1.1" - 200 23388 "" "Mozilla/5.0 (Linux; U; Android 4.4.4; zh-cn; MX4 Pro Build/KTU84P) AppleWebKit/534.30 (KHTML, like Gecko) Version/4.0 Mobile Safari/534.30" 0.001 101.226.125.103
[elk@Vsftp logstash]$ cat logstash.conf
input {
stdin{}
}
filter {
grok {
match =>[
"message","%{IPORHOST:clientip} [%{HTTPDATE:time}] "%{WORD:verb} %{URIPATHPARAM:request}?.* HTTP/%{NUMBER:httpversion}" - %{NUMBER:http_status_code} %{NUMBER:bytes} "(?<http_referer>S+)" "(?<http_user_agent>(S+s+)*S+)" (%{BASE16FLOAT:request_time}) (%{IPORHOST:http_x_forwarded_for}|-)",
"message" , "%{IPORHOST:clientip} [%{HTTPDATE:time}] "%{WORD:verb} %{URIPATHPARAM:request} HTTP/%{NUMBER:httpversion}" - %{NUMBER:http_status_code} %{NUMBER:bytes} "(?<http_referer>S+)" "(?<http_user_agent>(S+s+)*S+)" (%{BASE16FLOAT:request_time}) (%{IPORHOST:http_x_forwarded_for}|-)",
"message","%{IPORHOST:clientip} [%{HTTPDATE:time}] "%{WORD:verb} (?<http_url>S+)s+HTTP/%{NUMBER:httpversion}"s+-s+%{NUMBER:http_status_code}s+%{NUMBER:bytes}s+"-"s+"(?<http_user_agent>(S+))"s+(%{BASE16FLOAT:request_time})s+(%{IPORHOST:http_x_forwarded_for}|-)",
"message","%{IPORHOST:clientip} [%{HTTPDATE:time}] "%{WORD:verb} %{URIPATHPARAM:request} HTTP/%{NUMBER:httpversion}" - %{NUMBER:http_status_code} %{NUMBER:bytes} "" "(?<http_user_agent>(S+s+)*S+)" (%{BASE16FLOAT:request_time}) (%{IPORHOST:http_x_forwarded_for}|-)"
]
}
}
output {
stdout {
codec => rubydebug
}
}
[elk@Vsftp logstash]$ logstash -f logstash.conf
Settings: Default pipeline workers: 4
Pipeline main started
10.168.255.134 [09/Oct/2016:15:28:52 +0800] "GET / HTTP/1.1" - 200 23388 "" "Mozilla/5.0 (Linux; U; Android 4.4.4; zh-cn; MX4 Pro Build/KTU84P) AppleWebKit/534.30 (KHTML, like Gecko) Version/4.0 Mobile Safari/534.30" 0.001 101.226.125.103
{
"message" => "10.168.255.134 [09/Oct/2016:15:28:52 +0800] "GET / HTTP/1.1" - 200 23388 "" "Mozilla/5.0 (Linux; U; Android 4.4.4; zh-cn; MX4 Pro Build/KTU84P) AppleWebKit/534.30 (KHTML, like Gecko) Version/4.0 Mobile Safari/534.30" 0.001 101.226.125.103",
"@version" => "1",
"@timestamp" => "2017-02-08T01:39:50.650Z",
"host" => "Vsftp",
"clientip" => "10.168.255.134",
"time" => "09/Oct/2016:15:28:52 +0800",
"verb" => "GET",
"request" => "/",
"httpversion" => "1.1",
"http_status_code" => "200",
"bytes" => "23388",
"http_user_agent" => "Mozilla/5.0 (Linux; U; Android 4.4.4; zh-cn; MX4 Pro Build/KTU84P) AppleWebKit/534.30 (KHTML, like Gecko) Version/4.0 Mobile Safari/534.30",
"request_time" => "0.001",
"http_x_forwarded_for" => "101.226.125.103"
}
[elk@Vsftp logstash]$ cat logstash.conf
input {
stdin{}
}
filter {
grok {
match =>[
"message","%{IPORHOST:clientip} [%{HTTPDATE:time}] "%{WORD:verb} %{URIPATHPARAM:request}?.* HTTP/%{NUMBER:httpversion}" - %{NUMBER:http_status_code} %{NUMBER:bytes} "(?<http_referer>S+)" "(?<http_user_agent>(S+s+)*S+)" (%{BASE16FLOAT:request_time}) (%{IPORHOST:http_x_forwarded_for}|-)",
"message" , "%{IPORHOST:clientip} [%{HTTPDATE:time}] "%{WORD:verb} %{URIPATHPARAM:request} HTTP/%{NUMBER:httpversion}" - %{NUMBER:http_status_code} %{NUMBER:bytes} "(?<http_referer>S+)" "(?<http_user_agent>(S+s+)*S+)" (%{BASE16FLOAT:request_time}) (%{IPORHOST:http_x_forwarded_for}|-)",
"message","%{IPORHOST:clientip} [%{HTTPDATE:time}] "%{WORD:verb} (?<http_url>S+)s+HTTP/%{NUMBER:httpversion}"s+-s+%{NUMBER:http_status_code}s+%{NUMBER:bytes}s+"-"s+"(?<http_user_agent>(S+))"s+(%{BASE16FLOAT:request_time})s+(%{IPORHOST:http_x_forwarded_for}|-)",
"message","%{IPORHOST:clientip} [%{HTTPDATE:time}] "%{WORD:verb} %{URIPATHPARAM:request} HTTP/%{NUMBER:httpversion}" - %{NUMBER:http_status_code} %{NUMBER:bytes} "" "(?<http_user_agent>(S+s+)*S+)" (%{BASE16FLOAT:request_time}) (%{IPORHOST:http_x_forwarded_for}|-)"
]
}
geoip {
source => "http_x_forwarded_for"
target => "geoip"
database => "/usr/local/logstash-2.3.4/etc/GeoLiteCity.dat"
add_field => [ "[geoip][coordinates]", "%{[geoip][longitude]}" ]
add_field => [ "[geoip][coordinates]", "%{[geoip][latitude]}" ]
}
}
output {
stdout {
codec => rubydebug
}
}
[elk@Vsftp logstash]$ logstash -f logstash.conf
Settings: Default pipeline workers: 4
Pipeline main started
10.168.255.134 [09/Oct/2016:15:28:52 +0800] "GET / HTTP/1.1" - 200 23388 "" "Mozilla/5.0 (Linux; U; Android 4.4.4; zh-cn; MX4 Pro Build/KTU84P) AppleWebKit/534.30 (KHTML, like Gecko) Version/4.0 Mobile Safari/534.30" 0.001 101.226.125.103
{
"message" => "10.168.255.134 [09/Oct/2016:15:28:52 +0800] "GET / HTTP/1.1" - 200 23388 "" "Mozilla/5.0 (Linux; U; Android 4.4.4; zh-cn; MX4 Pro Build/KTU84P) AppleWebKit/534.30 (KHTML, like Gecko) Version/4.0 Mobile Safari/534.30" 0.001 101.226.125.103",
"@version" => "1",
"@timestamp" => "2017-02-08T01:42:33.645Z",
"host" => "Vsftp",
"clientip" => "10.168.255.134",
"time" => "09/Oct/2016:15:28:52 +0800",
"verb" => "GET",
"request" => "/",
"httpversion" => "1.1",
"http_status_code" => "200",
"bytes" => "23388",
"http_user_agent" => "Mozilla/5.0 (Linux; U; Android 4.4.4; zh-cn; MX4 Pro Build/KTU84P) AppleWebKit/534.30 (KHTML, like Gecko) Version/4.0 Mobile Safari/534.30",
"request_time" => "0.001",
"http_x_forwarded_for" => "101.226.125.103",
"geoip" => {
"ip" => "101.226.125.103",
"country_code2" => "CN",
"country_code3" => "CHN",
"country_name" => "China",
"continent_code" => "AS",
"region_name" => "23",
"city_name" => "Shanghai",
"latitude" => 31.045600000000007,
"longitude" => 121.3997,
"timezone" => "Asia/Shanghai",
"real_region_name" => "Shanghai",
"location" => [
[0] 121.3997,
[1] 31.045600000000007
],
"coordinates" => [
[0] 121.3997,
[1] 31.045600000000007
]
}
}
字段引用
字段引用是Logstash::Event 对象的属性,我们之前提过事件就像一个哈希一样,所以你可以想象字段就像一个键值对
如果你想在Logstash 配置中使用字段的值,只需把字段的名字写在中括号[]里就行了,这就叫字段引用
[elk@Vsftp logstash]$ cat logstash.conf
input {
stdin{}
}
filter {
grok {
match =>[
"message","%{IPORHOST:clientip} [%{HTTPDATE:time}] "%{WORD:verb} %{URIPATHPARAM:request}?.* HTTP/%{NUMBER:httpversion}" - %{NUMBER:http_status_code} %{NUMBER:bytes} "(?<http_referer>S+)" "(?<http_user_agent>(S+s+)*S+)" (%{BASE16FLOAT:request_time}) (%{IPORHOST:http_x_forwarded_for}|-)",
"message" , "%{IPORHOST:clientip} [%{HTTPDATE:time}] "%{WORD:verb} %{URIPATHPARAM:request} HTTP/%{NUMBER:httpversion}" - %{NUMBER:http_status_code} %{NUMBER:bytes} "(?<http_referer>S+)" "(?<http_user_agent>(S+s+)*S+)" (%{BASE16FLOAT:request_time}) (%{IPORHOST:http_x_forwarded_for}|-)",
"message","%{IPORHOST:clientip} [%{HTTPDATE:time}] "%{WORD:verb} (?<http_url>S+)s+HTTP/%{NUMBER:httpversion}"s+-s+%{NUMBER:http_status_code}s+%{NUMBER:bytes}s+"-"s+"(?<http_user_agent>(S+))"s+(%{BASE16FLOAT:request_time})s+(%{IPORHOST:http_x_forwarded_for}|-)",
"message","%{IPORHOST:clientip} [%{HTTPDATE:time}] "%{WORD:verb} %{URIPATHPARAM:request} HTTP/%{NUMBER:httpversion}" - %{NUMBER:http_status_code} %{NUMBER:bytes} "" "(?<http_user_agent>(S+s+)*S+)" (%{BASE16FLOAT:request_time}) (%{IPORHOST:http_x_forwarded_for}|-)"
]
}
geoip {
source => "http_x_forwarded_for"
target => "geoip"
database => "/usr/local/logstash-2.3.4/etc/GeoLiteCity.dat"
add_field => [ "aaaaaa", "%{[geoip][location][0]}" ]
add_field => [ "bbbbbb", "%{[geoip][location][1]}" ]
}
}
output {
stdout {
codec => rubydebug
}
}
[elk@Vsftp logstash]$ logstash -f logstash.conf
Settings: Default pipeline workers: 4
Pipeline main started
10.168.255.134 [09/Oct/2016:15:28:52 +0800] "GET / HTTP/1.1" - 200 23388 "" "Mozilla/5.0 (Linux; U; Android 4.4.4; zh-cn; MX4 Pro Build/KTU84P) AppleWebKit/534.30 (KHTML, like Gecko) Version/4.0 Mobile Safari/534.30" 0.001 101.226.125.103
{
"message" => "10.168.255.134 [09/Oct/2016:15:28:52 +0800] "GET / HTTP/1.1" - 200 23388 "" "Mozilla/5.0 (Linux; U; Android 4.4.4; zh-cn; MX4 Pro Build/KTU84P) AppleWebKit/534.30 (KHTML, like Gecko) Version/4.0 Mobile Safari/534.30" 0.001 101.226.125.103",
"@version" => "1",
"@timestamp" => "2017-02-08T01:47:32.656Z",
"host" => "Vsftp",
"clientip" => "10.168.255.134",
"time" => "09/Oct/2016:15:28:52 +0800",
"verb" => "GET",
"request" => "/",
"httpversion" => "1.1",
"http_status_code" => "200",
"bytes" => "23388",
"http_user_agent" => "Mozilla/5.0 (Linux; U; Android 4.4.4; zh-cn; MX4 Pro Build/KTU84P) AppleWebKit/534.30 (KHTML, like Gecko) Version/4.0 Mobile Safari/534.30",
"request_time" => "0.001",
"http_x_forwarded_for" => "101.226.125.103",
"geoip" => {
"ip" => "101.226.125.103",
"country_code2" => "CN",
"country_code3" => "CHN",
"country_name" => "China",
"continent_code" => "AS",
"region_name" => "23",
"city_name" => "Shanghai",
"latitude" => 31.045600000000007,
"longitude" => 121.3997,
"timezone" => "Asia/Shanghai",
"real_region_name" => "Shanghai",
"location" => [
[0] 121.3997,
[1] 31.045600000000007
]
},
"aaaaaa" => 121.3997,
"bbbbbb" => 31.045600000000007
}
变量值内插:
[elk@Vsftp logstash]$ cat logstash.conf
input {
stdin{}
}
filter {
grok {
match =>[
"message","%{IPORHOST:clientip} [%{HTTPDATE:time}] "%{WORD:verb} %{URIPATHPARAM:request}?.* HTTP/%{NUMBER:httpversion}" - %{NUMBER:http_status_code} %{NUMBER:bytes} "(?<http_referer>S+)" "(?<http_user_agent>(S+s+)*S+)" (%{BASE16FLOAT:request_time}) (%{IPORHOST:http_x_forwarded_for}|-)",
"message" , "%{IPORHOST:clientip} [%{HTTPDATE:time}] "%{WORD:verb} %{URIPATHPARAM:request} HTTP/%{NUMBER:httpversion}" - %{NUMBER:http_status_code} %{NUMBER:bytes} "(?<http_referer>S+)" "(?<http_user_agent>(S+s+)*S+)" (%{BASE16FLOAT:request_time}) (%{IPORHOST:http_x_forwarded_for}|-)",
"message","%{IPORHOST:clientip} [%{HTTPDATE:time}] "%{WORD:verb} (?<http_url>S+)s+HTTP/%{NUMBER:httpversion}"s+-s+%{NUMBER:http_status_code}s+%{NUMBER:bytes}s+"-"s+"(?<http_user_agent>(S+))"s+(%{BASE16FLOAT:request_time})s+(%{IPORHOST:http_x_forwarded_for}|-)",
"message","%{IPORHOST:clientip} [%{HTTPDATE:time}] "%{WORD:verb} %{URIPATHPARAM:request} HTTP/%{NUMBER:httpversion}" - %{NUMBER:http_status_code} %{NUMBER:bytes} "" "(?<http_user_agent>(S+s+)*S+)" (%{BASE16FLOAT:request_time}) (%{IPORHOST:http_x_forwarded_for}|-)"
]
}
geoip {
source => "http_x_forwarded_for"
target => "geoip"
database => "/usr/local/logstash-2.3.4/etc/GeoLiteCity.dat"
add_field => [ "kkkkkkk", "[geoip][location][0]"]
add_field => [ "hhhhhhh", "[geoip][location][1]" ]
}
}
output {
stdout {
codec => rubydebug
}
}
[elk@Vsftp logstash]$ logstash -f logstash.conf
Settings: Default pipeline workers: 4
Pipeline main started
10.168.255.134 [09/Oct/2016:15:28:52 +0800] "GET / HTTP/1.1" - 200 23388 "" "Mozilla/5.0 (Linux; U; Android 4.4.4; zh-cn; MX4 Pro Build/KTU84P) AppleWebKit/534.30 (KHTML, like Gecko) Version/4.0 Mobile Safari/534.30" 0.001 101.226.125.103
{
"message" => "10.168.255.134 [09/Oct/2016:15:28:52 +0800] "GET / HTTP/1.1" - 200 23388 "" "Mozilla/5.0 (Linux; U; Android 4.4.4; zh-cn; MX4 Pro Build/KTU84P) AppleWebKit/534.30 (KHTML, like Gecko) Version/4.0 Mobile Safari/534.30" 0.001 101.226.125.103",
"@version" => "1",
"@timestamp" => "2017-02-08T01:49:49.034Z",
"host" => "Vsftp",
"clientip" => "10.168.255.134",
"time" => "09/Oct/2016:15:28:52 +0800",
"verb" => "GET",
"request" => "/",
"httpversion" => "1.1",
"http_status_code" => "200",
"bytes" => "23388",
"http_user_agent" => "Mozilla/5.0 (Linux; U; Android 4.4.4; zh-cn; MX4 Pro Build/KTU84P) AppleWebKit/534.30 (KHTML, like Gecko) Version/4.0 Mobile Safari/534.30",
"request_time" => "0.001",
"http_x_forwarded_for" => "101.226.125.103",
"geoip" => {
"ip" => "101.226.125.103",
"country_code2" => "CN",
"country_code3" => "CHN",
"country_name" => "China",
"continent_code" => "AS",
"region_name" => "23",
"city_name" => "Shanghai",
"latitude" => 31.045600000000007,
"longitude" => 121.3997,
"timezone" => "Asia/Shanghai",
"real_region_name" => "Shanghai",
"location" => [
[0] 121.3997,
[1] 31.045600000000007
]
},
"kkkkkkk" => "[geoip][location][0]",
"hhhhhhh" => "[geoip][location][1]"
必须使用 add_field => [ "aaaaaa", "%{[geoip][location][0]}" ]
add_field => [ "bbbbbb", "%{[geoip][location][1]}" ]
}