ELKStack-生产案例项目实战（十一）

ELKStack-生产案例项目实战

1、收集ES和apache日志，入redis

input {
    file {
        path => "/etc/httpd/logs/access_log"
        start_position => "beginning"
        type => "apache-accesslog"
    }
	file{
        path => "/var/log/elasticsearch/myes.log"
        type => "es-log"
        start_position => "beginning"
        codec => multiline{
            pattern => "^["
            negate => true
            what => "previous"
        }
    }
}


output{
    if [type] == "es-log" {
	    redis {
            host => ["192.168.137.11"]
            port => 6379
            db => 1
            data_type => "list"
            key => "es-log"
            timeout => 10
        }
    }
    if [type] == "apache-accesslog" {
        redis {
            host => ["192.168.137.11"]
            port => 6379
            db => 1
            data_type => "list"
            key => "apache-accesslog"
            timeout => 10
        }
    }
}

启动/opt/logstash/bin/logstash -f /etc/logstash/conf.d/shipper.conf

2、通过syslog服务端主机，获取所有的客户端主机的syslog和redis中数据，写入ES

input{
    syslog {
        type => "system-syslog"
        port => 514
    }
	redis {
        type => "es-log"
		host => ["192.168.137.11"]
		port => 6379
		db => 1
		data_type => "list"
		key => "es-log"
		timeout => 10
	}
	redis {
        type => "apache-accesslog"
		host => ["192.168.137.11"]
		port => 6379
		db => 1
		data_type => "list"
		key => "apache-accesslog"
		timeout => 10
	}
}

filter {
    if [type] == "apache-accesslog" {
        grok {
            match => { "message" => "%{COMBINEDAPACHELOG}" }
        }
    }
}

output{
    if [type] == "apache-accesslog" {
	    elasticsearch {
            hosts => ["192.168.137.11:9200"]
            index => "apache-accesslog-%{+YYYY.MM.dd}"
        }
    }
    if [type] == "es-log" {
        elasticsearch {
            hosts => ["192.168.137.11:9200"]
            index => "es-log-%{+YYYY.MM}"
        }
    }
    if [type] == "system-syslog" {
        elasticsearch {
            hosts => ["192.168.137.11:9200"]
            index => "system-syslog-%{+YYYY.MM}"
        }
    }
}

启动/opt/logstash/bin/logstash -f /etc/logstash/conf.d/redis-es.conf