elk+redis

一 简介

在上一篇中介绍了elk安装和收集日志，这一篇我们在中间加个数据库，让filebeat直接把日志发送给redis,然后logstash在去redis里面取出来
大致流程:filebeat---redis---logstash---es--kibana

二 redis安装

下载  https://redis.io/download

三 logstash配置

用redis，配置文件需要更改一下

[root@elk logstash]# cat conf.d/logstash.conf
input {
  redis {
     host => "172.17.199.231"
     port => 6379
     key => "filebeat"
     db => "0"
     data_type => "list"
  }
}


filter{
    grok {
       match => ["message", "%{SYSLOGBASE} %{GREEDYDATA:message}"]
       overwrite => ["message"]
    }
}

output {
    if [app] == "www" {
        if [type] == "tiantian-system-service-node2" {
           elasticsearch {
              hosts => ["http://127.0.0.1:9200"]
              index => "tiantian-system-service-node2-%{+YYYY.MM.dd}"
           }
        }
        else if [type] == "tiantian-system-service-node4" {
           elasticsearch {
              hosts => ["http://127.0.0.1:9200"]
              index => "tiantian-system-service-node4-%{+YYYY.MM.dd}"
           }
        }
        else if [type] == "tiantian-assets-service-node2" {
           elasticsearch {
              hosts => ["http://127.0.0.1:9200"]
              index => "tiantian-assets-service-node2-%{+YYYY.MM.dd}"
           }
        }
   

        else if [type] == "tiantian-collection-admin-node6" {
           elasticsearch {
              hosts => ["http://127.0.0.1:9200"]
              index => "tiantian-collection-admin-node6-%{+YYYY.MM.dd}"
           }
        }
        else if [type] == "tiantian-collection-job-node5" {
           elasticsearch {
              hosts => ["http://127.0.0.1:9200"]
              index => "tiantian-collection-job-node5-%{+YYYY.MM.dd}"
           }
        }
 
        else if [type] == "xxd-credit-service-node2" {
           elasticsearch {
              hosts => ["http://127.0.0.1:9200"]
              index => "xxd-credit-service-node2-%{+YYYY.MM.dd}"
           }
        }
        else if [type] == "xxd-jinbaodai-api-node4" {
           elasticsearch {
              hosts => ["http://127.0.0.1:9200"]
              index => "xxd-jinbaodai-api-node4-%{+YYYY.MM.dd}"
           }
        }
    
    
    }
  stdout { codec=> rubydebug }
}

四 filebeat设置

https://www.elastic.co/guide/en/beats/filebeat/1.3/redis-output.html #filebeat设置redis

[root@node1 ~]# grep -v "^ *#" /etc/filebeat/filebeat.yml |grep -v "^$"
filebeat.inputs:
- type: log
  paths:
     - /datalog/service/jz-asset/node1/nohup.out
  encoding: utf-8
  tail_files: true
  fields:
    app: www
    type: jz-asset-node1
  fields_under_root: true
  multiline:
     pattern: '^[0-2][0-9]:[0-5][0-9]:[0-5][0-9]'
     negate: true
     match: after
filebeat.config.modules:
  path: ${path.config}/modules.d/*.yml
  reload.enabled: false
setup.template.settings:
  index.number_of_shards: 1
setup.kibana:
output:
  redis:
    hosts: ["172.17.199.231:6379"] #发送给redis
    save_topology: true
    index: "filebeat"
    db: 0
    db_topology: 1
    timeout: 5
    reconnect_interval: 1
processors:
  - add_host_metadata: ~
  - add_cloud_metadata: ~

只有这两个地方需要更改一下，其余地方不用设置