1 安装一个redis服务
[root@node4 ~]# yum -y install redis
直接启动
[root@node4 ~]# systemctl restart redis
[root@node4 ~]# systemctl status redis
[root@node4 ~]# redis-cli -h 127.0.0.1
2 配置filebeat，把数据传给redis
[root@node4 ~]# vim /etc/filebeat/filebeat.yml
filebeat.inputs: ##################################################### ## Nginx log ##################################################### - type: log enabled: true paths: - /usr/local/nginx/logs/access.log json.key_under_root: true json.overwrite_keys: true tags: ["access"] #- type: log # enabled: true # paths: # - /usr/local/nginx/logs/error.log # tags: ["error"] ##################################################### ## tomcat log ##################################################### - type: log enabled: true paths: - /var/log/tomcat/localhost_access_log.*.txt json.key_under_root: true json.overwrite_keys: true tags: ["tomcat"] ##################################################### ## java log ##################################################### - type: log enabled: true paths: - /usr/local/elasticsearch/logs/my-elktest-cluster.log tags: ["es-java"] multiline.pattern: '^[' multiline.negate: true multiline.match: "after" ##################################################### ## docker log ##################################################### - type: docker containers.ids: - '*' json.key_under_root: true json.overwrite_keys: true tags: ["docker"] ##################################################### ## outout redis ##################################################### output.redis: hosts: ["127.0.0.1"] key: "filebeat" db: 0 timeout: 5
[root@node4 ~]# systemctl restart filebeat
访问产生日志
3 查看redis
127.0.0.1:6379> keys * 1) "filebeat" 127.0.0.1:6379> 127.0.0.1:6379> keys * 1) "filebeat" 127.0.0.1:6379> type filebeat #查看类型 list 127.0.0.1:6379> llen filebeat #查看长度 (integer) 22 127.0.0.1:6379> LRANGE filebeat 1 22 1) "{"@timestamp":"2020-01-20T14:22:15.291Z","@metadata":{"beat":"filebeat","type":"_doc","version":"7.4.2"},"agent":{"hostname":"node4","id":"bb3818f9-66e2-4eb2-8f0c-3f35b543e025","version":"7.4.2","type":"filebeat","ephemeral_id":"663027a7-1bdc-4a9f-b9d3-1297ef06c0b0"},"log":{"offset":21185,"file":{"path":"/usr/local/nginx/logs/error.log"}},"message":"2020/01/20 09:22:08 [error] 2790#0: *32 open() \"/usr/local/nginx/html/favicon.ico\" failed (2: No such file or directory), client: 192.168.132.1, server: localhost, request: \"GET /favicon.ico HTTP/1.1\", host: \"192.168.132.134\", referrer: \"http://192.168.132.134/\"","tags":["error"],"input":{"type":"log"},"ecs":{"version":"1.1.0"},"host":{"name":"node4"}}"
使用json解析
{ "@timestamp": "2020-01-20T14:22:15.293Z", "@metadata": { "beat": "filebeat", "type": "_doc", "version": "7.4.2" }, "log": { "offset": 21460, "file": { "path": "/usr/local/nginx/logs/access.log" } }, "json": { "size": 612, "xff": "-", "upstreamhost": "-", "url": "/index.html", "domain": "192.168.132.134", "upstreamtime": "-", "@timestamp": "2020-01-20T09:22:08-05:00", "clientip": "192.168.132.1", "host": "192.168.132.134", "status": "200", "http_host": "192.168.132.134", "Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", "responsetime": 0, "referer": "-" }, "tags": ["access"], "input": { "type": "log" }, "ecs": { "version": "1.1.0" }, "host": { "name": "node4" }, "agent": { "id": "bb3818f9-66e2-4eb2-8f0c-3f35b543e025", "version": "7.4.2", "type": "filebeat", "ephemeral_id": "663027a7-1bdc-4a9f-b9d3-1297ef06c0b0", "hostname": "node4" } }
4 使用logstash收集消费resdis的数据
再node4节点安装logstash
[root@node4 ~]# wget https://artifacts.elastic.co/downloads/logstash/logstash-7.5.1.rpm
[root@node4 ~]# rpm -ivh logstash-7.5.1.rpm
[root@node4 ~]# vim /etc/logstash/conf.d/logsatsh.conf
input { redis { host => "127.0.0.1" port => "6379" db => "0" key => "filebeat" data_type => "list" } } filter{ mutate { convert => ["upstream_time","float"] convert => ["request_time","float"] } } output{ stdout {} elasticsearch { hosts => "192.168.132.131:9200" manage_template => false index => "nginx_access-%{+yyyy.MM.dd}" } }
[root@node4 ~]# /usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/logsatsh.conf
已经有索引
访问
[root@node5 ~]# ab -n 20000 -c 20 http://192.168.132.134
查看redis
127.0.0.1:6379> keys * (empty list or set) 127.0.0.1:6379> keys * 1) "filebeat" 127.0.0.1:6379> LLEN filebeat (integer) 16875 127.0.0.1:6379> LLEN filebeat (integer) 16000 127.0.0.1:6379> LLEN filebeat (integer) 15125 127.0.0.1:6379> LLEN filebeat (integer) 14375 127.0.0.1:6379> LLEN filebeat (integer) 13625 127.0.0.1:6379> LLEN filebeat (integer) 13000 127.0.0.1:6379> LLEN filebeat (integer) 12375
使用kibana查看
{ "_index": "nginx_access-2020.01.20", "_type": "_doc", "_id": "A1J-w28BOF7DoSFdyQr8", "_version": 1, "_score": null, "_source": { "host": { "name": "node4" }, "tags": [ "access" ], "input": { "type": "log" }, "ecs": { "version": "1.1.0" }, "log": { "file": { "path": "/usr/local/nginx/logs/access.log" }, "offset": 12386215 }, "json": { "host": "192.168.132.134", "upstreamtime": "-", "xff": "-", "status": "200", "referer": "-", "http_host": "192.168.132.134", "Agent": "ApacheBench/2.3", "url": "/index.html", "responsetime": 0, "domain": "192.168.132.134", "size": 612, "clientip": "192.168.132.135", "upstreamhost": "-", "@timestamp": "2020-01-20T10:07:11-05:00" }, "agent": { "hostname": "node4", "id": "bb3818f9-66e2-4eb2-8f0c-3f35b543e025", "type": "filebeat", "ephemeral_id": "efddca40-1d19-4036-9724-410b1b6d4c8b", "version": "7.4.2" }, "@version": "1", "@timestamp": "2020-01-20T15:07:16.439Z" }, "fields": { "json.@timestamp": [ "2020-01-20T15:07:11.000Z" ], "@timestamp": [ "2020-01-20T15:07:16.439Z" ] }, "sort": [ 1579532836439 ] }
5 filebeat添加错误日志
- type: log enabled: true paths: - /usr/local/nginx/logs/error.log tags: ["error"] output.redis: hosts: ["127.0.0.1"] keys: - key: "nginx_access" when.contains: tags: "access" - key: "nginx_error" when.contains: tags: "error"
访问的错误日志
[root@node5 ~]# ab -n 20000 -c 200 http://192.168.132.134/hehe
127.0.0.1:6379> keys * 1) "nginx_error" 2) "nginx_access"
6 有两个key，配置logstash
[root@node4 ~]# cat /etc/logstash/conf.d/logsatsh.conf
input { redis { host => "127.0.0.1" port => "6379" db => "0" key => "nginx_access" data_type => "list" } redis { host => "127.0.0.1" port => "6379" db => "0" key => "nginx_error" data_type => "list" } } filter{ mutate { convert => ["upstream_time","float"] convert => ["request_time","float"] } } output{ stdout {} if "access" in [tags]{ elasticsearch { hosts => "192.168.132.131:9200" manage_template => false index => "nginx_access-%{+yyyy.MM.dd}" } } if "error" in [tags]{ elasticsearch { hosts => "192.168.132.131:9200" manage_template => false index => "nginx_error-%{+yyyy.MM.dd}" } }
}
[root@node4 ~]# /usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/logsatsh.conf
7 启动后，redis被消费
127.0.0.1:6379> keys * 1) "nginx_error" 2) "nginx_access" 127.0.0.1:6379> LLEN nginx_error (integer) 20000 127.0.0.1:6379> LLEN nginx_error (integer) 20000 127.0.0.1:6379> LLEN nginx_error (integer) 20000 127.0.0.1:6379> LLEN nginx_error (integer) 20000 127.0.0.1:6379> LLEN nginx_error (integer) 16125 127.0.0.1:6379> LLEN nginx_error (integer) 15750 127.0.0.1:6379> LLEN nginx_access (integer) 14625 127.0.0.1:6379> LLEN nginx_access (integer) 14375 127.0.0.1:6379> LLEN nginx_error (integer) 14000 127.0.0.1:6379> LLEN nginx_error
查看索引
8 优化配置
通过上面的传输，logstah重视通过tags来区分日志，所以再logstash的input中可以不配置两个key，只需要配置一个key即可
[root@node4 ~]# vim /etc/logstash/conf.d/logsatsh.conf
input { redis { host => "127.0.0.1" port => "6379" db => "0" key => "nginx" data_type => "list" } } filter{ mutate { convert => ["upstream_time","float"] convert => ["request_time","float"] } } output{ stdout {} if "access" in [tags]{ elasticsearch { hosts => "192.168.132.131:9200" manage_template => false index => "nginx_access-%{+yyyy.MM.dd}" } } if "error" in [tags]{ elasticsearch { hosts => "192.168.132.131:9200" manage_template => false index => "nginx_error-%{+yyyy.MM.dd}" } } }
filebeat配置
##################################################### ## outout redis ##################################################### output.redis: hosts: ["127.0.0.1"] keys: "nginx"
[root@node4 ~]# systemctl restart filebeat
[root@node4 ~]# /usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/logsatsh.conf
访问生成日志
[root@node5 ~]# ab -n 20000 -c 200 http://192.168.132.134/hehe
查看redis
127.0.0.1:6379> keys * 1) "nginx" 127.0.0.1:6379> LLEN nginx (integer) 20125 127.0.0.1:6379> LLEN nginx (integer) 19625 127.0.0.1:6379> LLEN nginx (integer) 18250 127.0.0.1:6379> LLEN nginx (integer) 17500 127.0.0.1:6379> LLEN nginx (integer) 16750 127.0.0.1:6379> LLEN nginx (integer) 0
查看索引
索引生成，实验完成