k8s filebeat收集容器中的日志文件

k8s-filebeat收集容器中的日志文件

1. K8s-收集容器中的日志文件

收集容器文件图解

针对容器中日志文件：在Pod中增加一个容器运行日志采集器，使用emtyDir共享日志目录让日志采集器读取到日志文件

示例代码

[root@k8s-master elk]# vi app-log-logfile.yaml
[root@k8s-master elk]# cat app-log-logfile.yaml 
apiVersion: apps/v1
kind: Deployment
metadata:
  name: app-log-logfile
spec:
  replicas: 3
  selector:
    matchLabels:
      project: microservice
      app: nginx-logfile
  template:
    metadata:
      labels:
        project: microservice
        app: nginx-logfile
    spec:
      containers:
      # 应用容器
      - name: nginx 
        image: lizhenliang/nginx-php
        # 将数据卷挂载到日志目录
        volumeMounts:
        - name: nginx-logs 
          mountPath: /usr/local/nginx/logs
      # 日志采集器容器
      - name: filebeat
        image: elastic/filebeat:7.10.1 
        args: [
          "-c", "/etc/filebeat.yml",
          "-e",
        ]
        resources:
          requests:
            cpu: 100m
            memory: 100Mi
          limits:
            memory: 500Mi
        securityContext:
          runAsUser: 0
        volumeMounts:
        # 挂载filebeat配置文件
        - name: filebeat-config
          mountPath: /etc/filebeat.yml
          subPath: filebeat.yml
        # 将数据卷挂载到日志目录
        - name: nginx-logs 
          mountPath: /usr/local/nginx/logs
      # 数据卷共享日志目录
      volumes:
      - name: nginx-logs
        emptyDir: {}
      - name: filebeat-config
        configMap:
          name: filebeat-nginx-config
---
apiVersion: v1
kind: Service
metadata:
  name: app-log-logfile
spec:
  ports:
  - port: 80
    protocol: TCP
    targetPort: 80
  selector:
    project: microservice
    app: nginx-logfile
---
apiVersion: v1
kind: ConfigMap
metadata:
  name: filebeat-nginx-config
  
data:
  # 配置文件保存在ConfigMap
  filebeat.yml: |-
    filebeat.inputs:
      - type: log
        paths:
          - /usr/local/nginx/logs/access.log
        # tags: ["access"]
        fields_under_root: true
        fields:
          project: microservice
          app: nginx

    setup.ilm.enabled: false
    setup.template.name: "nginx-access"
    setup.template.pattern: "nginx-access-*"

    output.elasticsearch:
      hosts: ['127.0.0.1:9200']
      username: "admin"
      password: "12345678"
      index: "nginx-access-%{+yyyy.MM.dd}"

2. 案例

编写配置文件

[root@k8s-master elk]# vi app-log-logfile.yaml
[root@k8s-master elk]# cat app-log-logfile.yaml 
apiVersion: apps/v1
kind: Deployment
metadata:
  name: app-log-logfile
spec:
  replicas: 3
  selector:
    matchLabels:
      project: microservice
      app: nginx-logfile
  template:
    metadata:
      labels:
        project: microservice
        app: nginx-logfile
    spec:
      containers:
      # 应用容器
      - name: nginx 
        image: lizhenliang/nginx-php
        # 将数据卷挂载到日志目录
        volumeMounts:
        - name: nginx-logs 
          mountPath: /usr/local/nginx/logs
      # 日志采集器容器
      - name: filebeat
        image: elastic/filebeat:7.10.1 
        args: [
          "-c", "/etc/filebeat.yml",
          "-e",
        ]
        resources:
          requests:
            cpu: 100m
            memory: 100Mi
          limits:
            memory: 500Mi
        securityContext:
          runAsUser: 0
        volumeMounts:
        # 挂载filebeat配置文件
        - name: filebeat-config
          mountPath: /etc/filebeat.yml
          subPath: filebeat.yml
        # 将数据卷挂载到日志目录
        - name: nginx-logs 
          mountPath: /usr/local/nginx/logs
      # 数据卷共享日志目录
      volumes:
      - name: nginx-logs
        emptyDir: {}
      - name: filebeat-config
        configMap:
          name: filebeat-nginx-config
---
apiVersion: v1
kind: Service
metadata:
  name: app-log-logfile
spec:
  ports:
  - port: 80
    protocol: TCP
    targetPort: 80
  selector:
    project: microservice
    app: nginx-logfile
---
apiVersion: v1
kind: ConfigMap
metadata:
  name: filebeat-nginx-config
  
data:
  # 配置文件保存在ConfigMap
  filebeat.yml: |-
    filebeat.inputs:
      - type: log
        paths:
          - /usr/local/nginx/logs/access.log
        # tags: ["access"]
        fields_under_root: true
        fields:
          project: microservice
          app: nginx

    setup.ilm.enabled: false
    setup.template.name: "nginx-access"
    setup.template.pattern: "nginx-access-*"

    output.elasticsearch:
      hosts: ['127.0.0.1:9200']
      username: "admin"
      password: "12345678"
      index: "nginx-access-%{+yyyy.MM.dd}"

运行配置文件

[root@k8s-master elk]# kubectl apply -f app-log-logfile.yaml 
deployment.apps/app-log-logfile created
service/app-log-logfile created
configmap/filebeat-nginx-config created

查看容器是否启动

[root@k8s-master elk]# kubectl get pods
NAME                               READY   STATUS    RESTARTS   AGE
app-log-logfile-68b99bf447-fvvdj   2/2     Running   0          2m21s

访问nginx

[root@k8s-master elk]# kubectl get pods -o wide
NAME                               READY   STATUS    RESTARTS   AGE     IP              NODE         NOMINATED NODE   READINESS GATES
app-log-logfile-68b99bf447-fvvdj   2/2     Running   0          5m53s   10.244.85.199   k8s-node01   <none>           <none>
nginx                              1/1     Running   0          33h     10.244.85.196   k8s-node01   <none>           <none>
tomcat                             1/1     Running   0          33h     10.244.85.197   k8s-node01   <none>           <none>
web-5df8b97c79-hksfc               1/1     Running   0          3d4h    10.244.85.195   k8s-node01   <none>           <none>
[root@k8s-master elk]# curl -I 10.244.85.199/status.html
HTTP/1.1 200 OK
Server: nginx
Date: Thu, 08 Jul 2021 14:41:48 GMT
Content-Type: text/html
Content-Length: 3
Last-Modified: Tue, 01 Jan 2019 20:08:29 GMT
Connection: close
ETag: "5c2bc8bd-3"
Accept-Ranges: bytes
[root@k8s-master elk]# curl -I 10.244.85.199/status.html
HTTP/1.1 200 OK
Server: nginx
Date: Thu, 08 Jul 2021 14:41:48 GMT
Content-Type: text/html
Content-Length: 3
Last-Modified: Tue, 01 Jan 2019 20:08:29 GMT
Connection: close
ETag: "5c2bc8bd-3"
Accept-Ranges: bytes

kibana验证是否有索引
创建索引验证数据