背景使用 EFK 收集 K8S 日志(K8S 容器运行时是 containerd)
安装 es 集群
# 创建目录
mkdir -p /data/yaml/kube-logging/{elasticsearch,filebeat,kibana}
cd /data/yaml/kube-logging/
# 创建命名空间
cat > kube-logging.yaml << EOF
kind: Namespace
apiVersion: v1
metadata:
name: kube-logging
EOF
kubectl apply -f kube-logging.yaml
cd /data/yaml/kube-logging/elasticsearch
# 创建es后端存储，这里使用 ceph
cat > es-rbd-sc.yaml << EOF
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
name: es-rbd-sc
namespace: kube-logging
provisioner: rbd.csi.ceph.com
parameters:
clusterID: 24c6d785-eec0-44f7-8ae1-738d05a1c89d
pool: kubernetes
csi.storage.k8s.io/provisioner-secret-name: csi-rbd-secret
csi.storage.k8s.io/provisioner-secret-namespace: default
csi.storage.k8s.io/node-stage-secret-name: csi-rbd-secret
csi.storage.k8s.io/node-stage-secret-namespace: default
imageFormat: "2"
imageFeatures: "layering"
reclaimPolicy: Delete
mountOptions:
- discard
EOF
kubectl apply -f es-rbd-sc.yaml
# 创建 es svc
cat > elasticsearch_svc.yaml << EOF
kind: Service
apiVersion: v1
metadata:
name: elasticsearch
namespace: kube-logging
labels:
app: elasticsearch
spec:
selector:
app: elasticsearch
clusterIP: None
ports:
- port: 9200
name: rest
- port: 9300
name: inter-node
EOF
kubectl apply -f elasticsearch_svc.yaml
# 创建 es pod
cat > elasticsearch_statefulset.yaml << EOF
apiVersion: apps/v1
kind: StatefulSet
metadata:
name: es-cluster
namespace: kube-logging
spec:
serviceName: elasticsearch
replicas: 3
selector:
matchLabels:
app: elasticsearch
template:
metadata:
labels:
app: elasticsearch
spec:
containers:
- name: elasticsearch
image: elasticsearch:7.2.0
imagePullPolicy: IfNotPresent
resources:
limits:
cpu: 1000m
requests:
cpu: 100m
ports:
- containerPort: 9200
name: rest
protocol: TCP
- containerPort: 9300
name: inter-node
protocol: TCP
volumeMounts:
- name: data
mountPath: /usr/share/elasticsearch/data
env:
- name: cluster.name
value: k8s-logs
- name: node.name
valueFrom:
fieldRef:
fieldPath: metadata.name
- name: discovery.seed_hosts
value: "es-cluster-0.elasticsearch,es-cluster-1.elasticsearch,es-cluster-2.elasticsearch"
- name: cluster.initial_master_nodes
value: "es-cluster-0,es-cluster-1,es-cluster-2"
- name: ES_JAVA_OPTS
value: "-Xms512m -Xmx512m"
initContainers:
- name: fix-permissions
image: busybox
imagePullPolicy: IfNotPresent
command: ["sh", "-c", "chown -R 1000:1000 /usr/share/elasticsearch/data"]
securityContext:
privileged: true
volumeMounts:
- name: data
mountPath: /usr/share/elasticsearch/data
- name: increase-vm-max-map
image: busybox
imagePullPolicy: IfNotPresent
command: ["sysctl", "-w", "vm.max_map_count=262144"]
securityContext:
privileged: true
- name: increase-fd-ulimit
image: busybox
imagePullPolicy: IfNotPresent
command: ["sh", "-c", "ulimit -n 65536"]
securityContext:
privileged: true
volumeClaimTemplates:
- metadata:
name: data
labels:
app: elasticsearch
spec:
accessModes: [ "ReadWriteOnce" ]
storageClassName: es-rbd-sc
resources:
requests:
storage: 5Gi
kubectl apply -f elasticsearch_statefulset.yaml
安装 Kibana
cd /data/yaml/kube-logging/kibana
cat > kibana.yaml << EOF
apiVersion: v1
kind: Service
metadata:
name: kibana
namespace: kube-logging
labels:
app: kibana
spec:
type: NodePort
ports:
- port: 5601
targetPort: 5601
nodePort: 30056
selector:
app: kibana
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: kibana
namespace: kube-logging
labels:
app: kibana
spec:
replicas: 1
selector:
matchLabels:
app: kibana
template:
metadata:
labels:
app: kibana
spec:
containers:
- name: kibana
image: kibana:7.2.0
imagePullPolicy: IfNotPresent
resources:
limits:
cpu: 1000m
requests:
cpu: 100m
env:
- name: ELASTICSEARCH_URL
value: http://elasticsearch:9200
ports:
- containerPort: 5601
EOF
kubectl apply -f kibana.yaml
安装 filebeat
cd /data/yaml/kube-logging/filebeat
# 配置权限
cat > rbac.yaml << EOF
---
# Source: filebeat/templates/filebeat-service-account.yaml
apiVersion: v1
kind: ServiceAccount
metadata:
name: filebeat
namespace: kube-logging
labels:
k8s-app: filebeat
---
# Source: filebeat/templates/filebeat-role.yaml
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRole
metadata:
name: filebeat
labels:
k8s-app: filebeat
rules:
- apiGroups: [""] # "" indicates the core API group
resources:
- namespaces
- pods
verbs:
- get
- watch
- list
---
# Source: filebeat/templates/filebeat-role-binding.yaml
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRoleBinding
metadata:
name: filebeat
subjects:
- kind: ServiceAccount
name: filebeat
namespace: kube-logging
roleRef:
kind: ClusterRole
name: filebeat
apiGroup: rbac.authorization.k8s.io
EOF
kubectl apply -f rbac.yaml
# 创建配置
cat > filebeat-cm.yaml << EOF
---
apiVersion: v1
kind: ConfigMap
metadata:
name: filebeat-config
namespace: kube-logging
labels:
k8s-app: filebeat
data:
filebeat.yml: |-
setup.ilm.enabled: false
filebeat.inputs:
- type: container
paths:
- /var/log/containers/*.log
processors:
- add_kubernetes_metadata:
host: ${NODE_NAME}
matchers:
- logs_path:
logs_path: "/var/log/containers/"
multiline.pattern: '^[[:space:]]+(at|.{3})|^Caused by:'
multiline.negate: false
multiline.match: after
processors:
- add_cloud_metadata:
- add_host_metadata:
cloud.id: ${ELASTIC_CLOUD_ID}
cloud.auth: ${ELASTIC_CLOUD_AUTH}
setup.template.name: "k8s"
setup.template.pattern: "k8s-*"
setup.template.enabled: false
# 如果是第一次则不需要, 如果 index-template 已经存在需要更新, 则需要
setup.template.overwrite: false
setup.template.settings:
# 根据收集的日志量级, 因为日志会每天一份, 如果一天的日志量小于 30g, 一个 shard 足够
index.number_of_shards: 2
# 这个日志并不是那么重要, 并且如果是单节点的话, 直接设置为 0 个副本
index.number_of_replicas: 0
output.elasticsearch:
hosts: ['${ELASTICSEARCH_HOST:elasticsearch}:${ELASTICSEARCH_PORT:9200}']
# 启动进程数
worker: 20
# 发送重试的次数取决于max_retries的设置默认为3
max_retries: 3
# 单个elasticsearch批量API索引请求的最大事件数。默认是50。
bulk_max_size: 800
#index: "k8s-%{[kubernetes.namespace]}-%{[kubernetes.container.name]}-%{+yyyy.MM.dd}"
indices:
- index: "k8s-%{[kubernetes.namespace]}-%{[kubernetes.container.name]}-%{+yyyy.MM.dd}"
setup.kibana:
host: '${KIBANA_HOST:kibana-logging}:${KIBANA_PORT:5601}'
# 设置 ilm 的 policy life, 日志保留 45 天, 每 7 天一轮训, 最大 30g
setup.ilm:
policy_file: /etc/indice-lifecycle.json
---
apiVersion: v1
kind: ConfigMap
metadata:
name: filebeat-index-rules
namespace: kube-logging
labels:
k8s-app: filebeat
data:
indice-lifecycle.json: |-
{
"policy": {
"phases": {
"hot": {
"actions": {
"rollover": {
"max_size": "5GB" ,
"max_age": "1d"
}
}
},
"delete": {
"min_age": "7d",
"actions": {
"delete": {}
}
}
}
}
}
EOF
kubectl apply -f filebeat-cm.yaml
# 创建 pod
cat > filebeat-ds.yaml << EOF
apiVersion: apps/v1
kind: DaemonSet
metadata:
name: filebeat
namespace: kube-logging
labels:
k8s-app: filebeat
spec:
selector:
matchLabels:
k8s-app: filebeat
template:
metadata:
labels:
k8s-app: filebeat
spec:
serviceAccountName: filebeat
terminationGracePeriodSeconds: 30
hostNetwork: true
dnsPolicy: ClusterFirstWithHostNet
containers:
- name: filebeat
image: elastic/filebeat:7.4.2
args: [
"-c", "/etc/filebeat.yml",
"-e",
]
env:
- name: ELASTICSEARCH_HOST
value: "elasticsearch"
- name: ELASTICSEARCH_PORT
value: "9200"
- name: ELASTICSEARCH_USERNAME
value:
- name: ELASTICSEARCH_PASSWORD
value:
- name: ELASTIC_CLOUD_ID
value:
- name: ELASTIC_CLOUD_AUTH
value:
- name: NODE_NAME
valueFrom:
fieldRef:
fieldPath: spec.nodeName
securityContext:
runAsUser: 0
resources:
requests:
cpu: 100m
memory: 500Mi
volumeMounts:
- name: config
mountPath: /etc/filebeat.yml
readOnly: true
subPath: filebeat.yml
- name: rules
mountPath: /etc/indice-lifecycle.json
readOnly: true
subPath: indice-lifecycle.json
- name: data
mountPath: /usr/share/filebeat/data
- name: varlibdockercontainers
mountPath: /var/lib/docker/containers
readOnly: true
- name: varlog
mountPath: /var/log
readOnly: true
volumes:
- name: config
configMap:
defaultMode: 0600
name: filebeat-config
- name: rules
configMap:
defaultMode: 0600
name: filebeat-index-rules
- name: varlibdockercontainers
hostPath:
path: /var/log/pods
- name: varlog
hostPath:
path: /var/log
# data folder stores a registry of read status for all files, so we don't send everything again on a Filebeat pod restart
- name: data
hostPath:
path: /var/lib/filebeat-data
type: DirectoryOrCreate
EOF
kubectl apply -f filebeat-ds.yaml
