k8s日志收集方案
三种收集方案的优缺点：
下面我们就实践第二种日志收集方案：
一、安装ELK
下面直接采用yum的方式安装ELK（源码包安装参考：https://www.cnblogs.com/Dev0ps/p/9309103.html）
1.安装jdk
yum install -y java
2.添加ELK源
cat << EOF >>/etc/yum.repos.d/logstash.repo [logstash-7.x] name=Elastic repository for 7.x packages baseurl=https://artifacts.elastic.co/packages/7.x/yum gpgcheck=1 gpgkey=https://artifacts.elastic.co/GPG-KEY-elasticsearch enabled=1 autorefresh=1 type=rpm-md EOF
3.安装ELK组件
yum install elasticsearch logstash kibana -y
4.修改kibana的配置文件
[root@master ~]# grep -Ev '^$|^#' /etc/kibana/kibana.yml server.port: 5601 server.host: "0.0.0.0" elasticsearch.hosts: ["http://localhost:9200"]
5.启动服务
systemctl start elasticsearch systemctl start kibana
二、收集k8s的日志
1.添加logstash配置文件
input {
beats {
port => 5044
}
}
filter {
}
output {
if [app] == "k8s" {
if [type] == "module" {
elasticsearch {
hosts => ["http://127.0.0.1:9200"]
index => "k8s-log-%{+YYYY.MM.dd}"
}
}
}
}
2.启动logstash
/usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/logstash-to-es.conf &
3.创建filebeat的yaml文件
[root@master logs]# cat k8s-logs.yaml
apiVersion: v1
kind: ConfigMap
metadata:
name: k8s-logs-filebeat-config
namespace: kube-system
data:
filebeat.yml: |-
filebeat.prospectors:
- type: log
paths:
- /messages
fields:
app: k8s
type: module
fields_under_root: true
output.logstash:
hosts: ['172.31.182.140:5044']
---
apiVersion: apps/v1
kind: DaemonSet
metadata:
name: k8s-logs
namespace: kube-system
spec:
selector:
matchLabels:
project: k8s
app: filebeat
template:
metadata:
labels:
project: k8s
app: filebeat
spec:
containers:
- name: filebeat
image: docker.elastic.co/beats/filebeat:6.4.2
args: [
"-c", "/etc/filebeat.yml",
"-e",
]
resources:
requests:
cpu: 100m
memory: 100Mi
limits:
cpu: 500m
memory: 500Mi
securityContext:
runAsUser: 0
volumeMounts:
- name: filebeat-config
mountPath: /etc/filebeat.yml
subPath: filebeat.yml
- name: k8s-logs
mountPath: /messages
volumes:
- name: k8s-logs
hostPath:
path: /var/log/messages
type: File
- name: filebeat-config
configMap:
name: k8s-logs-filebeat-config
4.创建pod
kubectl apply -f k8s-logs.yaml
5.kibana添加索引
三、收集nginx日志
1.创建nginx-deployment
默认的日志路径为：/var/log/nginx/
[root@master logs]# cat nginx-deployment.yaml
apiVersion: apps/v1beta1
kind: Deployment
metadata:
name: nginx-demo
# namespace: test
spec:
replicas: 3
selector:
matchLabels:
project: www
app: www
template:
metadata:
labels:
project: www
app: www
spec:
imagePullSecrets:
- name: registry-pull-secret
containers:
- name: nginx
image: nginx:latest
imagePullPolicy: Always
ports:
- containerPort: 80
name: web
protocol: TCP
resources:
requests:
cpu: 0.5
memory: 256Mi
limits:
cpu: 1
memory: 1Gi
resources:
requests:
cpu: 0.5
memory: 256Mi
limits:
cpu: 1
memory: 1Gi
livenessProbe:
httpGet:
path: /index.html
port: 80
initialDelaySeconds: 6
timeoutSeconds: 20
volumeMounts:
- name: nginx-logs
mountPath: /var/log/nginx/
- name: filebeat
image: docker.elastic.co/beats/filebeat:6.4.2
args: [
"-c", "/etc/filebeat.yml",
"-e",
]
resources:
limits:
memory: 500Mi
requests:
cpu: 100m
memory: 100Mi
securityContext:
runAsUser: 0
volumeMounts:
- name: filebeat-config
mountPath: /etc/filebeat.yml
subPath: filebeat.yml
- name: nginx-logs
mountPath: /var/log/nginx/
volumes:
- name: nginx-logs
emptyDir: {}
- name: filebeat-config
configMap:
name: filebeat-nginx-config
2.创建filebest收集日志
[root@master logs]# cat filebeat-nginx-configmap.yaml
apiVersion: v1
kind: ConfigMap
metadata:
name: filebeat-nginx-config
# namespace: test
data:
filebeat.yml: |-
filebeat.prospectors:
- type: log
paths:
- /var/log/nginx/access.log
# tags: ["access"]
fields:
app: www
type: nginx-access
fields_under_root: true
- type: log
paths:
- /var/log/nginx/error.log
# tags: ["error"]
fields:
app: www
type: nginx-error
fields_under_root: true
output.logstash:
hosts: ['172.31.182.140:5044']
3.添加logstas配置
[root@master logs]# cat /etc/logstash/conf.d/logstash-to-es.conf
input {
beats {
port => 5044
}
}
filter {
}
output {
if [app] == "www" {
if [type] == "nginx-access" {
elasticsearch {
hosts => ["http://127.0.0.1:9200"]
index => "nginx-access-%{+YYYY.MM.dd}"
}
}
else if [type] == "nginx-error" {
elasticsearch {
hosts => ["http://127.0.0.1:9200"]
index => "nginx-error-%{+YYYY.MM.dd}"
}
}
} else if [app] == "k8s" {
if [type] == "module" {
elasticsearch {
hosts => ["http://127.0.0.1:9200"]
index => "k8s-log-%{+YYYY.MM.dd}"
}
}
}
# stdout { codec=> rubydebug }
}
4.kibana添加索引
四、收集tomcat日志
1.1.创建tomcat-deployment
默认的日志路径为：/usr/local/tomcat/logs
[root@master logs]# cat tomcat-deployment.yaml
apiVersion: apps/v1beta1
kind: Deployment
metadata:
name: tomcat-java-demo
# namespace: test
spec:
replicas: 3
selector:
matchLabels:
project: www
app: www
template:
metadata:
labels:
project: www
app: www
spec:
imagePullSecrets:
- name: registry-pull-secret
containers:
- name: tomcat
image: tomcat:latest
imagePullPolicy: Always
ports:
- containerPort: 8080
name: web
protocol: TCP
resources:
requests:
cpu: 0.5
memory: 1Gi
limits:
cpu: 1
memory: 2Gi
livenessProbe:
httpGet:
path: /
port: 8080
initialDelaySeconds: 60
timeoutSeconds: 20
readinessProbe:
httpGet:
path: /
port: 8080
initialDelaySeconds: 60
timeoutSeconds: 20
volumeMounts:
- name: tomcat-logs
mountPath: /usr/local/tomcat/logs
- name: filebeat
image: docker.elastic.co/beats/filebeat:6.4.2
args: [
"-c", "/etc/filebeat.yml",
"-e",
]
resources:
limits:
memory: 500Mi
requests:
cpu: 100m
memory: 100Mi
securityContext:
runAsUser: 0
volumeMounts:
- name: filebeat-config
mountPath: /etc/filebeat.yml
subPath: filebeat.yml
- name: tomcat-logs
mountPath: /usr/local/tomcat/logs
volumes:
- name: tomcat-logs
emptyDir: {}
- name: filebeat-config
configMap:
name: filebeat-config
2.创建filebest收集tomcat日志
[root@master logs]# cat filebeat-tomcat-configmap.yaml
apiVersion: v1
kind: ConfigMap
metadata:
name: filebeat-config
# namespace: test
data:
filebeat.yml: |-
filebeat.prospectors:
- type: log
paths:
- /usr/local/tomcat/logs/catalina.*
# tags: ["tomcat"]
fields:
app: www
type: tomcat-catalina
fields_under_root: true
multiline:
pattern: '^['
negate: true
match: after
output.logstash:
hosts: ['172.31.182.140:5044']
3.添加logstas配置后重启
[root@master logs]# cat /etc/logstash/conf.d/logstash-to-es.conf
input {
beats {
port => 5044
}
}
filter {
}
output {
if [app] == "www" {
if [type] == "nginx-access" {
elasticsearch {
hosts => ["http://127.0.0.1:9200"]
index => "nginx-access-%{+YYYY.MM.dd}"
}
}
else if [type] == "nginx-error" {
elasticsearch {
hosts => ["http://127.0.0.1:9200"]
index => "nginx-error-%{+YYYY.MM.dd}"
}
}
else if [type] == "tomcat-catalina" {
elasticsearch {
hosts => ["http://127.0.0.1:9200"]
index => "tomcat-catalina-%{+YYYY.MM.dd}"
}
}
} else if [app] == "k8s" {
if [type] == "module" {
elasticsearch {
hosts => ["http://127.0.0.1:9200"]
index => "k8s-log-%{+YYYY.MM.dd}"
}
}
}
# stdout { codec=> rubydebug }
}
4.kibana添加索引