k8s实现elk日志收集系统
说明：es搭建集群，3台master，3台node，node角色分别为热数据，温数据，冷数据
kibana进行数据展示，同时连接es的工具使用es-head。
此处使用deployment+service结合搭建,logstash还未进行试验，等我继续更新。。。
master节点搭建
######### 创建名称空间 ###########
kind: Namespace
apiVersion: v1
metadata:
name: elastic
---
######## service类型为nodeport用以暴露es端口供外部访问 ########
kind: Service
apiVersion: v1
metadata:
name: es-cluster-nodeport
namespace: elastic
spec:
type: NodePort
selector:
app: es-cluster
ports:
- port: 9200
targetPort: 9200
nodePort: 32700
---
####### es集群间通讯端口，使用service进行暴露，但不像nodeport可对集群外服务器提供访问 #############
kind: Service
apiVersion: v1
metadata:
name: es-svc
namespace: elastic
labels:
app: es-cluster
annotations:
es.kubernetes.io/es: es-cluster
spec:
selector:
app: es-cluster
role: master
ports:
- port: 9300
targetPort: 9300
---
###### es的master创建 #############################
kind: Deployment # 资源类型
apiVersion: apps/v1 # 版本
metadata: # 元数据
namespace: elastic # 名称空间
name: es-cluster-master # deployment类型的元数据名
labels: # 标签
app: es-cluster-master # 定义deployment的标签
role: master
spec: # 说明
replicas: 3 # 副本数为3,表示创建3个pod
selector: # 选择器
matchLabels: # 精确选择
app: es-cluster # 选择标签为 app: es-cluster
role: master # 选择标签为 role: master
template: # 模板
metadata: # 元数据信息
labels: # 标签
app: es-cluster # 此模板的信息
role: master
spec: # 模板对应的说明
containers: # 容器
- name: elasticsearch-master # 名称
image: registry.cn-hangzhou.aliyuncs.com/k8sos/es:v1 # 镜像
ports: # 端口
- containerPort: 9200 # 这里的端口不提供暴露，仅说明
protocol: TCP
- containerPort: 9300
protocol: TCP
lifecycle: # 生命周期，也是回调钩子
postStart: # 当容器启动后执行的操作
exec: # 执行命令
command:
- "/bin/sh"
- "-c"
- "sysctl -w vm.max_map_count=262144; ulimit -l unlimited;"
securityContext: # 安全上下文，指具备对系统的操作权限
privileged: true
readinessProbe: # 就绪性探测
httpGet: # 探测9200端口是否就绪
port: 9200
volumeMounts: # 存储性挂载
- mountPath: /usr/share/elasticsearch/data # 容器中需要挂载的路径
name: es-data # 挂载到的地方，这个名称对应下方的挂载
env: # 此处定义的环境变量相当于在命令行执行命令，替代了配置文件的操作
- name: "cluster.name"
value: "es-cluster"
- name: "bootstrap.memory_lock"
value: "true"
- name: "discovery.zen.ping.unicast.hosts"
value: "es-svc"
- name: "discovery.zen.minimum_master_nodes"
value: "2"
- name: "discovery.zen.ping_timeout"
value: "5s"
- name: "node.master"
value: "true"
- name: "node.data"
value: "false"
- name: "ES_JAVA_OPTS"
value: "-Xms256m -Xmx256m"
- name: "http.cors.enabled"
value: "true"
- name: "http.cors.allow-origin"
value: "*"
volumes: # 对应上方的挂载名
- name: es-data
emptyDir: {}
head插件
kind: Deployment
apiVersion: apps/v1
metadata:
namespace: elastic
name: es-head
spec:
selector:
matchLabels: # deployment精确匹配选择对应的模板
app: es-head
template:
metadata:
labels:
app: es-head # 模板标签信息，供deployment选择
spec:
containers:
- name: head
image: alvinos/elasticsearch-head
ports:
- containerPort: 9100
protocol: TCP
---
##### 使用nodeport型的service，将pod中的端口进行暴露 ##########
kind: Service
apiVersion: v1
metadata:
name: es-head-svc
namespace: elastic
spec:
type: NodePort
selector:
app: es-head
ports:
- port: 9100
targetPort: 9100
nodePort: 32701
node节点--hot
kind: Deployment
apiVersion: apps/v1
metadata:
name: es-cluster-data-hot
namespace: elastic
labels: # deployment的标签
app: es-cluster-data-hot
role: data
spec:
selector:
matchLabels: # deployment选择器精确匹配模板中的信息
app: es-cluster
role: data
datatype: hot
template:
metadata:
labels: # 供上方deployment选择器选择
app: es-cluster
role: data
datatype: hot
spec:
containers:
- name: es-cluster-hot
image: registry.cn-hangzhou.aliyuncs.com/k8sos/es:v1
ports:
- containerPort: 9200
protocol: TCP
- containerPort: 9300
protocol: TCP
env: # 环境变量
- name: "cluster.name"
value: "es-cluster"
- name: "bootstrap.memory_lock"
value: "true"
- name: "discovery.zen.ping.unicast.hosts"
value: "es-svc"
- name: "node.master"
value: "false"
- name: "node.data"
value: "true"
- name: "ES_JAVA_OPTS"
value: "-Xms256m -Xmx256m"
- name: "node.attr.box_type"
value: "hot" # 指定为hot数据类型
lifecycle:
postStart: # 容器启动后的操作
exec:
command:
- "/bin/sh"
- "-c"
- "sysctl -w vm.max_map_count=262144; ulimit -l unlimited;"
securityContext:
privileged: true
volumeMounts:
- mountPath: /usr/share/elasticsearch/data
name: es-data
volumes:
- name: es-data
emptyDir: {}
node节点--warm
kind: Deployment
apiVersion: apps/v1
metadata:
name: es-cluster-data-warm
namespace: elastic
labels:
app: es-cluster-data-warm
role: data
spec:
selector:
matchLabels:
app: es-cluster
role: data
datatype: warm
template:
metadata:
labels:
app: es-cluster
role: data
datatype: warm
spec:
containers:
- name: es-cluster-warm
image: registry.cn-hangzhou.aliyuncs.com/k8sos/es:v1
ports:
- containerPort: 9200
protocol: TCP
- containerPort: 9300
protocol: TCP
env:
- name: "cluster.name"
value: "es-cluster"
- name: "bootstrap.memory_lock"
value: "true"
- name: "discovery.zen.ping.unicast.hosts"
value: "es-svc"
- name: "node.master"
value: "false"
- name: "node.data"
value: "true"
- name: "ES_JAVA_OPTS"
value: "-Xms256m -Xmx256m"
- name: "node.attr.box_type"
value: "warm"
lifecycle:
postStart:
exec:
command:
- "/bin/sh"
- "-c"
- "sysctl -w vm.max_map_count=262144; ulimit -l unlimited;"
securityContext:
privileged: true
readinessProbe:
httpGet:
port: 9200
volumeMounts:
- mountPath: /usr/share/elasticsearch/data
name: es-data
volumes:
- name: es-data
emptyDir: {}
node节点--cold
kind: Deployment
apiVersion: apps/v1
metadata:
name: es-cluster-data-cold
namespace: elastic
labels:
app: es-cluster-data-cold
role: data
spec:
selector:
matchLabels:
app: es-cluster
role: data
datatype: cold
template:
metadata:
labels:
app: es-cluster
role: data
datatype: cold
spec:
containers:
- name: es-cluster-cold
image: registry.cn-hangzhou.aliyuncs.com/k8sos/es:v1
ports:
- containerPort: 9200
protocol: TCP
- containerPort: 9300
protocol: TCP
env:
- name: "cluster.name"
value: "es-cluster"
- name: "bootstrap.memory_lock"
value: "true"
- name: "discovery.zen.ping.unicast.hosts"
value: "es-svc"
- name: "node.master"
value: "false"
- name: "node.data"
value: "true"
- name: "ES_JAVA_OPTS"
value: "-Xms256m -Xmx256m"
- name: "node.attr.box_type"
value: "cold"
lifecycle:
postStart:
exec:
command:
- "/bin/sh"
- "-c"
- "sysctl -w vm.max_map_count=262144; ulimit -l unlimited;"
securityContext:
privileged: true
readinessProbe:
httpGet:
port: 9200
volumeMounts:
- mountPath: /usr/share/elasticsearch/data
name: es-data
volumes:
- name: es-data
emptyDir: {}