简介

本集群使用 nginx + keepalived 实现高可用

nginx 安装配置

下载编译nginx

nginx 只需要编译一次，把编译后的文件拷贝到其他master机器上即可

cd /opt/k8s/work
wget http://nginx.org/download/nginx-1.15.3.tar.gz
tar -xzvf nginx-1.15.3.tar.gz

#编译
cd /opt/k8s/work/nginx-1.15.3
mkdir nginx-prefix
./configure --with-stream --without-http --prefix=$(pwd)/nginx-prefix --without-http_uwsgi_module 
make && make install

#############
--without-http_scgi_module --without-http_fastcgi_module
--with-stream：开启 4 层透明转发(TCP Proxy)功能；
--without-xxx：关闭所有其他功能，这样生成的动态链接二进制程序依赖最小；

查看 nginx 动态链接的库：

[root@node01 nginx-1.15.3]# ldd ./nginx-prefix/sbin/nginx
	linux-vdso.so.1 =>  (0x00007ffee18cc000)
	libdl.so.2 => /lib64/libdl.so.2 (0x00007f5e89daa000)
	libpthread.so.0 => /lib64/libpthread.so.0 (0x00007f5e89b8e000)
	libc.so.6 => /lib64/libc.so.6 (0x00007f5e897c0000)
	/lib64/ld-linux-x86-64.so.2 (0x00007f5e89fae000)

由于只开启了 4 层透明转发功能，所以除了依赖 libc 等操作系统核心 lib 库外，没有对其它 lib 的依赖(如 libz、libssl 等)，这样可以方便部署到各版本操作系统中

创建目录结构

cd /opt/k8s/work
source /opt/k8s/bin/environment.sh
for node_ip in ${MASTER_IPS[@]}
  do
    echo ">>> ${node_ip}"
    ssh root@${node_ip} "mkdir -p /opt/k8s/kube-nginx/{conf,logs,sbin}"
  done

拷贝二进制程序到其他主机 (有报错执行2遍就可以)

cd /opt/k8s/work
source /opt/k8s/bin/environment.sh
for node_ip in ${MASTER_IPS[@]}
  do
    echo ">>> ${node_ip}"
    scp /opt/k8s/work/nginx-1.15.3/nginx-prefix/sbin/nginx  root@${node_ip}:/opt/k8s/kube-nginx/sbin/kube-nginx
    ssh root@${node_ip} "chmod a+x /opt/k8s/kube-nginx/sbin/*"
  done

配置Nginx文件，开启4层透明转发

cd /opt/k8s/work
cat > kube-nginx.conf <<EOF
worker_processes 1;
events {
    worker_connections  1024;
}
stream {
    upstream backend {
        hash $remote_addr consistent;
        server 10.0.20.11:6443        max_fails=3 fail_timeout=30s;
        server 10.0.20.12:6443        max_fails=3 fail_timeout=30s;
        server 10.0.20.13:6443        max_fails=3 fail_timeout=30s;
    }
    server {
        listen *:8443;
        proxy_connect_timeout 1s;
        proxy_pass backend;
    }
}
EOF

#这里需要将server替换我们自己的地址

分发配置文件

cd /opt/k8s/work
source /opt/k8s/bin/environment.sh
for node_ip in ${MASTER_IPS[@]}
  do
    echo ">>> ${node_ip}"
    scp kube-nginx.conf  root@${node_ip}:/opt/k8s/kube-nginx/conf/kube-nginx.conf
  done

配置Nginx启动文件

cd /opt/k8s/work
cat > kube-nginx.service <<EOF
[Unit]
Description=kube-apiserver nginx proxy
After=network.target
After=network-online.target
Wants=network-online.target
[Service]
Type=forking
ExecStartPre=/opt/k8s/kube-nginx/sbin/kube-nginx -c /opt/k8s/kube-nginx/conf/kube-nginx.conf -p /opt/k8s/kube-nginx -t
ExecStart=/opt/k8s/kube-nginx/sbin/kube-nginx -c /opt/k8s/kube-nginx/conf/kube-nginx.conf -p /opt/k8s/kube-nginx
ExecReload=/opt/k8s/kube-nginx/sbin/kube-nginx -c /opt/k8s/kube-nginx/conf/kube-nginx.conf -p /opt/k8s/kube-nginx -s reload
PrivateTmp=true
Restart=always
RestartSec=5
StartLimitInterval=0
LimitNOFILE=65536
[Install]
WantedBy=multi-user.target
EOF

分发nginx启动文件

cd /opt/k8s/work
source /opt/k8s/bin/environment.sh
for node_ip in ${MASTER_IPS[@]}
  do
    echo ">>> ${node_ip}"
    scp kube-nginx.service  root@${node_ip}:/etc/systemd/system/
  done

启动 kube-nginx 服务

cd /opt/k8s/work
source /opt/k8s/bin/environment.sh
for node_ip in ${MASTER_IPS[@]}
  do
    echo ">>> ${node_ip}"
    ssh root@${node_ip} "systemctl daemon-reload && systemctl enable kube-nginx && systemctl start kube-nginx"
  done

检查 kube-nginx 服务运行状态

cd /opt/k8s/work
source /opt/k8s/bin/environment.sh
for node_ip in ${MASTER_IPS[@]}
  do
    echo ">>> ${node_ip}"
    ssh root@${node_ip} "systemctl status kube-nginx |grep 'Active:'"
  done

[root@node01 work]# for node_ip in ${MASTER_IPS[@]}
>   do
>     echo ">>> ${node_ip}"
>     ssh root@${node_ip} "systemctl status kube-nginx |grep 'Active:'"
>   done
>>> 10.0.20.11
   Active: active (running) since Thu 2019-12-05 15:13:19 CST; 3s ago
>>> 10.0.20.12
   Active: active (running) since Thu 2019-12-05 15:13:19 CST; 3s ago
>>> 10.0.20.13
   Active: active (running) since Thu 2019-12-05 15:13:19 CST; 3s ago

检查 kube-nginx 端口

cd /opt/k8s/work
source /opt/k8s/bin/environment.sh
for node_ip in ${MASTER_IPS[@]}
  do
    echo ">>> ${node_ip}"
    ssh root@${node_ip} "netstat -lntup | grep 8443"
  done

[root@node01 work]# for node_ip in ${MASTER_IPS[@]}
>   do
>     echo ">>> ${node_ip}"
>     ssh root@${node_ip} "netstat -lntup | grep 8443"
>   done
>>> 10.0.20.11
tcp        0      0 0.0.0.0:8443            0.0.0.0:*               LISTEN      5356/nginx: master  
>>> 10.0.20.12
tcp        0      0 0.0.0.0:8443            0.0.0.0:*               LISTEN      2586/nginx: master  
>>> 10.0.20.13
tcp        0      0 0.0.0.0:8443            0.0.0.0:*               LISTEN      2630/nginx: master

keepalived 安装配置

安装keeplive服务

前面我们也说了，高可用方案需要一个VIP，供集群内部访问

在所有master节点安装keeplived

yum  install -y keepalived

配置keeplive服务

配置文件模板

cd /opt/k8s/work
source /opt/k8s/bin/environment.sh
cat > keepalived.conf.template <<EOF
! Configuration File for keepalived
global_defs {
   router_id ##MASTER_IP##
}
vrrp_script chk_nginx {
    script "/etc/keepalived/check_port.sh 8443"
    interval 2
    weight -20
}
vrrp_instance VI_1 {
    state MASTER
    interface ##IFACE##
    virtual_router_id 251
    priority 100
    advert_int 1
    mcast_src_ip ##MASTER_IP##
    nopreempt
    authentication {
        auth_type PASS
        auth_pass 11111111
    }
    track_script {
         chk_nginx
    }
    virtual_ipaddress {
        ##KEEP_VIP##
    }
}
EOF

替换模板文件的变量，为各个节点生成配置文件

cd /opt/k8s/work
source /opt/k8s/bin/environment.sh
for (( i=0; i < 3; i++ ))
  do
    sed -e "s/##MASTER_IP##/${MASTER_IPS[i]}/" -e "s/##KEEP_VIP##/${KEEP_VIP_ADDR}/" -e "s/##IFACE##/${IFACE}/" keepalived.conf.template >  keepalived-${MASTER_IPS[i]}.conf
  done
ls keepalived-*.conf

将对应的keepalived配置文件拷贝到对应的节点上

cd /opt/k8s/work
source /opt/k8s/bin/environment.sh
for (( i=0; i < 3; i++ ))
  do
    echo ">>> ${node_ip}"
    scp  keepalived-${MASTER_IPS[i]}.conf ${MASTER_NAMES[i]}:/etc/keepalived/keepalived.conf
  done

创建健康检查脚本

cd /opt/k8s/work
cat > check_port.sh <<EOF
#!/bin/sh
CHK_PORT=$1
if [ -n "$CHK_PORT" ];then
  PORT_PROCESS=\`ss -lntup|grep ${CHK_PORT}|wc -l\`
    if [ $PORT_PROCESS -eq 0 ];then
      echo -e "33[31m ERROR: Port $CHK_PORT Is Not Used,End. 33[0m"
      exit 1
  fi
fi
EOF

分发脚本到所有keepalived节点

cd /opt/k8s/work
source /opt/k8s/bin/environment.sh
for node in ${MASTER_IPS[@]}
do
  echo ">>> ${node}"
  scp check_port.sh ${node}:/etc/keepalived/
done

启动keeplived

cd /opt/k8s/work
source /opt/k8s/bin/environment.sh
for node in ${MASTER_IPS[@]}
do
  echo ">>> ${node}"
  ssh ${node} "systemctl enable keepalived && systemctl start keepalived && systemctl status keepalived | grep active"
done

查看VIP地址

cd /opt/k8s/work
source /opt/k8s/bin/environment.sh
for node in ${MASTER_IPS[@]}
do
  echo ">>> ${node}"
  ssh ${node} "ip a | grep 20.10"
done

输出结果

[root@node01 work]# for node in ${MASTER_IPS[@]}
> do
>   echo ">>> ${node}"
>   ssh ${node} "ip a | grep 20.10"
> done
>>> 10.0.20.11
>>> 10.0.20.12
>>> 10.0.20.13
    inet 10.0.20.10/32 scope global bond0