一、安装环境、主机信息及软件版本
Nginx：1.12.2
keepalived:2.0.12
时间同步(同步后确认各服务器时间是否一致，不一致需要修改一下时区)
关闭防火墙
二、编译安装Nginx
1.编译安装Nginx
[root@k8s-node-207 ~]# wget http://nginx.org/download/nginx-1.12.2.tar.gz
[root@k8s-node-207 ~]# yum -y install gcc gcc-c++ automake pcre pcre-devel zlib zlib-devel open openssl-devel #安装相关依赖包
[root@k8s-node-207 ~]# tar xf nginx-1.12.2.tar.gz
[root@k8s-node-207 ~]# cd nginx-1.12.2/
[root@k8s-node-207 nginx-1.12.2]# ./configure --prefix=/usr/local/nginx --with-http_ssl_module --with-http_stub_status_module --with-http_realip_module --with-threads --with-pcre --with-http_gzip_static_module #设置编译参数，特别是--prefix
[root@k8s-node-207 nginx-1.12.2]# make && make install
[root@k8s-node-207 nginx-1.12.2]# scp -r /usr/local/nginx 172.16.155.208:/usr/local/ #拷贝到另外一台Nginx服务器
2.配置Nginx为系统服务
[root@k8s-node-207 nginx-1.12.2]# vim /lib/systemd/system/nginx.service #创建Nginx服务系统启动文件
[Unit]
Description=nginx
After=network.target
[Service]
Type=forking
ExecStart=/usr/local/nginx/sbin/nginx #注意要替换为自己编译安装的路径
ExecReload=/usr/local/nginx/sbin/nginx -s reload
ExecStop=/usr/local/nginx/sbin/nginx -s quit
PrivateTmp=true
[Install]
WantedBy=multi-user.target
[root@k8s-node-207 nginx-1.12.2]# systemctl start nginx #测试脚本，启动
[root@k8s-node-207 nginx-1.12.2]# netstat -tnlp|grep :80 #检查端口
tcp 0 0 0.0.0.0:80 0.0.0.0:* LISTEN 7834/nginx: master
[root@k8s-node-207 nginx-1.12.2]# ps aux|grep nginx #检查进程
root 7834 0.0 0.0 45956 1124 ? Ss 16:24 0:00 nginx: master process /usr/local/nginx/sbin/nginx
nobody 7835 0.0 0.0 48484 1976 ? S 16:24 0:00 nginx: worker process
root 7944 0.0 0.0 112720 968 pts/0 S+ 16:25 0:00 grep -E --color=auto nginx
[root@k8s-node-207 nginx-1.12.2]# systemctl enable nginx #设置开机自启
[root@k8s-node-207 nginx-1.12.2]# scp /lib/systemd/system/nginx.service 172.16.155.208:/lib/systemd/system/nginx.service #拷贝到另一台机器
#在另一台机器上启动Nginx
[root@k8s-node-208 ~]# systemctl start nginx
[root@k8s-node-208 ~]# systemctl status nginx
[root@k8s-node-208 ~]# systemctl enable nginx
三、编译安装配置keepalived
1.编译安装keepalived
[root@k8s-node-207 ~]# wget https://www.keepalived.org/software/keepalived-2.0.12.tar.gz #下载源码
[root@k8s-node-207 ~]# yum install -y openssl openssl-devel libnl libnl-devel #安装依赖文件
[root@k8s-node-207 ~]# tar xf keepalived-2.0.12.tar.gz
[root@k8s-node-207 ~]# cd keepalived-2.0.12/
[root@k8s-node-207 keepalived-2.0.12]# ./configure --prefix=/usr/local/keepalived #只有编译安装的目录
[root@k8s-node-207 keepalived-2.0.12]# make && make install
2.配置keepalived master节点
[root@k8s-node-207 keepalived-2.0.12]# cd /usr/local/keepalived/etc/keepalived/ #进入keepalived配置目录 [root@k8s-node-207 keepalived]# cp keepalived.conf keepalived.conf-$(date +%F-%H:%M:%S) #拷贝默认的配置 ! Configuration File for keepalived global_defs { #默认该字段是配置发送邮件通知，由于我使用微信进行通知所以忽略 } vrrp_script chk_http_port { #配置服务的健康检查 script "/mnt/chk_nginx.sh" #检查是使用的脚本路径 interval 2 #监控间隔 weight -5 #每检测失败一次，如果weight大于0则当前节点的priority增加该配置的值，否则减少 fall 2 #执行几次才会认为是失败 rise 1 #执行多少次才会认为是成功 } vrrp_instance VI_1 { state MASTER #指定当前节点的初始状态 interface eth0 #vrrp实例绑定的网卡接口 用于发送vrrp包
nopreempt #设置为非抢占模式，优先级高的设置 解决优先级高的恢复后再次抢断 测试时可以先不配置 virtual_router_id 51 #指定vrrp实例的ID 范围是0-255 主备节点必须一致 priority 101 #指定当前节点的优先级 优先级高的为MASTER advert_int 1 #指定发送vrrp间隔时间 主备必须一致 authentication { #主备必须一致 auth_type PASS #指定认证方式 这里使用简单密码认证 auth_pass 1111 #指定认证使用的密码 最大为8位 } virtual_ipaddress { 172.16.155.209 #指定VIP地址 } notify_master "/root/script_dir/wechat.py master test keepalived状态发送改变,master切换至172.16.155.207" #设置通知脚本路径及通知信息 此处根据自己实际情况自定义，此处配置不影响启动，如果没有告警通知可以稍后创建 notify_backup "/root/script_dir/wechat.py backup test keepalived状态发送改变,backup切换至172.16.168.207" notify_fault "/root/script_dir/wechat.py fault test keepalived发送故障,故障主机为：172.16.168.207" #以上配置为：当节点成为master时执行的操作 成为backup是执行的操作 当发生故障时执行的操作 track_script { #监控脚本执行的状态 chk_http_port } } [root@k8s-node-207 keepalived]# cd /usr/local/keepalived/etc/sysconfig/ #由于启动命令默认会去/etc/keepalived/下读取keepalived.conf，并且默认日志会写入/var/log/messages文件中，所以我们需要修改相关配置 [root@k8s-node-207 sysconfig]# cat keepalived KEEPALIVED_OPTIONS="-f /usr/local/keepalived/etc/keepalived/keepalived.conf -S 0 -D" #-f 指定配置文件路径 -S 指定日志路径 0 表示local0.* [root@k8s-node-207 sysconfig]# cat /etc/rsyslog.conf #修改rsyslog配置文件 #Save keepalived log local0.* /var/log/keepalived.log #指定日志文件路径
#暂时不拷贝至另一台服务器
3.配置keepalived backup节点
#207上拷贝相关文件到208上
[root@k8s-node-207 sysconfig]# scp -r /usr/local/keepalived/ 172.16.155.208:/usr/local/ [root@k8s-node-207 sysconfig]# scp /etc/rsyslog.conf 172.16.155.208:/etc/rsyslog.conf
[root@k8s-node-207 sysconfig]# scp /lib/systemd/system/keepalived.service 172.16.155.208:/lib/systemd/system/keepalived.service #默认编译keepalived时会自动生成系统服务配置文件
[root@k8s-node-207 sysconfig]# systemctl restart rsyslog.service
#208上修改配置文件为backup
[root@k8s-node-208 ~]# cd /usr/local/keepalived/etc/keepalived/
[root@k8s-node-208 keepalived]# vim keepalived.conf
! Configuration File for keepalived
global_defs { #可自定义
}
vrrp_script chk_http_port { #可自定义
script "/mnt/chk_nginx.sh"
interval 2
weight -5
fall 2
rise 1
}
vrrp_instance VI_1 {
state backup #指定当前节点状态
interface eth0
nopreempt
virtual_router_id 51 #确保与master保持一致
priority 100 #确保小于master设置的值
advert_int 1 #确保与master保持一致
authentication { #确保与master保持一致
auth_type PASS
auth_pass 1111
}
virtual_ipaddress { #确保与master保持一致
172.16.155.209
}
notify_master "/root/script_dir/wechat.py master test keepalived状态发送改变,master切换至172.16.155.208" #可自定义
notify_backup "/root/script_dir/wechat.py backup test keepalived状态发送改变,backup切换至172.16.168.208"
notify_fault "/root/script_dir/wechat.py fault test keepalived发送故障,故障主机为：172.16.168.208"
track_script { #可自定义
chk_http_port
}
}
默认编译后会自动生成系统服务配置文件，如果没有参考以下配置创建
[root@k8s-node-207 sysconfig]# vim /lib/systemd/system/keepalived.service
[Unit]
Description=LVS and VRRP High Availability Monitor
After= network-online.target syslog.target
Wants=network-online.target
[Service]
Type=forking
PIDFile=/var/run/keepalived.pid
KillMode=process
EnvironmentFile=-/usr/local/keepalived/etc/sysconfig/keepalived
ExecStart=/usr/local/keepalived/sbin/keepalived $KEEPALIVED_OPTIONS
ExecReload=/bin/kill -HUP $MAINPID
[Install]
WantedBy=multi-user.target
创建Nginx状态检查脚本
[root@k8s-node-207 sysconfig]# cat /mnt/chk_nginx.sh
#!/bin/bash
ngix_status=`ps -C nginx --no-header |wc -l`
if [[ ${ngix_status} -eq 0 ]];then
/usr/local/nginx/sbin/nginx
sleep 2
new_nginx_status=$(ps -C nginx --no-header |wc -l)
if [[ ${new_nginx_status} -eq 0 ]];then
killall keepalived
fi
fi
[root@k8s-node-207 sysconfig]# scp /mnt/chk_nginx.sh 172.16.155.208:/mnt/chk_nginx.sh
[root@k8s-node-207 sysconfig]# chmod +x /mnt/chk_nginx.sh
[root@k8s-node-207 sysconfig]# ssh 172.16.155.208 "chmod +x /mnt/chk_nginx.sh"
4.启动服务并检查当前状态
[root@k8s-node-207 sysconfig]# systemctl restart rsyslog.service #重启rsyslog服务，使keepalived的日志配置生效
[root@k8s-node-207 sysconfig]# echo "k8s-node-207" > /usr/local/nginx/html/index.html #修改Nginx index文件 稍后测试使用
[root@k8s-node-207 sysconfig]# systemctl start keepalived.service #启动keepalived服务
#208上 同207操作
[root@k8s-node-208 keepalived]# systemctl restart rsyslog.service
[root@k8s-node-208 keepalived]# echo "k8s-node-208" > /usr/local/nginx/html/index.html
[root@k8s-node-208 keepalived]# systemctl start keepalived.service
[root@k8s-node-207 sysconfig]# ip a|grep 172.16 #207上检查当前的IP地址
inet 172.16.155.207/24 brd 172.16.155.255 scope global eth0
inet 172.16.155.209/32 scope global eth0 #VIP地址当前绑定在当前主机eth0网卡上
[root@k8s-node-208 keepalived]# ip a|grep 172.16 #208上检查当前用有的IP地址 主要检查是否发生脑裂问题
inet 172.16.155.208/24 brd 172.16.155.255 scope global eth0
[root@k8s-node-207 sysconfig]# curl http://172.16.155.209:80 #使用crul命令范围VIP查看访问到哪个节点 显示为207节点
k8s-node-207
5.测试keepalived主从是否可以切换
[root@k8s-node-207 sysconfig]# systemctl stop keepalived.service #在207上关闭keepalived服务
[root@k8s-node-207 sysconfig]# ip a|grep 172.16 #查看207上的拥有的IP地址 VIP已经没有了
inet 172.16.155.207/24 brd 172.16.155.255 scope global eth0
[root@k8s-node-208 keepalived]# ip a|grep 172.16 #查看208上的拥有的IP地址 VIP已经漂移到208上了
inet 172.16.155.208/24 brd 172.16.155.255 scope global eth0
inet 172.16.155.209/32 scope global eth0
[root@k8s-node-207 sysconfig]# curl http://172.16.155.209:80 #访问一下VIP 显示为208节点
k8s-node-208
[root@k8s-node-207 sysconfig]# systemctl start keepalived.service #207上重新启动keepalived
[root@k8s-node-207 sysconfig]# ip a|grep 172.16 #VIP没有漂移回来，符合预期，如果想在master恢复后VIP漂移回来则可以删除nopreempt配置
inet 172.16.155.207/24 brd 172.16.155.255 scope global eth0
5.测试Nginx服务故障时keepalived是否可以切换
[root@k8s-node-208 mnt]# ip a|grep 172.16 #当前VIP在208上
inet 172.16.155.208/24 brd 172.16.155.255 scope global eth0
inet 172.16.155.209/32 scope global eth0
[root@k8s-node-208 mnt]# systemctl stop nginx #关闭208上Nginx服务 稍等几秒钟
[root@k8s-node-208 mnt]# ip a|grep 172.16 #再次检查208上的IP，VIP已经没有了
inet 172.16.155.208/24 brd 172.16.155.255 scope global eth0
[root@k8s-node-207 mnt]# ip a|grep 172.16 #207上查看，VIP已经漂移到207上了
inet 172.16.155.207/24 brd 172.16.155.255 scope global eth0
inet 172.16.155.209/32 scope global eth0