Nginx+Keepalived实现负载均衡的高可用

一、环境说明

系统：Centos 7.6

主机：四台，负载均衡两台：node1(192.168.145.27/24)，node2(192.168.145.37/24)；后端服务器两台：real1(192.168.145.47/24)，real2(192.168.145.57/24)

VIP：192.168.145.100

软件：nginx-1.18.0.tar.gz，keepalived-1.3.5(光盘yum源安装）

二、实现过程

2.1、Nginx安装与配置

(1) Nginx的安装

　两台主机都要安装，以node1为例。

[root@node1 ~]# wget https://nginx.org/download/nginx-1.18.0.tar.gz
[root@node1 ~]# useradd -r -s /sbin/nologin nginx
[root@node1 ~]# tar -xf nginx-1.18.0.tar.gz 
[root@node1 ~]# cd nginx-1.18.0/
[root@node1 nginx-1.18.0]# yum install -y gcc pcre-devel openssl-devel zlib-devel
[root@node1 nginx-1.18.0]# ./configure --prefix=/usr/local/nginx 
> --user=nginx 
> --group=nginx 
> --with-http_ssl_module 
> --with-http_v2_module 
> --with-http_realip_module 
> --with-http_stub_status_module 
> --with-http_gzip_static_module 
> --with-pcre 
> --with-stream 
> --with-stream_ssl_module 
> --with-stream_realip_module
[root@node1 nginx-1.18.0]# make && make install
[root@node1 nginx-1.18.0]# ln -s /usr/local/nginx/sbin/nginx /usr/sbin/
[root@node1 nginx-1.18.0]# nginx

(2) 修改nginx配置文件，两台配置一样

[root@node1 ~]# grep -Ev "^$|#" /usr/local/nginx/conf/nginx.conf
user nginx nginx;
worker_processes  1;
events {
    worker_connections  1024;
}
http {
    include       mime.types;
    default_type  application/octet-stream;
    log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
                      '$status $body_bytes_sent "$http_referer" '
                      '"$http_user_agent" "$http_x_forwarded_for"';
    sendfile        on;
    keepalive_timeout  65;
    upstream web {
        server 192.168.145.47:80 weight=1 fail_timeout=5s max_fails=3;
        server 192.168.145.57:80 weight=1 fail_timeout=5s max_fails=3;
    }
    server {
        listen 80;
        server_name  www.aaa.com;
        charset utf-8;
        access_log  logs/access_www.log  main;
        location / {
            proxy_pass http://web;
            proxy_set_header Host $host;
            proxy_set_header X-Forwarded-For $remote_addr;
        }
    }
}

(3) 修改内核参数，让nginx可以绑定在vip上

#不修改内核参数，nginx会报以下错误：
[root@node1 ~]# nginx -t
nginx: the configuration file /usr/local/nginx/conf/nginx.conf syntax is ok
nginx: [emerg] bind() to 192.168.145.100:80 failed (99: Cannot assign requested address)
nginx: configuration file /usr/local/nginx/conf/nginx.conf test failed
[root@node1 ~]# cat >> /etc/sysctl.conf <<EOF
> net.ipv4.ip_nonlocal_bind = 1
> net.ipv4.ip_forward = 1
> EOF
[root@node1 ~]# sysctl -p
net.ipv4.ip_nonlocal_bind = 1
net.ipv4.ip_forward = 1
[root@node1 ~]# nginx -t
nginx: the configuration file /usr/local/nginx/conf/nginx.conf syntax is ok
nginx: configuration file /usr/local/nginx/conf/nginx.conf test is successful
[root@node1 ~]# nginx -s reload

(4) 修改两台主机的/etc/hosts文件，使其能够解析www.aaa.com域名

[root@node1 ~]# vim /etc/hosts
#添加以下内容
192.168.145.100 www.aaa.com

(5) 后端服务器配置访问页面

[root@real1 ~]# yum install -y httpd
[root@real1 ~]# echo "192.168.145.47" > /var/www/html/index.html
[root@real1 ~]# systemctl start httpd
[root@real2 ~]# yum install -y httpd
[root@real2 ~]# echo "192.168.145.57" > /var/www/html/index.htm
[root@real2 ~]# systemctl start httpd

2.2、Keepalived的安装与配置

(1) Keepalived的安装

　两台主机都要安装，使用yum源的安装方式。

[root@node1 ~]# yum install -y keepalived
[root@node2 ~]# yum install -y keepalived

(2) 修改keepalived的配置文件

[root@node1 ~]# vim /etc/keepalived/keepalived.conf 
global_defs {
   notification_email {
     root@localhost
   }
   notification_email_from root@localhost
   smtp_server 127.0.0.1
   smtp_connect_timeout 30
   router_id node1
   vrrp_skip_check_adv_addr
   vrrp_strict
   vrrp_iptables
   vrrp_garp_interval 0
   vrrp_gna_interval 0
}

vrrp_script check_nginx {
    script "/etc/keepalived/check_nginx.sh"
}

vrrp_instance VI_1 {
    state MASTER
    interface eth0
    virtual_router_id 51
    priority 100
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 1111
    }
    virtual_ipaddress {
        192.168.145.100/24 dev eth0 label eth0:1
    }
    track_script {
        check_nginx
    }
}

[root@node2 ~]# vim /etc/keepalived/keepalived.conf 
global_defs {
   notification_email {
     root@localhost
   }
   notification_email_from root@localhost
   smtp_server 127.0.0.1
   smtp_connect_timeout 30
   router_id node2
   vrrp_skip_check_adv_addr
   vrrp_strict
   vrrp_iptables
   vrrp_garp_interval 0
   vrrp_gna_interval 0
}

vrrp_script check_nginx {
    script "/etc/keepalived/check_nginx.sh"
}

vrrp_instance VI_1 {
    state BACKUP
    interface eth0
    virtual_router_id 51
    priority 80
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 1111
    }
    virtual_ipaddress {
        192.168.145.100/24 dev eth0 label eth0:1
    }
    track_script {
        check_nginx
    }
}

(3) 编写监测nginx的脚本

[root@node1 ~]# vim /etc/keepalived/check_nginx.sh
#!/bin/bash

if [ `ps -C nginx --no-header|wc -l` -eq 0 ];then
   /usr/local/nginx/sbin/nginx        #尝试重新启动nginx
   sleep 2
   if [ `ps -C nginx --no-header|wc -l` -eq 0 ];then
       /usr/bin/systemctl stop keepalived        #启动失败，将keepalived服务关闭，让vip漂移到其它节点
       #killall keepalived
   fi
fi
[root@node1 ~]# chmod +x /etc/keepalived/check_nginx.sh
[root@node1 ~]# scp /etc/keepalived/check_nginx.sh 192.168.145.37:/etc/keepalived/check_nginx.sh