EMQ集群搭建实现高可用和负载均衡(百万级设备连接)

一．EMQ集群搭建实现高可用和负载均衡

架构服务器规划

服务器IP	部署业务	作用
192.168.81.13	EMQTTD	EMQ集群
192.168.81.22	EMQTTD	EMQ集群
192.168.81.23	EMQTTD	EMQ集群
192.168.81.12(VIP:192.168.81.101)	haproxy、keepalived	HA和LB
192.168.81.21(VIP:192.168.81.101)	haproxy、keepalived	HA和LB

二．架构图

三．EMQ集群搭建

192.168.81.13 , 192.168.81.22 , 192.168.81.23 三台服务器作为emq集成服务器，三台都部署emqttd服务

3.1 环境安装

https://www.erlang-solutions.com/resources/download.html;https://github.com/rabbitmq/erlang-rpm/releases （erlang下载）

yum -y remove erlang

rpm -qa | grep erlang | xargs -I {} rpm -e {}

rpm -ivh erlang-21.3.8.1-1.el7.x86_64.rpm

socat安装

yum -y install socat

nginx安装

yum -y install nginx

3.2 SSL证书准备

mkdir –p /etc/nginx/ssl

mkdir –p /etc/nginx/ssl/ca

mkdir –p /etc/nginx/ssl/server

mkdir –p /etc/nginx/ssl/client

mkdir –p /etc/nginx/ssl/certs

mkdir –p /etc/nginx/ssl/crl

touch /etc/nginx/ssl/index.txt

cat <<EOF >/etc/nginx/ssl/serial

EOF

cat <<EOF >/etc/nginx/ssl/crlnumber

EOF

mkdir -p /etc/nginx/ssl/newcerts/server

生成ca证书:

openssl genrsa -out /etc/nginx/ssl/ca/ca.key 1024

openssl req -out /etc/nginx/ssl/ca/ca.req -key /etc/nginx/ssl/ca/ca.key -new -subj "/C=CN/ST=GuangDong/L=ShenZhen/O=Xxxxxx/OU=Xxxxxx/CN=xxxxxx/email=info@xxxxxx.com"

openssl x509 -req -in /etc/nginx/ssl/ca/ca.req -out /etc/nginx/ssl/ca/ca.crt -sha1 -days 5000 -signkey /etc/nginx/ssl/ca/ca.key

rm -f /etc/nginx/ssl/ca/ca.req

生成server服务端证书:

openssl genrsa -out /etc/nginx/ssl/server/dev.xxxxxx.com.key 1024

openssl req -out /etc/nginx/ssl/server/dev.xxxxxx.com.req -key /etc/nginx/ssl/server/dev.xxxxxx.com.key -new -subj "/C=CN/ST=GuangDong/L=ShenZhen/O=Xxxxxx/OU=Medc IoT/CN=dev.xxxxxx.com/Email=info@xxxxxx.com"

openssl x509 -req -in /etc/nginx/ssl/server/dev.xxxxxx.com.req -out /etc/nginx/ssl/server/dev.xxxxxx.com.crt -sha1 -CAcreateserial -days 5000 -CA /etc/nginx/ssl/ca/ca.crt -CAkey /etc/nginx/ssl/ca/ca.key

rm -f /etc/nginx/ssl/server/dev.xxxxxx.com.req

生成client客户端证书（若有先吊销然后再生成）:

openssl genrsa -out /etc/nginx/ssl/client/client.key 1024

openssl req -out /etc/nginx/ssl/client/client.req -key /etc/nginx/ssl/client/client.key -new -subj "/C=CN/ST=GuangDong/L=ShenZhen/O=Xxxxxx/OU=Medc IoT/Email=info@xxxxxx.com"

openssl x509 -req -in /etc/nginx/ssl/client/client.req -out /etc/nginx/ssl/client/client.crt -sha1 -CAcreateserial -days 5000 -CA /etc/nginx/ssl/ca/ca.crt -CAkey /etc/nginx/ssl/ca/ca.key

rm -f /etc/nginx/ssl/client/client.req

3.3 修改openssl配置文件(centos7)

sed -i "s//etc/pki/CA//etc/nginx/ssl/g" /etc/pki/tls/openssl.cnf

sed -i "s/cacert.pem/ca/ca.crt/g" /etc/pki/tls/openssl.cnf

sed -i "s/private/cakey.pem/ca/ca.key/g" /etc/pki/tls/openssl.cnf

sed -i "s/private/.rand/ca/.rand/g" /etc/pki/tls/openssl.cnf

3.4 准备nginx的配置文件

配置nginx

cat <<EOF >/etc/nginx/nginx.conf

# For more information on configuration, see:

# * Official English Documentation: http://nginx.org/en/docs/

# * Official Russian Documentation: http://nginx.org/ru/docs/

user nginx;

worker_processes auto;

error_log /var/log/nginx/error.log;

pid /run/nginx.pid;

# Load dynamic modules. See /usr/share/nginx/README.dynamic.

include /usr/share/nginx/modules/*.conf;

events {

worker_connections 1024;

}

http {

log_format main '$remote_addr - $remote_user [$time_local] "$request" '

'$status $body_bytes_sent "$http_referer" '

'"$http_user_agent" "$http_x_forwarded_for"';

access_log /var/log/nginx/access.log main;

sendfile on;

tcp_nopush on;

tcp_nodelay on;

keepalive_timeout 65;

types_hash_max_size 2048;

include /etc/nginx/mime.types;

default_type application/octet-stream;

include /etc/nginx/conf.d/http/*.conf;

}

stream {

include /etc/nginx/conf.d/tcp/*.conf;

}

EOF

新建相关文件夹

mkdir -p /etc/nginx/conf.d/{http,tcp}

cat <<EOF >/etc/nginx/conf.d/http/80.conf

upstream xxxxxx{

ip_hash;

server ${gateway_ip_address}:8800; #网关微服务地址和端口号

}

server {

error_log /var/log/nginx/80_error.log;

listen 80;

server_name xxxxxx.com;

location / {

index index.html index.htm;

root /usr/share/nginx/html/dist;

try_files $uri $uri/ /index.html;

}

location /api/ {

add_header 'Access-Control-Allow-Origin' '*';

add_header 'Access-Control-Allow-Credentials' 'true';

add_header 'Access-Control-Allow-Methods' '*';

add_header 'Access-Control-Max-Age' '18000L';

add_header 'Access-Control-Allow-Headers' '*';

client_max_body_size 1000m;

proxy_http_version 1.1;

proxy_set_header Upgrade $http_upgrade;

proxy_set_header Connection "upgrade";

proxy_set_header Host $host;

proxy_set_header X-Real-IP $remote_addr;

proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;

proxy_pass http://xxxxxx/;

}

error_page 500 502 503 504 /50x.html;

location = /50x.html {

root html;

}

EOF

cat <<EOF >/etc/nginx/conf.d/tcp/1882.conf

server {

error_log /var/log/nginx/1882_error.log;

listen 1882;

proxy_connect_timeout 600s;

proxy_pass xxxxxx;

}

EOF

cat <<EOF >/etc/nginx/conf.d/tcp/1883.conf

upstream xxxxxx{

server ${emq_ip_address}:1885; #这个配置无论写哪个tcp的配置文件里都可以，emq的端口必须写无鉴权的

}

server {

error_log /var/log/nginx/1883_error.log;

listen 1883 ssl;

proxy_connect_timeout 600s;

#proxy_timeout 3s;

proxy_pass xxxxxx;

ssl_certificate /etc/nginx/ssl/server/dev.xxxxxx.com.crt;

ssl_certificate_key /etc/nginx/ssl/server/dev.xxxxxx.com.key;

ssl_protocols SSLv3 TLSv1 TLSv1.1 TLSv1.2;

ssl_ciphers HIGH:!aNULL:!MD5;

ssl_session_cache shared:SSL:20m;

ssl_session_timeout 4h;

ssl_handshake_timeout 30s;

}

EOF

cat <<EOF >/etc/nginx/conf.d/tcp/1884.conf

server {

error_log /var/log/nginx/1884_error.log;

listen 1884 ssl;

proxy_connect_timeout 600s;

#proxy_timeout 3s;

proxy_pass xxxxxx;

ssl_verify_client on;

ssl_client_certificate /etc/nginx/ssl/ca/ca.crt;

ssl_certificate /etc/nginx/ssl/server/dev.xxxxxx.com.crt;

ssl_certificate_key /etc/nginx/ssl/server/dev.xxxxxx.com.key;

ssl_protocols SSLv3 TLSv1 TLSv1.1 TLSv1.2;

ssl_ciphers HIGH:!aNULL:!MD5;

ssl_session_cache shared:SSL:20m;

ssl_session_timeout 4h;

ssl_handshake_timeout 30s;

}

EOF

3.5 修改emq的配置文件

修改插件配置文件，更改域名地址

sed -i "s/test.xxxxxx.com/dev.xxxxxx.com/g" /usr/local/src/emqx-rel/deps/emqx_plugin_template/src/emqx_plugin_template.erl

cd /usr/local/src/emqx-rel/

重新编译，此操作会重新生成配置文件

make -C /usr/local/src/emqx-rel

修改emq配置文件，开启双向认证

sed -i "/^node.name = emqx@/cnode.name = emqx@192.168.81.XX" /usr/local/src/emqx-rel/_rel/emqx/etc/emqx.conf

配置端口号和证书路径

sed -i "/^listener.tcp.external =/clistener.tcp.external = 0.0.0.0:1885" /usr/local/src/emqx-rel/_rel/emqx/etc/emqx.conf

sed -i "/^{deny, all, subscribe, /c%%{deny, all, subscribe, ["$SYS/#", {eq, "#"}]}." /usr/local/src/emqx-rel/_rel/emqx/etc/acl.conf

sed -i"/^cluster.discovery=/ccluster.discovery=static"/usr/local/src/emqx-rel/emqx/etc/emqx.conf

sed-i"/^##cluster.static.seeds=/ccluster.static.seeds=emqx@192.168.81.13,emqx@192.168.81.22,emqx@192.168.81.23" /usr/local/src/emqx-rel/_rel/emqx/etc/emqx.conf

启动emq

/usr/local/src/emqx-rel/_rel/emqx/bin/emqx start

启动nginx

/usr/sbin/nginx -c /etc/nginx/nginx.conf

查看集群状态：

[root@dev-13 bin]# ./emqx_ctl status

Node 'emqx@192.168.81.13' is started

emqx a2a8e428 is running

[root@dev-13 bin]# ./emqx_ctl cluster status

Cluster status: [{running_nodes,['emqx@192.168.81.22','emqx@192.168.81.23',

'emqx@192.168.81.13']}]

3.6 Liunx和Erlang虚拟机调优

Linux 系统参数优化

修改系统所有进程可打开的文件数量:

vim /etc/sysctl.conf

fs.file-max = 2097152

fs.nr_open = 2097152

### backlog - Socket 监听队列长度:

net.core.somaxconn=32768

net.ipv4.tcp_max_syn_backlog=16384

net.core.netdev_max_backlog=16384

## 可用知名端口范围:

net.ipv4.ip_local_port_range='1000 65535'

## TCP Socket 读写 Buffer 设置:

net.core.rmem_default=262144

net.core.wmem_default=262144

net.core.rmem_max=16777216

net.core.wmem_max=16777216

net.core.optmem_max=16777216

#sysctl -w net.ipv4.tcp_mem='16777216 16777216 16777216'

net.ipv4.tcp_rmem='1024 4096 16777216'

net.ipv4.tcp_wmem='1024 4096 16777216'

## TCP 连接追踪设置:

net.nf_conntrack_max=1000000

net.netfilter.nf_conntrack_max=1000000

net.netfilter.nf_conntrack_tcp_timeout_time_wait=30

## FIN-WAIT-2 Socket 超时设置:

net.ipv4.tcp_fin_timeout = 15

## TIME-WAIT Socket 最大数量、回收与重用设置:

net.ipv4.tcp_max_tw_buckets=1048576

# 注意: 不建议开启該设置，NAT模式下可能引起连接RST

# net.ipv4.tcp_tw_recycle = 1

# net.ipv4.tcp_tw_reuse = 1

设置服务最大文件句柄数:

vim /etc/systemd/system.conf

DefaultLimitNOFILE=1048576

持久化设置允许用户/进程打开文件句柄数:

vim /etc/security/limits.conf

* soft nofile 1048576

* hard nofile 1048576

Erlang 虚拟机参数:

vim /usr/local/emqttd/etc/emq.conf

## Erlang Process Limit

node.process_limit = 2097152

## Sets the maximum number of simultaneously existing ports for this system

node.max_ports = 1048576

## EMQ 最大允许连接数

## Size of acceptor pool

listener.tcp.external.acceptors = 64

## Maximum number of concurrent clients(以1G内存比5W进行配置)

listener.tcp.external.max_clients = 1000000

四．haproxy部署

192.168.81.12和192.168.81.21服务器部署和配置一样

4.1 haproxy安装

yum install -y pcre-devel bzip2-devel gcc gcc-c++ make

tar –zxvf haproxy-1.8.26.tar.gz

cd haproxy-1.8.26

make TARGET=linux2628 PREFIX=/usr/local/haproxy

make install PREFIX=/usr/local/haproxy

/usr/local/haproxy/sbin/haproxy -v

HA-Proxy version 1.8.26 2020/08/03

mkdir /etc/haproxy

groupadd haproxy

useradd -s /sbin/nologin -M -g haproxy haproxy //添加haproxy运行haproxy账号并设置及属主与属组

cp examples/haproxy.init /etc/init.d/haproxy

chmod 755 /etc/init.d/haproxy

chkconfig --add haproxy

cp /usr/local/haproxy/sbin/haproxy /usr/sbin/

4.2 增加配置文件

cat <<EOF>>/etc/haproxy/haproxy.cfg

#---------------------------------------------------------------------

# common defaults that all the 'listen' and 'backend' sections will

# use if not designated in their block

#---------------------------------------------------------------------

defaults

log global

option dontlognull

option http-server-close

# option forwardfor

retries 3

timeout http-request 10s

timeout queue 1m

timeout connect 60s

timeout client 2m

timeout server 2m

timeout http-keep-alive 10s

timeout check 10s

frontend emqtt-front

bind *:1885

maxconn 1000000

mode tcp

default_backend emqtt-backend

backend emqtt-backend

balance roundrobin

# balance source

server emq1 192.168.81.13:1885 check inter 100000 fall 2 rise 5 weight 1

server emq2 192.168.81.22:1885 check inter 100000 fall 2 rise 5 weight 1

server emq3 192.168.81.23:1885 check inter 100000 fall 2 rise 5 weight 1

# source 0.0.0.0 usesrc clientip

frontend emqtt-admin-front

bind *:18083

mode http

default_backend emqtt-admin-backend

backend emqtt-admin-backend

mode http

balance roundrobin

server emq1 192.168.81.13:18083 check

server emq2 192.168.81.22:18083 check

server emq3 192.168.81.23:18083 check

listen admin_stats

stats enable

bind *:8080

mode http

option httplog

log global

maxconn 10

stats refresh 30s

stats uri /admin

stats realm haproxy

stats auth admin:admin

stats hide-version

stats admin if TRUE

4.3 启动haproxy

systemctl start haproxy

chkconfig haproxy on

五．keepalived部署

192.168.81.12和192.168.81.21服务器部署和配置略有差别

5.1 keepalived安装

yum –y Install keepalived

5.2 增加配置文件

! Configuration File for keepalived

global_defs {

router_id mqtt81

vrrp_skip_check_adv_addr

vrrp_garp_interval 0

vrrp_gna_interval 0

}

vrrp_instance VI_1 {

state BACKUP

interface ens192

virtual_router_id 81

nopreempt

priority 100

advert_int 1

authentication {

auth_type PASS

auth_pass 1111

}

mcast_src_ip 192.168.81.12

virtual_ipaddress {

192.168.81.101/24

}

5.3 启动keepalived并设置开机自启动

systemctl start keepalived

systemctl enable keepalived

六．架构测试

6.1 keepalived架构测试

关闭192.168.81.12或者192.168.81.21其中一台，查看服务是否还可以正常访问

6.2 haproxy负载均衡

客户端工具发起pub，看是否按照轮询方式发送到后端3台EMQ服务上面。