生产环境中zookeeper分布式集群部署实战案例
作者：尹正杰
版权声明：原创作品，谢绝转载！否则将追究法律责任。
一.zookeeper概述
1>.访问zookeeper的官网
官方地址: https://zookeeper.apache.org/
2>.选择zookeeper的版本进行下载
二.本地模式部署
1>.部署JDK
博主推荐阅读: https://www.cnblogs.com/yinzhengjie/p/12199413.html
2>.将下载的zookeeper软件包并解压到指定路径
[root@hadoop101.yinzhengjie.org.cn ~]# ll total 12148 -rw-r--r-- 1 root root 12436328 May 12 05:50 apache-zookeeper-3.6.1-bin.tar.gz [root@hadoop101.yinzhengjie.org.cn ~]# [root@hadoop101.yinzhengjie.org.cn ~]# tar -zxf apache-zookeeper-3.6.1-bin.tar.gz -C /yinzhengjie/softwares/ [root@hadoop101.yinzhengjie.org.cn ~]# [root@hadoop101.yinzhengjie.org.cn ~]# ll /yinzhengjie/softwares/ total 0 drwxr-xr-x 6 root root 133 May 12 05:54 apache-zookeeper-3.6.1-bin drwxr-xr-x 11 root root 173 Mar 12 01:18 fully-mode drwxr-xr-x 12 root root 184 Mar 12 04:35 hadoop-2.10.0 drwxr-xr-x 7 10 143 245 Dec 16 2018 jdk1.8.0_201 drwxr-xr-x 9 root root 149 Mar 10 23:38 local-mode drwxr-xr-x 11 root root 173 Mar 11 01:15 pseudo-mode [root@hadoop101.yinzhengjie.org.cn ~]#
[root@hadoop101.yinzhengjie.org.cn ~]# ll /yinzhengjie/softwares/apache-zookeeper-3.6.1-bin/ total 32 drwxr-xr-x 2 yinzhengjie yinzhengjie 289 Apr 21 22:59 bin drwxr-xr-x 2 yinzhengjie yinzhengjie 77 Apr 21 22:59 conf drwxr-xr-x 5 yinzhengjie yinzhengjie 4096 Apr 21 23:00 docs drwxr-xr-x 2 root root 4096 May 12 05:54 lib -rw-r--r-- 1 yinzhengjie yinzhengjie 11358 Apr 21 22:59 LICENSE.txt -rw-r--r-- 1 yinzhengjie yinzhengjie 432 Apr 21 22:59 NOTICE.txt -rw-r--r-- 1 yinzhengjie yinzhengjie 1963 Apr 21 22:59 README.md -rw-r--r-- 1 yinzhengjie yinzhengjie 3166 Apr 21 22:59 README_packaging.md [root@hadoop101.yinzhengjie.org.cn ~]#
3>.修改配置文件
[root@hadoop101.yinzhengjie.org.cn ~]# cd /yinzhengjie/softwares/apache-zookeeper-3.6.1-bin/conf/ [root@hadoop101.yinzhengjie.org.cn /yinzhengjie/softwares/apache-zookeeper-3.6.1-bin/conf]# [root@hadoop101.yinzhengjie.org.cn /yinzhengjie/softwares/apache-zookeeper-3.6.1-bin/conf]# ll total 12 -rw-r--r-- 1 yinzhengjie yinzhengjie 535 Apr 21 22:59 configuration.xsl -rw-r--r-- 1 yinzhengjie yinzhengjie 3435 Apr 21 22:59 log4j.properties -rw-r--r-- 1 yinzhengjie yinzhengjie 1148 Apr 21 22:59 zoo_sample.cfg [root@hadoop101.yinzhengjie.org.cn /yinzhengjie/softwares/apache-zookeeper-3.6.1-bin/conf]# [root@hadoop101.yinzhengjie.org.cn /yinzhengjie/softwares/apache-zookeeper-3.6.1-bin/conf]# cp zoo_sample.cfg zoo.cfg [root@hadoop101.yinzhengjie.org.cn /yinzhengjie/softwares/apache-zookeeper-3.6.1-bin/conf]# [root@hadoop101.yinzhengjie.org.cn /yinzhengjie/softwares/apache-zookeeper-3.6.1-bin/conf]# vim zoo.cfg
[root@hadoop101.yinzhengjie.org.cn /yinzhengjie/softwares/apache-zookeeper-3.6.1-bin/conf]# cat zoo.cfg #通信心跳数，Zookeeper服务器与客户端心跳时间，单位毫秒 tickTime=2000 #集群中的Follower跟随者服务器与Leader领导者服务器之间初始连接时能容忍的最多心跳数（tickTime的数量），用它来限定集群中的Zookeeper服务器连接到Leader的时限。 initLimit=10 #集群中Leader与Follower之间的最大响应时间单位，假如响应超过syncLimit * tickTime，Leader认为Follwer死掉，从服务器列表中删除Follwer。 syncLimit=5 #主要用于保存Zookeeper中的数据。 dataDir=/yinzhengjie/data/zookeeper #监听客户端连接的端口 clientPort=2181 # the maximum number of client connections. # increase this if you need to handle more clients #maxClientCnxns=60 # # Be sure to read the maintenance section of the # administrator guide before turning on autopurge. # # http://zookeeper.apache.org/doc/current/zookeeperAdmin.html#sc_maintenance # # The number of snapshots to retain in dataDir #autopurge.snapRetainCount=3 # Purge task interval in hours # Set to "0" to disable auto purge feature #autopurge.purgeInterval=1 ## Metrics Providers # # https://prometheus.io Metrics Exporter #metricsProvider.className=org.apache.zookeeper.metrics.prometheus.PrometheusMetricsProvider #metricsProvider.httpPort=7000 #metricsProvider.exportJvmInfo=true [root@hadoop101.yinzhengjie.org.cn /yinzhengjie/softwares/apache-zookeeper-3.6.1-bin/conf]#
4>.配置zookeeper的环境变量
[root@hadoop101.yinzhengjie.org.cn ~]# ln -sv /yinzhengjie/softwares/apache-zookeeper-3.6.1-bin /yinzhengjie/softwares/zookeeper ‘/yinzhengjie/softwares/zookeeper’ -> ‘/yinzhengjie/softwares/apache-zookeeper-3.6.1-bin’ [root@hadoop101.yinzhengjie.org.cn ~]# [root@hadoop101.yinzhengjie.org.cn ~]# vim /etc/profile.d/zookeeper.sh [root@hadoop101.yinzhengjie.org.cn ~]# [root@hadoop101.yinzhengjie.org.cn ~]# cat /etc/profile.d/zookeeper.sh #!/bin/bash ZOOKEEPER=/yinzhengjie/softwares/zookeeper PATH=$PATH:$ZOOKEEPER/bin [root@hadoop101.yinzhengjie.org.cn ~]# [root@hadoop101.yinzhengjie.org.cn ~]# source /etc/profile.d/zookeeper.sh #使得环境变量生效~
5>.启动zookeeper
[root@hadoop101.yinzhengjie.org.cn ~]# zkServer.sh start #启动zookeeper服务 /yinzhengjie/softwares/jdk1.8.0_201/bin/java ZooKeeper JMX enabled by default Using config: /yinzhengjie/softwares/zookeeper/bin/../conf/zoo.cfg Starting zookeeper ... STARTED [root@hadoop101.yinzhengjie.org.cn ~]# [root@hadoop101.yinzhengjie.org.cn ~]# ll /yinzhengjie/data/ total 0 drwxr-xr-x 3 root root 51 May 12 05:59 zookeeper [root@hadoop101.yinzhengjie.org.cn ~]# [root@hadoop101.yinzhengjie.org.cn ~]# zkServer.sh status #查看zookeeper的运行状态 /yinzhengjie/softwares/jdk1.8.0_201/bin/java ZooKeeper JMX enabled by default Using config: /yinzhengjie/softwares/zookeeper/bin/../conf/zoo.cfg Client port found: 2181. Client address: localhost. Mode: standalone #很显然，现在的zookeeper并不是以集群的方式运行,而是以本地模式运行的 [root@hadoop101.yinzhengjie.org.cn ~]#
6>.验证zookeeper服务是否正常启动
[root@hadoop101.yinzhengjie.org.cn ~]# ll /yinzhengjie/data/ total 0 drwxr-xr-x 3 root root 51 May 12 05:59 zookeeper [root@hadoop101.yinzhengjie.org.cn ~]# [root@hadoop101.yinzhengjie.org.cn ~]# ll /yinzhengjie/data/zookeeper/ total 4 drwxr-xr-x 2 root root 24 May 12 05:59 version-2 -rw-r--r-- 1 root root 4 May 12 05:59 zookeeper_server.pid [root@hadoop101.yinzhengjie.org.cn ~]# [root@hadoop101.yinzhengjie.org.cn ~]# more /yinzhengjie/data/zookeeper/zookeeper_server.pid #查看当前zookeeper启动的进程ID 5944 [root@hadoop101.yinzhengjie.org.cn ~]# [root@hadoop101.yinzhengjie.org.cn ~]# jps #使用jps命令查看 5991 Jps 5944 QuorumPeerMain [root@hadoop101.yinzhengjie.org.cn ~]# [root@hadoop101.yinzhengjie.org.cn ~]# ss -ntl | grep 2181 #端口也是监听成功的~ LISTEN 0 50 *:2181 *:* [root@hadoop101.yinzhengjie.org.cn ~]#
7>.停止zookeeper服务
[root@hadoop101.yinzhengjie.org.cn ~]# jps 5944 QuorumPeerMain 15658 Jps [root@hadoop101.yinzhengjie.org.cn ~]# [root@hadoop101.yinzhengjie.org.cn ~]# zkServer.sh stop /yinzhengjie/softwares/jdk1.8.0_201/bin/java ZooKeeper JMX enabled by default Using config: /yinzhengjie/softwares/zookeeper/bin/../conf/zoo.cfg Stopping zookeeper ... STOPPED [root@hadoop101.yinzhengjie.org.cn ~]# [root@hadoop101.yinzhengjie.org.cn ~]# jps 15691 Jps [root@hadoop101.yinzhengjie.org.cn ~]# [root@hadoop101.yinzhengjie.org.cn ~]#
三.分布式集群部署
1>.实验环境说明(准备3台测试服务器,其基本配置信息如下所示)
[root@hadoop101.yinzhengjie.org.cn ~]# uname -r 3.10.0-957.el7.x86_64 [root@hadoop101.yinzhengjie.org.cn ~]# [root@hadoop101.yinzhengjie.org.cn ~]# uname -m x86_64 [root@hadoop101.yinzhengjie.org.cn ~]# [root@hadoop101.yinzhengjie.org.cn ~]# cat /etc/redhat-release CentOS Linux release 7.6.1810 (Core) [root@hadoop101.yinzhengjie.org.cn ~]# [root@hadoop101.yinzhengjie.org.cn ~]# free -h total used free shared buff/cache available Mem: 3.8G 171M 3.4G 11M 231M 3.4G Swap: 0B 0B 0B [root@hadoop101.yinzhengjie.org.cn ~]#
2>.创建配置zookeeper的堆内存配置文件
[root@hadoop101.yinzhengjie.org.cn ~]# vim $ZOOKEEPER/conf/java.env [root@hadoop101.yinzhengjie.org.cn ~]# [root@hadoop101.yinzhengjie.org.cn ~]# cat $ZOOKEEPER/conf/java.env #!/bin/bash #@author :yinzhengjie #blog:http://www.cnblogs.com/yinzhengjie #EMAIL:y1053419035@qq.com #指定JDK的安装路径 export JAVA_HOME=/yinzhengjie/softwares/jdk1.8.0_201 #指定zookeeper的heap内存大小 export JVMFLAGS="-Xms2048m -Xmx2048m $JVMFLAGS" [root@hadoop101.yinzhengjie.org.cn ~]#
3>.修改zookeeper的配置文件zoo.cfg
[root@hadoop101.yinzhengjie.org.cn ~]# vim $ZOOKEEPER/conf/zoo.cfg [root@hadoop101.yinzhengjie.org.cn ~]# [root@hadoop101.yinzhengjie.org.cn ~]# cat $ZOOKEEPER/conf/zoo.cfg # 滴答，计时的基本单位，默认是2000毫秒，即2秒。它是zookeeper最小的时间单位，用于丈量心跳时间和超时时间等，通常设置成默认2秒即可。 tickTime=2000 # 初始化限制是10滴答，默认是10个滴答，即默认是20秒。指定follower节点初始化是链接leader节点的最大tick次数。 initLimit=5 # 数据同步的时间限制，默认是5个滴答，即默认时间是10秒。设定了follower节点与leader节点进行同步的最大时间。与initLimit类似，它也是以tickTime为单位进行指定的。 syncLimit=2 # 指定zookeeper的工作目录，这是一个非常重要的参数，zookeeper会在内存中在内存只能中保存系统快照，并定期写入该路径指定的文件夹中。生产环境中需要注意该文件夹的磁盘占用情况。 dataDir=/yinzhengjie/data/zookeeper # 监听zookeeper的默认端口。zookeeper监听客户端链接的端口，一般设置成默认2181即可。 clientPort=2181 # 这个操作将限制连接到 ZooKeeper 的客户端的数量，限制并发连接的数量，它通过 IP 来区分不同的客户端。此配置选项可以用来阻止某些类别的 Dos 攻击。将它设置为 0 或者忽略而不进行设置将会取消对并发连接的限制。 #maxClientCnxns=60 # zookeeper 3.4.0及之后版本，ZK提供了自动清理事务日志和快照文件的功能，这个参数指定了清理频率，单位是小时，需要配置一个1或更大的整数，默认是0，表示不开启自动清理功能。 #autopurge.purgeInterval=1 # 这个参数和上面的参数搭配使用，这个参数指定了需要保留的文件数目。默认是保留3个。 #autopurge.snapRetainCount=3 #server.x=[hostname]:nnnnn[:nnnnn]，这里的x是一个数字，与myid文件中的id是一致的。右边可以配置两个端口，第一个端口用于F和L之间的数据同步和其它通信，第二个端口用于Leader选举过程中投票通信。 server.101=hadoop101.yinzhengjie.org.cn:2888:3888 server.102=hadoop102.yinzhengjie.org.cn:2888:3888 server.103=hadoop103.yinzhengjie.org.cn:2888:3888 [root@hadoop101.yinzhengjie.org.cn ~]#
4>.将配置文件及环境变量同步到其它节点
[root@hadoop101.yinzhengjie.org.cn ~]# vim /etc/ansible/hosts [root@hadoop101.yinzhengjie.org.cn ~]# [root@hadoop101.yinzhengjie.org.cn ~]# tail -2 /etc/ansible/hosts [zk] hadoop[101:103].yinzhengjie.org.cn [root@hadoop101.yinzhengjie.org.cn ~]#
[root@hadoop101.yinzhengjie.org.cn ~]# ansible zk -m copy -a 'src=/yinzhengjie/softwares/zookeeper dest=/yinzhengjie/softwares' #拷贝目录建议大家不要在虚拟机上执行，效率极低，但在真实的服务器上执行速度还是相当的块。 [root@hadoop101.yinzhengjie.org.cn ~]# [root@hadoop101.yinzhengjie.org.cn ~]# scp -r /yinzhengjie/softwares/zookeeper hadoop102.yinzhengjie.org.cn:/yinzhengjie/softwares #虚拟机拷贝目录推荐使用scp，或者你也基于rsync服务写一个同步脚本 [root@hadoop101.yinzhengjie.org.cn ~]# [root@hadoop101.yinzhengjie.org.cn ~]# scp -r /yinzhengjie/softwares/zookeeper hadoop102.yinzhengjie.org.cn:/yinzhengjie/softwares [root@hadoop101.yinzhengjie.org.cn ~]#
[root@hadoop101.yinzhengjie.org.cn ~]# ansible zk -m copy -a 'src=/etc/profile.d/zookeeper.sh dest=/etc/profile.d' hadoop103.yinzhengjie.org.cn | SUCCESS => { "changed": true, "checksum": "df143023ba24250e7ea17cf590dc0ff03ddf0c48", "dest": "/etc/profile.d/zookeeper.sh", "gid": 0, "group": "root", "md5sum": "2ad2824abe1539ed5c1a6fdbafcf3138", "mode": "0644", "owner": "root", "size": 82, "src": "/root/.ansible/tmp/ansible-tmp-1589236581.53-224194132507684/source", "state": "file", "uid": 0 } hadoop102.yinzhengjie.org.cn | SUCCESS => { "changed": true, "checksum": "df143023ba24250e7ea17cf590dc0ff03ddf0c48", "dest": "/etc/profile.d/zookeeper.sh", "gid": 0, "group": "root", "md5sum": "2ad2824abe1539ed5c1a6fdbafcf3138", "mode": "0644", "owner": "root", "size": 82, "src": "/root/.ansible/tmp/ansible-tmp-1589236581.53-112267348533965/source", "state": "file", "uid": 0 } hadoop101.yinzhengjie.org.cn | SUCCESS => { "changed": false, "checksum": "df143023ba24250e7ea17cf590dc0ff03ddf0c48", "gid": 0, "group": "root", "mode": "0644", "owner": "root", "path": "/etc/profile.d/zookeeper.sh", "size": 82, "state": "file", "uid": 0 } [root@hadoop101.yinzhengjie.org.cn ~]#
关于ansible的快速入门使用，博主推荐阅读: https://www.cnblogs.com/yinzhengjie/p/10447587.html
5>.编写zookeeper的启动脚本
[root@hadoop101.yinzhengjie.org.cn ~]# vim /usr/local/bin/zookeeper.sh [root@hadoop101.yinzhengjie.org.cn ~]# [root@hadoop101.yinzhengjie.org.cn ~]# cat /usr/local/bin/zookeeper.sh #!/bin/bash #@author :yinzhengjie #blog:http://www.cnblogs.com/yinzhengjie #EMAIL:y1053419035@qq.com #判断用户是否传参 if [ $# -ne 1 ];then echo "无效参数，用法为: $0 {start|stop|restart|status}" exit fi #获取用户输入的命令 cmd=$1 #定义函数功能 function zookeeperManger(){ case $cmd in start) echo "启动服务" remoteExecution start ;; stop) echo "停止服务" remoteExecution stop ;; restart) echo "重启服务" remoteExecution restart ;; status) echo "查看状态" remoteExecution status ;; *) echo "无效参数，用法为: $0 {start|stop|restart|status}" ;; esac } #定义执行的命令 function remoteExecution(){ for (( i=101 ; i<=103 ; i++ )) ; do tput setaf 2 echo ========== zookeeper${i}.yinzhengjie.org.cn zkServer.sh $1 ================ tput setaf 9 ssh hadoop${i}.yinzhengjie.org.cn "source /etc/profile ; zkServer.sh $1" done } #调用函数 zookeeperManger [root@hadoop101.yinzhengjie.org.cn ~]#
[root@hadoop101.yinzhengjie.org.cn ~]# ll /usr/local/bin/zookeeper.sh -rw-r--r-- 1 root root 1132 May 12 06:18 /usr/local/bin/zookeeper.sh [root@hadoop101.yinzhengjie.org.cn ~]# [root@hadoop101.yinzhengjie.org.cn ~]# chmod +x /usr/local/bin/zookeeper.sh #别忘记为脚本添加执行权限哟~ [root@hadoop101.yinzhengjie.org.cn ~]# [root@hadoop101.yinzhengjie.org.cn ~]# ll /usr/local/bin/zookeeper.sh -rwxr-xr-x 1 root root 1132 May 12 06:18 /usr/local/bin/zookeeper.sh [root@hadoop101.yinzhengjie.org.cn ~]#
[root@hadoop101.yinzhengjie.org.cn ~]# zookeeper.sh status 查看状态 ========== zookeeper101.yinzhengjie.org.cn zkServer.sh status ================ /yinzhengjie/softwares/jdk1.8.0_201/bin/java ZooKeeper JMX enabled by default Using config: /yinzhengjie/softwares/zookeeper/bin/../conf/zoo.cfg Client port found: 2181. Client address: localhost. Error contacting service. It is probably not running. ========== zookeeper102.yinzhengjie.org.cn zkServer.sh status ================ /yinzhengjie/softwares/jdk1.8.0_201/bin/java ZooKeeper JMX enabled by default Using config: /yinzhengjie/softwares/zookeeper/bin/../conf/zoo.cfg Client port found: 2181. Client address: localhost. Error contacting service. It is probably not running. ========== zookeeper103.yinzhengjie.org.cn zkServer.sh status ================ /yinzhengjie/softwares/jdk1.8.0_201/bin/java ZooKeeper JMX enabled by default Using config: /yinzhengjie/softwares/zookeeper/bin/../conf/zoo.cfg Client port found: 2181. Client address: localhost. Error contacting service. It is probably not running. [root@hadoop101.yinzhengjie.org.cn ~]#
6>.配置服务器编号
[root@hadoop101.yinzhengjie.org.cn ~]# for (( i=101;i<=103;i++ )) do ssh hadoop${i}.yinzhengjie.org.cn "echo -n $i > /yinzhengjie/data/zookeeper/myid" ;done [root@hadoop101.yinzhengjie.org.cn ~]# [root@hadoop101.yinzhengjie.org.cn ~]# ansible zk -m shell -a 'cat /yinzhengjie/data/zookeeper/myid' hadoop102.yinzhengjie.org.cn | SUCCESS | rc=0 >> 102 hadoop101.yinzhengjie.org.cn | SUCCESS | rc=0 >> 101 hadoop103.yinzhengjie.org.cn | SUCCESS | rc=0 >> 103 [root@hadoop101.yinzhengjie.org.cn ~]#
7>.启动zookeeper集群
[root@hadoop101.yinzhengjie.org.cn ~]# zookeeper.sh start 启动服务 ========== zookeeper101.yinzhengjie.org.cn zkServer.sh start ================ /yinzhengjie/softwares/jdk1.8.0_201/bin/java ZooKeeper JMX enabled by default Using config: /yinzhengjie/softwares/zookeeper/bin/../conf/zoo.cfg Starting zookeeper ... STARTED ========== zookeeper102.yinzhengjie.org.cn zkServer.sh start ================ /yinzhengjie/softwares/jdk1.8.0_201/bin/java ZooKeeper JMX enabled by default Using config: /yinzhengjie/softwares/zookeeper/bin/../conf/zoo.cfg Starting zookeeper ... STARTED ========== zookeeper103.yinzhengjie.org.cn zkServer.sh start ================ /yinzhengjie/softwares/jdk1.8.0_201/bin/java ZooKeeper JMX enabled by default Using config: /yinzhengjie/softwares/zookeeper/bin/../conf/zoo.cfg Starting zookeeper ... STARTED [root@hadoop101.yinzhengjie.org.cn ~]#
[root@hadoop101.yinzhengjie.org.cn ~]# zookeeper.sh status 查看状态 ========== zookeeper101.yinzhengjie.org.cn zkServer.sh status ================ /yinzhengjie/softwares/jdk1.8.0_201/bin/java ZooKeeper JMX enabled by default Using config: /yinzhengjie/softwares/zookeeper/bin/../conf/zoo.cfg Client port found: 2181. Client address: localhost. Mode: follower ========== zookeeper102.yinzhengjie.org.cn zkServer.sh status ================ /yinzhengjie/softwares/jdk1.8.0_201/bin/java ZooKeeper JMX enabled by default Using config: /yinzhengjie/softwares/zookeeper/bin/../conf/zoo.cfg Client port found: 2181. Client address: localhost. Mode: leader ========== zookeeper103.yinzhengjie.org.cn zkServer.sh status ================ /yinzhengjie/softwares/jdk1.8.0_201/bin/java ZooKeeper JMX enabled by default Using config: /yinzhengjie/softwares/zookeeper/bin/../conf/zoo.cfg Client port found: 2181. Client address: localhost. Mode: follower [root@hadoop101.yinzhengjie.org.cn ~]#
8>.客户端命令行操作
博主推荐阅读: https://www.cnblogs.com/yinzhengjie2020/p/12501295.html