Centos7 部署GlusterFS

环境准备

服务器	hostname
192.168.10.107	node1
192.168.10.108	node2
192.168.10.106	node3

安装yum源

# 每台服务器执行

# yum -y install centos-release-gluster

安装glusterfs

# 每台服务器执行

# yum -y install glusterfs glusterfs-fuse glusterfs-server

#启动glusterfs

# systemctl start glusterd
# systemctl enable glusterd

# 组建gluster集群
# 如果使用heketi 提供restapi ,则不用gluster组建集群
# gluster peer probe node2
# gluster peer probe node3

# 查看集群状态

# gluster peer status
Number of Peers: 2

Hostname: node1
Uuid: 15bcdaa7-83bc-4c58-be79-bc17492979df
State: Peer in Cluster (Connected)

Hostname: node2
Uuid: ac815e24-a6e8-4e00-881e-a8c104bf79e7
State: Peer in Cluster (Connected)

Kubernetes使用GlusterFS卷

自动方式需要先部署Heketi软件，Heketi用来管理GlusterFS，并提供RESTful API接口供Kubernetes调用。Heketi需要使用裸磁盘(挂在的磁盘不需要做任何处理，否则Heketi连接会出错)

# 安装heketi
yum -y install heketi heketi-client

# 配置Heketi用户能ssh密钥方式连接集群各个节点

ssh-keygen -f /etc/heketi/heketi_key -g rsa -N ''
chown heketi:heketi /etc/heketi/heketi_key*

for host in node1 node2 node3;do ssh-copy-id -i /etc/heketi/heketi_key.pub root@${host};done

# 编辑heketi.json

{
  "port": "18080",
  "use_auth": false,
  "jwt": {
    "admin": {
      "key": "My Secret"
    },
    "user": {
      "key": "My Secret"
    }
  },
  "glusterfs": {
    "executor": "ssh",
    "sshexec": {
      "keyfile": "/etc/heketi/heketi_key",
      "user": "root",
      "port": "22",
      "fstab": "/etc/fstab"
    },
    "db": "/var/lib/heketi/heketi.db",
    "loglevel" : "debug"
  }
}

# 启动heketi
# systemctl start heketi
# systemctl enable heketi

# 测试访问
# curl http://node3:18080/hello
Hello from Heketi

设置Heketi系统拓扑

{
    "clusters":[
        {
            "nodes":[
                {
                    "node": {
                        "hostnames":{
                            "manage":[
                              "node1"
                            ],
                            "storage":[
                                "192.168.1.107"
                            ]
                        },
                        "zone":1
                    },
                    "devices":[
                        "/dev/mapper/vgglusterfs-glusterfs_data"
                    ]
                },
                {
                    "node": {
                        "hostnames":{
                            "manage":[
                              "node2"
                            ],
                            "storage":[
                                "192.168.1.108"
                            ]
                        },
                        "zone":1
                    },
                    "devices":[
                        "/dev/mapper/vgglusterfs-glusterfs_data"
                    ]
                },
                {
                    "node": {
                        "hostnames":{
                            "manage":[
                              "node3"
                            ],
                            "storage":[
                                "192.168.1.106"
                            ]
                        },
                        "zone":1
                    },
                    "devices":[
                        "/dev/mapper/vgglusterfs-glusterfs_data"
                    ]
                }
            ]
        }
    ]
}

# 加载拓扑信息 生成集群  
# export HEKETI_CLI_SERVER=http://192.168.10.106:18080

# heketi-cli topology load --json=/etc/heketi/topology.json 

Creating cluster ... ID: bd85e222e97b72362d208d086d9f264f
    Allowing file volumes on cluster.
    Allowing block volumes on cluster.
    Creating node node3 ... ID: b289deb036e4ff87d951eda840217bc9
        Adding device /dev/sdb ... OK
    Creating node node2 ... ID: 36d4ccb95d7ae08eff0203df4cd9fcdd
        Adding device /dev/sdb ... OK
    Creating node node1 ... ID: 3ac2efae858119a8653585c6f4332b23
        Adding device /dev/sdb ... OK

# 查看集群状态
# heketi-cli cluster list
Clusters:
Id:bd85e222e97b72362d208d086d9f264f [file][block]

# heketi-cli cluster info bd85e222e97b72362d208d086d9f264f
Cluster id: bd85e222e97b72362d208d086d9f264f
Nodes:
36d4ccb95d7ae08eff0203df4cd9fcdd
3ac2efae858119a8653585c6f4332b23
b289deb036e4ff87d951eda840217bc9
Volumes:
90546735a4d59c0aaff6f09889886733
Block: true

File: true

# heketi-cli node list
Id:36d4ccb95d7ae08eff0203df4cd9fcdd Cluster:bd85e222e97b72362d208d086d9f264f
Id:3ac2efae858119a8653585c6f4332b23 Cluster:bd85e222e97b72362d208d086d9f264f
Id:b289deb036e4ff87d951eda840217bc9 Cluster:bd85e222e97b72362d208d086d9f264f