主机配置
[root@localhost ~]# hostname k8s-master
[root@localhost ~]# bash
[root@k8s-master ~]#
[root@localhost ~]# hostname k8s-node01
[root@localhost ~]# bash
[root@k8s-node01 ~]#
[root@localhost ~]# hostname k8s-node02
[root@localhost ~]# bash
[root@k8s-node02 ~]#
三台主机上修改 hosts 文件添加地址解析记录
cat << EOF >> /etc/hosts
192.168.200.14 k8s-master
192.168.200.10 k8s-node01
192.168.200.11 k8s-node02
EOF
iptables -I INPUT -s 192.168.200.0/24 -j ACCEPT
sed -i '/^SELINUX=/s/enforcing/disabled/' /etc/selinux/config
生成CA证书
[root@k8s-master ~]# mkdir -p /root/software/ssl
[root@k8s-master ~]# cd /root/software/ssl/
[root@k8s-master ssl]# rz
[root@k8s-master ssl]# ls
cfssl-certinfo_linux-amd64 cfssljson_linux-amd64 cfssl_linux-amd64
[root@k8s-master ssl]# chmod +x *
[root@k8s-master ssl]# mv cfssl_linux-amd64 /usr/local/bin/cfssl
[root@k8s-master ssl]# mv cfssljson_linux-amd64 /usr/local/bin/cfssljson
[root@k8s-master ssl]# mv cfssl-certinfo_linux-amd64 /usr/local/bin/cfssl-certinfo
[root@k8s-master ssl]# cfssl --help
Usage:
Available commands:
revoke
serve
genkey
gencrl
selfsign
sign
gencert
ocsprefresh
version
ocspserve
scan
ocspsign
info
print-defaults
bundle
certinfo
ocspdump
Top-level flags:
-allow_verification_with_non_compliant_keys
Allow a SignatureVerifier to use keys which are technically non-compliant with RFC6962.
-loglevel int
Log level (0 = DEBUG, 5 = FATAL) (default 1)
执行以下命令，拷贝证书生成脚本。
[root@k8s-master ssl]# cat << EOF > ca-config.json
> {
> "signing": {
> "default": {
> "expiry": "87600h"
> },
> "profiles": {
> "kubernetes": {
> "expiry": "87600h",
> "usages": [
> "signing",
> "key encipherment",
> "server auth",
> "client auth"
> ]
> }
> }
> }
> }
> EOF
[root@k8s-master ssl]# cat << EOF > ca-csr.json
> {
> "CN": "kubernetes",
> "key": {
> "algo": "rsa",
> "size": 2048
> },
> "names": [
> {
> "C": "CN",
> "L": "Beijing",
> "ST": "Beijing",
> "O": "k8s",
> "OU": "System"
> }
> ]
> }
> EOF
执行以下操作，生成 CA 证书。
[root@k8s-master ssl]# cfssl gencert -initca ca-csr.json | cfssljson -bare ca -
2021/11/12 04:55:08 [INFO] generating a new CA key and certificate from CSR
2021/11/12 04:55:08 [INFO] generate received request
2021/11/12 04:55:08 [INFO] received CSR
2021/11/12 04:55:08 [INFO] generating key: rsa-2048
2021/11/12 04:55:09 [INFO] encoded CSR
2021/11/12 04:55:09 [INFO] signed certificate with serial number 653190800918052544492132829215453908423014945556
[root@k8s-master ssl]# ls
ca-config.json ca.csr ca-csr.json ca-key.pem ca.pem
生成server证书
[root@k8s-master ssl]# cat << EOF > server-csr.json
> {
> "CN": "kubernetes",
> "hosts": [
> "127.0.0.1",
> "192.168.200.111",
> "192.168.200.112",
> "192.168.200.113",
> "10.10.10.1",
> "kubernetes",
> "kubernetes.default",
> "kubernetes.default.svc",
> "kubernetes.default.svc.cluster",
> "kubernetes.default.svc.cluster.local"
> ],
> "key": {
> "algo": "rsa",
> "size": 2048
> },
> "names": [
> {
> "C": "CN",
> "L": "BeiJing",
> cfssl gencert -initca ca-csr.json | cfssljson -bare ca -^C
[root@k8s-master ssl]# cat << EOF > server-csr.json
{
"CN": "kubernetes",
"hosts": [
"127.0.0.1",
"192.168.200.14",
"192.168.200.10",
"192.168.200.11",
"10.10.10.1",
"kubernetes",
"kubernetes.default",
"kubernetes.default.svc",
"kubernetes.default.svc.cluster",
"kubernetes.default.svc.cluster.local"
],
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"L": "BeiJing",
"O": "k8s",
> "OU": "System"
> }
> ]
> }
> EOF
[root@k8s-master ssl]# ls
ca-config.json ca.csr ca-csr.json ca-key.pem ca.pem server-csr.json
[root@k8s-master ssl]# cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=kubernetes server-csr.json | cfssljson -bare server
2021/11/12 04:58:29 [INFO] generate received request
2021/11/12 04:58:29 [INFO] received CSR
2021/11/12 04:58:29 [INFO] generating key: rsa-2048
2021/11/12 04:58:29 [INFO] encoded CSR
2021/11/12 04:58:29 [INFO] signed certificate with serial number 85562378331608519833702058069174740363634879214
2021/11/12 04:58:29 [WARNING] This certificate lacks a "hosts" field. This makes it unsuitable for
websites. For more information see the Baseline Requirements for the Issuance and Management
of Publicly-Trusted Certificates, v.1.1.6, from the CA/Browser Forum (https://cabforum.org);
specifically, section 10.2.3 ("Information Requirements").
生成 admin 证书
[root@k8s-master ssl]# cat << EOF > admin-csr.json
> {
> "CN": "admin",
> "hosts": [],
> "key": {
> "algo": "rsa",
> "size": 2048
> },
> "names": [
> {
> "C": "CN",
> "L": "BeiJing",
> "ST": "BeiJing",
> "O": "system:masters",
> "OU": "System"
> }
> ]
> }
>
> EOF
[root@k8s-master ssl]# cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json - ^C
[root@k8s-master ssl]# cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=kubernetes admin-csr.json | cfssljson -bare admin
2021/11/12 05:01:52 [INFO] generate received request
2021/11/12 05:01:52 [INFO] received CSR
2021/11/12 05:01:52 [INFO] generating key: rsa-2048
2021/11/12 05:01:52 [INFO] encoded CSR
2021/11/12 05:01:52 [INFO] signed certificate with serial number 643833933703109524978384066858919303910102994915
2021/11/12 05:01:52 [WARNING] This certificate lacks a "hosts" field. This makes it unsuitable for
websites. For more information see the Baseline Requirements for the Issuance and Management
of Publicly-Trusted Certificates, v.1.1.6, from the CA/Browser Forum (https://cabforum.org);
specifically, section 10.2.3 ("Information Requirements").
[root@k8s-master ssl]# ls
admin.csr admin.pem ca-csr.json server.csr server.pem
admin-csr.json ca-config.json ca-key.pem server-csr.json
admin-key.pem ca.csr ca.pem server-key.pem
生成proxy证书
[root@k8s-master ssl]# cat << EOF > kube-proxy-csr.json
> {
> "CN": "system:kube-proxy",
> "hosts": [],
> "key": {
> "algo": "rsa",
> "size": 2048
> },
> "names": [
> {
> "C": "CN",
> "L": "BeiJing",
> "ST": "BeiJing",
> "O": "k8s",
> "OU": "System"
> }
> ]
> }
> EOF
[root@k8s-master ssl]# cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=kubernetes kube-proxy-csr.json | cfssljson -bare kube-proxy
2021/11/12 05:04:10 [INFO] generate received request
2021/11/12 05:04:10 [INFO] received CSR
2021/11/12 05:04:10 [INFO] generating key: rsa-2048
2021/11/12 05:04:10 [INFO] encoded CSR
2021/11/12 05:04:10 [INFO] signed certificate with serial number 193018966224499611445645373222754334130237086881
2021/11/12 05:04:10 [WARNING] This certificate lacks a "hosts" field. This makes it unsuitable for
websites. For more information see the Baseline Requirements for the Issuance and Management
of Publicly-Trusted Certificates, v.1.1.6, from the CA/Browser Forum (https://cabforum.org);
specifically, section 10.2.3 ("Information Requirements").
[root@k8s-master ssl]# ls
admin.csr ca-config.json ca.pem kube-proxy.pem server.pem
admin-csr.json ca.csr kube-proxy.csr server.csr
admin-key.pem ca-csr.json kube-proxy-csr.json server-csr.json
admin.pem ca-key.pem kube-proxy-key.pem server-key.pem
[root@k8s-master ssl]# ls | grep -v pem | xargs -i rm {} #删除证书以外的 json 文件，只保留 pem 证书
[root@k8s-master ssl]# ls -l
总用量 32
-rw------- 1 root root 1679 11月 12 05:01 admin-key.pem
-rw-r--r-- 1 root root 1399 11月 12 05:01 admin.pem
-rw------- 1 root root 1675 11月 12 04:55 ca-key.pem
-rw-r--r-- 1 root root 1359 11月 12 04:55 ca.pem
-rw------- 1 root root 1679 11月 12 05:04 kube-proxy-key.pem
-rw-r--r-- 1 root root 1403 11月 12 05:04 kube-proxy.pem
-rw------- 1 root root 1675 11月 12 04:58 server-key.pem
-rw-r--r-- 1 root root 1602 11月 12 04:58 server.pem
部署Etcd集群
创建文件
[root@k8s-master ssl]# mkdir /opt/kubernetes
[root@k8s-master ssl]# mkdir /opt/kubernetes/{bin,cfg,ssl}
[root@k8s-master ~]# tar xf etcd-v3.3.18-linux-amd64.tar.gz
[root@k8s-master ~]# cd etcd-v3.3.18-linux-amd64/
[root@k8s-master etcd-v3.3.18-linux-amd64]# mv etcd /opt/kubernetes/bin/
[root@k8s-master etcd-v3.3.18-linux-amd64]# mv etcdctl /opt/kubernetes/bin/
在k8s-master 上部署Etcd节点
[root@k8s-master etcd-v3.3.18-linux-amd64]# vim /opt/kubernetes/cfg/etcd
#[Member]
ETCD_NAME="etcd01"
ETCD_DATA_DIR="/var/lib/etcd/default.etcd"
ETCD_LISTEN_PEER_URLS="https://192.168.200.14:2380"
ETCD_LISTEN_CLIENT_URLS="https://192.168.200.14:2379"
#[Clustering]
ETCD_INITIAL_ADVERTISE_PEER_URLS="https://192.168.200.14:2380"
ETCD_ADVERTISE_CLIENT_URLS="https://192.168.200.14:2379"
ETCD_INITIAL_CLUSTER="etcd01=https://192.168.200.14:2380,etcd02=https://192.168.200.10:2380,etcd03=https://192.168.200.11:2380"
ETCD_INITIAL_CLUSTER_TOKEN="etcd-cluster"
ETCD_INITIAL_CLUSTER_STATE="new"
创建脚本配置文件
[root@k8s-master etcd-v3.3.18-linux-amd64]# vim /usr/lib/systemd/system/etcd.service
[Unit]
Description=Etcd Server
After=network.target
After=network-online.target
Wants=network-online.target
[Service]
Type=notify
EnvironmentFile=-/opt/kubernetes/cfg/etcd
ExecStart=/opt/kubernetes/bin/etcd
--name=${ETCD_NAME}
--data-dir=${ETCD_DATA_DIR}
--listen-peer-urls=${ETCD_LISTEN_PEER_URLS}
--listen-client-urls=${ETCD_LISTEN_CLIENT_URLS},http://127.0.0.1:2379
--advertise-client-urls=${ETCD_ADVERTISE_CLIENT_URLS}
--initial-advertise-peer-urls=${ETCD_INITIAL_ADVERTISE_PEER_URLS}
--initial-cluster=${ETCD_INITIAL_CLUSTER}
--initial-cluster-token=${ETCD_INITIAL_CLUSTER}
--initial-cluster-state=new
--cert-file=/opt/kubernetes/ssl/server.pem
--key-file=/opt/kubernetes/ssl/server-key.pem
--peer-cert-file=/opt/kubernetes/ssl/server.pem
--peer-key-file=/opt/kubernetes/ssl/server-key.pem
--trusted-ca-file=/opt/kubernetes/ssl/ca.pem
--peer-trusted-ca-file=/opt/kubernetes/ssl/ca.pem
Restart=on-failure
LimitNOFILE=65536
[Install]
WantedBy=multi-user.target
拷贝Etcd启动所依赖的证书
[root@k8s-master etcd-v3.3.18-linux-amd64]# cd /root/software/
[root@k8s-master software]# cp ssl/server*pem ssl/ca*.pem /opt/kubernetes/ssl/
启动Etcd主节点。若主节点卡顿，直接 ctrl +c 终止即可。实际 Etcd 进程已经启动，在连接另外两个节点时会超时，因为另外两个节点尚未启动。
[root@k8s-master software]# systemctl start etcd
^C
[root@k8s-master software]# systemctl enable etcd
Created symlink from /etc/systemd/system/multi-user.target.wants/etcd.service to /usr/lib/systemd/system/etcd.service.
[root@k8s-master software]# ps aux | grep etcd
root 11025 4.1 1.8 10610508 18036 ? Ssl 05:17 0:01 /opt/kubernetes/bin/etc --name=etcd01 --data-dir=/var/lib/etcd/default.etcd --listen-peer-urls=https://192.168.200.14:2380 --listen-client-urls=https://192.168.200.14:2379,http://127.0.0.1:2379 --advertise-client-urls=https://192.168.200.14:2379 --initial-advertise-peer-urls=https://192.168.200.14:2380 --initial-cluster=etcd01=https://192.168.200.14:2380,etcd02=https://192.168.200.10:2380,etcd03=https://192.168.200.11:2380 --initial-cluster-token=etcd01=https://192.168.200.14:2380,etcd02=https://192.168.200.10:2380,etcd03=https://192.168.200.11:2380 --initial-cluster-state=new --cert-file=/opt/kubernetes/ssl/server.pem --key-file=/opt/kubernetes/ssl/server-key.pem --peer-cert-file=/opt/kubernetes/ssl/server.pem --peer-key-file=/opt/kubernetes/ssl/server-key.pem --trusted-ca-file=/opt/kubernetes/ssl/ca.pem --peer-trusted-ca-file=/opt/kubernetes/ssl/ca.pem
root 11070 0.0 0.0 112728 992 pts/2 R+ 05:17 0:00 grep --color=auto etcd
在node节点上部署etcd节点
[root@k8s-master ~]# rsync -avcz /opt/kubernetes/* 192.168.200.10:/opt/kubernetes/
root@192.168.200.10's password:
sending incremental file list
created directory /opt/kubernetes
bin/
bin/etcd
bin/etcdctl
cfg/
cfg/etcd
ssl/
ssl/ca-key.pem
ssl/ca.pem
ssl/server-key.pem
ssl/server.pem
sent 13,942,496 bytes received 199 bytes 2,535,035.45 bytes/sec
total size is 40,371,722 speedup is 2.90
[root@k8s-master ~]# rsync -avcz /opt/kubernetes/* 192.168.200.11:/opt/kubernetes/
The authenticity of host '192.168.200.11 (192.168.200.11)' can't be established.
ECDSA key fingerprint is SHA256:Ch6vBCnRcTQNR6+DnYMKJR2jvtB7y1/bB3zMjtBe3Xk.
ECDSA key fingerprint is MD5:c7:39:97:f1:e5:1f:ce:74:87:88:f2:05:83:0f:1b:5f.
Are you sure you want to continue connecting (yes/no)? yes
Warning: Permanently added '192.168.200.11' (ECDSA) to the list of known hosts.
root@192.168.200.11's password:
sending incremental file list
created directory /opt/kubernetes
bin/
bin/etcd
bin/etcdctl
cfg/
cfg/etcd
ssl/
ssl/ca-key.pem
ssl/ca.pem
ssl/server-key.pem
ssl/server.pem
sent 13,942,496 bytes received 199 bytes 1,640,317.06 bytes/sec
total size is 40,371,722 speedup is 2.90
#修改对应下的Etcd
拷贝启动脚本
[root@k8s-master ~]# scp /usr/lib/systemd/system/etcd.service 192.168.200.10:/usr/lib/systemd/system/
root@192.168.200.10's password:
etcd.service 100% 994 547.8KB/s 00:00
[root@k8s-master ~]# scp /usr/lib/systemd/system/etcd.service 192.168.200.11:/usr/lib/systemd/system/
root@192.168.200.11's password:
etcd.service 100% 994 572.8KB/s 00:00
启动node上的Etcd
[root@k8s-node01 ~]# systemctl start etcd
[root@k8s-node01 ~]# systemctl enable etcd
Created symlink from /etc/systemd/system/multi-user.target.wants/etcd.service to /usr/lib/systemd/system/etcd.service.
查看Etcd集群部署
[root@k8s-master ~]# vim /etc/profile
export PATH=$PATH:/opt/kubernetes/bin
[root@k8s-master ~]# source /etc/profile
[root@k8s-master ~]# cd /root/software/ssl/
[root@k8s-master ssl]# etcdctl --ca-file=ca.pem --cert-file=server.pem --key-file=server-key.pem --endpoints="https://192.168.200.14:2379,https://192.168.200.10,https://192.168.200.11:2379" cluster-health
member 7c28f3ecc1788ca9 is healthy: got healthy result from https://192.168.200.11:2379
member a1441fe9e75a6508 is healthy: got healthy result from https://192.168.200.14:2379
member ba62de56b2cc4d06 is healthy: got healthy result from https://192.168.200.10:2379
cluster is healthy
部署Flannel网络
主节点写入分配网络到Etcd，给flanneld使用
[root@k8s-master ssl]# etcdctl -ca-file=ca.pem --cert-file=server.pem --key-file=server-key.pem --endpoints="https://192.168.200.14:2379,https://192.168.200.10:2379,https://192.168.200.11:2379" set /coreos.com/network/config '{"Network":"172.17.0.0/16","Backend":{"Type":"vxlan"} }'
{"Network":"172.17.0.0/16","Backend":{"Type":"vxlan"} }
#上传flannel包，拷贝到node节点
[root@k8s-master ~]# tar xf flannel-v0.12.0-linux-amd64.tar.gz
[root@k8s-master ~]# ls
flanneld mk-docker-opts.sh
[root@k8s-master ~]# scp flannel mk-docker-opts.sh 192.168.200.10:/opt/kubernetes/bin/
root@192.168.200.10's password:
flannel: No such file or directory
mk-docker-opts.sh 100% 2139 1.2MB/s 00:00
[root@k8s-master ~]# scp flannel mk-docker-opts.sh 192.168.200.11:/opt/kubernetes/bin/
root@192.168.200.11's password:
flannel: No such file or directory
mk-docker-opts.sh 100% 2139 1.1MB/s 00:00
node上配置flannel
[root@k8s-node01 ~]# vim /opt/kubernetes/cfg/flanneld
FLANNEL_OPTIONS="--etcdendpoints=https://192.168.200.14:2379,https://192.168.200.10:2379,https://192.168.200.11:2379 -etcd-cafile=/opt/kubernetes/ssl/ca.pem -etcdcertfile=/opt/kubernetes/ssl/server.pem -etcd-keyfile=/opt/kubernetes/ssl/server-key.pem"
[root@k8s-node01 ~]# scp /opt/kubernetes/cfg/flanneld 192.168.200.11:/opt/kubernetes/cfg/
在node节点上分别创建 flanneld.service 脚本文件管理 Flanneld。
cat <<EOF >/usr/lib/systemd/system/flanneld.service
[Unit]
Description=Flanneld overlay address etcd agent
After=network-online.target network.target
Before=docker.service
[Service]
Type=notify
EnvironmentFile=/opt/kubernetes/cfg/flanneld
ExecStart=/opt/kubernetes/bin/flanneld --ip-masq $FLANNEL_OPTIONS
ExecStartPost=/opt/kubernetes/bin/mk-docker-opts.sh -k DOCKER_NETWORK_OPTIONS -d
/run/flannel/subnet.env
Restart=on-failure
[Install]
WantedBy=multi-user.target
EOF
在node上配置docker启动指定网段
[root@k8s-node01 ~]# vim /usr/lib/systemd/system/docker.service
EnvironmentFile=/run/flannel/subnet.env //新添加[Service]块内，目的是让 Docker 网桥分
发的 ip 地址与 flanned 网桥在同一个网段
ExecStart=/usr/bin/dockerd $DOCKER_NETWORK_OPTIONS //添加$
DOCKER_NETWORK_OPTIONS 变量，替换原来的 ExecStart，目的是调用 Flannel 网桥 IP
地址
#ExecStart=/usr/bin/dockerd -H fd:// --containerd=/run/containerd/containerd.sock