root@centos7 ~]# kubectl get pod -A The connection to the server 10.10.16.251:6443 was refused - did you specify the right host or port? [root@centos7 ~]#
[root@centos7 ~]# kubectl get pod -A The connection to the server 10.10.16.251:6443 was refused - did you specify the right host or port? [root@centos7 ~]# netstat -lpn | grep 6443 tcp6 1 0 :::6443 :::* LISTEN 26035/kube-apiserve [root@centos7 ~]#
systemctl restart kube-apiserver
journalctl -xefu kubelet
Sep 10 02:47:48 centos7 kubelet[26092]: I0910 02:47:48.815353 26092 kubelet_node_status.go:294] Setting node annotation to enable volume controller attach/detach Sep 10 02:47:48 centos7 kubelet[26092]: E0910 02:47:48.837179 26092 kubelet.go:2267] node "centos7" not found Sep 10 02:47:48 centos7 kubelet[26092]: E0910 02:47:48.937284 26092 kubelet.go:2267] node "centos7" not found Sep 10 02:47:49 centos7 kubelet[26092]: E0910 02:47:49.037380 26092 kubelet.go:2267] node "centos7" not found Sep 10 02:47:49 centos7 kubelet[26092]: E0910 02:47:49.137483 26092 kubelet.go:2267] node "centos7" not found Sep 10 02:47:49 centos7 kubelet[26092]: E0910 02:47:49.237581 26092 kubelet.go:2267] node "centos7" not found Sep 10 02:47:49 centos7 kubelet[26092]: E0910 02:47:49.337726 26092 kubelet.go:2267] node "centos7" not found Sep 10 02:47:49 centos7 kubelet[26092]: E0910 02:47:49.437837 26092 kubelet.go:2267] node "centos7" not found Sep 10 02:47:49 centos7 kubelet[26092]: E0910 02:47:49.537958 26092 kubelet.go:2267] node "centos7" not found Sep 10 02:47:49 centos7 kubelet[26092]: E0910 02:47:49.638079 26092 kubelet.go:2267] node "centos7" not found Sep 10 02:47:49 centos7 kubelet[26092]: E0910 02:47:49.738177 26092 kubelet.go:2267] node "centos7" not found Sep 10 02:47:49 centos7 kubelet[26092]: E0910 02:47:49.838267 26092 kubelet.go:2267] node "centos7" not found Sep 10 02:47:49 centos7 kubelet[26092]: E0910 02:47:49.938347 26092 kubelet.go:2267] node "centos7" not found Sep 10 02:47:50 centos7 kubelet[26092]: E0910 02:47:50.038442 26092 kubelet.go:2267] node "centos7" not found Sep 10 02:47:50 centos7 kubelet[26092]: E0910 02:47:50.138540 26092 kubelet.go:2267] node "centos7" not found Sep 10 02:47:50 centos7 kubelet[26092]: E0910 02:47:50.238639 26092 kubelet.go:2267] node "centos7" not found Sep 10 02:47:50 centos7 kubelet[26092]: E0910 02:47:50.338739 26092 kubelet.go:2267] node "centos7" not found Sep 10 02:47:50 centos7 kubelet[26092]: E0910 02:47:50.438835 26092 kubelet.go:2267] node "centos7" not found Sep 10 02:47:50 centos7 kubelet[26092]: E0910 02:47:50.538921 26092 kubelet.go:2267] node "centos7" not found Sep 10 02:47:50 centos7 kubelet[26092]: E0910 02:47:50.639039 26092 kubelet.go:2267] node "centos7" not found Sep 10 02:47:50 centos7 kubelet[26092]: E0910 02:47:50.739140 26092 kubelet.go:2267] node "centos7" not found Sep 10 02:47:50 centos7 kubelet[26092]: E0910 02:47:50.839224 26092 kubelet.go:2267] node "centos7" not found ^C [root@centos7 ~]# journalctl -xefu kubelet
[root@centos7 ~]# ps -elf | grep kube 4 S root 18186 18147 0 80 0 - 3237 futex_ 02:35 ? 00:00:02 kube-controller-manager --allocate-node-cidrs=true --authentication-kubeconfig=/etc/kubernetes/controller-manager.conf --authorization-kubeconfig=/etc/kubernetes/controller-manager.conf --bind-address=127.0.0.1 --client-ca-file=/etc/kubernetes/pki/ca.crt --cluster-cidr=10.251.0.0/16 --cluster-name=kubernetes --cluster-signing-cert-file=/etc/kubernetes/pki/ca.crt --cluster-signing-key-file=/etc/kubernetes/pki/ca.key --controllers=*,bootstrapsigner,tokencleaner --kubeconfig=/etc/kubernetes/controller-manager.conf --leader-elect=true --node-cidr-mask-size=24 --requestheader-client-ca-file=/etc/kubernetes/pki/front-proxy-ca.crt --root-ca-file=/etc/kubernetes/pki/ca.crt --service-account-private-key-file=/etc/kubernetes/pki/sa.key --service-cluster-ip-range=10.96.0.0/12 --use-service-account-credentials=true 4 S root 18212 18166 0 80 0 - 2290 futex_ 02:35 ? 00:00:02 kube-scheduler --authentication-kubeconfig=/etc/kubernetes/scheduler.conf --authorization-kubeconfig=/etc/kubernetes/scheduler.conf --bind-address=127.0.0.1 --kubeconfig=/etc/kubernetes/scheduler.conf --leader-elect=true 4 S root 24523 1 3 80 0 - 68253 futex_ 02:44 ? 00:00:02 /usr/bin/kubelet --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --kubeconfig=/etc/kubernetes/kubelet.conf --config=/var/lib/kubelet/config.yaml --cgroup-driver=systemd --network-plugin=cni --pod-infra-container-image=registry.aliyuncs.com/google_containers/pause:3.2 4 S root 25116 25096 2 80 0 - 3381 futex_ 02:45 ? 00:00:00 kube-apiserver --advertise-address=10.10.16.251 --allow-privileged=true --authorization-mode=Node,RBAC --client-ca-file=/etc/kubernetes/pki/ca.crt --enable-admission-plugins=NodeRestriction --enable-bootstrap-token-auth=true --etcd-cafile=/etc/kubernetes/pki/etcd/ca.crt --etcd-certfile=/etc/kubernetes/pki/apiserver-etcd-client.crt --etcd-keyfile=/etc/kubernetes/pki/apiserver-etcd-client.key --etcd-servers=https://127.0.0.1:2379 --insecure-port=0 --kubelet-client-certificate=/etc/kubernetes/pki/apiserver-kubelet-client.crt --kubelet-client-key=/etc/kubernetes/pki/apiserver-kubelet-client.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --proxy-client-cert-file=/etc/kubernetes/pki/front-proxy-client.crt --proxy-client-key-file=/etc/kubernetes/pki/front-proxy-client.key --requestheader-allowed-names=front-proxy-client --requestheader-client-ca-file=/etc/kubernetes/pki/front-proxy-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-key-file=/etc/kubernetes/pki/sa.pub --service-cluster-ip-range=10.96.0.0/12 --tls-cert-file=/etc/kubernetes/pki/apiserver.crt --tls-private-key-file=/etc/kubernetes/pki/apiserver.key
journalctl -xefu docker
58534) (total time: 10.000619494s): Sep 10 02:44:17 centos7 dockerd-current[9671]: Trace[1545735573]: [10.000619494s] [10.000619494s] END Sep 10 02:44:17 centos7 dockerd-current[9671]: E0910 06:44:17.014125 1 reflector.go:178] k8s.io/kubernetes/cmd/kube-scheduler/app/server.go:233: Failed to list *v1.Pod: Get https://10.10.16.251:6443/api/v1/pods?fieldSelector=status.phase%21%3DFailed%2Cstatus.phase%21%3DSucceeded&limit=500&resourceVersion=0: net/http: TLS handshake timeout Sep 10 02:44:18 centos7 dockerd-current[9671]: panic: context deadline exceeded Sep 10 02:44:18 centos7 dockerd-current[9671]: Sep 10 02:44:18 centos7 dockerd-current[9671]: goroutine 1 [running]: Sep 10 02:44:18 centos7 dockerd-current[9671]: k8s.io/kubernetes/vendor/k8s.io/apiextensions-apiserver/pkg/registry/customresourcedefinition.NewREST(0x4000775730, 0x46fea20, 0x400028aea0, 0x4000282108) Sep 10 02:44:18 centos7 dockerd-current[9671]: /workspace/anago-v1.18.1-beta.0.38+49aac775931dd1/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/vendor/k8s.io/apiextensions-apiserver/pkg/registry/customresourcedefinition/etcd.go:56 +0x328 Sep 10 02:44:18 centos7 dockerd-current[9671]: k8s.io/kubernetes/vendor/k8s.io/apiextensions-apiserver/pkg/apiserver.completedConfig.New(0x4000d4aac0, 0x40003eefc8, 0x47bd420, 0x6cfb260, 0x10, 0x10, 0x0) Sep 10 02:44:18 centos7 dockerd-current[9671]: /workspace/anago-v1.18.1-beta.0.38+49aac775931dd1/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/vendor/k8s.io/apiextensions-apiserver/pkg/apiserver/apiserver.go:145 +0x10f0 Sep 10 02:44:18 centos7 dockerd-current[9671]: k8s.io/kubernetes/cmd/kube-apiserver/app.createAPIExtensionsServer(0x40003eefc0, 0x47bd420, 0x6cfb260, 0x0, 0x46fe5e0, 0x40007135c0) Sep 10 02:44:18 centos7 dockerd-current[9671]: /workspace/anago-v1.18.1-beta.0.38+49aac775931dd1/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/cmd/kube-apiserver/app/apiextensions.go:102 +0x50 Sep 10 02:44:18 centos7 dockerd-current[9671]: k8s.io/kubernetes/cmd/kube-apiserver/app.CreateServerChain(0x40008a5600, 0x4000545320, 0x3b4fe29, 0xc, 0x4000c3fc28) Sep 10 02:44:18 centos7 dockerd-current[9671]: /workspace/anago-v1.18.1-beta.0.38+49aac775931dd1/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/cmd/kube-apiserver/app/server.go:186 +0x1cc Sep 10 02:44:18 centos7 dockerd-current[9671]: k8s.io/kubernetes/cmd/kube-apiserver/app.Run(0x40008a5600, 0x4000545320, 0x0, 0x0) Sep 10 02:44:18 centos7 dockerd-current[9671]: /workspace/anago-v1.18.1-beta.0.38+49aac775931dd1/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/cmd/kube-apiserver/app/server.go:155 +0xc0 Sep 10 02:44:18 centos7 dockerd-current[9671]: k8s.io/kubernetes/cmd/kube-apiserver/app.NewAPIServerCommand.func1(0x4000974500, 0x4000a08000, 0x0, 0x1a, 0x0, 0x0) Sep 10 02:44:18 centos7 dockerd-current[9671]: /workspace/anago-v1.18.1-beta.0.38+49aac775931dd1/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/cmd/kube-apiserver/app/server.go:122 +0xdc Sep 10 02:44:18 centos7 dockerd-current[9671]: k8s.io/kubernetes/vendor/github.com/spf13/cobra.(*Command).execute(0x4000974500, 0x400001e1d0, 0x1a, 0x1b, 0x4000974500, 0x400001e1d0) Sep 10 02:44:18 centos7 dockerd-current[9671]: /workspace/anago-v1.18.1-beta.0.38+49aac775931dd1/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/vendor/github.com/spf13/cobra/command.go:826 +0x314 Sep 10 02:44:18 centos7 dockerd-current[9671]: k8s.io/kubernetes/vendor/github.com/spf13/cobra.(*Command).ExecuteC(0x4000974500, 0x16a362e7e522a728, 0x6cdd080, 0x15670) Sep 10 02:44:18 centos7 dockerd-current[9671]: /workspace/anago-v1.18.1-beta.0.38+49aac775931dd1/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/vendor/github.com/spf13/cobra/command.go:914 +0x22c Sep 10 02:44:18 centos7 dockerd-current[9671]: k8s.io/kubernetes/vendor/github.com/spf13/cobra.(*Command).Execute(...) Sep 10 02:44:18 centos7 dockerd-current[9671]: /workspace/anago-v1.18.1-beta.0.38+49aac775931dd1/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/vendor/github.com/spf13/cobra/command.go:864 Sep 10 02:44:18 centos7 dockerd-current[9671]: main.main() Sep 10 02:44:18 centos7 dockerd-current[9671]: _output/dockerized/go/src/k8s.io/kubernetes/cmd/kube-apiserver/apiserver.go:43 +0xc0 Sep 10 02:44:18 centos7 dockerd-current[9671]: time="2021-09-10T02:44:18.945071161-04:00" level=warning msg="b14197ff830fd13aba475f357dfe1e28c75343e59bdfb56b673768cb4440665e cleanup: failed to unmount secrets: invalid argument" Sep 10 02:44:19 centos7 dockerd-current[9671]: E0910 06:44:19.746062 1 leaderelection.go:320] error retrieving resource lock kube-system/kube-controller-manager: Get https://10.10.16.251:6443/api/v1/namespaces/kube-system/endpoints/kube-controller-manager?timeout=10s: dial tcp 10.10.16.251:6443: connect: connection refused Sep 10 02:44:19 centos7 dockerd-current[9671]: E0910 06:44:19.746097 1 reflector.go:178] k8s.io/client-go/informers/factory.go:135: Failed to list *v1.Node: Get https://10.10.16.251:6443/api/v1/nodes?limit=500&resourceVersion=0: dial tcp 10.10.16.251:6443: connect: connection refused Sep 10 02:44:19 centos7 dockerd-current[9671]: I0910 06:44:19.746159 1 trace.go:116] Trace[1510189219]: "Reflector ListAndWatch" name:k8s.io/client-go/informers/factory.go:135 (started: 2021-09-10 06:44:09.472908343 +0000 UTC m=+530.406010893) (total time: 10.273214971s): Sep 10 02:44:19 centos7 dockerd-current[9671]: Trace[1510189219]: [10.273214971s] [10.273214971s] END Sep 10 02:44:19 centos7 dockerd-current[9671]: E0910 06:44:19.746194 1 reflector.go:178] k8s.io/client-go/informers/factory.go:135: Failed to list *v1.StorageClass: Get https://10.10.16.251:6443/apis/storage.k8s.io/v1/storageclasses?limit=500&resourceVersion=0: dial tcp 10.10.16.251:6443: connect: connection refused Sep 10 02:44:23 centos7 dockerd-current[9671]: E0910 06:44:23.432360 1 leaderelection.go:320] error retrieving resource lock kube-system/kube-controller-manager: Get https://10.10.16.251:6443/api/v1/namespaces/kube-system/endpoints/kube-controller-manager?timeout=10s: dial tcp 10.10.16.251:6443: connect: connection refused Sep 10 02:44:25 centos7 dockerd-current[9671]: E0910 06:44:25.860773 1 leaderelection.go:320] error retrieving resource lock kube-system/kube-controller-manager: Get https://10.10.16.251:6443/api/v1/namespaces/kube-system/endpoints/kube-controller-manager?timeout=10s: dial tcp 10.10.16.251:6443: connect: connection refused
telnet 10.10.16.251 6443
[root@centos7 ~]# telnet 10.10.16.251 6443 Trying 10.10.16.251... Connected to 10.10.16.251. Escape character is '^]'. ^C ^C Connection closed by foreign host. [root@centos7 ~]#
kubectl -v=8 get nodes
[root@centos7 ~]# docker ps CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES bc207e344e55 registry.aliyuncs.com/google_containers/pause:3.2 "/pause" 1 second ago Up Less than a second k8s_POD_kube-controller-manager-centos7_kube-system_60fc8c543d8227eaee711b0029f6a305_5 9bde8e1d8fbd registry.aliyuncs.com/google_containers/pause:3.2 "/pause" 1 second ago Up Less than a second k8s_POD_kube-apiserver-centos7_kube-system_d074386824dbcee6927fa7bdc297683b_3 d6db8dae983d f0a76adbc244 "kube-scheduler --..." 5 seconds ago Up 4 seconds k8s_kube-scheduler_kube-scheduler-centos7_kube-system_2c04fc5e4761bd2ada4d5c31bd4317ad_5 55b788630193 registry.aliyuncs.com/google_containers/pause:3.2 "/pause" 6 seconds ago Up 5 seconds k8s_POD_kube-scheduler-centos7_kube-system_2c04fc5e4761bd2ada4d5c31bd4317ad_6 [root@centos7 ~]# kubectl -v=8 get nodes I0910 02:57:10.960284 33823 loader.go:375] Config loaded from file: /root/.kube/config I0910 02:57:10.961032 33823 round_trippers.go:420] GET https://10.10.16.251:6443/api?timeout=32s I0910 02:57:10.961045 33823 round_trippers.go:427] Request Headers: I0910 02:57:10.961054 33823 round_trippers.go:431] Accept: application/json, */* I0910 02:57:10.961062 33823 round_trippers.go:431] User-Agent: kubectl/v1.18.1 (linux/arm64) kubernetes/7879fc1 I0910 02:57:10.961337 33823 round_trippers.go:446] Response Status: in 0 milliseconds I0910 02:57:10.961358 33823 round_trippers.go:449] Response Headers: I0910 02:57:10.961406 33823 cached_discovery.go:121] skipped caching discovery info due to Get https://10.10.16.251:6443/api?timeout=32s: dial tcp 10.10.16.251:6443: connect: connection refused I0910 02:57:10.961776 33823 round_trippers.go:420] GET https://10.10.16.251:6443/api?timeout=32s I0910 02:57:10.961787 33823 round_trippers.go:427] Request Headers: I0910 02:57:10.961796 33823 round_trippers.go:431] Accept: application/json, */* I0910 02:57:10.961803 33823 round_trippers.go:431] User-Agent: kubectl/v1.18.1 (linux/arm64) kubernetes/7879fc1 I0910 02:57:10.961982 33823 round_trippers.go:446] Response Status: in 0 milliseconds I0910 02:57:10.961993 33823 round_trippers.go:449] Response Headers: I0910 02:57:10.962018 33823 cached_discovery.go:121] skipped caching discovery info due to Get https://10.10.16.251:6443/api?timeout=32s: dial tcp 10.10.16.251:6443: connect: connection refused I0910 02:57:10.962033 33823 shortcut.go:89] Error loading discovery information: Get https://10.10.16.251:6443/api?timeout=32s: dial tcp 10.10.16.251:6443: connect: connection refused I0910 02:57:10.962086 33823 round_trippers.go:420] GET https://10.10.16.251:6443/api?timeout=32s I0910 02:57:10.962096 33823 round_trippers.go:427] Request Headers: I0910 02:57:10.962104 33823 round_trippers.go:431] Accept: application/json, */* I0910 02:57:10.962111 33823 round_trippers.go:431] User-Agent: kubectl/v1.18.1 (linux/arm64) kubernetes/7879fc1 I0910 02:57:10.962245 33823 round_trippers.go:446] Response Status: in 0 milliseconds I0910 02:57:10.962255 33823 round_trippers.go:449] Response Headers: I0910 02:57:10.962278 33823 cached_discovery.go:121] skipped caching discovery info due to Get https://10.10.16.251:6443/api?timeout=32s: dial tcp 10.10.16.251:6443: connect: connection refused I0910 02:57:10.962330 33823 round_trippers.go:420] GET https://10.10.16.251:6443/api?timeout=32s I0910 02:57:10.962339 33823 round_trippers.go:427] Request Headers: I0910 02:57:10.962347 33823 round_trippers.go:431] Accept: application/json, */* I0910 02:57:10.962355 33823 round_trippers.go:431] User-Agent: kubectl/v1.18.1 (linux/arm64) kubernetes/7879fc1 I0910 02:57:10.962480 33823 round_trippers.go:446] Response Status: in 0 milliseconds I0910 02:57:10.962490 33823 round_trippers.go:449] Response Headers: I0910 02:57:10.962512 33823 cached_discovery.go:121] skipped caching discovery info due to Get https://10.10.16.251:6443/api?timeout=32s: dial tcp 10.10.16.251:6443: connect: connection refused I0910 02:57:10.962561 33823 round_trippers.go:420] GET https://10.10.16.251:6443/api?timeout=32s I0910 02:57:10.962569 33823 round_trippers.go:427] Request Headers: I0910 02:57:10.962578 33823 round_trippers.go:431] Accept: application/json, */* I0910 02:57:10.962586 33823 round_trippers.go:431] User-Agent: kubectl/v1.18.1 (linux/arm64) kubernetes/7879fc1 I0910 02:57:10.962712 33823 round_trippers.go:446] Response Status: in 0 milliseconds I0910 02:57:10.962723 33823 round_trippers.go:449] Response Headers: I0910 02:57:10.962744 33823 cached_discovery.go:121] skipped caching discovery info due to Get https://10.10.16.251:6443/api?timeout=32s: dial tcp 10.10.16.251:6443: connect: connection refused I0910 02:57:10.962768 33823 helpers.go:234] Connection error: Get https://10.10.16.251:6443/api?timeout=32s: dial tcp 10.10.16.251:6443: connect: connection refused F0910 02:57:10.962786 33823 helpers.go:115] The connection to the server 10.10.16.251:6443 was refused - did you specify the right host or port? [root@centos7 ~]#
swapoff -a解决
[root@centos7 ~]# swapoff -a [root@centos7 ~]# kubectl get pod NAME READY STATUS RESTARTS AGE antrea-debian 0/1 Completed 3 49m apache-app-68cbd94946-gk7hv 0/1 Completed 1 14d apache-app-68cbd94946-p92jh 0/1 Completed 1 14d nginx-app-56b5bb67cc-6hjgt 0/1 Completed 1 14d nginx-app-56b5bb67cc-hnhm9 0/1 Completed 1 14d nginx-karmada-f89759699-8xmfw 0/1 Completed 1 16d [root@centos7 ~]#