client
root@cloud:~# ip a 1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000 link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 inet 127.0.0.1/8 scope host lo valid_lft forever preferred_lft forever inet6 ::1/128 scope host valid_lft forever preferred_lft forever 2: enahisic2i0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq state UP group default qlen 1000 link/ether 9c:52:f8:67:c4:d3 brd ff:ff:ff:ff:ff:ff inet 10.10.16.47/24 brd 10.10.16.255 scope global enahisic2i0 valid_lft forever preferred_lft forever inet6 fe80::9e52:f8ff:fe67:c4d3/64 scope link valid_lft forever preferred_lft forever
root@ubuntu:~# kubectl get svc NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE kubernetes ClusterIP 10.96.0.1 <none> 443/TCP 243d my-nginx ClusterIP 10.110.79.116 <none> 8280/TCP 35d my-nginx-np NodePort 10.99.1.231 <none> 8081:31199/TCP 35d web2 NodePort 10.110.171.213 <none> 8097:31866/TCP 19d root@ubuntu:~# kubectl get svc my-nginx-np -o yaml apiVersion: v1 kind: Service metadata: annotations: kubectl.kubernetes.io/last-applied-configuration: | {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{},"labels":{"run":"my-nginx"},"name":"my-nginx-np","namespace":"default"},"spec":{"ports":[{"nodePort":31199,"port":8080,"protocol":"TCP","targetPort":80}],"selector":{"run":"my-nginx"},"type":"NodePort"},"status":{"loadBalancer":{}}} creationTimestamp: "2021-05-12T08:39:09Z" labels: run: my-nginx managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:metadata: f:annotations: .: {} f:kubectl.kubernetes.io/last-applied-configuration: {} f:labels: .: {} f:run: {} f:spec: f:externalTrafficPolicy: {} f:ports: .: {} k:{"port":8081,"protocol":"TCP"}: .: {} f:nodePort: {} f:port: {} f:protocol: {} f:targetPort: {} f:selector: .: {} f:run: {} f:sessionAffinity: {} f:type: {} manager: kubectl operation: Update time: "2021-05-12T08:47:48Z" name: my-nginx-np namespace: default resourceVersion: "39417422" selfLink: /api/v1/namespaces/default/services/my-nginx-np uid: 1ef49071-e7b5-4a92-8785-b5158606731e spec: clusterIP: 10.99.1.231 externalTrafficPolicy: Cluster ports: - nodePort: 31199 port: 8081 protocol: TCP targetPort: 80 selector: run: my-nginx sessionAffinity: None type: NodePort status: loadBalancer: {} root@ubuntu:~#
root@ubuntu:~# kubectl get pod NAME READY STATUS RESTARTS AGE debian-6c44fc6956-ltsrt 0/1 CrashLoopBackOff 4773 16d mc1 2/2 Running 0 16d my-deployment-68bdbbb5cc-bbszv 0/1 ImagePullBackOff 0 35d my-deployment-68bdbbb5cc-nrst9 0/1 ImagePullBackOff 0 35d my-deployment-68bdbbb5cc-rlgzt 0/1 ImagePullBackOff 0 35d my-nginx-5dc4865748-jqx54 1/1 Running 2 35d my-nginx-5dc4865748-pcrbg 1/1 Running 2 35d nginx 0/1 ImagePullBackOff 0 35d nginx-deployment-6b474476c4-r6z5b 1/1 Running 0 8d nginx-deployment-6b474476c4-w6xh9 1/1 Running 0 8d web2-6d784f67bf-4gqq2 1/1 Running 0 19d root@ubuntu:~#
root@ubuntu:~# kubectl -n default describe pod my-nginx-5dc4865748-jqx54 | grep Container Containers: Container ID: containerd://270994804d4c87638dcfd98a9ca1489f8c268998a0930cc5d3eef52887797777 ContainersReady True root@ubuntu:~# kubectl -n default describe pod my-nginx-5dc4865748-pcrbg | grep Container Containers: Container ID: containerd://7617bb50d622d23fb26feaffba96c832ea53d48bad66502e18dd4f77480d3d98 ContainersReady True root@ubuntu:~#
root@ubuntu:~# crictl inspect 270994804d4c87638dcfd98a9ca1489f8c268998a0930cc5d3eef52887797777 | grep -i pid "pid": 27134, "pid": 1 "type": "pid" root@ubuntu:~# crictl inspect 7617bb50d622d23fb26feaffba96c832ea53d48bad66502e18dd4f77480d3d98 | grep -i pid "pid": 27324, "pid": 1 "type": "pid" root@ubuntu:~#
从客户端ping
root@cloud:~# telnet 10.10.16.82 31199 Trying 10.10.16.82... Connected to 10.10.16.82. Escape character is '^]'. ^CConnection closed by foreign host. root@cloud:~# telnet 10.10.16.82 31199 Trying 10.10.16.82... Connected to 10.10.16.82. Escape character is '^]'. ^CConnection closed by foreign host. root@cloud:~#
root@ubuntu:~# nsenter -n --target 27134 root@ubuntu:~# ip a 1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000 link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 inet 127.0.0.1/8 scope host lo valid_lft forever preferred_lft forever inet6 ::1/128 scope host valid_lft forever preferred_lft forever 3: eth0@if641: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1450 qdisc noqueue state UP group default link/ether fa:66:b3:ab:05:9f brd ff:ff:ff:ff:ff:ff link-netnsid 0 inet 10.244.0.19/24 brd 10.244.0.255 scope global eth0 valid_lft forever preferred_lft forever inet6 fe80::f866:b3ff:feab:59f/64 scope link valid_lft forever preferred_lft forever root@ubuntu:~# tcpdump -i eth0 icmp -nnvv tcpdump: listening on eth0, link-type EN10MB (Ethernet), capture size 262144 bytes ^C 0 packets captured 0 packets received by filter 0 packets dropped by kernel root@ubuntu:~# tcpdump -i eth0 tcp -nnvv tcpdump: listening on eth0, link-type EN10MB (Ethernet), capture size 262144 bytes 09:25:08.212080 IP (tos 0x10, ttl 63, id 25166, offset 0, flags [DF], proto TCP (6), length 60) 10.244.0.1.10054 > 10.244.0.20.80: Flags [S], cksum 0x0331 (correct), seq 1619805106, win 64240, options [mss 1460,sackOK,TS val 3487078452 ecr 0,nop,wscale 7], length 0
root@ubuntu:~# nsenter -n --target 27324 root@ubuntu:~# ip a 1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000 link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 inet 127.0.0.1/8 scope host lo valid_lft forever preferred_lft forever inet6 ::1/128 scope host valid_lft forever preferred_lft forever 3: eth0@if642: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1450 qdisc noqueue state UP group default link/ether 6e:5a:30:bc:6d:5b brd ff:ff:ff:ff:ff:ff link-netnsid 0 inet 10.244.0.20/24 brd 10.244.0.255 scope global eth0 valid_lft forever preferred_lft forever inet6 fe80::6c5a:30ff:febc:6d5b/64 scope link valid_lft forever preferred_lft forever root@ubuntu:~# tcpdump -i eth0 icmp -nnvv tcpdump: listening on eth0, link-type EN10MB (Ethernet), capture size 262144 bytes ^C 0 packets captured 0 packets received by filter 0 packets dropped by kernel root@ubuntu:~# tcpdump -i eth0 tcp -nnvv tcpdump: listening on eth0, link-type EN10MB (Ethernet), capture size 262144 bytes 09:25:08.212027 IP (tos 0x10, ttl 63, id 25166, offset 0, flags [DF], proto TCP (6), length 60) 10.244.0.1.10054 > 10.244.0.20.80: Flags [S], cksum 0x0331 (correct), seq 1619805106, win 64240, options [mss 1460,sackOK,TS val 3487078452 ecr 0,nop,wscale 7], length 0 09:25:08.212074 IP (tos 0x0, ttl 64, id 0, offset 0, flags [DF], proto TCP (6), length 60) 10.244.0.20.80 > 10.244.0.1.10054: Flags [S.], cksum 0x162b (incorrect -> 0x6794), seq 465847927, ack 1619805107, win 64308, options [mss 1410,sackOK,TS val 2479727984 ecr 3487078452,nop,wscale 7], length 0 09:25:08.212185 IP (tos 0x10, ttl 63, id 25167, offset 0, flags [DF], proto TCP (6), length 52) 10.244.0.1.10054 > 10.244.0.20.80: Flags [.], cksum 0x8f6c (correct), seq 1, ack 1, win 502, options [nop,nop,TS val 3487078453 ecr 2479727984], length 0 09:25:13.484043 IP (tos 0x10, ttl 63, id 25168, offset 0, flags [DF], proto TCP (6), length 57) 10.244.0.1.10054 > 10.244.0.20.80: Flags [P.], cksum 0x74d4 (correct), seq 1:6, ack 1, win 502, options [nop,nop,TS val 3487083725 ecr 2479727984], length 5: HTTP 09:25:13.484079 IP (tos 0x0, ttl 64, id 51806, offset 0, flags [DF], proto TCP (6), length 52) 10.244.0.20.80 > 10.244.0.1.10054: Flags [.], cksum 0x1623 (incorrect -> 0x6636), seq 1, ack 6, win 503, options [nop,nop,TS val 2479733256 ecr 3487083725], length 0 09:25:13.484223 IP (tos 0x0, ttl 64, id 51807, offset 0, flags [DF], proto TCP (6), length 363) 10.244.0.20.80 > 10.244.0.1.10054: Flags [P.], cksum 0x175a (incorrect -> 0x1b1c), seq 1:312, ack 6, win 503, options [nop,nop,TS val 2479733257 ecr 3487083725], length 311: HTTP, length: 311 HTTP/1.1 400 Bad Request Server: nginx/1.19.10 Date: Thu, 17 Jun 2021 01:25:13 GMT Content-Type: text/html Content-Length: 158 Connection: close <html> <head><title>400 Bad Request</title></head> <body> <center><h1>400 Bad Request</h1></center> <hr><center>nginx/1.19.10</center> </body> </html> 09:25:13.484321 IP (tos 0x0, ttl 64, id 51808, offset 0, flags [DF], proto TCP (6), length 52) 10.244.0.20.80 > 10.244.0.1.10054: Flags [F.], cksum 0x1623 (incorrect -> 0x64fd), seq 312, ack 6, win 503, options [nop,nop,TS val 2479733257 ecr 3487083725], length 0 09:25:13.484367 IP (tos 0x10, ttl 63, id 25169, offset 0, flags [DF], proto TCP (6), length 52) 10.244.0.1.10054 > 10.244.0.20.80: Flags [.], cksum 0x6500 (correct), seq 6, ack 312, win 501, options [nop,nop,TS val 3487083725 ecr 2479733257], length 0 09:25:13.484412 IP (tos 0x10, ttl 63, id 25170, offset 0, flags [DF], proto TCP (6), length 52) 10.244.0.1.10054 > 10.244.0.20.80: Flags [F.], cksum 0x64fe (correct), seq 6, ack 313, win 501, options [nop,nop,TS val 3487083725 ecr 2479733257], length 0 09:25:13.484427 IP (tos 0x0, ttl 64, id 51809, offset 0, flags [DF], proto TCP (6), length 52) 10.244.0.20.80 > 10.244.0.1.10054: Flags [.], cksum 0x1623 (incorrect -> 0x64fc), seq 313, ack 7, win 503, options [nop,nop,TS val 2479733257 ecr 3487083725], length 0
再telnet 一次
root@cloud:~# telnet 10.10.16.82 31199 Trying 10.10.16.82... Connected to 10.10.16.82. Escape character is '^]'. ^CConnection closed by foreign host. root@cloud:~#
root@ubuntu:~# nsenter -n --target 27134 root@ubuntu:~# ip a 1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000 link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 inet 127.0.0.1/8 scope host lo valid_lft forever preferred_lft forever inet6 ::1/128 scope host valid_lft forever preferred_lft forever 3: eth0@if641: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1450 qdisc noqueue state UP group default link/ether fa:66:b3:ab:05:9f brd ff:ff:ff:ff:ff:ff link-netnsid 0 inet 10.244.0.19/24 brd 10.244.0.255 scope global eth0 valid_lft forever preferred_lft forever inet6 fe80::f866:b3ff:feab:59f/64 scope link valid_lft forever preferred_lft forever
09:27:27.397163 IP (tos 0x10, ttl 63, id 60223, offset 0, flags [DF], proto TCP (6), length 60) 10.244.0.1.28407 > 10.244.0.19.80: Flags [S], cksum 0x6fd3 (correct), seq 882875289, win 64240, options [mss 1460,sackOK,TS val 3487217637 ecr 0,nop,wscale 7], length 0 09:27:27.397214 IP (tos 0x0, ttl 64, id 0, offset 0, flags [DF], proto TCP (6), length 60) 10.244.0.19.80 > 10.244.0.1.28407: Flags [S.], cksum 0x162a (incorrect -> 0xceca), seq 1817758467, ack 882875290, win 64308, options [mss 1410,sackOK,TS val 1135551962 ecr 3487217637,nop,wscale 7], length 0 09:27:27.397305 IP (tos 0x10, ttl 63, id 60224, offset 0, flags [DF], proto TCP (6), length 52) 10.244.0.1.28407 > 10.244.0.19.80: Flags [.], cksum 0xf6a2 (correct), seq 1, ack 1, win 502, options [nop,nop,TS val 3487217638 ecr 1135551962], length 0 09:27:29.680979 IP (tos 0x10, ttl 63, id 60225, offset 0, flags [DF], proto TCP (6), length 57) 10.244.0.1.28407 > 10.244.0.19.80: Flags [P.], cksum 0xe7b7 (correct), seq 1:6, ack 1, win 502, options [nop,nop,TS val 3487219921 ecr 1135551962], length 5: HTTP 09:27:29.681014 IP (tos 0x0, ttl 64, id 57110, offset 0, flags [DF], proto TCP (6), length 52) 10.244.0.19.80 > 10.244.0.1.28407: Flags [.], cksum 0x1622 (incorrect -> 0xe4c6), seq 1, ack 6, win 503, options [nop,nop,TS val 1135554245 ecr 3487219921], length 0 09:27:29.681118 IP (tos 0x0, ttl 64, id 57111, offset 0, flags [DF], proto TCP (6), length 363) 10.244.0.19.80 > 10.244.0.1.28407: Flags [P.], cksum 0x1759 (incorrect -> 0x96a7), seq 1:312, ack 6, win 503, options [nop,nop,TS val 1135554245 ecr 3487219921], length 311: HTTP, length: 311 HTTP/1.1 400 Bad Request Server: nginx/1.19.10 Date: Thu, 17 Jun 2021 01:27:29 GMT Content-Type: text/html Content-Length: 158 Connection: close <html> <head><title>400 Bad Request</title></head> <body> <center><h1>400 Bad Request</h1></center> <hr><center>nginx/1.19.10</center> </body> </html> 09:27:29.681157 IP (tos 0x0, ttl 64, id 57112, offset 0, flags [DF], proto TCP (6), length 52) 10.244.0.19.80 > 10.244.0.1.28407: Flags [F.], cksum 0x1622 (incorrect -> 0xe38e), seq 312, ack 6, win 503, options [nop,nop,TS val 1135554245 ecr 3487219921], length 0 09:27:29.681210 IP (tos 0x10, ttl 63, id 60226, offset 0, flags [DF], proto TCP (6), length 52) 10.244.0.1.28407 > 10.244.0.19.80: Flags [.], cksum 0xe390 (correct), seq 6, ack 312, win 501, options [nop,nop,TS val 3487219922 ecr 1135554245], length 0 09:27:29.681255 IP (tos 0x10, ttl 63, id 60227, offset 0, flags [DF], proto TCP (6), length 52) 10.244.0.1.28407 > 10.244.0.19.80: Flags [F.], cksum 0xe38e (correct), seq 6, ack 313, win 501, options [nop,nop,TS val 3487219922 ecr 1135554245], length 0 09:27:29.681270 IP (tos 0x0, ttl 64, id 57113, offset 0, flags [DF], proto TCP (6), length 52) 10.244.0.19.80 > 10.244.0.1.28407: Flags [.], cksum 0x1622 (incorrect -> 0xe38b), seq 313, ack 7, win 503, options [nop,nop,TS val 1135554246 ecr 3487219922], length 0
externalTrafficPolicy: Local
root@ubuntu:~# kubectl edit svc/my-nginx-np
externalTrafficPolicy: Local
root@ubuntu:~# kubectl get svc my-nginx-np -o yaml apiVersion: v1 kind: Service metadata: annotations: kubectl.kubernetes.io/last-applied-configuration: | {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{},"labels":{"run":"my-nginx"},"name":"my-nginx-np","namespace":"default"},"spec":{"ports":[{"nodePort":31199,"port":8080,"protocol":"TCP","targetPort":80}],"selector":{"run":"my-nginx"},"type":"NodePort"},"status":{"loadBalancer":{}}} creationTimestamp: "2021-05-12T08:39:09Z" labels: run: my-nginx managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:metadata: f:annotations: .: {} f:kubectl.kubernetes.io/last-applied-configuration: {} f:labels: .: {} f:run: {} f:spec: f:externalTrafficPolicy: {} f:ports: .: {} k:{"port":8081,"protocol":"TCP"}: .: {} f:nodePort: {} f:port: {} f:protocol: {} f:targetPort: {} f:selector: .: {} f:run: {} f:sessionAffinity: {} f:type: {} manager: kubectl operation: Update time: "2021-06-17T01:30:58Z" name: my-nginx-np namespace: default resourceVersion: "46377335" selfLink: /api/v1/namespaces/default/services/my-nginx-np uid: 1ef49071-e7b5-4a92-8785-b5158606731e spec: clusterIP: 10.99.1.231 externalTrafficPolicy: Local ports: - nodePort: 31199 port: 8081 protocol: TCP targetPort: 80 selector: run: my-nginx sessionAffinity: None type: NodePort status: loadBalancer: {}
root@cloud:~# telnet 10.10.16.82 31199 Trying 10.10.16.82... Connected to 10.10.16.82. Escape character is '^]'. Connection closed by foreign host. root@cloud:~#
root@cloud:~# telnet 10.10.16.82 31199 Trying 10.10.16.82... Connected to 10.10.16.82. Escape character is '^]'. Connection closed by foreign host. root@cloud:~#
root@ubuntu:~# ip a 1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000 link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 inet 127.0.0.1/8 scope host lo valid_lft forever preferred_lft forever inet6 ::1/128 scope host valid_lft forever preferred_lft forever 3: eth0@if642: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1450 qdisc noqueue state UP group default link/ether 6e:5a:30:bc:6d:5b brd ff:ff:ff:ff:ff:ff link-netnsid 0 inet 10.244.0.20/24 brd 10.244.0.255 scope global eth0 valid_lft forever preferred_lft forever inet6 fe80::6c5a:30ff:febc:6d5b/64 scope link valid_lft forever preferred_lft forever
root@ubuntu:~# ip a 1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000 link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 inet 127.0.0.1/8 scope host lo valid_lft forever preferred_lft forever inet6 ::1/128 scope host valid_lft forever preferred_lft forever 3: eth0@if642: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1450 qdisc noqueue state UP group default link/ether 6e:5a:30:bc:6d:5b brd ff:ff:ff:ff:ff:ff link-netnsid 0 inet 10.244.0.20/24 brd 10.244.0.255 scope global eth0 valid_lft forever preferred_lft forever inet6 fe80::6c5a:30ff:febc:6d5b/64 scope link valid_lft forever preferred_lft forever
root@ubuntu:~# tcpdump -i eth0 tcp -nnvv tcpdump: listening on eth0, link-type EN10MB (Ethernet), capture size 262144 bytes 09:32:14.837750 IP (tos 0x10, ttl 63, id 54404, offset 0, flags [DF], proto TCP (6), length 60) 10.10.16.47.44812 > 10.244.0.20.80: Flags [S], cksum 0x75c0 (correct), seq 1162007257, win 64240, options [mss 1460,sackOK,TS val 3487505078 ecr 0,nop,wscale 7], length 0 09:32:14.837802 IP (tos 0x0, ttl 64, id 0, offset 0, flags [DF], proto TCP (6), length 60) 10.244.0.20.80 > 10.10.16.47.44812: Flags [S.], cksum 0x256f (incorrect -> 0xf825), seq 3731283071, ack 1162007258, win 64308, options [mss 1410,sackOK,TS val 954370478 ecr 3487505078,nop,wscale 7], length 0 09:32:14.837925 IP (tos 0x10, ttl 63, id 54405, offset 0, flags [DF], proto TCP (6), length 52) 10.10.16.47.44812 > 10.244.0.20.80: Flags [.], cksum 0x1fff (correct), seq 1, ack 1, win 502, options [nop,nop,TS val 3487505078 ecr 954370478], length 0 ^C 3 packets captured 3 packets received by filter 0 packets dropped by kernel root@ubuntu:~#
root@ubuntu:~# ip a 1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000 link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 inet 127.0.0.1/8 scope host lo valid_lft forever preferred_lft forever inet6 ::1/128 scope host valid_lft forever preferred_lft forever 3: eth0@if641: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1450 qdisc noqueue state UP group default link/ether fa:66:b3:ab:05:9f brd ff:ff:ff:ff:ff:ff link-netnsid 0 inet 10.244.0.19/24 brd 10.244.0.255 scope global eth0 valid_lft forever preferred_lft forever inet6 fe80::f866:b3ff:feab:59f/64 scope link valid_lft forever preferred_lft forever root@ubuntu:~# tcpdump -i eth0 tcp -nnvv tcpdump: listening on eth0, link-type EN10MB (Ethernet), capture size 262144 bytes
再telnet一次
root@ubuntu:~# ip a 1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000 link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 inet 127.0.0.1/8 scope host lo valid_lft forever preferred_lft forever inet6 ::1/128 scope host valid_lft forever preferred_lft forever 3: eth0@if641: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1450 qdisc noqueue state UP group default link/ether fa:66:b3:ab:05:9f brd ff:ff:ff:ff:ff:ff link-netnsid 0 inet 10.244.0.19/24 brd 10.244.0.255 scope global eth0 valid_lft forever preferred_lft forever inet6 fe80::f866:b3ff:feab:59f/64 scope link valid_lft forever preferred_lft forever root@ubuntu:~# tcpdump -i eth0 tcp -nnvv tcpdump: listening on eth0, link-type EN10MB (Ethernet), capture size 262144 bytes 09:35:56.586113 IP (tos 0x10, ttl 63, id 41459, offset 0, flags [DF], proto TCP (6), length 60) 10.10.16.47.44814 > 10.244.0.19.80: Flags [S], cksum 0x09d3 (correct), seq 2463076097, win 64240, options [mss 1460,sackOK,TS val 3487726826 ecr 0,nop,wscale 7], length 0 09:35:56.586173 IP (tos 0x0, ttl 64, id 0, offset 0, flags [DF], proto TCP (6), length 60) 10.244.0.19.80 > 10.10.16.47.44814: Flags [S.], cksum 0x256e (incorrect -> 0x2b5c), seq 3412747048, ack 2463076098, win 64308, options [mss 1410,sackOK,TS val 2384339362 ecr 3487726826,nop,wscale 7], length 0 09:35:56.586262 IP (tos 0x10, ttl 63, id 41460, offset 0, flags [DF], proto TCP (6), length 52) 10.10.16.47.44814 > 10.244.0.19.80: Flags [.], cksum 0x5335 (correct), seq 1, ack 1, win 502, options [nop,nop,TS val 3487726826 ecr 2384339362], length 0 09:35:59.525713 IP (tos 0x10, ttl 63, id 41461, offset 0, flags [DF], proto TCP (6), length 57) 10.10.16.47.44814 > 10.244.0.19.80: Flags [P.], cksum 0x41b9 (correct), seq 1:6, ack 1, win 502, options [nop,nop,TS val 3487729766 ecr 2384339362], length 5: HTTP 09:35:59.525758 IP (tos 0x0, ttl 64, id 26237, offset 0, flags [DF], proto TCP (6), length 52) 10.244.0.19.80 > 10.10.16.47.44814: Flags [.], cksum 0x2566 (incorrect -> 0x3c37), seq 1, ack 6, win 503, options [nop,nop,TS val 2384342302 ecr 3487729766], length 0 09:35:59.525865 IP (tos 0x0, ttl 64, id 26238, offset 0, flags [DF], proto TCP (6), length 363) 10.244.0.19.80 > 10.10.16.47.44814: Flags [P.], cksum 0x269d (incorrect -> 0xed16), seq 1:312, ack 6, win 503, options [nop,nop,TS val 2384342302 ecr 3487729766], length 311: HTTP, length: 311 HTTP/1.1 400 Bad Request Server: nginx/1.19.10 Date: Thu, 17 Jun 2021 01:35:59 GMT Content-Type: text/html Content-Length: 158 Connection: close <html> <head><title>400 Bad Request</title></head> <body> <center><h1>400 Bad Request</h1></center> <hr><center>nginx/1.19.10</center> </body> </html> 09:35:59.525900 IP (tos 0x0, ttl 64, id 26239, offset 0, flags [DF], proto TCP (6), length 52) 10.244.0.19.80 > 10.10.16.47.44814: Flags [F.], cksum 0x2566 (incorrect -> 0x3aff), seq 312, ack 6, win 503, options [nop,nop,TS val 2384342302 ecr 3487729766], length 0 09:35:59.525940 IP (tos 0x10, ttl 63, id 41462, offset 0, flags [DF], proto TCP (6), length 52) 10.10.16.47.44814 > 10.244.0.19.80: Flags [.], cksum 0x3b02 (correct), seq 6, ack 312, win 501, options [nop,nop,TS val 3487729766 ecr 2384342302], length 0 09:35:59.525993 IP (tos 0x10, ttl 63, id 41463, offset 0, flags [DF], proto TCP (6), length 52) 10.10.16.47.44814 > 10.244.0.19.80: Flags [F.], cksum 0x3b00 (correct), seq 6, ack 313, win 501, options [nop,nop,TS val 3487729766 ecr 2384342302], length 0 09:35:59.526005 IP (tos 0x0, ttl 64, id 26240, offset 0, flags [DF], proto TCP (6), length 52) 10.244.0.19.80 > 10.10.16.47.44814: Flags [.], cksum 0x2566 (incorrect -> 0x3afe), seq 313, ack 7, win 503, options [nop,nop,TS val 2384342302 ecr 3487729766], length
root@ubuntu:~# tcpdump -i eth0 tcp -nnvv tcpdump: listening on eth0, link-type EN10MB (Ethernet), capture size 262144 bytes 09:47:02.283093 IP (tos 0x10, ttl 63, id 41444, offset 0, flags [DF], proto TCP (6), length 60) 10.10.16.47.44818 > 10.244.0.19.80: Flags [S], cksum 0xd4ae (correct), seq 3850455300, win 64240, options [mss 1460,sackOK,TS val 3488392523 ecr 0,nop,wscale 7], length 0 09:47:02.283146 IP (tos 0x0, ttl 64, id 0, offset 0, flags [DF], proto TCP (6), length 60) 10.244.0.19.80 > 10.10.16.47.44818: Flags [S.], cksum 0x256e (incorrect -> 0xa3fb), seq 2900667263, ack 3850455301, win 64308, options [mss 1410,sackOK,TS val 2385005059 ecr 3488392523,nop,wscale 7], length 0 09:47:02.283249 IP (tos 0x10, ttl 63, id 41445, offset 0, flags [DF], proto TCP (6), length 52) 10.10.16.47.44818 > 10.244.0.19.80: Flags [.], cksum 0xcbd4 (correct), seq 1, ack 1, win 502, options [nop,nop,TS val 3488392523 ecr 2385005059], length 0 09:47:03.661387 IP (tos 0x10, ttl 63, id 41446, offset 0, flags [DF], proto TCP (6), length 57) 10.10.16.47.44818 > 10.244.0.19.80: Flags [P.], cksum 0xc072 (correct), seq 1:6, ack 1, win 502, options [nop,nop,TS val 3488393901 ecr 2385005059], length 5: HTTP 09:47:03.661427 IP (tos 0x0, ttl 64, id 9452, offset 0, flags [DF], proto TCP (6), length 52) 10.244.0.19.80 > 10.10.16.47.44818: Flags [.], cksum 0x2566 (incorrect -> 0xc109), seq 1, ack 6, win 503, options [nop,nop,TS val 2385006438 ecr 3488393901], length 0 09:47:03.661592 IP (tos 0x0, ttl 64, id 9453, offset 0, flags [DF], proto TCP (6), length 363) 10.244.0.19.80 > 10.10.16.47.44818: Flags [P.], cksum 0x269d (incorrect -> 0x74ee), seq 1:312, ack 6, win 503, options [nop,nop,TS val 2385006438 ecr 3488393901], length 311: HTTP, length: 311 HTTP/1.1 400 Bad Request Server: nginx/1.19.10 Date: Thu, 17 Jun 2021 01:47:03 GMT Content-Type: text/html Content-Length: 158 Connection: close <html> <head><title>400 Bad Request</title></head> <body> <center><h1>400 Bad Request</h1></center> <hr><center>nginx/1.19.10</center> </body> </html> 09:47:03.661646 IP (tos 0x0, ttl 64, id 9454, offset 0, flags [DF], proto TCP (6), length 52) 10.244.0.19.80 > 10.10.16.47.44818: Flags [F.], cksum 0x2566 (incorrect -> 0xbfd1), seq 312, ack 6, win 503, options [nop,nop,TS val 2385006438 ecr 3488393901], length 0 09:47:03.661673 IP (tos 0x10, ttl 63, id 41447, offset 0, flags [DF], proto TCP (6), length 52) 10.10.16.47.44818 > 10.244.0.19.80: Flags [.], cksum 0xbfd4 (correct), seq 6, ack 312, win 501, options [nop,nop,TS val 3488393901 ecr 2385006438], length 0 09:47:03.661733 IP (tos 0x10, ttl 63, id 41448, offset 0, flags [DF], proto TCP (6), length 52) 10.10.16.47.44818 > 10.244.0.19.80: Flags [F.], cksum 0xbfd2 (correct), seq 6, ack 313, win 501, options [nop,nop,TS val 3488393901 ecr 2385006438], length 0 09:47:03.661744 IP (tos 0x0, ttl 64, id 9455, offset 0, flags [DF], proto TCP (6), length 52) 10.244.0.19.80 > 10.10.16.47.44818: Flags [.], cksum 0x2566 (incorrect -> 0xbfd0), seq 313, ack 7, win 503, options [nop,nop,TS val 2385006438 ecr 3488393901], length 0
tcp 6 108 TIME_WAIT src=10.10.16.47 dst=10.10.16.82 sport=44816 dport=31199 src=10.244.0.20 dst=10.10.16.47 sport=80 dport=44816 [ASSURED] mark=0 use=1 conntrack v1.4.4 (conntrack-tools): 181 flow entries have been shown. tcp 6 111 TIME_WAIT src=10.10.16.47 dst=10.10.16.82 sport=44818 dport=31199 src=10.244.0.19 dst=10.10.16.47 sport=80 dport=44818 [ASSURED] mark=0 use=1 root@ubuntu:~# conntrack -L -o ktimestamp | grep 10.10.16.47 tcp 6 1 TIME_WAIT src=10.10.16.47 dst=10.10.16.82 sport=44816 dport=31199 src=10.244.0.20 dst=10.10.16.47 sport=80 dport=44816 [ASSURED] mark=0 use=1 conntrack v1.4.4 (conntrack-tools): 185 flow entries have been shown. tcp 6 4 TIME_WAIT src=10.10.16.47 dst=10.10.16.82 sport=44818 dport=31199 src=10.244.0.19 dst=10.10.16.47 sport=80 dport=44818 [ASSURED] mark=0 use=1 root@ubuntu:~#
為什麼需要 SNAT
當設定 ExternalTrafficPolicy 為 Cluster 時，會在 Node 上經過一次的 SNAT 然後才到達 Pod，所以我們在 Pod 上看到的來源 IP 永遠會是 Node IP，那 Kube Proxy 為甚麼要這樣做呢，我們來看看如果沒有 NAT 的話會發生什麼事情，為了簡化我們把 Node2 移除，假設客戶端從 Node3 想要存取服務：
Routing without SNAT at arrival node
- 首先 Client 向 Node3 (假設這個 Node 沒有 Pod 跑在上面) 發送要求。
- Node3 將封包轉向 Pod IP (只有 DNAT)。
- Pod 回應。
- Node1 將封包轉向 Client。
這張圖可以很明顯的看出來，Client 向 Node3 發送要求，卻收到 Node1 的回應，在大多數情況這樣是行不通的，甚至在防火牆就會被擋掉。所以 Kube Proxy 才會讓 Pod 先回傳給原始的 Node，再由 Node 負責回應：
Routing with SNAT at arrival node
雖然會產生額外的步驟，但也只有這樣能確保路由順暢。