qemu-system-aarch64 -m 1024 -kernel vmlinuz-5.4.34-88 -nographic -initrd kata-containers.img -append console=ttyS0 -machine virt
root@ubuntu:/etc/docker# docker ps -a CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES 4e9f50721226 busybox "sh" 2 hours ago Up 2 hours funny_diffie root@ubuntu:/etc/docker# id=$(sudo docker ps -q --no-trunc) root@ubuntu:/etc/docker# echo $id 4e9f5072122614c41960d734e1b977fa882a641a84e8b263ddba1664effe0e6f root@ubuntu:/etc/docker# ls /var/run/vc/vm/ 4e9f5072122614c41960d734e1b977fa882a641a84e8b263ddba1664effe0e6f root@ubuntu:/etc/docker# ls /var/run/vc/vm/4e9f5072122614c41960d734e1b977fa882a641a84e8b263ddba1664effe0e6f/ console.sock kata.sock pid qemu.log qmp.sock root@ubuntu:/etc/docker# ls /var/run/vc/vm/4e9f5072122614c41960d734e1b977fa882a641a84e8b263ddba1664effe0e6f/console.sock /var/run/vc/vm/4e9f5072122614c41960d734e1b977fa882a641a84e8b263ddba1664effe0e6f/console.sock root@ubuntu:/etc/docker# console="/var/run/vc/vm/4e9f5072122614c41960d734e1b977fa882a641a84e8b263ddba1664effe0e6f/console.sock" root@ubuntu:/etc/docker# socat "stdin,raw,echo=0,escape=0x11" "unix-connect:${console}"
root@ubuntu:/etc/docker# kata-runtime list ID PID STATUS BUNDLE CREATED OWNER e12a7db6fb05df044a59a19bb03c39fe7752e4d684a8e2e58822b88606d3ac3e 12665 running /run/containerd/io.containerd.runtime.v1.linux/moby/e12a7db6fb05df044a59a19bb03c39fe7752e4d684a8e2e58822b88606d3ac3e 2020-10-10T08:40:39.593030395Z #0
root@ubuntu:/home/ubuntu# kata-runtime exec e12a7db6fb05df044a59a19bb03c39fe7752e4d684a8e2e58822b88606d3ac3e rpc error: code = Internal desc = Could not run process: container_linux.go:349: starting container process caused "panic from initialization: runtime error: index out of range, goroutine 1 [running, locked to thread]: runtime/debug.Stack(0x400018fbd8, 0xaaaab1b68260, 0xaaaab21de220) /usr/go/src/runtime/debug/stack.go:24 +0x88 github.com/kata-containers/agent/vendor/github.com/opencontainers/runc/libcontainer.(*LinuxFactory).StartInitialization.func2(0x400018fea0) /root/go/src/github.com/kata-containers/agent/vendor/github.com/opencontainers/runc/libcontainer/factory_linux.go:370 +0x40 panic(0xaaaab1b68260, 0xaaaab21de220) /usr/go/src/runtime/panic.go:513 +0x18c github.com/kata-containers/agent/vendor/github.com/opencontainers/runc/libcontainer.(*linuxSetnsInit).Init(0x400012d9c0, 0x0, 0x0) /root/go/src/github.com/kata-containers/agent/vendor/github.com/opencontainers/runc/libcontainer/setns_init_linux.go:91 +0x434 github.com/kata-containers/agent/vendor/github.com/opencontainers/runc/libcontainer.(*LinuxFactory).StartInitialization(0x4000164090, 0x0, 0x0) /root/go/src/github.com/kata-containers/agent/vendor/github.com/opencontainers/runc/libcontainer/factory_linux.go:380 +0x2ec main.init.0() /root/go/src/github.com/kata-containers/agent/agent.go:1506 +0x88 " root@ubuntu:/home/ubuntu#
root@ubuntu:/usr/share/kata-containers/binary# ip netns cnitest-bb252624-2184-af78-6ee7-e54ec40635fb (id: 0) root@ubuntu:/usr/share/kata-containers/binary# p netns pids cnitest-bb252624-2184-af78-6ee7-e54ec40635fb p: command not found root@ubuntu:/usr/share/kata-containers/binary# ip netns pids cnitest-bb252624-2184-af78-6ee7-e54ec40635fb 13373 13391 root@ubuntu:/usr/share/kata-containers/binary# ps -p 13373 13391 PID TTY STAT TIME COMMAND 13373 ? Sl 0:01 /usr/share/kata-containers/binary/qemu-system-aarch64 -name sandbox-ca5ca0fe6fd3ce12e8a 13391 pts/4 Ssl+ 0:00 /usr/libexec/kata-containers/kata-shim -agent unix:///run/vc/sbs/ca5ca0fe6fd3ce12e8a1c0 root@ubuntu:/usr/share/kata-containers/binary#
root@25a725e7599e:/# ls run/ kata-containers libcontainer lock mount sandbox-ns systemd
// Shared path handling: // 1. create two directories for each sandbox: // -. /run/kata-containers/shared/sandboxes/$sbx_id/mounts/, a directory to hold all host/guest shared mounts // -. /run/kata-containers/shared/sandboxes/$sbx_id/shared/, a host/guest shared directory (9pfs/virtiofs source dir) // // 2. /run/kata-containers/shared/sandboxes/$sbx_id/mounts/ is bind mounted readonly to /run/kata-containers/shared/sandboxes/$sbx_id/shared/, so guest cannot modify it // // 3. host-guest shared files/directories are mounted one-level under /run/kata-containers/shared/sandboxes/$sbx_id/mounts/ and thus present to guest at one level under /run/kata-containers/shared/sandboxes/$sbx_id/shared/
root@ubuntu:/home/ubuntu# ls /run/kata-containers/ containers-mapping shared root@ubuntu:/home/ubuntu# ls /run/kata-containers/containers-mapping/ cf09483176d1a73c6e21d0b69a1de3b2d06b2cde4ec63f62d5e6608e0ab7a93c root@ubuntu:/home/ubuntu# ls /run/kata-containers/containers-mapping/cf09483176d1a73c6e21d0b69a1de3b2d06b2cde4ec63f62d5e6608e0ab7a93c/ cf09483176d1a73c6e21d0b69a1de3b2d06b2cde4ec63f62d5e6608e0ab7a93c root@ubuntu:/home/ubuntu# ls /run/kata-containers/containers-mapping/cf09483176d1a73c6e21d0b69a1de3b2d06b2cde4ec63f62d5e6608e0ab7a93c/cf09483176d1a73c6e21d0b69a1de3b2d06b2cde4ec63f62d5e6608e0ab7a93c/ root@ubuntu:/home/ubuntu# ls /var/run/vc/sbs/ cf09483176d1a73c6e21d0b69a1de3b2d06b2cde4ec63f62d5e6608e0ab7a93c root@ubuntu:/home/ubuntu# ls /var/run/vc/sbs/cf09483176d1a73c6e21d0b69a1de3b2d06b2cde4ec63f62d5e6608e0ab7a93c/ cf09483176d1a73c6e21d0b69a1de3b2d06b2cde4ec63f62d5e6608e0ab7a93c persist.json proxy.sock root@ubuntu:/home/ubuntu# mount | grep cf09483176d1a7 tmpfs on /run/kata-containers/shared/sandboxes/cf09483176d1a73c6e21d0b69a1de3b2d06b2cde4ec63f62d5e6608e0ab7a93c/shared type tmpfs (ro,relatime,size=26334908k,mode=755) overlay on /run/kata-containers/shared/sandboxes/cf09483176d1a73c6e21d0b69a1de3b2d06b2cde4ec63f62d5e6608e0ab7a93c/mounts/cf09483176d1a73c6e21d0b69a1de3b2d06b2cde4ec63f62d5e6608e0ab7a93c/rootfs type overlay (rw,relatime,lowerdir=/var/lib/docker/overlay2/l/EUZLGUHBQ7L3655EHGCEMALDBZ:/var/lib/docker/overlay2/l/QYAY6NY35IL5RIM4PE5ZLY7C44:/var/lib/docker/overlay2/l/MN4IC4KQI4FAGG4ZIPEYHSPJGW:/var/lib/docker/overlay2/l/PWRZKLZFFPTNF76EUWJQWMXDXN:/var/lib/docker/overlay2/l/OHD7XJ4JW7PEYZRGBIBJZYTLYY:/var/lib/docker/overlay2/l/XKGKOR5GBTIGTO6EHG22MIZ7NE,upperdir=/var/lib/docker/overlay2/ff47992f102aa5baa3e56614a739411ae30dea2c68dafa33d31218bf763cb660/diff,workdir=/var/lib/docker/overlay2/ff47992f102aa5baa3e56614a739411ae30dea2c68dafa33d31218bf763cb660/work,xino=off) overlay on /run/kata-containers/shared/sandboxes/cf09483176d1a73c6e21d0b69a1de3b2d06b2cde4ec63f62d5e6608e0ab7a93c/shared/cf09483176d1a73c6e21d0b69a1de3b2d06b2cde4ec63f62d5e6608e0ab7a93c/rootfs type overlay (rw,relatime,lowerdir=/var/lib/docker/overlay2/l/EUZLGUHBQ7L3655EHGCEMALDBZ:/var/lib/docker/overlay2/l/QYAY6NY35IL5RIM4PE5ZLY7C44:/var/lib/docker/overlay2/l/MN4IC4KQI4FAGG4ZIPEYHSPJGW:/var/lib/docker/overlay2/l/PWRZKLZFFPTNF76EUWJQWMXDXN:/var/lib/docker/overlay2/l/OHD7XJ4JW7PEYZRGBIBJZYTLYY:/var/lib/docker/overlay2/l/XKGKOR5GBTIGTO6EHG22MIZ7NE,upperdir=/var/lib/docker/overlay2/ff47992f102aa5baa3e56614a739411ae30dea2c68dafa33d31218bf763cb660/diff,workdir=/var/lib/docker/overlay2/ff47992f102aa5baa3e56614a739411ae30dea2c68dafa33d31218bf763cb660/work,xino=off) /dev/sdc3 on /run/kata-containers/shared/sandboxes/cf09483176d1a73c6e21d0b69a1de3b2d06b2cde4ec63f62d5e6608e0ab7a93c/mounts/cf09483176d1a73c6e21d0b69a1de3b2d06b2cde4ec63f62d5e6608e0ab7a93c-3c5f4f0362a80b0d-resolv.conf type ext4 (rw,relatime,errors=remount-ro,stripe=64) /dev/sdc3 on /run/kata-containers/shared/sandboxes/cf09483176d1a73c6e21d0b69a1de3b2d06b2cde4ec63f62d5e6608e0ab7a93c/shared/cf09483176d1a73c6e21d0b69a1de3b2d06b2cde4ec63f62d5e6608e0ab7a93c-3c5f4f0362a80b0d-resolv.conf type ext4 (rw,relatime,errors=remount-ro,stripe=64) /dev/sdc3 on /run/kata-containers/shared/sandboxes/cf09483176d1a73c6e21d0b69a1de3b2d06b2cde4ec63f62d5e6608e0ab7a93c/mounts/cf09483176d1a73c6e21d0b69a1de3b2d06b2cde4ec63f62d5e6608e0ab7a93c-f0398696c4ccbad5-hostname type ext4 (rw,relatime,errors=remount-ro,stripe=64) /dev/sdc3 on /run/kata-containers/shared/sandboxes/cf09483176d1a73c6e21d0b69a1de3b2d06b2cde4ec63f62d5e6608e0ab7a93c/shared/cf09483176d1a73c6e21d0b69a1de3b2d06b2cde4ec63f62d5e6608e0ab7a93c-f0398696c4ccbad5-hostname type ext4 (rw,relatime,errors=remount-ro,stripe=64) /dev/sdc3 on /run/kata-containers/shared/sandboxes/cf09483176d1a73c6e21d0b69a1de3b2d06b2cde4ec63f62d5e6608e0ab7a93c/mounts/cf09483176d1a73c6e21d0b69a1de3b2d06b2cde4ec63f62d5e6608e0ab7a93c-889f0c6b7cf0823e-hosts type ext4 (rw,relatime,errors=remount-ro,stripe=64) /dev/sdc3 on /run/kata-containers/shared/sandboxes/cf09483176d1a73c6e21d0b69a1de3b2d06b2cde4ec63f62d5e6608e0ab7a93c/shared/cf09483176d1a73c6e21d0b69a1de3b2d06b2cde4ec63f62d5e6608e0ab7a93c-889f0c6b7cf0823e-hosts type ext4 (rw,relatime,errors=remount-ro,stripe=64) root@ubuntu:/home/ubuntu#
kata虚拟机
root@25a725e7599e:/# ls run/kata-containers/ sandbox shared root@25a725e7599e:/# ls run/kata-containers/sandbox/ resolv.conf shm root@25a725e7599e:/# ls run/kata-containers/sandbox/shm/ root@25a725e7599e:/# ls ./run/kata-containers/shared/containers/ cf09483176d1a73c6e21d0b69a1de3b2d06b2cde4ec63f62d5e6608e0ab7a93c cf09483176d1a73c6e21d0b69a1de3b2d06b2cde4ec63f62d5e6608e0ab7a93c-3c5f4f0362a80b0d-resolv.conf cf09483176d1a73c6e21d0b69a1de3b2d06b2cde4ec63f62d5e6608e0ab7a93c-889f0c6b7cf0823e-hosts cf09483176d1a73c6e21d0b69a1de3b2d06b2cde4ec63f62d5e6608e0ab7a93c-f0398696c4ccbad5-hostname 6e21d0b69a1de3b2d06b2cde4ec63f62d5e6608e0ab7a93cared/containers/cf09483176d1a73c6
application挂载实现
-chardev socket,id=charch0,path=/run/vc/sbs/2ed4a3afed3c3d3269ca230d87da940bcdb85a6f239fab015b2710b83253dc02/kata.sock,server,nowait
-device virtio-9p-pci,fsdev=extra-9p-kataShared,mount_tag=kataShared -fsdev local,id=extra-9p-kataShared,path=/run/kata-containers/shared/sandboxes/2ed4a3afed3c3d3269ca230d87da940bcdb85a6f239