https://mails.dpdk.org/archives/dts/2016-August/001780.html
virt-sysprep -a vhuser-test1.qcow2 --network --install "telnet*"
#qemu-system-aarch64 -name vm2 -nographic qemu-system-aarch64 -name vm2 -daemonize -enable-kvm -M virt -cpu host -smp 2 -m 4096 -global virtio-blk-device.scsi=off -device virtio-scsi-device,id=scsi -kernel vmlinuz-4.18 --append "console=ttyAMA0 root=UUID=6a09973e-e8fd-4a6d-a8c0-1deb9556f477" -initrd initramfs-4.18 -drive file=vhuser-test1.qcow2 -netdev user,id=unet,hostfwd=tcp:127.0.0.1:1122-:22 -device virtio-net-device,netdev=unet -monitor telnet::3333,server,nowait -serial telnet:localhost:5432,server,nowait -incoming tcp:0:4444 -vnc :10
Last login: Wed Nov 4 23:54:54 2020 from 192.168.117.59 [root@localhost ~]# telnet localhost 3333 Trying ::1... Connected to localhost. Escape character is '^]'. QEMU 5.1.90 monitor - type 'help' for more information (qemu) ls unknown command: 'ls' (qemu) info info balloon -- show balloon information info block [-n] [-v] [device] -- show info of one block device or all block devices (-n: show named nodes; -v: show details) info block-jobs -- show progress of ongoing block device operations info blockstats -- show block device statistics info capture -- show capture information info chardev -- show the character devices info cpus -- show infos for each CPU info cpustats -- show CPU statistics info dump -- Display the latest dump status info history -- show the command line history info hotpluggable-cpus -- Show information about hotpluggable CPUs info iothreads -- show iothreads info irq -- show the interrupts statistics (if available) info jit -- show dynamic compiler info info kvm -- show KVM information info memdev -- show memory backends info memory-devices -- show memory devices info memory_size_summary -- show the amount of initially allocated and present hotpluggable (if enabled) memory in bytes. info mice -- show which guest mouse is receiving events info migrate -- show migration status info migrate_cache_size -- show current migration xbzrle cache size info migrate_capabilities -- show current migration capabilities info migrate_parameters -- show current migration parameters info mtree [-f][-d][-o][-D] -- show memory tree (-f: dump flat view for address spaces;-d: dump dispatch tree, valid with -f only);-o: dump region owners/parents;-D: dump disabled regions info name -- show the current VM name info network -- show the network state info numa -- show NUMA information info opcount -- show dynamic compiler opcode counters info pci -- show PCI info info pic -- show PIC state info profile -- show profiling information info qdm -- show qdev device model list info qom-tree [path] -- show QOM composition tree info qtree -- show device tree info ramblock -- Display system ramblock information info rdma -- show RDMA state info registers [-a] -- show the cpu registers (-a: all - show register info for all cpus) info replay -- show record/replay information info rocker name -- Show rocker switch info rocker-of-dpa-flows name [tbl_id] -- Show rocker OF-DPA flow tables info rocker-of-dpa-groups name [type] -- Show rocker OF-DPA groups info rocker-ports name -- Show rocker ports info roms -- show roms info snapshots -- show the currently saved VM snapshots info status -- show the current VM status (running|paused) info sync-profile [-m] [-n] [max] -- show synchronization profiling info, up to max entries (default: 10), sorted by total wait time. (-m: sort by mean wait time; -n: do not coalesce objects with the same call site) info tpm -- show the TPM device info trace-events [name] [vcpu] -- show available trace-events & their state (name: event name pattern; vcpu: vCPU to query, default is any) info usb -- show guest USB devices info usbhost -- show host USB devices info usernet -- show user network stack connection states info uuid -- show the current VM UUID info version -- show the version of QEMU info vm-generation-id -- Show Virtual Machine Generation ID info vnc -- show the vnc server status (qemu) info cpus * CPU #0: thread_id=40195 CPU #1: thread_id=40197 (qemu) info kvm kvm support: enabled (qemu)
Configure script: configure.sh
git clone https://github.com/intel/nemu
cd nemu
git checkout -b topic/virt-x86 origin/topic/virt-x86
mkdir build-x86-64
cd build-x86-64
./configure.sh
make -j `nproc`
Testing
We can test the NEMU virt machine type through either direct kernel boot (a.k.a. nofw
) or through a regular cloud or server image as long as it supports EFI boot.
Direct kernel boot
Kernel
I use the v4.9.34-75 branch from the Clear Containers kernel with a custom .config
Initramfs image
See the busybox initramfs build instructions, and then configure the above kernel build to point to your local rootfs.
Command line
#!/bin/bash
# -*- mode: shell-script; indent-tabs-mode: nil; sh-basic-offset: 4; -*-
# ex: ts=8 sw=4 sts=4 et filetype=sh
sudo /home/samuel/devlp/hypervisor/nemu/build-x86-64/x86_64-softmmu/qemu-system-x86_64
-nographic
-nodefaults
-L .
-net none
-machine virt,accel=kvm,kernel_irqchip,nofw
-smp sockets=1,cpus=4,cores=2,maxcpus=8 -cpu host
-m 1G,slots=3,maxmem=4G
-kernel /home/samuel/devlp/kernels/nfc/arch/x86/boot/compressed/vmlinux.bin -append 'console=hvc0 single iommu=false root=/dev/ram0'
-device virtio-serial-pci,id=virtio-serial0 -device virtconsole,chardev=charconsole0,id=console0 -chardev stdio,id=charconsole0
-monitor telnet:127.0.0.1:55555,server,nowait
Cloud image
Warning Most cloud or server images kernels do not have HW reduced ACPI enabled so they won't support CPU and memory hotplug with our HW reduced ACPI virt machine type.
We have tested the latest Clear Linux cloud image with our custom OVMF binary.
Command line
#!/bin/bash
# -*- mode: shell-script; indent-tabs-mode: nil; sh-basic-offset: 4; -*-
# ex: ts=8 sw=4 sts=4 et filetype=sh
VMN=${VMN:=1}
sudo /home/samuel/devlp/hypervisor/nemu/build-x86-64/x86_64-softmmu/qemu-system-x86_64
-bios $HOME/devlp/hypervisor/nemu/clear-cloud/OVMF.fd
-nographic
-nodefaults
-L .
-machine virt,accel=kvm,kernel_irqchip
-smp sockets=1,cpus=4,cores=2,maxcpus=8 -cpu host
-m 512
-device sysbus-debugcon,iobase=0x402,chardev=debugcon -chardev file,path=/tmp/debug-log,id=debugcon
-device virtio-blk-pci,drive=image -drive if=none,id=image,file=$HOME/devlp/hypervisor/nemu/clear-cloud/clear-cloud.img
-device virtio-serial-pci,id=virtio-serial0 -device virtconsole,chardev=charconsole0,id=console0 -chardev stdio,id=charconsole0
-netdev user,id=mynet0,hostfwd=tcp::${VMN}0022-:22,hostfwd=tcp::${VMN}2375-:2375 -device virtio-net-pci,netdev=mynet0
-monitor telnet:127.0.0.1:55555,server,nowait
Hotplug
Hotplug is done through the QEMU monitoring socket (-monitor telnet:127.0.0.1:55555,server,nowait
from the above command lines). The socket is reachable through port 23 (telnet):
# telnet 127.0.0.1 55555
CPU hotplug
Plug
(qemu) device_add host-x86_64-cpu,id=core4,socket-id=1,core-id=1,thread-id=0
Unplug
(qemu) device_del core4
Memory hotplug
Plug
(qemu) object_add memory-backend-ram,id=mem1,size=1G
(qemu) device_add pc-dimm,id=dimm1,memdev=mem1
Unplug
(qemu) device_del dimm1
step
[root@localhost cloud_images]# vi sriov2.sh #qemu-system-aarch64 -name vm2 -daemonize qemu-system-aarch64 -name vm2 -nographic -enable-kvm -M virt -cpu host -smp 2 -m 4096 -object memory-backend-file,id=mem,size=4096M,mem-path=/mnt/huge,share=on -numa node,memdev=mem -mem-prealloc -global virtio-blk-device.scsi=off -device virtio-scsi-device,id=scsi -kernel vmlinuz-4.18 --append "console=ttyAMA0 root=UUID=6a09973e-e8fd-4a6d-a8c0-1deb9556f477 iommu=pt intel_iommu=on iommu.passthrough=1" -initrd initramfs-4.18 -drive file=vhuser-test1.qcow2 -device vfio-pci,host=0000:05:00.0 -net nic,macaddr=00:00:00:99:99:01 -serial telnet:localhost:4321,server,nowait -vnc :10
qemu-system-aarch64 -name vm2 -daemonize -enable-kvm -M virt -cpu host -smp 2 -m 4096 -object memory-backend-file,id=mem,size=4096M,mem-path=/mnt/huge,share=on -numa node,memdev=mem -mem-prealloc -global virtio-blk-device.scsi=off -device virtio-scsi-device,id=scsi -kernel vmlinuz-4.18 --append "console=ttyAMA0 root=UUID=6a09973e-e8fd-4a6d-a8c0-1deb9556f477 iommu=pt intel_iommu=on iommu.passthrough=1" -initrd initramfs-4.18 -drive file=vhuser-test1.qcow2 -device vfio-pci,host=0000:05:00.0 -net nic,macaddr=00:00:00:99:99:01 -serial telnet:localhost:4321,server,nowait -vnc :10
step --host
[root@localhost cloud_images]# netstat -lpn | grep 4321 tcp 0 0 127.0.0.1:4321 0.0.0.0:* LISTEN 46099/qemu-system-a tcp6 0 0 ::1:4321 :::* LISTEN 46099/qemu-system-a [root@localhost cloud_images]# telnet localhost 4321 Trying ::1... Connected to localhost. Escape character is '^]'.
[root@localhost ~]# ip a 1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000 link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 inet 127.0.0.1/8 scope host lo valid_lft forever preferred_lft forever inet6 ::1/128 scope host valid_lft forever preferred_lft forever 2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc fq_codel state UP group default qlen 1000 link/ether 00:00:00:99:99:01 brd ff:ff:ff:ff:ff:ff inet6 fe80::200:ff:fe99:9901/64 scope link valid_lft forever preferred_lft forever [root@localhost ~]# cat /proc/cmdline console=ttyAMA0 root=UUID=6a09973e-e8fd-4a6d-a8c0-1deb9556f477 iommu=pt intel_iommu=on iommu.passthrough=1 [root@localhost ~]# lspci | grep -i ether 00:01.0 Ethernet controller: Red Hat, Inc. Virtio network device 00:02.0 Ethernet controller: Huawei Technologies Co., Ltd. Hi1822 Family (2*100GE) (rev 45) ---透传的网卡不见了 [root@localhost ~]#