vfio

root@zj-x86:~# lspci | grep -i ether
1a:00.0 Ethernet controller: Intel Corporation Ethernet Connection X722 for 10GbE SFP+ (rev 09)
1a:00.1 Ethernet controller: Intel Corporation Ethernet Connection X722 for 10GbE SFP+ (rev 09)
1a:00.2 Ethernet controller: Intel Corporation Ethernet Connection X722 for 1GbE (rev 09)
1a:00.3 Ethernet controller: Intel Corporation Ethernet Connection X722 for 1GbE (rev 09)
root@zj-x86:~# lspci -n -s 1a:00.2
1a:00.2 0200: 8086:37d1 (rev 09)
root@zj-x86:~# dmesg |grep -i iommu|grep 1a:00.2
root@zj-x86:~# ls /sys/bus/pci/devices/0000:1a:00.2
aer_dev_correctable       net
aer_dev_fatal             numa_node
aer_dev_nonfatal          power
ari_enabled               ptp
broken_parity_status      remove
class                     rescan
config                    reset
consistent_dma_mask_bits  resource
current_link_speed        resource0
current_link_width        resource0_wc
d3cold_allowed            resource3
device                    resource3_wc
dma_mask_bits             revision
driver                    rom
driver_override           sriov_drivers_autoprobe
enable                    sriov_numvfs
firmware_node             sriov_offset
index                     sriov_stride
irq                       sriov_totalvfs
label                     sriov_vf_device
local_cpulist             subsystem
local_cpus                subsystem_device
max_link_speed            subsystem_vendor
max_link_width            uevent
modalias                  vendor
msi_bus                   vpd
msi_irqs

arm64

[root@localhost ~]# find /sys/kernel/iommu_groups/*/devices/*
/sys/kernel/iommu_groups/0/devices/0000:06:00.0
/sys/kernel/iommu_groups/10/devices/0000:00:04.0
/sys/kernel/iommu_groups/11/devices/0000:00:08.0
/sys/kernel/iommu_groups/12/devices/0000:00:0c.0
/sys/kernel/iommu_groups/13/devices/0000:00:10.0
/sys/kernel/iommu_groups/14/devices/0000:00:11.0
/sys/kernel/iommu_groups/15/devices/0000:00:12.0
/sys/kernel/iommu_groups/16/devices/0000:7c:00.0
/sys/kernel/iommu_groups/17/devices/0000:80:00.0
/sys/kernel/iommu_groups/18/devices/0000:80:04.0
/sys/kernel/iommu_groups/19/devices/0000:80:08.0
/sys/kernel/iommu_groups/1/devices/0000:74:03.0
/sys/kernel/iommu_groups/20/devices/0000:80:10.0
/sys/kernel/iommu_groups/21/devices/0000:83:00.0
/sys/kernel/iommu_groups/22/devices/0000:84:00.0
/sys/kernel/iommu_groups/23/devices/0000:84:01.0
/sys/kernel/iommu_groups/24/devices/0000:bc:00.0
/sys/kernel/iommu_groups/25/devices/0000:03:00.0
/sys/kernel/iommu_groups/26/devices/0000:74:02.0
/sys/kernel/iommu_groups/27/devices/0000:7d:00.0
/sys/kernel/iommu_groups/28/devices/0000:85:00.0
/sys/kernel/iommu_groups/29/devices/0000:7d:00.1
/sys/kernel/iommu_groups/2/devices/0000:b4:03.0
/sys/kernel/iommu_groups/30/devices/0000:7d:00.2
/sys/kernel/iommu_groups/31/devices/0000:7d:00.3
/sys/kernel/iommu_groups/32/devices/0000:bd:00.0
/sys/kernel/iommu_groups/33/devices/0000:bd:00.1
/sys/kernel/iommu_groups/34/devices/0000:bd:00.2
/sys/kernel/iommu_groups/35/devices/0000:bd:00.3
/sys/kernel/iommu_groups/36/devices/0000:86:00.0
/sys/kernel/iommu_groups/37/devices/0000:74:04.0
/sys/kernel/iommu_groups/38/devices/0000:b4:02.0
/sys/kernel/iommu_groups/39/devices/0000:b4:04.0
/sys/kernel/iommu_groups/3/devices/0000:7a:01.0
/sys/kernel/iommu_groups/4/devices/0000:ba:01.0
/sys/kernel/iommu_groups/5/devices/0000:7a:00.0
/sys/kernel/iommu_groups/6/devices/0000:ba:00.0
/sys/kernel/iommu_groups/7/devices/0000:7a:02.0
/sys/kernel/iommu_groups/8/devices/0000:ba:02.0
/sys/kernel/iommu_groups/9/devices/0000:00:00.0
[root@localhost ~]# uname -a
Linux localhost.localdomain 4.19.90-9.aarch64

[root@localhost ~]# find /sys/kernel/iommu_groups/ -type l
/sys/kernel/iommu_groups/17/devices/0000:80:00.0
/sys/kernel/iommu_groups/35/devices/0000:bd:00.3
/sys/kernel/iommu_groups/7/devices/0000:7a:02.0
/sys/kernel/iommu_groups/25/devices/0000:03:00.0
/sys/kernel/iommu_groups/15/devices/0000:00:12.0
/sys/kernel/iommu_groups/33/devices/0000:bd:00.1
/sys/kernel/iommu_groups/5/devices/0000:7a:00.0
/sys/kernel/iommu_groups/23/devices/0000:84:01.0
/sys/kernel/iommu_groups/13/devices/0000:00:10.0
/sys/kernel/iommu_groups/31/devices/0000:7d:00.3
/sys/kernel/iommu_groups/3/devices/0000:7a:01.0
/sys/kernel/iommu_groups/21/devices/0000:83:00.0
/sys/kernel/iommu_groups/11/devices/0000:00:08.0
/sys/kernel/iommu_groups/1/devices/0000:74:03.0
/sys/kernel/iommu_groups/38/devices/0000:b4:02.0
/sys/kernel/iommu_groups/28/devices/0000:85:00.0
/sys/kernel/iommu_groups/18/devices/0000:80:04.0
/sys/kernel/iommu_groups/36/devices/0000:86:00.0
/sys/kernel/iommu_groups/8/devices/0000:ba:02.0
/sys/kernel/iommu_groups/26/devices/0000:74:02.0
/sys/kernel/iommu_groups/16/devices/0000:7c:00.0
/sys/kernel/iommu_groups/34/devices/0000:bd:00.2
/sys/kernel/iommu_groups/6/devices/0000:ba:00.0
/sys/kernel/iommu_groups/24/devices/0000:bc:00.0
/sys/kernel/iommu_groups/14/devices/0000:00:11.0
/sys/kernel/iommu_groups/32/devices/0000:bd:00.0
/sys/kernel/iommu_groups/4/devices/0000:ba:01.0
/sys/kernel/iommu_groups/22/devices/0000:84:00.0
/sys/kernel/iommu_groups/12/devices/0000:00:0c.0
/sys/kernel/iommu_groups/30/devices/0000:7d:00.2
/sys/kernel/iommu_groups/2/devices/0000:b4:03.0
/sys/kernel/iommu_groups/20/devices/0000:80:10.0
/sys/kernel/iommu_groups/10/devices/0000:00:04.0
/sys/kernel/iommu_groups/39/devices/0000:b4:04.0
/sys/kernel/iommu_groups/29/devices/0000:7d:00.1
/sys/kernel/iommu_groups/0/devices/0000:06:00.0
/sys/kernel/iommu_groups/19/devices/0000:80:08.0
/sys/kernel/iommu_groups/37/devices/0000:74:04.0
/sys/kernel/iommu_groups/9/devices/0000:00:00.0
/sys/kernel/iommu_groups/27/devices/0000:7d:00.0
[root@localhost ~]#

[root@localhost ~]# ls  /sys/bus/pci/devices/0000:00:00.0/
ari_enabled               max_link_speed
broken_parity_status      max_link_width
class                     modalias
config                    msi_bus
consistent_dma_mask_bits  numa_node
current_link_speed        pci_bus
current_link_width        power
d3cold_allowed            remove
device                    rescan
devspec                   resource
dma_mask_bits             revision
driver_override           secondary_bus_number
enable                    subordinate_bus_number
iommu                     subsystem
iommu_group               subsystem_device
irq                       subsystem_vendor
local_cpulist             uevent
local_cpus                vendor
[root@localhost ~]#

[root@localhost ~]# ls -al  /sys/bus/pci/devices/0000:00:00.0/iommu_group/devices
总用量 0
drwxr-xr-x 2 root root 0  8月 22 12:48 .
drwxr-xr-x 3 root root 0  7月  8 17:57 ..
lrwxrwxrwx 1 root root 0  7月  8 17:57 0000:00:00.0 -> ../../../../devices/pci0000:00/0000:00:00.0
[root@localhost ~]# lspci -n -s 0000:00:00.0
00:00.0 0604: 19e5:a120 (rev 21)

[root@localhost ~]# ls /sys/bus/pci/drivers/vfio-pci
ls: 无法访问 '/sys/bus/pci/drivers/vfio-pci': 没有那个文件或目录
[root@localhost ~]# modprobe vfio-pci
[root@localhost ~]# ls /sys/bus/pci/drivers/vfio-pci
bind  module  new_id  remove_id  uevent  unbind
[root@localhost ~]# 

root@zj-x86:/sys/bus/pci/devices# ls /sys/bus/pci/drivers/{igb,pci-stub,vfio-pci}
ls: cannot access '/sys/bus/pci/drivers/igb': No such file or directory
ls: cannot access '/sys/bus/pci/drivers/pci-stub': No such file or directory
ls: cannot access '/sys/bus/pci/drivers/vfio-pci': No such file or directory
root@zj-x86:/sys/bus/pci/devices# modprobe vfio_pci
root@zj-x86:/sys/bus/pci/devices# ls /sys/bus/pci/drivers/{igb,pci-stub,vfio-pci}
ls: cannot access '/sys/bus/pci/drivers/igb': No such file or directory
ls: cannot access '/sys/bus/pci/drivers/pci-stub': No such file or directory
/sys/bus/pci/drivers/vfio-pci:
bind  module  new_id  remove_id  uevent  unbind
root@zj-x86:/sys/bus/pci/devices# 

VFIO no-IOMMU

[root@localhost ~]# cat /sys/module/vfio/parameters/enable_unsafe_noiommu_mode
N

[root@localhost ~]# modprobe vfio enable_unsafe_noiommu_mode=1
[root@localhost ~]# modprobe vfio-pci
[root@localhost ~]# 

[root@localhost ~]# find /sys/kernel/iommu_groups/ -type l
/sys/kernel/iommu_groups/17/devices/0000:7c:00.0
/sys/kernel/iommu_groups/35/devices/0000:b4:02.0
/sys/kernel/iommu_groups/7/devices/0000:ba:02.0
/sys/kernel/iommu_groups/25/devices/0000:05:00.0
/sys/kernel/iommu_groups/15/devices/0000:04:00.0
/sys/kernel/iommu_groups/33/devices/0000:7d:00.3
/sys/kernel/iommu_groups/5/devices/0000:ba:00.0
/sys/kernel/iommu_groups/23/devices/0000:81:00.0
/sys/kernel/iommu_groups/13/devices/0000:00:12.0
/sys/kernel/iommu_groups/31/devices/0000:06:00.0
/sys/kernel/iommu_groups/3/devices/0000:ba:01.0
/sys/kernel/iommu_groups/21/devices/0000:80:10.0
/sys/kernel/iommu_groups/11/devices/0000:00:10.0
/sys/kernel/iommu_groups/1/devices/0000:b4:03.0
/sys/kernel/iommu_groups/28/devices/0000:08:00.0
/sys/kernel/iommu_groups/18/devices/0000:80:00.0
/sys/kernel/iommu_groups/36/devices/0000:b4:04.0
/sys/kernel/iommu_groups/8/devices/0000:00:00.0
/sys/kernel/iommu_groups/26/devices/0000:7d:00.0
/sys/kernel/iommu_groups/16/devices/0000:04:01.0
/sys/kernel/iommu_groups/34/devices/0000:74:04.0
/sys/kernel/iommu_groups/6/devices/0000:7a:02.0
/sys/kernel/iommu_groups/24/devices/0000:74:02.0
/sys/kernel/iommu_groups/14/devices/0000:03:00.0
/sys/kernel/iommu_groups/32/devices/0000:84:00.0
/sys/kernel/iommu_groups/4/devices/0000:7a:00.0
/sys/kernel/iommu_groups/22/devices/0000:bc:00.0
/sys/kernel/iommu_groups/12/devices/0000:00:11.0
/sys/kernel/iommu_groups/30/devices/0000:7d:00.2
/sys/kernel/iommu_groups/2/devices/0000:7a:01.0
/sys/kernel/iommu_groups/20/devices/0000:80:0c.0
/sys/kernel/iommu_groups/10/devices/0000:00:0c.0
/sys/kernel/iommu_groups/29/devices/0000:7d:00.1
/sys/kernel/iommu_groups/0/devices/0000:74:03.0
/sys/kernel/iommu_groups/19/devices/0000:80:08.0
/sys/kernel/iommu_groups/9/devices/0000:00:08.0
/sys/kernel/iommu_groups/27/devices/0000:02:00.0
[root@localhost ~]# readlink -f  /sys/bus/pci/devices/0000:00:11.0/i
iommu/       iommu_group/ irq          
[root@localhost ~]# readlink -f  /sys/bus/pci/devices/0000:00:11.0/iommu_group
/sys/kernel/iommu_groups/12
[root@localhost ~]# ls  /sys/bus/pci/devices/0000:00:11.0/iommu_group/devices/
0000:00:11.0
[root@localhost ~]# 

00:11.0这个是什么pci设备

[root@localhost ~]# lspci -vvv | grep '00:11.0' -A 2 -B 2
        Kernel driver in use: pcieport

00:11.0 PCI bridge: Huawei Technologies Co., Ltd. HiSilicon PCIe Root Port with Gen4 (rev 21) (prog-if 00 [Normal decode])
        Control: I/O+ Mem+ BusMaster+ SpecCycle- MemWINV- VGASnoop- ParErr+ Stepping- SERR+ FastB2B- DisINTx+
        Status: Cap+ 66MHz- UDF- FastB2B- ParErr- DEVSEL=fast >TAbort- <TAbort- <MAbort- >SERR- <PERR- INTx-
[root@localhost ~]# 

[root@localhost ~]# find /sys/bus/pci/devices/*/iommu_group/devices/ -type l
/sys/bus/pci/devices/0000:00:00.0/iommu_group/devices/0000:00:00.0
/sys/bus/pci/devices/0000:00:08.0/iommu_group/devices/0000:00:08.0
/sys/bus/pci/devices/0000:00:0c.0/iommu_group/devices/0000:00:0c.0
/sys/bus/pci/devices/0000:00:10.0/iommu_group/devices/0000:00:10.0
/sys/bus/pci/devices/0000:00:11.0/iommu_group/devices/0000:00:11.0
/sys/bus/pci/devices/0000:00:12.0/iommu_group/devices/0000:00:12.0
/sys/bus/pci/devices/0000:02:00.0/iommu_group/devices/0000:02:00.0
/sys/bus/pci/devices/0000:03:00.0/iommu_group/devices/0000:03:00.0
/sys/bus/pci/devices/0000:04:00.0/iommu_group/devices/0000:04:00.0
/sys/bus/pci/devices/0000:04:01.0/iommu_group/devices/0000:04:01.0
/sys/bus/pci/devices/0000:05:00.0/iommu_group/devices/0000:05:00.0
/sys/bus/pci/devices/0000:06:00.0/iommu_group/devices/0000:06:00.0
/sys/bus/pci/devices/0000:08:00.0/iommu_group/devices/0000:08:00.0
/sys/bus/pci/devices/0000:74:02.0/iommu_group/devices/0000:74:02.0
/sys/bus/pci/devices/0000:74:03.0/iommu_group/devices/0000:74:03.0
/sys/bus/pci/devices/0000:74:04.0/iommu_group/devices/0000:74:04.0
/sys/bus/pci/devices/0000:7a:00.0/iommu_group/devices/0000:7a:00.0
/sys/bus/pci/devices/0000:7a:01.0/iommu_group/devices/0000:7a:01.0
/sys/bus/pci/devices/0000:7a:02.0/iommu_group/devices/0000:7a:02.0
/sys/bus/pci/devices/0000:7c:00.0/iommu_group/devices/0000:7c:00.0
/sys/bus/pci/devices/0000:7d:00.0/iommu_group/devices/0000:7d:00.0
/sys/bus/pci/devices/0000:7d:00.1/iommu_group/devices/0000:7d:00.1
/sys/bus/pci/devices/0000:7d:00.2/iommu_group/devices/0000:7d:00.2
/sys/bus/pci/devices/0000:7d:00.3/iommu_group/devices/0000:7d:00.3
/sys/bus/pci/devices/0000:80:00.0/iommu_group/devices/0000:80:00.0
/sys/bus/pci/devices/0000:80:08.0/iommu_group/devices/0000:80:08.0
/sys/bus/pci/devices/0000:80:0c.0/iommu_group/devices/0000:80:0c.0
/sys/bus/pci/devices/0000:80:10.0/iommu_group/devices/0000:80:10.0
/sys/bus/pci/devices/0000:81:00.0/iommu_group/devices/0000:81:00.0
/sys/bus/pci/devices/0000:84:00.0/iommu_group/devices/0000:84:00.0
/sys/bus/pci/devices/0000:b4:02.0/iommu_group/devices/0000:b4:02.0
/sys/bus/pci/devices/0000:b4:03.0/iommu_group/devices/0000:b4:03.0
/sys/bus/pci/devices/0000:b4:04.0/iommu_group/devices/0000:b4:04.0
/sys/bus/pci/devices/0000:ba:00.0/iommu_group/devices/0000:ba:00.0
/sys/bus/pci/devices/0000:ba:01.0/iommu_group/devices/0000:ba:01.0
/sys/bus/pci/devices/0000:ba:02.0/iommu_group/devices/0000:ba:02.0
/sys/bus/pci/devices/0000:bc:00.0/iommu_group/devices/0000:bc:00.0
[root@localhost ~]# 

OMMU groups are visible to the user through sysfs:

$ find /sys/kernel/iommu_groups/ -type l
/sys/kernel/iommu_groups/0/devices/0000:00:00.0
/sys/kernel/iommu_groups/1/devices/0000:00:02.0
/sys/kernel/iommu_groups/2/devices/0000:00:14.0
/sys/kernel/iommu_groups/3/devices/0000:00:16.0
/sys/kernel/iommu_groups/4/devices/0000:00:19.0
/sys/kernel/iommu_groups/5/devices/0000:00:1a.0
/sys/kernel/iommu_groups/6/devices/0000:00:1b.0
/sys/kernel/iommu_groups/7/devices/0000:00:1c.0
/sys/kernel/iommu_groups/7/devices/0000:00:1c.1
/sys/kernel/iommu_groups/7/devices/0000:00:1c.2
/sys/kernel/iommu_groups/7/devices/0000:02:00.0
/sys/kernel/iommu_groups/7/devices/0000:03:00.0
/sys/kernel/iommu_groups/8/devices/0000:00:1d.0
/sys/kernel/iommu_groups/9/devices/0000:00:1f.0
/sys/kernel/iommu_groups/9/devices/0000:00:1f.2

/sys/kernel/iommu_groups/9/devices/0000:00:1f.3

Here we see that devices like the audio controller (0000:00:1b.0) have their own IOMMU group, while a wireless adapter (0000:03:00.0) and flash card reader (0000:02:00.0) share an IOMMU group.  The later is a result of lack of ACS support at the PCIe root ports (0000:00:1c.*).  Each device also has links back to its IOMMU group:

$ readlink -f /sys/bus/pci/devices/0000:03:00.0/iommu_group/
/sys/kernel/iommu_groups/7

The set of devices can thus be found using:

$ ls /sys/bus/pci/devices/0000:03:00.0/iommu_group/devices/     ----------这个查看iommu_group下的设备

0000:00:1c.0  0000:00:1c.1  0000:00:1c.2  0000:02:00.0  0000:03:00.0

Using this example, if I wanted to assign the wireless adapter (0000:03:00.0) to a guest, I would also need to bind the flash card reader (0000:02:00.0) to either vfio-pci or pci-stub in order to make the group viable.  An important point here is that the flash card reader does not also need to be assigned to the guest, it simply needs to be held by a device which is known to either participate in VFIO, like vfio-pci, or known not to do DMA, like pci-stub.  Newer kernels than used for this example will split this IOMMU group as support has been added to expose the isolation capabilities of this chipset, even though it does not support PCIe ACS directly.

[root@localhost ~]# ls /dev/vfio/
25 31 vfio ---------------25和31是group id
[root@localhost ~]# ls /dev/vfio/25
/dev/vfio/25
[root@localhost ~]# ls /dev/vfio/31
/dev/vfio/31
[root@localhost ~]# cat /dev/vfio/31
cat: /dev/vfio/31: Invalid argument
[root@localhost ~]# ls -l /sys/bus/pci/drivers/vfio-pci
total 0
lrwxrwxrwx. 1 root root 0 Aug 22 08:54 0000:05:00.0 -> ../../../../devices/pci0000:00/0000:00:0c.0/0000:03:00.0/0000:04:00.0/0000:05:00.0
lrwxrwxrwx. 1 root root 0 Aug 22 08:54 0000:06:00.0 -> ../../../../devices/pci0000:00/0000:00:0c.0/0000:03:00.0/0000:04:01.0/0000:06:00.0
--w-------. 1 root root 65536 Apr 21 05:26 bind
lrwxrwxrwx. 1 root root 0 Apr 28 23:45 module -> ../../../../module/vfio_pci
--w-------. 1 root root 65536 Aug 22 08:54 new_id
--w-------. 1 root root 65536 Aug 22 08:54 remove_id
--w-------. 1 root root 65536 Apr 21 05:15 uevent
--w-------. 1 root root 65536 Aug 22 08:54 unbind
[root@localhost ~]#

[root@localhost ~]# lspci -vvv | grep '05:00.0' 
05:00.0 Ethernet controller: Huawei Technologies Co., Ltd. Hi1822 Family (2*25GE) (rev 45)
[root@localhost ~]# lspci -vvv | grep '06:00.0' 
06:00.0 Ethernet controller: Huawei Technologies Co., Ltd. Hi1822 Family (2*25GE) (rev 45)
[root@localhost ~]#

[root@localhost ~]# find /sys/bus/pci/devices/*/iommu_group/devices/ -type l | grep 05:00.0
/sys/bus/pci/devices/0000:05:00.0/iommu_group/devices/0000:05:00.0
[root@localhost ~]# find /sys/bus/pci/devices/*/iommu_group/devices/ -type l | grep 06:00.0
/sys/bus/pci/devices/0000:06:00.0/iommu_group/devices/0000:06:00.0
[root@localhost ~]#

[root@localhost ~]# readlink -f /sys/bus/pci/devices/0000:06:00.0/iommu_group
/sys/kernel/iommu_groups/31
[root@localhost ~]# readlink -f /sys/bus/pci/devices/0000:05:00.0/iommu_group
/sys/kernel/iommu_groups/25
[root@localhost ~]# 

#!/bin/bash

modprobe vfio-pci

for dev in "$@"; do
        vendor=$(cat /sys/bus/pci/devices/$dev/vendor)
        device=$(cat /sys/bus/pci/devices/$dev/device)
        if [ -e /sys/bus/pci/devices/$dev/driver ]; then
                echo $dev > /sys/bus/pci/devices/$dev/driver/unbind
        fi
        echo $vendor $device > /sys/bus/pci/drivers/vfio-pci/new_id
done

• Configure the card types through sysfs.
  1. Unless it is compiled into the kernel, load the vfio-pci device driver.

     # modprobe vfio_pci

  2. Set the PCI card types to be controlled by the vfio-pci device driver by writing the vendor and device code to /sys/bus/pci/drivers/vfio-pci/new_id.

     # echo <vendor_code> <device_code> > /sys/bus/pci/drivers/vfio-pci/new_id

     Separate the vendor code and the device code with a blank. Repeat the command to specify multiple card types.
• If the vfio-pci device driver is compiled as a separate module, you can configure the card types with the ids= module parameter.

  # modprobe vfio_pci ids=<vendor_code>:<device_code>

  You can specify multiple card types as a comma-separated value.
• If the vfio-pci device driver is compiled into the kernel, you can configure the card types with the vfio_pci.ids= module parameter.

  vfio_pci.ids=<vendor_code>:<device_code>

  You can specify multiple card types as a comma-separated value.

[root@localhost ~]# ls /sys/bus/pci/devices
0000:00:00.0  0000:05:00.0  0000:7b:00.0  0000:81:00.0
0000:00:08.0  0000:06:00.0  0000:7c:00.0  0000:84:00.0
0000:00:0c.0  0000:07:00.0  0000:7d:00.0  0000:b4:02.0
0000:00:10.0  0000:08:00.0  0000:7d:00.1  0000:b4:03.0
0000:00:11.0  0000:74:02.0  0000:7d:00.2  0000:b4:04.0
0000:00:12.0  0000:74:03.0  0000:7d:00.3  0000:ba:00.0
0000:02:00.0  0000:74:04.0  0000:80:00.0  0000:ba:01.0
0000:03:00.0  0000:7a:00.0  0000:80:08.0  0000:ba:02.0
0000:04:00.0  0000:7a:01.0  0000:80:0c.0  0000:bb:00.0
0000:04:01.0  0000:7a:02.0  0000:80:10.0  0000:bc:00.0
[root@localhost ~]# ls /sys/bus/pci/devices//0000:05:00.0
broken_parity_status      msi_irqs
class                     numa_node
config                    power
consistent_dma_mask_bits  remove
current_link_speed        rescan
current_link_width        reset
d3cold_allowed            resource
device                    resource0
devspec                   resource2
dma_mask_bits             resource4
driver                    revision
driver_override           rom
enable                    sriov_drivers_autoprobe
iommu                     sriov_numvfs
iommu_group               sriov_totalvfs
irq                       subsystem
local_cpulist             subsystem_device
local_cpus                subsystem_vendor
max_link_speed            uevent
max_link_width            vendor
modalias                  vpd
msi_bus
[root@localhost ~]# cat  /sys/bus/pci/devices//0000:05:00.0/vendor 
0x19e5
[root@localhost ~]# cat  /sys/bus/pci/devices//0000:05:00.0/device 
0x0200
[root@localhost ~]# ls  /sys/bus/pci/devices//0000:05:00.0/driver
0000:05:00.0  bind    new_id     uevent
0000:06:00.0  module  remove_id  unbind
[root@localhost ~]# ls  /sys/bus/pci/devices//0000:05:00.0/driver -al
lrwxrwxrwx. 1 root root 0 Apr 21 05:26 /sys/bus/pci/devices//0000:05:00.0/driver -> ../../../../../../bus/pci/drivers/vfio-pci
[root@localhost ~]# ls  /sys/bus/pci/devices//0000:05:00.0/driver/* -al
lrwxrwxrwx. 1 root root     0 Aug 22 08:54 /sys/bus/pci/devices//0000:05:00.0/driver/0000:05:00.0 -> ../../../../devices/pci0000:00/0000:00:0c.0/0000:03:00.0/0000:04:00.0/0000:05:00.0
lrwxrwxrwx. 1 root root     0 Aug 22 08:54 /sys/bus/pci/devices//0000:05:00.0/driver/0000:06:00.0 -> ../../../../devices/pci0000:00/0000:00:0c.0/0000:03:00.0/0000:04:01.0/0000:06:00.0
--w-------. 1 root root 65536 Aug 22 08:54 /sys/bus/pci/devices//0000:05:00.0/driver/bind
lrwxrwxrwx. 1 root root     0 Apr 28 23:45 /sys/bus/pci/devices//0000:05:00.0/driver/module -> ../../../../module/vfio_pci
--w-------. 1 root root 65536 Aug 22 08:54 /sys/bus/pci/devices//0000:05:00.0/driver/new_id
--w-------. 1 root root 65536 Aug 22 08:54 /sys/bus/pci/devices//0000:05:00.0/driver/remove_id
--w-------. 1 root root 65536 Aug 22 08:54 /sys/bus/pci/devices//0000:05:00.0/driver/uevent
--w-------. 1 root root 65536 Aug 22 08:54 /sys/bus/pci/devices//0000:05:00.0/driver/unbind
[root@localhost ~]# cat /sys/bus/pci/devices//0000:05:00.0/driver/bind
cat: /sys/bus/pci/devices//0000:05:00.0/driver/bind: Permission denied
[root@localhost ~]# cat /sys/bus/pci/devices//0000:05:00.0/driver/unbind
cat: /sys/bus/pci/devices//0000:05:00.0/driver/unbind: Permission denied
[root@localhost ~]#

pci passthrough(VFIO)2
 
VFIO可以用于实现高效的用户态驱动。在虚拟化场景可以用于device passthrough。
通过用户态配置IOMMU接口，可以将DMA地址空间映射限制在进程虚拟空间中。
这对高性能驱动和虚拟化场景device passthrough尤其重要。

相对于传统方式，VFIO对UEFI支持更好。
VFIO技术实现了用户空间直接访问设备。无须root特权，更安全，功能更多。
 

操作方法
安装kernel module
sudo modprobe vfio
sudo modprobe vfio-pci
设备unbind
  1. 查看iommu_group及其下所有设备
  cd /sys/bus/pci/devices/0000:0d:00.0/
  readlink iommu_group    #查看iommu_group名字
  ll iommu_group/devices  #查看iommu_group下设备
  2. 需要将iommu_group下所有设备unbind并添加到iommu_group中  
  echo 0000:0d:00.0 > /sys/bus/pci/devices/0000:0d:00.0/driver/unbind   
  echo 1180 e823 > /sys/bus/pci/drivers/vfio-pci/new_id