agent createDeviceNode
容器侧
kata 虚拟机
Nov 25 11:31:21 pcl-01 kata-runtime[2103926]: time="2020-11-25T11:31:21.185001827+08:00"
level=info msg="Device has not been passed to the container" arch=arm64
command=create container=29713c5cffb627585cc9def39792688b68968a831455c94252c16658a2f080ea device=/dev/binder59 name=kata-runtime pid=2103926 source=virtcontainers subsystem=device
drivers.NewGenericDevice(&devInfo), nil
}
Nov 25 11:31:21 pcl-01 kata-runtime[2103926]: time="2020-11-25T11:31:21.18526109+08:00" level=info msg="normal attach devices"
arch=arm64 command=create container=29713c5cffb627585cc9def39792688b68968a831455c94252c16658a2f080ea devices="[{6e2dcf9ae1fc8164 /dev/binder59 -rw------- 0 0}]"
machine_type=virt name=kata-runtime pid=2103926 sandbox=29713c5cffb627585cc9def39792688b68968a831455c94252c16658a2f080ea source=virtcontainers subsystem=container
func (dm *deviceManager) AttachDevice(id string, dr api.DeviceReceiver) error { dm.Lock() defer dm.Unlock() d, ok := dm.devices[id] if !ok { return ErrDeviceNotExist } if err := d.Attach(dr); err != nil { return err } return nil }
grep ' Attach(' -rn * | grep api.DeviceReceiver virtcontainers/device/drivers/vhost_user_blk.go:39:func (device *VhostUserBlkDevice) Attach(devReceiver api.DeviceReceiver) (err error) { virtcontainers/device/drivers/block.go:38:func (device *BlockDevice) Attach(devReceiver api.DeviceReceiver) (err error) { virtcontainers/device/drivers/vhost_user_scsi.go:30:func (device *VhostUserSCSIDevice) Attach(devReceiver api.DeviceReceiver) (err error) { virtcontainers/device/drivers/generic.go:35:func (device *GenericDevice) Attach(devReceiver api.DeviceReceiver) error { virtcontainers/device/drivers/vhost_user_net.go:30:func (device *VhostUserNetDevice) Attach(devReceiver api.DeviceReceiver) (err error) { virtcontainers/device/drivers/vfio.go:59:func (device *VFIODevice) Attach(devReceiver api.DeviceReceiver) (retErr error) { virtcontainers/device/drivers/vhost_user_fs.go:24:func (device *VhostUserFSDevice) Attach(devReceiver api.DeviceReceiver) (err error) { ubuntu@nshost:/opt/gopath/src/github.com/kata-containers/runtime$
// Attach is standard interface of api.Device func (device *GenericDevice) Attach(devReceiver api.DeviceReceiver) error { _, err := device.bumpAttachCount(true) return err }
devices map[string]api.Device sync.RWMutex } func deviceLogger() *logrus.Entry { return api.DeviceLogger().WithField("subsystem", "device") } // NewDeviceManager creates a deviceManager object behaved as api.DeviceManager func NewDeviceManager(blockDriver string, vhostUserStoreEnabled bool, vhostUserStorePath string, devices []api.Device) api.DeviceManager { dm := &deviceManager{ vhostUserStoreEnabled: vhostUserStoreEnabled, vhostUserStorePath: vhostUserStorePath, devices: make(map[string]api.Device), } if blockDriver == VirtioMmio { dm.blockDriver = VirtioMmio } else if blockDriver == VirtioBlock { dm.blockDriver = VirtioBlock } else if blockDriver == Nvdimm { dm.blockDriver = Nvdimm } else if blockDriver == VirtioBlockCCW { dm.blockDriver = VirtioBlockCCW } else { dm.blockDriver = VirtioSCSI } drivers.AllPCIeDevs = make(map[string]bool) for _, dev := range devices { dm.devices[dev.DeviceID()] = dev } return dm }
root@nshost:/home/ubuntu# docker run -it --runtime=kata-runtime --rm --device /dev/binder:/dev/binder debian /bin/bash docker: Error response from daemon: error gathering device information while adding custom device "/dev/binder": no such file or directory. root@nshost:/home/ubuntu# docker run -it --runtime=kata-runtime --rm --device /dev/binder -v /dev/binder debian /bin/bash docker: Error response from daemon: error gathering device information while adding custom device "/dev/binder": no such file or directory. root@nshost:/home/ubuntu# docker run -it --runtime=kata-runtime --rm --device /dev/binder -v /dev/binder debian /bin/bash
root@nshost:/home/ubuntu# docker run -it --runtime=kata-runtime --rm --device /dev/vfio/0 -v /dev:/dev debian /bin/bash Unable to find image 'debian:latest' locally latest: Pulling from library/debian 22518ad4a7da: Pull complete Digest: sha256:e2cc6fb403be437ef8af68bdc3a89fd58e80b4e390c58f14c77c466002391193 Status: Downloaded newer image for debian:latest docker: Error response from daemon: error gathering device information while adding custom device "/dev/vfio/0": no such file or directory. root@nshost:/home/ubuntu#
hotplugAddBlockDevice
root@nshost:/home/ubuntu# docker run -it --runtime=kata-runtime --rm --device /dev/loop0:/dev/loop0 debian /bin/bash docker: Error response from daemon: OCI runtime create failed: rpc error: code = DeadlineExceeded desc = Timeout reached after 3s waiting for device 0:0:0:0/block: unknown. root@nshost:/home/ubuntu# ls /dev/loop0 /dev/loop0 root@nshost:/home/ubuntu#
Nov 25 11:24:50 nshost kata-runtime[3126]: time="2020-11-25T11:24:50.098759451+08:00" level=info
msg="{"arguments":{"driver":"raw","file":{"driver":"file","filename":"/dev/loop0"},"node-name":"drive-8ace5ee3a2698802"},"execute":"blockdev-add"}"
arch=arm64 command=create container=5755fc7c31f428bd28c8dd6dc76468f897410ca2e3d13ae64d1cf55109ec370f name=kata-runtime pid=3126 source=virtcontainers subsystem=qmp
virtcontainers/hypervisor.go:783: hotplugAddDevice(devInfo interface{}, devType deviceType) (interface{}, error) virtcontainers/sandbox.go:1706: if _, err := s.hypervisor.hotplugAddDevice(dev, vfioDev); err != nil { virtcontainers/sandbox.go:1722: _, err := s.hypervisor.hotplugAddDevice(blockDevice.BlockDrive, blockDev) virtcontainers/sandbox.go:1729: _, err := s.hypervisor.hotplugAddDevice(vhostUserBlkDevice.VhostUserDeviceAttrs, vhostuserDev) virtcontainers/acrn_test.go:147: _, err := a.hotplugAddDevice(&memoryDevice{0, 128, uint64(0), false}, fsDev)
// HotplugAddDevice is used for add a device to sandbox // Sandbox implement DeviceReceiver interface from device/api/interface.go func (s *Sandbox) HotplugAddDevice(device api.Device, devType config.DeviceType) error { span, _ := s.trace("HotplugAddDevice") defer span.Finish() if s.config.SandboxCgroupOnly { // We are about to add a device to the hypervisor, // the device cgroup MUST be updated since the hypervisor // will need access to such device hdev := device.GetHostPath() if err := s.cgroupMgr.AddDevice(hdev); err != nil { s.Logger().WithError(err).WithField("device", hdev). Warn("Could not add device to cgroup") } } switch devType { case config.DeviceVFIO: vfioDevices, ok := device.GetDeviceInfo().([]*config.VFIODev) if !ok { return fmt.Errorf("device type mismatch, expect device type to be %s", devType) } // adding a group of VFIO devices for _, dev := range vfioDevices { if _, err := s.hypervisor.hotplugAddDevice(dev, vfioDev); err != nil { s.Logger(). WithFields(logrus.Fields{ "sandbox": s.id, "vfio-device-ID": dev.ID, "vfio-device-BDF": dev.BDF, }).WithError(err).Error("failed to hotplug VFIO device") return err } } return nil case config.DeviceBlock: blockDevice, ok := device.(*drivers.BlockDevice) if !ok { return fmt.Errorf("device type mismatch, expect device type to be %s", devType) } _, err := s.hypervisor.hotplugAddDevice(blockDevice.BlockDrive, blockDev) return err case config.VhostUserBlk: vhostUserBlkDevice, ok := device.(*drivers.VhostUserBlkDevice) if !ok { return fmt.Errorf("device type mismatch, expect device type to be %s", devType) } _, err := s.hypervisor.hotplugAddDevice(vhostUserBlkDevice.VhostUserDeviceAttrs, vhostuserDev) return err case config.DeviceGeneric: // TODO: what? return nil } return nil }
func (q *qemu) hotplugAddDevice(devInfo interface{}, devType deviceType) (interface{}, error) { span, _ := q.trace("hotplugAddDevice") defer span.Finish() data, err := q.hotplugDevice(devInfo, devType, addDevice) if err != nil { return data, err } return data, nil }
func (q *qemu) hotplugDevice(devInfo interface{}, devType deviceType, op operation) (interface{}, error) { switch devType { case blockDev: drive := devInfo.(*config.BlockDrive) return nil, q.hotplugBlockDevice(drive, op) case cpuDev: vcpus := devInfo.(uint32) return q.hotplugCPUs(vcpus, op) case vfioDev: device := devInfo.(*config.VFIODev) return nil, q.hotplugVFIODevice(device, op) case memoryDev: memdev := devInfo.(*memoryDevice) return q.hotplugMemory(memdev, op) case netDev: device := devInfo.(Endpoint) return nil, q.hotplugNetDevice(device, op) case vhostuserDev: vAttr := devInfo.(*config.VhostUserDeviceAttrs) return nil, q.hotplugVhostUserDevice(vAttr, op) default: return nil, fmt.Errorf("cannot hotplug device: unsupported device type '%v'", devType) } }
// ExecuteBlockdevAdd sends a blockdev-add to the QEMU instance. device is the // path of the device to add, e.g., /dev/rdb0, and blockdevID is an identifier // used to name the device. As this identifier will be passed directly to QMP, // it must obey QMP's naming rules, e,g., it must start with a letter. func (q *QMP) ExecuteBlockdevAdd(ctx context.Context, device, blockdevID string) error { args, _ := q.blockdevAddBaseArgs(device, blockdevID) return q.executeCommand(ctx, "blockdev-add", args, nil) }
// DevicesFromPath computes a list of devices and device permissions from paths (pathOnHost and pathInContainer) and cgroup permissions. func DevicesFromPath(pathOnHost, pathInContainer, cgroupPermissions string) (devs []specs.LinuxDevice, devPermissions []specs.LinuxDeviceCgroup, err error) { resolvedPathOnHost := pathOnHost // check if it is a symbolic link if src, e := os.Lstat(pathOnHost); e == nil && src.Mode()&os.ModeSymlink == os.ModeSymlink { if linkedPathOnHost, e := filepath.EvalSymlinks(pathOnHost); e == nil { resolvedPathOnHost = linkedPathOnHost } } device, err := devices.DeviceFromPath(resolvedPathOnHost, cgroupPermissions) // if there was no error, return the device if err == nil { device.Path = pathInContainer return append(devs, Device(device)), append(devPermissions, deviceCgroup(device)), nil } // if the device is not a device node // try to see if it's a directory holding many devices if err == devices.ErrNotADevice { // check if it is a directory if src, e := os.Stat(resolvedPathOnHost); e == nil && src.IsDir() { // mount the internal devices recursively // TODO check if additional errors should be handled or logged _ = filepath.Walk(resolvedPathOnHost, func(dpath string, f os.FileInfo, _ error) error { childDevice, e := devices.DeviceFromPath(dpath, cgroupPermissions) if e != nil { // ignore the device return nil } // add the device to userSpecified devices childDevice.Path = strings.Replace(dpath, resolvedPathOnHost, pathInContainer, 1) devs = append(devs, Device(childDevice)) devPermissions = append(devPermissions, deviceCgroup(childDevice)) return nil }) } } if len(devs) > 0 { return devs, devPermissions, nil } return devs, devPermissions, fmt.Errorf("error gathering device information while adding custom device %q: %s", pathOnHost, err) }
// WithDevices sets the container's devices func WithDevices(daemon *Daemon, c *container.Container) coci.SpecOpts { return func(ctx context.Context, _ coci.Client, _ *containers.Container, s *coci.Spec) error { // Build lists of devices allowed and created within the container. var devs []specs.LinuxDevice devPermissions := s.Linux.Resources.Devices if c.HostConfig.Privileged && !sys.RunningInUserNS() { hostDevices, err := devices.HostDevices() if err != nil { return err } for _, d := range hostDevices { devs = append(devs, oci.Device(d)) } // adding device mappings in privileged containers for _, deviceMapping := range c.HostConfig.Devices { // issue a warning that custom cgroup permissions are ignored in privileged mode if deviceMapping.CgroupPermissions != "rwm" { logrus.WithField("container", c.ID).Warnf("custom %s permissions for device %s are ignored in privileged mode", deviceMapping.CgroupPermissions, deviceMapping.PathOnHost) } // issue a warning that the device path already exists via /dev mounting in privileged mode if deviceMapping.PathOnHost == deviceMapping.PathInContainer { logrus.WithField("container", c.ID).Warnf("path in container %s already exists in privileged mode", deviceMapping.PathInContainer) continue } d, _, err := oci.DevicesFromPath(deviceMapping.PathOnHost, deviceMapping.PathInContainer, "rwm") if err != nil { return err } devs = append(devs, d...) } devPermissions = []specs.LinuxDeviceCgroup{ { Allow: true, Access: "rwm", }, } } else { for _, deviceMapping := range c.HostConfig.Devices { d, dPermissions, err := oci.DevicesFromPath(deviceMapping.PathOnHost, deviceMapping.PathInContainer, deviceMapping.CgroupPermissions) if err != nil { return err } devs = append(devs, d...) devPermissions = append(devPermissions, dPermissions...) } var err error devPermissions, err = oci.AppendDevicePermissionsFromCgroupRules(devPermissions, c.HostConfig.DeviceCgroupRules) if err != nil { return err } } s.Linux.Devices = append(s.Linux.Devices, devs...) s.Linux.Resources.Devices = devPermissions for _, req := range c.HostConfig.DeviceRequests { if err := daemon.handleDevice(req, s); err != nil { return err } } return nil } }
func (daemon *Daemon) handleDevice(req container.DeviceRequest, spec *specs.Spec) error { if req.Driver == "" { for _, dd := range deviceDrivers { if selected := dd.capset.Match(req.Capabilities); selected != nil { return dd.updateSpec(spec, &deviceInstance{req: req, selectedCaps: selected}) } } } else if dd := deviceDrivers[req.Driver]; dd != nil { if selected := dd.capset.Match(req.Capabilities); selected != nil { return dd.updateSpec(spec, &deviceInstance{req: req, selectedCaps: selected}) } } return incompatibleDeviceRequest{req.Driver, req.Capabilities} }