From 691e45a727267f2e66339646d23d2b3a4f3563d8 Mon Sep 17 00:00:00 2001
From: liu chengshan <atomprimitive@foxmail.com>
Date: Thu, 23 Feb 2023 10:51:03 +0800
Subject: [PATCH] support vdpa generic feature

---
 src/conf/domain_conf.c          | 10 +++++++
 src/conf/domain_conf.h          |  1 +
 src/conf/network_conf.c         |  1 +
 src/conf/network_conf.h         |  2 +-
 src/hypervisor/virhostdev.c     |  5 +++-
 src/libvirt_private.syms        |  1 +
 src/qemu/qemu_command.c         | 27 ++++++++++++------
 src/qemu/qemu_domain.c          |  5 +++-
 src/qemu/qemu_driver.c          |  4 ++-
 src/qemu/qemu_hostdev.c         |  8 ++++++
 src/qemu/qemu_hotplug.c         |  1 +
 src/qemu/qemu_migration.c       |  3 ++
 src/qemu/qemu_process.c         |  1 -
 src/security/security_dac.c     | 11 ++++++++
 src/security/security_selinux.c | 11 ++++++++
 src/util/virpci.c               | 50 +++++++++++++++++++++++++++++++++
 src/util/virpci.h               |  3 ++
 17 files changed, 131 insertions(+), 13 deletions(-)

diff --git a/src/conf/domain_conf.c b/src/conf/domain_conf.c
index b8cf1ad..3b21cff 100644
--- a/src/conf/domain_conf.c
+++ b/src/conf/domain_conf.c
@@ -919,6 +919,7 @@ VIR_ENUM_IMPL(virDomainHostdevSubsysPCIBackend,
               "kvm",
               "vfio",
               "xen",
+              "vdpa-generic",
 );
 
 VIR_ENUM_IMPL(virDomainHostdevSubsysSCSIProtocol,
@@ -31589,6 +31590,11 @@ virDomainNetDefActualFromNetworkPort(virDomainNetDefPtr iface,
                 VIR_DOMAIN_HOSTDEV_PCI_BACKEND_VFIO;
             break;
 
+    case VIR_NETWORK_FORWARD_DRIVER_NAME_VDPA_GENERIC:
+            actual->data.hostdev.def.source.subsys.u.pci.backend =
+                VIR_DOMAIN_HOSTDEV_PCI_BACKEND_VDPA_GENERIC;
+            break;
+
         case VIR_NETWORK_FORWARD_DRIVER_NAME_LAST:
         default:
             virReportEnumRangeError(virNetworkForwardDriverNameType,
@@ -31716,6 +31722,10 @@ virDomainNetDefActualToNetworkPort(virDomainDefPtr dom,
                            _("Unexpected PCI backend 'xen'"));
             break;
 
+    case VIR_DOMAIN_HOSTDEV_PCI_BACKEND_VDPA_GENERIC:
+        port->plug.hostdevpci.driver = VIR_NETWORK_FORWARD_DRIVER_NAME_VDPA_GENERIC;
+        break;
+
         case VIR_DOMAIN_HOSTDEV_PCI_BACKEND_TYPE_LAST:
         default:
             virReportEnumRangeError(virDomainHostdevSubsysPCIBackendType,
diff --git a/src/conf/domain_conf.h b/src/conf/domain_conf.h
index 98fa323..c416cf7 100644
--- a/src/conf/domain_conf.h
+++ b/src/conf/domain_conf.h
@@ -209,6 +209,7 @@ typedef enum {
     VIR_DOMAIN_HOSTDEV_PCI_BACKEND_KVM,    /* force legacy kvm style */
     VIR_DOMAIN_HOSTDEV_PCI_BACKEND_VFIO,   /* force vfio */
     VIR_DOMAIN_HOSTDEV_PCI_BACKEND_XEN,    /* force legacy xen style, use pciback */
+    VIR_DOMAIN_HOSTDEV_PCI_BACKEND_VDPA_GENERIC, /* force vdpa generic */
 
     VIR_DOMAIN_HOSTDEV_PCI_BACKEND_TYPE_LAST
 } virDomainHostdevSubsysPCIBackendType;
diff --git a/src/conf/network_conf.c b/src/conf/network_conf.c
index c379042..5483717 100644
--- a/src/conf/network_conf.c
+++ b/src/conf/network_conf.c
@@ -63,6 +63,7 @@ VIR_ENUM_IMPL(virNetworkForwardDriverName,
               "default",
               "kvm",
               "vfio",
+              "vdpa-generic",
 );
 
 VIR_ENUM_IMPL(virNetworkTaint,
diff --git a/src/conf/network_conf.h b/src/conf/network_conf.h
index db7243e..6070b48 100644
--- a/src/conf/network_conf.h
+++ b/src/conf/network_conf.h
@@ -88,7 +88,7 @@ typedef enum {
     VIR_NETWORK_FORWARD_DRIVER_NAME_DEFAULT, /* kvm now, could change */
     VIR_NETWORK_FORWARD_DRIVER_NAME_KVM,    /* force legacy kvm style */
     VIR_NETWORK_FORWARD_DRIVER_NAME_VFIO,   /* force vfio */
-
+    VIR_NETWORK_FORWARD_DRIVER_NAME_VDPA_GENERIC, /* vdpa generic */
     VIR_NETWORK_FORWARD_DRIVER_NAME_LAST
 } virNetworkForwardDriverNameType;
 
diff --git a/src/hypervisor/virhostdev.c b/src/hypervisor/virhostdev.c
index 4057f7b..e3fee33 100644
--- a/src/hypervisor/virhostdev.c
+++ b/src/hypervisor/virhostdev.c
@@ -247,6 +247,8 @@ virHostdevGetPCIHostDevice(const virDomainHostdevDef *hostdev,
         virPCIDeviceSetStubDriver(actual, VIR_PCI_STUB_DRIVER_VFIO);
     } else if (pcisrc->backend == VIR_DOMAIN_HOSTDEV_PCI_BACKEND_XEN) {
         virPCIDeviceSetStubDriver(actual, VIR_PCI_STUB_DRIVER_XEN);
+    } else if (pcisrc->backend == VIR_DOMAIN_HOSTDEV_PCI_BACKEND_VDPA_GENERIC) {
+        virPCIDeviceSetStubDriver(actual, VIR_PCI_STUB_DRIVER_VDPA_GENERIC);
     } else {
         virReportError(VIR_ERR_CONFIG_UNSUPPORTED,
                        _("pci backend driver '%s' is not supported"),
@@ -705,6 +707,7 @@ virHostdevPreparePCIDevicesImpl(virHostdevManagerPtr mgr,
         virPCIDevicePtr pci = virPCIDeviceListGet(pcidevs, i);
         bool strict_acs_check = !!(flags & VIR_HOSTDEV_STRICT_ACS_CHECK);
         bool usesVFIO = (virPCIDeviceGetStubDriver(pci) == VIR_PCI_STUB_DRIVER_VFIO);
+        bool usesVDPA = (virPCIDeviceGetStubDriver(pci) == VIR_PCI_STUB_DRIVER_VDPA_GENERIC);
         struct virHostdevIsPCINodeDeviceUsedData data = {mgr, drv_name, dom_name, false};
         int hdrType = -1;
 
@@ -718,7 +721,7 @@ virHostdevPreparePCIDevicesImpl(virHostdevManagerPtr mgr,
             goto cleanup;
         }
 
-        if (!usesVFIO && !virPCIDeviceIsAssignable(pci, strict_acs_check)) {
+        if (!usesVFIO && !usesVDPA && !virPCIDeviceIsAssignable(pci, strict_acs_check)) {
             virReportError(VIR_ERR_OPERATION_INVALID,
                            _("PCI device %s is not assignable"),
                            virPCIDeviceGetName(pci));
diff --git a/src/libvirt_private.syms b/src/libvirt_private.syms
index a00e354..b96ae66 100644
--- a/src/libvirt_private.syms
+++ b/src/libvirt_private.syms
@@ -2763,6 +2763,7 @@ virPCIDeviceAddressGetIOMMUGroupAddresses;
 virPCIDeviceAddressGetIOMMUGroupDev;
 virPCIDeviceAddressGetIOMMUGroupNum;
 virPCIDeviceAddressGetSysfsFile;
+virPCIDeviceAddressGetVDPAGroupDev;
 virPCIDeviceAddressIOMMUGroupIterate;
 virPCIDeviceAddressIsEmpty;
 virPCIDeviceAddressIsValid;
diff --git a/src/qemu/qemu_command.c b/src/qemu/qemu_command.c
index 5f7e847..e151c27 100644
--- a/src/qemu/qemu_command.c
+++ b/src/qemu/qemu_command.c
@@ -4825,7 +4825,6 @@ qemuBuildVideoCommandLine(virCommandPtr cmd,
     return 0;
 }
 
-
 char *
 qemuBuildPCIHostdevDevStr(const virDomainDef *def,
                           virDomainHostdevDefPtr dev,
@@ -4842,6 +4841,10 @@ qemuBuildPCIHostdevDevStr(const virDomainDef *def,
         virBufferAddLit(&buf, "vfio-pci");
         break;
 
+    case VIR_DOMAIN_HOSTDEV_PCI_BACKEND_VDPA_GENERIC:
+        virBufferAddLit(&buf, "vhost-vdpa-device-pci");
+        break;
+
     case VIR_DOMAIN_HOSTDEV_PCI_BACKEND_KVM:
     case VIR_DOMAIN_HOSTDEV_PCI_BACKEND_DEFAULT:
     case VIR_DOMAIN_HOSTDEV_PCI_BACKEND_XEN:
@@ -4852,13 +4855,21 @@ qemuBuildPCIHostdevDevStr(const virDomainDef *def,
         return NULL;
     }
 
-    virBufferAddLit(&buf, ",host=");
-    virBufferAsprintf(&buf,
-                      VIR_PCI_DEVICE_ADDRESS_FMT,
-                      pcisrc->addr.domain,
-                      pcisrc->addr.bus,
-                      pcisrc->addr.slot,
-                      pcisrc->addr.function);
+    if (backend == VIR_DOMAIN_HOSTDEV_PCI_BACKEND_VDPA_GENERIC) {
+        g_autofree char *path = NULL;
+        if (qemuDomainGetHostdevPath(dev, &path, NULL) < 0)
+            return NULL;
+        VIR_DEBUG("vdpa dev path: %s", path);
+        virBufferAsprintf(&buf, ",vhostdev=%s", path);
+    } else {
+         virBufferAddLit(&buf, ",host=");
+         virBufferAsprintf(&buf,
+                           VIR_PCI_DEVICE_ADDRESS_FMT,
+                           pcisrc->addr.domain,
+                           pcisrc->addr.bus,
+                           pcisrc->addr.slot,
+                           pcisrc->addr.function);
+    }
     virBufferAsprintf(&buf, ",id=%s", dev->info->alias);
     if (dev->info->bootIndex)
         bootIndex = dev->info->bootIndex;
diff --git a/src/qemu/qemu_domain.c b/src/qemu/qemu_domain.c
index 70835e4..ddd7b7d 100644
--- a/src/qemu/qemu_domain.c
+++ b/src/qemu/qemu_domain.c
@@ -14383,7 +14383,6 @@ qemuDomainNeedsVFIO(const virDomainDef *def)
         virDomainDefHasNVMeDisk(def);
 }
 
-
 /**
  * qemuDomainGetHostdevPath:
  * @dev: host device definition
@@ -14425,6 +14424,10 @@ qemuDomainGetHostdevPath(virDomainHostdevDefPtr dev,
                     return -1;
 
                 perm = VIR_CGROUP_DEVICE_RW;
+            } else if (pcisrc->backend == VIR_DOMAIN_HOSTDEV_PCI_BACKEND_VDPA_GENERIC) {
+                if (!(tmpPath = virPCIDeviceAddressGetVDPAGroupDev(&pcisrc->addr)))
+                    return -1;
+                perm = VIR_CGROUP_DEVICE_RW;
             }
             break;
 
diff --git a/src/qemu/qemu_driver.c b/src/qemu/qemu_driver.c
index 32b3ef3..c3b5e78 100644
--- a/src/qemu/qemu_driver.c
+++ b/src/qemu/qemu_driver.c
@@ -12937,7 +12937,9 @@ qemuNodeDeviceDetachFlags(virNodeDevicePtr dev,
                        _("KVM device assignment is no longer "
                          "supported on this system"));
         goto cleanup;
-    } else {
+    } else if (STREQ(driverName, "vp-vdpa")) {
+       virPCIDeviceSetStubDriver(pci, VIR_PCI_STUB_DRIVER_VDPA_GENERIC);
+    }  else {
         virReportError(VIR_ERR_INVALID_ARG,
                        _("unknown driver name '%s'"), driverName);
         goto cleanup;
diff --git a/src/qemu/qemu_hostdev.c b/src/qemu/qemu_hostdev.c
index fb63da2..d7f4d0a 100644
--- a/src/qemu/qemu_hostdev.c
+++ b/src/qemu/qemu_hostdev.c
@@ -204,6 +204,14 @@ qemuHostdevPreparePCIDevicesCheckSupport(virDomainHostdevDefPtr *hostdevs,
             return false;
             break;
 
+    case VIR_DOMAIN_HOSTDEV_PCI_BACKEND_VDPA_GENERIC:
+        if (!supportsPassthroughVFIO) {
+                virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s",
+                               _("host doesn't support VDPA GENERIC VFIO PCI passthrough"));
+                return false;
+            }
+            break;
+
         case VIR_DOMAIN_HOSTDEV_PCI_BACKEND_XEN:
         case VIR_DOMAIN_HOSTDEV_PCI_BACKEND_TYPE_LAST:
             break;
diff --git a/src/qemu/qemu_hotplug.c b/src/qemu/qemu_hotplug.c
index e13879c..f94a5dd 100644
--- a/src/qemu/qemu_hotplug.c
+++ b/src/qemu/qemu_hotplug.c
@@ -1586,6 +1586,7 @@ qemuDomainAttachHostPCIDevice(virQEMUDriverPtr driver,
         }
         break;
 
+    case VIR_DOMAIN_HOSTDEV_PCI_BACKEND_VDPA_GENERIC:
     case VIR_DOMAIN_HOSTDEV_PCI_BACKEND_DEFAULT:
     case VIR_DOMAIN_HOSTDEV_PCI_BACKEND_KVM:
         break;
diff --git a/src/qemu/qemu_migration.c b/src/qemu/qemu_migration.c
index c2a694e..512239a 100644
--- a/src/qemu/qemu_migration.c
+++ b/src/qemu/qemu_migration.c
@@ -1132,6 +1132,9 @@ qemuMigrationSrcIsAllowedHostdev(const virDomainDef *def)
                  * the device will be auto-unplugged by QEMU during
                  * migration.
                  */
+                if (hostdev->source.subsys.u.pci.backend == VIR_DOMAIN_HOSTDEV_PCI_BACKEND_VDPA_GENERIC) {
+                    continue;
+                }
                 if (hostdev->parentnet &&
                     hostdev->parentnet->teaming.type == VIR_DOMAIN_NET_TEAMING_TYPE_TRANSIENT) {
                     continue;
diff --git a/src/qemu/qemu_process.c b/src/qemu/qemu_process.c
index 48d39ba..5b6ed0e 100644
--- a/src/qemu/qemu_process.c
+++ b/src/qemu/qemu_process.c
@@ -6985,7 +6985,6 @@ qemuProcessSetupDiskThrottlingBlockdev(virQEMUDriverPtr driver,
     return ret;
 }
 
-
 /**
  * qemuProcessLaunch:
  *
diff --git a/src/security/security_dac.c b/src/security/security_dac.c
index 51cabf1..4737b08 100644
--- a/src/security/security_dac.c
+++ b/src/security/security_dac.c
@@ -1303,6 +1303,17 @@ virSecurityDACSetHostdevLabel(virSecurityManagerPtr mgr,
                                                       false,
                                                       &cbdata);
             VIR_FREE(vfioGroupDev);
+        } else if (pcisrc->backend == VIR_DOMAIN_HOSTDEV_PCI_BACKEND_VDPA_GENERIC)  {
+            char *vdpaGroupDev = virPCIDeviceAddressGetVDPAGroupDev(&pcisrc->addr);
+
+            if (!vdpaGroupDev) {
+            virPCIDeviceFree(pci);
+                return -1;
+        }
+            ret = virSecurityDACSetHostdevLabelHelper(vdpaGroupDev,
+                                                      false,
+                                                      &cbdata);
+            VIR_FREE(vdpaGroupDev);
         } else {
             ret = virPCIDeviceFileIterate(pci,
                                           virSecurityDACSetPCILabel,
diff --git a/src/security/security_selinux.c b/src/security/security_selinux.c
index c91c7ca..9a7ebb2 100644
--- a/src/security/security_selinux.c
+++ b/src/security/security_selinux.c
@@ -2161,6 +2161,17 @@ virSecuritySELinuxSetHostdevSubsysLabel(virSecurityManagerPtr mgr,
                                                           false,
                                                           &data);
             VIR_FREE(vfioGroupDev);
+        } else if (pcisrc->backend == VIR_DOMAIN_HOSTDEV_PCI_BACKEND_VDPA_GENERIC) {
+            char *vdpaGroupDev = virPCIDeviceAddressGetVDPAGroupDev(&pcisrc->addr);
+
+            if (!vdpaGroupDev) {
+        virPCIDeviceFree(pci);
+                return -1;
+        }
+            ret = virSecuritySELinuxSetHostdevLabelHelper(vdpaGroupDev,
+                                                          false,
+                                                          &data);
+            VIR_FREE(vdpaGroupDev);
         } else {
             ret = virPCIDeviceFileIterate(pci, virSecuritySELinuxSetPCILabel, &data);
         }
diff --git a/src/util/virpci.c b/src/util/virpci.c
index 6c7e6bb..62044e4 100644
--- a/src/util/virpci.c
+++ b/src/util/virpci.c
@@ -53,6 +53,7 @@ VIR_ENUM_IMPL(virPCIStubDriver,
               "none",
               "pciback", /* XEN */
               "vfio-pci", /* VFIO */
+              "vp-vdpa", /* vdpa */
 );
 
 VIR_ENUM_IMPL(virPCIHeader,
@@ -1994,6 +1995,55 @@ virPCIDeviceGetIOMMUGroupDev(virPCIDevicePtr dev)
     return g_strdup_printf("/dev/vfio/%s", groupFile);
 }
 
+static char*
+subDirPrefixWith(const char *path, const char *prefix)
+{
+    DIR *dir;
+    struct dirent *entry;
+    char *dirName = NULL;
+
+    if (virDirOpenIfExists(&dir, path) <= 0)
+        return NULL;
+
+    while (virDirRead(dir, &entry, path) > 0) {
+        if (STREQLEN(entry->d_name, prefix, strlen(prefix))) {
+            dirName = g_strdup(entry->d_name);
+            goto cleanup;
+        }
+    }
+
+cleanup:
+    VIR_DIR_CLOSE(dir);
+    return dirName;
+}
+
+char*
+virPCIDeviceAddressGetVDPAGroupDev(const virPCIDeviceAddress *devAddr)
+{
+    g_autofree char *pciDir = NULL;
+    g_autofree char *vdpaDir = NULL;
+    char *tmpPath = NULL;
+    char *devPath= NULL;
+
+    pciDir = g_strdup_printf("/sys/bus/pci/devices/%04x:%02x:%02x.%x",
+                             devAddr->domain,
+                             devAddr->bus,
+                             devAddr->slot,
+                             devAddr->function);
+    if ((tmpPath = subDirPrefixWith(pciDir, "vdpa")) == NULL)
+        return NULL;
+
+    vdpaDir = g_strdup_printf("%s/%s", pciDir, tmpPath);
+    VIR_FREE(tmpPath);
+    if ((tmpPath = subDirPrefixWith(vdpaDir, "vhost-vdpa-")) == NULL)
+        return NULL;
+
+    devPath = g_strdup_printf("/dev/%s", tmpPath);
+    VIR_FREE(tmpPath);
+
+    return devPath;
+}
+
 static int
 virPCIDeviceDownstreamLacksACS(virPCIDevicePtr dev)
 {
diff --git a/src/util/virpci.h b/src/util/virpci.h
index f16d236..3445e49 100644
--- a/src/util/virpci.h
+++ b/src/util/virpci.h
@@ -63,6 +63,7 @@ typedef enum {
     VIR_PCI_STUB_DRIVER_NONE = 0,
     VIR_PCI_STUB_DRIVER_XEN,
     VIR_PCI_STUB_DRIVER_VFIO,
+    VIR_PCI_STUB_DRIVER_VDPA_GENERIC,
     VIR_PCI_STUB_DRIVER_LAST
 } virPCIStubDriver;
 
@@ -199,6 +200,8 @@ int virPCIDeviceAddressGetIOMMUGroupNum(virPCIDeviceAddressPtr addr);
 char *virPCIDeviceAddressGetIOMMUGroupDev(const virPCIDeviceAddress *devAddr);
 char *virPCIDeviceGetIOMMUGroupDev(virPCIDevicePtr dev);
 
+char *virPCIDeviceAddressGetVDPAGroupDev(const virPCIDeviceAddress *devAddr);
+
 int virPCIDeviceIsAssignable(virPCIDevicePtr dev,
                              int strict_acs_check);
 
-- 
2.33.0