From c98bd215cdd2ac3e4296d5ff6264eae18a8cda90 Mon Sep 17 00:00:00 2001
From: zhengchuan <zhengchuan@huawei.com>
Date: Wed, 30 Nov 2022 15:01:12 +0800
Subject: [PATCH] migration/migration-pin: add domainMigrationPid for
 qemuMonitorCallbacks

add domainMigrationPid for qemuMonitorCallbacks

Signed-off-by:zhengchuan<zhengchuan@huawei.com>
---
 src/qemu/qemu_process.c | 184 ++++++++++++++++++++++++++++++++++++++++
 src/qemu/qemu_process.h |  12 +++
 2 files changed, 196 insertions(+)

diff --git a/src/qemu/qemu_process.c b/src/qemu/qemu_process.c
index fc05b4b24f..5be6710ea7 100644
--- a/src/qemu/qemu_process.c
+++ b/src/qemu/qemu_process.c
@@ -1819,6 +1819,7 @@ static qemuMonitorCallbacks monitorCallbacks = {
     .domainMemoryFailure = qemuProcessHandleMemoryFailure,
     .domainMemoryDeviceSizeChange = qemuProcessHandleMemoryDeviceSizeChange,
     .domainDeviceUnplugError = qemuProcessHandleDeviceUnplugErr,
+    .domainMigrationPid = qemuProcessHandleMigrationPid,
     .domainNetdevStreamDisconnected = qemuProcessHandleNetdevStreamDisconnected,
 };
 
@@ -2676,6 +2677,189 @@ qemuProcessResctrlCreate(virQEMUDriver *driver,
 }
 
 
+int
+qemuProcessSetupMigration(virDomainObj *vm,
+                          virDomainMigrationIDDefPtr migration)
+{
+    return qemuProcessSetupPid(vm, migration->thread_id,
+                               VIR_CGROUP_THREAD_MIGRATION_THREAD,
+                               0,
+                               vm->def->cputune.emulatorpin,
+                               vm->def->cputune.emulator_period,
+                               vm->def->cputune.emulator_quota,
+                               &migration->sched);
+}
+
+
+unsigned char *
+virParseCPUList(int *cpumaplen, const char *cpulist, int maxcpu)
+{
+    int lastcpu;
+    unsigned char *cpumap = NULL;
+    virBitmap *map = NULL;
+
+    if (cpulist[0] == 'r') {
+        map = virBitmapNew(maxcpu);
+        if (!map)
+            return NULL;
+        virBitmapSetAll(map);
+    } else {
+        if (virBitmapParse(cpulist, &map, 1024) < 0 ||
+            virBitmapIsAllClear(map)) {
+                goto cleanup;
+            }
+
+            lastcpu = virBitmapLastSetBit(map);
+            if (lastcpu >= maxcpu)
+                goto cleanup;
+    }
+
+    if (virBitmapToData(map, &cpumap, cpumaplen) < 0)
+        VIR_ERROR(_("Bitmap to data failure"));
+
+ cleanup:
+    virBitmapFree(map);
+    return cpumap;
+}
+
+
+/*
+ * If priv->pcpumap is NULL, it means migrationpin command is not called,
+ * otherwise we set the affinity of migration thread by migrationpin.
+ */
+static virBitmap *
+qemuProcessGetPcpumap(qemuDomainObjPrivate *priv)
+{
+    int cpumaplen = 0;
+    int maxcpu = 0;
+    g_autofree unsigned char *cpumap = NULL;
+    virBitmap *pcpumap = NULL;
+
+    if(priv->pcpumap)
+        return priv->pcpumap;
+
+    if (!(priv->migrationThreadPinList) || STREQ(priv->migrationThreadPinList, "")) {
+        VIR_ERROR(_("didn't set the migratin thread pin"));
+        return NULL;
+    }
+
+    /* judge whether migration.pin is default value or not */
+    if (STREQ(priv->migrationThreadPinList, "none"))
+        return NULL;
+
+    maxcpu = virHostCPUGetCount();
+    if (maxcpu < 0) {
+        VIR_ERROR(_("get the cpu count of host failure"));
+        return NULL;
+    }
+
+    cpumap = virParseCPUList(&cpumaplen, priv->migrationThreadPinList, maxcpu);
+    if (!cpumap) {
+        VIR_ERROR(_("parse migration.pin params failure : migration.pin = %s"),
+                  priv->migrationThreadPinList);
+        return NULL;
+    }
+
+    if (!(pcpumap = virBitmapNewData(cpumap, cpumaplen))) {
+        VIR_ERROR(_("Bitmap data failure"));
+        return pcpumap;
+    }
+
+    return pcpumap;
+}
+
+
+/*
+ * In order to set migration thread affinity when vm is migrating,
+ * we should create the cgroup for migration thread.
+ */
+static void
+qemuProcessSetMigthreadAffinity(qemuDomainObjPrivate *priv,
+                                virBitmap *pcpumap,
+                                int mpid)
+{
+    int migration_id = 0;
+    virCgroup *cgroup_migthread = NULL;
+
+    if (!pcpumap)
+        return;
+
+    if (virCgroupHasController(priv->cgroup,
+                               VIR_CGROUP_CONTROLLER_CPUSET)) {
+        if (virCgroupNewThread(priv->cgroup, VIR_CGROUP_THREAD_MIGRATION_THREAD,
+                               migration_id, false, &cgroup_migthread) < 0)
+            goto cleanup;
+
+        if (virCgroupSetupCpusetCpus(cgroup_migthread, pcpumap) < 0) {
+            virReportError(VIR_ERR_OPERATION_INVALID,
+                           _("failed to set cpuset.cpus in cgroup"
+                             " for migration%d thread"), migration_id);
+            goto cleanup;
+        }
+   }
+
+   if (virProcessSetAffinity(mpid, pcpumap, false) < 0)
+       VIR_WARN("failed to set affinity in migration");
+
+ cleanup:
+    if (cgroup_migthread)
+        virCgroupFree(cgroup_migthread);
+    return;
+}
+
+
+void
+qemuProcessHandleMigrationPid(qemuMonitor *mon ATTRIBUTE_UNUSED,
+                              virDomainObj *vm,
+                              int mpid)
+{
+    qemuDomainObjPrivate *priv;
+    char *mpidStr = NULL;
+    virDomainMigrationIDDefPtr migration = NULL;
+    virBitmap *pcpumap = NULL;
+    virObjectLock(vm);
+
+    VIR_INFO("Migrating domain %p %s, migration pid %d",
+              vm, vm->def->name, mpid);
+
+    priv = vm->privateData;
+    if (vm->job->asyncJob == VIR_ASYNC_JOB_NONE) {
+        VIR_DEBUG("got MIGRATION_PID event without a migration job");
+        goto cleanup;
+    }
+
+    migration = g_malloc0(sizeof(*migration));
+    migration->thread_id = mpid;
+
+    if (qemuProcessSetupMigration(vm, migration) < 0) {
+        VIR_ERROR(_("fail to setup migration cgroup"));
+        goto cleanup;
+    }
+
+    mpidStr = g_strdup_printf("%d", mpid);
+
+    VIR_FREE(priv->migrationPids);
+    priv->migrationPids = mpidStr;
+
+    pcpumap = qemuProcessGetPcpumap(priv);
+
+    if (!pcpumap)
+        goto cleanup;
+
+    qemuProcessSetMigthreadAffinity(priv, pcpumap, mpid);
+
+ cleanup:
+    /*
+     * If the value of pcpumap is setted by priv->migrationThreadPinList,
+     * we need to free pcpumap.
+     */
+    if (pcpumap != priv->pcpumap)
+        virBitmapFree(pcpumap);
+    virDomainMigrationIDDefFree(migration);
+    virObjectUnlock(vm);
+}
+
+
 static char *
 qemuProcessBuildPRHelperPidfilePathOld(virDomainObj *vm)
 {
diff --git a/src/qemu/qemu_process.h b/src/qemu/qemu_process.h
index c1ea949215..fff976f6f7 100644
--- a/src/qemu/qemu_process.h
+++ b/src/qemu/qemu_process.h
@@ -25,6 +25,7 @@
 #include "qemu_domain.h"
 #include "qemu_saveimage.h"
 #include "vireventthread.h"
+#include "domain_conf.h"
 
 int qemuProcessPrepareMonitorChr(virDomainChrSourceDef *monConfig,
                                  const char *domainDir);
@@ -250,6 +251,17 @@ int qemuProcessQMPStart(qemuProcessQMP *proc);
 
 bool qemuProcessRebootAllowed(const virDomainDef *def);
 
+int qemuProcessSetupMigration(virDomainObj *vm,
+                              virDomainMigrationIDDefPtr migration);
+
+unsigned char * virParseCPUList(int *cpumaplen,
+                                const char *cpulist,
+                                int maxcpu);
+
+void qemuProcessHandleMigrationPid(qemuMonitor *mon ATTRIBUTE_UNUSED,
+                                   virDomainObj *vm,
+                                   int mpid);
+
 void qemuProcessCleanupMigrationJob(virQEMUDriver *driver,
                                     virDomainObj *vm);
 
-- 
2.27.0