From a2d8cf86a379bb161cdae850824c9e80fb370599 Mon Sep 17 00:00:00 2001
From: Keqian Zhu <zhukeqian1@huawei.com>
Date: Fri, 10 Apr 2020 14:16:40 +0800
Subject: [PATCH] arm/virt: Start up CPU hot-plug and cold-plug

All the CPU hotplug facilities are ready. Assemble them
to start up CPU hot-plug capability for arm/virt.

This also adds CPU cold plug support to arm virt machine
board. CPU cold plug means adding CPU by using "-device
xx-arm-cpu" when we bring up Qemu.

Signed-off-by: Salil Mehta <salil.mehta@huawei.com>
Signed-off-by: Keqian Zhu <zhukeqian1@huawei.com>
---
 hw/arm/virt.c         | 110 ++++++++++++++++++++++++++++++++++++++++--
 hw/core/cpu-common.c  |   4 ++
 include/hw/arm/virt.h |   1 +
 target/arm/cpu.c      |   2 +
 4 files changed, 113 insertions(+), 4 deletions(-)

diff --git a/hw/arm/virt.c b/hw/arm/virt.c
index 4eb1b44729..b81d22d68f 100644
--- a/hw/arm/virt.c
+++ b/hw/arm/virt.c
@@ -52,6 +52,8 @@
 #include "sysemu/tpm.h"
 #include "sysemu/kvm.h"
 #include "sysemu/hvf.h"
+#include "sysemu/cpus.h"
+#include "sysemu/hw_accel.h"
 #include "hw/loader.h"
 #include "qapi/error.h"
 #include "qemu/bitops.h"
@@ -703,9 +705,9 @@ static inline DeviceState *create_acpi_ged(VirtMachineState *vms)
         event |= ACPI_GED_NVDIMM_HOTPLUG_EVT;
     }
 
-    /* event |= ACPI_GED_CPU_HOTPLUG_EVT;
-     * Currently CPU hotplug is not enabled.
-     */
+    if (vms->cpu_hotplug_enabled) {
+        event |= ACPI_GED_CPU_HOTPLUG_EVT;
+    }
 
     dev = qdev_new(TYPE_ACPI_GED);
     qdev_prop_set_uint32(dev, "ged-event", event);
@@ -2555,11 +2557,18 @@ static void virt_cpu_pre_plug(HotplugHandler *hotplug_dev,
     VirtMachineState *vms = VIRT_MACHINE(hotplug_dev);
     VirtMachineClass *vmc = VIRT_MACHINE_GET_CLASS(hotplug_dev);
     const CPUArchIdList *possible_cpus = mc->possible_cpu_arch_ids(ms);
+    const CPUArchId *cpu_slot = NULL;
     MemoryRegion *sysmem = get_system_memory();
     int smp_clusters = ms->smp.clusters;
     int smp_cores = ms->smp.cores;
     int smp_threads = ms->smp.threads;
 
+    if (!object_dynamic_cast(OBJECT(cpu), ms->cpu_type)) {
+        error_setg(errp, "Invalid CPU type, expected cpu type: '%s'",
+                   ms->cpu_type);
+        return;
+    }
+
     /* if cpu idx is not set, set it based on socket/cluster/core/thread
      * properties
      */
@@ -2593,6 +2602,20 @@ static void virt_cpu_pre_plug(HotplugHandler *hotplug_dev,
         cs->cpu_index = idx_from_topo_ids(smp_clusters, smp_cores, smp_threads, &topo);
     }
 
+    /* Some hotplug capability checks */
+    if (cs->cpu_index >= ms->smp.cpus) {
+        if (!vms->acpi_dev) {
+            error_setg(errp, "CPU cold/hot plug is disabled: "
+                             "missing acpi device.");
+            return;
+        }
+        if (!vms->cpu_hotplug_enabled) {
+            error_setg(errp, "CPU cold/hot plug is disabled: "
+                             "should use AArch64 CPU and GICv3.");
+            return;
+        }
+    }
+
     /* if 'address' properties socket-id/cluster-id/core-id/thread-id are not
      * set, set them so that machine_query_hotpluggable_cpus would show correct
      * values
@@ -2631,6 +2654,13 @@ static void virt_cpu_pre_plug(HotplugHandler *hotplug_dev,
     object_property_set_int(cpuobj, "mp-affinity",
                             possible_cpus->cpus[cs->cpu_index].arch_id, NULL);
 
+    cpu_slot = &possible_cpus->cpus[cs->cpu_index];
+    if (cpu_slot->cpu) {
+        error_setg(errp, "CPU[%d] with mp_affinity %" PRIu64 " exists",
+                   cs->cpu_index, cpu->mp_affinity);
+        return;
+    }
+
     numa_cpu_pre_plug(&possible_cpus->cpus[cs->cpu_index], DEVICE(cpuobj),
                       &error_fatal);
 
@@ -2716,12 +2746,83 @@ static void virt_cpu_pre_plug(HotplugHandler *hotplug_dev,
                                      &error_abort);
         }
     }
+
+    /* If we use KVM accel, we should pause all vcpus to
+     * allow hot access of vcpu registers.
+     */
+    if (dev->hotplugged && kvm_enabled()) {
+        pause_all_vcpus();
+    }
 }
 
 static void virt_cpu_plug(HotplugHandler *hotplug_dev,
                           DeviceState *dev, Error **errp)
 {
-    /* Currently nothing to do */
+    CPUArchId *cpu_slot;
+    CPUState *cs = CPU(dev);
+    int ncpu = cs->cpu_index;
+    MachineState *ms = MACHINE(hotplug_dev);
+    VirtMachineState *vms = VIRT_MACHINE(hotplug_dev);
+    bool pmu = object_property_get_bool(OBJECT(first_cpu), "pmu", NULL);
+    bool steal_time = object_property_get_bool(OBJECT(first_cpu),
+                                               "kvm-steal-time", NULL);
+    GICv3State *gicv3;
+    ARMGICv3CommonClass *agcc;
+    Error *local_err = NULL;
+
+    /* For CPU that is cold/hot plugged */
+    if (ncpu >= ms->smp.cpus) {
+        /* Realize GIC related parts of CPU */
+        assert(vms->gic_version == 3);
+        gicv3 = ARM_GICV3_COMMON(vms->gic);
+        agcc = ARM_GICV3_COMMON_GET_CLASS(gicv3);
+        agcc->cpu_hotplug_realize(gicv3, ncpu);
+        connect_gic_cpu_irqs(vms, ncpu);
+
+        /* Init PMU and steal_time part */
+        if (kvm_enabled()) {
+            hwaddr pvtime_reg_base = vms->memmap[VIRT_PVTIME].base;
+
+            if (pmu) {
+                assert(arm_feature(&ARM_CPU(cs)->env, ARM_FEATURE_PMU));
+                if (kvm_irqchip_in_kernel()) {
+                    kvm_arm_pmu_set_irq(cs, PPI(VIRTUAL_PMU_IRQ));
+                }
+                kvm_arm_pmu_init(cs);
+            }
+            if (steal_time) {
+                kvm_arm_pvtime_init(cs, pvtime_reg_base +
+                                        ncpu * PVTIME_SIZE_PER_CPU);
+            }
+        }
+
+        /* Register CPU reset and trigger it manually */
+        cpu_synchronize_state(cs);
+        cpu_hotplug_register_reset(ncpu);
+        cpu_hotplug_reset_manually(ncpu);
+        cpu_synchronize_post_reset(cs);
+    }
+
+    if (dev->hotplugged && kvm_enabled()) {
+        resume_all_vcpus();
+    }
+
+    if (vms->acpi_dev) {
+        hotplug_handler_plug(HOTPLUG_HANDLER(vms->acpi_dev), dev, &local_err);
+        if (local_err) {
+            goto out;
+        }
+    }
+
+    vms->boot_cpus++;
+    if (vms->fw_cfg) {
+        fw_cfg_modify_i16(vms->fw_cfg, FW_CFG_NB_CPUS, vms->boot_cpus);
+    }
+
+    cpu_slot = &ms->possible_cpus->cpus[ncpu];
+    cpu_slot->cpu = OBJECT(dev);
+out:
+    error_propagate(errp, local_err);
 }
 
 static void virt_machine_device_pre_plug_cb(HotplugHandler *hotplug_dev,
@@ -2940,6 +3041,7 @@ static void virt_machine_class_init(ObjectClass *oc, void *data)
     mc->default_cpu_type = ARM_CPU_TYPE_NAME("cortex-a15");
     mc->get_default_cpu_node_id = virt_get_default_cpu_node_id;
     mc->kvm_type = virt_kvm_type;
+    mc->has_hotpluggable_cpus = true;
     assert(!mc->get_hotplug_handler);
     mc->get_hotplug_handler = virt_machine_get_hotplug_handler;
     hc->pre_plug = virt_machine_device_pre_plug_cb;
diff --git a/hw/core/cpu-common.c b/hw/core/cpu-common.c
index 9e3241b430..b8d1d820cb 100644
--- a/hw/core/cpu-common.c
+++ b/hw/core/cpu-common.c
@@ -208,6 +208,10 @@ static void cpu_common_realizefn(DeviceState *dev, Error **errp)
 
     if (dev->hotplugged) {
         cpu_synchronize_post_init(cpu);
+
+#ifdef __aarch64__
+    if (!kvm_enabled())
+#endif
         cpu_resume(cpu);
     }
 
diff --git a/include/hw/arm/virt.h b/include/hw/arm/virt.h
index c371d377e0..4ddee19b18 100644
--- a/include/hw/arm/virt.h
+++ b/include/hw/arm/virt.h
@@ -168,6 +168,7 @@ struct VirtMachineState {
     uint32_t msi_phandle;
     uint32_t iommu_phandle;
     int psci_conduit;
+    uint32_t boot_cpus;
     hwaddr highest_gpa;
     DeviceState *gic;
     DeviceState *acpi_dev;
diff --git a/target/arm/cpu.c b/target/arm/cpu.c
index 9fd8e57971..d550022f18 100644
--- a/target/arm/cpu.c
+++ b/target/arm/cpu.c
@@ -2580,6 +2580,8 @@ static void arm_cpu_class_init(ObjectClass *oc, void *data)
     device_class_set_props(dc, arm_cpu_properties);
     device_class_set_parent_reset(dc, arm_cpu_reset, &acc->parent_reset);
 
+    dc->user_creatable = true;
+
     cc->class_by_name = arm_cpu_class_by_name;
     cc->has_work = arm_cpu_has_work;
     cc->dump_state = arm_cpu_dump_state;
-- 
2.27.0