From 0a75312c069d89be94bcaa688429d8f60a0c528b Mon Sep 17 00:00:00 2001
From: Keqian Zhu <zhukeqian1@huawei.com>
Date: Fri, 10 Apr 2020 13:15:35 +0800
Subject: [PATCH] intc/gicv3: Add pre-sizing capability to GICv3

Currently GICv3 supports fixed smp_cpus CPUs, and all CPUs are
present always. Now we want to pre-sizing GICv3 to support max_cpus
CPUs and not all of them are present always, so some sizing codes
should be concerned.

GIC irqs, GICR and GICC are pre-created for all possible CPUs at
start, but only smp_cpus CPUs are realize and irqs of smp_cpus CPUs
are connected.

Other code changes are mainly for arm_gicv3, and we do little about
kvm_arm_gicv3 becasue KVM will deal with the sizing information properly.

Signed-off-by: Keqian Zhu <zhukeqian1@huawei.com>
Signed-off-by: Salil Mehta <salil.mehta@huawei.com>
---
 hw/arm/virt.c              | 17 +++++++++++----
 hw/intc/arm_gicv3.c        | 43 +++++++++++++++++++++++++-------------
 hw/intc/arm_gicv3_common.c | 23 ++++++++++++++++++--
 hw/intc/arm_gicv3_cpuif.c  |  4 ++++
 hw/intc/arm_gicv3_kvm.c    | 28 ++++++++++++++++++++++++-
 include/hw/arm/virt.h      |  3 ++-
 6 files changed, 96 insertions(+), 22 deletions(-)

diff --git a/hw/arm/virt.c b/hw/arm/virt.c
index 55d403bad6..dda22194b5 100644
--- a/hw/arm/virt.c
+++ b/hw/arm/virt.c
@@ -761,14 +761,19 @@ static void create_gic(VirtMachineState *vms)
     SysBusDevice *gicbusdev;
     const char *gictype;
     int type = vms->gic_version, i;
+    /* The max number of CPUs suppored by GIC */
+    unsigned int num_cpus = ms->smp.cpus;
+    /* The number of CPUs present before boot */
     unsigned int smp_cpus = ms->smp.cpus;
     uint32_t nb_redist_regions = 0;

+    assert(num_cpus >= smp_cpus);
+
     gictype = (type == 3) ? gicv3_class_name() : gic_class_name();

     vms->gic = qdev_create(NULL, gictype);
     qdev_prop_set_uint32(vms->gic, "revision", type);
-    qdev_prop_set_uint32(vms->gic, "num-cpu", smp_cpus);
+    qdev_prop_set_uint32(vms->gic, "num-cpu", num_cpus);
     /* Note that the num-irq property counts both internal and external
      * interrupts; there are always 32 of the former (mandated by GIC spec).
      */
@@ -780,7 +785,7 @@ static void create_gic(VirtMachineState *vms)
     if (type == 3) {
         uint32_t redist0_capacity =
                     vms->memmap[VIRT_GIC_REDIST].size / GICV3_REDIST_SIZE;
-        uint32_t redist0_count = MIN(smp_cpus, redist0_capacity);
+        uint32_t redist0_count = MIN(num_cpus, redist0_capacity);

         nb_redist_regions = virt_gicv3_redist_region_count(vms);

@@ -793,7 +798,7 @@ static void create_gic(VirtMachineState *vms)
                     vms->memmap[VIRT_HIGH_GIC_REDIST2].size / GICV3_REDIST_SIZE;

             qdev_prop_set_uint32(vms->gic, "redist-region-count[1]",
-                MIN(smp_cpus - redist0_count, redist1_capacity));
+                MIN(num_cpus - redist0_count, redist1_capacity));
         }
     } else {
         if (!kvm_irqchip_in_kernel()) {
@@ -820,7 +825,11 @@ static void create_gic(VirtMachineState *vms)

     /* Wire the outputs from each CPU's generic timer and the GICv3
      * maintenance interrupt signal to the appropriate GIC PPI inputs,
-     * and the GIC's IRQ/FIQ/VIRQ/VFIQ interrupt outputs to the CPU's inputs.
+     * and the GIC's IRQ/FIQ/VIRQ/VFIQ interrupt outputs to the CPU's
+     * inputs.
+     *
+     * The irqs of remaining CPUs (if we has) will be connected during
+     * hotplugging.
      */
     for (i = 0; i < smp_cpus; i++) {
         connect_gic_cpu_irqs(vms, i);
diff --git a/hw/intc/arm_gicv3.c b/hw/intc/arm_gicv3.c
index cacef26546..a60185113f 100644
--- a/hw/intc/arm_gicv3.c
+++ b/hw/intc/arm_gicv3.c
@@ -20,6 +20,7 @@
 #include "qemu/module.h"
 #include "hw/sysbus.h"
 #include "hw/intc/arm_gicv3.h"
+#include "qom/cpu.h"
 #include "gicv3_internal.h"

 static bool irqbetter(GICv3CPUState *cs, int irq, uint8_t prio)
@@ -206,7 +207,9 @@ static void gicv3_update_noirqset(GICv3State *s, int start, int len)
     assert(len > 0);

     for (i = 0; i < s->num_cpu; i++) {
-        s->cpu[i].seenbetter = false;
+        if (qemu_get_cpu(i)) {
+            s->cpu[i].seenbetter = false;
+        }
     }

     /* Find the highest priority pending interrupt in this range. */
@@ -248,16 +251,18 @@ static void gicv3_update_noirqset(GICv3State *s, int start, int len)
      * now be the new best one).
      */
     for (i = 0; i < s->num_cpu; i++) {
-        GICv3CPUState *cs = &s->cpu[i];
+        if (qemu_get_cpu(i)) {
+            GICv3CPUState *cs = &s->cpu[i];

-        if (cs->seenbetter) {
-            cs->hppi.grp = gicv3_irq_group(cs->gic, cs, cs->hppi.irq);
-        }
+            if (cs->seenbetter) {
+                cs->hppi.grp = gicv3_irq_group(cs->gic, cs, cs->hppi.irq);
+            }

-        if (!cs->seenbetter && cs->hppi.prio != 0xff &&
-            cs->hppi.irq >= start && cs->hppi.irq < start + len) {
-            gicv3_full_update_noirqset(s);
-            break;
+            if (!cs->seenbetter && cs->hppi.prio != 0xff &&
+                cs->hppi.irq >= start && cs->hppi.irq < start + len) {
+                gicv3_full_update_noirqset(s);
+                break;
+            }
         }
     }
 }
@@ -268,7 +273,9 @@ void gicv3_update(GICv3State *s, int start, int len)

     gicv3_update_noirqset(s, start, len);
     for (i = 0; i < s->num_cpu; i++) {
-        gicv3_cpuif_update(&s->cpu[i]);
+        if (qemu_get_cpu(i)) {
+            gicv3_cpuif_update(&s->cpu[i]);
+        }
     }
 }

@@ -280,7 +287,9 @@ void gicv3_full_update_noirqset(GICv3State *s)
     int i;

     for (i = 0; i < s->num_cpu; i++) {
-        s->cpu[i].hppi.prio = 0xff;
+        if (qemu_get_cpu(i)) {
+            s->cpu[i].hppi.prio = 0xff;
+        }
     }

     /* Note that we can guarantee that these functions will not
@@ -291,7 +300,9 @@ void gicv3_full_update_noirqset(GICv3State *s)
     gicv3_update_noirqset(s, GIC_INTERNAL, s->num_irq - GIC_INTERNAL);

     for (i = 0; i < s->num_cpu; i++) {
-        gicv3_redist_update_noirqset(&s->cpu[i]);
+        if (qemu_get_cpu(i)) {
+            gicv3_redist_update_noirqset(&s->cpu[i]);
+        }
     }
 }

@@ -304,7 +315,9 @@ void gicv3_full_update(GICv3State *s)

     gicv3_full_update_noirqset(s);
     for (i = 0; i < s->num_cpu; i++) {
-        gicv3_cpuif_update(&s->cpu[i]);
+        if (qemu_get_cpu(i)) {
+            gicv3_cpuif_update(&s->cpu[i]);
+        }
     }
 }

@@ -401,7 +414,9 @@ static void arm_gic_realize(DeviceState *dev, Error **errp)
     }

     for (i = 0; i < s->num_cpu; i++) {
-        gicv3_cpu_realize(s, i);
+        if (qemu_get_cpu(i)) {
+            gicv3_cpu_realize(s, i);
+        }
     }
 }

diff --git a/hw/intc/arm_gicv3_common.c b/hw/intc/arm_gicv3_common.c
index 8740a52c9f..913bf068be 100644
--- a/hw/intc/arm_gicv3_common.c
+++ b/hw/intc/arm_gicv3_common.c
@@ -24,10 +24,12 @@
 #include "qemu/osdep.h"
 #include "qapi/error.h"
 #include "qemu/module.h"
+#include "qemu/error-report.h"
 #include "qom/cpu.h"
 #include "hw/intc/arm_gicv3_common.h"
 #include "gicv3_internal.h"
 #include "hw/arm/linux-boot-if.h"
+#include "hw/boards.h"
 #include "sysemu/kvm.h"


@@ -363,10 +365,15 @@ static void arm_gicv3_common_realize(DeviceState *dev, Error **errp)
     for (i = 0; i < s->num_cpu; i++) {
         CPUState *cpu = qemu_get_cpu(i);

+        MachineState *ms = MACHINE(qdev_get_machine());
+        MachineClass *mc = MACHINE_GET_CLASS(ms);
+        const CPUArchIdList *possible_cpus = NULL;
         uint64_t cpu_affid;
         int last;

-        arm_gicv3_common_cpu_realize(s, i);
+        if (cpu) {
+            arm_gicv3_common_cpu_realize(s, i);
+        }

         /* Pre-construct the GICR_TYPER:
          * For our implementation:
@@ -380,7 +387,19 @@ static void arm_gicv3_common_realize(DeviceState *dev, Error **errp)
          *  VLPIS == 0 (virtual LPIs not supported)
          *  PLPIS == 0 (physical LPIs not supported)
          */
-        cpu_affid = object_property_get_uint(OBJECT(cpu), "mp-affinity", NULL);
+        if (cpu) {
+            cpu_affid = object_property_get_uint(OBJECT(cpu), "mp-affinity", NULL);
+        } else {
+            if (!mc->possible_cpu_arch_ids) {
+                error_report("MachineClass must implement possible_cpu_arch_ids "
+                             "hook to support pre-sizing GICv3");
+                exit(1);
+            }
+
+            possible_cpus = mc->possible_cpu_arch_ids(ms);
+            cpu_affid = possible_cpus->cpus[i].arch_id;
+        }
+
         last = (i == s->num_cpu - 1);

         /* The CPU mp-affinity property is in MPIDR register format; squash
diff --git a/hw/intc/arm_gicv3_cpuif.c b/hw/intc/arm_gicv3_cpuif.c
index 56aa5efede..a20aa693ea 100644
--- a/hw/intc/arm_gicv3_cpuif.c
+++ b/hw/intc/arm_gicv3_cpuif.c
@@ -1648,6 +1648,10 @@ static void icc_generate_sgi(CPUARMState *env, GICv3CPUState *cs,
                                  aff, targetlist);

     for (i = 0; i < s->num_cpu; i++) {
+        if (!qemu_get_cpu(i)) {
+            continue;
+        }
+
         GICv3CPUState *ocs = &s->cpu[i];

         if (irm) {
diff --git a/hw/intc/arm_gicv3_kvm.c b/hw/intc/arm_gicv3_kvm.c
index f8d7be5479..8eea7c9dd9 100644
--- a/hw/intc/arm_gicv3_kvm.c
+++ b/hw/intc/arm_gicv3_kvm.c
@@ -341,6 +341,10 @@ static void kvm_arm_gicv3_put(GICv3State *s)
         for (ncpu = 0; ncpu < s->num_cpu; ncpu++) {
             GICv3CPUState *c = &s->cpu[ncpu];

+            if (!qemu_get_cpu(ncpu)) {
+                continue;
+            }
+
             reg64 = c->gicr_propbaser;
             regl = (uint32_t)reg64;
             kvm_gicr_access(s, GICR_PROPBASER, ncpu, &regl, true);
@@ -366,6 +370,10 @@ static void kvm_arm_gicv3_put(GICv3State *s)
     for (ncpu = 0; ncpu < s->num_cpu; ncpu++) {
         GICv3CPUState *c = &s->cpu[ncpu];

+        if (!qemu_get_cpu(ncpu)) {
+            continue;
+        }
+
         reg = c->gicr_ctlr;
         kvm_gicr_access(s, GICR_CTLR, ncpu, &reg, true);

@@ -462,6 +470,10 @@ static void kvm_arm_gicv3_put(GICv3State *s)
         GICv3CPUState *c = &s->cpu[ncpu];
         int num_pri_bits;

+        if (!qemu_get_cpu(ncpu)) {
+            continue;
+        }
+
         kvm_gicc_access(s, ICC_SRE_EL1, ncpu, &c->icc_sre_el1, true);
         kvm_gicc_access(s, ICC_CTLR_EL1, ncpu,
                         &c->icc_ctlr_el1[GICV3_NS], true);
@@ -525,6 +537,10 @@ static void kvm_arm_gicv3_get(GICv3State *s)
     /* Redistributor state (one per CPU) */

     for (ncpu = 0; ncpu < s->num_cpu; ncpu++) {
+        if (!qemu_get_cpu(ncpu)) {
+            continue;
+        }
+
         GICv3CPUState *c = &s->cpu[ncpu];

         kvm_gicr_access(s, GICR_CTLR, ncpu, &reg, false);
@@ -560,6 +576,10 @@ static void kvm_arm_gicv3_get(GICv3State *s)

     if (redist_typer & GICR_TYPER_PLPIS) {
         for (ncpu = 0; ncpu < s->num_cpu; ncpu++) {
+            if (!qemu_get_cpu(ncpu)) {
+                continue;
+            }
+
             GICv3CPUState *c = &s->cpu[ncpu];

             kvm_gicr_access(s, GICR_PROPBASER, ncpu, &regl, false);
@@ -613,6 +633,10 @@ static void kvm_arm_gicv3_get(GICv3State *s)
      */

     for (ncpu = 0; ncpu < s->num_cpu; ncpu++) {
+        if (!qemu_get_cpu(ncpu)) {
+            continue;
+        }
+
         GICv3CPUState *c = &s->cpu[ncpu];
         int num_pri_bits;

@@ -806,7 +830,9 @@ static void kvm_arm_gicv3_realize(DeviceState *dev, Error **errp)
     }

     for (i = 0; i < s->num_cpu; i++) {
-        kvm_arm_gicv3_cpu_realize(s, i);
+        if (qemu_get_cpu(i)) {
+            kvm_arm_gicv3_cpu_realize(s, i);
+        }
     }

     /* Try to create the device via the device control API */
diff --git a/include/hw/arm/virt.h b/include/hw/arm/virt.h
index 6880ebe07c..beef4c8002 100644
--- a/include/hw/arm/virt.h
+++ b/include/hw/arm/virt.h
@@ -168,8 +168,9 @@ static inline int virt_gicv3_redist_region_count(VirtMachineState *vms)
                 vms->memmap[VIRT_GIC_REDIST].size / GICV3_REDIST_SIZE;

     assert(vms->gic_version == 3);
+    GICv3State *s = ARM_GICV3_COMMON(vms->gic);

-    return vms->smp_cpus > redist0_capacity ? 2 : 1;
+    return s->num_cpu > redist0_capacity ? 2 : 1;
 }

 #endif /* QEMU_ARM_VIRT_H */
--
2.19.1