Re: [PATCH v4 5/6] target/riscv: update APLIC and IMSIC to support KVM AIA

[Andrew Jones]

On Wed, Jun 21, 2023 at 02:54:55PM +0000, Yong-Xuan Wang wrote:
> KVM AIA can't emulate APLIC only.

I think you mean "KVM AIA can't emulate the APLIC alone." ?

> When "aia=aplic" parameter is passed,
> APLIC devices is emulated by QEMU. For "aia=aplic-imsic", remove the
> mmio operations of APLIC when using KVM AIA and send wired interrupt
> signal via KVM_IRQ_LINE API.
> After KVM AIA enabled, MSI messages are delivered by KVM_SIGNAL_MSI API
> when the IMSICs receive mmio write requests.
> 
> Signed-off-by: Yong-Xuan Wang <yongxuan.wang@sifive.com>
> Reviewed-by: Jim Shu <jim.shu@sifive.com>
> ---
>  hw/intc/riscv_aplic.c | 58 +++++++++++++++++++++++++++++++------------
>  hw/intc/riscv_imsic.c | 26 +++++++++++++++----
>  2 files changed, 63 insertions(+), 21 deletions(-)
> 
> diff --git a/hw/intc/riscv_aplic.c b/hw/intc/riscv_aplic.c
> index afc5b54dbb..b1949636b6 100644
> --- a/hw/intc/riscv_aplic.c
> +++ b/hw/intc/riscv_aplic.c
> @@ -31,6 +31,7 @@
>  #include "hw/irq.h"
>  #include "target/riscv/cpu.h"
>  #include "sysemu/sysemu.h"
> +#include "sysemu/kvm.h"
>  #include "migration/vmstate.h"
>  
>  #define APLIC_MAX_IDC                  (1UL << 14)
> @@ -148,6 +149,15 @@
>  
>  #define APLIC_IDC_CLAIMI               0x1c
>  
> +/*
> + * KVM AIA only supports APLIC.m, fallback to QEMU emulation if we want to 
> use
> + * APLIC.w.

I assume APLIC.m and APLIC.w mean "APLIC MSI" and "APLIC Wired". Can you
please spell that out?

> + */
> +static bool is_kvm_aia(bool msimode)
> +{
> +    return kvm_irqchip_in_kernel() && msimode;
> +}
> +
>  static uint32_t riscv_aplic_read_input_word(RISCVAPLICState *aplic,
>                                              uint32_t word)
>  {
> @@ -471,6 +481,13 @@ static uint32_t riscv_aplic_idc_claimi(RISCVAPLICState 
> *aplic, uint32_t idc)
>      return topi;
>  }
>  
> +static void riscv_kvm_aplic_request(void *opaque, int irq, int level)
> +{
> +    kvm_set_irq(kvm_state, irq, !!level);
> +
> +    return;

Unnecessary 'return'

> +}
> +
>  static void riscv_aplic_request(void *opaque, int irq, int level)
>  {
>      bool update = false;
> @@ -801,29 +818,35 @@ static void riscv_aplic_realize(DeviceState *dev, Error 
> **errp)
>      uint32_t i;
>      RISCVAPLICState *aplic = RISCV_APLIC(dev);
>  
> -    aplic->bitfield_words = (aplic->num_irqs + 31) >> 5;
> -    aplic->sourcecfg = g_new0(uint32_t, aplic->num_irqs);
> -    aplic->state = g_new0(uint32_t, aplic->num_irqs);
> -    aplic->target = g_new0(uint32_t, aplic->num_irqs);
> -    if (!aplic->msimode) {
> -        for (i = 0; i < aplic->num_irqs; i++) {
> -            aplic->target[i] = 1;
> +    if (!is_kvm_aia(aplic->msimode)) {
> +        aplic->bitfield_words = (aplic->num_irqs + 31) >> 5;
> +        aplic->sourcecfg = g_new0(uint32_t, aplic->num_irqs);
> +        aplic->state = g_new0(uint32_t, aplic->num_irqs);
> +        aplic->target = g_new0(uint32_t, aplic->num_irqs);
> +        if (!aplic->msimode) {
> +            for (i = 0; i < aplic->num_irqs; i++) {
> +                aplic->target[i] = 1;
> +            }
>          }
> -    }
> -    aplic->idelivery = g_new0(uint32_t, aplic->num_harts);
> -    aplic->iforce = g_new0(uint32_t, aplic->num_harts);
> -    aplic->ithreshold = g_new0(uint32_t, aplic->num_harts);
> +        aplic->idelivery = g_new0(uint32_t, aplic->num_harts);
> +        aplic->iforce = g_new0(uint32_t, aplic->num_harts);
> +        aplic->ithreshold = g_new0(uint32_t, aplic->num_harts);
>  
> -    memory_region_init_io(&aplic->mmio, OBJECT(dev), &riscv_aplic_ops, aplic,
> -                          TYPE_RISCV_APLIC, aplic->aperture_size);
> -    sysbus_init_mmio(SYS_BUS_DEVICE(dev), &aplic->mmio);
> +        memory_region_init_io(&aplic->mmio, OBJECT(dev), &riscv_aplic_ops,
> +                             aplic, TYPE_RISCV_APLIC, aplic->aperture_size);
> +        sysbus_init_mmio(SYS_BUS_DEVICE(dev), &aplic->mmio);
> +    }
>  
>      /*
>       * Only root APLICs have hardware IRQ lines. All non-root APLICs
>       * have IRQ lines delegated by their parent APLIC.
>       */
>      if (!aplic->parent) {
> -        qdev_init_gpio_in(dev, riscv_aplic_request, aplic->num_irqs);
> +        if (is_kvm_aia(aplic->msimode)) {
> +            qdev_init_gpio_in(dev, riscv_kvm_aplic_request, aplic->num_irqs);
> +        } else {
> +            qdev_init_gpio_in(dev, riscv_aplic_request, aplic->num_irqs);
> +        }
>      }
>  
>      /* Create output IRQ lines for non-MSI mode */
> @@ -958,7 +981,10 @@ DeviceState *riscv_aplic_create(hwaddr addr, hwaddr size,
>      qdev_prop_set_bit(dev, "mmode", mmode);
>  
>      sysbus_realize_and_unref(SYS_BUS_DEVICE(dev), &error_fatal);
> -    sysbus_mmio_map(SYS_BUS_DEVICE(dev), 0, addr);
> +
> +    if (!is_kvm_aia(msimode)) {
> +        sysbus_mmio_map(SYS_BUS_DEVICE(dev), 0, addr);
> +    }
>  
>      if (parent) {
>          riscv_aplic_add_child(parent, dev);
> diff --git a/hw/intc/riscv_imsic.c b/hw/intc/riscv_imsic.c
> index fea3385b51..00fdb60fc6 100644
> --- a/hw/intc/riscv_imsic.c
> +++ b/hw/intc/riscv_imsic.c
> @@ -32,6 +32,7 @@
>  #include "target/riscv/cpu.h"
>  #include "target/riscv/cpu_bits.h"
>  #include "sysemu/sysemu.h"
> +#include "sysemu/kvm.h"
>  #include "migration/vmstate.h"
>  
>  #define IMSIC_MMIO_PAGE_LE             0x00
> @@ -283,6 +284,20 @@ static void riscv_imsic_write(void *opaque, hwaddr addr, 
> uint64_t value,
>          goto err;
>      }
>  
> +#if defined(CONFIG_KVM)
> +    if (kvm_irqchip_in_kernel()) {
> +        struct kvm_msi msi;
> +
> +        msi.address_lo = extract64(imsic->mmio.addr + addr, 0, 32);
> +        msi.address_hi = extract64(imsic->mmio.addr + addr, 32, 32);
> +        msi.data = le32_to_cpu(value);
> +
> +        kvm_vm_ioctl(kvm_state, KVM_SIGNAL_MSI, &msi);
> +
> +        return;
> +    }
> +#endif
> +
>      /* Writes only supported for MSI little-endian registers */
>      page = addr >> IMSIC_MMIO_PAGE_SHIFT;
>      if ((addr & (IMSIC_MMIO_PAGE_SZ - 1)) == IMSIC_MMIO_PAGE_LE) {
> @@ -296,7 +311,6 @@ static void riscv_imsic_write(void *opaque, hwaddr addr, 
> uint64_t value,
>      riscv_imsic_update(imsic, page);
>  
>      return;
> -

Stray line removal. IMO it's better to leave it.

>  err:
>      qemu_log_mask(LOG_GUEST_ERROR,
>                    "%s: Invalid register write 0x%" HWADDR_PRIx "\n",
> @@ -320,10 +334,12 @@ static void riscv_imsic_realize(DeviceState *dev, Error 
> **errp)
>      CPUState *cpu = cpu_by_arch_id(imsic->hartid);
>      CPURISCVState *env = cpu ? cpu->env_ptr : NULL;
>  
> -    imsic->num_eistate = imsic->num_pages * imsic->num_irqs;
> -    imsic->eidelivery = g_new0(uint32_t, imsic->num_pages);
> -    imsic->eithreshold = g_new0(uint32_t, imsic->num_pages);
> -    imsic->eistate = g_new0(uint32_t, imsic->num_eistate);
> +    if (!kvm_irqchip_in_kernel()) {
> +        imsic->num_eistate = imsic->num_pages * imsic->num_irqs;
> +        imsic->eidelivery = g_new0(uint32_t, imsic->num_pages);
> +        imsic->eithreshold = g_new0(uint32_t, imsic->num_pages);
> +        imsic->eistate = g_new0(uint32_t, imsic->num_eistate);
> +    }
>  
>      memory_region_init_io(&imsic->mmio, OBJECT(dev), &riscv_imsic_ops,
>                            imsic, TYPE_RISCV_IMSIC,
> -- 
> 2.17.1
> 
>

Otherwise,

Reviewed-by: Andrew Jones <ajones@ventanamicro.com>

Thanks,
drew