Re: [PULL 16/45] tests/qtest/libqos: Add generic pci host bridge in arm-virt machine

[Eric Auger]

Hi Michael,

On 3/7/22 11:02 AM, Michael S. Tsirkin wrote:
> On Fri, Mar 04, 2022 at 08:39:36AM -0500, Michael S. Tsirkin wrote:
>> From: Eric Auger <eric.auger@redhat.com>
>>
>> Up to now the virt-machine node contains a virtio-mmio node.
>> However no driver produces any PCI interface node. Hence, PCI
>> tests cannot be run with aarch64 binary.
>>
>> Add a GPEX driver node that produces a pci interface node. This latter
>> then can be consumed by all the pci tests. One of the first motivation
>> was to be able to run the virtio-iommu-pci tests.
>>
>> We still face an issue with pci hotplug tests as hotplug cannot happen
>> on the pcie root bus and require a generic root port. This will be
>> addressed later on.
>>
>> We force cpu=max along with aarch64/virt machine as some PCI tests
>> require high MMIO regions to be available.
>>
>> Signed-off-by: Eric Auger <eric.auger@redhat.com>
>>
>> Message-Id: <20220210145254.157790-6-eric.auger@redhat.com>
>> Signed-off-by: Eric Auger <eric.auger@redhat.com>
> Pls note I dropped this in v2 due to issues on powerpc.  Feel free to 
> resubmit. Thanks!
noted, thanks. Working on it... Sorry for the inconvenience.

Eric
>
>> ---
>>  tests/qtest/libqos/generic-pcihost.h  |  54 ++++++
>>  tests/qtest/libqos/arm-virt-machine.c |  19 ++-
>>  tests/qtest/libqos/generic-pcihost.c  | 231 ++++++++++++++++++++++++++
>>  tests/qtest/libqos/meson.build        |   1 +
>>  4 files changed, 301 insertions(+), 4 deletions(-)
>>  create mode 100644 tests/qtest/libqos/generic-pcihost.h
>>  create mode 100644 tests/qtest/libqos/generic-pcihost.c
>>
>> diff --git a/tests/qtest/libqos/generic-pcihost.h 
>> b/tests/qtest/libqos/generic-pcihost.h
>> new file mode 100644
>> index 0000000000..c693c769df
>> --- /dev/null
>> +++ b/tests/qtest/libqos/generic-pcihost.h
>> @@ -0,0 +1,54 @@
>> +/*
>> + * libqos Generic PCI bindings and generic pci host bridge
>> + *
>> + * Copyright Red Hat Inc., 2022
>> + *
>> + * Authors:
>> + *  Eric Auger <eric.auger@redhat.com>
>> + *
>> + * This work is licensed under the terms of the GNU GPL, version 2 or later.
>> + * See the COPYING file in the top-level directory.
>> + */
>> +
>> +#ifndef LIBQOS_GENERIC_PCIHOST_H
>> +#define LIBQOS_GENERIC_PCIHOST_H
>> +
>> +#include "pci.h"
>> +#include "malloc.h"
>> +#include "qgraph.h"
>> +
>> +typedef struct QGenericPCIBus {
>> +    QOSGraphObject obj;
>> +    QPCIBus bus;
>> +    uint64_t gpex_pio_base;
>> +    uint64_t ecam_alloc_ptr;
>> +} QGenericPCIBus;
>> +
>> +/*
>> + * qpci_init_generic():
>> + * @ret: A valid QGenericPCIBus * pointer
>> + * @qts: The %QTestState
>> + * @alloc: A previously initialized @alloc providing memory for @qts
>> + * @bool: devices can be hotplugged on this bus
>> + *
>> + * This function initializes an already allocated
>> + * QGenericPCIBus object.
>> + */
>> +void qpci_init_generic(QGenericPCIBus *ret, QTestState *qts,
>> +                       QGuestAllocator *alloc, bool hotpluggable);
>> +
>> +/* QGenericPCIHost */
>> +
>> +typedef struct QGenericPCIHost QGenericPCIHost;
>> +
>> +struct QGenericPCIHost {
>> +    QOSGraphObject obj;
>> +    QGenericPCIBus pci;
>> +};
>> +
>> +QOSGraphObject *generic_pcihost_get_device(void *obj, const char *device);
>> +void qos_create_generic_pcihost(QGenericPCIHost *host,
>> +                                QTestState *qts,
>> +                                QGuestAllocator *alloc);
>> +
>> +#endif
>> diff --git a/tests/qtest/libqos/arm-virt-machine.c 
>> b/tests/qtest/libqos/arm-virt-machine.c
>> index e0f5932284..96da0dde54 100644
>> --- a/tests/qtest/libqos/arm-virt-machine.c
>> +++ b/tests/qtest/libqos/arm-virt-machine.c
>> @@ -22,6 +22,8 @@
>>  #include "malloc.h"
>>  #include "qgraph.h"
>>  #include "virtio-mmio.h"
>> +#include "generic-pcihost.h"
>> +#include "hw/pci/pci_regs.h"
>>  
>>  #define ARM_PAGE_SIZE               4096
>>  #define VIRTIO_MMIO_BASE_ADDR       0x0A003E00
>> @@ -35,6 +37,7 @@ struct QVirtMachine {
>>      QOSGraphObject obj;
>>      QGuestAllocator alloc;
>>      QVirtioMMIODevice virtio_mmio;
>> +    QGenericPCIHost bridge;
>>  };
>>  
>>  static void virt_destructor(QOSGraphObject *obj)
>> @@ -57,11 +60,13 @@ static void *virt_get_driver(void *object, const char 
>> *interface)
>>  static QOSGraphObject *virt_get_device(void *obj, const char *device)
>>  {
>>      QVirtMachine *machine = obj;
>> -    if (!g_strcmp0(device, "virtio-mmio")) {
>> +    if (!g_strcmp0(device, "generic-pcihost")) {
>> +        return &machine->bridge.obj;
>> +    } else if (!g_strcmp0(device, "virtio-mmio")) {
>>          return &machine->virtio_mmio.obj;
>>      }
>>  
>> -    fprintf(stderr, "%s not present in arm/virtio\n", device);
>> +    fprintf(stderr, "%s not present in arm/virt\n", device);
>>      g_assert_not_reached();
>>  }
>>  
>> @@ -76,16 +81,22 @@ static void *qos_create_machine_arm_virt(QTestState *qts)
>>      qvirtio_mmio_init_device(&machine->virtio_mmio, qts, 
>> VIRTIO_MMIO_BASE_ADDR,
>>                               VIRTIO_MMIO_SIZE);
>>  
>> +    qos_create_generic_pcihost(&machine->bridge, qts, &machine->alloc);
>> +
>>      machine->obj.get_device = virt_get_device;
>>      machine->obj.get_driver = virt_get_driver;
>>      machine->obj.destructor = virt_destructor;
>>      return machine;
>>  }
>>  
>> -static void virtio_mmio_register_nodes(void)
>> +static void virt_machine_register_nodes(void)
>>  {
>>      qos_node_create_machine("arm/virt", qos_create_machine_arm_virt);
>>      qos_node_contains("arm/virt", "virtio-mmio", NULL);
>> +
>> +    qos_node_create_machine_args("aarch64/virt", 
>> qos_create_machine_arm_virt,
>> +                                 " -cpu max");
>> +    qos_node_contains("aarch64/virt", "generic-pcihost", NULL);
>>  }
>>  
>> -libqos_init(virtio_mmio_register_nodes);
>> +libqos_init(virt_machine_register_nodes);
>> diff --git a/tests/qtest/libqos/generic-pcihost.c 
>> b/tests/qtest/libqos/generic-pcihost.c
>> new file mode 100644
>> index 0000000000..704bbc3473
>> --- /dev/null
>> +++ b/tests/qtest/libqos/generic-pcihost.c
>> @@ -0,0 +1,231 @@
>> +/*
>> + * libqos PCI bindings for generic PCI
>> + *
>> + * Copyright Red Hat Inc., 2022
>> + *
>> + * Authors:
>> + *  Eric Auger   <eric.auger@redhat.com>
>> + *
>> + * This work is licensed under the terms of the GNU GPL, version 2 or later.
>> + * See the COPYING file in the top-level directory.
>> + */
>> +
>> +#include "qemu/osdep.h"
>> +#include "libqtest.h"
>> +#include "generic-pcihost.h"
>> +#include "qapi/qmp/qdict.h"
>> +#include "hw/pci/pci_regs.h"
>> +
>> +#include "qemu/module.h"
>> +
>> +/* QGenericPCIHost */
>> +
>> +QOSGraphObject *generic_pcihost_get_device(void *obj, const char *device)
>> +{
>> +    QGenericPCIHost *host = obj;
>> +    if (!g_strcmp0(device, "pci-bus-generic")) {
>> +        return &host->pci.obj;
>> +    }
>> +    fprintf(stderr, "%s not present in generic-pcihost\n", device);
>> +    g_assert_not_reached();
>> +}
>> +
>> +void qos_create_generic_pcihost(QGenericPCIHost *host,
>> +                                QTestState *qts,
>> +                                QGuestAllocator *alloc)
>> +{
>> +    host->obj.get_device = generic_pcihost_get_device;
>> +    qpci_init_generic(&host->pci, qts, alloc, false);
>> +}
>> +
>> +static uint8_t qpci_generic_pio_readb(QPCIBus *bus, uint32_t addr)
>> +{
>> +    QGenericPCIBus *s = container_of(bus, QGenericPCIBus, bus);
>> +
>> +    return qtest_readb(bus->qts, s->gpex_pio_base + addr);
>> +}
>> +
>> +static void qpci_generic_pio_writeb(QPCIBus *bus, uint32_t addr, uint8_t 
>> val)
>> +{
>> +    QGenericPCIBus *s = container_of(bus, QGenericPCIBus, bus);
>> +
>> +    qtest_writeb(bus->qts, s->gpex_pio_base + addr,  val);
>> +}
>> +
>> +static uint16_t qpci_generic_pio_readw(QPCIBus *bus, uint32_t addr)
>> +{
>> +    QGenericPCIBus *s = container_of(bus, QGenericPCIBus, bus);
>> +
>> +    return qtest_readw(bus->qts, s->gpex_pio_base + addr);
>> +}
>> +
>> +static void qpci_generic_pio_writew(QPCIBus *bus, uint32_t addr, uint16_t 
>> val)
>> +{
>> +    QGenericPCIBus *s = container_of(bus, QGenericPCIBus, bus);
>> +
>> +    qtest_writew(bus->qts, s->gpex_pio_base + addr, val);
>> +}
>> +
>> +static uint32_t qpci_generic_pio_readl(QPCIBus *bus, uint32_t addr)
>> +{
>> +    QGenericPCIBus *s = container_of(bus, QGenericPCIBus, bus);
>> +
>> +    return qtest_readl(bus->qts, s->gpex_pio_base + addr);
>> +}
>> +
>> +static void qpci_generic_pio_writel(QPCIBus *bus, uint32_t addr, uint32_t 
>> val)
>> +{
>> +    QGenericPCIBus *s = container_of(bus, QGenericPCIBus, bus);
>> +
>> +    qtest_writel(bus->qts, s->gpex_pio_base + addr, val);
>> +}
>> +
>> +static uint64_t qpci_generic_pio_readq(QPCIBus *bus, uint32_t addr)
>> +{
>> +    QGenericPCIBus *s = container_of(bus, QGenericPCIBus, bus);
>> +
>> +    return qtest_readq(bus->qts, s->gpex_pio_base + addr);
>> +}
>> +
>> +static void qpci_generic_pio_writeq(QPCIBus *bus, uint32_t addr, uint64_t 
>> val)
>> +{
>> +    QGenericPCIBus *s = container_of(bus, QGenericPCIBus, bus);
>> +
>> +    qtest_writeq(bus->qts, s->gpex_pio_base + addr, val);
>> +}
>> +
>> +static void qpci_generic_memread(QPCIBus *bus, uint32_t addr, void *buf, 
>> size_t len)
>> +{
>> +    qtest_memread(bus->qts, addr, buf, len);
>> +}
>> +
>> +static void qpci_generic_memwrite(QPCIBus *bus, uint32_t addr,
>> +                                  const void *buf, size_t len)
>> +{
>> +    qtest_memwrite(bus->qts, addr, buf, len);
>> +}
>> +
>> +static uint8_t qpci_generic_config_readb(QPCIBus *bus, int devfn, uint8_t 
>> offset)
>> +{
>> +    QGenericPCIBus *gbus = container_of(bus, QGenericPCIBus, bus);
>> +    uint64_t addr = gbus->ecam_alloc_ptr + ((0 << 20) | (devfn << 12) | 
>> offset);
>> +    uint8_t val;
>> +
>> +    qtest_memread(bus->qts, addr, &val, 1);
>> +    return val;
>> +}
>> +
>> +static uint16_t qpci_generic_config_readw(QPCIBus *bus, int devfn, uint8_t 
>> offset)
>> +{
>> +    QGenericPCIBus *gbus = container_of(bus, QGenericPCIBus, bus);
>> +    uint64_t addr = gbus->ecam_alloc_ptr + ((0 << 20) | (devfn << 12) | 
>> offset);
>> +    uint16_t val;
>> +
>> +    qtest_memread(bus->qts, addr, &val, 2);
>> +    return val;
>> +}
>> +
>> +static uint32_t qpci_generic_config_readl(QPCIBus *bus, int devfn, uint8_t 
>> offset)
>> +{
>> +    QGenericPCIBus *gbus = container_of(bus, QGenericPCIBus, bus);
>> +    uint64_t addr = gbus->ecam_alloc_ptr + ((0 << 20) | (devfn << 12) | 
>> offset);
>> +    uint32_t val;
>> +
>> +    qtest_memread(bus->qts, addr, &val, 4);
>> +    return val;
>> +}
>> +
>> +static void
>> +qpci_generic_config_writeb(QPCIBus *bus, int devfn, uint8_t offset, uint8_t 
>> value)
>> +{
>> +    QGenericPCIBus *gbus = container_of(bus, QGenericPCIBus, bus);
>> +    uint64_t addr = gbus->ecam_alloc_ptr + ((0 << 20) | (devfn << 12) | 
>> offset);
>> +    uint32_t val = value;
>> +
>> +    qtest_memwrite(bus->qts, addr, &val, 1);
>> +}
>> +
>> +static void
>> +qpci_generic_config_writew(QPCIBus *bus, int devfn, uint8_t offset, 
>> uint16_t value)
>> +{
>> +    QGenericPCIBus *gbus = container_of(bus, QGenericPCIBus, bus);
>> +    uint64_t addr = gbus->ecam_alloc_ptr + ((0 << 20) | (devfn << 12) | 
>> offset);
>> +    uint32_t val = value;
>> +
>> +    qtest_memwrite(bus->qts, addr, &val, 2);
>> +}
>> +
>> +static void
>> +qpci_generic_config_writel(QPCIBus *bus, int devfn, uint8_t offset, 
>> uint32_t value)
>> +{
>> +    QGenericPCIBus *gbus = container_of(bus, QGenericPCIBus, bus);
>> +    uint64_t addr = gbus->ecam_alloc_ptr + ((0 << 20) | (devfn << 12) | 
>> offset);
>> +    uint32_t val = value;
>> +
>> +    qtest_memwrite(bus->qts, addr, &val, 4);
>> +}
>> +
>> +static void *qpci_generic_get_driver(void *obj, const char *interface)
>> +{
>> +    QGenericPCIBus *qpci = obj;
>> +    if (!g_strcmp0(interface, "pci-bus")) {
>> +        return &qpci->bus;
>> +    }
>> +    fprintf(stderr, "%s not present in pci-bus-generic\n", interface);
>> +    g_assert_not_reached();
>> +}
>> +
>> +void qpci_init_generic(QGenericPCIBus *qpci, QTestState *qts,
>> +                       QGuestAllocator *alloc, bool hotpluggable)
>> +{
>> +    assert(qts);
>> +
>> +    qpci->gpex_pio_base = 0x3eff0000;
>> +    qpci->bus.not_hotpluggable = !hotpluggable;
>> +    qpci->bus.has_buggy_msi = false;
>> +
>> +    qpci->bus.pio_readb = qpci_generic_pio_readb;
>> +    qpci->bus.pio_readw = qpci_generic_pio_readw;
>> +    qpci->bus.pio_readl = qpci_generic_pio_readl;
>> +    qpci->bus.pio_readq = qpci_generic_pio_readq;
>> +
>> +    qpci->bus.pio_writeb = qpci_generic_pio_writeb;
>> +    qpci->bus.pio_writew = qpci_generic_pio_writew;
>> +    qpci->bus.pio_writel = qpci_generic_pio_writel;
>> +    qpci->bus.pio_writeq = qpci_generic_pio_writeq;
>> +
>> +    qpci->bus.memread = qpci_generic_memread;
>> +    qpci->bus.memwrite = qpci_generic_memwrite;
>> +
>> +    qpci->bus.config_readb = qpci_generic_config_readb;
>> +    qpci->bus.config_readw = qpci_generic_config_readw;
>> +    qpci->bus.config_readl = qpci_generic_config_readl;
>> +
>> +    qpci->bus.config_writeb = qpci_generic_config_writeb;
>> +    qpci->bus.config_writew = qpci_generic_config_writew;
>> +    qpci->bus.config_writel = qpci_generic_config_writel;
>> +
>> +    qpci->bus.qts = qts;
>> +    qpci->bus.pio_alloc_ptr = 0x0000;
>> +    qpci->bus.pio_limit = 0x10000;
>> +    qpci->bus.mmio_alloc_ptr = 0x10000000;
>> +    qpci->bus.mmio_limit = 0x2eff0000;
>> +    qpci->ecam_alloc_ptr = 0x4010000000;
>> +
>> +    qpci->obj.get_driver = qpci_generic_get_driver;
>> +}
>> +
>> +static void qpci_generic_register_nodes(void)
>> +{
>> +    qos_node_create_driver("pci-bus-generic", NULL);
>> +    qos_node_produces("pci-bus-generic", "pci-bus");
>> +}
>> +
>> +static void qpci_generic_pci_register_nodes(void)
>> +{
>> +    qos_node_create_driver("generic-pcihost", NULL);
>> +    qos_node_contains("generic-pcihost", "pci-bus-generic", NULL);
>> +}
>> +
>> +libqos_init(qpci_generic_register_nodes);
>> +libqos_init(qpci_generic_pci_register_nodes);
>> diff --git a/tests/qtest/libqos/meson.build b/tests/qtest/libqos/meson.build
>> index e988d15791..8c8ee15553 100644
>> --- a/tests/qtest/libqos/meson.build
>> +++ b/tests/qtest/libqos/meson.build
>> @@ -42,6 +42,7 @@ libqos_srcs = files('../libqtest.c',
>>          'virtio-scsi.c',
>>          'virtio-serial.c',
>>          'virtio-iommu.c',
>> +        'generic-pcihost.c',
>>  
>>          # qgraph machines:
>>          'aarch64-xlnx-zcu102-machine.c',
>> -- 
>> MST
>>