[Qemu-devel] [RFC PATCH] qemu vfio-powerpc: added initial support

[Alexey Kardashevskiy]

The idea of the patch is to demonstrate what POWER needs to support VFIO.
It will not compile as many other patches are needed and they are not
posted anywhere yet.

The patch includes:

1) sPAPR PCI bus got an additional flag "is_vfio" which defines IOMMU
behavior

2) Added POWER-specific IOMMU handling, at the moment to vfio_pci.c as it
accesses VFIODevice and VFIOIOMMU structs in ordet to call
vfio_dma_map/vfio_dma_unmap.

3) #ifdef/#else for EOI handlers as POWER uses XICS interrupt controller
rather than APIC. Also, sys/io.h has been moved under #ifdef.

4) IRQ update notifier is disabled for POWER. Cannot see how IRQ can
change on POWER dynamically.
---
 hw/spapr_iommu.c |    4 +++
 hw/spapr_pci.c   |   12 ++++++++++
 hw/spapr_pci.h   |    2 +
 hw/vfio_pci.c    |   65 +++++++++++++++++++++++++++++++++++++++++++++++++++++-
 hw/vfio_pci.h    |    3 ++
 5 files changed, 85 insertions(+), 1 deletions(-)

diff --git a/hw/spapr_iommu.c b/hw/spapr_iommu.c
index e97abf1..aa08368 100644
--- a/hw/spapr_iommu.c
+++ b/hw/spapr_iommu.c
@@ -23,6 +23,7 @@
 #include "dma.h"

 #include "hw/spapr.h"
+#include "hw/vfio_pci.h"

 #include <libfdt.h>

@@ -198,6 +199,9 @@ static target_ulong h_put_tce(CPUPPCState *env, 
sPAPREnvironment *spapr,
     if (tcet) {
         return put_tce_emu(tcet, ioba, tce);
     }
+    if (!put_tce_vfio(liobn, ioba, tce)) {
+        return H_SUCCESS;
+    }
 #ifdef DEBUG_TCE
     fprintf(stderr, "%s on liobn=" TARGET_FMT_lx /*%s*/
             "  ioba 0x" TARGET_FMT_lx "  TCE 0x" TARGET_FMT_lx "\n",
diff --git a/hw/spapr_pci.c b/hw/spapr_pci.c
index 2bc0e49..de14fdb 100644
--- a/hw/spapr_pci.c
+++ b/hw/spapr_pci.c
@@ -29,6 +29,7 @@
 #include "pci_host.h"
 #include "hw/spapr.h"
 #include "hw/spapr_pci.h"
+#include "hw/vfio_pci.h"
 #include "exec-memory.h"
 #include <libfdt.h>

@@ -563,6 +564,7 @@ static Property spapr_emulatedpci_properties[] = {
     DEFINE_PROP_UINT8("min_busid", sPAPREmulatedPCIState, s.min_busid, 0),
     DEFINE_PROP_UINT8("max_busid", sPAPREmulatedPCIState, s.max_busid, 0xff),
     DEFINE_PROP_HEX64("msi_win_addr", sPAPREmulatedPCIState, msi_win_addr, 0),
+    DEFINE_PROP_UINT8("vfio", sPAPREmulatedPCIState, is_vfio, 0),
     DEFINE_PROP_END_OF_LIST(),
 };

@@ -606,6 +608,16 @@ void spapr_create_phb(sPAPREnvironment *spapr,
 /* Finalize PCI setup, called when all devices are already created */
 int spapr_finalize_pci_setup(sPAPREmulatedPCIState *ephb)
 {
+    if (ephb->is_vfio) {
+        /* Add VFIO DMA window configuration! */
+        ephb->s.dma_window_start = 0;
+        ephb->s.dma_window_size = 0x40000000;
+        spapr_vfio_create_dma_context(ephb->host_state.bus, ephb->s.dma_liobn);
+        /* ephb->s.dma = spapr_tce_new_dma_context(ephb->s.dma_liobn,
+           ephb->s.dma_window_size);*/
+        return 0;
+    }
+
     ephb->s.dma_window_start = 0;
     ephb->s.dma_window_size = 0x40000000;
     ephb->s.dma = spapr_tce_new_dma_context(ephb->s.dma_liobn,
diff --git a/hw/spapr_pci.h b/hw/spapr_pci.h
index f469484..f104d15 100644
--- a/hw/spapr_pci.h
+++ b/hw/spapr_pci.h
@@ -64,6 +64,8 @@ typedef struct sPAPREmulatedPCIState {
     target_phys_addr_t msi_win_addr;
     MemoryRegion msiwindow;

+    uint8_t is_vfio;
+
     QLIST_ENTRY(sPAPREmulatedPCIState) list;

 } sPAPREmulatedPCIState;
diff --git a/hw/vfio_pci.c b/hw/vfio_pci.c
index 6389cb9..5746984 100644
--- a/hw/vfio_pci.c
+++ b/hw/vfio_pci.c
@@ -25,7 +25,6 @@
 #include <netlink/genl/genl.h>
 #include <netlink/msg.h>
 #include <netlink/netlink.h>
-#include <sys/io.h>
 #include <sys/ioctl.h>
 #include <sys/mman.h>
 #include <sys/types.h>
@@ -50,6 +49,14 @@
 #include <linux/types.h>
 #include "linux-vfio.h"

+#ifndef TARGET_PPC64
+#include <sys/io.h>
+#else
+#include "hw/pci_internals.h"
+#include "hw/xics.h"
+#include "hw/spapr.h"
+#endif
+
 //#define DEBUG_VFIO
 #ifdef DEBUG_VFIO
 #define DPRINTF(fmt, ...) \
@@ -209,6 +216,7 @@ static void vfio_eoi(Notifier *notify, void *data)
     vfio_unmask_intx(vdev);
 }

+#ifndef TARGET_PPC64
 static void vfio_update_irq(Notifier *notify, void *data)
 {
     VFIODevice *vdev = container_of(notify, VFIODevice, intx.update_irq);
@@ -236,6 +244,7 @@ static void vfio_update_irq(Notifier *notify, void *data)
     /* Re-enable the interrupt in cased we missed an EOI */
     vfio_eoi(&vdev->intx.eoi, NULL);
 }
+#endif

 static int vfio_enable_intx(VFIODevice *vdev)
 {
@@ -261,10 +270,14 @@ static int vfio_enable_intx(VFIODevice *vdev)
     vdev->intx.pin = pin - 1; /* Pin A (1) -> irq[0] */
     vdev->intx.irq = pci_get_irq(&vdev->pdev, vdev->intx.pin);
     vdev->intx.eoi.notify = vfio_eoi;
+#ifndef TARGET_PPC64
     ioapic_add_gsi_eoi_notifier(&vdev->intx.eoi, vdev->intx.irq);

     vdev->intx.update_irq.notify = vfio_update_irq;
     pci_add_irq_update_notifier(&vdev->pdev, &vdev->intx.update_irq);
+#else
+    xics_add_eoi_notifier(&vdev->intx.eoi, vdev->intx.irq);
+#endif

     if (event_notifier_init(&vdev->intx.interrupt, 0)) {
         fprintf(stderr, "vfio: Error: event_notifier_init failed\n");
@@ -306,8 +319,12 @@ static void vfio_disable_intx(VFIODevice *vdev)

     ioctl(vdev->fd, VFIO_DEVICE_SET_IRQS, &irq_set);

+#ifndef TARGET_PPC64
     pci_remove_irq_update_notifier(&vdev->pdev, &vdev->intx.update_irq);
     ioapic_remove_gsi_eoi_notifier(&vdev->intx.eoi, vdev->intx.irq);
+#else
+    xics_remove_eoi_notifier(&vdev->intx.eoi);
+#endif

     fd = event_notifier_get_fd(&vdev->intx.interrupt);
     qemu_set_fd_handler(fd, NULL, NULL, vdev);
@@ -1702,3 +1719,49 @@ static void register_vfio_pci_dev_type(void)

 type_init(register_vfio_pci_dev_type)

+#ifdef TARGET_PPC64
+
+typedef struct sPAPRVFIOTable {
+    struct VFIOContainer *vfio_container; /* used from the 1st device on the 
bus */
+    uint32_t liobn;
+    QLIST_ENTRY(sPAPRVFIOTable) list;
+} sPAPRVFIOTable;
+
+QLIST_HEAD(vfio_tce_tables, sPAPRVFIOTable) vfio_tce_tables;
+
+void spapr_vfio_create_dma_context(struct PCIBus *pbus, uint32_t liobn)
+{
+    sPAPRVFIOTable *t;
+
+    t = g_malloc0(sizeof(*t));
+
+    DeviceState *qdev1st = QTAILQ_FIRST(&pbus->qbus.children);
+    VFIODevice *vdev1st = container_of(qdev1st, VFIODevice, pdev.qdev);
+
+    t->vfio_container = vdev1st->group->container;
+    t->liobn = liobn;
+
+    QLIST_INSERT_HEAD(&vfio_tce_tables, t, list);
+}
+
+int put_tce_vfio(uint32_t liobn, target_ulong ioba, target_ulong tce)
+{
+    sPAPRVFIOTable *t;
+
+    QLIST_FOREACH(t, &vfio_tce_tables, list) {
+        if (t->liobn != liobn) {
+            continue;
+        }
+        if (tce) {
+            vfio_dma_map(t->vfio_container, ioba, SPAPR_TCE_PAGE_SIZE,
+                         (tce & ~SPAPR_TCE_PAGE_MASK));
+        } else {
+            vfio_dma_unmap(t->vfio_container, ioba, SPAPR_TCE_PAGE_SIZE, 0);
+        }
+        return 0;
+    }
+
+    return -1;
+}
+
+#endif
diff --git a/hw/vfio_pci.h b/hw/vfio_pci.h
index f091850..e17be5e 100644
--- a/hw/vfio_pci.h
+++ b/hw/vfio_pci.h
@@ -104,4 +104,7 @@ typedef struct VFIOGroup {
 #define VFIO_FLAG_IOMMU_SHARED_BIT 0
 #define VFIO_FLAG_IOMMU_SHARED (1U << VFIO_FLAG_UIOMMU_SHARED_BIT)

+void spapr_vfio_create_dma_context(struct PCIBus *pbus, uint32_t liobn);
+int put_tce_vfio(uint32_t liobn, target_ulong ioba, target_ulong tce);
+
 #endif /* __VFIO_H__ */
-- 
1.7.0.4

-- 
Alexey