[RFC PATCH 2/2] hw/arm/sbsa-ref: Support CXL Fixed Memory Window

[Yuquan Wang]

In order to provide CFMWs on sbsa-ref, this extends 1TB space
from the hole above RAM Memory [SBSA_MEM] for CXL Fixed Memory
Window. 0xA0000000000 is chosen as the base address of this
space because of 3 reasons:

1) It is more suitable to choose a static address instead of that
implementation in virt, since a dynamic address space layout of
sbsa-ref is not appropriate for its original purpose as a reference
platform.

2) The Hotplug Memory address range should in the range of maximum
addressable range of sbsa-ref platform(0x10000000000-0x80ffffffffff).
It is satisfied the requirements of memory hotplug in linux kernel.

3) The start pfn of CFMW should exceed the reserved_pfn_range for
onlined numa node.

Based on 'cxl_fmws_link_targets', this adds a new function
'sbsa_cxl_fmws_link_targets' for binding cfmws.target with the default
pxb-cxl-bus on sbsa-ref.

In addition, this also adds 'create_cxl_fixed_window_region' which
based on 'machine_set_cfmw' to support creating a static cfmw region on
sbsa-ref.

Thus, to run sbsa-ref with a cxl device could use:
qemu-system-aarch64 \
-machine sbsa-ref \
-cpu cortex-a57 \
-smp 4 \
-m 4G \
-object memory-backend-ram,size=2G,id=mem0 \
-numa node,nodeid=0,cpus=0-1,memdev=mem0 \
-object memory-backend-ram,size=2G,id=mem1 \
-numa node,nodeid=1,cpus=2-3,memdev=mem1 \
-object memory-backend-file,id=mem2,mem-path=/tmp/mem2,size=256M,share=true \
-device cxl-type3,bus=cxl.0,volatile-memdev=mem2,id=cxl-mem1 \
-hda ubuntu.ext4 \
-pflash SBSA_FLASH0.fd \
-pflash SBSA_FLASH1.fd \

I'm not sure if the new space layout would bring a series of bad
influence, this patch is here to hopefully some comments to guide me!

Signed-off-by: Yuquan Wang <wangyuquan1236@phytium.com.cn>
---
 hw/arm/sbsa-ref.c                   | 73 +++++++++++++++++++++++++++++
 hw/cxl/cxl-host-stubs.c             |  1 +
 hw/cxl/cxl-host.c                   |  2 +-
 hw/pci-bridge/pci_expander_bridge.c |  1 -
 include/hw/cxl/cxl_host.h           |  1 +
 include/hw/pci/pci_bridge.h         |  1 +
 6 files changed, 77 insertions(+), 2 deletions(-)

diff --git a/hw/arm/sbsa-ref.c b/hw/arm/sbsa-ref.c
index dc924d181e..b10865920e 100644
--- a/hw/arm/sbsa-ref.c
+++ b/hw/arm/sbsa-ref.c
@@ -100,6 +100,7 @@ enum {
     SBSA_AHCI,
     SBSA_XHCI,
     SBSA_CXL_HOST,
+    SBSA_CXL_FIXED_WINDOW,
 };
 
 struct SBSAMachineState {
@@ -152,6 +153,8 @@ static const MemMapEntry sbsa_ref_memmap[] = {
     /* ~1TB PCIE MMIO space (4GB to 1024GB boundary) */
     [SBSA_PCIE_MMIO_HIGH] =     { 0x100000000ULL, 0xFF00000000ULL },
     [SBSA_MEM] =                { 0x10000000000ULL, RAMLIMIT_BYTES },
+    /* 1TB CXL FIXED WINDOW space */
+    [SBSA_CXL_FIXED_WINDOW] =   { 0xA0000000000ULL, 0x10000000000ULL },
 };
 
 static const int sbsa_ref_irqmap[] = {
@@ -734,6 +737,72 @@ static void create_cxl_host_reg_region(SBSAMachineState 
*sms)
     memory_region_add_subregion(sysmem, sbsa_ref_memmap[SBSA_CXL_HOST].base, 
mr);
 }
 
+static void create_cxl_fixed_window_region(SBSAMachineState *sms, MemoryRegion 
*mem)
+{
+    char cxl_host[] = "pxb-cxl-bus.0";
+    CXLFixedMemoryWindowOptionsList *list;
+    hwaddr base, end;
+    GList *it;
+    strList host_target = { NULL, cxl_host };
+    CXLFixedMemoryWindowOptions sbsa_ref_cfmwoptions = {
+        .size = 1 * TiB,
+        .has_interleave_granularity = false,
+        .targets = &host_target,
+    };
+    CXLFixedMemoryWindowOptionsList sbsa_ref_cfmwlist = { NULL, 
&sbsa_ref_cfmwoptions };
+
+    for (list = &sbsa_ref_cfmwlist; list; list = list->next) {
+        cxl_fixed_memory_window_config(&sms->cxl_devices_state, list->value, 
&error_fatal);
+    }
+
+    base = sbsa_ref_memmap[SBSA_CXL_FIXED_WINDOW].base;
+    end = base + sbsa_ref_memmap[SBSA_CXL_FIXED_WINDOW].size;
+
+    for (it = sms->cxl_devices_state.fixed_windows; it; it = it->next) {
+        CXLFixedWindow *fw = it->data;
+        if (base + fw->size > end) {
+            error_report("CFMWS does not fit under PA limit");
+            exit(EXIT_FAILURE);
+        }
+
+        fw->base = base;
+        memory_region_init_io(&fw->mr, OBJECT(sms), &cfmws_ops, fw,
+                                "cxl-fixed-memory-region", fw->size);
+
+        memory_region_add_subregion(mem, fw->base, &fw->mr);
+        base += fw->size;
+    }
+}
+
+static void sbsa_cxl_fmws_link_targets(SBSAMachineState *sms,
+                                CXLState *cxl_state, Error **errp)
+{
+    PXBCXLDev *pxb =  PXB_CXL_DEV(pci_bridge_get_device(sms->cxlbus));
+    if (cxl_state && cxl_state->fixed_windows) {
+        GList *it;
+
+        for (it = cxl_state->fixed_windows; it; it = it->next) {
+            CXLFixedWindow *fw = it->data;
+            int i;
+
+            for (i = 0; i < fw->num_targets; i++) {
+                Object *o;
+                bool ambig;
+
+                o = object_resolve_path_type(fw->targets[i],
+                                             TYPE_PXB_CXL_BUS,
+                                             &ambig);
+                if (!o) {
+                    error_setg(errp, "Could not resolve CXLFM target %s",
+                               fw->targets[i]);
+                    return;
+                }
+                fw->target_hbs[i] = pxb;
+            }
+        }
+    }
+}
+
 static void *sbsa_ref_dtb(const struct arm_boot_info *binfo, int *fdt_size)
 {
     const SBSAMachineState *board = container_of(binfo, SBSAMachineState,
@@ -869,8 +938,12 @@ static void sbsa_ref_init(MachineState *machine)
 
     create_cxl_host_reg_region(sms);
 
+    create_cxl_fixed_window_region(sms, sysmem);
+
     cxl_hook_up_pxb_registers(sms->bus, &sms->cxl_devices_state, &error_fatal);
 
+    sbsa_cxl_fmws_link_targets(sms, &sms->cxl_devices_state, &error_fatal);
+
     create_secure_ec(secure_sysmem);
 
     sms->bootinfo.ram_size = machine->ram_size;
diff --git a/hw/cxl/cxl-host-stubs.c b/hw/cxl/cxl-host-stubs.c
index cae4afcdde..d523be24a2 100644
--- a/hw/cxl/cxl-host-stubs.c
+++ b/hw/cxl/cxl-host-stubs.c
@@ -11,5 +11,6 @@
 void cxl_fmws_link_targets(CXLState *stat, Error **errp) {};
 void cxl_machine_init(Object *obj, CXLState *state) {};
 void cxl_hook_up_pxb_registers(PCIBus *bus, CXLState *state, Error **errp) {};
+void cxl_fixed_memory_window_config(CXLState *cxl_state, 
CXLFixedMemoryWindowOptions *object, Error **errp) {};
 
 const MemoryRegionOps cfmws_ops;
diff --git a/hw/cxl/cxl-host.c b/hw/cxl/cxl-host.c
index e9f2543c43..d408c7db15 100644
--- a/hw/cxl/cxl-host.c
+++ b/hw/cxl/cxl-host.c
@@ -22,7 +22,7 @@
 #include "hw/pci/pcie_port.h"
 #include "hw/pci-bridge/pci_expander_bridge.h"
 
-static void cxl_fixed_memory_window_config(CXLState *cxl_state,
+void cxl_fixed_memory_window_config(CXLState *cxl_state,
                                            CXLFixedMemoryWindowOptions *object,
                                            Error **errp)
 {
diff --git a/hw/pci-bridge/pci_expander_bridge.c 
b/hw/pci-bridge/pci_expander_bridge.c
index 0411ad31ea..f5431443b9 100644
--- a/hw/pci-bridge/pci_expander_bridge.c
+++ b/hw/pci-bridge/pci_expander_bridge.c
@@ -38,7 +38,6 @@ DECLARE_INSTANCE_CHECKER(PXBBus, PXB_BUS,
 DECLARE_INSTANCE_CHECKER(PXBBus, PXB_PCIE_BUS,
                          TYPE_PXB_PCIE_BUS)
 
-#define TYPE_PXB_CXL_BUS "pxb-cxl-bus"
 DECLARE_INSTANCE_CHECKER(PXBBus, PXB_CXL_BUS,
                          TYPE_PXB_CXL_BUS)
 
diff --git a/include/hw/cxl/cxl_host.h b/include/hw/cxl/cxl_host.h
index c9bc9c7c50..5fbf5a9347 100644
--- a/include/hw/cxl/cxl_host.h
+++ b/include/hw/cxl/cxl_host.h
@@ -16,6 +16,7 @@
 void cxl_machine_init(Object *obj, CXLState *state);
 void cxl_fmws_link_targets(CXLState *stat, Error **errp);
 void cxl_hook_up_pxb_registers(PCIBus *bus, CXLState *state, Error **errp);
+void cxl_fixed_memory_window_config(CXLState *cxl_state, 
CXLFixedMemoryWindowOptions *object, Error **errp);
 
 extern const MemoryRegionOps cfmws_ops;
 
diff --git a/include/hw/pci/pci_bridge.h b/include/hw/pci/pci_bridge.h
index 5cd452115a..5456e24883 100644
--- a/include/hw/pci/pci_bridge.h
+++ b/include/hw/pci/pci_bridge.h
@@ -102,6 +102,7 @@ typedef struct PXBPCIEDev {
     PXBDev parent_obj;
 } PXBPCIEDev;
 
+#define TYPE_PXB_CXL_BUS "pxb-cxl-bus"
 #define TYPE_PXB_DEV "pxb"
 OBJECT_DECLARE_SIMPLE_TYPE(PXBDev, PXB_DEV)
 
-- 
2.34.1