[RFC PATCH v2 QEMU 3/4] hw/cxl: Provide a means to get the interleave set position for an EP

[Jonathan Cameron]

CXL interleave decoding is hieriarchical in a fashion that means the CXL
memory devices only need to know the interleave granularity and interleave
ways to figure out which address bits to drop from incoming translactions.
Unfortunately to provide the right information to the hotness monitoring
plugin, which filters transactions in Host Physical Address space, it is
necessary to know which interleave set position a given device is in.

I tried various more sophisticated solutions to provide this information
but they were all rather complex. The solution used here is the brute force
one.  Every time an HDM Decoder is committed (these are the address routing
elements) it checks every Type 3 Device HDM Decoder to find a HPA range. It
then uses the address routing HPA to Device matching heirarchical routing
at a series of addresses corresponding to the first byte of each interleave
set. When the same device is reached then we know we have the correct
Interleave Set Position and pass that to the CHMU.

Signed-off-by: Jonathan Cameron <Jonathan.Cameron@huawei.com>

---
RFC v2: New patch. Note lightly tested only so far.
---
 include/hw/cxl/cxl.h         |  1 +
 hw/cxl/cxl-component-utils.c |  4 ++
 hw/cxl/cxl-host.c            | 72 ++++++++++++++++++++++++++++++++++++
 hw/mem/cxl_type3.c           |  2 +
 4 files changed, 79 insertions(+)

diff --git a/include/hw/cxl/cxl.h b/include/hw/cxl/cxl.h
index 12844d3418..b4b83c0b63 100644
--- a/include/hw/cxl/cxl.h
+++ b/include/hw/cxl/cxl.h
@@ -71,4 +71,5 @@ CXLComponentState *cxl_usp_to_cstate(CXLUpstreamPort *usp);
 typedef struct CXLDownstreamPort CXLDownstreamPort;
 DECLARE_INSTANCE_CHECKER(CXLDownstreamPort, CXL_DSP, TYPE_CXL_DSP)
 
+void cxl_update_isp(void);
 #endif
diff --git a/hw/cxl/cxl-component-utils.c b/hw/cxl/cxl-component-utils.c
index 473895948b..f53ce1ebaa 100644
--- a/hw/cxl/cxl-component-utils.c
+++ b/hw/cxl/cxl-component-utils.c
@@ -116,6 +116,10 @@ static void dumb_hdm_handler(CXLComponentState 
*cxl_cstate, hwaddr offset,
         value = FIELD_DP32(value, CXL_HDM_DECODER0_CTRL, COMMITTED, 0);
     }
     stl_le_p((uint8_t *)cache_mem + offset, value);
+
+    if (should_commit) {
+        cxl_update_isp();
+    }
 }
 
 static void cxl_cache_mem_write_reg(void *opaque, hwaddr offset, uint64_t 
value,
diff --git a/hw/cxl/cxl-host.c b/hw/cxl/cxl-host.c
index 5239555f6c..893ef7f7fa 100644
--- a/hw/cxl/cxl-host.c
+++ b/hw/cxl/cxl-host.c
@@ -279,6 +279,78 @@ static MemTxResult cxl_write_cfmws(void *opaque, hwaddr 
addr,
     return cxl_type3_write(d, addr + fw->base, data, size, attrs);
 }
 
+/*
+ * Updating the end point decoder stashed Interleave Set Positions (ISP)
+ * that is needed for the CHMU to pass to the cache plugin + hotness tracker
+ * is tricky as the decoders can be committed in any order.
+ *
+ * Brute force the problem by finding any endpoints below a cfmws and for
+ * each enabled decoder, probing until we get a match - if any upstream
+ * decoders are not commited this will fail but that is fine as we try again
+ * later when the situation is resolved by commiting upstream decoders.
+ *
+ * This is a rare operation, so not worth complexity of walking down from
+ * the Fixed memory windows. Just compare all with all.
+ */
+
+/* Update ISP for a given Type 3 memory device */
+static int cxl_type3_update_isp(Object *obj, void *opaque)
+{
+    CXLType3Dev *ct3d;
+    CXLFixedWindow *fw = opaque;
+    int i;
+
+    /*
+     * From the CXL Type 3 HDM decoders we need interleave info.
+     * That will let us then find out for each decoder what hits it....
+     */
+    if (!object_dynamic_cast(obj, TYPE_CXL_TYPE3)) {
+        return 0;
+    }
+
+    ct3d = CXL_TYPE3(obj);
+
+    for (i = 0; i < CXL_HDM_DECODER_COUNT; i++) {
+        uint64_t hpa_base;
+        uint16_t granual;
+        uint8_t ways, w;
+        PCIDevice *d;
+
+        if (!cxl_type3_get_hdm_interleave_props(ct3d, i, &hpa_base, &granual,
+                                                &ways)) {
+            continue; /* commit in order, but teardown can be messy */
+        }
+
+        for (w = 0; w < ways; w++) {
+            d = cxl_cfmws_find_device(fw, hpa_base + w * granual - fw->base);
+            if (d == PCI_DEVICE(ct3d)) {
+                cxl_type3_set_hdm_isp(ct3d, i, w);
+            }
+        }
+    }
+    return 0;
+}
+
+static int cxl_fmw_update_isp(Object *obj, void *priv)
+{
+    struct CXLFixedWindow *fw;
+
+    if (!object_dynamic_cast(obj, TYPE_CXL_FMW)) {
+        return 0;
+    }
+    fw = CXL_FMW(obj);
+    object_child_foreach_recursive(object_get_root(),
+                                   cxl_type3_update_isp, fw);
+    return 0;
+}
+
+/* Update all Interleave Set Positions on all EP HDM decoders */
+void cxl_update_isp(void)
+{
+    object_child_foreach_recursive(object_get_root(),
+                                   cxl_fmw_update_isp, NULL);
+}
+
 const MemoryRegionOps cfmws_ops = {
     .read_with_attrs = cxl_read_cfmws,
     .write_with_attrs = cxl_write_cfmws,
diff --git a/hw/mem/cxl_type3.c b/hw/mem/cxl_type3.c
index 43f4cd8023..8e9f76a07a 100644
--- a/hw/mem/cxl_type3.c
+++ b/hw/mem/cxl_type3.c
@@ -484,6 +484,8 @@ static void hdm_decoder_commit(CXLType3Dev *ct3d, int which)
 
     stl_le_p(cache_mem + R_CXL_HDM_DECODER0_CTRL + which * hdm_inc, ctrl);
 
+    cxl_update_isp();
+
     low = ldl_le_p(cache_mem + R_CXL_HDM_DECODER0_BASE_LO + which * hdm_inc);
     high = ldl_le_p(cache_mem + R_CXL_HDM_DECODER0_BASE_HI + which * hdm_inc);
     ct3d->cxl_dstate.chmu[0].decoder[which].base =
-- 
2.48.1