[RFC PATCH v1 00/01] PCIe DOE for PCIe and CXL 2.0

[Chris Browy]

PCIe Data Object Exchange (DOE) protocol for PCIe and CXL is available

https://gitlab.com/avery-qemu/cxl2.0-v3-doe/

based on Ben Widawsky's CXL QEMU cxl2.0-v3 gitlab branch

20210202005948.241655-1-ben.widawsky@intel.com">https://lore.kernel.org/qemu-devel/20210202005948.241655-1-ben.widawsky@intel.com

which is located at

https://gitlab.com/bwidawsk/qemu

The changes from Ben’s latest cxl-2.0v3 are:

 MAINTAINERS                               |   7 +
 hw/cxl/cxl-component-utils.c              | 151 +++++++++++
 hw/mem/cxl_type3.c                        | 121 +++++++++
 hw/pci/meson.build                        |   1 +
 hw/pci/pcie.c                             |   4 +-
 hw/pci/pcie_doe.c                         | 360 +++++++++++++++++++++++++
 include/hw/cxl/cxl_component.h            | 120 +++++++++
 include/hw/cxl/cxl_pci.h                  | 428 ++++++++++++++++++++++++++++++
 include/hw/pci/pcie.h                     |   5 +
 include/hw/pci/pcie_doe.h                 | 130 +++++++++
 include/hw/pci/pcie_regs.h                |   4 +
 include/standard-headers/linux/pci_regs.h |   3 +-
 12 files changed, 1332 insertions(+), 2 deletions(-)

The DOE protocol defines a mailbox method that allows either UEFI or OS methods
to read the device and do further setup of ACPI tables, etc.

There are 2 PCIe DOE protocols (PCI-SIG ECN Data Object Exchange (DOE) March 
2020)
- Discovery
- Component Measurement (CMA)

And 2 CXL specific ones:
- Compliance Mode (Compute Express Link Specification September 2, 2020
                   Revision: 2.0, Version 1.0)
- CDAT (Coherent Device Attribute Table (CDAT) Specification
                   October 2020 Revision 1.02)

For CXL, the CDAT table defines the memory device so that UEFI or OS can 
read it out of device using DOE and then can configure the system’s ACPI SRAT/HMAT
tables for system memory, and DEVSEC, Component, and Device registers in CXL
device.

Current version provides fixed CDAT table defined in the CXL Type3 device
model. Updates are planned shortly to allow for user to provide CDAT tables
through -device option property to vary from run to run.  The format will be
ASCII with structure/field-values pairs that are read by the device during
initialization as shown here:

-device 
cxl-type3,bus=rp0,memdev=cxl-mem1,id=cxl-pmem0,size=256M,cdat_file=<filename>

For testing, a cxl_app.c user program is enhanced to test all supported 
DOE protocols which are comprised of sequences of CFG RD/WR to various DOE cap
registers.  The Linux kernel updates and CXL Type3 Device driver 
provide sufficient ioctl() support to exercise the DOE protocol. See

https://lore.kernel.org/linux-cxl/20210130002438.1872527-1-ben.widawsky@intel.com

======== cxl_app.c
#include <stdio.h>
#include <unistd.h>
#include <sys/ioctl.h>
#include <string.h>
#include <fcntl.h>

#include "cxl_mem_wrapper.h"

const char* help= "\
-h                           help message\n\
-query                       IOCTL CXL_MEM_QUERY_COMMANDS\n\
-cfg_rd [0xoffset]           IOCTL CXL_MEM_CONFIG_WR Read Hex\n\
-cfg_wr [0xoffset] [0xaddr]  IOCTL CXL_MEM_CONFIG_WR Write Hex\n\
-doe_discovery [0xindex=0-3] IOCTL CXL_MEM_CONFIG_WR Write Hex\n\
-doe_cxl [0xprotocol=0 or 2] [0xreq_code=0,1 for protocol=0]\n\
-doe_cma [0xnum = 0]         IOCTL CXL_MEM_CONFIG_WR Write Hex\n\
example:\n\
./cxl_app.exe -cfg_rd 0x00\n\
./cxl_app.exe -cfg_wr 0x10 0x00ff0004\n\
./cxl_app.exe -doe_discovery 0\n\
./cxl_app.exe -doe_cxl 2\n\
./cxl_app.exe -doe_cxl 0 1\n\
./cxl_app.exe -doe_cma 0\n\
 ";
#define READ  0 
#define WRITE 1

int FD;
typedef struct cxl_pdev_config cxl_pdev_config;

int cxl_query() {
    typedef struct cxl_mem_query_commands cxl_mem_query_commands;
    typedef struct cxl_command_info cxl_command_info;
    int n_cmds= 0;
    // QUERY with n_commands == 0 to get command size
    ioctl(FD, CXL_MEM_QUERY_COMMANDS, &n_cmds);
    printf("Querying\n");

    cxl_mem_query_commands* cmds= malloc(sizeof(cxl_mem_query_commands)
                                + n_cmds * sizeof(cxl_command_info));
    cmds->n_commands= n_cmds;
    // QUERY with command size & pre-alloc memory
    ioctl(FD, CXL_MEM_QUERY_COMMANDS, cmds);

    for (int i= 0; i < (int)cmds->n_commands; i++) {
        printf(" id %d", cmds->commands[i].id);
        printf(" flags %d", cmds->commands[i].flags);
        printf(" size_in %d", cmds->commands[i].size_in);
        printf(" size_out %d\n", cmds->commands[i].size_out);
    }

    return 0;
};

int cxl_config(char* offset_s, char* data_s) {
    int offset, data, is_write;
    cxl_pdev_config* config_payload= malloc(sizeof(cxl_pdev_config));
    if (data_s == NULL)
        is_write= 0;
    else {
        is_write= 1;
        data= strtol(data_s, NULL, 16);
    }
    offset= strtol(offset_s, NULL, 16);

    config_payload->offset= offset;
    config_payload->data= data;
    config_payload->is_write= is_write;
    ioctl(FD, CXL_MEM_CONFIG_WR, config_payload);
    printf("CONFIG_WR %s [%0x] ", (is_write)? "write" : "read",
                    config_payload->offset);
    for (int i= 0; i < 32; i += 8) printf(" %02x", (config_payload->data >> i) 
& 0xff);
    printf("\n");

    return 0;
};

void doe_config(cxl_pdev_config* config_payload, uint32_t offset, uint32_t 
data, uint32_t is_write) {
    config_payload->offset= offset;
    config_payload->data= data;
    config_payload->is_write= is_write;
    ioctl(FD, CXL_MEM_CONFIG_WR, config_payload);
    printf("CONFIG_%s [%0x] ", (is_write)? "WR": "RD",
                    config_payload->offset);
    for (int i= 0; i < 32; i += 8) printf(" %02x", (config_payload->data >> i) 
& 0xff);
    printf("\n");
};

int cxl_doe_cxl(char* entry_s, char* data_s) {
    cxl_pdev_config* config_payload= malloc(sizeof(cxl_pdev_config));
    uint32_t data_obj[3];
    int length;
    uint32_t do_type;
    uint32_t req_type;

    do_type = strtol(entry_s, NULL, 16);
    req_type = strtol(data_s, NULL, 16);
    printf("DOE TYPE=%0x\n", do_type);
    printf("REQ TYPE=%0x\n", req_type);
    if (do_type == 0)
        data_obj[0] = 0x00001e98;
    if (do_type == 2)
        data_obj[0] = 0x00021e98;
    data_obj[1] = 0x3;
    data_obj[2] = req_type;
    printf("data_obj[0]=%x\n", data_obj[0]);
    printf("data_obj[1]=%x\n", data_obj[1]);
    printf("data_obj[2]=%x\n", data_obj[2]);

    printf("DOE\n");

    doe_config(config_payload, 0x170, data_obj[0], WRITE);
    doe_config(config_payload, 0x170, data_obj[1], WRITE);
    doe_config(config_payload, 0x170, data_obj[2], WRITE);

    /* Set GO */
    doe_config(config_payload, 0x168, 0x80000000, WRITE);
    /* check status READY is set 16c */
    doe_config(config_payload, 0x16c, 0, READ);

    /* read cdat response */
    doe_config(config_payload, 0x174, 0, READ);

    /* write cdat response success */
    doe_config(config_payload, 0x174, 0x00000001, WRITE);

    /* read cdat response */
    doe_config(config_payload, 0x174, 0, READ);

    length = config_payload->data & 0x0000ffff;
    printf("DOE RSP LENGTH = %0d\n", length);

    /* write cdat response success */
    doe_config(config_payload, 0x174, 0x00000001, WRITE);

    for (int j= 0; j < length-2; j=j+1) {
        /* read cdat response */
        doe_config(config_payload, 0x174, 0, READ);

        /* write cdat response success */
        doe_config(config_payload, 0x174, 0x00000001, WRITE);
    }
    return 0;

};


int cxl_doe_cma(char* entry_s, char* data_s) {
    cxl_pdev_config* config_payload= malloc(sizeof(cxl_pdev_config));
    uint32_t data_obj[3];

    data_obj[0] = 0x00010001;
    data_obj[1] = 0x3;
    data_obj[2] = strtol(entry_s, NULL, 16);

    printf("DOE\n");
    doe_config(config_payload, 0x170, data_obj[0], WRITE);
    doe_config(config_payload, 0x170, data_obj[1], WRITE);
    doe_config(config_payload, 0x170, data_obj[2], WRITE);

    /* Set GO */
    doe_config(config_payload, 0x168, 0x80000000, WRITE);

    /* check status READY is set 16c */
    doe_config(config_payload, 0x16c, 0, READ);

    /* read Discovery response */
    doe_config(config_payload, 0x174, 0, READ);

    doe_config(config_payload, 0x168, 0x1, WRITE);

    /* check status READY is set 16c */
    doe_config(config_payload, 0x16c, 0, READ);
};

int cxl_doe_discovery(char* entry_s, char* data_s) {
    cxl_pdev_config* config_payload= malloc(sizeof(cxl_pdev_config));
    uint32_t data_obj[3];

    data_obj[0] = 0x00000001;
    data_obj[1] = 0x3;
    data_obj[2] = strtol(entry_s, NULL, 16);

    printf("DOE\n");
    doe_config(config_payload, 0x170, data_obj[0], WRITE);
    doe_config(config_payload, 0x170, data_obj[1], WRITE);
    doe_config(config_payload, 0x170, data_obj[2], WRITE);

    /* Set GO */
    doe_config(config_payload, 0x168, 0x80000000, WRITE);

    /* check status READY is set 16c */
    doe_config(config_payload, 0x16c, 0, READ);

    /* read Discovery response */
    doe_config(config_payload, 0x174, 0, READ);

    /* write Discovery response success */
    doe_config(config_payload, 0x174, 0x00000001, WRITE);

    doe_config(config_payload, 0x174, 0, READ);

    /* write Discovery response success */
    doe_config(config_payload, 0x174, 0x00000002, WRITE);

    doe_config(config_payload, 0x174, 0, READ);

    /* write Discovery response success */
    doe_config(config_payload, 0x174, 0x00000003, WRITE);

    return 0;
};

int parse_input(int argc, char** argv) {
    int idx;
    if (argc < 2) return -1;
    for (idx= 0; idx < argc; idx++) {
        if (strcmp(argv[idx], "-h") == 0)
            return -1;
        if (strcmp(argv[idx], "-query") == 0)
            return cxl_query();
        if (strcmp(argv[idx], "-cfg_rd") == 0)
            return cxl_config(argv[idx + 1], NULL);
        if (strcmp(argv[idx], "-cfg_wr") == 0)
            return cxl_config(argv[idx + 1], argv[idx + 2]);
        if (strcmp(argv[idx], "-doe_discovery") == 0)
            return cxl_doe_discovery(argv[idx + 1], NULL);
        if (strcmp(argv[idx], "-doe_cxl") == 0)
            return cxl_doe_cxl(argv[idx + 1], argv[idx + 2]);
        if (strcmp(argv[idx], "-doe_cma") == 0)
            return cxl_doe_cma(argv[idx + 1], NULL);
    }
    return 0;
};

int main(int argc, char** argv) {
    int ret;
    char* dev_path= "/dev/cxl/mem0";

    if ((FD= open(dev_path, O_RDWR)) < 0) {
        printf("Open error loc: %s\n", dev_path);
        printf("Try sudo %s\n", argv[0]);
        exit(0);
    }

    if ((ret= parse_input(argc, argv)) < 0) {
        printf("Please specify input ");
        for (int i= 0; i < argc; i++) printf(" %s", argv[i]);;
          printf("\n%s\n", help);
    }
    printf("DOE\n");

    close(FD);
    exit(0);
}

========

The QEMU script to run is

sudo $QEMU_HOME/qemu/build/x86_64-softmmu/qemu-system-x86_64 \
-nic user,hostfwd=tcp::2222-:22,hostfwd=tcp::1234-:1234 \
-machine q35,accel=kvm,nvdimm,cxl \
-m 8192M,slots=4,maxmem=40964M \
-smp 8,sockets=2,cores=2,threads=2 \
-enable-kvm \
-boot order=d \
-k 'en-us' \
-vga virtio \
-drive 
if=pflash,format=raw,readonly,file=$TIANOCORE_HOME/edk2/Build/OvmfX64/DEBUG_GCC5/FV/OVMF_CODE.fd
 \
-drive 
if=pflash,format=raw,file=$TIANOCORE_HOME/edk2/Build/OvmfX64/DEBUG_GCC5/FV/OVMF_VARS.fd
 \
-drive file=$QCOW_HOME/ubuntu_20.qcow,format=qcow2 \
-object memory-backend-file,id=cxl-mem1,share,mem-path=cxl-type3,size=512M \
-object memory-backend-file,id=cxl-mem2,share,mem-path=cxl-mem2,size=512M" \
-device 
pxb-cxl,id=cxl.0,bus=pcie.0,bus_nr=52,uid=0,len-window-base=1,window-base[0]=0x4c0000000,memdev[0]=cxl-mem1
 \
-device cxl-rp,id=rp0,bus=cxl.0,addr=0.0,chassis=0,slot=0 \
-device cxl-type3,bus=rp0,memdev=cxl-mem1,id=cxl-pmem0,size=256M,lsa=cxl-mem2 \
-object memory-backend-ram,id=mem0,size=2048M \
-numa node,nodeid=0,memdev=mem0, \
-numa cpu,node-id=0,socket-id=0 \
-object memory-backend-ram,id=mem1,size=2048M \
-numa node,nodeid=1,memdev=mem1, \
-numa cpu,node-id=1,socket-id=1 \
-object memory-backend-ram,id=mem2,size=2048M \
-numa node,nodeid=2,memdev=mem2, \
-object memory-backend-ram,id=mem3,size=2048M \
-numa node,nodeid=3,memdev=mem3, \
-numa node,nodeid=4, \
-object 
memory-backend-file,id=nvmem0,share,mem-path=nvdimm-0,size=16384M,align=1G \
-device nvdimm,memdev=nvmem0,id=nv0,label-size=2M,node=4 \
-numa node,nodeid=5, \
-object 
memory-backend-file,id=nvmem1,share,mem-path=nvdimm-1,size=16384M,align=1G \
-device nvdimm,memdev=nvmem1,id=nv1,label-size=2M,node=5 \
-numa dist,src=0,dst=0,val=10 \
-numa dist,src=0,dst=1,val=21 \
-numa dist,src=0,dst=2,val=12 \
-numa dist,src=0,dst=3,val=21 \
-numa dist,src=0,dst=4,val=17 \
-numa dist,src=0,dst=5,val=28 \
-numa dist,src=1,dst=1,val=10 \
-numa dist,src=1,dst=2,val=21 \
-numa dist,src=1,dst=3,val=12 \
-numa dist,src=1,dst=4,val=28 \
-numa dist,src=1,dst=5,val=17 \
-numa dist,src=2,dst=2,val=10 \
-numa dist,src=2,dst=3,val=21 \
-numa dist,src=2,dst=4,val=28 \
-numa dist,src=2,dst=5,val=28 \ 
-numa dist,src=3,dst=3,val=10 \
-numa dist,src=3,dst=4,val=28 \
-numa dist,src=3,dst=5,val=28 \
-numa dist,src=4,dst=4,val=10 \
-numa dist,src=4,dst=5,val=28 \
-numa dist,src=5,dst=5,val=10