[Qemu-devel] [PATCH RFC 1/1] tests: Add migration test for aarch64

[Wei Huang]

This patch adds the migration test support for aarch64. The test code,
which implements the same functionality as x86, is compiled into a binary
and booted as a kernel to qemu. Here are the design ideas:

* We choose this -kernel design because aarch64 QEMU doesn't provide a
  built-in fw like x86 does. So instead of relying on a boot loader, we
  use -kernel approach for aarch64.
* The serial output is sent to PL011 directly.
* The physical memory base for mach-virt machine is 0x40000000. We have
  to change the start_address and end_address for aarch64.
* The downtime is changed from 0.001 to 0.1. Without this change, we saw
  migration stalled. This problem is still under analysis and needs to be
  resolved before removing RFC for this patch.

The test code is as the following:

.section .text

        .globl  start

start:
        /* disable MMU to use phys mem address, just in case */
        mrs     x0, sctlr_el1
        bic     x0, x0, #(1<<0)
        msr     sctlr_el1, x0
        isb

        /* output char 'A' to PL011 */
        mov     w4, #65
        mov     x5, #0x9000000
        strb    w4, [x5]

        /* w6 keeps a counter so we limit the output speed */
        mov     w6, #0

        mov     x3, #(0x40000000 + 100 * 1024 * 1024)
mainloop:
        mov     x2, #(0x40000000 + 1024*1024) /* base addr = 0x40000000 */

innerloop:
        ldrb    w1, [x2]
        add     w1, w1, #1
        strb    w1, [x2]

        add     x2, x2, #(4096)
        cmp     x2, x3
        blt     innerloop

        add     w6, w6, #1
        and     w6, w6, #(0xff)
        cmp     w6, #0
        bne     mainloop

        /* output char 'B' to PL011 */
        mov     w4, #66
        mov     x5, #0x9000000
        strb    w4, [x5]

        bl      mainloop

The code is compiled with the following command:
 * gcc -c -o fill.o fill.s
 * gcc -O2 -o fill.elf -Wl,-T,/tmp/flat.lds,--build-id=none,-Ttext=40080000 \
   -nostdlib fill.o
 * objcopy -O binary fill.elf fill.flat
 * xxd -i fill.flat

The linker file (borrowed from KVM unit test) is defined as:

SECTIONS
{
    .text : { *(.init) *(.text) *(.text.*) }
    . = ALIGN(64K);
    etext = .;
    .data : {
        *(.data)
    }
    . = ALIGN(16);
    .rodata : { *(.rodata) }
    . = ALIGN(16);
    .bss : { *(.bss) }
    . = ALIGN(64K);
    edata = .;
    . += 64K;
    . = ALIGN(64K);
    /*
     * stack depth is 16K for arm and PAGE_SIZE for arm64, see THREAD_SIZE
     * sp must be 16 byte aligned for arm64, and 8 byte aligned for arm
     * sp must always be strictly less than the true stacktop
     */
    stackptr = . - 16;
    stacktop = .;
}

ENTRY(start)

Signed-off-by: Wei Huang <address@hidden>
---
 tests/Makefile.include |  1 +
 tests/migration-test.c | 43 ++++++++++++++++++++++++++++++++++++++++---
 2 files changed, 41 insertions(+), 3 deletions(-)

diff --git a/tests/Makefile.include b/tests/Makefile.include
index c002352..d5828c4 100644
--- a/tests/Makefile.include
+++ b/tests/Makefile.include
@@ -357,6 +357,7 @@ check-qtest-arm-y += tests/test-arm-mptimer$(EXESUF)
 gcov-files-arm-y += hw/timer/arm_mptimer.c
 
 check-qtest-aarch64-y = tests/numa-test$(EXESUF)
+check-qtest-aarch64-y += tests/migration-test$(EXESUF)
 
 check-qtest-microblazeel-y = $(check-qtest-microblaze-y)
 
diff --git a/tests/migration-test.c b/tests/migration-test.c
index be598d3..b0dd365 100644
--- a/tests/migration-test.c
+++ b/tests/migration-test.c
@@ -22,8 +22,8 @@
 
 #define MIN_NVRAM_SIZE 8192 /* from spapr_nvram.c */
 
-const unsigned start_address = 1024 * 1024;
-const unsigned end_address = 100 * 1024 * 1024;
+unsigned start_address = 1024 * 1024;
+unsigned end_address = 100 * 1024 * 1024;
 bool got_stop;
 
 #if defined(__linux__)
@@ -125,6 +125,18 @@ unsigned char bootsect[] = {
   0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x55, 0xaa
 };
 
+unsigned char aarch64_kernel[] = {
+    0x00, 0x10, 0x38, 0xd5, 0x00, 0xf8, 0x7f, 0x92, 0x00, 0x10, 0x18, 0xd5,
+    0xdf, 0x3f, 0x03, 0xd5, 0x24, 0x08, 0x80, 0x52, 0x05, 0x20, 0xa1, 0xd2,
+    0xa4, 0x00, 0x00, 0x39, 0x06, 0x00, 0x80, 0x52, 0x03, 0xc8, 0xa8, 0xd2,
+    0x02, 0x02, 0xa8, 0xd2, 0x41, 0x00, 0x40, 0x39, 0x21, 0x04, 0x00, 0x11,
+    0x41, 0x00, 0x00, 0x39, 0x42, 0x04, 0x40, 0x91, 0x5f, 0x00, 0x03, 0xeb,
+    0x6b, 0xff, 0xff, 0x54, 0xc6, 0x04, 0x00, 0x11, 0xc6, 0x1c, 0x00, 0x12,
+    0xdf, 0x00, 0x00, 0x71, 0xc1, 0xfe, 0xff, 0x54, 0x44, 0x08, 0x80, 0x52,
+    0x05, 0x20, 0xa1, 0xd2, 0xa4, 0x00, 0x00, 0x39, 0xf2, 0xff, 0xff, 0x97
+};
+unsigned int aarch64_kernel_len = 96;
+
 static void init_bootfile_x86(const char *bootpath)
 {
     FILE *bootfile = fopen(bootpath, "wb");
@@ -163,6 +175,15 @@ static void init_bootfile_ppc(const char *bootpath)
     fclose(bootfile);
 }
 
+static void init_bootfile_aarch64(const char *bootpath)
+{
+    FILE *kernelfile = fopen(bootpath, "wb");
+
+    g_assert_cmpint(fwrite(aarch64_kernel, aarch64_kernel_len, 1, kernelfile),
+                    ==, 1);
+    fclose(kernelfile);
+}
+
 /*
  * Wait for some output in the serial output file,
  * we get an 'A' followed by an endless string of 'B's
@@ -470,6 +491,22 @@ static void test_migrate_start(QTestState **from, 
QTestState **to,
                                   " -serial file:%s/dest_serial"
                                   " -incoming %s",
                                   accel, tmpfs, uri);
+    } else if (strcmp(arch, "aarch64") == 0) {
+        init_bootfile_aarch64(bootpath);
+        cmd_src = g_strdup_printf("-machine virt,accel=kvm:cg -m 1024M"
+                                  " -name vmsource,debug-threads=on -cpu host"
+                                  " -serial file:%s/src_serial "
+                                  " -kernel %s ",
+                                  tmpfs, bootpath);
+        cmd_dst = g_strdup_printf("-machine virt,accel=kvm:tcg -m 1024M"
+                                  " -name vmdest,debug-threads=on -cpu host"
+                                  " -serial file:%s/dest_serial"
+                                  " -kernel %s"
+                                  " -incoming %s",
+                                  tmpfs, bootpath, uri);
+        /* aarch64 virt machine physical mem started from 0x40000000 */
+        start_address += 0x40000000;
+        end_address += 0x40000000;
     } else {
         g_assert_not_reached();
     }
@@ -530,7 +567,7 @@ static void test_migrate(void)
      * machine, so also set the downtime.
      */
     migrate_set_speed(from, "100000000");
-    migrate_set_downtime(from, 0.001);
+    migrate_set_downtime(from, 0.1);
 
     /* Wait for the first serial output from the source */
     wait_for_serial("src_serial");
-- 
1.8.3.1