[Qemu-devel] [PATCH 6/9 v3] Add API to create page

[Qiao Nuohan]

Functions in this patch are used to gather data of page desc and page data in
kdump-compressed format. The following patch will use these functions to gather
data of page, then cache them into tmp files

Signed-off-by: Qiao Nuohan <address@hidden>
Reviewed-by: Zhang Xiaohe <address@hidden>
---
 dump.c                |  259 +++++++++++++++++++++++++++++++++++++++++++++++++
 include/sysemu/dump.h |   32 ++++++
 2 files changed, 291 insertions(+), 0 deletions(-)

diff --git a/dump.c b/dump.c
index 998d71e..ebfb190 100644
--- a/dump.c
+++ b/dump.c
@@ -877,6 +877,265 @@ static int create_dump_bitmap(DumpState *s)
     return 0;
 }
 
+/*
+ * create two tmpfile and save page_desc and page_data
+ */
+static int prepare_pages(DumpState *s)
+{
+    int ret;
+    struct cache_data *page_desc;
+    struct cache_data *page_data;
+
+    page_desc = g_malloc0(sizeof(struct cache_data));
+
+    page_data = g_malloc0(sizeof(struct cache_data));
+
+    ret = init_cache_data(page_desc, FILENAME_PAGE_DESC);
+    if (ret < 0) {
+        dump_error(s, "dump: failed to init page_desc.\n");
+        return -1;
+    }
+    s->page_desc = page_desc;
+
+    ret = init_cache_data(page_data, FILENAME_PAGE_DATA);
+    if (ret < 0) {
+        dump_error(s, "dump: failed to init page_desc.\n");
+        return -1;
+    }
+    s->page_data = page_data;
+
+    return 0;
+}
+
+/*
+ * memory should be read page by page, or it may exceed the boundary and
+ * fail to read
+ */
+static int readmem(void *bufptr, ram_addr_t addr, size_t size, DumpState *s)
+{
+    RAMBlock *block;
+
+    block = s->block;
+
+    while (block) {
+        if ((addr >= block->offset) &&
+            (addr + size <= block->offset + block->length)) {
+            memcpy(bufptr, block->host + (addr - block->offset), size);
+            return 0;
+        } else {
+            block = QTAILQ_NEXT(block, next);
+        }
+    }
+
+    return -1;
+}
+
+/*
+ * check if the page is all 0
+ */
+static inline int is_zero_page(unsigned char *buf, long page_size)
+{
+    size_t i;
+
+    for (i = 0; i < page_size; i++) {
+        if (buf[i]) {
+            return 0;
+        }
+    }
+
+    return 1;
+}
+
+static int create_pages(DumpState *s)
+{
+    int ret;
+    unsigned long long pfn;
+    unsigned char buf[s->page_size];
+    unsigned char *buf_out = NULL;
+    unsigned long len_buf_out;
+    unsigned long size_out;
+    int zero_page;
+    struct page_desc pd, pd_zero;
+    off_t offset_desc, offset_data;
+    unsigned long len_buf_out_zlib, len_buf_out_lzo, len_buf_out_snappy;
+
+    ret = 0;
+
+    ret = prepare_pages(s);
+    if (ret < 0) {
+        dump_error(s, "dump: failed to prepare pages.\n");
+        goto out;
+    }
+
+    /* init buf_out */
+    len_buf_out_zlib = len_buf_out_lzo = len_buf_out_snappy = 0;
+
+    /* buf size for zlib */
+    len_buf_out_zlib = compressBound(s->page_size);
+
+    /* buf size for lzo */
+#ifdef CONFIG_LZO
+    if (s->flag_compress & DUMP_DH_COMPRESSED_LZO) {
+        if (lzo_init() != LZO_E_OK) {
+            ret = -1;
+            dump_error(s, "dump: failed to use lzo compression");
+            goto out;
+        }
+    }
+
+    lzo_bytep wrkmem;
+
+    wrkmem = g_malloc(LZO1X_1_MEM_COMPRESS);
+
+    len_buf_out_lzo = s->page_size + s->page_size / 16 + 64 + 3;
+#endif
+
+    /* buf size for snappy */
+#ifdef CONFIG_SNAPPY
+    len_buf_out_snappy = snappy_max_compressed_length(s->page_size);
+#endif
+
+    /* get the biggest that can store all kinds of compressed page */
+    len_buf_out = MAX(len_buf_out_zlib,
+                    MAX(len_buf_out_lzo, len_buf_out_snappy));
+
+    buf_out = g_malloc(len_buf_out);
+
+    /* get offset of page_desc and page_data in dump file */
+    offset_desc = s->offset_page;
+    offset_data = offset_desc + sizeof(page_desc_t) * s->num_dumpable;
+
+    /*
+     * init zero page's page_desc and page_data, and all zero pages
+     * will use the same page_data
+     */
+    pd_zero.size = s->page_size;
+    pd_zero.flags = 0;
+    pd_zero.offset = offset_data;
+    pd_zero.page_flags = 0;
+    memset(buf, 0, pd_zero.size);
+    write_cache(s->page_data, buf, pd_zero.size);
+    offset_data += pd_zero.size;
+
+    for (pfn = 0; pfn < s->max_mapnr; pfn++) {
+        /* check whether the page is dumpable */
+        if (!is_bit_set(s->dump_bitmap2, pfn)) {
+            continue;
+        }
+
+        memset(buf, 0, s->page_size);
+        ret = readmem(buf, pfn_to_paddr(pfn, s->page_shift), s->page_size, s);
+        if (ret < 0) {
+            dump_error(s, "dump: failed to read memory ");
+            goto out;
+        }
+
+        /* check zero page */
+        zero_page = is_zero_page(buf, s->page_size);
+        if (zero_page) {
+            write_cache(s->page_desc, &pd_zero, sizeof(page_desc_t));
+        } else {
+            /*
+             * not zero page, then:
+             * 1. compress the page
+             * 2. write the compressed page into the cache of page_data
+             * 3. get page desc of the compressed page and write it into the
+             *    cache of page_desc
+             */
+            size_out = len_buf_out;
+            if ((s->flag_compress & DUMP_DH_COMPRESSED_ZLIB) &&
+                    (compress2(buf_out, &size_out, buf, s->page_size,
+                    Z_BEST_SPEED) == Z_OK) && (size_out < s->page_size)) {
+                pd.flags = DUMP_DH_COMPRESSED_ZLIB;
+                pd.size  = size_out;
+
+                ret = write_cache(s->page_data, buf_out, pd.size);
+                if (ret < 0) {
+                    dump_error(s, "dump: failed to write page data(zlib).\n");
+                    goto out;
+                }
+#ifdef CONFIG_LZO
+            } else if ((s->flag_compress & DUMP_DH_COMPRESSED_LZO) &&
+                    (lzo1x_1_compress(buf, s->page_size, buf_out, &size_out,
+                    wrkmem) == LZO_E_OK) && (size_out < s->page_size)) {
+                pd.flags = DUMP_DH_COMPRESSED_LZO;
+                pd.size  = size_out;
+
+                ret = write_cache(s->page_data, buf_out, pd.size);
+                if (ret < 0) {
+                    dump_error(s, "dump: failed to write page data(lzo).\n");
+                    goto out;
+                }
+#endif
+#ifdef CONFIG_SNAPPY
+            } else if ((s->flag_compress & DUMP_DH_COMPRESSED_SNAPPY) &&
+                    (snappy_compress((char *)buf, s->page_size, (char 
*)buf_out,
+                    (size_t *)&size_out) == SNAPPY_OK) &&
+                    (size_out < s->page_size)) {
+                pd.flags = DUMP_DH_COMPRESSED_SNAPPY;
+                pd.size  = size_out;
+
+                ret = write_cache(s->page_data, buf_out, pd.size);
+                if (ret < 0) {
+                    dump_error(s, "dump: failed to write page 
data(snappy).\n");
+                    goto out;
+                }
+#endif
+            } else {
+                pd.flags = 0;
+                pd.size = s->page_size;
+
+                ret = write_cache(s->page_data, buf, pd.size);
+                if (ret < 0) {
+                    dump_error(s, "dump: failed to write page data.\n");
+                    goto out;
+                }
+            }
+
+            /* get and write page desc here */
+            pd.page_flags = 0;
+            pd.offset = offset_data;
+            offset_data += pd.size;
+
+            ret = write_cache(s->page_desc, &pd, sizeof(page_desc_t));
+            if (ret < 0) {
+                dump_error(s, "dump: failed to write page desc.\n");
+                goto out;
+            }
+        }
+    }
+
+    ret = sync_cache(s->page_desc);
+    if (ret < 0) {
+        dump_error(s, "dump: failed to sync cache for page_desc.\n");
+        goto out;
+    }
+    ret = sync_cache(s->page_data);
+    if (ret < 0) {
+        dump_error(s, "dump: failed to sync cache for page_data.\n");
+        goto out;
+    }
+
+   /* get size of page_desc and page_data, then reset their offset */
+    s->page_desc_size = s->page_desc->offset;
+    s->page_data_size = s->page_data->offset;
+    s->page_desc->offset = 0;
+    s->page_data->offset = 0;
+
+out:
+#ifdef CONFIG_LZO
+    if (wrkmem) {
+        g_free(wrkmem);
+    }
+#endif
+
+    if (buf_out) {
+        g_free(buf_out);
+    }
+
+    return ret;
+}
+
 static int dump_init(DumpState *s, int fd, bool paging, bool has_filter,
                      int64_t begin, int64_t length, Error **errp)
 {
diff --git a/include/sysemu/dump.h b/include/sysemu/dump.h
index 597b19e..27ee3fa 100644
--- a/include/sysemu/dump.h
+++ b/include/sysemu/dump.h
@@ -25,8 +25,23 @@
 #include "qapi/error.h"
 #include "qmp-commands.h"
 #include "dump_bitmap.h"
+#include "cache_data.h"
 
 #include <sys/utsname.h>
+#include <zlib.h>
+#ifdef CONFIG_LZO
+#include <lzo/lzo1x.h>
+#endif
+#ifdef CONFIG_SNAPPY
+#include <snappy-c.h>
+#endif
+
+/*
+ * flag used in page desc of kdump-compressed format
+ */
+#define DUMP_DH_COMPRESSED_ZLIB     (0x1)
+#define DUMP_DH_COMPRESSED_LZO      (0x2)
+#define DUMP_DH_COMPRESSED_SNAPPY   (0x4)
 
 #define KDUMP_SIGNATURE             "KDUMP   "
 #define SIG_LEN                     (sizeof(KDUMP_SIGNATURE) - 1)
@@ -36,10 +51,14 @@
 #define ARCH_PFN_OFFSET             (0)
 #define FILENAME_BITMAP1            "kdump_bitmap1_XXXXXX"
 #define FILENAME_BITMAP2            "kdump_bitmap2_XXXXXX"
+#define FILENAME_PAGE_DESC          "kdump_page_desc_XXXXXX"
+#define FILENAME_PAGE_DATA          "kdump_page_data_XXXXXX"
 
 #define divideup(x, y)              (((x) + ((y) - 1)) / (y))
 #define paddr_to_pfn(X, page_shift) \
     (((unsigned long long)(X) >> (page_shift)) - ARCH_PFN_OFFSET)
+#define pfn_to_paddr(X, page_shift) \
+    (((unsigned long long)(X) + ARCH_PFN_OFFSET) << (page_shift))
 
 typedef struct ArchDumpInfo {
     int d_machine;  /* Architecture */
@@ -120,6 +139,14 @@ struct kdump_sub_header64 {
     uint64_t size_eraseinfo;        /* header_version 5 and later */
 };
 
+/* descriptor of each page for vmcore */
+typedef struct page_desc {
+    off_t offset;                   /* the offset of the page data*/
+    unsigned int size;              /* the size of this dump page */
+    unsigned int flags;             /* flags */
+    unsigned long long page_flags;  /* page flags */
+} page_desc_t;
+
 typedef struct DumpState {
     ArchDumpInfo dump_info;
     MemoryMappingList list;
@@ -146,6 +173,7 @@ typedef struct DumpState {
     void *kh;
     unsigned long long num_dumpable;
     off_t offset_sub_header;
+    int flag_compress;
 
     off_t offset_dump_bitmap;
     unsigned long len_dump_bitmap;
@@ -153,6 +181,10 @@ typedef struct DumpState {
     struct dump_bitmap *dump_bitmap2;
 
     off_t offset_page;
+    unsigned long long page_desc_size;
+    unsigned long long page_data_size;
+    struct cache_data *page_desc;
+    struct cache_data *page_data;
 } DumpState;
 
 int cpu_get_dump_info(ArchDumpInfo *info);
-- 
1.7.1