[Qemu-devel] [RFC PATCH 05/17] COLO save: integrate COLO checkpointed save into qemu migration

[Yang Hongyang]

  Integrate COLO checkpointed save flow into qemu migration.
  Add a migrate state: MIG_STATE_COLO, enter this migrate state
after the first live migration successfully finished.
  Create a colo thread to do the checkpointed save.

Signed-off-by: Yang Hongyang <address@hidden>
---
 include/migration/migration-colo.h |  4 ++++
 include/migration/migration.h      | 13 +++++++++++
 migration-colo-comm.c              |  2 +-
 migration-colo.c                   | 48 ++++++++++++++++++++++++++++++++++++++
 migration.c                        | 36 ++++++++++++++++------------
 stubs/migration-colo.c             |  4 ++++
 6 files changed, 91 insertions(+), 16 deletions(-)

diff --git a/include/migration/migration-colo.h 
b/include/migration/migration-colo.h
index e3735d8..24589c0 100644
--- a/include/migration/migration-colo.h
+++ b/include/migration/migration-colo.h
@@ -18,4 +18,8 @@ void colo_info_mig_init(void);
 
 bool colo_supported(void);
 
+/* save */
+bool migrate_use_colo(void);
+void colo_init_checkpointer(MigrationState *s);
+
 #endif
diff --git a/include/migration/migration.h b/include/migration/migration.h
index 3cb5ba8..3e81a27 100644
--- a/include/migration/migration.h
+++ b/include/migration/migration.h
@@ -64,6 +64,19 @@ struct MigrationState
     int64_t dirty_sync_count;
 };
 
+enum {
+    MIG_STATE_ERROR = -1,
+    MIG_STATE_NONE,
+    MIG_STATE_SETUP,
+    MIG_STATE_CANCELLING,
+    MIG_STATE_CANCELLED,
+    MIG_STATE_ACTIVE,
+    MIG_STATE_COLO,
+    MIG_STATE_COMPLETED,
+};
+
+void migrate_set_state(MigrationState *s, int old_state, int new_state);
+
 void process_incoming_migration(QEMUFile *f);
 
 void qemu_start_incoming_migration(const char *uri, Error **errp);
diff --git a/migration-colo-comm.c b/migration-colo-comm.c
index ccbc246..4504ceb 100644
--- a/migration-colo-comm.c
+++ b/migration-colo-comm.c
@@ -25,7 +25,7 @@ static bool colo_requested;
 
 /* save */
 
-static bool migrate_use_colo(void)
+bool migrate_use_colo(void)
 {
     MigrationState *s = migrate_get_current();
     return s->enabled_capabilities[MIGRATION_CAPABILITY_COLO];
diff --git a/migration-colo.c b/migration-colo.c
index 1d3bef8..0cef8bd 100644
--- a/migration-colo.c
+++ b/migration-colo.c
@@ -8,9 +8,57 @@
  * the COPYING file in the top-level directory.
  */
 
+#include "qemu/main-loop.h"
+#include "qemu/thread.h"
 #include "migration/migration-colo.h"
 
+static QEMUBH *colo_bh;
+
 bool colo_supported(void)
 {
     return true;
 }
+
+/* save */
+
+static void *colo_thread(void *opaque)
+{
+    MigrationState *s = opaque;
+
+    /*TODO: COLO checkpointed save loop*/
+
+    if (s->state != MIG_STATE_ERROR) {
+        migrate_set_state(s, MIG_STATE_COLO, MIG_STATE_COMPLETED);
+    }
+
+    qemu_mutex_lock_iothread();
+    qemu_bh_schedule(s->cleanup_bh);
+    qemu_mutex_unlock_iothread();
+
+    return NULL;
+}
+
+static void colo_start_checkpointer(void *opaque)
+{
+    MigrationState *s = opaque;
+
+    if (colo_bh) {
+        qemu_bh_delete(colo_bh);
+        colo_bh = NULL;
+    }
+
+    qemu_mutex_unlock_iothread();
+    qemu_thread_join(&s->thread);
+    qemu_mutex_lock_iothread();
+
+    migrate_set_state(s, MIG_STATE_ACTIVE, MIG_STATE_COLO);
+
+    qemu_thread_create(&s->thread, "colo", colo_thread, s,
+                       QEMU_THREAD_JOINABLE);
+}
+
+void colo_init_checkpointer(MigrationState *s)
+{
+    colo_bh = qemu_bh_new(colo_start_checkpointer, s);
+    qemu_bh_schedule(colo_bh);
+}
diff --git a/migration.c b/migration.c
index ca83310..b7f8e7e 100644
--- a/migration.c
+++ b/migration.c
@@ -27,16 +27,6 @@
 #include "trace.h"
 #include "migration/migration-colo.h"
 
-enum {
-    MIG_STATE_ERROR = -1,
-    MIG_STATE_NONE,
-    MIG_STATE_SETUP,
-    MIG_STATE_CANCELLING,
-    MIG_STATE_CANCELLED,
-    MIG_STATE_ACTIVE,
-    MIG_STATE_COMPLETED,
-};
-
 #define MAX_THROTTLE  (32 << 20)      /* Migration speed throttling */
 
 /* Amount of time to allocate to each "chunk" of bandwidth-throttled
@@ -229,6 +219,11 @@ MigrationInfo *qmp_query_migrate(Error **errp)
 
         get_xbzrle_cache_stats(info);
         break;
+    case MIG_STATE_COLO:
+        info->has_status = true;
+        info->status = g_strdup("colo");
+        /* TODO: display COLO specific informations(checkpoint info etc.),*/
+        break;
     case MIG_STATE_COMPLETED:
         get_xbzrle_cache_stats(info);
 
@@ -272,7 +267,8 @@ void 
qmp_migrate_set_capabilities(MigrationCapabilityStatusList *params,
     MigrationState *s = migrate_get_current();
     MigrationCapabilityStatusList *cap;
 
-    if (s->state == MIG_STATE_ACTIVE || s->state == MIG_STATE_SETUP) {
+    if (s->state == MIG_STATE_ACTIVE || s->state == MIG_STATE_SETUP ||
+        s->state == MIG_STATE_COLO) {
         error_set(errp, QERR_MIGRATION_ACTIVE);
         return;
     }
@@ -289,7 +285,7 @@ void 
qmp_migrate_set_capabilities(MigrationCapabilityStatusList *params,
 
 /* shared migration helpers */
 
-static void migrate_set_state(MigrationState *s, int old_state, int new_state)
+void migrate_set_state(MigrationState *s, int old_state, int new_state)
 {
     if (atomic_cmpxchg(&s->state, old_state, new_state) == new_state) {
         trace_migrate_set_state(new_state);
@@ -423,7 +419,7 @@ void qmp_migrate(const char *uri, bool has_blk, bool blk,
     params.shared = has_inc && inc;
 
     if (s->state == MIG_STATE_ACTIVE || s->state == MIG_STATE_SETUP ||
-        s->state == MIG_STATE_CANCELLING) {
+        s->state == MIG_STATE_CANCELLING || s->state == MIG_STATE_COLO) {
         error_set(errp, QERR_MIGRATION_ACTIVE);
         return;
     }
@@ -591,6 +587,7 @@ static void *migration_thread(void *opaque)
     int64_t max_size = 0;
     int64_t start_time = initial_time;
     bool old_vm_running = false;
+    bool use_colo = migrate_use_colo();
 
     qemu_savevm_state_begin(s->file, &s->params);
 
@@ -627,7 +624,10 @@ static void *migration_thread(void *opaque)
                 }
 
                 if (!qemu_file_get_error(s->file)) {
-                    migrate_set_state(s, MIG_STATE_ACTIVE, 
MIG_STATE_COMPLETED);
+                    if (!use_colo) {
+                        migrate_set_state(s, MIG_STATE_ACTIVE,
+                                          MIG_STATE_COMPLETED);
+                    }
                     break;
                 }
             }
@@ -677,11 +677,17 @@ static void *migration_thread(void *opaque)
         }
         runstate_set(RUN_STATE_POSTMIGRATE);
     } else {
+        if (s->state == MIG_STATE_ACTIVE && use_colo) {
+            colo_init_checkpointer(s);
+        }
         if (old_vm_running) {
             vm_start();
         }
     }
-    qemu_bh_schedule(s->cleanup_bh);
+
+    if (!use_colo) {
+        qemu_bh_schedule(s->cleanup_bh);
+    }
     qemu_mutex_unlock_iothread();
 
     return NULL;
diff --git a/stubs/migration-colo.c b/stubs/migration-colo.c
index b9ee6a0..9013c40 100644
--- a/stubs/migration-colo.c
+++ b/stubs/migration-colo.c
@@ -14,3 +14,7 @@ bool colo_supported(void)
 {
     return false;
 }
+
+void colo_init_checkpointer(MigrationState *s)
+{
+}
-- 
1.9.1