[Qemu-devel] [PULL 12/18] replay: checkpoints

[Paolo Bonzini]

From: Pavel Dovgalyuk <address@hidden>

This patch introduces checkpoints that synchronize cpu thread and iothread.
When checkpoint is met in the code all asynchronous events from the queue
are executed.

Signed-off-by: Pavel Dovgalyuk <address@hidden>
Message-Id: <address@hidden>
Signed-off-by: Paolo Bonzini <address@hidden>
---
 cpus.c                   | 12 ++++++++++++
 include/sysemu/replay.h  | 20 ++++++++++++++++++++
 qemu-timer.c             | 41 ++++++++++++++++++++++++++++++++++-------
 replay/replay-internal.h |  4 ++++
 replay/replay.c          | 34 ++++++++++++++++++++++++++++++++++
 stubs/replay.c           |  6 ++++++
 vl.c                     | 30 +++++++++++++++++++++++++-----
 7 files changed, 135 insertions(+), 12 deletions(-)

diff --git a/cpus.c b/cpus.c
index 1a7318f..2619fd0 100644
--- a/cpus.c
+++ b/cpus.c
@@ -410,6 +410,18 @@ void qemu_clock_warp(QEMUClockType type)
         return;
     }
 
+    /* Nothing to do if the VM is stopped: QEMU_CLOCK_VIRTUAL timers
+     * do not fire, so computing the deadline does not make sense.
+     */
+    if (!runstate_is_running()) {
+        return;
+    }
+
+    /* warp clock deterministically in record/replay mode */
+    if (!replay_checkpoint(CHECKPOINT_CLOCK_WARP)) {
+        return;
+    }
+
     if (icount_sleep) {
         /*
          * If the CPUs have been sleeping, advance QEMU_CLOCK_VIRTUAL timer 
now.
diff --git a/include/sysemu/replay.h b/include/sysemu/replay.h
index fcc93d1..e2696fe 100644
--- a/include/sysemu/replay.h
+++ b/include/sysemu/replay.h
@@ -26,6 +26,20 @@ enum ReplayClockKind {
 };
 typedef enum ReplayClockKind ReplayClockKind;
 
+/* IDs of the checkpoints */
+enum ReplayCheckpoint {
+    CHECKPOINT_CLOCK_WARP,
+    CHECKPOINT_RESET_REQUESTED,
+    CHECKPOINT_SUSPEND_REQUESTED,
+    CHECKPOINT_CLOCK_VIRTUAL,
+    CHECKPOINT_CLOCK_HOST,
+    CHECKPOINT_CLOCK_VIRTUAL_RT,
+    CHECKPOINT_INIT,
+    CHECKPOINT_RESET,
+    CHECKPOINT_COUNT
+};
+typedef enum ReplayCheckpoint ReplayCheckpoint;
+
 extern ReplayMode replay_mode;
 
 /* Processing the instructions */
@@ -70,6 +84,12 @@ int64_t replay_read_clock(ReplayClockKind kind);
 
 /*! Called when qemu shutdown is requested. */
 void replay_shutdown_request(void);
+/*! Should be called at check points in the execution.
+    These check points are skipped, if they were not met.
+    Saves checkpoint in the SAVE mode and validates in the PLAY mode.
+    Returns 0 in PLAY mode if checkpoint was not found.
+    Returns 1 in all other cases. */
+bool replay_checkpoint(ReplayCheckpoint checkpoint);
 
 /* Asynchronous events queue */
 
diff --git a/qemu-timer.c b/qemu-timer.c
index 3dc847b..f16e422 100644
--- a/qemu-timer.c
+++ b/qemu-timer.c
@@ -25,6 +25,7 @@
 #include "qemu/main-loop.h"
 #include "qemu/timer.h"
 #include "sysemu/replay.h"
+#include "sysemu/sysemu.h"
 
 #ifdef CONFIG_POSIX
 #include <pthread.h>
@@ -478,10 +479,31 @@ bool timerlist_run_timers(QEMUTimerList *timer_list)
     void *opaque;
 
     qemu_event_reset(&timer_list->timers_done_ev);
-    if (!timer_list->clock->enabled) {
+    if (!timer_list->clock->enabled || !timer_list->active_timers) {
         goto out;
     }
 
+    switch (timer_list->clock->type) {
+    case QEMU_CLOCK_REALTIME:
+        break;
+    default:
+    case QEMU_CLOCK_VIRTUAL:
+        if (!replay_checkpoint(CHECKPOINT_CLOCK_VIRTUAL)) {
+            goto out;
+        }
+        break;
+    case QEMU_CLOCK_HOST:
+        if (!replay_checkpoint(CHECKPOINT_CLOCK_HOST)) {
+            goto out;
+        }
+        break;
+    case QEMU_CLOCK_VIRTUAL_RT:
+        if (!replay_checkpoint(CHECKPOINT_CLOCK_VIRTUAL_RT)) {
+            goto out;
+        }
+        break;
+    }
+
     current_time = qemu_clock_get_ns(timer_list->clock->type);
     for(;;) {
         qemu_mutex_lock(&timer_list->active_timers_lock);
@@ -545,11 +567,17 @@ int64_t timerlistgroup_deadline_ns(QEMUTimerListGroup 
*tlg)
 {
     int64_t deadline = -1;
     QEMUClockType type;
+    bool play = replay_mode == REPLAY_MODE_PLAY;
     for (type = 0; type < QEMU_CLOCK_MAX; type++) {
-        if (qemu_clock_use_for_deadline(tlg->tl[type]->clock->type)) {
-            deadline = qemu_soonest_timeout(deadline,
-                                            timerlist_deadline_ns(
-                                                tlg->tl[type]));
+        if (qemu_clock_use_for_deadline(type)) {
+            if (!play || type == QEMU_CLOCK_REALTIME) {
+                deadline = qemu_soonest_timeout(deadline,
+                                                
timerlist_deadline_ns(tlg->tl[type]));
+            } else {
+                /* Read clock from the replay file and
+                   do not calculate the deadline, based on virtual clock. */
+                qemu_clock_get_ns(type);
+            }
         }
     }
     return deadline;
@@ -574,8 +602,7 @@ int64_t qemu_clock_get_ns(QEMUClockType type)
         now = REPLAY_CLOCK(REPLAY_CLOCK_HOST, get_clock_realtime());
         last = clock->last;
         clock->last = now;
-        if ((now < last || now > (last + get_max_clock_jump()))
-            && replay_mode == REPLAY_MODE_NONE) {
+        if (now < last || now > (last + get_max_clock_jump())) {
             notifier_list_notify(&clock->reset_notifiers, &now);
         }
         return now;
diff --git a/replay/replay-internal.h b/replay/replay-internal.h
index 4414695..bf64be5 100644
--- a/replay/replay-internal.h
+++ b/replay/replay-internal.h
@@ -29,6 +29,10 @@ enum ReplayEvents {
     /* some of greater codes are reserved for clocks */
     EVENT_CLOCK,
     EVENT_CLOCK_LAST = EVENT_CLOCK + REPLAY_CLOCK_COUNT - 1,
+    /* for checkpoint event */
+    /* some of greater codes are reserved for checkpoints */
+    EVENT_CHECKPOINT,
+    EVENT_CHECKPOINT_LAST = EVENT_CHECKPOINT + CHECKPOINT_COUNT - 1,
     EVENT_COUNT
 };
 
diff --git a/replay/replay.c b/replay/replay.c
index 65dca7f..44fbed9 100644
--- a/replay/replay.c
+++ b/replay/replay.c
@@ -160,3 +160,37 @@ void replay_shutdown_request(void)
         replay_mutex_unlock();
     }
 }
+
+bool replay_checkpoint(ReplayCheckpoint checkpoint)
+{
+    bool res = false;
+    assert(EVENT_CHECKPOINT + checkpoint <= EVENT_CHECKPOINT_LAST);
+    replay_save_instructions();
+
+    if (!replay_file) {
+        return true;
+    }
+
+    replay_mutex_lock();
+
+    if (replay_mode == REPLAY_MODE_PLAY) {
+        if (replay_next_event_is(EVENT_CHECKPOINT + checkpoint)) {
+            replay_finish_event();
+        } else if (replay_data_kind != EVENT_ASYNC) {
+            res = false;
+            goto out;
+        }
+        replay_read_events(checkpoint);
+        /* replay_read_events may leave some unread events.
+           Return false if not all of the events associated with
+           checkpoint were processed */
+        res = replay_data_kind != EVENT_ASYNC;
+    } else if (replay_mode == REPLAY_MODE_RECORD) {
+        replay_put_event(EVENT_CHECKPOINT + checkpoint);
+        replay_save_events(checkpoint);
+        res = true;
+    }
+out:
+    replay_mutex_unlock();
+    return res;
+}
diff --git a/stubs/replay.c b/stubs/replay.c
index f0d95b0..3354fcf 100644
--- a/stubs/replay.c
+++ b/stubs/replay.c
@@ -1,5 +1,6 @@
 #include "sysemu/replay.h"
 #include <stdlib.h>
+#include "sysemu/sysemu.h"
 
 ReplayMode replay_mode;
 
@@ -14,3 +15,8 @@ int64_t replay_read_clock(unsigned int kind)
     abort();
     return 0;
 }
+
+bool replay_checkpoint(ReplayCheckpoint checkpoint)
+{
+    return true;
+}
diff --git a/vl.c b/vl.c
index 1856180..1cb7550 100644
--- a/vl.c
+++ b/vl.c
@@ -122,6 +122,7 @@ int main(int argc, char **argv)
 #include "qapi-event.h"
 #include "exec/semihost.h"
 #include "crypto/init.h"
+#include "sysemu/replay.h"
 
 #define MAX_VIRTIO_CONSOLES 1
 #define MAX_SCLP_CONSOLES 1
@@ -1640,15 +1641,21 @@ static void qemu_kill_report(void)
 static int qemu_reset_requested(void)
 {
     int r = reset_requested;
-    reset_requested = 0;
-    return r;
+    if (r && replay_checkpoint(CHECKPOINT_RESET_REQUESTED)) {
+        reset_requested = 0;
+        return r;
+    }
+    return false;
 }
 
 static int qemu_suspend_requested(void)
 {
     int r = suspend_requested;
-    suspend_requested = 0;
-    return r;
+    if (r && replay_checkpoint(CHECKPOINT_SUSPEND_REQUESTED)) {
+        suspend_requested = 0;
+        return r;
+    }
+    return false;
 }
 
 static WakeupReason qemu_wakeup_requested(void)
@@ -1796,7 +1803,12 @@ void qemu_system_killed(int signal, pid_t pid)
     shutdown_signal = signal;
     shutdown_pid = pid;
     no_shutdown = 0;
-    qemu_system_shutdown_request();
+
+    /* Cannot call qemu_system_shutdown_request directly because
+     * we are in a signal handler.
+     */
+    shutdown_requested = 1;
+    qemu_notify_event();
 }
 
 void qemu_system_shutdown_request(void)
@@ -4488,6 +4500,10 @@ int main(int argc, char **argv, char **envp)
     }
     qemu_add_globals();
 
+    /* This checkpoint is required by replay to separate prior clock
+       reading from the other reads, because timer polling functions query
+       clock values from the log. */
+    replay_checkpoint(CHECKPOINT_INIT);
     qdev_machine_init();
 
     current_machine->ram_size = ram_size;
@@ -4606,6 +4622,10 @@ int main(int argc, char **argv, char **envp)
         exit(1);
     }
 
+    /* This checkpoint is required by replay to separate prior clock
+       reading from the other reads, because timer polling functions query
+       clock values from the log. */
+    replay_checkpoint(CHECKPOINT_RESET);
     qemu_system_reset(VMRESET_SILENT);
     register_global_state();
     if (loadvm) {
-- 
1.8.3.1