[PATCH experiment 14/16] util: introduce C++ stackless coroutine backend

[Paolo Bonzini]

Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>
---
 configure                                     |  44 +-
 include/qemu/coroutine.h                      | 445 +++++++++++++-----
 include/qemu/coroutine_int.h                  |   8 +
 util/coroutine-stackless.cc                   | 145 ++++++
 util/meson.build                              |   6 +-
 ...oroutine-lock.c => qemu-coroutine-lock.cc} |  70 +--
 ...outine-sleep.c => qemu-coroutine-sleep.cc} |  12 +-
 util/{qemu-coroutine.c => qemu-coroutine.cc}  |  16 -
 8 files changed, 530 insertions(+), 216 deletions(-)
 create mode 100644 util/coroutine-stackless.cc
 rename util/{qemu-coroutine-lock.c => qemu-coroutine-lock.cc} (87%)
 rename util/{qemu-coroutine-sleep.c => qemu-coroutine-sleep.cc} (89%)
 rename util/{qemu-coroutine.c => qemu-coroutine.cc} (94%)

diff --git a/configure b/configure
index 091710ec03..c02b5edcba 100755
--- a/configure
+++ b/configure
@@ -1220,8 +1220,6 @@ Advanced options (experts only):
   --with-trace-file=NAME   Full PATH,NAME of file to store traces
                            Default:trace-<pid>
   --cpu=CPU                Build for host CPU [$cpu]
-  --with-coroutine=BACKEND coroutine backend. Supported options:
-                           ucontext, sigaltstack, windows
   --enable-gcov            enable test coverage analysis with gcov
   --tls-priority           default TLS protocol/cipher priority string
   --enable-plugins
@@ -1242,7 +1240,7 @@ cat << EOF
   debug-info      debugging information
   lto             Enable Link-Time Optimization.
   safe-stack      SafeStack Stack Smash Protection. Depends on
-                  clang/llvm >= 3.7 and requires coroutine backend ucontext.
+                  clang/llvm >= 3.7
   rdma            Enable RDMA-based migration
   pvrdma          Enable PVRDMA support
   vhost-net       vhost-net kernel acceleration support
@@ -2338,39 +2336,7 @@ EOF
   fi
 fi
 
-if test "$coroutine" = ""; then
-  if test "$mingw32" = "yes"; then
-    coroutine=win32
-  elif test "$ucontext_works" = "yes"; then
-    coroutine=ucontext
-  else
-    coroutine=sigaltstack
-  fi
-else
-  case $coroutine in
-  windows)
-    if test "$mingw32" != "yes"; then
-      error_exit "'windows' coroutine backend only valid for Windows"
-    fi
-    # Unfortunately the user visible backend name doesn't match the
-    # coroutine-*.c filename for this case, so we have to adjust it here.
-    coroutine=win32
-    ;;
-  ucontext)
-    if test "$ucontext_works" != "yes"; then
-      feature_not_found "ucontext"
-    fi
-    ;;
-  sigaltstack)
-    if test "$mingw32" = "yes"; then
-      error_exit "only the 'windows' coroutine backend is valid for Windows"
-    fi
-    ;;
-  *)
-    error_exit "unknown coroutine backend $coroutine"
-    ;;
-  esac
-fi
+coroutine=stackless
 
 ##################################################
 # SafeStack
@@ -2395,9 +2361,6 @@ EOF
   else
     error_exit "SafeStack not supported by your compiler"
   fi
-  if test "$coroutine" != "ucontext"; then
-    error_exit "SafeStack is only supported by the coroutine backend ucontext"
-  fi
 else
 cat > $TMPC << EOF
 int main(int argc, char *argv[])
@@ -2427,9 +2390,6 @@ else # "$safe_stack" = ""
     safe_stack="no"
   else
     safe_stack="yes"
-    if test "$coroutine" != "ucontext"; then
-      error_exit "SafeStack is only supported by the coroutine backend 
ucontext"
-    fi
   fi
 fi
 fi
diff --git a/include/qemu/coroutine.h b/include/qemu/coroutine.h
index ac9891502e..0f89fbafa0 100644
--- a/include/qemu/coroutine.h
+++ b/include/qemu/coroutine.h
@@ -48,25 +48,6 @@ G_BEGIN_DECLS
 
 typedef struct Coroutine Coroutine;
 
-/**
- * Coroutine entry point
- *
- * When the coroutine is entered for the first time, opaque is passed in as an
- * argument.
- *
- * When this function returns, the coroutine is destroyed automatically and
- * execution continues in the caller who last entered the coroutine.
- */
-typedef void coroutine_fn CoroutineEntry(void *opaque);
-
-/**
- * Create a new coroutine
- *
- * Use qemu_coroutine_enter() to actually transfer control to the coroutine.
- * The opaque argument is passed as the argument to the entry point.
- */
-Coroutine *qemu_coroutine_create(CoroutineEntry *entry, void *opaque);
-
 /**
  * Transfer control to a coroutine
  */
@@ -83,14 +64,6 @@ void qemu_coroutine_enter_if_inactive(Coroutine *co);
  */
 void qemu_aio_coroutine_enter(AioContext *ctx, Coroutine *co);
 
-/**
- * Transfer control back to a coroutine's caller
- *
- * This function does not return until the coroutine is re-entered using
- * qemu_coroutine_enter().
- */
-void coroutine_fn qemu_coroutine_yield(void);
-
 /**
  * Get the AioContext of the given coroutine
  */
@@ -157,18 +130,6 @@ struct CoMutex {
  */
 void qemu_co_mutex_init(CoMutex *mutex);
 
-/**
- * Locks the mutex. If the lock cannot be taken immediately, control is
- * transferred to the caller of the current coroutine.
- */
-void coroutine_fn qemu_co_mutex_lock(CoMutex *mutex);
-
-/**
- * Unlocks the mutex and schedules the next coroutine that was waiting for this
- * lock to be run.
- */
-void coroutine_fn qemu_co_mutex_unlock(CoMutex *mutex);
-
 /**
  * Assert that the current coroutine holds @mutex.
  */
@@ -200,17 +161,6 @@ typedef struct CoQueue {
  */
 void qemu_co_queue_init(CoQueue *queue);
 
-#if 0
-/**
- * Adds the current coroutine to the CoQueue and transfers control to the
- * caller of the coroutine.  The mutex is unlocked during the wait and
- * locked again afterwards.
- */
-#define qemu_co_queue_wait(queue, lock) \
-    qemu_co_queue_wait_impl(queue, QEMU_MAKE_CO_LOCKABLE(lock))
-void coroutine_fn qemu_co_queue_wait_impl(CoQueue *queue, QemuCoLockable 
*lock);
-#endif
-
 /**
  * Removes the next coroutine from the CoQueue, and wake it up.
  * Returns true if a coroutine was removed, false if the queue is empty.
@@ -260,66 +210,10 @@ typedef struct CoRwlock {
  */
 void qemu_co_rwlock_init(CoRwlock *lock);
 
-/**
- * Read locks the CoRwlock. If the lock cannot be taken immediately because
- * of a parallel writer, control is transferred to the caller of the current
- * coroutine.
- */
-void coroutine_fn qemu_co_rwlock_rdlock(CoRwlock *lock);
-
-/**
- * Write Locks the CoRwlock from a reader.  This is a bit more efficient than
- * @qemu_co_rwlock_unlock followed by a separate @qemu_co_rwlock_wrlock.
- * Note that if the lock cannot be upgraded immediately, control is transferred
- * to the caller of the current coroutine; another writer might run while
- * @qemu_co_rwlock_upgrade blocks.
- */
-void coroutine_fn qemu_co_rwlock_upgrade(CoRwlock *lock);
-
-/**
- * Downgrades a write-side critic section to a reader.  Downgrading with
- * @qemu_co_rwlock_downgrade never blocks, unlike @qemu_co_rwlock_unlock
- * followed by @qemu_co_rwlock_rdlock.  This makes it more efficient, but
- * may also sometimes be necessary for correctness.
- */
-void coroutine_fn qemu_co_rwlock_downgrade(CoRwlock *lock);
-
-/**
- * Write Locks the mutex. If the lock cannot be taken immediately because
- * of a parallel reader, control is transferred to the caller of the current
- * coroutine.
- */
-void coroutine_fn qemu_co_rwlock_wrlock(CoRwlock *lock);
-
-/**
- * Unlocks the read/write lock and schedules the next coroutine that was
- * waiting for this lock to be run.
- */
-void coroutine_fn qemu_co_rwlock_unlock(CoRwlock *lock);
-
 typedef struct QemuCoSleep {
     Coroutine *to_wake;
 } QemuCoSleep;
 
-/**
- * Yield the coroutine for a given duration. Initializes @w so that,
- * during this yield, it can be passed to qemu_co_sleep_wake() to
- * terminate the sleep.
- */
-void coroutine_fn qemu_co_sleep_ns_wakeable(QemuCoSleep *w,
-                                            QEMUClockType type, int64_t ns);
-
-/**
- * Yield the coroutine until the next call to qemu_co_sleep_wake.
- */
-void coroutine_fn qemu_co_sleep(QemuCoSleep *w);
-
-static inline void coroutine_fn qemu_co_sleep_ns(QEMUClockType type, int64_t 
ns)
-{
-    QemuCoSleep w = { 0 };
-    qemu_co_sleep_ns_wakeable(&w, type, ns);
-}
-
 /**
  * Wake a coroutine if it is sleeping in qemu_co_sleep_ns. The timer will be
  * deleted. @sleep_state must be the variable whose address was given to
@@ -328,13 +222,6 @@ static inline void coroutine_fn 
qemu_co_sleep_ns(QEMUClockType type, int64_t ns)
  */
 void qemu_co_sleep_wake(QemuCoSleep *w);
 
-/**
- * Yield until a file descriptor becomes readable
- *
- * Note that this function clobbers the handlers for the file descriptor.
- */
-void coroutine_fn yield_until_fd_readable(int fd);
-
 /**
  * Increase coroutine pool size
  */
@@ -348,6 +235,338 @@ void qemu_coroutine_decrease_pool_batch_size(unsigned int 
additional_pool_size);
 G_END_DECLS
 
 #include "qemu/lockable.h"
+
+#ifdef __cplusplus
+#include <cstdint>
+#include <coroutine>
+#include <exception>
+
+// BaseCoroutine is a simple wrapper type for a Promise.  It mostly
+// exists because C++ says so, but it also provides two extra features:
+// RAII destruction of the coroutine (which is more efficient but
+// beware, the promise's final_suspend must always suspend to avoid
+// double free) and a cast to std::coroutine_handle<>, which makes
+// it resumable.
+
+template<typename Promise> struct BaseCoroutine
+{
+    using promise_type = Promise;
+
+    BaseCoroutine() = default;
+    explicit BaseCoroutine (Promise &promise) :
+        _coroutine{std::coroutine_handle<Promise>::from_promise(promise)} {}
+
+    BaseCoroutine(BaseCoroutine const&) = delete;
+    BaseCoroutine(BaseCoroutine&& other) : _coroutine{other._coroutine} {
+        other._coroutine = nullptr;
+    }
+
+    BaseCoroutine& operator=(BaseCoroutine const&) = delete;
+    BaseCoroutine& operator=(BaseCoroutine&& other) {
+        if (&other != this) {
+            _coroutine = other._coroutine;
+            other._coroutine = nullptr;
+        }
+        return *this;
+    }
+
+    ~BaseCoroutine() {
+        //printf("!!!! destroying %p\n", _coroutine);
+        if (_coroutine) _coroutine.destroy();
+    }
+
+    operator bool() const noexcept {
+        return _coroutine;
+    }
+    operator std::coroutine_handle<>() const noexcept {
+        return _coroutine;
+    }
+    Promise &promise() const noexcept {
+        return _coroutine.promise();
+    }
+
+private:
+    std::coroutine_handle<Promise> _coroutine = nullptr;
+};
+
+// This is a simple awaitable object that takes care of resuming a
+// parent coroutine.  It's needed because co_await suspends all
+// parent coroutines on the stack.  It does not need a specific
+// "kind" of coroutine_handle, so no need to put it inside the
+// templates below.
+//
+// If next is NULL, then this degrades to std::suspend_always.
+
+struct ResumeAndFinish {
+    explicit ResumeAndFinish(std::coroutine_handle<> next) noexcept :
+        _next{next} {}
+
+    bool await_ready() const noexcept {
+        return false;
+    }
+    bool await_suspend(std::coroutine_handle<> ch) const noexcept {
+        if (_next) {
+            _next.resume();
+        }
+        return true;
+    }
+    void await_resume() const noexcept {}
+
+private:
+    std::coroutine_handle<> _next;
+};
+
+// ------------------------
+
+// CoroutineFn does not even need anything more than what
+// BaseCoroutine provides, so it's just a type alias.  The magic
+// is all in ValuePromise<T>.
+//
+// Suspended CoroutineFns are chained between themselves.  Whenever a
+// coroutine is suspended, all those that have done a co_await are
+// also suspended, and whenever a coroutine finishes, it has to
+// check if its parent can now be resumed.
+//
+// The two auxiliary classes Awaiter and ResumeAndFinish take
+// care of the two sides of this.  Awaiter's await_suspend() stores
+// the parent coroutine into ValuePromise; ResumeAndFinish's runs
+// after a coroutine returns, and resumes the parent coroutine.
+
+template<typename T> struct ValuePromise;
+template<typename T>
+using CoroutineFn = BaseCoroutine<ValuePromise<T>>;
+
+typedef CoroutineFn<void> CoroutineFunc(void *);
+
+// Unfortunately it is forbidden to define both return_void() and
+// return_value() in the same class.  In order to cut on the
+// code duplication, define a superclass for both ValuePromise<T>
+// and ValuePromise<void>.
+//
+// The "curiously recurring template pattern" is used to substitute
+// ValuePromise<T> into the methods of the base class and its Awaited.
+// For example await_resume() needs to retrieve a value with the
+// correct type from the subclass's value() method.
+
+template<typename T, typename Derived>
+struct BasePromise
+{
+    using coro_handle_type = std::coroutine_handle<Derived>;
+
+#if 0
+    // Same as get_return_object().address() but actually works.
+    // Useful as an identifier to identify the promise in debugging
+    // output, because it matches the values passed to await_suspend().
+    void *coro_address() const {
+        return __builtin_coro_promise((char *)this, __alignof(*this), true);
+    }
+
+    BasePromise() {
+        printf("!!!! created %p\n", coro_address());
+    }
+    ~BasePromise() {
+        printf("!!!! destroyed %p\n", coro_address());
+    }
+#endif
+
+    CoroutineFn<T> get_return_object() noexcept { return 
CoroutineFn<T>{downcast()}; }
+    void unhandled_exception()                  { std::terminate(); }
+    auto initial_suspend() const noexcept       { return std::suspend_never{}; 
}
+    auto final_suspend() noexcept               {
+        auto continuation = ResumeAndFinish{_next};
+        mark_ready();
+        return continuation;
+    }
+private:
+    std::coroutine_handle<> _next = nullptr;
+
+    static const std::uintptr_t READY_MARKER = 1;
+    void mark_ready() {
+        _next = std::coroutine_handle<>::from_address((void *)READY_MARKER);
+    }
+
+    bool is_ready() const {
+        return _next.address() == (void *)READY_MARKER;
+    }
+
+    Derived& downcast() noexcept                { return 
*static_cast<Derived*>(this); }
+    Derived const& downcast() const noexcept    { return *static_cast<const 
Derived*>(this); }
+
+    // This records the parent coroutine, before a co_await suspends
+    // all parent coroutines on the stack.
+    void then(std::coroutine_handle<> parent)  { _next = parent; }
+
+    // This is the object that lets us co_await a CoroutineFn<T> (of which
+    // this class is the corresponding promise object).  This is just mapping
+    // C++ awaitable naming into the more convention promise naming.
+    struct Awaiter {
+        Derived &_promise;
+
+        explicit Awaiter(Derived &promise) : _promise{promise} {}
+
+        bool await_ready() const noexcept {
+            return _promise.is_ready();
+        }
+
+        void await_suspend(std::coroutine_handle<> parent) const noexcept {
+            _promise.then(parent);
+        }
+
+        Derived::await_resume_type await_resume() const noexcept {
+            return _promise.value();
+        }
+    };
+
+    // C++ connoisseurs will tell you that this is not private.
+    friend Awaiter operator co_await(CoroutineFn<T> co) {
+        return Awaiter{co.promise()};
+    }
+};
+
+// The actu promises, respectively for non-void and void types.
+// All that's left is storing and retrieving the value.
+
+template<typename T>
+struct ValuePromise: BasePromise<T, ValuePromise<T>>
+{
+    using await_resume_type = T&&;
+    T _value;
+    void return_value(T&& value)       { _value = std::move(value); }
+    void return_value(T const& value)  { _value = value; }
+    T&& value() noexcept               { return static_cast<T&&>(_value); }
+};
+
+template<>
+struct ValuePromise<void>: BasePromise<void, ValuePromise<void>>
+{
+    using await_resume_type = void;
+    void return_void() const           {}
+    void value() const                 {}
+};
+
+
+// ---------------------------
+
+// This class takes care of yielding, which is just a matter of doing
+// "co_await Yield{}".  This always suspends, and also stores the
+// suspending CoroutineFn in current->top.
+struct Yield: std::suspend_always {
+    void await_suspend(std::coroutine_handle<> parent) const noexcept;
+};
+
+// ---------------------------
+
+// Make it possible to write "co_await qemu_coroutine_yield()"
+static inline Yield qemu_coroutine_yield()
+{
+    return Yield{};
+}
+
+/**
+ * Coroutine entry point
+ *
+ * When the coroutine is entered for the first time, opaque is passed in as an
+ * argument.
+ *
+ * When this function returns, the coroutine is destroyed automatically and
+ * execution continues in the caller who last entered the coroutine.
+ */
+typedef CoroutineFn<void> CoroutineEntry(void *opaque);
+
+/**
+ * Create a new coroutine
+ *
+ * Use qemu_coroutine_enter() to actually transfer control to the coroutine.
+ * The opaque argument is passed as the argument to the entry point.
+ */
+Coroutine *qemu_coroutine_create(CoroutineEntry *entry, void *opaque);
+
+/**
+ * Adds the current coroutine to the CoQueue and transfers control to the
+ * caller of the coroutine.  The mutex is unlocked during the wait and
+ * locked again afterwards.
+ */
+#define qemu_co_queue_wait(queue, lock) \
+    qemu_co_queue_wait_impl(queue, QEMU_MAKE_CO_LOCKABLE(lock))
+CoroutineFn<void> qemu_co_queue_wait_impl(CoQueue *queue, QemuCoLockable 
*lock);
+
+/**
+ * Locks the mutex. If the lock cannot be taken immediately, control is
+ * transferred to the caller of the current coroutine.
+ */
+CoroutineFn<void> qemu_co_mutex_lock(CoMutex *mutex);
+
+/**
+ * Unlocks the mutex and schedules the next coroutine that was waiting for this
+ * lock to be run.
+ */
+CoroutineFn<void> qemu_co_mutex_unlock(CoMutex *mutex);
+
+/**
+ * Read locks the CoRwlock. If the lock cannot be taken immediately because
+ * of a parallel writer, control is transferred to the caller of the current
+ * coroutine.
+ */
+CoroutineFn<void> qemu_co_rwlock_rdlock(CoRwlock *lock);
+
+/**
+ * Write Locks the CoRwlock from a reader.  This is a bit more efficient than
+ * @qemu_co_rwlock_unlock followed by a separate @qemu_co_rwlock_wrlock.
+ * Note that if the lock cannot be upgraded immediately, control is transferred
+ * to the caller of the current coroutine; another writer might run while
+ * @qemu_co_rwlock_upgrade blocks.
+ */
+CoroutineFn<void> qemu_co_rwlock_upgrade(CoRwlock *lock);
+
+/**
+ * Downgrades a write-side critic section to a reader.  Downgrading with
+ * @qemu_co_rwlock_downgrade never blocks, unlike @qemu_co_rwlock_unlock
+ * followed by @qemu_co_rwlock_rdlock.  This makes it more efficient, but
+ * may also sometimes be necessary for correctness.
+ */
+CoroutineFn<void> qemu_co_rwlock_downgrade(CoRwlock *lock);
+
+/**
+ * Write Locks the mutex. If the lock cannot be taken immediately because
+ * of a parallel reader, control is transferred to the caller of the current
+ * coroutine.
+ */
+CoroutineFn<void> qemu_co_rwlock_wrlock(CoRwlock *lock);
+
+/**
+ * Unlocks the read/write lock and schedules the next coroutine that was
+ * waiting for this lock to be run.
+ */
+CoroutineFn<void> qemu_co_rwlock_unlock(CoRwlock *lock);
+
+/**
+ * Yield the coroutine for a given duration. Initializes @w so that,
+ * during this yield, it can be passed to qemu_co_sleep_wake() to
+ * terminate the sleep.
+ */
+CoroutineFn<void> qemu_co_sleep_ns_wakeable(QemuCoSleep *w,
+                                            QEMUClockType type, int64_t ns);
+
+/**
+ * Yield the coroutine until the next call to qemu_co_sleep_wake.
+ */
+CoroutineFn<void> qemu_co_sleep(QemuCoSleep *w);
+
+static inline CoroutineFn<void> qemu_co_sleep_ns(QEMUClockType type, int64_t 
ns)
+{
+    QemuCoSleep w = { 0 };
+    co_await qemu_co_sleep_ns_wakeable(&w, type, ns);
+}
+
+/**
+ * Yield until a file descriptor becomes readable
+ *
+ * Note that this function clobbers the handlers for the file descriptor.
+ */
+CoroutineFn<void> yield_until_fd_readable(int fd);
+
+
 #include "qemu/co-lockable.h"
+#endif
 
 #endif /* QEMU_COROUTINE_H */
diff --git a/include/qemu/coroutine_int.h b/include/qemu/coroutine_int.h
index 1da148552f..67d6586997 100644
--- a/include/qemu/coroutine_int.h
+++ b/include/qemu/coroutine_int.h
@@ -28,6 +28,8 @@
 #include "qemu/queue.h"
 #include "qemu/coroutine.h"
 
+G_BEGIN_DECLS
+
 #ifdef CONFIG_SAFESTACK
 /* Pointer to the unsafe stack, defined by the compiler */
 extern __thread void *__safestack_unsafe_stack_ptr;
@@ -41,6 +43,10 @@ typedef enum {
     COROUTINE_ENTER = 3,
 } CoroutineAction;
 
+#ifndef __cplusplus
+typedef struct IncompleteType *CoroutineEntry;
+#endif
+
 struct Coroutine {
     CoroutineEntry *entry;
     void *entry_arg;
@@ -74,4 +80,6 @@ void qemu_coroutine_delete(Coroutine *co);
 CoroutineAction qemu_coroutine_switch(Coroutine *from, Coroutine *to,
                                       CoroutineAction action);
 
+G_END_DECLS
+
 #endif
diff --git a/util/coroutine-stackless.cc b/util/coroutine-stackless.cc
new file mode 100644
index 0000000000..ce2f284663
--- /dev/null
+++ b/util/coroutine-stackless.cc
@@ -0,0 +1,145 @@
+/*
+ * stackless coroutine initialization code
+ *
+ * Copyright (C) 2022 Paolo BOnzini <pbonzini@redhat.com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser Gener Public
+ * License as published by the Free Software Foundation; either
+ * version 2.0 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser Gener Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser Gener Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "qemu/osdep.h"
+#include "trace.h"
+#include "qemu/coroutine_int.h"
+
+// CoroutineImpl is the entry point into a coroutine.  It stores the
+// coroutine_handle that last called qemu_coroutine_yield(), and
+// Coroutine::resume() then resumes from the last yield point.
+//
+// Together with a thread-loc variable "current", the "caller"
+// member establishes a stack of active coroutines, so that
+// qemu_coroutine_yield() knows which coroutine has yielded.
+//
+// Its promise type, EntryPromise, is pretty much bog-standard.
+// It always suspends on entry, so that the coroutine is only
+// entered by the first call to qemu_coroutine_enter(); and it
+// always suspends on exit too, because we want to clean up the
+// coroutine explicitly in BaseCoroutine's destructor.
+
+struct EntryPromise;
+struct CoroutineImpl: BaseCoroutine<EntryPromise> {
+    std::coroutine_handle<> top;
+    explicit CoroutineImpl(promise_type &promise) :
+        BaseCoroutine{promise}, top{*this} {}
+
+    CoroutineAction resume();
+};
+
+struct EntryPromise
+{
+    CoroutineImpl get_return_object() noexcept          { return 
CoroutineImpl{*this}; }
+    void unhandled_exception()                          { std::terminate(); }
+    auto initial_suspend() const noexcept               { return 
std::suspend_always{}; }
+    auto final_suspend() const noexcept                 { return 
std::suspend_always{}; }
+    void return_void() const noexcept                   {}
+};
+
+typedef struct {
+    Coroutine base;
+    CoroutineImpl *impl;
+} CoroutineStackless;
+
+static __thread CoroutineStackless leader;
+static __thread Coroutine *current;
+
+// ---------------------------
+
+// Change the type from CoroutineFn<void> to Coroutine,
+// so that it does not start until qemu_coroutine_enter()
+CoroutineImpl coroutine_trampoline(Coroutine *co)
+{
+    co_await co->entry(co->entry_arg);
+}
+
+CoroutineAction CoroutineImpl::resume() {
+    std::coroutine_handle<> old_top = top;
+    top = nullptr;
+    old_top.resume();
+    return top ? COROUTINE_YIELD : COROUTINE_TERMINATE;
+}
+
+void Yield::await_suspend(std::coroutine_handle<> parent) const noexcept {
+    CoroutineStackless *curr = DO_UPCAST(CoroutineStackless, base, current);
+    curr->impl->top = parent;
+}
+
+// ---------------------------
+
+Coroutine *qemu_coroutine_new(void)
+{
+    CoroutineStackless *co;
+
+    co = g_new0(CoroutineStackless, 1);
+    return &co->base;
+}
+
+void qemu_coroutine_delete(Coroutine *co_)
+{
+    CoroutineStackless *co = DO_UPCAST(CoroutineStackless, base, co_);
+
+    g_free(co);
+}
+
+// RAII wrapper to set and restore the current coroutine
+struct WithCurrent {
+    Coroutine &_co;
+    WithCurrent(Coroutine &co): _co(co) {
+        current = &_co;
+    }
+    ~WithCurrent() {
+        current = _co.caller;
+        _co.caller = NULL;
+    }
+};
+
+CoroutineAction
+qemu_coroutine_switch(Coroutine *from, Coroutine *to_,
+                      CoroutineAction action)
+{
+    CoroutineStackless *to = DO_UPCAST(CoroutineStackless, base, to_);
+
+    assert(action == COROUTINE_ENTER);
+    assert(to->base.caller);
+    auto w = WithCurrent{*to_};
+    if (!to->impl) {
+        to->impl = new CoroutineImpl(coroutine_trampoline(to_));
+    }
+    if (to->impl->resume() == COROUTINE_YIELD) {
+        return COROUTINE_YIELD;
+    }
+    delete to->impl;
+    to->impl = NULL;
+    return COROUTINE_TERMINATE;
+}
+
+Coroutine *qemu_coroutine_self(void)
+{
+    if (!current) {
+        current = &leader.base;
+    }
+    return current;
+}
+
+bool qemu_in_coroutine(void)
+{
+    return current && current->caller;
+}
diff --git a/util/meson.build b/util/meson.build
index 30949cd481..11ec6534b9 100644
--- a/util/meson.build
+++ b/util/meson.build
@@ -68,7 +68,7 @@ if have_block
   util_ss.add(files('base64.c'))
   util_ss.add(files('buffer.c'))
   util_ss.add(files('bufferiszero.c'))
-  
util_ss.add(files('coroutine-@0@.c'.format(config_host['CONFIG_COROUTINE_BACKEND'])))
+  util_ss.add(files('coroutine-stackless.cc'))
   util_ss.add(files('hbitmap.c'))
   util_ss.add(files('hexdump.c'))
   util_ss.add(files('iova-tree.c'))
@@ -76,12 +76,12 @@ if have_block
   util_ss.add(files('lockcnt.c'))
   util_ss.add(files('main-loop.c'))
   util_ss.add(files('nvdimm-utils.c'))
-  util_ss.add(files('qemu-coroutine.c', 'qemu-coroutine-lock.c')) # 
'qemu-coroutine-io.c'
+  util_ss.add(files('qemu-coroutine.cc', 'qemu-coroutine-lock.cc')) # 
'qemu-coroutine-io.c'
 # util_ss.add(when: 'CONFIG_LINUX', if_true: [
 #   files('vhost-user-server.c'), vhost_user
 # ])
   util_ss.add(files('block-helpers.c'))
-  util_ss.add(files('qemu-coroutine-sleep.c'))
+  util_ss.add(files('qemu-coroutine-sleep.cc'))
 # util_ss.add(files('qemu-co-shared-resource.c'))
   util_ss.add(files('thread-pool.c', 'qemu-timer.c'))
   util_ss.add(files('readline.c'))
diff --git a/util/qemu-coroutine-lock.c b/util/qemu-coroutine-lock.cc
similarity index 87%
rename from util/qemu-coroutine-lock.c
rename to util/qemu-coroutine-lock.cc
index d6c0565ba5..86c51604b6 100644
--- a/util/qemu-coroutine-lock.c
+++ b/util/qemu-coroutine-lock.cc
@@ -120,6 +120,7 @@ bool qemu_co_queue_empty(CoQueue *queue)
 {
     return QSIMPLEQ_FIRST(&queue->entries) == NULL;
 }
+#endif
 
 /* The wait records are handled with a multiple-producer, single-consumer
  * lock-free queue.  There cannot be two concurrent pop_waiter() calls
@@ -187,7 +188,7 @@ void qemu_co_mutex_init(CoMutex *mutex)
     memset(mutex, 0, sizeof(*mutex));
 }
 
-static void coroutine_fn qemu_co_mutex_wake(CoMutex *mutex, Coroutine *co)
+static void qemu_co_mutex_wake(CoMutex *mutex, Coroutine *co)
 {
     /* Read co before co->ctx; pairs with smp_wmb() in
      * qemu_coroutine_enter().
@@ -197,7 +198,7 @@ static void coroutine_fn qemu_co_mutex_wake(CoMutex *mutex, 
Coroutine *co)
     aio_co_wake(co);
 }
 
-static void coroutine_fn qemu_co_mutex_lock_slowpath(AioContext *ctx,
+static CoroutineFn<void> qemu_co_mutex_lock_slowpath(AioContext *ctx,
                                                      CoMutex *mutex)
 {
     Coroutine *self = qemu_coroutine_self();
@@ -223,17 +224,17 @@ static void coroutine_fn 
qemu_co_mutex_lock_slowpath(AioContext *ctx,
             /* We got the lock ourselves!  */
             assert(to_wake == &w);
             mutex->ctx = ctx;
-            return;
+            co_return;
         }
 
         qemu_co_mutex_wake(mutex, co);
     }
 
-    qemu_coroutine_yield();
+    co_await qemu_coroutine_yield();
     trace_qemu_co_mutex_lock_return(mutex, self);
 }
 
-void coroutine_fn qemu_co_mutex_lock(CoMutex *mutex)
+CoroutineFn<void> qemu_co_mutex_lock(CoMutex *mutex)
 {
     AioContext *ctx = qemu_get_current_aio_context();
     Coroutine *self = qemu_coroutine_self();
@@ -267,13 +268,13 @@ retry_fast_path:
         trace_qemu_co_mutex_lock_uncontended(mutex, self);
         mutex->ctx = ctx;
     } else {
-        qemu_co_mutex_lock_slowpath(ctx, mutex);
+        co_await qemu_co_mutex_lock_slowpath(ctx, mutex);
     }
     mutex->holder = self;
     self->locks_held++;
 }
 
-void coroutine_fn qemu_co_mutex_unlock(CoMutex *mutex)
+CoroutineFn<void> qemu_co_mutex_unlock(CoMutex *mutex)
 {
     Coroutine *self = qemu_coroutine_self();
 
@@ -288,7 +289,7 @@ void coroutine_fn qemu_co_mutex_unlock(CoMutex *mutex)
     self->locks_held--;
     if (qatomic_fetch_dec(&mutex->locked) == 1) {
         /* No waiting qemu_co_mutex_lock().  Pfew, that was easy!  */
-        return;
+        co_return;
     }
 
     for (;;) {
@@ -342,7 +343,7 @@ void qemu_co_rwlock_init(CoRwlock *lock)
 }
 
 /* Releases the intern CoMutex.  */
-static void qemu_co_rwlock_maybe_wake_one(CoRwlock *lock)
+static CoroutineFn<void> qemu_co_rwlock_maybe_wake_one(CoRwlock *lock)
 {
     CoRwTicket *tkt = QSIMPLEQ_FIRST(&lock->tickets);
     Coroutine *co = NULL;
@@ -368,46 +369,46 @@ static void qemu_co_rwlock_maybe_wake_one(CoRwlock *lock)
 
     if (co) {
         QSIMPLEQ_REMOVE_HEAD(&lock->tickets, next);
-        qemu_co_mutex_unlock(&lock->mutex);
+        co_await qemu_co_mutex_unlock(&lock->mutex);
         aio_co_wake(co);
     } else {
-        qemu_co_mutex_unlock(&lock->mutex);
+        co_await qemu_co_mutex_unlock(&lock->mutex);
     }
 }
 
-void qemu_co_rwlock_rdlock(CoRwlock *lock)
+CoroutineFn<void> qemu_co_rwlock_rdlock(CoRwlock *lock)
 {
     Coroutine *self = qemu_coroutine_self();
 
-    qemu_co_mutex_lock(&lock->mutex);
+    co_await qemu_co_mutex_lock(&lock->mutex);
     /* For fairness, wait if a writer is in line.  */
     if (lock->owners == 0 || (lock->owners > 0 && 
QSIMPLEQ_EMPTY(&lock->tickets))) {
         lock->owners++;
-        qemu_co_mutex_unlock(&lock->mutex);
+        co_await qemu_co_mutex_unlock(&lock->mutex);
     } else {
         CoRwTicket my_ticket = { true, self };
 
         QSIMPLEQ_INSERT_TAIL(&lock->tickets, &my_ticket, next);
-        qemu_co_mutex_unlock(&lock->mutex);
-        qemu_coroutine_yield();
+        co_await qemu_co_mutex_unlock(&lock->mutex);
+        co_await qemu_coroutine_yield();
         assert(lock->owners >= 1);
 
         /* Possibly wake another reader, which will wake the next in line.  */
-        qemu_co_mutex_lock(&lock->mutex);
-        qemu_co_rwlock_maybe_wake_one(lock);
+        co_await qemu_co_mutex_lock(&lock->mutex);
+        co_await qemu_co_rwlock_maybe_wake_one(lock);
     }
 
     self->locks_held++;
 }
 
-void qemu_co_rwlock_unlock(CoRwlock *lock)
+CoroutineFn<void> qemu_co_rwlock_unlock(CoRwlock *lock)
 {
     Coroutine *self = qemu_coroutine_self();
 
     assert(qemu_in_coroutine());
     self->locks_held--;
 
-    qemu_co_mutex_lock(&lock->mutex);
+    co_await qemu_co_mutex_lock(&lock->mutex);
     if (lock->owners > 0) {
         lock->owners--;
     } else {
@@ -415,55 +416,54 @@ void qemu_co_rwlock_unlock(CoRwlock *lock)
         lock->owners = 0;
     }
 
-    qemu_co_rwlock_maybe_wake_one(lock);
+    co_await qemu_co_rwlock_maybe_wake_one(lock);
 }
 
-void qemu_co_rwlock_downgrade(CoRwlock *lock)
+CoroutineFn<void> qemu_co_rwlock_downgrade(CoRwlock *lock)
 {
-    qemu_co_mutex_lock(&lock->mutex);
+    co_await qemu_co_mutex_lock(&lock->mutex);
     assert(lock->owners == -1);
     lock->owners = 1;
 
     /* Possibly wake another reader, which will wake the next in line.  */
-    qemu_co_rwlock_maybe_wake_one(lock);
+    co_await qemu_co_rwlock_maybe_wake_one(lock);
 }
 
-void qemu_co_rwlock_wrlock(CoRwlock *lock)
+CoroutineFn<void> qemu_co_rwlock_wrlock(CoRwlock *lock)
 {
     Coroutine *self = qemu_coroutine_self();
 
-    qemu_co_mutex_lock(&lock->mutex);
+    co_await qemu_co_mutex_lock(&lock->mutex);
     if (lock->owners == 0) {
         lock->owners = -1;
-        qemu_co_mutex_unlock(&lock->mutex);
+        co_await qemu_co_mutex_unlock(&lock->mutex);
     } else {
         CoRwTicket my_ticket = { false, self };
 
         QSIMPLEQ_INSERT_TAIL(&lock->tickets, &my_ticket, next);
-        qemu_co_mutex_unlock(&lock->mutex);
-        qemu_coroutine_yield();
+        co_await qemu_co_mutex_unlock(&lock->mutex);
+        co_await qemu_coroutine_yield();
         assert(lock->owners == -1);
     }
 
     self->locks_held++;
 }
 
-void qemu_co_rwlock_upgrade(CoRwlock *lock)
+CoroutineFn<void> qemu_co_rwlock_upgrade(CoRwlock *lock)
 {
-    qemu_co_mutex_lock(&lock->mutex);
+    co_await qemu_co_mutex_lock(&lock->mutex);
     assert(lock->owners > 0);
     /* For fairness, wait if a writer is in line.  */
     if (lock->owners == 1 && QSIMPLEQ_EMPTY(&lock->tickets)) {
         lock->owners = -1;
-        qemu_co_mutex_unlock(&lock->mutex);
+        co_await qemu_co_mutex_unlock(&lock->mutex);
     } else {
         CoRwTicket my_ticket = { false, qemu_coroutine_self() };
 
         lock->owners--;
         QSIMPLEQ_INSERT_TAIL(&lock->tickets, &my_ticket, next);
-        qemu_co_rwlock_maybe_wake_one(lock);
-        qemu_coroutine_yield();
+        co_await qemu_co_rwlock_maybe_wake_one(lock);
+        co_await qemu_coroutine_yield();
         assert(lock->owners == -1);
     }
 }
-#endif
diff --git a/util/qemu-coroutine-sleep.c b/util/qemu-coroutine-sleep.cc
similarity index 89%
rename from util/qemu-coroutine-sleep.c
rename to util/qemu-coroutine-sleep.cc
index b5bfb4ad18..8bb8d91109 100644
--- a/util/qemu-coroutine-sleep.c
+++ b/util/qemu-coroutine-sleep.cc
@@ -17,7 +17,6 @@
 #include "qemu/timer.h"
 #include "block/aio.h"
 
-#if 0
 static const char *qemu_co_sleep_ns__scheduled = "qemu_co_sleep_ns";
 
 void qemu_co_sleep_wake(QemuCoSleep *w)
@@ -38,11 +37,11 @@ void qemu_co_sleep_wake(QemuCoSleep *w)
 
 static void co_sleep_cb(void *opaque)
 {
-    QemuCoSleep *w = opaque;
+    QemuCoSleep *w = (QemuCoSleep *)opaque;
     qemu_co_sleep_wake(w);
 }
 
-void coroutine_fn qemu_co_sleep(QemuCoSleep *w)
+CoroutineFn<void> qemu_co_sleep(QemuCoSleep *w)
 {
     Coroutine *co = qemu_coroutine_self();
 
@@ -56,13 +55,13 @@ void coroutine_fn qemu_co_sleep(QemuCoSleep *w)
     }
 
     w->to_wake = co;
-    qemu_coroutine_yield();
+    co_await qemu_coroutine_yield();
 
     /* w->to_wake is cleared before resuming this coroutine.  */
     assert(w->to_wake == NULL);
 }
 
-void coroutine_fn qemu_co_sleep_ns_wakeable(QemuCoSleep *w,
+CoroutineFn<void> qemu_co_sleep_ns_wakeable(QemuCoSleep *w,
                                             QEMUClockType type, int64_t ns)
 {
     AioContext *ctx = qemu_get_current_aio_context();
@@ -76,7 +75,6 @@ void coroutine_fn qemu_co_sleep_ns_wakeable(QemuCoSleep *w,
      * must happen after qemu_co_sleep yields and there is no race
      * between timer_mod and qemu_co_sleep.
      */
-    qemu_co_sleep(w);
+    co_await qemu_co_sleep(w);
     timer_del(&ts);
 }
-#endif
diff --git a/util/qemu-coroutine.c b/util/qemu-coroutine.cc
similarity index 94%
rename from util/qemu-coroutine.c
rename to util/qemu-coroutine.cc
index 9f2bd96fa0..0ae2a4090f 100644
--- a/util/qemu-coroutine.c
+++ b/util/qemu-coroutine.cc
@@ -179,22 +179,6 @@ void qemu_coroutine_enter_if_inactive(Coroutine *co)
     }
 }
 
-void coroutine_fn qemu_coroutine_yield(void)
-{
-    Coroutine *self = qemu_coroutine_self();
-    Coroutine *to = self->caller;
-
-    trace_qemu_coroutine_yield(self, to);
-
-    if (!to) {
-        fprintf(stderr, "Co-routine is yielding to no one\n");
-        abort();
-    }
-
-    self->caller = NULL;
-    qemu_coroutine_switch(self, to, COROUTINE_YIELD);
-}
-
 bool qemu_coroutine_entered(Coroutine *co)
 {
     return co->caller;
-- 
2.35.1