[Qemu-devel] [PATCH 11/13] net: add qemu_send_packet_async()

[Mark McLoughlin]

Add a qemu_send_packet() variant which will queue up the packet
if it cannot be sent when all client queues are full. It later
invokes the supplied callback when the packet has been sent.

If qemu_send_packet_async() returns zero, the caller is expected
to not send any more packets until the queued packet has been
sent.

Packets are queued iff a receive() handler returns zero (indicating
queue full) and the caller has provided a sent notification callback
(indicating it will stop and start its own queue).

We need the packet sending API to support queueing because:

  - a sending client should process all available packets in one go
    (e.g. virtio-net emptying its tx ring)

  - a receiving client may not be able to handle the packet
    (e.g. -EAGAIN from write() to tapfd)

  - the sending client could detect this condition in advance
    (e.g. by select() for writable on tapfd)

  - that's too much overhead (e.g. a select() call per packet)

  - therefore the sending client must handle the condition by
    dropping the packet or queueing it

  - dropping packets is poor form; we should queue.

However, we don't want queueing to be completely transparent. We
want the sending client to stop sending packets as soon as a
packet is queued. This allows the sending client to be throttled
by the receiver.

Signed-off-by: Mark McLoughlin <address@hidden>
---
 net.c |   83 ++++++++++++++++++++++++++++++++++++++++++++++++----------------
 net.h |    8 ++++++
 2 files changed, 70 insertions(+), 21 deletions(-)

diff --git a/net.c b/net.c
index 2f4743e..3675bc5 100644
--- a/net.c
+++ b/net.c
@@ -439,19 +439,32 @@ qemu_deliver_packet(VLANClientState *sender, const 
uint8_t *buf, int size)
     return ret;
 }
 
-static void qemu_flush_queued_packets(VLANClientState *vc)
+void qemu_flush_queued_packets(VLANClientState *vc)
 {
     VLANPacket *packet;
 
     while ((packet = vc->vlan->send_queue) != NULL) {
+        int ret;
+
         vc->vlan->send_queue = packet->next;
-        qemu_deliver_packet(packet->sender, packet->data, packet->size);
+
+        ret = qemu_deliver_packet(packet->sender, packet->data, packet->size);
+        if (ret == 0 && packet->sent_cb != NULL) {
+            packet->next = vc->vlan->send_queue;
+            vc->vlan->send_queue = packet;
+            break;
+        }
+
+        if (packet->sent_cb)
+            packet->sent_cb(packet->sender);
+
         qemu_free(packet);
     }
 }
 
-static void
-qemu_enqueue_packet(VLANClientState *sender, const uint8_t *buf, int size)
+static void qemu_enqueue_packet(VLANClientState *sender,
+                                const uint8_t *buf, int size,
+                                NetPacketSent *sent_cb)
 {
     VLANPacket *packet;
 
@@ -459,28 +472,45 @@ qemu_enqueue_packet(VLANClientState *sender, const 
uint8_t *buf, int size)
     packet->next = sender->vlan->send_queue;
     packet->sender = sender;
     packet->size = size;
+    packet->sent_cb = sent_cb;
     memcpy(packet->data, buf, size);
     sender->vlan->send_queue = packet;
 }
 
-void qemu_send_packet(VLANClientState *vc, const uint8_t *buf, int size)
+ssize_t qemu_send_packet_async(VLANClientState *sender,
+                               const uint8_t *buf, int size,
+                               NetPacketSent *sent_cb)
 {
-    VLANState *vlan = vc->vlan;
+    int ret;
 
-    if (vc->link_down)
-        return;
+    if (sender->link_down) {
+        return size;
+    }
 
 #ifdef DEBUG_NET
-    printf("vlan %d send:\n", vlan->id);
+    printf("vlan %d send:\n", sender->vlan->id);
     hex_dump(stdout, buf, size);
 #endif
-    if (vlan->delivering) {
-        qemu_enqueue_packet(vc, buf, size);
-        return;
+
+    if (sender->vlan->delivering) {
+        qemu_enqueue_packet(sender, buf, size, NULL);
+        return size;
     }
 
-    qemu_deliver_packet(vc, buf, size);
-    qemu_flush_queued_packets(vc);
+    ret = qemu_deliver_packet(sender, buf, size);
+    if (ret == 0 && sent_cb != NULL) {
+        qemu_enqueue_packet(sender, buf, size, sent_cb);
+        return 0;
+    }
+
+    qemu_flush_queued_packets(sender);
+
+    return ret;
+}
+
+void qemu_send_packet(VLANClientState *vc, const uint8_t *buf, int size)
+{
+    qemu_send_packet_async(vc, buf, size, NULL);
 }
 
 static ssize_t vc_sendv_compat(VLANClientState *vc, const struct iovec *iov,
@@ -498,9 +528,7 @@ static ssize_t vc_sendv_compat(VLANClientState *vc, const 
struct iovec *iov,
         offset += len;
     }
 
-    vc->receive(vc, buffer, offset);
-
-    return offset;
+    return vc->receive(vc, buffer, offset);
 }
 
 static ssize_t calc_iov_length(const struct iovec *iov, int iovcnt)
@@ -548,7 +576,8 @@ static int qemu_deliver_packet_iov(VLANClientState *sender,
 }
 
 static ssize_t qemu_enqueue_packet_iov(VLANClientState *sender,
-                                       const struct iovec *iov, int iovcnt)
+                                       const struct iovec *iov, int iovcnt,
+                                       NetPacketSent *sent_cb)
 {
     VLANPacket *packet;
     size_t max_len = 0;
@@ -559,6 +588,7 @@ static ssize_t qemu_enqueue_packet_iov(VLANClientState 
*sender,
     packet = qemu_malloc(sizeof(VLANPacket) + max_len);
     packet->next = sender->vlan->send_queue;
     packet->sender = sender;
+    packet->sent_cb = sent_cb;
     packet->size = 0;
 
     for (i = 0; i < iovcnt; i++) {
@@ -573,8 +603,9 @@ static ssize_t qemu_enqueue_packet_iov(VLANClientState 
*sender,
     return packet->size;
 }
 
-ssize_t qemu_sendv_packet(VLANClientState *sender, const struct iovec *iov,
-                          int iovcnt)
+ssize_t qemu_sendv_packet_async(VLANClientState *sender,
+                                const struct iovec *iov, int iovcnt,
+                                NetPacketSent *sent_cb)
 {
     int ret;
 
@@ -583,16 +614,26 @@ ssize_t qemu_sendv_packet(VLANClientState *sender, const 
struct iovec *iov,
     }
 
     if (sender->vlan->delivering) {
-        return qemu_enqueue_packet_iov(sender, iov, iovcnt);
+        return qemu_enqueue_packet_iov(sender, iov, iovcnt, NULL);
     }
 
     ret = qemu_deliver_packet_iov(sender, iov, iovcnt);
+    if (ret == 0 && sent_cb != NULL) {
+        qemu_enqueue_packet_iov(sender, iov, iovcnt, sent_cb);
+        return 0;
+    }
 
     qemu_flush_queued_packets(sender);
 
     return ret;
 }
 
+ssize_t
+qemu_sendv_packet(VLANClientState *vc, const struct iovec *iov, int iovcnt)
+{
+    return qemu_sendv_packet_async(vc, iov, iovcnt, NULL);
+}
+
 #if defined(CONFIG_SLIRP)
 
 /* slirp network adapter */
diff --git a/net.h b/net.h
index 766edbe..82ca255 100644
--- a/net.h
+++ b/net.h
@@ -32,10 +32,13 @@ struct VLANClientState {
 
 typedef struct VLANPacket VLANPacket;
 
+typedef void (NetPacketSent) (VLANClientState *);
+
 struct VLANPacket {
     struct VLANPacket *next;
     VLANClientState *sender;
     int size;
+    NetPacketSent *sent_cb;
     uint8_t data[0];
 };
 
@@ -62,7 +65,12 @@ VLANClientState *qemu_find_vlan_client(VLANState *vlan, void 
*opaque);
 int qemu_can_send_packet(VLANClientState *vc);
 ssize_t qemu_sendv_packet(VLANClientState *vc, const struct iovec *iov,
                           int iovcnt);
+ssize_t qemu_sendv_packet_async(VLANClientState *vc, const struct iovec *iov,
+                                int iovcnt, NetPacketSent *sent_cb);
 void qemu_send_packet(VLANClientState *vc, const uint8_t *buf, int size);
+ssize_t qemu_send_packet_async(VLANClientState *vc, const uint8_t *buf,
+                               int size, NetPacketSent *sent_cb);
+void qemu_flush_queued_packets(VLANClientState *vc);
 void qemu_format_nic_info_str(VLANClientState *vc, uint8_t macaddr[6]);
 void qemu_check_nic_model(NICInfo *nd, const char *model);
 void qemu_check_nic_model_list(NICInfo *nd, const char * const *models,
-- 
1.6.0.6