[GNUnet-SVN] [gnurl] 102/153: upload: allocate upload buffer on-demand

[gnunet]

This is an automated email from the git hooks/post-receive script.

ng0 pushed a commit to branch master
in repository gnurl.

commit e6e9b006f770ef104fbcdef32dd6e7f42eb114b7
Author: Daniel Stenberg <address@hidden>
AuthorDate: Fri Aug 17 00:49:37 2018 +0200

    upload: allocate upload buffer on-demand
    
    Saves 16KB on the easy handle for operations that don't need that
    buffer.
    
    Part 1 of #2888
---
 lib/http.c     | 29 ++++++++++++++++++-----------
 lib/multi.c    |  3 +++
 lib/smb.c      | 21 ++++++++++++++-------
 lib/transfer.c | 20 ++++++++++++++++++--
 lib/transfer.h |  1 +
 lib/url.c      |  5 ++---
 lib/urldata.h  |  4 +++-
 7 files changed, 59 insertions(+), 24 deletions(-)

diff --git a/lib/http.c b/lib/http.c
index 2e9f40719..e727ed823 100644
--- a/lib/http.c
+++ b/lib/http.c
@@ -1123,7 +1123,8 @@ CURLcode Curl_add_buffer_send(Curl_send_buffer *in,
   CURLcode result;
   char *ptr;
   size_t size;
-  struct HTTP *http = conn->data->req.protop;
+  struct Curl_easy *data = conn->data;
+  struct HTTP *http = data->req.protop;
   size_t sendsize;
   curl_socket_t sockfd;
   size_t headersize;
@@ -1143,7 +1144,7 @@ CURLcode Curl_add_buffer_send(Curl_send_buffer *in,
 
   DEBUGASSERT(size > included_body_bytes);
 
-  result = Curl_convert_to_network(conn->data, ptr, headersize);
+  result = Curl_convert_to_network(data, ptr, headersize);
   /* Curl_convert_to_network calls failf if unsuccessful */
   if(result) {
     /* conversion failed, free memory and return to the caller */
@@ -1168,8 +1169,14 @@ CURLcode Curl_add_buffer_send(Curl_send_buffer *in,
        must copy the data to the uploadbuffer first, since that is the buffer
        we will be using if this send is retried later.
     */
-    memcpy(conn->data->state.uploadbuffer, ptr, sendsize);
-    ptr = conn->data->state.uploadbuffer;
+    result = Curl_get_upload_buffer(data);
+    if(result) {
+      /* malloc failed, free memory and return to the caller */
+      Curl_add_buffer_free(in);
+      return result;
+    }
+    memcpy(data->state.ulbuf, ptr, sendsize);
+    ptr = data->state.ulbuf;
   }
   else
     sendsize = size;
@@ -1186,13 +1193,13 @@ CURLcode Curl_add_buffer_send(Curl_send_buffer *in,
     size_t headlen = (size_t)amount>headersize ? headersize : (size_t)amount;
     size_t bodylen = amount - headlen;
 
-    if(conn->data->set.verbose) {
+    if(data->set.verbose) {
       /* this data _may_ contain binary stuff */
-      Curl_debug(conn->data, CURLINFO_HEADER_OUT, ptr, headlen);
+      Curl_debug(data, CURLINFO_HEADER_OUT, ptr, headlen);
       if(bodylen) {
         /* there was body data sent beyond the initial header part, pass that
            on to the debug callback too */
-        Curl_debug(conn->data, CURLINFO_DATA_OUT,
+        Curl_debug(data, CURLINFO_DATA_OUT,
                    ptr + headlen, bodylen);
       }
     }
@@ -1217,14 +1224,14 @@ CURLcode Curl_add_buffer_send(Curl_send_buffer *in,
         ptr = in->buffer + amount;
 
         /* backup the currently set pointers */
-        http->backup.fread_func = conn->data->state.fread_func;
-        http->backup.fread_in = conn->data->state.in;
+        http->backup.fread_func = data->state.fread_func;
+        http->backup.fread_in = data->state.in;
         http->backup.postdata = http->postdata;
         http->backup.postsize = http->postsize;
 
         /* set the new pointers for the request-sending */
-        conn->data->state.fread_func = (curl_read_callback)readmoredata;
-        conn->data->state.in = (void *)conn;
+        data->state.fread_func = (curl_read_callback)readmoredata;
+        data->state.in = (void *)conn;
         http->postdata = ptr;
         http->postsize = (curl_off_t)size;
 
diff --git a/lib/multi.c b/lib/multi.c
index c2c5e1c27..0caf94322 100644
--- a/lib/multi.c
+++ b/lib/multi.c
@@ -592,6 +592,7 @@ static CURLcode multi_done(struct connectdata **connp,
     conn->dns_entry = NULL;
   }
   Curl_hostcache_prune(data);
+  Curl_safefree(data->state.ulbuf);
 
   /* if the transfer was completed in a paused state there can be buffered
      data left to free */
@@ -1575,6 +1576,8 @@ static CURLMcode multi_runsingle(struct Curl_multi *multi,
           multistate(data, CURLM_STATE_SENDPROTOCONNECT);
         }
       }
+      else if(result)
+        stream_error = TRUE;
       break;
 #endif
 
diff --git a/lib/smb.c b/lib/smb.c
index 09aa8efc7..32f0ac889 100644
--- a/lib/smb.c
+++ b/lib/smb.c
@@ -367,7 +367,7 @@ static CURLcode smb_send(struct connectdata *conn, ssize_t 
len,
   ssize_t bytes_written;
   CURLcode result;
 
-  result = Curl_write(conn, FIRSTSOCKET, conn->data->state.uploadbuffer,
+  result = Curl_write(conn, FIRSTSOCKET, conn->data->state.ulbuf,
                       len, &bytes_written);
   if(result)
     return result;
@@ -393,7 +393,7 @@ static CURLcode smb_flush(struct connectdata *conn)
     return CURLE_OK;
 
   result = Curl_write(conn, FIRSTSOCKET,
-                      conn->data->state.uploadbuffer + smbc->sent,
+                      conn->data->state.ulbuf + smbc->sent,
                       len, &bytes_written);
   if(result)
     return result;
@@ -409,9 +409,12 @@ static CURLcode smb_flush(struct connectdata *conn)
 static CURLcode smb_send_message(struct connectdata *conn, unsigned char cmd,
                                  const void *msg, size_t msg_len)
 {
-  smb_format_message(conn, (struct smb_header *)conn->data->state.uploadbuffer,
+  CURLcode result = Curl_get_upload_buffer(conn->data);
+  if(result)
+    return result;
+  smb_format_message(conn, (struct smb_header *)conn->data->state.ulbuf,
                      cmd, msg_len);
-  memcpy(conn->data->state.uploadbuffer + sizeof(struct smb_header),
+  memcpy(conn->data->state.ulbuf + sizeof(struct smb_header),
          msg, msg_len);
 
   return smb_send(conn, sizeof(struct smb_header) + msg_len, 0);
@@ -572,11 +575,15 @@ static CURLcode smb_send_read(struct connectdata *conn)
 
 static CURLcode smb_send_write(struct connectdata *conn)
 {
-  struct smb_write *msg = (struct smb_write *)conn->data->state.uploadbuffer;
+  struct smb_write *msg;
   struct smb_request *req = conn->data->req.protop;
   curl_off_t offset = conn->data->req.offset;
-
   curl_off_t upload_size = conn->data->req.size - conn->data->req.bytecount;
+  CURLcode result = Curl_get_upload_buffer(conn->data);
+  if(result)
+    return result;
+  msg = (struct smb_write *)conn->data->state.ulbuf;
+
   if(upload_size >= MAX_PAYLOAD_SIZE - 1) /* There is one byte of padding */
     upload_size = MAX_PAYLOAD_SIZE - 1;
 
@@ -605,7 +612,7 @@ static CURLcode smb_send_and_recv(struct connectdata *conn, 
void **msg)
   if(!smbc->send_size && smbc->upload_size) {
     int nread = smbc->upload_size > UPLOAD_BUFSIZE ? UPLOAD_BUFSIZE :
       (int) smbc->upload_size;
-    conn->data->req.upload_fromhere = conn->data->state.uploadbuffer;
+    conn->data->req.upload_fromhere = conn->data->state.ulbuf;
     result = Curl_fillreadbuffer(conn, nread, &nread);
     if(result && result != CURLE_AGAIN)
       return result;
diff --git a/lib/transfer.c b/lib/transfer.c
index 49f32568f..96d8ab48f 100644
--- a/lib/transfer.c
+++ b/lib/transfer.c
@@ -106,6 +106,16 @@ char *Curl_checkheaders(const struct connectdata *conn,
 }
 #endif
 
+CURLcode Curl_get_upload_buffer(struct Curl_easy *data)
+{
+  if(!data->state.ulbuf) {
+    data->state.ulbuf = malloc(data->set.upload_buffer_size);
+    if(!data->state.ulbuf)
+      return CURLE_OUT_OF_MEMORY;
+  }
+  return CURLE_OK;
+}
+
 /*
  * This function will call the read callback to fill our buffer with data
  * to upload.
@@ -914,8 +924,11 @@ static CURLcode readwrite_upload(struct Curl_easy *data,
     /* only read more data if there's no upload data already
        present in the upload buffer */
     if(0 == k->upload_present) {
+      result = Curl_get_upload_buffer(data);
+      if(result)
+        return result;
       /* init the "upload from here" pointer */
-      k->upload_fromhere = data->state.uploadbuffer;
+      k->upload_fromhere = data->state.ulbuf;
 
       if(!k->upload_done) {
         /* HTTP pollution, this should be written nicer to become more
@@ -1071,7 +1084,10 @@ static CURLcode readwrite_upload(struct Curl_easy *data,
     }
     else {
       /* we've uploaded that buffer now */
-      k->upload_fromhere = data->state.uploadbuffer;
+      result = Curl_get_upload_buffer(data);
+      if(result)
+        return result;
+      k->upload_fromhere = data->state.ulbuf;
       k->upload_present = 0; /* no more bytes left */
 
       if(k->upload_done) {
diff --git a/lib/transfer.h b/lib/transfer.h
index 9ba398d27..df75f9a97 100644
--- a/lib/transfer.h
+++ b/lib/transfer.h
@@ -54,6 +54,7 @@ CURLcode Curl_readrewind(struct connectdata *conn);
 CURLcode Curl_fillreadbuffer(struct connectdata *conn, int bytes, int *nreadp);
 CURLcode Curl_retry_request(struct connectdata *conn, char **url);
 bool Curl_meets_timecondition(struct Curl_easy *data, time_t timeofdoc);
+CURLcode Curl_get_upload_buffer(struct Curl_easy *data);
 
 /* This sets up a forthcoming transfer */
 void
diff --git a/lib/url.c b/lib/url.c
index dcb7fcf6c..12f0f8ae7 100644
--- a/lib/url.c
+++ b/lib/url.c
@@ -367,11 +367,9 @@ CURLcode Curl_close(struct Curl_easy *data)
 
   Curl_safefree(data->state.buffer);
   Curl_safefree(data->state.headerbuff);
-
+  Curl_safefree(data->state.ulbuf);
   Curl_flush_cookies(data, 1);
-
   Curl_digest_cleanup(data);
-
   Curl_safefree(data->info.contenttype);
   Curl_safefree(data->info.wouldredirect);
 
@@ -534,6 +532,7 @@ CURLcode Curl_init_userdefined(struct Curl_easy *data)
   set->expect_100_timeout = 1000L; /* Wait for a second by default. */
   set->sep_headers = TRUE; /* separated header lists by default */
   set->buffer_size = READBUFFER_SIZE;
+  set->upload_buffer_size = UPLOAD_BUFSIZE;
   set->happy_eyeballs_timeout = CURL_HET_DEFAULT;
 
   Curl_http2_init_userset(set);
diff --git a/lib/urldata.h b/lib/urldata.h
index 6119136d4..502cff362 100644
--- a/lib/urldata.h
+++ b/lib/urldata.h
@@ -1220,7 +1220,7 @@ struct UrlState {
   size_t headersize;   /* size of the allocation */
 
   char *buffer; /* download buffer */
-  char uploadbuffer[UPLOAD_BUFSIZE + 1]; /* upload buffer */
+  char *ulbuf; /* alloced upload buffer or NULL */
   curl_off_t current_speed;  /* the ProgressShow() function sets this,
                                 bytes / second */
   bool this_is_a_follow; /* this is a followed Location: request */
@@ -1557,6 +1557,8 @@ struct UserDefined {
   curl_proxytype proxytype; /* what kind of proxy that is in use */
   long dns_cache_timeout; /* DNS cache timeout */
   long buffer_size;      /* size of receive buffer to use */
+  long upload_buffer_size; /* size of upload buffer to use,
+                              keep it >= CURL_MAX_WRITE_SIZE */
   void *private_data; /* application-private data */
 
   struct curl_slist *http200aliases; /* linked list of aliases for http200 */

-- 
To stop receiving notification emails like this one, please contact
address@hidden