[libmicrohttpd] Re: libmicrohttpd - question on tutorial

[Christian Grothoff]

Now, this really looks like a problem in libc to me -- MHD calls sendfile using

off_t off = 0;
sendfile (5, 6, &off, 72313) 

and strace reports

sendfile64(5, 6, [617678926377910272], 72313) = -1 EOVERFLOW 

so clearly the translation into a system call by libc is not right...

I've found this online for protfpd:

- Bug 3003 - Fallback to normal transmission in case of sendfile EOVERFLOW
  error missing.

http://bugs.proftpd.org/show_bug.cgi?id=3003

And something similar for Apache:

http://bugs.debian.org/cgi-bin/bugreport.cgi?bug=494768

So this seems to be a difficult libc/kernel/file-system interaction problem.  
Now, for most of those reports, the strace reports do not contain a "strange" 
offset value but simply [0], but in any case clearly sendfile can fail in odd 
ways and we should recover.

As a workaround for you for now, you can try
1) kernel update
2) libc update
3) not use MHD_create_response_from_fd_at_offset but some other method 
   (which will then not end up using sendfile)

In any case, I'll add code to MHD to fall back to standard reading and writing 
in the case where 'sendfile' returns an "odd" error.

Happy hacking,

Christian

On Thursday 17 February 2011 18:41:38 Neil D'Souza wrote:
> I modified daemon.c with context printed below from line 719 - rebuilt the
> library and re-installed.
> 
> 
> Checked timestamp of created file in /usr/local/lib - as below - which
> matches the time on my pc.
> -rw-r--r-- 1 root root   254516 2011-02-17 22:55 libmicrohttpd.a
> -rwxr-xr-x 1 root root     1093 2011-02-17 22:55 libmicrohttpd.la
> lrwxrwxrwx 1 root root       23 2011-02-17 22:55 libmicrohttpd.so ->
> libmicrohttpd.so.10.5.0
> lrwxrwxrwx 1 root root       23 2011-02-17 22:55 libmicrohttpd.so.10 ->
> libmicrohttpd.so.10.5.0
> -------------------------
> daemon.c context from line 719
> -------------
> if ( (connection->write_buffer_append_offset ==
>         connection->write_buffer_send_offset) &&
>        (NULL != connection->response) &&
>        (-1 != (fd = connection->response->fd)) )
>     {
>       /* can use sendfile */
>         fprintf (stderr, "offset: %llu, size: %llu, delta: %llu\n",
>                         (unsigned long long) offset,
>                         (unsigned long long)
> connection->response->total_size, (unsigned long long)
> (connection->response->total_size - offset));
>       offset = (off_t) connection->response_write_position +
> connection->response->fd_off;
>       ret = sendfile (connection->socket_fd,
>                       fd,
>                       &offset,
>                       connection->response->total_size - offset);
> 
> 
> -------------------------
> end of context from line daemon.c
> ---------------
> 
> below is the response from the server once I try to connect from the
> browser. I dont think that when I highlight anything in the yahoo using
> the html highlighter  - it arrives at your end so instead i will mark the
> interesting lines with ">>"
> -----------------------
> [pid 10922] select(5, [4], [], [], {1, 0}) = 1 (in [4], left {0, 919215})
> [pid 10922] accept(4, {sa_family=AF_INET, sin_port=htons(39599),
> sin_addr=inet_addr("127.0.0.1")}, [16]) = 5
> [pid 10922] time(NULL)                  = 1297963621
> [pid 10922] mmap2(NULL, 32768, PROT_READ|PROT_WRITE,
> MAP_FILE|MAP_ANONYMOUS, -1, 0) = -1 EINVAL (Invalid argument)
> [pid 10922] select(6, [4 5], [], [], {1, 0}) = 1 (in [5], left {0, 999995})
> [pid 10922] time(NULL)                  = 1297963621
> [pid 10922] recv(5, "GET / HTTP/1.1\r\nHost: localhost:"..., 2048,
> MSG_DONTWAIT|MSG_NOSIGNAL) = 383
> [pid 10922] fstat64(1, {st_mode=S_IFCHR|0620, st_rdev=makedev(136, 1),
> ...}) = 0 [pid 10922] mmap2(NULL, 4096, PROT_READ|PROT_WRITE,
> MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb78d3000
> [pid 10922] write(1, "New GET request for / using vers"..., 45New GET
> request for / using version HTTP/1.1
> ) = 45
> [pid 10922] open("after-the-rain.jpg", O_RDONLY) = 6
> [pid 10922] fstat64(6, {st_mode=S_IFREG|0777, st_size=72313, ...}) = 0
> [pid 10922] shutdown(5, 0 /* receive */) = 0
> [pid 10922] write(1, "queued response\n", 16queued response
> ) = 16
> [pid 10922] time(NULL)                  = 1297963621
> [pid 10922] open("/etc/localtime", O_RDONLY) = 7
> [pid 10922] fstat64(7, {st_mode=S_IFREG|0644, st_size=265, ...}) = 0
> [pid 10922] fstat64(7, {st_mode=S_IFREG|0644, st_size=265, ...}) = 0
> [pid 10922] mmap2(NULL, 4096, PROT_READ|PROT_WRITE,
> MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb78d2000
> [pid 10922] read(7,
> "TZif2\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\4\0\0\0\4\0\0\0\0"..., 4096) =
> 265 [pid 10922] _llseek(7, -10, [255], SEEK_CUR) = 0
> [pid 10922] read(7, "\nIST-5:30\n", 4096) = 10
> [pid 10922] close(7)                    = 0
> [pid 10922] munmap(0xb78d2000, 4096)    = 0
> [pid 10922] setsockopt(5, SOL_TCP, TCP_CORK, [1], 4) = 0
> [pid 10922] select(6, [4], [5], [], {1, 0}) = 1 (out [5], left {0, 999996})
> [pid 10922] time(NULL)                  = 1297963621
> [pid 10922] send(5, "HTTP/1.1 200 OK\r\nContent-Length:"..., 104,
> MSG_DONTWAIT|MSG_NOSIGNAL) = 104
> [pid 10922] select(6, [4], [5], [], {1, 0}) = 1 (out [5], left {0, 999996})
> [pid 10922] time(NULL)                  = 1297963621
> 
> >>[pid 10922] write(2, "offset: 0, size: 72313, delta: 7"..., 37offset: 0,
> >>size: 72313, delta: 72313
> 
> ) = 37
> 
> >>[pid 10922] sendfile64(5, 6, [617678926377910272], 72313) = -1 EOVERFLOW
> >>(Value too large for defined data type)
> 
> [pid 10922] shutdown(5, 2 /* send and receive */) = 0
> [pid 10922] close(5)                    = 0
> [pid 10922] close(6)                    = 0
> [pid 10922] select(5, [4], [], [], {1, 0}) = 0 (Timeout)
> 
> Thanks V Much for your support.
> 
> Regards,
> Neil
> 
> 
> 
> ________________________________
> From: Christian Grothoff <address@hidden>
> To: Neil D'Souza <address@hidden>
> Cc: address@hidden
> Sent: Thu, February 17, 2011 10:45:26 PM
> Subject: Re: libmicrohttpd - question on tutorial
> 
> sendfile64 here refers to 64-bit 'off_t' ("large files"), not to 64-bit
> 'size_t'
> 
> (heap/pointer size).   Still, this could be related to 32 vs. 64-bit
> issues.
> 
> Based on what I've read so far, MHD's calculation of the length argument
> ("connection->response->total_size - offset") would not seem to be
> correct/working for files larger than 2^32 (and the 'ret' value would be
> insufficient for values > 2^31, as I've seen in some discussions on
> sendfile online now).  The solution to this will be to cap the length
> argument at 2 GB, I'll probably take care of that later once we understand
> the real issue here better:
> 
> Since  your file is not larger than 2 GB, this "minor" issue does NOT
> explain your problem.  Could you add
> 
> fprintf (stderr, "offset: %llu, size: %llu, delta: %llu\n",
>           (unsigned long long) offset,
>          (unsigned long long) connection->response->total_size,
>       (unsigned long long) (connection->response->total_size - offset));
> 
> on the line before the 'sendfile' call?  That way, we can see if the odd
> length argument comes from within MHD (=> MHD bug) or from libc's handling
> of the arguments (=> libc bug).
> 
> Thanks!
> 
> Christian
> 
> On Thursday 17 February 2011 17:27:14 Neil D'Souza wrote:
> > I think I have the culprit. Pasted below is the output with the offending
> > linehighlighted. But I do not know why this is happening? I was using an
> > image file about 1.3 mb earlier - I changed it to a file 72 kb - just to
> > check that there are no internal restrictions on the file size which have
> > to be changed by some config parameter. It has failed again for a 72 kb
> > file in the sendfile64 call.
> > 
> > I quickly ran this check to verify that my system is 32 bit
> > 
> > #include <iostream>
> > 
> > using namespace std;
> > 
> > int main()
> > {
> > 
> >     char * ptr;
> >     cout << "sizeof(ptr): " << sizeof(ptr) << endl;
> > 
> > }
> > 
> > 
> > /media/sda3/home/nxd/Prog/C/microhttpd/tutorial>./sizeof_ptr
> > sizeof(ptr): 4
> > 
> > I find it odd that the kernel is calling sendfile64 when the system is
> > 32bit - because sendfile64 shows up in the man page, however my knowledge
> > of the linux kernel is close to 0. So I should not hazard any guesses.
> > 
> > 
> > 
> > pid  5695] select(5, [4], [], [], {1, 0}) = 0 (Timeout)
> > [pid  5695] select(5, [4], [], [], {1, 0}) = 0 (Timeout)
> > [pid  5695] select(5, [4], [], [], {1, 0}) = 0 (Timeout)
> > [pid  5695] select(5, [4], [], [], {1, 0}) = 0 (Timeout)
> > [pid  5695] select(5, [4], [], [], {1, 0}) = 0 (Timeout)
> > [pid  5695] select(5, [4], [], [], {1, 0}) = 1 (in [4], left {0, 564047})
> > [pid  5695] accept(4, {sa_family=AF_INET, sin_port=htons(49723),
> > sin_addr=inet_addr("127.0.0.1")}, [16]) = 5
> > [pid  5695] time(NULL)                  = 1297958855
> > [pid  5695] mmap2(NULL, 32768, PROT_READ|PROT_WRITE,
> > MAP_FILE|MAP_ANONYMOUS, -1, 0) = -1 EINVAL (Invalid argument)
> > [pid  5695] select(6, [4 5], [], [], {1, 0}) = 1 (in [5], left {0,
> > 999995}) [pid  5695] time(NULL)                  = 1297958855
> > [pid  5695] recv(5, "GET / HTTP/1.1\r\nHost: localhost:"..., 2048,
> > MSG_DONTWAIT|MSG_NOSIGNAL) = 383
> > [pid  5695] fstat64(1, {st_mode=S_IFCHR|0620, st_rdev=makedev(136, 1),
> > ...}) = 0 [pid  5695] mmap2(NULL, 4096, PROT_READ|PROT_WRITE,
> > MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb789f000
> > [pid  5695] write(1, "New GET request for / using vers"..., 45New GET
> > request for / using version HTTP/1.1
> > ) = 45
> > [pid  5695] open("after-the-rain.jpg", O_RDONLY) = 6
> > [pid  5695] fstat64(6, {st_mode=S_IFREG|0644, st_size=72313, ...}) = 0
> > [pid  5695] shutdown(5, 0 /* receive */) = 0
> > [pid  5695] write(1, "queued response\n", 16queued response
> > ) = 16
> > [pid  5695] time(NULL)                  = 1297958855
> > [pid  5695] open("/etc/localtime", O_RDONLY) = 7
> > [pid  5695] fstat64(7, {st_mode=S_IFREG|0644, st_size=265, ...}) = 0
> > [pid  5695] fstat64(7, {st_mode=S_IFREG|0644, st_size=265, ...}) = 0
> > [pid  5695] mmap2(NULL, 4096, PROT_READ|PROT_WRITE,
> > MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb789e000
> > [pid  5695] read(7,
> > "TZif2\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\4\0\0\0\4\0\0\0\0"..., 4096) =
> > 265 [pid  5695] _llseek(7, -10, [255], SEEK_CUR) = 0
> > [pid  5695] read(7, "\nIST-5:30\n", 4096) = 10
> > [pid  5695] close(7)                    = 0
> > [pid  5695] munmap(0xb789e000, 4096)    = 0
> > [pid  5695] setsockopt(5, SOL_TCP, TCP_CORK, [1], 4) = 0
> > [pid  5695] select(6, [4], [5], [], {1, 0}) = 1 (out [5], left {0,
> > 999996}) [pid  5695] time(NULL)                  = 1297958855
> > [pid  5695] send(5, "HTTP/1.1 200 OK\r\nContent-Length:"..., 104,
> > MSG_DONTWAIT|MSG_NOSIGNAL) = 104
> > [pid  5695] select(6, [4], [5], [], {1, 0}) = 1 (out [5], left {0,
> > 999996}) [pid  5695] time(NULL)                  = 1297958855
> > [pid  5695] sendfile64(5, 6, [683649624044470272], 72313) = -1 EOVERFLOW
> > (Value too large for defined data type)
> > [pid  5695] shutdown(5, 2 /* send and receive */) = 0
> > [pid  5695] close(5)                    = 0
> > [pid  5695] close(6)                    = 0
> > 
> > 
> > Thanks
> > Neil
> > 
> > 
> > ________________________________
> > From: Christian Grothoff <address@hidden>
> > To: Neil D'Souza <address@hidden>
> > Cc: address@hidden
> > Sent: Thu, February 17, 2011 8:31:19 PM
> > Subject: Re: libmicrohttpd - question on tutorial
> > 
> > Hi!
> > 
> > I just tried this:
> > 
> > $ echo test > picture.png
> > $ gcc -o srv -g -I /usr/local/include/ \
> > 
> >     -L/usr/local/lib/ -lmicrohttpd responseheaders.c
> > 
> > $ ./srv &
> > $ telnet localhost 8888
> > Trying ::1...
> > Trying 127.0.0.1...
> > Connected to localhost.
> > Escape character is '^]'.
> > GET /dontcare HTTP/1.1
> > Host: itsme
> > 
> > HTTP/1.1 200 OK
> > Content-Length: 5
> > Content-Type: image/png
> > Date: Thu, 17 Feb 2011 14:47:53 GMT
> > 
> > test
> > Connection closed by foreign host.
> > 
> > 
> > So this worked on my system (which is Debian, but should make no
> > difference) using the (except #includes) unmodified responseheaders.c
> > against MHD 0.9.7.
> > 
> > So this is rather confusing, especially given that you did use the same
> > code (!?).  Does your image file have some access permissions that might
> > cause problems? Try running the server with 'strace -f' and see what the
> > system calls do -- there SHOULD be one to 'sendfile' for the actual
> > transmission (shortly after your 'open' call to picture.png).  That might
> > help.  Here is what mine looks like:
> > 
> > After telnet accept, select waits for 'GET' (I post the entire request at
> > once with middle-mouse click):
> > [pid 19650] select(6, [4 5], [], [], {1, 0}) = 1 (in [5], left {0,
> > 556551}) [pid 19650] recvfrom(5, "GET / HTTP/1.1\r\nHost: me\r\n\r\n",
> > 2048, MSG_DONTWAIT|MSG_NOSIGNAL, NULL, NULL) = 28
> > 
> > Here is the open call:
> > [pid 19650] open("picture.png", O_RDONLY) = 6
> > [pid 19650] fstat(6, {st_mode=S_IFREG|0644, st_size=5, ...}) = 0
> > [pid 19650] shutdown(5, 0 /* receive */) = 0
> > 
> > Part of MHD header response creation involves getting system time:
> > [pid 19650] open("/etc/localtime", O_RDONLY) = 7
> > [pid 19650] fstat(7, {st_mode=S_IFREG|0644, st_size=2309, ...}) = 0
> > [pid 19650] fstat(7, {st_mode=S_IFREG|0644, st_size=2309, ...}) = 0
> > [pid 19650] mmap(NULL, 4096, PROT_READ|PROT_WRITE,
> > MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7fe5da3b7000
> > [pid 19650] read(7,
> > "TZif2\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\10\0\0\0\10\0\0\0\0"..., 4096)
> > = 2309
> > [pid 19650] lseek(7, -1467, SEEK_CUR)   = 842
> > [pid 19650] read(7,
> > "TZif2\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\t\0\0\0\t\0\0\0\0"..., 4096) =
> > 1467 [pid 19650] close(7)                    = 0
> > [pid 19650] munmap(0x7fe5da3b7000, 4096) = 0
> > 
> > Then MHD sends the header (cork on):
> > [pid 19650] setsockopt(5, SOL_TCP, TCP_CORK, [1], 4) = 0
> > [pid 19650] select(6, [4], [5], [], {1, 0}) = 1 (out [5], left {0,
> > 999998}) [pid 19650] sendto(5, "HTTP/1.1 200 OK\r\nContent-Length:"...,
> > 100, MSG_DONTWAIT|MSG_NOSIGNAL, NULL, 0) = 100
> > 
> > Then MHD waits again to send the body (cork off):
> > [pid 19650] select(6, [4], [5], [], {1, 0}) = 1 (out [5], left {0,
> > 999998}) [pid 19650] sendfile(5, 6, [0], 5)      = 5
> > [pid 19650] setsockopt(5, SOL_TCP, TCP_CORK, [0], 4) = 0
> > 
> > Finally, we're done and close the file and socket:
> > [pid 19650] close(6)                    = 0
> > [pid 19650] shutdown(5, 2 /* send and receive */) = 0
> > [pid 19650] close(5)                    = 0
> > 
> > Back to waiting for more:
> > [pid 19650] select(5, [4], [], [], {1, 0}) = 0 (Timeout)
> > 
> > 
> > Any changes from this sequence (return values, etc.) might help explain
> > what's going on...
> > 
> > Happy hacking,
> > 
> > Christian
> > 
> > On Thursday 17 February 2011 15:26:05 Neil D'Souza wrote:
> > > Hello Christian,
> > > 
> > > First of all Thank you for the excellently written tutorial. I myself
> > > am working on a language for scripting questionnaires for the Market
> > > Research industry. You can see the project here
> > > http://sourceforge.net/projects/xtcc - the compiler is called qscript
> > > (for Questionnaire Script).
> > > 
> > > It currently generates code for simple data entry in an ncurses
> > > interface. Installing the compiler requires the presence of gcc,
> > > ncurses and compiling qscript from source - which can be big barriers
> > > for adoption/trying. I am trying to embed the generated code inside a
> > > web-server and demo what the interface looks like and the programming
> > > language looks like - hence my interest in libmicrohttpd. I am going
> > > through the tutorial so that I can understand libmicrohttpd usage.
> > > 
> > > I have reached Chapter 4 where we send the png image. below is a
> > > slightly modified function just for me to be sure that things were
> > > working. However the image did not appear in the browser. So I
> > > telnetted to port 8888. As you can see the png file is not sent -
> > > otherwise there should have been a bytestream of the image data. It
> > > seems that after getting enqueued the file is not transmitted. Just
> > > wanted to check if I am missing something. I tried with Konqueror and
> > > Firefox before resorting to telnet.
> > > 
> > > Also wanted to add that you need to include
> > > #include <sys/stat.h>
> > > #include <fcntl.h>
> > > 
> > >  to responseheaders.c to compile with gcc version 4.4.1 . I am using
> > >  ubuntu
> > > 
> > > 9.10 in case it is of any use and microhttpd version is 0.9.7 and was
> > > compiled by me from source and installed in /usr/local .
> > > 
> > > Many thanks for your help in advance.
> > > 
> > > Kind Regards,
> > > Neil
> > > 
> > > 
> > > /media/sda3/home/nxd/Prog/C/microhttpd/tutorial>telnet localhost 8888
> > > Trying 127.0.0.1...
> > > Connected to localhost.
> > > Escape character is '^]'.
> > > GET /dontcare HTTP/1.1
> > > Host: itsme
> > > 
> > > HTTP/1.1 200 OK
> > > Content-Length: 1340341
> > > Content-Type: image/png
> > > Date: Thu, 17 Feb 2011 13:54:52 GMT
> > > 
> > > Connection closed by foreign host.
> > > 
> > > 
> > > 
> > > // ==================
> > > static int
> > > answer_to_connection (void *cls, struct MHD_Connection *connection,
> > > 
> > >                       const char *url, const char *method,
> > >                       const char *version, const char *upload_data,
> > >                       size_t *upload_data_size, void **con_cls)
> > > 
> > > {
> > > 
> > >   struct MHD_Response *response;
> > >   int fd;
> > >   int ret;
> > >   struct stat sbuf;
> > >   printf ("New %s request for %s using version %s\n", method, url,
> > > 
> > > version);
> > > 
> > >   if (0 != strcmp (method, "GET"))
> > >   
> > >     return MHD_NO;
> > >   
> > >   if ( (-1 == (fd = open (FILENAME, O_RDONLY))) ||
> > >   
> > >        (0 != fstat (fd, &sbuf)) )
> > >     
> > >     {
> > >     
> > >       /* error accessing file */
> > >       if (fd != -1) close (fd);
> > >       const char *errorstr =
> > >       
> > >         "<html><body>An internal server error has occured!\
> > >         
> > >                               </body></html>";
> > >       
> > >       response =
> > >     
> > >     MHD_create_response_from_buffer (strlen (errorstr),
> > >     
> > >                      (void *) errorstr,
> > >                      MHD_RESPMEM_PERSISTENT);
> > >       
> > >       if (response)
> > >       
> > >         {
> > >         
> > >           ret =
> > >           
> > >             MHD_queue_response (connection,
> > >             MHD_HTTP_INTERNAL_SERVER_ERROR,
> > >             
> > >                                 response);
> > >           
> > >           MHD_destroy_response (response);
> > >           
> > >           return MHD_YES;
> > >         
> > >         }
> > >       
> > >       else
> > >       
> > >         return MHD_NO;
> > >     
> > >     }
> > >   
> > >   response =
> > >   
> > >     MHD_create_response_from_fd_at_offset (sbuf.st_size, fd, 0);
> > >   
> > >   MHD_add_response_header (response, "Content-Type", MIMETYPE);
> > >   ret = MHD_queue_response (connection, MHD_HTTP_OK, response);
> > >   if (ret == MHD_YES)
> > >   
> > >       printf ("queued response\n");
> > >   
> > >   else
> > >   
> > >       printf ("could not queue response\n");
> > >   
> > >   MHD_destroy_response (response);
> > >   
> > >   return ret;
> > > 
> > > }
> > > 
> > > // ===============