[PATCH 2/2] dup2, dup3: work around another cygwin crasher

[Eric Blake]

Cygwin 1.7.25 can crash due to an off-by-one bug on an attempt to
duplicate a file into the current RLIMIT_NOFILE soft limit, when
that limit is smaller than the hard limit.  The intent in the
cygwin code was to allow the dup and auto-increase the soft limit,
which is itself questionable (and which we work around in the
gnulib getdtablesize module); but avoiding the crash is worth
doing even if the soft limit semantics are wrong.

http://cygwin.com/ml/cygwin/2013-09/msg00397.html
http://cygwin.com/ml/cygwin-developers/2013-q3/msg00010.html

* m4/dup2.m4 (gl_FUNC_DUP2): Expose the bug.
* m4/dup3.m4 (gl_FUNC_DUP3): Likewise.
* tests/test-dup2.c (main): Likewise.
* lib/dup2.c (rpl_dup2): Use setdtablesize to avoid it.
* lib/dup3.c (dup3): Likewise.
* doc/posix-functions/dup2.texi (dup2): Document it.
* doc/glibc-functions/dup3.texi (dup3): Likewise.

Signed-off-by: Eric Blake <address@hidden>
---
 ChangeLog                     |    9 +++++++++
 doc/glibc-functions/dup3.texi |    4 ++++
 doc/posix-functions/dup2.texi |    4 ++++
 lib/dup2.c                    |    4 ++++
 lib/dup3.c                    |    4 ++++
 m4/dup2.m4                    |    7 +++++--
 m4/dup3.m4                    |    4 ++--
 tests/test-dup2.c             |    7 +++++++
 tests/test-dup3.c             |    7 +++++++
 9 files changed, 46 insertions(+), 4 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index 812ea54..5934344 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,5 +1,14 @@
 2013-09-26  Eric Blake  <address@hidden>

+       dup2, dup3: work around another cygwin crasher
+       * m4/dup2.m4 (gl_FUNC_DUP2): Expose the bug.
+       * m4/dup3.m4 (gl_FUNC_DUP3): Likewise.
+       * tests/test-dup2.c (main): Likewise.
+       * lib/dup2.c (rpl_dup2): Use setdtablesize to avoid it.
+       * lib/dup3.c (dup3): Likewise.
+       * doc/posix-functions/dup2.texi (dup2): Document it.
+       * doc/glibc-functions/dup3.texi (dup3): Likewise.
+
        getdtablesize: work around cygwin issue
        * m4/getdtablesize.m4 (gl_FUNC_GETDTABLESIZE): Detect problem.
        * modules/getdtablesize (configure.ac): Build replacement.
diff --git a/doc/glibc-functions/dup3.texi b/doc/glibc-functions/dup3.texi
index 52b8abb..cc3ca0d 100644
--- a/doc/glibc-functions/dup3.texi
+++ b/doc/glibc-functions/dup3.texi
@@ -10,6 +10,10 @@ dup3
 This function is missing on many non-glibc platforms:
 Mac OS X 10.5, FreeBSD 6.0, NetBSD 5.0, OpenBSD 3.8, Minix 3.1.8, AIX 5.1, 
HP-UX 11,
 IRIX 6.5, OSF/1 5.1, Solaris 11 2011-11, Cygwin 1.7.1, mingw, MSVC 9, Interix 
3.5, BeOS.
+
address@hidden
+This function can crash on some platforms:
+Cygwin 1.7.25.
 @end itemize

 Portability problems not fixed by Gnulib:
diff --git a/doc/posix-functions/dup2.texi b/doc/posix-functions/dup2.texi
index 14e5236..cab793f 100644
--- a/doc/posix-functions/dup2.texi
+++ b/doc/posix-functions/dup2.texi
@@ -21,6 +21,10 @@ dup2
 Cygwin 1.7.17, MSVC 9.

 @item
+This function crashes when invoked with valid arguments on some platforms:
+Cygwin 1.7.25.
+
address@hidden
 This function resets the @code{FD_CLOEXEC} flag when duplicating an fd
 to itself on some platforms:
 Haiku.
diff --git a/lib/dup2.c b/lib/dup2.c
index 9219eb3..f128e7a 100644
--- a/lib/dup2.c
+++ b/lib/dup2.c
@@ -96,7 +96,11 @@ rpl_dup2 (int fd, int desired_fd)
   /* On Linux kernels 2.6.26-2.6.29, dup2 (fd, fd) returns -EBADF.
      On Cygwin 1.5.x, dup2 (1, 1) returns 0.
      On Cygwin 1.7.17, dup2 (1, -1) dumps core.
+     On Cygwin 1.7.25, dup2 (1, 256) can dump core.
      On Haiku, dup2 (fd, fd) mistakenly clears FD_CLOEXEC.  */
+#  if HAVE_SETDTABLESIZE
+  setdtablesize (desired_fd + 1);
+#  endif
   if (desired_fd < 0)
     fd = desired_fd;
   if (fd == desired_fd)
diff --git a/lib/dup3.c b/lib/dup3.c
index b2b5705..3073189 100644
--- a/lib/dup3.c
+++ b/lib/dup3.c
@@ -30,6 +30,10 @@ dup3 (int oldfd, int newfd, int flags)
 {
 #if HAVE_DUP3
 # undef dup3
+# if HAVE_SETDTABLESIZE
+  /* Avoid a cygwin crasher. */
+  setdtablesize (newfd + 1);
+# endif
   /* Try the system call first, if it exists.  (We may be running with a glibc
      that has the function but with an older kernel that lacks it.)  */
   {
diff --git a/m4/dup2.m4 b/m4/dup2.m4
index 269cfdc..c65089f 100644
--- a/m4/dup2.m4
+++ b/m4/dup2.m4
@@ -1,4 +1,4 @@
-#serial 19
+#serial 20
 dnl Copyright (C) 2002, 2005, 2007, 2009-2013 Free Software Foundation, Inc.
 dnl This file is free software; the Free Software Foundation
 dnl gives unlimited permission to copy and/or distribute it,
@@ -39,9 +39,11 @@ AC_DEFUN([gl_FUNC_DUP2],
             /* Many gnulib modules require POSIX conformance of EBADF.  */
             if (dup2 (2, 1000000) == -1 && errno != EBADF)
               result |= 16;
-            /* Flush out a cygwin core dump.  */
+            /* Flush out some cygwin core dumps.  */
             if (dup2 (2, -1) != -1 || errno != EBADF)
               result |= 32;
+           dup2 (2, 255);
+           dup2 (2, 256);
             return result;
            ])
         ],
@@ -65,6 +67,7 @@ AC_DEFUN([gl_FUNC_DUP2],
       *yes) ;;
       *)
         REPLACE_DUP2=1
+       AC_CHECK_FUNCS([setdtablesize])
         ;;
     esac
   fi
diff --git a/m4/dup3.m4 b/m4/dup3.m4
index 706d726..23b6bb1 100644
--- a/m4/dup3.m4
+++ b/m4/dup3.m4
@@ -1,4 +1,4 @@
-# dup3.m4 serial 4
+# dup3.m4 serial 5
 dnl Copyright (C) 2009-2013 Free Software Foundation, Inc.
 dnl This file is free software; the Free Software Foundation
 dnl gives unlimited permission to copy and/or distribute it,
@@ -11,7 +11,7 @@ AC_DEFUN([gl_FUNC_DUP3],
   dnl Persuade glibc <unistd.h> to declare dup3().
   AC_REQUIRE([AC_USE_SYSTEM_EXTENSIONS])

-  AC_CHECK_FUNCS_ONCE([dup3])
+  AC_CHECK_FUNCS_ONCE([dup3 setdtablesize])
   if test $ac_cv_func_dup3 != yes; then
     HAVE_DUP3=0
   fi
diff --git a/tests/test-dup2.c b/tests/test-dup2.c
index ecb2692..19aaf66 100644
--- a/tests/test-dup2.c
+++ b/tests/test-dup2.c
@@ -150,6 +150,13 @@ main (void)
   errno = 0;
   ASSERT (dup2 (fd, -2) == -1);
   ASSERT (errno == EBADF);
+  if (bad_fd > 256)
+    {
+      ASSERT (dup2 (fd, 256) == 256);
+      ASSERT (close (256) == 0);
+    }
+  ASSERT (dup2 (fd, bad_fd - 1) == bad_fd - 1);
+  ASSERT (close (bad_fd - 1) == 0);
   errno = 0;
   ASSERT (dup2 (fd, bad_fd) == -1);
   ASSERT (errno == EBADF);
diff --git a/tests/test-dup3.c b/tests/test-dup3.c
index eb26c8f..b6074d8 100644
--- a/tests/test-dup3.c
+++ b/tests/test-dup3.c
@@ -124,6 +124,13 @@ main ()
       errno = 0;
       ASSERT (dup3 (fd, -2, o_flags) == -1);
       ASSERT (errno == EBADF);
+      if (bad_fd > 256)
+       {
+         ASSERT (dup3 (fd, 256, 0) == 256);
+         ASSERT (close (256) == 0);
+       }
+      ASSERT (dup3 (fd, bad_fd - 1, 0) == bad_fd - 1);
+      ASSERT (close (bad_fd - 1) == 0);
       errno = 0;
       ASSERT (dup3 (fd, bad_fd, o_flags) == -1);
       ASSERT (errno == EBADF);
-- 
1.7.1