fcntl: Work around NetBSD bug with F_DUPFD_CLOEXEC

[Bruno Haible]

When I added a check to the fcntl tests that verifies that a file
descriptor created by fcntl(, F_DUPFD_CLOEXEC,) is actually closed
by exec*(), I noticed that this test fails on NetBSD 7.0 and 9.0, at
least.

This patch adds a workaround to make fcntl(, F_DUPFD_CLOEXEC,) work
right, and enhances the unit test accordingly.


2020-11-29  Bruno Haible  <bruno@clisp.org>

        fcntl: Work around NetBSD bug with F_DUPFD_CLOEXEC.
        * m4/fcntl.m4 (gl_FUNC_FCNTL): Test whether F_DUPFD_CLOEXEC actually
        works.
        * lib/fcntl.c (rpl_fcntl_DUPFD_CLOEXEC): On NetBSD, use the same
        fallback implementation as on Haiku.
        * tests/test-fcntl.c (main): Add a test whether F_DUPFD_CLOEXEC is
        effective.
        * doc/posix-functions/fcntl.texi: Mention the NetBSD bug.

diff --git a/doc/posix-functions/fcntl.texi b/doc/posix-functions/fcntl.texi
index 887f8d4..831a194 100644
--- a/doc/posix-functions/fcntl.texi
+++ b/doc/posix-functions/fcntl.texi
@@ -23,6 +23,11 @@ IRIX 6.5, Solaris 11 2010-11, Cygwin 1.7.1.
 Note that the gnulib replacement code is functional but not atomic.
 
 @item
+The @code{F_DUPFD_CLOEXEC} action of this function does not set the
+@code{FD_CLOEXEC} flag on some platforms:
+NetBSD 9.0.
+
+@item
 The @code{F_DUPFD_CLOEXEC} action of this function sets the
 @code{FD_CLOEXEC} flag on the wrong file descriptor on some platforms:
 Haiku.
diff --git a/lib/fcntl.c b/lib/fcntl.c
index 8cd1531..59bb3ca 100644
--- a/lib/fcntl.c
+++ b/lib/fcntl.c
@@ -491,7 +491,9 @@ rpl_fcntl_DUPFD_CLOEXEC (int fd, int target)
 #if !HAVE_FCNTL
   result = dupfd (fd, target, O_CLOEXEC);
 #else /* HAVE_FCNTL */
-# if defined __HAIKU__
+# if defined __NetBSD__ || defined __HAIKU__
+  /* On NetBSD 9.0, the system fcntl (fd, F_DUPFD_CLOEXEC, target)
+     has only the same effect as fcntl (fd, F_DUPFD, target).  */
   /* On Haiku, the system fcntl (fd, F_DUPFD_CLOEXEC, target) sets
      the FD_CLOEXEC flag on fd, not on target.  Therefore avoid the
      system fcntl in this case.  */
diff --git a/m4/fcntl.m4 b/m4/fcntl.m4
index ea24f3d..547ff40 100644
--- a/m4/fcntl.m4
+++ b/m4/fcntl.m4
@@ -1,4 +1,4 @@
-# fcntl.m4 serial 10
+# fcntl.m4 serial 11
 dnl Copyright (C) 2009-2020 Free Software Foundation, Inc.
 dnl This file is free software; the Free Software Foundation
 dnl gives unlimited permission to copy and/or distribute it,
@@ -81,15 +81,29 @@ AC_DEFUN([gl_FUNC_FCNTL],
           behavior does not match POSIX]) ;;
     esac
 
-    dnl Many systems lack F_DUPFD_CLOEXEC
+    dnl Many systems lack F_DUPFD_CLOEXEC.
+    dnl NetBSD 9.0 declares F_DUPFD_CLOEXEC but it works only like F_DUPFD.
     AC_CACHE_CHECK([whether fcntl understands F_DUPFD_CLOEXEC],
       [gl_cv_func_fcntl_f_dupfd_cloexec],
-      [AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[
-#include <fcntl.h>
-#ifndef F_DUPFD_CLOEXEC
-choke me
-#endif
-         ]])],
+      [AC_RUN_IFELSE(
+         [AC_LANG_SOURCE(
+            [[#include <fcntl.h>
+              #include <unistd.h>
+              int main (int argc, char *argv[])
+              {
+                if (argc == 1)
+                  /* parent process */
+                  {
+                    if (fcntl (1, F_DUPFD_CLOEXEC, 10) < 0)
+                      return 1;
+                    return execl ("./conftest", "./conftest", "child", NULL);
+                  }
+                else
+                  /* child process */
+                  return (fcntl (10, F_GETFL) < 0 ? 0 : 42);
+              }
+            ]])
+         ],
          [AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[
 #ifdef __linux__
 /* The Linux kernel only added F_DUPFD_CLOEXEC in 2.6.24, so we always replace
@@ -98,12 +112,22 @@ choke me
 #endif
            ]])],
            [gl_cv_func_fcntl_f_dupfd_cloexec=yes],
-           [gl_cv_func_fcntl_f_dupfd_cloexec="needs runtime check"])],
-         [gl_cv_func_fcntl_f_dupfd_cloexec=no])])
-    if test "$gl_cv_func_fcntl_f_dupfd_cloexec" != yes; then
-      gl_REPLACE_FCNTL
-      dnl No witness macro needed for this bug.
-    fi
+           [gl_cv_func_fcntl_f_dupfd_cloexec="needs runtime check"])
+         ],
+         [gl_cv_func_fcntl_f_dupfd_cloexec=no],
+         [case "$host_os" in
+                     # Guess no on NetBSD.
+            netbsd*) gl_cv_func_fcntl_f_dupfd_cloexec="guessing no" ;;
+            *)       gl_cv_func_fcntl_f_dupfd_cloexec="$gl_cross_guess_normal" 
;;
+          esac
+         ])
+      ])
+    case "$gl_cv_func_fcntl_f_dupfd_cloexec" in
+      *yes) ;;
+      *)    gl_REPLACE_FCNTL
+            dnl No witness macro needed for this bug.
+            ;;
+    esac
   fi
   dnl Replace fcntl() for supporting the gnulib-defined fchdir() function,
   dnl to keep fchdir's bookkeeping up-to-date.
diff --git a/tests/test-fcntl.c b/tests/test-fcntl.c
index 9ef354c..c734e41 100644
--- a/tests/test-fcntl.c
+++ b/tests/test-fcntl.c
@@ -212,8 +212,12 @@ check_flags (void)
 }
 
 int
-main (void)
+main (int argc, char *argv[])
 {
+  if (argc > 1)
+    /* child process */
+    return (is_open (10) ? 42 : 0);
+
   const char *file = "test-fcntl.tmp";
   int fd;
   int bad_fd = getdtablesize ();
@@ -411,5 +415,11 @@ main (void)
   ASSERT (close (fd) == 0);
   ASSERT (unlink (file) == 0);
 
-  return 0;
+  /* Test whether F_DUPFD_CLOEXEC is effective.  */
+  ASSERT (fcntl (1, F_DUPFD_CLOEXEC, 10) >= 0);
+#if defined _WIN32 && !defined __CYGWIN__
+  return _execl ("./test-fcntl", "./test-fcntl", "child", NULL);
+#else
+  return execl ("./test-fcntl", "./test-fcntl", "child", NULL);
+#endif
 }