[Emacs-diffs] master 32e5366: Fix "not a tty" bug on Solaris 10

[Paul Eggert]

branch: master
commit 32e53667a91ed479743175d5698a89b163c8be94
Author: Paul Eggert <address@hidden>
Commit: Paul Eggert <address@hidden>

    Fix "not a tty" bug on Solaris 10
    
    * configure.ac (PTY_OPEN): Define to plain 'open'
    on SVR4-derived hosts, so that the O_CLOEXEC flag isn't set.
    * src/process.c (allocate_pty): Set the O_CLOEXEC flag after
    calling PTY_TTY_NAME_SPRINTF, for the benefit of SVR4-derived
    hosts that call grantpt which does its work via a setuid subcommand
    (Bug#19191, Bug#19927, Bug#20555, Bug#20686).
    Also, set O_CLOEXEC even if PTY_OPEN is not defined, since it
    seems relevant in that case too.
---
 configure.ac  |    5 ++++-
 src/process.c |   20 +++++++++++---------
 2 files changed, 15 insertions(+), 10 deletions(-)

diff --git a/configure.ac b/configure.ac
index 9c6a74a..070b061 100644
--- a/configure.ac
+++ b/configure.ac
@@ -4397,14 +4397,17 @@ case $opsys in
     ;;
 
   sol2* )
-    dnl On SysVr4, grantpt(3) forks a subprocess, so keep sigchld_handler()
+    dnl On SysVr4, grantpt(3) forks a subprocess, so do not use
+    dnl O_CLOEXEC when opening the pty, and keep the SIGCHLD handler
     dnl from intercepting that death.  If any child but grantpt's should die
     dnl within, it should be caught after sigrelse(2).
+    AC_DEFINE(PTY_OPEN, [fd = open (pty_name, O_RDWR | O_NONBLOCK)])
     AC_DEFINE(PTY_TTY_NAME_SPRINTF, [{ char *ptsname (int), *ptyname; int 
grantpt_result; sigset_t blocked; sigemptyset (&blocked); sigaddset (&blocked, 
SIGCHLD); pthread_sigmask (SIG_BLOCK, &blocked, 0); grantpt_result = grantpt 
(fd); pthread_sigmask (SIG_UNBLOCK, &blocked, 0); if (grantpt_result == -1 || 
unlockpt (fd) == -1 || !(ptyname = ptsname (fd))) { emacs_close (fd); return 
-1; } snprintf (pty_name, PTY_NAME_SIZE, "%s", ptyname); }])
     ;;
 
   unixware )
     dnl Comments are as per sol2*.
+    AC_DEFINE(PTY_OPEN, [fd = open (pty_name, O_RDWR | O_NONBLOCK)])
     AC_DEFINE(PTY_TTY_NAME_SPRINTF, [{ char *ptsname (int), *ptyname; int 
grantpt_result; sigset_t blocked; sigemptyset (&blocked); sigaddset (&blocked, 
SIGCHLD); pthread_sigmask (SIG_BLOCK, &blocked, 0); grantpt_result = grantpt 
(fd); pthread_sigmask (SIG_UNBLOCK, &blocked, 0); if (grantpt_result == -1) 
fatal("could not grant slave pty"); if (unlockpt(fd) == -1) fatal("could not 
unlock slave pty"); if (!(ptyname = ptsname(fd))) fatal ("could not enable 
slave pty"); snprintf (pty_name, P [...]
     ;;
 esac
diff --git a/src/process.c b/src/process.c
index 17fe708..b4f979f 100644
--- a/src/process.c
+++ b/src/process.c
@@ -658,22 +658,24 @@ allocate_pty (char pty_name[PTY_NAME_SIZE])
 
        if (fd >= 0)
          {
-#ifdef PTY_OPEN
+#ifdef PTY_TTY_NAME_SPRINTF
+           PTY_TTY_NAME_SPRINTF
+#else
+           sprintf (pty_name, "/dev/tty%c%x", c, i);
+#endif /* no PTY_TTY_NAME_SPRINTF */
+
            /* Set FD's close-on-exec flag.  This is needed even if
               PT_OPEN calls posix_openpt with O_CLOEXEC, since POSIX
               doesn't require support for that combination.
+              Do this after PTY_TTY_NAME_SPRINTF, which on some platforms
+              doesn't work if the close-on-exec flag is set (Bug#20555).
               Multithreaded platforms where posix_openpt ignores
               O_CLOEXEC (or where PTY_OPEN doesn't call posix_openpt)
               have a race condition between the PTY_OPEN and here.  */
            fcntl (fd, F_SETFD, FD_CLOEXEC);
-#endif
-           /* Check to make certain that both sides are available
-              this avoids a nasty yet stupid bug in rlogins.  */
-#ifdef PTY_TTY_NAME_SPRINTF
-           PTY_TTY_NAME_SPRINTF
-#else
-           sprintf (pty_name, "/dev/tty%c%x", c, i);
-#endif /* no PTY_TTY_NAME_SPRINTF */
+
+           /* Check to make certain that both sides are available.
+              This avoids a nasty yet stupid bug in rlogins.  */
            if (faccessat (AT_FDCWD, pty_name, R_OK | W_OK, AT_EACCESS) != 0)
              {
                emacs_close (fd);