emacs-27 d87a4d1: Limit RLIMIT_NOFILE to FD_SETSIZE on macOS

[Robert Pluim]

branch: emacs-27
commit d87a4d1f4e1a84b2a527ebe5a04516f47d4717ea
Author: YAMAMOTO Mitsuharu <address@hidden>
Commit: Robert Pluim <address@hidden>

    Limit RLIMIT_NOFILE to FD_SETSIZE on macOS
    
    * src/nsterm.m ([EmacsApp applicationDidFinishLaunching:]): Call
    CoreFoundation functions that increase RLIMIT_NOFILE behind our back
    during startup, and then set RLIMIT_NOFILE back to FD_SETSIZE to avoid
    crashes in setup_process_coding_system  (Bug#39164).
---
 src/nsterm.m | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/src/nsterm.m b/src/nsterm.m
index e92e3d5..ac46784 100644
--- a/src/nsterm.m
+++ b/src/nsterm.m
@@ -5812,6 +5812,21 @@ ns_term_shutdown (int sig)
 #endif
 
 #ifdef NS_IMPL_COCOA
+  /* Some functions/methods in CoreFoundation/Foundation increase the
+     maximum number of open files for the process in their first call.
+     We make dummy calls to them and then reduce the resource limit
+     here, since pselect cannot handle file descriptors that are
+     greater than or equal to FD_SETSIZE.  */
+  CFSocketGetTypeID ();
+  CFFileDescriptorGetTypeID ();
+  [[NSFileHandle alloc] init];
+  struct rlimit rlim;
+  if (getrlimit (RLIMIT_NOFILE, &rlim) == 0
+      && rlim.rlim_cur > FD_SETSIZE)
+    {
+      rlim.rlim_cur = FD_SETSIZE;
+      setrlimit (RLIMIT_NOFILE, &rlim);
+    }
   if ([NSApp activationPolicy] == NSApplicationActivationPolicyProhibited) {
     /* Set the app's activation policy to regular when we run outside
        of a bundle.  This is already done for us by Info.plist when we