Re: [Bug-tar] "Cannot open: Too many open files" extraction error with --files-from option tar-1.15.90/91

[Sergey Poznyakoff]

Hi Ed,

> Last night I created a test tarfile that apparently
> causes a failure when extracting many small files to multiple
> directories in round-robin order.
[...]
> tar: html/group__pangommEnums.html: Cannot open: Too many open files
> tar: Skipping to next header

Yes, it is possible. The reason behind this is that when changing to a
directory tar saves current wd using save_cwd, which returns the directory
descriptor. So, given sufficiently large number of -C options, the
descriptor pool can be exhausted. 

Please try the following patch. It keeps track of the number of
descriptors opened by save_cwd and falls back to xgetcwd() when it
reaches the limit.

Please inform me if it works for you.

Regards,
Sergey

Index: src/common.h
===================================================================
RCS file: /cvsroot/tar/tar/src/common.h,v
retrieving revision 1.89
diff -p -u -r1.89 common.h
--- src/common.h        25 Jun 2006 12:45:16 -0000      1.89
+++ src/common.h        3 Jul 2006 08:23:56 -0000
@@ -582,6 +582,7 @@ void undo_last_backup (void);
 
 int deref_stat (bool deref, char const *name, struct stat *buf);
 
+void closeopen (void);
 int chdir_arg (char const *dir);
 void chdir_do (int dir);
 
Index: src/misc.c
===================================================================
RCS file: /cvsroot/tar/tar/src/misc.c,v
retrieving revision 1.35
diff -p -u -r1.35 misc.c
--- src/misc.c  20 Jun 2006 15:14:00 -0000      1.35
+++ src/misc.c  3 Jul 2006 08:23:56 -0000
@@ -18,10 +18,13 @@
    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.  */
 
 #include <system.h>
+#include <sys/time.h>
+#include <sys/resource.h>
 #include <rmt.h>
 #include "common.h"
 #include <quotearg.h>
 #include <save-cwd.h>
+#include <xgetcwd.h>
 #include <unlinkdir.h>
 #include <utimens.h>
 
@@ -568,13 +571,41 @@ chdir_arg (char const *dir)
   return wds++;
 }
 
+/* Return maximum number of open files */
+int
+get_max_open_files ()
+{
+#if defined(_SC_OPEN_MAX)
+  return sysconf (_SC_OPEN_MAX);
+#elif defined(RLIMIT_NOFILE)
+  struct rlimit rlim;
+
+  if (getrlimit(RLIMIT_NOFILE, &rlim) == 0)
+    return rlim.rlim_max;
+#elif defined(HAVE_GETDTABLESIZE)
+  return getdtablesize ();
+#endif
+  return -1;
+}
+
+/* Close all descriptors, except first three */
+void
+closeopen ()
+{
+  int i;
+
+  for (i = get_max_open_files () - 1; i > 2; i--)
+    close (i);
+}
+  
 /* Change to directory I.  If I is 0, change to the initial working
    directory; otherwise, I must be a value returned by chdir_arg.  */
 void
 chdir_do (int i)
 {
   static int previous;
-
+  static int saved_count;
+  
   if (previous != i)
     {
       struct wd *prev = &wd[previous];
@@ -583,7 +614,15 @@ chdir_do (int i)
       if (! prev->saved)
        {
          prev->saved = 1;
-         if (save_cwd (&prev->saved_cwd) != 0)
+         saved_count++;
+         /* Make sure we still have at least one descriptor available */
+         if (saved_count >= get_max_open_files () - 4)
+           {
+             /* Force restore_cwd to use chdir_long */
+             prev->saved_cwd.desc = -1;
+             prev->saved_cwd.name = xgetcwd ();
+           }
+         else if (save_cwd (&prev->saved_cwd) != 0)
            FATAL_ERROR ((0, 0, _("Cannot save working directory")));
        }
 
Index: src/tar.c
===================================================================
RCS file: /cvsroot/tar/tar/src/tar.c,v
retrieving revision 1.152
diff -p -u -r1.152 tar.c
--- src/tar.c   25 Jun 2006 12:46:12 -0000      1.152
+++ src/tar.c   3 Jul 2006 08:23:57 -0000
@@ -2240,6 +2240,9 @@ main (int argc, char **argv)
   /* Make sure we have first three descriptors available */
   stdopen ();
 
+  /* Close all open files, except for the first three */
+  closeopen ();
+  
   /* Pre-allocate a few structures.  */
 
   allocated_archive_names = 10;