bug-coreutils
[Top][All Lists]
Advanced

[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]

Re: Core utils 5.0 - "csplit" pipe memory error


From: Jim Meyering
Subject: Re: Core utils 5.0 - "csplit" pipe memory error
Date: Sat, 27 Sep 2003 09:04:00 +0200

Thanks again for the report.
I was able to reproduce it consistently with this:

  (while :; do echo =; yes jjjjjjjjjjjjjjjjjjjjjjjjjj|head -n10000; done) \
   | csplit - '/=$/' '{*}'

I got output like this:

  0
  270002
  253910
  csplit: memory exhausted

Here's the patch:

        Don't exhaust virtual memory when processing large inputs.
        Fix this by removing csplit's internal free-list management;
        instead rely on malloc for that.

        * src/csplit.c (free_list): Remove global.
        (clear_all_line_control): Remove function.
        (get_new_buffer): Always use create_new_buffer to obtain a
        new buffer, rather than searching free_list.
        (free_buffer): Just call free.
        Reported by Nikola Milutinovic.

Index: src/csplit.c
===================================================================
RCS file: /fetish/cu/src/csplit.c,v
retrieving revision 1.119
retrieving revision 1.120
diff -u -p -u -r1.119 -r1.120
--- src/csplit.c        23 Sep 2003 21:57:13 -0000      1.119
+++ src/csplit.c        27 Sep 2003 06:57:07 -0000      1.120
@@ -134,9 +134,6 @@ static const unsigned int bytes_to_octal
 /* Input file descriptor. */
 static int input_desc = 0;
 
-/* List of available buffers. */
-static struct buffer_record *free_list = NULL;
-
 /* Start of buffer list. */
 static struct buffer_record *head = NULL;
 
@@ -302,17 +299,6 @@ clear_line_control (struct line *p)
   p->retrieve_index = 0;
 }
 
-/* Initialize all line records in B. */
-
-static void
-clear_all_line_control (struct buffer_record *b)
-{
-  struct line *l;
-
-  for (l = b->line_start; l; l = l->next)
-    clear_line_control (l);
-}
-
 /* Return a new, initialized line record. */
 
 static struct line *
@@ -436,7 +422,6 @@ create_new_buffer (unsigned int size)
 static struct buffer_record *
 get_new_buffer (unsigned int min_size)
 {
-  struct buffer_record *p, *q;
   struct buffer_record *new_buffer; /* Buffer to return. */
   unsigned int alloc_size;     /* Actual size that will be requested. */
 
@@ -444,34 +429,7 @@ get_new_buffer (unsigned int min_size)
   while (min_size > alloc_size)
     alloc_size += INCR_SIZE;
 
-  if (free_list == NULL)
-    new_buffer = create_new_buffer (alloc_size);
-  else
-    {
-      /* Use first-fit to find a buffer. */
-      p = new_buffer = NULL;
-      q = free_list;
-
-      do
-       {
-         if (q->bytes_alloc >= min_size)
-           {
-             if (p == NULL)
-               free_list = q->next;
-             else
-               p->next = q->next;
-             break;
-           }
-         p = q;
-         q = q->next;
-       }
-      while (q);
-
-      new_buffer = (q ? q : create_new_buffer (alloc_size));
-
-      new_buffer->curr_line = new_buffer->line_start;
-      clear_all_line_control (new_buffer);
-    }
+  new_buffer = create_new_buffer (alloc_size);
 
   new_buffer->num_lines = 0;
   new_buffer->bytes_used = 0;
@@ -481,13 +439,10 @@ get_new_buffer (unsigned int min_size)
   return new_buffer;
 }
 
-/* Add buffer BUF to the list of free buffers. */
-
 static void
 free_buffer (struct buffer_record *buf)
 {
-  buf->next = free_list;
-  free_list = buf;
+  free (buf->buffer);
 }
 
 /* Append buffer BUF to the linked list of buffers that contain




reply via email to

[Prev in Thread] Current Thread [Next in Thread]