[Top][All Lists]
[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]
Re: Core utils 5.0 - "csplit" pipe memory error
From: |
Jim Meyering |
Subject: |
Re: Core utils 5.0 - "csplit" pipe memory error |
Date: |
Sat, 27 Sep 2003 09:04:00 +0200 |
Thanks again for the report.
I was able to reproduce it consistently with this:
(while :; do echo =; yes jjjjjjjjjjjjjjjjjjjjjjjjjj|head -n10000; done) \
| csplit - '/=$/' '{*}'
I got output like this:
0
270002
253910
csplit: memory exhausted
Here's the patch:
Don't exhaust virtual memory when processing large inputs.
Fix this by removing csplit's internal free-list management;
instead rely on malloc for that.
* src/csplit.c (free_list): Remove global.
(clear_all_line_control): Remove function.
(get_new_buffer): Always use create_new_buffer to obtain a
new buffer, rather than searching free_list.
(free_buffer): Just call free.
Reported by Nikola Milutinovic.
Index: src/csplit.c
===================================================================
RCS file: /fetish/cu/src/csplit.c,v
retrieving revision 1.119
retrieving revision 1.120
diff -u -p -u -r1.119 -r1.120
--- src/csplit.c 23 Sep 2003 21:57:13 -0000 1.119
+++ src/csplit.c 27 Sep 2003 06:57:07 -0000 1.120
@@ -134,9 +134,6 @@ static const unsigned int bytes_to_octal
/* Input file descriptor. */
static int input_desc = 0;
-/* List of available buffers. */
-static struct buffer_record *free_list = NULL;
-
/* Start of buffer list. */
static struct buffer_record *head = NULL;
@@ -302,17 +299,6 @@ clear_line_control (struct line *p)
p->retrieve_index = 0;
}
-/* Initialize all line records in B. */
-
-static void
-clear_all_line_control (struct buffer_record *b)
-{
- struct line *l;
-
- for (l = b->line_start; l; l = l->next)
- clear_line_control (l);
-}
-
/* Return a new, initialized line record. */
static struct line *
@@ -436,7 +422,6 @@ create_new_buffer (unsigned int size)
static struct buffer_record *
get_new_buffer (unsigned int min_size)
{
- struct buffer_record *p, *q;
struct buffer_record *new_buffer; /* Buffer to return. */
unsigned int alloc_size; /* Actual size that will be requested. */
@@ -444,34 +429,7 @@ get_new_buffer (unsigned int min_size)
while (min_size > alloc_size)
alloc_size += INCR_SIZE;
- if (free_list == NULL)
- new_buffer = create_new_buffer (alloc_size);
- else
- {
- /* Use first-fit to find a buffer. */
- p = new_buffer = NULL;
- q = free_list;
-
- do
- {
- if (q->bytes_alloc >= min_size)
- {
- if (p == NULL)
- free_list = q->next;
- else
- p->next = q->next;
- break;
- }
- p = q;
- q = q->next;
- }
- while (q);
-
- new_buffer = (q ? q : create_new_buffer (alloc_size));
-
- new_buffer->curr_line = new_buffer->line_start;
- clear_all_line_control (new_buffer);
- }
+ new_buffer = create_new_buffer (alloc_size);
new_buffer->num_lines = 0;
new_buffer->bytes_used = 0;
@@ -481,13 +439,10 @@ get_new_buffer (unsigned int min_size)
return new_buffer;
}
-/* Add buffer BUF to the list of free buffers. */
-
static void
free_buffer (struct buffer_record *buf)
{
- buf->next = free_list;
- free_list = buf;
+ free (buf->buffer);
}
/* Append buffer BUF to the linked list of buffers that contain