[PATCH] stdbuf work in progress

[Pádraig Brady]

I've started to work on the stdbuf util referenced in this thread:
http://lists.gnu.org/archive/html/bug-coreutils/2008-11/threads.html#00134
I've a few questions...

The main one is should I introduce a dependency on libtool or use
something else. I'll be installing an unversioned shared lib (plugin)
on ELF systems, so I'm not sure the libtool dependency is desired?
Perhaps it's needed considering cygwin supports LD_PRELOAD?

The stdbuf util is built only on ELF systems. Is my check for such
systems in configure.ac appropriate?
(Note libstdbuf.so is built unconditionally for the moment,
and is not installed. But the lib will be used if it's in the
same directory as the stdbuf util).

I've added code to stdbuf to try and specify the full path
to the lib to LD_PRELOAD, rather than letting the dynamic
linker search for it. This is so as to remove the need to
add our lib install dir to the system search path, and
also simplifies testing as we can run stdbuf through the path
without having to setup LD_LIBRARY_PATH. Any gotchas with doing this?

I've currently split the util and shared lib into stdbuf.c
and libstdbuf.c. It might be better to just have a stdbuf.c
and build both binary and shared lib from that?

cheers,
Pádraig.

---
 Makefile.am     |    2 +-
 README          |    6 +-
 configure.ac    |   10 ++
 man/.gitignore  |    1 +
 man/Makefile.am |    1 +
 man/stdbuf.x    |   13 ++
 src/.gitignore  |    2 +
 src/Makefile.am |   10 +-
 src/libstdbuf.c |   73 ++++++++++++
 src/stdbuf.c    |  354 +++++++++++++++++++++++++++++++++++++++++++++++++++++++
 10 files changed, 465 insertions(+), 7 deletions(-)
 create mode 100644 man/stdbuf.x
 create mode 100644 src/libstdbuf.c
 create mode 100644 src/stdbuf.c

diff --git a/Makefile.am b/Makefile.am
index 3964888..847aec5 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -1,6 +1,6 @@
 # Make coreutils.                                      -*-Makefile-*-

-# Copyright (C) 1990, 1993-2008 Free Software Foundation, Inc.
+# Copyright (C) 1990, 1993-2009 Free Software Foundation, Inc.

 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
diff --git a/README b/README
index 157432e..323ec50 100644
--- a/README
+++ b/README
@@ -13,9 +13,9 @@ The programs that can be built with this package are:
   link ln logname ls md5sum mkdir mkfifo mknod mktemp mv nice nl nohup
   od paste pathchk pinky pr printenv printf ptx pwd readlink rm rmdir
   runcon seq sha1sum sha224sum sha256sum sha384sum sha512sum shred shuf
-  sleep sort split stat stty su sum sync tac tail tee test timeout touch tr
-  true truncate tsort tty uname unexpand uniq unlink uptime users vdir wc who
-  whoami yes
+  sleep sort split stat stdbuf stty su sum sync tac tail tee test timeout
+  touch tr true truncate tsort tty uname unexpand uniq unlink uptime users
+  vdir wc who whoami yes

 See the file NEWS for a list of major changes in the current release.

diff --git a/configure.ac b/configure.ac
index 99452c8..2485858 100644
--- a/configure.ac
+++ b/configure.ac
@@ -284,6 +284,16 @@ if test $gl_cv_list_mounted_fs = yes && test 
$gl_cv_fs_space = yes; then
   gl_ADD_PROG([optional_bin_progs], [df])
 fi

+# Limit stdbuf to ELF systems
+AC_MSG_CHECKING([whether this is an ELF system])
+AC_EGREP_CPP([yes], [#if __ELF__
+yes
+#endif], elf_sys=yes, elf_sys=no)
+AC_MSG_RESULT([$elf_sys])
+if test "$elf_sys" = "yes"; then
+  gl_ADD_PROG([optional_bin_progs], [stdbuf])
+fi
+
 ############################################################################
 mk="$srcdir/src/Makefile.am"
 # Extract all literal names from the definition of $(EXTRA_PROGRAMS)
diff --git a/man/.gitignore b/man/.gitignore
index e9e270d..1085ff0 100644
--- a/man/.gitignore
+++ b/man/.gitignore
@@ -72,6 +72,7 @@ sleep.1
 sort.1
 split.1
 stat.1
+stdbuf.1
 stty.1
 su.1
 sum.1
diff --git a/man/Makefile.am b/man/Makefile.am
index 415eb82..d496940 100644
--- a/man/Makefile.am
+++ b/man/Makefile.am
@@ -105,6 +105,7 @@ sleep.1:    $(common_dep)   $(srcdir)/sleep.x       
../src/sleep.c
 sort.1:                $(common_dep)   $(srcdir)/sort.x        ../src/sort.c
 split.1:       $(common_dep)   $(srcdir)/split.x       ../src/split.c
 stat.1:                $(common_dep)   $(srcdir)/stat.x        ../src/stat.c
+stdbuf.1:      $(common_dep)   $(srcdir)/stdbuf.x      ../src/stdbuf.c
 stty.1:                $(common_dep)   $(srcdir)/stty.x        ../src/stty.c
 su.1:          $(common_dep)   $(srcdir)/su.x          ../src/su.c
 sum.1:         $(common_dep)   $(srcdir)/sum.x         ../src/sum.c
diff --git a/man/stdbuf.x b/man/stdbuf.x
new file mode 100644
index 0000000..bdd132c
--- /dev/null
+++ b/man/stdbuf.x
@@ -0,0 +1,13 @@
+'\" Copyright (C) 2009 Free Software Foundation, Inc.
+'\"
+'\" This is free software.  You may redistribute copies of it under the terms
+'\" of the GNU General Public License <http://www.gnu.org/licenses/gpl.html>.
+'\" There is NO WARRANTY, to the extent permitted by law.
+[NAME]
+stdbuf \- Run COMMAND, with modified buffering operations for its standard 
streams.
+[DESCRIPTION]
+.\" Add any additional description here
+[EXAMPLES]
+.B tail -f access.log | stdbuf -oL cut -d \(aq \(aq -f1 | uniq
+.br
+This will immedidately display unique entries from access.log
diff --git a/src/.gitignore b/src/.gitignore
index bc14523..f2886de 100644
--- a/src/.gitignore
+++ b/src/.gitignore
@@ -40,6 +40,7 @@ hostname
 id
 join
 kill
+libstdbuf.so
 libver.a
 link
 ln
@@ -81,6 +82,7 @@ sleep
 sort
 split
 stat
+stdbuf
 stty
 su
 sum
diff --git a/src/Makefile.am b/src/Makefile.am
index 555700b..9cee4cc 100644
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -1,6 +1,6 @@
 ## Process this file with automake to produce Makefile.in -*-Makefile-*-

-## Copyright (C) 1990, 1991, 1993-2008 Free Software Foundation, Inc.
+## Copyright (C) 1990, 1991, 1993-2009 Free Software Foundation, Inc.

 ## This program is free software: you can redistribute it and/or modify
 ## it under the terms of the GNU General Public License as published by
@@ -24,7 +24,7 @@ no_install__progs = \
   arch hostname su

 build_if_possible__progs = \
-  chroot df hostid nice pinky stty su uname uptime users who
+  chroot df hostid nice pinky stdbuf stty su uname uptime users who

 AM_CFLAGS = $(WARN_CFLAGS) $(WERROR_CFLAGS)

@@ -44,9 +44,12 @@ EXTRA_PROGRAMS = \
   test timeout true truncate tty whoami yes \
   base64

+libstdbuf.so: libstdbuf.c
+       gcc -fPIC -shared -o libstdbuf.so libstdbuf.c
+
 bin_PROGRAMS = $(OPTIONAL_BIN_PROGS)

-noinst_PROGRAMS = setuidgid getlimits
+noinst_PROGRAMS = setuidgid getlimits libstdbuf.so

 noinst_HEADERS = \
   chown-core.h \
@@ -87,6 +90,7 @@ du_LDADD = $(LDADD)
 getlimits_LDADD = $(LDADD)
 ptx_LDADD = $(LDADD)
 split_LDADD = $(LDADD)
+stdbuf_LDADD = $(LDADD)
 timeout_LDADD = $(LDADD)
 truncate_LDADD = $(LDADD)

diff --git a/src/libstdbuf.c b/src/libstdbuf.c
new file mode 100644
index 0000000..34e0b58
--- /dev/null
+++ b/src/libstdbuf.c
@@ -0,0 +1,73 @@
+/* Assume all input validation has been done by stdbuf.  */
+
+#include <stdio.h>
+//#include <stdio_ext.h>
+#include <stdlib.h>
+#include <errno.h>
+
+/*
+    Note currently for glibc (2.3.5) the following call does not change the
+    the buffer size, and more problematically does not give any indication
+    that the new size request was ignored:
+
+        setvbuf(stdout,(char*)NULL,_IOFBF,8192);
+
+    The ISO C99 standard section 7.19.5.6 on the setvbuf function says:
+
+    ... If buf is not a null pointer, the array it points to _may_ be used
+    instead of a buffer allocated by the setvbuf function and the argument
+    size specifies the size of the array; otherwise, size _may_ determine
+    the size of a buffer allocated by the setvbuf function. ...
+
+    Obviously some interpret the above to mean setvbuf(....,size)
+    is only a hint from the application which I don't agree with.
+
+    FreeBSD's libc seems more sensible in this regard. From the man page:
+
+    The size argument may be given as zero to obtain deferred optimal-size
+    buffer allocation as usual.  If it is not zero, then except for
+    unbuffered files, the buf argument should point to a buffer at least size
+    bytes long; this buffer will be used instead of the current buffer.  (If
+    the size argument is not zero but buf is NULL, a buffer of the given size
+    will be allocated immediately, and released on close.  This is an extension
+    to ANSI C; portable code should use a size of 0 with any NULL buffer.)
+*/
+
+static void
+apply_mode(FILE* stream, const char* mode)
+{
+  char* buf = NULL;
+  int setvbuf_mode;
+  size_t size = 0;
+
+  if (*mode == '0')
+    setvbuf_mode=_IONBF;
+  else if (*mode == 'L')
+    setvbuf_mode=_IOLBF; //TODO: should we allow 1ML
+  else
+    {
+      setvbuf_mode=_IOFBF;
+      /* Note this is done elsewhere in coreutils:
+        verify (SIZE_MAX <= ULONG_MAX);  */
+      size = strtoul(mode, NULL, 10);
+      buf = malloc (size); /* will be free by fclose()  */
+    }
+
+  if (setvbuf (stream, buf, setvbuf_mode, size) != 0)
+    fprintf(stderr, "Warning: error setting %d to %s\n", fileno(stream), mode);
+  //fprintf(stderr,"specified buf_size = %d\n",__fbufsize(stream));
+}
+
+__attribute__ ((constructor)) static void
+stdbuf ()
+{
+  char* e_mode=getenv("_STDBUF_E");
+  char* i_mode=getenv("_STDBUF_I");
+  char* o_mode=getenv("_STDBUF_O");
+  if (e_mode) /* Do first so can write errors to stderr  */
+    apply_mode(stderr, e_mode);
+  if (i_mode)
+    apply_mode(stdin, i_mode);
+  if (o_mode)
+    apply_mode(stdout, o_mode);
+}
diff --git a/src/stdbuf.c b/src/stdbuf.c
new file mode 100644
index 0000000..8472412
--- /dev/null
+++ b/src/stdbuf.c
@@ -0,0 +1,354 @@
+/* stdbuf -- setup the standard streams for a command
+   Copyright (C) 2009 Free Software Foundation, Inc.
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.  */
+
+/* Written by Pádraig Brady.  */
+
+/* XXX: note df, nice, stdbuf etc. are only built on certain conditions.
+ * However tests seem to be run unconditionally? */
+
+#include <config.h>
+#include <configmake.h>
+#include <stdio.h>
+#include <getopt.h>
+#include <sys/types.h>
+
+#include "system.h"
+#include "error.h"
+#include "posixver.h"
+#include "quote.h"
+#include "xstrtol.h"
+
+/* The official name of this program (e.g., no `g' prefix).  */
+#define PROGRAM_NAME "stdbuf"
+#define LIB_NAME "libstdbuf.so" //XXX: don't hardcode?
+
+#define AUTHORS proper_name_utf8 ("Padraig Brady", "P\303\241draig Brady")
+
+static char *program_path;
+
+extern char **environ;
+
+/* Set size to the value of STR, interpreted as a decimal integer,
+   optionally multiplied by various values.
+   Return -1 on error, 0 on success.
+
+   This supports dd BLOCK size suffixes.
+   Note we don't support dd's b=512, c=1, w=2 or 21x512MiB formats.  */
+static int
+parse_size (char const *str, size_t *size)
+{
+  enum strtol_error e;
+  uintmax_t tmp_size;
+  e = xstrtoumax (str, NULL, 10, &tmp_size, "EGkKMPTYZ0");
+  if (e == LONGINT_OK && tmp_size > SIZE_MAX)
+    e = LONGINT_OVERFLOW;
+
+  if (e == LONGINT_OK)
+    {
+      errno = 0;
+      *size = tmp_size;
+      return 0;
+    }
+
+  errno = (e == LONGINT_OVERFLOW ? EOVERFLOW : 0);
+  return -1;
+}
+
+static struct option const longopts[] =
+{
+  {"input", required_argument, NULL, 'i'},
+  {"output", required_argument, NULL, 'o'},
+  {"error", required_argument, NULL, 'e'},
+  {GETOPT_HELP_OPTION_DECL},
+  {GETOPT_VERSION_OPTION_DECL},
+  {NULL, 0, NULL, 0}
+};
+
+void
+usage (int status)
+{
+  if (status != EXIT_SUCCESS)
+    fprintf (stderr, _("Try `%s --help' for more information.\n"),
+             program_name);
+  else
+    {
+      printf (_("Usage: %s OPTION... COMMAND\n"), program_name);
+      fputs (_("\
+Run COMMAND, with modified buffering operations for its standard streams.\n\
+\n\
+"), stdout);
+      fputs (_("\
+Mandatory arguments to long options are mandatory for short options too.\n\
+"), stdout);
+      fputs (_("\
+  -i, --input=MODE  Setup standard input stream buffering\n\
+  -o, --output=MODE Setup standard output stream buffering\n\
+  -e, --error=MODE  Setup standard error stream buffering\n\
+"), stdout);
+      fputs (HELP_OPTION_DESCRIPTION, stdout);
+      fputs (VERSION_OPTION_DESCRIPTION, stdout);
+      fputs (_("\n\
+If MODE is `L' then corresponding stream will be line buffered.\n\
+This option is invalid with standard input.\n"), stdout);
+      fputs (_("\n\
+If MODE is `0' then corresponding stream will be unbuffered.\n\
+"), stdout);
+      fputs (_("\n\
+Otherwise MODE is a number which may be followed by one of the following:\n\
+KB 1000, K 1024, MB 1000*1000, M 1024*1024, and so on for G, T, P, E, Z, Y.\n\
+In this case the corresponding stream will be fully buffered with the buffer\n\
+size set to MODE bytes.\n\
+"), stdout);
+      fputs (_("\n\
+NOTE: If COMMAND controls the buffering of its standard streams (like `tee')\n\
+then this will override corresponding settings changed by this command.\n\
+Also some filters (like `dd' and `cat' etc.) don't use streams for I/O,\n\
+and are thus unaffected.\n\
+"), stdout);
+      emit_bug_reporting_address ();
+    }
+  exit (status);
+}
+
+static struct {
+    size_t size;
+    int optc;
+    char *optarg;
+} stdbuf[3];
+
+static int
+optc_to_fileno (int c)
+{
+  if (c == 'e')
+    return STDERR_FILENO;
+  if (c == 'i')
+    return STDIN_FILENO;
+  if (c == 'o')
+    return STDOUT_FILENO;
+  return -1;
+}
+
+/* Internal error  */
+#define EXIT_CANCELED 125
+
+static void
+set_LD_PRELOAD(void)
+{
+  int ret;
+  char* old_libs = getenv("LD_PRELOAD");
+  char* LD_PRELOAD;
+
+  //Note this would auto add the appropriate search path for "libstdbuf.so":
+  //  gcc stdbuf.c -Wl,-rpath,'$ORIGIN' -Wl,-rpath,$PKGLIBDIR
+  //However we want the lookup done for the exec'd command not stdbuf.
+  //
+  //Not linked against, but is possibly searched for, so add to LIBDIR rather 
than LIBEXECDIR
+  char* search_path[] = {
+      program_path,
+      PKGLIBDIR,
+      "", /* sys default */
+      NULL
+  };
+
+  char** path = search_path;
+  char* libstdbuf;
+
+  do
+    {
+      struct stat sb;
+
+      if (!**path) /* system default  */
+        {
+          libstdbuf = xstrdup(LIB_NAME);
+          break;
+        }
+      ret = asprintf (&libstdbuf, "%s/%s", *path, LIB_NAME);
+      if (ret < 0)
+        xalloc_die ();
+      if (stat (libstdbuf, &sb) == 0) /* file_exists  */
+        break;
+      free (libstdbuf);
+    } while (*++path);
+
+  //XXX: ask do we need to support libstdbuf.dll, c:, '\' separators etc.
+
+  if (old_libs)
+    ret = asprintf (&LD_PRELOAD, "LD_PRELOAD=%s:%s", old_libs, libstdbuf);
+  else
+    ret = asprintf (&LD_PRELOAD, "LD_PRELOAD=%s", libstdbuf);
+
+  if (ret < 0)
+    xalloc_die ();
+
+  free(libstdbuf);
+
+  ret = putenv(LD_PRELOAD);
+
+  if (ret < 0)
+    xalloc_die ();
+}
+
+/* argv[0] can be anything really, but generally it contains
+   the path to the executable or just a name if it was executed
+   using $PATH. In the latter case to get the path we can:
+   search getenv("PATH"), readlink("/prof/self/exe"), getenv("_"),
+   dladdr(), pstat_getpathname(), etc.  */
+void set_program_path(const char* arg)
+{
+  if (strchr(arg, '/')) /* Use absolute or relative paths directly.  */
+    {
+      program_path = dir_name (arg);
+    }
+  else
+    {
+      char* path;
+      char tmppath[PATH_MAX+1];
+      int len = readlink("/proc/self/exe", tmppath, sizeof(tmppath)-1);
+      if (len > 0)
+       {
+         tmppath[len]='\0';
+         program_path = dir_name (tmppath);
+       }
+      else if ((path = getenv("PATH")))
+       {
+          char* dir;
+          path = strdup(path);
+          for (dir = strtok(path, ":"); dir != NULL; dir = strtok(NULL, ":"))
+            {
+             int req = snprintf(tmppath, sizeof(tmppath), "%s/%s", dir, arg);
+             if (req >= sizeof(tmppath))
+               {
+                 error (0, 0, _("Path truncated when looking for %s"),
+                        quote(arg));
+               }
+              else if (access(tmppath, X_OK) == 0)
+                {
+                 program_path = dir_name (tmppath);
+                  break;
+                }
+            }
+          free(path);
+       }
+    }
+}
+
+int
+main (int argc, char **argv)
+{
+  int c,i;
+
+  initialize_main (&argc, &argv);
+  set_program_name (argv[0]);
+  setlocale (LC_ALL, "");
+  bindtextdomain (PACKAGE, LOCALEDIR);
+  textdomain (PACKAGE);
+
+  atexit (close_stdout);
+
+  /* Try to get path to this program, so we can load
+     libstdbuf.so from the same directory as a convenience.  */
+  set_program_path (argv[0]);
+  if (!program_path)
+    program_path = strdup(PKGLIBDIR); /* Need to init to non NULL.  */
+
+  while ((c = getopt_long (argc, argv, "+i:o:e:", longopts, NULL)) != -1)
+    {
+      int opt_fileno;
+
+      switch (c)
+        {
+        /* Old McDonald had a farm ei...  */
+        case 'e':
+        case 'i':
+        case 'o':
+          opt_fileno = optc_to_fileno(c);
+          stdbuf[opt_fileno].optc=c;
+          while (isspace (*optarg))
+            optarg++;
+          stdbuf[opt_fileno].optarg = optarg;
+          if (c == 'i' && *optarg == 'L')
+            {
+              /* -oL will be by far the most common use of this utility,
+               * but one could easily think -iL might have the same affect,
+               * so disallow it as it could be confusing. */
+              error (0, 0, _("Line buffering stdin is meaninglesss"));
+              usage (EXIT_CANCELED);
+            }
+
+          if (!STREQ(optarg,"L") &&
+              parse_size (optarg, &stdbuf[opt_fileno].size) == -1)
+            error (EXIT_CANCELED, errno, _("invalid mode %s"),
+                   quote (optarg));
+
+          break;
+
+        case_GETOPT_HELP_CHAR;
+
+        case_GETOPT_VERSION_CHAR (PROGRAM_NAME, AUTHORS);
+
+        default:
+          usage (EXIT_CANCELED);
+        }
+    }
+
+  argv += optind;
+  argc -= optind;
+
+  /* must specify at least 1 command.  */
+  if (argc < 1)
+    {
+      error (0, 0, _("missing operand"));
+      usage (EXIT_CANCELED);
+    }
+
+  /* XXX: Should we mandate at least one option?  */
+
+  set_LD_PRELOAD();
+
+  /* Populate environ with _STDBUF_I=$MODE _STDBUF_O=$MODE _STDBUF_E=$MODE  */
+  for (i = 0; i < sizeof (stdbuf) / sizeof *(stdbuf); i++)
+    {
+      if (stdbuf[i].optarg)
+        {
+          char *var;
+          int ret;
+
+          if (*stdbuf[i].optarg == 'L')
+            ret = asprintf (&var, "%s%c=L", "_STDBUF_",
+                           toupper(stdbuf[i].optc));
+          else
+            ret = asprintf (&var, "%s%c=%"PRIuMAX, "_STDBUF_",
+                           toupper(stdbuf[i].optc), (uintmax_t) 
stdbuf[i].size);
+          if (ret < 0)
+            xalloc_die ();
+          putenv(var);
+        }
+    }
+
+  execvp (*argv, argv);
+
+  {
+    int exit_status = (errno == ENOENT ? EXIT_ENOENT : EXIT_CANNOT_INVOKE);
+    error (0, errno, "%s", *argv);
+    exit (exit_status);
+  }
+}
+
+/*
+ * Local variables:
+ *  indent-tabs-mode: nil
+ * End:
+ */
-- 
1.5.3.6