[PATCH v2] build: Option for building all tools in a single binary

[Alex Deymo]

Add the --enable-single-binary option to the configure file. When
enabled, this option builds a single binary file with all the
selected tools in it. Which tools gets executed depends on the value
of argv[0] which is intended to use together with symlinks to the
single program.

This setup reduces significantly the size of the installed binaries
since all the redundant code from lib/libcoreutils.a installed on
every one of the more than 100 binaries will be only once. This also
favors the cache, since the result binary is smaller, so after
running a couple of commands all the code is cached.

When installing, the makefile will create symlinks from each
configured tool to a single "coreutils" binary installed on the
same directory.
---
New in v2:
 * More comments on coreutils.c
 * Added some documentation to the gen-single-binary.sh
 * Added a flag to get-lists-of-program.sh to just print the list
   of programs. It feels less dirty that the previous stuff.
 * Fixed/improved gen-single-binary.sh to include all the required
   flags.

Tested this patch with:
./configure --prefix=`pwd`/foo --enable-single-binary && make && make install
...and "it works for me".

I think this is ready for patch review if you are not against having this
feature on coreutils. I'd like to remind you that this is a configure
flag that's disabled by default, so the tradeoff between having
"true" linking against pthreads and the 5x disk space required is what
determines if you want to enable this. In our case (ChromeOS) we will
probably enable it. So here you have the patch =). I also think that
having this feature will increase performance rather than decrease it
because less disk space means faster access/less cached data. But
again, that's up to the user.

 Makefile.am                        |  6 +++
 bootstrap                          |  3 ++
 build-aux/gen-lists-of-programs.sh |  7 +++
 build-aux/gen-single-binary.sh     | 99 ++++++++++++++++++++++++++++++++++++++
 configure.ac                       | 26 ++++++++++
 gnulib                             |  2 +-
 src/coreutils.c                    | 82 +++++++++++++++++++++++++++++++
 src/getlimits.c                    |  2 +
 src/local.mk                       | 31 ++++++++++++
 9 files changed, 257 insertions(+), 1 deletion(-)
 create mode 100755 build-aux/gen-single-binary.sh
 create mode 100644 src/coreutils.c

diff --git a/Makefile.am b/Makefile.am
index e88dc9c..eb467c7 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -180,6 +180,12 @@ check-git-hook-script-sync:
        rm -rf $$t;                                                     \
        test $$fail = 0
 
+# If we are building a single-binary, create symlinks for them.
+install-exec-hook:
+       @for i in $(single_binary_progs); do \
+               $(LN_S) -s coreutils $(DESTDIR)$(bindir)/$$i$(EXEEXT); \
+       done
+
 noinst_LIBRARIES =
 MOSTLYCLEANFILES =
 CLEANFILES =
diff --git a/bootstrap b/bootstrap
index a3e68f0..87f97a2 100755
--- a/bootstrap
+++ b/bootstrap
@@ -807,6 +807,9 @@ version_controlled_file() {
   fi
 }
 
+# Regenerate src/single-binary.sh
+./build-aux/gen-single-binary.sh src/local.mk
+
 # Avoid boostrap failure with gettext/autopoint bug in version 0.18.3.1
 # http://lists.gnu.org/archive/html/coreutils/2013-11/msg00038.html
 # Remove in 2015 when distros have upgraded to >= 0.18.3.2
diff --git a/build-aux/gen-lists-of-programs.sh 
b/build-aux/gen-lists-of-programs.sh
index bf63ee3..c95e598 100755
--- a/build-aux/gen-lists-of-programs.sh
+++ b/build-aux/gen-lists-of-programs.sh
@@ -17,6 +17,7 @@ set -e
 # use "--enable-install-program=A,B" when invoking configure.
 disabled_by_default_progs='
     arch
+    coreutils
     hostname
 '
 
@@ -178,6 +179,12 @@ END
       echo default__progs += $progsdir/$p
     done
     ;;
+  1,--list-progs)
+    for p in $disabled_by_default_progs $build_if_possible_progs \
+        $normal_progs; do
+      echo $p
+    done
+    ;;
   *)
     echo "$0: invalid usage" >&2; exit 2
     ;;
diff --git a/build-aux/gen-single-binary.sh b/build-aux/gen-single-binary.sh
new file mode 100755
index 0000000..4df1180
--- /dev/null
+++ b/build-aux/gen-single-binary.sh
@@ -0,0 +1,99 @@
+#!/bin/sh
+
+# Generate the list of rules for the single-binary option based on all the 
other
+# binaries found in src/local.mk.
+#
+# We need to duplicate the specific rules to build each program into a new
+# static library target. We can't reuse the existing target since we need to
+# create a .a file instead of linking the program. We can't do this at
+# ./configure since the file names need to available when automake runs to let
+# it generate all the required rules in Makefile.in. The configure step will
+# select which ones will be used to build, but they need to be generated
+# beforehand.
+#
+# Instead of maintaining a duplicated list of rules, we generate the
+# single-binary required rules based on the normal configuration found on
+# src/local.mk with this script.
+
+if [ "x$1" == "x" ]; then
+  echo "Usage: $0 path/to/src/local.mk" >&2
+  exit 1
+fi
+
+set -e
+
+LOCAL_MK=$1
+SINGLE_BINARY_MK="`dirname $1`/single-binary.mk"
+GEN_LISTS_OF_PROGRAMS="`dirname $0`/gen-lists-of-programs.sh"
+
+ALL_PROGRAMS=$($GEN_LISTS_OF_PROGRAMS --list-progs \
+    | grep -v -F -e coreutils -e libstdbuf.so \
+    | tr '.[' '_')
+
+# Compute default SOURCES. automake will assume the source file for the
+# src_${cmd} target to be src/${cmd}.c, but we will add rules to generate
+# the lib src_libsinglebin_${cmd}_a which won't match the autogenerated source
+# file. This loop will initialize the default source file and will be reset
+# later if needed.
+for cmd in $ALL_PROGRAMS; do
+  eval "src_${cmd}_SOURCES=src/${cmd}.c"
+done
+
+# Load actual values from src/local.mk. This will read all the variables from
+# the local.mk matching the src_${cmd}_... case.
+while read l; do
+  if echo "$l" | grep -E '^src_\w+ +\+?=' > /dev/null; then
+    var=$(echo $l | cut -f 1 -d ' ')
+    value=$(echo $l | cut -f 2- -d =)
+    if [ "$value" != " \$(LDADD)" ]; then
+      oldvalue=""
+      if echo $l | grep -F '+=' >/dev/null; then
+        eval "oldvalue=\${$var}"
+      fi
+      eval "$var='$oldvalue "${value//\'/\'\"\'\"\'}"'"
+    fi
+  fi
+done < $LOCAL_MK
+
+me=`echo "$0" | sed 's,.*/,,'`
+msg="Automatically generated by $me.  DO NOT EDIT BY HAND!"
+echo "## $msg" >$SINGLE_BINARY_MK
+
+for cmd in $ALL_PROGRAMS; do
+  echo "# Command $cmd"
+  echo noinst_LIBRARIES += src/libsinglebin_${cmd}.a
+  base="src_libsinglebin_${cmd}_a"
+  # SOURCES
+  var=src_${cmd}_SOURCES
+  eval "value=\$$var"
+  echo "${base}_SOURCES = $value"
+  echo "EXTRA_src_coreutils_SOURCES += $value"
+  echo "EXTRA_src_coreutils_DEPENDENCIES += src/libsinglebin_${cmd}.a"
+
+  # CFLAGS
+  # Hack any other program defining a main() replacing its main by
+  # _single_binary_main_$PROGRAM_NAME.
+  echo "${base}_CFLAGS = -Dmain=_single_binary_main_${cmd}" \
+       "-Dusage=_usage_${cmd} \$(src_coreutils_CFLAGS)"
+  var=src_${cmd}_CFLAGS
+  eval "value=\$$var"
+  if [ "x$value" != "x" ]; then
+    echo "${base}_CFLAGS += $value"
+  fi
+
+  # CPPFLAGS
+  var=src_${cmd}_CPPFLAGS
+  eval "value=\$$var"
+  if [ "x$value" != "x" ]; then
+    echo "${base}_CPPFLAGS = $value"
+  fi
+
+  # LIBADD
+  var=src_${cmd}_LDADD
+  eval "value=\$$var"
+  if [ "x$value" != "x" ]; then
+    echo "src_coreutils_LDADD += $value"
+  fi
+done >> $SINGLE_BINARY_MK
+
+exit 0
diff --git a/configure.ac b/configure.ac
index 7c210d9..81e21a3 100644
--- a/configure.ac
+++ b/configure.ac
@@ -221,6 +221,17 @@ if test "$gl_gcc_warnings" = yes; then
   AC_SUBST([GNULIB_TEST_WARN_CFLAGS])
 fi
 
+AC_ARG_ENABLE([single-binary],
+  [AS_HELP_STRING([--enable-single-binary],
+     [Compile all the tools in a single binary (reduces overall size)])],
+  [case $enableval in
+     yes|no) ;;
+     *)      AC_MSG_ERROR([bad value $enableval for single-binary option]) ;;
+   esac
+   gl_single_binary=$enableval],
+  [gl_single_binary=no]
+)
+
 AC_FUNC_FORK
 
 optional_bin_progs=
@@ -487,6 +498,21 @@ man1_MANS=`
 # a distribution tarball.
 EXTRA_MANS=`for p in $no_install_progs_default; do echo man/$p.1; done`
 
+# Replace all the programs by the single binary and simlinks if specified.
+single_binary_progs=
+single_binary_libs=
+if test "$gl_single_binary" = yes; then
+  single_binary_progs=`echo $optional_bin_progs`
+  single_binary_libs=`
+    for p in $single_binary_progs; do
+      echo -n "src/libsinglebin_$p.a " | tr "\133" _
+    done`
+  optional_bin_progs="coreutils"
+  echo $single_binary_libs
+fi
+AC_SUBST([single_binary_progs], [$single_binary_progs])
+AC_SUBST([single_binary_libs], [$single_binary_libs])
+
 # The programs built and installed by "make && make install".
 # Since this is AC_SUBST'd, Automake won't be able to perform rewrite
 # with $(EXEEXT) appending on it, so we have to do it ourselves -- in
diff --git a/gnulib b/gnulib
index a10acfb..a008d62 160000
--- a/gnulib
+++ b/gnulib
@@ -1 +1 @@
-Subproject commit a10acfb1d2118f9a180181d3fed5399dbbe1df3c
+Subproject commit a008d625b7854d2c08c6606d90f6f2f48263f973
diff --git a/src/coreutils.c b/src/coreutils.c
new file mode 100644
index 0000000..ee38ae9
--- /dev/null
+++ b/src/coreutils.c
@@ -0,0 +1,82 @@
+/* Copyright (C) 2014 Free Software Foundation, Inc.
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.  */
+
+/* coreutils.c aggregates the functionality of every other tool into a single
+   binary multiplexed by the value of argv[0]. This is enabled by passing
+   --enable-single-binary to configure.
+
+   By Alex Deymo <address@hidden> */
+
+#include <config.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+/* Declare the main() function on each one of the selected tools. This name
+ * needs to match the one passed as CFLAGS on single-binary.mk (generated
+ * byt gen-single-binary.sh). */
+#define SINGLE_BINARY_PROGRAM(prog_name_str, main_name) \
+  int _single_binary_main_##main_name(int argc, char** argv, char** envp);
+#include "coreutils.h"
+#undef SINGLE_BINARY_PROGRAM
+
+void
+usage (int status);
+
+void
+usage (int status)
+{
+  printf ("Usage: coreutils <command> ...\n");
+  printf ("\nCompiled commands:\n"
+#define SINGLE_BINARY_PROGRAM(prog_name_str, main_name) " " prog_name_str
+#include "coreutils.h"
+#undef SINGLE_BINARY_PROGRAM
+  "\n");
+  exit (status);
+}
+
+int
+main (int argc, char **argv, char** envp)
+{
+  char *name = argv[0];
+  char *p;
+
+  /* Replace argv[0] by the tool name, removing the path from it. */
+  for (p = name; *p; ++p) {
+    if (*p == '/')
+      name = p + 1;
+  }
+  argv[0] = name;
+
+  /* If this program is called directly as "coreutils" instead of using a
+   * symlink, we use argv[1] as the name of the tool, shifting all the
+   * arguments one position. */
+  if (!strcmp(argv[0], "coreutils")) {
+    argv++;
+    argc--;
+  }
+  if (argc < 1 || !argv[0])
+    usage(EXIT_FAILURE);
+
+  /* Lookup the right main program */
+#define SINGLE_BINARY_PROGRAM(prog_name_str, main_name) \
+  if (!strcmp(prog_name_str, argv[0])) \
+    return _single_binary_main_##main_name(argc, argv, envp);
+#include "coreutils.h"
+#undef SINGLE_BINARY_PROGRAM
+
+  fprintf(stderr, "%s: program not found.\n", argv[0]);
+  usage(EXIT_FAILURE);
+}
diff --git a/src/getlimits.c b/src/getlimits.c
index 597efd8..62da2bb 100644
--- a/src/getlimits.c
+++ b/src/getlimits.c
@@ -167,4 +167,6 @@ main (int argc, char **argv)
   print_float (FLT);
   print_float (DBL);
   print_float (LDBL);
+
+  exit (EXIT_SUCCESS);
 }
diff --git a/src/local.mk b/src/local.mk
index 865dd74..43a9923 100644
--- a/src/local.mk
+++ b/src/local.mk
@@ -42,6 +42,7 @@ noinst_PROGRAMS =             \
 noinst_HEADERS =               \
   src/chown-core.h             \
   src/copy.h                   \
+  src/coreutils.h              \
   src/cp-hash.h                        \
   src/dircolors.h              \
   src/fiemap.h                 \
@@ -150,6 +151,7 @@ src_mv_LDADD = $(LDADD)
 src_nice_LDADD = $(LDADD)
 src_nl_LDADD = $(LDADD)
 src_nohup_LDADD = $(LDADD)
+src_numfmt_LDADD = $(LDADD)
 src_od_LDADD = $(LDADD)
 src_paste_LDADD = $(LDADD)
 src_pathchk_LDADD = $(LDADD)
@@ -395,6 +397,19 @@ src_libstdbuf_so_LDADD = $(LIBINTL)
 src_libstdbuf_so_LDFLAGS = -shared
 src_libstdbuf_so_CFLAGS = -fPIC $(AM_CFLAGS)
 
+# Single binary dependencies
+src_coreutils_SOURCES = src/coreutils.c src/coreutils.h
+src_coreutils_CFLAGS = $(AM_CFLAGS) -Wno-suggest-attribute=noreturn
+src_coreutils_LDFLAGS = $(AM_LDFLAGS)
+src_coreutils_LDADD = $(single_binary_libs) $(LDADD)
+
+# We need this EXTRA_* variables so automake generates all the rules to build
+# src/coreutils.
+EXTRA_src_coreutils_SOURCES =
+EXTRA_src_coreutils_DEPENDENCIES =
+include $(top_srcdir)/src/single-binary.mk
+
+
 BUILT_SOURCES += src/dircolors.h
 src/dircolors.h: src/dcgen src/dircolors.hin
        $(AM_V_GEN)rm -f $@ $@-t
@@ -515,6 +530,22 @@ src/version.h: Makefile
        $(AM_V_at)chmod a-w $@t
        $(AM_V_at)mv $@t $@
 
+# Generates a list of macro invocations like:
+#   SINGLE_BINARY_PROGRAM(program_name_str, main_name)
+# once for each program list on $(single_binary_progs). Note that
+# for ] the macro invocation is:
+#   SINGLE_BINARY_PROGRAM("[", _)
+BUILT_SOURCES += src/coreutils.h
+src/coreutils.h: Makefile
+       $(AM_V_GEN)rm -f $@
+       $(AM_V_at)for prog in $(single_binary_progs); do                       \
+               prog="$$(basename $$prog)"                                   ; \
+               main="$$(echo $$prog | tr '[' '_')"                          ; \
+               echo "SINGLE_BINARY_PROGRAM(\"$$prog\", $$main)"             ; \
+       done | sort > $@t
+       $(AM_V_at)chmod a-w $@t
+       $(AM_V_at)mv $@t $@
+
 DISTCLEANFILES += src/version.c src/version.h
 MAINTAINERCLEANFILES += $(BUILT_SOURCES)
 
-- 
2.0.0.526.g5318336