RFC: [PATCH] build: Option for building all tools in a single binary

[Alex Deymo]

Add the --enable-single-binary option to the configure file. When
enabled, this option builds a single binary file with all the
selected tools in it. Which tools gets executed depends on the value
of argv[0] which is intended to use together with symlinks to the
single program.

This setup reduces significantly the size of the installed binaries
since all the redundant code from lib/libcoreutils.a installed on
every one of the more than 100 binaries will be only once. This also
favors the cache, since the result binary is smaller.

When installing, the makefile will create symlinks from each
configured tool to a single "coreutils" binary installed on the
same directory.

---
My tests on a x86_64 linux system show an overall decrease from
5.0MiB after stripping on the case of building all the binaries to
just 924KiB with the single-binary option enabled.

I'd like to have this kind of feature in coreutils since the
reduction in size significant. The following patch is a proposal to
implement this feature that I used to compute the results. I'm new
to this project so I'd love to hear some comments and work on this
idea.

Thanks for reviewing.

 Makefile.am                        |  6 +++
 bootstrap                          |  3 ++
 build-aux/gen-lists-of-programs.sh |  1 +
 build-aux/gen-single-binary.sh     | 76 +++++++++++++++++++++++++++++++++++++
 configure.ac                       | 26 +++++++++++++
 gnulib                             |  2 +-
 src/coreutils.c                    | 78 ++++++++++++++++++++++++++++++++++++++
 src/getlimits.c                    |  2 +
 src/local.mk                       | 31 +++++++++++++++
 9 files changed, 224 insertions(+), 1 deletion(-)
 create mode 100755 build-aux/gen-single-binary.sh
 create mode 100644 src/coreutils.c

diff --git a/Makefile.am b/Makefile.am
index e88dc9c..eb467c7 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -180,6 +180,12 @@ check-git-hook-script-sync:
        rm -rf $$t;                                                     \
        test $$fail = 0
 
+# If we are building a single-binary, create symlinks for them.
+install-exec-hook:
+       @for i in $(single_binary_progs); do \
+               $(LN_S) -s coreutils $(DESTDIR)$(bindir)/$$i$(EXEEXT); \
+       done
+
 noinst_LIBRARIES =
 MOSTLYCLEANFILES =
 CLEANFILES =
diff --git a/bootstrap b/bootstrap
index a3e68f0..87f97a2 100755
--- a/bootstrap
+++ b/bootstrap
@@ -807,6 +807,9 @@ version_controlled_file() {
   fi
 }
 
+# Regenerate src/single-binary.sh
+./build-aux/gen-single-binary.sh src/local.mk
+
 # Avoid boostrap failure with gettext/autopoint bug in version 0.18.3.1
 # http://lists.gnu.org/archive/html/coreutils/2013-11/msg00038.html
 # Remove in 2015 when distros have upgraded to >= 0.18.3.2
diff --git a/build-aux/gen-lists-of-programs.sh 
b/build-aux/gen-lists-of-programs.sh
index bf63ee3..a3491ed 100755
--- a/build-aux/gen-lists-of-programs.sh
+++ b/build-aux/gen-lists-of-programs.sh
@@ -17,6 +17,7 @@ set -e
 # use "--enable-install-program=A,B" when invoking configure.
 disabled_by_default_progs='
     arch
+    coreutils
     hostname
 '
 
diff --git a/build-aux/gen-single-binary.sh b/build-aux/gen-single-binary.sh
new file mode 100755
index 0000000..2885bd3
--- /dev/null
+++ b/build-aux/gen-single-binary.sh
@@ -0,0 +1,76 @@
+#!/bin/sh
+
+# Generate the list of rules for the single-binary option based on all the 
other
+# binaries found in src/local.mk
+
+if [ "x$1" == "x" ]; then
+  echo "Usage: $0 path/to/src/local.mk" >&2
+  exit 1
+fi
+
+set -u
+set -e
+
+LOCAL_MK=$1
+SINGLE_BINARY_MK="`dirname $1`/single-binary.mk"
+
+ALL_COMMANDS=$(grep -E '^src_([a-z0-9_]+)_LDADD = \$\(' $LOCAL_MK -o | sort -u 
\
+  | sed 's/^src_\([a-z0-9_]\+\)_LDADD .*/\1/' | grep -v coreutils)
+
+# Compute default variables
+for cmd in $ALL_COMMANDS; do
+  eval "src_${cmd}_SOURCES=src/${cmd}.c"
+  eval "src_${cmd}_CFLAGS=''"
+  eval "src_${cmd}_CPPFLAGS=''"
+  eval "src_${cmd}_LDADD=''"
+done
+
+# Load actual values from src/local.mk
+while read l; do
+  if echo $l | grep -E '^src_\w+ +\+?=' > /dev/null; then
+    var=$(echo $l | cut -f 1 -d ' ')
+    value=$(echo $l | cut -f 2- -d =)
+    if [ "$value" != " \$(LDADD)" ]; then
+      eval "$var='"${value//\'/\'\"\'\"\'}"'"
+    fi
+  fi
+done < $LOCAL_MK
+
+for cmd in $ALL_COMMANDS; do
+  echo "# Command $cmd"
+  echo noinst_LIBRARIES += src/libsinglebin_${cmd}.a
+  base="src_libsinglebin_${cmd}_a"
+  # SOURCES
+  var=src_${cmd}_SOURCES
+  eval "value=\$$var"
+  echo "${base}_SOURCES = $value"
+  echo "EXTRA_src_coreutils_SOURCES += $value"
+  echo "EXTRA_src_coreutils_DEPENDENCIES += src/libsinglebin_${cmd}.a"
+
+  # CFLAGS
+  # Hack any other program defining a main() replacing its main by
+  # _single_binary_main_$PROGRAM_NAME.
+  echo "${base}_CFLAGS = -Dmain=_single_binary_main_${cmd}" \
+       "-Dusage=_usage_${cmd} \$(src_coreutils_CFLAGS)"
+  var=src_${cmd}_CFLAGS
+  eval "value=\$$var"
+  if [ "x$value" != "x" ]; then
+    echo "${base}_CFLAGS += $value"
+  fi
+
+  # CPPFLAGS
+  var=src_${cmd}_CPPFLAGS
+  eval "value=\$$var"
+  if [ "x$value" != "x" ]; then
+    echo "${base}_CPPFLAGS = $value"
+  fi
+
+  # LIBADD
+  var=src_${cmd}_LDADD
+  eval "value=\$$var"
+  if [ "x$value" != "x" ]; then
+    echo "src_coreutils_LDADD += $value"
+  fi
+done > $SINGLE_BINARY_MK
+
+exit 0
diff --git a/configure.ac b/configure.ac
index 7c210d9..81e21a3 100644
--- a/configure.ac
+++ b/configure.ac
@@ -221,6 +221,17 @@ if test "$gl_gcc_warnings" = yes; then
   AC_SUBST([GNULIB_TEST_WARN_CFLAGS])
 fi
 
+AC_ARG_ENABLE([single-binary],
+  [AS_HELP_STRING([--enable-single-binary],
+     [Compile all the tools in a single binary (reduces overall size)])],
+  [case $enableval in
+     yes|no) ;;
+     *)      AC_MSG_ERROR([bad value $enableval for single-binary option]) ;;
+   esac
+   gl_single_binary=$enableval],
+  [gl_single_binary=no]
+)
+
 AC_FUNC_FORK
 
 optional_bin_progs=
@@ -487,6 +498,21 @@ man1_MANS=`
 # a distribution tarball.
 EXTRA_MANS=`for p in $no_install_progs_default; do echo man/$p.1; done`
 
+# Replace all the programs by the single binary and simlinks if specified.
+single_binary_progs=
+single_binary_libs=
+if test "$gl_single_binary" = yes; then
+  single_binary_progs=`echo $optional_bin_progs`
+  single_binary_libs=`
+    for p in $single_binary_progs; do
+      echo -n "src/libsinglebin_$p.a " | tr "\133" _
+    done`
+  optional_bin_progs="coreutils"
+  echo $single_binary_libs
+fi
+AC_SUBST([single_binary_progs], [$single_binary_progs])
+AC_SUBST([single_binary_libs], [$single_binary_libs])
+
 # The programs built and installed by "make && make install".
 # Since this is AC_SUBST'd, Automake won't be able to perform rewrite
 # with $(EXEEXT) appending on it, so we have to do it ourselves -- in
diff --git a/gnulib b/gnulib
index a10acfb..0658e50 160000
--- a/gnulib
+++ b/gnulib
@@ -1 +1 @@
-Subproject commit a10acfb1d2118f9a180181d3fed5399dbbe1df3c
+Subproject commit 0658e5039f75053480a7599edca236fecffe100d
diff --git a/src/coreutils.c b/src/coreutils.c
new file mode 100644
index 0000000..46c51d6
--- /dev/null
+++ b/src/coreutils.c
@@ -0,0 +1,78 @@
+/* Copyright (C) 2014 Free Software Foundation, Inc.
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.  */
+
+/* coreutils.c aggregates the functionality of every other tool into a single
+   binary multiplexed by the value of argv[0]. This is enabled by passing
+   --enable-single-binary to configure.
+
+   By Alex Deymo <address@hidden> */
+
+#include <config.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+
+/* Declare the main() function on the tools */
+#define SINGLE_BINARY_PROGRAM(prog_name_str, main_name) \
+  int _single_binary_main_##main_name(int argc, char** argv, char** envp);
+#include "coreutils.h"
+#undef SINGLE_BINARY_PROGRAM
+
+void
+usage (int status);
+
+void
+usage (int status)
+{
+  printf ("Usage: coreutils <command> ...\n");
+  printf ("\nCompiled commands:\n"
+#define SINGLE_BINARY_PROGRAM(prog_name_str, main_name) " " prog_name_str
+#include "coreutils.h"
+#undef SINGLE_BINARY_PROGRAM
+  "\n");
+  exit (status);
+}
+
+int
+main (int argc, char **argv, char** envp)
+{
+  char *name = argv[0];
+  char *p;
+
+  /* Replace argv[0] by the tool name. */
+  for (p = name; *p; ++p) {
+    if (*p == '/')
+      name = p + 1;
+  }
+  argv[0] = name;
+
+  if (!strcmp(argv[0], "coreutils")) {
+    argv++;
+    argc--;
+  }
+  if (argc < 1 || !argv[0])
+    usage(EXIT_FAILURE);
+
+  /* Lookup the right main program */
+#define SINGLE_BINARY_PROGRAM(prog_name_str, main_name) \
+  if (!strcmp(prog_name_str, argv[0])) \
+    return _single_binary_main_##main_name(argc, argv, envp);
+#include "coreutils.h"
+#undef SINGLE_BINARY_PROGRAM
+
+  fprintf(stderr, "%s: program not found.\n", argv[0]);
+  usage(EXIT_FAILURE);
+}
diff --git a/src/getlimits.c b/src/getlimits.c
index 597efd8..62da2bb 100644
--- a/src/getlimits.c
+++ b/src/getlimits.c
@@ -167,4 +167,6 @@ main (int argc, char **argv)
   print_float (FLT);
   print_float (DBL);
   print_float (LDBL);
+
+  exit (EXIT_SUCCESS);
 }
diff --git a/src/local.mk b/src/local.mk
index 865dd74..43a9923 100644
--- a/src/local.mk
+++ b/src/local.mk
@@ -42,6 +42,7 @@ noinst_PROGRAMS =             \
 noinst_HEADERS =               \
   src/chown-core.h             \
   src/copy.h                   \
+  src/coreutils.h              \
   src/cp-hash.h                        \
   src/dircolors.h              \
   src/fiemap.h                 \
@@ -150,6 +151,7 @@ src_mv_LDADD = $(LDADD)
 src_nice_LDADD = $(LDADD)
 src_nl_LDADD = $(LDADD)
 src_nohup_LDADD = $(LDADD)
+src_numfmt_LDADD = $(LDADD)
 src_od_LDADD = $(LDADD)
 src_paste_LDADD = $(LDADD)
 src_pathchk_LDADD = $(LDADD)
@@ -395,6 +397,19 @@ src_libstdbuf_so_LDADD = $(LIBINTL)
 src_libstdbuf_so_LDFLAGS = -shared
 src_libstdbuf_so_CFLAGS = -fPIC $(AM_CFLAGS)
 
+# Single binary dependencies
+src_coreutils_SOURCES = src/coreutils.c src/coreutils.h
+src_coreutils_CFLAGS = $(AM_CFLAGS) -Wno-suggest-attribute=noreturn
+src_coreutils_LDFLAGS = $(AM_LDFLAGS)
+src_coreutils_LDADD = $(single_binary_libs) $(LDADD)
+
+# We need this EXTRA_* variables so automake generates all the rules to build
+# src/coreutils.
+EXTRA_src_coreutils_SOURCES =
+EXTRA_src_coreutils_DEPENDENCIES =
+include $(top_srcdir)/src/single-binary.mk
+
+
 BUILT_SOURCES += src/dircolors.h
 src/dircolors.h: src/dcgen src/dircolors.hin
        $(AM_V_GEN)rm -f $@ $@-t
@@ -515,6 +530,22 @@ src/version.h: Makefile
        $(AM_V_at)chmod a-w $@t
        $(AM_V_at)mv $@t $@
 
+# Generates a list of macro invocations like:
+#   SINGLE_BINARY_PROGRAM(program_name_str, main_name)
+# once for each program list on $(single_binary_progs). Note that
+# for ] the macro invocation is:
+#   SINGLE_BINARY_PROGRAM("[", _)
+BUILT_SOURCES += src/coreutils.h
+src/coreutils.h: Makefile
+       $(AM_V_GEN)rm -f $@
+       $(AM_V_at)for prog in $(single_binary_progs); do                       \
+               prog="$$(basename $$prog)"                                   ; \
+               main="$$(echo $$prog | tr '[' '_')"                          ; \
+               echo "SINGLE_BINARY_PROGRAM(\"$$prog\", $$main)"             ; \
+       done | sort > $@t
+       $(AM_V_at)chmod a-w $@t
+       $(AM_V_at)mv $@t $@
+
 DISTCLEANFILES += src/version.c src/version.h
 MAINTAINERCLEANFILES += $(BUILT_SOURCES)
 
-- 
2.0.0.526.g5318336