From 529a2c5c838438f3cd5c4686cfa32e192917be63 Mon Sep 17 00:00:00 2001 From: Alex Deymo
Date: Thu, 5 Jun 2014 19:50:32 -0700 Subject: [PATCH] build: support building all tools in a single binary Add the --enable-single-binary option to the configure file. When enabled, this option builds a single binary file containing the selected tools. Which tool gets executed depends on the value of argv[0] which can be set implicitly through symlinks to the single program. This setup reduces significantly the size of a complete coreutils install, since code from lib/libcoreutils.a is not duplicated in every one of the more than 100 binaries. Runtime overhead is increased due to more dynamic libraries being loaded, and extra initialization being performed for all utils. Also initially a larger binary is loaded from storage, though this is usually alleviated due to caching and lazy mmaping of unused blocks, and in fact the single binary should have better caching characteristics. Comparing the size of the individual versus single binary on x86_64: $ cd src $ size coreutils $ size -t $(../build-aux/gen-lists-of-programs.sh --list-progs | | grep -Ev '(coreutils|libstdbuf)') | tail -n1 text data bss dec hex filename 1207992 6316 94384 1308692 13f814 coreutils 4901010 124964 163768 5189742 4f306e (TOTALS) Storage requirements are reduced similarly: $ cd src $ du -h coreutils $ du -ch $(../build-aux/gen-lists-of-programs.sh --list-progs | grep -Ev '(coreutils|libstdbuf)') | tail -n1 1.2M coreutils 5.3M total When installing, the makefile will create symlinks from each configured tool to a single "coreutils" binary installed on the same directory. * .gitignore: Added new generated files. * Makefile.am: New rules to generate build-aux/gen-single-binary.sh and install symlinks. * NEWS: Mention the new feature. * README: Added "coreutils" to the list of utils. * bootstrap.conf: Regenerate src/single-binary.mk * build-aux/gen-lists-of-programs.sh: New --list-progs option. * build-aux/gen-single-binary.sh: Regenerate * configure.ac: New --enable-single-binary option and other variables. * man/coreutils.x: Manpage hook. * man/local.mk: Added manpage hook. * src/coreutils.c: Multicall implementation. * src/local.mk: New rules for the single binary option. * tests/misc/help-version.sh: Added exception for coreutils. * src/kill.c: Changes to call exit() in main. * src/readlink.c: Likewise. * src/shuf.c: Likewise. * src/timeout.c: Likewise. * src/truncate.c: Likewise. --- .gitignore | 3 + AUTHORS | 1 + Makefile.am | 13 ++++ NEWS | 7 ++ README | 18 ++--- bootstrap.conf | 7 ++ build-aux/gen-lists-of-programs.sh | 7 ++ build-aux/gen-single-binary.sh | 110 ++++++++++++++++++++++++++++++ configure.ac | 62 +++++++++++++++++ man/coreutils.x | 4 ++ man/local.mk | 3 +- po/POTFILES.in | 1 + src/coreutils-arch.c | 14 ++++ src/coreutils-dir.c | 14 ++++ src/coreutils-vdir.c | 14 ++++ src/coreutils.c | 136 +++++++++++++++++++++++++++++++++++++ src/kill.c | 6 +- src/local.mk | 41 +++++++++++ src/readlink.c | 2 +- src/shuf.c | 2 +- src/timeout.c | 8 +-- src/truncate.c | 2 +- tests/misc/help-version.sh | 2 +- 23 files changed, 456 insertions(+), 21 deletions(-) create mode 100755 build-aux/gen-single-binary.sh create mode 100644 man/coreutils.x create mode 100644 src/coreutils-arch.c create mode 100644 src/coreutils-dir.c create mode 100644 src/coreutils-vdir.c create mode 100644 src/coreutils.c diff --git a/.gitignore b/.gitignore index 558577d..e5c07fe 100644 --- a/.gitignore +++ b/.gitignore @@ -155,10 +155,13 @@ /po/remove-potcdate.sed /po/remove-potcdate.sin /po/stamp-po +/src/coreutils.h /src/cu-progs.mk /src/fs-latest-magic.h +/src/libsinglebin_*.a /src/make-prime-list /src/primes.h +/src/single-binary.mk /src/version.c /src/version.h /stamp-h1 diff --git a/AUTHORS b/AUTHORS index df21e90..0296830 100644 --- a/AUTHORS +++ b/AUTHORS @@ -12,6 +12,7 @@ chown: David MacKenzie, Jim Meyering chroot: Roland McGrath cksum: Q. Frank Xia comm: Richard M. Stallman, David MacKenzie +coreutils: Alex Deymo cp: Torbjörn Granlund, David MacKenzie, Jim Meyering csplit: Stuart Kemp, David MacKenzie cut: David M. Ihnat, David MacKenzie, Jim Meyering diff --git a/Makefile.am b/Makefile.am index e88dc9c..1351434 100644 --- a/Makefile.am +++ b/Makefile.am @@ -50,6 +50,7 @@ EXTRA_DIST = \ bootstrap \ bootstrap.conf \ build-aux/gen-lists-of-programs.sh \ + build-aux/gen-single-binary.sh \ cfg.mk \ dist-check.mk \ maint.mk \ @@ -57,6 +58,7 @@ EXTRA_DIST = \ thanks-gen gen_progs_lists = $(top_srcdir)/build-aux/gen-lists-of-programs.sh +gen_single_binary = $(top_srcdir)/build-aux/gen-single-binary.sh # Keep these in sync with bootstrap.conf:bootstrap_post_import_hook(). # Use '$(top_srcdir)/m4' and '$(srcdir)/src' for the benefit of non-GNU @@ -70,6 +72,10 @@ $(srcdir)/src/cu-progs.mk: $(gen_progs_lists) $(AM_V_GEN)rm -f $@ $@-t \ && $(SHELL) $(gen_progs_lists) --automake >$@-t \ && chmod a-w $@-t && mv -f $@-t $@ +$(srcdir)/src/single-binary.mk: $(gen_single_binary) $(srcdir)/src/local.mk + $(AM_V_GEN)rm -f $@ $@-t \ + && $(SHELL) $(gen_single_binary) $(srcdir)/src/local.mk >$@-t \ + && chmod a-w $@-t && mv -f $@-t $@ ACLOCAL_AMFLAGS = -I m4 @@ -180,6 +186,13 @@ check-git-hook-script-sync: rm -rf $$t; \ test $$fail = 0 +# If we are building a single-binary, create symlinks for them when installing. +install-exec-hook: + @for i in $(single_binary_progs); do \ + rm -f $(DESTDIR)$(bindir)/$$i$(EXEEXT); \ + $(LN_S) -s coreutils $(DESTDIR)$(bindir)/$$i$(EXEEXT); \ + done + noinst_LIBRARIES = MOSTLYCLEANFILES = CLEANFILES = diff --git a/NEWS b/NEWS index 5d1fe99..603bdd7 100644 --- a/NEWS +++ b/NEWS @@ -85,6 +85,13 @@ GNU coreutils NEWS -*- outline -*- od accepts a new option: --endian=TYPE to handle inputs with different byte orders, or to provide consistent output on systems with disparate endianness. + configure accepts the new option --enable-single-binary to build all the + selected programs in a single binary called "coreutils" with symlinks to it + for each program. The functionality of each program is not affected but this + single binary will depend on all the required dynamic libraries even to run + simple programs. This flag considerably reduces the overall size of the + installed binaries. + ** Changes in behavior chroot with an argument of "/" no longer implicitly changes the current diff --git a/README b/README index fa1a429..bd0bce5 100644 --- a/README +++ b/README @@ -7,15 +7,15 @@ arbitrary limits. The programs that can be built with this package are: - [ arch base64 basename cat chcon chgrp chmod chown chroot cksum comm cp - csplit cut date dd df dir dircolors dirname du echo env expand expr - factor false fmt fold groups head hostid hostname id install join kill - link ln logname ls md5sum mkdir mkfifo mknod mktemp mv nice nl nohup - nproc numfmt od paste pathchk pinky pr printenv printf ptx pwd readlink - realpath rm rmdir runcon seq sha1sum sha224sum sha256sum sha384sum sha512sum - shred shuf sleep sort split stat stdbuf stty sum sync tac tail tee test - timeout touch tr true truncate tsort tty uname unexpand uniq unlink - uptime users vdir wc who whoami yes + [ arch base64 basename cat chcon chgrp chmod chown chroot cksum comm + coreutils cp csplit cut date dd df dir dircolors dirname du echo env + expand expr factor false fmt fold groups head hostid hostname id install + join kill link ln logname ls md5sum mkdir mkfifo mknod mktemp mv nice nl + nohup nproc numfmt od paste pathchk pinky pr printenv printf ptx pwd + readlink realpath rm rmdir runcon seq sha1sum sha224sum sha256sum sha384sum + sha512sum shred shuf sleep sort split stat stdbuf stty sum sync tac tail + tee test timeout touch tr true truncate tsort tty uname unexpand uniq + unlink uptime users vdir wc who whoami yes See the file NEWS for a list of major changes in the current release. diff --git a/bootstrap.conf b/bootstrap.conf index 98e83bb..c0b5f02 100644 --- a/bootstrap.conf +++ b/bootstrap.conf @@ -346,6 +346,13 @@ bootstrap_post_import_hook () # Massage lib/gnulib.mk before using it later in the bootstrapping process. build-aux/prefix-gnulib-mk --lib-name=$gnulib_name lib/$gnulib_mk + + # Regenerate src/single-binary.mk + (mkf=src/single-binary.mk tmp=single-binary.tmp \ + && rm -f $mkf $tmp \ + && build-aux/gen-single-binary.sh src/local.mk >$tmp \ + && chmod a-w $tmp \ + && mv -f $tmp $mkf) } diff --git a/build-aux/gen-lists-of-programs.sh b/build-aux/gen-lists-of-programs.sh index bf63ee3..c95e598 100755 --- a/build-aux/gen-lists-of-programs.sh +++ b/build-aux/gen-lists-of-programs.sh @@ -17,6 +17,7 @@ set -e # use "--enable-install-program=A,B" when invoking configure. disabled_by_default_progs=' arch + coreutils hostname ' @@ -178,6 +179,12 @@ END echo default__progs += $progsdir/$p done ;; + 1,--list-progs) + for p in $disabled_by_default_progs $build_if_possible_progs \ + $normal_progs; do + echo $p + done + ;; *) echo "$0: invalid usage" >&2; exit 2 ;; diff --git a/build-aux/gen-single-binary.sh b/build-aux/gen-single-binary.sh new file mode 100755 index 0000000..f3958bb --- /dev/null +++ b/build-aux/gen-single-binary.sh @@ -0,0 +1,110 @@ +#!/bin/sh + +# Generate the list of rules for the single-binary option based on all the other +# binaries found in src/local.mk. +# +# We need to duplicate the specific rules to build each program into a new +# static library target. We can't reuse the existing target since we need to +# create a .a file instead of linking the program. We can't do this at +# ./configure since the file names need to available when automake runs to let +# it generate all the required rules in Makefile.in. The configure step will +# select which ones will be used to build, but they need to be generated +# beforehand. +# +# Instead of maintaining a duplicated list of rules, we generate the +# single-binary required rules based on the normal configuration found on +# src/local.mk with this script. + +if test "x$1" = "x"; then + echo "Usage: $0 path/to/src/local.mk" >&2 + exit 1 +fi + +set -e + +LOCAL_MK=$1 +GEN_LISTS_OF_PROGRAMS="`dirname "$0"`/gen-lists-of-programs.sh" + +ALL_PROGRAMS=$($GEN_LISTS_OF_PROGRAMS --list-progs \ + | grep -v -F -e coreutils -e libstdbuf.so \ + | tr '[' '_') + +# Compute default SOURCES. automake will assume the source file for the +# src_${cmd} target to be src/${cmd}.c, but we will add rules to generate +# the lib src_libsinglebin_${cmd}_a which won't match the autogenerated source +# file. This loop will initialize the default source file and will be reset +# later if needed. +for cmd in $ALL_PROGRAMS; do + eval "src_${cmd}_SOURCES=src/${cmd}.c" +done + +# Load actual values from src/local.mk. This will read all the variables from +# the local.mk matching the src_${cmd}_... case. +while read l; do + if echo "$l" | grep -E '^src_\w+ +\+?=' > /dev/null; then + var=$(echo $l | cut -f 1 -d ' ') + value=$(echo $l | cut -f 2- -d =) + if [ "$value" != " \$(LDADD)" ]; then + oldvalue="" + if echo $l | grep -F '+=' >/dev/null; then + eval "oldvalue=\${$var}" + fi + eval "$var='$oldvalue "${value//\'/\'\"\'\"\'}"'" + fi + fi +done < $LOCAL_MK + +me=`echo "$0" | sed 's,.*/,,'` +echo "## Automatically generated by $me. DO NOT EDIT BY HAND!" + +# Override the sources for dir and vdir. We use a smaller version of dir and +# vdir that relies on the ls main. +src_dir_SOURCES="src/coreutils-dir.c" +src_dir_LDADD+=" src/libsinglebin_ls.a" +echo src_libsinglebin_dir_a_DEPENDENCIES = src/libsinglebin_ls.a +src_vdir_SOURCES="src/coreutils-vdir.c" +src_vdir_LDADD+=" src/libsinglebin_ls.a" +echo src_libsinglebin_vdir_a_DEPENDENCIES = src/libsinglebin_ls.a + +# Override the sources for arch likewise, using the main from uname. +src_arch_SOURCES="src/coreutils-arch.c" +src_arch_LDADD+=" src/libsinglebin_uname.a" +echo src_libsinglebin_arch_a_DEPENDENCIES = src/libsinglebin_uname.a + +for cmd in $ALL_PROGRAMS; do + echo "# Command $cmd" + echo noinst_LIBRARIES += src/libsinglebin_${cmd}.a + base="src_libsinglebin_${cmd}_a" + # SOURCES + var=src_${cmd}_SOURCES + eval "value=\$$var" + echo "${base}_SOURCES = $value" + + # LDADD + var=src_${cmd}_LDADD + eval "value=\$$var" + if [ "x$value" != "x" ]; then + echo "${base}_ldadd = $value" + fi + + # CFLAGS + # Hack any other program defining a main() replacing its main by + # _single_binary_main_$PROGRAM_NAME. + echo "${base}_CFLAGS = \"-Dmain=_single_binary_main_${cmd}(int, char**) " \ + "ATTRIBUTE_NORETURN; int _single_binary_main_${cmd}\" " \ + "-Dusage=_usage_${cmd} \$(src_coreutils_CFLAGS)" + var=src_${cmd}_CFLAGS + eval "value=\$$var" + if [ "x$value" != "x" ]; then + echo "${base}_CFLAGS += $value" + fi + + # CPPFLAGS + var=src_${cmd}_CPPFLAGS + eval "value=\$$var" + if [ "x$value" != "x" ]; then + echo "${base}_CPPFLAGS = $value" + fi +done + +exit 0 diff --git a/configure.ac b/configure.ac index 01098cf..3c44be3 100644 --- a/configure.ac +++ b/configure.ac @@ -213,6 +213,26 @@ if test "$gl_gcc_warnings" = yes; then AC_SUBST([GNULIB_TEST_WARN_CFLAGS]) fi +AC_ARG_ENABLE([single-binary], + [AS_HELP_STRING([--enable-single-binary], + [Compile all the tools in a single binary (reduces overall size)])], + [case $enableval in + yes|no) ;; + *) AC_MSG_ERROR([bad value $enableval for single-binary option]) ;; + esac + gl_single_binary=$enableval], + [gl_single_binary=no] +) +AC_ARG_ENABLE([single-binary-exceptions], + [AS_HELP_STRING([--enable-single-binary-exceptions=PROG_LIST], + [When used with --enable-single-binary, exclude the PROG_LIST from + it, so these programs are compiled as separated files + (comma-separated, default none))])], + [gl_single_binary_exceptions=$enableval], + [gl_single_binary_exceptions=] +) +AM_CONDITIONAL([SINGLE_BINARY], [test "$gl_single_binary" = yes]) + AC_FUNC_FORK optional_bin_progs= @@ -479,6 +499,48 @@ man1_MANS=` # a distribution tarball. EXTRA_MANS=`for p in $no_install_progs_default; do echo man/$p.1; done` +# Replace all the programs by the single binary and simlinks if specified. +single_binary_progs= +single_binary_libs= +single_binary_deps= +if test "$gl_single_binary" = yes; then + # Convert the list to a space separated list + gl_single_binary_exceptions=`echo $gl_single_binary_exceptions | tr ',' ' '` + + single_binary_progs=`echo $optional_bin_progs` + optional_bin_progs="coreutils" + for prog in $gl_single_binary_exceptions; do + # Fail if requested to exclude a program than was not part of coreutils. + case " $single_binary_progs " in + *" $prog "*) + gl_REMOVE_PROG([single_binary_progs], [$prog]) ; + gl_ADD_PROG([optional_bin_progs], [$prog]) ;; + *) AC_MSG_ERROR(['$prog' is not being compiled.]) ;; + esac + done + + # single_binary_libs holds the list of libs required by the selected + # programs, such as for example -lrt. + single_binary_libs=` + for p in $single_binary_progs; do + # Convert '[' to '_' + test x"$p" = x'@<:@' && p='_' + printf '$(src_libsinglebin_%s_a_ldadd) ' "$p" + done` + # single_binary_deps holds the list of libsinglebin_*.a files that have the + # compiled code of each selected program in a "library" format. + single_binary_deps=` + for p in $single_binary_progs; do + # Convert '[' to '_' + test x"$p" = x'@<:@' && p='_' + printf 'src/libsinglebin_%s.a ' "$p" + done` +fi +AC_SUBST([single_binary_progs], [$single_binary_progs]) +AC_SUBST([single_binary_libs], [$single_binary_libs]) +AC_SUBST([single_binary_deps], [$single_binary_deps]) + + # The programs built and installed by "make && make install". # Since this is AC_SUBST'd, Automake won't be able to perform rewrite # with $(EXEEXT) appending on it, so we have to do it ourselves -- in diff --git a/man/coreutils.x b/man/coreutils.x new file mode 100644 index 0000000..a7a4879 --- /dev/null +++ b/man/coreutils.x @@ -0,0 +1,4 @@ +[NAME] +coreutils \- single binary for coreutils programs +[DESCRIPTION] +.\" Add any additional description here diff --git a/man/local.mk b/man/local.mk index 7ce426b..ac19190 100644 --- a/man/local.mk +++ b/man/local.mk @@ -77,6 +77,7 @@ man/chown.1: src/chown man/chroot.1: src/chroot man/cksum.1: src/cksum man/comm.1: src/comm +man/coreutils.1: src/coreutils man/cp.1: src/cp man/csplit.1: src/csplit man/cut.1: src/cut @@ -189,7 +190,7 @@ man/yes.1: src/yes && $(run_help2man) \ --source='$(PACKAGE_STRING)' \ --include=$(srcdir)/man/$$name.x \ - --output=$$t/$$name.1 $$t/$$name \ + --output=$$t/$$name.1 '$(abs_top_builddir)/src/'$$prog \ --info-page='coreutils \(aq'$$name' invocation\(aq' \ && sed \ -e 's|$*\.td/||g' \ diff --git a/po/POTFILES.in b/po/POTFILES.in index 07bbdd8..d380dee 100644 --- a/po/POTFILES.in +++ b/po/POTFILES.in @@ -45,6 +45,7 @@ src/chroot.c src/cksum.c src/comm.c src/copy.c +src/coreutils.c src/cp.c src/csplit.c src/cut.c diff --git a/src/coreutils-arch.c b/src/coreutils-arch.c new file mode 100644 index 0000000..b165b44 --- /dev/null +++ b/src/coreutils-arch.c @@ -0,0 +1,14 @@ +#include