[PATCH 1/2] build: accommodate newer bootstrap from gnulib

[Jim Meyering]

"make check" with automake-from-git/master was failing all tests.
Even after the lib/Makefile.am tweak, it was still failing four.
The four scripts that were not executable.  That's the 2nd patch.

The newer bootstrap from gnulib is cleaner, but required a
few client-side adjustments:

>From 9e61483eb1f62e4bd4b1c2a2492d4b46294abd0c Mon Sep 17 00:00:00 2001
From: Jim Meyering <address@hidden>
Date: Wed, 11 Jan 2012 21:31:50 +0100
Subject: [PATCH 1/2] build: accommodate newer bootstrap from gnulib

* bootstrap.conf (gnulib_tool_option_extras): Add both --symlink
and --makefile-name=gnulib.mk.
* bootstrap: Update from gnulib.
* tests/init.sh: Update from gnulib.
* lib/Makefile.am: Initialize all of the following so that
generated code in gnulib.mk may use += to append to those variables:
AM_CFLAGS, BUILT_SOURCES, CLEANFILES, EXTRA_DIST, MOSTLYCLEANDIRS,
MOSTLYCLEANFILES, SUFFIXES, noinst_LTLIBRARIES.
---
 bootstrap       |  251 ++++++++++++++++--------------------------------------
 bootstrap.conf  |    3 +
 lib/Makefile.am |    8 ++
 tests/init.sh   |  117 ++++++++++++++++++++++++--
 4 files changed, 195 insertions(+), 184 deletions(-)

diff --git a/bootstrap b/bootstrap
index 5c4eec3..66da981 100755
--- a/bootstrap
+++ b/bootstrap
@@ -1,6 +1,6 @@
 #! /bin/sh
 # Print a version string.
-scriptversion=2011-06-22.06; # UTC
+scriptversion=2012-01-06.07; # UTC

 # Bootstrap this package from checked-out sources.

@@ -38,10 +38,6 @@ export LC_ALL

 local_gl_dir=gl

-# Temporary directory names.
-bt='._bootmp'
-bt_regex=`echo "$bt"| sed 's/\./[.]/g'`
-bt2=${bt}2
 me=$0

 usage() {
@@ -88,6 +84,9 @@ gnulib_modules=
 # Any gnulib files needed that are not in modules.
 gnulib_files=

+: ${AUTOPOINT=autopoint}
+: ${AUTORECONF=autoreconf}
+
 # A function to be called to edit gnulib.mk right after it's created.
 # Override it via your own definition in bootstrap.conf.
 gnulib_mk_hook() { :; }
@@ -105,6 +104,11 @@ po_download_command_format=\
 "rsync --delete --exclude '*.s1' -Lrtvz \
  'translationproject.org::tp/latest/%s/' '%s'"

+# Fallback for downloading .po files (if rsync fails).
+po_download_command_format2=\
+"wget --mirror -nd -q -np -A.po -P '%s' \
+ http://translationproject.org/latest/%s/";
+
 extract_package_name='
   /^AC_INIT(/{
      /.*,.*,.*, */{
@@ -312,8 +316,8 @@ insert_vc_ignore() {
   pattern="$2"
   case $vc_ignore_file in
   *.gitignore)
-    # A .gitignore entry that does not start with `/' applies
-    # recursively to subdirectories, so prepend `/' to every
+    # A .gitignore entry that does not start with '/' applies
+    # recursively to subdirectories, so prepend '/' to every
     # .gitignore entry.
     pattern=`echo "$pattern" | sed s,^,/,`;;
   esac
@@ -473,6 +477,32 @@ if test $use_libtool = 1; then
   find_tool LIBTOOLIZE glibtoolize libtoolize
 fi

+# gnulib-tool requires at least automake and autoconf.
+# If either is not listed, add it (with minimum version) as a prerequisite.
+case $buildreq in
+  *automake*) ;;
+  *) buildreq="automake 1.9
+$buildreq" ;;
+esac
+case $buildreq in
+  *autoconf*) ;;
+  *) buildreq="autoconf 2.59
+$buildreq" ;;
+esac
+
+# When we can deduce that gnulib-tool will require patch,
+# and when patch is not already listed as a prerequisite, add it, too.
+if test ! -d "$local_gl_dir" \
+    || find "$local_gl_dir" -name '*.diff' -exec false {} +; then
+  :
+else
+  case $buildreq in
+    *patch*) ;;
+    *) buildreq="patch -
+$buildreq" ;;
+  esac
+fi
+
 if ! printf "$buildreq" | check_versions; then
   echo >&2
   if test -f README-prereq; then
@@ -583,6 +613,9 @@ download_po_files() {
   domain=$2
   echo "$me: getting translations into $subdir for $domain..."
   cmd=`printf "$po_download_command_format" "$domain" "$subdir"`
+  eval "$cmd" && return
+  # Fallback to HTTP.
+  cmd=`printf "$po_download_command_format2" "$subdir" "$domain"`
   eval "$cmd"
 }

@@ -703,56 +736,32 @@ symlink_to_dir()
   }
 }

-cp_mark_as_generated()
-{
-  cp_src=$1
-  cp_dst=$2
+# NOTE: we have to be careful to run both autopoint and libtoolize
+# before gnulib-tool, since gnulib-tool is likely to provide newer
+# versions of files "installed" by these two programs.
+# Then, *after* gnulib-tool (see below), we have to be careful to
+# run autoreconf in such a way that it does not run either of these
+# two just-pre-run programs.

-  if cmp -s "$cp_src" "$GNULIB_SRCDIR/$cp_dst"; then
-    symlink_to_dir "$GNULIB_SRCDIR" "$cp_dst"
-  elif cmp -s "$cp_src" "$local_gl_dir/$cp_dst"; then
-    symlink_to_dir $local_gl_dir "$cp_dst"
-  else
-    case $cp_dst in
-      *.[ch])             c1='/* '; c2=' */';;
-      *.texi)             c1='@c '; c2=     ;;
-      *.m4|*/Make*|Make*) c1='# ' ; c2=     ;;
-      *)                  c1=     ; c2=     ;;
-    esac
+# Import from gettext.
+with_gettext=yes
+grep '^[        ]*AM_GNU_GETTEXT_VERSION(' configure.ac >/dev/null || \
+    with_gettext=no

-    # If the destination directory doesn't exist, create it.
-    # This is required at least for "lib/uniwidth/cjk.h".
-    dst_dir=`dirname "$cp_dst"`
-    test -d "$dst_dir" || mkdir -p "$dst_dir"
-
-    if test -z "$c1"; then
-      cmp -s "$cp_src" "$cp_dst" || {
-        # Copy the file first to get proper permissions if it
-        # doesn't already exist.  Then overwrite the copy.
-        echo "$me: cp -f $cp_src $cp_dst" &&
-        rm -f "$cp_dst" &&
-        cp "$cp_src" "$cp_dst-t" &&
-        sed "s!$bt_regex/!!g" "$cp_src" > "$cp_dst-t" &&
-        mv -f "$cp_dst-t" "$cp_dst"
-      }
-    else
-      # Copy the file first to get proper permissions if it
-      # doesn't already exist.  Then overwrite the copy.
-      cp "$cp_src" "$cp_dst-t" &&
-      (
-        echo "$c1-*- buffer-read-only: t -*- vi: set ro:$c2" &&
-        echo "${c1}DO NOT EDIT! GENERATED AUTOMATICALLY!$c2" &&
-        sed "s!$bt_regex/!!g" "$cp_src"
-      ) > $cp_dst-t &&
-      if cmp -s "$cp_dst-t" "$cp_dst"; then
-        rm -f "$cp_dst-t"
-      else
-        echo "$me: cp $cp_src $cp_dst # with edits" &&
-        mv -f "$cp_dst-t" "$cp_dst"
-      fi
-    fi
-  fi
-}
+if test $with_gettext = yes; then
+  # Released autopoint has the tendency to install macros that have been
+  # obsoleted in current gnulib, so run this before gnulib-tool.
+  echo "$0: $AUTOPOINT --force"
+  $AUTOPOINT --force || exit
+fi
+
+# Autoreconf runs aclocal before libtoolize, which causes spurious
+# warnings if the initial aclocal is confused by the libtoolized
+# (or worse out-of-date) macro directory.
+if grep '^[    ]*LT_INIT' configure.ac >/dev/null; then
+  echo "running: $LIBTOOLIZE --copy --install"
+  $LIBTOOLIZE --copy --install
+fi

 version_controlled_file() {
   dir=$1
@@ -771,96 +780,17 @@ version_controlled_file() {
   test $found = yes
 }

-slurp() {
-  for dir in . `(cd $1 && find * -type d -print)`; do
-    copied=
-    sep=
-    for file in `ls -a $1/$dir`; do
-      case $file in
-      .|..) continue;;
-      # FIXME: should all file names starting with "." be ignored?
-      .*) continue;;
-      esac
-      test -d $1/$dir/$file && continue
-      for excluded_file in $excluded_files; do
-        test "$dir/$file" = "$excluded_file" && continue 2
-      done
-      if test $file = Makefile.am && test "X$gnulib_mk" != XMakefile.am; then
-        copied=$copied${sep}$gnulib_mk; sep=$nl
-        remove_intl='/^[^#].*\/intl/s/^/#/;'"s!$bt_regex/!!g"
-        sed "$remove_intl" $1/$dir/$file |
-        cmp - $dir/$gnulib_mk > /dev/null || {
-          echo "$me: Copying $1/$dir/$file to $dir/$gnulib_mk ..." &&
-          rm -f $dir/$gnulib_mk &&
-          sed "$remove_intl" $1/$dir/$file >$dir/$gnulib_mk &&
-          gnulib_mk_hook $dir/$gnulib_mk
-        }
-      elif { test "${2+set}" = set && test -r $2/$dir/$file; } ||
-           version_controlled_file $dir $file; then
-        echo "$me: $dir/$file overrides $1/$dir/$file"
-      else
-        copied=$copied$sep$file; sep=$nl
-        if test $file = gettext.m4; then
-          echo "$me: patching m4/gettext.m4 to remove need for intl/* ..."
-          rm -f $dir/$file
-          sed '
-            /^AC_DEFUN(\[AM_INTL_SUBDIR],/,/^]/c\
-              AC_DEFUN([AM_INTL_SUBDIR], [])
-            /^AC_DEFUN(\[gt_INTL_SUBDIR_CORE],/,/^]/c\
-              AC_DEFUN([gt_INTL_SUBDIR_CORE], [])
-            $a\
-              AC_DEFUN([gl_LOCK_EARLY], [])
-          ' $1/$dir/$file >$dir/$file
-        else
-          cp_mark_as_generated $1/$dir/$file $dir/$file
-        fi
-      fi || exit
-    done
-
-    for dot_ig in x $vc_ignore; do
-      test $dot_ig = x && continue
-      ig=$dir/$dot_ig
-      if test -n "$copied"; then
-        insert_vc_ignore $ig "$copied"
-        # If an ignored file name ends with .in.h, then also add
-        # the name with just ".h".  Many gnulib headers are generated,
-        # e.g., stdint.in.h -> stdint.h, dirent.in.h ->..., etc.
-        # Likewise for .gperf -> .h, .y -> .c, and .sin -> .sed
-        f=`echo "$copied" |
-          sed '
-            s/\.in\.h$/.h/
-            s/\.sin$/.sed/
-            s/\.y$/.c/
-            s/\.gperf$/.h/
-          '
-        `
-        insert_vc_ignore $ig "$f"
-
-        # For files like sys_stat.in.h and sys_time.in.h, record as
-        # ignorable the directory we might eventually create: sys/.
-        f=`echo "$copied"|sed 's/sys_.*\.in\.h$/sys/'`
-        insert_vc_ignore $ig "$f"
-      fi
-    done
-  done
-}
-
-
-# Create boot temporary directories to import from gnulib and gettext.
-rm -fr $bt $bt2 &&
-mkdir $bt $bt2 || exit
-
 # Import from gnulib.

 gnulib_tool_options="\
  --import\
  --no-changelog\
- --aux-dir $bt/$build_aux\
- --doc-base $bt/$doc_base\
+ --aux-dir $build_aux\
+ --doc-base $doc_base\
  --lib $gnulib_name\
- --m4-base $bt/$m4_base/\
- --source-base $bt/$source_base/\
- --tests-base $bt/$tests_base\
+ --m4-base $m4_base/\
+ --source-base $source_base/\
+ --tests-base $tests_base\
  --local-dir $local_gl_dir\
  $gnulib_tool_option_extras\
 "
@@ -872,26 +802,11 @@ if test $use_libtool = 1; then
 fi
 echo "$0: $gnulib_tool $gnulib_tool_options --import ..."
 $gnulib_tool $gnulib_tool_options --import $gnulib_modules &&
-slurp $bt || exit

 for file in $gnulib_files; do
   symlink_to_dir "$GNULIB_SRCDIR" $file || exit
 done

-
-# Import from gettext.
-with_gettext=yes
-grep '^[        ]*AM_GNU_GETTEXT_VERSION(' configure.ac >/dev/null || \
-    with_gettext=no
-
-if test $with_gettext = yes; then
-  echo "$0: (cd $bt2; ${AUTOPOINT-autopoint}) ..."
-  cp configure.ac $bt2 &&
-  (cd $bt2 && ${AUTOPOINT-autopoint} && rm configure.ac) &&
-  slurp $bt2 $bt || exit
-fi
-rm -fr $bt $bt2 || exit
-
 # Remove any dangling symlink matching "*.m4" or "*.[ch]" in some
 # gnulib-populated directories.  Such .m4 files would cause aclocal to fail.
 # The following requires GNU find 4.2.3 or newer.  Considering the usual
@@ -904,28 +819,12 @@ find "$m4_base" "$source_base" \
   -depth \( -name '*.m4' -o -name '*.[ch]' \) \
   -type l -xtype l -delete > /dev/null 2>&1

-# Reconfigure, getting other files.
-
-# Skip autoheader if it's not needed.
-grep -E '^[     ]*AC_CONFIG_HEADERS?\>' configure.ac >/dev/null ||
-  AUTOHEADER=true
-
-for command in \
-  libtool \
-  "${ACLOCAL-aclocal} --force -I '$m4_base' $ACLOCAL_FLAGS" \
-  "${AUTOCONF-autoconf} --force" \
-  "${AUTOHEADER-autoheader} --force" \
-  "${AUTOMAKE-automake} --add-missing --copy --force-missing"
-do
-  if test "$command" = libtool; then
-    test $use_libtool = 0 \
-      && continue
-    command="${LIBTOOLIZE-libtoolize} -c -f"
-  fi
-  echo "$0: $command ..."
-  eval "$command" || exit
-done
-
+# Tell autoreconf not to invoke autopoint or libtoolize; they were run above.
+echo "running: AUTOPOINT=true LIBTOOLIZE=true " \
+    "$AUTORECONF --verbose --install --no-recursive -I $m4_base $ACLOCAL_FLAGS"
+AUTOPOINT=true LIBTOOLIZE=true \
+    $AUTORECONF --verbose --install --no-recursive -I $m4_base $ACLOCAL_FLAGS \
+  || exit 1

 # Get some extra files from gnulib, overriding existing files.
 for file in $gnulib_extra_files; do
diff --git a/bootstrap.conf b/bootstrap.conf
index 9f3e16f..8e66054 100644
--- a/bootstrap.conf
+++ b/bootstrap.conf
@@ -63,8 +63,11 @@ yesno
 gnulib_tool_option_extras="\
  --avoid getline\
  --avoid rpmatch\
+ --symlink\
+ --makefile-name=gnulib.mk\
 "

+
 # Create build-aux, since 'bootstrap' won't do that for us.
 mkdir -p build-aux

diff --git a/lib/Makefile.am b/lib/Makefile.am
index fb11166..f31fd9a 100644
--- a/lib/Makefile.am
+++ b/lib/Makefile.am
@@ -16,6 +16,14 @@
 # with this program; if not, write to the Free Software Foundation, Inc.,
 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.

+AM_CFLAGS =
+BUILT_SOURCES =
+CLEANFILES =
+EXTRA_DIST =
+MOSTLYCLEANDIRS =
+MOSTLYCLEANFILES =
+SUFFIXES =
+noinst_LIBRARIES =

 include gnulib.mk

diff --git a/tests/init.sh b/tests/init.sh
index 544bf67..5985552 100644
--- a/tests/init.sh
+++ b/tests/init.sh
@@ -61,7 +61,7 @@ ME_=`expr "./$0" : '.*/\(.*\)$'`

 # We use a trap below for cleanup.  This requires us to go through
 # hoops to get the right exit status transported through the handler.
-# So use `Exit STATUS' instead of `exit STATUS' inside of the tests.
+# So use 'Exit STATUS' instead of 'exit STATUS' inside of the tests.
 # Turn off errexit here so that we don't trip the bug with OSF1/Tru64
 # sh inside this function.
 Exit () { set +e; (exit $1); exit $1; }
@@ -74,7 +74,20 @@ Exit () { set +e; (exit $1); exit $1; }
 # the reason for skip/failure to console, rather than to the .log files.
 : ${stderr_fileno_=2}

-warn_ () { echo "$@" 1>&$stderr_fileno_; }
+# Note that correct expansion of "$*" depends on IFS starting with ' '.
+# Always write the full diagnostic to stderr.
+# When stderr_fileno_ is not 2, also emit the first line of the
+# diagnostic to that file descriptor.
+warn_ ()
+{
+  # If IFS does not start with ' ', set it and emit the warning in a subshell.
+  case $IFS in
+    ' '*) printf '%s\n' "$*" >&2
+          test $stderr_fileno_ = 2 \
+            || { printf '%s\n' "$*" | sed 1q >&$stderr_fileno_ ; } ;;
+    *) (IFS=' '; warn_ "$@");;
+  esac
+}
 fail_ () { warn_ "$ME_: failed test: $@"; Exit 1; }
 skip_ () { warn_ "$ME_: skipped test: $@"; Exit 77; }
 fatal_ () { warn_ "$ME_: hard error: $@"; Exit 99; }
@@ -159,7 +172,7 @@ else
     if test "$re_shell_" = __current__; then
       # 'eval'ing this code makes Solaris 10's /bin/sh exit with
       # $? set to 2.  It does not evaluate any of the code after the
-      # "unexpected" first `('.  Thus, we must run it in a subshell.
+      # "unexpected" first '('.  Thus, we must run it in a subshell.
       ( eval "$gl_shell_test_script_" ) > /dev/null 2>&1
     else
       "$re_shell_" -c "$gl_shell_test_script_" 2>/dev/null
@@ -208,14 +221,102 @@ export MALLOC_PERTURB_
 # a partition, or to undo any other global state changes.
 cleanup_ () { :; }

-if ( diff --version < /dev/null 2>&1 | grep GNU ) > /dev/null 2>&1; then
-  compare () { diff -u "$@"; }
+# Emit a header similar to that from diff -u;  Print the simulated "diff"
+# command so that the order of arguments is clear.  Don't bother with @@ lines.
+emit_diff_u_header_ ()
+{
+  printf '%s\n' "diff -u $*" \
+    "--- $1    1970-01-01" \
+    "+++ $2    1970-01-01"
+}
+
+# Arrange not to let diff or cmp operate on /dev/null,
+# since on some systems (at least OSF/1 5.1), that doesn't work.
+# When there are not two arguments, or no argument is /dev/null, return 2.
+# When one argument is /dev/null and the other is not empty,
+# cat the nonempty file to stderr and return 1.
+# Otherwise, return 0.
+compare_dev_null_ ()
+{
+  test $# = 2 || return 2
+
+  if test "x$1" = x/dev/null; then
+    test -s "$2" || return 0
+    emit_diff_u_header_ "$@"; sed 's/^/+/' "$2"
+    return 1
+  fi
+
+  if test "x$2" = x/dev/null; then
+    test -s "$1" || return 0
+    emit_diff_u_header_ "$@"; sed 's/^/-/' "$1"
+    return 1
+  fi
+
+  return 2
+}
+
+if diff_out_=`exec 2>/dev/null; diff -u "$0" "$0" < /dev/null`; then
+  if test -z "$diff_out_"; then
+    compare_ () { diff -u "$@"; }
+  else
+    compare_ ()
+    {
+      if diff -u "$@" > diff.out; then
+        # No differences were found, but Solaris 'diff' produces output
+        # "No differences encountered". Hide this output.
+        rm -f diff.out
+        true
+      else
+        cat diff.out
+        rm -f diff.out
+        false
+      fi
+    }
+  fi
+elif diff_out_=`exec 2>/dev/null; diff -c "$0" "$0" < /dev/null`; then
+  if test -z "$diff_out_"; then
+    compare_ () { diff -c "$@"; }
+  else
+    compare_ ()
+    {
+      if diff -c "$@" > diff.out; then
+        # No differences were found, but AIX and HP-UX 'diff' produce output
+        # "No differences encountered" or "There are no differences between the
+        # files.". Hide this output.
+        rm -f diff.out
+        true
+      else
+        cat diff.out
+        rm -f diff.out
+        false
+      fi
+    }
+  fi
 elif ( cmp --version < /dev/null 2>&1 | grep GNU ) > /dev/null 2>&1; then
-  compare () { cmp -s "$@"; }
+  compare_ () { cmp -s "$@"; }
 else
-  compare () { cmp "$@"; }
+  compare_ () { cmp "$@"; }
 fi

+# Usage: compare EXPECTED ACTUAL
+#
+# Given compare_dev_null_'s preprocessing, defer to compare_ if 2 or more.
+# Otherwise, propagate $? to caller: any diffs have already been printed.
+compare ()
+{
+  # This looks like it can be factored to use a simple "case $?"
+  # after unchecked compare_dev_null_ invocation, but that would
+  # fail in a "set -e" environment.
+  if compare_dev_null_ "$@"; then
+    return 0
+  else
+    case $? in
+      1) return 1;;
+      *) compare_ "$@";;
+    esac
+  fi
+}
+
 # An arbitrary prefix to help distinguish test directories.
 testdir_prefix_ () { printf gt; }

@@ -426,7 +527,7 @@ mktempd_ ()
   esac

   # First, try to use mktemp.
-  d=`unset TMPDIR; mktemp -d -t -p "$destdir_" "$template_" 2>/dev/null` \
+  d=`unset TMPDIR; { mktemp -d -t -p "$destdir_" "$template_"; } 2>/dev/null` \
     || fail=1

   # The resulting name must be in the specified directory.
--
1.7.9.rc0.13.gbee72


>From fcdcc8474b6210caf8be867f4eeeee250b0d489d Mon Sep 17 00:00:00 2001
From: Jim Meyering <address@hidden>
Date: Wed, 11 Jan 2012 21:35:41 +0100
Subject: [PATCH 2/2] tests: make all test scripts executable; work with
 automake-1.12

* tests/Makefile.am (TESTS_ENVIRONMENT): Adapt to work with upcoming
automake-1.12.
* tests/mixed: Make executable.
* tests/zgrep-f: Likewise.
* tests/zgrep-signal: Likewise.
* tests/znew-k: Likewise.
---
 tests/Makefile.am |    3 +--
 1 files changed, 1 insertions(+), 2 deletions(-)
 mode change 100644 => 100755 tests/mixed
 mode change 100644 => 100755 tests/zgrep-f
 mode change 100644 => 100755 tests/zgrep-signal
 mode change 100644 => 100755 tests/znew-k

diff --git a/tests/Makefile.am b/tests/Makefile.am
index 9e4b241..3b6ec31 100644
--- a/tests/Makefile.am
+++ b/tests/Makefile.am
@@ -52,7 +52,6 @@ built_programs = \
 TESTS_ENVIRONMENT =                            \
   tmp__=$$TMPDIR; test -d "$$tmp__" || tmp__=.;        \
   TMPDIR=$$tmp__; export TMPDIR;               \
-  exec 9>&2;                                   \
   shell_or_perl_() {                           \
     if grep '^\#!/usr/bin/perl' "$$1" > /dev/null; then                        
\
       if $(PERL) -e 'use warnings' > /dev/null 2>&1; then              \
@@ -85,6 +84,6 @@ TESTS_ENVIRONMENT =                           \
   PERL='$(PERL)'                               \
   SHELL='$(SHELL)'                             \
   PATH='$(abs_top_builddir)$(PATH_SEPARATOR)'"$$PATH" \
-  ; shell_or_perl_
+  ; 9>&2

 VERBOSE = yes
diff --git a/tests/mixed b/tests/mixed
old mode 100644
new mode 100755
diff --git a/tests/zgrep-f b/tests/zgrep-f
old mode 100644
new mode 100755
diff --git a/tests/zgrep-signal b/tests/zgrep-signal
old mode 100644
new mode 100755
diff --git a/tests/znew-k b/tests/znew-k
old mode 100644
new mode 100755
--
1.7.9.rc0.13.gbee72