[FYI] {micro} tests: run_make: options to do command redirection

[Stefano Lattarini]

Let's improve the API of the 'run_make()' helper shell function by
adding three new options:

  -O   Save the standard output from make on disk, in a regular file
       named 'stdout'.

  -E   Save the standard error from make on disk, in a regular file
       named 'stderr'.

  -M   Save both the standard output and standard error from make on
       disk, in a regular file named 'output'. This option supersedes
       both the '-O' and '-E' options.

This new API has two main advantages.

  1. Its use will allow us to get rid of more cumbersome idioms
     like, e.g.,

       $MAKE check >stdout && { cat stdout; exit 1; }
       cat stdout

     That can now be substituted with a simpler one:

       run_make -e FAIL -O check

  2. More importantly, using the new API we will prevent any extra output
     from the shell traces of the code in run_make to be redirected along
     with the make stderr (where that was redirected).  This problem was
     present in usages like, e.g.,

       run_make TESTS=foo.test check 2>stderr && exit 1
       grep 'expected error message' stderr

     Such usages are now to be rewritten as follows:

       run_make -e FAIL -E TESTS=foo.test check
       grep 'expected error message' stderr

     ensuring that 'stderr' won't end up containing unrelated stuff.

Note that we do not convert in bulk the old idioms and the use of
redirected 'run_make' invocations with this patch.  We only convert
some occurrences, to ensure that the new implementation of 'run_make'
is sound enough.  More sweeping conversions will likely be done in
follow-up patches.

* t/ax/am-test-lib.sh (run_make): Enhance and implement the extended API.
* t/tap-xfail-tests.sh: Use the new 'run_make' API.
* t/test-driver-cond.sh: Likewise.
* t/tests-environment-fd-redirect.sh: Likewise.
* t/uninstall-fail.sh: Likewise.
* t/yacc-dist-nobuild.sh: Likewise.

Signed-off-by: Stefano Lattarini <address@hidden>
---
 t/ax/am-test-lib.sh                | 46 ++++++++++++++++++++++++++++++++------
 t/tap-xfail-tests.sh               |  8 ++-----
 t/test-driver-cond.sh              |  8 +++----
 t/tests-environment-fd-redirect.sh |  3 +--
 t/uninstall-fail.sh                | 10 ++++-----
 t/yacc-dist-nobuild.sh             |  5 ++---
 6 files changed, 51 insertions(+), 29 deletions(-)

diff --git a/t/ax/am-test-lib.sh b/t/ax/am-test-lib.sh
index 4f5c895..74311f3 100644
--- a/t/ax/am-test-lib.sh
+++ b/t/ax/am-test-lib.sh
@@ -159,16 +159,33 @@ is_valid_varname ()
   return 0
 }
 
-# run_make [-e STATUS] [--] [VAR=VAL ...] [MAKE-ARGS...]
-# ------------------------------------------------------
+# run_make [-e STATUS] [-O] [-E] [-M] [--] [VAR=VAL ...] [MAKE-ARGS...]
+# ---------------------------------------------------------------------
+#
 # Run $MAKE with the given command-line, and fail if it doesn't exit with
 # STATUS (default: 0).  If STATUS is "FAIL", then any exit status > 0 is
 # acceptable.  If STATUS is "IGNORE", any exit value is acceptable.
+#
+# Other options:
+#
+#  -O   save the standard output from make on disk, in a regular file
+#       named 'stdout'.
+#
+#  -E   save the standard error from make on disk, in a regular file
+#       named 'stderr'.
+#
+#  -M   save both the standard output and standard error from make on
+#       disk, in a regular file named 'output'. This option supersedes
+#       both the '-O' and '-E' options.
+#
 # This function also handle command-line override of variable definition
 # in a smart way, using AM_MAKEFLAGS if a non-GNU make implementation
 # is in use.
+#
 run_make ()
 {
+  am__make_redirect=
+  am__make_flags=
   # Follow-up code might want to analyse these, so don't make them as
   # private, nor unset them later.
   am_make_rc_exp=0
@@ -177,12 +194,15 @@ run_make ()
   while test $# -gt 0; do
     case $1 in
       -e) am_make_rc_exp=$2; shift;;
+      -O) am__make_redirect="$am__make_redirect >stdout";;
+      -E) am__make_redirect="$am__make_redirect 2>stderr";;
+      -M) am__make_redirect=">output 2>&1";;
       --) shift; break;;
        *) break;;
     esac
     shift
   done
-  am__make_flags=
+
   if using_gmake; then
     # We can trust GNU make to correctly pass macro definitions given
     # on the command line down to sub-make invocations, and this allow
@@ -221,12 +241,23 @@ run_make ()
     done
     unset am__x
   fi
+
   if test x"$am__make_flags" != x; then
-    $MAKE AM_MAKEFLAGS="$am__make_flags" ${1+"$@"} || am_make_rc_got=$?
-  else
-    $MAKE ${1+"$@"} || am_make_rc_got=$?
+     set AM_MAKEFLAGS="$am__make_flags" ${1+"$@"}
+     unset am__make_flags
   fi
-  unset am__make_flags
+
+  eval "\$MAKE${am__make_redirect}"' ${1+"$@"}' || am_make_rc_got=$?
+
+  case $am__make_redirect in
+           *output*) cat output;;
+    *stderr*stdout*) cat stdout && cat stderr >&2;;
+    *stdout*stderr*) cat stdout && cat stderr >&2;;
+           *stdout*) cat stdout;;
+           *stderr*) cat stderr >&2;;
+  esac \
+    || fatal_ "displaying make output"
+
   case $am_make_rc_exp in
     IGNORE)
       : Ignore exit status
@@ -240,6 +271,7 @@ run_make ()
      test $am_make_rc_exp -eq $am_make_rc_got || return 1
      ;;
   esac
+  unset am__make_redirect
 }
 
 # AUTOMAKE_run [-e STATUS] [-d DESCRIPTION] [--] [AUTOMAKE-ARGS...]
diff --git a/t/tap-xfail-tests.sh b/t/tap-xfail-tests.sh
index 79bde30..b8b785e 100644
--- a/t/tap-xfail-tests.sh
+++ b/t/tap-xfail-tests.sh
@@ -36,9 +36,7 @@ not ok 6 # SKIP
 Bail out!
 END
 
-$MAKE check >stdout && { cat stdout; exit 1; }
-cat stdout
-
+run_make -O -e FAIL check
 count_test_results total=7 pass=0 fail=0 xpass=2 xfail=3 skip=1 error=1
 
 grep '^XPASS: all\.test 1$' stdout
@@ -59,9 +57,7 @@ ok 2 # SKIP
 not ok 3 # TODO
 END
 
-$MAKE check >stdout || { cat stdout; exit 1; }
-cat stdout
-
+run_make -O check
 count_test_results total=3 pass=0 fail=0 xpass=0 xfail=2 skip=1 error=0
 
 :
diff --git a/t/test-driver-cond.sh b/t/test-driver-cond.sh
index cf408d0..cf204c4 100644
--- a/t/test-driver-cond.sh
+++ b/t/test-driver-cond.sh
@@ -102,17 +102,15 @@ do_count ()
   $EGREP 'XFAIL: baz\.sh 3( |$)' stdout
 }
 
-st=0; $MAKE check >stdout || st=$?
-cat stdout
+run_make -O -e IGNORE check
 cat test-suite.log
 cat foo.log
 cat bar.log
 cat baz.log
-test $st -eq 0 || exit 1
+test $am_make_rc_got -eq 0 || exit 1
 do_count
 
-$MAKE distcheck >stdout || { cat stdout; exit 1; }
-cat stdout
+run_make -O distcheck
 do_count
 
 :
diff --git a/t/tests-environment-fd-redirect.sh 
b/t/tests-environment-fd-redirect.sh
index 243174a..c08d5c6 100644
--- a/t/tests-environment-fd-redirect.sh
+++ b/t/tests-environment-fd-redirect.sh
@@ -79,8 +79,7 @@ for sh in "$SHELL" "$bin_ksh"; do
 END
     $AUTOMAKE -a
     CONFIG_SHELL="$sh" $sh ./configure CONFIG_SHELL="$sh"
-    VERBOSE=y $MAKE check >stdout || { cat stdout; exit 1; }
-    cat stdout
+    run_make -O VERBOSE=y check
     grep '[ /]foo\.test: foofoofoo$' stdout
     grep '[ /]foo\.test: barbarbar$' stdout
     grep '[ /]bar\.test: 8888$' stdout
diff --git a/t/uninstall-fail.sh b/t/uninstall-fail.sh
index f002e08..1c7f9da 100644
--- a/t/uninstall-fail.sh
+++ b/t/uninstall-fail.sh
@@ -68,8 +68,7 @@ mkdir $inst $inst/share
 : > $inst/share/foobar.txt
 
 chmod a-w $inst/share
-$MAKE uninstall >output 2>&1 && { cat output; exit 1; }
-cat output
+run_make -M -e FAIL uninstall
 if test $rm_f_is_silent_on_error = yes; then
   : "rm -f" is silent on errors, skip the grepping of make output
 else
@@ -79,9 +78,8 @@ fi
 chmod a-rwx $inst/share
 (cd $inst/share) && skip_ "cannot make directories fully unreadable"
 
-$MAKE uninstall >output 2>&1 && { cat output; exit 1; }
-cat output
-#
+run_make -M -e FAIL uninstall
+
 # Some shells, like Solaris 10 /bin/ksh and /usr/xpg4/bin/sh, do not
 # report the name of the 'cd' builtin upon a chdir error:
 #
@@ -97,7 +95,7 @@ cat output
 #   > \
 #   > cd unreadable'
 #   /bin/ksh[3]: unreadable: permission denied
-#
+
 $EGREP "(cd|sh)(\[[0-9]*[0-9]\])?: .*$inst/share" output
 
 :
diff --git a/t/yacc-dist-nobuild.sh b/t/yacc-dist-nobuild.sh
index f0e0ea6..54164b8 100644
--- a/t/yacc-dist-nobuild.sh
+++ b/t/yacc-dist-nobuild.sh
@@ -83,8 +83,7 @@ chmod a-w $distdir
 mkdir build2
 cd build2
 ../$distdir/configure
-$MAKE >out 2>&1 && { cat out; exit 1; }
-cat out
-$FGREP parse.c out
+run_make -e FAIL -M
+$FGREP parse.c output
 
 :
-- 
1.8.3.rc2