Re: Libtool stresstest: echo marker with loop variables

[Ralf Wildenhues]

Hi Peter,

* Peter Ekberg wrote on Tue, Sep 20, 2005 at 12:07:00PM CEST:
> * Ralf Wildenhues wrote on Monday, September 19, 2005 19:22 CEST:
> 
*big snip*
> > If any link tests failed, you might be able to paste and run them from
> > the failures file (given that the prerequisite files are present).
> 
> Wishlist here is to dump the state of all active loop variables,
> preferably in copy-pasteable form, so that it is dead easy to
> recreate all the prerequisites. But just a wish...

Hmm.  We'd have to re-quote them then.
How about we just put the commands to reproduce this in the failure
file?  Then, after running the test with the test suite once, you can
then go in, and just give the shell pieces of `failures' or `skips' to
eat again?

*snip*
> > +# LT_AT_EXEC_NOFAIL_CHECK(EXECUTABLE)
> > +# -----------------------------------
> > +# EXECUTABLE always gives 0 in the good case.
> > +m4_define([LT_AT_EXEC_NOFAIL_CHECK],
> > +[AT_CHECK(
> > +  [if $1; then :; else
> > +     if test "X$host" != "X$build" && { test -x "$1" || test 
> > -x "$1"$EXEEXT; }
> > +     then out=skips; else out=failures; fi
> > +   eval \$ECHO \'\# failure of $1 after: $last_link\' >>$out
> > +   fi], [ignore], [ignore], [ignore])
> > +])
> 
> Would be nice if the failure line was also echoed to stdout, but
> perhaps that disrupts the stdout file?

No, I think that is no problem at all, we could just use `tee' here.
But I don't think it would help much: there is way too much noise
to actually see the failures.  Or do you mean without using `-v'
in the TESTSUITE_FLAGS?  That's not really how autotest meant us
to use it. ;)

Note that we currently don't use a `stdout' file here at all.

> Otherwise you don't see
> the failures/skips until after the whole test (unless you tail -f
> the files). Also valid for previous macro I suppose. The exit
> code should perhaps be logged?

Agreed to both.

I've scratched much of the last patch and decided to reduce the noise
of TESTSUITE_FLAGS=-v.  Here's what I am at now.  It's not ready for
inclusion by far, can be done much nicer and factored more, but it'd be
nice if you could give it a try.  :)

Cheers,
Ralf

Index: tests/testsuite.at
===================================================================
RCS file: /cvsroot/libtool/libtool/tests/testsuite.at,v
retrieving revision 1.19
diff -u -r1.19 testsuite.at
--- tests/testsuite.at  16 Sep 2005 08:33:24 -0000      1.19
+++ tests/testsuite.at  20 Sep 2005 16:37:12 -0000
@@ -25,7 +25,7 @@
 : ${AUTOCONF=autoconf}
 export LIBTOOLIZE LIBTOOL ACLOCAL AUTOCONF
 eval `$LIBTOOL --config | grep ^EGREP=`
-eval `$LIBTOOL --config | $EGREP '^(host|host_os|build)='`
+eval `$LIBTOOL --config | $EGREP '^(host|host_os|build|ECHO)='`
 m4_divert_pop([PREPARE_TESTS])dnl
 
 # LT_AT_LIBTOOLIZE([ARGS])
Index: tests/link-order.at
===================================================================
RCS file: /cvsroot/libtool/libtool/tests/link-order.at,v
retrieving revision 1.4
diff -u -r1.4 link-order.at
--- tests/link-order.at 13 Aug 2005 06:57:39 -0000      1.4
+++ tests/link-order.at 20 Sep 2005 16:37:12 -0000
@@ -19,7 +19,6 @@
 # link-order.test - make sure that library linking order matches
 
 AT_SETUP([Link order test.])
-eval `$LIBTOOL --config | grep ECHO=`
 LDFLAGS="$LDFLAGS -no-undefined"
 
 prefix_old=`pwd`/old
Index: tests/stresstest.at
===================================================================
RCS file: /cvsroot/libtool/libtool/tests/stresstest.at,v
retrieving revision 1.6
diff -u -r1.6 stresstest.at
--- tests/stresstest.at 13 Sep 2005 07:28:14 -0000      1.6
+++ tests/stresstest.at 20 Sep 2005 16:37:12 -0000
@@ -170,32 +170,89 @@
 AT_CHECK([$LIBTOOL --mode=compile $CC $CFLAGS -c main.c],[0],[ignore],[ignore])
 AT_CHECK([$LIBTOOL --mode=compile $CC $CFLAGS -c dlself.c -o 
sub3/dlself.lo],[0],[ignore],[ignore])
 
+
+# LT_AT_FAIL_HIST(LEVEL, [EXECUTABLE])
+# ------------------------------------
+m4_define([LT_AT_FAIL_HIST],
+[echo "# failure, status $lt_status in sequence:"
+m4_for([index], [1], [$1], [1], [echo "$lt_cmd_[]index"
+])
+])
+
+
+# LT_AT_EXPAND_NOFAIL_CHECK(LEVEL, COMMANDS)
+# ------------------------------------------
+# Do not use single quotes in the m4 literal COMMANDS, please.
+m4_define([LT_AT_EXPAND_NOFAIL_CHECK],
+[eval lt_cmd_$1=\'$2\'
+AT_CHECK([echo "$lt_cmd_$1"; $lt_cmd_$1; echo $? > status],
+        [ignore], [ignore], [ignore])
+lt_status=`cat status`
+if test $lt_status -ne 0; then
+  echo "failure, exit status $lt_status." >&2
+  ( LT_AT_FAIL_HIST([$1]) ) >>failures
+fi
+])
+
+
+# LT_AT_EXEC_NOFAIL_CHECK(LEVEL, EXECUTABLE)
+# ------------------------------------------
+# EXECUTABLE always gives 0 in the good case.
+m4_define([LT_AT_EXEC_NOFAIL_CHECK],
+[lt_cmd_$1='$2'
+AT_CHECK([$2; echo $? > status], [ignore], [ignore], [ignore])
+lt_status=`cat status`
+if test $lt_status -ne 0; then
+  if test "X$host" != "X$build" && { test -x "$2" || test -x "$2"$EXEEXT; }; 
then
+    out=skip
+  else
+    out=failure
+  fi
+  echo "$out, exit status $lt_status." >&2
+  ( LT_AT_FAIL_HIST([$1]) ) >>${out}s
+fi
+])
+
+
+# list of arguments we want to check
 case $host_os in
-  cygwin* | mingw* | pw32*) undef_opts=-no-undefined ;;
-  *)                        undef_opts='"" -no-undefined' ;;
+cygwin*|mingw*|pw32*)
+   : ${stresstest_undef_opts="-no-undefined"} ;;
+*) : ${stresstest_undef_opts="'' -no-undefined"} ;;
 esac
+: ${stresstest_expsyms="'' '-export-symbols-regex v.*' '-export-symbols 
asyms'"}
+: ${stresstest_rpath="'' '-rpath /nonexistent'"}
+: ${stresstest_relpath="'' ./ `pwd`/"}
+: ${stresstest_static="'' '-static'"}
+: ${stresstest_selfsyms="'' '-export-symbols dlselfsyms'"}
 
-for l1 in $undef_opts
+eval "set x $stresstest_undef_opts"; shift
+for l1
 do
-  for l2 in '' '-export-symbols-regex v.*' '-export-symbols asyms'
+  eval "set x $stresstest_expsyms"; shift
+  for l2
   do
-    for l3 in '' '-rpath /nonexistent'
+    eval "set x $stresstest_rpath"; shift
+    for l3
     do
       linkargs="$l1 $l2 $l3"
-      for rel in '' ./ `pwd`/
+      eval "set x $stresstest_relpath"; shift
+      for rel
       do
-       AT_CHECK([$LIBTOOL --mode=link $CC $CFLAGS $LDFLAGS -o 
"$rel"sub2/liba.la "$rel"sub/a.lo $linkargs],
-                [0],[ignore],[ignore])
-       for st in '' '-static'
+       LT_AT_EXPAND_NOFAIL_CHECK([1],
+           [$LIBTOOL --mode=link $CC $CFLAGS $LDFLAGS -o "$rel"sub2/liba.la 
"$rel"sub/a.lo $linkargs])
+       eval "set x $stresstest_static"; shift
+       for st
        do
-         AT_CHECK([$LIBTOOL --mode=link $CC $CFLAGS $LDFLAGS $st -o "$rel"main 
"$rel"main.lo "$rel"sub2/liba.la],
-                  [0],[ignore],[ignore])
-         LT_AT_EXEC_CHECK([./main],[0])
-         for l10 in '' '-export-symbols dlselfsyms'
+         LT_AT_EXPAND_NOFAIL_CHECK([2],
+             [$LIBTOOL --mode=link $CC $CFLAGS $LDFLAGS $st -o "$rel"main 
"$rel"main.lo "$rel"sub2/liba.la])
+         LT_AT_EXEC_NOFAIL_CHECK([3], [./main])
+         eval "set x $stresstest_selfsyms"; shift
+         for l10
          do
-           AT_CHECK([$LIBTOOL --mode=link $CC $CFLAGS $LDFLAGS $st -o 
"$rel"sub3/dlself "$rel"sub3/dlself.lo "$rel"sub2/liba.la sub/b.lo -dlopen self 
$l10],
-                    [0],[ignore],[ignore])
-           LT_AT_EXEC_CHECK([./sub3/dlself],[0])
+           LT_AT_EXPAND_NOFAIL_CHECK([3],
+               [$LIBTOOL --mode=link $CC $CFLAGS $LDFLAGS $st -o 
"$rel"sub3/dlself "$rel"sub3/dlself.lo "$rel"sub2/liba.la sub/b.lo -dlopen self 
$l10])
+           LT_AT_EXEC_NOFAIL_CHECK([4], [./sub3/dlself])
          done
        done
       done
@@ -203,4 +260,8 @@
   done
 done
 
+AT_CHECK([test -s failures], [1], [ignore], [ignore],
+        [echo "Please check the \`failures' file in the test directory"])
+AT_CHECK([test -s skips && (exit 77)], [1], [ignore], [ignore],
+        [echo "Please check the \`skips' file in the test directory"])
 AT_CLEANUP