Test suite and autotest

[Tim Landscheidt]

Hi,

TODO.private reads since 1999:

| [...]

| The Top Item
| ------------

| If you know perl (or want to learn DejaGNU or similar), the number one
| priority on my list of things I don't have time to do right now is
| fixing up the GNU make test suite.  Most importantly it needs to be made
| "parallelizable", so more than one regression can run at the same time
| (essentially, make the "work" directory local).  Also, the CWD during
| the test should be in the work directory or, better, a test-specific
| temporary directory so each test gets a new directory; right now
| sometimes tests leak files into the main directory which causes
| subsequent tests to fail (some tests may need to be tweaked).  Beyond
| that, any cleanup done to make writing, reading, or handling tests
| simpler would be great!  Please feel free to make whatever changes you
| like to the current tests, given some high-level goals, and that you'll
| port the current tests to whatever you do :).

| [...]

What is the sentiment concerning converting the test suite
to autotest?  Attached is an initial, unpolished patch for
one test.  NB: tests/run_make_tests.pl deletes
tests/Makefile, so you have to run config.status after
(every) "make check".

Tim

>From 3ef11dc16e176c248b5ffab057d46d00b9a5ac8f Mon Sep 17 00:00:00 2001
From: Tim Landscheidt <address@hidden>
Date: Mon, 5 Mar 2012 20:18:05 +0000
Subject: [PATCH] Snapshot of conversion to autotest.

---
 Makefile.am                  |    2 +-
 configure.in                 |    3 +-
 tests/Makefile.am            |   37 ++++++++++++++++++
 tests/local.at               |   18 +++++++++
 tests/options/dash-n.at      |   62 ++++++++++++++++++++++++++++++
 tests/scripts/options/dash-n |   85 ------------------------------------------
 tests/testsuite.at           |    3 +
 7 files changed, 123 insertions(+), 87 deletions(-)
 create mode 100644 tests/Makefile.am
 create mode 100644 tests/atlocal.in
 create mode 100644 tests/local.at
 create mode 100644 tests/options/dash-n.at
 delete mode 100644 tests/scripts/options/dash-n
 create mode 100644 tests/testsuite.at

diff --git a/Makefile.am b/Makefile.am
index 133a579..c35406d 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -29,7 +29,7 @@ if WINDOWSENV
   W32LIB =     -Lw32 -lw32
 endif
 
-SUBDIRS =      glob config po doc $(MAYBE_W32)
+SUBDIRS =      glob config po doc tests $(MAYBE_W32)
 
 bin_PROGRAMS = make
 
diff --git a/configure.in b/configure.in
index d2d3387..d4edda0 100644
--- a/configure.in
+++ b/configure.in
@@ -461,7 +461,8 @@ AS_IF([test "x$make_cv_job_server" = xno && test 
"x$user_job_server" = xyes],
 
 # Specify what files are to be created.
 AC_CONFIG_FILES([Makefile glob/Makefile po/Makefile.in config/Makefile \
-                 doc/Makefile w32/Makefile])
+                 doc/Makefile tests/atlocal tests/Makefile w32/Makefile])
+AC_CONFIG_TESTDIR([tests])
 
 # OK, do it!
 
diff --git a/tests/Makefile.am b/tests/Makefile.am
new file mode 100644
index 0000000..dbefa42
--- /dev/null
+++ b/tests/Makefile.am
@@ -0,0 +1,37 @@
+# The `:;' works around a Bash 3.2 bug when the output is not writeable.
+$(srcdir)/package.m4: $(top_srcdir)/configure.in
+       :;{ \
+         echo '# Signature of the current package.' && \
+         echo 'm4_define([AT_PACKAGE_NAME],' && \
+         echo '  [$(PACKAGE_NAME)])' && \
+         echo 'm4_define([AT_PACKAGE_TARNAME],' && \
+         echo '  [$(PACKAGE_TARNAME)])' && \
+         echo 'm4_define([AT_PACKAGE_VERSION],' && \
+         echo '  [$(PACKAGE_VERSION)])' && \
+         echo 'm4_define([AT_PACKAGE_STRING],' && \
+         echo '  [$(PACKAGE_STRING)])' && \
+         echo 'm4_define([AT_PACKAGE_BUGREPORT],' && \
+         echo '  [$(PACKAGE_BUGREPORT)])'; \
+         echo 'm4_define([AT_PACKAGE_URL],' && \
+         echo '  [$(PACKAGE_URL)])'; \
+       } >'$(srcdir)/package.m4'
+
+EXTRA_DIST = $(srcdir)/testsuite.at $(srcdir)/local.at $(srcdir)/package.m4 
$(TESTSUITE) atlocal.in
+TESTSUITE = $(srcdir)/testsuite
+
+check-local: atconfig atlocal $(TESTSUITE)
+       $(SHELL) '$(TESTSUITE)' $(TESTSUITEFLAGS)
+
+installcheck-local: atconfig atlocal $(TESTSUITE)
+       $(SHELL) '$(TESTSUITE)' AUTOTEST_PATH='$(bindir)' \
+         $(TESTSUITEFLAGS)
+
+clean-local:
+       test ! -f '$(TESTSUITE)' || \
+         $(SHELL) '$(TESTSUITE)' --clean
+
+AUTOM4TE = $(SHELL) $(top_srcdir)/config/missing --run autom4te
+AUTOTEST = $(AUTOM4TE) --language=autotest
+$(TESTSUITE): $(srcdir)/testsuite.at $(srcdir)/local.at 
$(srcdir)/options/dash-n.at $(srcdir)/package.m4
+       $(AUTOTEST) -I '$(srcdir)' -o address@hidden address@hidden
+       mv address@hidden $@
diff --git a/tests/atlocal.in b/tests/atlocal.in
new file mode 100644
index 0000000..e69de29
diff --git a/tests/local.at b/tests/local.at
new file mode 100644
index 0000000..f1f4618
--- /dev/null
+++ b/tests/local.at
@@ -0,0 +1,18 @@
+# AT_TEST_MAKE(MAKEFILE, OPTIONS, EXPOUT)
+# ---------------------------------------
+# Try MAKEFILE with OPTIONS and expect EXPOUT.
+m4_define([AT_TEST_MAKE],
+[AT_CHECK([MAKELEVEL= ../../../make --no-print-directory $2 -f $1], 0, [$3], 
[])])
+
+# AT_TOUCH(FILE)
+# --------------
+# Create/overwrite FILE with a newline.
+m4_define([AT_TOUCH],
+[AT_CHECK([echo > $1], 0, [], [])])
+
+# AT_UTOUCH(OFF, FILE)
+# --------------------
+# Create/overwrite FILE with a newline last modified OFF seconds ago.
+m4_define([AT_UTOUCH],
+[AT_CHECK([echo > $2], 0, [], [])
+AT_CHECK([touch -d 'm4_eval(-1 * $1) seconds ago' $2])])
diff --git a/tests/options/dash-n.at b/tests/options/dash-n.at
new file mode 100644
index 0000000..1baed50
--- /dev/null
+++ b/tests/options/dash-n.at
@@ -0,0 +1,62 @@
+# Try various uses of -n and ensure they all give the correct results.
+
+AT_SETUP([[Test the -n option.]])
+
+AT_DATA([[Makefile]], [[
+final: intermediate ; echo >> $@
+intermediate: orig ; echo >> $@
+
+]])
+
+AT_TOUCH([[orig]])
+
+# Test 0.
+
+AT_TEST_MAKE([[Makefile]], [[]], [[echo >> intermediate
+echo >> final
+]])
+
+# Test 1.
+
+AT_TEST_MAKE([[Makefile]], [[-Worig -n]], [[echo >> intermediate
+echo >> final
+]])
+AT_CHECK([[rm -f orig intermediate final]])
+
+# We consider the actual updated timestamp of targets with all
+# recursive commands, even with -n.
+
+AT_DATA([[Makefile2]], [[.SUFFIXES:
+BAR =     # nothing
+FOO = +$(BAR)
+a: b; echo > $@
+b: c; $(FOO)
+]])
+AT_UTOUCH([[-20]], [[b]])
+AT_UTOUCH([[-10]], [[a]])
+AT_TOUCH([[c]])
+
+# Test 2.
+
+AT_TEST_MAKE([[Makefile2]], [[]], [[make: `a' is up to date.
+]])
+
+# Test 3.
+
+AT_TEST_MAKE([[Makefile2]], [[-n]], [[make: `a' is up to date.
+]])
+
+# Test 4.
+
+AT_CHECK([[rm -f a b]])
+
+AT_TEST_MAKE([[Makefile2]], [[-t -n]], [[touch b
+touch a
+]])
+
+AT_CHECK([test ! -e a])
+AT_CHECK([test ! -e b])
+
+# Cleanup.
+
+AT_CLEANUP
diff --git a/tests/scripts/options/dash-n b/tests/scripts/options/dash-n
deleted file mode 100644
index 248e0c8..0000000
--- a/tests/scripts/options/dash-n
+++ /dev/null
@@ -1,85 +0,0 @@
-#                                                                    -*-perl-*-
-$description = "Test the -n option.\n";
-
-$details = "Try various uses of -n and ensure they all give the correct 
results.\n";
-
-open(MAKEFILE, "> $makefile");
-
-# The Contents of the MAKEFILE ...
-
-print MAKEFILE <<'EOMAKE';
-
-final: intermediate ; echo >> $@
-intermediate: orig ; echo >> $@
-
-EOMAKE
-
-close(MAKEFILE);
-
-&touch('orig');
-
-# TEST 0
-
-&run_make_with_options($makefile, "", &get_logfile);
-$answer = "echo >> intermediate\necho >> final\n";
-&compare_output($answer, &get_logfile(1));
-
-# TEST 1
-
-&run_make_with_options($makefile, "-Worig -n", &get_logfile);
-$answer = "echo >> intermediate\necho >> final\n";
-&compare_output($answer, &get_logfile(1));
-
-unlink('orig', 'intermediate', 'final');
-
-# We consider the actual updated timestamp of targets with all
-# recursive commands, even with -n.
-
-$makefile2 = &get_tmpfile;
-
-open(MAKEFILE, "> $makefile2");
-
-print MAKEFILE <<'EOF';
-.SUFFIXES:
-BAR =     # nothing
-FOO = +$(BAR)
-a: b; echo > $@
-b: c; $(FOO)
-EOF
-
-close(MAKEFILE);
-
-&utouch(-20, 'b');
-&utouch(-10, 'a');
-&touch('c');
-
-# TEST 2
-
-&run_make_with_options($makefile2, "", &get_logfile);
-$answer = "$make_name: `a' is up to date.\n";
-&compare_output($answer, &get_logfile(1));
-
-# TEST 3
-
-&run_make_with_options($makefile2, "-n", &get_logfile);
-$answer = "$make_name: `a' is up to date.\n";
-&compare_output($answer, &get_logfile(1));
-
-# TEST 4
-
-unlink(qw(a b));
-
-&run_make_with_options($makefile2, "-t -n", &get_logfile);
-
-open(DASH_N_LOG, ">>" . &get_logfile(1));
-print DASH_N_LOG "a exists but should not!\n" if -e 'a';
-print DASH_N_LOG "b exists but should not!\n" if -e 'b';
-close(DASH_N_LOG);
-
-&compare_output("touch b\ntouch a\n", &get_logfile(1));
-
-# CLEANUP
-
-unlink(qw(a b c));
-
-1;
diff --git a/tests/testsuite.at b/tests/testsuite.at
new file mode 100644
index 0000000..7ef2c23
--- /dev/null
+++ b/tests/testsuite.at
@@ -0,0 +1,3 @@
+AT_INIT
+
+m4_include([options/dash-n.at])
-- 
1.6.2.5