[Qemu-devel] [RFC PATCH] check: integrate with virt-test (formerly known as KVM Autotest)

[Paolo Bonzini]

This adds a new "check-autotest" target that calls out to the
Autotest-based virt-test testsuite.  The testsuite is very comprehensive,
however the current state of the "developer- oriented" tools only allows a
small subset of tests to be performed.  The tests take around 30 minutes
to run.

To use them, you need a clone of git://github.com/autotest/virt-test.git;
then you can point configure to that clone, and just run "make
check-autotest".  For now, the tests are just run on x86_64, but this
is just because no one bothered to add guest OS descriptions to Autotest
for other devices.  I plan to look into Highbank and/or Versatile Express
and/or pSeries, but help from maintainers is welcome. :)

After adding adding a few more architectures and tests, the full testsuite
shall take no more than 3-4 hours to run (1 hour per target).  The plan
is that it will run (at least for non-x86 targets) on both KVM and TCG.

Signed-off-by: Paolo Bonzini <address@hidden>
---
 configure      |  4 ++++
 tests/Makefile | 69 +++++++++++++++++++++++++++++++++++++++++++++++++++++-----
 2 files changed, 67 insertions(+), 6 deletions(-)

diff --git a/configure b/configure
index 8789324..00c5960 100755
--- a/configure
+++ b/configure
@@ -626,6 +626,8 @@ for opt do
   ;;
   --with-trace-file=*) trace_file="$optarg"
   ;;
+  --with-virt-tests=*) autotest_dir="$optarg"
+  ;;
   --enable-gprof) gprof="yes"
   ;;
   --enable-gcov) gcov="yes"
@@ -3330,6 +3332,7 @@ echo "libcap-ng support $cap_ng"
 echo "vhost-net support $vhost_net"
 echo "Trace backend     $trace_backend"
 echo "Trace output file $trace_file-<pid>"
+echo "virt-tests repo   ${autotest_dir:-no}"
 echo "spice support     $spice ($spice_protocol_version/$spice_server_version)"
 echo "rbd support       $rbd"
 echo "xfsctl support    $xfs"
@@ -3790,6 +3793,7 @@ echo "EXESUF=$EXESUF" >> $config_host_mak
 echo "LIBS_QGA+=$libs_qga" >> $config_host_mak
 echo "POD2MAN=$POD2MAN" >> $config_host_mak
 echo "TRANSLATE_OPT_CFLAGS=$TRANSLATE_OPT_CFLAGS" >> $config_host_mak
+echo "AUTOTEST_DIR=$autotest_dir" >> $config_host_mak
 if test "$gcov" = "yes" ; then
   echo "CONFIG_GCOV=y" >> $config_host_mak
   echo "GCOV=$gcov_tool" >> $config_host_mak
diff --git a/tests/Makefile b/tests/Makefile
index a2d62b8..c1f4560 100644
--- a/tests/Makefile
+++ b/tests/Makefile
@@ -140,16 +140,73 @@ qtest-obj-y = tests/libqtest.o libqemuutil.a libqemustub.a
 qtest-obj-y += tests/libi2c.o tests/libi2c-omap.o
 $(check-qtest-y): $(qtest-obj-y)
 
+# Autotest rules
+
+AUTOTEST_TARGETS = x86_64
+AUTOTEST_TARGETS := $(filter $(AUTOTEST_TARGETS), $(TARGETS))
+
+# Other interesting tests that are currently broken:
+#      block_mirror
+#      cdrom_test
+#      chardev_hotplug
+#      vmstop
+#      block_hotplug
+#      ethtool (needs bridge, slirp is not enough; pretty long, too)
+#      guest_s4
+#      linux_s3
+#      mac_change
+#      nic_bonding
+#      ping.multi_nics
+#      nic_promisc
+#      vlan
+#
+# Some of them due to using slirp or for JeOS-specific failures.
+
+AUTOTEST_TESTS = \
+       block_stream \
+       floppy_test \
+       getfd \
+       migrate.default \
+       nic_hotplug.default \
+       numa_opts \
+       qemu_img \
+       qmp_basic \
+       seabios \
+       system_powerdown \
+       system_reset \
+       usb..usb_tablet \
+       usb..usb_storage \
+       
virtio_console.spread_linear.specifiable.virtserialport.virtio_console_smoke \
+       boot \
+       reboot \
+       shutdown
+
+ifneq ($(AUTOTEST_DIR),)
+check-autotest: $(patsubst %, check-autotest-%,$(AUTOTEST_TARGETS))
+$(patsubst %, check-autotest-%,$(AUTOTEST_TARGETS)): check-autotest-%:
+       mkdir -p $(BUILD_DIR)/log-autotest-$* && \
+         cd $(AUTOTEST_DIR) && \
+         git clean -Xf qemu/cfg && \
+         ./run -tqemu --arch=$* -m 256 --smp=1 \
+           --logs-dir=$(BUILD_DIR)/log-autotest-$* \
+           --qemu-bin=$(BUILD_DIR)/$*-softmmu/qemu-system-$* \
+           --tests '$(AUTOTEST_TESTS)'
+endif
+
 .PHONY: check-help
 check-help:
        @echo "Regression testing targets:"
        @echo
-       @echo " make check                Run all tests"
-       @echo " make check-qtest-TARGET   Run qtest tests for given target"
-       @echo " make check-qtest          Run qtest tests"
-       @echo " make check-unit           Run qobject tests"
-       @echo " make check-block          Run block tests"
-       @echo " make check-report.html    Generates an HTML test report"
+       @echo " make check                 Run all tests"
+       @echo " make check-qtest-TARGET    Run qtest tests for given target"
+       @echo " make check-qtest           Run qtest tests"
+       @echo " make check-unit            Run qobject tests"
+       @echo " make check-block           Run block tests"
+ifneq ($(AUTOTEST_DIR),)
+       @echo " make check-autotest-TARGET Run virt-test suite for given target"
+       @echo " make check-autotest        Run virt-test suite"
+endif
+       @echo " make check-report.html     Generates an HTML test report"
        @echo
        @echo "Please note that HTML reports do not regenerate if the unit 
tests"
        @echo "has not changed."
-- 
1.8.0.2