[PATCH v3 07/10] iotests: add testenv.py

[Vladimir Sementsov-Ogievskiy]

Add TestEnv class, which will handle test environment in a new python
iotests running framework.

Difference with current ./check interface:
- -v (verbose) option dropped, as it is unused

- -xdiff option is dropped, until somebody complains that it is needed
- same for -n option

Signed-off-by: Vladimir Sementsov-Ogievskiy <address@hidden>
---

RFC question here:

What about moving to classic double-dash long options (and short
one-dash-one-letter options)?

So, to convert

-qcow2  --> -f qcow2
-misalign  -->  --misalign
etc.

This may be done later, or I can do it in next version, if all agree
that it's good idea.

 tests/qemu-iotests/testenv.py | 332 ++++++++++++++++++++++++++++++++++
 1 file changed, 332 insertions(+)
 create mode 100755 tests/qemu-iotests/testenv.py

diff --git a/tests/qemu-iotests/testenv.py b/tests/qemu-iotests/testenv.py
new file mode 100755
index 0000000000..d1c2d41974
--- /dev/null
+++ b/tests/qemu-iotests/testenv.py
@@ -0,0 +1,332 @@
+#!/usr/bin/env python3
+#
+# Parse command line options to manage test environment variables.
+#
+# Copyright (c) 2020 Virtuozzo International GmbH
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+import os
+import sys
+import tempfile
+from pathlib import Path
+import shutil
+import collections
+import subprocess
+import argparse
+
+
+def get_default_machine(qemu_prog):
+    outp = subprocess.run([qemu_prog, '-machine', 'help'], check=True,
+                          text=True, stdout=subprocess.PIPE).stdout
+
+    machines = outp.split('\n')
+    default_machine = next(m for m in machines if m.endswith(' (default)'))
+    default_machine = default_machine.split(' ', 1)[0]
+
+    alias_suf = ' (alias of {})'.format(default_machine)
+    alias = next((m for m in machines if m.endswith(alias_suf)), None)
+    if alias is not None:
+        default_machine = alias.split(' ', 1)[0]
+
+    return default_machine
+
+
+""" EnvVarDescriptor
+name - variable name in upper case.
+inherit - variable defaults to same variable from os.environ.
+default - default value for the variable. None means absence by default.
+help - description. Unused now, just in-place documentation. May take place in
+       help message at some point.
+"""
+EnvVarDescriptor = collections.namedtuple('EnvVarDescriptor',
+                                          ('name', 'inherit', 'default',
+                                           'help'))
+
+
+class TestEnv:
+    """
+    Manage system environment for running tests
+
+    The following variables are supported/provided. They are represented by
+    lower-cased TestEnv attributes.
+    """
+    env_descriptors = [EnvVarDescriptor(*d) for d in (
+        # name, inherit, default, description
+
+        # Directories
+        ('TEST_DIR', True, os.path.join(os.getcwd(), 'scratch'),
+         'directory for all intermediate test files'),
+        ('SOCK_DIR', True, None, 'directory for unix sockets'),
+        ('SAMPLE_IMG_DIR', True, None, 'TODO: ???'),
+        ('OUTPUT_DIR', False, os.getcwd(), 'TODO: ???'),
+
+        # Binaries
+        ('PYTHON', False, '/usr/bin/python3 -B', 'python3 for bash tests'),
+        ('QEMU_PROG', True, None, 'qemu binary'),
+        ('QEMU_IMG_PROG', True, None, 'qemu-img binary'),
+        ('QEMU_IO_PROG', True, None, 'qemu-io binary'),
+        ('QEMU_NBD_PROG', True, None, 'qemu-nbd binary'),
+        ('SOCKET_SCM_HELPER', False, None, 'socket_scm_helper binary'),
+
+        # Options for binaries
+        # RFC: Interesting, that only IMG and NBD options may be passed by
+        # user, QEMU and IO options are only calculated. Looks inconsistent.
+        ('QEMU_OPTIONS', False, None, 'qemu command line arguments'),
+        ('QEMU_IMG_OPTIONS', True, None, 'qemu command line arguments'),
+        ('QEMU_IO_OPTIONS', False, None, 'qemu command line arguments'),
+        ('QEMU_NBD_OPTIONS', True, None, 'qemu command line arguments'),
+
+        ('IMGOPTS', False, None, 'options to pass to qemu-img create/convert'),
+
+        ('IMGFMT', False, 'raw', 'image format, set by cmdline'),
+        ('IMGPROTO', False, 'file', 'image protocol, set by cmdline'),
+        ('AIOMODE', False, 'threads', 'image protocol, set by cmdline'),
+        ('CACHEMODE', False, 'writeback', 'cache mode, set by cmdline'),
+
+        ('VALGRIND_QEMU', False, None, 'use valgrind, set by cmdline'),
+
+        # Helping variables, not passed by user, only calculated
+        ('CACHEMODE_IS_DEFAULT', False, None,
+         'cache mode was not set by user'),
+        ('IMGFMT_GENERIC', False, None, 'TODO: ???'),
+        ('IMGOPTSSYNTAX', False, None, 'TODO: ???'),
+        ('IMGKEYSECRET', False, None, 'TODO: ???'),
+        ('QEMU_DEFAULT_MACHINE', False, None, 'TODO: ???'),
+    )]
+
+    @staticmethod
+    def create_argparser():
+        p = argparse.ArgumentParser(description="Test environment preparation",
+                                    add_help=False, usage=argparse.SUPPRESS)
+
+        p.add_argument('-d', dest='debug', action='store_true', help='debug')
+        p.add_argument('-misalign', action='store_true',
+                       help='misalign memory allocations')
+
+        format_list = ['raw', 'bochs', 'parallels', 'qcow', 'qcow2', 'qed',
+                       'vdi', 'vpc', 'vhdx', 'vmdk', 'luks', 'dmg']
+        g = p.add_argument_group(
+            'image format options',
+            'The following options sets IMGFMT environment variable. '
+            'At most one chose is allowed, default is "raw"')
+        g = g.add_mutually_exclusive_group()
+        for fmt in format_list:
+            g.add_argument('-' + fmt, dest='imgfmt', action='store_const',
+                           const=fmt)
+
+        protocol_list = ['file', 'rbd', 'sheepdoc', 'nbd', 'ssh', 'nfs',
+                         'vxhs']
+        g = p.add_argument_group(
+            'image protocol options',
+            'The following options sets IMGPROTO environment variably. '
+            'At most one chose is allowed, default is "file"')
+        g = g.add_mutually_exclusive_group()
+        for prt in protocol_list:
+            g.add_argument('-' + prt, dest='imgproto', action='store_const',
+                           const=prt)
+
+        g = p.add_mutually_exclusive_group()
+        g.add_argument('-nocache', dest='cachemode', action='store_const',
+                       const='none', help='set cache mode "none" (O_DIRECT), '
+                       'sets CACHEMODE environment variable')
+        g.add_argument('-c', dest='cachemode',
+                       help='sets CACHEMODE environment variable')
+
+        p.add_argument('-i', dest='aiomode', default='threads',
+                       help='sets AIOMODE environment variable')
+
+        g = p.add_argument_group('bash tests options',
+                                 'The following options are ignored by '
+                                 'python tests. TODO: support them in '
+                                 'iotests.py')
+        g.add_argument('-o', dest='imgopts',
+                       help='options to pass to qemu-img create/convert, sets '
+                       'IMGOPTS environment variable')
+        p.add_argument('-valgrind', dest='VALGRIND_QEMU', action='store_const',
+                       const='y', help='use valgrind, sets VALGRIND_QEMU '
+                       'environment variable')
+
+        return p
+
+    argparser = create_argparser.__func__()
+
+    def __contains__(self, item):
+        return hasattr(self, item)
+
+    def __setitem__(self, key, value):
+        setattr(self, key, value)
+
+    def __getitem__(self, key):
+        return getattr(self, key)
+
+    def setdefault(self, attr, value):
+        if attr not in self:
+            self[attr] = value
+
+    def init_paths(self):
+        self.build_iotests = os.getcwd()
+
+        if os.path.islink(sys.argv[0]):
+            # called from the build tree
+            self.source_iotests = os.path.dirname(os.readlink(sys.argv[0]))
+        else:
+            self.source_iotests = self.build_iotests
+
+        self.build_root = os.path.join(self.build_iotests, '..', '..')
+
+    def init_handle_argv(self, argv):
+        self.args, self.remaining_argv = self.argparser.parse_known_args(argv)
+
+        for k, v in vars(self.args).items():
+            if v is not None:
+                self[k] = v
+
+    def init_handle_env_descriptors(self):
+        for d in self.env_descriptors:
+            if d.name.lower() in self:
+                continue  # set by command line argument
+
+            if d.inherit and d.name in os.environ:
+                self[d.name.lower()] = os.environ[d.name]
+            elif d.default is not None:
+                self[d.name.lower()] = d.default
+
+    def init_find_binaries(self):
+        self.setdefault('qemu_img_prog',
+                        os.path.join(self.build_root, 'qemu-img'))
+        self.setdefault('qemu_io_prog',
+                        os.path.join(self.build_root, 'qemu-io'))
+        self.setdefault('qemu_nbd_prog',
+                        os.path.join(self.build_root, 'qemu-nbd'))
+
+        if 'qemu_prog' not in self:
+            arch = os.uname().machine
+            if 'ppc64' in arch:
+                arch = 'ppc64'
+            self.qemu_prog = os.path.join(self.build_root, arch + '-softmmu',
+                                          'qemu-system-' + arch)
+
+        for b in [self.qemu_img_prog, self.qemu_io_prog, self.qemu_nbd_prog,
+                  self.qemu_prog]:
+            if not os.path.exists(b):
+                exit('Not such file: ' + b)
+            if not os.access(b, os.X_OK):
+                exit('Not executable: ' + b)
+
+        helper_path = os.path.join(self.build_iotests, 'socket_scm_helper')
+        if os.access(helper_path, os.X_OK):
+            self.socket_scm_helper = helper_path
+
+    def __init__(self, argv):
+        """ Parse args and environment """
+
+        self.init_paths()
+        self.init_handle_argv(argv)
+
+        self.cachemode_is_default = 'false' if 'cachemode' in self else 'true'
+
+        self.init_handle_env_descriptors()
+        self.init_find_binaries()
+
+        self.imgfmt_generic = self.imgfmt not in ['bochs', 'cloop', 'dmg']
+        self.imgfmt_generic = 'true' if self.imgfmt_generic else 'false'
+
+        self.qemu_io_options = f'--cache {self.cachemode} --aio {self.aiomode}'
+        if self.misalign:
+            self.qemu_io_options += ' --misalign'
+
+        self.qemu_io_options_no_fmt = self.qemu_io_options
+
+        if self.imgfmt == 'luks':
+            self.imgoptssyntax = 'true'
+            self.imgkeysecret = '123456'
+            if 'imgopts' not in self:
+                self.imgopts = 'iter-time=10'
+            elif 'iter-time=' not in self.imgopts:
+                self.imgopts += ',iter-time=10'
+        else:
+            self.imgoptssyntax = 'false'
+            self.qemu_io_options += ' -f ' + self.imgfmt
+
+        self.qemu_options = '-nodefaults -display none -accel qtest'
+        if self.qemu_prog.endswith(('qemu-system-arm', 'qemu-system-aarch64')):
+            self.qemu_options += ' -machine virt'
+        elif self.qemu_prog.endswith('qemu-system-tricore'):
+            self.qemu_options += ' -machine tricore_testboard'
+
+        self.setdefault('sample_img_dir',
+                        os.path.join(self.source_iotests, 'sample_images'))
+
+        self.qemu_default_machine = get_default_machine(self.qemu_prog)
+
+        self.tmp_sock_dir = False
+        if 'sock_dir' not in self:
+            self.sock_dir = tempfile.mkdtemp()
+            self.tmp_sock_dir = True
+
+        for d in (self.test_dir, self.sock_dir):
+            Path(d).mkdir(parents=True, exist_ok=True)
+
+    def close(self):
+        if self.tmp_sock_dir:
+            shutil.rmtree(self.sock_dir)
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, *args):
+        self.close()
+
+    def get_env(self):
+        env = {}
+        for d in self.env_descriptors:
+            if d.name.lower() in self:
+                env[d.name] = self[d.name.lower()]
+
+        return env
+
+    def print_env(self):
+        template = """\
+QEMU          -- "{QEMU_PROG}" {QEMU_OPTIONS}
+QEMU_IMG      -- "{QEMU_IMG_PROG}" {QEMU_IMG_OPTIONS}
+QEMU_IO       -- "{QEMU_IO_PROG}" {QEMU_IO_OPTIONS}
+QEMU_NBD      -- "{QEMU_NBD_PROG}" {QEMU_NBD_OPTIONS}
+IMGFMT        -- {IMGFMT}{imgopts}
+IMGPROTO      -- {IMGPROTO}
+PLATFORM      -- {platform}
+TEST_DIR      -- {TEST_DIR}
+SOCK_DIR      -- {SOCK_DIR}
+SOCKET_SCM_HELPER -- {SOCKET_SCM_HELPER}"""
+
+        args = collections.defaultdict(str, self.get_env())
+
+        if 'IMGOPTS' in args:
+            args['imgopts'] = ' (' + args['IMGOPTS'] + ')'
+
+        u = os.uname()
+        args['platform'] = f'{u.sysname}/{u.machine} {u.nodename} {u.release}'
+
+        print(template.format_map(args))
+
+
+if __name__ == '__main__':
+    if len(sys.argv) == 2 and sys.argv[1] in ['-h', '--help']:
+        TestEnv.argparser.print_help()
+        exit()
+
+    with TestEnv(sys.argv) as te:
+        te.print_env()
+        print('\nUnhandled options: ', te.remaining_argv)
-- 
2.21.0