[PATCH 17/27] configure: use 'mkvenv ensure meson' to bootstrap meson

[John Snow]

This commit changes how we detect and install meson. It notably removes
'--meson='.

Currently, configure creates a lightweight Python virtual environment
unconditionally using the user's configured $python that inherits system
packages. Temporarily, we forced the use of meson source present via git
submodule or in the release tarball.

With this patch, we restore the ability to use a system-provided meson:

If Meson is installed in the build venv and meets our minimum version
requirements, we will use that Meson. This includes a system provided
meson, which would be visible via system-site packages inside the venv.

In the event that Meson is installed but *not for the chosen Python
interpreter*, not found, or of insufficient version, we will attempt to
install Meson from vendored source into the newly created Python virtual
environment. This vendored installation replaces both the git submodule
and tarball source mechanisms for sourcing meson.

As a result of this patch, the Python interpreter we use for both our
own build scripts *and* Meson extensions are always known to be the
exact same Python. As a further benefit, there will also be a symlink
available in the build directory that points to the correct, configured
python and can be used by e.g. manual tests to invoke the correct,
configured Python unambiguously.

Signed-off-by: John Snow <jsnow@redhat.com>
---
 configure                           | 80 ++++++++++-------------------
 .gitlab-ci.d/buildtest-template.yml |  4 +-
 python/scripts/mkvenv.py            |  4 ++
 3 files changed, 32 insertions(+), 56 deletions(-)

diff --git a/configure b/configure
index 1d7db92ee3..6e4499a68a 100755
--- a/configure
+++ b/configure
@@ -731,8 +731,6 @@ for opt do
   ;;
   --skip-meson) skip_meson=yes
   ;;
-  --meson=*) meson="$optarg"
-  ;;
   --ninja=*) ninja="$optarg"
   ;;
   --smbd=*) smbd="$optarg"
@@ -1017,7 +1015,6 @@ Advanced options (experts only):
   --cross-prefix-ARCH=PREFIX cross compiler prefix when building ARCH guest 
test cases
   --make=MAKE              use specified make [$make]
   --python=PYTHON          use specified python [$python]
-  --meson=MESON            use specified meson [$meson]
   --ninja=NINJA            use specified ninja [$ninja]
   --smbd=SMBD              use specified smbd [$smbd]
   --with-git=GIT           use specified git [$git]
@@ -1090,7 +1087,6 @@ fi
 
 # Resolve PATH
 python="$(command -v "$python")"
-explicit_python=yes
 
 # Create a Python virtual environment using our configured python.
 # The stdout of this script will be the location of a symlink that
@@ -1102,7 +1098,6 @@ explicit_python=yes
 # - venv is cleared if it exists already;
 # - venv is allowed to use system packages;
 # - all setup is performed **offline**;
-# - No packages are installed by default;
 # - pip is not installed into the venv when possible,
 #   but ensurepip is called as a fallback when necessary.
 
@@ -1117,59 +1112,36 @@ fi
 # Suppress writing compiled files
 python="$python -B"
 
-has_meson() {
-  local python_dir=$(dirname "$python")
-  # PEP405: pyvenv.cfg is either adjacent to the Python executable
-  # or one directory above
-  if test -f $python_dir/pyvenv.cfg || test -f $python_dir/../pyvenv.cfg; then
-    # Ensure that Meson and Python come from the same virtual environment
-    test -x "$python_dir/meson" &&
-      test "$(command -v meson)" -ef "$python_dir/meson"
-  else
-    has meson
-  fi
-}
 
-if test -z "$meson"; then
-    if test "$explicit_python" = no && has_meson && version_ge "$(meson 
--version)" 0.61.5; then
-        meson=meson
-    elif test "$git_submodules_action" != 'ignore' ; then
-        meson=git
-    elif test -e "${source_path}/meson/meson.py" ; then
-        meson=internal
-    else
-        if test "$explicit_python" = yes; then
-            error_exit "--python requires using QEMU's embedded Meson 
distribution, but it was not found."
-        else
-            error_exit "Meson not found.  Use --meson=/path/to/meson"
-        fi
-    fi
+if ! $python "${source_path}/python/scripts/mkvenv.py" ensure \
+     --dir "${source_path}/python/wheels" \
+     --diagnose "meson" \
+     "meson>=0.61.5" ;
+then
+    exit 1
+fi
+
+# At this point, we expect Meson to be installed and available.
+# We expect mkvenv or pip to have created pyvenv/bin/meson for us.
+# We ignore PATH completely here: we want to use the venv's Meson
+# *exclusively*.
+
+# "mkvenv ensure" has a limitation compared to "pip install": it is not
+# able to create launcher .exe files on Windows.  This limitation exists
+# because "py.exe" is not guaranteed to exist on the machine (pip/setuptools
+# work around the issue by bundling the .exe files as resources).
+# This is not a problem for msys, since it emulates a POSIX environment;
+# it is also okay for programs that meson.build looks up with find_program(),
+# because in that case Meson checks the file for a shebang line.  However,
+# when meson wants to invoke itself as part of a build recipe, we need
+# to convince it to put the python interpreter in front of the path to the
+# script.  To do so, run it using '-m'.
+if test "$targetos" = windows; then
+  meson="$python -m mesonbuild.mesonmain"
 else
-    # Meson uses its own Python interpreter to invoke other Python scripts,
-    # but the user wants to use the one they specified with --python.
-    #
-    # We do not want to override the distro Python interpreter (and sometimes
-    # cannot: for example in Homebrew /usr/bin/meson is a bash script), so
-    # just require --meson=git|internal together with --python.
-    if test "$explicit_python" = yes; then
-        case "$meson" in
-            git | internal) ;;
-            *) error_exit "--python requires using QEMU's embedded Meson 
distribution." ;;
-        esac
-    fi
+  meson="$(cd pyvenv/bin; pwd)/meson"
 fi
 
-if test "$meson" = git; then
-    git_submodules="${git_submodules} meson"
-fi
-
-case "$meson" in
-    git | internal)
-        meson="$python ${source_path}/meson/meson.py"
-        ;;
-    *) meson=$(command -v "$meson") ;;
-esac
-
 # Probe for ninja
 
 if test -z "$ninja"; then
diff --git a/.gitlab-ci.d/buildtest-template.yml 
b/.gitlab-ci.d/buildtest-template.yml
index a6cfe9be97..7edb50b760 100644
--- a/.gitlab-ci.d/buildtest-template.yml
+++ b/.gitlab-ci.d/buildtest-template.yml
@@ -12,12 +12,12 @@
     - mkdir build
     - cd build
     - ../configure --enable-werror --disable-docs --enable-fdt=system
-          ${LD_JOBS:+--meson=git} ${TARGETS:+--target-list="$TARGETS"}
+          ${TARGETS:+--target-list="$TARGETS"}
           $CONFIGURE_ARGS ||
       { cat config.log meson-logs/meson-log.txt && exit 1; }
     - if test -n "$LD_JOBS";
       then
-        ../meson/meson.py configure . -Dbackend_max_links="$LD_JOBS" ;
+        pyvenv/bin/meson configure . -Dbackend_max_links="$LD_JOBS" ;
       fi || exit 1;
     - make -j"$JOBS"
     - if test -n "$MAKE_CHECK_ARGS";
diff --git a/python/scripts/mkvenv.py b/python/scripts/mkvenv.py
index 1e072b3df3..31193b3b22 100644
--- a/python/scripts/mkvenv.py
+++ b/python/scripts/mkvenv.py
@@ -530,6 +530,10 @@ def _get_entry_points() -> Iterator[Dict[str, str]]:
         # launcher copied to foo.exe.  Unfortunately there is no guarantee that
         # py.exe exists on the machine.  Creating the script like this is
         # enough for msys and meson, both of which understand shebang lines.
+        # It does requires some care when invoking meson however, which is
+        # worked around in configure.  Note that a .exe launcher is needed
+        # and not for example a batch file, because the CreateProcess API
+        # (used by Ninja) cannot start them.
         with open(script_path, "w", encoding="UTF-8") as file:
             file.write(script)
         mode = os.stat(script_path).st_mode | stat.S_IEXEC
-- 
2.40.0