[Emacs-diffs] emacs-26 3fc05cf: Scripts to automate windows binary distribution

[Phillip Lord]

branch: emacs-26
commit 3fc05cfaecf03a249bb9df1a9566d16ec005c7c3
Author: Phillip Lord <address@hidden>
Commit: Phillip Lord <address@hidden>

    Scripts to automate windows binary distribution
    
    * admin/nt/dist-build/README-windows-binaries,
      admin/nt/dist-build/build-dep-zips.py,
      admin/nt/dist-build/build-zips.sh: New Files
---
 admin/nt/dist-build/README-windows-binaries |  45 ++++++
 admin/nt/dist-build/build-dep-zips.py       | 223 ++++++++++++++++++++++++++++
 admin/nt/dist-build/build-zips.sh           | 143 ++++++++++++++++++
 3 files changed, 411 insertions(+)

diff --git a/admin/nt/dist-build/README-windows-binaries 
b/admin/nt/dist-build/README-windows-binaries
new file mode 100644
index 0000000..27a5483
--- /dev/null
+++ b/admin/nt/dist-build/README-windows-binaries
@@ -0,0 +1,45 @@
+Windows Binaries
+================
+
+Currently, we provide four different binary packages for Emacs, which
+are:
+
+emacs-$VERSION-x86_64.zip
+
+Contains a 64-bit build of Emacs with dependencies. Mostly, this is
+the best one to install.
+
+emacs-$VERSION-x86_64-no-deps.zip
+
+Contains a 64-bit build of Emacs without any dependencies. This may be
+useful if you wish to install where the dependencies are already
+available, or if you want the small possible Emacs.
+
+emacs-$VERSION-i686.zip
+
+Contains a 32-bit build of Emacs with dependencies. This is useful for
+running on a 32-bit machine.
+
+emacs-$VERSION-i686-no-deps.zip
+
+Contains a 32-bit build of Emacs without dependencies
+
+In addition, we provide the following files which will not be useful
+for most end-users.
+
+emacs-26-x86_64-deps.zip
+
+The dependencies. Unzipping this file on top of
+emacs-$VERSION-x86_64-no-deps.zip should result in the same install as
+emacs-$VERSION-x86_64.zip.
+
+emacs-26-i686-deps.zip
+
+The 32-bit version of the dependencies.
+
+emacs-26-deps-mingw-w64-src.zip
+
+The source for the dependencies. Source for Emacs itself is available
+in the main distribution tarball. These dependencies were produced
+from an updated msys2 at the point of the first pre-test. It is not
+intended that these will be updated after that point.
\ No newline at end of file
diff --git a/admin/nt/dist-build/build-dep-zips.py 
b/admin/nt/dist-build/build-dep-zips.py
new file mode 100755
index 0000000..33bc4b5
--- /dev/null
+++ b/admin/nt/dist-build/build-dep-zips.py
@@ -0,0 +1,223 @@
+#!/usr/bin/python3
+
+## Copyright (C) 2017 Free Software Foundation, Inc.
+
+## This file is part of GNU Emacs.
+
+## GNU Emacs is free software: you can redistribute it and/or modify
+## it under the terms of the GNU General Public License as published by
+## the Free Software Foundation, either version 3 of the License, or
+## (at your option) any later version.
+
+## GNU Emacs is distributed in the hope that it will be useful,
+## but WITHOUT ANY WARRANTY; without even the implied warranty of
+## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+## GNU General Public License for more details.
+
+## You should have received a copy of the GNU General Public License
+## along with GNU Emacs.  If not, see <https://www.gnu.org/licenses/>.
+import argparse
+import multiprocessing as mp
+import glob
+import os
+import shutil
+import re
+
+from subprocess import check_output
+
+## Constants
+EMACS_MAJOR_VERSION="26"
+
+
+## Options
+DRY_RUN=False
+
+## Packages to fiddle with
+SKIP_PKGS=["mingw-w64-gcc-libs"]
+MUNGE_PKGS ={"mingw-w64-libwinpthread-git":"mingw-w64-winpthreads-git"}
+ARCH_PKGS=["mingw-w64-mpc",
+           "mingw-w64-termcap",
+           "mingw-w64-xpm-nox"]
+SRC_REPO="https://sourceforge.net/projects/msys2/files/REPOS/MINGW/Sources";
+
+
+def check_output_maybe(*args,**kwargs):
+    if(DRY_RUN):
+        print("Calling: {}{}".format(args,kwargs))
+    else:
+        return check_output(*args,**kwargs)
+
+def extract_deps():
+
+    # This list derives from the features we want Emacs to compile with.
+    PKG_REQ='''mingw-w64-x86_64-giflib
+mingw-w64-x86_64-gnutls
+mingw-w64-x86_64-libjpeg-turbo
+mingw-w64-x86_64-libpng
+mingw-w64-x86_64-librsvg
+mingw-w64-x86_64-libtiff
+mingw-w64-x86_64-libxml2
+mingw-w64-x86_64-xpm-nox
+mingw-w64-x86_64-lcms2'''.split()
+
+    # Get a list of all dependencies needed for packages mentioned above.
+    # Run `pactree -lu' for each elment of $PKG_REQ
+    pkgs = set()
+    for x in PKG_REQ:
+        pkgs.update(
+            check_output(["pactree", "-lu", x]).decode("utf-8").split()
+        )
+
+    return sorted(pkgs)
+
+def gather_deps(deps, arch, directory):
+
+    os.mkdir(arch)
+    os.chdir(arch)
+
+    ## Replace the architecture with the correct one
+    deps = [re.sub(r"x86_64",arch,x) for x in deps]
+
+    ## find all files the transitive dependencies
+    deps_files = check_output(
+        ["pacman", "-Ql"] + deps
+    ).decode("utf-8").split("\n")
+
+    ## Produces output like
+    ## mingw-w64-x86_64-zlib /mingw64/lib/libminizip.a
+
+    ## drop the package name
+    tmp = deps_files.copy()
+    deps_files=[]
+    for d in tmp:
+        slt = d.split()
+        if(not slt==[]):
+            deps_files.append(slt[1])
+
+    ## sort uniq
+    deps_files = sorted(list(set(deps_files)))
+    ## copy all files into local
+    print("Copying dependencies: {}".format(arch))
+    check_output_maybe(["rsync", "-R"] + deps_files + ["."])
+
+    ## And package them up
+    os.chdir(directory)
+    print("Zipping: {}".format(arch))
+    check_output_maybe("zip -9r ../../emacs-26-{}-deps.zip *".format(arch),
+                       shell=True)
+    os.chdir("../../")
+
+
+def download_source(tarball):
+    print("Downloading {}...".format(tarball))
+    check_output_maybe(
+        "wget -a ../download.log -O {} {}/{}/download"
+        .format(tarball, SRC_REPO, tarball),
+        shell=True
+    )
+    print("Downloading {}... done".format(tarball))
+
+def gather_source(deps):
+
+
+    ## Source for gcc-libs is part of gcc
+    ## Source for libwinpthread is in libwinpthreads
+    ## mpc, termcap, xpm -- has x86_64, and i686 versions
+
+    ## This needs to have been run first at the same time as the
+    ## system was udpated.
+    os.mkdir("emacs-src")
+    os.chdir("emacs-src")
+
+    to_download = []
+    for pkg in deps:
+        pkg_name_and_version= \
+            check_output(["pacman","-Q", pkg]).decode("utf-8").strip()
+
+        ## Produces output like:
+        ## mingw-w64-x86_64-zlib 2.43.2
+        pkg_name_components = pkg_name_and_version.split()
+        pkg_name=pkg_name_components[0]
+        pkg_version=pkg_name_components[1]
+
+        ## make a simple name to make lookup easier
+        simple_pkg_name = re.sub(r"x86_64-","",pkg_name)
+
+        if(simple_pkg_name in SKIP_PKGS):
+            continue
+
+        ## Some packages have different source files for different
+        ## architectures. For these we need two downloads.
+        if(simple_pkg_name in ARCH_PKGS):
+            downloads = [pkg_name,
+                         re.sub(r"x86_64","i686",pkg_name)]
+        else:
+            downloads = [simple_pkg_name]
+
+        for d in downloads:
+            ## Switch names if necessary
+            d = MUNGE_PKGS.get(d,d)
+
+            tarball = "{}-{}.src.tar.gz".format(d,pkg_version)
+
+            to_download.append(tarball)
+
+    ## Download in parallel or it is just too slow
+    p = mp.Pool(16)
+    p.map(download_source,to_download)
+
+    print("Zipping")
+    check_output_maybe("zip -9 ../emacs-{}-deps-mingw-w64-src.zip *"
+                       .format(EMACS_MAJOR_VERSION),
+                       shell=True)
+
+    os.chdir("..")
+
+
+def clean():
+    print("Cleaning")
+    os.path.isdir("emacs-src") and shutil.rmtree("emacs-src")
+    os.path.isdir("i686") and shutil.rmtree("i686")
+    os.path.isdir("x86_64") and shutil.rmtree("x86_64")
+    os.path.isfile("download.log") and os.remove("download.log")
+
+
+if(os.environ["MSYSTEM"] != "MSYS"):
+    print("Run this script in an MSYS-shell!")
+    exit(1)
+
+
+parser = argparse.ArgumentParser()
+parser.add_argument("-t", help="32 bit deps only",
+                    action="store_true")
+
+parser.add_argument("-f", help="64 bit deps only",
+                    action="store_true")
+
+parser.add_argument("-s", help="source code only",
+                    action="store_true")
+
+parser.add_argument("-c", help="clean only",
+                    action="store_true")
+
+parser.add_argument("-d", help="dry run",
+                    action="store_true")
+
+args = parser.parse_args()
+do_all=not (args.c or args.s or args.f or args.t)
+
+deps=extract_deps()
+
+DRY_RUN=args.d
+
+if( do_all or args.t ):
+    gather_deps(deps,"i686","mingw32")
+
+if( do_all or args.f ):
+    gather_deps(deps,"x86_64","mingw64")
+
+if( do_all or args.s ):
+    gather_source(deps)
+
+if( args.c ):
+    clean()
diff --git a/admin/nt/dist-build/build-zips.sh 
b/admin/nt/dist-build/build-zips.sh
new file mode 100755
index 0000000..e78f72c
--- /dev/null
+++ b/admin/nt/dist-build/build-zips.sh
@@ -0,0 +1,143 @@
+#!/bin/bash
+
+## Copyright (C) 2017 Free Software Foundation, Inc.
+
+## This file is part of GNU Emacs.
+
+## GNU Emacs is free software: you can redistribute it and/or modify
+## it under the terms of the GNU General Public License as published by
+## the Free Software Foundation, either version 3 of the License, or
+## (at your option) any later version.
+
+## GNU Emacs is distributed in the hope that it will be useful,
+## but WITHOUT ANY WARRANTY; without even the implied warranty of
+## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+## GNU General Public License for more details.
+
+## You should have received a copy of the GNU General Public License
+## along with GNU Emacs.  If not, see <https://www.gnu.org/licenses/>.
+
+
+function git_up {
+    echo Making git worktree for Emacs $VERSION
+    cd $HOME/emacs-build/git/emacs-$MAJOR_VERSION
+    git pull
+    git worktree add ../emacs-$BRANCH emacs-$BRANCH
+
+    cd ../emacs-$BRANCH
+    ./autogen.sh
+
+}
+
+function build_zip {
+
+    ARCH=$1
+    PKG=$2
+    HOST=$3
+
+    echo Building Emacs-$VERSION for $ARCH
+    if [ $ARCH == "i686" ]
+    then
+        PATH=/mingw32/bin:$PATH
+        MSYSTEM=MINGW32
+    fi
+
+    mkdir --parents $HOME/emacs-build/build/emacs-$VERSION/$ARCH
+    cd $HOME/emacs-build/build/emacs-$VERSION/$ARCH
+
+    export PKG_CONFIG_PATH=$PKG
+    ../../../git/emacs-$BRANCH/configure \
+        --without-dbus \
+        --host=$HOST --without-compress-install \
+        CFLAGS="-O2 -static -g3"
+    make -j 8 install \
+         prefix=$HOME/emacs-build/install/emacs-$VERSION/$ARCH
+    cd $HOME/emacs-build/install/emacs-$VERSION/$ARCH
+    cp $HOME/emacs-build/deps/libXpm/$ARCH/libXpm-noX4.dll bin
+    zip -r -9 emacs-$VERSION-$ARCH-no-deps.zip *
+    mv emacs-$VERSION-$ARCH.zip $HOME/emacs-upload
+    rm bin/libXpm-noX4.dll
+    unzip $HOME/emacs-build/deps/emacs-26-$ARCH-deps.zip
+    zip -r -9 emacs-$VERSION-$ARCH.zip *
+    mv emacs-$VERSION-$ARCH.zip ~/emacs-upload
+}
+
+
+##set -o xtrace
+set -o errexit
+
+SNAPSHOT=
+
+BUILD_32=1
+BUILD_64=1
+GIT_UP=0
+
+while getopts "36ghsV:" opt; do
+  case $opt in
+    3)
+        BUILD_32=1
+        BUILD_64=0
+        GIT_UP=0
+        ;;
+    6)
+        BUILD_32=0
+        BUILD_64=1
+        GIT_UP=0
+        ;;
+
+    g)
+        BUILD_32=0
+        BUILD_64=0
+        GIT_UP=1
+        ;;
+    V)
+        VERSION=$OPTARG
+        ;;
+    s)
+        SNAPSHOT="-snapshot"
+        ;;
+    h)
+        echo "build-zips.sh"
+        echo "  -3 32 bit build only"
+        echo "  -6 64 bit build only"
+        echo "  -g git update and worktree only"
+        exit 0
+        ;;
+    \?)
+        echo "Invalid option: -$OPTARG" >&2
+        ;;
+  esac
+done
+
+if [ -z $VERSION ];
+then
+    echo "doing version thing"
+    VERSION=`
+  sed -n 's/^AC_INIT(GNU Emacs,[        ]*\([^  ,)]*\).*/\1/p' < 
../../../configure.ac
+`
+fi
+
+if [ -z $VERSION ];
+then
+    echo Cannot determine Emacs version
+    exit 1
+fi
+
+MAJOR_VERSION="$(echo $VERSION | cut -d'.' -f1)"
+BRANCH=$VERSION
+VERSION=$VERSION$SNAPSHOT
+
+if (($GIT_UP))
+then
+    git_up
+fi
+
+if (($BUILD_32))
+then
+    build_zip i686 /mingw32/lib/pkgconfig i686-w64-mingw32
+fi
+
+if (($BUILD_64))
+then
+    build_zip x86_64 /mingw64/lib/pkgconfig x86_64-w64-mingw32
+fi