feature/dll-only-windows 2bf9ee9: Update dependency capture

[Phillip Lord]

branch: feature/dll-only-windows
commit 2bf9ee9b997a688fdd6cfbcb0e60db465e76acda
Author: Phillip Lord <phillip.lord@russet.org.uk>
Commit: Phillip Lord <phillip.lord@russet.org.uk>

    Update dependency capture
    
    * admin/nt/dist-build/build-dep-zips.py: Use ntldd to directly
      determine DLL dependencies
---
 admin/nt/dist-build/build-dep-zips.py | 122 +++++++++++++++++++++-------------
 admin/nt/dist-build/build-zips.sh     |   6 +-
 2 files changed, 76 insertions(+), 52 deletions(-)

diff --git a/admin/nt/dist-build/build-dep-zips.py 
b/admin/nt/dist-build/build-dep-zips.py
index 47185db..ec99bd6 100755
--- a/admin/nt/dist-build/build-dep-zips.py
+++ b/admin/nt/dist-build/build-dep-zips.py
@@ -40,10 +40,77 @@ mingw-w64-x86_64-libtiff
 mingw-w64-x86_64-libxml2
 mingw-w64-x86_64-xpm-nox'''.split()
 
+DLL_REQ='''libgif
+libgnutls
+libharfbuzz
+libjansson
+liblcms2
+libturbojpeg
+libpng
+librsvg
+libtiff
+libxml
+libXpm'''.split()
+
 
 ## Options
 DRY_RUN=False
 
+
+def check_output_maybe(*args,**kwargs):
+    if(DRY_RUN):
+        print("Calling: {}{}".format(args,kwargs))
+    else:
+        return check_output(*args,**kwargs)
+
+## DLL Capture
+def gather_deps(arch, directory):
+    os.mkdir(arch)
+    os.chdir(arch)
+
+    for dep in full_dll_dependency(directory):
+        check_output_maybe(["cp /{}/bin/{}*.dll .".format(directory, dep)],
+                           shell=True)
+
+    ## And package them up
+    ## os.chdir(arch)
+    print("Zipping: {}".format(arch))
+    check_output_maybe("zip -9r ../emacs-{}-{}{}-deps.zip *"
+                       .format(EMACS_MAJOR_VERSION, DATE, arch),
+                       shell=True)
+    os.chdir("../")
+
+## Return all Emacs dependencies
+def full_dll_dependency(directory):
+    deps = [dll_dependency(dep, directory) for dep in DLL_REQ]
+    return set(sum(deps, []) + DLL_REQ)
+
+## Dependencies for a given DLL
+def dll_dependency(dll, directory):
+    output = check_output(["/mingw64/bin/ntldd", "--recursive",
+                           "/{}/bin/{}*.dll".format(directory, 
dll)]).decode("utf-8")
+    ## munge output
+    return ntldd_munge(output)
+
+def ntldd_munge(out):
+    deps = out.splitlines()
+    rtn = []
+    for dep in deps:
+        ## Output looks something like this
+
+        ## KERNEL32.dll => C:\Windows\SYSTEM32\KERNEL32.dll 
(0x0000000002a30000)
+        ## libwinpthread-1.dll => C:\msys64\mingw64\bin\libwinpthread-1.dll 
(0x0000000000090000)
+
+        ## if it's the former, we want it, if its the later we don't
+        splt = dep.split()
+        if len(splt) > 2 and "msys64" in splt[2]:
+            print("Adding dep", splt[0])
+            rtn.append(splt[0].split(".")[0])
+
+    return rtn
+
+#### Source Capture
+
 ## Packages to fiddle with
 ## Source for gcc-libs is part of gcc
 SKIP_SRC_PKGS=["mingw-w64-gcc-libs"]
@@ -62,12 +129,6 @@ ARCH_PKGS=[]
 SRC_REPO="https://sourceforge.net/projects/msys2/files/REPOS/MINGW/Sources";
 
 
-def check_output_maybe(*args,**kwargs):
-    if(DRY_RUN):
-        print("Calling: {}{}".format(args,kwargs))
-    else:
-        return check_output(*args,**kwargs)
-
 def immediate_deps(pkg):
     package_info = check_output(["pacman", "-Si", 
pkg]).decode("utf-8").split("\n")
 
@@ -87,6 +148,7 @@ def immediate_deps(pkg):
     return dependencies
 
 
+## Extract all the msys2 packages that are dependencies of our direct 
dependencies
 def extract_deps():
 
     print( "Extracting deps" )
@@ -105,44 +167,6 @@ def extract_deps():
 
     return sorted(pkgs)
 
-def gather_deps(deps, arch, directory):
-
-    os.mkdir(arch)
-    os.chdir(arch)
-
-    ## Replace the architecture with the correct one
-    deps = [re.sub(r"x86_64",arch,x) for x in deps]
-
-    ## find all files the transitive dependencies
-    deps_files = check_output(
-        ["pacman", "-Ql"] + deps
-    ).decode("utf-8").split("\n")
-
-    ## Produces output like
-    ## mingw-w64-x86_64-zlib /mingw64/lib/libminizip.a
-
-    ## drop the package name
-    tmp = deps_files.copy()
-    deps_files=[]
-    for d in tmp:
-        slt = d.split()
-        if(not slt==[]):
-            deps_files.append(slt[1])
-
-    ## sort uniq
-    deps_files = sorted(list(set(deps_files)))
-    ## copy all files into local
-    print("Copying dependencies: {}".format(arch))
-    check_output_maybe(["rsync", "-R"] + deps_files + ["."])
-
-    ## And package them up
-    os.chdir(directory)
-    print("Zipping: {}".format(arch))
-    check_output_maybe("zip -9r ../../emacs-{}-{}{}-deps.zip *"
-                       .format(EMACS_MAJOR_VERSION, DATE, arch),
-                       shell=True)
-    os.chdir("../../")
-
 
 def download_source(tarball):
     print("Acquiring {}...".format(tarball))
@@ -160,6 +184,7 @@ def download_source(tarball):
         )
         print("Downloading {}... done".format(tarball))
 
+## Fetch all the source code
 def gather_source(deps):
 
 
@@ -206,7 +231,7 @@ def gather_source(deps):
             to_download.append(tarball)
 
     ## Download in parallel or it is just too slow
-    p = mp.Pool(16)
+    p = mp.Pool(1)
     p.map(download_source,to_download)
 
     print("Zipping")
@@ -255,7 +280,7 @@ parser.add_argument("-l", help="list dependencies only",
 args = parser.parse_args()
 do_all=not (args.c or args.r or args.f or args.t)
 
-deps=extract_deps()
+
 
 DRY_RUN=args.d
 
@@ -270,12 +295,13 @@ else:
     DATE=""
 
 if( do_all or args.t ):
-    gather_deps(deps,"i686","mingw32")
+    gather_deps("i686","mingw32")
 
 if( do_all or args.f ):
-    gather_deps(deps,"x86_64","mingw64")
+    gather_deps("x86_64","mingw64")
 
 if( do_all or args.r ):
+    deps=extract_deps()
     gather_source(deps)
 
 if( args.c ):
diff --git a/admin/nt/dist-build/build-zips.sh 
b/admin/nt/dist-build/build-zips.sh
index 4a9a7b5..fbb9889 100755
--- a/admin/nt/dist-build/build-zips.sh
+++ b/admin/nt/dist-build/build-zips.sh
@@ -64,10 +64,8 @@ function build_zip {
     make -j 4 $INSTALL_TARGET \
          prefix=$HOME/emacs-build/install/emacs-$VERSION/$ARCH
     cd $HOME/emacs-build/install/emacs-$VERSION/$ARCH
-    cp $HOME/emacs-build/deps/libXpm/$ARCH/libXpm-noX4.dll bin
     zip -r -9 emacs-$OF_VERSION-$ARCH-no-deps.zip *
     mv emacs-$OF_VERSION-$ARCH-no-deps.zip $HOME/emacs-upload
-    rm bin/libXpm-noX4.dll
 
     if [ -z $SNAPSHOT ];
     then
@@ -78,7 +76,7 @@ function build_zip {
     fi
 
     echo [build] Using $DEPS_FILE
-    unzip $DEPS_FILE
+    unzip -d bin $DEPS_FILE
 
     zip -r -9 emacs-$OF_VERSION-$ARCH.zip *
     mv emacs-$OF_VERSION-$ARCH.zip ~/emacs-upload
@@ -208,7 +206,7 @@ then
 else
     BRANCH=$REQUIRED_BRANCH
     echo [build] Building from Branch $BRANCH
-    VERSION=$VERSION-$BRANCH
+    VERSION=$VERSION-${BRANCH/\//_}
     OF_VERSION="$VERSION-`date +%Y-%m-%d`"
     ## Use snapshot dependencies
     SNAPSHOT=1