bug#46330: Guile-provided GMP allocators interfere with GnuTLS

[Ludovic Courtès]

Ludovic Courtès <ludo@gnu.org> skribis:

> In a nutshell, Guile installs its own GMP memory allocation routines
> (when ‘scm_install_gmp_memory_functions’ is true, which is the default)
> so that GMP allocates via libgc.  GnuTLS uses Nettle, which uses GMP, so
> Nettle too ends up allocating via libgc; however, since pointers to that
> memory are not scanned by libgc, they end up being reclaimed early.

For the record, one option I considered is to link GnuTLS against
Nettle-with-mini-GMP, whereby Nettle uses a bundled mini-GMP instead of
mini-GMP.

Another option is to build GnuTLS --with-nettle-mini, where GnuTLS links
against a bundled mini-Nettle, instead linked against a bundled mini-GMP (!).

Currently both options lead to build failures in GnuTLS:

  https://lists.gnutls.org/pipermail/gnutls-help/2021-February/004680.html

(Package definitions attached for posterity.)

Ludo’.

diff --git a/gnu/packages/nettle.scm b/gnu/packages/nettle.scm
index f5e7188ff0..875a858946 100644
--- a/gnu/packages/nettle.scm
+++ b/gnu/packages/nettle.scm
@@ -1,5 +1,5 @@
 ;;; GNU Guix --- Functional package management for GNU
-;;; Copyright © 2012, 2013, 2014, 2015 Ludovic Courtès <ludo@gnu.org>
+;;; Copyright © 2012, 2013, 2014, 2015, 2021 Ludovic Courtès <ludo@gnu.org>
 ;;; Copyright © 2016 Mark H Weaver <mhw@netris.org>
 ;;; Copyright © 2017 Efraim Flashner <efraim@flashner.co.il>
 ;;;
@@ -89,3 +89,12 @@ themselves.")
         ;; Build "fat" binaries where the right implementation is chosen
         ;; at run time based on CPU features (starting from 3.1.)
         `(cons "--enable-fat" ,flags))))))
+
+(define-public nettle/mini-gmp
+  (package/inherit nettle
+    (name "nettle-mini-gmp")
+    (arguments
+     (substitute-keyword-arguments (package-arguments nettle)
+       ((#:configure-flags flags)
+        `(cons "--enable-mini-gmp" ,flags))))
+    (propagated-inputs '())))
diff --git a/gnu/packages/tls.scm b/gnu/packages/tls.scm
index 775e915534..fe2ec88a9e 100644
--- a/gnu/packages/tls.scm
+++ b/gnu/packages/tls.scm
@@ -1,5 +1,5 @@
 ;;; GNU Guix --- Functional package management for GNU
-;;; Copyright © 2012, 2013, 2014, 2015, 2016, 2017, 2019, 2020 Ludovic Courtès 
<ludo@gnu.org>
+;;; Copyright © 2012, 2013, 2014, 2015, 2016, 2017, 2019, 2020, 2021 Ludovic 
Courtès <ludo@gnu.org>
 ;;; Copyright © 2014, 2015, 2016, 2017, 2018 Mark H Weaver <mhw@netris.org>
 ;;; Copyright © 2014 Ian Denhardt <ian@zenhack.net>
 ;;; Copyright © 2013, 2015 Andreas Enge <andreas@enge.fr>
@@ -165,6 +165,7 @@ living in the same process.")
   (package
     (name "gnutls")
     (version "3.6.15")
+    (replacement gnutls/mini-nettle)
     (source (origin
               (method url-fetch)
               ;; Note: Releases are no longer on ftp.gnu.org since the
@@ -256,6 +257,31 @@ required structures.")
     (properties '((ftp-server . "ftp.gnutls.org")
                   (ftp-directory . "/gcrypt/gnutls")))))
 
+;; (define-public gnutls/nettle-mini-gmp
+;;   (package/inherit gnutls
+;;     (arguments
+;;      (substitute-keyword-arguments (package-arguments gnutls)
+;;        ((#:phases phases '%standard-phases)
+;;         `(modify-phases ,phases
+;;            (add-before 'configure 'dont-link-against-libgmp
+;;              (lambda _
+;;                (substitute* "configure"
+;;                  (("GMP_LIBS=\"-lgmp\"")
+;;                   "GMP_LIBS=\"\""))))))))
+;;     (propagated-inputs
+;;      `(("nettle" ,nettle/mini-gmp)
+;;        ,@(alist-delete "nettle" (package-propagated-inputs gnutls))))))
+
+(define-public gnutls/mini-nettle
+  (package/inherit gnutls
+    (arguments
+      (substitute-keyword-arguments (package-arguments gnutls)
+        ((#:configure-flags flags ''())
+         `(cons "--with-nettle-mini" ,flags))))
+    ;; (propagated-inputs
+    ;;  (alist-delete "nettle" (package-propagated-inputs gnutls)))
+    ))
+
 (define-public gnutls/guile-2.0
   ;; GnuTLS for Guile 2.0.
   (package/inherit gnutls