branch master updated: hydra: bayfront: Add the Guix Build Coordinator.

[Christopher Baines]

This is an automated email from the git hooks/post-receive script.

cbaines pushed a commit to branch master
in repository maintenance.

The following commit(s) were added to refs/heads/master by this push:
     new 72cafee  hydra: bayfront: Add the Guix Build Coordinator.
72cafee is described below

commit 72cafeeb423a6510e5d7afb7b8b8109d9b4cc0ed
Author: Christopher Baines <mail@cbaines.net>
AuthorDate: Fri Apr 9 11:43:08 2021 +0100

    hydra: bayfront: Add the Guix Build Coordinator.
    
    Both the coordinator component, and an agent running locally on
    bayfront.
    
    This commit also makes other changes to enable this.
    
    * hydra/bayfront.scm (%nginx-deploy-hook): Fix location of the NGinx
    pid file.
    (%certbot-configuration): Add coordinator.bayfront.guix.gnu.org.
    (%bayfront.guix.gnu.org-nginx-servers): Adjust for serving narinfos
    and nars from /var/lib/nars.
    (%coordinator.bayfront.guix.gnu.org-nginx-servers,
    %guix-build-coordinator-configuration): New variables.
    (operating-system)[packages]: Add guix-build-coordinator.
    [services]: Adjust NGinx and Guix, add the Guix Build Coordinator and
    Guix Build Coordinator agent.
---
 hydra/bayfront.scm | 327 +++++++++++++++++++++++++++++++++++------------------
 1 file changed, 217 insertions(+), 110 deletions(-)

diff --git a/hydra/bayfront.scm b/hydra/bayfront.scm
index 9f1d6ae..2c1e3f2 100644
--- a/hydra/bayfront.scm
+++ b/hydra/bayfront.scm
@@ -4,7 +4,7 @@
              (sysadmin people) (sysadmin services)
              (sysadmin dns) (sysadmin web))
 (use-service-modules admin avahi base certbot databases dns
-                     monitoring networking shepherd ssh web mcron)
+                     monitoring networking shepherd ssh web mcron guix)
 (use-package-modules admin certs ci databases linux ssh tls vim
                      package-management web wget)
 
@@ -59,7 +59,7 @@
 (define %nginx-deploy-hook
   ;; Hook that restarts nginx when a new certificate is deployed.
   (program-file "nginx-deploy-hook"
-                #~(let ((pid (call-with-input-file "/var/run/nginx.pid"
+                #~(let ((pid (call-with-input-file "/var/run/nginx/pid"
                                read)))
                     (kill pid SIGHUP))))
 
@@ -73,7 +73,8 @@
                       "logs.guix.gnu.org"
                       "bayfront.guix.info"
                       "hpc.guix.info"
-                      "guix-hpc.bordeaux.inria.fr"))
+                      "guix-hpc.bordeaux.inria.fr"
+                      "coordinator.bayfront.guix.gnu.org"))
            (deploy-hook %nginx-deploy-hook))))))
 
 (define guix-hpc-web-site
@@ -382,123 +383,102 @@ ssl_dhparam         /etc/dhparams.pem;
 access_log   /var/log/nginx/logs.access.log;"))
       (locations common-locations)))))
 
-(define %bayfront.guix.gnu.org-nginx-servers
-  (let ((common-locations
-         (list
-          (nginx-location-configuration
-           (uri "~ ^/admin")
-           (body '("if ($ssl_client_verify != SUCCESS) {
-                        return 403;
-                    }
-                    proxy_pass http://localhost:8081;";)))
-          (nginx-location-configuration
-           (uri "/nix-cache-info")
-           (body '("proxy_pass http://localhost:3000/nix-cache-info;";
-                   ;; Cache this file since that's always the first
-                   ;; thing we ask for.
-                   "proxy_cache static;
-                    proxy_cache_valid 200 100d;  # cache hits for a looong 
time.
-                    proxy_cache_valid any 5m;    # cache misses/others for 5 
min.
-                    proxy_ignore_client_abort on;"
-                   ;; We need to hide and ignore the Set-Cookie header
-                   ;; to enable caching.
-                   "proxy_hide_header    Set-Cookie;
-                    proxy_ignore_headers Set-Cookie;")))
-          (nginx-location-configuration
-           (uri "/nar/")
-           (body '("proxy_pass http://localhost:3000;";
+(define %coordinator.bayfront.guix.gnu.org-nginx-servers
+  (list
+   (nginx-server-configuration
+    (server-name '("coordinator.bayfront.guix.gnu.org"))
+    (listen '("80"))
+    (locations
+     (list
+      (nginx-location-configuration  ; For use by Certbot
+       (uri "/.well-known")
+       (body '(("root /var/www;")))))))
+
+   (nginx-server-configuration
+    (server-name '("coordinator.bayfront.guix.gnu.org"))
+    (listen '("443 ssl"))
+    (ssl-certificate
+     "/etc/letsencrypt/live/bayfront.guix.gnu.org/fullchain.pem")
+    (ssl-certificate-key
+     "/etc/letsencrypt/live/bayfront.guix.gnu.org/privkey.pem")
+    (raw-content
+     '("
+# Make sure SSL is disabled.
+ssl_protocols       TLSv1 TLSv1.1 TLSv1.2;
 
-                   "client_body_buffer_size 256k;"
+# Disable weak cipher suites.
+ssl_ciphers         HIGH:!aNULL:!MD5;
+ssl_prefer_server_ciphers on;
 
-                   ;; Be more tolerant of delays when fetching a nar.
-                   "proxy_read_timeout 60s;
-                    proxy_send_timeout 60s;"
+# Use our own DH parameters created with:
+#    openssl dhparam -out dhparams.pem 2048
+# as suggested at <https://weakdh.org/sysadmin.html>.
+ssl_dhparam         /etc/dhparams.pem;
 
-                   ;; Enable caching for nar files, to avoid ;;
-                   ;; reconstructing and recompressing archives.
-                   "proxy_cache nar;"
-                   "proxy_cache_valid 200 30d;" ; cache hits for 1 month
-                   "proxy_cache_valid 504 3m;" ; timeout, when
-                                               ; hydra.gnu.org is
-                                               ; overloaded
-                   "proxy_cache_valid any 1h;" ; cache misses/others for 1h.
+client_max_body_size 0;
+client_body_buffer_size 128K;
 
-                   "proxy_ignore_client_abort on;"
+access_log  /var/log/nginx/coordinator.access.log;
 
-                   ;; Nars are already compressed.
-                   "gzip off;"
+proxy_set_header X-Forwarded-Host $host;
+proxy_set_header X-Forwarded-Port $server_port;
+proxy_set_header X-Forwarded-For  $proxy_add_x_forwarded_for;"))
+    (locations
+     (list (nginx-location-configuration
+            (uri "/")
+            (body '("try_files $uri $uri/ @guix-build-coordinator;")))
+           (nginx-named-location-configuration
+            (name "guix-build-coordinator")
+            (body '("proxy_pass http://guix-build-coordinator-proxy;";
+                    "proxy_http_version               1.1;"
+                    "proxy_set_header Host            $host;"
+                    "proxy_set_header X-Forwarded-For $remote_addr;"
+                    "proxy_request_buffering          off;"
+
+                    "gzip on;"
+                    "gzip_types text/html application/json;"
+                    "gzip_proxied any;"))))))))
 
-                   ;; We need to hide and ignore the Set-Cookie header
-                   ;; to enable caching.
-                   "proxy_hide_header    Set-Cookie;
-                    proxy_ignore_headers Set-Cookie;")))
+(define %bayfront.guix.gnu.org-nginx-servers
+  (let ((common-locations
+         (list
+          (nginx-location-configuration
+           (uri "= /nix-cache-info")
+           (body '("
+    return 200 'StoreDir: /gnu/store\nWantMassQuery: 0\nPriority: 100\n';
+    add_header Content-Type text/plain;")))
+          (nginx-location-configuration
+           (uri "= /404")
+           (body '("
+return 404 '404';
+add_header Content-Type text/plain;")))
           (nginx-location-configuration
            (uri "~ \\.narinfo$")
-           (body '("proxy_pass http://localhost:3000;";
-
-                   ;; Since 'guix publish' has its own caching, and
-                   ;; since it relies on the atime of cached narinfos
-                   ;; to determine whether a narinfo can be removed
-                   ;; from the cache, don't do any caching here.
+           (body '("
+    root /var/lib/nars;
 
-                   ;; For HTTP pipelining.  This has a dramatic impact
-                   ;; on performance.
-                   "client_body_buffer_size 128k;"
+    error_page 404 /404;
 
-                   ;; Narinfos requests are short, serve many of them
-                   ;; on a connection.
-                   "keepalive_requests 600;"
-
-                   ;; Do not tolerate slowness of hydra.gnu.org when
-                   ;; fetching narinfos: better return 504 quickly
-                   ;; than wait forever.
-                   "proxy_connect_timeout 2s;
-                    proxy_read_timeout 2s;
-                    proxy_send_timeout 2s;"
+    # For HTTP pipelining.  This has a dramatic impact on performance.
+    client_body_buffer_size 128k;
 
-                   ;; 'guix publish --ttl' produces a 'Cache-Control'
-                   ;; header for use by 'guix substitute'.  Let it
-                   ;; through rather than use nginx's "expire"
-                   ;; directive since the expiration time defined by
-                   ;; 'guix publish' is the right one.
-                   "proxy_pass_header Cache-Control;
-
-                    proxy_ignore_client_abort on;"
-
-                   ;; We need to hide and ignore the Set-Cookie header
-                   ;; to enable caching.
-                   "proxy_hide_header    Set-Cookie;
-                    proxy_ignore_headers Set-Cookie;")))
+    # Narinfos requests are short, serve many of them on a connection.
+    keepalive_requests 20000;
 
+    add_header Content-Type text/plain;
+")))
           (nginx-location-configuration
-           (uri "/log/")
-           (body '("proxy_pass http://localhost:3000;";
-
-                   ;; Enable caching for build logs.
-                   "proxy_cache logs;"
-                   "proxy_cache_valid 200 60d;" ; cache hits.
-                   "proxy_cache_valid 504 3m;" ; timeout, when
-                                               ; hydra.gnu.org is
-                                               ; overloaded
-                   "proxy_cache_valid any 1h;" ; cache misses/others.
-
-                   "proxy_ignore_client_abort on;"
-
-                   ;; We need to hide and ignore the Set-Cookie header
-                   ;; to enable caching.
-                   "proxy_hide_header    Set-Cookie;
-                    proxy_ignore_headers Set-Cookie;")))
+           (uri "/nar/")
+           (body '("
+    root /var/lib/nars;
 
-          ;; Content-addressed files served by 'guix publish'.
-          (nginx-location-configuration
-           (uri "/file/")
-           (body '("proxy_pass http://localhost:3000;";
+    error_page 404 /404;
 
-                   "proxy_cache cas;"
-                   "proxy_cache_valid 200 200d;" ; cache hits
-                   "proxy_cache_valid any 5m;"   ; cache misses/others
+    client_body_buffer_size 256k;
 
-                   "proxy_ignore_client_abort on;"))))))
+    # Nars are already compressed.
+    gzip off;
+"))))))
 
     (list
      (nginx-server-configuration
@@ -547,6 +527,101 @@ proxy_set_header X-Forwarded-Port $server_port;
 proxy_set_header X-Forwarded-For  $proxy_add_x_forwarded_for;"))
       (locations common-locations)))))
 
+(define %guix-build-coordinator-configuration
+  (let* ((data.guix.gnu.org-build-event-destination
+          #~(string-append
+             "https://data.guix.gnu.org";
+             "/build-server/2/build-events?token="
+             (string-trim-right
+              (call-with-input-file
+                  "/etc/guix-build-coordinator/data.guix.gnu.org-token"
+                get-string-all))))
+         (data.guix-patches.cbaines.net-build-event-destination
+          #~(string-append
+             "https://data.guix-patches.cbaines.net";
+             "/build-server/2/build-events?token="
+             (string-trim-right
+              (call-with-input-file
+                  
"/etc/guix-build-coordinator/data.guix-patches.cbaines.net-token"
+                get-string-all))))
+         (publish-directory
+          "/var/lib/nars")
+         (recompress-log-file-hook
+          #~(apply ((@ (guix-build-coordinator hooks)
+                       build-recompress-log-file-hook)
+                    #:recompress-to 'gzip)
+                   args)))
+
+    (define (send-guix-data-service-event event-hook)
+      #~(begin
+          (use-modules (ice-9 textual-ports))
+
+          (apply ((@ (guix-build-coordinator hooks)
+                     #$event-hook)
+                  #$data.guix.gnu.org-build-event-destination)
+                 args)
+          (apply ((@ (guix-build-coordinator hooks)
+                     #$event-hook)
+                  #$data.guix-patches.cbaines.net-build-event-destination)
+                 args)))
+
+    (guix-build-coordinator-configuration
+     (agent-communication-uri-string
+      "http://127.0.0.1:8745";)          ; only listen locally
+     (allocation-strategy
+      #~derivation-ordered-build-allocation-strategy)
+     (hooks
+      `((build-submitted
+         . ,#~(lambda args
+                #$(send-guix-data-service-event
+                   'build-submitted-send-event-to-guix-data-service-hook)))
+        (build-started
+         . ,#~(lambda args
+                #$(send-guix-data-service-event
+                   'build-started-send-event-to-guix-data-service-hook)))
+        (build-success
+         . ,#~(lambda args
+                (use-modules (gcrypt pk-crypto)) ; for read-file-sexp
+
+                #$recompress-log-file-hook
+                (apply ((@ (guix-build-coordinator hooks)
+                           build-success-publish-hook)
+                        #$publish-directory
+                        ;; These should be the same as
+                        ;; /etc/guix/... but are copied here so that
+                        ;; they can be read by the Guix Build
+                        ;; Coordinantor
+                        #:public-key
+                        (read-file-sexp
+                         "/etc/guix-build-coordinator/signing-key.pub")
+                        #:private-key
+                        (read-file-sexp
+                         "/etc/guix-build-coordinator/signing-key.sec"))
+                       args)
+                #$(send-guix-data-service-event
+                   'build-success-send-event-to-guix-data-service-hook)
+                (apply (@ (guix-build-coordinator hooks)
+                          default-build-success-hook)
+                       args)))
+        (build-failure
+         . ,#~(lambda args
+                #$recompress-log-file-hook
+                (apply ((@ (guix-build-coordinator hooks)
+                           build-failure-retry-hook))
+                       args)
+                #$(send-guix-data-service-event
+                   'build-failure-send-event-to-guix-data-service-hook)
+                (apply (@ (guix-build-coordinator hooks)
+                          default-build-failure-hook)
+                       args)))
+        (build-canceled
+         . ,#~(lambda args
+                #$(send-guix-data-service-event
+                   'build-canceled-send-event-to-guix-data-service-hook)
+                (apply (@ (guix-build-coordinator hooks)
+                          default-build-canceled-hook)
+                       args))))))))
+
 (operating-system
   (host-name "bayfront")
   (timezone "Europe/Paris")
@@ -573,7 +648,7 @@ proxy_set_header X-Forwarded-For  
$proxy_add_x_forwarded_for;"))
 
   (packages (cons* certbot wget iptables jnettop
                    mdadm vim lm-sensors openssh
-                   nss-certs
+                   nss-certs guix-build-coordinator
                    %base-packages))
 
   (services
@@ -687,11 +762,34 @@ proxy_set_header X-Forwarded-For  
$proxy_add_x_forwarded_for;"))
     (service nginx-service-type
              (nginx-configuration
               (extra-content %bayfront-nginx-service-extra-config)
+              (upstream-blocks
+               (list (nginx-upstream-configuration
+                      (name "guix-build-coordinator-proxy")
+                      (servers '("localhost:8745")))))
               (server-blocks
                (append %hpc.guix.info-nginx-servers
                        %guix-hpc.bordeaux.inria.fr-nginx-servers
                        %logs.guix.gnu.org-nginx-servers
-                       %bayfront.guix.gnu.org-nginx-servers))))
+                       %bayfront.guix.gnu.org-nginx-servers
+                       %coordinator.bayfront.guix.gnu.org-nginx-servers))))
+
+    (service guix-build-coordinator-service-type
+             %guix-build-coordinator-configuration)
+
+    (service guix-build-coordinator-agent-service-type
+             (guix-build-coordinator-agent-configuration
+              (coordinator "https://coordinator.bayfront.guix.gnu.org";)
+              (authentication
+               (guix-build-coordinator-agent-password-file-auth
+                (uuid "0c973ac8-4e62-4889-87b4-31c9536641db")
+                (password-file
+                 "/etc/guix-build-coordinator-agent-password")))
+              (max-parallel-builds 4)
+              (systems '("x86_64-linux" "i686-linux"))
+              (derivation-substitute-urls
+               (list "https://data.guix.gnu.org";))
+              (non-derivation-substitute-urls
+               (list "https://bayfront.guix.gnu.org";))))
 
     (modify-services %base-services
       (guix-service-type
@@ -699,20 +797,29 @@ proxy_set_header X-Forwarded-For  
$proxy_add_x_forwarded_for;"))
                   (substitute-urls
                    '("https://bayfront.guix.gnu.org";))
                   (authorized-keys
-                   (cons
+                   (cons*
                     (local-file "keys/guix/bayfront.guix.info-export.pub")
+                    ;; for substituting derivations
+                    (plain-file
+                     "data.guix.gnu.org.pub"
+                     "
+(public-key
+ (ecc
+  (curve Ed25519)
+  (q #1EEE5340C3AAD6E062A1395A88A86FC75982E8BC7DCBAE171858EEAAB14AAB77#)
+  )
+ )")
                     %build-node-keys))
 
                   (max-silent-time 3600)
-                  (timeout (* 6 3600))
+                  (timeout (* 12 3600))
 
                   ;; be friendly to 'guix publish' users
                   (log-compression 'gzip)
 
                   (build-accounts 64)
                   (extra-options
-                   (list "--max-jobs" "4"
-                         "--cores" "6"))))
+                   (list "--max-jobs" "4"))))
       (login-service-type
        config => (login-configuration
                   (inherit config)