02/02: hydra: bayfront: Use the Nar Herder for bordeaux.guix.gnu.org.

[Christopher Baines]

cbaines pushed a commit to branch master
in repository maintenance.

commit fcb31b2cd923adfebf784853ef9c902929e70511
Author: Christopher Baines <mail@cbaines.net>
AuthorDate: Tue Dec 21 11:45:18 2021 +0000

    hydra: bayfront: Use the Nar Herder for bordeaux.guix.gnu.org.
    
    This changes were made last week.
    
    This commit switches from serving narinfo files directly from the
    disk, and instead has NGinx forward those requests to the
    nar-herder. For nar requests, NGinx will check the local storage, and
    if the nar isn't there, it'll forward the requests to
    lakefront.cbaines.net. Additionally, the nars are now cached by
    NGinx. The Guix Build Coordinator configuration is changed to call the
    nar-herder to import new nars.
    
    * hydra/bayfront.scm: Adapt the NGinx and Guix Build Coordinator
    configuration for the introduction of the Nar Herder.
---
 hydra/bayfront.scm | 62 ++++++++++++++++++++++++++++++++++++++++++++++++------
 1 file changed, 55 insertions(+), 7 deletions(-)

diff --git a/hydra/bayfront.scm b/hydra/bayfront.scm
index 628f937..4d3e403 100644
--- a/hydra/bayfront.scm
+++ b/hydra/bayfront.scm
@@ -199,6 +199,12 @@ proxy_cache_path /var/cache/nginx/static
                  inactive=10d          # inactive keys removed after 10d
                  keys_zone=static:1m   # nar cache meta data: ~8K keys
                  max_size=200m;        # total cache data size max
+
+proxy_cache_path /var/cache/nginx/bordeaux/nar
+                 levels=2
+                 inactive=28d          # inactive keys removed after 28d
+                 keys_zone=bordeaux-nar:64m
+                 max_size=2048g;       # total cache data size max
 ")
 
 (define %hpc.guix.info-nginx-servers
@@ -512,23 +518,24 @@ add_header Content-Type text/plain;")))
           (nginx-location-configuration
            (uri "~ \\.narinfo$")
            (body '("
-    root /var/lib/nars;
-
-    error_page 404 /404;
+    proxy_pass http://nar-herder;
 
     # For HTTP pipelining.  This has a dramatic impact on performance.
     client_body_buffer_size 128k;
 
     # Narinfos requests are short, serve many of them on a connection.
     keepalive_requests 20000;
-
-    add_header Content-Type text/plain;
 ")))
           (nginx-location-configuration
+           (uri "~ \\.narinfo/info$")
+           (body '("proxy_pass http://nar-herder;";)))
+          (nginx-location-configuration
            (uri "/nar/")
            (body '("
     root /var/lib/nars;
 
+    try_files $uri @nar-storage;
+
     error_page 404 /404;
 
     client_body_buffer_size 256k;
@@ -536,6 +543,22 @@ add_header Content-Type text/plain;")))
     # Nars are already compressed.
     gzip off;
 ")))
+          (nginx-named-location-configuration
+           (name "nar-storage")
+           (body '("proxy_pass https://nar-storage;";
+                   "proxy_cache bordeaux-nar;"
+                   "proxy_cache_valid 200 28d;"
+                   "proxy_ignore_client_abort on;")))
+          (nginx-location-configuration
+           (uri "= /latest-database-dump")
+           (body '("proxy_pass http://nar-herder;";)))
+          (nginx-location-configuration
+           (uri "= /recent-changes")
+           (body '("proxy_pass http://nar-herder;";)))
+          (nginx-location-configuration
+           (uri "~ ^/internal/database/(.*)$")
+           (body '("internal;"
+                   "alias /var/lib/nar-herder/$1;")))
           (nginx-location-configuration
            (uri "~ \"\\/build\\/([a-z0-9-]{36})$\"")
            ;; At some point a detailed page for each build would be
@@ -657,7 +680,24 @@ access_log  /var/log/nginx/bordeaux.access.log;"))
                          "/etc/guix-build-coordinator/signing-key.pub")
                         #:private-key
                         (read-file-sexp
-                         "/etc/guix-build-coordinator/signing-key.sec"))
+                         "/etc/guix-build-coordinator/signing-key.sec")
+                        #:post-publish-hook
+                        (lambda (directory narinfo-filename nar-filename)
+                          (let ((command
+                                 (list #$(file-append nar-herder 
"/bin/nar-herder")
+                                       "import"
+                                       
"--database=/var/lib/nar-herder/nar_herder.db"
+                                       (string-append directory "/" 
narinfo-filename))))
+                            (let ((exit-code
+                                   (status:exit-val
+                                    (apply system* command))))
+                              (unless (zero? exit-code)
+                                (error
+                                 (simple-format
+                                  #f
+                                  "error: command failed (~A): ~A\n"
+                                  exit-code
+                                  command)))))))
                        args)
                 #$(send-guix-data-service-event
                    'build-success-send-event-to-guix-data-service-hook)
@@ -865,7 +905,15 @@ access_log  /var/log/nginx/bordeaux.access.log;"))
               (upstream-blocks
                (list (nginx-upstream-configuration
                       (name "guix-build-coordinator-proxy")
-                      (servers '("localhost:8745")))))
+                      (servers '("localhost:8745")))
+                     (nginx-upstream-configuration
+                      (name "nar-herder")
+                      ;; TODO: This is currently running in a screen
+                      ;; session under the cbaines user
+                      (servers '("localhost:8080")))
+                     (nginx-upstream-configuration
+                      (name "nar-storage")
+                      (servers '("lakefront.cbaines.net:443")))))
               (server-blocks
                (append %hpc.guix.info-nginx-servers
                        %guix-hpc.bordeaux.inria.fr-nginx-servers