04/04: Process derivations in chunks

[Christopher Baines]

cbaines pushed a commit to branch master
in repository data-service.

commit a7c9daab6ace33858be1dcde60f147c3c84afa81
Author: Christopher Baines <mail@cbaines.net>
AuthorDate: Fri Jan 14 15:25:53 2022 +0000

    Process derivations in chunks
    
    Which should reduce the peak memory usage.
---
 guix-data-service/model/derivation.scm | 42 ++++++++++++++++++++++------------
 1 file changed, 28 insertions(+), 14 deletions(-)

diff --git a/guix-data-service/model/derivation.scm 
b/guix-data-service/model/derivation.scm
index fddf0e3..a1ae821 100644
--- a/guix-data-service/model/derivation.scm
+++ b/guix-data-service/model/derivation.scm
@@ -1801,20 +1801,34 @@ INNER JOIN derivation_source_files
                                            derivation-ids-hash-table
                                            derivation-file-names)
 
-        (let ((missing-derivations
-               (with-time-logging "reading missing derivations"
-                 (map read-derivation-from-file
-                      (deduplicate-strings
-                       (filter (lambda (derivation-file-name)
-                                 (not (hash-ref derivation-ids-hash-table
-                                                derivation-file-name)))
-                               derivation-file-names))))))
-
-          (unless (null? missing-derivations)
-            (insert-missing-derivations conn
-                                        derivation-ids-hash-table
-                                        missing-derivations))
-
+        (let ((missing-derivation-filenames
+               (deduplicate-strings
+                (filter (lambda (derivation-file-name)
+                          (not (hash-ref derivation-ids-hash-table
+                                         derivation-file-name)))
+                        derivation-file-names))))
+
+          (chunk-for-each!
+           (lambda (missing-derivation-filenames-chunk)
+             (let ((missing-derivations-chunk
+                    (with-time-logging
+                        (simple-format #f "reading ~A missing derivations"
+                                       (length 
missing-derivation-filenames-chunk))
+                      (map read-derivation-from-file
+                           ;; Do the filter again, since processing the last
+                           ;; chunk might have inserted some of the
+                           ;; derivations in this chunk
+                           (filter (lambda (derivation-file-name)
+                                     (not (hash-ref derivation-ids-hash-table
+                                                    derivation-file-name)))
+                                   missing-derivation-filenames-chunk)))))
+
+               (unless (null? missing-derivations-chunk)
+                 (insert-missing-derivations conn
+                                             derivation-ids-hash-table
+                                             missing-derivations-chunk))))
+           1000
+           missing-derivation-filenames)
 
           (let ((all-ids
                  (map (lambda (derivation-file-name)