02/03: Insert derivations in chunks

[Christopher Baines]

cbaines pushed a commit to branch master
in repository data-service.

commit b28d338de75a6470097b9c12769dcb1ba143f09d
Author: Christopher Baines <mail@cbaines.net>
AuthorDate: Sun Oct 3 14:54:43 2021 +0100

    Insert derivations in chunks
    
    To avoid making a very large query when inserting lots of derivations.
---
 guix-data-service/model/derivation.scm | 13 ++++++++-----
 1 file changed, 8 insertions(+), 5 deletions(-)

diff --git a/guix-data-service/model/derivation.scm 
b/guix-data-service/model/derivation.scm
index 3178e83..b258803 100644
--- a/guix-data-service/model/derivation.scm
+++ b/guix-data-service/model/derivation.scm
@@ -1514,7 +1514,7 @@ LIMIT $1"
            (map read-derivation-from-file
                 missing-derivations-filenames))))))
 
-  (define (insert-into-derivations)
+  (define (insert-into-derivations dervs)
     (string-append
      "INSERT INTO derivations "
      "(file_name, builder, args, env_vars, system_id) VALUES "
@@ -1535,7 +1535,7 @@ LIMIT $1"
                                  env-vars)
                             ",")
                (system->system-id conn system))))
-           derivations)
+           dervs)
       ",")
      " RETURNING id"
      ";"))
@@ -1544,9 +1544,12 @@ LIMIT $1"
    #t "debug: insert-missing-derivations: inserting ~A derivations\n"
    (length derivations))
   (let ((derivation-ids
-         (map (lambda (result)
-                (string->number (car result)))
-              (exec-query conn (insert-into-derivations)))))
+         (append-map
+          (lambda (chunk)
+            (map (lambda (result)
+                   (string->number (car result)))
+                 (exec-query conn (insert-into-derivations chunk))))
+          (chunk derivations 5000))))
 
     (simple-format
      #t "debug: insert-missing-derivations: updating hash table\n")