01/05: build: Makefile splits Scheme compilation in four steps.

[guix-commits]

civodul pushed a commit to branch master
in repository guix.

commit ef82ba9dd94369926eb13325d5e7da4306d23dd7
Author: Ludovic Courtès <ludo@gnu.org>
AuthorDate: Wed Jun 23 22:43:00 2021 +0200

    build: Makefile splits Scheme compilation in four steps.
    
    Fixes <https://bugs.gnu.org/48963>.
    Reported by Julien Lepiller <julien@lepiller.eu>.
    
    This reduces peak memory consumption to something less unreasonable.
    
    * Makefile.am (make-go): Depend on 'make-*-go' targets; remove body.
    (guile-compilation-rule): New function.
    (MODULES_CORE, MODULES_PACKAGES, MODULES_SYSTEM, MODULES_CLI): New
    variables.
    <top level>: Call 'guile-compilation-rule' 4 times.
    * build-aux/compile-all.scm <top level>: Expect "--total" and
    "--processed".  Take them into account when displaying progress
    reports.
---
 Makefile.am               | 56 +++++++++++++++++++++++++++++++++++++++--------
 build-aux/compile-all.scm | 46 +++++++++++++++++++++++---------------
 2 files changed, 75 insertions(+), 27 deletions(-)

diff --git a/Makefile.am b/Makefile.am
index aa21b53..7bb5de0 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -663,15 +663,53 @@ CLEANFILES =                                      \
 # the whole thing.  Likewise, set 'XDG_CACHE_HOME' to avoid loading possibly
 # stale files from ~/.cache/guile/ccache.
 %.go: make-go ; @:
-make-go: $(MODULES) guix/config.scm $(dist_noinst_DATA)
-       $(AM_V_at)echo "Compiling Scheme modules..." ;                  \
-       unset GUILE_LOAD_COMPILED_PATH ;                                \
-       XDG_CACHE_HOME=/nowhere                                         \
-       host=$(host) srcdir="$(top_srcdir)"                             \
-       $(top_builddir)/pre-inst-env                                    \
-       $(GUILE) -L "$(top_builddir)" -L "$(top_srcdir)"                \
-         --no-auto-compile                                             \
-         -s "$(top_srcdir)"/build-aux/compile-all.scm $^
+make-go: make-core-go make-packages-go make-system-go make-cli-go
+
+# Define a rule to build a subset of the .go files.
+define guile-compilation-rule
+
+$(1): $(2)
+       $(AM_V_at)echo "Compiling Scheme modules..." ;          \
+       unset GUILE_LOAD_COMPILED_PATH ;                        \
+       XDG_CACHE_HOME=/nowhere                                 \
+       host=$(host) srcdir="$(top_srcdir)"                     \
+       $(top_builddir)/pre-inst-env                            \
+       $(GUILE) -L "$(top_builddir)" -L "$(top_srcdir)"        \
+         --no-auto-compile                                     \
+         -s "$(top_srcdir)"/build-aux/compile-all.scm          \
+         --total $(words $(MODULES))                           \
+         --completed $(3)                                      \
+         $$(filter %.scm,$$^)
+
+.PHONY: $(1)
+
+endef
+
+# Split compilation in several steps, each of which building a subset of
+# $(MODULES).  The main goal is to reduce peak memory consumption, as reported
+# in <https://issues.guix.gnu.org/48963>.  Each 'eval' call below creates a
+# 'make-*-go' phony target that builds the corresponding subset.
+
+MODULES_CORE     = $(filter-out guix/scripts/%,$(filter guix/%,$(MODULES)))
+MODULES_PACKAGES = $(filter gnu/packages/%,$(MODULES))
+MODULES_SYSTEM   = $(filter-out gnu/packages/%,$(filter gnu/%,$(MODULES)))
+MODULES_CLI      = $(filter guix/scripts/%,$(MODULES))
+
+$(eval $(call guile-compilation-rule,make-core-go,     \
+  $(MODULES_CORE) guix/config.scm $(dist_noinst_DATA), \
+  0))
+
+$(eval $(call guile-compilation-rule,make-packages-go, \
+  $(MODULES_PACKAGES) make-core-go,                    \
+  $(words $(MODULES_CORE))))
+
+$(eval $(call guile-compilation-rule,make-system-go,   \
+  $(MODULES_SYSTEM) make-packages-go make-core-go,     \
+  $(words $(MODULES_CORE) $(MODULES_PACKAGES))))
+
+$(eval $(call guile-compilation-rule,make-cli-go,                      \
+  $(MODULES_CLI) make-system-go make-packages-go make-core-go,         \
+  $(words $(MODULES_CORE) $(MODULES_PACKAGES) $(MODULES_SYSTEM))))
 
 SUFFIXES = .go
 
diff --git a/build-aux/compile-all.scm b/build-aux/compile-all.scm
index e6982f5..9ffbce4 100644
--- a/build-aux/compile-all.scm
+++ b/build-aux/compile-all.scm
@@ -98,26 +98,36 @@ to 'make'."
     (exit 1)))
 
 (match (command-line)
-  ((_ . files)
+  ((_ "--total" (= string->number grand-total)
+      "--completed" (= string->number processed)
+      . files)
+   ;; GRAND-TOTAL is the total number of .scm files in the project; PROCESSED
+   ;; is the total number of .scm files already compiled in previous
+   ;; invocations of this script.
    (catch #t
      (lambda ()
-       (compile-files srcdir (getcwd)
-                      (filter file-needs-compilation? files)
-                      #:workers (parallel-job-count*)
-                      #:host host
-                      #:report-load (lambda (file total completed)
-                                      (when file
-                                        (format #t "[~3d%] LOAD     ~a~%"
-                                                (% (+ 1 completed) (* 2 total))
-                                                file)
-                                        (force-output)))
-                      #:report-compilation (lambda (file total completed)
-                                             (when file
-                                               (format #t "[~3d%] GUILEC   
~a~%"
-                                                       (% (+ total completed 1)
-                                                          (* 2 total))
-                                                       (scm->go file))
-                                               (force-output)))))
+       (let* ((to-build  (filter file-needs-compilation? files))
+              (processed (+ processed
+                            (- (length files) (length to-build)))))
+         (compile-files srcdir (getcwd) to-build
+                        #:workers (parallel-job-count*)
+                        #:host host
+                        #:report-load (lambda (file total completed)
+                                        (when file
+                                          (format #t "[~3d%] LOAD     ~a~%"
+                                                  (% (+ 1 completed
+                                                          (* 2 processed))
+                                                     (* 2 grand-total))
+                                                  file)
+                                          (force-output)))
+                        #:report-compilation (lambda (file total completed)
+                                               (when file
+                                                 (format #t "[~3d%] GUILEC   
~a~%"
+                                                         (% (+ total completed 
1
+                                                                     (* 2 
processed))
+                                                            (* 2 grand-total))
+                                                         (scm->go file))
+                                                 (force-output))))))
      (lambda _
        (primitive-exit 1))
      (lambda args