[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]
[bug#58235] [PATCH] rakdo-build-system
|
From: |
Daniel Sockwell |
|
Subject: |
[bug#58235] [PATCH] rakdo-build-system |
|
Date: |
Sat, 01 Oct 2022 22:01:20 +0000 |
I previously submitted patch #46352, which was an attempt to update
the Raku ecosystem and received very helpful feedback from Maxime
Devos. While revising my patch based on that feedback, I realized
that the issue preventing updates to the Raku ecosystem were larger
than I had realized.
In fact, I discovered that I wasn't the first person to try to solve
them: patch #55179 by Paul A. Patience was another attempt in April.
And Paul _also_ wasn't the first attempt; his patch mentioned that
he didn't know about patch #46352 by Alexandru-Sergiu Marton from
February 2021.
Given that two previous patches had attempted to update the Raku
ecosystem without success and that my patch was running into some
of the same issues that had been a problem for Paul, I decided to
take a step back and figure out what was posing such a problem.
I realized that the problem was that the rakudo-build-system didn't
have a complete solution for handling Raku precompilation. To fix
this, I ended up rewiting the rakudo-build-system to add that support;
doing so seems to have resolved the issue.
This patch is part 1 of three, and contains only the changes to the
build system. The second patch will have the changes to the core Raku
packages that the rest of the ecosystem depend on (Rakudo, NQP, Zef,
etc). Finally, the third will adapt the remaining ecosystem packages
to the new version of the build system.
I hope that this 4th patch series will result in Guix being able to
package an up-to-date version of Rakudo!
Best regards,
Daniel
From f03ba44cdab7c3a200cd8f71624848490fdd1099 Mon Sep 17 00:00:00 2001
Message-Id:
<f03ba44cdab7c3a200cd8f71624848490fdd1099.1664660088.git.daniel@codesections.com>
From: Daniel Sockwell <daniel@codesections.com>
Date: Sat, 1 Oct 2022 17:24:20 -0400
Subject: [PATCH] build-system/rakudo: rewrite to support precomp
Significantly rewrite the Rakudo build system to support precompiling
Raku programs and to work with modern versions of Rakudo. This has
previously been a significant blocker to updating Raku programs, see
<https://issues.guix.gnu.org/46352>,
<https://issues.guix.gnu.org/55179>, and
<https://issues.guix.gnu.org/57491>.
* guix/build/rakudo-build-system.scm: significant changes
* guix/build-system/rakudo.scm: minor corresponding changes
---
guix/build-system/rakudo.scm | 31 +--
guix/build/rakudo-build-system.scm | 392 +++++++++++++++++++++--------
2 files changed, 309 insertions(+), 114 deletions(-)
diff --git a/guix/build-system/rakudo.scm b/guix/build-system/rakudo.scm
index 05a4d9c2ad..b9c9d3c612 100644
--- a/guix/build-system/rakudo.scm
+++ b/guix/build-system/rakudo.scm
@@ -54,7 +54,7 @@ (define (default-rakudo)
(define (default-prove6)
"Return the default perl6-tap-harness package for tests."
(let ((module (resolve-interface '(gnu packages perl6))))
- (module-ref module 'perl6-tap-harness)))
+ (module-ref module 'perl6-prove6)))
(define (default-zef)
"Return the default perl6-zef package."
@@ -62,19 +62,20 @@ (define (default-zef)
(module-ref module 'perl6-zef)))
(define* (lower name
- #:key source inputs native-inputs outputs
+ #:key source inputs native-inputs
+ outputs
system target
(rakudo (default-rakudo))
(prove6 (default-prove6))
(zef (default-zef))
- (with-prove6? #t)
- (with-zef? #t)
+ (tests? #t)
+ (test-runner "zef")
#:allow-other-keys
#:rest arguments)
"Return a bag for NAME."
(define private-keywords
'(#:target #:rakudo #:prove6 #:zef #:inputs #:native-inputs))
-
+
(and (not target) ;XXX: no cross-compilation
(bag
(name name)
@@ -87,12 +88,14 @@ (define private-keywords
;; Keep the standard inputs of 'gnu-build-system'.
,@(standard-packages)))
(build-inputs `(("rakudo" ,rakudo)
- ,@(if with-prove6?
- `(("perl6-tap-harness" ,prove6)
- ,@(if with-zef?
- `(("perl6-zef" ,zef))
- '()))
- '())
+ ,@(cond ((not tests?)
+ '())
+ ((and tests? (eq? test-runner "zef"))
+ `(("test-runner-zef" ,zef)))
+ ((and tests? (eq? test-runner "prove6"))
+ `(("test-runner-prove6" ,prove6)))
+ (else '()))
+
,@native-inputs))
(outputs outputs)
(build rakudo-build)
@@ -104,11 +107,10 @@ (define* (rakudo-build name inputs
(search-paths '())
(tests? #t)
(phases '%standard-phases)
- (outputs '("out"))
+ outputs
(system (%current-system))
(guile #f)
- (with-zef? #t)
- (with-prove6? #t)
+ (test-runner "zef")
(imported-modules %rakudo-build-system-modules)
(modules '((guix build rakudo-build-system)
(guix build utils))))
@@ -125,6 +127,7 @@ (define builder
#:phases #$phases
#:system #$system
#:tests? #$tests?
+ #:test-runner #$test-runner
#:outputs #$(outputs->gexp outputs)
#:inputs #$(input-tuples->gexp inputs)))))
diff --git a/guix/build/rakudo-build-system.scm
b/guix/build/rakudo-build-system.scm
index 5cf1cc55bc..82f08f56c3 100644
--- a/guix/build/rakudo-build-system.scm
+++ b/guix/build/rakudo-build-system.scm
@@ -22,132 +22,324 @@ (define-module (guix build rakudo-build-system)
#:use-module (guix build utils)
#:use-module (ice-9 ftw)
#:use-module (ice-9 match)
+ #:use-module (ice-9 regex)
+ #:use-module (ice-9 rdelim)
#:use-module (srfi srfi-1)
#:use-module (srfi srfi-26)
+ #:use-module (srfi srfi-34)
+ #:use-module (srfi srfi-35)
#:export (%standard-phases
rakudo-build))
;; Commentary:
;;
-;; Builder-side code of the standard Rakudo package build procedure.
+;; Builder-side code of the Rakudo build procedure for Raku packages.
+;;
+;;
+;; The rakudo-build-system uses the Rakudo compiler to compile and install Raku
+;; programs, and then uses a Raku test runner (e.g., Zef) to test those
+;; programs. The Rakudo build system is an extension of the GNU build system
+;; that is customized ways to support Raku packages, especially for
+;; precompilation. The Rakudo build system accepts one non-standard argument:
+;;
+;; #:test-runner - The Test Anything Protocol program used to run tests.
+;; (ignored if #:tests? #f)
+;; Valid values: "zef" - [default] Use perl6-zef, which is closest to how
+;; tests are normally run in the Raku ecosystem.
+;; "prove6" - Use perl6-prove6, a Raku TAP test runner
+;; "rakudo" - Run tests directly via Rakudo (no TAP test runner)
+;;
+;; This build system configures Raku programs to write a metadata cache to
+;; $XDG_DATA_HOME/raku/repo
+;;
+;; The above should be all you need to know to package Raku programs with the
+;; Rakudo build system; the rest of this commentary provides background on the
+;; build system and Raku precompilation generally.
+;;
+;; Raku's strategy for dependency management is remarkably similar to Guix's:
+;; Raku compiles each program into an immutable output; then, whenever that
+;; program's (recursive) inputs change, Raku generates a new output and
+;; redirects the relevant references to that new output. This gives Raku many
+;; of the same advantages that Guix enjoys, including allowing simultaneous
+;; installation of multiple package versions.
+;;
+;; But the similarity between Guix and Raku creates an issue for packaging Raku
+;; programs: Raku expects to be able to manage its equivalent to /gnu/store/ by
+;; recompiling Raku programs in their install location. This doesn't work when
+;; those programs are installed in Guix's actual, read-only /gnu/store/. Thus,
+;; integrating Raku and Guix requires some care.
+;;
+;; Specifically, because Raku can't update precomp files in /gnu/store/, Guix
+;; needs replace Raku as the director of Raku's precomp process, including by
+;; guaranteeing that every Raku package is fully precompiled at install time.
+;; This is different from how Raku normally handles precomp (partly lazily,
with
+;; some precomp delayed until a module is first loaded).
+;;
+;; Accordingly, the Rakudo build system installs all precomp files to the
store.
+;; The only non-store data a Raku package should generate is a metadata index
+;; consisting of *.repo-id files and *.lock files that gets written to
+;; $XDG_DATA_HOME/raku/repo. Raku reads this index to avoid the need to check
+;; input integrity, which is purely a performance optimization – Raku
+;; automatically rebuilds the index if it's missing.
+;;
+;; The Rakudo build system supports using Guix-installed packages in
combination
+;; with Zef-installed ones. (Zef is the LPM (language package manager) that
+;; Raku developers use, i.e., Raku's equivalent to Rust's Cargo or JavaScript's
+;; NPM/Yarn/PNPM.) Programs installed via Zef have all their data installed to
+;; the same ./raku/repo directory used for Guix-installed packages' metadata.
+;; After installation, Zef-installed packages be used interchangeably with Raku
+;; packages installed via Guix – though, of course, they lack access to
+;; rollbacks, build transformations, and the rest of Guix's superpowers.
+;;
+;; Including Raku precomp files in the Guix store creates one minor issue: it
+;; decreases the number of Raku packages that pass "guix build --check". This
+;; shouldn't be the case: In theory, Raku precomp files are bit-for-bit
+;; reproducible but, in practice, a few precomp files show (extremely minor)
+;; differences. Until that's fixed, excluding precomp files from Guix's store
+;; would let some Raku packages appear to be fully reproducible. But doing
that
+;; wouldn't *actually* help reproduciblity: those slightly non-reproducible
+;; precomp files would still be generated and executed outside the store.
+;;
+;; The Rakudo build system does not yet have an importer, but creating one is
+;; conceptually simple and is planned.
+;;
;;
;; Code:
-(define* (check #:key tests? inputs with-prove6? #:allow-other-keys)
- (if (and tests? (assoc-ref inputs "perl6-tap-harness"))
- ;(if (and tests? with-prove6?)
- (invoke "prove6" "-I=lib" "t/")
- (format #t "test suite not run~%"))
- #t)
-
-(define* (install #:key inputs outputs with-zef? #:allow-other-keys)
- "Install a given Perl6 package."
- (let* ((out (assoc-ref outputs "out"))
- (perl6 (string-append out "/share/perl6")))
- (if (assoc-ref inputs "perl6-zef")
- ;(if with-zef?
- (begin
- (let ((zef (string-append (assoc-ref inputs "perl6-zef")
- "/bin/zef")))
- (setenv "HOME" (getcwd))
- (mkdir-p perl6)
- (invoke zef "install" "--verbose" "."
- ;; Don't install any of the following:
- "--/depends" "--/build-depends" "--/test-depends"
- (string-append "--install-to=" perl6))
- (delete-file (string-append perl6 "/repo.lock")))
- #t)
- (begin
- (let ((inst (string-append (assoc-ref inputs "rakudo")
- "/share/perl6/tools/install-dist.p6")))
- (setenv "RAKUDO_RERESOLVE_DEPENDENCIES" "0")
- (setenv "RAKUDO_MODULE_DEBUG" "1") ; be verbose while building
- (invoke inst (string-append "--to=" perl6) "--for=site"))))))
-
-(define* (install-libs #:key outputs #:allow-other-keys)
- (let ((out (assoc-ref outputs "out"))
- (lock "lib/.precomp/.lock"))
- (when (file-exists? lock)
- (delete-file "lib/.precomp/.lock"))
- (copy-recursively "lib" (string-append out "/share/perl6/lib"))
- #t))
-(define* (install-bins #:key outputs #:allow-other-keys)
- (let ((out (assoc-ref outputs "out")))
- (when (file-exists? "bin")
- (for-each (lambda (file)
- (install-file file (string-append out "/bin"))
- (chmod (string-append out "/" file) #o555))
- (find-files "bin" ".*")))
- (when (file-exists? "sbin")
- (for-each (lambda (file)
- (install-file file (string-append out "/sbin"))
- (chmod (string-append out "/" file) #o555))
- (find-files "sbin" ".*")))
- #t))
+(define (raku-input? input)
+ "Test if an input appears to be a Rakudo program based on its name."
+ (or (string-prefix? "raku-" (car input))
+ (string-prefix? "perl6-" (car input))))
+
+(define (with-vendor-path dir)
+ "Append the /share/perl6/vendor path to a directory path"
+ (string-append dir "/share/perl6/vendor"))
+
+(define (inputs->raku-vendor-paths inputs)
+ "Map a list of Guix inputs into a list of Raku vendor paths."
+ (let* ((raku-inputs (filter raku-input? inputs))
+ (raku-input-paths (map cdr raku-inputs)))
+ (map with-vendor-path raku-input-paths)))
+
+(define (dir->inst dir-path)
+ "Produce a CompUnit::Repository::Installation path from a directory path.
+Raku provides several types of Repositories, most relevantly 'FileSystem' repos
+and 'Installation' repos. Guix should use Installation repos, which support
+installation of multiple versions. To specify an Installation repo, the path
+should be prefixed with 'inst#'"
+ (string-append "inst#" dir-path))
+
+(define (copy-raku-dependencies inputs out-dir)
+ "Recursively copy files from each Raku dependency to the out directory. This
+lets Rakudo find the precomp files without recursively checking the integrity
of
+each one (which is prohibitively slow). Copying the files doesn't increase
disk
+usage thanks to Guix's deduplication via hard links."
+ (let* ((raku-inputs (filter raku-input? inputs))
+ (raku-input-paths (map (lambda (input)
+ (with-vendor-path (cdr input)))
+ raku-inputs)))
+ (mkdir-p out-dir)
+ (for-each
+ (lambda (source-dir)
+ (for-each make-file-writable (find-files out-dir))
+ (copy-recursively source-dir out-dir))
+ raku-input-paths)))
+
+(define (set-repository-version repository-dir version)
+ "Set the repository format version for a Raku repository. Raku repositories
+have used incompatible formats and an unspecified version defaults to v1; thus,
+Raku packages must set the repo version. NOTE: the 'install-dist.raku' script
+cleans up the version marker, so you may need to set the version again.
+Forgetting to set the version currently triggers a cryptic error that includes
+the text `cannot do '.open' on a directory'."
+ ;; see https://github.com/Raku/old-issue-tracker/issues/6422
+ ;; TODO: Consider upstreaming a patch to provide a less cryptic error msg.
+ (let ((version-file (open-output-file "version")))
+ (format version-file "~a\n" version)
+ (close-port version-file)
+ (install-file "version" repository-dir)))
+
+(define* (validate-keyword-arg keyword allowed-values
+ #:key (keyword-name "a keyword argument"))
+ "Check that the keyword argument has one of the allowed values."
+ (unless (member keyword allowed-values)
+ (let ((allowed-strings (map (cut format #f "~s" <>) allowed-values)))
+ (raise
+ (condition
+ (&message (message
+ (format #f "Invalid value for ~a: '~s'"
+ keyword-name keyword)))
+ (&message (message
+ (format #f "Valid values: '~a'."
+ (string-join allowed-strings "', '")))))))))
+
+(define (extract-provided-modules meta6)
+ "Extract the list of provided modules from a packages META6.json file."
+ ;; Could alternatively use (json->scm), at the cost of a build dependency
+ (let* ((q-mark "\"")
+ (non-quote "[^\"]+")
+ (json-key (string-append q-mark "(" non-quote ")" q-mark ))
+ (json-value (string-append q-mark non-quote q-mark))
+ (json-key-value (make-regexp (string-append non-quote json-key
+ non-quote json-value)))
+ (json-provides-field "[,{]\\s*\"provides\"")
+ (provides-value-regexp
+ (make-regexp
+ (string-append json-provides-field
+ "\\s*:\\s*"
+ "\\{([^}]+)\\}")))
+ (provides-value (regexp-exec provides-value-regexp meta6)))
+ (map (cut match:substring <> 1)
+ (list-matches json-key-value
+ (match:substring provides-value 1)))))
+
+;;; Phases
+
+(define* (setup-rakudo-env #:key inputs outputs #:allow-other-keys)
+ "Set various RAKUDO* environment variables."
+ (let* ((out (assoc-ref outputs "out"))
+ (dest (with-vendor-path out))
+ (rakudo-dir (assoc-ref inputs "rakudo"))
+ (rakudo-home (string-append rakudo-dir "/share/perl6"))
+ (raku-vendor-paths (inputs->raku-vendor-paths inputs))
+ (raku-installations (map dir->inst raku-vendor-paths)))
+
+ (setenv "HOME" (getcwd))
+ (setenv "RAKUDO_HOME" rakudo-home)
+ (setenv "RAKUDO_RESOLVE_DEPENDENCIES" "0")
+ (setenv "RAKUDO_LOG_PRECOMP" "1")
+ (setenv "RAKUDO_MODULE_DEBUG" "1")
+ (setenv "RAKULIB"
+ (string-append (dir->inst dest) "," "home,"
+ (string-join raku-installations ",")))))
+
+
+(define* (check #:key tests? test-runner inputs #:allow-other-keys)
+ "Run the tests in ./t with the supplied #:test-runner (default: zef) unless
+#:tests? is #f."
+ (validate-keyword-arg test-runner '("zef" "prove6" "rakudo")
+ #:keyword-name "test-runner")
+ (when tests?
+ (cond
+ ((eq? test-runner "zef")
+ (let* ((zef-dir (assoc-ref inputs "test-runner-zef"))
+ (zef (string-append zef-dir "/bin/zef")))
+ (invoke zef "test" ".")))
+ ((eq? test-runner "prove6")
+ (let* ((prove6-dir (assoc-ref inputs "test-runner-prove6"))
+ (prove6 (string-append prove6-dir "/bin/prove6")))
+ (invoke prove6 "-Ilib" "t/")))
+ ((eq? test-runner "rakudo")
+ (let* ((rakudo-dir (assoc-ref inputs "rakudo"))
+ (rakudo (string-append rakudo-dir "/bin/rakudo")))
+ (for-each (cut invoke rakudo "-Ilib" <>)
+ (find-files "t/")))))))
+
+(define* (install #:key inputs outputs #:allow-other-keys)
+ "Install the code of a given Raku package."
+ (let* ((out (assoc-ref outputs "out"))
+ (vendor-dir (with-vendor-path out))
+ (rakudo-dir (assoc-ref inputs "rakudo"))
+ (install-dist-script (string-append rakudo-dir
+ "/share/perl6/tools/install-dist.raku")))
+
+ (copy-raku-dependencies inputs vendor-dir)
+ (set-repository-version vendor-dir 2)
+
+ (invoke install-dist-script
+ "--from=."
+ (string-append "--to=" vendor-dir)
+ "--for=vendor"
+ "--build"
+ "--precompile")))
(define* (install-resources #:key outputs #:allow-other-keys)
- (let ((out (assoc-ref outputs "out")))
- (when (file-exists? "resources")
- (copy-recursively "resources"
- (string-append out "/share/perl6/resources")))
- #t))
-
-(define* (wrap #:key inputs outputs #:allow-other-keys)
- (define (list-of-files dir)
- (map (cut string-append dir "/" <>)
- (or (scandir dir (lambda (f)
- (let ((s (stat (string-append dir "/" f))))
- (and (eq? 'regular (stat:type s))
- (not (wrapped-program? f))))))
- '())))
-
- (define bindirs
- (append-map (match-lambda
- ((_ . dir)
- (list (string-append dir "/bin")
- (string-append dir "/sbin"))))
- outputs))
-
- ;; Do not require bash to be present in the package inputs
- ;; even when there is nothing to wrap.
- ;; Also, calculate (sh) only once to prevent some I/O.
- (define %sh (delay (search-input-file inputs "bin/bash")))
- (define (sh) (force %sh))
-
- (let* ((out (assoc-ref outputs "out"))
- (var `("PERL6LIB" "," prefix
- ,(cons (string-append out "/share/perl6/lib,"
- out "/share/perl6/site/lib,"
- out "/share/perl6/vendor/lib")
- (search-path-as-string->list
- (or (getenv "PERL6LIB") "") #\,)))))
- (for-each (lambda (dir)
- (let ((files (list-of-files dir)))
- (for-each (cut wrap-program <> #:sh (sh) var)
- files)))
- bindirs)
- #t))
+ "Install the resources supplied by a Raku package. Resources are typically
+items such as configuration or data files; for details, see
+https://docs.raku.org/language/variables#%?RESOURCES";
+ (when (file-exists? "resources")
+ (let* ((out (assoc-ref outputs "out"))
+ (resources-dir (string-append out "/share/perl6/resources")))
+ (copy-recursively "resources" resources-dir))))
+
+(define* (install-bins #:key outputs #:allow-other-keys)
+ "Install any binary programs supplied by a Raku package."
+
+ (define (install-file-list file-list target-dir)
+ (for-each (cut install-file <> target-dir) file-list))
+
+ (let* ((binary-dirs (filter file-exists? '("bin" "sbin")))
+ (out (assoc-ref outputs "out")))
+ (when binary-dirs
+ (for-each (lambda (binary-dir)
+ (install-file-list (find-files binary-dir)
+ (string-append out "/" binary-dir)))
+ binary-dirs))))
+
+(define* (wrap-raku-programs #:key inputs outputs #:allow-other-keys)
+ "Wrap executable Raku programs to ensure they always uses the correct inputs
+(including the correct Rakudo version) even if other versions are installed."
+
+ (define (with-rakulib program rakulib)
+ (wrap-program program #:sh (search-input-file inputs "bin/sh")
+ `("RAKULIB" "," prefix ,rakulib)))
+
+ (let ((out (assoc-ref outputs "out")))
+ (define bin-dirs
+ (filter directory-exists?
+ (list (string-append out "/bin")
+ (string-append out "/sbin"))))
+ (unless (eq? bin-dirs '())
+ (let* ((bin-files (fold append '() (map find-files bin-dirs)))
+ (wrappable-bin-files (filter (negate wrapped-program?) bin-files))
+ (raku-vendor-paths (inputs->raku-vendor-paths inputs))
+ (raku-installations (map dir->inst raku-vendor-paths))
+ (current-installation (dir->inst (with-vendor-path out)))
+ (xdg-data-home-inst "inst#${XDG_DATA_HOME:-$HOME/.local/share}")
+ (guix-pkg-index-file (string-append xdg-data-home-inst
+ "/raku/repo")))
+ (for-each (cut with-rakulib
+ <>
+ (append (list current-installation)
+ raku-installations
+ (list guix-pkg-index-file)))
+ wrappable-bin-files)))))
+
+(define* (precompile #:key inputs outputs #:allow-other-keys)
+ "Fully precompile all Raku code."
+ (let ((provides (extract-provided-modules (read-string (open-input-file
"META6.json")))))
+ (let* ((out (assoc-ref outputs "out"))
+ (rakudo-dir (assoc-ref inputs "rakudo"))
+ (rakudo (string-append rakudo-dir "/bin/.rakudo-real")))
+ (setenv "RAKULIB" (dir->inst (with-vendor-path out)))
+ (for-each (lambda (module)
+ (invoke rakudo
+ "-c" ; compile w/o running
+ "-e" (string-append "use " module)))
+ provides))))
(define %standard-phases
- ;; No need for 'bootstrap, 'configure or 'build.
(modify-phases gnu:%standard-phases
(delete 'bootstrap)
(delete 'configure)
(delete 'build)
+ (add-after 'unpack 'setup-rakudo-environment setup-rakudo-env)
(replace 'check check)
(replace 'install install)
- (add-before 'install 'install-lib-dir install-libs)
- (add-after 'install-lib-dir 'install-resources install-resources)
+ (add-after 'install 'install-resources install-resources)
(add-after 'install-resources 'install-binaries install-bins)
- ;; needs to be after 'install-binaries and all 'install phases
- (add-after 'install 'wrap wrap)))
+ ;; needs to be after 'install-binaries
+ (add-after 'install-binaries 'wrap-raku-programs wrap-raku-programs)
+ (add-after 'wrap-raku-programs 'precompile-raku-modules precompile)))
(define* (rakudo-build #:key inputs (phases %standard-phases)
#:allow-other-keys #:rest args)
- "Build the given Perl6 package, applying all of PHASES in order."
+ "Build the given Raku package, applying all of PHASES in order."
(apply gnu:gnu-build
- #:inputs inputs #:phases phases
+ #:inputs inputs
+ #:phases phases
args))
+
;;; rakudo-build-system.scm ends here
--
2.37.3
- [bug#58235] [PATCH] rakdo-build-system,
Daniel Sockwell <=