[no subject]

[Ludovic Courtès]

branch: master
commit a3025cad92054cb7399a7816a0696493396fc916
Author: Ludovic Courtès <address@hidden>
Date:   Mon Feb 5 14:59:29 2018 +0100

    cuirass: Add '--threads' and put an upper bound on the default.
    
    * bin/cuirass.in (show-help, %options): Add "--threads".
    (main): Honor it.  Pass #:parallelism to 'run-fibers'.
---
 bin/cuirass.in   | 17 ++++++++++++++++-
 doc/cuirass.texi |  9 +++++++++
 2 files changed, 25 insertions(+), 1 deletion(-)

diff --git a/bin/cuirass.in b/bin/cuirass.in
index 5c11ff0..ba10274 100644
--- a/bin/cuirass.in
+++ b/bin/cuirass.in
@@ -8,6 +8,7 @@ exec ${GUILE:address@hidden@} --no-auto-compile -e main -s "$0" 
"$@"
 ;;;; cuirass -- continuous integration tool
 ;;; Copyright © 2016 Mathieu Lirzin <address@hidden>
 ;;; Copyright © 2017 Mathieu Othacehe <address@hidden>
+;;; Copyright © 2018 Ludovic Courtès <address@hidden>
 ;;;
 ;;; This file is part of Cuirass.
 ;;;
@@ -31,6 +32,7 @@ exec ${GUILE:address@hidden@} --no-auto-compile -e main -s 
"$0" "$@"
              (guix ui)
              (fibers)
              (fibers channels)
+             (ice-9 threads)                    ;for 'current-processor-count'
              (ice-9 getopt-long))
 
 (define (show-help)
@@ -48,6 +50,7 @@ exec ${GUILE:address@hidden@} --no-auto-compile -e main -s 
"$0" "$@"
       --listen=HOST         Listen on the network interface for HOST
   -I, --interval=N          Wait N seconds between each poll
       --use-substitutes     Allow usage of pre-built substitutes
+      --threads=N           Use up to N kernel threads
   -V, --version             Display version
   -h, --help                Display this help message")
   (newline)
@@ -63,6 +66,7 @@ exec ${GUILE:address@hidden@} --no-auto-compile -e main -s 
"$0" "$@"
     (listen                           (value #t))
     (interval       (single-char #\I) (value #t))
     (use-substitutes                  (value #f))
+    (threads                          (value #t))
     (fallback                         (value #f))
     (version        (single-char #\V) (value #f))
     (help           (single-char #\h) (value #f))))
@@ -95,8 +99,18 @@ exec ${GUILE:address@hidden@} --no-auto-compile -e main -s 
"$0" "$@"
               (port      (string->number (option-ref opts 'port "8080")))
               (host      (option-ref opts 'listen "localhost"))
               (interval  (string->number (option-ref opts 'interval "300")))
-              (specfile  (option-ref opts 'specifications #f)))
+              (specfile  (option-ref opts 'specifications #f))
+
+              ;; Since our work is mostly I/O-bound, default to a maximum of 4
+              ;; kernel threads.  Going beyond that can increase overhead (GC
+              ;; may not scale well, work-stealing may become detrimental,
+              ;; etc.) for little in return.
+              (threads   (or (and=> (option-ref opts 'threads #f)
+                                    string->number)
+                             (min (current-processor-count) 4))))
           (prepare-git)
+
+          (log-message "running Fibers on ~a kernel threads" threads)
           (run-fibers
            (lambda ()
              (with-database db
@@ -158,4 +172,5 @@ exec ${GUILE:address@hidden@} --no-auto-compile -e main -s 
"$0" "$@"
            ;; continuations.)  Thus, reduce the tick rate.
            #:hz 10
 
+           #:parallelism threads
            #:drain? #t)))))))
diff --git a/doc/cuirass.texi b/doc/cuirass.texi
index ec81245..fcebef6 100644
--- a/doc/cuirass.texi
+++ b/doc/cuirass.texi
@@ -198,6 +198,15 @@ Wait @var{n} seconds between each poll.
 This can be useful when you are not interested in building the
 dependencies of a particular job.
 
address@hidden address@hidden
+Use up to @var{n} kernel threads.
+
address@hidden should be lower than or equal to the number of CPU cores on the
+machine.  In general though, having a large @var{n} is not very useful
+since the work of Cuirass is primarily I/O-bound---on the contrary,
+large values of @var{n} may increase overhead.  The default value should
+be appropriate for most cases.
+
 @item --version
 @itemx -V
 Display the actual version of @code{cuirass}.