[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]
[no subject]
|
From: |
Mathieu Othacehe |
|
Date: |
Wed, 16 Sep 2020 10:02:24 -0400 (EDT) |
branch: master
commit 78de0da831217513723f83147e7276b9c3652b69
Author: Mathieu Othacehe <othacehe@gnu.org>
AuthorDate: Wed Sep 16 15:58:54 2020 +0200
metrics: Add evaluation related metrics.
* src/cuirass/metrics.scm (db-average-build-start-time-per-eval,
db-average-build-complete-time-per-eval, db-evaluation-completion-speed,
db-latest-evaluations): New procedures.
(%metrics): Add 'average-eval-build-start-time,
'average-eval-build-complete-time, 'evaluation-completion-speed.
(db-update-metrics): Update evaluation related metrics for the evaluations
added the past three days.
* tests/metrics.scm ("average-eval-build-start-time",
"average-eval-build-complete-time", "evaluation-completion-speed"): Add new
tests.
---
src/cuirass/metrics.scm | 100 +++++++++++++++++++++++++++++++++++++++++++++++-
tests/metrics.scm | 47 +++++++++++++++++------
2 files changed, 133 insertions(+), 14 deletions(-)
diff --git a/src/cuirass/metrics.scm b/src/cuirass/metrics.scm
index ba5ed95..a08bcf5 100644
--- a/src/cuirass/metrics.scm
+++ b/src/cuirass/metrics.scm
@@ -96,6 +96,58 @@ SELECT 100 * CAST(SUM(status > 0) as float) / COUNT(*) FROM
" ORDER BY rowid DESC LIMIT " (or limit -1) ");")))
(and=> (expect-one-row rows) (cut vector-ref <> 0)))))
+(define* (db-average-build-start-time-per-eval eval)
+ "Return the average build start time for the given EVAL."
+ (with-db-worker-thread db
+ (let ((rows (sqlite-exec db "\
+SELECT AVG(B.starttime - E.evaltime) FROM
+(SELECT id, evaltime
+FROM Evaluations WHERE id = " eval ") E
+LEFT JOIN
+(SELECT id, evaluation, starttime FROM Builds) B
+ON E.id = B.evaluation and B.starttime > 0
+GROUP BY E.id;")))
+ (and=> (expect-one-row rows) (cut vector-ref <> 0)))))
+
+(define* (db-average-build-complete-time-per-eval eval)
+ "Return the average build complete time for the given EVAL."
+ (with-db-worker-thread db
+ (let ((rows (sqlite-exec db "\
+SELECT AVG(B.stoptime - E.evaltime) FROM
+(SELECT id, evaltime
+FROM Evaluations WHERE id = " eval ") E
+LEFT JOIN
+(SELECT id, evaluation, stoptime FROM Builds) B
+ON E.id = B.evaluation and B.stoptime > 0
+GROUP BY E.id;")))
+ (and=> (expect-one-row rows) (cut vector-ref <> 0)))))
+
+(define* (db-evaluation-completion-speed eval)
+ "Return the evaluation completion speed of the given EVAL. The speed is
+expressed in builds per minute."
+ ;; completion_speed = 60 * completed_builds / eval_duration.
+ ;;
+ ;; evaluation_duration (seconds) = current_time - eval_start_time
+ ;; If some evaluations builds are not completed.
+ ;;
+ ;; evaluation_duration (seconds) = max(build_stop_time) - eval_start_time
+ ;; If the evaluation builds are all completed.
+ (with-db-worker-thread db
+ (let ((rows (sqlite-exec db "\
+SELECT
+60.0 * SUM(B.status = 0) /
+(CASE SUM(status < 0)
+ WHEN 0 THEN MAX(stoptime)
+ ELSE strftime('%s', 'now')
+END - E.evaltime) FROM
+(SELECT id, evaltime
+FROM Evaluations WHERE id = " eval ") E
+LEFT JOIN
+(SELECT id, evaluation, status, stoptime FROM Builds) B
+ON E.id = B.evaluation and B.stoptime > 0
+GROUP BY E.id;")))
+ (and=> (expect-one-row rows) (cut vector-ref <> 0)))))
+
(define (db-previous-day-timestamp)
"Return the timestamp of the previous day."
(with-db-worker-thread db
@@ -110,6 +162,20 @@ date('now', '-1 day'));")))
date('now'));")))
(and=> (expect-one-row rows) (cut vector-ref <> 0)))))
+(define* (db-latest-evaluations #:key (days 3))
+ "Return the successful evaluations added during the previous DAYS."
+ (with-db-worker-thread db
+ (let ((query (format #f "SELECT id from Evaluations
+WHERE date(timestamp, 'unixepoch') > date('now', '-~a day') AND
+status = 0 ORDER BY rowid DESC" days)))
+ (let loop ((rows (sqlite-exec db query))
+ (evaluations '()))
+ (match rows
+ (() (reverse evaluations))
+ ((#(id) . rest)
+ (loop rest
+ (cons id evaluations))))))))
+
;;;
;;; Definitions.
@@ -165,7 +231,22 @@ date('now'));")))
(metric
(id 'percentage-failed-eval-per-spec)
- (compute-proc db-percentage-failed-eval-per-spec))))
+ (compute-proc db-percentage-failed-eval-per-spec))
+
+ ;; Average time to start a build for an evaluation.
+ (metric
+ (id 'average-eval-build-start-time)
+ (compute-proc db-average-build-start-time-per-eval))
+
+ ;; Average time to complete a build for an evaluation.
+ (metric
+ (id 'average-eval-build-complete-time)
+ (compute-proc db-average-build-complete-time-per-eval))
+
+ ;; Evaluation completion speed in builds/minute.
+ (metric
+ (id 'evaluation-completion-speed)
+ (compute-proc db-evaluation-completion-speed))))
(define (metric->type metric)
"Return the index of the given METRIC in %metrics list. This index is used
@@ -251,6 +332,11 @@ timestamp) VALUES ("
(define specifications
(map (cut assq-ref <> #:name) (db-get-specifications)))
+ ;; We can not update all evaluations metrics for performance reasons. Limit
+ ;; to the evaluations that were added during the past three days.
+ (define evaluations
+ (db-latest-evaluations))
+
(db-update-metric 'builds-per-day)
(db-update-metric 'new-derivations-per-day)
(db-update-metric 'pending-builds)
@@ -270,4 +356,14 @@ timestamp) VALUES ("
'percentage-failure-100-last-eval-per-spec spec)
(db-update-metric
'percentage-failed-eval-per-spec spec))
- specifications))
+ specifications)
+
+ ;; Update evaluation related metrics.
+ (for-each (lambda (evaluation)
+ (db-update-metric
+ 'average-eval-build-start-time evaluation)
+ (db-update-metric
+ 'average-eval-build-complete-time evaluation)
+ (db-update-metric
+ 'evaluation-completion-speed evaluation))
+ evaluations))
diff --git a/tests/metrics.scm b/tests/metrics.scm
index 9b9a29f..dfe3728 100644
--- a/tests/metrics.scm
+++ b/tests/metrics.scm
@@ -60,36 +60,41 @@
(sqlite-exec (%db) "\
INSERT INTO Evaluations (specification, status,
timestamp, checkouttime, evaltime) VALUES ('guix', -1, 1600174547, 0, 0);")
- (sqlite-exec (%db) "\
+ (sqlite-exec (%db) (format #f "\
INSERT INTO Evaluations (specification, status,
-timestamp, checkouttime, evaltime) VALUES ('guix', 0, 1600174547, 1600174548,
-1600260947);")
+timestamp, checkouttime, evaltime) VALUES ('guix', 0, ~a, ~a, ~a);\
+" yesterday (+ yesterday 100) (+ yesterday 600)))
(sqlite-exec (%db) "\
INSERT INTO Evaluations (specification, status,
timestamp, checkouttime, evaltime) VALUES ('guix', 1, 1600174547,
1600174548, 0);")
(sqlite-exec (%db) "\
INSERT INTO Evaluations (specification, status,
-timestamp, checkouttime, evaltime) VALUES ('guix', 2, 1600174547,
-1600174548, 0);")
+timestamp, checkouttime, evaltime) VALUES ('guix', 1, 1600174547,
+1600174548, 1600174647);")
+ (sqlite-exec (%db) (format #f "\
+INSERT INTO Builds (id, derivation, evaluation, job_name, system,
+nix_name, log, status, timestamp, starttime, stoptime) VALUES
+(1, '/gnu/store/1.drv', 2, '', '', '', '', 0, ~a, ~a, ~a);\
+" yesterday (+ yesterday 1600) (+ yesterday 2600)))
(sqlite-exec (%db) (format #f "\
INSERT INTO Builds (id, derivation, evaluation, job_name, system,
nix_name, log, status, timestamp, starttime, stoptime) VALUES
-(1, '/gnu/store/xxx.drv', 1, '', '', '', '', 0, ~a, ~a, ~a);\
-" yesterday (+ yesterday 500) (+ yesterday 1500)))
+(2, '/gnu/store/2.drv', 2, '', '', '', '', -2, 0, 0, 0);"))
(sqlite-exec (%db) (format #f "\
INSERT INTO Builds (id, derivation, evaluation, job_name, system,
nix_name, log, status, timestamp, starttime, stoptime) VALUES
-(2, '/gnu/store/yyy.drv', 1, '', '', '', '', -2, 0, 0, 0);"))))
+(3, '/gnu/store/3.drv', 4, '', '', '', '', 0, 1600174451, 1600174451,
+ 1600174651);"))))
(test-equal "average-eval-duration-per-spec"
- `(("guix" . 86400.0))
+ `(("guix" . 350.0))
(begin
(db-update-metric 'average-eval-duration-per-spec "guix")
(db-get-metrics-with-id 'average-eval-duration-per-spec)))
(test-equal "builds-per-day"
- 1.0
+ 2.0
(begin
(db-update-metric 'builds-per-day)
(db-get-metric 'builds-per-day yesterday)))
@@ -101,7 +106,7 @@ nix_name, log, status, timestamp, starttime, stoptime)
VALUES
(db-get-metrics-with-id 'pending-builds)))
(test-equal "new-derivations-per-day"
- `((,yesterday . 1.0))
+ `((,yesterday . 2.0))
(begin
(db-update-metric 'new-derivations-per-day)
(db-get-metrics-with-id 'new-derivations-per-day)))
@@ -118,10 +123,28 @@ nix_name, log, status, timestamp, starttime, stoptime)
VALUES
(sqlite-exec (%db) (format #f "\
INSERT INTO Builds (id, derivation, evaluation, job_name, system,
nix_name, log, status, timestamp, starttime, stoptime) VALUES
-(3, '/gnu/store/zzz.drv', 1, '', '', '', '', -2, 0, 0, 0);"))
+(4, '/gnu/store/4.drv', 1, '', '', '', '', -2, 0, 0, 0);"))
(db-update-metrics)
(db-get-metrics-with-id 'pending-builds)))
+ (test-equal "average-eval-build-start-time"
+ `((2 . 1000.0))
+ (begin
+ (db-update-metric 'average-eval-build-start-time 2)
+ (db-get-metrics-with-id 'average-eval-build-start-time)))
+
+ (test-equal "average-eval-build-complete-time"
+ `((2 . 2000.0))
+ (begin
+ (db-update-metric 'average-eval-build-complete-time 2)
+ (db-get-metrics-with-id 'average-eval-build-complete-time)))
+
+ (test-equal "evaluation-completion-speed"
+ 15.0
+ (begin
+ (db-update-metric 'evaluation-completion-speed 4)
+ (db-get-metric 'evaluation-completion-speed 4)))
+
(test-assert "db-close"
(db-close (%db)))