[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]
01/02: hydra: Add config for data.guix.gnu.org.
|
From: |
Christopher Baines |
|
Subject: |
01/02: hydra: Add config for data.guix.gnu.org. |
|
Date: |
Wed, 8 Feb 2023 12:28:07 -0500 (EST) |
cbaines pushed a commit to branch master
in repository maintenance.
commit e672670e9a732f529185b3f2850019f95b3d056e
Author: Christopher Baines <mail@cbaines.net>
AuthorDate: Tue Feb 7 14:30:34 2023 +0100
hydra: Add config for data.guix.gnu.org.
This has existed for a while, but the config has been just on the
machine. Since I'm looking at trying to sort out the hosting for this
going forward, it seems sensible to bring the configuration in to
maintenance.git, even though that's very overdue.
* hydra/data-guix-gnu-org.scm: New file.
---
hydra/data-guix-gnu-org.scm | 444 ++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 444 insertions(+)
diff --git a/hydra/data-guix-gnu-org.scm b/hydra/data-guix-gnu-org.scm
new file mode 100644
index 0000000..63c9a6e
--- /dev/null
+++ b/hydra/data-guix-gnu-org.scm
@@ -0,0 +1,444 @@
+;; Configuration for the machine hosting data.guix.gnu.org
+;;
+;; Currently this is deployed on a Hetzner VM hosted in Nuremberg It
+;; has 8 cores, 32GB of RAM and 240GB root storage with 600GB of
+;; attached storage
+;;
+;; Copyright © 2020, 2021, 2022 Christopher Baines <mail@cbaines.net>
+;; Released under the GNU GPLv3 or any later version.
+
+(use-modules (srfi srfi-1)
+ (gnu)
+ (guix packages)
+ (guix build-system trivial)
+ (guix git-download)
+ (guix gexp))
+(use-service-modules admin databases getmail guix networking mcron ssh
+ virtualization web mail monitoring certbot)
+(use-package-modules certs screen ssh linux web bash version-control
+ haskell-apps python-web databases guile-xyz)
+
+(define backup-script
+ (package
+ (name "data-guix-gnu-org-backup-script")
+ (version "0")
+ (source
+ (plain-file
+ "data-guix-gnu-org-backup"
+ "#!/bin/sh
+
+set -eux
+
+export GUIX_DATA_SERVICE_BACKUP_DIRECTORY=/var/lib/guix-data-service/dumps
+export TMPDIR=/mnt/guix-data-service-temp-data
+
+export GIT_SSH_COMMAND=\"ssh -F
/etc/guix-data-service-backup-ssh-config/config\"
+
+S3_ENDPOINT_URL=\"https://s3.eu-central-1.wasabisys.com\";
+DATE=\"$(date \"+%Y-%m-%d\")\"
+
+source /etc/wasabi-env-vars
+
+cd \"$GUIX_DATA_SERVICE_BACKUP_DIRECTORY\"
+
+##
+## Delete old backups
+##
+
+olderThan=`date -d\"-3 months\" +%s`
+
+find \"$GUIX_DATA_SERVICE_BACKUP_DIRECTORY/\" -maxdepth 1 -not -path '*/\\.*'
-printf \"%f\\n\" | grep -v dumps | sort | while read -r line;
+ do
+ fileName=\"$line\"
+ createDate=`date -d\"$fileName\" +%s`
+ if [[ $createDate -lt $olderThan ]]
+ then
+ echo \"Deleting $fileName\"
+ if [[ $fileName != \"\" ]]
+ then
+ rm -r \"${GUIX_DATA_SERVICE_BACKUP_DIRECTORY}/${fileName}\"
+ git reset # Just to be careful not to commit anything else
+ git add \"${GUIX_DATA_SERVICE_BACKUP_DIRECTORY}/${fileName}\"
+ git commit -m \"Delete ${fileName}\"
+ aws s3 --endpoint-url=https://s3.eu-central-1.wasabisys.com rm
s3://data-guix-gnu-org-database-dumps/ --recursive --exclude \"*\" --include
\"${fileName}*\"
+ fi
+ fi
+ done;
+
+##
+## Main backup
+##
+
+guix-data-service-backup-database
+
+# Add files to the annex
+
+git annex add \"$DATE\"
+git commit -m \"Add data for $DATE\"
+
+git push
+
+# Copy files manually to S3, as git annex export doesn't work reliably
+
+aws s3 \"--endpoint-url=$S3_ENDPOINT_URL\" \\
+ cp
\"$GUIX_DATA_SERVICE_BACKUP_DIRECTORY/$DATE/guix_data_service_full.dump\" \\
+ \"s3://data-guix-gnu-org-database-dumps/$DATE/\"
+
+# Run git annex export
+
+git annex export master --to=wasabi-export || true
+
+cleanup() {
+ echo \"cleaning up\"
+ git annex drop --force \"$DATE\"
+}
+
+trap \"cleanup\" ERR
+
+##
+## Small backup
+##
+
+#guix-data-service-create-small-backup \"$DATE\"
+
+# Add files to the annex
+
+#git annex add \"$DATE\"
+#git commit -m \"Add small backup for $DATE\"
+
+#git push
+
+# Copy files manually to S3, as git annex export doesn't work reliably
+
+#S3_ENDPOINT_URL=\"https://s3.eu-central-1.wasabisys.com\";
+
+#aws s3 \"--endpoint-url=$S3_ENDPOINT_URL\" \\
+ cp
\"$GUIX_DATA_SERVICE_BACKUP_DIRECTORY/$DATE/guix_data_service_small.dump\" \\
+ \"s3://data-guix-gnu-org-database-dumps/$DATE/\"
+
+# Run git annex export
+
+git annex export master --to=wasabi-export || true
+
+##
+## Cleanup
+##
+
+cleanup
+"))
+
+ (build-system trivial-build-system)
+ (arguments
+ `(#:modules ((guix build utils)
+ (srfi srfi-26))
+ #:builder
+ (begin
+ (use-modules (guix build utils)
+ (srfi srfi-26))
+ (let* ((source (assoc-ref %build-inputs "source"))
+ (out (assoc-ref %outputs "out"))
+ (bin (string-append out "/bin"))
+ (script (string-append bin "/data-guix-gnu-org-backup")))
+
+ (setenv "PATH"
+ (string-append (assoc-ref %build-inputs "bash")
+ "/bin"))
+
+ (mkdir-p bin)
+ (copy-file source script)
+ (chmod script #o555)
+ (wrap-program
+ script
+ `("PATH" ":" prefix
+ ,(map (lambda (dir)
+ (string-append dir "/bin"))
+ (map (lambda (input)
+ (assoc-ref %build-inputs input))
+ '("coreutils"
+ "git"
+ "git-annex"
+ "awscli"
+ "guix-data-service"))))))
+ #t)))
+ (inputs
+ `(("bash" ,bash)
+ ("coreutils" ,coreutils)
+ ("git" ,git)
+ ("git-annex" ,git-annex)
+ ("awscli" ,awscli)
+ ("guix-data-service" ,guix-data-service)))
+ (home-page "")
+ (synopsis "")
+ (description "")
+ (license #f)))
+
+(define %nginx-configuration
+ (nginx-configuration
+ (server-names-hash-bucket-size 64)
+ (extra-content
+ "
+ proxy_cache_path /var/cache/nginx/guix-data-service
+ levels=2
+ inactive=2d
+ keys_zone=guix-data-service:4m # ~32K keys
+ max_size=1g
+ use_temp_path=off;
+
+ limit_req_zone $binary_remote_addr zone=webpagezone:1m rate=1r/s;
+")
+ (upstream-blocks
+ (list
+ (nginx-upstream-configuration
+ (name "guix-data-service-proxy")
+ (servers '("localhost:8765")))
+ (nginx-upstream-configuration
+ (name "wasabi-proxy")
+ (servers '("s3.eu-central-1.wasabisys.com:443")))))
+ (server-blocks
+ (list
+ (nginx-server-configuration
+ (listen '("80"))
+ (raw-content
+ (list "return 404;")))
+ (nginx-server-configuration
+ (server-name '("data.guix.gnu.org"))
+ (listen '("80" "443 ssl"))
+ (ssl-certificate
+ "/etc/letsencrypt/live/data.guix.gnu.org/fullchain.pem")
+ (ssl-certificate-key
+ "/etc/letsencrypt/live/data.guix.gnu.org/privkey.pem")
+ (locations
+ (list
+ (nginx-location-configuration
+ (uri "~ /(builds|statistics|revision|package|gnu)/")
+ (body '("proxy_pass http://guix-data-service-proxy;";
+ "proxy_http_version 1.1;"
+ "proxy_set_header Host $host;"
+ "proxy_set_header X-Forwarded-For $remote_addr;"
+
+ "proxy_cache guix-data-service;"
+ "proxy_cache_revalidate on;"
+ "proxy_cache_min_uses 3;"
+ "proxy_cache_use_stale error timeout updating http_500
http_502 http_503 http_504;"
+ "proxy_cache_background_update on;"
+ "proxy_cache_lock on;"
+ "add_header X-Cache-Status $upstream_cache_status;"
+
+ "keepalive_requests 20000;"
+
+ "limit_req zone=webpagezone burst=20 nodelay;"
+ "limit_req_status 429;"
+
+ "gzip on;"
+ "gzip_types text/html text/css application/json;"
+ "gzip_proxied any;")))
+ (nginx-location-configuration
+ (uri "/")
+ (body '("proxy_pass http://guix-data-service-proxy;";
+ "proxy_http_version 1.1;"
+ "proxy_set_header Host $host;"
+ "proxy_set_header X-Forwarded-For $remote_addr;"
+
+ "proxy_cache guix-data-service;"
+ "proxy_cache_revalidate on;"
+ "proxy_cache_min_uses 3;"
+ "proxy_cache_use_stale error timeout updating http_500
http_502 http_503 http_504;"
+ "proxy_cache_background_update on;"
+ "proxy_cache_lock on;"
+ "add_header X-Cache-Status $upstream_cache_status;"
+
+ "keepalive_requests 20000;"
+
+ "gzip on;"
+ "gzip_types text/html text/css application/json;"
+ "gzip_proxied any;")))
+ (nginx-location-configuration
+ (uri "~ ^/dumps/(?!latest)(.+)")
+ (body '("rewrite /dumps/(.+) /data-guix-gnu-org-database-dumps/$1
break;"
+ "proxy_pass https://wasabi-proxy;";
+ "proxy_set_header Host
s3.eu-central-1.wasabisys.com;")))
+ (nginx-location-configuration
+ (uri "^~ /.well-known/acme-challenge/")
+ (body '("root /srv/http/data.guix.gnu.org;")))
+ (nginx-location-configuration
+ (uri "= /.well-known/acme-challenge/")
+ (body '("return 404;"))))))))))
+
+(operating-system
+ (host-name "guix-hetzner-1")
+ (timezone "Europe/Berlin")
+ (locale "en_US.utf8")
+
+ (initrd-modules (cons "virtio_scsi" %base-initrd-modules))
+
+ ;; Boot in "legacy" BIOS mode, assuming /dev/sdX is the
+ ;; target hard disk, and "my-root" is the label of the target
+ ;; root file system.
+ (bootloader (bootloader-configuration
+ (bootloader grub-bootloader)
+ (targets '("/dev/sda"))))
+ (file-systems (cons* (file-system
+ (device (file-system-label "root"))
+ (mount-point "/")
+ (type "ext4"))
+ ;; sudo mount -o discard,defaults
/dev/disk/by-id/scsi-0HC_Volume_4469096 /mnt/guix-data-service-temp-data
+ (file-system
+ (device "/dev/disk/by-id/scsi-0HC_Volume_4469096")
+ (mount-point "/mnt/guix-data-service-temp-data")
+ (type "ext4")
+ (options "discard"))
+ ;; sudo mount -o discard,defaults
/dev/disk/by-id/scsi-0HC_Volume_5374344
/mnt/guix-data-service-postgresql-additional-data
+ (file-system
+ (device "/dev/disk/by-id/scsi-0HC_Volume_5374344")
+ (mount-point
"/mnt/guix-data-service-postgresql-additional-data")
+ (type "ext4")
+ (options "discard"))
+ %base-file-systems))
+
+ (swap-devices
+ (list (swap-space
+ (target "/swapfile"))))
+
+ (users (cons* (user-account
+ (name "chris")
+ (group "users")
+ (supplementary-groups '("wheel"
+ "audio" "video")))
+ %base-user-accounts))
+
+ (packages (cons* screen
+ nss-certs ; Needed for the Guix Data Service
+ %base-packages))
+
+ (services
+ (append (list
+ (service dhcp-client-service-type)
+ (service ntp-service-type)
+ (service openssh-service-type
+ (openssh-configuration
+ (password-authentication? #f)))
+ (service prometheus-node-exporter-service-type)
+ (service opensmtpd-service-type
+ (opensmtpd-configuration
+ (config-file
+ (plain-file "smtpd.conf" (simple-format #f "
+table cred file:/etc/smtpd-secrets
+listen on lo
+action \"relay\" relay host
\"smtp+tls://guixdataservice@smtp.cbaines.net:587\" auth <cred>
+match for any action \"relay\"")))))
+
+ (service postgresql-service-type
+ (postgresql-configuration
+ (postgresql postgresql-13)
+ (config-file
+ (postgresql-config-file
+ (log-destination "stderr")
+ (hba-file
+ (plain-file "pg_hba.conf"
+ "
+local all all trust
+host all all 127.0.0.1/32 md5
+host all all ::1/128 md5
+"))
+ (extra-config
+ '(("session_preload_libraries" "auto_explain")
+ ("auto_explain.log_min_duration" "500ms")
+ ("random_page_cost" 1.0)
+ ("work_mem" "500MB")
+ ;; ("shared_buffers" "1G")
+ ("max_worker_processes" 4)
+ ("max_parallel_workers_per_gather" 2)
+ ("effective_io_concurrency" 200)
+ ("autovacuum_vacuum_threshold" 0)
+ ("autovacuum_vacuum_cost_limit" 2000)
+ ("logging_collector" #t)
+ ("log_directory" "/var/log/postgresql")))))))
+ (service mcron-service-type
+ (mcron-configuration
+ (jobs (list
+ #~(job "30 2 * * *"
+ "guix gc -F 50G")
+ #~(job "0 1 */14 * *"
+ "guix gc")
+ #~(job "45 23 * * */2"
+ "data-guix-gnu-org-backup"
+ #:user "guix-data-service")))))
+
+ (simple-service 'guix-data-service-backup-script
+ profile-service-type
+ (list backup-script))
+
+ (service qemu-binfmt-service-type
+ (qemu-binfmt-configuration
+ (platforms
+ (lookup-qemu-platforms
+ "arm"
+ "aarch64"
+ "ppc64le"
+ ;;"riscv64"
+ ;; I'm not sure which mips thing is relevant
+ ;; "mips" "mipsel" "mipsn32" "mipsn32el" "mips64"
"mips64el"
+ ))))
+
+ (service guix-data-service-type
+ (guix-data-service-configuration
+ (extra-options
+ '("--thread-pool-threads=32"))
+ (extra-process-jobs-options
+ '("--max-processes=2"
+ "--latest-branch-revision-max-processes=4"))
+ (getmail-idle-mailboxes '("Commits"))
+ (commits-getmail-retriever-configuration
+ (getmail-retriever-configuration
+ (type "SimpleIMAPSSLRetriever")
+ (server "imap.cbaines.net")
+ (port 993)
+ (username "patchwork")
+ (password-command
+ (list (file-append coreutils "/bin/cat")
+ "/etc/guix-data-service-commits-imap-password"))
+ (extra-parameters
+ '((mailboxes . ("Commits"))))))))
+
+ (service certbot-service-type
+ (certbot-configuration
+ (certificates
+ (list (certificate-configuration
+ (domains '("data.guix.gnu.org")))))
+ (email "mail@cbaines.net")
+ (webroot "/srv/http/data.guix.gnu.org")))
+
+ (service nginx-service-type %nginx-configuration))
+
+ (modify-services %base-services
+ (rottlog-service-type
+ config => (rottlog-configuration
+ (inherit config)
+ (rc-file
+ (plain-file
+ "rottlog-rc"
+ "
+packer=gzip
+compress=\"-9c\"
+unpacker=gunzip
+uncompress=\"-c\"
+extension=\"gz\"
+pager=\"less\"
+packdir=\"/var/log\"
+fromuser=\"root@data.guix.gnu.org\"
+touser=\"mail@cbaines.net\"
+notifempty=1
+mail=\"sendmail -t\"
+maxdepth=0
+dir_own=root
+dir_grp=root
+dir_perm=0700
+remove_missing
+"))))
+ (guix-service-type
+ config => (guix-configuration
+ (inherit config)
+ (timeout (* 60 60 6))
+ (max-silent-time (* 60 60 2)) ; max 2 hours of silence
+ (extra-options '("--max-jobs=3"
+ "--cache-failures"))))))))