[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]
[taler-grid5k] 36/73: Update image build, add install script to install
From: |
gnunet |
Subject: |
[taler-grid5k] 36/73: Update image build, add install script to install taler binaries in running experiment |
Date: |
Tue, 14 Dec 2021 15:10:18 +0100 |
This is an automated email from the git hooks/post-receive script.
marco-boss pushed a commit to branch master
in repository grid5k.
commit 600c67872d2ff51290209c0f4d5190f2b254c714
Author: Boss Marco <bossm8@bfh.ch>
AuthorDate: Wed Dec 1 11:47:00 2021 +0100
Update image build, add install script to install taler binaries in running
experiment
---
configs/etc/taler/conf.d/exchange-business.conf | 2 +
.../systemd/system/taler-exchange-httpd.service | 3 +-
.../systemd/system/taler-exchange-httpd@.service | 3 +-
docker/Dockerfile | 6 ++
docker/entrypoint.sh | 85 +++++++++++++---------
experiment/README.md | 19 +++++
experiment/env | 2 +
experiment/scripts/database.sh | 4 +-
experiment/scripts/exchange.sh | 2 +-
experiment/scripts/helpers.sh | 2 +-
experiment/scripts/install.sh | 70 ++++++++++++++++++
experiment/scripts/setup.sh | 8 +-
image/README.md | 2 +
image/taler-debian11.yaml | 4 +
14 files changed, 170 insertions(+), 42 deletions(-)
diff --git a/configs/etc/taler/conf.d/exchange-business.conf
b/configs/etc/taler/conf.d/exchange-business.conf
index 0e6dd1e..7a8b299 100644
--- a/configs/etc/taler/conf.d/exchange-business.conf
+++ b/configs/etc/taler/conf.d/exchange-business.conf
@@ -12,6 +12,8 @@ MASTER_PUBLIC_KEY = <MASTER_KEY_HERE>
# BASE_URL = https://example.com/
BASE_URL = <BASE_URL_HERE>
+MAX_REQUESTS = 1024
+
# For your terms of service and privacy policy, you should specify
# an Etag that must be updated whenever there are significant
# changes to either document. The format is up to you, what matters
diff --git a/configs/usr/lib/systemd/system/taler-exchange-httpd.service
b/configs/usr/lib/systemd/system/taler-exchange-httpd.service
index 51e62d1..2df2697 100644
--- a/configs/usr/lib/systemd/system/taler-exchange-httpd.service
+++ b/configs/usr/lib/systemd/system/taler-exchange-httpd.service
@@ -8,7 +8,8 @@ PartOf=taler-exchange.target
[Service]
User=taler-exchange-httpd
Type=simple
-Restart=on-failure
+Restart=always
+RestartSec=1ms
EnvironmentFile=/etc/environment
ExecStart=<CMD_PREFIX_HERE>/usr/bin/taler-exchange-httpd -c
/etc/taler/taler.conf $TALER_ARGS $EXCHANGE_ARGS
StandardOutput=journal
diff --git a/configs/usr/lib/systemd/system/taler-exchange-httpd@.service
b/configs/usr/lib/systemd/system/taler-exchange-httpd@.service
index e0930f0..3f51bcc 100644
--- a/configs/usr/lib/systemd/system/taler-exchange-httpd@.service
+++ b/configs/usr/lib/systemd/system/taler-exchange-httpd@.service
@@ -8,7 +8,8 @@ PartOf=taler-exchange.target
[Service]
User=taler-exchange-httpd
Type=simple
-Restart=on-failure
+Restart=always
+RestartSec=1ms
EnvironmentFile=/etc/environment
ExecStart=<CMD_PREFIX_HERE>/usr/bin/taler-exchange-httpd -c
/etc/taler/taler.conf $TALER_ARGS $EXCHANGE_ARGS
StandardOutput=journal
diff --git a/docker/Dockerfile b/docker/Dockerfile
index 8805d3e..f735e49 100644
--- a/docker/Dockerfile
+++ b/docker/Dockerfile
@@ -1,5 +1,9 @@
FROM debian:11
+LABEL author="Boss Marco" \
+ description="Build the Taler Grid5000 environment image" \
+ company="Taler"
+
ENV DEBIAN_FRONTEND=noninteractive
ENV GRID5K_DEST=lyon,lille
@@ -9,6 +13,8 @@ ENV MERCHANT_COMMIT_SHA=master
ENV WALLET_COMMIT_SHA=master
ENV GRID5K_COMMIT_SHA=master
+ENV TALER_HOME=/taler
+
RUN apt update && \
apt upgrade -y && \
apt install -y \
diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh
index 2862722..ea91a20 100644
--- a/docker/entrypoint.sh
+++ b/docker/entrypoint.sh
@@ -1,4 +1,8 @@
#!/bin/bash
+# Docker Entrypoint script
+# Automatically builds the Taler performance
+# environment image for Grid5000
+
set -e
while [[ $# -gt 0 ]]; do
@@ -30,71 +34,79 @@ while [[ $# -gt 0 ]]; do
esac
done
-TALER_HOME=/taler
+# Run a command and fail when there was an error
+# $1: Command to run
+# $2: Log file to write to (default /dev/null)
+function run_command() {
+ if ! $(eval ${1} > "${2:-/dev/null}" 2>&1) ; then
+ echo "ERROR cmd ${1} failed"
+ cat "${2}" || true
+ exit 1
+ fi
+}
+# Prepare a git repo
+# $1: Git repo name
+# $2: Git commit to checkout to
function prepare() {
cd "${TALER_HOME}/$1"
git checkout master > /dev/null && (git pull > /dev/null 2>&1 || true)
git checkout "$2" > /dev/null && (git pull > /dev/null 2>&1 || true)
}
-function _make() {
+# Build from a repo
+function build() {
echo "INFO running bootstrap and configure"
- if ! ./bootstrap > bootstrap.log 2>&1; then
- echo "ERROR bootstrap failed"
- cat bootstrap.log
- exit 1
- fi
+ run_command "./bootstrap" "bootstrap.log"
if [ -f contrib/gana.sh ]; then
- ./contrib/gana.sh > gana.log 2>&1
- fi
- if ! (./configure --enable-logging=verbose --prefix=/usr > configure.log
2>&1 || \
- ./configure > configure.log 2>&1); then
- echo "ERROR configure failed"
- cat configure.log
- exit 1
+ run_command "./contrib/gana.sh" "gana.log"
fi
+ run_command \
+ "(./configure --enable-logging=verbose --prefix=/usr || ./configure)" \
+ "configure.log"
+ run_command "make" "make.log"
if [ "$VERBOSE" = true ]; then
cat bootstrap.log || true
cat gana.log || true
cat configure.log || true
+ cat make.log || true
fi
}
-function build() {
+# Build a dist version (build is run first)
+# $1: Git repo to build
+# $2: Git commit to checkout to
+# $3: Custom command to run before running make dist
+function dist() {
prepare "$1" "$2"
if [ "$CLEAN" = true ]; then
echo "INFO cleaning previous build"
make clean > /dev/null 2>&1 || true
fi
if ! make > /dev/null 2>&1; then
- _make
+ build
fi
if [ ! -z "$3" ]; then
echo "INFO running custom command '$3'"
/bin/bash -c "$3"
fi
echo "INFO running 'make dist'"
- if ! make dist > dist.log 2>&1; then
- echo "ERROR make dist failed"
- cat dist.log
- exit 1
- fi
+ run_command "make dist" "dist.log"
if [ "$VERBOSE" = true ]; then
cat dist.log || true
fi
}
+# Install from a git repo (dist is run first)
+# $1: Git repo to build
+# $2: Git commit to checkout to
+# $3: Custom command to run before running make dist
function install() {
- build "$1" "$2" "$3"
+ dist "$1" "$2" "$3"
echo "INFO installing"
if [[ $(ldconfig -p | grep "$1") == "" ]] || \
[ "$CLEAN" = true ]; then
- if ! make install > install.log 2>&1; then
- echo "ERROR install failed"
- cat install.log
- exit 1
- fi
+ run_command "make install" "install.log"
ldconfig
fi
if [ "$VERBOSE" = true ]; then
@@ -104,6 +116,8 @@ function install() {
if [ "$COPY" != false ];
then
+ # Add the certificate to the agent
+ # no more passwords are needed when it succeeds
eval $(ssh-agent)
if [[ -f "/root/cert.pem" ]]; then
@@ -140,15 +154,16 @@ fi
echo "INFO preparing Grid5000 repository"
prepare grid5k \
"${GRID5K_COMMIT_SHA}"
+
+# Test if the image specification is correct
+# To do this the files copied into it must be present
touch
${TALER_HOME}/grid5k/image/grid5000/steps/data/{gnunet,wallet,exchange,merchant}.tar.gz
-if ! kameleon dryrun ${TALER_HOME}/grid5k/image/taler-debian11.yaml >
dryrun.log 2>&1; then
- echo "ERROR Grid5000 image configuration contains errors"
- cat dryrun.log
- exit 1
-else
- rm ${TALER_HOME}/grid5k/image/grid5000/steps/data/*.tar.gz
-fi
+run_command \
+ "kameleon dryrun ${TALER_HOME}/grid5k/image/taler-debian11.yaml" \
+ "dryrun.log"
+rm ${TALER_HOME}/grid5k/image/grid5000/steps/data/*.tar.gz
+# Prepare and package all applications
echo "INFO preparing GNUnet"
install gnunet \
"${GNUNET_COMMIT_SHA}"
@@ -163,7 +178,7 @@ install merchant \
"${MERCHANT_COMMIT_SHA}"
echo "INFO preparing Taler Wallet"
-build wallet-core \
+dist wallet-core \
"${WALLET_COMMIT_SHA}"
cd ${TALER_HOME}/grid5k/image
diff --git a/experiment/README.md b/experiment/README.md
index 4c6de59..c34bb9a 100644
--- a/experiment/README.md
+++ b/experiment/README.md
@@ -35,6 +35,25 @@ TODO - DOES NOT WORK YET - DNS host needs to be known ..
* Click (Re)Run ESpec for the wallet job (type directory)
* Run `talet-perf update prometheus` on any deployed node to make prometheus
aware of the freshly added wallets
+## Rebuild Taler Binaries
+
+To quickly test fixes of new commits in gnunet,exchange,merchant and wallet,
there is a script `scripts/instal.sh`
+which can be run inside a running experiment rather than rebuilding the whole
image.
+
+To do so copy the following snippet into the `Multi Command` window in jFed:
+(Please adjust commits as needed)
+
+```bash
+#!/bin/bash
+
+export GNUNET_COMMIT_SHA=master
+export EXCHANGE_COMMIT_SHA=master
+export MERCHANT_COMMIT_SHA=master
+export WALLET_COMMIT_SHA=master
+
+./scripts/install.sh
+```
+
## Actions in running Experiment
To add more exchange processes run `taler-perf stop ecxchange <NUM>` on any
node
diff --git a/experiment/env b/experiment/env
index 9fa06af..2f5875a 100644
--- a/experiment/env
+++ b/experiment/env
@@ -54,6 +54,8 @@ DB_NAME=taler-exchange
DB_USER=taler
# Database password
DB_PASSWORD=taler
+# If exchanges should connect to pgBouncer rather than to the db directly
+USE_PGBOUNCER=false
# Initial number of wallets to start in parallel per host.
# This is a bulk size with default = 10
diff --git a/experiment/scripts/database.sh b/experiment/scripts/database.sh
index 17276f3..072c605 100755
--- a/experiment/scripts/database.sh
+++ b/experiment/scripts/database.sh
@@ -41,7 +41,7 @@ function setup_config() {
# Enable password for taler since this is commonly the case
# For the postgres user do not enable authentication (used in metrics)
echo "
- # host all ${DB_USER} 172.16.0.0/12 md5
+ host all ${DB_USER} 172.16.0.0/12 md5
host all postgres 172.16.0.0/12 trust
" >> /etc/postgresql/13/main/pg_hba.conf
@@ -60,7 +60,7 @@ fi
rm -f /var/run/postgresql/pgbouncer.pid
rm -f /var/run/postgresql/.s.PGSQL.6432
-systemctl restart postgresql pgbouncer
+systemctl restart postgresql # pgbouncer
su postgres << EOF
psql postgres -tAc "SELECT 1 FROM pg_roles WHERE
rolname='taler-exchange-httpd'" | \
diff --git a/experiment/scripts/exchange.sh b/experiment/scripts/exchange.sh
index d6ed1ed..e5f3b65 100755
--- a/experiment/scripts/exchange.sh
+++ b/experiment/scripts/exchange.sh
@@ -2,7 +2,7 @@
set -eux
function setup_config() {
- sed -i
"s\<DB_URL_HERE>\postgresql://${DB_USER}:${DB_PASSWORD}@${DATABASE_DOMAIN}:6432/${DB_NAME}\g"
\
+ sed -i
"s\<DB_URL_HERE>\postgresql://${DB_USER}:${DB_PASSWORD}@${DATABASE_DOMAIN}:${DB_PORT}/${DB_NAME}\g"
\
/etc/taler/secrets/exchange-db.secret.conf
sed -i "s/<BANK_HOST_HERE>/${BANK_DOMAIN}/g" \
diff --git a/experiment/scripts/helpers.sh b/experiment/scripts/helpers.sh
index 00ecac9..3c64be4 100755
--- a/experiment/scripts/helpers.sh
+++ b/experiment/scripts/helpers.sh
@@ -43,7 +43,7 @@ function wait_for_db() {
-h "${DATABASE_DOMAIN}" \
-U "${DB_USER}" \
-d "${DB_NAME}" \
- -p "6432" \
+ -p "${DB_PORT}" \
-c '\q';
do
echo "Database not ready yet"
diff --git a/experiment/scripts/install.sh b/experiment/scripts/install.sh
new file mode 100644
index 0000000..63ee692
--- /dev/null
+++ b/experiment/scripts/install.sh
@@ -0,0 +1,70 @@
+#!/bin/bash
+
+TALER_HOME=~/taler
+
+# Run a command and fail when there was an error
+# $1: Command to run
+# $2: Log file to write to (default /dev/null)
+function run_command() {
+ if ! $(eval ${1} > "${2:-/dev/null}" 2>&1) ; then
+ echo "ERROR cmd ${1} failed"
+ cat "${2}" || true
+ exit 1
+ fi
+}
+
+# Prepare the repository
+# $1: Git repo to clone
+# $2: Commit to checkout to
+function prepare() {
+ DIR="${TALER_HOME}/$(basename ${1%.*})"
+ test -d "${DIR}" || git clone "${1}" "${DIR}"
+ cd "${DIR}"
+ git checkout master > /dev/null && \
+ (git pull > /dev/null 2>&1 || true)
+ git checkout "$2" > /dev/null && \
+ (git pull > /dev/null 2>&1 || true)
+}
+
+function build() {
+ echo "INFO running bootstrap and configure"
+ run_command "./bootstrap" "bootstrap.log"
+ if [ -f contrib/gana.sh ]; then
+ run_command "./contrib/gana.sh" "gana.log"
+ fi
+ run_command \
+ "(./configure --enable-logging=verbose --prefix=/usr || ./configure)" \
+ "configure.log"
+ run_command "make" "make.log"
+}
+
+# Install from a git repo
+# $1: Git repo to clone
+# $2: Commit to checkout to
+function install() {
+ prepare "$1" "$2"
+ build
+ echo "INFO installing"
+ run_command "make install" "install.log"
+ ldconfig
+}
+
+if [ ! -d "${TALER_HOME}" ]; then
+ mkdir "${TALER_HOME}"
+fi
+
+echo "INFO preparing GNUnet"
+install "https://git.gnunet.org/gnunet.git" \
+ "${GNUNET_COMMIT_SHA:-master}"
+
+echo "INFO preparing Taler Exchange"
+install "https://git.taler.net/exchange.git" \
+ "${EXCHANGE_COMMIT_SHA:-master}"
+
+echo "INFO preparing Taler Merchant"
+install "https://git.taler.net/merchant.git" \
+ "${MERCHANT_COMMIT_SHA:-master}"
+
+echo "INFO preparing Taler Wallet"
+install "https://git.taler.net/wallet-core.git" \
+ "${WALLET_COMMIT_SHA:-master}"
diff --git a/experiment/scripts/setup.sh b/experiment/scripts/setup.sh
index d4f2c04..9274dc4 100644
--- a/experiment/scripts/setup.sh
+++ b/experiment/scripts/setup.sh
@@ -44,13 +44,19 @@ function setup_log() {
function setup_environment() {
set +x
+ if [ "$USE_PGBOUNCER" = "true" ]; then
+ echo "DB_PORT=6432" > ~/.env
+ else
+ echo "DB_PORT=5432" > ~/.env
+ fi
+
# Needed for envsubst to work
export DNS_ZONE=${DNS_ZONE}
# Set the hostnames completely with substituting ${DNS_ZONE}
cat ~/.env | envsubst > .env && mv .env ~/.env
# Add the environment config for following shells
cat ~/.env | grep -v API_KEY | tee /etc/environment
-
+
# Reload the env since HOST_* and *_DOMAIN was added
source ~/.env
diff --git a/image/README.md b/image/README.md
index 866af19..16e0e7c 100644
--- a/image/README.md
+++ b/image/README.md
@@ -18,6 +18,8 @@ To build the image, the following repositories need to be
added as dist archives
* [taler-merchant](https://git.taler.net/merchant.git): merchant.tar.gz
* [taler-wallet-core](https://git.taler.net/wallet-core.git): wallet.tar.gz
+This is necessary since bootstrap seems not to work when creating the image.
+
Replace `<G5K_USER> `and `<G5K_HOST>` in `taler-debian11.yaml` in the
following line:
`g5k_tar_path:
"http://public.<G5K_HOST>.grid5000.fr/~<G5K_USER>/taler-debian11.tar.zst"`
diff --git a/image/taler-debian11.yaml b/image/taler-debian11.yaml
index ce70f84..50cac31 100644
--- a/image/taler-debian11.yaml
+++ b/image/taler-debian11.yaml
@@ -101,6 +101,10 @@ setup:
- exec_in: |
cd $${taler_path}
+ git clone https://git.gnunet.org/gnunet.git
+ cd gnunet
+ ./bootstrap
+
tar -xvf $${gnunet_tar}
cd gnunet-*
CFLAGS="-O0 -g" ./configure --enable-logging=verbose --prefix=/usr #
--enable-sanitizer
--
To stop receiving notification emails like this one, please contact
gnunet@gnunet.org.
- [taler-grid5k] 48/73: update install script, (continued)
- [taler-grid5k] 48/73: update install script, gnunet, 2021/12/14
- [taler-grid5k] 37/73: bootstrap seems to work with sudo, gnunet, 2021/12/14
- [taler-grid5k] 47/73: fix stupid bug and fix arguments for bank, gnunet, 2021/12/14
- [taler-grid5k] 55/73: db disk mount, gnunet, 2021/12/14
- [taler-grid5k] 42/73: fix setup.sh, gnunet, 2021/12/14
- [taler-grid5k] 50/73: note about postgres huge pages, gnunet, 2021/12/14
- [taler-grid5k] 39/73: fix typo, gnunet, 2021/12/14
- [taler-grid5k] 60/73: fix logbackup so it gets rotated correctly, gnunet, 2021/12/14
- [taler-grid5k] 57/73: improve wallet script, gnunet, 2021/12/14
- [taler-grid5k] 58/73: sanitize scripts, gnunet, 2021/12/14
- [taler-grid5k] 36/73: Update image build, add install script to install taler binaries in running experiment,
gnunet <=
- [taler-grid5k] 63/73: fix, gnunet, 2021/12/14
- [taler-grid5k] 46/73: remove previous builds, gnunet, 2021/12/14
- [taler-grid5k] 62/73: fix monitor, gnunet, 2021/12/14
- [taler-grid5k] 49/73: monitor dns too, change db settings, gnunet, 2021/12/14
- [taler-grid5k] 59/73: fix taler-perf.sh, improve other scripts, gnunet, 2021/12/14
- [taler-grid5k] 56/73: fix startup, gnunet, 2021/12/14
- [taler-grid5k] 45/73: take postgres config to seperate file, gnunet, 2021/12/14
- [taler-grid5k] 38/73: speedup image build, gnunet, 2021/12/14
- [taler-grid5k] 40/73: document variables, gnunet, 2021/12/14
- [taler-grid5k] 70/73: update plotter README and dashboards, gnunet, 2021/12/14