[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]
[taler-grid5k] 58/73: sanitize scripts
From: |
gnunet |
Subject: |
[taler-grid5k] 58/73: sanitize scripts |
Date: |
Tue, 14 Dec 2021 15:10:40 +0100 |
This is an automated email from the git hooks/post-receive script.
marco-boss pushed a commit to branch master
in repository grid5k.
commit 93009e27e8d6b0df70e5409f2be096baa013a33a
Author: Boss Marco <bossm8@bfh.ch>
AuthorDate: Thu Dec 9 20:36:01 2021 +0100
sanitize scripts
---
README.md | 10 +++++++
experiment/scripts/database.sh | 2 +-
experiment/scripts/exchange.sh | 59 +++++++++++++++++++++++++---------------
experiment/scripts/helpers.sh | 22 +++++++++++++++
experiment/scripts/monitor.sh | 41 +++++++++++++++++++++-------
experiment/scripts/proxy.sh | 56 +++++++++++++++++++++++++++++---------
experiment/scripts/taler-perf.sh | 58 +++++++++++++++++++++++++++++++++++++--
experiment/scripts/wallet.sh | 23 ++++------------
8 files changed, 204 insertions(+), 67 deletions(-)
diff --git a/README.md b/README.md
index 7a86c9b..3f170a0 100644
--- a/README.md
+++ b/README.md
@@ -35,3 +35,13 @@ Contains the configurations for the applications in the
environment.
They will be adjusted copied to '/' (some make sure to add the correct
directory strucutre)
once an experiment is started.
+## Quick Start
+
+To run an experiment, you must
+
+* (optionally) have a grafana instance
+* Make sure the environment exists in the public direcory which is configured
in
+ `experiment/taler.rspec`. If its not in the grid, use `image/README.md` or
`docker/README.md`
+ to see how to build such an environment.
+* Read `experiment/README.md` for instructions on how to run an experiment
inside the grid
+
diff --git a/experiment/scripts/database.sh b/experiment/scripts/database.sh
index c07517f..3f0631e 100755
--- a/experiment/scripts/database.sh
+++ b/experiment/scripts/database.sh
@@ -109,7 +109,7 @@ function setup_config() {
if [[ "$1" == "init" ]]; then
setup_config
- setup_disks
+# setup_disks
fi
systemctl restart postgresql
diff --git a/experiment/scripts/exchange.sh b/experiment/scripts/exchange.sh
index 4d747c0..6300b45 100755
--- a/experiment/scripts/exchange.sh
+++ b/experiment/scripts/exchange.sh
@@ -1,6 +1,8 @@
#!/bin/bash
set -eux
+source ~/scripts/helpers.sh
+
function setup_config() {
sed -i
"s\<DB_URL_HERE>\postgresql://${DB_USER}:${DB_PASSWORD}@${DATABASE_DOMAIN}:${DB_PORT}/${DB_NAME}\g"
\
/etc/taler/secrets/exchange-db.secret.conf
@@ -33,9 +35,7 @@ function setup_exchange() {
taler-exchange-offline upload < fee-res.json
}
-if [[ "$1" == "init" ]];
-then
- source ~/scripts/helpers.sh
+function init_exchanges() {
restart_rsyslog
setup_config
@@ -44,29 +44,44 @@ then
# Wait another second to make sure user has permissions
sleep 5
- NUM_PROCESSES=$((${NUM_EXCHANGE_PROCESSES:-10}-1))
-
setup_exchange
+}
+
+function start_exchanges() {
+ RUNNING=$(ps -aux | grep "[taler]-exchange-httpd" | wc -l)
+
+ for i in $(seq ${1}); do
+ let "i+=${RUNNING}-1" || true
+ let "i+=10000"
+ systemctl restart taler-exchange-httpd@"${i}".socket \
+ taler-exchange-httpd@"${i}".service
+ sleep 0.05
+ done
+}
- # || true is needed since when 1-1 is run let returns an error
- let "NUM_WIREWATCH_PROCESSES-=1" || true
- for i in $(seq ${NUM_WIREWATCH_PROCESSES});
- do
- systemctl restart taler-exchange-wirewatch@${i}.service
+function start_wirewatchers() {
+ for i in $(seq ${1}); do
+ systemctl restart taler-exchange-wirewatch@"${i}".service
done
+}
-else
- NUM_PROCESSES=$1
-fi
-
-RUNNING=$(ps -aux | grep "[taler]-exchange-httpd" | wc -l)
+function stop_exchanges() {
+ stop_numbered_services "taler-exchange-httpd" $1
+}
-for i in $(seq ${NUM_PROCESSES});
-do
- let "i+=${RUNNING}-1" || true
- let "i+=10000"
- systemctl restart taler-exchange-httpd@"${i}".socket
taler-exchange-httpd@"${i}".service
- sleep 0.05
-done
+case $1 in
+ init)
+ init_exchanges
+ # TODO adjust when started manually and not with .target
+ start_exchanges "$((${NUM_EXCHANGE_PROCESSES}-1))"
+ start_wirewatchers "$((${NUM_WIREWATCH_PROCESSES}-1))"
+ ;;
+ start)
+ start_exchanges $2
+ ;;
+ stop)
+ stop_exchanges $2
+ ;;
+esac
exit 0
diff --git a/experiment/scripts/helpers.sh b/experiment/scripts/helpers.sh
index d6a05c0..5557a5c 100755
--- a/experiment/scripts/helpers.sh
+++ b/experiment/scripts/helpers.sh
@@ -82,6 +82,28 @@ function restart_rsyslog() {
systemctl restart rsyslog
}
+# Stop services which have the form name@number.service
+# will stop the services starting from the biggest numbers
+# Expects the following arguments:
+# $1: service base name, e.g. taler-exchange-httpd
+# $2: amount of services to stop
+function stop_numbered_services() {
+ # Get all running processes of the service $1,
+ # extract their number, limit by number of
+ # to stop and stop those
+ N=$(\
+ systemctl status "${1}"@*.service | \
+ grep -E "${1}@[0-9]+.service -" | \
+ awk '{print $2}' | \
+ sed 's/[^0-9]*//g' | \
+ sort -n -r | \
+ head -n $2 \
+ )
+ for i in ${N}; do
+ systemctl stop ${1}@${i}.service
+ done
+}
+
function get_wallet_domains() {
IFS=$'\n' read -r -d '' -a WALLETS < <(\
dig -t AXFR "${DNS_ZONE}" "@${DNS_HOSTS}" \
diff --git a/experiment/scripts/monitor.sh b/experiment/scripts/monitor.sh
index 8d9dd56..9992827 100755
--- a/experiment/scripts/monitor.sh
+++ b/experiment/scripts/monitor.sh
@@ -89,7 +89,21 @@ function add_exchanges_to_prometheus() {
fi
}
-function init() {
+function remove_exchanges_from_prometheus() {
+ TO_STOP=$(\
+ get_exchanges | \
+ tr " " "\n" | \
+ sort -t ":" -k 2 -n -r | \
+ head -n ${1}\
+ )
+
+ for EXCH in ${TO_STOP}; do
+ sed -i "/${EXCH}/d" \
+ /etc/monitor/prometheus.yaml
+ done
+}
+
+function init_monitor() {
update_grafana
@@ -109,15 +123,22 @@ function init() {
add_exchanges_to_prometheus "init"
systemctl restart prometheus-nginx-exporter \
- prometheus-postgres-exporter
+ prometheus-postgres-exporter \
+ prometheus
}
-if [[ "$1" == "init" ]]; then
- init
-else
- add_wallet_nodes_to_prometheus ""
- add_exchanges_to_prometheus ""
-fi
-
-systemctl restart prometheus
+case $1 in
+ init)
+ init_monitor
+ ;;
+ start)
+ add_wallet_nodes_to_prometheus ""
+ add_exchanges_to_prometheus ""
+ ;;
+ stop-exchanges)
+ remove_exchanges_from_prometheus $2
+ ;;
+esac
+
+systemctl reload prometheus
diff --git a/experiment/scripts/proxy.sh b/experiment/scripts/proxy.sh
index 5e04919..f6543c1 100755
--- a/experiment/scripts/proxy.sh
+++ b/experiment/scripts/proxy.sh
@@ -1,28 +1,26 @@
#!/bin/bash
set -eux
-if [[ "$1" == "init" ]];
-then
source ~/scripts/helpers.sh
- sed -e "/<SERVERS_HERE>/a \ \ server ${EXCHANGE_DOMAIN}:80;" \
- -e "s/<MONITOR_HOST_HERE>/${MONITOR_DOMAIN}/g" \
- /etc/nginx/sites-available/proxy > /etc/nginx/sites-enabled/proxy
+function setup_config() {
+ sed "/<SERVERS_HERE>/a \ \ server ${EXCHANGE_DOMAIN}:80;" \
+ /etc/nginx/sites-available/proxy > /etc/nginx/sites-enabled/proxy
# We want n processes, one is already enabled on port 80
- for (( i=1; i < ${NUM_EXCHANGE_PROCESSES}; i++ ))
- do
+ for i in $(seq $((${NUM_EXCHANGE_PROCESSES:-10}-1)) ); do
let "PORT=i+10000"
sed -i "/<SERVERS_HERE>/a \ \ server ${EXCHANGE_DOMAIN}:${PORT};" \
/etc/nginx/sites-enabled/proxy
done
+ # Nginx will log to our rsyslog directly an then from there it will be
+ # redirected to promtail - there was an issue doing it directly that's
+ # why it is done this way
sed -i -e '/module(load="imudp")/s/^#//g' \
-e '/input(type="imudp" port="514")/s/^#//g' \
/etc/rsyslog.conf
- systemctl restart rsyslog
-
echo "
fs.file-max=50000
" >> /etc/sysctl.conf
@@ -33,23 +31,55 @@ then
" >> /etc/security/limits.conf
sysctl -p
+}
+
+function init_proxy() {
+ setup_config
+
+ restart_rsyslog
# Nginx does not start until the destination server is reachable - wait here
# nginx: [emerg] host not found in upstream "exch.perf.taler" ...
wait_for_keys "${EXCHANGE_DOMAIN}"
systemctl restart nginx
-else
+}
+function add_exchanges() {
ADDED=$(grep -r " server ${EXCHANGE_DOMAIN}:"
/etc/nginx/sites-enabled/proxy | wc -l)
- for i in $(seq $1); do
+ for i in $(seq ${1}); do
let "i+=${ADDED}-1"
let "i+=10000"
sed -i "/<SERVERS_HERE>/a \ \ server ${EXCHANGE_DOMAIN}:${i};" \
- /etc/nginx/sites-enabled/proxy
+ /etc/nginx/sites-enabled/proxy
+ done
+}
+
+function remove_exchanges() {
+ TO_STOP=$(\
+ get_exchanges | \
+ tr " " "\n" | \
+ sort -t ":" -k 2 -n -r | \
+ head -n ${1}\
+ )
+
+ for EXCH in ${TO_STOP}; do
+ sed -i "/${EXCH};/d" \
+ /etc/nginx/sites-enabled/proxy
done
+}
-fi
+case $1 in
+ init)
+ init_proxy
+ ;;
+ start)
+ add_exchanges $2
+ ;;
+ stop)
+ remove_exchanges $2
+ ;;
+esac
systemctl reload nginx
diff --git a/experiment/scripts/taler-perf.sh b/experiment/scripts/taler-perf.sh
index 253038a..8171e0b 100644
--- a/experiment/scripts/taler-perf.sh
+++ b/experiment/scripts/taler-perf.sh
@@ -6,7 +6,8 @@ source ~/scripts/helpers.sh
function update_processes() {
case "$1" in
prometheus)
- ssh -A -o StrictHostKeyChecking=no ${MONITOR_DOMAIN} "/bin/bash -c
/root/scripts/monitor.sh"
+ ssh -A -o StrictHostKeyChecking=no ${MONITOR_DOMAIN} \
+ "/bin/bash /root/scripts/monitor.sh start"
;;
*)
echo "Unknown argument '$1' for function ${FUNCNAME[0]}"
@@ -33,12 +34,24 @@ function stop_wallets() {
function start_exchanges() {
ssh -o StrictHostKeyChecking=no ${EXCHANGE_DOMAIN} \
- "/bin/bash /root/scripts/exchange.sh ${1:-${NUM_EXCHANGE_PROCESSES}}"
+ "/bin/bash /root/scripts/exchange.sh start
${1:-${NUM_EXCHANGE_PROCESSES}}"
ssh -o StrictHostKeyChecking=no ${PROXY_DOMAIN} \
- "/bin/bash /root/scripts/proxy.sh ${1:-${NUM_EXCHANGE_PROCESSES}}"
+ "/bin/bash /root/scripts/proxy.sh start
${1:-${NUM_EXCHANGE_PROCESSES}}"
update_processes "prometheus"
}
+function stop_exchanges() {
+ # must remove the exchange form the monitor host before the one from the
nginx
+ # since helpers.sh gets all exchanges from the nginx config
+ ssh -A -o StrictHostKeyChecking=no ${MONITOR_DOMAIN} \
+ "/bin/bash /root/scripts/monitor.sh stop-exchanges
${1:-${NUM_EXCHANGE_PROCESSES}}"
+ ssh -A -o StrictHostKeyChecking=no ${PROXY_DOMAIN} \
+ "/bin/bash /root/scripts/proxy.sh stop
${1:-${NUM_EXCHANGE_PROCESSES}}"
+ sleep 5
+ ssh -o StrictHostKeyChecking=no ${EXCHANGE_DOMAIN} \
+ "/bin/bash /root/scripts/exchange.sh stop
${1:-${NUM_EXCHANGE_PROCESSES}}"
+}
+
function start_processes() {
case "$1" in
wallet)
@@ -59,6 +72,9 @@ function stop_processes() {
wallet)
stop_wallets $2
;;
+ exchange)
+ stop_exchanges $2
+ ;;
*)
echo "Unknown argument '$1' for function ${FUNCNAME[0]}"
echo "Usage: stop [wallet] NUM"
@@ -66,6 +82,39 @@ function stop_processes() {
esac
}
+function rebuild() {
+ while [[ $# -gt 0 ]]; do
+ case "$1" in
+ --exchange|-e)
+ EXCHANGE_COMMIT_SHA=$2
+ shift 2
+ ;;
+ --gnunet|-g)
+ GNUNET_COMMIT_SHA=$2
+ shift 2
+ ;;
+ --wallet|-w)
+ WALLET_COMMIT_SHA=$2
+ shift 2
+ ;;
+ --merchant|-m)
+ MERCHANT_COMMIT_SHA=$2
+ shift 2
+ ;;
+ *)
+ echo "Unkown argument $1"
+ echo "Usage rebuilt [target]"
+ echo "Targets: "
+ echo "-e|--exchange <commit-sha>"
+ echo "-g|--gnunet <commit-sha>"
+ echo "-w|--wallet <commit-sha>"
+ echo "-m|--merchant <commit-sha>"
+ exit 1
+ esac
+ done
+ # TODO
+}
+
case "$1" in
start)
shift
@@ -78,6 +127,9 @@ case "$1" in
update)
shift
update_processes $@
+ rebuild)
+ shift
+ rebuild
;;
esac
diff --git a/experiment/scripts/wallet.sh b/experiment/scripts/wallet.sh
index aa38e98..29df965 100755
--- a/experiment/scripts/wallet.sh
+++ b/experiment/scripts/wallet.sh
@@ -1,10 +1,9 @@
#!/bin/bash
set -eux
-NUM_PROCESSES=${NUM_WALLET_PROCESSES:-10}
+source ~/scripts/helpers.sh
function init_wallets() {
- source ~/scripts/helpers.sh
restart_rsyslog
wait_for_keys "${PROXY_DOMAIN}"
sleep 5
@@ -29,33 +28,21 @@ function start_wallets() {
done
}
-function stop_wallet() {
+function stop_wallets() {
if [[ "$1" == "all" ]]; then
systemctl stop taler-wallet@*.service
else
- # Get all running wallet processes,
- # extract their number, limit by number of
- # wallets to stop and stop those
- N=$(\
- systemctl status taler-wallet@*.service | \
- grep -E "taler-wallet@[0-9]+.service -" | \
- sed 's/[^0-9]*//g' | \
- sort -n -r | \
- head -n $1 \
- )
- for i in ${N}; do
- systemctl stop taler-wallet@${i}.service
- done
+ stop_numbered_services "taler-wallet" $1
fi
}
case "$1" in
init)
init_wallets
- start_wallets $NUM_PROCESSES
+ start_wallets ${NUM_WALLET_PROCESSES:-10}
;;
stop)
- stop_wallet $2
+ stop_wallets $2
;;
start)
start_wallets $2
--
To stop receiving notification emails like this one, please contact
gnunet@gnunet.org.
- [taler-grid5k] 54/73: enable huge_pages, implement exchange scrape logic, (continued)
- [taler-grid5k] 54/73: enable huge_pages, implement exchange scrape logic, gnunet, 2021/12/14
- [taler-grid5k] 48/73: update install script, gnunet, 2021/12/14
- [taler-grid5k] 37/73: bootstrap seems to work with sudo, gnunet, 2021/12/14
- [taler-grid5k] 47/73: fix stupid bug and fix arguments for bank, gnunet, 2021/12/14
- [taler-grid5k] 55/73: db disk mount, gnunet, 2021/12/14
- [taler-grid5k] 42/73: fix setup.sh, gnunet, 2021/12/14
- [taler-grid5k] 50/73: note about postgres huge pages, gnunet, 2021/12/14
- [taler-grid5k] 39/73: fix typo, gnunet, 2021/12/14
- [taler-grid5k] 60/73: fix logbackup so it gets rotated correctly, gnunet, 2021/12/14
- [taler-grid5k] 57/73: improve wallet script, gnunet, 2021/12/14
- [taler-grid5k] 58/73: sanitize scripts,
gnunet <=
- [taler-grid5k] 36/73: Update image build, add install script to install taler binaries in running experiment, gnunet, 2021/12/14
- [taler-grid5k] 63/73: fix, gnunet, 2021/12/14
- [taler-grid5k] 46/73: remove previous builds, gnunet, 2021/12/14
- [taler-grid5k] 62/73: fix monitor, gnunet, 2021/12/14
- [taler-grid5k] 49/73: monitor dns too, change db settings, gnunet, 2021/12/14
- [taler-grid5k] 59/73: fix taler-perf.sh, improve other scripts, gnunet, 2021/12/14
- [taler-grid5k] 56/73: fix startup, gnunet, 2021/12/14
- [taler-grid5k] 45/73: take postgres config to seperate file, gnunet, 2021/12/14
- [taler-grid5k] 38/73: speedup image build, gnunet, 2021/12/14
- [taler-grid5k] 40/73: document variables, gnunet, 2021/12/14