[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]
[taler-grid5k] 24/73: cleanup with functions - try work_mem and idle_in_
From: |
gnunet |
Subject: |
[taler-grid5k] 24/73: cleanup with functions - try work_mem and idle_in_transaction |
Date: |
Tue, 14 Dec 2021 15:10:06 +0100 |
This is an automated email from the git hooks/post-receive script.
marco-boss pushed a commit to branch master
in repository grid5k.
commit 01793c1134d093c4475a086de12911131a470951
Author: Boss Marco <bossm8@bfh.ch>
AuthorDate: Tue Nov 30 10:16:25 2021 +0100
cleanup with functions - try work_mem and idle_in_transaction
---
experiment/scripts/database.sh | 16 +++++++--
experiment/scripts/exchange.sh | 30 +++++++++++------
experiment/scripts/monitor.sh | 76 +++++++++++++++++++++++-------------------
3 files changed, 74 insertions(+), 48 deletions(-)
diff --git a/experiment/scripts/database.sh b/experiment/scripts/database.sh
index 69503c0..fd20388 100755
--- a/experiment/scripts/database.sh
+++ b/experiment/scripts/database.sh
@@ -6,8 +6,7 @@ source ~/scripts/helpers.sh
# move to tmp to prevent change dir errors
cd /tmp
-if [[ "$1" == "init" ]];
-then
+function setup_config() {
sed -i "s\<DB_URL_HERE>\postgresql:///${DB_NAME}\g" \
/etc/taler/secrets/exchange-db.secret.conf
@@ -22,14 +21,21 @@ then
# (https://www.postgresql.org/docs/13/runtime-config-resource.html)
shared_buffers=$(($(awk '/MemTotal/ {print $2}' /proc/meminfo) / 4 ))kB
effective_cache_size=$(($(awk '/MemTotal/ {print $2}' /proc/meminfo) *
3/4))kB
+
#
(https://www.postgresql.org/docs/current/runtime-config-wal.html#GUC-MAX-WAL-SIZE)
max_wal_size=2GB
wal_buffers=16MB
+
max_worker_processes=$(lscpu | grep "CPU(s)" | head -n 1 | awk '{print $2}')
max_connections=200
+
# out of shared memory
max_locks_per_transaction=85
- # idle_in_transaction_session_timeout=5000
+
+ # Increase work mem to lower I/O utilization (max used =~ work_mem *
max_connections)
+ # NOTE: This formula is not completely correct
+ work_mem=128MB
+ idle_in_transaction_session_timeout=3000
" >> /etc/postgresql/13/main/postgresql.conf
# Enable password for taler since this is commonly the case
@@ -38,6 +44,10 @@ then
host all ${DB_USER} 172.16.0.0/12 md5
host all postgres 172.16.0.0/12 trust
" >> /etc/postgresql/13/main/pg_hba.conf
+}
+
+if [[ "$1" == "init" ]]; then
+ setup_config
fi
systemctl restart postgresql
diff --git a/experiment/scripts/exchange.sh b/experiment/scripts/exchange.sh
index 0094231..bb37d69 100755
--- a/experiment/scripts/exchange.sh
+++ b/experiment/scripts/exchange.sh
@@ -1,11 +1,7 @@
#!/bin/bash
set -eux
-if [[ "$1" == "init" ]];
-then
- source ~/scripts/helpers.sh
- restart_rsyslog
-
+function setup_config() {
sed -i
"s\<DB_URL_HERE>\postgresql://${DB_USER}:${DB_PASSWORD}@${DATABASE_DOMAIN}/${DB_NAME}\g"
\
/etc/taler/secrets/exchange-db.secret.conf
@@ -18,13 +14,9 @@ then
-e "s\<BASE_URL_HERE>\http://${EXCHANGE_DOMAIN}/\g" \
-e "s/<MASTER_KEY_HERE>/${MASTER_KEY}/g" \
/etc/taler/conf.d/exchange-business.conf
-
- wait_for_db
- # Wait another second to make sure user has permissions
- sleep 5
-
- NUM_PROCESSES=$((${NUM_EXCHANGE_PROCESSES:-10}-1))
+}
+function setup_exchange() {
systemctl restart taler-exchange.target
wait_for_keys "${EXCHANGE_DOMAIN}/management"
@@ -38,6 +30,22 @@ then
taler-exchange-offline upload < sig-res.json
taler-exchange-offline upload < acct-res.json
taler-exchange-offline upload < fee-res.json
+}
+
+if [[ "$1" == "init" ]];
+then
+ source ~/scripts/helpers.sh
+ restart_rsyslog
+
+ setup_config
+
+ wait_for_db
+ # Wait another second to make sure user has permissions
+ sleep 5
+
+ NUM_PROCESSES=$((${NUM_EXCHANGE_PROCESSES:-10}-1))
+
+ setup_exchange
# || true is needed since when 1-1 is run let returns an error
let "NUM_WIREWATCH_PROCESSES-=1" || true
diff --git a/experiment/scripts/monitor.sh b/experiment/scripts/monitor.sh
index cab6045..034f78e 100755
--- a/experiment/scripts/monitor.sh
+++ b/experiment/scripts/monitor.sh
@@ -17,7 +17,7 @@ function update_datasource() {
}
function update_grafana() {
- if [ -z "${GRAFANA_HOST}" || -z ${GRAFANA_API_KEY} ]; then
+ if [ -z "${GRAFANA_HOST}" ] || [ -z ${GRAFANA_API_KEY} ]; then
return
fi
AUTH_HEADER="Authorization: Bearer ${GRAFANA_API_KEY}"
@@ -34,15 +34,8 @@ function update_grafana() {
update_datasource "${PROMETHEUS_DATASOURCE_NAME}"
"${PROMETHEUS_G5K_PROXY_PORT}"
update_datasource "${LOKI_DATASOURCE_NAME}" "${LOKI_G5K_PROXY_PORT}"
}
-
-if [[ "$1" == "init" ]];
-then
- update_grafana
-
- systemctl restart loki \
- promtail
-
+function configure_prometheus_and_exporters() {
sed -i "s/<MERCHANT_HOST_HERE>/${MERCHANT_DOMAIN}/g" \
/etc/monitor/prometheus.yaml
@@ -51,39 +44,54 @@ then
sed -i "s\<PROXY_URL_HERE>\http://${PROXY_DOMAIN}/stub_status\g" \
/etc/default/prometheus-nginx-exporter
-
- wait_for_db
-
- # Initialize prometheus after the db is ready, then all dns records have
been set for sure
- if [[ "${ENABLE_EXPORTERS}" == "true" ]];
- then
- cat /etc/monitor/node-exporters.yaml.tpl | envsubst >>
/etc/monitor/prometheus.yaml
- for WALLET in $(get_wallet_domains);
- do
- sed -i "/<WALLETS_HERE>/a \ \ \ \ \ \ -
'${WALLET_DOMAIN//\*/${WALLET}}:9100'" \
- /etc/monitor/prometheus.yaml
- done
- fi
-
- # Proxy takes longer to start
- wait_for_keys "${PROXY_DOMAIN}"
-
- systemctl restart prometheus-postgres-exporter \
- prometheus-nginx-exporter
-else
+}
+
+
+function add_wallet_nodes_to_prometheus() {
+ if [[ "${ENABLE_EXPORTERS}" == "true" ]]; then
+
+ if [[ "$1" == "init" ]]; then
+ cat /etc/monitor/node-exporters.yaml.tpl | envsubst >>
/etc/monitor/prometheus.yaml
+ fi
+
+ for WALLET in $(get_wallet_domains); do
- if [[ "${ENABLE_EXPORTERS}" == "true" ]];
- then
- for WALLET in $(get_wallet_domains);
- do
if ! grep -q "${WALLET_DOMAIN//\*/${WALLET}}:9100"
/etc/monitor/prometheus.yaml;
then
- sed -i "/<WALLETS_HERE>/a \ \ \ \ \ \ -
'wallet.${WALLET}.${DNS_ZONE}:9100'" \
+ sed -i "/<WALLETS_HERE>/a \ \ \ \ \ \ -
'${WALLET_DOMAIN//\*/${WALLET}}:9100'" \
/etc/monitor/prometheus.yaml
fi
+
done
+
fi
+}
+function init() {
+
+ update_grafana
+
+ systemctl restart loki \
+ promtail
+
+ configure_prometheus_and_exporters
+
+ wait_for_db
+
+ add_wallet_nodes_to_prometheus "init"
+
+ # Proxy takes longer to start
+ wait_for_keys "${PROXY_DOMAIN}"
+
+ systemctl restart prometheus-nginx-exporter
+ # prometheus-postgres-exporter
+
+}
+
+if [[ "$1" == "init" ]]; then
+ init
+else
+ add_wallet_nodes_to_prometheus ""
fi
systemctl restart prometheus
--
To stop receiving notification emails like this one, please contact
gnunet@gnunet.org.
- [taler-grid5k] 26/73: add pgbouncer config, (continued)
- [taler-grid5k] 26/73: add pgbouncer config, gnunet, 2021/12/14
- [taler-grid5k] 18/73: wallet core still not building in one step, gnunet, 2021/12/14
- [taler-grid5k] 28/73: increase pgbouncer max connections, gnunet, 2021/12/14
- [taler-grid5k] 31/73: pgbouncer still occupying 6432 even when stopped, gnunet, 2021/12/14
- [taler-grid5k] 35/73: some more pgbouncer config, gnunet, 2021/12/14
- [taler-grid5k] 27/73: rename userlist, gnunet, 2021/12/14
- [taler-grid5k] 20/73: issue in logbackup, gnunet, 2021/12/14
- [taler-grid5k] 19/73: running make two times seems to work, gnunet, 2021/12/14
- [taler-grid5k] 10/73: update service files, gnunet, 2021/12/14
- [taler-grid5k] 34/73: exporter for merchant now on monitor too, gnunet, 2021/12/14
- [taler-grid5k] 24/73: cleanup with functions - try work_mem and idle_in_transaction,
gnunet <=
- [taler-grid5k] 15/73: build wallet from source since dist does not work, gnunet, 2021/12/14
- [taler-grid5k] 25/73: enable postgres-exporter again, gnunet, 2021/12/14
- [taler-grid5k] 29/73: switch to transaction mode since otherwise its all the same as when pgbouncer not used at all, gnunet, 2021/12/14
- [taler-grid5k] 23/73: disable pgbouncer per default, gnunet, 2021/12/14
- [taler-grid5k] 52/73: possible to configure max_requests with env file, gnunet, 2021/12/14
- [taler-grid5k] 22/73: add pgbouncer, tune pg config, gnunet, 2021/12/14
- [taler-grid5k] 16/73: fix wallet build - broken on master, gnunet, 2021/12/14
- [taler-grid5k] 61/73: refactor db.sh, gnunet, 2021/12/14
- [taler-grid5k] 30/73: increase default pool size, gnunet, 2021/12/14
- [taler-grid5k] 43/73: update config of db and exch, gnunet, 2021/12/14