mirror of
https://github.com/fscotto/infra.git
synced 2026-10-03 13:29:58 +00:00
Feature/prometheus npm quadlet (#15)
* Stage Prometheus NPM Quadlet with backup-safe cutover * Complete Prometheus NPM Quadlet cutover
This commit is contained in:
committed by
GitHub
parent
1577eec19d
commit
7bc7f0e645
106
scripts/cutover_prometheus_npm_quadlet.sh
Normal file
106
scripts/cutover_prometheus_npm_quadlet.sh
Normal file
@@ -0,0 +1,106 @@
|
||||
#!/usr/bin/env bash
|
||||
# Run on Prometheus as root with the exact verified source-export version.
|
||||
set -Eeuo pipefail
|
||||
|
||||
expected_export=${1:?Pass the verified Prometheus backup export version}
|
||||
mode=${2:---preflight}
|
||||
[[ $expected_export =~ ^[0-9]{8}T[0-9]{6}Z$ ]] || exit 2
|
||||
[[ $mode == --preflight || $mode == --execute ]] || exit 2
|
||||
[[ $EUID -eq 0 ]] || { echo 'Run as root on Prometheus' >&2; exit 2; }
|
||||
|
||||
compose_unit=podman-compose-server.service
|
||||
quadlet_unit=prometheus-npm.service
|
||||
backup_timer=prometheus-backup-export.timer
|
||||
versions=/var/lib/prometheus-backup-export/versions
|
||||
quadlet_file=/etc/containers/systemd/prometheus-npm.container
|
||||
|
||||
exec 9>/run/lock/prometheus-backup-export.lock
|
||||
flock -n 9 || { echo 'Backup/export lock is busy' >&2; exit 1; }
|
||||
|
||||
systemctl is-active --quiet "$compose_unit"
|
||||
if systemctl is-active --quiet "$quadlet_unit"; then
|
||||
echo 'NPM Quadlet is already active; refusing overlapping cutover' >&2
|
||||
exit 1
|
||||
fi
|
||||
[[ $(systemctl show "$quadlet_unit" -p LoadState --value) == loaded ]]
|
||||
[[ $(systemctl is-enabled "$compose_unit") == enabled ]]
|
||||
[[ $(readlink "$versions/current") == "$expected_export" ]]
|
||||
image=$(sed -n 's/^Image=//p' "$quadlet_file")
|
||||
[[ $image =~ ^docker\.io/jc21/nginx-proxy-manager@sha256:[a-f0-9]{64}$ ]]
|
||||
podman image exists "$image"
|
||||
(cd "$versions/current" && sha256sum -c payload.sha256 && tar -tf payload.tar >/dev/null)
|
||||
curl -fsS --connect-timeout 2 --max-time 5 -o /dev/null http://127.0.0.1:81/
|
||||
old_image=$(podman inspect nginx-proxy-manager --format '{{.Image}}')
|
||||
|
||||
data_signature() {
|
||||
python3 - <<'PY'
|
||||
import hashlib, os, sqlite3
|
||||
db = sqlite3.connect('file:/opt/npm/data/database.sqlite?mode=ro', uri=True)
|
||||
assert db.execute('pragma quick_check').fetchone()[0] == 'ok'
|
||||
counts = [db.execute('select count(*) from ' + table).fetchone()[0]
|
||||
for table in ('proxy_host', 'certificate', 'user')]
|
||||
db.close()
|
||||
digest = hashlib.sha256()
|
||||
for root, dirs, files in os.walk('/opt/npm/letsencrypt'):
|
||||
dirs.sort()
|
||||
for name in sorted(files):
|
||||
path = os.path.join(root, name)
|
||||
with open(path, 'rb') as stream:
|
||||
digest.update(path.encode() + b'\0' + stream.read())
|
||||
print(*counts, digest.hexdigest())
|
||||
PY
|
||||
}
|
||||
before=$(data_signature)
|
||||
if [[ $mode == --preflight ]]; then
|
||||
echo 'NPM Quadlet cutover preflight passed; no service was changed'
|
||||
exit 0
|
||||
fi
|
||||
|
||||
stopped_old=false
|
||||
rollback() {
|
||||
rc=$?
|
||||
trap - EXIT
|
||||
if (( rc != 0 )) && "$stopped_old"; then
|
||||
echo 'NPM Quadlet cutover failed; restoring Compose' >&2
|
||||
systemctl stop "$quadlet_unit" || true
|
||||
systemctl enable "$compose_unit" || true
|
||||
systemctl start "$compose_unit" || true
|
||||
systemctl start "$backup_timer" || true
|
||||
curl -fsS --connect-timeout 2 --max-time 10 -o /dev/null http://127.0.0.1:81/ || true
|
||||
fi
|
||||
exit "$rc"
|
||||
}
|
||||
trap rollback EXIT
|
||||
|
||||
stopped_old=true
|
||||
systemctl stop "$backup_timer"
|
||||
systemctl stop "$compose_unit"
|
||||
if podman container exists nginx-proxy-manager; then
|
||||
echo 'Compose left the NPM container behind; refusing duplicate ownership' >&2
|
||||
exit 1
|
||||
fi
|
||||
systemctl disable "$compose_unit"
|
||||
systemctl start "$quadlet_unit"
|
||||
|
||||
ready=false
|
||||
for _ in {1..60}; do
|
||||
if curl -fsS --connect-timeout 2 --max-time 3 -o /dev/null http://127.0.0.1:81/; then
|
||||
ready=true
|
||||
break
|
||||
fi
|
||||
sleep 2
|
||||
done
|
||||
"$ready"
|
||||
systemctl is-active --quiet "$quadlet_unit"
|
||||
[[ $(podman inspect nginx-proxy-manager --format '{{.Image}}') == "$old_image" ]]
|
||||
podman exec nginx-proxy-manager nginx -t
|
||||
[[ $(data_signature) == "$before" ]]
|
||||
for hostname in git.fscotto.duckdns.org syncthing.fscotto.duckdns.org; do
|
||||
status=$(curl -ksS --connect-timeout 3 --max-time 10 \
|
||||
--resolve "$hostname:443:127.0.0.1" -o /dev/null -w '%{http_code}' \
|
||||
"https://$hostname/")
|
||||
[[ $status == 200 ]]
|
||||
done
|
||||
systemctl start "$backup_timer"
|
||||
stopped_old=false
|
||||
echo 'NPM Quadlet cutover passed local application and data checks'
|
||||
Reference in New Issue
Block a user