mirror of
https://github.com/fscotto/infra.git
synced 2026-10-03 21:39:50 +00:00
7bc7f0e645
* Stage Prometheus NPM Quadlet with backup-safe cutover * Complete Prometheus NPM Quadlet cutover
107 lines
3.7 KiB
Bash
107 lines
3.7 KiB
Bash
#!/usr/bin/env bash
|
|
# Run on Prometheus as root with the exact verified source-export version.
|
|
set -Eeuo pipefail
|
|
|
|
expected_export=${1:?Pass the verified Prometheus backup export version}
|
|
mode=${2:---preflight}
|
|
[[ $expected_export =~ ^[0-9]{8}T[0-9]{6}Z$ ]] || exit 2
|
|
[[ $mode == --preflight || $mode == --execute ]] || exit 2
|
|
[[ $EUID -eq 0 ]] || { echo 'Run as root on Prometheus' >&2; exit 2; }
|
|
|
|
compose_unit=podman-compose-server.service
|
|
quadlet_unit=prometheus-npm.service
|
|
backup_timer=prometheus-backup-export.timer
|
|
versions=/var/lib/prometheus-backup-export/versions
|
|
quadlet_file=/etc/containers/systemd/prometheus-npm.container
|
|
|
|
exec 9>/run/lock/prometheus-backup-export.lock
|
|
flock -n 9 || { echo 'Backup/export lock is busy' >&2; exit 1; }
|
|
|
|
systemctl is-active --quiet "$compose_unit"
|
|
if systemctl is-active --quiet "$quadlet_unit"; then
|
|
echo 'NPM Quadlet is already active; refusing overlapping cutover' >&2
|
|
exit 1
|
|
fi
|
|
[[ $(systemctl show "$quadlet_unit" -p LoadState --value) == loaded ]]
|
|
[[ $(systemctl is-enabled "$compose_unit") == enabled ]]
|
|
[[ $(readlink "$versions/current") == "$expected_export" ]]
|
|
image=$(sed -n 's/^Image=//p' "$quadlet_file")
|
|
[[ $image =~ ^docker\.io/jc21/nginx-proxy-manager@sha256:[a-f0-9]{64}$ ]]
|
|
podman image exists "$image"
|
|
(cd "$versions/current" && sha256sum -c payload.sha256 && tar -tf payload.tar >/dev/null)
|
|
curl -fsS --connect-timeout 2 --max-time 5 -o /dev/null http://127.0.0.1:81/
|
|
old_image=$(podman inspect nginx-proxy-manager --format '{{.Image}}')
|
|
|
|
data_signature() {
|
|
python3 - <<'PY'
|
|
import hashlib, os, sqlite3
|
|
db = sqlite3.connect('file:/opt/npm/data/database.sqlite?mode=ro', uri=True)
|
|
assert db.execute('pragma quick_check').fetchone()[0] == 'ok'
|
|
counts = [db.execute('select count(*) from ' + table).fetchone()[0]
|
|
for table in ('proxy_host', 'certificate', 'user')]
|
|
db.close()
|
|
digest = hashlib.sha256()
|
|
for root, dirs, files in os.walk('/opt/npm/letsencrypt'):
|
|
dirs.sort()
|
|
for name in sorted(files):
|
|
path = os.path.join(root, name)
|
|
with open(path, 'rb') as stream:
|
|
digest.update(path.encode() + b'\0' + stream.read())
|
|
print(*counts, digest.hexdigest())
|
|
PY
|
|
}
|
|
before=$(data_signature)
|
|
if [[ $mode == --preflight ]]; then
|
|
echo 'NPM Quadlet cutover preflight passed; no service was changed'
|
|
exit 0
|
|
fi
|
|
|
|
stopped_old=false
|
|
rollback() {
|
|
rc=$?
|
|
trap - EXIT
|
|
if (( rc != 0 )) && "$stopped_old"; then
|
|
echo 'NPM Quadlet cutover failed; restoring Compose' >&2
|
|
systemctl stop "$quadlet_unit" || true
|
|
systemctl enable "$compose_unit" || true
|
|
systemctl start "$compose_unit" || true
|
|
systemctl start "$backup_timer" || true
|
|
curl -fsS --connect-timeout 2 --max-time 10 -o /dev/null http://127.0.0.1:81/ || true
|
|
fi
|
|
exit "$rc"
|
|
}
|
|
trap rollback EXIT
|
|
|
|
stopped_old=true
|
|
systemctl stop "$backup_timer"
|
|
systemctl stop "$compose_unit"
|
|
if podman container exists nginx-proxy-manager; then
|
|
echo 'Compose left the NPM container behind; refusing duplicate ownership' >&2
|
|
exit 1
|
|
fi
|
|
systemctl disable "$compose_unit"
|
|
systemctl start "$quadlet_unit"
|
|
|
|
ready=false
|
|
for _ in {1..60}; do
|
|
if curl -fsS --connect-timeout 2 --max-time 3 -o /dev/null http://127.0.0.1:81/; then
|
|
ready=true
|
|
break
|
|
fi
|
|
sleep 2
|
|
done
|
|
"$ready"
|
|
systemctl is-active --quiet "$quadlet_unit"
|
|
[[ $(podman inspect nginx-proxy-manager --format '{{.Image}}') == "$old_image" ]]
|
|
podman exec nginx-proxy-manager nginx -t
|
|
[[ $(data_signature) == "$before" ]]
|
|
for hostname in git.fscotto.duckdns.org syncthing.fscotto.duckdns.org; do
|
|
status=$(curl -ksS --connect-timeout 3 --max-time 10 \
|
|
--resolve "$hostname:443:127.0.0.1" -o /dev/null -w '%{http_code}' \
|
|
"https://$hostname/")
|
|
[[ $status == 200 ]]
|
|
done
|
|
systemctl start "$backup_timer"
|
|
stopped_old=false
|
|
echo 'NPM Quadlet cutover passed local application and data checks'
|