mirror of
https://github.com/fscotto/infra.git
synced 2026-10-03 21:39:50 +00:00
76 lines
2.9 KiB
Django/Jinja
76 lines
2.9 KiB
Django/Jinja
#!/usr/bin/env bash
|
|
set -Eeuo pipefail
|
|
umask 077
|
|
|
|
backup_root={{ atlas_backup_prometheus_mountpoint | quote }}
|
|
snapshots="$backup_root/snapshots"
|
|
stage=''
|
|
exec 9>/run/lock/atlas-prometheus-pull.lock
|
|
flock -n 9 || { echo 'A Prometheus pull is already running' >&2; exit 1; }
|
|
|
|
cleanup() {
|
|
local rc=$?
|
|
trap - EXIT
|
|
if (( rc != 0 )) && [[ -n "$stage" && -d "$stage" ]]; then
|
|
rm -rf -- "$stage"
|
|
fi
|
|
exit "$rc"
|
|
}
|
|
trap cleanup EXIT
|
|
|
|
zpool list -H -o name {{ atlas_zfs_pool | quote }} >/dev/null
|
|
findmnt -rn --mountpoint "$backup_root" >/dev/null
|
|
stage=$(mktemp -d "$backup_root/.staging.XXXXXXXX")
|
|
ssh_cmd='/usr/bin/ssh -F /dev/null -o BatchMode=yes -o StrictHostKeyChecking=yes -o UserKnownHostsFile={{ atlas_prometheus_pull_known_hosts_path }} -o IdentitiesOnly=yes -i {{ atlas_prometheus_pull_private_key_path }} -p {{ atlas_prometheus_pull_source_port }}'
|
|
rsync -a --partial --delay-updates -e "$ssh_cmd" \
|
|
{{ (atlas_prometheus_pull_source_user ~ '@' ~ hostvars['prometheus'].ansible_host ~ ':current/') | quote }} \
|
|
"$stage/"
|
|
|
|
test -s "$stage/payload.tar"
|
|
test -s "$stage/payload.sha256"
|
|
test -s "$stage/metadata.json"
|
|
(cd "$stage" && sha256sum -c payload.sha256)
|
|
tar -tf "$stage/payload.tar" >/dev/null
|
|
stamp=$(python3 - "$stage/metadata.json" <<'PY'
|
|
import json
|
|
import datetime as dt
|
|
import re
|
|
import sys
|
|
|
|
with open(sys.argv[1], encoding="utf-8") as stream:
|
|
metadata = json.load(stream)
|
|
stamp = metadata.get("created_utc", "")
|
|
if metadata.get("schema") != 1 or metadata.get("host") != "prometheus":
|
|
raise SystemExit("Unexpected Prometheus backup metadata")
|
|
if not re.fullmatch(r"[0-9]{8}T[0-9]{6}Z", stamp):
|
|
raise SystemExit("Invalid Prometheus backup timestamp")
|
|
created = dt.datetime.strptime(stamp, "%Y%m%dT%H%M%SZ").replace(tzinfo=dt.timezone.utc)
|
|
age = dt.datetime.now(dt.timezone.utc) - created
|
|
if age.total_seconds() < -300 or age > dt.timedelta(hours={{ atlas_prometheus_pull_max_age_hours }}):
|
|
raise SystemExit("Prometheus backup is outside the configured freshness window")
|
|
print(stamp)
|
|
PY
|
|
)
|
|
if [[ -e "$snapshots/$stamp" ]]; then
|
|
cmp "$stage/payload.sha256" "$snapshots/$stamp/payload.sha256"
|
|
cmp "$stage/metadata.json" "$snapshots/$stamp/metadata.json"
|
|
(cd "$snapshots/$stamp" && sha256sum -c payload.sha256)
|
|
rm -rf -- "${stage:?}"
|
|
stage=''
|
|
else
|
|
chown -R root:root "$stage"
|
|
chmod 0700 "$stage"
|
|
chmod 0600 "$stage/payload.tar" "$stage/payload.sha256" "$stage/metadata.json"
|
|
mv -- "$stage" "$snapshots/$stamp"
|
|
stage=''
|
|
fi
|
|
latest_link=$(readlink "$backup_root/latest" 2>/dev/null || true)
|
|
latest_stamp=${latest_link##*/}
|
|
if [[ -z "$latest_stamp" || "$stamp" > "$latest_stamp" ]]; then
|
|
ln -s "snapshots/$stamp" "$backup_root/.latest.new"
|
|
mv -Tf -- "$backup_root/.latest.new" "$backup_root/latest"
|
|
fi
|
|
python3 /usr/local/libexec/atlas-prometheus-prune "$snapshots" \
|
|
{{ atlas_prometheus_pull_keep_daily }} {{ atlas_prometheus_pull_keep_weekly }} {{ atlas_prometheus_pull_keep_monthly }}
|
|
echo "Verified and published Prometheus backup $stamp"
|