mirror of
https://github.com/fscotto/infra.git
synced 2026-10-03 13:29:58 +00:00
Prepare consistent final Gitea export on Prometheus
This commit is contained in:
@@ -61,6 +61,8 @@ Ansible-driven personal infrastructure repo for Fedora and Void desktops, Fedora
|
|||||||
`ansible-playbook ansible/site.yml --limit atlas --tags gitea --check --diff`
|
`ansible-playbook ansible/site.yml --limit atlas --tags gitea --check --diff`
|
||||||
- Atlas explicit isolated Gitea restore rehearsal (not part of normal runs):
|
- Atlas explicit isolated Gitea restore rehearsal (not part of normal runs):
|
||||||
`ansible-playbook ansible/site.yml --limit atlas --tags gitea_restore -e atlas_gitea_restore_test=true`
|
`ansible-playbook ansible/site.yml --limit atlas --tags gitea_restore -e atlas_gitea_restore_test=true`
|
||||||
|
- Prometheus final Gitea export helper (dry-run installs only; outage action remains opt-in):
|
||||||
|
`ansible-playbook ansible/site.yml --limit prometheus --tags gitea_final_export --check --diff`
|
||||||
- Atlas network/share hardening:
|
- Atlas network/share hardening:
|
||||||
`ansible-playbook ansible/site.yml --limit atlas --tags hardening,sharing --check --diff`
|
`ansible-playbook ansible/site.yml --limit atlas --tags hardening,sharing --check --diff`
|
||||||
- Atlas ZFS snapshot retention and scrub timers:
|
- Atlas ZFS snapshot retention and scrub timers:
|
||||||
@@ -286,6 +288,10 @@ successfully. The first monthly scrub remains a runtime check.
|
|||||||
- [ ] Include the new Gitea dataset in the next UUID-bound offline USB version and test a file restore
|
- [ ] Include the new Gitea dataset in the next UUID-bound offline USB version and test a file restore
|
||||||
from that version before accepting production writes; the UUID-bound disk is connected but its
|
from that version before accepting production writes; the UUID-bound disk is connected but its
|
||||||
LUKS mapper is closed, so the manual backup still requires interactive unlock.
|
LUKS mapper is closed, so the manual backup still requires interactive unlock.
|
||||||
|
- [x] Install a separate opt-in final Gitea export helper on Prometheus. Its 2026-10-01 targeted
|
||||||
|
deployment and `bash -n` passed while Gitea and NPM stayed running. It refuses an active export
|
||||||
|
timer, stops only Gitea, verifies SQLite, publishes a checksum-verified Gitea-only version for
|
||||||
|
Atlas' existing pull, and leaves the source stopped on success; it has **not** been invoked.
|
||||||
- [ ] After an explicit outage approval, perform the final consistent copy and HTTPS/SSH cutover,
|
- [ ] After an explicit outage approval, perform the final consistent copy and HTTPS/SSH cutover,
|
||||||
then remove Gitea from Prometheus' desired stack and backup export without deleting source data.
|
then remove Gitea from Prometheus' desired stack and backup export without deleting source data.
|
||||||
- [ ] Design and deploy Nextcloud as another explicitly temporary Atlas service before Uranus. Give it
|
- [ ] Design and deploy Nextcloud as another explicitly temporary Atlas service before Uranus. Give it
|
||||||
|
|||||||
@@ -87,6 +87,9 @@ server_backup_export_root: /var/lib/prometheus-backup-export
|
|||||||
server_backup_rrsync_path: /usr/share/doc/rsync/support/rrsync
|
server_backup_rrsync_path: /usr/share/doc/rsync/support/rrsync
|
||||||
server_backup_export_calendar: "*-*-* 02:00:00 Europe/Rome"
|
server_backup_export_calendar: "*-*-* 02:00:00 Europe/Rome"
|
||||||
server_backup_export_start_timer: false
|
server_backup_export_start_timer: false
|
||||||
|
# Explicit Gitea cutover helper: installed separately from any outage action.
|
||||||
|
server_gitea_cutover_tools_enabled: false
|
||||||
|
server_gitea_final_export: false
|
||||||
server_backup_export_source_keep: 3
|
server_backup_export_source_keep: 3
|
||||||
server_backup_export_paths:
|
server_backup_export_paths:
|
||||||
- opt/npm/data
|
- opt/npm/data
|
||||||
|
|||||||
@@ -8,6 +8,8 @@ ansible_ssh_private_key_file: /home/fscotto/.ssh/id_ed25519
|
|||||||
server_username: rocky
|
server_username: rocky
|
||||||
server_backup_export_enabled: true
|
server_backup_export_enabled: true
|
||||||
server_backup_export_start_timer: true
|
server_backup_export_start_timer: true
|
||||||
|
# Install the final-copy helper only; it is never run by a normal playbook invocation.
|
||||||
|
server_gitea_cutover_tools_enabled: true
|
||||||
server_duckdns_domain: fscotto
|
server_duckdns_domain: fscotto
|
||||||
server_ssh_authorized_keys:
|
server_ssh_authorized_keys:
|
||||||
- name: ikaros
|
- name: ikaros
|
||||||
|
|||||||
34
ansible/roles/profile_server/tasks/gitea_final_export.yml
Normal file
34
ansible/roles/profile_server/tasks/gitea_final_export.yml
Normal file
@@ -0,0 +1,34 @@
|
|||||||
|
---
|
||||||
|
- name: Install the explicit Gitea final-export helper
|
||||||
|
tags: [services, gitea_final_export]
|
||||||
|
ansible.builtin.template:
|
||||||
|
src: prometheus-gitea-final-export.sh.j2
|
||||||
|
dest: /usr/local/sbin/prometheus-gitea-final-export
|
||||||
|
owner: root
|
||||||
|
group: root
|
||||||
|
mode: "0750"
|
||||||
|
when: server_gitea_cutover_tools_enabled | bool
|
||||||
|
|
||||||
|
- name: Require the prepared source and explicit final-export approval
|
||||||
|
tags: [services, gitea_final_export]
|
||||||
|
ansible.builtin.assert:
|
||||||
|
that:
|
||||||
|
- server_gitea_cutover_tools_enabled | bool
|
||||||
|
- server_backup_export_enabled | bool
|
||||||
|
- not ansible_check_mode
|
||||||
|
fail_msg: >-
|
||||||
|
Install the cutover helper and perform an explicit non-check-mode run
|
||||||
|
only after the Gitea outage gate has been approved.
|
||||||
|
when: server_gitea_final_export | bool
|
||||||
|
|
||||||
|
- name: Stop source Gitea and publish the final consistent export
|
||||||
|
tags: [services, gitea_final_export]
|
||||||
|
ansible.builtin.command:
|
||||||
|
argv:
|
||||||
|
- /usr/local/sbin/prometheus-gitea-final-export
|
||||||
|
register: server_gitea_final_export_result
|
||||||
|
changed_when: server_gitea_final_export_result.rc == 0
|
||||||
|
no_log: true
|
||||||
|
when:
|
||||||
|
- server_gitea_final_export | bool
|
||||||
|
- not ansible_check_mode
|
||||||
@@ -59,6 +59,9 @@
|
|||||||
- name: Import Prometheus backup export job tasks
|
- name: Import Prometheus backup export job tasks
|
||||||
ansible.builtin.import_tasks: backup_export_job.yml
|
ansible.builtin.import_tasks: backup_export_job.yml
|
||||||
|
|
||||||
|
- name: Import explicit Prometheus Gitea final-export tasks
|
||||||
|
ansible.builtin.import_tasks: gitea_final_export.yml
|
||||||
|
|
||||||
- name: Ensure server SSH authorized key fragments directory exists
|
- name: Ensure server SSH authorized key fragments directory exists
|
||||||
tags: [services, ssh]
|
tags: [services, ssh]
|
||||||
ansible.builtin.file:
|
ansible.builtin.file:
|
||||||
|
|||||||
@@ -0,0 +1,80 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
set -Eeuo pipefail
|
||||||
|
umask 077
|
||||||
|
|
||||||
|
export_root={{ server_backup_export_root | quote }}
|
||||||
|
versions="$export_root/versions"
|
||||||
|
stamp=$(date -u +%Y%m%dT%H%M%SZ)
|
||||||
|
stage=''
|
||||||
|
gitea_stopped=false
|
||||||
|
|
||||||
|
exec 9>/run/lock/prometheus-backup-export.lock
|
||||||
|
flock -n 9 || { echo 'A Prometheus backup export is already running' >&2; exit 1; }
|
||||||
|
|
||||||
|
cleanup() {
|
||||||
|
local rc=$?
|
||||||
|
trap - EXIT
|
||||||
|
if (( rc != 0 )) && "$gitea_stopped"; then
|
||||||
|
podman start gitea >/dev/null || rc=1
|
||||||
|
fi
|
||||||
|
if (( rc != 0 )) && [[ -n "$stage" && -d "$stage" ]]; then
|
||||||
|
rm -rf -- "$stage"
|
||||||
|
fi
|
||||||
|
exit "$rc"
|
||||||
|
}
|
||||||
|
trap cleanup EXIT
|
||||||
|
trap 'exit 129' HUP
|
||||||
|
trap 'exit 130' INT
|
||||||
|
trap 'exit 143' TERM
|
||||||
|
|
||||||
|
systemctl is-active --quiet podman-compose-server.service || {
|
||||||
|
echo 'Prometheus Compose stack is not active' >&2; exit 1;
|
||||||
|
}
|
||||||
|
if systemctl is-active --quiet prometheus-backup-export.timer; then
|
||||||
|
echo 'Stop the scheduled export timer for the cutover first' >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
[[ $(podman inspect --format '{{ '{{.State.Running}}' }}' gitea) == true ]] || {
|
||||||
|
echo 'Source Gitea must be running before the final export' >&2; exit 1;
|
||||||
|
}
|
||||||
|
[[ -d /opt/gitea/data && -d /home/git/.ssh ]] || {
|
||||||
|
echo 'Required source Gitea paths are missing' >&2; exit 1;
|
||||||
|
}
|
||||||
|
[[ ! -e "$versions/$stamp" ]] || {
|
||||||
|
echo 'Final export timestamp already exists' >&2; exit 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
gitea_stopped=true
|
||||||
|
podman stop --time 30 gitea >/dev/null
|
||||||
|
[[ $(podman inspect --format '{{ '{{.State.Running}}' }}' gitea) == false ]] || {
|
||||||
|
echo 'Source Gitea did not stop' >&2; exit 1;
|
||||||
|
}
|
||||||
|
python3 - <<'PY'
|
||||||
|
import sqlite3
|
||||||
|
path = '/opt/gitea/data/gitea/gitea.db'
|
||||||
|
with sqlite3.connect(f'file:{path}?mode=ro', uri=True) as database:
|
||||||
|
if database.execute('PRAGMA quick_check').fetchone()[0] != 'ok':
|
||||||
|
raise SystemExit('Source Gitea SQLite quick_check failed')
|
||||||
|
PY
|
||||||
|
|
||||||
|
stage=$(mktemp -d "$export_root/.staging.XXXXXXXX")
|
||||||
|
tar --acls --xattrs --selinux -C / -cf "$stage/payload.tar" \
|
||||||
|
opt/gitea/data home/git/.ssh
|
||||||
|
tar -tf "$stage/payload.tar" >/dev/null
|
||||||
|
(cd "$stage" && sha256sum payload.tar >payload.sha256)
|
||||||
|
printf '{"schema":1,"host":"prometheus","purpose":"gitea-cutover","created_utc":"%s"}\n' \
|
||||||
|
"$stamp" >"$stage/metadata.json"
|
||||||
|
|
||||||
|
[[ $(podman inspect --format '{{ '{{.State.Running}}' }}' gitea) == false ]] || {
|
||||||
|
echo 'Source Gitea restarted during final export' >&2; exit 1;
|
||||||
|
}
|
||||||
|
chown root:{{ server_backup_username }} "$stage" "$stage/payload.tar" \
|
||||||
|
"$stage/payload.sha256" "$stage/metadata.json"
|
||||||
|
chmod 0750 "$stage"
|
||||||
|
chmod 0640 "$stage/payload.tar" "$stage/payload.sha256" "$stage/metadata.json"
|
||||||
|
mv -- "$stage" "$versions/$stamp"
|
||||||
|
stage=''
|
||||||
|
ln -s "$stamp" "$versions/.current.new"
|
||||||
|
mv -Tf -- "$versions/.current.new" "$versions/current"
|
||||||
|
|
||||||
|
echo "Prepared final Gitea export $stamp; source Gitea remains stopped"
|
||||||
@@ -104,6 +104,15 @@ restored this new dataset.
|
|||||||
|
|
||||||
## Phase 2: explicit final cutover
|
## Phase 2: explicit final cutover
|
||||||
|
|
||||||
|
The opt-in `/usr/local/sbin/prometheus-gitea-final-export` helper was installed
|
||||||
|
on 2026-10-01 and passed `bash -n`; it has **not** been invoked. It refuses to
|
||||||
|
run while the scheduled Prometheus export timer is active. When explicitly
|
||||||
|
triggered, it stops only the source Gitea container, checks SQLite, publishes
|
||||||
|
a checksum-verified Gitea-only version for Atlas' existing pull, and leaves
|
||||||
|
the source stopped on success. NPM remains running. A failure before
|
||||||
|
completion restarts source Gitea. Its Ansible gate is
|
||||||
|
`--tags gitea_final_export -e server_gitea_final_export=true`.
|
||||||
|
|
||||||
1. Agree on an outage and record source/target versions, pool health, the
|
1. Agree on an outage and record source/target versions, pool health, the
|
||||||
latest backups, SSH host-key fingerprints, and both current NPM routes.
|
latest backups, SSH host-key fingerprints, and both current NPM routes.
|
||||||
Stop the Prometheus export timer for the change window so it cannot
|
Stop the Prometheus export timer for the change window so it cannot
|
||||||
|
|||||||
Reference in New Issue
Block a user