diff --git a/AGENTS.md b/AGENTS.md index 8cd76dc..8c5349f 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -61,6 +61,8 @@ Ansible-driven personal infrastructure repo for Fedora and Void desktops, Fedora `ansible-playbook ansible/site.yml --limit atlas --tags gitea --check --diff` - Atlas explicit isolated Gitea restore rehearsal (not part of normal runs): `ansible-playbook ansible/site.yml --limit atlas --tags gitea_restore -e atlas_gitea_restore_test=true` + - Prometheus final Gitea export helper (dry-run installs only; outage action remains opt-in): + `ansible-playbook ansible/site.yml --limit prometheus --tags gitea_final_export --check --diff` - Atlas network/share hardening: `ansible-playbook ansible/site.yml --limit atlas --tags hardening,sharing --check --diff` - Atlas ZFS snapshot retention and scrub timers: @@ -286,6 +288,10 @@ successfully. The first monthly scrub remains a runtime check. - [ ] Include the new Gitea dataset in the next UUID-bound offline USB version and test a file restore from that version before accepting production writes; the UUID-bound disk is connected but its LUKS mapper is closed, so the manual backup still requires interactive unlock. +- [x] Install a separate opt-in final Gitea export helper on Prometheus. Its 2026-10-01 targeted + deployment and `bash -n` passed while Gitea and NPM stayed running. It refuses an active export + timer, stops only Gitea, verifies SQLite, publishes a checksum-verified Gitea-only version for + Atlas' existing pull, and leaves the source stopped on success; it has **not** been invoked. - [ ] After an explicit outage approval, perform the final consistent copy and HTTPS/SSH cutover, then remove Gitea from Prometheus' desired stack and backup export without deleting source data. - [ ] Design and deploy Nextcloud as another explicitly temporary Atlas service before Uranus. Give it diff --git a/ansible/inventory/group_vars/server.yml b/ansible/inventory/group_vars/server.yml index d1b8e19..8092dae 100644 --- a/ansible/inventory/group_vars/server.yml +++ b/ansible/inventory/group_vars/server.yml @@ -87,6 +87,9 @@ server_backup_export_root: /var/lib/prometheus-backup-export server_backup_rrsync_path: /usr/share/doc/rsync/support/rrsync server_backup_export_calendar: "*-*-* 02:00:00 Europe/Rome" server_backup_export_start_timer: false +# Explicit Gitea cutover helper: installed separately from any outage action. +server_gitea_cutover_tools_enabled: false +server_gitea_final_export: false server_backup_export_source_keep: 3 server_backup_export_paths: - opt/npm/data diff --git a/ansible/inventory/host_vars/prometheus.yml b/ansible/inventory/host_vars/prometheus.yml index bb41b77..538af8d 100644 --- a/ansible/inventory/host_vars/prometheus.yml +++ b/ansible/inventory/host_vars/prometheus.yml @@ -8,6 +8,8 @@ ansible_ssh_private_key_file: /home/fscotto/.ssh/id_ed25519 server_username: rocky server_backup_export_enabled: true server_backup_export_start_timer: true +# Install the final-copy helper only; it is never run by a normal playbook invocation. +server_gitea_cutover_tools_enabled: true server_duckdns_domain: fscotto server_ssh_authorized_keys: - name: ikaros diff --git a/ansible/roles/profile_server/tasks/gitea_final_export.yml b/ansible/roles/profile_server/tasks/gitea_final_export.yml new file mode 100644 index 0000000..4a5ec81 --- /dev/null +++ b/ansible/roles/profile_server/tasks/gitea_final_export.yml @@ -0,0 +1,34 @@ +--- +- name: Install the explicit Gitea final-export helper + tags: [services, gitea_final_export] + ansible.builtin.template: + src: prometheus-gitea-final-export.sh.j2 + dest: /usr/local/sbin/prometheus-gitea-final-export + owner: root + group: root + mode: "0750" + when: server_gitea_cutover_tools_enabled | bool + +- name: Require the prepared source and explicit final-export approval + tags: [services, gitea_final_export] + ansible.builtin.assert: + that: + - server_gitea_cutover_tools_enabled | bool + - server_backup_export_enabled | bool + - not ansible_check_mode + fail_msg: >- + Install the cutover helper and perform an explicit non-check-mode run + only after the Gitea outage gate has been approved. + when: server_gitea_final_export | bool + +- name: Stop source Gitea and publish the final consistent export + tags: [services, gitea_final_export] + ansible.builtin.command: + argv: + - /usr/local/sbin/prometheus-gitea-final-export + register: server_gitea_final_export_result + changed_when: server_gitea_final_export_result.rc == 0 + no_log: true + when: + - server_gitea_final_export | bool + - not ansible_check_mode diff --git a/ansible/roles/profile_server/tasks/main.yml b/ansible/roles/profile_server/tasks/main.yml index 73bda6f..5ceffef 100644 --- a/ansible/roles/profile_server/tasks/main.yml +++ b/ansible/roles/profile_server/tasks/main.yml @@ -59,6 +59,9 @@ - name: Import Prometheus backup export job tasks ansible.builtin.import_tasks: backup_export_job.yml +- name: Import explicit Prometheus Gitea final-export tasks + ansible.builtin.import_tasks: gitea_final_export.yml + - name: Ensure server SSH authorized key fragments directory exists tags: [services, ssh] ansible.builtin.file: diff --git a/ansible/roles/profile_server/templates/prometheus-gitea-final-export.sh.j2 b/ansible/roles/profile_server/templates/prometheus-gitea-final-export.sh.j2 new file mode 100644 index 0000000..567bcdc --- /dev/null +++ b/ansible/roles/profile_server/templates/prometheus-gitea-final-export.sh.j2 @@ -0,0 +1,80 @@ +#!/usr/bin/env bash +set -Eeuo pipefail +umask 077 + +export_root={{ server_backup_export_root | quote }} +versions="$export_root/versions" +stamp=$(date -u +%Y%m%dT%H%M%SZ) +stage='' +gitea_stopped=false + +exec 9>/run/lock/prometheus-backup-export.lock +flock -n 9 || { echo 'A Prometheus backup export is already running' >&2; exit 1; } + +cleanup() { + local rc=$? + trap - EXIT + if (( rc != 0 )) && "$gitea_stopped"; then + podman start gitea >/dev/null || rc=1 + fi + if (( rc != 0 )) && [[ -n "$stage" && -d "$stage" ]]; then + rm -rf -- "$stage" + fi + exit "$rc" +} +trap cleanup EXIT +trap 'exit 129' HUP +trap 'exit 130' INT +trap 'exit 143' TERM + +systemctl is-active --quiet podman-compose-server.service || { + echo 'Prometheus Compose stack is not active' >&2; exit 1; +} +if systemctl is-active --quiet prometheus-backup-export.timer; then + echo 'Stop the scheduled export timer for the cutover first' >&2 + exit 1 +fi +[[ $(podman inspect --format '{{ '{{.State.Running}}' }}' gitea) == true ]] || { + echo 'Source Gitea must be running before the final export' >&2; exit 1; +} +[[ -d /opt/gitea/data && -d /home/git/.ssh ]] || { + echo 'Required source Gitea paths are missing' >&2; exit 1; +} +[[ ! -e "$versions/$stamp" ]] || { + echo 'Final export timestamp already exists' >&2; exit 1; +} + +gitea_stopped=true +podman stop --time 30 gitea >/dev/null +[[ $(podman inspect --format '{{ '{{.State.Running}}' }}' gitea) == false ]] || { + echo 'Source Gitea did not stop' >&2; exit 1; +} +python3 - <<'PY' +import sqlite3 +path = '/opt/gitea/data/gitea/gitea.db' +with sqlite3.connect(f'file:{path}?mode=ro', uri=True) as database: + if database.execute('PRAGMA quick_check').fetchone()[0] != 'ok': + raise SystemExit('Source Gitea SQLite quick_check failed') +PY + +stage=$(mktemp -d "$export_root/.staging.XXXXXXXX") +tar --acls --xattrs --selinux -C / -cf "$stage/payload.tar" \ + opt/gitea/data home/git/.ssh +tar -tf "$stage/payload.tar" >/dev/null +(cd "$stage" && sha256sum payload.tar >payload.sha256) +printf '{"schema":1,"host":"prometheus","purpose":"gitea-cutover","created_utc":"%s"}\n' \ + "$stamp" >"$stage/metadata.json" + +[[ $(podman inspect --format '{{ '{{.State.Running}}' }}' gitea) == false ]] || { + echo 'Source Gitea restarted during final export' >&2; exit 1; +} +chown root:{{ server_backup_username }} "$stage" "$stage/payload.tar" \ + "$stage/payload.sha256" "$stage/metadata.json" +chmod 0750 "$stage" +chmod 0640 "$stage/payload.tar" "$stage/payload.sha256" "$stage/metadata.json" +mv -- "$stage" "$versions/$stamp" +stage='' +ln -s "$stamp" "$versions/.current.new" +mv -Tf -- "$versions/.current.new" "$versions/current" + +echo "Prepared final Gitea export $stamp; source Gitea remains stopped" diff --git a/docs/atlas-gitea-migration.md b/docs/atlas-gitea-migration.md index 934c67d..0e6c065 100644 --- a/docs/atlas-gitea-migration.md +++ b/docs/atlas-gitea-migration.md @@ -104,6 +104,15 @@ restored this new dataset. ## Phase 2: explicit final cutover +The opt-in `/usr/local/sbin/prometheus-gitea-final-export` helper was installed +on 2026-10-01 and passed `bash -n`; it has **not** been invoked. It refuses to +run while the scheduled Prometheus export timer is active. When explicitly +triggered, it stops only the source Gitea container, checks SQLite, publishes +a checksum-verified Gitea-only version for Atlas' existing pull, and leaves +the source stopped on success. NPM remains running. A failure before +completion restarts source Gitea. Its Ansible gate is +`--tags gitea_final_export -e server_gitea_final_export=true`. + 1. Agree on an outage and record source/target versions, pool health, the latest backups, SSH host-key fingerprints, and both current NPM routes. Stop the Prometheus export timer for the change window so it cannot