Prepare consistent final Gitea export on Prometheus

ansible/inventory/group_vars/server.yml

@@ -87,6 +87,9 @@ server_backup_export_root: /var/lib/prometheus-backup-export
 server_backup_rrsync_path: /usr/share/doc/rsync/support/rrsync
 server_backup_export_calendar: "*-*-* 02:00:00 Europe/Rome"
 server_backup_export_start_timer: false
+# Explicit Gitea cutover helper: installed separately from any outage action.
+server_gitea_cutover_tools_enabled: false
+server_gitea_final_export: false
 server_backup_export_source_keep: 3
 server_backup_export_paths:
   - opt/npm/data

ansible/inventory/host_vars/prometheus.yml

@@ -8,6 +8,8 @@ ansible_ssh_private_key_file: /home/fscotto/.ssh/id_ed25519
 server_username: rocky
 server_backup_export_enabled: true
 server_backup_export_start_timer: true
+# Install the final-copy helper only; it is never run by a normal playbook invocation.
+server_gitea_cutover_tools_enabled: true
 server_duckdns_domain: fscotto
 server_ssh_authorized_keys:
   - name: ikaros

ansible/roles/profile_server/tasks/gitea_final_export.yml

@@ -0,0 +1,34 @@
+---
+- name: Install the explicit Gitea final-export helper
+  tags: [services, gitea_final_export]
+  ansible.builtin.template:
+    src: prometheus-gitea-final-export.sh.j2
+    dest: /usr/local/sbin/prometheus-gitea-final-export
+    owner: root
+    group: root
+    mode: "0750"
+  when: server_gitea_cutover_tools_enabled | bool
+
+- name: Require the prepared source and explicit final-export approval
+  tags: [services, gitea_final_export]
+  ansible.builtin.assert:
+    that:
+      - server_gitea_cutover_tools_enabled | bool
+      - server_backup_export_enabled | bool
+      - not ansible_check_mode
+    fail_msg: >-
+      Install the cutover helper and perform an explicit non-check-mode run
+      only after the Gitea outage gate has been approved.
+  when: server_gitea_final_export | bool
+
+- name: Stop source Gitea and publish the final consistent export
+  tags: [services, gitea_final_export]
+  ansible.builtin.command:
+    argv:
+      - /usr/local/sbin/prometheus-gitea-final-export
+  register: server_gitea_final_export_result
+  changed_when: server_gitea_final_export_result.rc == 0
+  no_log: true
+  when:
+    - server_gitea_final_export | bool
+    - not ansible_check_mode

ansible/roles/profile_server/tasks/main.yml

@@ -59,6 +59,9 @@
 - name: Import Prometheus backup export job tasks
   ansible.builtin.import_tasks: backup_export_job.yml
 
+- name: Import explicit Prometheus Gitea final-export tasks
+  ansible.builtin.import_tasks: gitea_final_export.yml
+
 - name: Ensure server SSH authorized key fragments directory exists
   tags: [services, ssh]
   ansible.builtin.file:

ansible/roles/profile_server/templates/prometheus-gitea-final-export.sh.j2

@@ -0,0 +1,80 @@
+#!/usr/bin/env bash
+set -Eeuo pipefail
+umask 077
+
+export_root={{ server_backup_export_root | quote }}
+versions="$export_root/versions"
+stamp=$(date -u +%Y%m%dT%H%M%SZ)
+stage=''
+gitea_stopped=false
+
+exec 9>/run/lock/prometheus-backup-export.lock
+flock -n 9 || { echo 'A Prometheus backup export is already running' >&2; exit 1; }
+
+cleanup() {
+  local rc=$?
+  trap - EXIT
+  if (( rc != 0 )) && "$gitea_stopped"; then
+    podman start gitea >/dev/null || rc=1
+  fi
+  if (( rc != 0 )) && [[ -n "$stage" && -d "$stage" ]]; then
+    rm -rf -- "$stage"
+  fi
+  exit "$rc"
+}
+trap cleanup EXIT
+trap 'exit 129' HUP
+trap 'exit 130' INT
+trap 'exit 143' TERM
+
+systemctl is-active --quiet podman-compose-server.service || {
+  echo 'Prometheus Compose stack is not active' >&2; exit 1;
+}
+if systemctl is-active --quiet prometheus-backup-export.timer; then
+  echo 'Stop the scheduled export timer for the cutover first' >&2
+  exit 1
+fi
+[[ $(podman inspect --format '{{ '{{.State.Running}}' }}' gitea) == true ]] || {
+  echo 'Source Gitea must be running before the final export' >&2; exit 1;
+}
+[[ -d /opt/gitea/data && -d /home/git/.ssh ]] || {
+  echo 'Required source Gitea paths are missing' >&2; exit 1;
+}
+[[ ! -e "$versions/$stamp" ]] || {
+  echo 'Final export timestamp already exists' >&2; exit 1;
+}
+
+gitea_stopped=true
+podman stop --time 30 gitea >/dev/null
+[[ $(podman inspect --format '{{ '{{.State.Running}}' }}' gitea) == false ]] || {
+  echo 'Source Gitea did not stop' >&2; exit 1;
+}
+python3 - <<'PY'
+import sqlite3
+path = '/opt/gitea/data/gitea/gitea.db'
+with sqlite3.connect(f'file:{path}?mode=ro', uri=True) as database:
+    if database.execute('PRAGMA quick_check').fetchone()[0] != 'ok':
+        raise SystemExit('Source Gitea SQLite quick_check failed')
+PY
+
+stage=$(mktemp -d "$export_root/.staging.XXXXXXXX")
+tar --acls --xattrs --selinux -C / -cf "$stage/payload.tar" \
+  opt/gitea/data home/git/.ssh
+tar -tf "$stage/payload.tar" >/dev/null
+(cd "$stage" && sha256sum payload.tar >payload.sha256)
+printf '{"schema":1,"host":"prometheus","purpose":"gitea-cutover","created_utc":"%s"}\n' \
+  "$stamp" >"$stage/metadata.json"
+
+[[ $(podman inspect --format '{{ '{{.State.Running}}' }}' gitea) == false ]] || {
+  echo 'Source Gitea restarted during final export' >&2; exit 1;
+}
+chown root:{{ server_backup_username }} "$stage" "$stage/payload.tar" \
+  "$stage/payload.sha256" "$stage/metadata.json"
+chmod 0750 "$stage"
+chmod 0640 "$stage/payload.tar" "$stage/payload.sha256" "$stage/metadata.json"
+mv -- "$stage" "$versions/$stamp"
+stage=''
+ln -s "$stamp" "$versions/.current.new"
+mv -Tf -- "$versions/.current.new" "$versions/current"
+
+echo "Prepared final Gitea export $stamp; source Gitea remains stopped"