mirror of
https://github.com/fscotto/infra.git
synced 2026-10-03 13:29:58 +00:00
Prepare guarded final Gitea restore on Atlas
This commit is contained in:
@@ -61,6 +61,8 @@ Ansible-driven personal infrastructure repo for Fedora and Void desktops, Fedora
|
|||||||
`ansible-playbook ansible/site.yml --limit atlas --tags gitea --check --diff`
|
`ansible-playbook ansible/site.yml --limit atlas --tags gitea --check --diff`
|
||||||
- Atlas explicit isolated Gitea restore rehearsal (not part of normal runs):
|
- Atlas explicit isolated Gitea restore rehearsal (not part of normal runs):
|
||||||
`ansible-playbook ansible/site.yml --limit atlas --tags gitea_restore -e atlas_gitea_restore_test=true`
|
`ansible-playbook ansible/site.yml --limit atlas --tags gitea_restore -e atlas_gitea_restore_test=true`
|
||||||
|
- Atlas final Gitea replacement gate (dry-run only until a stopped-source export is pulled):
|
||||||
|
`ansible-playbook ansible/site.yml --limit atlas --tags gitea_final_restore --check --diff -e atlas_gitea_final_restore=true`
|
||||||
- Prometheus final Gitea export helper (dry-run installs only; outage action remains opt-in):
|
- Prometheus final Gitea export helper (dry-run installs only; outage action remains opt-in):
|
||||||
`ansible-playbook ansible/site.yml --limit prometheus --tags gitea_final_export --check --diff`
|
`ansible-playbook ansible/site.yml --limit prometheus --tags gitea_final_export --check --diff`
|
||||||
- Atlas network/share hardening:
|
- Atlas network/share hardening:
|
||||||
@@ -292,6 +294,11 @@ successfully. The first monthly scrub remains a runtime check.
|
|||||||
deployment and `bash -n` passed while Gitea and NPM stayed running. It refuses an active export
|
deployment and `bash -n` passed while Gitea and NPM stayed running. It refuses an active export
|
||||||
timer, stops only Gitea, verifies SQLite, publishes a checksum-verified Gitea-only version for
|
timer, stops only Gitea, verifies SQLite, publishes a checksum-verified Gitea-only version for
|
||||||
Atlas' existing pull, and leaves the source stopped on success; it has **not** been invoked.
|
Atlas' existing pull, and leaves the source stopped on success; it has **not** been invoked.
|
||||||
|
- [x] Prepare the Atlas final-restore gate without replacing the rehearsal: it accepts only a
|
||||||
|
checksum-verified `gitea-cutover` export, refuses a running target, stages and validates the new
|
||||||
|
layout before replacing the marked rehearsal, and rolls back a failed swap. Synthetic success
|
||||||
|
and rollback tests and a second idempotent rehearsal run passed on 2026-10-01; the final gate
|
||||||
|
has **not** been invoked.
|
||||||
- [ ] After an explicit outage approval, perform the final consistent copy and HTTPS/SSH cutover,
|
- [ ] After an explicit outage approval, perform the final consistent copy and HTTPS/SSH cutover,
|
||||||
then remove Gitea from Prometheus' desired stack and backup export without deleting source data.
|
then remove Gitea from Prometheus' desired stack and backup export without deleting source data.
|
||||||
- [ ] Design and deploy Nextcloud as another explicitly temporary Atlas service before Uranus. Give it
|
- [ ] Design and deploy Nextcloud as another explicitly temporary Atlas service before Uranus. Give it
|
||||||
|
|||||||
@@ -182,6 +182,7 @@ atlas_gitea_staging_bind_address: 127.0.0.1
|
|||||||
atlas_gitea_staging_http_port: 3001
|
atlas_gitea_staging_http_port: 3001
|
||||||
atlas_gitea_staging_ssh_port: 2223
|
atlas_gitea_staging_ssh_port: 2223
|
||||||
atlas_gitea_restore_test: false
|
atlas_gitea_restore_test: false
|
||||||
|
atlas_gitea_final_restore: false
|
||||||
atlas_gitea_restore_helper: /usr/local/libexec/atlas-gitea-restore-test
|
atlas_gitea_restore_helper: /usr/local/libexec/atlas-gitea-restore-test
|
||||||
|
|
||||||
atlas_45drives_repo_url: https://repo.45drives.com/repofiles/rocky/45drives-enterprise.repo
|
atlas_45drives_repo_url: https://repo.45drives.com/repofiles/rocky/45drives-enterprise.repo
|
||||||
|
|||||||
@@ -3,6 +3,7 @@
|
|||||||
|
|
||||||
import argparse
|
import argparse
|
||||||
import hashlib
|
import hashlib
|
||||||
|
import json
|
||||||
import os
|
import os
|
||||||
from pathlib import Path, PurePosixPath
|
from pathlib import Path, PurePosixPath
|
||||||
import re
|
import re
|
||||||
@@ -136,12 +137,41 @@ def chown_tree(root, uid, gid):
|
|||||||
os.chown(os.path.join(directory, name), uid, gid)
|
os.chown(os.path.join(directory, name), uid, gid)
|
||||||
|
|
||||||
|
|
||||||
|
def replace_rehearsal(target, stage, digest, uid, gid):
|
||||||
|
previous_data = target / ".previous-rehearsal-data"
|
||||||
|
previous_config = target / ".previous-rehearsal-config"
|
||||||
|
if previous_data.exists() or previous_config.exists():
|
||||||
|
raise ValueError("An interrupted Gitea replacement needs manual recovery")
|
||||||
|
os.rename(target / "data", previous_data)
|
||||||
|
try:
|
||||||
|
os.rename(target / "config", previous_config)
|
||||||
|
os.rename(stage / "data", target / "data")
|
||||||
|
os.rename(stage / "config", target / "config")
|
||||||
|
final_marker = target / ".final-sha256"
|
||||||
|
final_marker.write_text(digest + "\n")
|
||||||
|
final_marker.chmod(0o600)
|
||||||
|
os.chown(final_marker, uid, gid)
|
||||||
|
(target / ".rehearsal-sha256").unlink()
|
||||||
|
except Exception:
|
||||||
|
for name, previous in (("data", previous_data), ("config", previous_config)):
|
||||||
|
current = target / name
|
||||||
|
if previous.exists():
|
||||||
|
if current.exists():
|
||||||
|
shutil.rmtree(current)
|
||||||
|
os.rename(previous, current)
|
||||||
|
(target / ".final-sha256").unlink(missing_ok=True)
|
||||||
|
raise
|
||||||
|
shutil.rmtree(previous_data)
|
||||||
|
shutil.rmtree(previous_config)
|
||||||
|
|
||||||
|
|
||||||
def main():
|
def main():
|
||||||
parser = argparse.ArgumentParser()
|
parser = argparse.ArgumentParser()
|
||||||
parser.add_argument("--backup", type=Path, required=True)
|
parser.add_argument("--backup", type=Path, required=True)
|
||||||
parser.add_argument("--target", type=Path, required=True)
|
parser.add_argument("--target", type=Path, required=True)
|
||||||
parser.add_argument("--uid", type=int, required=True)
|
parser.add_argument("--uid", type=int, required=True)
|
||||||
parser.add_argument("--gid", type=int, required=True)
|
parser.add_argument("--gid", type=int, required=True)
|
||||||
|
parser.add_argument("--replace-rehearsal", action="store_true")
|
||||||
args = parser.parse_args()
|
args = parser.parse_args()
|
||||||
|
|
||||||
backup = args.backup.resolve(strict=True)
|
backup = args.backup.resolve(strict=True)
|
||||||
@@ -156,17 +186,28 @@ def main():
|
|||||||
if sha256(backup / "payload.tar") != expected:
|
if sha256(backup / "payload.tar") != expected:
|
||||||
raise ValueError("Prometheus backup SHA-256 mismatch")
|
raise ValueError("Prometheus backup SHA-256 mismatch")
|
||||||
|
|
||||||
marker = target / ".rehearsal-sha256"
|
marker = target / (".final-sha256" if args.replace_rehearsal else ".rehearsal-sha256")
|
||||||
if marker.exists():
|
if marker.exists():
|
||||||
if marker.read_text().strip() != expected:
|
if marker.read_text().strip() != expected:
|
||||||
raise ValueError("A different Gitea rehearsal already occupies this dataset")
|
raise ValueError("A different Gitea restore already occupies this dataset")
|
||||||
validate(target / "data", target / "config")
|
validate(target / "data", target / "config")
|
||||||
print("unchanged")
|
print("unchanged")
|
||||||
return
|
return
|
||||||
for name in ("data", "config"):
|
if args.replace_rehearsal:
|
||||||
directory = target / name
|
metadata = json.loads((backup / "metadata.json").read_text())
|
||||||
if not directory.is_dir() or any(directory.iterdir()):
|
if metadata.get("purpose") != "gitea-cutover":
|
||||||
raise ValueError("Gitea target is not empty; refusing overwrite")
|
raise ValueError("Final restore requires an explicit Gitea cutover export")
|
||||||
|
if not (target / ".rehearsal-sha256").is_file():
|
||||||
|
raise ValueError("Only a marked rehearsal may be replaced")
|
||||||
|
if not all((target / name).is_dir() for name in ("data", "config")):
|
||||||
|
raise ValueError("Prepared Gitea volume paths are missing")
|
||||||
|
else:
|
||||||
|
if (target / ".final-sha256").exists():
|
||||||
|
raise ValueError("Refusing a rehearsal restore over final Gitea data")
|
||||||
|
for name in ("data", "config"):
|
||||||
|
directory = target / name
|
||||||
|
if not directory.is_dir() or any(directory.iterdir()):
|
||||||
|
raise ValueError("Gitea target is not empty; refusing overwrite")
|
||||||
|
|
||||||
with tempfile.TemporaryDirectory(prefix=".rehearsal-", dir=target) as temporary:
|
with tempfile.TemporaryDirectory(prefix=".rehearsal-", dir=target) as temporary:
|
||||||
stage = Path(temporary)
|
stage = Path(temporary)
|
||||||
@@ -183,12 +224,15 @@ def main():
|
|||||||
convert_config(staged_config / "app.ini")
|
convert_config(staged_config / "app.ini")
|
||||||
validate(staged_data, staged_config)
|
validate(staged_data, staged_config)
|
||||||
chown_tree(stage, args.uid, args.gid)
|
chown_tree(stage, args.uid, args.gid)
|
||||||
for name in ("data", "config"):
|
if args.replace_rehearsal:
|
||||||
(target / name).rmdir()
|
replace_rehearsal(target, stage, expected, args.uid, args.gid)
|
||||||
os.rename(stage / name, target / name)
|
else:
|
||||||
marker.write_text(expected + "\n")
|
for name in ("data", "config"):
|
||||||
marker.chmod(0o600)
|
(target / name).rmdir()
|
||||||
os.chown(marker, args.uid, args.gid)
|
os.rename(stage / name, target / name)
|
||||||
|
marker.write_text(expected + "\n")
|
||||||
|
marker.chmod(0o600)
|
||||||
|
os.chown(marker, args.uid, args.gid)
|
||||||
print("restored")
|
print("restored")
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -1,12 +1,13 @@
|
|||||||
---
|
---
|
||||||
- name: Rehearse an isolated rootless Gitea restore from the verified Prometheus backup
|
- name: Restore Gitea from a verified Prometheus backup only on explicit request
|
||||||
tags: [atlas, gitea_restore]
|
tags: [atlas, gitea_restore, gitea_final_restore]
|
||||||
when: atlas_gitea_restore_test | bool
|
when: atlas_gitea_restore_test | bool or atlas_gitea_final_restore | bool
|
||||||
block:
|
block:
|
||||||
- name: Require the prepared rootless Gitea target
|
- name: Require the prepared rootless Gitea target
|
||||||
ansible.builtin.assert:
|
ansible.builtin.assert:
|
||||||
that:
|
that:
|
||||||
- atlas_manage_gitea | bool
|
- atlas_manage_gitea | bool
|
||||||
|
- not (atlas_gitea_restore_test | bool and atlas_gitea_final_restore | bool)
|
||||||
- atlas_gitea_staging_bind_address == '127.0.0.1'
|
- atlas_gitea_staging_bind_address == '127.0.0.1'
|
||||||
- atlas_gitea_mountpoint == atlas_app_data_mountpoint ~ '/gitea'
|
- atlas_gitea_mountpoint == atlas_app_data_mountpoint ~ '/gitea'
|
||||||
fail_msg: Prepare the isolated, loopback-only rootless Gitea target first.
|
fail_msg: Prepare the isolated, loopback-only rootless Gitea target first.
|
||||||
@@ -81,4 +82,26 @@
|
|||||||
register: atlas_gitea_restore_result
|
register: atlas_gitea_restore_result
|
||||||
changed_when: atlas_gitea_restore_result.stdout == 'restored'
|
changed_when: atlas_gitea_restore_result.stdout == 'restored'
|
||||||
no_log: true
|
no_log: true
|
||||||
when: not ansible_check_mode
|
when:
|
||||||
|
- atlas_gitea_restore_test | bool
|
||||||
|
- not ansible_check_mode
|
||||||
|
|
||||||
|
- name: Replace the marked rehearsal with the final consistent Gitea export
|
||||||
|
ansible.builtin.command:
|
||||||
|
argv:
|
||||||
|
- "{{ atlas_gitea_restore_helper }}"
|
||||||
|
- --backup
|
||||||
|
- "{{ atlas_backup_prometheus_mountpoint }}/latest"
|
||||||
|
- --target
|
||||||
|
- "{{ atlas_gitea_mountpoint }}"
|
||||||
|
- --uid
|
||||||
|
- "{{ atlas_gitea_uid | string }}"
|
||||||
|
- --gid
|
||||||
|
- "{{ atlas_gitea_gid | string }}"
|
||||||
|
- --replace-rehearsal
|
||||||
|
register: atlas_gitea_final_restore_result
|
||||||
|
changed_when: atlas_gitea_final_restore_result.stdout == 'restored'
|
||||||
|
no_log: true
|
||||||
|
when:
|
||||||
|
- atlas_gitea_final_restore | bool
|
||||||
|
- not ansible_check_mode
|
||||||
|
|||||||
@@ -112,6 +112,11 @@ a checksum-verified Gitea-only version for Atlas' existing pull, and leaves
|
|||||||
the source stopped on success. NPM remains running. A failure before
|
the source stopped on success. NPM remains running. A failure before
|
||||||
completion restarts source Gitea. Its Ansible gate is
|
completion restarts source Gitea. Its Ansible gate is
|
||||||
`--tags gitea_final_export -e server_gitea_final_export=true`.
|
`--tags gitea_final_export -e server_gitea_final_export=true`.
|
||||||
|
After Atlas pulls that version, its separate
|
||||||
|
`--tags gitea_final_restore -e atlas_gitea_final_restore=true` gate accepts
|
||||||
|
only metadata marked `gitea-cutover`, validates a private staged replacement,
|
||||||
|
and swaps it for the marked rehearsal. The swap and its rollback path passed
|
||||||
|
synthetic tests on 2026-10-01; the gate has not been used on live Gitea data.
|
||||||
|
|
||||||
1. Agree on an outage and record source/target versions, pool health, the
|
1. Agree on an outage and record source/target versions, pool health, the
|
||||||
latest backups, SSH host-key fingerprints, and both current NPM routes.
|
latest backups, SSH host-key fingerprints, and both current NPM routes.
|
||||||
|
|||||||
Reference in New Issue
Block a user