Add verified Prometheus backup pull to Atlas

ansible/roles/profile_atlas/tasks/main.yml

@@ -23,6 +23,12 @@
 - name: Import Atlas offline USB backup tasks
   ansible.builtin.import_tasks: usb_backup.yml
 
+- name: Import Atlas Prometheus backup pull identity tasks
+  ansible.builtin.import_tasks: prometheus_pull_identity.yml
+
+- name: Import Atlas Prometheus backup pull job tasks
+  ansible.builtin.import_tasks: prometheus_pull_job.yml
+
 - name: Import Atlas health monitoring tasks
   ansible.builtin.import_tasks: monitoring.yml
 

ansible/roles/profile_atlas/tasks/monitoring.yml

@@ -7,8 +7,8 @@
       - atlas_zfs_pool != 'CHANGEME_ZFS_POOL'
       - atlas_monitor_calendar | length > 0
       - atlas_monitor_smart_devices | length > 0
-      - atlas_monitor_timers | length > 0
-      - atlas_monitor_failure_units | length > 0
+      - atlas_monitor_effective_timers | length > 0
+      - atlas_monitor_effective_failure_units | length > 0
       - atlas_monitor_remote_capacity.user == atlas_borg_repository_user
       - atlas_monitor_remote_capacity.host == atlas_borg_repository_host
       - atlas_monitor_remote_capacity.run_as == atlas_borg_username
@@ -49,7 +49,7 @@
     that:
       - item.name is match('^[a-zA-Z0-9@_.-]+\\.timer$')
       - item.max_age_hours | int >= 0
-  loop: "{{ atlas_monitor_timers }}"
+  loop: "{{ atlas_monitor_effective_timers }}"
   loop_control:
     label: "{{ item.name }}"
   when: atlas_manage_monitoring | bool
@@ -59,7 +59,7 @@
   ansible.builtin.assert:
     that:
       - item is match('^[a-zA-Z0-9@_.-]+\\.service$')
-  loop: "{{ atlas_monitor_failure_units }}"
+  loop: "{{ atlas_monitor_effective_failure_units }}"
   when: atlas_manage_monitoring | bool
 
 - name: Validate Atlas health monitor calendar
@@ -144,7 +144,7 @@
     owner: root
     group: root
     mode: "0755"
-  loop: "{{ atlas_monitor_failure_units }}"
+  loop: "{{ atlas_monitor_effective_failure_units }}"
   when: atlas_manage_monitoring | bool
 
 - name: Notify 45Drives Alerts when an Atlas job fails
@@ -155,7 +155,7 @@
     owner: root
     group: root
     mode: "0644"
-  loop: "{{ atlas_monitor_failure_units }}"
+  loop: "{{ atlas_monitor_effective_failure_units }}"
   when: atlas_manage_monitoring | bool
 
 - name: Reload systemd after installing Atlas monitoring

ansible/roles/profile_atlas/tasks/prometheus_pull_identity.yml

@@ -0,0 +1,66 @@
+---
+- name: Validate Atlas Prometheus pull identity inputs
+  tags: [atlas, backup, prometheus_backup, prometheus_backup_key]
+  ansible.builtin.assert:
+    that:
+      - atlas_prometheus_pull_ssh_dir.startswith('/etc/')
+      - atlas_prometheus_pull_private_key_path.startswith(atlas_prometheus_pull_ssh_dir ~ '/')
+      - atlas_prometheus_pull_known_hosts_path.startswith(atlas_prometheus_pull_ssh_dir ~ '/')
+      - atlas_prometheus_ssh_host_key.startswith(
+          (hostvars['prometheus'].ansible_host | string) ~ ' ssh-ed25519 '
+        )
+    fail_msg: Pin the verified Prometheus ED25519 SSH host key before enabling the pull.
+  when: atlas_manage_prometheus_backup_pull | bool
+
+- name: Create private Atlas Prometheus pull SSH directory
+  tags: [atlas, backup, prometheus_backup, prometheus_backup_key]
+  ansible.builtin.file:
+    path: "{{ atlas_prometheus_pull_ssh_dir }}"
+    state: directory
+    owner: root
+    group: root
+    mode: "0700"
+  when: atlas_manage_prometheus_backup_pull | bool
+
+- name: Generate Atlas-only Prometheus pull SSH identity
+  tags: [atlas, backup, prometheus_backup, prometheus_backup_key]
+  ansible.builtin.command:
+    argv:
+      - ssh-keygen
+      - -q
+      - -t
+      - ed25519
+      - -N
+      - ""
+      - -C
+      - atlas-prometheus-pull@atlas
+      - -f
+      - "{{ atlas_prometheus_pull_private_key_path }}"
+    creates: "{{ atlas_prometheus_pull_private_key_path }}"
+  when: atlas_manage_prometheus_backup_pull | bool
+
+- name: Protect Atlas-only Prometheus pull SSH identity
+  tags: [atlas, backup, prometheus_backup, prometheus_backup_key]
+  ansible.builtin.file:
+    path: "{{ item.path }}"
+    owner: root
+    group: root
+    mode: "{{ item.mode }}"
+  loop:
+    - { path: "{{ atlas_prometheus_pull_private_key_path }}", mode: "0600" }
+    - { path: "{{ atlas_prometheus_pull_private_key_path }}.pub", mode: "0644" }
+  loop_control:
+    label: "{{ item.path }}"
+  when:
+    - atlas_manage_prometheus_backup_pull | bool
+    - not ansible_check_mode
+
+- name: Pin Prometheus SSH host key on Atlas
+  tags: [atlas, backup, prometheus_backup, prometheus_backup_key]
+  ansible.builtin.copy:
+    content: "{{ atlas_prometheus_ssh_host_key }}\n"
+    dest: "{{ atlas_prometheus_pull_known_hosts_path }}"
+    owner: root
+    group: root
+    mode: "0600"
+  when: atlas_manage_prometheus_backup_pull | bool

ansible/roles/profile_atlas/tasks/prometheus_pull_job.yml

@@ -0,0 +1,90 @@
+---
+- name: Validate Atlas Prometheus backup pull inputs
+  tags: [atlas, backup, prometheus_backup]
+  ansible.builtin.assert:
+    that:
+      - atlas_manage_storage | bool
+      - atlas_prometheus_pull_source_user is match('^[a-z_][a-z0-9_-]*$')
+      - atlas_prometheus_pull_source_port | int > 0
+      - atlas_prometheus_pull_source_port | int < 65536
+      - atlas_prometheus_pull_keep_daily | int > 0
+      - atlas_prometheus_pull_keep_weekly | int > 0
+      - atlas_prometheus_pull_keep_monthly | int > 0
+      - atlas_prometheus_pull_max_age_hours | int > 0
+      - atlas_backup_prometheus_mountpoint.startswith(atlas_mount_root ~ '/')
+    fail_msg: Define the Atlas backup destination, source account, and retention before enabling the pull.
+  when: atlas_manage_prometheus_backup_pull | bool
+
+- name: Validate Atlas Prometheus backup pull calendar
+  tags: [atlas, backup, prometheus_backup]
+  ansible.builtin.command:
+    argv: [systemd-analyze, calendar, "{{ atlas_prometheus_pull_calendar }}"]
+  changed_when: false
+  check_mode: false
+  when: atlas_manage_prometheus_backup_pull | bool
+
+- name: Create private Atlas Prometheus backup version directory
+  tags: [atlas, backup, prometheus_backup]
+  ansible.builtin.file:
+    path: "{{ atlas_backup_prometheus_mountpoint }}/snapshots"
+    state: directory
+    owner: root
+    group: root
+    mode: "0700"
+  when: atlas_manage_prometheus_backup_pull | bool
+
+- name: Install Atlas Prometheus backup pull helper
+  tags: [atlas, backup, prometheus_backup]
+  ansible.builtin.template:
+    src: atlas-prometheus-pull.sh.j2
+    dest: /usr/local/sbin/atlas-prometheus-pull
+    owner: root
+    group: root
+    mode: "0750"
+  when: atlas_manage_prometheus_backup_pull | bool
+
+- name: Install Atlas Prometheus backup retention helper
+  tags: [atlas, backup, prometheus_backup]
+  ansible.builtin.copy:
+    src: atlas-prometheus-prune.py
+    dest: /usr/local/libexec/atlas-prometheus-prune
+    owner: root
+    group: root
+    mode: "0750"
+  when: atlas_manage_prometheus_backup_pull | bool
+
+- name: Install Atlas Prometheus backup pull systemd units
+  tags: [atlas, backup, prometheus_backup]
+  ansible.builtin.template:
+    src: "{{ item }}.j2"
+    dest: "/etc/systemd/system/{{ item }}"
+    owner: root
+    group: root
+    mode: "0644"
+  loop:
+    - atlas-prometheus-pull.service
+    - atlas-prometheus-pull.timer
+  loop_control:
+    label: "{{ item }}"
+  register: atlas_prometheus_pull_units
+  when: atlas_manage_prometheus_backup_pull | bool
+
+- name: Reload systemd after Atlas Prometheus pull unit changes
+  tags: [atlas, backup, prometheus_backup]
+  ansible.builtin.systemd:
+    daemon_reload: true
+  when:
+    - atlas_manage_prometheus_backup_pull | bool
+    - atlas_prometheus_pull_units is changed
+    - not ansible_check_mode
+
+- name: Enable Atlas Prometheus pull timer only after explicit activation
+  tags: [atlas, backup, prometheus_backup]
+  ansible.builtin.systemd:
+    name: atlas-prometheus-pull.timer
+    enabled: true
+    state: started
+  when:
+    - atlas_manage_prometheus_backup_pull | bool
+    - atlas_prometheus_pull_start_timer | bool
+    - not ansible_check_mode