Feature/atlas icloudpd migration (#14)

* Design gated Atlas iCloudPD migration target

* Target Atlas iCloudPD photos to Photobook

* Record isolated iCloudPD Photobook ACL validation

* Record Aegis iCloudPD source audit gap

* Verify iCloudPD backup source scope and Borg access

* Record Atlas iCloudPD deployment gate checks

* Pin iCloudPD photo file and directory modes

* Validate inactive iCloudPD Quadlet on Atlas generator

* Keep iCloudPD in Archive and reserve Photobook for Immich

* Prepare guarded Aegis iCloudPD retirement

* Declare inactive Atlas iCloudPD storage and Quadlet

* Retire Aegis iCloudPD from desired state

* Clear retired Aegis iCloudPD failed-unit state

* Remove completed iCloudPD retirement tasks from Aegis

* Record initial Atlas iCloudPD service start

* Manage Atlas iCloudPD config from Vault

* Fix Atlas iCloudPD traceroute startup and config drift

* Use Atlas Vault key for iCloudPD Apple ID

* Add HEIC decoding to Fedora desktops

* Record completed iCloudPD ingestion and remaining recovery checks

ansible/roles/packages_fedora/tasks/main.yml

@@ -39,11 +39,41 @@
     state: enabled
   when: "'workstation_dev_wsl' in group_names"
 
+- name: Install distribution signing keys for Fedora desktop codecs
+  tags: [packages, heic]
+  ansible.builtin.dnf:
+    name: distribution-gpg-keys
+    state: present
+  when: "'graphical_desktop' in group_names"
+
+- name: Import RPM Fusion Free signing key for Fedora desktop codecs
+  tags: [packages, heic]
+  ansible.builtin.rpm_key:
+    key: /usr/share/distribution-gpg-keys/rpmfusion/RPM-GPG-KEY-rpmfusion-free-fedora-2020
+    fingerprint: E9A491A3DE247814E7E067EAE06F8ECDD651FF2E
+    state: present
+  when: "'graphical_desktop' in group_names"
+
+- name: Enable RPM Fusion Free for Fedora desktop codecs
+  tags: [packages, heic]
+  ansible.builtin.dnf:
+    name: "https://download1.rpmfusion.org/free/fedora/rpmfusion-free-release-{{ ansible_facts['distribution_major_version'] }}.noarch.rpm"
+    state: present
+  when: "'graphical_desktop' in group_names"
+
 - name: Refresh dnf package metadata
   tags: [packages]
   ansible.builtin.dnf:
     update_cache: true
 
+- name: Install HEIC decoder on Fedora desktops
+  tags: [packages, heic]
+  ansible.builtin.dnf:
+    name: libheif-freeworld
+    state: present
+    update_cache: true
+  when: "'graphical_desktop' in group_names"
+
 - name: Install packages on Fedora
   tags: [packages]
   ansible.builtin.dnf:

ansible/roles/profile_aegis/defaults/main.yml

@@ -8,10 +8,6 @@ aegis_network_connection_uuid: ""
 aegis_host_dns_servers: []
 aegis_host_dns_search_domains: []
 aegis_adguard_image: docker.io/adguard/adguardhome:latest
-aegis_icloudpd_image: docker.io/boredazfcuk/icloudpd:latest
-aegis_icloudpd_folder_structure: '{:%Y/%m/%d}'
-aegis_icloudpd_synchronisation_interval: 86400
-aegis_icloudpd_apple_id: ""
 aegis_ikaros_mac_address: aa:bb:cc:dd:ee:ff
 aegis_wol_port: 9
 

ansible/roles/profile_aegis/handlers/main.yml

@@ -9,13 +9,12 @@
     name: sshd.service
     state: reloaded
 
-- name: Restart Aegis Quadlet services
+- name: Restart Aegis AdGuard Quadlet
   ansible.builtin.systemd:
     name: "{{ item }}"
     state: restarted
     daemon_reload: true
   loop:
     - adguardhome.service
-    - icloudpd.service
   loop_control:
     label: "{{ item }}"

ansible/roles/profile_aegis/tasks/main.yml

@@ -13,14 +13,6 @@
     msg: Reboot Aegis to activate the newly layered packages, then rerun the playbook.
   when: aegis_layered_packages_result.needs_reboot | default(false)
 
-- name: Require Aegis iCloudPD Apple ID
-  tags: [aegis, icloudpd]
-  ansible.builtin.assert:
-    that:
-      - aegis_icloudpd_apple_id | length > 0
-    fail_msg: Define vault_aegis_icloudpd_apple_id before applying the Aegis profile.
-  no_log: true
-
 - name: Require completed Aegis network placeholders
   tags: [aegis, dns, firewall, network, services]
   ansible.builtin.assert:
@@ -119,8 +111,6 @@
   loop:
     - /var/lib/adguard/work
     - /var/lib/adguard/conf
-    - /var/lib/icloudpd/data
-    - /var/lib/icloudpd/config
 
 - name: Create Quadlet configuration directory
   tags: [aegis, containers]
@@ -142,12 +132,9 @@
   loop:
     - src: adguardhome.container.j2
       dest: adguardhome.container
-    - src: icloudpd.container.j2
-      dest: icloudpd.container
   loop_control:
     label: "{{ item.dest }}"
-  no_log: "{{ item.dest == 'icloudpd.container' }}"
-  notify: Restart Aegis Quadlet services
+  notify: Restart Aegis AdGuard Quadlet
 
 - name: Create Aegis systemd-resolved configuration directory
   tags: [aegis, adguard, dns, services]
@@ -168,7 +155,7 @@
     mode: "0644"
   notify:
     - Restart Aegis systemd-resolved
-    - Restart Aegis Quadlet services
+    - Restart Aegis AdGuard Quadlet
 
 - name: Point Aegis resolver at the full systemd-resolved configuration
   tags: [aegis, adguard, dns, services]
@@ -361,7 +348,7 @@
     group: root
     mode: "0755"
 
-- name: Enable Aegis Quadlet services and automatic updates
+- name: Enable Aegis AdGuard Quadlet and automatic updates
   tags: [aegis, containers, services]
   ansible.builtin.systemd:
     name: "{{ item }}"
@@ -370,7 +357,6 @@
     daemon_reload: true
   loop:
     - adguardhome.service
-    - icloudpd.service
     - podman-auto-update.timer
   loop_control:
     label: "{{ item }}"

ansible/roles/profile_aegis/templates/icloudpd.container.j2

@@ -1,20 +0,0 @@
-# Managed by Ansible. Do not edit manually.
-[Unit]
-Description=iCloud Photos Downloader
-Wants=network-online.target
-After=network-online.target
-
-[Container]
-Image={{ aegis_icloudpd_image }}
-Environment=apple_id={{ aegis_icloudpd_apple_id }}
-Environment=folder_structure={{ aegis_icloudpd_folder_structure }}
-Environment=synchronisation_interval={{ aegis_icloudpd_synchronisation_interval }}
-Volume=/var/lib/icloudpd/data:/home/root/iCloud:Z
-Volume=/var/lib/icloudpd/config:/config:Z
-AutoUpdate=registry
-
-[Service]
-Restart=always
-
-[Install]
-WantedBy=multi-user.target

ansible/roles/profile_atlas/defaults/main.yml

@@ -194,6 +194,17 @@ atlas_gitea_restore_test: false
 atlas_gitea_final_restore: false
 atlas_gitea_restore_helper: /usr/local/libexec/atlas-gitea-restore-test
 
+# Declare storage and an inactive Quadlet only. The operator supplies the
+# private configuration, handles MFA, and starts the user service manually.
+atlas_icloudpd_dataset: "{{ atlas_zfs_pool }}/services/data/icloudpd"
+atlas_icloudpd_state_dir: "{{ atlas_app_data_mountpoint }}/icloudpd"
+atlas_icloudpd_config_dir: "{{ atlas_icloudpd_state_dir }}/config"
+atlas_icloudpd_photos_dir: "{{ atlas_archive_mountpoint }}/Pictures/iCloudPD"
+atlas_icloudpd_image: >-
+  docker.io/boredazfcuk/icloudpd@sha256:9966c31ddf0b5b306ac2410b4edd5d626806d96e80c92b83cbb689972dc9389f
+atlas_icloudpd_quadlet_dir: "{{ atlas_admin_home }}/.config/containers/systemd"
+atlas_icloudpd_timezone: Europe/Rome
+
 atlas_45drives_repo_url: https://repo.45drives.com/repofiles/rocky/45drives-enterprise.repo
 atlas_45drives_repo_file: /etc/yum.repos.d/45drives-enterprise.repo
 atlas_45drives_packages:

ansible/roles/profile_atlas/handlers/main.yml

@@ -1,4 +1,14 @@
 ---
+- name: Reload Atlas admin user manager
+  become_user: "{{ atlas_admin_username }}"
+  ansible.builtin.systemd:
+    scope: user
+    daemon_reload: true
+  environment:
+    XDG_RUNTIME_DIR: "/run/user/{{ atlas_admin_uid }}"
+    DBUS_SESSION_BUS_ADDRESS: "unix:path=/run/user/{{ atlas_admin_uid }}/bus"
+  when: not ansible_check_mode
+
 - name: Reload SSH service
   ansible.builtin.systemd:
     name: sshd

ansible/roles/profile_atlas/tasks/icloudpd.yml

@@ -0,0 +1,165 @@
+---
+- name: Require exact Atlas iCloudPD paths and rootless identity
+  tags: [atlas, icloudpd]
+  ansible.builtin.assert:
+    that:
+      - atlas_manage_storage | bool
+      - atlas_icloudpd_dataset == atlas_zfs_pool ~ '/services/data/icloudpd'
+      - atlas_icloudpd_state_dir == atlas_app_data_mountpoint ~ '/icloudpd'
+      - atlas_icloudpd_config_dir == atlas_icloudpd_state_dir ~ '/config'
+      - atlas_icloudpd_photos_dir == atlas_archive_mountpoint ~ '/Pictures/iCloudPD'
+      - atlas_admin_uid | int == 1000
+      - atlas_admin_gid | int == 1000
+      - atlas_icloudpd_image is search('@sha256:[0-9a-f]{64}$')
+    fail_msg: Verify the fixed, separate Atlas iCloudPD photo and state paths.
+
+- name: Declare inactive rootless Atlas iCloudPD storage and Quadlet
+  tags: [atlas, icloudpd]
+  block:
+    - name: Inspect the existing Archive and application-data datasets
+      community.general.zfs_facts:
+        name: "{{ item.dataset }}"
+        properties: name,mounted,mountpoint
+      loop:
+        - dataset: "{{ atlas_zfs_pool }}/{{ atlas_zfs_dataset_archive }}"
+          mountpoint: "{{ atlas_archive_mountpoint }}"
+        - dataset: "{{ atlas_zfs_pool }}/{{ atlas_zfs_dataset_app_data }}"
+          mountpoint: "{{ atlas_app_data_mountpoint }}"
+      loop_control:
+        label: "{{ item.dataset }}"
+      register: atlas_icloudpd_parent_datasets
+
+    - name: Refuse missing or unmounted iCloudPD parent datasets
+      ansible.builtin.assert:
+        that:
+          - item.ansible_facts.ansible_zfs_datasets | length == 1
+          - item.ansible_facts.ansible_zfs_datasets[0].mounted == 'yes'
+          - item.ansible_facts.ansible_zfs_datasets[0].mountpoint == item.item.mountpoint
+      loop: "{{ atlas_icloudpd_parent_datasets.results }}"
+      loop_control:
+        label: "{{ item.item.dataset }}"
+
+    - name: Inspect the existing Pictures namespace and proposed target
+      ansible.builtin.stat:
+        path: "{{ item }}"
+        follow: false
+      loop:
+        - "{{ atlas_archive_mountpoint }}/Pictures"
+        - "{{ atlas_icloudpd_photos_dir }}"
+        - "{{ atlas_icloudpd_photos_dir }}/.atlas-icloudpd-managed"
+      register: atlas_icloudpd_photo_paths
+
+    - name: Refuse to adopt unrelated Pictures data or a symlink
+      ansible.builtin.assert:
+        that:
+          - atlas_icloudpd_photo_paths.results[0].stat.isdir | default(false)
+          - atlas_icloudpd_photo_paths.results[0].stat.uid | int == atlas_admin_uid | int
+          - >-
+            not atlas_icloudpd_photo_paths.results[1].stat.exists or
+            (atlas_icloudpd_photo_paths.results[1].stat.isdir | default(false) and
+             atlas_icloudpd_photo_paths.results[2].stat.isreg | default(false))
+        fail_msg: >-
+          Pictures must exist and be admin-owned; an existing iCloudPD target
+          must carry its managed marker. Never adopt or replace unrelated data.
+
+    - name: Create a dedicated ZFS dataset for iCloudPD configuration and MFA
+      community.general.zfs:
+        name: "{{ atlas_icloudpd_dataset }}"
+        state: present
+        extra_zfs_properties:
+          compression: zstd
+          mountpoint: "{{ atlas_icloudpd_state_dir }}"
+
+    - name: Restrict iCloudPD state and the new photo subtree
+      ansible.builtin.file:
+        path: "{{ item.path }}"
+        state: directory
+        owner: "{{ atlas_admin_username }}"
+        group: "{{ atlas_admin_group }}"
+        mode: "{{ item.mode }}"
+      loop:
+        - path: "{{ atlas_icloudpd_state_dir }}"
+          mode: "0700"
+        - path: "{{ atlas_icloudpd_config_dir }}"
+          mode: "0700"
+        - path: "{{ atlas_icloudpd_photos_dir }}"
+          mode: "0750"
+        - path: "{{ atlas_icloudpd_quadlet_dir }}"
+          mode: "0700"
+      loop_control:
+        label: "{{ item.path }}"
+
+    - name: Mark only the newly managed iCloudPD photo subtree
+      ansible.builtin.copy:
+        content: "Atlas iCloudPD photo subtree; do not remove source photos.\n"
+        dest: "{{ atlas_icloudpd_photos_dir }}/.atlas-icloudpd-managed"
+        owner: "{{ atlas_admin_username }}"
+        group: "{{ atlas_admin_group }}"
+        mode: "0600"
+        force: false
+
+    - name: Install the image's required mounted-filesystem failsafe
+      ansible.builtin.copy:
+        content: ""
+        dest: "{{ atlas_icloudpd_photos_dir }}/.mounted"
+        owner: "{{ atlas_admin_username }}"
+        group: "{{ atlas_admin_group }}"
+        mode: "0644"
+        force: false
+
+    - name: Require the Vault-backed iCloudPD Apple ID
+      ansible.builtin.assert:
+        that:
+          - vault_atlas_icloudpd_apple_id is defined
+          - vault_atlas_icloudpd_apple_id | length > 0
+          - vault_atlas_icloudpd_apple_id != 'REPLACE_ME'
+          - vault_atlas_icloudpd_apple_id.splitlines() | length == 1
+        fail_msg: Configure the existing iCloudPD Apple ID in Vault.
+      no_log: true
+
+    - name: Seed private Atlas iCloudPD configuration when absent
+      ansible.builtin.template:
+        src: atlas-icloudpd.conf.j2
+        dest: "{{ atlas_icloudpd_config_dir }}/icloudpd.conf"
+        owner: "{{ atlas_admin_username }}"
+        group: "{{ atlas_admin_group }}"
+        mode: "0600"
+        force: false
+      no_log: true
+      diff: false
+
+    - name: Keep declared iCloudPD options in the image-managed configuration
+      ansible.builtin.lineinfile:
+        path: "{{ atlas_icloudpd_config_dir }}/icloudpd.conf"
+        regexp: "^{{ item.key }}="
+        line: "{{ item.key }}={{ item.value }}"
+        owner: "{{ atlas_admin_username }}"
+        group: "{{ atlas_admin_group }}"
+        mode: "0600"
+      loop:
+        - {key: apple_id, value: "{{ vault_atlas_icloudpd_apple_id }}"}
+        - {key: authentication_type, value: MFA}
+        - {key: user, value: user}
+        - {key: user_id, value: "1000"}
+        - {key: group, value: group}
+        - {key: group_id, value: "1000"}
+        - {key: download_path, value: /home/user/iCloud}
+        - {key: folder_structure, value: "{:%Y/%m/%d}"}
+        - {key: directory_permissions, value: "750"}
+        - {key: file_permissions, value: "640"}
+        - {key: download_interval, value: "86400"}
+        - {key: auto_delete, value: "false"}
+        - {key: delete_after_download, value: "false"}
+      loop_control:
+        label: "{{ item.key }}"
+      no_log: true
+      diff: false
+
+    - name: Render the rootless Atlas iCloudPD Quadlet
+      ansible.builtin.template:
+        src: atlas-icloudpd.container.j2
+        dest: "{{ atlas_icloudpd_quadlet_dir }}/atlas-icloudpd.container"
+        owner: "{{ atlas_admin_username }}"
+        group: "{{ atlas_admin_group }}"
+        mode: "0644"
+      notify: Reload Atlas admin user manager

ansible/roles/profile_atlas/tasks/main.yml

@@ -20,6 +20,9 @@
 - name: Import staged Atlas rootless Gitea tasks
   ansible.builtin.import_tasks: gitea.yml
 
+- name: Import Atlas iCloudPD storage and inactive Quadlet tasks
+  ansible.builtin.import_tasks: icloudpd.yml
+
 - name: Import explicit Atlas Gitea restore rehearsal tasks
   ansible.builtin.import_tasks: gitea_restore.yml
 

ansible/roles/profile_atlas/templates/atlas-icloudpd.conf.j2

@@ -0,0 +1,14 @@
+# Managed by Ansible. Password, keyring and MFA cookies are stored separately in /config.
+apple_id={{ vault_atlas_icloudpd_apple_id }}
+authentication_type=MFA
+user=user
+user_id=1000
+group=group
+group_id=1000
+download_path=/home/user/iCloud
+folder_structure={:%Y/%m/%d}
+directory_permissions=750
+file_permissions=640
+download_interval=86400
+auto_delete=false
+delete_after_download=false

ansible/roles/profile_atlas/templates/atlas-icloudpd.container.j2

@@ -0,0 +1,22 @@
+# Managed by Ansible. No install target or automatic start.
+[Unit]
+Description=Atlas rootless iCloud Photos Downloader
+RequiresMountsFor={{ atlas_icloudpd_state_dir }} {{ atlas_icloudpd_photos_dir }}
+
+[Container]
+ContainerName=atlas-icloudpd
+Image={{ atlas_icloudpd_image }}
+UserNS=keep-id:uid=1000,gid=1000
+# The image initialises its unprivileged UID 1000 account as container root.
+User=0
+# Upstream launcher requires traceroute for its iCloud reachability check.
+AddCapability=NET_RAW
+Environment=TZ={{ atlas_icloudpd_timezone }}
+Volume={{ atlas_icloudpd_photos_dir }}:/home/user/iCloud:z
+Volume={{ atlas_icloudpd_config_dir }}:/config:Z
+NoNewPrivileges=true
+
+[Service]
+Restart=on-failure
+RestartSec=300
+TimeoutStartSec=900