diff --git a/ansible/roles/podman/defaults/main.yml b/ansible/roles/podman/defaults/main.yml index d8db091..c232b88 100644 --- a/ansible/roles/podman/defaults/main.yml +++ b/ansible/roles/podman/defaults/main.yml @@ -213,3 +213,14 @@ podman_prune_users: # Keep 30 days of unused images so a rollback needs no rebuild or re-pull. podman_prune_until: 720h podman_prune_oncalendar: "Sun *-*-* 02:00:00" + +# Hardened CIFS options for the TrueNAS shares (see containers/home/photos.yml). +# x-systemd.automount is what makes a failed mount recoverable without a human. +cifs_mount_opts: >- + credentials=/etc/cifs/photos.creds,uid={{ podman_subuid.stdout }},gid={{ podman_subuid.stdout }},_netdev,nofail,soft,x-systemd.automount,x-systemd.mount-timeout=30,x-systemd.idle-timeout=600,vers=3.1.1,resilienthandles,echo_interval=10 +# How often the watchdog checks mount health and heals immich. +cifs_watchdog_oncalendar: "*:0/5" +cifs_watchdog_mounts: + - "{{ photos_path }}/storage" + - "{{ photos_path }}/immich" + diff --git a/ansible/roles/podman/tasks/containers/home/photos.yml b/ansible/roles/podman/tasks/containers/home/photos.yml index c8b3b5a..127c64f 100644 --- a/ansible/roles/podman/tasks/containers/home/photos.yml +++ b/ansible/roles/podman/tasks/containers/home/photos.yml @@ -17,13 +17,56 @@ - name: flush handlers ansible.builtin.meta: flush_handlers +- name: create cifs credentials directory + become: true + ansible.builtin.file: + path: /etc/cifs + state: directory + owner: root + group: root + mode: 0700 + +# /etc/fstab is 0644 by design, so an inline `password=` is readable by every +# local user. Keep the credential in a 0600 file and reference it instead. +- name: deploy photos cifs credentials + become: true + ansible.builtin.template: + src: cifs-credentials.j2 + dest: /etc/cifs/photos.creds + owner: root + group: root + mode: 0600 + vars: + cifs_username: photos + cifs_password: "{{ photos_cifs_pass }}" + no_log: true + +# These mounts previously failed permanently whenever TrueNAS was power-cycled: +# a systemd .mount unit does NOT retry after a failed attempt, so the share came +# back and nothing remounted, leaving immich serving an empty library while its +# database still referenced 14k assets. +# +# x-systemd.automount the actual fix -- any ACCESS to the path re-attempts the +# mount, so a failure stops being terminal +# _netdev / nofail order after network-online; a dead NAS must not block boot +# soft fail I/O with an error instead of blocking forever, so +# immich-server can be restarted during an outage rather +# than wedging in uninterruptible sleep. Accepted trade-off: +# a write interrupted mid-flight fails and is retried. +# idle-timeout unmount when unused, which clears stale handles instead +# of nursing a half-dead connection +# resilienthandles SMB3 rides out brief server blips transparently +# +# See also cifs-watchdog.sh.j2, which restarts immich-server when a mount that +# was unhealthy becomes healthy again -- the automount restores the FILESYSTEM, +# but the container still holds the old, empty view until it is bounced. - name: mount photos cifs become: true ansible.posix.mount: src: "{{ photos_cifs_src }}" path: "{{ photos_path }}/storage" fstype: cifs - opts: "username=photos,password={{ photos_cifs_pass }},uid={{ podman_subuid.stdout }},gid={{ podman_subuid.stdout }}" + opts: "{{ cifs_mount_opts }}" state: mounted - name: mount immich cifs @@ -32,9 +75,23 @@ src: "{{ immich_cifs_src }}" path: "{{ photos_path }}/immich" fstype: cifs - opts: "username=photos,password={{ photos_cifs_pass }},uid={{ podman_subuid.stdout }},gid={{ podman_subuid.stdout }}" + opts: "{{ cifs_mount_opts }}" state: mounted +# systemd-fstab-generator creates the .automount unit from x-systemd.automount, +# but ansible.posix.mount mounts the path directly and never starts it, leaving +# the on-access trigger INACTIVE -- so a dropped mount stayed dropped, which is +# the exact failure this work exists to fix. Enable it explicitly. +- name: enable cifs automount units + become: true + ansible.builtin.systemd: + name: "{{ item }}" + enabled: true + state: started + daemon_reload: true + loop: "{{ cifs_watchdog_mounts | map('regex_replace', '^/', '') | map('regex_replace', '/', '-') | map('regex_replace', '$', '.automount') | list }}" + failed_when: false + - import_tasks: podman/podman-check.yml vars: container_name: immich-machine-learning diff --git a/ansible/roles/podman/tasks/main.yml b/ansible/roles/podman/tasks/main.yml index 6221098..f0710b3 100644 --- a/ansible/roles/podman/tasks/main.yml +++ b/ansible/roles/podman/tasks/main.yml @@ -5,6 +5,11 @@ - import_tasks: podman/podman-prune.yml tags: podman-prune +# Heals the TrueNAS CIFS mounts and bounces immich when they return. +# Must be deployable independently of the photos containers. +- import_tasks: podman/cifs-watchdog.yml + tags: cifs-watchdog, photos + # WEB SERVER: Caddy is the default and only web server # nginx has been completely replaced and removed diff --git a/ansible/roles/podman/tasks/podman/cifs-watchdog.yml b/ansible/roles/podman/tasks/podman/cifs-watchdog.yml new file mode 100644 index 0000000..e2bce68 --- /dev/null +++ b/ansible/roles/podman/tasks/podman/cifs-watchdog.yml @@ -0,0 +1,36 @@ +--- +- name: template cifs watchdog script + become: true + ansible.builtin.template: + src: cifs-watchdog.sh.j2 + dest: /usr/local/bin/cifs-watchdog.sh + owner: root + group: root + mode: 0755 + setype: bin_t + +- name: template cifs watchdog systemd service + become: true + ansible.builtin.template: + src: cifs-watchdog.service.j2 + dest: /etc/systemd/system/cifs-watchdog.service + owner: root + group: root + mode: 0644 + +- name: template cifs watchdog systemd timer + become: true + ansible.builtin.template: + src: cifs-watchdog.timer.j2 + dest: /etc/systemd/system/cifs-watchdog.timer + owner: root + group: root + mode: 0644 + +- name: enable and start cifs watchdog timer + become: true + ansible.builtin.systemd: + name: cifs-watchdog.timer + enabled: true + state: started + daemon_reload: true diff --git a/ansible/roles/podman/templates/cifs-credentials.j2 b/ansible/roles/podman/templates/cifs-credentials.j2 new file mode 100644 index 0000000..c0ce80a --- /dev/null +++ b/ansible/roles/podman/templates/cifs-credentials.j2 @@ -0,0 +1,8 @@ +# {{ ansible_managed }} +# CIFS credentials for the TrueNAS shares consumed by immich. +# +# This file exists so the password stops living in /etc/fstab, which is +# world-readable (0644) by design -- every local user could read the SMB +# credential straight out of it. Mounted via credentials= instead. +username={{ cifs_username }} +password={{ cifs_password }} diff --git a/ansible/roles/podman/templates/cifs-watchdog.service.j2 b/ansible/roles/podman/templates/cifs-watchdog.service.j2 new file mode 100644 index 0000000..73fb279 --- /dev/null +++ b/ansible/roles/podman/templates/cifs-watchdog.service.j2 @@ -0,0 +1,13 @@ +[Unit] +Description=Verify TrueNAS CIFS mounts and heal immich after recovery +After=network-online.target +Wants=network-online.target + +[Service] +Type=oneshot +ExecStart=/usr/local/bin/cifs-watchdog.sh +# Type=oneshot disables the start timeout by default. Every external call in +# the script is already wrapped in `timeout`, but bound the unit too so a +# wedged run cannot block every subsequent trigger. +TimeoutStartSec={{ cifs_watchdog_timeout | default('5m') }} +Nice=10 diff --git a/ansible/roles/podman/templates/cifs-watchdog.sh.j2 b/ansible/roles/podman/templates/cifs-watchdog.sh.j2 new file mode 100644 index 0000000..06cd141 --- /dev/null +++ b/ansible/roles/podman/templates/cifs-watchdog.sh.j2 @@ -0,0 +1,97 @@ +#!/bin/bash +# {{ ansible_managed }} +# Keep the TrueNAS CIFS mounts healthy and heal immich when they come back. +# +# Why this exists at all: the immich containers are systemd USER units under +# "{{ podman_user }}", while the mounts are SYSTEM units. A user unit cannot +# declare RequiresMountsFor= against a system mount, so there is no native way +# to say "restart immich when this mount returns". This bridges the two scopes. +# +# The failure it addresses: TrueNAS was power-cycled, the .mount units failed, +# and systemd never retried -- mount units are not restarted on failure. SMB +# came back, nothing remounted, and immich-server kept serving an empty library +# while its database still listed 14k assets. +# +# Deliberately NOT `set -e`: one unhealthy mount must not stop the others from +# being checked or healed. Same reasoning as nextcloud-backup-alert.sh.j2. +set -uo pipefail + +TAG=cifs-watchdog +STATE_DIR=/var/lib/cifs-watchdog + +log() { logger -t "$TAG" -p daemon.info -- "$*"; echo "$TAG: $*"; } +warn() { logger -t "$TAG" -p daemon.err -- "$*"; echo "$TAG: $*" >&2; } + +install -d -m 0755 "$STATE_DIR" + +healed=0 + +for mp in {{ cifs_watchdog_mounts | map('quote') | join(' ') }}; do + key="$STATE_DIR/$(systemd-escape -p "$mp")" + prev="unknown" + [ -f "$key" ] && prev="$(cat "$key" 2>/dev/null)" + + # `mountpoint -q` alone is NOT sufficient: a CIFS mount whose server vanished + # stays "mounted" while every read returns EIO. The directory listing is the + # real health check. timeout guards against a hang despite `soft`. + if timeout 15 ls -1 "$mp" >/dev/null 2>&1 && mountpoint -q "$mp" 2>/dev/null; then + state=healthy + else + state=unhealthy + fi + + recovered=0 + if [ "$state" = unhealthy ]; then + warn "mount=$mp status=unhealthy action=recovering" + unit="$(systemd-escape -p --suffix=mount "$mp")" + automount="$(systemd-escape -p --suffix=automount "$mp")" + # A unit left in `failed` refuses to start again until it is reset. + systemctl reset-failed "$unit" "$automount" 2>/dev/null + # Start the .mount unit DIRECTLY -- do not try to trigger the automount by + # listing the path. An unmounted mount point is still an ordinary empty + # directory, so `ls` succeeds and tells us nothing; relying on it silently + # skipped recovery entirely in testing. + timeout 40 systemctl start "$unit" 2>/dev/null + # Re-arm the on-access trigger too, so a drop between watchdog ticks is + # repaired by the next process that touches the path rather than waiting. + systemctl start "$automount" 2>/dev/null + if timeout 15 ls -1 "$mp" >/dev/null 2>&1 && mountpoint -q "$mp" 2>/dev/null; then + state=healthy + recovered=1 + log "mount=$mp status=recovered" + else + warn "mount=$mp status=still-unhealthy" + fi + fi + + # Restart when the mount became healthy after being down -- either across + # ticks (prev=unhealthy) or WITHIN this run (recovered=1). The second case is + # not redundant: a drop that is detected and repaired by the same invocation + # leaves prev=healthy, so checking only the stored state silently skips the + # restart while the container still holds its stale, empty view. + # + # Still gated on actually reaching healthy, so a run that fails to recover + # does not bounce immich on every tick while the NAS is down. + if [ "$state" = healthy ] && { [ "$prev" = unhealthy ] || [ "$recovered" -eq 1 ]; }; then + healed=1 + fi + + echo "$state" > "$key" +done + +if [ "$healed" -eq 1 ]; then + # ONLY immich-server: it is the sole consumer of both CIFS paths. Postgres, + # redis and machine-learning use local volumes and must not be bounced. + # + # Rootless podman under "{{ podman_user }}": -H for HOME, the `cd;` preamble + # is required (see CLAUDE.md), and XDG_RUNTIME_DIR for systemctl --user. + if sudo -H -u {{ podman_user }} bash -c \ + 'cd; export XDG_RUNTIME_DIR=/run/user/$(id -u) + systemctl --user restart {{ cifs_watchdog_restart_unit | default("immich-server.service") }}' 2>/dev/null; then + log "status=healed action=restarted unit={{ cifs_watchdog_restart_unit | default('immich-server.service') }}" + else + warn "status=healed action=restart-FAILED unit={{ cifs_watchdog_restart_unit | default('immich-server.service') }}" + fi +fi + +exit 0 diff --git a/ansible/roles/podman/templates/cifs-watchdog.timer.j2 b/ansible/roles/podman/templates/cifs-watchdog.timer.j2 new file mode 100644 index 0000000..4bb941c --- /dev/null +++ b/ansible/roles/podman/templates/cifs-watchdog.timer.j2 @@ -0,0 +1,13 @@ +[Unit] +Description=Periodic TrueNAS CIFS mount health check + +[Timer] +OnBootSec={{ cifs_watchdog_onbootsec | default('2m') }} +OnCalendar={{ cifs_watchdog_oncalendar }} +RandomizedDelaySec=30s +# Not Persistent: this is a liveness check, so a run missed while the host was +# off carries no meaning -- the next tick reflects current reality. +Persistent=false + +[Install] +WantedBy=timers.target