Bastian de Byl
773a2bbc9c
TrueNAS was power-cycled, the CIFS mounts failed, and systemd never retried
-- mount units are not restarted on failure. SMB came back, nothing
remounted, and immich-server served an empty library for days while its
database still listed 14,181 assets pointing at /mnt/media/originals.
fstab carried no _netdev, no nofail and no automount, so there was no path
back without a human. Now:
x-systemd.automount any access re-attempts the mount; failure stops
being terminal
_netdev / nofail ordered after network-online, dead NAS cannot block boot
soft I/O errors instead of blocking forever, so the
container can be restarted rather than wedging in
uninterruptible sleep
idle-timeout unmount when unused, clearing stale handles
resilienthandles SMB3 rides out brief blips
ansible.posix.mount mounts directly and never starts the generated
.automount unit, leaving the on-access trigger inactive -- enable it
explicitly, or the headline fix silently does nothing.
The containers are systemd USER units while the mounts are SYSTEM units, so
RequiresMountsFor= is unavailable. cifs-watchdog bridges the scopes: checks
health, recovers, and restarts ONLY immich-server (the sole consumer of both
paths; postgres/redis/ML use local volumes).
Two bugs the umount test caught, both worth knowing:
- `ls` cannot test mountedness. An unmounted mount point is an ordinary
empty directory, so ls succeeds and recovery was skipped entirely.
- A drop repaired within a single run leaves prev=healthy, so keying the
restart solely on the stored state skipped it while the container still
held its stale view.
Also moves the SMB password out of /etc/fstab, which is 0644 and was
readable by every local user, into a 0600 credentials file.
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
199 lines
6.0 KiB
YAML
199 lines
6.0 KiB
YAML
---
|
|
- name: create required photos volumes
|
|
become: true
|
|
ansible.builtin.file:
|
|
path: "{{ item }}"
|
|
state: directory
|
|
owner: "{{ podman_subuid.stdout }}"
|
|
group: "{{ podman_user }}"
|
|
mode: 0755
|
|
notify: restorecon podman
|
|
loop:
|
|
- "{{ photos_path }}/psql"
|
|
- "{{ photos_path }}/mlcache"
|
|
- "{{ photos_path }}/storage"
|
|
- "{{ photos_path }}/immich"
|
|
|
|
- name: flush handlers
|
|
ansible.builtin.meta: flush_handlers
|
|
|
|
- name: create cifs credentials directory
|
|
become: true
|
|
ansible.builtin.file:
|
|
path: /etc/cifs
|
|
state: directory
|
|
owner: root
|
|
group: root
|
|
mode: 0700
|
|
|
|
# /etc/fstab is 0644 by design, so an inline `password=` is readable by every
|
|
# local user. Keep the credential in a 0600 file and reference it instead.
|
|
- name: deploy photos cifs credentials
|
|
become: true
|
|
ansible.builtin.template:
|
|
src: cifs-credentials.j2
|
|
dest: /etc/cifs/photos.creds
|
|
owner: root
|
|
group: root
|
|
mode: 0600
|
|
vars:
|
|
cifs_username: photos
|
|
cifs_password: "{{ photos_cifs_pass }}"
|
|
no_log: true
|
|
|
|
# These mounts previously failed permanently whenever TrueNAS was power-cycled:
|
|
# a systemd .mount unit does NOT retry after a failed attempt, so the share came
|
|
# back and nothing remounted, leaving immich serving an empty library while its
|
|
# database still referenced 14k assets.
|
|
#
|
|
# x-systemd.automount the actual fix -- any ACCESS to the path re-attempts the
|
|
# mount, so a failure stops being terminal
|
|
# _netdev / nofail order after network-online; a dead NAS must not block boot
|
|
# soft fail I/O with an error instead of blocking forever, so
|
|
# immich-server can be restarted during an outage rather
|
|
# than wedging in uninterruptible sleep. Accepted trade-off:
|
|
# a write interrupted mid-flight fails and is retried.
|
|
# idle-timeout unmount when unused, which clears stale handles instead
|
|
# of nursing a half-dead connection
|
|
# resilienthandles SMB3 rides out brief server blips transparently
|
|
#
|
|
# See also cifs-watchdog.sh.j2, which restarts immich-server when a mount that
|
|
# was unhealthy becomes healthy again -- the automount restores the FILESYSTEM,
|
|
# but the container still holds the old, empty view until it is bounced.
|
|
- name: mount photos cifs
|
|
become: true
|
|
ansible.posix.mount:
|
|
src: "{{ photos_cifs_src }}"
|
|
path: "{{ photos_path }}/storage"
|
|
fstype: cifs
|
|
opts: "{{ cifs_mount_opts }}"
|
|
state: mounted
|
|
|
|
- name: mount immich cifs
|
|
become: true
|
|
ansible.posix.mount:
|
|
src: "{{ immich_cifs_src }}"
|
|
path: "{{ photos_path }}/immich"
|
|
fstype: cifs
|
|
opts: "{{ cifs_mount_opts }}"
|
|
state: mounted
|
|
|
|
# systemd-fstab-generator creates the .automount unit from x-systemd.automount,
|
|
# but ansible.posix.mount mounts the path directly and never starts it, leaving
|
|
# the on-access trigger INACTIVE -- so a dropped mount stayed dropped, which is
|
|
# the exact failure this work exists to fix. Enable it explicitly.
|
|
- name: enable cifs automount units
|
|
become: true
|
|
ansible.builtin.systemd:
|
|
name: "{{ item }}"
|
|
enabled: true
|
|
state: started
|
|
daemon_reload: true
|
|
loop: "{{ cifs_watchdog_mounts | map('regex_replace', '^/', '') | map('regex_replace', '/', '-') | map('regex_replace', '$', '.automount') | list }}"
|
|
failed_when: false
|
|
|
|
- import_tasks: podman/podman-check.yml
|
|
vars:
|
|
container_name: immich-machine-learning
|
|
container_image: "{{ ml_image }}"
|
|
|
|
- name: create immich-ml container
|
|
become: true
|
|
become_user: "{{ podman_user }}"
|
|
containers.podman.podman_container:
|
|
name: immich-machine-learning
|
|
image: "{{ ml_image }}"
|
|
restart_policy: on-failure:3
|
|
log_driver: journald
|
|
network:
|
|
- shared
|
|
volumes:
|
|
- "{{ photos_path }}/mlcache:/cache"
|
|
|
|
- name: create systemd startup job for immich-machine-learning
|
|
include_tasks: podman/systemd-generate.yml
|
|
vars:
|
|
container_name: immich-machine-learning
|
|
|
|
- import_tasks: podman/podman-check.yml
|
|
vars:
|
|
container_name: immich-redis
|
|
container_image: "{{ redis_image }}"
|
|
|
|
- name: create immich-redis container
|
|
become: true
|
|
become_user: "{{ podman_user }}"
|
|
containers.podman.podman_container:
|
|
name: immich-redis
|
|
image: "{{ redis_image }}"
|
|
restart_policy: on-failure:3
|
|
log_driver: journald
|
|
network:
|
|
- shared
|
|
|
|
- name: create systemd startup job for immich-redis
|
|
include_tasks: podman/systemd-generate.yml
|
|
vars:
|
|
container_name: immich-redis
|
|
|
|
- import_tasks: podman/podman-check.yml
|
|
vars:
|
|
container_name: immich-postgres
|
|
container_image: "{{ db_image }}"
|
|
|
|
- name: create immich-db container
|
|
become: true
|
|
become_user: "{{ podman_user }}"
|
|
containers.podman.podman_container:
|
|
name: immich-postgres
|
|
image: "{{ db_image }}"
|
|
restart_policy: on-failure:3
|
|
log_driver: journald
|
|
network:
|
|
- shared
|
|
env:
|
|
POSTGRES_DATABASE: photos
|
|
POSTGRES_USER: photos
|
|
POSTGRES_PASSWORD: "{{ photos_db_pass }}"
|
|
POSTGRES_INITDB_ARGS: "--data-checksums"
|
|
volumes:
|
|
- "{{ photos_path }}/psql:/var/lib/postgresql/data"
|
|
|
|
- name: create systemd startup job for immich-postgres
|
|
include_tasks: podman/systemd-generate.yml
|
|
vars:
|
|
container_name: immich-postgres
|
|
|
|
- import_tasks: podman/podman-check.yml
|
|
vars:
|
|
container_name: immich-server
|
|
container_image: "{{ image }}"
|
|
|
|
- name: create immich container
|
|
become: true
|
|
become_user: "{{ podman_user }}"
|
|
containers.podman.podman_container:
|
|
name: immich-server
|
|
image: "{{ image }}"
|
|
restart_policy: on-failure:3
|
|
log_driver: journald
|
|
network:
|
|
- shared
|
|
env:
|
|
REDIS_HOSTNAME: immich-redis
|
|
DB_HOSTNAME: immich-postgres
|
|
DB_DATABASE_NAME: photos
|
|
DB_USERNAME: photos
|
|
DB_PASSWORD: "{{ photos_db_pass }}"
|
|
IMMICH_PORT: 8088
|
|
volumes:
|
|
- "{{ photos_path }}/storage:/mnt/media/originals"
|
|
- "{{ photos_path }}/immich:/usr/src/app/upload"
|
|
ports:
|
|
- "8088:8088"
|
|
|
|
- name: create systemd startup job for immich-server
|
|
include_tasks: podman/systemd-generate.yml
|
|
vars:
|
|
container_name: immich-server
|