#!/bin/bash
# {{ ansible_managed }}
# Keep the TrueNAS CIFS mounts healthy and heal immich when they come back.
#
# Why this exists at all: the immich containers are systemd USER units under
# "{{ podman_user }}", while the mounts are SYSTEM units. A user unit cannot
# declare RequiresMountsFor= against a system mount, so there is no native way
# to say "restart immich when this mount returns". This bridges the two scopes.
#
# The failure it addresses: TrueNAS was power-cycled, the .mount units failed,
# and systemd never retried -- mount units are not restarted on failure. SMB
# came back, nothing remounted, and immich-server kept serving an empty library
# while its database still listed 14k assets.
#
# Deliberately NOT `set -e`: one unhealthy mount must not stop the others from
# being checked or healed. Same reasoning as nextcloud-backup-alert.sh.j2.
set -uo pipefail

TAG=cifs-watchdog
STATE_DIR=/var/lib/cifs-watchdog

log()  { logger -t "$TAG" -p daemon.info -- "$*"; echo "$TAG: $*"; }
warn() { logger -t "$TAG" -p daemon.err  -- "$*"; echo "$TAG: $*" >&2; }

install -d -m 0755 "$STATE_DIR"

healed=0

for mp in {{ cifs_watchdog_mounts | map('quote') | join(' ') }}; do
  key="$STATE_DIR/$(systemd-escape -p "$mp")"
  prev="unknown"
  [ -f "$key" ] && prev="$(cat "$key" 2>/dev/null)"

  # `mountpoint -q` alone is NOT sufficient: a CIFS mount whose server vanished
  # stays "mounted" while every read returns EIO. The directory listing is the
  # real health check. timeout guards against a hang despite `soft`.
  if timeout 15 ls -1 "$mp" >/dev/null 2>&1 && mountpoint -q "$mp" 2>/dev/null; then
    state=healthy
  else
    state=unhealthy
  fi

  recovered=0
  if [ "$state" = unhealthy ]; then
    warn "mount=$mp status=unhealthy action=recovering"
    unit="$(systemd-escape -p --suffix=mount "$mp")"
    automount="$(systemd-escape -p --suffix=automount "$mp")"
    # A unit left in `failed` refuses to start again until it is reset.
    systemctl reset-failed "$unit" "$automount" 2>/dev/null
    # Start the .mount unit DIRECTLY -- do not try to trigger the automount by
    # listing the path. An unmounted mount point is still an ordinary empty
    # directory, so `ls` succeeds and tells us nothing; relying on it silently
    # skipped recovery entirely in testing.
    timeout 40 systemctl start "$unit" 2>/dev/null
    # Re-arm the on-access trigger too, so a drop between watchdog ticks is
    # repaired by the next process that touches the path rather than waiting.
    systemctl start "$automount" 2>/dev/null
    if timeout 15 ls -1 "$mp" >/dev/null 2>&1 && mountpoint -q "$mp" 2>/dev/null; then
      state=healthy
      recovered=1
      log "mount=$mp status=recovered"
    else
      warn "mount=$mp status=still-unhealthy"
    fi
  fi

  # Restart when the mount became healthy after being down -- either across
  # ticks (prev=unhealthy) or WITHIN this run (recovered=1). The second case is
  # not redundant: a drop that is detected and repaired by the same invocation
  # leaves prev=healthy, so checking only the stored state silently skips the
  # restart while the container still holds its stale, empty view.
  #
  # Still gated on actually reaching healthy, so a run that fails to recover
  # does not bounce immich on every tick while the NAS is down.
  if [ "$state" = healthy ] && { [ "$prev" = unhealthy ] || [ "$recovered" -eq 1 ]; }; then
    healed=1
  fi

  echo "$state" > "$key"
done

if [ "$healed" -eq 1 ]; then
  # ONLY immich-server: it is the sole consumer of both CIFS paths. Postgres,
  # redis and machine-learning use local volumes and must not be bounced.
  #
  # Rootless podman under "{{ podman_user }}": -H for HOME, the `cd;` preamble
  # is required (see CLAUDE.md), and XDG_RUNTIME_DIR for systemctl --user.
  if sudo -H -u {{ podman_user }} bash -c \
       'cd; export XDG_RUNTIME_DIR=/run/user/$(id -u)
        systemctl --user restart {{ cifs_watchdog_restart_unit | default("immich-server.service") }}' 2>/dev/null; then
    log "status=healed action=restarted unit={{ cifs_watchdog_restart_unit | default('immich-server.service') }}"
  else
    warn "status=healed action=restart-FAILED unit={{ cifs_watchdog_restart_unit | default('immich-server.service') }}"
  fi
fi

exit 0