From ae99ba415e32c3401a15554eda3396697d3f6082 Mon Sep 17 00:00:00 2001 From: Bastian de Byl Date: Fri, 28 Aug 2026 11:39:57 -0400 Subject: [PATCH] fix(immich): keep the ML models resident, and pin the ML URL explicitly Smart search, face detection and OCR were all failing with Machine learning request to "http://immich-machine-learning:3003" failed while the container still reported Up. Podman only sees PID 1: gunicorn's master was alive and holding the listening socket, but its worker had died at a WORKER TIMEOUT and was never respawned -- hence a connect timeout rather than a refusal. The dead worker was a zombie whose remaining thread was stuck in uninterruptible sleep in exit_mmap, so it survived SIGKILL, podman rm -f and rm -f -t 0, and kept the container name and network alias until the host was rebooted. MACHINE_LEARNING_MODEL_TTL=0 addresses the cause rather than the symptom. The default unloads models after 300s idle, so every search following a gap reloaded four of them (CLIP, buffalo_l detection + recognition, PP-OCRv5) on a CPU-only 4-core box and then tore those mappings back down -- and that teardown is what wedged. Keeping them resident costs ~1-2 GB and removes the path. IMMICH_MACHINE_LEARNING_URL is now set explicitly instead of relying on immich's implicit default, so the ML container can be renamed without silently losing search, and the name is a variable so a replacement can be stood up beside a broken one without editing tasks. Verified after deploy: zero ML failures, "in-memory cache with unloading disabled", ping 200 from immich-server, 26/26 containers healthy. Co-Authored-By: Claude Opus 5 --- ansible/roles/podman/defaults/main.yml | 8 ++++++++ .../podman/tasks/containers/home/photos.yml | 19 ++++++++++++++++--- 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/ansible/roles/podman/defaults/main.yml b/ansible/roles/podman/defaults/main.yml index 77869c9..32c6725 100644 --- a/ansible/roles/podman/defaults/main.yml +++ b/ansible/roles/podman/defaults/main.yml @@ -27,6 +27,14 @@ hass_path: "{{ podman_volumes }}/hass" partsy_path: "{{ podman_volumes }}/partsy" partsy_skudak_path: "{{ podman_volumes }}/partsy-skudak" photos_path: "{{ podman_volumes }}/photos" +# Named rather than hardcoded so the ML service can be stood up beside a broken +# one without editing tasks. On 2026-08-28 a worker thread wedged in +# uninterruptible sleep (D state) in exit_mmap, which made the container +# unremovable -- it survived SIGKILL and `podman rm -f` and kept its name and +# network alias until the host was rebooted. The reboot cleared it, so this +# stays on the canonical name; see MACHINE_LEARNING_MODEL_TTL in photos.yml for +# the change that stops it recurring. +immich_ml_container: immich-machine-learning uptime_kuma_path: "{{ podman_volumes }}/uptime-kuma" uptime_kuma_personal_path: "{{ podman_volumes }}/uptime-kuma-personal" zomboid_path: "{{ podman_volumes }}/zomboid" diff --git a/ansible/roles/podman/tasks/containers/home/photos.yml b/ansible/roles/podman/tasks/containers/home/photos.yml index 127c64f..21dddc5 100644 --- a/ansible/roles/podman/tasks/containers/home/photos.yml +++ b/ansible/roles/podman/tasks/containers/home/photos.yml @@ -94,26 +94,35 @@ - import_tasks: podman/podman-check.yml vars: - container_name: immich-machine-learning + container_name: "{{ immich_ml_container }}" container_image: "{{ ml_image }}" +# MACHINE_LEARNING_MODEL_TTL=0 keeps the models resident instead of unloading +# them after 300s idle. The default made every search following an idle gap +# reload four models (CLIP, buffalo_l detection + recognition, PP-OCRv5) on a +# CPU-only 4-core box, then tear those mappings down again -- and it is that +# teardown, in exit_mmap, that wedged a worker thread in uninterruptible sleep +# on 2026-08-27 and took smart search, face detection and OCR down with it. +# Costs ~1-2 GB resident to remove the code path entirely. - name: create immich-ml container become: true become_user: "{{ podman_user }}" containers.podman.podman_container: - name: immich-machine-learning + name: "{{ immich_ml_container }}" image: "{{ ml_image }}" restart_policy: on-failure:3 log_driver: journald network: - shared + env: + MACHINE_LEARNING_MODEL_TTL: "0" volumes: - "{{ photos_path }}/mlcache:/cache" - name: create systemd startup job for immich-machine-learning include_tasks: podman/systemd-generate.yml vars: - container_name: immich-machine-learning + container_name: "{{ immich_ml_container }}" - import_tasks: podman/podman-check.yml vars: @@ -186,6 +195,10 @@ DB_USERNAME: photos DB_PASSWORD: "{{ photos_db_pass }}" IMMICH_PORT: 8088 + # Set explicitly rather than relying on immich's default of + # http://immich-machine-learning:3003, so the ML container can be renamed + # without silently losing search, faces and OCR. + IMMICH_MACHINE_LEARNING_URL: "http://{{ immich_ml_container }}:3003" volumes: - "{{ photos_path }}/storage:/mnt/media/originals" - "{{ photos_path }}/immich:/usr/src/app/upload"