From 5d0d15f4148691cef7f65f2a3f6b59f081e56cb0 Mon Sep 17 00:00:00 2001
From: Bastian de Byl <bastian@debyl.io>
Date: Sun, 28 Jun 2026 15:35:22 -0400
Subject: [PATCH] SCRUM-97: Healthcheck + restart-on-unhealthy for fulfillr
 containers

After a power cycle a transient HMAC Secrets Manager blip leaves
go-fulfillr's gated routes unregistered (404) with the process still up,
so nothing restarts it. Add a podman healthcheck probing the new
dependency-free /api/v1/health/startup (503 until those routes register)
with healthcheck_failure_action: restart, so podman restarts the
container in place and the next boot self-heals.

- fulfillr.yml + fulfillr-dev.yml: healthcheck via busybox wget (ships in
  the alpine image), interval 30s / timeout 5s / retries 3 /
  start_period 30s (covers the ~14s HMAC retry backoff), failure_action
  restart. Existing restart_policy on-failure:3 kept (process-exit case).
- main.yml: bump fulfillr + fulfillr-dev image to 20260628.1930 (the
  build carrying the /health/startup probe).

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
---
 .../tasks/containers/debyltech/fulfillr-dev.yml       | 11 +++++++++++
 .../podman/tasks/containers/debyltech/fulfillr.yml    | 11 +++++++++++
 ansible/roles/podman/tasks/main.yml                   |  6 +++---
 3 files changed, 25 insertions(+), 3 deletions(-)

diff --git a/ansible/roles/podman/tasks/containers/debyltech/fulfillr-dev.yml b/ansible/roles/podman/tasks/containers/debyltech/fulfillr-dev.yml
index d009108..61ee062 100644
--- a/ansible/roles/podman/tasks/containers/debyltech/fulfillr-dev.yml
+++ b/ansible/roles/podman/tasks/containers/debyltech/fulfillr-dev.yml
@@ -55,6 +55,17 @@
     image_strict: true
     command: --config /config/dev.json
     restart_policy: on-failure:3
+    # Self-heal the SCRUM-97 boot race: if a transient Secrets-Manager blip left the
+    # gated outreach/newsletter/cases routes unregistered, /health/startup returns 503
+    # and podman restarts the container in place (busybox wget ships in the alpine
+    # image; start_period covers the ~14s HMAC retry backoff so a healthy boot is never
+    # flagged).
+    healthcheck: "wget -q -O /dev/null http://localhost:8080/api/v1/health/startup || exit 1"
+    healthcheck_interval: 30s
+    healthcheck_timeout: 5s
+    healthcheck_retries: 3
+    healthcheck_start_period: 30s
+    healthcheck_failure_action: restart
     log_driver: journald
     volumes:
       - "{{ fulfillr_dev_path }}:/config"
diff --git a/ansible/roles/podman/tasks/containers/debyltech/fulfillr.yml b/ansible/roles/podman/tasks/containers/debyltech/fulfillr.yml
index 0caa5ff..a442b2c 100644
--- a/ansible/roles/podman/tasks/containers/debyltech/fulfillr.yml
+++ b/ansible/roles/podman/tasks/containers/debyltech/fulfillr.yml
@@ -52,6 +52,17 @@
     image_strict: true
     command: --config /config/production.json
     restart_policy: on-failure:3
+    # Self-heal the SCRUM-97 boot race: if a transient Secrets-Manager blip left the
+    # gated outreach/newsletter/cases routes unregistered, /health/startup returns 503
+    # and podman restarts the container in place (busybox wget ships in the alpine
+    # image; start_period covers the ~14s HMAC retry backoff so a healthy boot is never
+    # flagged).
+    healthcheck: "wget -q -O /dev/null http://localhost:8080/api/v1/health/startup || exit 1"
+    healthcheck_interval: 30s
+    healthcheck_timeout: 5s
+    healthcheck_retries: 3
+    healthcheck_start_period: 30s
+    healthcheck_failure_action: restart
     log_driver: journald
     volumes:
       - "{{ fulfillr_path }}:/config"
diff --git a/ansible/roles/podman/tasks/main.yml b/ansible/roles/podman/tasks/main.yml
index 3f1d549..9187f5c 100644
--- a/ansible/roles/podman/tasks/main.yml
+++ b/ansible/roles/podman/tasks/main.yml
@@ -78,13 +78,13 @@
 
 - import_tasks: containers/debyltech/fulfillr.yml
   vars:
-    image: git.debyl.io/debyltech/fulfillr:20260614.1925
+    image: git.debyl.io/debyltech/fulfillr:20260628.1930
   tags: debyltech, fulfillr
 
 # Staging back-office (fulfillr-dev.debyltech.com) — same image, staging Turso config.
 - import_tasks: containers/debyltech/fulfillr-dev.yml
   vars:
-    image: git.debyl.io/debyltech/fulfillr:20260614.1925
+    image: git.debyl.io/debyltech/fulfillr:20260628.1930
   tags: debyltech, fulfillr-dev
 
 - import_tasks: containers/debyltech/uptime-kuma.yml
@@ -109,7 +109,7 @@
 
 - import_tasks: containers/home/gregtime.yml
   vars:
-    image: localhost/greg-time-bot:3.9.6
+    image: localhost/greg-time-bot:3.9.14
   tags: gregtime
 
 - import_tasks: containers/home/zomboid.yml