UPS monitoring, Nextcloud cron, container image bumps

ups role: NUT server on home.debyl.io for the CyberPower PR1500RT2U that backs both it and truenas.localdomain, with staged shutdown (TrueNAS sheds at t+2min, host at 10% charge) and best-effort IPMI power-on when mains returns. The 10% threshold leans on ignorelb + override.battery.charge.low rather than a custom poller, because CyberPower asserts its own low-battery flag far too early. Credentials come from vault vars; nothing sensitive is templated in the clear. Nextcloud background jobs: both instances have backgroundjobs_mode "cron", which expects an external caller every ~5 minutes, and nothing was calling. The personal instance had not run a background job since 2026-05-14 and skudak since 2024-11-20. Consequently trash and file versions never expired, stale chunked uploads accumulated, calendar reminders never fired, and nextcloud.log was never rotated -- which quietly made the existing log_rotate_size cap inert. Added a systemd timer per instance, skipping cleanly when the container is down or in maintenance so deploy windows don't show up as failed units. Trash retention on the personal instance: the default "auto" only expires when disk space demands it, so 66 GB of >30-day deletions sat on a host with 1.3 TB free -- effectively unbounded. "auto, 30" makes the 30-day expiry unconditional while still purging early under pressure. Image bumps: nextcloud 33.0.0 -> 34.0.2 (both cloud and skudak-cloud) greg-time-bot 3.9.25 -> 3.10.0 fulfillr 20260723.2044 -> 20260728.2155 (prod and dev) The fulfillr bump records what is already deployed: both containers were rolled to that image on 2026-07-29 for SCRUM-156 (digital product releases + customer update campaign). Committing it keeps the repo from claiming an older tag than the host is actually running, which would otherwise roll fulfillr backwards on the next clean-checkout deploy. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-07-30 13:00:51 -04:00
parent 16145fb6bd
commit 5776dbe1bf
27 changed files with 768 additions and 5 deletions
@@ -0,0 +1,40 @@
+---
+- name: template {{ cron_name }} cron script
+  become: true
+  ansible.builtin.template:
+    src: nextcloud/cloud-cron.sh.j2
+    dest: "{{ cron_script_path }}"
+    owner: root
+    group: root
+    mode: 0755
+    setype: bin_t
+
+- name: template {{ cron_name }} cron systemd service
+  become: true
+  ansible.builtin.template:
+    src: nextcloud/cloud-cron.service.j2
+    dest: "/etc/systemd/system/{{ cron_name }}-cron.service"
+    owner: root
+    group: root
+    mode: 0644
+  vars:
+    instance_name: "{{ cron_name }}"
+
+- name: template {{ cron_name }} cron systemd timer
+  become: true
+  ansible.builtin.template:
+    src: nextcloud/cloud-cron.timer.j2
+    dest: "/etc/systemd/system/{{ cron_name }}-cron.timer"
+    owner: root
+    group: root
+    mode: 0644
+  vars:
+    instance_name: "{{ cron_name }}"
+
+- name: enable and start {{ cron_name }} cron timer
+  become: true
+  ansible.builtin.systemd:
+    name: "{{ cron_name }}-cron.timer"
+    enabled: true
+    state: started
+    daemon_reload: true
@@ -96,6 +96,26 @@
  changed_when: "'System config value log_rotate_size' in cloud_log_rotate.stdout"
  failed_when: false

+# Nextcloud's default ('auto') only expires trash when disk space is needed,
+# so 66 GB of >30-day deletions sat untouched on a host with 1.3 TB free --
+# the retention was effectively unbounded. 'auto, 30' makes the 30-day
+# expiry unconditional while still purging early under space pressure.
+- name: set nextcloud trashbin retention for cloud
+  become: true
+  become_user: "{{ podman_user }}"
+  ansible.builtin.command: >
+    podman exec -u www-data cloud
+    php occ config:system:set trashbin_retention_obligation --value "auto, 30"
+  register: cloud_trashbin_retention
+  changed_when: "'System config value trashbin_retention_obligation' in cloud_trashbin_retention.stdout"
+  failed_when: false
+
+- include_tasks: containers/cloud-cron.yml
+  vars:
+    cron_name: cloud
+    cron_container: cloud
+    cron_script_path: /usr/local/bin/cloud-cron.sh
+
 - include_tasks: containers/cloud-backup.yml
  vars:
    backup_name: cloud
@@ -155,6 +155,12 @@
  changed_when: "'System config value log_rotate_size' in skudak_log_rotate.stdout"
  failed_when: false

+- include_tasks: containers/cloud-cron.yml
+  vars:
+    cron_name: skudak-cloud
+    cron_container: skudak-cloud
+    cron_script_path: /usr/local/bin/skudak-cloud-cron.sh
+
 - include_tasks: containers/cloud-backup.yml
  vars:
    backup_name: skudak-cloud
@@ -67,24 +67,24 @@
 - import_tasks: containers/home/cloud.yml
  vars:
    db_image: docker.io/library/mariadb:10.6
-    image: docker.io/library/nextcloud:33.0.0-apache
+    image: docker.io/library/nextcloud:34.0.2-apache
  tags: cloud

 - import_tasks: containers/skudak/cloud.yml
  vars:
    db_image: docker.io/library/mariadb:10.6
-    image: docker.io/library/nextcloud:33.0.0-apache
+    image: docker.io/library/nextcloud:34.0.2-apache
  tags: skudak, skudak-cloud

 - import_tasks: containers/debyltech/fulfillr.yml
  vars:
-    image: git.debyl.io/debyltech/fulfillr:20260723.2044
+    image: git.debyl.io/debyltech/fulfillr:20260728.2155
  tags: debyltech, fulfillr

 # Staging back-office (fulfillr-dev.debyltech.com) — same image, staging Turso config.
 - import_tasks: containers/debyltech/fulfillr-dev.yml
  vars:
-    image: git.debyl.io/debyltech/fulfillr:20260723.2044
+    image: git.debyl.io/debyltech/fulfillr:20260728.2155
  tags: debyltech, fulfillr-dev

 - import_tasks: containers/debyltech/uptime-kuma.yml
@@ -109,7 +109,7 @@

 - import_tasks: containers/home/gregtime.yml
  vars:
-    image: localhost/greg-time-bot:3.9.25
+    image: localhost/greg-time-bot:3.10.0
  tags: gregtime

 - import_tasks: containers/home/zomboid.yml
@@ -0,0 +1,13 @@
+[Unit]
+Description=Nextcloud {{ instance_name }} background jobs
+After=network-online.target
+Wants=network-online.target
+
+[Service]
+Type=oneshot
+ExecStart={{ cron_script_path }}
+# Type=oneshot disables the start timeout by default, so a wedged cron.php
+# would leave the unit "activating" forever and every subsequent 5-minute
+# trigger would be silently skipped. Bound it.
+TimeoutStartSec={{ cron_timeout | default('30m') }}
+Nice=10
@@ -0,0 +1,51 @@
+#!/bin/bash
+# {{ ansible_managed }}
+# Nextcloud "{{ cron_name }}" background jobs (cron.php).
+#
+# backgroundjobs_mode is "cron" on both instances, which means Nextcloud
+# expects an external caller to run cron.php every ~5 minutes. Nothing was:
+# the personal instance had not run a background job since 2026-05-14 and
+# skudak since 2024-11-20. Without it Nextcloud never expires trash or file
+# versions, never cleans stale chunked uploads, never sends calendar
+# reminders, and -- easy to miss -- never rotates nextcloud.log, which makes
+# the log_rotate_size cap set in containers/*/cloud.yml inert.
+set -euo pipefail
+
+TAG=nextcloud-cron
+INSTANCE={{ cron_name }}
+
+log()  { logger -t "$TAG" -p daemon.info -- "instance=$INSTANCE $*"; }
+fail() { logger -t "$TAG" -p daemon.err  -- "instance=$INSTANCE status=failed $*"
+         echo "$TAG: FAILED: $*" >&2; exit 1; }
+
+# The Nextcloud containers are ROOTLESS podman owned by "{{ podman_user }}",
+# but this script runs as root under systemd. Same sudo/cd/XDG_RUNTIME_DIR
+# preamble as cloud-backup.sh -- see CLAUDE.md for why `cd;` is required.
+pexec() {
+  sudo -H -u {{ podman_user }} bash -c \
+    'cd; d=/run/user/$(id -u); [ -d "$d" ] && export XDG_RUNTIME_DIR="$d"
+     exec podman "$@"' _ "$@"
+}
+
+# A container that is down (deploy, image bump, host reboot) is not a failure
+# worth flagging -- the next tick picks it up five minutes later.
+if ! pexec container exists {{ cron_container }} 2>/dev/null; then
+  log "status=skipped reason=container-absent"
+  exit 0
+fi
+
+# occ and cron.php both refuse to do anything useful mid-upgrade. Skipping
+# keeps a deploy window from parading as a run of failed units.
+if pexec exec -u www-data {{ cron_container }} php occ status 2>/dev/null \
+    | grep -q 'maintenance: true'; then
+  log "status=skipped reason=maintenance"
+  exit 0
+fi
+
+set +e
+pexec exec -u www-data {{ cron_container }} php -f /var/www/html/cron.php
+rc=$?
+set -e
+[ "$rc" -eq 0 ] || fail "cron.php exited $rc"
+
+log "status=ok"
@@ -0,0 +1,13 @@
+[Unit]
+Description=Nextcloud {{ instance_name }} background jobs every 5 minutes
+
+[Timer]
+OnBootSec={{ cron_onbootsec | default('5m') }}
+OnUnitActiveSec={{ cron_interval | default('5m') }}
+RandomizedDelaySec={{ cron_randomized_delay | default('30s') }}
+# Deliberately NOT Persistent: this runs every 5 minutes, so replaying runs
+# missed while the host was off buys nothing and just stampedes at boot.
+Persistent=false
+
+[Install]
+WantedBy=timers.target