UPS monitoring, Nextcloud cron, container image bumps

ups role: NUT server on home.debyl.io for the CyberPower PR1500RT2U that backs
both it and truenas.localdomain, with staged shutdown (TrueNAS sheds at t+2min,
host at 10% charge) and best-effort IPMI power-on when mains returns. The 10%
threshold leans on ignorelb + override.battery.charge.low rather than a custom
poller, because CyberPower asserts its own low-battery flag far too early.
Credentials come from vault vars; nothing sensitive is templated in the clear.

Nextcloud background jobs: both instances have backgroundjobs_mode "cron", which
expects an external caller every ~5 minutes, and nothing was calling. The
personal instance had not run a background job since 2026-05-14 and skudak since
2024-11-20. Consequently trash and file versions never expired, stale chunked
uploads accumulated, calendar reminders never fired, and nextcloud.log was never
rotated -- which quietly made the existing log_rotate_size cap inert. Added a
systemd timer per instance, skipping cleanly when the container is down or in
maintenance so deploy windows don't show up as failed units.

Trash retention on the personal instance: the default "auto" only expires when
disk space demands it, so 66 GB of >30-day deletions sat on a host with 1.3 TB
free -- effectively unbounded. "auto, 30" makes the 30-day expiry unconditional
while still purging early under pressure.

Image bumps:
  nextcloud       33.0.0  -> 34.0.2   (both cloud and skudak-cloud)
  greg-time-bot   3.9.25  -> 3.10.0
  fulfillr        20260723.2044 -> 20260728.2155  (prod and dev)

The fulfillr bump records what is already deployed: both containers were rolled
to that image on 2026-07-29 for SCRUM-156 (digital product releases + customer
update campaign). Committing it keeps the repo from claiming an older tag than
the host is actually running, which would otherwise roll fulfillr backwards on
the next clean-checkout deploy.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

ansible/deploy_home.yml

@@ -11,3 +11,5 @@
     - role: github-actions
     - role: graylog-config
       tags: graylog-config
+    - role: ups
+      tags: ups

ansible/roles/podman/tasks/containers/cloud-cron.yml

@@ -0,0 +1,40 @@
+---
+- name: template {{ cron_name }} cron script
+  become: true
+  ansible.builtin.template:
+    src: nextcloud/cloud-cron.sh.j2
+    dest: "{{ cron_script_path }}"
+    owner: root
+    group: root
+    mode: 0755
+    setype: bin_t
+
+- name: template {{ cron_name }} cron systemd service
+  become: true
+  ansible.builtin.template:
+    src: nextcloud/cloud-cron.service.j2
+    dest: "/etc/systemd/system/{{ cron_name }}-cron.service"
+    owner: root
+    group: root
+    mode: 0644
+  vars:
+    instance_name: "{{ cron_name }}"
+
+- name: template {{ cron_name }} cron systemd timer
+  become: true
+  ansible.builtin.template:
+    src: nextcloud/cloud-cron.timer.j2
+    dest: "/etc/systemd/system/{{ cron_name }}-cron.timer"
+    owner: root
+    group: root
+    mode: 0644
+  vars:
+    instance_name: "{{ cron_name }}"
+
+- name: enable and start {{ cron_name }} cron timer
+  become: true
+  ansible.builtin.systemd:
+    name: "{{ cron_name }}-cron.timer"
+    enabled: true
+    state: started
+    daemon_reload: true

ansible/roles/podman/tasks/containers/home/cloud.yml

@@ -96,6 +96,26 @@
   changed_when: "'System config value log_rotate_size' in cloud_log_rotate.stdout"
   failed_when: false
 
+# Nextcloud's default ('auto') only expires trash when disk space is needed,
+# so 66 GB of >30-day deletions sat untouched on a host with 1.3 TB free --
+# the retention was effectively unbounded. 'auto, 30' makes the 30-day
+# expiry unconditional while still purging early under space pressure.
+- name: set nextcloud trashbin retention for cloud
+  become: true
+  become_user: "{{ podman_user }}"
+  ansible.builtin.command: >
+    podman exec -u www-data cloud
+    php occ config:system:set trashbin_retention_obligation --value "auto, 30"
+  register: cloud_trashbin_retention
+  changed_when: "'System config value trashbin_retention_obligation' in cloud_trashbin_retention.stdout"
+  failed_when: false
+
+- include_tasks: containers/cloud-cron.yml
+  vars:
+    cron_name: cloud
+    cron_container: cloud
+    cron_script_path: /usr/local/bin/cloud-cron.sh
+
 - include_tasks: containers/cloud-backup.yml
   vars:
     backup_name: cloud

ansible/roles/podman/tasks/containers/skudak/cloud.yml

@@ -155,6 +155,12 @@
   changed_when: "'System config value log_rotate_size' in skudak_log_rotate.stdout"
   failed_when: false
 
+- include_tasks: containers/cloud-cron.yml
+  vars:
+    cron_name: skudak-cloud
+    cron_container: skudak-cloud
+    cron_script_path: /usr/local/bin/skudak-cloud-cron.sh
+
 - include_tasks: containers/cloud-backup.yml
   vars:
     backup_name: skudak-cloud

ansible/roles/podman/tasks/main.yml

@@ -67,24 +67,24 @@
 - import_tasks: containers/home/cloud.yml
   vars:
     db_image: docker.io/library/mariadb:10.6
-    image: docker.io/library/nextcloud:33.0.0-apache
+    image: docker.io/library/nextcloud:34.0.2-apache
   tags: cloud
 
 - import_tasks: containers/skudak/cloud.yml
   vars:
     db_image: docker.io/library/mariadb:10.6
-    image: docker.io/library/nextcloud:33.0.0-apache
+    image: docker.io/library/nextcloud:34.0.2-apache
   tags: skudak, skudak-cloud
 
 - import_tasks: containers/debyltech/fulfillr.yml
   vars:
-    image: git.debyl.io/debyltech/fulfillr:20260723.2044
+    image: git.debyl.io/debyltech/fulfillr:20260728.2155
   tags: debyltech, fulfillr
 
 # Staging back-office (fulfillr-dev.debyltech.com) — same image, staging Turso config.
 - import_tasks: containers/debyltech/fulfillr-dev.yml
   vars:
-    image: git.debyl.io/debyltech/fulfillr:20260723.2044
+    image: git.debyl.io/debyltech/fulfillr:20260728.2155
   tags: debyltech, fulfillr-dev
 
 - import_tasks: containers/debyltech/uptime-kuma.yml
@@ -109,7 +109,7 @@
 
 - import_tasks: containers/home/gregtime.yml
   vars:
-    image: localhost/greg-time-bot:3.9.25
+    image: localhost/greg-time-bot:3.10.0
   tags: gregtime
 
 - import_tasks: containers/home/zomboid.yml

ansible/roles/podman/templates/nextcloud/cloud-cron.service.j2

@@ -0,0 +1,13 @@
+[Unit]
+Description=Nextcloud {{ instance_name }} background jobs
+After=network-online.target
+Wants=network-online.target
+
+[Service]
+Type=oneshot
+ExecStart={{ cron_script_path }}
+# Type=oneshot disables the start timeout by default, so a wedged cron.php
+# would leave the unit "activating" forever and every subsequent 5-minute
+# trigger would be silently skipped. Bound it.
+TimeoutStartSec={{ cron_timeout | default('30m') }}
+Nice=10

ansible/roles/podman/templates/nextcloud/cloud-cron.sh.j2

@@ -0,0 +1,51 @@
+#!/bin/bash
+# {{ ansible_managed }}
+# Nextcloud "{{ cron_name }}" background jobs (cron.php).
+#
+# backgroundjobs_mode is "cron" on both instances, which means Nextcloud
+# expects an external caller to run cron.php every ~5 minutes. Nothing was:
+# the personal instance had not run a background job since 2026-05-14 and
+# skudak since 2024-11-20. Without it Nextcloud never expires trash or file
+# versions, never cleans stale chunked uploads, never sends calendar
+# reminders, and -- easy to miss -- never rotates nextcloud.log, which makes
+# the log_rotate_size cap set in containers/*/cloud.yml inert.
+set -euo pipefail
+
+TAG=nextcloud-cron
+INSTANCE={{ cron_name }}
+
+log()  { logger -t "$TAG" -p daemon.info -- "instance=$INSTANCE $*"; }
+fail() { logger -t "$TAG" -p daemon.err  -- "instance=$INSTANCE status=failed $*"
+         echo "$TAG: FAILED: $*" >&2; exit 1; }
+
+# The Nextcloud containers are ROOTLESS podman owned by "{{ podman_user }}",
+# but this script runs as root under systemd. Same sudo/cd/XDG_RUNTIME_DIR
+# preamble as cloud-backup.sh -- see CLAUDE.md for why `cd;` is required.
+pexec() {
+  sudo -H -u {{ podman_user }} bash -c \
+    'cd; d=/run/user/$(id -u); [ -d "$d" ] && export XDG_RUNTIME_DIR="$d"
+     exec podman "$@"' _ "$@"
+}
+
+# A container that is down (deploy, image bump, host reboot) is not a failure
+# worth flagging -- the next tick picks it up five minutes later.
+if ! pexec container exists {{ cron_container }} 2>/dev/null; then
+  log "status=skipped reason=container-absent"
+  exit 0
+fi
+
+# occ and cron.php both refuse to do anything useful mid-upgrade. Skipping
+# keeps a deploy window from parading as a run of failed units.
+if pexec exec -u www-data {{ cron_container }} php occ status 2>/dev/null \
+    | grep -q 'maintenance: true'; then
+  log "status=skipped reason=maintenance"
+  exit 0
+fi
+
+set +e
+pexec exec -u www-data {{ cron_container }} php -f /var/www/html/cron.php
+rc=$?
+set -e
+[ "$rc" -eq 0 ] || fail "cron.php exited $rc"
+
+log "status=ok"

ansible/roles/podman/templates/nextcloud/cloud-cron.timer.j2

@@ -0,0 +1,13 @@
+[Unit]
+Description=Nextcloud {{ instance_name }} background jobs every 5 minutes
+
+[Timer]
+OnBootSec={{ cron_onbootsec | default('5m') }}
+OnUnitActiveSec={{ cron_interval | default('5m') }}
+RandomizedDelaySec={{ cron_randomized_delay | default('30s') }}
+# Deliberately NOT Persistent: this runs every 5 minutes, so replaying runs
+# missed while the host was off buys nothing and just stampedes at boot.
+Persistent=false
+
+[Install]
+WantedBy=timers.target

ansible/roles/ups/README.md

@@ -0,0 +1,114 @@
+# ups
+
+UPS monitoring and staged shutdown for the home rack.
+
+A CyberPower PR1500RT2U (`0764:0601`) is cabled by USB to `home.debyl.io` and
+backs both that host and `truenas.localdomain` (Dell PowerEdge R415). This role
+makes `home.debyl.io` the NUT server and gives it the ability to power TrueNAS
+back on over IPMI.
+
+## Outage sequence
+
+| When | What happens | Driven by |
+| --- | --- | --- |
+| t+0 | UPS goes on battery, `ONBATT` logged to journald → Graylog | `upsmon` |
+| t+2min | TrueNAS shuts itself down cleanly, shedding ~200 W | TrueNAS UPS service, Slave mode, `Shutdown Timer 120` |
+| 10% charge | `home.debyl.io` shuts itself down and tells the UPS to cut its output | `upsmon` `SHUTDOWNCMD` + `/lib/systemd/system-shutdown/nutshutdown` |
+| mains returns | UPS re-energizes; both machines power themselves back up | R415 `always-on` restore policy; `home.debyl.io` BIOS *After Power Loss → Power On* |
+| mains back +5min | Best-effort IPMI power-on, if a dedicated iDRAC is ever fitted | `upssched` → `ups-restore.sh` |
+| host boot | Same restore check, for the deep-drain case | `ups-restore.service` |
+
+The 10% threshold is not a custom poller. CyberPower asserts its own low-battery
+flag around 20–35%, so `ups.conf` sets `ignorelb` plus
+`override.battery.charge.low`, and stock `upsmon` fires at exactly the
+configured percentage.
+
+Likewise, the 2-minute TrueNAS shed is TrueNAS's own native "shutdown timer"
+setting — no SSH key and no shutdown script from this side.
+
+## Powering TrueNAS back on
+
+Neither Wake-on-LAN nor IPMI works on this box today, so restore is done with
+the chassis power restore policy instead.
+
+**Wake-on-LAN is out.** `bce0` advertises no WOL capability (`ifconfig -m bce0`
+has no `WOL_MAGIC`) and both NICs are bonded into an LACP `lagg0`.
+
+**IPMI is out too, for now.** The iDRAC is in `shared / LOM1` mode and the
+Enterprise card that would provide a dedicated management port is not fitted:
+
+```
+$ ipmitool sdr elist | grep -i idrac
+iDRAC6 Ent Pres  | 70h | ok  |  7.1 | Absent
+```
+
+A shared-LOM iDRAC6 Express has no standby power. Measured directly: with the
+chassis powered off the BMC does not even answer ARP, and it only reappears
+~200s into POST, at the moment the host brings the NIC link up. That is a
+hardware limitation, not a switch or BIOS problem.
+
+**So restore works like this instead.** `ipmitool chassis policy always-on` is
+set on the R415. On a deep outage `home.debyl.io` halts at 10% and NUT's
+shutdown hook tells the UPS to cut its output; when mains returns the UPS
+re-energizes, the R415 sees AC and boots itself.
+
+The gap is the medium outage — mains returns after TrueNAS has shed but before
+the battery reaches 10%. The UPS never cuts power, so TrueNAS stays off and
+needs a manual power button press. Fitting a used iDRAC6 Enterprise card and
+running `ipmitool delloem lan set dedicated` (plus a cable to the dedicated
+port) closes that gap, and `ups-restore.sh` starts working with no code
+changes — it is already deployed and simply logs and exits while the BMC is
+unreachable.
+
+## One-time setup outside Ansible
+
+These are not managed by this role.
+
+**iDRAC (already done, via `ipmitool` on TrueNAS):**
+
+```
+ipmitool lan set 1 ipsrc static
+ipmitool lan set 1 ipaddr 192.168.1.12
+ipmitool lan set 1 netmask 255.255.255.0
+ipmitool lan set 1 defgw ipaddr 192.168.1.1
+ipmitool lan set 1 access on               # was disabled - nothing answers without this
+ipmitool channel setaccess 1 2 callin=on ipmi=on link=on privilege=4
+ipmitool user set password 2 '<idrac_password>'
+ipmitool chassis policy always-on          # this is what restores power after an outage
+```
+
+`idrac_password` is also the iDRAC web UI password for `root` - they share a
+user database.
+
+**TrueNAS UI → Services → UPS** (enable + start automatically):
+
+| Field | Value |
+| --- | --- |
+| UPS Mode | Slave |
+| Remote Host | `192.168.1.10` |
+| Remote Port | `3493` |
+| Identifier | `cyberpower` |
+| Monitor User | `truenas` |
+| Monitor Password | vault `nut_truenas_password` |
+| Shutdown Mode | UPS goes on battery |
+| Shutdown Timer | `120` |
+| Shutdown Command | `/sbin/shutdown -p now` |
+| Power Off UPS | unchecked |
+
+**BIOS on `home.debyl.io`** (Lenovo 10MR0004US): Power → *After Power Loss* →
+**Power On**. On a full drain the NUT shutdown hook
+(`/lib/systemd/system-shutdown/nutshutdown`) tells the UPS to cut its own
+output; this BIOS setting is what brings the host back when mains returns.
+
+## Vault keys
+
+`nut_upsmon_password`, `nut_truenas_password`, `idrac_password`
+
+## Operating
+
+```
+upsc cyberpower                      # full UPS status
+upsc cyberpower battery.charge
+sudo /usr/local/bin/truenas-power.sh status|on|soft|off
+journalctl -t ups-restore -t ups-sched
+```

ansible/roles/ups/defaults/main.yml

@@ -0,0 +1,36 @@
+---
+# CyberPower PR1500LCDRT2U cabled by USB to this host (0764:0601).
+# This host is the NUT server; truenas.localdomain is a NUT slave.
+ups_name: cyberpower
+# lsusb reports the product string as PR1500LCDRT2U, but the device itself
+# reports device.model CP1500PFCRM2U. The latter is what it actually is.
+ups_desc: CyberPower CP1500PFCRM2U
+ups_vendorid: "0764"
+ups_productid: "0601"
+
+ups_listen_addr: 192.168.1.10
+ups_listen_port: 3493
+
+# truenas.localdomain - the only host allowed to reach upsd
+ups_slave_ip: 192.168.1.11
+
+# Battery charge at which THIS host shuts itself down. TrueNAS sheds much
+# earlier via its own "shutdown timer" setting (see roles/ups/README.md).
+ups_low_charge_pct: 10
+
+# Seconds of stable mains after ONLINE before TrueNAS is powered back on.
+ups_restore_stable_secs: 300
+
+# R415 iDRAC6. Shares LOM1 with bce0.
+idrac_host: 192.168.1.12
+idrac_user: root
+
+ups_deps:
+  [
+    ipmitool,
+    nut,
+    nut-client,
+  ]
+
+# Secrets live in ansible/vars/vault.yml (no vault_ prefix, per repo
+# convention): nut_upsmon_password, nut_truenas_password, idrac_password

ansible/roles/ups/handlers/main.yml

@@ -0,0 +1,35 @@
+---
+# Regenerates the nut-driver@<name> unit instances from ups.conf. Oneshot,
+# so "restarted" just means "run it again".
+- name: reload nut driver units
+  become: true
+  ansible.builtin.systemd:
+    name: nut-driver-enumerator.service
+    state: restarted
+    daemon_reload: true
+
+# The enumerator only writes unit definitions; it will not pick up changed
+# driver options in an already-running driver. Restart the instance itself.
+- name: restart nut driver
+  become: true
+  ansible.builtin.systemd:
+    name: "nut-driver@{{ ups_name }}.service"
+    state: restarted
+
+- name: restart nut server
+  become: true
+  ansible.builtin.systemd:
+    name: nut-server.service
+    state: restarted
+
+- name: restart nut monitor
+  become: true
+  ansible.builtin.systemd:
+    name: nut-monitor.service
+    state: restarted
+
+- name: restart firewalld
+  become: true
+  ansible.builtin.systemd:
+    name: firewalld
+    state: restarted

ansible/roles/ups/tasks/deps.yml

@@ -0,0 +1,7 @@
+---
+- name: install NUT and ipmitool
+  become: true
+  ansible.builtin.dnf:
+    name: "{{ ups_deps }}"
+    state: present
+  tags: ups

ansible/roles/ups/tasks/firewall.yml

@@ -0,0 +1,12 @@
+---
+- name: allow NUT from the truenas slave only
+  become: true
+  ansible.posix.firewalld:
+    rich_rule: >-
+      rule family="ipv4" source address="{{ ups_slave_ip }}/32"
+      port port="{{ ups_listen_port }}" protocol="tcp" accept
+    permanent: true
+    immediate: true
+    state: enabled
+  notify: restart firewalld
+  tags: [ups, firewall]

ansible/roles/ups/tasks/ipmi.yml

@@ -0,0 +1,68 @@
+---
+# IPMI power control for truenas.localdomain (Dell R415, iDRAC6).
+# WOL is not an option there: bce0 advertises no WOL capability and is an
+# LACP lagg member, so IPMI is the only remote power-on path.
+
+- name: deploy iDRAC credential
+  become: true
+  ansible.builtin.copy:
+    content: "{{ idrac_password }}\n"
+    dest: /etc/ups/idrac.pw
+    owner: root
+    group: nut
+    mode: '0640'
+  no_log: true
+  tags: ups
+
+- name: deploy truenas power helper
+  become: true
+  ansible.builtin.template:
+    src: truenas-power.sh.j2
+    dest: /usr/local/bin/truenas-power.sh
+    owner: root
+    group: root
+    mode: '0755'
+    setype: bin_t
+  tags: ups
+
+- name: deploy truenas restore script
+  become: true
+  ansible.builtin.template:
+    src: ups-restore.sh.j2
+    dest: /usr/local/bin/ups-restore.sh
+    owner: root
+    group: root
+    mode: '0755'
+    setype: bin_t
+  tags: ups
+
+- name: deploy upssched command dispatcher
+  become: true
+  ansible.builtin.template:
+    src: ups-sched-cmd.sh.j2
+    dest: /usr/local/bin/ups-sched-cmd.sh
+    owner: root
+    group: root
+    mode: '0755'
+    setype: bin_t
+  tags: ups
+
+# Covers the deep-drain case: if the battery ran out, this host was itself
+# powered off when mains returned, so nothing was running to restore TrueNAS.
+- name: deploy boot-time truenas restore unit
+  become: true
+  ansible.builtin.template:
+    src: ups-restore.service.j2
+    dest: /etc/systemd/system/ups-restore.service
+    owner: root
+    group: root
+    mode: '0644'
+  tags: ups
+
+- name: enable boot-time truenas restore unit
+  become: true
+  ansible.builtin.systemd:
+    name: ups-restore.service
+    enabled: true
+    daemon_reload: true
+  tags: ups

ansible/roles/ups/tasks/main.yml

@@ -0,0 +1,5 @@
+---
+- import_tasks: deps.yml
+- import_tasks: nut.yml
+- import_tasks: ipmi.yml
+- import_tasks: firewall.yml

ansible/roles/ups/tasks/nut.yml

@@ -0,0 +1,129 @@
+---
+# NUT server. The UPS is on USB here, so this host drives it and serves
+# status to truenas.localdomain over the network.
+
+- name: set NUT run mode to netserver
+  become: true
+  ansible.builtin.template:
+    src: nut.conf.j2
+    dest: /etc/ups/nut.conf
+    owner: root
+    group: nut
+    mode: '0640'
+  notify:
+    - reload nut driver units
+    - restart nut driver
+    - restart nut server
+    - restart nut monitor
+  tags: ups
+
+- name: deploy NUT driver configuration
+  become: true
+  ansible.builtin.template:
+    src: ups.conf.j2
+    dest: /etc/ups/ups.conf
+    owner: root
+    group: nut
+    mode: '0640'
+  notify:
+    - reload nut driver units
+    - restart nut driver
+    - restart nut server
+  tags: ups
+
+- name: deploy upsd listener configuration
+  become: true
+  ansible.builtin.template:
+    src: upsd.conf.j2
+    dest: /etc/ups/upsd.conf
+    owner: root
+    group: nut
+    mode: '0640'
+  notify: restart nut server
+  tags: ups
+
+- name: deploy upsd users
+  become: true
+  ansible.builtin.template:
+    src: upsd.users.j2
+    dest: /etc/ups/upsd.users
+    owner: root
+    group: nut
+    mode: '0640'
+  notify:
+    - restart nut server
+    - restart nut monitor
+  tags: ups
+
+- name: deploy upsmon configuration
+  become: true
+  ansible.builtin.template:
+    src: upsmon.conf.j2
+    dest: /etc/ups/upsmon.conf
+    owner: root
+    group: nut
+    mode: '0640'
+  notify: restart nut monitor
+  tags: ups
+
+- name: deploy upssched configuration
+  become: true
+  ansible.builtin.template:
+    src: upssched.conf.j2
+    dest: /etc/ups/upssched.conf
+    owner: root
+    group: nut
+    mode: '0640'
+  notify: restart nut monitor
+  tags: ups
+
+# The nut package ships /usr/lib/udev/rules.d/62-nut-usbups.rules, which
+# hands the UPS USB device to the nut user. Reload and retrigger so the
+# driver can claim it without physically replugging the UPS.
+- name: reload udev rules for NUT USB access
+  become: true
+  ansible.builtin.command:
+    cmd: udevadm control --reload-rules
+  changed_when: false
+  tags: ups
+
+- name: retrigger UPS USB device
+  become: true
+  ansible.builtin.command:
+    cmd: >-
+      udevadm trigger --subsystem-match=usb
+      --attr-match=idVendor={{ ups_vendorid }}
+  changed_when: false
+  tags: ups
+
+# nut-driver-enumerator reads ups.conf and generates nut-driver@{{ ups_name }}.
+# The .service is oneshot (so never "started" for long) and is triggered by
+# the .path unit watching ups.conf - enable both, but only start the .path.
+- name: enable NUT driver enumerator
+  become: true
+  ansible.builtin.systemd:
+    name: nut-driver-enumerator.service
+    enabled: true
+    daemon_reload: true
+  tags: ups
+
+- name: enable and start NUT driver enumerator path trigger
+  become: true
+  ansible.builtin.systemd:
+    name: nut-driver-enumerator.path
+    enabled: true
+    state: started
+  tags: ups
+
+- name: enable and start NUT services
+  become: true
+  ansible.builtin.systemd:
+    name: "{{ item }}"
+    enabled: true
+    state: started
+  loop:
+    - "nut-driver@{{ ups_name }}.service"
+    - nut-server.service
+    - nut-monitor.service
+    - nut.target
+  tags: ups

ansible/roles/ups/templates/nut.conf.j2

@@ -0,0 +1,3 @@
+# {{ ansible_managed }}
+# netserver: run the driver + upsd locally and serve slaves over the network.
+MODE=netserver

ansible/roles/ups/templates/truenas-power.sh.j2

@@ -0,0 +1,34 @@
+#!/bin/bash
+# {{ ansible_managed }}
+# Remote power control for truenas.localdomain via the R415 iDRAC6.
+# Usage: truenas-power.sh {status|on|soft|off|cycle}
+set -euo pipefail
+
+PW_FILE=/etc/ups/idrac.pw
+
+if [ ! -r "$PW_FILE" ]; then
+    echo "cannot read $PW_FILE" >&2
+    exit 1
+fi
+
+# -f keeps the password out of the process argument list.
+ipmi() {
+    ipmitool -I lanplus -H {{ idrac_host }} -U {{ idrac_user }} \
+        -f "$PW_FILE" "$@"
+}
+
+case "${1:-status}" in
+    status) ipmi chassis power status ;;
+    on)     ipmi chassis power on ;;
+    # ACPI soft-off. FreeBSD has hw.acpi.power_button_state=S5, so this is
+    # a clean TrueNAS shutdown. Normally unused: TrueNAS shuts itself down
+    # as a NUT slave. This is the manual escape hatch.
+    soft)   ipmi chassis power soft ;;
+    # Hard cut, last resort only.
+    off)    ipmi chassis power off ;;
+    cycle)  ipmi chassis power cycle ;;
+    *)
+        echo "usage: $0 {status|on|soft|off|cycle}" >&2
+        exit 2
+        ;;
+esac

ansible/roles/ups/templates/ups-restore.service.j2

@@ -0,0 +1,13 @@
+# {{ ansible_managed }}
+[Unit]
+Description=Restore TrueNAS power after an outage
+After=network-online.target nut-server.service nut-monitor.service
+Wants=network-online.target
+Requires=nut-server.service
+
+[Service]
+Type=oneshot
+ExecStart=/usr/local/bin/ups-restore.sh
+
+[Install]
+WantedBy=multi-user.target

ansible/roles/ups/templates/ups-restore.sh.j2

@@ -0,0 +1,63 @@
+#!/bin/bash
+# {{ ansible_managed }}
+# Power truenas.localdomain back on after an outage, but only when it is
+# genuinely safe to do so. Called by upssched once mains has been stable,
+# and once at boot by ups-restore.service (deep-drain recovery).
+#
+# Deliberately stateless: we ask the iDRAC whether the chassis is off
+# rather than tracking whether we were the ones who shut it down. A NAS
+# powered off by hand while on mains is safe, because no ONLINE event
+# fires in that case and the boot path checks UPS status first.
+#
+# NOTE: as of now this is a best-effort secondary path. The R415 has no
+# iDRAC6 Enterprise card ("iDRAC6 Ent Pres ... Absent"), so its shared-LOM
+# BMC has no standby power and goes unreachable whenever the chassis is
+# off - exactly when we would want it. An unreachable iDRAC is therefore
+# the EXPECTED case here, and we exit quietly rather than alarming.
+#
+# The primary restore path needs no IPMI: the R415 power restore policy is
+# set to always-on, so when the UPS cuts and then restores its output the
+# server powers itself back up. Fit an iDRAC6 Enterprise card and switch it
+# to dedicated mode and this script starts working with no changes.
+set -uo pipefail
+
+TAG=ups-restore
+UPS={{ ups_name }}@localhost
+
+log() { logger -t "$TAG" -- "$*"; echo "$TAG: $*"; }
+
+# At boot, upsd may not be serving yet. Wait a bounded amount of time.
+for _ in $(seq 1 30); do
+    if upsc "$UPS" ups.status >/dev/null 2>&1; then
+        break
+    fi
+    sleep 2
+done
+
+status=$(upsc "$UPS" ups.status 2>/dev/null || echo UNKNOWN)
+if [[ "$status" != *OL* ]]; then
+    log "UPS status is '$status', not on line - refusing to power TrueNAS on"
+    exit 0
+fi
+
+power=$(/usr/local/bin/truenas-power.sh status 2>&1 || true)
+case "$power" in
+    *"is off"*)
+        log "mains stable and chassis off - powering TrueNAS on"
+        if /usr/local/bin/truenas-power.sh on; then
+            log "power-on command accepted"
+        else
+            log "power-on command FAILED - check iDRAC at {{ idrac_host }}"
+            exit 1
+        fi
+        ;;
+    *"is on"*)
+        log "TrueNAS already on, nothing to do"
+        ;;
+    *)
+        # Expected while the box has no iDRAC6 Enterprise card: the BMC is
+        # simply not on the network with the chassis powered down.
+        log "iDRAC at {{ idrac_host }} unreachable - relying on the" \
+            "always-on power restore policy instead ($power)"
+        ;;
+esac

ansible/roles/ups/templates/ups-sched-cmd.sh.j2

@@ -0,0 +1,14 @@
+#!/bin/bash
+# {{ ansible_managed }}
+# upssched CMDSCRIPT. Runs as the unprivileged nut user.
+set -uo pipefail
+
+case "${1:-}" in
+    truenas-restore)
+        exec /usr/local/bin/ups-restore.sh
+        ;;
+    *)
+        logger -t ups-sched -- "unknown timer '${1:-}'"
+        exit 1
+        ;;
+esac

ansible/roles/ups/templates/ups.conf.j2

@@ -0,0 +1,18 @@
+# {{ ansible_managed }}
+
+[{{ ups_name }}]
+    driver = usbhid-ups
+    port = auto
+    vendorid = {{ ups_vendorid }}
+    productid = {{ ups_productid }}
+    desc = "{{ ups_desc }}"
+    # CyberPower asserts its own low-battery flag around 20-35%, far too
+    # early for us. Ignore it and derive LB from our own threshold so
+    # upsmon fires SHUTDOWNCMD at exactly {{ ups_low_charge_pct }}%.
+    ignorelb
+    override.battery.charge.low = {{ ups_low_charge_pct }}
+    # Unlock driver.killpower. Without this the driver refuses to cut UPS
+    # output at shutdown, and the R415's always-on power restore policy
+    # would never see AC drop and return - i.e. nothing comes back after a
+    # deep outage. See README.md.
+    allow_killpower

ansible/roles/ups/templates/upsd.conf.j2

@@ -0,0 +1,3 @@
+# {{ ansible_managed }}
+LISTEN 127.0.0.1 {{ ups_listen_port }}
+LISTEN {{ ups_listen_addr }} {{ ups_listen_port }}

ansible/roles/ups/templates/upsd.users.j2

@@ -0,0 +1,11 @@
+# {{ ansible_managed }}
+
+# Local upsmon on this host.
+[upsmon]
+    password = {{ nut_upsmon_password }}
+    upsmon master
+
+# truenas.localdomain, running the TrueNAS UPS service in Slave mode.
+[truenas]
+    password = {{ nut_truenas_password }}
+    upsmon slave

ansible/roles/ups/templates/upsmon.conf.j2

@@ -0,0 +1,40 @@
+# {{ ansible_managed }}
+
+MONITOR {{ ups_name }}@localhost 1 upsmon {{ nut_upsmon_password }} master
+
+MINSUPPLIES 1
+SHUTDOWNCMD "/usr/bin/systemctl poweroff"
+NOTIFYCMD /usr/bin/upssched
+
+# upsmon drops this file before halting; /lib/systemd/system-shutdown/nutshutdown
+# reads it late in shutdown and, if present, tells the UPS to cut its output.
+# That AC drop-and-return is what triggers the R415's always-on restore policy
+# and this host's BIOS "After Power Loss: Power On". upsmon has NO compiled-in
+# default for this - leave it unset and the UPS never powers down.
+# Must be on tmpfs: a persistent path can go stale and make every ordinary
+# reboot look like a forced shutdown.
+POWERDOWNFLAG /run/nut/killpower
+
+POLLFREQ 5
+POLLFREQALERT 5
+
+# Wait up to 30s for the truenas slave to disconnect before we halt.
+HOSTSYNC 30
+DEADTIME 15
+RBWARNTIME 43200
+NOCOMMWARNTIME 300
+FINALDELAY 5
+
+# SYSLOG puts every UPS event in the journal, which fluent-bit already
+# forwards to Graylog (see roles/common/tasks/fluent-bit.yml).
+# EXEC runs NOTIFYCMD, i.e. upssched, which drives the TrueNAS restore.
+NOTIFYFLAG ONLINE    SYSLOG+EXEC
+NOTIFYFLAG ONBATT    SYSLOG+EXEC
+NOTIFYFLAG LOWBATT   SYSLOG+EXEC
+NOTIFYFLAG FSD       SYSLOG+EXEC
+NOTIFYFLAG COMMOK    SYSLOG+EXEC
+NOTIFYFLAG COMMBAD   SYSLOG+EXEC
+NOTIFYFLAG SHUTDOWN  SYSLOG+EXEC
+NOTIFYFLAG REPLBATT  SYSLOG
+NOTIFYFLAG NOCOMM    SYSLOG
+NOTIFYFLAG NOPARENT  SYSLOG

ansible/roles/ups/templates/upssched.conf.j2

@@ -0,0 +1,13 @@
+# {{ ansible_managed }}
+
+CMDSCRIPT /usr/local/bin/ups-sched-cmd.sh
+PIPEFN /run/nut/upssched.pipe
+LOCKFN /run/nut/upssched.lock
+
+# Mains is back: wait for it to hold for {{ ups_restore_stable_secs }}s
+# before powering TrueNAS back on, so we do not flap on unstable power.
+AT ONLINE * CANCEL-TIMER truenas-restore
+AT ONLINE * START-TIMER truenas-restore {{ ups_restore_stable_secs }}
+
+# Power dropped again while the restore timer was pending - stand down.
+AT ONBATT * CANCEL-TIMER truenas-restore