Backup hardening, UPS monitoring, Nextcloud cron, image bumps

2026-07-30 13:05:20 -04:00
parent 7abec05cc2 5776dbe1bf
commit a63bf5edec
38 changed files with 1097 additions and 11 deletions
@@ -11,3 +11,5 @@
    - role: github-actions
    - role: graylog-config
      tags: graylog-config
+    - role: ups
+      tags: ups
@@ -0,0 +1,34 @@
+---
+# Outbound mail for system notifications (backup failures, cron output).
+#
+# msmtp rather than the esmtp already on the box: the OpenSRS relay uses port
+# 465, which is IMPLICIT TLS, and libesmtp/esmtp only speaks STARTTLS. msmtp
+# handles implicit TLS via `tls_starttls off` and mirrors the settings the
+# TrueNAS box already uses.
+# On Fedora the single `msmtp` package already ships /usr/bin/sendmail and
+# /usr/lib/sendmail; there is no separate msmtp-sendmail package (Debian split).
+- name: install msmtp
+  become: true
+  ansible.builtin.package:
+    name: msmtp
+    state: present
+  tags: mail
+
+- name: configure msmtp
+  become: true
+  ansible.builtin.template:
+    src: msmtp/msmtprc.j2
+    dest: /etc/msmtprc
+    owner: root
+    group: root
+    mode: 0600
+  no_log: true
+  tags: mail
+
+- name: point the mta alternative at msmtp
+  become: true
+  community.general.alternatives:
+    name: mta
+    path: /usr/bin/msmtp-sendmail
+  failed_when: false
+  tags: mail
@@ -3,6 +3,9 @@
 - import_tasks: security.yml
 - import_tasks: service.yml

+- import_tasks: mail.yml
+  tags: mail
+
 - import_tasks: fluent-bit.yml
  tags: fluent-bit, graylog

@@ -0,0 +1,17 @@
+# {{ ansible_managed }}
+defaults
+auth           on
+tls            on
+tls_trust_file /etc/pki/tls/certs/ca-bundle.crt
+logfile        /var/log/msmtp.log
+
+account        home
+host           {{ system_smtp_host | default('mail.b.hostedemail.com') }}
+port           {{ system_smtp_port | default(465) }}
+# Port 465 is implicit TLS (SMTPS), not STARTTLS.
+tls_starttls   off
+from           {{ system_smtp_from | default('home@bdebyl.net') }}
+user           {{ system_smtp_user | default('home@bdebyl.net') }}
+password       {{ home_smtp }}
+
+account default : home
@@ -1,16 +1,23 @@
 # ESP-IDF firmware job image (managed by ansible: roles/gitea-actions).
 # Adds node (required by actions/checkout and other JS actions), the AWS CLI
-# (firmware artifacts ship to S3), and the common-yaml header generator's Python
-# deps on top of the official Espressif toolchain.
+# (firmware artifacts ship to S3), jq (the release script rewrites the protocol
+# manifest with it), and the common-yaml header generator's Python deps on top
+# of the official Espressif toolchain.
 # IDF lives at /opt/esp/idf — firmware jobs source /opt/esp/idf/export.sh.
 # python3-yaml + python3-jinja2 are installed as distro packages so the
 # common-yaml generator runs with a plain `python3 generate.py` — no pip at job
 # time (the base image's system Python is PEP 668 externally-managed) and no
 # need to source the IDF venv just to generate headers.
+#
+# jq is required by esp32-stm32-vcu scripts/release.sh, which publishes the
+# protocol manifest and BLE_API.md to S3 after the firmware upload. Without it
+# the release aborts *after* the firmware and version.json are already live —
+# clients get the new build while the tag, Gitea release and protocol manifest
+# are never written. Keep it installed.
 FROM espressif/idf:{{ esp_idf_version }}

 RUN apt-get update && apt-get install -y --no-install-recommends \
-        curl ca-certificates unzip python3-yaml python3-jinja2 \
+        curl ca-certificates unzip jq python3-yaml python3-jinja2 \
    && curl -fsSL https://deb.nodesource.com/setup_20.x | bash - \
    && apt-get install -y --no-install-recommends nodejs \
    && rm -rf /var/lib/apt/lists/*
@@ -1,4 +1,6 @@
 ---
+# Where Nextcloud backup failure alerts are mailed (see containers/cloud-backup.yml).
+backup_alert_email: bastian@debyl.io
 bookstack_path: "{{ podman_volumes }}/bookstack"
 cam2ip_path: "{{ podman_volumes }}/cam2ip"
 cloud_path: "{{ podman_volumes }}/cloud"
@@ -28,6 +28,27 @@
    mode: 0755
    setype: bin_t

+# Shared by every backup instance. Rendered once per include; the second and
+# later renders are no-ops.
+- name: template nextcloud backup alert script
+  become: true
+  ansible.builtin.template:
+    src: nextcloud/nextcloud-backup-alert.sh.j2
+    dest: /usr/local/bin/nextcloud-backup-alert.sh
+    owner: root
+    group: root
+    mode: 0755
+    setype: bin_t
+
+- name: template nextcloud backup failure handler unit
+  become: true
+  ansible.builtin.template:
+    src: nextcloud/nextcloud-backup-failed@.service.j2
+    dest: /etc/systemd/system/nextcloud-backup-failed@.service
+    owner: root
+    group: root
+    mode: 0644
+
 - name: template {{ backup_name }} backup systemd service
  become: true
  ansible.builtin.template:
@@ -0,0 +1,40 @@
+---
+- name: template {{ cron_name }} cron script
+  become: true
+  ansible.builtin.template:
+    src: nextcloud/cloud-cron.sh.j2
+    dest: "{{ cron_script_path }}"
+    owner: root
+    group: root
+    mode: 0755
+    setype: bin_t
+
+- name: template {{ cron_name }} cron systemd service
+  become: true
+  ansible.builtin.template:
+    src: nextcloud/cloud-cron.service.j2
+    dest: "/etc/systemd/system/{{ cron_name }}-cron.service"
+    owner: root
+    group: root
+    mode: 0644
+  vars:
+    instance_name: "{{ cron_name }}"
+
+- name: template {{ cron_name }} cron systemd timer
+  become: true
+  ansible.builtin.template:
+    src: nextcloud/cloud-cron.timer.j2
+    dest: "/etc/systemd/system/{{ cron_name }}-cron.timer"
+    owner: root
+    group: root
+    mode: 0644
+  vars:
+    instance_name: "{{ cron_name }}"
+
+- name: enable and start {{ cron_name }} cron timer
+  become: true
+  ansible.builtin.systemd:
+    name: "{{ cron_name }}-cron.timer"
+    enabled: true
+    state: started
+    daemon_reload: true
@@ -84,10 +84,44 @@
  vars:
    container_name: cloud

+# Unbounded by default: nextcloud.log.1 had reached 1.12 GB and was being
+# rsynced to TrueNAS and pushed to S3 on every run. Cap at 10 MiB.
+- name: cap nextcloud log rotation size for cloud
+  become: true
+  become_user: "{{ podman_user }}"
+  ansible.builtin.command: >
+    podman exec -u www-data cloud
+    php occ config:system:set log_rotate_size --value 10485760 --type integer
+  register: cloud_log_rotate
+  changed_when: "'System config value log_rotate_size' in cloud_log_rotate.stdout"
+  failed_when: false
+
+# Nextcloud's default ('auto') only expires trash when disk space is needed,
+# so 66 GB of >30-day deletions sat untouched on a host with 1.3 TB free --
+# the retention was effectively unbounded. 'auto, 30' makes the 30-day
+# expiry unconditional while still purging early under space pressure.
+- name: set nextcloud trashbin retention for cloud
+  become: true
+  become_user: "{{ podman_user }}"
+  ansible.builtin.command: >
+    podman exec -u www-data cloud
+    php occ config:system:set trashbin_retention_obligation --value "auto, 30"
+  register: cloud_trashbin_retention
+  changed_when: "'System config value trashbin_retention_obligation' in cloud_trashbin_retention.stdout"
+  failed_when: false
+
+- include_tasks: containers/cloud-cron.yml
+  vars:
+    cron_name: cloud
+    cron_container: cloud
+    cron_script_path: /usr/local/bin/cloud-cron.sh
+
 - include_tasks: containers/cloud-backup.yml
  vars:
    backup_name: cloud
    data_path: "{{ cloud_path }}/data"
+    config_path: "{{ cloud_path }}/config"
+    db_container: cloud-db
    ssh_key_path: /etc/ssh/backup_keys/cloud
    ssh_key_content: "{{ cloud_backup_ssh_key }}"
    ssh_user: cloud
@@ -131,12 +131,50 @@
  changed_when: "'System config value trusted_domains' in trusted_domain_result.stdout"
  failed_when: false

+# This instance was left at loglevel 0 (DEBUG) and had written a 64 GB
+# nextcloud.log, almost entirely repeated deprecation notices. 2 = Warning,
+# which is both the Nextcloud default and what the home instance already uses.
+- name: set nextcloud loglevel for skudak-cloud
+  become: true
+  become_user: "{{ podman_user }}"
+  ansible.builtin.command: >
+    podman exec -u www-data skudak-cloud
+    php occ config:system:set loglevel --value 2 --type integer
+  register: skudak_loglevel
+  changed_when: "'System config value loglevel' in skudak_loglevel.stdout"
+  failed_when: false
+
+# Unbounded by default; see the equivalent task in containers/home/cloud.yml.
+- name: cap nextcloud log rotation size for skudak-cloud
+  become: true
+  become_user: "{{ podman_user }}"
+  ansible.builtin.command: >
+    podman exec -u www-data skudak-cloud
+    php occ config:system:set log_rotate_size --value 10485760 --type integer
+  register: skudak_log_rotate
+  changed_when: "'System config value log_rotate_size' in skudak_log_rotate.stdout"
+  failed_when: false
+
+- include_tasks: containers/cloud-cron.yml
+  vars:
+    cron_name: skudak-cloud
+    cron_container: skudak-cloud
+    cron_script_path: /usr/local/bin/skudak-cloud-cron.sh
+
 - include_tasks: containers/cloud-backup.yml
  vars:
    backup_name: skudak-cloud
    data_path: "{{ cloud_skudak_path }}/data"
+    config_path: "{{ cloud_skudak_path }}/config"
+    db_container: skudak-cloud-db
    ssh_key_path: /etc/ssh/backup_keys/skudak-cloud
    ssh_key_content: "{{ cloud_skudak_backup_ssh_key }}"
    ssh_user: skucloud
    remote_path: /mnt/glacier/skudakcloud
    script_path: /usr/local/bin/skudak-cloud-backup.sh
+    # skudakcloud/data is mode 770, so the receiving side needs traversable
+    # dirs. This flag was hand-added on the host and was being silently
+    # reverted by every `make deploy TAGS=skudak-cloud`; it now lives in git.
+    backup_rsync_extra_args: "--chmod=Du=rwx,Dgo=rx"
+    # Staggered so both instances finish before the 05:00 TrueNAS snapshot.
+    backup_oncalendar: "*-*-* 04:30:00"
@@ -67,24 +67,24 @@
 - import_tasks: containers/home/cloud.yml
  vars:
    db_image: docker.io/library/mariadb:10.6
-    image: docker.io/library/nextcloud:33.0.0-apache
+    image: docker.io/library/nextcloud:34.0.2-apache
  tags: cloud

 - import_tasks: containers/skudak/cloud.yml
  vars:
    db_image: docker.io/library/mariadb:10.6
-    image: docker.io/library/nextcloud:33.0.0-apache
+    image: docker.io/library/nextcloud:34.0.2-apache
  tags: skudak, skudak-cloud

 - import_tasks: containers/debyltech/fulfillr.yml
  vars:
-    image: git.debyl.io/debyltech/fulfillr:20260723.2044
+    image: git.debyl.io/debyltech/fulfillr:20260728.2155
  tags: debyltech, fulfillr

 # Staging back-office (fulfillr-dev.debyltech.com) — same image, staging Turso config.
 - import_tasks: containers/debyltech/fulfillr-dev.yml
  vars:
-    image: git.debyl.io/debyltech/fulfillr:20260723.2044
+    image: git.debyl.io/debyltech/fulfillr:20260728.2155
  tags: debyltech, fulfillr-dev

 - import_tasks: containers/debyltech/uptime-kuma.yml
@@ -109,7 +109,7 @@

 - import_tasks: containers/home/gregtime.yml
  vars:
-    image: localhost/greg-time-bot:3.9.25
+    image: localhost/greg-time-bot:3.10.0
  tags: gregtime

 - import_tasks: containers/home/zomboid.yml
@@ -1,6 +1,14 @@
 [Unit]
 Description=Nextcloud {{ instance_name }} backup to TrueNAS
+After=network-online.target
+Wants=network-online.target
+OnFailure=nextcloud-backup-failed@%n.service

 [Service]
 Type=oneshot
 ExecStart={{ script_path }}
+# Type=oneshot disables the start timeout by default, so a wedged rsync would
+# leave the unit "activating" forever and every subsequent daily trigger would
+# be silently skipped. Bound it.
+TimeoutStartSec={{ backup_timeout | default('4h') }}
+Nice=10
@@ -1,4 +1,117 @@
 #!/bin/bash
+# {{ ansible_managed }}
+# Nextcloud "{{ backup_name }}" -> truenas.localdomain.
+#
+# Ordering is deliberate: the database is dumped BEFORE the file tree is
+# synced. A DB snapshot slightly OLDER than the files degrades to "files
+# Nextcloud has not indexed yet" and is repaired with `occ files:scan`. A DB
+# snapshot NEWER than the files references blobs that never made it into the
+# backup, which surfaces as broken shares and dead file entries on restore.
 set -euo pipefail
-rsync -az --exclude .ssh -e "ssh -i {{ ssh_key_path }} -o StrictHostKeyChecking=accept-new" \
-  {{ data_path }}/ {{ ssh_user }}@truenas.localdomain:{{ remote_path }}/
+
+TAG=nextcloud-backup
+INSTANCE={{ backup_name }}
+STAGE={{ backup_stage_path | default('/var/backups/nextcloud/' ~ backup_name) }}
+KEEP={{ backup_db_keep | default(7) }}
+DUMP="$STAGE/db/${INSTANCE}-$(date +%Y%m%d).sql.gz"
+
+log()  { logger -t "$TAG" -p daemon.info -- "instance=$INSTANCE $*"; echo "$TAG: $*"; }
+fail() { logger -t "$TAG" -p daemon.err  -- "instance=$INSTANCE status=failed $*"
+         echo "$TAG: FAILED: $*" >&2; exit 1; }
+
+SSH="ssh -i {{ ssh_key_path }} -o StrictHostKeyChecking=accept-new -o ServerAliveInterval=30 -o ServerAliveCountMax=6"
+DEST={{ ssh_user }}@truenas.localdomain
+
+log "status=start"
+
+{% if db_container | default('') %}
+# ------------------------------------------------------------ 1. database
+# The Nextcloud containers are ROOTLESS podman owned by "{{ podman_user }}",
+# but this script runs as root under systemd. Every podman call therefore
+# goes through sudo:
+#   -H    HOME becomes the podman user's home, so podman finds its rootless
+#         graph root under ~/.local/share/containers
+#   cd;   required preamble (see CLAUDE.md) so the shell starts in that home
+#   XDG_RUNTIME_DIR  the podman user's runtime dir. Lingering is enabled by
+#         roles/podman/tasks/podman/podman.yml so /run/user/<uid> exists;
+#         guarded anyway so podman falls back cleanly if it ever does not.
+#
+# No credential is stored in this file or placed on a host command line:
+# $MYSQL_ROOT_PASSWORD and $MYSQL_DATABASE are expanded by the shell INSIDE
+# the database container, which already carries them in its environment.
+#
+# --routines and --events initially aborted the dump here: mysql.proc read as
+# corrupted (error 1728) and the event scheduler reported disabled (1577),
+# both artefacts of an image bump without mariadb-upgrade. `mariadb-upgrade
+# --force` has since been run against both instances and repaired the system
+# tables, so the full flag set works and is kept for completeness.
+#
+# Note that upgrade still exits non-zero on these containers: it cannot
+# install the `sys` schema because the datadir root (/var/lib/mysql) is owned
+# by daemon rather than mysql, so mysqld may not create new top-level
+# databases. `sys` is purely diagnostic and unused by Nextcloud, so this is
+# cosmetic -- but it does mean creating a NEW database would fail too.
+pexec() {
+  sudo -H -u {{ podman_user }} bash -c \
+    'cd; d=/run/user/$(id -u); [ -d "$d" ] && export XDG_RUNTIME_DIR="$d"
+     exec podman "$@"' _ "$@"
+}
+
+install -d -m 0700 "$STAGE" "$STAGE/db"
+tmp="$DUMP.tmp"
+rm -f "$tmp"
+
+log "dumping {{ db_container }}"
+set +e
+pexec exec {{ db_container }} sh -c '
+  exec env MYSQL_PWD="$MYSQL_ROOT_PASSWORD" mariadb-dump -u root \
+    --single-transaction --quick --routines --events --triggers \
+    --no-tablespaces --default-character-set=utf8mb4 "$MYSQL_DATABASE"
+' | gzip -6 > "$tmp"
+dump_rc=${PIPESTATUS[0]}
+set -e
+[ "$dump_rc" -eq 0 ] || fail "mariadb-dump {{ db_container }} exited $dump_rc"
+
+# A new dump is promoted over yesterday's only after it proves complete:
+# a valid gzip stream AND the "-- Dump completed" trailer that mariadb-dump
+# writes only on a clean finish. `mv` is atomic within the staging
+# filesystem, so a failed or truncated run can never replace a good dump.
+gzip -t "$tmp" || fail "dump is not a valid gzip stream"
+gunzip -c "$tmp" | tail -c 512 | grep -q 'Dump completed' \
+  || fail "dump is truncated (no completion trailer)"
+mv -f "$tmp" "$DUMP"
+log "db_dump=ok bytes=$(stat -c %s "$DUMP")"
+
+# Local retention. The staging sync below mirrors with --delete, so remote
+# retention follows the same window; deeper history comes from the TrueNAS
+# periodic ZFS snapshots (see roles/podman/README.md).
+ls -1t "$STAGE"/db/"$INSTANCE"-*.sql.gz | tail -n +$((KEEP + 1)) | xargs -r rm -f
+{% endif %}
+
+# ----------------------------------------------------------- 2. file tree
+# --exclude .ssh is load-bearing: {{ remote_path }} IS {{ ssh_user }}'s home
+# on TrueNAS and its authorized_keys lives there, so a `.ssh` directory
+# appearing in the data tree must never be shipped. No --delete here: the
+# data tree is append-mostly and a source-side mishap must not propagate.
+log "syncing data"
+rsync -az --timeout=1800 --exclude .ssh \
+  {{ backup_rsync_excludes | default("--exclude '/nextcloud.log*' --exclude '/updater.log'") }} \
+  {{ backup_rsync_extra_args | default('') }} \
+  -e "$SSH" {{ data_path }}/ "$DEST:{{ remote_path }}/"
+
+# ------------------------------------------- 3. config/ and database dumps
+{# --mkpath creates the nested _backup/<x>/ destination; rsync will not build
+   more than one missing level on its own. Requires rsync >= 3.2.3 on both
+   ends (galactica 3.4.1, truenas 3.2.7). #}
+{% if config_path | default('') %}
+log "syncing config"
+rsync -az --timeout=600 --delete --mkpath {{ backup_rsync_extra_args | default('') }} \
+  -e "$SSH" {{ config_path }}/ "$DEST:{{ remote_path }}/_backup/config/"
+{% endif %}
+{% if db_container | default('') %}
+log "syncing db dumps"
+rsync -az --timeout=600 --delete --mkpath {{ backup_rsync_extra_args | default('') }} \
+  -e "$SSH" "$STAGE/db/" "$DEST:{{ remote_path }}/_backup/db/"
+{% endif %}
+
+log "status=ok"
@@ -2,7 +2,8 @@
 Description=Daily Nextcloud {{ instance_name }} backup

 [Timer]
-OnCalendar=*-*-* 04:00:00
+OnCalendar={{ backup_oncalendar | default('*-*-* 04:00:00') }}
+RandomizedDelaySec={{ backup_randomized_delay | default('5m') }}
 Persistent=true

 [Install]
@@ -0,0 +1,13 @@
+[Unit]
+Description=Nextcloud {{ instance_name }} background jobs
+After=network-online.target
+Wants=network-online.target
+
+[Service]
+Type=oneshot
+ExecStart={{ cron_script_path }}
+# Type=oneshot disables the start timeout by default, so a wedged cron.php
+# would leave the unit "activating" forever and every subsequent 5-minute
+# trigger would be silently skipped. Bound it.
+TimeoutStartSec={{ cron_timeout | default('30m') }}
+Nice=10
@@ -0,0 +1,51 @@
+#!/bin/bash
+# {{ ansible_managed }}
+# Nextcloud "{{ cron_name }}" background jobs (cron.php).
+#
+# backgroundjobs_mode is "cron" on both instances, which means Nextcloud
+# expects an external caller to run cron.php every ~5 minutes. Nothing was:
+# the personal instance had not run a background job since 2026-05-14 and
+# skudak since 2024-11-20. Without it Nextcloud never expires trash or file
+# versions, never cleans stale chunked uploads, never sends calendar
+# reminders, and -- easy to miss -- never rotates nextcloud.log, which makes
+# the log_rotate_size cap set in containers/*/cloud.yml inert.
+set -euo pipefail
+
+TAG=nextcloud-cron
+INSTANCE={{ cron_name }}
+
+log()  { logger -t "$TAG" -p daemon.info -- "instance=$INSTANCE $*"; }
+fail() { logger -t "$TAG" -p daemon.err  -- "instance=$INSTANCE status=failed $*"
+         echo "$TAG: FAILED: $*" >&2; exit 1; }
+
+# The Nextcloud containers are ROOTLESS podman owned by "{{ podman_user }}",
+# but this script runs as root under systemd. Same sudo/cd/XDG_RUNTIME_DIR
+# preamble as cloud-backup.sh -- see CLAUDE.md for why `cd;` is required.
+pexec() {
+  sudo -H -u {{ podman_user }} bash -c \
+    'cd; d=/run/user/$(id -u); [ -d "$d" ] && export XDG_RUNTIME_DIR="$d"
+     exec podman "$@"' _ "$@"
+}
+
+# A container that is down (deploy, image bump, host reboot) is not a failure
+# worth flagging -- the next tick picks it up five minutes later.
+if ! pexec container exists {{ cron_container }} 2>/dev/null; then
+  log "status=skipped reason=container-absent"
+  exit 0
+fi
+
+# occ and cron.php both refuse to do anything useful mid-upgrade. Skipping
+# keeps a deploy window from parading as a run of failed units.
+if pexec exec -u www-data {{ cron_container }} php occ status 2>/dev/null \
+    | grep -q 'maintenance: true'; then
+  log "status=skipped reason=maintenance"
+  exit 0
+fi
+
+set +e
+pexec exec -u www-data {{ cron_container }} php -f /var/www/html/cron.php
+rc=$?
+set -e
+[ "$rc" -eq 0 ] || fail "cron.php exited $rc"
+
+log "status=ok"
@@ -0,0 +1,13 @@
+[Unit]
+Description=Nextcloud {{ instance_name }} background jobs every 5 minutes
+
+[Timer]
+OnBootSec={{ cron_onbootsec | default('5m') }}
+OnUnitActiveSec={{ cron_interval | default('5m') }}
+RandomizedDelaySec={{ cron_randomized_delay | default('30s') }}
+# Deliberately NOT Persistent: this runs every 5 minutes, so replaying runs
+# missed while the host was off buys nothing and just stampedes at boot.
+Persistent=false
+
+[Install]
+WantedBy=timers.target
@@ -0,0 +1,65 @@
+#!/bin/bash
+# {{ ansible_managed }}
+# OnFailure= handler for the Nextcloud backup units. Invoked as:
+#   nextcloud-backup-alert.sh <failed-unit-name>
+#
+# Deliberately NOT `set -e`: an alert handler that dies partway through
+# reports nothing, which is worse than a partial report. Same reasoning as
+# roles/ups/templates/ups-restore.sh.j2.
+set -uo pipefail
+
+TAG=nextcloud-backup
+UNIT="${1:-unknown}"
+TO="{{ backup_alert_email | default('root') }}"
+HOST="$(hostname -f 2>/dev/null || hostname)"
+
+result="$(systemctl show -p Result --value "$UNIT" 2>/dev/null)"
+code="$(systemctl show -p ExecMainStatus --value "$UNIT" 2>/dev/null)"
+
+# Only claim a failure when the unit actually reports one. Starting this
+# handler by hand (or any other spurious trigger) would otherwise mail out a
+# subject line saying FAILED about a run that succeeded. Keeping status=failed
+# exact also stops such triggers matching the Graylog alert rule.
+if [ "${result:-success}" = "success" ]; then
+  state=spurious
+  prio=daemon.warning
+  headline="$(printf 'Nextcloud backup alert handler was invoked on %s, but %s reports SUCCESS.\nThis is not a backup failure -- most likely the handler was started manually.' "$HOST" "$UNIT")"
+  subject="[$HOST] Nextcloud backup alert (spurious, unit OK): $UNIT"
+else
+  state=failed
+  prio=daemon.err
+  headline="Nextcloud backup FAILED on $HOST"
+  subject="[$HOST] Nextcloud backup FAILED: $UNIT"
+fi
+
+# One machine-parseable line for Graylog, then the context.
+logger -t "$TAG" -p "$prio" -- \
+  "status=$state unit=$UNIT result=${result:-unknown} exit=${code:-unknown}"
+
+body="$(printf '%s\n\nunit:   %s\nresult: %s\nexit:   %s\n\n--- last 40 journal lines ---\n' \
+  "$headline" "$UNIT" "${result:-unknown}" "${code:-unknown}")
+$(journalctl -u "$UNIT" -n 40 --no-pager -o cat 2>/dev/null)"
+
+echo "$body" | logger -t "$TAG" -p "$prio"
+
+# Only genuine failures are worth an email. A spurious invocation carries no
+# action for a human, and mailing it trains the reader to ignore the subject
+# line -- which defeats the point of having the alert at all. The journald
+# record above is kept either way, so spurious triggers stay greppable.
+if [ "$state" != "failed" ]; then
+  logger -t "$TAG" -p daemon.info -- "alert_mail=skipped reason=$state"
+  exit 0
+fi
+
+# Mail is best-effort: if the MTA is not configured the journald record above
+# is still the authoritative signal, so never fail the handler on this.
+if command -v sendmail >/dev/null 2>&1; then
+  printf 'To: %s\nSubject: %s\nContent-Type: text/plain; charset=UTF-8\n\n%s\n' \
+    "$TO" "$subject" "$body" | sendmail -t \
+    && logger -t "$TAG" -p daemon.info -- "alert_mail=sent to=$TO" \
+    || logger -t "$TAG" -p daemon.err  -- "alert_mail=failed to=$TO"
+else
+  logger -t "$TAG" -p daemon.err -- "alert_mail=skipped reason=no-sendmail"
+fi
+
+exit 0
@@ -0,0 +1,6 @@
+[Unit]
+Description=Report failure of %i
+
+[Service]
+Type=oneshot
+ExecStart=/usr/local/bin/nextcloud-backup-alert.sh %i
@@ -0,0 +1,114 @@
+# ups
+
+UPS monitoring and staged shutdown for the home rack.
+
+A CyberPower PR1500RT2U (`0764:0601`) is cabled by USB to `home.debyl.io` and
+backs both that host and `truenas.localdomain` (Dell PowerEdge R415). This role
+makes `home.debyl.io` the NUT server and gives it the ability to power TrueNAS
+back on over IPMI.
+
+## Outage sequence
+
+| When | What happens | Driven by |
+| --- | --- | --- |
+| t+0 | UPS goes on battery, `ONBATT` logged to journald → Graylog | `upsmon` |
+| t+2min | TrueNAS shuts itself down cleanly, shedding ~200 W | TrueNAS UPS service, Slave mode, `Shutdown Timer 120` |
+| 10% charge | `home.debyl.io` shuts itself down and tells the UPS to cut its output | `upsmon` `SHUTDOWNCMD` + `/lib/systemd/system-shutdown/nutshutdown` |
+| mains returns | UPS re-energizes; both machines power themselves back up | R415 `always-on` restore policy; `home.debyl.io` BIOS *After Power Loss → Power On* |
+| mains back +5min | Best-effort IPMI power-on, if a dedicated iDRAC is ever fitted | `upssched` → `ups-restore.sh` |
+| host boot | Same restore check, for the deep-drain case | `ups-restore.service` |
+
+The 10% threshold is not a custom poller. CyberPower asserts its own low-battery
+flag around 20–35%, so `ups.conf` sets `ignorelb` plus
+`override.battery.charge.low`, and stock `upsmon` fires at exactly the
+configured percentage.
+
+Likewise, the 2-minute TrueNAS shed is TrueNAS's own native "shutdown timer"
+setting — no SSH key and no shutdown script from this side.
+
+## Powering TrueNAS back on
+
+Neither Wake-on-LAN nor IPMI works on this box today, so restore is done with
+the chassis power restore policy instead.
+
+**Wake-on-LAN is out.** `bce0` advertises no WOL capability (`ifconfig -m bce0`
+has no `WOL_MAGIC`) and both NICs are bonded into an LACP `lagg0`.
+
+**IPMI is out too, for now.** The iDRAC is in `shared / LOM1` mode and the
+Enterprise card that would provide a dedicated management port is not fitted:
+
+```
+$ ipmitool sdr elist | grep -i idrac
+iDRAC6 Ent Pres  | 70h | ok  |  7.1 | Absent
+```
+
+A shared-LOM iDRAC6 Express has no standby power. Measured directly: with the
+chassis powered off the BMC does not even answer ARP, and it only reappears
+~200s into POST, at the moment the host brings the NIC link up. That is a
+hardware limitation, not a switch or BIOS problem.
+
+**So restore works like this instead.** `ipmitool chassis policy always-on` is
+set on the R415. On a deep outage `home.debyl.io` halts at 10% and NUT's
+shutdown hook tells the UPS to cut its output; when mains returns the UPS
+re-energizes, the R415 sees AC and boots itself.
+
+The gap is the medium outage — mains returns after TrueNAS has shed but before
+the battery reaches 10%. The UPS never cuts power, so TrueNAS stays off and
+needs a manual power button press. Fitting a used iDRAC6 Enterprise card and
+running `ipmitool delloem lan set dedicated` (plus a cable to the dedicated
+port) closes that gap, and `ups-restore.sh` starts working with no code
+changes — it is already deployed and simply logs and exits while the BMC is
+unreachable.
+
+## One-time setup outside Ansible
+
+These are not managed by this role.
+
+**iDRAC (already done, via `ipmitool` on TrueNAS):**
+
+```
+ipmitool lan set 1 ipsrc static
+ipmitool lan set 1 ipaddr 192.168.1.12
+ipmitool lan set 1 netmask 255.255.255.0
+ipmitool lan set 1 defgw ipaddr 192.168.1.1
+ipmitool lan set 1 access on               # was disabled - nothing answers without this
+ipmitool channel setaccess 1 2 callin=on ipmi=on link=on privilege=4
+ipmitool user set password 2 '<idrac_password>'
+ipmitool chassis policy always-on          # this is what restores power after an outage
+```
+
+`idrac_password` is also the iDRAC web UI password for `root` - they share a
+user database.
+
+**TrueNAS UI → Services → UPS** (enable + start automatically):
+
+| Field | Value |
+| --- | --- |
+| UPS Mode | Slave |
+| Remote Host | `192.168.1.10` |
+| Remote Port | `3493` |
+| Identifier | `cyberpower` |
+| Monitor User | `truenas` |
+| Monitor Password | vault `nut_truenas_password` |
+| Shutdown Mode | UPS goes on battery |
+| Shutdown Timer | `120` |
+| Shutdown Command | `/sbin/shutdown -p now` |
+| Power Off UPS | unchecked |
+
+**BIOS on `home.debyl.io`** (Lenovo 10MR0004US): Power → *After Power Loss* →
+**Power On**. On a full drain the NUT shutdown hook
+(`/lib/systemd/system-shutdown/nutshutdown`) tells the UPS to cut its own
+output; this BIOS setting is what brings the host back when mains returns.
+
+## Vault keys
+
+`nut_upsmon_password`, `nut_truenas_password`, `idrac_password`
+
+## Operating
+
+```
+upsc cyberpower                      # full UPS status
+upsc cyberpower battery.charge
+sudo /usr/local/bin/truenas-power.sh status|on|soft|off
+journalctl -t ups-restore -t ups-sched
+```
@@ -0,0 +1,36 @@
+---
+# CyberPower PR1500LCDRT2U cabled by USB to this host (0764:0601).
+# This host is the NUT server; truenas.localdomain is a NUT slave.
+ups_name: cyberpower
+# lsusb reports the product string as PR1500LCDRT2U, but the device itself
+# reports device.model CP1500PFCRM2U. The latter is what it actually is.
+ups_desc: CyberPower CP1500PFCRM2U
+ups_vendorid: "0764"
+ups_productid: "0601"
+
+ups_listen_addr: 192.168.1.10
+ups_listen_port: 3493
+
+# truenas.localdomain - the only host allowed to reach upsd
+ups_slave_ip: 192.168.1.11
+
+# Battery charge at which THIS host shuts itself down. TrueNAS sheds much
+# earlier via its own "shutdown timer" setting (see roles/ups/README.md).
+ups_low_charge_pct: 10
+
+# Seconds of stable mains after ONLINE before TrueNAS is powered back on.
+ups_restore_stable_secs: 300
+
+# R415 iDRAC6. Shares LOM1 with bce0.
+idrac_host: 192.168.1.12
+idrac_user: root
+
+ups_deps:
+  [
+    ipmitool,
+    nut,
+    nut-client,
+  ]
+
+# Secrets live in ansible/vars/vault.yml (no vault_ prefix, per repo
+# convention): nut_upsmon_password, nut_truenas_password, idrac_password
@@ -0,0 +1,35 @@
+---
+# Regenerates the nut-driver@<name> unit instances from ups.conf. Oneshot,
+# so "restarted" just means "run it again".
+- name: reload nut driver units
+  become: true
+  ansible.builtin.systemd:
+    name: nut-driver-enumerator.service
+    state: restarted
+    daemon_reload: true
+
+# The enumerator only writes unit definitions; it will not pick up changed
+# driver options in an already-running driver. Restart the instance itself.
+- name: restart nut driver
+  become: true
+  ansible.builtin.systemd:
+    name: "nut-driver@{{ ups_name }}.service"
+    state: restarted
+
+- name: restart nut server
+  become: true
+  ansible.builtin.systemd:
+    name: nut-server.service
+    state: restarted
+
+- name: restart nut monitor
+  become: true
+  ansible.builtin.systemd:
+    name: nut-monitor.service
+    state: restarted
+
+- name: restart firewalld
+  become: true
+  ansible.builtin.systemd:
+    name: firewalld
+    state: restarted
@@ -0,0 +1,7 @@
+---
+- name: install NUT and ipmitool
+  become: true
+  ansible.builtin.dnf:
+    name: "{{ ups_deps }}"
+    state: present
+  tags: ups
@@ -0,0 +1,12 @@
+---
+- name: allow NUT from the truenas slave only
+  become: true
+  ansible.posix.firewalld:
+    rich_rule: >-
+      rule family="ipv4" source address="{{ ups_slave_ip }}/32"
+      port port="{{ ups_listen_port }}" protocol="tcp" accept
+    permanent: true
+    immediate: true
+    state: enabled
+  notify: restart firewalld
+  tags: [ups, firewall]
@@ -0,0 +1,68 @@
+---
+# IPMI power control for truenas.localdomain (Dell R415, iDRAC6).
+# WOL is not an option there: bce0 advertises no WOL capability and is an
+# LACP lagg member, so IPMI is the only remote power-on path.
+
+- name: deploy iDRAC credential
+  become: true
+  ansible.builtin.copy:
+    content: "{{ idrac_password }}\n"
+    dest: /etc/ups/idrac.pw
+    owner: root
+    group: nut
+    mode: '0640'
+  no_log: true
+  tags: ups
+
+- name: deploy truenas power helper
+  become: true
+  ansible.builtin.template:
+    src: truenas-power.sh.j2
+    dest: /usr/local/bin/truenas-power.sh
+    owner: root
+    group: root
+    mode: '0755'
+    setype: bin_t
+  tags: ups
+
+- name: deploy truenas restore script
+  become: true
+  ansible.builtin.template:
+    src: ups-restore.sh.j2
+    dest: /usr/local/bin/ups-restore.sh
+    owner: root
+    group: root
+    mode: '0755'
+    setype: bin_t
+  tags: ups
+
+- name: deploy upssched command dispatcher
+  become: true
+  ansible.builtin.template:
+    src: ups-sched-cmd.sh.j2
+    dest: /usr/local/bin/ups-sched-cmd.sh
+    owner: root
+    group: root
+    mode: '0755'
+    setype: bin_t
+  tags: ups
+
+# Covers the deep-drain case: if the battery ran out, this host was itself
+# powered off when mains returned, so nothing was running to restore TrueNAS.
+- name: deploy boot-time truenas restore unit
+  become: true
+  ansible.builtin.template:
+    src: ups-restore.service.j2
+    dest: /etc/systemd/system/ups-restore.service
+    owner: root
+    group: root
+    mode: '0644'
+  tags: ups
+
+- name: enable boot-time truenas restore unit
+  become: true
+  ansible.builtin.systemd:
+    name: ups-restore.service
+    enabled: true
+    daemon_reload: true
+  tags: ups
@@ -0,0 +1,5 @@
+---
+- import_tasks: deps.yml
+- import_tasks: nut.yml
+- import_tasks: ipmi.yml
+- import_tasks: firewall.yml
@@ -0,0 +1,129 @@
+---
+# NUT server. The UPS is on USB here, so this host drives it and serves
+# status to truenas.localdomain over the network.
+
+- name: set NUT run mode to netserver
+  become: true
+  ansible.builtin.template:
+    src: nut.conf.j2
+    dest: /etc/ups/nut.conf
+    owner: root
+    group: nut
+    mode: '0640'
+  notify:
+    - reload nut driver units
+    - restart nut driver
+    - restart nut server
+    - restart nut monitor
+  tags: ups
+
+- name: deploy NUT driver configuration
+  become: true
+  ansible.builtin.template:
+    src: ups.conf.j2
+    dest: /etc/ups/ups.conf
+    owner: root
+    group: nut
+    mode: '0640'
+  notify:
+    - reload nut driver units
+    - restart nut driver
+    - restart nut server
+  tags: ups
+
+- name: deploy upsd listener configuration
+  become: true
+  ansible.builtin.template:
+    src: upsd.conf.j2
+    dest: /etc/ups/upsd.conf
+    owner: root
+    group: nut
+    mode: '0640'
+  notify: restart nut server
+  tags: ups
+
+- name: deploy upsd users
+  become: true
+  ansible.builtin.template:
+    src: upsd.users.j2
+    dest: /etc/ups/upsd.users
+    owner: root
+    group: nut
+    mode: '0640'
+  notify:
+    - restart nut server
+    - restart nut monitor
+  tags: ups
+
+- name: deploy upsmon configuration
+  become: true
+  ansible.builtin.template:
+    src: upsmon.conf.j2
+    dest: /etc/ups/upsmon.conf
+    owner: root
+    group: nut
+    mode: '0640'
+  notify: restart nut monitor
+  tags: ups
+
+- name: deploy upssched configuration
+  become: true
+  ansible.builtin.template:
+    src: upssched.conf.j2
+    dest: /etc/ups/upssched.conf
+    owner: root
+    group: nut
+    mode: '0640'
+  notify: restart nut monitor
+  tags: ups
+
+# The nut package ships /usr/lib/udev/rules.d/62-nut-usbups.rules, which
+# hands the UPS USB device to the nut user. Reload and retrigger so the
+# driver can claim it without physically replugging the UPS.
+- name: reload udev rules for NUT USB access
+  become: true
+  ansible.builtin.command:
+    cmd: udevadm control --reload-rules
+  changed_when: false
+  tags: ups
+
+- name: retrigger UPS USB device
+  become: true
+  ansible.builtin.command:
+    cmd: >-
+      udevadm trigger --subsystem-match=usb
+      --attr-match=idVendor={{ ups_vendorid }}
+  changed_when: false
+  tags: ups
+
+# nut-driver-enumerator reads ups.conf and generates nut-driver@{{ ups_name }}.
+# The .service is oneshot (so never "started" for long) and is triggered by
+# the .path unit watching ups.conf - enable both, but only start the .path.
+- name: enable NUT driver enumerator
+  become: true
+  ansible.builtin.systemd:
+    name: nut-driver-enumerator.service
+    enabled: true
+    daemon_reload: true
+  tags: ups
+
+- name: enable and start NUT driver enumerator path trigger
+  become: true
+  ansible.builtin.systemd:
+    name: nut-driver-enumerator.path
+    enabled: true
+    state: started
+  tags: ups
+
+- name: enable and start NUT services
+  become: true
+  ansible.builtin.systemd:
+    name: "{{ item }}"
+    enabled: true
+    state: started
+  loop:
+    - "nut-driver@{{ ups_name }}.service"
+    - nut-server.service
+    - nut-monitor.service
+    - nut.target
+  tags: ups
@@ -0,0 +1,3 @@
+# {{ ansible_managed }}
+# netserver: run the driver + upsd locally and serve slaves over the network.
+MODE=netserver
@@ -0,0 +1,34 @@
+#!/bin/bash
+# {{ ansible_managed }}
+# Remote power control for truenas.localdomain via the R415 iDRAC6.
+# Usage: truenas-power.sh {status|on|soft|off|cycle}
+set -euo pipefail
+
+PW_FILE=/etc/ups/idrac.pw
+
+if [ ! -r "$PW_FILE" ]; then
+    echo "cannot read $PW_FILE" >&2
+    exit 1
+fi
+
+# -f keeps the password out of the process argument list.
+ipmi() {
+    ipmitool -I lanplus -H {{ idrac_host }} -U {{ idrac_user }} \
+        -f "$PW_FILE" "$@"
+}
+
+case "${1:-status}" in
+    status) ipmi chassis power status ;;
+    on)     ipmi chassis power on ;;
+    # ACPI soft-off. FreeBSD has hw.acpi.power_button_state=S5, so this is
+    # a clean TrueNAS shutdown. Normally unused: TrueNAS shuts itself down
+    # as a NUT slave. This is the manual escape hatch.
+    soft)   ipmi chassis power soft ;;
+    # Hard cut, last resort only.
+    off)    ipmi chassis power off ;;
+    cycle)  ipmi chassis power cycle ;;
+    *)
+        echo "usage: $0 {status|on|soft|off|cycle}" >&2
+        exit 2
+        ;;
+esac
@@ -0,0 +1,13 @@
+# {{ ansible_managed }}
+[Unit]
+Description=Restore TrueNAS power after an outage
+After=network-online.target nut-server.service nut-monitor.service
+Wants=network-online.target
+Requires=nut-server.service
+
+[Service]
+Type=oneshot
+ExecStart=/usr/local/bin/ups-restore.sh
+
+[Install]
+WantedBy=multi-user.target
@@ -0,0 +1,63 @@
+#!/bin/bash
+# {{ ansible_managed }}
+# Power truenas.localdomain back on after an outage, but only when it is
+# genuinely safe to do so. Called by upssched once mains has been stable,
+# and once at boot by ups-restore.service (deep-drain recovery).
+#
+# Deliberately stateless: we ask the iDRAC whether the chassis is off
+# rather than tracking whether we were the ones who shut it down. A NAS
+# powered off by hand while on mains is safe, because no ONLINE event
+# fires in that case and the boot path checks UPS status first.
+#
+# NOTE: as of now this is a best-effort secondary path. The R415 has no
+# iDRAC6 Enterprise card ("iDRAC6 Ent Pres ... Absent"), so its shared-LOM
+# BMC has no standby power and goes unreachable whenever the chassis is
+# off - exactly when we would want it. An unreachable iDRAC is therefore
+# the EXPECTED case here, and we exit quietly rather than alarming.
+#
+# The primary restore path needs no IPMI: the R415 power restore policy is
+# set to always-on, so when the UPS cuts and then restores its output the
+# server powers itself back up. Fit an iDRAC6 Enterprise card and switch it
+# to dedicated mode and this script starts working with no changes.
+set -uo pipefail
+
+TAG=ups-restore
+UPS={{ ups_name }}@localhost
+
+log() { logger -t "$TAG" -- "$*"; echo "$TAG: $*"; }
+
+# At boot, upsd may not be serving yet. Wait a bounded amount of time.
+for _ in $(seq 1 30); do
+    if upsc "$UPS" ups.status >/dev/null 2>&1; then
+        break
+    fi
+    sleep 2
+done
+
+status=$(upsc "$UPS" ups.status 2>/dev/null || echo UNKNOWN)
+if [[ "$status" != *OL* ]]; then
+    log "UPS status is '$status', not on line - refusing to power TrueNAS on"
+    exit 0
+fi
+
+power=$(/usr/local/bin/truenas-power.sh status 2>&1 || true)
+case "$power" in
+    *"is off"*)
+        log "mains stable and chassis off - powering TrueNAS on"
+        if /usr/local/bin/truenas-power.sh on; then
+            log "power-on command accepted"
+        else
+            log "power-on command FAILED - check iDRAC at {{ idrac_host }}"
+            exit 1
+        fi
+        ;;
+    *"is on"*)
+        log "TrueNAS already on, nothing to do"
+        ;;
+    *)
+        # Expected while the box has no iDRAC6 Enterprise card: the BMC is
+        # simply not on the network with the chassis powered down.
+        log "iDRAC at {{ idrac_host }} unreachable - relying on the" \
+            "always-on power restore policy instead ($power)"
+        ;;
+esac
@@ -0,0 +1,14 @@
+#!/bin/bash
+# {{ ansible_managed }}
+# upssched CMDSCRIPT. Runs as the unprivileged nut user.
+set -uo pipefail
+
+case "${1:-}" in
+    truenas-restore)
+        exec /usr/local/bin/ups-restore.sh
+        ;;
+    *)
+        logger -t ups-sched -- "unknown timer '${1:-}'"
+        exit 1
+        ;;
+esac
@@ -0,0 +1,18 @@
+# {{ ansible_managed }}
+
+[{{ ups_name }}]
+    driver = usbhid-ups
+    port = auto
+    vendorid = {{ ups_vendorid }}
+    productid = {{ ups_productid }}
+    desc = "{{ ups_desc }}"
+    # CyberPower asserts its own low-battery flag around 20-35%, far too
+    # early for us. Ignore it and derive LB from our own threshold so
+    # upsmon fires SHUTDOWNCMD at exactly {{ ups_low_charge_pct }}%.
+    ignorelb
+    override.battery.charge.low = {{ ups_low_charge_pct }}
+    # Unlock driver.killpower. Without this the driver refuses to cut UPS
+    # output at shutdown, and the R415's always-on power restore policy
+    # would never see AC drop and return - i.e. nothing comes back after a
+    # deep outage. See README.md.
+    allow_killpower
@@ -0,0 +1,3 @@
+# {{ ansible_managed }}
+LISTEN 127.0.0.1 {{ ups_listen_port }}
+LISTEN {{ ups_listen_addr }} {{ ups_listen_port }}
@@ -0,0 +1,11 @@
+# {{ ansible_managed }}
+
+# Local upsmon on this host.
+[upsmon]
+    password = {{ nut_upsmon_password }}
+    upsmon master
+
+# truenas.localdomain, running the TrueNAS UPS service in Slave mode.
+[truenas]
+    password = {{ nut_truenas_password }}
+    upsmon slave
@@ -0,0 +1,40 @@
+# {{ ansible_managed }}
+
+MONITOR {{ ups_name }}@localhost 1 upsmon {{ nut_upsmon_password }} master
+
+MINSUPPLIES 1
+SHUTDOWNCMD "/usr/bin/systemctl poweroff"
+NOTIFYCMD /usr/bin/upssched
+
+# upsmon drops this file before halting; /lib/systemd/system-shutdown/nutshutdown
+# reads it late in shutdown and, if present, tells the UPS to cut its output.
+# That AC drop-and-return is what triggers the R415's always-on restore policy
+# and this host's BIOS "After Power Loss: Power On". upsmon has NO compiled-in
+# default for this - leave it unset and the UPS never powers down.
+# Must be on tmpfs: a persistent path can go stale and make every ordinary
+# reboot look like a forced shutdown.
+POWERDOWNFLAG /run/nut/killpower
+
+POLLFREQ 5
+POLLFREQALERT 5
+
+# Wait up to 30s for the truenas slave to disconnect before we halt.
+HOSTSYNC 30
+DEADTIME 15
+RBWARNTIME 43200
+NOCOMMWARNTIME 300
+FINALDELAY 5
+
+# SYSLOG puts every UPS event in the journal, which fluent-bit already
+# forwards to Graylog (see roles/common/tasks/fluent-bit.yml).
+# EXEC runs NOTIFYCMD, i.e. upssched, which drives the TrueNAS restore.
+NOTIFYFLAG ONLINE    SYSLOG+EXEC
+NOTIFYFLAG ONBATT    SYSLOG+EXEC
+NOTIFYFLAG LOWBATT   SYSLOG+EXEC
+NOTIFYFLAG FSD       SYSLOG+EXEC
+NOTIFYFLAG COMMOK    SYSLOG+EXEC
+NOTIFYFLAG COMMBAD   SYSLOG+EXEC
+NOTIFYFLAG SHUTDOWN  SYSLOG+EXEC
+NOTIFYFLAG REPLBATT  SYSLOG
+NOTIFYFLAG NOCOMM    SYSLOG
+NOTIFYFLAG NOPARENT  SYSLOG
@@ -0,0 +1,13 @@
+# {{ ansible_managed }}
+
+CMDSCRIPT /usr/local/bin/ups-sched-cmd.sh
+PIPEFN /run/nut/upssched.pipe
+LOCKFN /run/nut/upssched.lock
+
+# Mains is back: wait for it to hold for {{ ups_restore_stable_secs }}s
+# before powering TrueNAS back on, so we do not flap on unstable power.
+AT ONLINE * CANCEL-TIMER truenas-restore
+AT ONLINE * START-TIMER truenas-restore {{ ups_restore_stable_secs }}
+
+# Power dropped again while the restore timer was pending - stand down.
+AT ONBATT * CANCEL-TIMER truenas-restore