diff --git a/ansible/deploy_home.yml b/ansible/deploy_home.yml index a40b77e..cc2064d 100644 --- a/ansible/deploy_home.yml +++ b/ansible/deploy_home.yml @@ -11,3 +11,5 @@ - role: github-actions - role: graylog-config tags: graylog-config + - role: ups + tags: ups diff --git a/ansible/roles/podman/tasks/containers/cloud-cron.yml b/ansible/roles/podman/tasks/containers/cloud-cron.yml new file mode 100644 index 0000000..1502453 --- /dev/null +++ b/ansible/roles/podman/tasks/containers/cloud-cron.yml @@ -0,0 +1,40 @@ +--- +- name: template {{ cron_name }} cron script + become: true + ansible.builtin.template: + src: nextcloud/cloud-cron.sh.j2 + dest: "{{ cron_script_path }}" + owner: root + group: root + mode: 0755 + setype: bin_t + +- name: template {{ cron_name }} cron systemd service + become: true + ansible.builtin.template: + src: nextcloud/cloud-cron.service.j2 + dest: "/etc/systemd/system/{{ cron_name }}-cron.service" + owner: root + group: root + mode: 0644 + vars: + instance_name: "{{ cron_name }}" + +- name: template {{ cron_name }} cron systemd timer + become: true + ansible.builtin.template: + src: nextcloud/cloud-cron.timer.j2 + dest: "/etc/systemd/system/{{ cron_name }}-cron.timer" + owner: root + group: root + mode: 0644 + vars: + instance_name: "{{ cron_name }}" + +- name: enable and start {{ cron_name }} cron timer + become: true + ansible.builtin.systemd: + name: "{{ cron_name }}-cron.timer" + enabled: true + state: started + daemon_reload: true diff --git a/ansible/roles/podman/tasks/containers/home/cloud.yml b/ansible/roles/podman/tasks/containers/home/cloud.yml index 23a2824..f67062e 100644 --- a/ansible/roles/podman/tasks/containers/home/cloud.yml +++ b/ansible/roles/podman/tasks/containers/home/cloud.yml @@ -96,6 +96,26 @@ changed_when: "'System config value log_rotate_size' in cloud_log_rotate.stdout" failed_when: false +# Nextcloud's default ('auto') only expires trash when disk space is needed, +# so 66 GB of >30-day deletions sat untouched on a host with 1.3 TB free -- +# the retention was effectively unbounded. 'auto, 30' makes the 30-day +# expiry unconditional while still purging early under space pressure. +- name: set nextcloud trashbin retention for cloud + become: true + become_user: "{{ podman_user }}" + ansible.builtin.command: > + podman exec -u www-data cloud + php occ config:system:set trashbin_retention_obligation --value "auto, 30" + register: cloud_trashbin_retention + changed_when: "'System config value trashbin_retention_obligation' in cloud_trashbin_retention.stdout" + failed_when: false + +- include_tasks: containers/cloud-cron.yml + vars: + cron_name: cloud + cron_container: cloud + cron_script_path: /usr/local/bin/cloud-cron.sh + - include_tasks: containers/cloud-backup.yml vars: backup_name: cloud diff --git a/ansible/roles/podman/tasks/containers/skudak/cloud.yml b/ansible/roles/podman/tasks/containers/skudak/cloud.yml index 1175948..9fe2d2f 100644 --- a/ansible/roles/podman/tasks/containers/skudak/cloud.yml +++ b/ansible/roles/podman/tasks/containers/skudak/cloud.yml @@ -155,6 +155,12 @@ changed_when: "'System config value log_rotate_size' in skudak_log_rotate.stdout" failed_when: false +- include_tasks: containers/cloud-cron.yml + vars: + cron_name: skudak-cloud + cron_container: skudak-cloud + cron_script_path: /usr/local/bin/skudak-cloud-cron.sh + - include_tasks: containers/cloud-backup.yml vars: backup_name: skudak-cloud diff --git a/ansible/roles/podman/tasks/main.yml b/ansible/roles/podman/tasks/main.yml index a981a85..cca4d9b 100644 --- a/ansible/roles/podman/tasks/main.yml +++ b/ansible/roles/podman/tasks/main.yml @@ -67,24 +67,24 @@ - import_tasks: containers/home/cloud.yml vars: db_image: docker.io/library/mariadb:10.6 - image: docker.io/library/nextcloud:33.0.0-apache + image: docker.io/library/nextcloud:34.0.2-apache tags: cloud - import_tasks: containers/skudak/cloud.yml vars: db_image: docker.io/library/mariadb:10.6 - image: docker.io/library/nextcloud:33.0.0-apache + image: docker.io/library/nextcloud:34.0.2-apache tags: skudak, skudak-cloud - import_tasks: containers/debyltech/fulfillr.yml vars: - image: git.debyl.io/debyltech/fulfillr:20260723.2044 + image: git.debyl.io/debyltech/fulfillr:20260728.2155 tags: debyltech, fulfillr # Staging back-office (fulfillr-dev.debyltech.com) — same image, staging Turso config. - import_tasks: containers/debyltech/fulfillr-dev.yml vars: - image: git.debyl.io/debyltech/fulfillr:20260723.2044 + image: git.debyl.io/debyltech/fulfillr:20260728.2155 tags: debyltech, fulfillr-dev - import_tasks: containers/debyltech/uptime-kuma.yml @@ -109,7 +109,7 @@ - import_tasks: containers/home/gregtime.yml vars: - image: localhost/greg-time-bot:3.9.25 + image: localhost/greg-time-bot:3.10.0 tags: gregtime - import_tasks: containers/home/zomboid.yml diff --git a/ansible/roles/podman/templates/nextcloud/cloud-cron.service.j2 b/ansible/roles/podman/templates/nextcloud/cloud-cron.service.j2 new file mode 100644 index 0000000..27bb1f0 --- /dev/null +++ b/ansible/roles/podman/templates/nextcloud/cloud-cron.service.j2 @@ -0,0 +1,13 @@ +[Unit] +Description=Nextcloud {{ instance_name }} background jobs +After=network-online.target +Wants=network-online.target + +[Service] +Type=oneshot +ExecStart={{ cron_script_path }} +# Type=oneshot disables the start timeout by default, so a wedged cron.php +# would leave the unit "activating" forever and every subsequent 5-minute +# trigger would be silently skipped. Bound it. +TimeoutStartSec={{ cron_timeout | default('30m') }} +Nice=10 diff --git a/ansible/roles/podman/templates/nextcloud/cloud-cron.sh.j2 b/ansible/roles/podman/templates/nextcloud/cloud-cron.sh.j2 new file mode 100644 index 0000000..672dbd1 --- /dev/null +++ b/ansible/roles/podman/templates/nextcloud/cloud-cron.sh.j2 @@ -0,0 +1,51 @@ +#!/bin/bash +# {{ ansible_managed }} +# Nextcloud "{{ cron_name }}" background jobs (cron.php). +# +# backgroundjobs_mode is "cron" on both instances, which means Nextcloud +# expects an external caller to run cron.php every ~5 minutes. Nothing was: +# the personal instance had not run a background job since 2026-05-14 and +# skudak since 2024-11-20. Without it Nextcloud never expires trash or file +# versions, never cleans stale chunked uploads, never sends calendar +# reminders, and -- easy to miss -- never rotates nextcloud.log, which makes +# the log_rotate_size cap set in containers/*/cloud.yml inert. +set -euo pipefail + +TAG=nextcloud-cron +INSTANCE={{ cron_name }} + +log() { logger -t "$TAG" -p daemon.info -- "instance=$INSTANCE $*"; } +fail() { logger -t "$TAG" -p daemon.err -- "instance=$INSTANCE status=failed $*" + echo "$TAG: FAILED: $*" >&2; exit 1; } + +# The Nextcloud containers are ROOTLESS podman owned by "{{ podman_user }}", +# but this script runs as root under systemd. Same sudo/cd/XDG_RUNTIME_DIR +# preamble as cloud-backup.sh -- see CLAUDE.md for why `cd;` is required. +pexec() { + sudo -H -u {{ podman_user }} bash -c \ + 'cd; d=/run/user/$(id -u); [ -d "$d" ] && export XDG_RUNTIME_DIR="$d" + exec podman "$@"' _ "$@" +} + +# A container that is down (deploy, image bump, host reboot) is not a failure +# worth flagging -- the next tick picks it up five minutes later. +if ! pexec container exists {{ cron_container }} 2>/dev/null; then + log "status=skipped reason=container-absent" + exit 0 +fi + +# occ and cron.php both refuse to do anything useful mid-upgrade. Skipping +# keeps a deploy window from parading as a run of failed units. +if pexec exec -u www-data {{ cron_container }} php occ status 2>/dev/null \ + | grep -q 'maintenance: true'; then + log "status=skipped reason=maintenance" + exit 0 +fi + +set +e +pexec exec -u www-data {{ cron_container }} php -f /var/www/html/cron.php +rc=$? +set -e +[ "$rc" -eq 0 ] || fail "cron.php exited $rc" + +log "status=ok" diff --git a/ansible/roles/podman/templates/nextcloud/cloud-cron.timer.j2 b/ansible/roles/podman/templates/nextcloud/cloud-cron.timer.j2 new file mode 100644 index 0000000..cf1d9a5 --- /dev/null +++ b/ansible/roles/podman/templates/nextcloud/cloud-cron.timer.j2 @@ -0,0 +1,13 @@ +[Unit] +Description=Nextcloud {{ instance_name }} background jobs every 5 minutes + +[Timer] +OnBootSec={{ cron_onbootsec | default('5m') }} +OnUnitActiveSec={{ cron_interval | default('5m') }} +RandomizedDelaySec={{ cron_randomized_delay | default('30s') }} +# Deliberately NOT Persistent: this runs every 5 minutes, so replaying runs +# missed while the host was off buys nothing and just stampedes at boot. +Persistent=false + +[Install] +WantedBy=timers.target diff --git a/ansible/roles/ups/README.md b/ansible/roles/ups/README.md new file mode 100644 index 0000000..0b402ad --- /dev/null +++ b/ansible/roles/ups/README.md @@ -0,0 +1,114 @@ +# ups + +UPS monitoring and staged shutdown for the home rack. + +A CyberPower PR1500RT2U (`0764:0601`) is cabled by USB to `home.debyl.io` and +backs both that host and `truenas.localdomain` (Dell PowerEdge R415). This role +makes `home.debyl.io` the NUT server and gives it the ability to power TrueNAS +back on over IPMI. + +## Outage sequence + +| When | What happens | Driven by | +| --- | --- | --- | +| t+0 | UPS goes on battery, `ONBATT` logged to journald → Graylog | `upsmon` | +| t+2min | TrueNAS shuts itself down cleanly, shedding ~200 W | TrueNAS UPS service, Slave mode, `Shutdown Timer 120` | +| 10% charge | `home.debyl.io` shuts itself down and tells the UPS to cut its output | `upsmon` `SHUTDOWNCMD` + `/lib/systemd/system-shutdown/nutshutdown` | +| mains returns | UPS re-energizes; both machines power themselves back up | R415 `always-on` restore policy; `home.debyl.io` BIOS *After Power Loss → Power On* | +| mains back +5min | Best-effort IPMI power-on, if a dedicated iDRAC is ever fitted | `upssched` → `ups-restore.sh` | +| host boot | Same restore check, for the deep-drain case | `ups-restore.service` | + +The 10% threshold is not a custom poller. CyberPower asserts its own low-battery +flag around 20–35%, so `ups.conf` sets `ignorelb` plus +`override.battery.charge.low`, and stock `upsmon` fires at exactly the +configured percentage. + +Likewise, the 2-minute TrueNAS shed is TrueNAS's own native "shutdown timer" +setting — no SSH key and no shutdown script from this side. + +## Powering TrueNAS back on + +Neither Wake-on-LAN nor IPMI works on this box today, so restore is done with +the chassis power restore policy instead. + +**Wake-on-LAN is out.** `bce0` advertises no WOL capability (`ifconfig -m bce0` +has no `WOL_MAGIC`) and both NICs are bonded into an LACP `lagg0`. + +**IPMI is out too, for now.** The iDRAC is in `shared / LOM1` mode and the +Enterprise card that would provide a dedicated management port is not fitted: + +``` +$ ipmitool sdr elist | grep -i idrac +iDRAC6 Ent Pres | 70h | ok | 7.1 | Absent +``` + +A shared-LOM iDRAC6 Express has no standby power. Measured directly: with the +chassis powered off the BMC does not even answer ARP, and it only reappears +~200s into POST, at the moment the host brings the NIC link up. That is a +hardware limitation, not a switch or BIOS problem. + +**So restore works like this instead.** `ipmitool chassis policy always-on` is +set on the R415. On a deep outage `home.debyl.io` halts at 10% and NUT's +shutdown hook tells the UPS to cut its output; when mains returns the UPS +re-energizes, the R415 sees AC and boots itself. + +The gap is the medium outage — mains returns after TrueNAS has shed but before +the battery reaches 10%. The UPS never cuts power, so TrueNAS stays off and +needs a manual power button press. Fitting a used iDRAC6 Enterprise card and +running `ipmitool delloem lan set dedicated` (plus a cable to the dedicated +port) closes that gap, and `ups-restore.sh` starts working with no code +changes — it is already deployed and simply logs and exits while the BMC is +unreachable. + +## One-time setup outside Ansible + +These are not managed by this role. + +**iDRAC (already done, via `ipmitool` on TrueNAS):** + +``` +ipmitool lan set 1 ipsrc static +ipmitool lan set 1 ipaddr 192.168.1.12 +ipmitool lan set 1 netmask 255.255.255.0 +ipmitool lan set 1 defgw ipaddr 192.168.1.1 +ipmitool lan set 1 access on # was disabled - nothing answers without this +ipmitool channel setaccess 1 2 callin=on ipmi=on link=on privilege=4 +ipmitool user set password 2 '' +ipmitool chassis policy always-on # this is what restores power after an outage +``` + +`idrac_password` is also the iDRAC web UI password for `root` - they share a +user database. + +**TrueNAS UI → Services → UPS** (enable + start automatically): + +| Field | Value | +| --- | --- | +| UPS Mode | Slave | +| Remote Host | `192.168.1.10` | +| Remote Port | `3493` | +| Identifier | `cyberpower` | +| Monitor User | `truenas` | +| Monitor Password | vault `nut_truenas_password` | +| Shutdown Mode | UPS goes on battery | +| Shutdown Timer | `120` | +| Shutdown Command | `/sbin/shutdown -p now` | +| Power Off UPS | unchecked | + +**BIOS on `home.debyl.io`** (Lenovo 10MR0004US): Power → *After Power Loss* → +**Power On**. On a full drain the NUT shutdown hook +(`/lib/systemd/system-shutdown/nutshutdown`) tells the UPS to cut its own +output; this BIOS setting is what brings the host back when mains returns. + +## Vault keys + +`nut_upsmon_password`, `nut_truenas_password`, `idrac_password` + +## Operating + +``` +upsc cyberpower # full UPS status +upsc cyberpower battery.charge +sudo /usr/local/bin/truenas-power.sh status|on|soft|off +journalctl -t ups-restore -t ups-sched +``` diff --git a/ansible/roles/ups/defaults/main.yml b/ansible/roles/ups/defaults/main.yml new file mode 100644 index 0000000..b8d8408 --- /dev/null +++ b/ansible/roles/ups/defaults/main.yml @@ -0,0 +1,36 @@ +--- +# CyberPower PR1500LCDRT2U cabled by USB to this host (0764:0601). +# This host is the NUT server; truenas.localdomain is a NUT slave. +ups_name: cyberpower +# lsusb reports the product string as PR1500LCDRT2U, but the device itself +# reports device.model CP1500PFCRM2U. The latter is what it actually is. +ups_desc: CyberPower CP1500PFCRM2U +ups_vendorid: "0764" +ups_productid: "0601" + +ups_listen_addr: 192.168.1.10 +ups_listen_port: 3493 + +# truenas.localdomain - the only host allowed to reach upsd +ups_slave_ip: 192.168.1.11 + +# Battery charge at which THIS host shuts itself down. TrueNAS sheds much +# earlier via its own "shutdown timer" setting (see roles/ups/README.md). +ups_low_charge_pct: 10 + +# Seconds of stable mains after ONLINE before TrueNAS is powered back on. +ups_restore_stable_secs: 300 + +# R415 iDRAC6. Shares LOM1 with bce0. +idrac_host: 192.168.1.12 +idrac_user: root + +ups_deps: + [ + ipmitool, + nut, + nut-client, + ] + +# Secrets live in ansible/vars/vault.yml (no vault_ prefix, per repo +# convention): nut_upsmon_password, nut_truenas_password, idrac_password diff --git a/ansible/roles/ups/handlers/main.yml b/ansible/roles/ups/handlers/main.yml new file mode 100644 index 0000000..e2af69c --- /dev/null +++ b/ansible/roles/ups/handlers/main.yml @@ -0,0 +1,35 @@ +--- +# Regenerates the nut-driver@ unit instances from ups.conf. Oneshot, +# so "restarted" just means "run it again". +- name: reload nut driver units + become: true + ansible.builtin.systemd: + name: nut-driver-enumerator.service + state: restarted + daemon_reload: true + +# The enumerator only writes unit definitions; it will not pick up changed +# driver options in an already-running driver. Restart the instance itself. +- name: restart nut driver + become: true + ansible.builtin.systemd: + name: "nut-driver@{{ ups_name }}.service" + state: restarted + +- name: restart nut server + become: true + ansible.builtin.systemd: + name: nut-server.service + state: restarted + +- name: restart nut monitor + become: true + ansible.builtin.systemd: + name: nut-monitor.service + state: restarted + +- name: restart firewalld + become: true + ansible.builtin.systemd: + name: firewalld + state: restarted diff --git a/ansible/roles/ups/tasks/deps.yml b/ansible/roles/ups/tasks/deps.yml new file mode 100644 index 0000000..2c640ce --- /dev/null +++ b/ansible/roles/ups/tasks/deps.yml @@ -0,0 +1,7 @@ +--- +- name: install NUT and ipmitool + become: true + ansible.builtin.dnf: + name: "{{ ups_deps }}" + state: present + tags: ups diff --git a/ansible/roles/ups/tasks/firewall.yml b/ansible/roles/ups/tasks/firewall.yml new file mode 100644 index 0000000..1004ef0 --- /dev/null +++ b/ansible/roles/ups/tasks/firewall.yml @@ -0,0 +1,12 @@ +--- +- name: allow NUT from the truenas slave only + become: true + ansible.posix.firewalld: + rich_rule: >- + rule family="ipv4" source address="{{ ups_slave_ip }}/32" + port port="{{ ups_listen_port }}" protocol="tcp" accept + permanent: true + immediate: true + state: enabled + notify: restart firewalld + tags: [ups, firewall] diff --git a/ansible/roles/ups/tasks/ipmi.yml b/ansible/roles/ups/tasks/ipmi.yml new file mode 100644 index 0000000..0a1518b --- /dev/null +++ b/ansible/roles/ups/tasks/ipmi.yml @@ -0,0 +1,68 @@ +--- +# IPMI power control for truenas.localdomain (Dell R415, iDRAC6). +# WOL is not an option there: bce0 advertises no WOL capability and is an +# LACP lagg member, so IPMI is the only remote power-on path. + +- name: deploy iDRAC credential + become: true + ansible.builtin.copy: + content: "{{ idrac_password }}\n" + dest: /etc/ups/idrac.pw + owner: root + group: nut + mode: '0640' + no_log: true + tags: ups + +- name: deploy truenas power helper + become: true + ansible.builtin.template: + src: truenas-power.sh.j2 + dest: /usr/local/bin/truenas-power.sh + owner: root + group: root + mode: '0755' + setype: bin_t + tags: ups + +- name: deploy truenas restore script + become: true + ansible.builtin.template: + src: ups-restore.sh.j2 + dest: /usr/local/bin/ups-restore.sh + owner: root + group: root + mode: '0755' + setype: bin_t + tags: ups + +- name: deploy upssched command dispatcher + become: true + ansible.builtin.template: + src: ups-sched-cmd.sh.j2 + dest: /usr/local/bin/ups-sched-cmd.sh + owner: root + group: root + mode: '0755' + setype: bin_t + tags: ups + +# Covers the deep-drain case: if the battery ran out, this host was itself +# powered off when mains returned, so nothing was running to restore TrueNAS. +- name: deploy boot-time truenas restore unit + become: true + ansible.builtin.template: + src: ups-restore.service.j2 + dest: /etc/systemd/system/ups-restore.service + owner: root + group: root + mode: '0644' + tags: ups + +- name: enable boot-time truenas restore unit + become: true + ansible.builtin.systemd: + name: ups-restore.service + enabled: true + daemon_reload: true + tags: ups diff --git a/ansible/roles/ups/tasks/main.yml b/ansible/roles/ups/tasks/main.yml new file mode 100644 index 0000000..bf19bd2 --- /dev/null +++ b/ansible/roles/ups/tasks/main.yml @@ -0,0 +1,5 @@ +--- +- import_tasks: deps.yml +- import_tasks: nut.yml +- import_tasks: ipmi.yml +- import_tasks: firewall.yml diff --git a/ansible/roles/ups/tasks/nut.yml b/ansible/roles/ups/tasks/nut.yml new file mode 100644 index 0000000..3f86819 --- /dev/null +++ b/ansible/roles/ups/tasks/nut.yml @@ -0,0 +1,129 @@ +--- +# NUT server. The UPS is on USB here, so this host drives it and serves +# status to truenas.localdomain over the network. + +- name: set NUT run mode to netserver + become: true + ansible.builtin.template: + src: nut.conf.j2 + dest: /etc/ups/nut.conf + owner: root + group: nut + mode: '0640' + notify: + - reload nut driver units + - restart nut driver + - restart nut server + - restart nut monitor + tags: ups + +- name: deploy NUT driver configuration + become: true + ansible.builtin.template: + src: ups.conf.j2 + dest: /etc/ups/ups.conf + owner: root + group: nut + mode: '0640' + notify: + - reload nut driver units + - restart nut driver + - restart nut server + tags: ups + +- name: deploy upsd listener configuration + become: true + ansible.builtin.template: + src: upsd.conf.j2 + dest: /etc/ups/upsd.conf + owner: root + group: nut + mode: '0640' + notify: restart nut server + tags: ups + +- name: deploy upsd users + become: true + ansible.builtin.template: + src: upsd.users.j2 + dest: /etc/ups/upsd.users + owner: root + group: nut + mode: '0640' + notify: + - restart nut server + - restart nut monitor + tags: ups + +- name: deploy upsmon configuration + become: true + ansible.builtin.template: + src: upsmon.conf.j2 + dest: /etc/ups/upsmon.conf + owner: root + group: nut + mode: '0640' + notify: restart nut monitor + tags: ups + +- name: deploy upssched configuration + become: true + ansible.builtin.template: + src: upssched.conf.j2 + dest: /etc/ups/upssched.conf + owner: root + group: nut + mode: '0640' + notify: restart nut monitor + tags: ups + +# The nut package ships /usr/lib/udev/rules.d/62-nut-usbups.rules, which +# hands the UPS USB device to the nut user. Reload and retrigger so the +# driver can claim it without physically replugging the UPS. +- name: reload udev rules for NUT USB access + become: true + ansible.builtin.command: + cmd: udevadm control --reload-rules + changed_when: false + tags: ups + +- name: retrigger UPS USB device + become: true + ansible.builtin.command: + cmd: >- + udevadm trigger --subsystem-match=usb + --attr-match=idVendor={{ ups_vendorid }} + changed_when: false + tags: ups + +# nut-driver-enumerator reads ups.conf and generates nut-driver@{{ ups_name }}. +# The .service is oneshot (so never "started" for long) and is triggered by +# the .path unit watching ups.conf - enable both, but only start the .path. +- name: enable NUT driver enumerator + become: true + ansible.builtin.systemd: + name: nut-driver-enumerator.service + enabled: true + daemon_reload: true + tags: ups + +- name: enable and start NUT driver enumerator path trigger + become: true + ansible.builtin.systemd: + name: nut-driver-enumerator.path + enabled: true + state: started + tags: ups + +- name: enable and start NUT services + become: true + ansible.builtin.systemd: + name: "{{ item }}" + enabled: true + state: started + loop: + - "nut-driver@{{ ups_name }}.service" + - nut-server.service + - nut-monitor.service + - nut.target + tags: ups diff --git a/ansible/roles/ups/templates/nut.conf.j2 b/ansible/roles/ups/templates/nut.conf.j2 new file mode 100644 index 0000000..d916a4c --- /dev/null +++ b/ansible/roles/ups/templates/nut.conf.j2 @@ -0,0 +1,3 @@ +# {{ ansible_managed }} +# netserver: run the driver + upsd locally and serve slaves over the network. +MODE=netserver diff --git a/ansible/roles/ups/templates/truenas-power.sh.j2 b/ansible/roles/ups/templates/truenas-power.sh.j2 new file mode 100644 index 0000000..310a189 --- /dev/null +++ b/ansible/roles/ups/templates/truenas-power.sh.j2 @@ -0,0 +1,34 @@ +#!/bin/bash +# {{ ansible_managed }} +# Remote power control for truenas.localdomain via the R415 iDRAC6. +# Usage: truenas-power.sh {status|on|soft|off|cycle} +set -euo pipefail + +PW_FILE=/etc/ups/idrac.pw + +if [ ! -r "$PW_FILE" ]; then + echo "cannot read $PW_FILE" >&2 + exit 1 +fi + +# -f keeps the password out of the process argument list. +ipmi() { + ipmitool -I lanplus -H {{ idrac_host }} -U {{ idrac_user }} \ + -f "$PW_FILE" "$@" +} + +case "${1:-status}" in + status) ipmi chassis power status ;; + on) ipmi chassis power on ;; + # ACPI soft-off. FreeBSD has hw.acpi.power_button_state=S5, so this is + # a clean TrueNAS shutdown. Normally unused: TrueNAS shuts itself down + # as a NUT slave. This is the manual escape hatch. + soft) ipmi chassis power soft ;; + # Hard cut, last resort only. + off) ipmi chassis power off ;; + cycle) ipmi chassis power cycle ;; + *) + echo "usage: $0 {status|on|soft|off|cycle}" >&2 + exit 2 + ;; +esac diff --git a/ansible/roles/ups/templates/ups-restore.service.j2 b/ansible/roles/ups/templates/ups-restore.service.j2 new file mode 100644 index 0000000..f8aeffd --- /dev/null +++ b/ansible/roles/ups/templates/ups-restore.service.j2 @@ -0,0 +1,13 @@ +# {{ ansible_managed }} +[Unit] +Description=Restore TrueNAS power after an outage +After=network-online.target nut-server.service nut-monitor.service +Wants=network-online.target +Requires=nut-server.service + +[Service] +Type=oneshot +ExecStart=/usr/local/bin/ups-restore.sh + +[Install] +WantedBy=multi-user.target diff --git a/ansible/roles/ups/templates/ups-restore.sh.j2 b/ansible/roles/ups/templates/ups-restore.sh.j2 new file mode 100644 index 0000000..666550a --- /dev/null +++ b/ansible/roles/ups/templates/ups-restore.sh.j2 @@ -0,0 +1,63 @@ +#!/bin/bash +# {{ ansible_managed }} +# Power truenas.localdomain back on after an outage, but only when it is +# genuinely safe to do so. Called by upssched once mains has been stable, +# and once at boot by ups-restore.service (deep-drain recovery). +# +# Deliberately stateless: we ask the iDRAC whether the chassis is off +# rather than tracking whether we were the ones who shut it down. A NAS +# powered off by hand while on mains is safe, because no ONLINE event +# fires in that case and the boot path checks UPS status first. +# +# NOTE: as of now this is a best-effort secondary path. The R415 has no +# iDRAC6 Enterprise card ("iDRAC6 Ent Pres ... Absent"), so its shared-LOM +# BMC has no standby power and goes unreachable whenever the chassis is +# off - exactly when we would want it. An unreachable iDRAC is therefore +# the EXPECTED case here, and we exit quietly rather than alarming. +# +# The primary restore path needs no IPMI: the R415 power restore policy is +# set to always-on, so when the UPS cuts and then restores its output the +# server powers itself back up. Fit an iDRAC6 Enterprise card and switch it +# to dedicated mode and this script starts working with no changes. +set -uo pipefail + +TAG=ups-restore +UPS={{ ups_name }}@localhost + +log() { logger -t "$TAG" -- "$*"; echo "$TAG: $*"; } + +# At boot, upsd may not be serving yet. Wait a bounded amount of time. +for _ in $(seq 1 30); do + if upsc "$UPS" ups.status >/dev/null 2>&1; then + break + fi + sleep 2 +done + +status=$(upsc "$UPS" ups.status 2>/dev/null || echo UNKNOWN) +if [[ "$status" != *OL* ]]; then + log "UPS status is '$status', not on line - refusing to power TrueNAS on" + exit 0 +fi + +power=$(/usr/local/bin/truenas-power.sh status 2>&1 || true) +case "$power" in + *"is off"*) + log "mains stable and chassis off - powering TrueNAS on" + if /usr/local/bin/truenas-power.sh on; then + log "power-on command accepted" + else + log "power-on command FAILED - check iDRAC at {{ idrac_host }}" + exit 1 + fi + ;; + *"is on"*) + log "TrueNAS already on, nothing to do" + ;; + *) + # Expected while the box has no iDRAC6 Enterprise card: the BMC is + # simply not on the network with the chassis powered down. + log "iDRAC at {{ idrac_host }} unreachable - relying on the" \ + "always-on power restore policy instead ($power)" + ;; +esac diff --git a/ansible/roles/ups/templates/ups-sched-cmd.sh.j2 b/ansible/roles/ups/templates/ups-sched-cmd.sh.j2 new file mode 100644 index 0000000..bf0cb90 --- /dev/null +++ b/ansible/roles/ups/templates/ups-sched-cmd.sh.j2 @@ -0,0 +1,14 @@ +#!/bin/bash +# {{ ansible_managed }} +# upssched CMDSCRIPT. Runs as the unprivileged nut user. +set -uo pipefail + +case "${1:-}" in + truenas-restore) + exec /usr/local/bin/ups-restore.sh + ;; + *) + logger -t ups-sched -- "unknown timer '${1:-}'" + exit 1 + ;; +esac diff --git a/ansible/roles/ups/templates/ups.conf.j2 b/ansible/roles/ups/templates/ups.conf.j2 new file mode 100644 index 0000000..b817d02 --- /dev/null +++ b/ansible/roles/ups/templates/ups.conf.j2 @@ -0,0 +1,18 @@ +# {{ ansible_managed }} + +[{{ ups_name }}] + driver = usbhid-ups + port = auto + vendorid = {{ ups_vendorid }} + productid = {{ ups_productid }} + desc = "{{ ups_desc }}" + # CyberPower asserts its own low-battery flag around 20-35%, far too + # early for us. Ignore it and derive LB from our own threshold so + # upsmon fires SHUTDOWNCMD at exactly {{ ups_low_charge_pct }}%. + ignorelb + override.battery.charge.low = {{ ups_low_charge_pct }} + # Unlock driver.killpower. Without this the driver refuses to cut UPS + # output at shutdown, and the R415's always-on power restore policy + # would never see AC drop and return - i.e. nothing comes back after a + # deep outage. See README.md. + allow_killpower diff --git a/ansible/roles/ups/templates/upsd.conf.j2 b/ansible/roles/ups/templates/upsd.conf.j2 new file mode 100644 index 0000000..2ce034e --- /dev/null +++ b/ansible/roles/ups/templates/upsd.conf.j2 @@ -0,0 +1,3 @@ +# {{ ansible_managed }} +LISTEN 127.0.0.1 {{ ups_listen_port }} +LISTEN {{ ups_listen_addr }} {{ ups_listen_port }} diff --git a/ansible/roles/ups/templates/upsd.users.j2 b/ansible/roles/ups/templates/upsd.users.j2 new file mode 100644 index 0000000..f070473 --- /dev/null +++ b/ansible/roles/ups/templates/upsd.users.j2 @@ -0,0 +1,11 @@ +# {{ ansible_managed }} + +# Local upsmon on this host. +[upsmon] + password = {{ nut_upsmon_password }} + upsmon master + +# truenas.localdomain, running the TrueNAS UPS service in Slave mode. +[truenas] + password = {{ nut_truenas_password }} + upsmon slave diff --git a/ansible/roles/ups/templates/upsmon.conf.j2 b/ansible/roles/ups/templates/upsmon.conf.j2 new file mode 100644 index 0000000..1fe98a7 --- /dev/null +++ b/ansible/roles/ups/templates/upsmon.conf.j2 @@ -0,0 +1,40 @@ +# {{ ansible_managed }} + +MONITOR {{ ups_name }}@localhost 1 upsmon {{ nut_upsmon_password }} master + +MINSUPPLIES 1 +SHUTDOWNCMD "/usr/bin/systemctl poweroff" +NOTIFYCMD /usr/bin/upssched + +# upsmon drops this file before halting; /lib/systemd/system-shutdown/nutshutdown +# reads it late in shutdown and, if present, tells the UPS to cut its output. +# That AC drop-and-return is what triggers the R415's always-on restore policy +# and this host's BIOS "After Power Loss: Power On". upsmon has NO compiled-in +# default for this - leave it unset and the UPS never powers down. +# Must be on tmpfs: a persistent path can go stale and make every ordinary +# reboot look like a forced shutdown. +POWERDOWNFLAG /run/nut/killpower + +POLLFREQ 5 +POLLFREQALERT 5 + +# Wait up to 30s for the truenas slave to disconnect before we halt. +HOSTSYNC 30 +DEADTIME 15 +RBWARNTIME 43200 +NOCOMMWARNTIME 300 +FINALDELAY 5 + +# SYSLOG puts every UPS event in the journal, which fluent-bit already +# forwards to Graylog (see roles/common/tasks/fluent-bit.yml). +# EXEC runs NOTIFYCMD, i.e. upssched, which drives the TrueNAS restore. +NOTIFYFLAG ONLINE SYSLOG+EXEC +NOTIFYFLAG ONBATT SYSLOG+EXEC +NOTIFYFLAG LOWBATT SYSLOG+EXEC +NOTIFYFLAG FSD SYSLOG+EXEC +NOTIFYFLAG COMMOK SYSLOG+EXEC +NOTIFYFLAG COMMBAD SYSLOG+EXEC +NOTIFYFLAG SHUTDOWN SYSLOG+EXEC +NOTIFYFLAG REPLBATT SYSLOG +NOTIFYFLAG NOCOMM SYSLOG +NOTIFYFLAG NOPARENT SYSLOG diff --git a/ansible/roles/ups/templates/upssched.conf.j2 b/ansible/roles/ups/templates/upssched.conf.j2 new file mode 100644 index 0000000..b668731 --- /dev/null +++ b/ansible/roles/ups/templates/upssched.conf.j2 @@ -0,0 +1,13 @@ +# {{ ansible_managed }} + +CMDSCRIPT /usr/local/bin/ups-sched-cmd.sh +PIPEFN /run/nut/upssched.pipe +LOCKFN /run/nut/upssched.lock + +# Mains is back: wait for it to hold for {{ ups_restore_stable_secs }}s +# before powering TrueNAS back on, so we do not flap on unstable power. +AT ONLINE * CANCEL-TIMER truenas-restore +AT ONLINE * START-TIMER truenas-restore {{ ups_restore_stable_secs }} + +# Power dropped again while the restore timer was pending - stand down. +AT ONBATT * CANCEL-TIMER truenas-restore diff --git a/ansible/vars/vault.yml b/ansible/vars/vault.yml index 371e48f..9c9d874 100644 Binary files a/ansible/vars/vault.yml and b/ansible/vars/vault.yml differ