diff --git a/ansible/deploy_home.yml b/ansible/deploy_home.yml index a40b77e..cc2064d 100644 --- a/ansible/deploy_home.yml +++ b/ansible/deploy_home.yml @@ -11,3 +11,5 @@ - role: github-actions - role: graylog-config tags: graylog-config + - role: ups + tags: ups diff --git a/ansible/roles/common/tasks/mail.yml b/ansible/roles/common/tasks/mail.yml new file mode 100644 index 0000000..5b14498 --- /dev/null +++ b/ansible/roles/common/tasks/mail.yml @@ -0,0 +1,34 @@ +--- +# Outbound mail for system notifications (backup failures, cron output). +# +# msmtp rather than the esmtp already on the box: the OpenSRS relay uses port +# 465, which is IMPLICIT TLS, and libesmtp/esmtp only speaks STARTTLS. msmtp +# handles implicit TLS via `tls_starttls off` and mirrors the settings the +# TrueNAS box already uses. +# On Fedora the single `msmtp` package already ships /usr/bin/sendmail and +# /usr/lib/sendmail; there is no separate msmtp-sendmail package (Debian split). +- name: install msmtp + become: true + ansible.builtin.package: + name: msmtp + state: present + tags: mail + +- name: configure msmtp + become: true + ansible.builtin.template: + src: msmtp/msmtprc.j2 + dest: /etc/msmtprc + owner: root + group: root + mode: 0600 + no_log: true + tags: mail + +- name: point the mta alternative at msmtp + become: true + community.general.alternatives: + name: mta + path: /usr/bin/msmtp-sendmail + failed_when: false + tags: mail diff --git a/ansible/roles/common/tasks/main.yml b/ansible/roles/common/tasks/main.yml index db08c36..8a7ba73 100644 --- a/ansible/roles/common/tasks/main.yml +++ b/ansible/roles/common/tasks/main.yml @@ -3,6 +3,9 @@ - import_tasks: security.yml - import_tasks: service.yml +- import_tasks: mail.yml + tags: mail + - import_tasks: fluent-bit.yml tags: fluent-bit, graylog diff --git a/ansible/roles/common/templates/msmtp/msmtprc.j2 b/ansible/roles/common/templates/msmtp/msmtprc.j2 new file mode 100644 index 0000000..20f29db --- /dev/null +++ b/ansible/roles/common/templates/msmtp/msmtprc.j2 @@ -0,0 +1,17 @@ +# {{ ansible_managed }} +defaults +auth on +tls on +tls_trust_file /etc/pki/tls/certs/ca-bundle.crt +logfile /var/log/msmtp.log + +account home +host {{ system_smtp_host | default('mail.b.hostedemail.com') }} +port {{ system_smtp_port | default(465) }} +# Port 465 is implicit TLS (SMTPS), not STARTTLS. +tls_starttls off +from {{ system_smtp_from | default('home@bdebyl.net') }} +user {{ system_smtp_user | default('home@bdebyl.net') }} +password {{ home_smtp }} + +account default : home diff --git a/ansible/roles/gitea-actions/templates/Containerfile.espidf.j2 b/ansible/roles/gitea-actions/templates/Containerfile.espidf.j2 index b849d61..6a64850 100644 --- a/ansible/roles/gitea-actions/templates/Containerfile.espidf.j2 +++ b/ansible/roles/gitea-actions/templates/Containerfile.espidf.j2 @@ -1,16 +1,23 @@ # ESP-IDF firmware job image (managed by ansible: roles/gitea-actions). # Adds node (required by actions/checkout and other JS actions), the AWS CLI -# (firmware artifacts ship to S3), and the common-yaml header generator's Python -# deps on top of the official Espressif toolchain. +# (firmware artifacts ship to S3), jq (the release script rewrites the protocol +# manifest with it), and the common-yaml header generator's Python deps on top +# of the official Espressif toolchain. # IDF lives at /opt/esp/idf — firmware jobs source /opt/esp/idf/export.sh. # python3-yaml + python3-jinja2 are installed as distro packages so the # common-yaml generator runs with a plain `python3 generate.py` — no pip at job # time (the base image's system Python is PEP 668 externally-managed) and no # need to source the IDF venv just to generate headers. +# +# jq is required by esp32-stm32-vcu scripts/release.sh, which publishes the +# protocol manifest and BLE_API.md to S3 after the firmware upload. Without it +# the release aborts *after* the firmware and version.json are already live — +# clients get the new build while the tag, Gitea release and protocol manifest +# are never written. Keep it installed. FROM espressif/idf:{{ esp_idf_version }} RUN apt-get update && apt-get install -y --no-install-recommends \ - curl ca-certificates unzip python3-yaml python3-jinja2 \ + curl ca-certificates unzip jq python3-yaml python3-jinja2 \ && curl -fsSL https://deb.nodesource.com/setup_20.x | bash - \ && apt-get install -y --no-install-recommends nodejs \ && rm -rf /var/lib/apt/lists/* diff --git a/ansible/roles/podman/defaults/main.yml b/ansible/roles/podman/defaults/main.yml index 70a8b88..7204e2d 100644 --- a/ansible/roles/podman/defaults/main.yml +++ b/ansible/roles/podman/defaults/main.yml @@ -1,4 +1,6 @@ --- +# Where Nextcloud backup failure alerts are mailed (see containers/cloud-backup.yml). +backup_alert_email: bastian@debyl.io bookstack_path: "{{ podman_volumes }}/bookstack" cam2ip_path: "{{ podman_volumes }}/cam2ip" cloud_path: "{{ podman_volumes }}/cloud" diff --git a/ansible/roles/podman/tasks/containers/cloud-backup.yml b/ansible/roles/podman/tasks/containers/cloud-backup.yml index 00bd37d..40fb012 100644 --- a/ansible/roles/podman/tasks/containers/cloud-backup.yml +++ b/ansible/roles/podman/tasks/containers/cloud-backup.yml @@ -28,6 +28,27 @@ mode: 0755 setype: bin_t +# Shared by every backup instance. Rendered once per include; the second and +# later renders are no-ops. +- name: template nextcloud backup alert script + become: true + ansible.builtin.template: + src: nextcloud/nextcloud-backup-alert.sh.j2 + dest: /usr/local/bin/nextcloud-backup-alert.sh + owner: root + group: root + mode: 0755 + setype: bin_t + +- name: template nextcloud backup failure handler unit + become: true + ansible.builtin.template: + src: nextcloud/nextcloud-backup-failed@.service.j2 + dest: /etc/systemd/system/nextcloud-backup-failed@.service + owner: root + group: root + mode: 0644 + - name: template {{ backup_name }} backup systemd service become: true ansible.builtin.template: diff --git a/ansible/roles/podman/tasks/containers/cloud-cron.yml b/ansible/roles/podman/tasks/containers/cloud-cron.yml new file mode 100644 index 0000000..1502453 --- /dev/null +++ b/ansible/roles/podman/tasks/containers/cloud-cron.yml @@ -0,0 +1,40 @@ +--- +- name: template {{ cron_name }} cron script + become: true + ansible.builtin.template: + src: nextcloud/cloud-cron.sh.j2 + dest: "{{ cron_script_path }}" + owner: root + group: root + mode: 0755 + setype: bin_t + +- name: template {{ cron_name }} cron systemd service + become: true + ansible.builtin.template: + src: nextcloud/cloud-cron.service.j2 + dest: "/etc/systemd/system/{{ cron_name }}-cron.service" + owner: root + group: root + mode: 0644 + vars: + instance_name: "{{ cron_name }}" + +- name: template {{ cron_name }} cron systemd timer + become: true + ansible.builtin.template: + src: nextcloud/cloud-cron.timer.j2 + dest: "/etc/systemd/system/{{ cron_name }}-cron.timer" + owner: root + group: root + mode: 0644 + vars: + instance_name: "{{ cron_name }}" + +- name: enable and start {{ cron_name }} cron timer + become: true + ansible.builtin.systemd: + name: "{{ cron_name }}-cron.timer" + enabled: true + state: started + daemon_reload: true diff --git a/ansible/roles/podman/tasks/containers/home/cloud.yml b/ansible/roles/podman/tasks/containers/home/cloud.yml index 0dddc81..f67062e 100644 --- a/ansible/roles/podman/tasks/containers/home/cloud.yml +++ b/ansible/roles/podman/tasks/containers/home/cloud.yml @@ -84,10 +84,44 @@ vars: container_name: cloud +# Unbounded by default: nextcloud.log.1 had reached 1.12 GB and was being +# rsynced to TrueNAS and pushed to S3 on every run. Cap at 10 MiB. +- name: cap nextcloud log rotation size for cloud + become: true + become_user: "{{ podman_user }}" + ansible.builtin.command: > + podman exec -u www-data cloud + php occ config:system:set log_rotate_size --value 10485760 --type integer + register: cloud_log_rotate + changed_when: "'System config value log_rotate_size' in cloud_log_rotate.stdout" + failed_when: false + +# Nextcloud's default ('auto') only expires trash when disk space is needed, +# so 66 GB of >30-day deletions sat untouched on a host with 1.3 TB free -- +# the retention was effectively unbounded. 'auto, 30' makes the 30-day +# expiry unconditional while still purging early under space pressure. +- name: set nextcloud trashbin retention for cloud + become: true + become_user: "{{ podman_user }}" + ansible.builtin.command: > + podman exec -u www-data cloud + php occ config:system:set trashbin_retention_obligation --value "auto, 30" + register: cloud_trashbin_retention + changed_when: "'System config value trashbin_retention_obligation' in cloud_trashbin_retention.stdout" + failed_when: false + +- include_tasks: containers/cloud-cron.yml + vars: + cron_name: cloud + cron_container: cloud + cron_script_path: /usr/local/bin/cloud-cron.sh + - include_tasks: containers/cloud-backup.yml vars: backup_name: cloud data_path: "{{ cloud_path }}/data" + config_path: "{{ cloud_path }}/config" + db_container: cloud-db ssh_key_path: /etc/ssh/backup_keys/cloud ssh_key_content: "{{ cloud_backup_ssh_key }}" ssh_user: cloud diff --git a/ansible/roles/podman/tasks/containers/skudak/cloud.yml b/ansible/roles/podman/tasks/containers/skudak/cloud.yml index ab2df47..9fe2d2f 100644 --- a/ansible/roles/podman/tasks/containers/skudak/cloud.yml +++ b/ansible/roles/podman/tasks/containers/skudak/cloud.yml @@ -131,12 +131,50 @@ changed_when: "'System config value trusted_domains' in trusted_domain_result.stdout" failed_when: false +# This instance was left at loglevel 0 (DEBUG) and had written a 64 GB +# nextcloud.log, almost entirely repeated deprecation notices. 2 = Warning, +# which is both the Nextcloud default and what the home instance already uses. +- name: set nextcloud loglevel for skudak-cloud + become: true + become_user: "{{ podman_user }}" + ansible.builtin.command: > + podman exec -u www-data skudak-cloud + php occ config:system:set loglevel --value 2 --type integer + register: skudak_loglevel + changed_when: "'System config value loglevel' in skudak_loglevel.stdout" + failed_when: false + +# Unbounded by default; see the equivalent task in containers/home/cloud.yml. +- name: cap nextcloud log rotation size for skudak-cloud + become: true + become_user: "{{ podman_user }}" + ansible.builtin.command: > + podman exec -u www-data skudak-cloud + php occ config:system:set log_rotate_size --value 10485760 --type integer + register: skudak_log_rotate + changed_when: "'System config value log_rotate_size' in skudak_log_rotate.stdout" + failed_when: false + +- include_tasks: containers/cloud-cron.yml + vars: + cron_name: skudak-cloud + cron_container: skudak-cloud + cron_script_path: /usr/local/bin/skudak-cloud-cron.sh + - include_tasks: containers/cloud-backup.yml vars: backup_name: skudak-cloud data_path: "{{ cloud_skudak_path }}/data" + config_path: "{{ cloud_skudak_path }}/config" + db_container: skudak-cloud-db ssh_key_path: /etc/ssh/backup_keys/skudak-cloud ssh_key_content: "{{ cloud_skudak_backup_ssh_key }}" ssh_user: skucloud remote_path: /mnt/glacier/skudakcloud script_path: /usr/local/bin/skudak-cloud-backup.sh + # skudakcloud/data is mode 770, so the receiving side needs traversable + # dirs. This flag was hand-added on the host and was being silently + # reverted by every `make deploy TAGS=skudak-cloud`; it now lives in git. + backup_rsync_extra_args: "--chmod=Du=rwx,Dgo=rx" + # Staggered so both instances finish before the 05:00 TrueNAS snapshot. + backup_oncalendar: "*-*-* 04:30:00" diff --git a/ansible/roles/podman/tasks/main.yml b/ansible/roles/podman/tasks/main.yml index a981a85..cca4d9b 100644 --- a/ansible/roles/podman/tasks/main.yml +++ b/ansible/roles/podman/tasks/main.yml @@ -67,24 +67,24 @@ - import_tasks: containers/home/cloud.yml vars: db_image: docker.io/library/mariadb:10.6 - image: docker.io/library/nextcloud:33.0.0-apache + image: docker.io/library/nextcloud:34.0.2-apache tags: cloud - import_tasks: containers/skudak/cloud.yml vars: db_image: docker.io/library/mariadb:10.6 - image: docker.io/library/nextcloud:33.0.0-apache + image: docker.io/library/nextcloud:34.0.2-apache tags: skudak, skudak-cloud - import_tasks: containers/debyltech/fulfillr.yml vars: - image: git.debyl.io/debyltech/fulfillr:20260723.2044 + image: git.debyl.io/debyltech/fulfillr:20260728.2155 tags: debyltech, fulfillr # Staging back-office (fulfillr-dev.debyltech.com) — same image, staging Turso config. - import_tasks: containers/debyltech/fulfillr-dev.yml vars: - image: git.debyl.io/debyltech/fulfillr:20260723.2044 + image: git.debyl.io/debyltech/fulfillr:20260728.2155 tags: debyltech, fulfillr-dev - import_tasks: containers/debyltech/uptime-kuma.yml @@ -109,7 +109,7 @@ - import_tasks: containers/home/gregtime.yml vars: - image: localhost/greg-time-bot:3.9.25 + image: localhost/greg-time-bot:3.10.0 tags: gregtime - import_tasks: containers/home/zomboid.yml diff --git a/ansible/roles/podman/templates/nextcloud/cloud-backup.service.j2 b/ansible/roles/podman/templates/nextcloud/cloud-backup.service.j2 index 626c635..4183855 100644 --- a/ansible/roles/podman/templates/nextcloud/cloud-backup.service.j2 +++ b/ansible/roles/podman/templates/nextcloud/cloud-backup.service.j2 @@ -1,6 +1,14 @@ [Unit] Description=Nextcloud {{ instance_name }} backup to TrueNAS +After=network-online.target +Wants=network-online.target +OnFailure=nextcloud-backup-failed@%n.service [Service] Type=oneshot ExecStart={{ script_path }} +# Type=oneshot disables the start timeout by default, so a wedged rsync would +# leave the unit "activating" forever and every subsequent daily trigger would +# be silently skipped. Bound it. +TimeoutStartSec={{ backup_timeout | default('4h') }} +Nice=10 diff --git a/ansible/roles/podman/templates/nextcloud/cloud-backup.sh.j2 b/ansible/roles/podman/templates/nextcloud/cloud-backup.sh.j2 index 50a656f..7ad1ed5 100644 --- a/ansible/roles/podman/templates/nextcloud/cloud-backup.sh.j2 +++ b/ansible/roles/podman/templates/nextcloud/cloud-backup.sh.j2 @@ -1,4 +1,117 @@ #!/bin/bash +# {{ ansible_managed }} +# Nextcloud "{{ backup_name }}" -> truenas.localdomain. +# +# Ordering is deliberate: the database is dumped BEFORE the file tree is +# synced. A DB snapshot slightly OLDER than the files degrades to "files +# Nextcloud has not indexed yet" and is repaired with `occ files:scan`. A DB +# snapshot NEWER than the files references blobs that never made it into the +# backup, which surfaces as broken shares and dead file entries on restore. set -euo pipefail -rsync -az --exclude .ssh -e "ssh -i {{ ssh_key_path }} -o StrictHostKeyChecking=accept-new" \ - {{ data_path }}/ {{ ssh_user }}@truenas.localdomain:{{ remote_path }}/ + +TAG=nextcloud-backup +INSTANCE={{ backup_name }} +STAGE={{ backup_stage_path | default('/var/backups/nextcloud/' ~ backup_name) }} +KEEP={{ backup_db_keep | default(7) }} +DUMP="$STAGE/db/${INSTANCE}-$(date +%Y%m%d).sql.gz" + +log() { logger -t "$TAG" -p daemon.info -- "instance=$INSTANCE $*"; echo "$TAG: $*"; } +fail() { logger -t "$TAG" -p daemon.err -- "instance=$INSTANCE status=failed $*" + echo "$TAG: FAILED: $*" >&2; exit 1; } + +SSH="ssh -i {{ ssh_key_path }} -o StrictHostKeyChecking=accept-new -o ServerAliveInterval=30 -o ServerAliveCountMax=6" +DEST={{ ssh_user }}@truenas.localdomain + +log "status=start" + +{% if db_container | default('') %} +# ------------------------------------------------------------ 1. database +# The Nextcloud containers are ROOTLESS podman owned by "{{ podman_user }}", +# but this script runs as root under systemd. Every podman call therefore +# goes through sudo: +# -H HOME becomes the podman user's home, so podman finds its rootless +# graph root under ~/.local/share/containers +# cd; required preamble (see CLAUDE.md) so the shell starts in that home +# XDG_RUNTIME_DIR the podman user's runtime dir. Lingering is enabled by +# roles/podman/tasks/podman/podman.yml so /run/user/ exists; +# guarded anyway so podman falls back cleanly if it ever does not. +# +# No credential is stored in this file or placed on a host command line: +# $MYSQL_ROOT_PASSWORD and $MYSQL_DATABASE are expanded by the shell INSIDE +# the database container, which already carries them in its environment. +# +# --routines and --events initially aborted the dump here: mysql.proc read as +# corrupted (error 1728) and the event scheduler reported disabled (1577), +# both artefacts of an image bump without mariadb-upgrade. `mariadb-upgrade +# --force` has since been run against both instances and repaired the system +# tables, so the full flag set works and is kept for completeness. +# +# Note that upgrade still exits non-zero on these containers: it cannot +# install the `sys` schema because the datadir root (/var/lib/mysql) is owned +# by daemon rather than mysql, so mysqld may not create new top-level +# databases. `sys` is purely diagnostic and unused by Nextcloud, so this is +# cosmetic -- but it does mean creating a NEW database would fail too. +pexec() { + sudo -H -u {{ podman_user }} bash -c \ + 'cd; d=/run/user/$(id -u); [ -d "$d" ] && export XDG_RUNTIME_DIR="$d" + exec podman "$@"' _ "$@" +} + +install -d -m 0700 "$STAGE" "$STAGE/db" +tmp="$DUMP.tmp" +rm -f "$tmp" + +log "dumping {{ db_container }}" +set +e +pexec exec {{ db_container }} sh -c ' + exec env MYSQL_PWD="$MYSQL_ROOT_PASSWORD" mariadb-dump -u root \ + --single-transaction --quick --routines --events --triggers \ + --no-tablespaces --default-character-set=utf8mb4 "$MYSQL_DATABASE" +' | gzip -6 > "$tmp" +dump_rc=${PIPESTATUS[0]} +set -e +[ "$dump_rc" -eq 0 ] || fail "mariadb-dump {{ db_container }} exited $dump_rc" + +# A new dump is promoted over yesterday's only after it proves complete: +# a valid gzip stream AND the "-- Dump completed" trailer that mariadb-dump +# writes only on a clean finish. `mv` is atomic within the staging +# filesystem, so a failed or truncated run can never replace a good dump. +gzip -t "$tmp" || fail "dump is not a valid gzip stream" +gunzip -c "$tmp" | tail -c 512 | grep -q 'Dump completed' \ + || fail "dump is truncated (no completion trailer)" +mv -f "$tmp" "$DUMP" +log "db_dump=ok bytes=$(stat -c %s "$DUMP")" + +# Local retention. The staging sync below mirrors with --delete, so remote +# retention follows the same window; deeper history comes from the TrueNAS +# periodic ZFS snapshots (see roles/podman/README.md). +ls -1t "$STAGE"/db/"$INSTANCE"-*.sql.gz | tail -n +$((KEEP + 1)) | xargs -r rm -f +{% endif %} + +# ----------------------------------------------------------- 2. file tree +# --exclude .ssh is load-bearing: {{ remote_path }} IS {{ ssh_user }}'s home +# on TrueNAS and its authorized_keys lives there, so a `.ssh` directory +# appearing in the data tree must never be shipped. No --delete here: the +# data tree is append-mostly and a source-side mishap must not propagate. +log "syncing data" +rsync -az --timeout=1800 --exclude .ssh \ + {{ backup_rsync_excludes | default("--exclude '/nextcloud.log*' --exclude '/updater.log'") }} \ + {{ backup_rsync_extra_args | default('') }} \ + -e "$SSH" {{ data_path }}/ "$DEST:{{ remote_path }}/" + +# ------------------------------------------- 3. config/ and database dumps +{# --mkpath creates the nested _backup// destination; rsync will not build + more than one missing level on its own. Requires rsync >= 3.2.3 on both + ends (galactica 3.4.1, truenas 3.2.7). #} +{% if config_path | default('') %} +log "syncing config" +rsync -az --timeout=600 --delete --mkpath {{ backup_rsync_extra_args | default('') }} \ + -e "$SSH" {{ config_path }}/ "$DEST:{{ remote_path }}/_backup/config/" +{% endif %} +{% if db_container | default('') %} +log "syncing db dumps" +rsync -az --timeout=600 --delete --mkpath {{ backup_rsync_extra_args | default('') }} \ + -e "$SSH" "$STAGE/db/" "$DEST:{{ remote_path }}/_backup/db/" +{% endif %} + +log "status=ok" diff --git a/ansible/roles/podman/templates/nextcloud/cloud-backup.timer.j2 b/ansible/roles/podman/templates/nextcloud/cloud-backup.timer.j2 index ca27957..0982d9f 100644 --- a/ansible/roles/podman/templates/nextcloud/cloud-backup.timer.j2 +++ b/ansible/roles/podman/templates/nextcloud/cloud-backup.timer.j2 @@ -2,7 +2,8 @@ Description=Daily Nextcloud {{ instance_name }} backup [Timer] -OnCalendar=*-*-* 04:00:00 +OnCalendar={{ backup_oncalendar | default('*-*-* 04:00:00') }} +RandomizedDelaySec={{ backup_randomized_delay | default('5m') }} Persistent=true [Install] diff --git a/ansible/roles/podman/templates/nextcloud/cloud-cron.service.j2 b/ansible/roles/podman/templates/nextcloud/cloud-cron.service.j2 new file mode 100644 index 0000000..27bb1f0 --- /dev/null +++ b/ansible/roles/podman/templates/nextcloud/cloud-cron.service.j2 @@ -0,0 +1,13 @@ +[Unit] +Description=Nextcloud {{ instance_name }} background jobs +After=network-online.target +Wants=network-online.target + +[Service] +Type=oneshot +ExecStart={{ cron_script_path }} +# Type=oneshot disables the start timeout by default, so a wedged cron.php +# would leave the unit "activating" forever and every subsequent 5-minute +# trigger would be silently skipped. Bound it. +TimeoutStartSec={{ cron_timeout | default('30m') }} +Nice=10 diff --git a/ansible/roles/podman/templates/nextcloud/cloud-cron.sh.j2 b/ansible/roles/podman/templates/nextcloud/cloud-cron.sh.j2 new file mode 100644 index 0000000..672dbd1 --- /dev/null +++ b/ansible/roles/podman/templates/nextcloud/cloud-cron.sh.j2 @@ -0,0 +1,51 @@ +#!/bin/bash +# {{ ansible_managed }} +# Nextcloud "{{ cron_name }}" background jobs (cron.php). +# +# backgroundjobs_mode is "cron" on both instances, which means Nextcloud +# expects an external caller to run cron.php every ~5 minutes. Nothing was: +# the personal instance had not run a background job since 2026-05-14 and +# skudak since 2024-11-20. Without it Nextcloud never expires trash or file +# versions, never cleans stale chunked uploads, never sends calendar +# reminders, and -- easy to miss -- never rotates nextcloud.log, which makes +# the log_rotate_size cap set in containers/*/cloud.yml inert. +set -euo pipefail + +TAG=nextcloud-cron +INSTANCE={{ cron_name }} + +log() { logger -t "$TAG" -p daemon.info -- "instance=$INSTANCE $*"; } +fail() { logger -t "$TAG" -p daemon.err -- "instance=$INSTANCE status=failed $*" + echo "$TAG: FAILED: $*" >&2; exit 1; } + +# The Nextcloud containers are ROOTLESS podman owned by "{{ podman_user }}", +# but this script runs as root under systemd. Same sudo/cd/XDG_RUNTIME_DIR +# preamble as cloud-backup.sh -- see CLAUDE.md for why `cd;` is required. +pexec() { + sudo -H -u {{ podman_user }} bash -c \ + 'cd; d=/run/user/$(id -u); [ -d "$d" ] && export XDG_RUNTIME_DIR="$d" + exec podman "$@"' _ "$@" +} + +# A container that is down (deploy, image bump, host reboot) is not a failure +# worth flagging -- the next tick picks it up five minutes later. +if ! pexec container exists {{ cron_container }} 2>/dev/null; then + log "status=skipped reason=container-absent" + exit 0 +fi + +# occ and cron.php both refuse to do anything useful mid-upgrade. Skipping +# keeps a deploy window from parading as a run of failed units. +if pexec exec -u www-data {{ cron_container }} php occ status 2>/dev/null \ + | grep -q 'maintenance: true'; then + log "status=skipped reason=maintenance" + exit 0 +fi + +set +e +pexec exec -u www-data {{ cron_container }} php -f /var/www/html/cron.php +rc=$? +set -e +[ "$rc" -eq 0 ] || fail "cron.php exited $rc" + +log "status=ok" diff --git a/ansible/roles/podman/templates/nextcloud/cloud-cron.timer.j2 b/ansible/roles/podman/templates/nextcloud/cloud-cron.timer.j2 new file mode 100644 index 0000000..cf1d9a5 --- /dev/null +++ b/ansible/roles/podman/templates/nextcloud/cloud-cron.timer.j2 @@ -0,0 +1,13 @@ +[Unit] +Description=Nextcloud {{ instance_name }} background jobs every 5 minutes + +[Timer] +OnBootSec={{ cron_onbootsec | default('5m') }} +OnUnitActiveSec={{ cron_interval | default('5m') }} +RandomizedDelaySec={{ cron_randomized_delay | default('30s') }} +# Deliberately NOT Persistent: this runs every 5 minutes, so replaying runs +# missed while the host was off buys nothing and just stampedes at boot. +Persistent=false + +[Install] +WantedBy=timers.target diff --git a/ansible/roles/podman/templates/nextcloud/nextcloud-backup-alert.sh.j2 b/ansible/roles/podman/templates/nextcloud/nextcloud-backup-alert.sh.j2 new file mode 100644 index 0000000..9598b8e --- /dev/null +++ b/ansible/roles/podman/templates/nextcloud/nextcloud-backup-alert.sh.j2 @@ -0,0 +1,65 @@ +#!/bin/bash +# {{ ansible_managed }} +# OnFailure= handler for the Nextcloud backup units. Invoked as: +# nextcloud-backup-alert.sh +# +# Deliberately NOT `set -e`: an alert handler that dies partway through +# reports nothing, which is worse than a partial report. Same reasoning as +# roles/ups/templates/ups-restore.sh.j2. +set -uo pipefail + +TAG=nextcloud-backup +UNIT="${1:-unknown}" +TO="{{ backup_alert_email | default('root') }}" +HOST="$(hostname -f 2>/dev/null || hostname)" + +result="$(systemctl show -p Result --value "$UNIT" 2>/dev/null)" +code="$(systemctl show -p ExecMainStatus --value "$UNIT" 2>/dev/null)" + +# Only claim a failure when the unit actually reports one. Starting this +# handler by hand (or any other spurious trigger) would otherwise mail out a +# subject line saying FAILED about a run that succeeded. Keeping status=failed +# exact also stops such triggers matching the Graylog alert rule. +if [ "${result:-success}" = "success" ]; then + state=spurious + prio=daemon.warning + headline="$(printf 'Nextcloud backup alert handler was invoked on %s, but %s reports SUCCESS.\nThis is not a backup failure -- most likely the handler was started manually.' "$HOST" "$UNIT")" + subject="[$HOST] Nextcloud backup alert (spurious, unit OK): $UNIT" +else + state=failed + prio=daemon.err + headline="Nextcloud backup FAILED on $HOST" + subject="[$HOST] Nextcloud backup FAILED: $UNIT" +fi + +# One machine-parseable line for Graylog, then the context. +logger -t "$TAG" -p "$prio" -- \ + "status=$state unit=$UNIT result=${result:-unknown} exit=${code:-unknown}" + +body="$(printf '%s\n\nunit: %s\nresult: %s\nexit: %s\n\n--- last 40 journal lines ---\n' \ + "$headline" "$UNIT" "${result:-unknown}" "${code:-unknown}") +$(journalctl -u "$UNIT" -n 40 --no-pager -o cat 2>/dev/null)" + +echo "$body" | logger -t "$TAG" -p "$prio" + +# Only genuine failures are worth an email. A spurious invocation carries no +# action for a human, and mailing it trains the reader to ignore the subject +# line -- which defeats the point of having the alert at all. The journald +# record above is kept either way, so spurious triggers stay greppable. +if [ "$state" != "failed" ]; then + logger -t "$TAG" -p daemon.info -- "alert_mail=skipped reason=$state" + exit 0 +fi + +# Mail is best-effort: if the MTA is not configured the journald record above +# is still the authoritative signal, so never fail the handler on this. +if command -v sendmail >/dev/null 2>&1; then + printf 'To: %s\nSubject: %s\nContent-Type: text/plain; charset=UTF-8\n\n%s\n' \ + "$TO" "$subject" "$body" | sendmail -t \ + && logger -t "$TAG" -p daemon.info -- "alert_mail=sent to=$TO" \ + || logger -t "$TAG" -p daemon.err -- "alert_mail=failed to=$TO" +else + logger -t "$TAG" -p daemon.err -- "alert_mail=skipped reason=no-sendmail" +fi + +exit 0 diff --git a/ansible/roles/podman/templates/nextcloud/nextcloud-backup-failed@.service.j2 b/ansible/roles/podman/templates/nextcloud/nextcloud-backup-failed@.service.j2 new file mode 100644 index 0000000..5da3b87 --- /dev/null +++ b/ansible/roles/podman/templates/nextcloud/nextcloud-backup-failed@.service.j2 @@ -0,0 +1,6 @@ +[Unit] +Description=Report failure of %i + +[Service] +Type=oneshot +ExecStart=/usr/local/bin/nextcloud-backup-alert.sh %i diff --git a/ansible/roles/ups/README.md b/ansible/roles/ups/README.md new file mode 100644 index 0000000..0b402ad --- /dev/null +++ b/ansible/roles/ups/README.md @@ -0,0 +1,114 @@ +# ups + +UPS monitoring and staged shutdown for the home rack. + +A CyberPower PR1500RT2U (`0764:0601`) is cabled by USB to `home.debyl.io` and +backs both that host and `truenas.localdomain` (Dell PowerEdge R415). This role +makes `home.debyl.io` the NUT server and gives it the ability to power TrueNAS +back on over IPMI. + +## Outage sequence + +| When | What happens | Driven by | +| --- | --- | --- | +| t+0 | UPS goes on battery, `ONBATT` logged to journald → Graylog | `upsmon` | +| t+2min | TrueNAS shuts itself down cleanly, shedding ~200 W | TrueNAS UPS service, Slave mode, `Shutdown Timer 120` | +| 10% charge | `home.debyl.io` shuts itself down and tells the UPS to cut its output | `upsmon` `SHUTDOWNCMD` + `/lib/systemd/system-shutdown/nutshutdown` | +| mains returns | UPS re-energizes; both machines power themselves back up | R415 `always-on` restore policy; `home.debyl.io` BIOS *After Power Loss → Power On* | +| mains back +5min | Best-effort IPMI power-on, if a dedicated iDRAC is ever fitted | `upssched` → `ups-restore.sh` | +| host boot | Same restore check, for the deep-drain case | `ups-restore.service` | + +The 10% threshold is not a custom poller. CyberPower asserts its own low-battery +flag around 20–35%, so `ups.conf` sets `ignorelb` plus +`override.battery.charge.low`, and stock `upsmon` fires at exactly the +configured percentage. + +Likewise, the 2-minute TrueNAS shed is TrueNAS's own native "shutdown timer" +setting — no SSH key and no shutdown script from this side. + +## Powering TrueNAS back on + +Neither Wake-on-LAN nor IPMI works on this box today, so restore is done with +the chassis power restore policy instead. + +**Wake-on-LAN is out.** `bce0` advertises no WOL capability (`ifconfig -m bce0` +has no `WOL_MAGIC`) and both NICs are bonded into an LACP `lagg0`. + +**IPMI is out too, for now.** The iDRAC is in `shared / LOM1` mode and the +Enterprise card that would provide a dedicated management port is not fitted: + +``` +$ ipmitool sdr elist | grep -i idrac +iDRAC6 Ent Pres | 70h | ok | 7.1 | Absent +``` + +A shared-LOM iDRAC6 Express has no standby power. Measured directly: with the +chassis powered off the BMC does not even answer ARP, and it only reappears +~200s into POST, at the moment the host brings the NIC link up. That is a +hardware limitation, not a switch or BIOS problem. + +**So restore works like this instead.** `ipmitool chassis policy always-on` is +set on the R415. On a deep outage `home.debyl.io` halts at 10% and NUT's +shutdown hook tells the UPS to cut its output; when mains returns the UPS +re-energizes, the R415 sees AC and boots itself. + +The gap is the medium outage — mains returns after TrueNAS has shed but before +the battery reaches 10%. The UPS never cuts power, so TrueNAS stays off and +needs a manual power button press. Fitting a used iDRAC6 Enterprise card and +running `ipmitool delloem lan set dedicated` (plus a cable to the dedicated +port) closes that gap, and `ups-restore.sh` starts working with no code +changes — it is already deployed and simply logs and exits while the BMC is +unreachable. + +## One-time setup outside Ansible + +These are not managed by this role. + +**iDRAC (already done, via `ipmitool` on TrueNAS):** + +``` +ipmitool lan set 1 ipsrc static +ipmitool lan set 1 ipaddr 192.168.1.12 +ipmitool lan set 1 netmask 255.255.255.0 +ipmitool lan set 1 defgw ipaddr 192.168.1.1 +ipmitool lan set 1 access on # was disabled - nothing answers without this +ipmitool channel setaccess 1 2 callin=on ipmi=on link=on privilege=4 +ipmitool user set password 2 '' +ipmitool chassis policy always-on # this is what restores power after an outage +``` + +`idrac_password` is also the iDRAC web UI password for `root` - they share a +user database. + +**TrueNAS UI → Services → UPS** (enable + start automatically): + +| Field | Value | +| --- | --- | +| UPS Mode | Slave | +| Remote Host | `192.168.1.10` | +| Remote Port | `3493` | +| Identifier | `cyberpower` | +| Monitor User | `truenas` | +| Monitor Password | vault `nut_truenas_password` | +| Shutdown Mode | UPS goes on battery | +| Shutdown Timer | `120` | +| Shutdown Command | `/sbin/shutdown -p now` | +| Power Off UPS | unchecked | + +**BIOS on `home.debyl.io`** (Lenovo 10MR0004US): Power → *After Power Loss* → +**Power On**. On a full drain the NUT shutdown hook +(`/lib/systemd/system-shutdown/nutshutdown`) tells the UPS to cut its own +output; this BIOS setting is what brings the host back when mains returns. + +## Vault keys + +`nut_upsmon_password`, `nut_truenas_password`, `idrac_password` + +## Operating + +``` +upsc cyberpower # full UPS status +upsc cyberpower battery.charge +sudo /usr/local/bin/truenas-power.sh status|on|soft|off +journalctl -t ups-restore -t ups-sched +``` diff --git a/ansible/roles/ups/defaults/main.yml b/ansible/roles/ups/defaults/main.yml new file mode 100644 index 0000000..b8d8408 --- /dev/null +++ b/ansible/roles/ups/defaults/main.yml @@ -0,0 +1,36 @@ +--- +# CyberPower PR1500LCDRT2U cabled by USB to this host (0764:0601). +# This host is the NUT server; truenas.localdomain is a NUT slave. +ups_name: cyberpower +# lsusb reports the product string as PR1500LCDRT2U, but the device itself +# reports device.model CP1500PFCRM2U. The latter is what it actually is. +ups_desc: CyberPower CP1500PFCRM2U +ups_vendorid: "0764" +ups_productid: "0601" + +ups_listen_addr: 192.168.1.10 +ups_listen_port: 3493 + +# truenas.localdomain - the only host allowed to reach upsd +ups_slave_ip: 192.168.1.11 + +# Battery charge at which THIS host shuts itself down. TrueNAS sheds much +# earlier via its own "shutdown timer" setting (see roles/ups/README.md). +ups_low_charge_pct: 10 + +# Seconds of stable mains after ONLINE before TrueNAS is powered back on. +ups_restore_stable_secs: 300 + +# R415 iDRAC6. Shares LOM1 with bce0. +idrac_host: 192.168.1.12 +idrac_user: root + +ups_deps: + [ + ipmitool, + nut, + nut-client, + ] + +# Secrets live in ansible/vars/vault.yml (no vault_ prefix, per repo +# convention): nut_upsmon_password, nut_truenas_password, idrac_password diff --git a/ansible/roles/ups/handlers/main.yml b/ansible/roles/ups/handlers/main.yml new file mode 100644 index 0000000..e2af69c --- /dev/null +++ b/ansible/roles/ups/handlers/main.yml @@ -0,0 +1,35 @@ +--- +# Regenerates the nut-driver@ unit instances from ups.conf. Oneshot, +# so "restarted" just means "run it again". +- name: reload nut driver units + become: true + ansible.builtin.systemd: + name: nut-driver-enumerator.service + state: restarted + daemon_reload: true + +# The enumerator only writes unit definitions; it will not pick up changed +# driver options in an already-running driver. Restart the instance itself. +- name: restart nut driver + become: true + ansible.builtin.systemd: + name: "nut-driver@{{ ups_name }}.service" + state: restarted + +- name: restart nut server + become: true + ansible.builtin.systemd: + name: nut-server.service + state: restarted + +- name: restart nut monitor + become: true + ansible.builtin.systemd: + name: nut-monitor.service + state: restarted + +- name: restart firewalld + become: true + ansible.builtin.systemd: + name: firewalld + state: restarted diff --git a/ansible/roles/ups/tasks/deps.yml b/ansible/roles/ups/tasks/deps.yml new file mode 100644 index 0000000..2c640ce --- /dev/null +++ b/ansible/roles/ups/tasks/deps.yml @@ -0,0 +1,7 @@ +--- +- name: install NUT and ipmitool + become: true + ansible.builtin.dnf: + name: "{{ ups_deps }}" + state: present + tags: ups diff --git a/ansible/roles/ups/tasks/firewall.yml b/ansible/roles/ups/tasks/firewall.yml new file mode 100644 index 0000000..1004ef0 --- /dev/null +++ b/ansible/roles/ups/tasks/firewall.yml @@ -0,0 +1,12 @@ +--- +- name: allow NUT from the truenas slave only + become: true + ansible.posix.firewalld: + rich_rule: >- + rule family="ipv4" source address="{{ ups_slave_ip }}/32" + port port="{{ ups_listen_port }}" protocol="tcp" accept + permanent: true + immediate: true + state: enabled + notify: restart firewalld + tags: [ups, firewall] diff --git a/ansible/roles/ups/tasks/ipmi.yml b/ansible/roles/ups/tasks/ipmi.yml new file mode 100644 index 0000000..0a1518b --- /dev/null +++ b/ansible/roles/ups/tasks/ipmi.yml @@ -0,0 +1,68 @@ +--- +# IPMI power control for truenas.localdomain (Dell R415, iDRAC6). +# WOL is not an option there: bce0 advertises no WOL capability and is an +# LACP lagg member, so IPMI is the only remote power-on path. + +- name: deploy iDRAC credential + become: true + ansible.builtin.copy: + content: "{{ idrac_password }}\n" + dest: /etc/ups/idrac.pw + owner: root + group: nut + mode: '0640' + no_log: true + tags: ups + +- name: deploy truenas power helper + become: true + ansible.builtin.template: + src: truenas-power.sh.j2 + dest: /usr/local/bin/truenas-power.sh + owner: root + group: root + mode: '0755' + setype: bin_t + tags: ups + +- name: deploy truenas restore script + become: true + ansible.builtin.template: + src: ups-restore.sh.j2 + dest: /usr/local/bin/ups-restore.sh + owner: root + group: root + mode: '0755' + setype: bin_t + tags: ups + +- name: deploy upssched command dispatcher + become: true + ansible.builtin.template: + src: ups-sched-cmd.sh.j2 + dest: /usr/local/bin/ups-sched-cmd.sh + owner: root + group: root + mode: '0755' + setype: bin_t + tags: ups + +# Covers the deep-drain case: if the battery ran out, this host was itself +# powered off when mains returned, so nothing was running to restore TrueNAS. +- name: deploy boot-time truenas restore unit + become: true + ansible.builtin.template: + src: ups-restore.service.j2 + dest: /etc/systemd/system/ups-restore.service + owner: root + group: root + mode: '0644' + tags: ups + +- name: enable boot-time truenas restore unit + become: true + ansible.builtin.systemd: + name: ups-restore.service + enabled: true + daemon_reload: true + tags: ups diff --git a/ansible/roles/ups/tasks/main.yml b/ansible/roles/ups/tasks/main.yml new file mode 100644 index 0000000..bf19bd2 --- /dev/null +++ b/ansible/roles/ups/tasks/main.yml @@ -0,0 +1,5 @@ +--- +- import_tasks: deps.yml +- import_tasks: nut.yml +- import_tasks: ipmi.yml +- import_tasks: firewall.yml diff --git a/ansible/roles/ups/tasks/nut.yml b/ansible/roles/ups/tasks/nut.yml new file mode 100644 index 0000000..3f86819 --- /dev/null +++ b/ansible/roles/ups/tasks/nut.yml @@ -0,0 +1,129 @@ +--- +# NUT server. The UPS is on USB here, so this host drives it and serves +# status to truenas.localdomain over the network. + +- name: set NUT run mode to netserver + become: true + ansible.builtin.template: + src: nut.conf.j2 + dest: /etc/ups/nut.conf + owner: root + group: nut + mode: '0640' + notify: + - reload nut driver units + - restart nut driver + - restart nut server + - restart nut monitor + tags: ups + +- name: deploy NUT driver configuration + become: true + ansible.builtin.template: + src: ups.conf.j2 + dest: /etc/ups/ups.conf + owner: root + group: nut + mode: '0640' + notify: + - reload nut driver units + - restart nut driver + - restart nut server + tags: ups + +- name: deploy upsd listener configuration + become: true + ansible.builtin.template: + src: upsd.conf.j2 + dest: /etc/ups/upsd.conf + owner: root + group: nut + mode: '0640' + notify: restart nut server + tags: ups + +- name: deploy upsd users + become: true + ansible.builtin.template: + src: upsd.users.j2 + dest: /etc/ups/upsd.users + owner: root + group: nut + mode: '0640' + notify: + - restart nut server + - restart nut monitor + tags: ups + +- name: deploy upsmon configuration + become: true + ansible.builtin.template: + src: upsmon.conf.j2 + dest: /etc/ups/upsmon.conf + owner: root + group: nut + mode: '0640' + notify: restart nut monitor + tags: ups + +- name: deploy upssched configuration + become: true + ansible.builtin.template: + src: upssched.conf.j2 + dest: /etc/ups/upssched.conf + owner: root + group: nut + mode: '0640' + notify: restart nut monitor + tags: ups + +# The nut package ships /usr/lib/udev/rules.d/62-nut-usbups.rules, which +# hands the UPS USB device to the nut user. Reload and retrigger so the +# driver can claim it without physically replugging the UPS. +- name: reload udev rules for NUT USB access + become: true + ansible.builtin.command: + cmd: udevadm control --reload-rules + changed_when: false + tags: ups + +- name: retrigger UPS USB device + become: true + ansible.builtin.command: + cmd: >- + udevadm trigger --subsystem-match=usb + --attr-match=idVendor={{ ups_vendorid }} + changed_when: false + tags: ups + +# nut-driver-enumerator reads ups.conf and generates nut-driver@{{ ups_name }}. +# The .service is oneshot (so never "started" for long) and is triggered by +# the .path unit watching ups.conf - enable both, but only start the .path. +- name: enable NUT driver enumerator + become: true + ansible.builtin.systemd: + name: nut-driver-enumerator.service + enabled: true + daemon_reload: true + tags: ups + +- name: enable and start NUT driver enumerator path trigger + become: true + ansible.builtin.systemd: + name: nut-driver-enumerator.path + enabled: true + state: started + tags: ups + +- name: enable and start NUT services + become: true + ansible.builtin.systemd: + name: "{{ item }}" + enabled: true + state: started + loop: + - "nut-driver@{{ ups_name }}.service" + - nut-server.service + - nut-monitor.service + - nut.target + tags: ups diff --git a/ansible/roles/ups/templates/nut.conf.j2 b/ansible/roles/ups/templates/nut.conf.j2 new file mode 100644 index 0000000..d916a4c --- /dev/null +++ b/ansible/roles/ups/templates/nut.conf.j2 @@ -0,0 +1,3 @@ +# {{ ansible_managed }} +# netserver: run the driver + upsd locally and serve slaves over the network. +MODE=netserver diff --git a/ansible/roles/ups/templates/truenas-power.sh.j2 b/ansible/roles/ups/templates/truenas-power.sh.j2 new file mode 100644 index 0000000..310a189 --- /dev/null +++ b/ansible/roles/ups/templates/truenas-power.sh.j2 @@ -0,0 +1,34 @@ +#!/bin/bash +# {{ ansible_managed }} +# Remote power control for truenas.localdomain via the R415 iDRAC6. +# Usage: truenas-power.sh {status|on|soft|off|cycle} +set -euo pipefail + +PW_FILE=/etc/ups/idrac.pw + +if [ ! -r "$PW_FILE" ]; then + echo "cannot read $PW_FILE" >&2 + exit 1 +fi + +# -f keeps the password out of the process argument list. +ipmi() { + ipmitool -I lanplus -H {{ idrac_host }} -U {{ idrac_user }} \ + -f "$PW_FILE" "$@" +} + +case "${1:-status}" in + status) ipmi chassis power status ;; + on) ipmi chassis power on ;; + # ACPI soft-off. FreeBSD has hw.acpi.power_button_state=S5, so this is + # a clean TrueNAS shutdown. Normally unused: TrueNAS shuts itself down + # as a NUT slave. This is the manual escape hatch. + soft) ipmi chassis power soft ;; + # Hard cut, last resort only. + off) ipmi chassis power off ;; + cycle) ipmi chassis power cycle ;; + *) + echo "usage: $0 {status|on|soft|off|cycle}" >&2 + exit 2 + ;; +esac diff --git a/ansible/roles/ups/templates/ups-restore.service.j2 b/ansible/roles/ups/templates/ups-restore.service.j2 new file mode 100644 index 0000000..f8aeffd --- /dev/null +++ b/ansible/roles/ups/templates/ups-restore.service.j2 @@ -0,0 +1,13 @@ +# {{ ansible_managed }} +[Unit] +Description=Restore TrueNAS power after an outage +After=network-online.target nut-server.service nut-monitor.service +Wants=network-online.target +Requires=nut-server.service + +[Service] +Type=oneshot +ExecStart=/usr/local/bin/ups-restore.sh + +[Install] +WantedBy=multi-user.target diff --git a/ansible/roles/ups/templates/ups-restore.sh.j2 b/ansible/roles/ups/templates/ups-restore.sh.j2 new file mode 100644 index 0000000..666550a --- /dev/null +++ b/ansible/roles/ups/templates/ups-restore.sh.j2 @@ -0,0 +1,63 @@ +#!/bin/bash +# {{ ansible_managed }} +# Power truenas.localdomain back on after an outage, but only when it is +# genuinely safe to do so. Called by upssched once mains has been stable, +# and once at boot by ups-restore.service (deep-drain recovery). +# +# Deliberately stateless: we ask the iDRAC whether the chassis is off +# rather than tracking whether we were the ones who shut it down. A NAS +# powered off by hand while on mains is safe, because no ONLINE event +# fires in that case and the boot path checks UPS status first. +# +# NOTE: as of now this is a best-effort secondary path. The R415 has no +# iDRAC6 Enterprise card ("iDRAC6 Ent Pres ... Absent"), so its shared-LOM +# BMC has no standby power and goes unreachable whenever the chassis is +# off - exactly when we would want it. An unreachable iDRAC is therefore +# the EXPECTED case here, and we exit quietly rather than alarming. +# +# The primary restore path needs no IPMI: the R415 power restore policy is +# set to always-on, so when the UPS cuts and then restores its output the +# server powers itself back up. Fit an iDRAC6 Enterprise card and switch it +# to dedicated mode and this script starts working with no changes. +set -uo pipefail + +TAG=ups-restore +UPS={{ ups_name }}@localhost + +log() { logger -t "$TAG" -- "$*"; echo "$TAG: $*"; } + +# At boot, upsd may not be serving yet. Wait a bounded amount of time. +for _ in $(seq 1 30); do + if upsc "$UPS" ups.status >/dev/null 2>&1; then + break + fi + sleep 2 +done + +status=$(upsc "$UPS" ups.status 2>/dev/null || echo UNKNOWN) +if [[ "$status" != *OL* ]]; then + log "UPS status is '$status', not on line - refusing to power TrueNAS on" + exit 0 +fi + +power=$(/usr/local/bin/truenas-power.sh status 2>&1 || true) +case "$power" in + *"is off"*) + log "mains stable and chassis off - powering TrueNAS on" + if /usr/local/bin/truenas-power.sh on; then + log "power-on command accepted" + else + log "power-on command FAILED - check iDRAC at {{ idrac_host }}" + exit 1 + fi + ;; + *"is on"*) + log "TrueNAS already on, nothing to do" + ;; + *) + # Expected while the box has no iDRAC6 Enterprise card: the BMC is + # simply not on the network with the chassis powered down. + log "iDRAC at {{ idrac_host }} unreachable - relying on the" \ + "always-on power restore policy instead ($power)" + ;; +esac diff --git a/ansible/roles/ups/templates/ups-sched-cmd.sh.j2 b/ansible/roles/ups/templates/ups-sched-cmd.sh.j2 new file mode 100644 index 0000000..bf0cb90 --- /dev/null +++ b/ansible/roles/ups/templates/ups-sched-cmd.sh.j2 @@ -0,0 +1,14 @@ +#!/bin/bash +# {{ ansible_managed }} +# upssched CMDSCRIPT. Runs as the unprivileged nut user. +set -uo pipefail + +case "${1:-}" in + truenas-restore) + exec /usr/local/bin/ups-restore.sh + ;; + *) + logger -t ups-sched -- "unknown timer '${1:-}'" + exit 1 + ;; +esac diff --git a/ansible/roles/ups/templates/ups.conf.j2 b/ansible/roles/ups/templates/ups.conf.j2 new file mode 100644 index 0000000..b817d02 --- /dev/null +++ b/ansible/roles/ups/templates/ups.conf.j2 @@ -0,0 +1,18 @@ +# {{ ansible_managed }} + +[{{ ups_name }}] + driver = usbhid-ups + port = auto + vendorid = {{ ups_vendorid }} + productid = {{ ups_productid }} + desc = "{{ ups_desc }}" + # CyberPower asserts its own low-battery flag around 20-35%, far too + # early for us. Ignore it and derive LB from our own threshold so + # upsmon fires SHUTDOWNCMD at exactly {{ ups_low_charge_pct }}%. + ignorelb + override.battery.charge.low = {{ ups_low_charge_pct }} + # Unlock driver.killpower. Without this the driver refuses to cut UPS + # output at shutdown, and the R415's always-on power restore policy + # would never see AC drop and return - i.e. nothing comes back after a + # deep outage. See README.md. + allow_killpower diff --git a/ansible/roles/ups/templates/upsd.conf.j2 b/ansible/roles/ups/templates/upsd.conf.j2 new file mode 100644 index 0000000..2ce034e --- /dev/null +++ b/ansible/roles/ups/templates/upsd.conf.j2 @@ -0,0 +1,3 @@ +# {{ ansible_managed }} +LISTEN 127.0.0.1 {{ ups_listen_port }} +LISTEN {{ ups_listen_addr }} {{ ups_listen_port }} diff --git a/ansible/roles/ups/templates/upsd.users.j2 b/ansible/roles/ups/templates/upsd.users.j2 new file mode 100644 index 0000000..f070473 --- /dev/null +++ b/ansible/roles/ups/templates/upsd.users.j2 @@ -0,0 +1,11 @@ +# {{ ansible_managed }} + +# Local upsmon on this host. +[upsmon] + password = {{ nut_upsmon_password }} + upsmon master + +# truenas.localdomain, running the TrueNAS UPS service in Slave mode. +[truenas] + password = {{ nut_truenas_password }} + upsmon slave diff --git a/ansible/roles/ups/templates/upsmon.conf.j2 b/ansible/roles/ups/templates/upsmon.conf.j2 new file mode 100644 index 0000000..1fe98a7 --- /dev/null +++ b/ansible/roles/ups/templates/upsmon.conf.j2 @@ -0,0 +1,40 @@ +# {{ ansible_managed }} + +MONITOR {{ ups_name }}@localhost 1 upsmon {{ nut_upsmon_password }} master + +MINSUPPLIES 1 +SHUTDOWNCMD "/usr/bin/systemctl poweroff" +NOTIFYCMD /usr/bin/upssched + +# upsmon drops this file before halting; /lib/systemd/system-shutdown/nutshutdown +# reads it late in shutdown and, if present, tells the UPS to cut its output. +# That AC drop-and-return is what triggers the R415's always-on restore policy +# and this host's BIOS "After Power Loss: Power On". upsmon has NO compiled-in +# default for this - leave it unset and the UPS never powers down. +# Must be on tmpfs: a persistent path can go stale and make every ordinary +# reboot look like a forced shutdown. +POWERDOWNFLAG /run/nut/killpower + +POLLFREQ 5 +POLLFREQALERT 5 + +# Wait up to 30s for the truenas slave to disconnect before we halt. +HOSTSYNC 30 +DEADTIME 15 +RBWARNTIME 43200 +NOCOMMWARNTIME 300 +FINALDELAY 5 + +# SYSLOG puts every UPS event in the journal, which fluent-bit already +# forwards to Graylog (see roles/common/tasks/fluent-bit.yml). +# EXEC runs NOTIFYCMD, i.e. upssched, which drives the TrueNAS restore. +NOTIFYFLAG ONLINE SYSLOG+EXEC +NOTIFYFLAG ONBATT SYSLOG+EXEC +NOTIFYFLAG LOWBATT SYSLOG+EXEC +NOTIFYFLAG FSD SYSLOG+EXEC +NOTIFYFLAG COMMOK SYSLOG+EXEC +NOTIFYFLAG COMMBAD SYSLOG+EXEC +NOTIFYFLAG SHUTDOWN SYSLOG+EXEC +NOTIFYFLAG REPLBATT SYSLOG +NOTIFYFLAG NOCOMM SYSLOG +NOTIFYFLAG NOPARENT SYSLOG diff --git a/ansible/roles/ups/templates/upssched.conf.j2 b/ansible/roles/ups/templates/upssched.conf.j2 new file mode 100644 index 0000000..b668731 --- /dev/null +++ b/ansible/roles/ups/templates/upssched.conf.j2 @@ -0,0 +1,13 @@ +# {{ ansible_managed }} + +CMDSCRIPT /usr/local/bin/ups-sched-cmd.sh +PIPEFN /run/nut/upssched.pipe +LOCKFN /run/nut/upssched.lock + +# Mains is back: wait for it to hold for {{ ups_restore_stable_secs }}s +# before powering TrueNAS back on, so we do not flap on unstable power. +AT ONLINE * CANCEL-TIMER truenas-restore +AT ONLINE * START-TIMER truenas-restore {{ ups_restore_stable_secs }} + +# Power dropped again while the restore timer was pending - stand down. +AT ONBATT * CANCEL-TIMER truenas-restore diff --git a/ansible/vars/vault.yml b/ansible/vars/vault.yml index 83294d1..9c9d874 100644 Binary files a/ansible/vars/vault.yml and b/ansible/vars/vault.yml differ