diff --git a/ansible/roles/common/tasks/mail.yml b/ansible/roles/common/tasks/mail.yml new file mode 100644 index 0000000..5b14498 --- /dev/null +++ b/ansible/roles/common/tasks/mail.yml @@ -0,0 +1,34 @@ +--- +# Outbound mail for system notifications (backup failures, cron output). +# +# msmtp rather than the esmtp already on the box: the OpenSRS relay uses port +# 465, which is IMPLICIT TLS, and libesmtp/esmtp only speaks STARTTLS. msmtp +# handles implicit TLS via `tls_starttls off` and mirrors the settings the +# TrueNAS box already uses. +# On Fedora the single `msmtp` package already ships /usr/bin/sendmail and +# /usr/lib/sendmail; there is no separate msmtp-sendmail package (Debian split). +- name: install msmtp + become: true + ansible.builtin.package: + name: msmtp + state: present + tags: mail + +- name: configure msmtp + become: true + ansible.builtin.template: + src: msmtp/msmtprc.j2 + dest: /etc/msmtprc + owner: root + group: root + mode: 0600 + no_log: true + tags: mail + +- name: point the mta alternative at msmtp + become: true + community.general.alternatives: + name: mta + path: /usr/bin/msmtp-sendmail + failed_when: false + tags: mail diff --git a/ansible/roles/common/tasks/main.yml b/ansible/roles/common/tasks/main.yml index db08c36..8a7ba73 100644 --- a/ansible/roles/common/tasks/main.yml +++ b/ansible/roles/common/tasks/main.yml @@ -3,6 +3,9 @@ - import_tasks: security.yml - import_tasks: service.yml +- import_tasks: mail.yml + tags: mail + - import_tasks: fluent-bit.yml tags: fluent-bit, graylog diff --git a/ansible/roles/common/templates/msmtp/msmtprc.j2 b/ansible/roles/common/templates/msmtp/msmtprc.j2 new file mode 100644 index 0000000..20f29db --- /dev/null +++ b/ansible/roles/common/templates/msmtp/msmtprc.j2 @@ -0,0 +1,17 @@ +# {{ ansible_managed }} +defaults +auth on +tls on +tls_trust_file /etc/pki/tls/certs/ca-bundle.crt +logfile /var/log/msmtp.log + +account home +host {{ system_smtp_host | default('mail.b.hostedemail.com') }} +port {{ system_smtp_port | default(465) }} +# Port 465 is implicit TLS (SMTPS), not STARTTLS. +tls_starttls off +from {{ system_smtp_from | default('home@bdebyl.net') }} +user {{ system_smtp_user | default('home@bdebyl.net') }} +password {{ home_smtp }} + +account default : home diff --git a/ansible/roles/podman/defaults/main.yml b/ansible/roles/podman/defaults/main.yml index 70a8b88..7204e2d 100644 --- a/ansible/roles/podman/defaults/main.yml +++ b/ansible/roles/podman/defaults/main.yml @@ -1,4 +1,6 @@ --- +# Where Nextcloud backup failure alerts are mailed (see containers/cloud-backup.yml). +backup_alert_email: bastian@debyl.io bookstack_path: "{{ podman_volumes }}/bookstack" cam2ip_path: "{{ podman_volumes }}/cam2ip" cloud_path: "{{ podman_volumes }}/cloud" diff --git a/ansible/roles/podman/tasks/containers/cloud-backup.yml b/ansible/roles/podman/tasks/containers/cloud-backup.yml index 00bd37d..40fb012 100644 --- a/ansible/roles/podman/tasks/containers/cloud-backup.yml +++ b/ansible/roles/podman/tasks/containers/cloud-backup.yml @@ -28,6 +28,27 @@ mode: 0755 setype: bin_t +# Shared by every backup instance. Rendered once per include; the second and +# later renders are no-ops. +- name: template nextcloud backup alert script + become: true + ansible.builtin.template: + src: nextcloud/nextcloud-backup-alert.sh.j2 + dest: /usr/local/bin/nextcloud-backup-alert.sh + owner: root + group: root + mode: 0755 + setype: bin_t + +- name: template nextcloud backup failure handler unit + become: true + ansible.builtin.template: + src: nextcloud/nextcloud-backup-failed@.service.j2 + dest: /etc/systemd/system/nextcloud-backup-failed@.service + owner: root + group: root + mode: 0644 + - name: template {{ backup_name }} backup systemd service become: true ansible.builtin.template: diff --git a/ansible/roles/podman/tasks/containers/home/cloud.yml b/ansible/roles/podman/tasks/containers/home/cloud.yml index 0dddc81..23a2824 100644 --- a/ansible/roles/podman/tasks/containers/home/cloud.yml +++ b/ansible/roles/podman/tasks/containers/home/cloud.yml @@ -84,10 +84,24 @@ vars: container_name: cloud +# Unbounded by default: nextcloud.log.1 had reached 1.12 GB and was being +# rsynced to TrueNAS and pushed to S3 on every run. Cap at 10 MiB. +- name: cap nextcloud log rotation size for cloud + become: true + become_user: "{{ podman_user }}" + ansible.builtin.command: > + podman exec -u www-data cloud + php occ config:system:set log_rotate_size --value 10485760 --type integer + register: cloud_log_rotate + changed_when: "'System config value log_rotate_size' in cloud_log_rotate.stdout" + failed_when: false + - include_tasks: containers/cloud-backup.yml vars: backup_name: cloud data_path: "{{ cloud_path }}/data" + config_path: "{{ cloud_path }}/config" + db_container: cloud-db ssh_key_path: /etc/ssh/backup_keys/cloud ssh_key_content: "{{ cloud_backup_ssh_key }}" ssh_user: cloud diff --git a/ansible/roles/podman/tasks/containers/skudak/cloud.yml b/ansible/roles/podman/tasks/containers/skudak/cloud.yml index ab2df47..1175948 100644 --- a/ansible/roles/podman/tasks/containers/skudak/cloud.yml +++ b/ansible/roles/podman/tasks/containers/skudak/cloud.yml @@ -131,12 +131,44 @@ changed_when: "'System config value trusted_domains' in trusted_domain_result.stdout" failed_when: false +# This instance was left at loglevel 0 (DEBUG) and had written a 64 GB +# nextcloud.log, almost entirely repeated deprecation notices. 2 = Warning, +# which is both the Nextcloud default and what the home instance already uses. +- name: set nextcloud loglevel for skudak-cloud + become: true + become_user: "{{ podman_user }}" + ansible.builtin.command: > + podman exec -u www-data skudak-cloud + php occ config:system:set loglevel --value 2 --type integer + register: skudak_loglevel + changed_when: "'System config value loglevel' in skudak_loglevel.stdout" + failed_when: false + +# Unbounded by default; see the equivalent task in containers/home/cloud.yml. +- name: cap nextcloud log rotation size for skudak-cloud + become: true + become_user: "{{ podman_user }}" + ansible.builtin.command: > + podman exec -u www-data skudak-cloud + php occ config:system:set log_rotate_size --value 10485760 --type integer + register: skudak_log_rotate + changed_when: "'System config value log_rotate_size' in skudak_log_rotate.stdout" + failed_when: false + - include_tasks: containers/cloud-backup.yml vars: backup_name: skudak-cloud data_path: "{{ cloud_skudak_path }}/data" + config_path: "{{ cloud_skudak_path }}/config" + db_container: skudak-cloud-db ssh_key_path: /etc/ssh/backup_keys/skudak-cloud ssh_key_content: "{{ cloud_skudak_backup_ssh_key }}" ssh_user: skucloud remote_path: /mnt/glacier/skudakcloud script_path: /usr/local/bin/skudak-cloud-backup.sh + # skudakcloud/data is mode 770, so the receiving side needs traversable + # dirs. This flag was hand-added on the host and was being silently + # reverted by every `make deploy TAGS=skudak-cloud`; it now lives in git. + backup_rsync_extra_args: "--chmod=Du=rwx,Dgo=rx" + # Staggered so both instances finish before the 05:00 TrueNAS snapshot. + backup_oncalendar: "*-*-* 04:30:00" diff --git a/ansible/roles/podman/templates/nextcloud/cloud-backup.service.j2 b/ansible/roles/podman/templates/nextcloud/cloud-backup.service.j2 index 626c635..4183855 100644 --- a/ansible/roles/podman/templates/nextcloud/cloud-backup.service.j2 +++ b/ansible/roles/podman/templates/nextcloud/cloud-backup.service.j2 @@ -1,6 +1,14 @@ [Unit] Description=Nextcloud {{ instance_name }} backup to TrueNAS +After=network-online.target +Wants=network-online.target +OnFailure=nextcloud-backup-failed@%n.service [Service] Type=oneshot ExecStart={{ script_path }} +# Type=oneshot disables the start timeout by default, so a wedged rsync would +# leave the unit "activating" forever and every subsequent daily trigger would +# be silently skipped. Bound it. +TimeoutStartSec={{ backup_timeout | default('4h') }} +Nice=10 diff --git a/ansible/roles/podman/templates/nextcloud/cloud-backup.sh.j2 b/ansible/roles/podman/templates/nextcloud/cloud-backup.sh.j2 index 50a656f..8153153 100644 --- a/ansible/roles/podman/templates/nextcloud/cloud-backup.sh.j2 +++ b/ansible/roles/podman/templates/nextcloud/cloud-backup.sh.j2 @@ -1,4 +1,111 @@ #!/bin/bash +# {{ ansible_managed }} +# Nextcloud "{{ backup_name }}" -> truenas.localdomain. +# +# Ordering is deliberate: the database is dumped BEFORE the file tree is +# synced. A DB snapshot slightly OLDER than the files degrades to "files +# Nextcloud has not indexed yet" and is repaired with `occ files:scan`. A DB +# snapshot NEWER than the files references blobs that never made it into the +# backup, which surfaces as broken shares and dead file entries on restore. set -euo pipefail -rsync -az --exclude .ssh -e "ssh -i {{ ssh_key_path }} -o StrictHostKeyChecking=accept-new" \ - {{ data_path }}/ {{ ssh_user }}@truenas.localdomain:{{ remote_path }}/ + +TAG=nextcloud-backup +INSTANCE={{ backup_name }} +STAGE={{ backup_stage_path | default('/var/backups/nextcloud/' ~ backup_name) }} +KEEP={{ backup_db_keep | default(7) }} +DUMP="$STAGE/db/${INSTANCE}-$(date +%Y%m%d).sql.gz" + +log() { logger -t "$TAG" -p daemon.info -- "instance=$INSTANCE $*"; echo "$TAG: $*"; } +fail() { logger -t "$TAG" -p daemon.err -- "instance=$INSTANCE status=failed $*" + echo "$TAG: FAILED: $*" >&2; exit 1; } + +SSH="ssh -i {{ ssh_key_path }} -o StrictHostKeyChecking=accept-new -o ServerAliveInterval=30 -o ServerAliveCountMax=6" +DEST={{ ssh_user }}@truenas.localdomain + +log "status=start" + +{% if db_container | default('') %} +# ------------------------------------------------------------ 1. database +# The Nextcloud containers are ROOTLESS podman owned by "{{ podman_user }}", +# but this script runs as root under systemd. Every podman call therefore +# goes through sudo: +# -H HOME becomes the podman user's home, so podman finds its rootless +# graph root under ~/.local/share/containers +# cd; required preamble (see CLAUDE.md) so the shell starts in that home +# XDG_RUNTIME_DIR the podman user's runtime dir. Lingering is enabled by +# roles/podman/tasks/podman/podman.yml so /run/user/ exists; +# guarded anyway so podman falls back cleanly if it ever does not. +# +# No credential is stored in this file or placed on a host command line: +# $MYSQL_ROOT_PASSWORD and $MYSQL_DATABASE are expanded by the shell INSIDE +# the database container, which already carries them in its environment. +# +# Flags are deliberately minimal. --events and --routines were both tried and +# both abort the dump on these instances: the event scheduler is disabled +# (error 1577) and mysql.proc reads as corrupted (error 1728, an artefact of +# an image bump without mariadb-upgrade). Nextcloud uses neither events nor +# stored routines, so dropping them loses nothing. +pexec() { + sudo -H -u {{ podman_user }} bash -c \ + 'cd; d=/run/user/$(id -u); [ -d "$d" ] && export XDG_RUNTIME_DIR="$d" + exec podman "$@"' _ "$@" +} + +install -d -m 0700 "$STAGE" "$STAGE/db" +tmp="$DUMP.tmp" +rm -f "$tmp" + +log "dumping {{ db_container }}" +set +e +pexec exec {{ db_container }} sh -c ' + exec env MYSQL_PWD="$MYSQL_ROOT_PASSWORD" mariadb-dump -u root \ + --single-transaction --quick --triggers \ + --no-tablespaces --default-character-set=utf8mb4 "$MYSQL_DATABASE" +' | gzip -6 > "$tmp" +dump_rc=${PIPESTATUS[0]} +set -e +[ "$dump_rc" -eq 0 ] || fail "mariadb-dump {{ db_container }} exited $dump_rc" + +# A new dump is promoted over yesterday's only after it proves complete: +# a valid gzip stream AND the "-- Dump completed" trailer that mariadb-dump +# writes only on a clean finish. `mv` is atomic within the staging +# filesystem, so a failed or truncated run can never replace a good dump. +gzip -t "$tmp" || fail "dump is not a valid gzip stream" +gunzip -c "$tmp" | tail -c 512 | grep -q 'Dump completed' \ + || fail "dump is truncated (no completion trailer)" +mv -f "$tmp" "$DUMP" +log "db_dump=ok bytes=$(stat -c %s "$DUMP")" + +# Local retention. The staging sync below mirrors with --delete, so remote +# retention follows the same window; deeper history comes from the TrueNAS +# periodic ZFS snapshots (see roles/podman/README.md). +ls -1t "$STAGE"/db/"$INSTANCE"-*.sql.gz | tail -n +$((KEEP + 1)) | xargs -r rm -f +{% endif %} + +# ----------------------------------------------------------- 2. file tree +# --exclude .ssh is load-bearing: {{ remote_path }} IS {{ ssh_user }}'s home +# on TrueNAS and its authorized_keys lives there, so a `.ssh` directory +# appearing in the data tree must never be shipped. No --delete here: the +# data tree is append-mostly and a source-side mishap must not propagate. +log "syncing data" +rsync -az --timeout=1800 --exclude .ssh \ + {{ backup_rsync_excludes | default("--exclude '/nextcloud.log*' --exclude '/updater.log'") }} \ + {{ backup_rsync_extra_args | default('') }} \ + -e "$SSH" {{ data_path }}/ "$DEST:{{ remote_path }}/" + +# ------------------------------------------- 3. config/ and database dumps +{# --mkpath creates the nested _backup// destination; rsync will not build + more than one missing level on its own. Requires rsync >= 3.2.3 on both + ends (galactica 3.4.1, truenas 3.2.7). #} +{% if config_path | default('') %} +log "syncing config" +rsync -az --timeout=600 --delete --mkpath {{ backup_rsync_extra_args | default('') }} \ + -e "$SSH" {{ config_path }}/ "$DEST:{{ remote_path }}/_backup/config/" +{% endif %} +{% if db_container | default('') %} +log "syncing db dumps" +rsync -az --timeout=600 --delete --mkpath {{ backup_rsync_extra_args | default('') }} \ + -e "$SSH" "$STAGE/db/" "$DEST:{{ remote_path }}/_backup/db/" +{% endif %} + +log "status=ok" diff --git a/ansible/roles/podman/templates/nextcloud/cloud-backup.timer.j2 b/ansible/roles/podman/templates/nextcloud/cloud-backup.timer.j2 index ca27957..0982d9f 100644 --- a/ansible/roles/podman/templates/nextcloud/cloud-backup.timer.j2 +++ b/ansible/roles/podman/templates/nextcloud/cloud-backup.timer.j2 @@ -2,7 +2,8 @@ Description=Daily Nextcloud {{ instance_name }} backup [Timer] -OnCalendar=*-*-* 04:00:00 +OnCalendar={{ backup_oncalendar | default('*-*-* 04:00:00') }} +RandomizedDelaySec={{ backup_randomized_delay | default('5m') }} Persistent=true [Install] diff --git a/ansible/roles/podman/templates/nextcloud/nextcloud-backup-alert.sh.j2 b/ansible/roles/podman/templates/nextcloud/nextcloud-backup-alert.sh.j2 new file mode 100644 index 0000000..c461bbb --- /dev/null +++ b/ansible/roles/podman/templates/nextcloud/nextcloud-backup-alert.sh.j2 @@ -0,0 +1,40 @@ +#!/bin/bash +# {{ ansible_managed }} +# OnFailure= handler for the Nextcloud backup units. Invoked as: +# nextcloud-backup-alert.sh +# +# Deliberately NOT `set -e`: an alert handler that dies partway through +# reports nothing, which is worse than a partial report. Same reasoning as +# roles/ups/templates/ups-restore.sh.j2. +set -uo pipefail + +TAG=nextcloud-backup +UNIT="${1:-unknown}" +TO="{{ backup_alert_email | default('root') }}" +HOST="$(hostname -f 2>/dev/null || hostname)" + +result="$(systemctl show -p Result --value "$UNIT" 2>/dev/null)" +code="$(systemctl show -p ExecMainStatus --value "$UNIT" 2>/dev/null)" + +# One machine-parseable line for Graylog, then the context. +logger -t "$TAG" -p daemon.err -- \ + "status=failed unit=$UNIT result=${result:-unknown} exit=${code:-unknown}" + +body="$(printf 'Nextcloud backup FAILED on %s\n\nunit: %s\nresult: %s\nexit: %s\n\n--- last 40 journal lines ---\n' \ + "$HOST" "$UNIT" "${result:-unknown}" "${code:-unknown}") +$(journalctl -u "$UNIT" -n 40 --no-pager -o cat 2>/dev/null)" + +echo "$body" | logger -t "$TAG" -p daemon.err + +# Mail is best-effort: if the MTA is not configured the journald record above +# is still the authoritative signal, so never fail the handler on this. +if command -v sendmail >/dev/null 2>&1; then + printf 'To: %s\nSubject: [%s] Nextcloud backup FAILED: %s\nContent-Type: text/plain; charset=UTF-8\n\n%s\n' \ + "$TO" "$HOST" "$UNIT" "$body" | sendmail -t \ + && logger -t "$TAG" -p daemon.info -- "alert_mail=sent to=$TO" \ + || logger -t "$TAG" -p daemon.err -- "alert_mail=failed to=$TO" +else + logger -t "$TAG" -p daemon.err -- "alert_mail=skipped reason=no-sendmail" +fi + +exit 0 diff --git a/ansible/roles/podman/templates/nextcloud/nextcloud-backup-failed@.service.j2 b/ansible/roles/podman/templates/nextcloud/nextcloud-backup-failed@.service.j2 new file mode 100644 index 0000000..5da3b87 --- /dev/null +++ b/ansible/roles/podman/templates/nextcloud/nextcloud-backup-failed@.service.j2 @@ -0,0 +1,6 @@ +[Unit] +Description=Report failure of %i + +[Service] +Type=oneshot +ExecStart=/usr/local/bin/nextcloud-backup-alert.sh %i