harden nextcloud backups: db dumps, alerting, drift fix

The data-only rsync left no way to restore a working instance: mysql/ and
config/ were never backed up, so a recovery would have files but no shares,
users or metadata. Dump the database before syncing files (a DB older than
the files is repairable with occ files:scan; a newer one references blobs
that never made it into the backup) and ship config/ alongside it.

Capture the --chmod=Du=rwx,Dgo=rx flag that had been hand-added to the
deployed skudak-cloud script. It was outside git, so every deploy silently
reverted it. It now lives in backup_rsync_extra_args.

Add OnFailure= alerting. The units failed silently before, which is how an
iDrive sync failure sat unnoticed since May. msmtp rather than the esmtp
already installed: the OpenSRS relay is port 465 (implicit TLS) and libesmtp
only speaks STARTTLS.

Exclude nextcloud.log* from the sync and cap log_rotate_size. skudak-cloud
was running at loglevel 0 and had written a 64 GB log that was being rsynced
and pushed to S3; set it to 2 to match the home instance.

Stagger the timers (04:00 / 04:30) so both finish before the 05:00 TrueNAS
snapshot task, and bound TimeoutStartSec so a wedged rsync cannot leave the
unit activating forever and skip every subsequent trigger.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

ansible/roles/common/tasks/mail.yml

@@ -0,0 +1,34 @@
+---
+# Outbound mail for system notifications (backup failures, cron output).
+#
+# msmtp rather than the esmtp already on the box: the OpenSRS relay uses port
+# 465, which is IMPLICIT TLS, and libesmtp/esmtp only speaks STARTTLS. msmtp
+# handles implicit TLS via `tls_starttls off` and mirrors the settings the
+# TrueNAS box already uses.
+# On Fedora the single `msmtp` package already ships /usr/bin/sendmail and
+# /usr/lib/sendmail; there is no separate msmtp-sendmail package (Debian split).
+- name: install msmtp
+  become: true
+  ansible.builtin.package:
+    name: msmtp
+    state: present
+  tags: mail
+
+- name: configure msmtp
+  become: true
+  ansible.builtin.template:
+    src: msmtp/msmtprc.j2
+    dest: /etc/msmtprc
+    owner: root
+    group: root
+    mode: 0600
+  no_log: true
+  tags: mail
+
+- name: point the mta alternative at msmtp
+  become: true
+  community.general.alternatives:
+    name: mta
+    path: /usr/bin/msmtp-sendmail
+  failed_when: false
+  tags: mail

ansible/roles/common/tasks/main.yml

@@ -3,6 +3,9 @@
 - import_tasks: security.yml
 - import_tasks: service.yml
 
+- import_tasks: mail.yml
+  tags: mail
+
 - import_tasks: fluent-bit.yml
   tags: fluent-bit, graylog
 

ansible/roles/common/templates/msmtp/msmtprc.j2

@@ -0,0 +1,17 @@
+# {{ ansible_managed }}
+defaults
+auth           on
+tls            on
+tls_trust_file /etc/pki/tls/certs/ca-bundle.crt
+logfile        /var/log/msmtp.log
+
+account        home
+host           {{ system_smtp_host | default('mail.b.hostedemail.com') }}
+port           {{ system_smtp_port | default(465) }}
+# Port 465 is implicit TLS (SMTPS), not STARTTLS.
+tls_starttls   off
+from           {{ system_smtp_from | default('home@bdebyl.net') }}
+user           {{ system_smtp_user | default('home@bdebyl.net') }}
+password       {{ home_smtp }}
+
+account default : home

ansible/roles/podman/defaults/main.yml

@@ -1,4 +1,6 @@
 ---
+# Where Nextcloud backup failure alerts are mailed (see containers/cloud-backup.yml).
+backup_alert_email: bastian@debyl.io
 bookstack_path: "{{ podman_volumes }}/bookstack"
 cam2ip_path: "{{ podman_volumes }}/cam2ip"
 cloud_path: "{{ podman_volumes }}/cloud"

ansible/roles/podman/tasks/containers/cloud-backup.yml

@@ -28,6 +28,27 @@
     mode: 0755
     setype: bin_t
 
+# Shared by every backup instance. Rendered once per include; the second and
+# later renders are no-ops.
+- name: template nextcloud backup alert script
+  become: true
+  ansible.builtin.template:
+    src: nextcloud/nextcloud-backup-alert.sh.j2
+    dest: /usr/local/bin/nextcloud-backup-alert.sh
+    owner: root
+    group: root
+    mode: 0755
+    setype: bin_t
+
+- name: template nextcloud backup failure handler unit
+  become: true
+  ansible.builtin.template:
+    src: nextcloud/nextcloud-backup-failed@.service.j2
+    dest: /etc/systemd/system/nextcloud-backup-failed@.service
+    owner: root
+    group: root
+    mode: 0644
+
 - name: template {{ backup_name }} backup systemd service
   become: true
   ansible.builtin.template:

ansible/roles/podman/tasks/containers/home/cloud.yml

@@ -84,10 +84,24 @@
   vars:
     container_name: cloud
 
+# Unbounded by default: nextcloud.log.1 had reached 1.12 GB and was being
+# rsynced to TrueNAS and pushed to S3 on every run. Cap at 10 MiB.
+- name: cap nextcloud log rotation size for cloud
+  become: true
+  become_user: "{{ podman_user }}"
+  ansible.builtin.command: >
+    podman exec -u www-data cloud
+    php occ config:system:set log_rotate_size --value 10485760 --type integer
+  register: cloud_log_rotate
+  changed_when: "'System config value log_rotate_size' in cloud_log_rotate.stdout"
+  failed_when: false
+
 - include_tasks: containers/cloud-backup.yml
   vars:
     backup_name: cloud
     data_path: "{{ cloud_path }}/data"
+    config_path: "{{ cloud_path }}/config"
+    db_container: cloud-db
     ssh_key_path: /etc/ssh/backup_keys/cloud
     ssh_key_content: "{{ cloud_backup_ssh_key }}"
     ssh_user: cloud

ansible/roles/podman/tasks/containers/skudak/cloud.yml

@@ -131,12 +131,44 @@
   changed_when: "'System config value trusted_domains' in trusted_domain_result.stdout"
   failed_when: false
 
+# This instance was left at loglevel 0 (DEBUG) and had written a 64 GB
+# nextcloud.log, almost entirely repeated deprecation notices. 2 = Warning,
+# which is both the Nextcloud default and what the home instance already uses.
+- name: set nextcloud loglevel for skudak-cloud
+  become: true
+  become_user: "{{ podman_user }}"
+  ansible.builtin.command: >
+    podman exec -u www-data skudak-cloud
+    php occ config:system:set loglevel --value 2 --type integer
+  register: skudak_loglevel
+  changed_when: "'System config value loglevel' in skudak_loglevel.stdout"
+  failed_when: false
+
+# Unbounded by default; see the equivalent task in containers/home/cloud.yml.
+- name: cap nextcloud log rotation size for skudak-cloud
+  become: true
+  become_user: "{{ podman_user }}"
+  ansible.builtin.command: >
+    podman exec -u www-data skudak-cloud
+    php occ config:system:set log_rotate_size --value 10485760 --type integer
+  register: skudak_log_rotate
+  changed_when: "'System config value log_rotate_size' in skudak_log_rotate.stdout"
+  failed_when: false
+
 - include_tasks: containers/cloud-backup.yml
   vars:
     backup_name: skudak-cloud
     data_path: "{{ cloud_skudak_path }}/data"
+    config_path: "{{ cloud_skudak_path }}/config"
+    db_container: skudak-cloud-db
     ssh_key_path: /etc/ssh/backup_keys/skudak-cloud
     ssh_key_content: "{{ cloud_skudak_backup_ssh_key }}"
     ssh_user: skucloud
     remote_path: /mnt/glacier/skudakcloud
     script_path: /usr/local/bin/skudak-cloud-backup.sh
+    # skudakcloud/data is mode 770, so the receiving side needs traversable
+    # dirs. This flag was hand-added on the host and was being silently
+    # reverted by every `make deploy TAGS=skudak-cloud`; it now lives in git.
+    backup_rsync_extra_args: "--chmod=Du=rwx,Dgo=rx"
+    # Staggered so both instances finish before the 05:00 TrueNAS snapshot.
+    backup_oncalendar: "*-*-* 04:30:00"

ansible/roles/podman/templates/nextcloud/cloud-backup.service.j2

@@ -1,6 +1,14 @@
 [Unit]
 Description=Nextcloud {{ instance_name }} backup to TrueNAS
+After=network-online.target
+Wants=network-online.target
+OnFailure=nextcloud-backup-failed@%n.service
 
 [Service]
 Type=oneshot
 ExecStart={{ script_path }}
+# Type=oneshot disables the start timeout by default, so a wedged rsync would
+# leave the unit "activating" forever and every subsequent daily trigger would
+# be silently skipped. Bound it.
+TimeoutStartSec={{ backup_timeout | default('4h') }}
+Nice=10

ansible/roles/podman/templates/nextcloud/cloud-backup.sh.j2

@@ -1,4 +1,111 @@
 #!/bin/bash
+# {{ ansible_managed }}
+# Nextcloud "{{ backup_name }}" -> truenas.localdomain.
+#
+# Ordering is deliberate: the database is dumped BEFORE the file tree is
+# synced. A DB snapshot slightly OLDER than the files degrades to "files
+# Nextcloud has not indexed yet" and is repaired with `occ files:scan`. A DB
+# snapshot NEWER than the files references blobs that never made it into the
+# backup, which surfaces as broken shares and dead file entries on restore.
 set -euo pipefail
-rsync -az --exclude .ssh -e "ssh -i {{ ssh_key_path }} -o StrictHostKeyChecking=accept-new" \
+
-  {{ data_path }}/ {{ ssh_user }}@truenas.localdomain:{{ remote_path }}/
+TAG=nextcloud-backup
+INSTANCE={{ backup_name }}
+STAGE={{ backup_stage_path | default('/var/backups/nextcloud/' ~ backup_name) }}
+KEEP={{ backup_db_keep | default(7) }}
+DUMP="$STAGE/db/${INSTANCE}-$(date +%Y%m%d).sql.gz"
+
+log()  { logger -t "$TAG" -p daemon.info -- "instance=$INSTANCE $*"; echo "$TAG: $*"; }
+fail() { logger -t "$TAG" -p daemon.err  -- "instance=$INSTANCE status=failed $*"
+         echo "$TAG: FAILED: $*" >&2; exit 1; }
+
+SSH="ssh -i {{ ssh_key_path }} -o StrictHostKeyChecking=accept-new -o ServerAliveInterval=30 -o ServerAliveCountMax=6"
+DEST={{ ssh_user }}@truenas.localdomain
+
+log "status=start"
+
+{% if db_container | default('') %}
+# ------------------------------------------------------------ 1. database
+# The Nextcloud containers are ROOTLESS podman owned by "{{ podman_user }}",
+# but this script runs as root under systemd. Every podman call therefore
+# goes through sudo:
+#   -H    HOME becomes the podman user's home, so podman finds its rootless
+#         graph root under ~/.local/share/containers
+#   cd;   required preamble (see CLAUDE.md) so the shell starts in that home
+#   XDG_RUNTIME_DIR  the podman user's runtime dir. Lingering is enabled by
+#         roles/podman/tasks/podman/podman.yml so /run/user/<uid> exists;
+#         guarded anyway so podman falls back cleanly if it ever does not.
+#
+# No credential is stored in this file or placed on a host command line:
+# $MYSQL_ROOT_PASSWORD and $MYSQL_DATABASE are expanded by the shell INSIDE
+# the database container, which already carries them in its environment.
+#
+# Flags are deliberately minimal. --events and --routines were both tried and
+# both abort the dump on these instances: the event scheduler is disabled
+# (error 1577) and mysql.proc reads as corrupted (error 1728, an artefact of
+# an image bump without mariadb-upgrade). Nextcloud uses neither events nor
+# stored routines, so dropping them loses nothing.
+pexec() {
+  sudo -H -u {{ podman_user }} bash -c \
+    'cd; d=/run/user/$(id -u); [ -d "$d" ] && export XDG_RUNTIME_DIR="$d"
+     exec podman "$@"' _ "$@"
+}
+
+install -d -m 0700 "$STAGE" "$STAGE/db"
+tmp="$DUMP.tmp"
+rm -f "$tmp"
+
+log "dumping {{ db_container }}"
+set +e
+pexec exec {{ db_container }} sh -c '
+  exec env MYSQL_PWD="$MYSQL_ROOT_PASSWORD" mariadb-dump -u root \
+    --single-transaction --quick --triggers \
+    --no-tablespaces --default-character-set=utf8mb4 "$MYSQL_DATABASE"
+' | gzip -6 > "$tmp"
+dump_rc=${PIPESTATUS[0]}
+set -e
+[ "$dump_rc" -eq 0 ] || fail "mariadb-dump {{ db_container }} exited $dump_rc"
+
+# A new dump is promoted over yesterday's only after it proves complete:
+# a valid gzip stream AND the "-- Dump completed" trailer that mariadb-dump
+# writes only on a clean finish. `mv` is atomic within the staging
+# filesystem, so a failed or truncated run can never replace a good dump.
+gzip -t "$tmp" || fail "dump is not a valid gzip stream"
+gunzip -c "$tmp" | tail -c 512 | grep -q 'Dump completed' \
+  || fail "dump is truncated (no completion trailer)"
+mv -f "$tmp" "$DUMP"
+log "db_dump=ok bytes=$(stat -c %s "$DUMP")"
+
+# Local retention. The staging sync below mirrors with --delete, so remote
+# retention follows the same window; deeper history comes from the TrueNAS
+# periodic ZFS snapshots (see roles/podman/README.md).
+ls -1t "$STAGE"/db/"$INSTANCE"-*.sql.gz | tail -n +$((KEEP + 1)) | xargs -r rm -f
+{% endif %}
+
+# ----------------------------------------------------------- 2. file tree
+# --exclude .ssh is load-bearing: {{ remote_path }} IS {{ ssh_user }}'s home
+# on TrueNAS and its authorized_keys lives there, so a `.ssh` directory
+# appearing in the data tree must never be shipped. No --delete here: the
+# data tree is append-mostly and a source-side mishap must not propagate.
+log "syncing data"
+rsync -az --timeout=1800 --exclude .ssh \
+  {{ backup_rsync_excludes | default("--exclude '/nextcloud.log*' --exclude '/updater.log'") }} \
+  {{ backup_rsync_extra_args | default('') }} \
+  -e "$SSH" {{ data_path }}/ "$DEST:{{ remote_path }}/"
+
+# ------------------------------------------- 3. config/ and database dumps
+{# --mkpath creates the nested _backup/<x>/ destination; rsync will not build
+   more than one missing level on its own. Requires rsync >= 3.2.3 on both
+   ends (galactica 3.4.1, truenas 3.2.7). #}
+{% if config_path | default('') %}
+log "syncing config"
+rsync -az --timeout=600 --delete --mkpath {{ backup_rsync_extra_args | default('') }} \
+  -e "$SSH" {{ config_path }}/ "$DEST:{{ remote_path }}/_backup/config/"
+{% endif %}
+{% if db_container | default('') %}
+log "syncing db dumps"
+rsync -az --timeout=600 --delete --mkpath {{ backup_rsync_extra_args | default('') }} \
+  -e "$SSH" "$STAGE/db/" "$DEST:{{ remote_path }}/_backup/db/"
+{% endif %}
+
+log "status=ok"

ansible/roles/podman/templates/nextcloud/cloud-backup.timer.j2

@@ -2,7 +2,8 @@
 Description=Daily Nextcloud {{ instance_name }} backup
 
 [Timer]
-OnCalendar=*-*-* 04:00:00
+OnCalendar={{ backup_oncalendar | default('*-*-* 04:00:00') }}
+RandomizedDelaySec={{ backup_randomized_delay | default('5m') }}
 Persistent=true
 
 [Install]

ansible/roles/podman/templates/nextcloud/nextcloud-backup-alert.sh.j2

@@ -0,0 +1,40 @@
+#!/bin/bash
+# {{ ansible_managed }}
+# OnFailure= handler for the Nextcloud backup units. Invoked as:
+#   nextcloud-backup-alert.sh <failed-unit-name>
+#
+# Deliberately NOT `set -e`: an alert handler that dies partway through
+# reports nothing, which is worse than a partial report. Same reasoning as
+# roles/ups/templates/ups-restore.sh.j2.
+set -uo pipefail
+
+TAG=nextcloud-backup
+UNIT="${1:-unknown}"
+TO="{{ backup_alert_email | default('root') }}"
+HOST="$(hostname -f 2>/dev/null || hostname)"
+
+result="$(systemctl show -p Result --value "$UNIT" 2>/dev/null)"
+code="$(systemctl show -p ExecMainStatus --value "$UNIT" 2>/dev/null)"
+
+# One machine-parseable line for Graylog, then the context.
+logger -t "$TAG" -p daemon.err -- \
+  "status=failed unit=$UNIT result=${result:-unknown} exit=${code:-unknown}"
+
+body="$(printf 'Nextcloud backup FAILED on %s\n\nunit:   %s\nresult: %s\nexit:   %s\n\n--- last 40 journal lines ---\n' \
+  "$HOST" "$UNIT" "${result:-unknown}" "${code:-unknown}")
+$(journalctl -u "$UNIT" -n 40 --no-pager -o cat 2>/dev/null)"
+
+echo "$body" | logger -t "$TAG" -p daemon.err
+
+# Mail is best-effort: if the MTA is not configured the journald record above
+# is still the authoritative signal, so never fail the handler on this.
+if command -v sendmail >/dev/null 2>&1; then
+  printf 'To: %s\nSubject: [%s] Nextcloud backup FAILED: %s\nContent-Type: text/plain; charset=UTF-8\n\n%s\n' \
+    "$TO" "$HOST" "$UNIT" "$body" | sendmail -t \
+    && logger -t "$TAG" -p daemon.info -- "alert_mail=sent to=$TO" \
+    || logger -t "$TAG" -p daemon.err  -- "alert_mail=failed to=$TO"
+else
+  logger -t "$TAG" -p daemon.err -- "alert_mail=skipped reason=no-sendmail"
+fi
+
+exit 0

ansible/roles/podman/templates/nextcloud/nextcloud-backup-failed@.service.j2

@@ -0,0 +1,6 @@
+[Unit]
+Description=Report failure of %i
+
+[Service]
+Type=oneshot
+ExecStart=/usr/local/bin/nextcloud-backup-alert.sh %i