harden nextcloud backups: db dumps, alerting, drift fix

The data-only rsync left no way to restore a working instance: mysql/ and config/ were never backed up, so a recovery would have files but no shares, users or metadata. Dump the database before syncing files (a DB older than the files is repairable with occ files:scan; a newer one references blobs that never made it into the backup) and ship config/ alongside it. Capture the --chmod=Du=rwx,Dgo=rx flag that had been hand-added to the deployed skudak-cloud script. It was outside git, so every deploy silently reverted it. It now lives in backup_rsync_extra_args. Add OnFailure= alerting. The units failed silently before, which is how an iDrive sync failure sat unnoticed since May. msmtp rather than the esmtp already installed: the OpenSRS relay is port 465 (implicit TLS) and libesmtp only speaks STARTTLS. Exclude nextcloud.log* from the sync and cap log_rotate_size. skudak-cloud was running at loglevel 0 and had written a 64 GB log that was being rsynced and pushed to S3; set it to 2 to match the home instance. Stagger the timers (04:00 / 04:30) so both finish before the 05:00 TrueNAS snapshot task, and bound TimeoutStartSec so a wedged rsync cannot leave the unit activating forever and skip every subsequent trigger. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-07-26 16:03:18 -04:00
parent 1e1d53ecd8
commit 0ab423ca55
12 changed files with 288 additions and 3 deletions
@@ -1,6 +1,14 @@
 [Unit]
 Description=Nextcloud {{ instance_name }} backup to TrueNAS
+After=network-online.target
+Wants=network-online.target
+OnFailure=nextcloud-backup-failed@%n.service

 [Service]
 Type=oneshot
 ExecStart={{ script_path }}
+# Type=oneshot disables the start timeout by default, so a wedged rsync would
+# leave the unit "activating" forever and every subsequent daily trigger would
+# be silently skipped. Bound it.
+TimeoutStartSec={{ backup_timeout | default('4h') }}
+Nice=10
@@ -1,4 +1,111 @@
 #!/bin/bash
+# {{ ansible_managed }}
+# Nextcloud "{{ backup_name }}" -> truenas.localdomain.
+#
+# Ordering is deliberate: the database is dumped BEFORE the file tree is
+# synced. A DB snapshot slightly OLDER than the files degrades to "files
+# Nextcloud has not indexed yet" and is repaired with `occ files:scan`. A DB
+# snapshot NEWER than the files references blobs that never made it into the
+# backup, which surfaces as broken shares and dead file entries on restore.
 set -euo pipefail
-rsync -az --exclude .ssh -e "ssh -i {{ ssh_key_path }} -o StrictHostKeyChecking=accept-new" \
-  {{ data_path }}/ {{ ssh_user }}@truenas.localdomain:{{ remote_path }}/
+
+TAG=nextcloud-backup
+INSTANCE={{ backup_name }}
+STAGE={{ backup_stage_path | default('/var/backups/nextcloud/' ~ backup_name) }}
+KEEP={{ backup_db_keep | default(7) }}
+DUMP="$STAGE/db/${INSTANCE}-$(date +%Y%m%d).sql.gz"
+
+log()  { logger -t "$TAG" -p daemon.info -- "instance=$INSTANCE $*"; echo "$TAG: $*"; }
+fail() { logger -t "$TAG" -p daemon.err  -- "instance=$INSTANCE status=failed $*"
+         echo "$TAG: FAILED: $*" >&2; exit 1; }
+
+SSH="ssh -i {{ ssh_key_path }} -o StrictHostKeyChecking=accept-new -o ServerAliveInterval=30 -o ServerAliveCountMax=6"
+DEST={{ ssh_user }}@truenas.localdomain
+
+log "status=start"
+
+{% if db_container | default('') %}
+# ------------------------------------------------------------ 1. database
+# The Nextcloud containers are ROOTLESS podman owned by "{{ podman_user }}",
+# but this script runs as root under systemd. Every podman call therefore
+# goes through sudo:
+#   -H    HOME becomes the podman user's home, so podman finds its rootless
+#         graph root under ~/.local/share/containers
+#   cd;   required preamble (see CLAUDE.md) so the shell starts in that home
+#   XDG_RUNTIME_DIR  the podman user's runtime dir. Lingering is enabled by
+#         roles/podman/tasks/podman/podman.yml so /run/user/<uid> exists;
+#         guarded anyway so podman falls back cleanly if it ever does not.
+#
+# No credential is stored in this file or placed on a host command line:
+# $MYSQL_ROOT_PASSWORD and $MYSQL_DATABASE are expanded by the shell INSIDE
+# the database container, which already carries them in its environment.
+#
+# Flags are deliberately minimal. --events and --routines were both tried and
+# both abort the dump on these instances: the event scheduler is disabled
+# (error 1577) and mysql.proc reads as corrupted (error 1728, an artefact of
+# an image bump without mariadb-upgrade). Nextcloud uses neither events nor
+# stored routines, so dropping them loses nothing.
+pexec() {
+  sudo -H -u {{ podman_user }} bash -c \
+    'cd; d=/run/user/$(id -u); [ -d "$d" ] && export XDG_RUNTIME_DIR="$d"
+     exec podman "$@"' _ "$@"
+}
+
+install -d -m 0700 "$STAGE" "$STAGE/db"
+tmp="$DUMP.tmp"
+rm -f "$tmp"
+
+log "dumping {{ db_container }}"
+set +e
+pexec exec {{ db_container }} sh -c '
+  exec env MYSQL_PWD="$MYSQL_ROOT_PASSWORD" mariadb-dump -u root \
+    --single-transaction --quick --triggers \
+    --no-tablespaces --default-character-set=utf8mb4 "$MYSQL_DATABASE"
+' | gzip -6 > "$tmp"
+dump_rc=${PIPESTATUS[0]}
+set -e
+[ "$dump_rc" -eq 0 ] || fail "mariadb-dump {{ db_container }} exited $dump_rc"
+
+# A new dump is promoted over yesterday's only after it proves complete:
+# a valid gzip stream AND the "-- Dump completed" trailer that mariadb-dump
+# writes only on a clean finish. `mv` is atomic within the staging
+# filesystem, so a failed or truncated run can never replace a good dump.
+gzip -t "$tmp" || fail "dump is not a valid gzip stream"
+gunzip -c "$tmp" | tail -c 512 | grep -q 'Dump completed' \
+  || fail "dump is truncated (no completion trailer)"
+mv -f "$tmp" "$DUMP"
+log "db_dump=ok bytes=$(stat -c %s "$DUMP")"
+
+# Local retention. The staging sync below mirrors with --delete, so remote
+# retention follows the same window; deeper history comes from the TrueNAS
+# periodic ZFS snapshots (see roles/podman/README.md).
+ls -1t "$STAGE"/db/"$INSTANCE"-*.sql.gz | tail -n +$((KEEP + 1)) | xargs -r rm -f
+{% endif %}
+
+# ----------------------------------------------------------- 2. file tree
+# --exclude .ssh is load-bearing: {{ remote_path }} IS {{ ssh_user }}'s home
+# on TrueNAS and its authorized_keys lives there, so a `.ssh` directory
+# appearing in the data tree must never be shipped. No --delete here: the
+# data tree is append-mostly and a source-side mishap must not propagate.
+log "syncing data"
+rsync -az --timeout=1800 --exclude .ssh \
+  {{ backup_rsync_excludes | default("--exclude '/nextcloud.log*' --exclude '/updater.log'") }} \
+  {{ backup_rsync_extra_args | default('') }} \
+  -e "$SSH" {{ data_path }}/ "$DEST:{{ remote_path }}/"
+
+# ------------------------------------------- 3. config/ and database dumps
+{# --mkpath creates the nested _backup/<x>/ destination; rsync will not build
+   more than one missing level on its own. Requires rsync >= 3.2.3 on both
+   ends (galactica 3.4.1, truenas 3.2.7). #}
+{% if config_path | default('') %}
+log "syncing config"
+rsync -az --timeout=600 --delete --mkpath {{ backup_rsync_extra_args | default('') }} \
+  -e "$SSH" {{ config_path }}/ "$DEST:{{ remote_path }}/_backup/config/"
+{% endif %}
+{% if db_container | default('') %}
+log "syncing db dumps"
+rsync -az --timeout=600 --delete --mkpath {{ backup_rsync_extra_args | default('') }} \
+  -e "$SSH" "$STAGE/db/" "$DEST:{{ remote_path }}/_backup/db/"
+{% endif %}
+
+log "status=ok"
@@ -2,7 +2,8 @@
 Description=Daily Nextcloud {{ instance_name }} backup

 [Timer]
-OnCalendar=*-*-* 04:00:00
+OnCalendar={{ backup_oncalendar | default('*-*-* 04:00:00') }}
+RandomizedDelaySec={{ backup_randomized_delay | default('5m') }}
 Persistent=true

 [Install]
@@ -0,0 +1,40 @@
+#!/bin/bash
+# {{ ansible_managed }}
+# OnFailure= handler for the Nextcloud backup units. Invoked as:
+#   nextcloud-backup-alert.sh <failed-unit-name>
+#
+# Deliberately NOT `set -e`: an alert handler that dies partway through
+# reports nothing, which is worse than a partial report. Same reasoning as
+# roles/ups/templates/ups-restore.sh.j2.
+set -uo pipefail
+
+TAG=nextcloud-backup
+UNIT="${1:-unknown}"
+TO="{{ backup_alert_email | default('root') }}"
+HOST="$(hostname -f 2>/dev/null || hostname)"
+
+result="$(systemctl show -p Result --value "$UNIT" 2>/dev/null)"
+code="$(systemctl show -p ExecMainStatus --value "$UNIT" 2>/dev/null)"
+
+# One machine-parseable line for Graylog, then the context.
+logger -t "$TAG" -p daemon.err -- \
+  "status=failed unit=$UNIT result=${result:-unknown} exit=${code:-unknown}"
+
+body="$(printf 'Nextcloud backup FAILED on %s\n\nunit:   %s\nresult: %s\nexit:   %s\n\n--- last 40 journal lines ---\n' \
+  "$HOST" "$UNIT" "${result:-unknown}" "${code:-unknown}")
+$(journalctl -u "$UNIT" -n 40 --no-pager -o cat 2>/dev/null)"
+
+echo "$body" | logger -t "$TAG" -p daemon.err
+
+# Mail is best-effort: if the MTA is not configured the journald record above
+# is still the authoritative signal, so never fail the handler on this.
+if command -v sendmail >/dev/null 2>&1; then
+  printf 'To: %s\nSubject: [%s] Nextcloud backup FAILED: %s\nContent-Type: text/plain; charset=UTF-8\n\n%s\n' \
+    "$TO" "$HOST" "$UNIT" "$body" | sendmail -t \
+    && logger -t "$TAG" -p daemon.info -- "alert_mail=sent to=$TO" \
+    || logger -t "$TAG" -p daemon.err  -- "alert_mail=failed to=$TO"
+else
+  logger -t "$TAG" -p daemon.err -- "alert_mail=skipped reason=no-sendmail"
+fi
+
+exit 0
@@ -0,0 +1,6 @@
+[Unit]
+Description=Report failure of %i
+
+[Service]
+Type=oneshot
+ExecStart=/usr/local/bin/nextcloud-backup-alert.sh %i