Bastian de Byl
0ab423ca55
The data-only rsync left no way to restore a working instance: mysql/ and config/ were never backed up, so a recovery would have files but no shares, users or metadata. Dump the database before syncing files (a DB older than the files is repairable with occ files:scan; a newer one references blobs that never made it into the backup) and ship config/ alongside it. Capture the --chmod=Du=rwx,Dgo=rx flag that had been hand-added to the deployed skudak-cloud script. It was outside git, so every deploy silently reverted it. It now lives in backup_rsync_extra_args. Add OnFailure= alerting. The units failed silently before, which is how an iDrive sync failure sat unnoticed since May. msmtp rather than the esmtp already installed: the OpenSRS relay is port 465 (implicit TLS) and libesmtp only speaks STARTTLS. Exclude nextcloud.log* from the sync and cap log_rotate_size. skudak-cloud was running at loglevel 0 and had written a 64 GB log that was being rsynced and pushed to S3; set it to 2 to match the home instance. Stagger the timers (04:00 / 04:30) so both finish before the 05:00 TrueNAS snapshot task, and bound TimeoutStartSec so a wedged rsync cannot leave the unit activating forever and skip every subsequent trigger. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
112 lines
5.1 KiB
Django/Jinja
112 lines
5.1 KiB
Django/Jinja
#!/bin/bash
|
|
# {{ ansible_managed }}
|
|
# Nextcloud "{{ backup_name }}" -> truenas.localdomain.
|
|
#
|
|
# Ordering is deliberate: the database is dumped BEFORE the file tree is
|
|
# synced. A DB snapshot slightly OLDER than the files degrades to "files
|
|
# Nextcloud has not indexed yet" and is repaired with `occ files:scan`. A DB
|
|
# snapshot NEWER than the files references blobs that never made it into the
|
|
# backup, which surfaces as broken shares and dead file entries on restore.
|
|
set -euo pipefail
|
|
|
|
TAG=nextcloud-backup
|
|
INSTANCE={{ backup_name }}
|
|
STAGE={{ backup_stage_path | default('/var/backups/nextcloud/' ~ backup_name) }}
|
|
KEEP={{ backup_db_keep | default(7) }}
|
|
DUMP="$STAGE/db/${INSTANCE}-$(date +%Y%m%d).sql.gz"
|
|
|
|
log() { logger -t "$TAG" -p daemon.info -- "instance=$INSTANCE $*"; echo "$TAG: $*"; }
|
|
fail() { logger -t "$TAG" -p daemon.err -- "instance=$INSTANCE status=failed $*"
|
|
echo "$TAG: FAILED: $*" >&2; exit 1; }
|
|
|
|
SSH="ssh -i {{ ssh_key_path }} -o StrictHostKeyChecking=accept-new -o ServerAliveInterval=30 -o ServerAliveCountMax=6"
|
|
DEST={{ ssh_user }}@truenas.localdomain
|
|
|
|
log "status=start"
|
|
|
|
{% if db_container | default('') %}
|
|
# ------------------------------------------------------------ 1. database
|
|
# The Nextcloud containers are ROOTLESS podman owned by "{{ podman_user }}",
|
|
# but this script runs as root under systemd. Every podman call therefore
|
|
# goes through sudo:
|
|
# -H HOME becomes the podman user's home, so podman finds its rootless
|
|
# graph root under ~/.local/share/containers
|
|
# cd; required preamble (see CLAUDE.md) so the shell starts in that home
|
|
# XDG_RUNTIME_DIR the podman user's runtime dir. Lingering is enabled by
|
|
# roles/podman/tasks/podman/podman.yml so /run/user/<uid> exists;
|
|
# guarded anyway so podman falls back cleanly if it ever does not.
|
|
#
|
|
# No credential is stored in this file or placed on a host command line:
|
|
# $MYSQL_ROOT_PASSWORD and $MYSQL_DATABASE are expanded by the shell INSIDE
|
|
# the database container, which already carries them in its environment.
|
|
#
|
|
# Flags are deliberately minimal. --events and --routines were both tried and
|
|
# both abort the dump on these instances: the event scheduler is disabled
|
|
# (error 1577) and mysql.proc reads as corrupted (error 1728, an artefact of
|
|
# an image bump without mariadb-upgrade). Nextcloud uses neither events nor
|
|
# stored routines, so dropping them loses nothing.
|
|
pexec() {
|
|
sudo -H -u {{ podman_user }} bash -c \
|
|
'cd; d=/run/user/$(id -u); [ -d "$d" ] && export XDG_RUNTIME_DIR="$d"
|
|
exec podman "$@"' _ "$@"
|
|
}
|
|
|
|
install -d -m 0700 "$STAGE" "$STAGE/db"
|
|
tmp="$DUMP.tmp"
|
|
rm -f "$tmp"
|
|
|
|
log "dumping {{ db_container }}"
|
|
set +e
|
|
pexec exec {{ db_container }} sh -c '
|
|
exec env MYSQL_PWD="$MYSQL_ROOT_PASSWORD" mariadb-dump -u root \
|
|
--single-transaction --quick --triggers \
|
|
--no-tablespaces --default-character-set=utf8mb4 "$MYSQL_DATABASE"
|
|
' | gzip -6 > "$tmp"
|
|
dump_rc=${PIPESTATUS[0]}
|
|
set -e
|
|
[ "$dump_rc" -eq 0 ] || fail "mariadb-dump {{ db_container }} exited $dump_rc"
|
|
|
|
# A new dump is promoted over yesterday's only after it proves complete:
|
|
# a valid gzip stream AND the "-- Dump completed" trailer that mariadb-dump
|
|
# writes only on a clean finish. `mv` is atomic within the staging
|
|
# filesystem, so a failed or truncated run can never replace a good dump.
|
|
gzip -t "$tmp" || fail "dump is not a valid gzip stream"
|
|
gunzip -c "$tmp" | tail -c 512 | grep -q 'Dump completed' \
|
|
|| fail "dump is truncated (no completion trailer)"
|
|
mv -f "$tmp" "$DUMP"
|
|
log "db_dump=ok bytes=$(stat -c %s "$DUMP")"
|
|
|
|
# Local retention. The staging sync below mirrors with --delete, so remote
|
|
# retention follows the same window; deeper history comes from the TrueNAS
|
|
# periodic ZFS snapshots (see roles/podman/README.md).
|
|
ls -1t "$STAGE"/db/"$INSTANCE"-*.sql.gz | tail -n +$((KEEP + 1)) | xargs -r rm -f
|
|
{% endif %}
|
|
|
|
# ----------------------------------------------------------- 2. file tree
|
|
# --exclude .ssh is load-bearing: {{ remote_path }} IS {{ ssh_user }}'s home
|
|
# on TrueNAS and its authorized_keys lives there, so a `.ssh` directory
|
|
# appearing in the data tree must never be shipped. No --delete here: the
|
|
# data tree is append-mostly and a source-side mishap must not propagate.
|
|
log "syncing data"
|
|
rsync -az --timeout=1800 --exclude .ssh \
|
|
{{ backup_rsync_excludes | default("--exclude '/nextcloud.log*' --exclude '/updater.log'") }} \
|
|
{{ backup_rsync_extra_args | default('') }} \
|
|
-e "$SSH" {{ data_path }}/ "$DEST:{{ remote_path }}/"
|
|
|
|
# ------------------------------------------- 3. config/ and database dumps
|
|
{# --mkpath creates the nested _backup/<x>/ destination; rsync will not build
|
|
more than one missing level on its own. Requires rsync >= 3.2.3 on both
|
|
ends (galactica 3.4.1, truenas 3.2.7). #}
|
|
{% if config_path | default('') %}
|
|
log "syncing config"
|
|
rsync -az --timeout=600 --delete --mkpath {{ backup_rsync_extra_args | default('') }} \
|
|
-e "$SSH" {{ config_path }}/ "$DEST:{{ remote_path }}/_backup/config/"
|
|
{% endif %}
|
|
{% if db_container | default('') %}
|
|
log "syncing db dumps"
|
|
rsync -az --timeout=600 --delete --mkpath {{ backup_rsync_extra_args | default('') }} \
|
|
-e "$SSH" "$STAGE/db/" "$DEST:{{ remote_path }}/_backup/db/"
|
|
{% endif %}
|
|
|
|
log "status=ok"
|