diff --git a/ansible/roles/podman/templates/nextcloud/cloud-backup.sh.j2 b/ansible/roles/podman/templates/nextcloud/cloud-backup.sh.j2 index 8153153..7ad1ed5 100644 --- a/ansible/roles/podman/templates/nextcloud/cloud-backup.sh.j2 +++ b/ansible/roles/podman/templates/nextcloud/cloud-backup.sh.j2 @@ -40,11 +40,17 @@ log "status=start" # $MYSQL_ROOT_PASSWORD and $MYSQL_DATABASE are expanded by the shell INSIDE # the database container, which already carries them in its environment. # -# Flags are deliberately minimal. --events and --routines were both tried and -# both abort the dump on these instances: the event scheduler is disabled -# (error 1577) and mysql.proc reads as corrupted (error 1728, an artefact of -# an image bump without mariadb-upgrade). Nextcloud uses neither events nor -# stored routines, so dropping them loses nothing. +# --routines and --events initially aborted the dump here: mysql.proc read as +# corrupted (error 1728) and the event scheduler reported disabled (1577), +# both artefacts of an image bump without mariadb-upgrade. `mariadb-upgrade +# --force` has since been run against both instances and repaired the system +# tables, so the full flag set works and is kept for completeness. +# +# Note that upgrade still exits non-zero on these containers: it cannot +# install the `sys` schema because the datadir root (/var/lib/mysql) is owned +# by daemon rather than mysql, so mysqld may not create new top-level +# databases. `sys` is purely diagnostic and unused by Nextcloud, so this is +# cosmetic -- but it does mean creating a NEW database would fail too. pexec() { sudo -H -u {{ podman_user }} bash -c \ 'cd; d=/run/user/$(id -u); [ -d "$d" ] && export XDG_RUNTIME_DIR="$d" @@ -59,7 +65,7 @@ log "dumping {{ db_container }}" set +e pexec exec {{ db_container }} sh -c ' exec env MYSQL_PWD="$MYSQL_ROOT_PASSWORD" mariadb-dump -u root \ - --single-transaction --quick --triggers \ + --single-transaction --quick --routines --events --triggers \ --no-tablespaces --default-character-set=utf8mb4 "$MYSQL_DATABASE" ' | gzip -6 > "$tmp" dump_rc=${PIPESTATUS[0]} diff --git a/ansible/roles/podman/templates/nextcloud/nextcloud-backup-alert.sh.j2 b/ansible/roles/podman/templates/nextcloud/nextcloud-backup-alert.sh.j2 index c461bbb..1e72623 100644 --- a/ansible/roles/podman/templates/nextcloud/nextcloud-backup-alert.sh.j2 +++ b/ansible/roles/podman/templates/nextcloud/nextcloud-backup-alert.sh.j2 @@ -16,21 +16,37 @@ HOST="$(hostname -f 2>/dev/null || hostname)" result="$(systemctl show -p Result --value "$UNIT" 2>/dev/null)" code="$(systemctl show -p ExecMainStatus --value "$UNIT" 2>/dev/null)" -# One machine-parseable line for Graylog, then the context. -logger -t "$TAG" -p daemon.err -- \ - "status=failed unit=$UNIT result=${result:-unknown} exit=${code:-unknown}" +# Only claim a failure when the unit actually reports one. Starting this +# handler by hand (or any other spurious trigger) would otherwise mail out a +# subject line saying FAILED about a run that succeeded. Keeping status=failed +# exact also stops such triggers matching the Graylog alert rule. +if [ "${result:-success}" = "success" ]; then + state=spurious + prio=daemon.warning + headline="$(printf 'Nextcloud backup alert handler was invoked on %s, but %s reports SUCCESS.\nThis is not a backup failure -- most likely the handler was started manually.' "$HOST" "$UNIT")" + subject="[$HOST] Nextcloud backup alert (spurious, unit OK): $UNIT" +else + state=failed + prio=daemon.err + headline="Nextcloud backup FAILED on $HOST" + subject="[$HOST] Nextcloud backup FAILED: $UNIT" +fi -body="$(printf 'Nextcloud backup FAILED on %s\n\nunit: %s\nresult: %s\nexit: %s\n\n--- last 40 journal lines ---\n' \ - "$HOST" "$UNIT" "${result:-unknown}" "${code:-unknown}") +# One machine-parseable line for Graylog, then the context. +logger -t "$TAG" -p "$prio" -- \ + "status=$state unit=$UNIT result=${result:-unknown} exit=${code:-unknown}" + +body="$(printf '%s\n\nunit: %s\nresult: %s\nexit: %s\n\n--- last 40 journal lines ---\n' \ + "$headline" "$UNIT" "${result:-unknown}" "${code:-unknown}") $(journalctl -u "$UNIT" -n 40 --no-pager -o cat 2>/dev/null)" -echo "$body" | logger -t "$TAG" -p daemon.err +echo "$body" | logger -t "$TAG" -p "$prio" # Mail is best-effort: if the MTA is not configured the journald record above # is still the authoritative signal, so never fail the handler on this. if command -v sendmail >/dev/null 2>&1; then - printf 'To: %s\nSubject: [%s] Nextcloud backup FAILED: %s\nContent-Type: text/plain; charset=UTF-8\n\n%s\n' \ - "$TO" "$HOST" "$UNIT" "$body" | sendmail -t \ + printf 'To: %s\nSubject: %s\nContent-Type: text/plain; charset=UTF-8\n\n%s\n' \ + "$TO" "$subject" "$body" | sendmail -t \ && logger -t "$TAG" -p daemon.info -- "alert_mail=sent to=$TO" \ || logger -t "$TAG" -p daemon.err -- "alert_mail=failed to=$TO" else