ansible/roles/podman/templates/nextcloud/cloud-backup.sh.j2

@@ -40,11 +40,17 @@ log "status=start"
 # $MYSQL_ROOT_PASSWORD and $MYSQL_DATABASE are expanded by the shell INSIDE
 # the database container, which already carries them in its environment.
 #
-# Flags are deliberately minimal. --events and --routines were both tried and
+# --routines and --events initially aborted the dump here: mysql.proc read as
-# both abort the dump on these instances: the event scheduler is disabled
+# corrupted (error 1728) and the event scheduler reported disabled (1577),
-# (error 1577) and mysql.proc reads as corrupted (error 1728, an artefact of
+# both artefacts of an image bump without mariadb-upgrade. `mariadb-upgrade
-# an image bump without mariadb-upgrade). Nextcloud uses neither events nor
+# --force` has since been run against both instances and repaired the system
-# stored routines, so dropping them loses nothing.
+# tables, so the full flag set works and is kept for completeness.
+#
+# Note that upgrade still exits non-zero on these containers: it cannot
+# install the `sys` schema because the datadir root (/var/lib/mysql) is owned
+# by daemon rather than mysql, so mysqld may not create new top-level
+# databases. `sys` is purely diagnostic and unused by Nextcloud, so this is
+# cosmetic -- but it does mean creating a NEW database would fail too.
 pexec() {
   sudo -H -u {{ podman_user }} bash -c \
     'cd; d=/run/user/$(id -u); [ -d "$d" ] && export XDG_RUNTIME_DIR="$d"
@@ -59,7 +65,7 @@ log "dumping {{ db_container }}"
 set +e
 pexec exec {{ db_container }} sh -c '
   exec env MYSQL_PWD="$MYSQL_ROOT_PASSWORD" mariadb-dump -u root \
-    --single-transaction --quick --triggers \
+    --single-transaction --quick --routines --events --triggers \
     --no-tablespaces --default-character-set=utf8mb4 "$MYSQL_DATABASE"
 ' | gzip -6 > "$tmp"
 dump_rc=${PIPESTATUS[0]}

ansible/roles/podman/templates/nextcloud/nextcloud-backup-alert.sh.j2

@@ -16,21 +16,37 @@ HOST="$(hostname -f 2>/dev/null || hostname)"
 result="$(systemctl show -p Result --value "$UNIT" 2>/dev/null)"
 code="$(systemctl show -p ExecMainStatus --value "$UNIT" 2>/dev/null)"
 
-# One machine-parseable line for Graylog, then the context.
+# Only claim a failure when the unit actually reports one. Starting this
-logger -t "$TAG" -p daemon.err -- \
+# handler by hand (or any other spurious trigger) would otherwise mail out a
-  "status=failed unit=$UNIT result=${result:-unknown} exit=${code:-unknown}"
+# subject line saying FAILED about a run that succeeded. Keeping status=failed
+# exact also stops such triggers matching the Graylog alert rule.
+if [ "${result:-success}" = "success" ]; then
+  state=spurious
+  prio=daemon.warning
+  headline="$(printf 'Nextcloud backup alert handler was invoked on %s, but %s reports SUCCESS.\nThis is not a backup failure -- most likely the handler was started manually.' "$HOST" "$UNIT")"
+  subject="[$HOST] Nextcloud backup alert (spurious, unit OK): $UNIT"
+else
+  state=failed
+  prio=daemon.err
+  headline="Nextcloud backup FAILED on $HOST"
+  subject="[$HOST] Nextcloud backup FAILED: $UNIT"
+fi
 
-body="$(printf 'Nextcloud backup FAILED on %s\n\nunit:   %s\nresult: %s\nexit:   %s\n\n--- last 40 journal lines ---\n' \
+# One machine-parseable line for Graylog, then the context.
-  "$HOST" "$UNIT" "${result:-unknown}" "${code:-unknown}")
+logger -t "$TAG" -p "$prio" -- \
+  "status=$state unit=$UNIT result=${result:-unknown} exit=${code:-unknown}"
+
+body="$(printf '%s\n\nunit:   %s\nresult: %s\nexit:   %s\n\n--- last 40 journal lines ---\n' \
+  "$headline" "$UNIT" "${result:-unknown}" "${code:-unknown}")
 $(journalctl -u "$UNIT" -n 40 --no-pager -o cat 2>/dev/null)"
 
-echo "$body" | logger -t "$TAG" -p daemon.err
+echo "$body" | logger -t "$TAG" -p "$prio"
 
 # Mail is best-effort: if the MTA is not configured the journald record above
 # is still the authoritative signal, so never fail the handler on this.
 if command -v sendmail >/dev/null 2>&1; then
-  printf 'To: %s\nSubject: [%s] Nextcloud backup FAILED: %s\nContent-Type: text/plain; charset=UTF-8\n\n%s\n' \
+  printf 'To: %s\nSubject: %s\nContent-Type: text/plain; charset=UTF-8\n\n%s\n' \
-    "$TO" "$HOST" "$UNIT" "$body" | sendmail -t \
+    "$TO" "$subject" "$body" | sendmail -t \
     && logger -t "$TAG" -p daemon.info -- "alert_mail=sent to=$TO" \
     || logger -t "$TAG" -p daemon.err  -- "alert_mail=failed to=$TO"
 else