bastian/deploy_home
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

bound log growth and reclaim ~52 GB of container disk

Browse Source 
Caddy was rotating on implicit defaults (100MiB/keep 10/90d) that were not
holding -- 20 rotated files per stream and a 190-day-old .gz, 1.3 GB across
16 log streams. Made explicit at 10MiB/keep 3/7d.

Note roll_size et al are subdirectives of `output file`, NOT of `log`.
Getting that wrong does not degrade gracefully: Caddy refuses to start on a
bad config, so every site went down until it was corrected. Worth a
`caddy validate` gate before reload.

journald had no SystemMaxUse and had reached 4 GB, drifting toward its
10%-of-filesystem default (~190 GB on this root). Capped at 500M.

Both are safe to keep short because fluent-bit ships the journal and every
Caddy access log into Graylog -- though note its GELF output has been
erroring for days, which weakens that premise and wants investigating.

The larger find was unrelated to logs: 896 images totalling 59.6 GB with
75% unused (94 tags of greg-time-bot, 73 of fulfillr -- one per deploy) and
5.4 GB of dangling volumes, mostly 804 MB Nextcloud /var/www/html trees
orphaned by container recreations. Pruned to 22 images / 15.4 GB, and added
a weekly timer keeping 30 days so a rollback still needs no rebuild.

Also dropped the decommissioned 6379/tcp redis rule (nothing listening;
Immich's redis is on the shared podman network) and the orphaned nosql, s3
and searxng volume dirs.

Backup log exclusions turned out to be unnecessary: Gitea logs to console
so its log dirs are empty, Nextcloud already excludes its own, BookStack
mounts only uploads, and Caddy is not backed up.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Bastian de Byl
2026-07-31 11:13:35 -04:00
parent bc110ce69e
commit f11391b28f
11 changed files with 260 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ansible/roles/podman/templates/caddy/Caddyfile.j2
+80 -16
View File
@@ -16,7 +16,11 @@
# Logging
log {
output file /var/log/caddy/caddy.log
output file /var/log/caddy/caddy.log {
roll_size {{ caddy_log_roll_size }}
roll_keep {{ caddy_log_roll_keep }}
roll_keep_for {{ caddy_log_roll_keep_for }}
}
format {{ caddy_log_format }}
level {{ caddy_log_level }}
}
@@ -73,7 +77,11 @@
reverse_proxy localhost:8088
log {
output file /var/log/caddy/photos.log
output file /var/log/caddy/photos.log {
roll_size {{ caddy_log_roll_size }}
roll_keep {{ caddy_log_roll_keep }}
roll_keep_for {{ caddy_log_roll_keep_for }}
}
format json
}
}
@@ -90,7 +98,11 @@
reverse_proxy localhost:6875
log {
output file /var/log/caddy/wiki.log
output file /var/log/caddy/wiki.log {
roll_size {{ caddy_log_roll_size }}
roll_keep {{ caddy_log_roll_keep }}
roll_keep_for {{ caddy_log_roll_keep_for }}
}
format json
}
}
@@ -122,7 +134,11 @@
}
log {
output file /var/log/caddy/assistant.log
output file /var/log/caddy/assistant.log {
roll_size {{ caddy_log_roll_size }}
roll_keep {{ caddy_log_roll_keep }}
roll_keep_for {{ caddy_log_roll_keep_for }}
}
format json
}
}
@@ -154,7 +170,11 @@
}
log {
output file /var/log/caddy/parts.log
output file /var/log/caddy/parts.log {
roll_size {{ caddy_log_roll_size }}
roll_keep {{ caddy_log_roll_keep }}
roll_keep_for {{ caddy_log_roll_keep_for }}
}
format json
}
}
@@ -164,7 +184,11 @@
import common_headers
reverse_proxy localhost:8082
log {
output file /var/log/caddy/parts-skudak.log
output file /var/log/caddy/parts-skudak.log {
roll_size {{ caddy_log_roll_size }}
roll_keep {{ caddy_log_roll_keep }}
roll_keep_for {{ caddy_log_roll_keep_for }}
}
format json
}
}
@@ -182,7 +206,11 @@
}
log {
output file /var/log/caddy/uptime-kuma.log
output file /var/log/caddy/uptime-kuma.log {
roll_size {{ caddy_log_roll_size }}
roll_keep {{ caddy_log_roll_keep }}
roll_keep_for {{ caddy_log_roll_keep_for }}
}
format json
}
}
@@ -200,7 +228,11 @@
}
log {
output file /var/log/caddy/uptime-kuma-personal.log
output file /var/log/caddy/uptime-kuma-personal.log {
roll_size {{ caddy_log_roll_size }}
roll_keep {{ caddy_log_roll_keep }}
roll_keep_for {{ caddy_log_roll_keep_for }}
}
format json
}
}
@@ -243,7 +275,11 @@
}
log {
output file /var/log/caddy/graylog.log
output file /var/log/caddy/graylog.log {
roll_size {{ caddy_log_roll_size }}
roll_keep {{ caddy_log_roll_keep }}
roll_keep_for {{ caddy_log_roll_keep_for }}
}
format json
}
}
@@ -281,7 +317,11 @@
redir /.well-known/caldav /remote.php/dav 301
log {
output file /var/log/caddy/cloud.log
output file /var/log/caddy/cloud.log {
roll_size {{ caddy_log_roll_size }}
roll_keep {{ caddy_log_roll_keep }}
roll_keep_for {{ caddy_log_roll_keep_for }}
}
format json
}
}
@@ -309,7 +349,11 @@
redir /.well-known/caldav /remote.php/dav 301
log {
output file /var/log/caddy/cloud-skudak.log
output file /var/log/caddy/cloud-skudak.log {
roll_size {{ caddy_log_roll_size }}
roll_keep {{ caddy_log_roll_keep }}
roll_keep_for {{ caddy_log_roll_keep_for }}
}
format json
}
}
@@ -323,7 +367,11 @@
}
log {
output file /var/log/caddy/gitea-debyl.log
output file /var/log/caddy/gitea-debyl.log {
roll_size {{ caddy_log_roll_size }}
roll_keep {{ caddy_log_roll_keep }}
roll_keep_for {{ caddy_log_roll_keep_for }}
}
format json
}
}
@@ -337,7 +385,11 @@
}
log {
output file /var/log/caddy/gitea-skudak.log
output file /var/log/caddy/gitea-skudak.log {
roll_size {{ caddy_log_roll_size }}
roll_keep {{ caddy_log_roll_keep }}
roll_keep_for {{ caddy_log_roll_keep_for }}
}
format json
}
}
@@ -384,7 +436,11 @@
}
log {
output file /var/log/caddy/fulfillr.log
output file /var/log/caddy/fulfillr.log {
roll_size {{ caddy_log_roll_size }}
roll_keep {{ caddy_log_roll_keep }}
roll_keep_for {{ caddy_log_roll_keep_for }}
}
format json
}
}
@@ -431,7 +487,11 @@
}
log {
output file /var/log/caddy/fulfillr-dev.log
output file /var/log/caddy/fulfillr-dev.log {
roll_size {{ caddy_log_roll_size }}
roll_keep {{ caddy_log_roll_keep }}
roll_keep_for {{ caddy_log_roll_keep_for }}
}
format json
}
}
@@ -453,7 +513,11 @@ test.debyl.io {
header Pragma "no-cache"
log {
output file /var/log/caddy/test.log
output file /var/log/caddy/test.log {
roll_size {{ caddy_log_roll_size }}
roll_keep {{ caddy_log_roll_keep }}
roll_keep_for {{ caddy_log_roll_keep_for }}
}
format json
level {{ caddy_log_level }}
}
ansible/roles/podman/templates/podman-prune.service.j2
+15
View File
@@ -0,0 +1,15 @@
[Unit]
Description=Prune unused podman images and volumes
After=network-online.target
[Service]
Type=oneshot
ExecStart=/usr/local/bin/podman-prune.sh
# Type=oneshot disables the start timeout by default, so a wedged prune would
# leave the unit "activating" forever and every later trigger would be
# silently skipped -- the same trap the backup units guard against.
TimeoutStartSec={{ podman_prune_timeout | default('1h') }}
# Pruning is I/O-heavy and competes with the 03:00-04:30 backups; keep it off
# the critical path.
Nice=15
IOSchedulingClass=idle
ansible/roles/podman/templates/podman-prune.sh.j2
+53
View File
@@ -0,0 +1,53 @@
#!/bin/bash
# {{ ansible_managed }}
# Weekly reclaim of unused podman images and volumes.
#
# Every image bump leaves the previous tag behind and nothing ever removed
# them: this was written after finding 896 images totalling 59.6 GB, 75% of it
# unused -- 94 tags of greg-time-bot and 73 of fulfillr, one per deploy.
#
# --filter until={{ podman_prune_until }} keeps recent images so a rollback
# does not require a rebuild or re-pull. Anything older is unused AND stale.
#
# Volumes pruned here are podman's ANONYMOUS volumes, not the bind mounts
# under {{ podman_volumes }} that hold real service data -- those are
# directories on the host and podman does not know about them. The dangling
# ones seen in practice were 804 MB copies of Nextcloud's /var/www/html left
# by container recreations, which are image content, not data.
set -uo pipefail
TAG=podman-prune
log() { logger -t "$TAG" -p daemon.info -- "$*"; echo "$TAG: $*"; }
total_before=0
total_after=0
for u in {{ podman_prune_users | join(' ') }}; do
# Rootless podman: -H so HOME points at the user's store, and the `cd;`
# preamble is required (see CLAUDE.md) or podman cannot find its graph root.
run() {
sudo -H -u "$u" bash -c \
'cd; d=/run/user/$(id -u); [ -d "$d" ] && export XDG_RUNTIME_DIR="$d"
exec podman "$@"' _ "$@"
}
if ! id "$u" >/dev/null 2>&1; then
log "user=$u status=skipped reason=no-such-user"
continue
fi
before=$(run system df --format '{{ '{{' }}.Size{{ '}}' }}' 2>/dev/null | head -1)
# Deliberately NOT `set -e`: a prune failing for one user must not stop the
# other, and a busy image is a normal, non-fatal outcome.
img=$(run image prune -af --filter "until={{ podman_prune_until }}" 2>&1 | tail -1)
vol=$(run volume prune -f 2>&1 | tail -1)
after=$(run system df --format '{{ '{{' }}.Size{{ '}}' }}' 2>/dev/null | head -1)
log "user=$u images_before=$before images_after=$after"
log "user=$u image_prune=${img:-none} volume_prune=${vol:-none}"
done
log "status=ok"
ansible/roles/podman/templates/podman-prune.timer.j2
+12
View File
@@ -0,0 +1,12 @@
[Unit]
Description=Weekly podman image and volume prune
[Timer]
OnCalendar={{ podman_prune_oncalendar | default('Sun *-*-* 02:00:00') }}
RandomizedDelaySec=15m
# Weekly, and growth is one tag per deploy, so a missed run is worth catching
# up on rather than skipping.
Persistent=true
[Install]
WantedBy=timers.target