From f11391b28f298ad3687e74d1a902237a02902b91 Mon Sep 17 00:00:00 2001 From: Bastian de Byl Date: Fri, 31 Jul 2026 11:13:35 -0400 Subject: [PATCH] bound log growth and reclaim ~52 GB of container disk Caddy was rotating on implicit defaults (100MiB/keep 10/90d) that were not holding -- 20 rotated files per stream and a 190-day-old .gz, 1.3 GB across 16 log streams. Made explicit at 10MiB/keep 3/7d. Note roll_size et al are subdirectives of `output file`, NOT of `log`. Getting that wrong does not degrade gracefully: Caddy refuses to start on a bad config, so every site went down until it was corrected. Worth a `caddy validate` gate before reload. journald had no SystemMaxUse and had reached 4 GB, drifting toward its 10%-of-filesystem default (~190 GB on this root). Capped at 500M. Both are safe to keep short because fluent-bit ships the journal and every Caddy access log into Graylog -- though note its GELF output has been erroring for days, which weakens that premise and wants investigating. The larger find was unrelated to logs: 896 images totalling 59.6 GB with 75% unused (94 tags of greg-time-bot, 73 of fulfillr -- one per deploy) and 5.4 GB of dangling volumes, mostly 804 MB Nextcloud /var/www/html trees orphaned by container recreations. Pruned to 22 images / 15.4 GB, and added a weekly timer keeping 30 days so a rollback still needs no rebuild. Also dropped the decommissioned 6379/tcp redis rule (nothing listening; Immich's redis is on the shared podman network) and the orphaned nosql, s3 and searxng volume dirs. Backup log exclusions turned out to be unnecessary: Gitea logs to console so its log dirs are empty, Nextcloud already excludes its own, BookStack mounts only uploads, and Caddy is not backed up. Co-Authored-By: Claude Opus 5 (1M context) --- ansible/roles/common/handlers/main.yml | 6 ++ ansible/roles/common/tasks/service.yml | 24 +++++ .../common/templates/journald-size.conf.j2 | 12 +++ ansible/roles/podman/defaults/main.yml | 19 ++++ ansible/roles/podman/tasks/firewall.yml | 2 - ansible/roles/podman/tasks/main.yml | 3 + .../podman/tasks/podman/podman-prune.yml | 36 +++++++ .../roles/podman/templates/caddy/Caddyfile.j2 | 96 +++++++++++++++---- .../podman/templates/podman-prune.service.j2 | 15 +++ .../roles/podman/templates/podman-prune.sh.j2 | 53 ++++++++++ .../podman/templates/podman-prune.timer.j2 | 12 +++ 11 files changed, 260 insertions(+), 18 deletions(-) create mode 100644 ansible/roles/common/templates/journald-size.conf.j2 create mode 100644 ansible/roles/podman/tasks/podman/podman-prune.yml create mode 100644 ansible/roles/podman/templates/podman-prune.service.j2 create mode 100644 ansible/roles/podman/templates/podman-prune.sh.j2 create mode 100644 ansible/roles/podman/templates/podman-prune.timer.j2 diff --git a/ansible/roles/common/handlers/main.yml b/ansible/roles/common/handlers/main.yml index 12eb5f6..341b298 100644 --- a/ansible/roles/common/handlers/main.yml +++ b/ansible/roles/common/handlers/main.yml @@ -16,3 +16,9 @@ ansible.builtin.systemd: name: fluent-bit state: restarted + +- name: restart_journald + become: true + ansible.builtin.systemd: + name: systemd-journald + state: restarted diff --git a/ansible/roles/common/tasks/service.yml b/ansible/roles/common/tasks/service.yml index 9b0c059..cdeb176 100644 --- a/ansible/roles/common/tasks/service.yml +++ b/ansible/roles/common/tasks/service.yml @@ -1,4 +1,28 @@ --- +# Bound the journal. Container stdout all lands here (log_driver=journald) and +# is shipped to Graylog by fluent-bit, so the local journal only needs to be a +# buffer -- see the template for the reasoning behind the size. +- name: ensure journald drop-in directory exists + become: true + ansible.builtin.file: + path: /etc/systemd/journald.conf.d + state: directory + owner: root + group: root + mode: 0755 + tags: security, service, journald + +- name: cap journald disk usage + become: true + ansible.builtin.template: + src: journald-size.conf.j2 + dest: /etc/systemd/journald.conf.d/99-size.conf + owner: root + group: root + mode: 0644 + notify: restart_journald + tags: security, service, journald + - name: ensure desired services are started and enabled become: true ansible.builtin.service: diff --git a/ansible/roles/common/templates/journald-size.conf.j2 b/ansible/roles/common/templates/journald-size.conf.j2 new file mode 100644 index 0000000..919bca7 --- /dev/null +++ b/ansible/roles/common/templates/journald-size.conf.j2 @@ -0,0 +1,12 @@ +### {{ ansible_managed }} +# Without a cap journald sizes itself at 10% of the filesystem, which on this +# 1.9 TB root means it will happily grow into the tens of gigabytes; it had +# reached 4 GB before this was set. +# +# Kept small on purpose: every container runs with log_driver=journald and +# fluent-bit drains the journal into Graylog continuously (systemd input, +# _COMM=conmon -- see templates/fluent-bit/fluent-bit.conf.j2), so Graylog is +# the system of record. What stays here is only the buffer that covers +# fluent-bit being down, and 500M is a long outage at this log rate. +[Journal] +SystemMaxUse={{ journald_max_use | default('500M') }} diff --git a/ansible/roles/podman/defaults/main.yml b/ansible/roles/podman/defaults/main.yml index c82e10e..9660ca9 100644 --- a/ansible/roles/podman/defaults/main.yml +++ b/ansible/roles/podman/defaults/main.yml @@ -134,6 +134,16 @@ caddy_local_networks: caddy_log_level: INFO caddy_log_format: json +# Log rotation. Caddy rotates on implicit defaults (100MiB / keep 10 / 90d) +# without these, and those were demonstrably not holding: 20 rotated files per +# stream for the busiest logs and a 190-day-old .gz, for 1.3 GB across 16 +# streams. Kept short deliberately -- fluent-bit tails every one of these into +# Graylog (see roles/common/templates/fluent-bit/fluent-bit.conf.j2), so +# Graylog is the system of record and the local files are only a buffer. +caddy_log_roll_size: 10MiB +caddy_log_roll_keep: 3 +caddy_log_roll_keep_for: 168h + # Caddy performance tuning caddy_max_request_body_mb: 500 @@ -175,3 +185,12 @@ geoip_path: "{{ graylog_path }}/geoip" geoip_database_edition: GeoLite2-City # geoip_maxmind_account_id: defined in vault # geoip_maxmind_license_key: defined in vault + +# Weekly podman prune (see templates/podman-prune.sh.j2). Both rootless users +# have their own image store; the git user runs the Gitea pods. +podman_prune_users: + - "{{ podman_user }}" + - "{{ git_user }}" +# Keep 30 days of unused images so a rollback needs no rebuild or re-pull. +podman_prune_until: 720h +podman_prune_oncalendar: "Sun *-*-* 02:00:00" diff --git a/ansible/roles/podman/tasks/firewall.yml b/ansible/roles/podman/tasks/firewall.yml index f90db4c..d93c6d3 100644 --- a/ansible/roles/podman/tasks/firewall.yml +++ b/ansible/roles/podman/tasks/firewall.yml @@ -15,8 +15,6 @@ - 443/tcp # Gitea Skudak SSH - 2222/tcp - # nosql/redis - - 6379/tcp # BookStack (wiki.skudak.com) -- container publishes 6875:8080 - 6875/tcp # Satisfactory diff --git a/ansible/roles/podman/tasks/main.yml b/ansible/roles/podman/tasks/main.yml index aaddf64..b770b8a 100644 --- a/ansible/roles/podman/tasks/main.yml +++ b/ansible/roles/podman/tasks/main.yml @@ -2,6 +2,9 @@ - import_tasks: firewall.yml - import_tasks: podman/podman.yml +- import_tasks: podman/podman-prune.yml + tags: podman-prune + # WEB SERVER: Caddy is the default and only web server # nginx has been completely replaced and removed diff --git a/ansible/roles/podman/tasks/podman/podman-prune.yml b/ansible/roles/podman/tasks/podman/podman-prune.yml new file mode 100644 index 0000000..047a28f --- /dev/null +++ b/ansible/roles/podman/tasks/podman/podman-prune.yml @@ -0,0 +1,36 @@ +--- +- name: template podman prune script + become: true + ansible.builtin.template: + src: podman-prune.sh.j2 + dest: /usr/local/bin/podman-prune.sh + owner: root + group: root + mode: 0755 + setype: bin_t + +- name: template podman prune systemd service + become: true + ansible.builtin.template: + src: podman-prune.service.j2 + dest: /etc/systemd/system/podman-prune.service + owner: root + group: root + mode: 0644 + +- name: template podman prune systemd timer + become: true + ansible.builtin.template: + src: podman-prune.timer.j2 + dest: /etc/systemd/system/podman-prune.timer + owner: root + group: root + mode: 0644 + +- name: enable and start podman prune timer + become: true + ansible.builtin.systemd: + name: podman-prune.timer + enabled: true + state: started + daemon_reload: true diff --git a/ansible/roles/podman/templates/caddy/Caddyfile.j2 b/ansible/roles/podman/templates/caddy/Caddyfile.j2 index 048696f..3caa72f 100644 --- a/ansible/roles/podman/templates/caddy/Caddyfile.j2 +++ b/ansible/roles/podman/templates/caddy/Caddyfile.j2 @@ -16,7 +16,11 @@ # Logging log { - output file /var/log/caddy/caddy.log + output file /var/log/caddy/caddy.log { + roll_size {{ caddy_log_roll_size }} + roll_keep {{ caddy_log_roll_keep }} + roll_keep_for {{ caddy_log_roll_keep_for }} + } format {{ caddy_log_format }} level {{ caddy_log_level }} } @@ -73,7 +77,11 @@ reverse_proxy localhost:8088 log { - output file /var/log/caddy/photos.log + output file /var/log/caddy/photos.log { + roll_size {{ caddy_log_roll_size }} + roll_keep {{ caddy_log_roll_keep }} + roll_keep_for {{ caddy_log_roll_keep_for }} + } format json } } @@ -90,7 +98,11 @@ reverse_proxy localhost:6875 log { - output file /var/log/caddy/wiki.log + output file /var/log/caddy/wiki.log { + roll_size {{ caddy_log_roll_size }} + roll_keep {{ caddy_log_roll_keep }} + roll_keep_for {{ caddy_log_roll_keep_for }} + } format json } } @@ -122,7 +134,11 @@ } log { - output file /var/log/caddy/assistant.log + output file /var/log/caddy/assistant.log { + roll_size {{ caddy_log_roll_size }} + roll_keep {{ caddy_log_roll_keep }} + roll_keep_for {{ caddy_log_roll_keep_for }} + } format json } } @@ -154,7 +170,11 @@ } log { - output file /var/log/caddy/parts.log + output file /var/log/caddy/parts.log { + roll_size {{ caddy_log_roll_size }} + roll_keep {{ caddy_log_roll_keep }} + roll_keep_for {{ caddy_log_roll_keep_for }} + } format json } } @@ -164,7 +184,11 @@ import common_headers reverse_proxy localhost:8082 log { - output file /var/log/caddy/parts-skudak.log + output file /var/log/caddy/parts-skudak.log { + roll_size {{ caddy_log_roll_size }} + roll_keep {{ caddy_log_roll_keep }} + roll_keep_for {{ caddy_log_roll_keep_for }} + } format json } } @@ -182,7 +206,11 @@ } log { - output file /var/log/caddy/uptime-kuma.log + output file /var/log/caddy/uptime-kuma.log { + roll_size {{ caddy_log_roll_size }} + roll_keep {{ caddy_log_roll_keep }} + roll_keep_for {{ caddy_log_roll_keep_for }} + } format json } } @@ -200,7 +228,11 @@ } log { - output file /var/log/caddy/uptime-kuma-personal.log + output file /var/log/caddy/uptime-kuma-personal.log { + roll_size {{ caddy_log_roll_size }} + roll_keep {{ caddy_log_roll_keep }} + roll_keep_for {{ caddy_log_roll_keep_for }} + } format json } } @@ -243,7 +275,11 @@ } log { - output file /var/log/caddy/graylog.log + output file /var/log/caddy/graylog.log { + roll_size {{ caddy_log_roll_size }} + roll_keep {{ caddy_log_roll_keep }} + roll_keep_for {{ caddy_log_roll_keep_for }} + } format json } } @@ -281,7 +317,11 @@ redir /.well-known/caldav /remote.php/dav 301 log { - output file /var/log/caddy/cloud.log + output file /var/log/caddy/cloud.log { + roll_size {{ caddy_log_roll_size }} + roll_keep {{ caddy_log_roll_keep }} + roll_keep_for {{ caddy_log_roll_keep_for }} + } format json } } @@ -309,7 +349,11 @@ redir /.well-known/caldav /remote.php/dav 301 log { - output file /var/log/caddy/cloud-skudak.log + output file /var/log/caddy/cloud-skudak.log { + roll_size {{ caddy_log_roll_size }} + roll_keep {{ caddy_log_roll_keep }} + roll_keep_for {{ caddy_log_roll_keep_for }} + } format json } } @@ -323,7 +367,11 @@ } log { - output file /var/log/caddy/gitea-debyl.log + output file /var/log/caddy/gitea-debyl.log { + roll_size {{ caddy_log_roll_size }} + roll_keep {{ caddy_log_roll_keep }} + roll_keep_for {{ caddy_log_roll_keep_for }} + } format json } } @@ -337,7 +385,11 @@ } log { - output file /var/log/caddy/gitea-skudak.log + output file /var/log/caddy/gitea-skudak.log { + roll_size {{ caddy_log_roll_size }} + roll_keep {{ caddy_log_roll_keep }} + roll_keep_for {{ caddy_log_roll_keep_for }} + } format json } } @@ -384,7 +436,11 @@ } log { - output file /var/log/caddy/fulfillr.log + output file /var/log/caddy/fulfillr.log { + roll_size {{ caddy_log_roll_size }} + roll_keep {{ caddy_log_roll_keep }} + roll_keep_for {{ caddy_log_roll_keep_for }} + } format json } } @@ -431,7 +487,11 @@ } log { - output file /var/log/caddy/fulfillr-dev.log + output file /var/log/caddy/fulfillr-dev.log { + roll_size {{ caddy_log_roll_size }} + roll_keep {{ caddy_log_roll_keep }} + roll_keep_for {{ caddy_log_roll_keep_for }} + } format json } } @@ -453,7 +513,11 @@ test.debyl.io { header Pragma "no-cache" log { - output file /var/log/caddy/test.log + output file /var/log/caddy/test.log { + roll_size {{ caddy_log_roll_size }} + roll_keep {{ caddy_log_roll_keep }} + roll_keep_for {{ caddy_log_roll_keep_for }} + } format json level {{ caddy_log_level }} } diff --git a/ansible/roles/podman/templates/podman-prune.service.j2 b/ansible/roles/podman/templates/podman-prune.service.j2 new file mode 100644 index 0000000..a7d8f54 --- /dev/null +++ b/ansible/roles/podman/templates/podman-prune.service.j2 @@ -0,0 +1,15 @@ +[Unit] +Description=Prune unused podman images and volumes +After=network-online.target + +[Service] +Type=oneshot +ExecStart=/usr/local/bin/podman-prune.sh +# Type=oneshot disables the start timeout by default, so a wedged prune would +# leave the unit "activating" forever and every later trigger would be +# silently skipped -- the same trap the backup units guard against. +TimeoutStartSec={{ podman_prune_timeout | default('1h') }} +# Pruning is I/O-heavy and competes with the 03:00-04:30 backups; keep it off +# the critical path. +Nice=15 +IOSchedulingClass=idle diff --git a/ansible/roles/podman/templates/podman-prune.sh.j2 b/ansible/roles/podman/templates/podman-prune.sh.j2 new file mode 100644 index 0000000..9ad1ca2 --- /dev/null +++ b/ansible/roles/podman/templates/podman-prune.sh.j2 @@ -0,0 +1,53 @@ +#!/bin/bash +# {{ ansible_managed }} +# Weekly reclaim of unused podman images and volumes. +# +# Every image bump leaves the previous tag behind and nothing ever removed +# them: this was written after finding 896 images totalling 59.6 GB, 75% of it +# unused -- 94 tags of greg-time-bot and 73 of fulfillr, one per deploy. +# +# --filter until={{ podman_prune_until }} keeps recent images so a rollback +# does not require a rebuild or re-pull. Anything older is unused AND stale. +# +# Volumes pruned here are podman's ANONYMOUS volumes, not the bind mounts +# under {{ podman_volumes }} that hold real service data -- those are +# directories on the host and podman does not know about them. The dangling +# ones seen in practice were 804 MB copies of Nextcloud's /var/www/html left +# by container recreations, which are image content, not data. +set -uo pipefail + +TAG=podman-prune + +log() { logger -t "$TAG" -p daemon.info -- "$*"; echo "$TAG: $*"; } + +total_before=0 +total_after=0 + +for u in {{ podman_prune_users | join(' ') }}; do + # Rootless podman: -H so HOME points at the user's store, and the `cd;` + # preamble is required (see CLAUDE.md) or podman cannot find its graph root. + run() { + sudo -H -u "$u" bash -c \ + 'cd; d=/run/user/$(id -u); [ -d "$d" ] && export XDG_RUNTIME_DIR="$d" + exec podman "$@"' _ "$@" + } + + if ! id "$u" >/dev/null 2>&1; then + log "user=$u status=skipped reason=no-such-user" + continue + fi + + before=$(run system df --format '{{ '{{' }}.Size{{ '}}' }}' 2>/dev/null | head -1) + + # Deliberately NOT `set -e`: a prune failing for one user must not stop the + # other, and a busy image is a normal, non-fatal outcome. + img=$(run image prune -af --filter "until={{ podman_prune_until }}" 2>&1 | tail -1) + vol=$(run volume prune -f 2>&1 | tail -1) + + after=$(run system df --format '{{ '{{' }}.Size{{ '}}' }}' 2>/dev/null | head -1) + + log "user=$u images_before=$before images_after=$after" + log "user=$u image_prune=${img:-none} volume_prune=${vol:-none}" +done + +log "status=ok" diff --git a/ansible/roles/podman/templates/podman-prune.timer.j2 b/ansible/roles/podman/templates/podman-prune.timer.j2 new file mode 100644 index 0000000..e74c66a --- /dev/null +++ b/ansible/roles/podman/templates/podman-prune.timer.j2 @@ -0,0 +1,12 @@ +[Unit] +Description=Weekly podman image and volume prune + +[Timer] +OnCalendar={{ podman_prune_oncalendar | default('Sun *-*-* 02:00:00') }} +RandomizedDelaySec=15m +# Weekly, and growth is one tag per deploy, so a missed run is worth catching +# up on rather than skipping. +Persistent=true + +[Install] +WantedBy=timers.target