Bastian de Byl
f11391b28f
Caddy was rotating on implicit defaults (100MiB/keep 10/90d) that were not holding -- 20 rotated files per stream and a 190-day-old .gz, 1.3 GB across 16 log streams. Made explicit at 10MiB/keep 3/7d. Note roll_size et al are subdirectives of `output file`, NOT of `log`. Getting that wrong does not degrade gracefully: Caddy refuses to start on a bad config, so every site went down until it was corrected. Worth a `caddy validate` gate before reload. journald had no SystemMaxUse and had reached 4 GB, drifting toward its 10%-of-filesystem default (~190 GB on this root). Capped at 500M. Both are safe to keep short because fluent-bit ships the journal and every Caddy access log into Graylog -- though note its GELF output has been erroring for days, which weakens that premise and wants investigating. The larger find was unrelated to logs: 896 images totalling 59.6 GB with 75% unused (94 tags of greg-time-bot, 73 of fulfillr -- one per deploy) and 5.4 GB of dangling volumes, mostly 804 MB Nextcloud /var/www/html trees orphaned by container recreations. Pruned to 22 images / 15.4 GB, and added a weekly timer keeping 30 days so a rollback still needs no rebuild. Also dropped the decommissioned 6379/tcp redis rule (nothing listening; Immich's redis is on the shared podman network) and the orphaned nosql, s3 and searxng volume dirs. Backup log exclusions turned out to be unnecessary: Gitea logs to console so its log dirs are empty, Nextcloud already excludes its own, BookStack mounts only uploads, and Caddy is not backed up. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
240 lines
8.6 KiB
YAML
240 lines
8.6 KiB
YAML
---
|
|
- import_tasks: firewall.yml
|
|
- import_tasks: podman/podman.yml
|
|
|
|
- import_tasks: podman/podman-prune.yml
|
|
tags: podman-prune
|
|
|
|
# WEB SERVER: Caddy is the default and only web server
|
|
# nginx has been completely replaced and removed
|
|
|
|
# ===== WEB SERVER CONFIGURATION =====
|
|
# Caddy is the default web server
|
|
- import_tasks: containers/base/conf-caddy.yml
|
|
tags:
|
|
- caddy
|
|
- web
|
|
|
|
- import_tasks: containers/base/caddy.yml
|
|
vars:
|
|
image: docker.io/library/caddy:2.11.2
|
|
tags:
|
|
- caddy
|
|
- web
|
|
|
|
# nginx cleanup completed - infrastructure removed
|
|
|
|
|
|
- import_tasks: containers/base/awsddns.yml
|
|
vars:
|
|
image: docker.io/bdebyl/awsddns:1.0.34
|
|
tags: ddns
|
|
|
|
# Drone CI infrastructure completely removed
|
|
|
|
- import_tasks: containers/home/hass.yml
|
|
vars:
|
|
image: ghcr.io/home-assistant/home-assistant:2026.5.1
|
|
tags: hass
|
|
|
|
- import_tasks: containers/home/partsy.yml
|
|
vars:
|
|
image: "git.debyl.io/debyltech/partsy:latest"
|
|
tags: partsy
|
|
|
|
- import_tasks: containers/skudak/partsy.yml
|
|
vars:
|
|
image: "git.debyl.io/debyltech/partsy:latest"
|
|
tags: skudak, partsy-skudak
|
|
|
|
- import_tasks: containers/skudak/wiki.yml
|
|
vars:
|
|
db_image: docker.io/library/mysql:5.7.21
|
|
image: docker.io/solidnerd/bookstack:26.3.4
|
|
tags: skudak, skudak-wiki
|
|
|
|
- import_tasks: containers/home/photos.yml
|
|
vars:
|
|
db_image: ghcr.io/immich-app/postgres:14-vectorchord0.4.3-pgvectors0.2.0@sha256:bcf63357191b76a916ae5eb93464d65c07511da41e3bf7a8416db519b40b1c23
|
|
ml_image: ghcr.io/immich-app/immich-machine-learning:v3.0.0
|
|
redis_image: docker.io/redis:6.2-alpine@sha256:eaba718fecd1196d88533de7ba49bf903ad33664a92debb24660a922ecd9cac8
|
|
image: ghcr.io/immich-app/immich-server:v3.0.0
|
|
tags: photos
|
|
|
|
- import_tasks: containers/home/cloud.yml
|
|
vars:
|
|
db_image: docker.io/library/mariadb:10.6
|
|
image: docker.io/library/nextcloud:34.0.2-apache
|
|
tags: cloud
|
|
|
|
- import_tasks: containers/skudak/cloud.yml
|
|
vars:
|
|
db_image: docker.io/library/mariadb:10.6
|
|
image: docker.io/library/nextcloud:34.0.2-apache
|
|
tags: skudak, skudak-cloud
|
|
|
|
- import_tasks: containers/debyltech/fulfillr.yml
|
|
vars:
|
|
image: git.debyl.io/debyltech/fulfillr:20260728.2155
|
|
tags: debyltech, fulfillr
|
|
|
|
# Staging back-office (fulfillr-dev.debyltech.com) — same image, staging Turso config.
|
|
- import_tasks: containers/debyltech/fulfillr-dev.yml
|
|
vars:
|
|
image: git.debyl.io/debyltech/fulfillr:20260728.2155
|
|
tags: debyltech, fulfillr-dev
|
|
|
|
- import_tasks: containers/debyltech/uptime-kuma.yml
|
|
vars:
|
|
image: docker.io/louislam/uptime-kuma:2.3.2
|
|
tags: debyltech, uptime-debyltech
|
|
|
|
- import_tasks: containers/home/uptime-kuma.yml
|
|
vars:
|
|
image: docker.io/louislam/uptime-kuma:2.3.2
|
|
tags: home, uptime
|
|
|
|
- import_tasks: data/geoip.yml
|
|
tags: graylog, geoip
|
|
|
|
- import_tasks: containers/debyltech/graylog.yml
|
|
vars:
|
|
mongo_image: docker.io/mongo:7.0
|
|
opensearch_image: docker.io/opensearchproject/opensearch:2
|
|
image: docker.io/graylog/graylog:7.0.1
|
|
tags: debyltech, graylog
|
|
|
|
- import_tasks: containers/home/gregtime.yml
|
|
vars:
|
|
image: localhost/greg-time-bot:3.10.0
|
|
tags: gregtime
|
|
|
|
- import_tasks: containers/home/zomboid.yml
|
|
vars:
|
|
image: docker.io/cm2network/steamcmd:root
|
|
tags: zomboid
|
|
|
|
# ---------------------------------------------------------- Gitea backups
|
|
# The Gitea pods themselves are owned by roles/git, but the backup machinery
|
|
# (containers/cloud-backup.yml plus templates/nextcloud/*) lives here, and an
|
|
# include_tasks reaching across roles would need a path outside the role. So
|
|
# the two Gitea backup instances are wired here alongside the container ones.
|
|
#
|
|
# Both run as ROOTLESS podman under "{{ git_user }}", not "{{ podman_user }}"
|
|
# -- hence backup_podman_user -- and use PostgreSQL rather than MariaDB.
|
|
#
|
|
# Scheduled ahead of the 04:00/04:30 Nextcloud runs so everything lands before
|
|
# the 05:00 TrueNAS snapshot.
|
|
# `apply` is required: tags on a dynamic include_tasks select whether the
|
|
# include runs, but do NOT propagate to the tasks inside it, so without this
|
|
# `make deploy TAGS=gitea-backup` includes the file and then filters out every
|
|
# task in it. The Nextcloud instances avoid this only because they are reached
|
|
# through a static import_tasks chain that tags their children at parse time.
|
|
- include_tasks:
|
|
file: containers/cloud-backup.yml
|
|
apply:
|
|
tags: gitea-backup
|
|
vars:
|
|
backup_name: gitea-debyl
|
|
backup_product: Gitea
|
|
backup_podman_user: "{{ git_user }}"
|
|
data_path: "{{ git_home }}/volumes/gitea/data"
|
|
db_container: gitea-debyl-postgres
|
|
backup_db_type: postgres
|
|
ssh_key_path: /etc/ssh/backup_keys/gitea
|
|
ssh_key_content: "{{ gitea_backup_ssh_key }}"
|
|
ssh_user: gitea
|
|
remote_path: /mnt/glacier/gitea
|
|
script_path: /usr/local/bin/gitea-backup.sh
|
|
# actions_log/artifacts are CI churn (510 MB and growing) and rebuildable;
|
|
# indexers/queues/tmp are derived state Gitea recreates on start. Note the
|
|
# default excludes are Nextcloud-specific, so this must be set explicitly.
|
|
backup_rsync_excludes: >-
|
|
--exclude '/gitea/actions_log' --exclude '/gitea/actions_artifacts'
|
|
--exclude '/gitea/tmp' --exclude '/gitea/indexers' --exclude '/gitea/queues'
|
|
backup_oncalendar: "*-*-* 03:30:00"
|
|
tags: gitea-backup
|
|
|
|
# ------------------------------------------------- Skudak app-data backups
|
|
# BookStack (wiki.skudak.com) and partsy-skudak are BUSINESS data. Both share
|
|
# one TrueNAS dataset (/mnt/glacier/skudakapps) so they need only one backup
|
|
# user, key and cloud-sync task between them; the personal "iDrive E2 Backup"
|
|
# task excludes /skudakapps/** and Skudak's own task pushes it to backup-all.
|
|
#
|
|
# Scheduled ahead of the 03:30+ Gitea/Nextcloud jobs and the 05:00 snapshot.
|
|
- include_tasks:
|
|
file: containers/cloud-backup.yml
|
|
apply:
|
|
tags: [skudak, skudak-apps-backup]
|
|
vars:
|
|
backup_name: bookstack
|
|
backup_product: BookStack
|
|
data_path: "{{ bookstack_path }}"
|
|
db_container: bookstack-db
|
|
# mysql:5.7 predates the mariadb-dump alias -- see cloud-backup.sh.j2.
|
|
backup_db_type: mysql
|
|
ssh_key_path: /etc/ssh/backup_keys/skudakapps
|
|
ssh_key_content: "{{ skudakapps_backup_ssh_key }}"
|
|
ssh_user: skudakapps
|
|
remote_path: /mnt/glacier/skudakapps/bookstack
|
|
script_path: /usr/local/bin/bookstack-backup.sh
|
|
# The wiki content is the DATABASE; /mysql is its raw datadir, which must
|
|
# not be rsynced live -- the dump above is the consistent copy. public/
|
|
# and storage/ hold the uploads and are the only file trees worth shipping.
|
|
backup_rsync_excludes: "--exclude '/mysql'"
|
|
backup_oncalendar: "*-*-* 03:00:00"
|
|
tags: skudak, skudak-apps-backup
|
|
|
|
- include_tasks:
|
|
file: containers/cloud-backup.yml
|
|
apply:
|
|
tags: [skudak, skudak-apps-backup]
|
|
vars:
|
|
backup_name: partsy-skudak
|
|
backup_product: Partsy
|
|
data_path: "{{ partsy_skudak_path }}"
|
|
# Live WAL-mode SQLite: snapshotted via `sqlite3 .backup` rather than
|
|
# rsynced, so the -wal/-shm sidecars are deliberately excluded from the
|
|
# file tree -- shipping them alongside a separately-taken snapshot would
|
|
# only invite a confusing restore.
|
|
backup_sqlite_dbs:
|
|
- "{{ partsy_skudak_path }}/data/partsy.db"
|
|
backup_rsync_excludes: "--exclude '*-wal' --exclude '*-shm'"
|
|
ssh_key_path: /etc/ssh/backup_keys/skudakapps
|
|
ssh_key_content: "{{ skudakapps_backup_ssh_key }}"
|
|
ssh_user: skudakapps
|
|
remote_path: /mnt/glacier/skudakapps/partsy-skudak
|
|
script_path: /usr/local/bin/partsy-skudak-backup.sh
|
|
backup_oncalendar: "*-*-* 03:10:00"
|
|
tags: skudak, skudak-apps-backup
|
|
|
|
# BUSINESS data. Rsynced to TrueNAS here, then pushed offsite to SKUDAK'S OWN
|
|
# iDrive e2 account (bucket `backup-all`) by TrueNAS cloud-sync task "Skudak
|
|
# iDrive - Gitea" (id 9, /mnt/glacier/skudakgit -> /skudakgit, daily 06:30).
|
|
#
|
|
# The personal "iDrive E2 Backup" task's /skudakgit/** exclude is PERMANENT:
|
|
# it is what keeps business data out of personal storage. Do not remove it --
|
|
# Skudak has its own task and bucket instead.
|
|
- include_tasks:
|
|
file: containers/cloud-backup.yml
|
|
apply:
|
|
tags: [skudak, gitea-backup-skudak]
|
|
vars:
|
|
backup_name: skudak-gitea
|
|
backup_product: Gitea
|
|
backup_podman_user: "{{ git_user }}"
|
|
data_path: "{{ git_home }}/volumes/gitea-skudak/data"
|
|
db_container: gitea-skudak-postgres
|
|
backup_db_type: postgres
|
|
ssh_key_path: /etc/ssh/backup_keys/skudak-gitea
|
|
ssh_key_content: "{{ skudakgit_backup_ssh_key }}"
|
|
ssh_user: skudakgit
|
|
remote_path: /mnt/glacier/skudakgit
|
|
script_path: /usr/local/bin/skudak-gitea-backup.sh
|
|
backup_rsync_excludes: >-
|
|
--exclude '/gitea/actions_log' --exclude '/gitea/actions_artifacts'
|
|
--exclude '/gitea/tmp' --exclude '/gitea/indexers' --exclude '/gitea/queues'
|
|
backup_oncalendar: "*-*-* 03:45:00"
|
|
tags: skudak, gitea-backup-skudak
|
|
|