diff --git a/ansible/roles/git/tasks/gitea.yml b/ansible/roles/git/tasks/gitea.yml index 601b297..b0cc991 100644 --- a/ansible/roles/git/tasks/gitea.yml +++ b/ansible/roles/git/tasks/gitea.yml @@ -59,6 +59,10 @@ GITEA__security__INSTALL_LOCK: "true" GITEA__service__DISABLE_REGISTRATION: "true" GITEA__service__REQUIRE_SIGNIN_VIEW: "false" + # Logging configuration - output to journald for fluent-bit capture + GITEA__log__MODE: console + GITEA__log__LEVEL: Info + GITEA__log__ENABLE_ACCESS_LOG: "true" volumes: - "{{ git_home }}/volumes/gitea/data:/data" - /etc/localtime:/etc/localtime:ro diff --git a/ansible/roles/podman/defaults/main.yml b/ansible/roles/podman/defaults/main.yml index c66759a..bafaf42 100644 --- a/ansible/roles/podman/defaults/main.yml +++ b/ansible/roles/podman/defaults/main.yml @@ -10,7 +10,7 @@ fulfillr_path: "{{ podman_volumes }}/fulfillr" gregtime_path: "{{ podman_volumes }}/gregtime" hass_path: "{{ podman_volumes }}/hass" # nginx_path: removed - nginx no longer used -nosql_path: "{{ podman_volumes }}/nosql" +# nosql_path: removed - nosql/redis no longer used partkeepr_path: "{{ podman_volumes }}/partkeepr" partsy_path: "{{ podman_volumes }}/partsy" photos_path: "{{ podman_volumes }}/photos" @@ -112,3 +112,25 @@ logs_server_name: logs.debyl.io # Fluent Bit is deployed as a systemd service (not container) # for direct journal access - see containers/base/fluent-bit.yml + +# Fluent-bit Caddy log forwarding +caddy_log_path: "{{ caddy_path }}/logs" +caddy_log_names: + - caddy + - photos + - wiki + - assistant + - parts + - uptime-kuma + - graylog + - cloud + - cloud-skudak + - gitea-debyl + - fulfillr + +# GeoIP configuration for Graylog +# Requires free MaxMind account: https://dev.maxmind.com/geoip/geolite2-free-geolocation-data +geoip_path: "{{ graylog_path }}/geoip" +geoip_database_edition: GeoLite2-City +# geoip_maxmind_account_id: defined in vault +# geoip_maxmind_license_key: defined in vault diff --git a/ansible/roles/podman/tasks/containers/base/fluent-bit.yml b/ansible/roles/podman/tasks/containers/base/fluent-bit.yml index 8cf2a09..bb81659 100644 --- a/ansible/roles/podman/tasks/containers/base/fluent-bit.yml +++ b/ansible/roles/podman/tasks/containers/base/fluent-bit.yml @@ -27,6 +27,25 @@ name: fluent-bit state: present +- name: create fluent-bit state directory for tail db files + become: true + ansible.builtin.file: + path: /var/lib/fluent-bit + state: directory + owner: root + group: root + mode: '0755' + +- name: deploy fluent-bit parsers configuration + become: true + ansible.builtin.template: + src: fluent-bit/parsers.conf.j2 + dest: /etc/fluent-bit/parsers.conf + owner: root + group: root + mode: '0644' + notify: restart fluent-bit + - name: deploy fluent-bit configuration become: true ansible.builtin.template: diff --git a/ansible/roles/podman/tasks/containers/debyltech/geoip.yml b/ansible/roles/podman/tasks/containers/debyltech/geoip.yml new file mode 100644 index 0000000..023e285 --- /dev/null +++ b/ansible/roles/podman/tasks/containers/debyltech/geoip.yml @@ -0,0 +1,59 @@ +--- +# Download MaxMind GeoLite2 database for Graylog GeoIP enrichment +# Requires free MaxMind account: https://dev.maxmind.com/geoip/geolite2-free-geolocation-data + +- name: create geoip directory + become: true + ansible.builtin.file: + path: "{{ geoip_path }}" + state: directory + owner: "{{ podman_subuid.stdout }}" + group: "{{ podman_subuid.stdout }}" + mode: '0755' + notify: restorecon podman + tags: graylog, geoip + +- name: download GeoLite2 database + become: true + ansible.builtin.get_url: + url: "https://download.maxmind.com/geoip/databases/{{ geoip_database_edition }}/download?suffix=tar.gz" + dest: "{{ geoip_path }}/{{ geoip_database_edition }}.tar.gz" + url_username: "{{ geoip_maxmind_account_id }}" + url_password: "{{ geoip_maxmind_license_key }}" + force: false + mode: '0644' + register: geoip_download + tags: graylog, geoip + +- name: extract GeoLite2 database + become: true + ansible.builtin.unarchive: + src: "{{ geoip_path }}/{{ geoip_database_edition }}.tar.gz" + dest: "{{ geoip_path }}" + remote_src: true + extra_opts: + - --strip-components=1 + - --wildcards + - "*/{{ geoip_database_edition }}.mmdb" + when: geoip_download.changed + tags: graylog, geoip + +# Fix ownership of downloaded files to podman user's subuid range +- name: fix geoip files ownership for podman user + become: true + ansible.builtin.file: + path: "{{ geoip_path }}" + state: directory + owner: "{{ podman_subuid.stdout }}" + group: "{{ podman_subuid.stdout }}" + recurse: true + tags: graylog, geoip + +# Graylog runs as UID 1100 inside the container +- name: fix geoip database ownership for graylog container + become: true + become_user: "{{ podman_user }}" + changed_when: false + ansible.builtin.command: | + podman unshare chown -R 1100:1100 {{ geoip_path }} + tags: graylog, geoip diff --git a/ansible/roles/podman/tasks/containers/debyltech/graylog.yml b/ansible/roles/podman/tasks/containers/debyltech/graylog.yml index a149833..043bde0 100644 --- a/ansible/roles/podman/tasks/containers/debyltech/graylog.yml +++ b/ansible/roles/podman/tasks/containers/debyltech/graylog.yml @@ -159,6 +159,7 @@ GRAYLOG_MONGODB_URI: "mongodb://127.0.0.1:27017/graylog" volumes: - "{{ graylog_path }}/graylog/data:/usr/share/graylog/data:z" + - "{{ geoip_path }}/{{ geoip_database_edition }}.mmdb:/etc/graylog/server/GeoLite2-City.mmdb:ro" requires: - graylog-mongo - graylog-opensearch diff --git a/ansible/roles/podman/tasks/containers/home/nosql.yml b/ansible/roles/podman/tasks/containers/home/nosql.yml deleted file mode 100644 index d58bfef..0000000 --- a/ansible/roles/podman/tasks/containers/home/nosql.yml +++ /dev/null @@ -1,44 +0,0 @@ ---- -- name: create nosql host directory volumes - become: true - ansible.builtin.file: - path: "{{ item }}" - state: directory - owner: "{{ podman_user }}" - group: "{{ podman_user }}" - mode: 0755 - notify: restorecon podman - loop: - - "{{ nosql_path }}/conf" - - "{{ nosql_path }}/data" - -- name: flush handlers - ansible.builtin.meta: flush_handlers - -- import_tasks: podman/podman-check.yml - vars: - container_name: nosql - container_image: "{{ image }}" - -- name: create nosql container - become: true - become_user: "{{ podman_user }}" - containers.podman.podman_container: - name: nosql - image: "{{ image }}" - command: redis-server --requirepass {{ nosql_password }} - restart_policy: on-failure:3 - log_driver: journald - volumes: - - "{{ nosql_path }}/conf:/usr/local/etc/redis/" - - "{{ nosql_path }}/data:/var/lib/redis" - env: - TZ: America/New_York - REDIS_REPLICATION_MODE: master - ports: - - 6379:6379/tcp - -- name: create systemd startup job for nosql - include_tasks: podman/systemd-generate.yml - vars: - container_name: nosql diff --git a/ansible/roles/podman/tasks/main.yml b/ansible/roles/podman/tasks/main.yml index 21390a3..9074a2f 100644 --- a/ansible/roles/podman/tasks/main.yml +++ b/ansible/roles/podman/tasks/main.yml @@ -73,7 +73,7 @@ - import_tasks: containers/debyltech/fulfillr.yml vars: - image: "git.debyl.io/debyltech/fulfillr:20260104.0001" + image: "git.debyl.io/debyltech/fulfillr:20260109.0522" tags: debyltech, fulfillr - import_tasks: containers/debyltech/uptime-kuma.yml @@ -81,17 +81,15 @@ image: docker.io/louislam/uptime-kuma:1 tags: debyltech, uptime-kuma +- import_tasks: containers/debyltech/geoip.yml + tags: debyltech, graylog, geoip + - import_tasks: containers/debyltech/graylog.yml tags: debyltech, graylog - import_tasks: containers/base/fluent-bit.yml tags: fluent-bit, graylog -- import_tasks: containers/home/nosql.yml - vars: - image: docker.io/redis:7.2.1-alpine - tags: nosql - - import_tasks: containers/home/gregtime.yml vars: image: localhost/greg-time-bot:1.5.2 diff --git a/ansible/roles/podman/templates/fluent-bit/fluent-bit.conf.j2 b/ansible/roles/podman/templates/fluent-bit/fluent-bit.conf.j2 index 165458e..77b902f 100644 --- a/ansible/roles/podman/templates/fluent-bit/fluent-bit.conf.j2 +++ b/ansible/roles/podman/templates/fluent-bit/fluent-bit.conf.j2 @@ -4,27 +4,72 @@ Log_Level info Parsers_File parsers.conf -# Read from systemd journal - filter for Podman container logs +# ============================================================================= +# INPUT: Podman container logs +# ============================================================================= # Container logs come from conmon process with CONTAINER_NAME field [INPUT] Name systemd - Tag journal.* + Tag podman.* Systemd_Filter _COMM=conmon Read_From_Tail On Strip_Underscores On -# Extract container name for better filtering in Graylog +# ============================================================================= +# INPUT: SSH logs for security monitoring +# ============================================================================= +[INPUT] + Name systemd + Tag ssh.* + Systemd_Filter _SYSTEMD_UNIT=sshd.service + Read_From_Tail On + Strip_Underscores On + +# ============================================================================= +# INPUT: Caddy access logs (JSON format) +# ============================================================================= +{% for log_name in caddy_log_names %} +[INPUT] + Name tail + Tag caddy.{{ log_name }} + Path {{ caddy_log_path }}/{{ log_name }}.log + Parser caddy_json + Read_From_Head False + Refresh_Interval 5 + DB /var/lib/fluent-bit/caddy_{{ log_name }}.db + +{% endfor %} +# ============================================================================= +# FILTERS: Add metadata for Graylog categorization +# ============================================================================= [FILTER] Name record_modifier - Match journal.* + Match podman.* Record host {{ ansible_hostname }} Record source podman + Record log_type container -# Output to Graylog GELF UDP (local, port 12203) -# Graylog needs a GELF UDP input configured on this port +[FILTER] + Name record_modifier + Match ssh.* + Record host {{ ansible_hostname }} + Record source sshd + Record log_type security + +[FILTER] + Name record_modifier + Match caddy.* + Record host {{ ansible_hostname }} + Record source caddy + Record log_type access + +# ============================================================================= +# OUTPUT: All logs to Graylog GELF UDP +# ============================================================================= +# Graylog needs a GELF UDP input configured on port 12203 [OUTPUT] Name gelf - Match journal.* + Match * Host 127.0.0.1 Port 12203 Mode udp diff --git a/ansible/roles/podman/templates/fluent-bit/parsers.conf.j2 b/ansible/roles/podman/templates/fluent-bit/parsers.conf.j2 new file mode 100644 index 0000000..bd8919a --- /dev/null +++ b/ansible/roles/podman/templates/fluent-bit/parsers.conf.j2 @@ -0,0 +1,5 @@ +[PARSER] + Name caddy_json + Format json + Time_Key ts + Time_Format %s.%L diff --git a/ansible/vars/vault.yml b/ansible/vars/vault.yml index 19e4a43..6133083 100644 Binary files a/ansible/vars/vault.yml and b/ansible/vars/vault.yml differ