graylog updates, test.debyl.io, scripts for reference

ansible/roles/graylog-config/defaults/main.yml

@@ -0,0 +1,170 @@
+---
+# Graylog API Configuration
+graylog_api_url: "https://logs.debyl.io/api"
+# graylog_api_token: defined in vault
+
+# Default index set for new streams (Default Stream index set)
+graylog_default_index_set: "6955a9d3cc3f442e78805871"
+
+# Stream definitions
+graylog_streams:
+  - title: "debyltech-api"
+    description: "Lambda API events from debyltech-api service"
+    rules:
+      - field: "service"
+        value: "debyltech-api"
+        type: 1  # EXACT match
+        inverted: false
+
+  - title: "caddy-access"
+    description: "Web traffic access logs from Caddy"
+    rules:
+      - field: "source"
+        value: "caddy"
+        type: 1
+        inverted: false
+      - field: "log_type"
+        value: "access"
+        type: 1
+        inverted: false
+
+  - title: "caddy-fulfillr"
+    description: "Fulfillr-specific web traffic"
+    rules:
+      - field: "source"
+        value: "caddy"
+        type: 1
+        inverted: false
+      - field: "tag"
+        value: "caddy.fulfillr"
+        type: 1
+        inverted: false
+
+  - title: "ssh-security"
+    description: "SSH access and security logs"
+    rules:
+      - field: "source"
+        value: "sshd"
+        type: 1
+        inverted: false
+
+  - title: "container-logs"
+    description: "Container stdout/stderr from Podman"
+    rules:
+      - field: "source"
+        value: "podman"
+        type: 1
+        inverted: false
+
+# Pipeline definitions
+graylog_pipelines:
+  - title: "GeoIP Enrichment"
+    description: "Add geographic information to access logs"
+    stages:
+      - stage: 0
+        match: "EITHER"
+        rules:
+          - "geoip_caddy_access"
+
+  - title: "Debyltech Event Classification"
+    description: "Categorize debyltech-api events"
+    stages:
+      - stage: 0
+        match: "EITHER"
+        rules:
+          - "classify_order_events"
+          - "classify_review_events"
+          - "classify_backinstock_events"
+          - "classify_shipping_events"
+          - "classify_product_events"
+      - stage: 1
+        match: "EITHER"
+        rules:
+          - "classify_default_events"
+
+# Pipeline rule definitions
+graylog_pipeline_rules:
+  - title: "geoip_caddy_access"
+    description: "GeoIP lookup for Caddy access logs"
+    source: |
+      rule "GeoIP for Caddy Access"
+      when
+        has_field("request_remote_ip")
+      then
+        let ip = to_string($message.request_remote_ip);
+        let geo = lookup("geoip-lookup", ip);
+        set_field("geo_country", geo["country"].iso_code);
+        set_field("geo_city", geo["city"].names.en);
+        set_field("geo_coordinates", geo["coordinates"]);
+      end
+
+  - title: "classify_order_events"
+    description: "Classify order events"
+    source: |
+      rule "Classify order events"
+      when
+        has_field("event") AND contains(to_string($message.event), "order")
+      then
+        set_field("event_category", "order");
+      end
+
+  - title: "classify_review_events"
+    description: "Classify review events"
+    source: |
+      rule "Classify review events"
+      when
+        has_field("event") AND contains(to_string($message.event), "review")
+      then
+        set_field("event_category", "review");
+      end
+
+  - title: "classify_backinstock_events"
+    description: "Classify back-in-stock events"
+    source: |
+      rule "Classify back-in-stock events"
+      when
+        has_field("event") AND contains(to_string($message.event), "backinstock")
+      then
+        set_field("event_category", "backinstock");
+      end
+
+  - title: "classify_shipping_events"
+    description: "Classify shipping events"
+    source: |
+      rule "Classify shipping events"
+      when
+        has_field("event") AND contains(to_string($message.event), "shipping")
+      then
+        set_field("event_category", "shipping");
+      end
+
+  - title: "classify_product_events"
+    description: "Classify product events"
+    source: |
+      rule "Classify product events"
+      when
+        has_field("event") AND contains(to_string($message.event), "product")
+      then
+        set_field("event_category", "product");
+      end
+
+  - title: "classify_default_events"
+    description: "Default category for unclassified events"
+    source: |
+      rule "Classify default events"
+      when
+        has_field("event") AND NOT has_field("event_category")
+      then
+        set_field("event_category", "other");
+      end
+
+# Pipeline to stream connections
+graylog_pipeline_connections:
+  - pipeline: "GeoIP Enrichment"
+    streams:
+      - "caddy-access"
+      - "caddy-fulfillr"
+
+  - pipeline: "Debyltech Event Classification"
+    streams:
+      - "debyltech-api"