graylog updates, test.debyl.io, scripts for reference

2026-01-13 16:08:38 -05:00
parent 364047558c
commit 34b45853e2
12 changed files with 1136 additions and 3 deletions
--- a/ansible/roles/graylog-config/tasks/pipelines.yml
+++ b/ansible/roles/graylog-config/tasks/pipelines.yml
@@ -0,0 +1,188 @@
+---
+# Graylog Pipeline Management via REST API
+# Idempotent: checks for existing pipelines/rules before creating
+
+# =============================================================================
+# Pipeline Rules
+# =============================================================================
+
+- name: get existing pipeline rules
+  ansible.builtin.uri:
+    url: "{{ graylog_api_url }}/system/pipelines/rule"
+    method: GET
+    user: "{{ graylog_api_token }}"
+    password: token
+    force_basic_auth: true
+    headers:
+      X-Requested-By: ansible
+      Accept: application/json
+    status_code: 200
+  register: existing_rules
+  tags: graylog-config, pipelines
+
+- name: build list of existing rule titles
+  ansible.builtin.set_fact:
+    existing_rule_titles: "{{ existing_rules.json | map(attribute='title') | list }}"
+    existing_rule_map: "{{ existing_rules.json | items2dict(key_name='title', value_name='id') }}"
+  tags: graylog-config, pipelines
+
+- name: create pipeline rules
+  ansible.builtin.uri:
+    url: "{{ graylog_api_url }}/system/pipelines/rule"
+    method: POST
+    user: "{{ graylog_api_token }}"
+    password: token
+    force_basic_auth: true
+    headers:
+      X-Requested-By: ansible
+      Content-Type: application/json
+    body_format: json
+    body:
+      title: "{{ item.title }}"
+      description: "{{ item.description | default('') }}"
+      source: "{{ item.source }}"
+    status_code: [200, 201]
+  loop: "{{ graylog_pipeline_rules }}"
+  loop_control:
+    label: "{{ item.title }}"
+  when: item.title not in existing_rule_titles
+  register: created_rules
+  tags: graylog-config, pipelines
+
+- name: refresh rule list after creation
+  ansible.builtin.uri:
+    url: "{{ graylog_api_url }}/system/pipelines/rule"
+    method: GET
+    user: "{{ graylog_api_token }}"
+    password: token
+    force_basic_auth: true
+    headers:
+      X-Requested-By: ansible
+      Accept: application/json
+    status_code: 200
+  register: all_rules
+  tags: graylog-config, pipelines
+
+- name: build rule ID lookup
+  ansible.builtin.set_fact:
+    rule_id_map: "{{ all_rules.json | items2dict(key_name='title', value_name='id') }}"
+  tags: graylog-config, pipelines
+
+# =============================================================================
+# Pipelines
+# =============================================================================
+
+- name: get existing pipelines
+  ansible.builtin.uri:
+    url: "{{ graylog_api_url }}/system/pipelines/pipeline"
+    method: GET
+    user: "{{ graylog_api_token }}"
+    password: token
+    force_basic_auth: true
+    headers:
+      X-Requested-By: ansible
+      Accept: application/json
+    status_code: 200
+  register: existing_pipelines
+  tags: graylog-config, pipelines
+
+- name: build list of existing pipeline titles
+  ansible.builtin.set_fact:
+    existing_pipeline_titles: "{{ existing_pipelines.json | map(attribute='title') | list }}"
+    existing_pipeline_map: "{{ existing_pipelines.json | items2dict(key_name='title', value_name='id') }}"
+  tags: graylog-config, pipelines
+
+- name: build pipeline source for each pipeline
+  ansible.builtin.set_fact:
+    pipeline_sources: "{{ pipeline_sources | default({}) | combine({item.title: lookup('template', 'pipeline_source.j2')}) }}"
+  loop: "{{ graylog_pipelines }}"
+  loop_control:
+    label: "{{ item.title }}"
+  vars:
+    pipeline: "{{ item }}"
+  tags: graylog-config, pipelines
+
+- name: create pipelines
+  ansible.builtin.uri:
+    url: "{{ graylog_api_url }}/system/pipelines/pipeline"
+    method: POST
+    user: "{{ graylog_api_token }}"
+    password: token
+    force_basic_auth: true
+    headers:
+      X-Requested-By: ansible
+      Content-Type: application/json
+    body_format: json
+    body:
+      title: "{{ item.title }}"
+      description: "{{ item.description | default('') }}"
+      source: "{{ pipeline_sources[item.title] }}"
+    status_code: [200, 201]
+  loop: "{{ graylog_pipelines }}"
+  loop_control:
+    label: "{{ item.title }}"
+  when: item.title not in existing_pipeline_titles
+  register: created_pipelines
+  tags: graylog-config, pipelines
+
+- name: refresh pipeline list after creation
+  ansible.builtin.uri:
+    url: "{{ graylog_api_url }}/system/pipelines/pipeline"
+    method: GET
+    user: "{{ graylog_api_token }}"
+    password: token
+    force_basic_auth: true
+    headers:
+      X-Requested-By: ansible
+      Accept: application/json
+    status_code: 200
+  register: all_pipelines
+  tags: graylog-config, pipelines
+
+- name: build pipeline ID lookup
+  ansible.builtin.set_fact:
+    pipeline_id_map: "{{ all_pipelines.json | items2dict(key_name='title', value_name='id') }}"
+  tags: graylog-config, pipelines
+
+# =============================================================================
+# Pipeline to Stream Connections
+# =============================================================================
+
+- name: get current pipeline connections
+  ansible.builtin.uri:
+    url: "{{ graylog_api_url }}/system/pipelines/connections"
+    method: GET
+    user: "{{ graylog_api_token }}"
+    password: token
+    force_basic_auth: true
+    headers:
+      X-Requested-By: ansible
+      Accept: application/json
+    status_code: 200
+  register: current_connections
+  tags: graylog-config, pipelines
+
+- name: connect pipelines to streams
+  ansible.builtin.uri:
+    url: "{{ graylog_api_url }}/system/pipelines/connections/to_stream"
+    method: POST
+    user: "{{ graylog_api_token }}"
+    password: token
+    force_basic_auth: true
+    headers:
+      X-Requested-By: ansible
+      Content-Type: application/json
+    body_format: json
+    body:
+      stream_id: "{{ stream_id_map[item.1] }}"
+      pipeline_ids:
+        - "{{ pipeline_id_map[item.0.pipeline] }}"
+    status_code: [200, 201]
+  loop: "{{ graylog_pipeline_connections | subelements('streams') }}"
+  loop_control:
+    label: "{{ item.0.pipeline }} -> {{ item.1 }}"
+  when:
+    - item.0.pipeline in pipeline_id_map
+    - item.1 in stream_id_map
+  ignore_errors: true
+  tags: graylog-config, pipelines