CU-251akbj added graylog and additional fixes from discovered logs

ansible/deploy_home.yml

@@ -12,6 +12,5 @@
     - role: drone
     - role: hass
     - role: motion
-      tags: motion
     - role: partkeepr
-      tags: partkeepr
+    - role: graylog

ansible/roles/common/defaults/main.yml

@@ -1,18 +1,7 @@
 ---
-deps: [
+deps: [cronie, docker, fail2ban, git, logrotate, python-docker, tmux, weechat]
-  cronie,
-  docker,
-  fail2ban,
-  git,
-  python-docker,
-  tmux,
-  weechat
-]
 
-fail2ban_jails: [
+fail2ban_jails: [sshd.local, nginx.local]
-  sshd.local,
-  nginx.local
-]
 
 services:
   - cronie

ansible/roles/drone/meta/main.yml

@@ -1,3 +1,4 @@
 ---
 dependencies:
   - role: http
+  - role: graylog

ansible/roles/drone/tasks/drone.yml

@@ -8,6 +8,11 @@
     restart: true
     restart_policy: on-failure
     restart_retries: 3
+    log_driver: syslog
+    log_options:
+      syslog-address: "udp://localhost:{{ syslog_udp_default }}"
+      syslog-facility: daemon
+      tag: "docker/{{'{{'}}.Name{{'}}'}}"
     env:
       DRONE_GITHUB_CLIENT_ID: "{{ drone_gh_client_id }}"
       DRONE_GITHUB_CLIENT_SECRET: "{{ drone_gh_client_sec }}"
@@ -31,6 +36,11 @@
     restart: true
     restart_policy: on-failure
     restart_retries: 3
+    log_driver: syslog
+    log_options:
+      syslog-address: "udp://localhost:{{ syslog_udp_default }}"
+      syslog-facility: daemon
+      tag: "docker/{{'{{'}} .Name {{'}}'}}"
     env:
       DRONE_RPC_SECRET: "{{ drone_rpc_secret }}"
       DRONE_RPC_HOST: "{{ ci_server_name }}"

ansible/roles/graylog/meta/main.yml

@@ -0,0 +1,3 @@
+---
+dependencies:
+  - role: common

ansible/roles/graylog/tasks/graylog.yml

@@ -0,0 +1,87 @@
+---
+- name: create graylog docker network
+  community.general.docker_network:
+    name: "graylog"
+  tags: graylog
+
+- name: create graylog required volumes
+  community.general.docker_volume:
+    name: "{{ item }}"
+  with_items:
+    - graylog-db
+    - graylog-es
+    - graylog-conf
+  tags: graylog
+
+- name: create graylog mongodb container
+  community.general.docker_container:
+    name: graylog-mongo
+    image: mongo:4.2
+    recreate: false
+    restart: false
+    restart_policy: on-failure
+    restart_retries: 3
+    networks:
+      - name: "graylog"
+    volumes:
+      - graylog-db:/data/db
+  tags: graylog
+
+- name: create graylog elasticsearch container
+  community.general.docker_container:
+    name: graylog-elastic
+    image: docker.elastic.co/elasticsearch/elasticsearch-oss:7.10.2
+    recreate: false
+    restart: false
+    restart_policy: on-failure
+    restart_retries: 3
+    networks:
+      - name: "graylog"
+    volumes:
+      - graylog-es:/usr/share/elasticsearch/data
+    env:
+      http.host: "0.0.0.0"
+      transport.host: "localhost"
+      network.host: "0.0.0.0"
+      cluster.name: "graylog"
+      ES_JAVA_OPTS: "-Dlog4j2.formatMsgNoLookups=true -Xms512m -Xmx512m"
+    ulimits:
+      - "memlock:-1:-1"
+    memory: 1G
+  tags: graylog
+
+- name: create graylog container
+  community.general.docker_container:
+    name: graylog
+    image: graylog/graylog:4.2
+    recreate: false
+    restart: true
+    restart_policy: on-failure
+    restart_retries: 3
+    networks:
+      - name: "graylog"
+    volumes:
+      - graylog-conf:/usr/share/graylog/data/config
+    env:
+      GRAYLOG_PASSWORD_SECRET: "{{ graylog_secret }}"
+      GRAYLOG_ROOT_PASSWORD_SHA2: "{{ graylog_root_pass_sha2 }}"
+      GRAYLOG_HTTP_EXTERNAL_URI: http://192.168.1.12:9000/
+      GRAYLOG_HTTP_BIND_ADDRESS: 0.0.0.0:9000
+      GRAYLOG_MONGODB_URI: mongodb://graylog-mongo/graylog
+      GRAYLOG_ELASTICSEARCH_HOSTS: http://graylog-elastic:9200
+    ports:
+      # Graylog web interface and REST API
+      - "{{ graylog_port }}:9000"
+      # Syslog TCP
+      #- 1514:1514
+      # Syslog UDP
+      - "0.0.0.0:{{ syslog_udp_default }}:{{ syslog_udp_default }}/udp"
+      # Syslog2 UDP
+      - "0.0.0.0:{{ syslog_udp_unifi }}:{{ syslog_udp_unifi }}/udp"
+      # Syslog2 UDP
+      - "0.0.0.0:{{ syslog_udp_error }}:{{ syslog_udp_error }}/udp"
+      # GELF TCP
+      #- 12201:12201
+      # GELF UDP
+      #- 12201:12201/udp
+  tags: graylog

ansible/roles/graylog/tasks/main.yml

@@ -0,0 +1,2 @@
+---
+- import_tasks: graylog.yml

ansible/roles/hass/meta/main.yml

@@ -1,3 +1,4 @@
 ---
 dependencies:
   - role: http
+  - role: graylog

ansible/roles/hass/tasks/hass.yml

@@ -27,10 +27,15 @@
   community.general.docker_container:
     name: hass
     image: ghcr.io/home-assistant/home-assistant:stable
-    recreate: true
+    recreate: false
     restart: true
     restart_policy: on-failure
     restart_retries: 3
+    log_driver: syslog
+    log_options:
+      syslog-address: "udp://localhost:{{ syslog_udp_default }}"
+      syslog-facility: daemon
+      tag: "docker/{{'{{'}}.Name{{'}}'}}"
     volumes:
       - /var/lib/hass:/config
       - /usr/share/hass:/share

ansible/roles/http/defaults/main.yml

@@ -1,9 +1,5 @@
 ---
-deps: [
+deps: [certbot, nginx, nginx-mod-modsecurity]
-  certbot,
-  nginx,
-  nginx-mod-modsecurity
-]
 
 ci_server_name: ci.bdebyl.net
 pi_server_name: pi.bdebyl.net
@@ -11,15 +7,15 @@ assistant_server_name: assistant.bdebyl.net
 home_server_name: home.bdebyl.net
 parts_server_name: parts.bdebyl.net
 video_server_name: video.bdebyl.net
+logs_server_name: logs.bdebyl.net
 install_path: /usr/share
 
 nginx_path: /etc/nginx
 nginx_conf_path: "{{ nginx_path }}/conf"
+modsec_log_path: /var/log/nginx/modsec_audit.log
 modsec_rules_path: "{{ nginx_conf_path }}/rules"
-modsec_crs_before_rule_conf:
+modsec_crs_before_rule_conf: "{{ modsec_rules_path }}/REQUEST-900-EXCLUSION-RULES-BEFORE-CRS.conf"
-  "{{ modsec_rules_path }}/REQUEST-900-EXCLUSION-RULES-BEFORE-CRS.conf"
+modsec_crs_after_rule_conf: "{{ modsec_rules_path }}/REQUEST-999-EXCLUSION-RULES-AFTER-CRS.conf"
-modsec_crs_after_rule_conf:
-  "{{ modsec_rules_path }}/REQUEST-999-EXCLUSION-RULES-AFTER-CRS.conf"
 modsec_path: "{{ install_path }}/modsecurity"
 crs_path: "{{ install_path }}/coreruleset"
 crs_rules_path: "{{ crs_path }}/rules"
@@ -39,6 +35,12 @@ modsec_git_urls:
     dest: "{{ modsec_path }}"
     ver: "v3.0.6"
 
+modsec_conf_replaces:
+  - regex: "^SecRuleEngine"
+    line: "SecRuleEngine On"
+  - regex: "^SecAuditLog"
+    line: "SecAuditLog {{ modsec_log_path }}"
+
 modsec_conf_links:
   - src: "{{ modsec_path }}/modsecurity.conf-recommended"
     dest: "{{ nginx_path }}/modsecurity.conf"
@@ -46,11 +48,9 @@ modsec_conf_links:
     dest: "{{ nginx_path }}/unicode.mapping"
   - src: "{{ crs_path }}/crs-setup.conf.example"
     dest: "{{ nginx_conf_path }}/crs-setup.conf"
-  - src:
+  - src: "{{ crs_rules_path }}/REQUEST-900-EXCLUSION-RULES-BEFORE-CRS.conf.example"
-      "{{ crs_rules_path }}/REQUEST-900-EXCLUSION-RULES-BEFORE-CRS.conf.example"
     dest: "{{ modsec_rules_path }}/REQUEST-900-EXCLUSION-RULES-BEFORE-CRS.conf"
-  - src:
+  - src: "{{ crs_rules_path }}/RESPONSE-999-EXCLUSION-RULES-AFTER-CRS.conf.example"
-      "{{ crs_rules_path }}/RESPONSE-999-EXCLUSION-RULES-AFTER-CRS.conf.example"
     dest: "{{ modsec_rules_path }}/RESPONSE-999-EXCLUSION-RULES-AFTER-CRS.conf"
 
 crs_rule_links:

ansible/roles/http/files/nginx/nginx.conf

@@ -1,48 +0,0 @@
-user http;
-worker_processes 1;
-
-load_module /usr/lib/nginx/modules/ngx_http_modsecurity_module.so;
-
-error_log /var/log/nginx/error.log notice;
-
-events {
-  worker_connections 1024;
-}
-
-http {
-  map $http_upgrade $connection_upgrade {
-    default upgrade;
-    '' close;
-  }
-
-  include mime.types;
-
-  default_type application/octet-stream;
-
-  log_format main '$remote_addr - $remote_user [$time_local] "$request" '
-                  '$status $body_bytes_sent "$http_referer" '
-                  '"$http_user_agent" "$http_x_forwarded_for"';
-
-  access_log /var/log/nginx/access.log main;
-
-  sendfile        on;
-  server_tokens   off;
-  #tcp_nopush     on;
-
-  keepalive_timeout 65;
-
-  gzip on;
-  gzip_disable "mise6";
-  gzip_min_length 1000;
-  gzip_proxied    expired no-cache no-store private auth;
-  gzip_types      text/plain application/xml application/json application/javascript application/octet-stream text/css;
-
-  # client_body_buffer_size     1k;
-  # client_header_buffer_size   1k;
-  # client_max_body_size        2k;
-  # large_client_header_buffers 2 1k;
-
-  limit_req_zone $binary_remote_addr zone=one:10m rate=10r/s;
-
-  include /etc/nginx/sites-enabled/*.conf;
-}

ansible/roles/http/tasks/http.yml

@@ -1,9 +1,11 @@
 ---
 - name: setup nginx base configuration
   become: true
-  ansible.builtin.copy:
+  ansible.builtin.template:
-    src: files/nginx/nginx.conf
+    src: templates/nginx/nginx.conf.j2
     dest: /etc/nginx/nginx.conf
+    owner: root
+    group: http
     mode: 0644
   notify: restart_nginx
   tags: http
@@ -26,7 +28,7 @@
     state: directory
     owner: http
     group: http
-    mode: 0644
+    mode: 0755
   loop:
     - /srv/http
     - /srv/http/letsencrypt
@@ -38,7 +40,7 @@
     path: /srv/http
     owner: http
     group: http
-    mode: 0644
+    mode: 0755
     recurse: true
   tags: http
 
@@ -55,6 +57,7 @@
     - "{{ assistant_server_name }}.conf"
     - "{{ video_server_name }}.conf"
     - "{{ parts_server_name }}.conf"
+    - "{{ logs_server_name }}.conf"
   notify: restart_nginx
   tags: http
 
@@ -78,6 +81,7 @@
     - "{{ home_server_name }}.conf"
     - "{{ assistant_server_name }}.conf"
     - "{{ video_server_name }}.conf"
+    - "{{ logs_server_name }}.conf"
   notify: restart_nginx
   tags: http
 

ansible/roles/http/tasks/logrotate.yml

@@ -0,0 +1,10 @@
+---
+- name: template nginx log rotation
+  become: true
+  ansible.builtin.template:
+    src: logrotate/nginx.j2
+    dest: /etc/logrotate.d/nginx
+    mode: 0644
+  tags:
+    - http
+    - logrotate

ansible/roles/http/tasks/main.yml

@@ -4,3 +4,4 @@
 - import_tasks: modsec.yml
 - import_tasks: http.yml
 - import_tasks: https.yml
+- import_tasks: logrotate.yml

ansible/roles/http/tasks/modsec.yml

@@ -84,7 +84,8 @@
   become: true
   ansible.builtin.lineinfile:
     path: /etc/nginx/modsecurity.conf
-    regexp: "^SecRuleEngine"
+    regexp: "{{ item.regex }}"
-    line: "SecRuleEngine On"
+    line: "{{ item.line }}"
+  loop: "{{ modsec_conf_replaces }} "
   notify: restart_nginx
   tags: modsec

ansible/roles/http/templates/logrotate/nginx.j2

@@ -0,0 +1,10 @@
+/var/log/nginx/*log {
+        daily
+        rotate 4
+        missingok
+        notifempty
+        create 640 http log
+        compress
+        delaycompress
+        copytruncate
+}

ansible/roles/http/templates/nginx/nginx.conf.j2

@@ -0,0 +1,71 @@
+user http;
+worker_processes 1;
+
+load_module /usr/lib/nginx/modules/ngx_http_modsecurity_module.so;
+
+error_log /var/log/nginx/error.log notice;
+error_log syslog:server=localhost:{{ syslog_udp_error }},tag=nginx,severity=info notice;
+
+events {
+  worker_connections 1024;
+}
+
+http {
+  map $http_upgrade $connection_upgrade {
+    default upgrade;
+    '' close;
+  }
+
+  include mime.types;
+
+  default_type application/octet-stream;
+
+  log_format main '$remote_addr - $connection : $connection_requests [$time_iso8601] "$request" '
+    '$status $body_bytes_sent "$http_referer" '
+    '"$http_user_agent" "$http_x_forwarded_for"';
+  log_format graylog_json escape=json '{ "nginx_timestamp": "[$time_iso8601]", '
+    '"remote_addr": "$remote_addr", '
+      '"connection": "$connection", '
+      '"connection_requests": $connection_requests, '
+      '"body_bytes_sent": $body_bytes_sent, '
+      '"request_length": $request_length, '
+      '"request_time": $request_time, '
+      '"response_status": $status, '
+      '"request": "$request", '
+      '"request_method": "$request_method", '
+      '"host": "$host", '
+      '"upstream_cache_status": "$upstream_cache_status", '
+      '"upstream_addr": "$upstream_addr", '
+      '"http_x_forwarded_for": "$http_x_forwarded_for", '
+      '"http_referrer": "$http_referer", '
+      '"http_user_agent": "$http_user_agent", '
+      '"http_version": "$server_protocol", '
+      '"remote_user": "$remote_user", '
+      '"http_x_forwarded_proto": "$http_x_forwarded_proto", '
+      '"upstream_response_time": "$upstream_response_time", '
+      '"nginx_access": true }';
+
+  access_log /var/log/nginx/access.log main;
+  access_log syslog:server=localhost:{{ syslog_udp_default }},tag=nginx,severity=info graylog_json;
+
+  sendfile        on;
+  server_tokens   off;
+  #tcp_nopush     on;
+
+  keepalive_timeout 65;
+
+  gzip on;
+  gzip_disable "mise6";
+  gzip_min_length 1000;
+  gzip_proxied    expired no-cache no-store private auth;
+  gzip_types      text/plain application/xml application/json application/javascript application/octet-stream text/css;
+
+  # client_body_buffer_size     1k;
+  # client_header_buffer_size   1k;
+  # client_max_body_size        2k;
+  # large_client_header_buffers 2 1k;
+
+  limit_req_zone $binary_remote_addr zone=one:10m rate=10r/s;
+
+  include /etc/nginx/sites-enabled/*.conf;
+}

ansible/roles/http/templates/nginx/sites/ci.bdebyl.net.https.conf.j2

@@ -2,6 +2,11 @@ upstream drone {
   server 127.0.0.1:8080;
 }
 
+geo $local_access {
+  default 0;
+  192.168.1.1 1;
+}
+
 server {
   modsecurity on;
   modsecurity_rules_file {{ nginx_path }}/modsec_includes.conf;
@@ -24,7 +29,12 @@ server {
   ssl_stapling              on;
   ssl_stapling_verify       on;
 
+  resolver 127.0.0.1 127.0.0.53 9.9.9.9 valid=60s;
+
   location / {
+    if ($local_access = 1) {
+      access_log off;
+    }
     add_header Allow                     "GET, POST, HEAD"                                                                                                                                                                        always;
     add_header Content-Security-Policy   "default-src 'self'; img-src 'self' https://*.githubusercontent.com; frame-ancestors 'self'; base-uri 'none',base-uri 'self'; form-action 'self'" always;
     add_header Referrer-Policy           "same-origin"                                                                                                                                                                            always;

ansible/roles/http/templates/nginx/sites/logs.bdebyl.net.conf.j2

@@ -0,0 +1,32 @@
+upstream graylog {
+  server localhost:{{ graylog_port }};
+}
+
+geo $local_access {
+  default 0;
+  192.168.1.0/24 1;
+}
+
+server {
+  modsecurity on;
+  modsecurity_rules_file {{ nginx_path }}/modsec_includes.conf;
+
+  listen 80;
+  server_name {{ logs_server_name }};
+
+  location / {
+    if ($local_access = 1) {
+      access_log off;
+    }
+    allow 192.168.1.0/24;
+    allow 127.0.0.1;
+    deny all;
+
+    proxy_set_header X-Forwarded-For $remote_addr;
+    proxy_set_header Upgrade         $http_upgrade;
+    proxy_set_header Connection      $connection_upgrade;
+
+    proxy_buffering off;
+    proxy_pass http://graylog;
+  }
+}

ansible/roles/http/templates/nginx/sites/parts.bdebyl.net.https.conf.j2

@@ -11,6 +11,8 @@ server {
   modsecurity on;
   modsecurity_rules_file {{ nginx_path }}/modsec_includes.conf;
 
+  resolver 127.0.0.1 127.0.0.53 9.9.9.9 valid=60s;
+
   listen 443 ssl http2;
   listen [::]:443 ssl http2;
   server_name {{ parts_server_name }};

ansible/roles/http/templates/nginx/sites/pi.bdebyl.net.conf.j2

@@ -12,7 +12,7 @@ server {
 	listen [::]:80;
 
 	root /srv/http/pihole;
-	server_name pi.bdebyl.net;
+	server_name {{ pi_server_name }};
 	autoindex off;
 
 	proxy_intercept_errors on;
@@ -21,6 +21,7 @@ server {
 	index pihole/index.php index.php index.html index.htm;
 
 	allow 192.168.1.0/24;
+	allow 127.0.0.1;
 	deny all;
 
 	location / {

ansible/roles/motion/meta/main.yml

@@ -1 +1,4 @@
 ---
+dependencies:
+  - role: http
+  - role: graylog

ansible/roles/motion/tasks/motion.yml

@@ -38,10 +38,15 @@
   community.general.docker_container:
     name: shinobi
     image: migoller/shinobidocker:latest
-    recreate: true
+    recreate: false
     restart: true
     restart_policy: on-failure
     restart_retries: 3
+    log_driver: syslog
+    log_options:
+      syslog-address: "udp://localhost:{{ syslog_udp_default }}"
+      syslog-facility: daemon
+      tag: "docker/{{'{{'}} .Name {{'}}'}}"
     volumes:
       - "shinobi_data:/var/lib/mysql"
       - "/mnt/shinobi:/opt/shinobi/videos"

ansible/roles/partkeepr/meta/main.yml

@@ -1,3 +1,4 @@
 ---
 dependencies:
   - role: http
+  - role: graylog

ansible/roles/partkeepr/tasks/main.yml

@@ -7,10 +7,12 @@
     - partkeepr-conf-vol
     - partkeepr-data-vol
     - partkeepr-db-vol
+  tags: partkeepr
 
 - name: create partkeepr network
   community.general.docker_network:
     name: "partkeepr"
+  tags: partkeepr
 
 - name: create partkeepr-db container
   diff: false
@@ -19,8 +21,13 @@
     image: mariadb:10.0
     recreate: false
     restart: true
-    restart_policy: on-failure
+    restart_policy: unless-stopped
     restart_retries: 3
+    log_driver: syslog
+    log_options:
+      syslog-address: "udp://localhost:{{ syslog_udp_default }}"
+      syslog-facility: daemon
+      tag: "docker/{{'{{'}}.Name{{'}}'}}"
     networks:
       - name: "partkeepr"
     env:
@@ -30,6 +37,7 @@
       MYSQL_PASSWORD: partkeepr
     volumes:
       - partkeepr-db-vol:/var/lib/mysql
+  tags: partkeepr
 
 - name: create partkeepr container
   diff: false
@@ -38,40 +46,13 @@
     image: mhubig/partkeepr:latest
     recreate: false
     restart: true
-    restart_policy: on-failure
+    restart_policy: unless-stopped
-    restart_retries: 3
-    networks:
-      - name: "partkeepr"
-    volumes:
-      - partkeepr-db-conf-vol:/var/www/html/app/config
-      - partkeepr-db-data-vol:/var/www/html/data
-      - partkeepr-db-web-vol:/var/www/html/web
-
-- name: create partkeepr-cron container
-  diff: false
-  community.general.docker_container:
-    name: partkeepr-cron
-    image: mhubig/partkeepr:latest
-    entrypoint: []
-    command: bash -c "crontab /etc/cron.d/partkeepr && cron -f"
-    recreate: false
-    restart: true
-    restart_policy: on-failure
-    restart_retries: 3
-    volumes:
-      - partkeepr-db-conf-vol:/var/www/html/app/config:ro
-      - partkeepr-db-data-vol:/var/www/html/data
-      - partkeepr-db-web-vol:/var/www/html/web
-
-- name: create partkeepr container
-  diff: false
-  community.general.docker_container:
-    name: partkeepr
-    image: mhubig/partkeepr:latest
-    recreate: false
-    restart: true
-    restart_policy: on-failure
     restart_retries: 3
+    log_driver: syslog
+    log_options:
+      syslog-address: "udp://localhost:{{ syslog_udp_default }}"
+      syslog-facility: daemon
+      tag: "docker/{{'{{'}}.Name{{'}}'}}"
     networks:
       - name: "partkeepr"
     ports:
@@ -80,3 +61,27 @@
       - partkeepr-db-conf-vol:/var/www/html/app/config
       - partkeepr-db-data-vol:/var/www/html/data
       - partkeepr-db-web-vol:/var/www/html/web
+  tags: partkeepr
+
+- name: create partkeepr-cron container
+  diff: false
+  community.general.docker_container:
+    name: partkeepr-cron
+    image: mhubig/partkeepr:latest
+    command_handling: correct
+    entrypoint: []
+    command: bash -c "crontab /etc/cron.d/partkeepr && cron -f"
+    recreate: false
+    restart: true
+    restart_policy: unless-stopped
+    restart_retries: 3
+    log_driver: syslog
+    log_options:
+      syslog-address: "udp://localhost:{{ syslog_udp_default }}"
+      syslog-facility: daemon
+      tag: "docker/{{'{{'}}.Name{{'}}'}}"
+    volumes:
+      - partkeepr-db-conf-vol:/var/www/html/app/config:ro
+      - partkeepr-db-data-vol:/var/www/html/data
+      - partkeepr-db-web-vol:/var/www/html/web
+  tags: partkeepr