bastian/deploy_home
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

CU-251akbj added graylog and additional fixes from discovered logs

Browse Source
This commit is contained in:
Bastian de Byl
2022-04-18 03:15:52 -04:00
parent f6af7e0bb1
commit 2360c82f98
27 changed files with 324 additions and 119 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
ansible/deploy_home.yml
View File

@@ -12,6 +12,5 @@
- role: drone
- role: hass
- role: motion
tags: motion
- role: partkeepr
tags: partkeepr
- role: graylog

15
ansible/roles/common/defaults/main.yml
View File

@@ -1,18 +1,7 @@
---
deps: [
cronie,
docker,
fail2ban,
git,
python-docker,
tmux,
weechat
]
deps: [cronie, docker, fail2ban, git, logrotate, python-docker, tmux, weechat]
fail2ban_jails: [
sshd.local,
nginx.local
]
fail2ban_jails: [sshd.local, nginx.local]
services:
- cronie

1
ansible/roles/drone/meta/main.yml
View File

@@ -1,3 +1,4 @@
---
dependencies:
- role: http
- role: graylog

10
ansible/roles/drone/tasks/drone.yml
View File

@@ -8,6 +8,11 @@
restart: true
restart_policy: on-failure
restart_retries: 3
log_driver: syslog
log_options:
syslog-address: "udp://localhost:{{ syslog_udp_default }}"
syslog-facility: daemon
tag: "docker/{{'{{'}}.Name{{'}}'}}"
env:
DRONE_GITHUB_CLIENT_ID: "{{ drone_gh_client_id }}"
DRONE_GITHUB_CLIENT_SECRET: "{{ drone_gh_client_sec }}"
@@ -31,6 +36,11 @@
restart: true
restart_policy: on-failure
restart_retries: 3
log_driver: syslog
log_options:
syslog-address: "udp://localhost:{{ syslog_udp_default }}"
syslog-facility: daemon
tag: "docker/{{'{{'}} .Name {{'}}'}}"
env:
DRONE_RPC_SECRET: "{{ drone_rpc_secret }}"
DRONE_RPC_HOST: "{{ ci_server_name }}"

3
ansible/roles/graylog/meta/main.yml Normal file
View File

@@ -0,0 +1,3 @@
---
dependencies:
- role: common

87
ansible/roles/graylog/tasks/graylog.yml Normal file
View File

@@ -0,0 +1,87 @@
---
- name: create graylog docker network
community.general.docker_network:
name: "graylog"
tags: graylog
- name: create graylog required volumes
community.general.docker_volume:
name: "{{ item }}"
with_items:
- graylog-db
- graylog-es
- graylog-conf
tags: graylog
- name: create graylog mongodb container
community.general.docker_container:
name: graylog-mongo
image: mongo:4.2
recreate: false
restart: false
restart_policy: on-failure
restart_retries: 3
networks:
- name: "graylog"
volumes:
- graylog-db:/data/db
tags: graylog
- name: create graylog elasticsearch container
community.general.docker_container:
name: graylog-elastic
image: docker.elastic.co/elasticsearch/elasticsearch-oss:7.10.2
recreate: false
restart: false
restart_policy: on-failure
restart_retries: 3
networks:
- name: "graylog"
volumes:
- graylog-es:/usr/share/elasticsearch/data
env:
http.host: "0.0.0.0"
transport.host: "localhost"
network.host: "0.0.0.0"
cluster.name: "graylog"
ES_JAVA_OPTS: "-Dlog4j2.formatMsgNoLookups=true -Xms512m -Xmx512m"
ulimits:
- "memlock:-1:-1"
memory: 1G
tags: graylog
- name: create graylog container
community.general.docker_container:
name: graylog
image: graylog/graylog:4.2
recreate: false
restart: true
restart_policy: on-failure
restart_retries: 3
networks:
- name: "graylog"
volumes:
- graylog-conf:/usr/share/graylog/data/config
env:
GRAYLOG_PASSWORD_SECRET: "{{ graylog_secret }}"
GRAYLOG_ROOT_PASSWORD_SHA2: "{{ graylog_root_pass_sha2 }}"
GRAYLOG_HTTP_EXTERNAL_URI: http://192.168.1.12:9000/
GRAYLOG_HTTP_BIND_ADDRESS: 0.0.0.0:9000
GRAYLOG_MONGODB_URI: mongodb://graylog-mongo/graylog
GRAYLOG_ELASTICSEARCH_HOSTS: http://graylog-elastic:9200
ports:
# Graylog web interface and REST API
- "{{ graylog_port }}:9000"
# Syslog TCP
#- 1514:1514
# Syslog UDP
- "0.0.0.0:{{ syslog_udp_default }}:{{ syslog_udp_default }}/udp"
# Syslog2 UDP
- "0.0.0.0:{{ syslog_udp_unifi }}:{{ syslog_udp_unifi }}/udp"
# Syslog2 UDP
- "0.0.0.0:{{ syslog_udp_error }}:{{ syslog_udp_error }}/udp"
# GELF TCP
#- 12201:12201
# GELF UDP
#- 12201:12201/udp
tags: graylog

2
ansible/roles/graylog/tasks/main.yml Normal file
View File

@@ -0,0 +1,2 @@
---
- import_tasks: graylog.yml

BIN
ansible/roles/hass/files/configuration.yaml
View File

Binary file not shown.

1
ansible/roles/hass/meta/main.yml
View File

@@ -1,3 +1,4 @@
---
dependencies:
- role: http
- role: graylog

7
ansible/roles/hass/tasks/hass.yml
View File

@@ -27,10 +27,15 @@
community.general.docker_container:
name: hass
image: ghcr.io/home-assistant/home-assistant:stable
recreate: true
recreate: false
restart: true
restart_policy: on-failure
restart_retries: 3
log_driver: syslog
log_options:
syslog-address: "udp://localhost:{{ syslog_udp_default }}"
syslog-facility: daemon
tag: "docker/{{'{{'}}.Name{{'}}'}}"
volumes:
- /var/lib/hass:/config
- /usr/share/hass:/share

26
ansible/roles/http/defaults/main.yml
View File

@@ -1,9 +1,5 @@
---
deps: [
certbot,
nginx,
nginx-mod-modsecurity
]
deps: [certbot, nginx, nginx-mod-modsecurity]
ci_server_name: ci.bdebyl.net
pi_server_name: pi.bdebyl.net
@@ -11,15 +7,15 @@ assistant_server_name: assistant.bdebyl.net
home_server_name: home.bdebyl.net
parts_server_name: parts.bdebyl.net
video_server_name: video.bdebyl.net
logs_server_name: logs.bdebyl.net
install_path: /usr/share
nginx_path: /etc/nginx
nginx_conf_path: "{{ nginx_path }}/conf"
modsec_log_path: /var/log/nginx/modsec_audit.log
modsec_rules_path: "{{ nginx_conf_path }}/rules"
modsec_crs_before_rule_conf:
"{{ modsec_rules_path }}/REQUEST-900-EXCLUSION-RULES-BEFORE-CRS.conf"
modsec_crs_after_rule_conf:
"{{ modsec_rules_path }}/REQUEST-999-EXCLUSION-RULES-AFTER-CRS.conf"
modsec_crs_before_rule_conf: "{{ modsec_rules_path }}/REQUEST-900-EXCLUSION-RULES-BEFORE-CRS.conf"
modsec_crs_after_rule_conf: "{{ modsec_rules_path }}/REQUEST-999-EXCLUSION-RULES-AFTER-CRS.conf"
modsec_path: "{{ install_path }}/modsecurity"
crs_path: "{{ install_path }}/coreruleset"
crs_rules_path: "{{ crs_path }}/rules"
@@ -39,6 +35,12 @@ modsec_git_urls:
dest: "{{ modsec_path }}"
ver: "v3.0.6"
modsec_conf_replaces:
- regex: "^SecRuleEngine"
line: "SecRuleEngine On"
- regex: "^SecAuditLog"
line: "SecAuditLog {{ modsec_log_path }}"
modsec_conf_links:
- src: "{{ modsec_path }}/modsecurity.conf-recommended"
dest: "{{ nginx_path }}/modsecurity.conf"
@@ -46,11 +48,9 @@ modsec_conf_links:
dest: "{{ nginx_path }}/unicode.mapping"
- src: "{{ crs_path }}/crs-setup.conf.example"
dest: "{{ nginx_conf_path }}/crs-setup.conf"
- src:
"{{ crs_rules_path }}/REQUEST-900-EXCLUSION-RULES-BEFORE-CRS.conf.example"
- src: "{{ crs_rules_path }}/REQUEST-900-EXCLUSION-RULES-BEFORE-CRS.conf.example"
dest: "{{ modsec_rules_path }}/REQUEST-900-EXCLUSION-RULES-BEFORE-CRS.conf"
- src:
"{{ crs_rules_path }}/RESPONSE-999-EXCLUSION-RULES-AFTER-CRS.conf.example"
- src: "{{ crs_rules_path }}/RESPONSE-999-EXCLUSION-RULES-AFTER-CRS.conf.example"
dest: "{{ modsec_rules_path }}/RESPONSE-999-EXCLUSION-RULES-AFTER-CRS.conf"
crs_rule_links:

48
ansible/roles/http/files/nginx/nginx.conf
View File

@@ -1,48 +0,0 @@
user http;
worker_processes 1;
load_module /usr/lib/nginx/modules/ngx_http_modsecurity_module.so;
error_log /var/log/nginx/error.log notice;
events {
worker_connections 1024;
}
http {
map $http_upgrade $connection_upgrade {
default upgrade;
'' close;
}
include mime.types;
default_type application/octet-stream;
log_format main '$remote_addr - $remote_user [$time_local] "$request" '
'$status $body_bytes_sent "$http_referer" '
'"$http_user_agent" "$http_x_forwarded_for"';
access_log /var/log/nginx/access.log main;
sendfile on;
server_tokens off;
#tcp_nopush on;
keepalive_timeout 65;
gzip on;
gzip_disable "mise6";
gzip_min_length 1000;
gzip_proxied expired no-cache no-store private auth;
gzip_types text/plain application/xml application/json application/javascript application/octet-stream text/css;
# client_body_buffer_size 1k;
# client_header_buffer_size 1k;
# client_max_body_size 2k;
# large_client_header_buffers 2 1k;
limit_req_zone $binary_remote_addr zone=one:10m rate=10r/s;
include /etc/nginx/sites-enabled/*.conf;
}

12
ansible/roles/http/tasks/http.yml
View File

@@ -1,9 +1,11 @@
---
- name: setup nginx base configuration
become: true
ansible.builtin.copy:
src: files/nginx/nginx.conf
ansible.builtin.template:
src: templates/nginx/nginx.conf.j2
dest: /etc/nginx/nginx.conf
owner: root
group: http
mode: 0644
notify: restart_nginx
tags: http
@@ -26,7 +28,7 @@
state: directory
owner: http
group: http
mode: 0644
mode: 0755
loop:
- /srv/http
- /srv/http/letsencrypt
@@ -38,7 +40,7 @@
path: /srv/http
owner: http
group: http
mode: 0644
mode: 0755
recurse: true
tags: http
@@ -55,6 +57,7 @@
- "{{ assistant_server_name }}.conf"
- "{{ video_server_name }}.conf"
- "{{ parts_server_name }}.conf"
- "{{ logs_server_name }}.conf"
notify: restart_nginx
tags: http
@@ -78,6 +81,7 @@
- "{{ home_server_name }}.conf"
- "{{ assistant_server_name }}.conf"
- "{{ video_server_name }}.conf"
- "{{ logs_server_name }}.conf"
notify: restart_nginx
tags: http

10
ansible/roles/http/tasks/logrotate.yml Normal file
View File

@@ -0,0 +1,10 @@
---
- name: template nginx log rotation
become: true
ansible.builtin.template:
src: logrotate/nginx.j2
dest: /etc/logrotate.d/nginx
mode: 0644
tags:
- http
- logrotate

1
ansible/roles/http/tasks/main.yml
View File

@@ -4,3 +4,4 @@
- import_tasks: modsec.yml
- import_tasks: http.yml
- import_tasks: https.yml
- import_tasks: logrotate.yml

5
ansible/roles/http/tasks/modsec.yml
View File

@@ -84,7 +84,8 @@
become: true
ansible.builtin.lineinfile:
path: /etc/nginx/modsecurity.conf
regexp: "^SecRuleEngine"
line: "SecRuleEngine On"
regexp: "{{ item.regex }}"
line: "{{ item.line }}"
loop: "{{ modsec_conf_replaces }} "
notify: restart_nginx
tags: modsec

10
ansible/roles/http/templates/logrotate/nginx.j2 Normal file
View File

@@ -0,0 +1,10 @@
/var/log/nginx/*log {
daily
rotate 4
missingok
notifempty
create 640 http log
compress
delaycompress
copytruncate
}

71
ansible/roles/http/templates/nginx/nginx.conf.j2 Normal file
View File

@@ -0,0 +1,71 @@
user http;
worker_processes 1;
load_module /usr/lib/nginx/modules/ngx_http_modsecurity_module.so;
error_log /var/log/nginx/error.log notice;
error_log syslog:server=localhost:{{ syslog_udp_error }},tag=nginx,severity=info notice;
events {
worker_connections 1024;
}
http {
map $http_upgrade $connection_upgrade {
default upgrade;
'' close;
}
include mime.types;
default_type application/octet-stream;
log_format main '$remote_addr - $connection : $connection_requests [$time_iso8601] "$request" '
'$status $body_bytes_sent "$http_referer" '
'"$http_user_agent" "$http_x_forwarded_for"';
log_format graylog_json escape=json '{ "nginx_timestamp": "[$time_iso8601]", '
'"remote_addr": "$remote_addr", '
'"connection": "$connection", '
'"connection_requests": $connection_requests, '
'"body_bytes_sent": $body_bytes_sent, '
'"request_length": $request_length, '
'"request_time": $request_time, '
'"response_status": $status, '
'"request": "$request", '
'"request_method": "$request_method", '
'"host": "$host", '
'"upstream_cache_status": "$upstream_cache_status", '
'"upstream_addr": "$upstream_addr", '
'"http_x_forwarded_for": "$http_x_forwarded_for", '
'"http_referrer": "$http_referer", '
'"http_user_agent": "$http_user_agent", '
'"http_version": "$server_protocol", '
'"remote_user": "$remote_user", '
'"http_x_forwarded_proto": "$http_x_forwarded_proto", '
'"upstream_response_time": "$upstream_response_time", '
'"nginx_access": true }';
access_log /var/log/nginx/access.log main;
access_log syslog:server=localhost:{{ syslog_udp_default }},tag=nginx,severity=info graylog_json;
sendfile on;
server_tokens off;
#tcp_nopush on;
keepalive_timeout 65;
gzip on;
gzip_disable "mise6";
gzip_min_length 1000;
gzip_proxied expired no-cache no-store private auth;
gzip_types text/plain application/xml application/json application/javascript application/octet-stream text/css;
# client_body_buffer_size 1k;
# client_header_buffer_size 1k;
# client_max_body_size 2k;
# large_client_header_buffers 2 1k;
limit_req_zone $binary_remote_addr zone=one:10m rate=10r/s;
include /etc/nginx/sites-enabled/*.conf;
}

10
ansible/roles/http/templates/nginx/sites/ci.bdebyl.net.https.conf.j2
View File

@@ -2,6 +2,11 @@ upstream drone {
server 127.0.0.1:8080;
}
geo $local_access {
default 0;
192.168.1.1 1;
}
server {
modsecurity on;
modsecurity_rules_file {{ nginx_path }}/modsec_includes.conf;
@@ -24,7 +29,12 @@ server {
ssl_stapling on;
ssl_stapling_verify on;
resolver 127.0.0.1 127.0.0.53 9.9.9.9 valid=60s;
location / {
if ($local_access = 1) {
access_log off;
}
add_header Allow "GET, POST, HEAD" always;
add_header Content-Security-Policy "default-src 'self'; img-src 'self' https://*.githubusercontent.com; frame-ancestors 'self'; base-uri 'none',base-uri 'self'; form-action 'self'" always;
add_header Referrer-Policy "same-origin" always;

32
ansible/roles/http/templates/nginx/sites/logs.bdebyl.net.conf.j2 Normal file
View File

@@ -0,0 +1,32 @@
upstream graylog {
server localhost:{{ graylog_port }};
}
geo $local_access {
default 0;
192.168.1.0/24 1;
}
server {
modsecurity on;
modsecurity_rules_file {{ nginx_path }}/modsec_includes.conf;
listen 80;
server_name {{ logs_server_name }};
location / {
if ($local_access = 1) {
access_log off;
}
allow 192.168.1.0/24;
allow 127.0.0.1;
deny all;
proxy_set_header X-Forwarded-For $remote_addr;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection $connection_upgrade;
proxy_buffering off;
proxy_pass http://graylog;
}
}

2
ansible/roles/http/templates/nginx/sites/parts.bdebyl.net.https.conf.j2
View File

@@ -11,6 +11,8 @@ server {
modsecurity on;
modsecurity_rules_file {{ nginx_path }}/modsec_includes.conf;
resolver 127.0.0.1 127.0.0.53 9.9.9.9 valid=60s;
listen 443 ssl http2;
listen [::]:443 ssl http2;
server_name {{ parts_server_name }};

3
ansible/roles/http/templates/nginx/sites/pi.bdebyl.net.conf.j2
View File

@@ -12,7 +12,7 @@ server {
listen [::]:80;
root /srv/http/pihole;
server_name pi.bdebyl.net;
server_name {{ pi_server_name }};
autoindex off;
proxy_intercept_errors on;
@@ -21,6 +21,7 @@ server {
index pihole/index.php index.php index.html index.htm;
allow 192.168.1.0/24;
allow 127.0.0.1;
deny all;
location / {

3
ansible/roles/motion/meta/main.yml
View File

@@ -1 +1,4 @@
---
dependencies:
- role: http
- role: graylog

7
ansible/roles/motion/tasks/motion.yml
View File

@@ -38,10 +38,15 @@
community.general.docker_container:
name: shinobi
image: migoller/shinobidocker:latest
recreate: true
recreate: false
restart: true
restart_policy: on-failure
restart_retries: 3
log_driver: syslog
log_options:
syslog-address: "udp://localhost:{{ syslog_udp_default }}"
syslog-facility: daemon
tag: "docker/{{'{{'}} .Name {{'}}'}}"
volumes:
- "shinobi_data:/var/lib/mysql"
- "/mnt/shinobi:/opt/shinobi/videos"

1
ansible/roles/partkeepr/meta/main.yml
View File

@@ -1,3 +1,4 @@
---
dependencies:
- role: http
- role: graylog

73
ansible/roles/partkeepr/tasks/main.yml
View File

@@ -7,10 +7,12 @@
- partkeepr-conf-vol
- partkeepr-data-vol
- partkeepr-db-vol
tags: partkeepr
- name: create partkeepr network
community.general.docker_network:
name: "partkeepr"
tags: partkeepr
- name: create partkeepr-db container
diff: false
@@ -19,8 +21,13 @@
image: mariadb:10.0
recreate: false
restart: true
restart_policy: on-failure
restart_policy: unless-stopped
restart_retries: 3
log_driver: syslog
log_options:
syslog-address: "udp://localhost:{{ syslog_udp_default }}"
syslog-facility: daemon
tag: "docker/{{'{{'}}.Name{{'}}'}}"
networks:
- name: "partkeepr"
env:
@@ -30,6 +37,7 @@
MYSQL_PASSWORD: partkeepr
volumes:
- partkeepr-db-vol:/var/lib/mysql
tags: partkeepr
- name: create partkeepr container
diff: false
@@ -38,40 +46,13 @@
image: mhubig/partkeepr:latest
recreate: false
restart: true
restart_policy: on-failure
restart_retries: 3
networks:
- name: "partkeepr"
volumes:
- partkeepr-db-conf-vol:/var/www/html/app/config
- partkeepr-db-data-vol:/var/www/html/data
- partkeepr-db-web-vol:/var/www/html/web
- name: create partkeepr-cron container
diff: false
community.general.docker_container:
name: partkeepr-cron
image: mhubig/partkeepr:latest
entrypoint: []
command: bash -c "crontab /etc/cron.d/partkeepr && cron -f"
recreate: false
restart: true
restart_policy: on-failure
restart_retries: 3
volumes:
- partkeepr-db-conf-vol:/var/www/html/app/config:ro
- partkeepr-db-data-vol:/var/www/html/data
- partkeepr-db-web-vol:/var/www/html/web
- name: create partkeepr container
diff: false
community.general.docker_container:
name: partkeepr
image: mhubig/partkeepr:latest
recreate: false
restart: true
restart_policy: on-failure
restart_policy: unless-stopped
restart_retries: 3
log_driver: syslog
log_options:
syslog-address: "udp://localhost:{{ syslog_udp_default }}"
syslog-facility: daemon
tag: "docker/{{'{{'}}.Name{{'}}'}}"
networks:
- name: "partkeepr"
ports:
@@ -80,3 +61,27 @@
- partkeepr-db-conf-vol:/var/www/html/app/config
- partkeepr-db-data-vol:/var/www/html/data
- partkeepr-db-web-vol:/var/www/html/web
tags: partkeepr
- name: create partkeepr-cron container
diff: false
community.general.docker_container:
name: partkeepr-cron
image: mhubig/partkeepr:latest
command_handling: correct
entrypoint: []
command: bash -c "crontab /etc/cron.d/partkeepr && cron -f"
recreate: false
restart: true
restart_policy: unless-stopped
restart_retries: 3
log_driver: syslog
log_options:
syslog-address: "udp://localhost:{{ syslog_udp_default }}"
syslog-facility: daemon
tag: "docker/{{'{{'}}.Name{{'}}'}}"
volumes:
- partkeepr-db-conf-vol:/var/www/html/app/config:ro
- partkeepr-db-data-vol:/var/www/html/data
- partkeepr-db-web-vol:/var/www/html/web
tags: partkeepr

BIN
ansible/vars/vault.yml
View File

Binary file not shown.