From 0d8dfb3b6c3c07c31d1bda564c9af357a7adbcdc Mon Sep 17 00:00:00 2001 From: Thomas Widhalm Date: Fri, 14 Mar 2025 11:37:37 +0100 Subject: [PATCH 1/9] Add output of Mermaid syntax overview to Logstash --- docs/logstash-pipelines.md | 8 ++++++ docs/role-logstash.md | 2 ++ roles/logstash/defaults/main.yml | 2 ++ roles/logstash/tasks/main.yml | 12 ++++++++ roles/logstash/templates/pipelines.mermaid.j2 | 28 +++++++++++++++++++ 5 files changed, 52 insertions(+) create mode 100644 roles/logstash/templates/pipelines.mermaid.j2 diff --git a/docs/logstash-pipelines.md b/docs/logstash-pipelines.md index 6b256761..13ee1e9d 100644 --- a/docs/logstash-pipelines.md +++ b/docs/logstash-pipelines.md @@ -1,5 +1,13 @@ # Pipelines # +## Keeping an overview ## + +It can be quite difficult to stay on top of your pipeline configuration because they tend to become very complex. + +This collection will leave some comments about how pipelines are interconnected within the `/etc/logstash/pipelines.yml` configuration file. + +If you set `logstash_mermaid` to `true` (which is the default), then you will also get a new file in `/etc/logstash/pipelines.mermaid`. You can paste it into a Mermaid editor in your documentation tool or in an [online Mermaid editor](https://mermaid.live/). + ## Git managed ## If you have pipeline code managed in (and available via) Git repositories, you can use this role to check them out and integrate them into `pipelines.yml`. diff --git a/docs/role-logstash.md b/docs/role-logstash.md index e0a8d142..6fd636fb 100644 --- a/docs/role-logstash.md +++ b/docs/role-logstash.md @@ -89,6 +89,8 @@ Aside from `logstash.yml` we can manage Logstashs pipelines. * *logstash_legacy_monitoring*: Enables legacy monitoring - ignored when `elasticstack_full_stack` is not set. (default: `true`) * *logstash_redis_password*: If set this will use this password when connecting our simple inputs and outputs to Redis. (default: not set) +* *logstash_mermaid*: Print overview over Logstash pipelines in Mermaid syntax into `/etc/logstash/pipelines.mermaid`. (default: `true`) + The following variables configure Log4j for Logstash. All default to `true` as this is the default after the installation. * *logstash_logging_console*: Log to console - syslog when run via systemd diff --git a/roles/logstash/defaults/main.yml b/roles/logstash/defaults/main.yml index 6c152923..0259db5e 100644 --- a/roles/logstash/defaults/main.yml +++ b/roles/logstash/defaults/main.yml @@ -88,6 +88,8 @@ logstash_pipeline_identifier: true logstash_pipeline_identifier_field_name: "[netways][pipeline]" logstash_pipeline_identifier_defaults: false +logstash_mermaid: true + # Only for internal use logstash_freshstart: diff --git a/roles/logstash/tasks/main.yml b/roles/logstash/tasks/main.yml index 84dc81b6..ac53766b 100644 --- a/roles/logstash/tasks/main.yml +++ b/roles/logstash/tasks/main.yml @@ -251,6 +251,18 @@ - configuration - logstash_configuration +- name: Print Logstash pipelines in Mermaid syntax + ansible.builtin.template: + src: pipelines.mermaid.j2 + dest: /etc/logstash/pipelines.mermaid + owner: root + group: root + mode: 0655 + when: + - logstash_mermaid | bool + tags: + - mermaid + - name: Install Logstash plugins community.general.logstash_plugin: state: present diff --git a/roles/logstash/templates/pipelines.mermaid.j2 b/roles/logstash/templates/pipelines.mermaid.j2 new file mode 100644 index 00000000..0409ead6 --- /dev/null +++ b/roles/logstash/templates/pipelines.mermaid.j2 @@ -0,0 +1,28 @@ +# Managed via Ansible role +# https://github.com/netways/ansible-role-logstash + +# Use the following code with your favorite Mermaid editor +# Or paste into: https://mermaid.live/ +# To get a graphical overview of your Logstash pipelines + +flowchart TD +{% if logstash_beats_input | bool %} +ansible-input[ansible-input] --> input{input} +{% endif %} +{% if logstash_elasticsearch_output | bool %} +forwarder{forwarder} --> ansible-forwarder[ansible-forwarder] +{% endif %} +{% if logstash_pipelines is defined %} +{% for item in logstash_pipelines %} +{% if item.input is defined %} +{% for input in item.input %} +{{ input.key }}{{ '{' }}{{ input.key }}{{ '}' }} --> {{ item.name }}{{ '[' }}{{item.name}}{{ ']' }} +{% endfor %} +{% endif %} +{% if item.output is defined %} +{% for output in item.output %} +{{ item.name }}{{ '[' }}{{ item.name }}{{ ']' }} --> {% if output.condition is defined %}{{ '|' }}if {{ output.condition | replace("][", ".") | replace("[","") | replace("]","") | replace('"', '')}}{{ '|' }}{% endif %}{{ output.key }}{{ '{' }}{{ output.key }}} +{% endfor %} +{% endif %} +{% endfor %} +{% endif %} From a2720a98cebd49cd398a5dd9b979bc48aaa205d9 Mon Sep 17 00:00:00 2001 From: Thomas Widhalm Date: Fri, 14 Mar 2025 12:00:12 +0100 Subject: [PATCH 2/9] Use prefix to distunquish key and pipeline in Mermaid --- roles/logstash/templates/pipelines.mermaid.j2 | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/roles/logstash/templates/pipelines.mermaid.j2 b/roles/logstash/templates/pipelines.mermaid.j2 index 0409ead6..33ceea74 100644 --- a/roles/logstash/templates/pipelines.mermaid.j2 +++ b/roles/logstash/templates/pipelines.mermaid.j2 @@ -7,21 +7,21 @@ flowchart TD {% if logstash_beats_input | bool %} -ansible-input[ansible-input] --> input{input} +p_ansible-input[ansible-input] --> k_input{input} {% endif %} {% if logstash_elasticsearch_output | bool %} -forwarder{forwarder} --> ansible-forwarder[ansible-forwarder] +k_forwarder{forwarder} --> p_ansible-forwarder[ansible-forwarder] {% endif %} {% if logstash_pipelines is defined %} {% for item in logstash_pipelines %} {% if item.input is defined %} {% for input in item.input %} -{{ input.key }}{{ '{' }}{{ input.key }}{{ '}' }} --> {{ item.name }}{{ '[' }}{{item.name}}{{ ']' }} +k_{{ input.key }}{{ '{' }}{{ input.key }}{{ '}' }} --> p_{{ item.name }}{{ '[' }}{{item.name}}{{ ']' }} {% endfor %} {% endif %} {% if item.output is defined %} {% for output in item.output %} -{{ item.name }}{{ '[' }}{{ item.name }}{{ ']' }} --> {% if output.condition is defined %}{{ '|' }}if {{ output.condition | replace("][", ".") | replace("[","") | replace("]","") | replace('"', '')}}{{ '|' }}{% endif %}{{ output.key }}{{ '{' }}{{ output.key }}} +p_{{ item.name }}{{ '[' }}{{ item.name }}{{ ']' }} --> {% if output.condition is defined %}{{ '|' }}if {{ output.condition | replace("][", ".") | replace("[","") | replace("]","") | replace('"', '')}}{{ '|' }}{% endif %}k_{{ output.key }}{{ '{' }}{{ output.key }}} {% endfor %} {% endif %} {% endfor %} From 9444e890dffb5e81dca52ff6fbbc104992e0cc51 Mon Sep 17 00:00:00 2001 From: Thomas Widhalm Date: Tue, 18 Mar 2025 16:14:48 +0100 Subject: [PATCH 3/9] Optionally put Mermaid syntax on control node, too --- docs/role-logstash.md | 4 ++- roles/logstash/defaults/main.yml | 2 ++ roles/logstash/tasks/logstash-mermaid.yml | 36 +++++++++++++++++++++++ roles/logstash/tasks/main.yml | 7 +---- 4 files changed, 42 insertions(+), 7 deletions(-) create mode 100644 roles/logstash/tasks/logstash-mermaid.yml diff --git a/docs/role-logstash.md b/docs/role-logstash.md index 6fd636fb..ea5c050f 100644 --- a/docs/role-logstash.md +++ b/docs/role-logstash.md @@ -89,7 +89,9 @@ Aside from `logstash.yml` we can manage Logstashs pipelines. * *logstash_legacy_monitoring*: Enables legacy monitoring - ignored when `elasticstack_full_stack` is not set. (default: `true`) * *logstash_redis_password*: If set this will use this password when connecting our simple inputs and outputs to Redis. (default: not set) -* *logstash_mermaid*: Print overview over Logstash pipelines in Mermaid syntax into `/etc/logstash/pipelines.mermaid`. (default: `true`) +* *logstash_mermaid*: Print overview over Logstash pipelines in Mermaid syntax. (default: `true`) +* *logstash_mermaid_logstash*: Place Mermaid syntax into `/etc/logstash/pipelines.mermaid` on Logstash hosts. (default: `true`) +* *logstash_mermaid_local*: Place Mermaid syntax into temporary file on control node. (default: `true`) The following variables configure Log4j for Logstash. All default to `true` as this is the default after the installation. diff --git a/roles/logstash/defaults/main.yml b/roles/logstash/defaults/main.yml index 0259db5e..3a16ba9c 100644 --- a/roles/logstash/defaults/main.yml +++ b/roles/logstash/defaults/main.yml @@ -89,6 +89,8 @@ logstash_pipeline_identifier_field_name: "[netways][pipeline]" logstash_pipeline_identifier_defaults: false logstash_mermaid: true +logstash_mermaid_local: true +logstash_mermaid_logstash: true # Only for internal use diff --git a/roles/logstash/tasks/logstash-mermaid.yml b/roles/logstash/tasks/logstash-mermaid.yml new file mode 100644 index 00000000..a3c13e07 --- /dev/null +++ b/roles/logstash/tasks/logstash-mermaid.yml @@ -0,0 +1,36 @@ +--- + +- name: Print Logstash pipelines in Mermaid syntax on Logstash hosts + ansible.builtin.template: + src: pipelines.mermaid.j2 + dest: /etc/logstash/pipelines.mermaid + owner: root + group: root + mode: 0655 + when: + - logstash_mermaid_logstash | bool + +- name: Provide Logstash pipelines in Mermaid syntax on control node + when: + - logstash_mermaid_local | bool + block: + + - name: Create temporary directory on control node + ansible.builtin.local_task: + module: tempfile + prefix: logstash_mermaid + register: mermaid_path + + - name: Print Logstash pipelines in Mermaid syntax on control node + ansible.builtin.local_task: + module: template + src: pipelines.mermaid.j2 + dest: "{{ mermaid_path.path }}" + owner: root + group: root + mode: 0655 + + - name: Send user to Mermaid file + ansible.builtin.debug: + msg: + - "You can find an overview of your pipeline configuration in Mermaid syntax at {{ mermaid_path.path }}" diff --git a/roles/logstash/tasks/main.yml b/roles/logstash/tasks/main.yml index ac53766b..52d87054 100644 --- a/roles/logstash/tasks/main.yml +++ b/roles/logstash/tasks/main.yml @@ -252,12 +252,7 @@ - logstash_configuration - name: Print Logstash pipelines in Mermaid syntax - ansible.builtin.template: - src: pipelines.mermaid.j2 - dest: /etc/logstash/pipelines.mermaid - owner: root - group: root - mode: 0655 + ansible.builtin.import_tasks: logstash-mermaid.yml when: - logstash_mermaid | bool tags: From 2cfd4fef4dbb333b7c4acc9a42ede5d327e2cfa5 Mon Sep 17 00:00:00 2001 From: Thomas Widhalm Date: Tue, 18 Mar 2025 16:31:47 +0100 Subject: [PATCH 4/9] Typo --- roles/logstash/tasks/logstash-mermaid.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/roles/logstash/tasks/logstash-mermaid.yml b/roles/logstash/tasks/logstash-mermaid.yml index a3c13e07..5f24e21c 100644 --- a/roles/logstash/tasks/logstash-mermaid.yml +++ b/roles/logstash/tasks/logstash-mermaid.yml @@ -16,13 +16,13 @@ block: - name: Create temporary directory on control node - ansible.builtin.local_task: + ansible.builtin.local_action: module: tempfile prefix: logstash_mermaid register: mermaid_path - name: Print Logstash pipelines in Mermaid syntax on control node - ansible.builtin.local_task: + ansible.builtin.local_action: module: template src: pipelines.mermaid.j2 dest: "{{ mermaid_path.path }}" From e364240aa21f55e32b8680899152e3d157266e9b Mon Sep 17 00:00:00 2001 From: Thomas Widhalm Date: Tue, 18 Mar 2025 16:39:42 +0100 Subject: [PATCH 5/9] Typo --- roles/logstash/tasks/logstash-mermaid.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/roles/logstash/tasks/logstash-mermaid.yml b/roles/logstash/tasks/logstash-mermaid.yml index 5f24e21c..ddbc820a 100644 --- a/roles/logstash/tasks/logstash-mermaid.yml +++ b/roles/logstash/tasks/logstash-mermaid.yml @@ -16,13 +16,13 @@ block: - name: Create temporary directory on control node - ansible.builtin.local_action: + local_action: module: tempfile prefix: logstash_mermaid register: mermaid_path - name: Print Logstash pipelines in Mermaid syntax on control node - ansible.builtin.local_action: + local_action: module: template src: pipelines.mermaid.j2 dest: "{{ mermaid_path.path }}" From 8069c177abcba92e7d0be40ce59168b03d087449 Mon Sep 17 00:00:00 2001 From: Thomas Widhalm Date: Tue, 18 Mar 2025 16:42:44 +0100 Subject: [PATCH 6/9] Lint --- roles/logstash/tasks/logstash-mermaid.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/roles/logstash/tasks/logstash-mermaid.yml b/roles/logstash/tasks/logstash-mermaid.yml index ddbc820a..b604d122 100644 --- a/roles/logstash/tasks/logstash-mermaid.yml +++ b/roles/logstash/tasks/logstash-mermaid.yml @@ -16,19 +16,19 @@ block: - name: Create temporary directory on control node - local_action: - module: tempfile + ansible.builtin.tempfile prefix: logstash_mermaid register: mermaid_path + delegate_to: localhost - name: Print Logstash pipelines in Mermaid syntax on control node - local_action: - module: template + ansible.builtin.template src: pipelines.mermaid.j2 dest: "{{ mermaid_path.path }}" owner: root group: root mode: 0655 + delegate_to: localhost - name: Send user to Mermaid file ansible.builtin.debug: From 89f4973e154629d49a1c8633d29b9c9283110cc8 Mon Sep 17 00:00:00 2001 From: Thomas Widhalm Date: Tue, 18 Mar 2025 16:44:21 +0100 Subject: [PATCH 7/9] Lint --- roles/logstash/tasks/logstash-mermaid.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/roles/logstash/tasks/logstash-mermaid.yml b/roles/logstash/tasks/logstash-mermaid.yml index b604d122..5d904237 100644 --- a/roles/logstash/tasks/logstash-mermaid.yml +++ b/roles/logstash/tasks/logstash-mermaid.yml @@ -16,13 +16,13 @@ block: - name: Create temporary directory on control node - ansible.builtin.tempfile + ansible.builtin.tempfile: prefix: logstash_mermaid register: mermaid_path delegate_to: localhost - name: Print Logstash pipelines in Mermaid syntax on control node - ansible.builtin.template + ansible.builtin.template: src: pipelines.mermaid.j2 dest: "{{ mermaid_path.path }}" owner: root From f94ecc2792fdb6e4dc1f29a3e40b9cc767e9eb73 Mon Sep 17 00:00:00 2001 From: Thomas Widhalm Date: Tue, 18 Mar 2025 16:55:52 +0100 Subject: [PATCH 8/9] Disable local Mermaid output on control node in Molecule You're not allowed to place files into the control node during a Molecule scenario --- docs/logstash-pipelines.md | 2 +- docs/role-logstash.md | 1 + molecule/elasticstack_default/converge.yml | 1 + roles/logstash/templates/pipelines.mermaid.j2 | 3 +++ 4 files changed, 6 insertions(+), 1 deletion(-) diff --git a/docs/logstash-pipelines.md b/docs/logstash-pipelines.md index 13ee1e9d..496d1ce2 100644 --- a/docs/logstash-pipelines.md +++ b/docs/logstash-pipelines.md @@ -6,7 +6,7 @@ It can be quite difficult to stay on top of your pipeline configuration because This collection will leave some comments about how pipelines are interconnected within the `/etc/logstash/pipelines.yml` configuration file. -If you set `logstash_mermaid` to `true` (which is the default), then you will also get a new file in `/etc/logstash/pipelines.mermaid`. You can paste it into a Mermaid editor in your documentation tool or in an [online Mermaid editor](https://mermaid.live/). +If you set `logstash_mermaid` to `true` (which is the default), then you will also get a new file in `/etc/logstash/pipelines.mermaid`. You can paste it into a Mermaid editor in your documentation tool or in an [online Mermaid editor](https://mermaid.live/). The same content will be available on your control node in a temporary file. You can even add arbitrary code to reflect manually managed pipelines by using `logstash_mermaid_extra`. ## Git managed ## diff --git a/docs/role-logstash.md b/docs/role-logstash.md index ea5c050f..fd961b70 100644 --- a/docs/role-logstash.md +++ b/docs/role-logstash.md @@ -92,6 +92,7 @@ Aside from `logstash.yml` we can manage Logstashs pipelines. * *logstash_mermaid*: Print overview over Logstash pipelines in Mermaid syntax. (default: `true`) * *logstash_mermaid_logstash*: Place Mermaid syntax into `/etc/logstash/pipelines.mermaid` on Logstash hosts. (default: `true`) * *logstash_mermaid_local*: Place Mermaid syntax into temporary file on control node. (default: `true`) +* *logstash_mermaid_extra*: You can add extra Mermaid syntax to the output by adding it to this variable. YAML-multiline is supported. (default: none) The following variables configure Log4j for Logstash. All default to `true` as this is the default after the installation. diff --git a/molecule/elasticstack_default/converge.yml b/molecule/elasticstack_default/converge.yml index 694db61d..0d47fcd5 100644 --- a/molecule/elasticstack_default/converge.yml +++ b/molecule/elasticstack_default/converge.yml @@ -19,6 +19,7 @@ elasticstack_no_log: false logstash_pipeline_unsafe_shutdown: true logstash_redis_password: "{{ lookup('ansible.builtin.password', '/tmp/redispassword', chars=['ascii_letters'], length=15) }}" + logstash_mermaid_local: false # no permission to write into Docker control node redis_requirepass: "{{ logstash_redis_password }}" beats_filebeat_journald: true beats_filebeat_modules: diff --git a/roles/logstash/templates/pipelines.mermaid.j2 b/roles/logstash/templates/pipelines.mermaid.j2 index 33ceea74..adc30895 100644 --- a/roles/logstash/templates/pipelines.mermaid.j2 +++ b/roles/logstash/templates/pipelines.mermaid.j2 @@ -26,3 +26,6 @@ p_{{ item.name }}{{ '[' }}{{ item.name }}{{ ']' }} --> {% if output.condition is {% endif %} {% endfor %} {% endif %} +{% if logstash_mermaid_extra is defined %} +{{ logstash_mermaid_extra }} +{% endif %} From 2a441035950d83f29cb0d35e8c9927711a533b5d Mon Sep 17 00:00:00 2001 From: Thomas Widhalm Date: Wed, 19 Mar 2025 10:46:17 +0100 Subject: [PATCH 9/9] Turn off local mermaid because of Molecule tests During Molecule tests you're not allowed to write Mermaid config to the control node. So we're turning it off by default. --- docs/role-logstash.md | 2 +- molecule/elasticstack_default/converge.yml | 1 - roles/logstash/defaults/main.yml | 2 +- 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/docs/role-logstash.md b/docs/role-logstash.md index fd961b70..1b4e4b78 100644 --- a/docs/role-logstash.md +++ b/docs/role-logstash.md @@ -91,7 +91,7 @@ Aside from `logstash.yml` we can manage Logstashs pipelines. * *logstash_mermaid*: Print overview over Logstash pipelines in Mermaid syntax. (default: `true`) * *logstash_mermaid_logstash*: Place Mermaid syntax into `/etc/logstash/pipelines.mermaid` on Logstash hosts. (default: `true`) -* *logstash_mermaid_local*: Place Mermaid syntax into temporary file on control node. (default: `true`) +* *logstash_mermaid_local*: Place Mermaid syntax into temporary file on control node. (default: `false`) * *logstash_mermaid_extra*: You can add extra Mermaid syntax to the output by adding it to this variable. YAML-multiline is supported. (default: none) The following variables configure Log4j for Logstash. All default to `true` as this is the default after the installation. diff --git a/molecule/elasticstack_default/converge.yml b/molecule/elasticstack_default/converge.yml index 0d47fcd5..694db61d 100644 --- a/molecule/elasticstack_default/converge.yml +++ b/molecule/elasticstack_default/converge.yml @@ -19,7 +19,6 @@ elasticstack_no_log: false logstash_pipeline_unsafe_shutdown: true logstash_redis_password: "{{ lookup('ansible.builtin.password', '/tmp/redispassword', chars=['ascii_letters'], length=15) }}" - logstash_mermaid_local: false # no permission to write into Docker control node redis_requirepass: "{{ logstash_redis_password }}" beats_filebeat_journald: true beats_filebeat_modules: diff --git a/roles/logstash/defaults/main.yml b/roles/logstash/defaults/main.yml index 3a16ba9c..88eead19 100644 --- a/roles/logstash/defaults/main.yml +++ b/roles/logstash/defaults/main.yml @@ -89,7 +89,7 @@ logstash_pipeline_identifier_field_name: "[netways][pipeline]" logstash_pipeline_identifier_defaults: false logstash_mermaid: true -logstash_mermaid_local: true +logstash_mermaid_local: false logstash_mermaid_logstash: true # Only for internal use