From bc970a2f80e3f94c9c68a3c4e5f0b2d42bbf3edf Mon Sep 17 00:00:00 2001 From: Afeef Ghannam Date: Sat, 27 Jun 2026 16:07:49 +0200 Subject: [PATCH 01/28] add meta/argument_specs.yml for all role variables --- roles/logstash/meta/argument_specs.yml | 456 +++++++++++++++++++++++++ 1 file changed, 456 insertions(+) create mode 100644 roles/logstash/meta/argument_specs.yml diff --git a/roles/logstash/meta/argument_specs.yml b/roles/logstash/meta/argument_specs.yml new file mode 100644 index 00000000..52e5719b --- /dev/null +++ b/roles/logstash/meta/argument_specs.yml @@ -0,0 +1,456 @@ +--- +# Role argument specification for the logstash role. +# This is the single source of truth for the role's variables: Ansible validates +# the supplied values against it at role start, and ansible-docsmith renders the +# README variable reference and the comments in defaults/main.yml from it. +# +# Only logstash_* variables live here. The shared elasticstack_* variables are +# documented centrally with the elasticstack role. + +argument_specs: + main: + short_description: Install and configure Logstash + description: + - Installs and configures Logstash on Linux systems. + - >- + Can manage logstash.yml, log4j2 logging, pipelines (default Redis/Beats + pipelines and pipelines from external git repositories), TLS and the + Elasticsearch writer role/user, and the JVM heap size. + author: + - NETWAYS GmbH + + options: + + # ----- Service & general ----- + logstash_enable: + type: bool + default: true + description: Start and enable the Logstash service. + + logstash_config_backup: + type: bool + default: false + description: Keep backups of every configuration file the role changes. + + logstash_manage_yaml: + type: bool + default: true + description: Manage and overwrite logstash.yml. + + logstash_manage_logging: + type: bool + default: false + description: Manage the log4j2 logging configuration. + + logstash_heap: + type: str + default: "1" + description: + - JVM heap size in GB. Sets both -Xms and -Xmx via an LS_JAVA_OPTS systemd drop-in. + - >- + Elastic recommends 4-8 GB for typical ingestion and staying below 50-75% + of physical memory. Keep it low when Logstash shares a host with + Elasticsearch so they do not compete for memory. + + logstash_plugins: + type: list + elements: str + description: List of Logstash plugins to install. Unset by default. + + # ----- logstash.yml settings ----- + logstash_config_autoreload: + type: bool + default: true + description: Enable automatic reload of the Logstash configuration. + + logstash_config_autoreload_interval: + type: str + description: >- + Interval between configuration reload checks (e.g. "3s"). Only used + when logstash_config_autoreload is enabled. Unset by default. + + logstash_config_path_data: + type: str + default: /var/lib/logstash + description: Logstash data directory (path.data). + + logstash_config_path_logs: + type: str + default: /var/log/logstash + description: Logstash log directory (path.logs). + + logstash_http_host: + type: str + description: Bind address of the Logstash monitoring API (http.host). Unset by default. + + logstash_http_port: + type: str + description: Port (or port range, e.g. "9600-9700") of the monitoring API (http.port). Unset by default. + + logstash_global_ecs: + type: str + choices: + - disabled + - v1 + description: Set the global ECS compatibility mode (pipeline.ecs_compatibility). Unset by default. + + logstash_pipeline_unsafe_shutdown: + type: bool + description: Force Logstash to exit during shutdown even if there are in-flight events. Unset by default. + + logstash_legacy_monitoring: + type: bool + default: true + description: >- + Enable legacy X-Pack monitoring. Ignored unless elasticstack_full_stack + is set and only effective on Elastic Stack releases lower than 8. + + # ----- Pipelines ----- + logstash_manage_pipelines: + type: bool + default: true + description: Manage pipelines.yml. + + logstash_no_pipelines: + type: bool + default: false + description: Disable all pipeline management entirely. + + logstash_pipelines: + type: list + elements: dict + description: + - List of pipelines to configure. + - >- + Each entry needs a name. A pipeline either points to an external git + repository (source/version) or defines simple input/output keys that + connect to Redis. See the pipelines documentation for details. + default: + - name: default + exclusive: false + queue_type: memory + queue_max_bytes: 1gb + input: + - name: default + key: input + output: + - name: default + key: forwarder + options: + name: + type: str + required: true + description: Unique name of the pipeline. Becomes the pipeline directory and pipeline.id. + exclusive: + type: bool + description: Mark this pipeline as the only one allowed to handle its events. + source: + type: str + description: URL of a git repository holding the pipeline configuration. + version: + type: str + description: Git branch/tag/commit to check out from source (default "main"). + queue_type: + type: str + choices: + - memory + - persisted + description: Queue type for this pipeline (default "memory"). + queue_max_bytes: + type: str + description: Maximum queue size for this pipeline (default "1gb"). + input: + type: list + elements: dict + description: Simple Redis inputs for this pipeline. + options: + name: + type: str + description: Name of the input. + key: + type: str + description: Redis key to read from. + output: + type: list + elements: dict + description: Simple Redis outputs for this pipeline. + options: + name: + type: str + description: Name of the output. + key: + type: str + description: Redis key to write to. + + logstash_elasticsearch_output: + type: bool + default: true + description: Create the default pipeline that forwards events to Elasticsearch. + + logstash_beats_input: + type: bool + default: true + description: Create the default pipeline with a Beats input. + + logstash_beats_input_congestion: + type: int + description: Congestion threshold (seconds) for the Beats input pipeline. Unset by default. + + logstash_beats_timeout: + type: str + description: Timeout for idle client connections on the Beats input (e.g. "60s"). Unset by default. + + logstash_beats_tls: + type: bool + description: >- + Activate TLS on the Beats input pipeline. Unset by default, but enabled + automatically in a full stack setup unless overridden. + + logstash_input_queue_type: + type: str + choices: + - memory + - persisted + default: memory + description: Queue type for the default Beats input pipeline. + + logstash_input_queue_max_bytes: + type: str + default: 1gb + description: Maximum queue size for the default Beats input pipeline. + + logstash_forwarder_queue_type: + type: str + choices: + - memory + - persisted + default: memory + description: Queue type for the default Elasticsearch forwarder pipeline. + + logstash_forwarder_queue_max_bytes: + type: str + default: 1gb + description: Maximum queue size for the default Elasticsearch forwarder pipeline. + + logstash_queue_type: + type: str + choices: + - persisted + - memory + default: persisted + description: Default queue type Logstash uses for pipelines. + + logstash_redis_password: + type: str + description: Password used when the simple inputs/outputs connect to Redis. Unset by default. + + # ----- Elasticsearch connection / output ----- + logstash_elasticsearch: + type: list + elements: str + description: >- + Elasticsearch hosts for the default output. Defaults to the nodes from + the elasticsearch group, or localhost when used standalone. + + logstash_validate_after_inactivity: + type: str + default: "300" + description: >- + Seconds Logstash waits before validating a previously idle connection to + Elasticsearch. + + logstash_sniffing: + type: bool + default: false + description: Enable sniffing for additional Elasticsearch nodes. + + logstash_sniffing_delay: + type: str + description: Seconds to wait between sniffing attempts. Unset by default. + + logstash_sniffing_path: + type: str + description: HTTP path used for the sniffing requests. Unset by default. + + # ----- Security: certificates ----- + logstash_security: + type: bool + description: >- + Enable X-Pack security (TLS) for Logstash. No default; enabled + automatically in full stack mode with the elastic variant. + + logstash_tls_key_passphrase: + type: str + default: LogstashChangeMe + description: Passphrase for the generated Logstash certificates. + + logstash_certs_dir: + type: str + default: /etc/logstash/certs + description: Directory holding the Logstash certificates. Used to build several file paths. + + logstash_cert_validity_period: + type: int + default: 1095 + description: Number of days the generated certificates are valid. + + logstash_cert_expiration_buffer: + type: int + default: 30 + description: >- + Renew the certificate when its remaining validity (in days) drops below + this value. + + logstash_cert_will_expire_soon: + type: bool + default: false + description: >- + Set to true to force renewal of the Logstash certificate. Alternatively + run the playbook with the renew_logstash_cert tag. + + # ----- Security: Elasticsearch role & user ----- + logstash_create_role: + type: bool + default: true + description: Create the Logstash writer role in Elasticsearch. + + logstash_role_name: + type: str + default: logstash_writer + description: Name of the Logstash writer role. + + logstash_role_cluster_privileges: + type: list + elements: str + default: + - manage_index_templates + - monitor + - manage_ilm + description: Cluster privileges granted to the Logstash writer role. + + logstash_role_indicies_names: + type: list + elements: str + default: + - "ecs-logstash*" + - "logstash*" + - "logs*" + description: Index patterns the Logstash writer role may access. + + logstash_role_indicies_privileges: + type: list + elements: str + default: + - write + - create + - delete + - create_index + - manage + - manage_ilm + description: Index privileges the Logstash writer role holds on its index patterns. + + logstash_reset_writer_role: + type: bool + default: true + description: Reset the writer role and user on every run. + + logstash_create_user: + type: bool + default: true + description: Create the Logstash writer user in Elasticsearch. + + logstash_user_name: + type: str + default: logstash_writer + description: Name of the Logstash user connecting to Elasticsearch. + + logstash_user_password: + type: str + default: password + description: Password of the Logstash user. Must be at least 6 characters long. + + logstash_user_email: + type: str + default: "" + description: Email address linked to the Logstash user. + + logstash_user_fullname: + type: str + default: Internal Logstash User + description: Full name linked to the Logstash user. + + # ----- log4j2 logging ----- + logstash_logging_console: + type: bool + default: true + description: Log to the console (syslog when run via systemd). + + logstash_logging_file: + type: bool + default: true + description: Log to the log file. + + logstash_logging_slow_console: + type: bool + default: true + description: Log the slowlog to the console (syslog when run via systemd). + + logstash_logging_slow_file: + type: bool + default: true + description: Log the slowlog to the log file. + + # ----- Identifying fields ----- + logstash_ident: + type: bool + default: true + description: Add a field identifying the node that processed an event. + + logstash_ident_field_name: + type: str + default: "[netways][instance]" + description: Name of the field that identifies the instance. + + logstash_pipeline_identifier: + type: bool + default: true + description: Add a field identifying which pipeline processed an event. + + logstash_pipeline_identifier_field_name: + type: str + default: "[netways][pipeline]" + description: Name of the pipeline identifier field. + + logstash_pipeline_identifier_defaults: + type: bool + default: false + description: >- + Also add pipeline identifiers in the default pipelines. This can let the + defaults dominate statistics with little value. + + # ----- Mermaid pipeline overview ----- + logstash_mermaid: + type: bool + default: true + description: Produce an overview of the Logstash pipelines in Mermaid syntax. + + logstash_mermaid_logstash: + type: bool + default: true + description: Place the Mermaid syntax into /etc/logstash/pipelines.mermaid on the Logstash hosts. + + logstash_mermaid_local: + type: bool + default: false + description: Place the Mermaid syntax into a temporary file on the control node. + + logstash_mermaid_extra: + type: str + description: Extra Mermaid syntax to append to the output (YAML multiline supported). Unset by default. + + # ----- Internal ----- + logstash_freshstart: + type: dict + default: + changed: false + description: Internal state used by the role to detect a fresh install. Do not set manually. From 3b2968ace106fc6ac8aadf5b155f87f69518ba29 Mon Sep 17 00:00:00 2001 From: Afeef Ghannam Date: Sat, 27 Jun 2026 16:36:04 +0200 Subject: [PATCH 02/28] Add shared compact ansible-docsmith README template --- .docsmith/readme.md.j2 | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 .docsmith/readme.md.j2 diff --git a/.docsmith/readme.md.j2 b/.docsmith/readme.md.j2 new file mode 100644 index 00000000..22a9f4d6 --- /dev/null +++ b/.docsmith/readme.md.j2 @@ -0,0 +1,17 @@ +{# Compact README variable reference for ansible-docsmith. + Shared by all roles in this collection. Renders ONE table with the full + description in each cell (no truncation, no per-variable detail sections, + no table of contents). Pass it with: --template-readme .docsmith/readme.md.j2 #} +## Role variables + +{% if has_options %} +The following variables can be configured for this role: + +| Variable | Type | Default | Choices | Description | +|----------|------|---------|---------|-------------| +{% for var_name, var_spec in options.items() %} +| `{{ var_name | ansible_escape }}` | `{{ var_spec.type }}`{% if var_spec.elements %} of `{{ var_spec.elements }}`{% endif %} | {{ var_spec.default | format_default }} | {% if var_spec.choices %}{{ var_spec.choices | map('string') | map('code_escape') | join(', ') }}{% else %}—{% endif %} | {{ var_spec.description | format_table_description(var_name, 0) }} | +{% endfor %} +{% else %} +No variables are defined for this role. +{% endif %} From 26512d76fd23a763e879b8124392998ab94fc8f2 Mon Sep 17 00:00:00 2001 From: Afeef Ghannam Date: Sat, 27 Jun 2026 16:37:04 +0200 Subject: [PATCH 03/28] Add README generated from argument_specs --- roles/logstash/README.md | 120 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 120 insertions(+) create mode 100644 roles/logstash/README.md diff --git a/roles/logstash/README.md b/roles/logstash/README.md new file mode 100644 index 00000000..fa932001 --- /dev/null +++ b/roles/logstash/README.md @@ -0,0 +1,120 @@ +# Ansible Role: Logstash + +![Test Role Logstash](https://github.com/netways/ansible-collection-elasticstack/actions/workflows/test_role_logstash.yml/badge.svg) + +Installs and configures [Logstash](https://www.elastic.co/products/logstash) on +Linux systems. The role can manage `logstash.yml`, the log4j2 logging, the JVM +heap, TLS, the Elasticsearch writer role/user, and pipelines — both the built-in +Redis/Beats pipelines and pipelines pulled from external git repositories. + +It works with the standard Elastic Stack packages and with Elastic's OSS variant. + +## Requirements + +* The `community.general` collection. +* `passlib` (Python, on the controller) unless you disable password hashing for the Logstash user. +* `curl` on the target host. +* `git` on the target host if you use git-based pipelines. +* Redis if you use the default pipeline or other pipelines that talk to Redis + (e.g. via the [`geerlingguy.redis`](https://galaxy.ansible.com/geerlingguy/redis) role). + +You also need the Elastic repositories configured — use the [`repos`](../repos) role. + +## Example + +```yaml +- name: Install Logstash + hosts: logstash + collections: + - netways.elasticstack + roles: + - repos + - logstash +``` + +## Pipelines + +For how to configure pipelines (built-in Redis/Beats and external git +repositories), see the [pipelines documentation](docs/pipelines.md). + + +## Role variables + +The following variables can be configured for this role: + +| Variable | Type | Default | Choices | Description | +|----------|------|---------|---------|-------------| +| `logstash_enable` | `bool` | `true` | — | Start and enable the Logstash service. | +| `logstash_config_backup` | `bool` | `false` | — | Keep backups of every configuration file the role changes. | +| `logstash_manage_yaml` | `bool` | `true` | — | Manage and overwrite logstash.yml. | +| `logstash_manage_logging` | `bool` | `false` | — | Manage the log4j2 logging configuration. | +| `logstash_heap` | `str` | `"1"` | — | JVM heap size in GB. Sets both -Xms and -Xmx via an LS_JAVA_OPTS systemd drop-in.

Elastic recommends 4-8 GB for typical ingestion and staying below 50-75% of physical memory. Keep it low when Logstash shares a host with Elasticsearch so they do not compete for memory. | +| `logstash_plugins` | `list` of `str` | N/A | — | List of Logstash plugins to install. Unset by default. | +| `logstash_config_autoreload` | `bool` | `true` | — | Enable automatic reload of the Logstash configuration. | +| `logstash_config_autoreload_interval` | `str` | N/A | — | Interval between configuration reload checks (e.g. "3s"). Only used when logstash_config_autoreload is enabled. Unset by default. | +| `logstash_config_path_data` | `str` | `"/var/lib/logstash"` | — | Logstash data directory (path.data). | +| `logstash_config_path_logs` | `str` | `"/var/log/logstash"` | — | Logstash log directory (path.logs). | +| `logstash_http_host` | `str` | N/A | — | Bind address of the Logstash monitoring API (http.host). Unset by default. | +| `logstash_http_port` | `str` | N/A | — | Port (or port range, e.g. "9600-9700") of the monitoring API (http.port). Unset by default. | +| `logstash_global_ecs` | `str` | N/A | `disabled`, `v1` | Set the global ECS compatibility mode (pipeline.ecs_compatibility). Unset by default. | +| `logstash_pipeline_unsafe_shutdown` | `bool` | N/A | — | Force Logstash to exit during shutdown even if there are in-flight events. Unset by default. | +| `logstash_legacy_monitoring` | `bool` | `true` | — | Enable legacy X-Pack monitoring. Ignored unless elasticstack_full_stack is set and only effective on Elastic Stack releases lower than 8. | +| `logstash_manage_pipelines` | `bool` | `true` | — | Manage pipelines.yml. | +| `logstash_no_pipelines` | `bool` | `false` | — | Disable all pipeline management entirely. | +| `logstash_pipelines` | `list` of `dict` | `[{'name': 'default', 'exclusive': False, 'queue_type': 'memory', 'queue_max_bytes': '1gb', 'input': [{'name': 'default', 'key': 'input'}], 'output': [{'name': 'default', 'key': 'forwarder'}]}]` | — | List of pipelines to configure.

Each entry needs a name. A pipeline either points to an external git repository (source/version) or defines simple input/output keys that connect to Redis. See the pipelines documentation for details. | +| `logstash_elasticsearch_output` | `bool` | `true` | — | Create the default pipeline that forwards events to Elasticsearch. | +| `logstash_beats_input` | `bool` | `true` | — | Create the default pipeline with a Beats input. | +| `logstash_beats_input_congestion` | `int` | N/A | — | Congestion threshold (seconds) for the Beats input pipeline. Unset by default. | +| `logstash_beats_timeout` | `str` | N/A | — | Timeout for idle client connections on the Beats input (e.g. "60s"). Unset by default. | +| `logstash_beats_tls` | `bool` | N/A | — | Activate TLS on the Beats input pipeline. Unset by default, but enabled automatically in a full stack setup unless overridden. | +| `logstash_input_queue_type` | `str` | `"memory"` | `memory`, `persisted` | Queue type for the default Beats input pipeline. | +| `logstash_input_queue_max_bytes` | `str` | `"1gb"` | — | Maximum queue size for the default Beats input pipeline. | +| `logstash_forwarder_queue_type` | `str` | `"memory"` | `memory`, `persisted` | Queue type for the default Elasticsearch forwarder pipeline. | +| `logstash_forwarder_queue_max_bytes` | `str` | `"1gb"` | — | Maximum queue size for the default Elasticsearch forwarder pipeline. | +| `logstash_queue_type` | `str` | `"persisted"` | `persisted`, `memory` | Default queue type Logstash uses for pipelines. | +| `logstash_redis_password` | `str` | N/A | — | Password used when the simple inputs/outputs connect to Redis. Unset by default. | +| `logstash_elasticsearch` | `list` of `str` | N/A | — | Elasticsearch hosts for the default output. Defaults to the nodes from the elasticsearch group, or localhost when used standalone. | +| `logstash_validate_after_inactivity` | `str` | `"300"` | — | Seconds Logstash waits before validating a previously idle connection to Elasticsearch. | +| `logstash_sniffing` | `bool` | `false` | — | Enable sniffing for additional Elasticsearch nodes. | +| `logstash_sniffing_delay` | `str` | N/A | — | Seconds to wait between sniffing attempts. Unset by default. | +| `logstash_sniffing_path` | `str` | N/A | — | HTTP path used for the sniffing requests. Unset by default. | +| `logstash_security` | `bool` | N/A | — | Enable X-Pack security (TLS) for Logstash. No default; enabled automatically in full stack mode with the elastic variant. | +| `logstash_tls_key_passphrase` | `str` | `"LogstashChangeMe"` | — | Passphrase for the generated Logstash certificates. | +| `logstash_certs_dir` | `str` | `"/etc/logstash/certs"` | — | Directory holding the Logstash certificates. Used to build several file paths. | +| `logstash_cert_validity_period` | `int` | `1095` | — | Number of days the generated certificates are valid. | +| `logstash_cert_expiration_buffer` | `int` | `30` | — | Renew the certificate when its remaining validity (in days) drops below this value. | +| `logstash_cert_will_expire_soon` | `bool` | `false` | — | Set to true to force renewal of the Logstash certificate. Alternatively run the playbook with the renew_logstash_cert tag. | +| `logstash_create_role` | `bool` | `true` | — | Create the Logstash writer role in Elasticsearch. | +| `logstash_role_name` | `str` | `"logstash_writer"` | — | Name of the Logstash writer role. | +| `logstash_role_cluster_privileges` | `list` of `str` | `['manage_index_templates', 'monitor', 'manage_ilm']` | — | Cluster privileges granted to the Logstash writer role. | +| `logstash_role_indicies_names` | `list` of `str` | `['ecs-logstash*', 'logstash*', 'logs*']` | — | Index patterns the Logstash writer role may access. | +| `logstash_role_indicies_privileges` | `list` of `str` | `['write', 'create', 'delete', 'create_index', 'manage', 'manage_ilm']` | — | Index privileges the Logstash writer role holds on its index patterns. | +| `logstash_reset_writer_role` | `bool` | `true` | — | Reset the writer role and user on every run. | +| `logstash_create_user` | `bool` | `true` | — | Create the Logstash writer user in Elasticsearch. | +| `logstash_user_name` | `str` | `"logstash_writer"` | — | Name of the Logstash user connecting to Elasticsearch. | +| `logstash_user_password` | `str` | `"password"` | — | Password of the Logstash user. Must be at least 6 characters long. | +| `logstash_user_email` | `str` | `""` | — | Email address linked to the Logstash user. | +| `logstash_user_fullname` | `str` | `"Internal Logstash User"` | — | Full name linked to the Logstash user. | +| `logstash_logging_console` | `bool` | `true` | — | Log to the console (syslog when run via systemd). | +| `logstash_logging_file` | `bool` | `true` | — | Log to the log file. | +| `logstash_logging_slow_console` | `bool` | `true` | — | Log the slowlog to the console (syslog when run via systemd). | +| `logstash_logging_slow_file` | `bool` | `true` | — | Log the slowlog to the log file. | +| `logstash_ident` | `bool` | `true` | — | Add a field identifying the node that processed an event. | +| `logstash_ident_field_name` | `str` | `"[netways][instance]"` | — | Name of the field that identifies the instance. | +| `logstash_pipeline_identifier` | `bool` | `true` | — | Add a field identifying which pipeline processed an event. | +| `logstash_pipeline_identifier_field_name` | `str` | `"[netways][pipeline]"` | — | Name of the pipeline identifier field. | +| `logstash_pipeline_identifier_defaults` | `bool` | `false` | — | Also add pipeline identifiers in the default pipelines. This can let the defaults dominate statistics with little value. | +| `logstash_mermaid` | `bool` | `true` | — | Produce an overview of the Logstash pipelines in Mermaid syntax. | +| `logstash_mermaid_logstash` | `bool` | `true` | — | Place the Mermaid syntax into /etc/logstash/pipelines.mermaid on the Logstash hosts. | +| `logstash_mermaid_local` | `bool` | `false` | — | Place the Mermaid syntax into a temporary file on the control node. | +| `logstash_mermaid_extra` | `str` | N/A | — | Extra Mermaid syntax to append to the output (YAML multiline supported). Unset by default. | +| `logstash_freshstart` | `dict` | `{'changed': False}` | — | Internal state used by the role to detect a fresh install. Do not set manually. | + + + +## Shared variables + +This role also uses the collection-wide `elasticstack_*` variables (e.g. +`elasticstack_full_stack`, `elasticstack_ca_host`, `elasticstack_ca_pass`, +`elasticstack_release`, `elasticstack_variant`). They are documented centrally +with the [elasticstack role](../../docs/role-elasticsearch.md). From 8f72f106ecedc70841b3aad7df400f6aea2624b9 Mon Sep 17 00:00:00 2001 From: Afeef Ghannam Date: Sat, 27 Jun 2026 16:41:14 +0200 Subject: [PATCH 04/28] Make logstash_config_backup default an explicit boolean --- roles/logstash/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/logstash/defaults/main.yml b/roles/logstash/defaults/main.yml index 007a4d15..477fd8fe 100644 --- a/roles/logstash/defaults/main.yml +++ b/roles/logstash/defaults/main.yml @@ -1,7 +1,7 @@ --- # defaults file for logstash logstash_enable: true -logstash_config_backup: no +logstash_config_backup: false logstash_manage_yaml: true logstash_manage_logging: false From 37afcab99ee868b718566a4c7bb099e7b3f8cb85 Mon Sep 17 00:00:00 2001 From: Afeef Ghannam Date: Sat, 27 Jun 2026 16:49:45 +0200 Subject: [PATCH 05/28] Add documentation gate (argument_specs validate + README freshness) --- .github/workflows/test_docs.yml | 73 +++++++++++++++++++++++++++++++++ 1 file changed, 73 insertions(+) create mode 100644 .github/workflows/test_docs.yml diff --git a/.github/workflows/test_docs.yml b/.github/workflows/test_docs.yml new file mode 100644 index 00000000..414f5268 --- /dev/null +++ b/.github/workflows/test_docs.yml @@ -0,0 +1,73 @@ +--- +name: Test Documentation +on: + workflow_dispatch: + pull_request: + paths: + - 'roles/**/meta/argument_specs.yml' + - 'roles/**/defaults/**' + - 'roles/**/README.md' + - '.docsmith/**' + - '.github/workflows/test_docs.yml' + +# Cancel outdated pipeline runs when a new push occurs in the pull request. +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + docs: + # For non-PR events (workflow_dispatch, etc.), github.event.pull_request.draft is null. + # GitHub Actions coerces null == false to true, so the job runs for all non-draft events. + if: github.event.pull_request.draft == false + runs-on: ubuntu-latest + env: + NO_COLOR: "1" # make ansible-docsmith print plain text so grep is reliable + steps: + - name: Check out the codebase. + uses: actions/checkout@v6 + + - name: Set up Python 3.11 + uses: actions/setup-python@v6 + with: + python-version: "3.11" + + # Pin the version: ansible-docsmith is a young, single-maintainer tool, so we + # upgrade deliberately rather than picking up changes on every CI run. + - name: Install ansible-docsmith + run: python3 -m pip install ansible-docsmith==2.0.2 + + # Consistency between meta/argument_specs.yml and defaults/main.yml. + # validate exits 1 on ERROR by itself; here we additionally treat any + # WARNING (e.g. a default that differs between spec and defaults) as a failure. + - name: Validate argument_specs (warnings = errors) + run: | + set -uo pipefail + status=0 + for role in roles/*/; do + [ -f "${role}meta/argument_specs.yml" ] || continue + echo "::group::validate ${role}" + out=$(ansible-docsmith validate "${role}" 2>&1) && rc=0 || rc=$? + echo "$out" + echo "::endgroup::" + if [ "$rc" -ne 0 ] || grep -q "Warnings:" <<<"$out"; then + echo "::error::ansible-docsmith reported errors or warnings for ${role}" + status=1 + fi + done + exit $status + + # README freshness: regenerate every role README and fail if anything changed, + # i.e. the committed README no longer matches argument_specs.yml. + - name: Check that every README is up to date + run: | + for role in roles/*/; do + [ -f "${role}meta/argument_specs.yml" ] || continue + ansible-docsmith generate "${role}" --no-defaults \ + --template-readme .docsmith/readme.md.j2 + done + git diff --exit-code -- 'roles/*/README.md' || { + echo "::error::A role README is out of date. Run locally:"; + echo " ansible-docsmith generate --no-defaults --template-readme .docsmith/readme.md.j2"; + exit 1; + } From 09dd2ef512350aa4233c5526984bb32eefb289bc Mon Sep 17 00:00:00 2001 From: Afeef Ghannam Date: Sat, 27 Jun 2026 17:22:18 +0200 Subject: [PATCH 06/28] Point docs to role README, ignore .docsmith in build --- README.md | 2 +- docs/getting-started.md | 2 +- docs/role-logstash.md | 136 ++-------------------------------------- galaxy.yml | 1 + 4 files changed, 9 insertions(+), 132 deletions(-) diff --git a/README.md b/README.md index 99b4389f..7106f822 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ Every role is documented with all variables, please refer to the documentation f * [Beats](docs/role-beats.md) * [Elasticsearch](docs/role-elasticsearch.md) * [Kibana](docs/role-kibana.md) -* [Logstash](docs/role-logstash.md) +* [Logstash](roles/logstash/README.md) * [Repos](docs/role-repos.md) ## Modules documentation diff --git a/docs/getting-started.md b/docs/getting-started.md index 79569b50..dc49fb40 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -50,7 +50,7 @@ Roles * [Beats](role-beats.md) * [Elasticsearch](role-elasticsearch.md) * [Kibana](role-kibana.md) -* [Logstash](role-logstash.md) +* [Logstash](../roles/logstash/README.md) * [Repos](role-repos.md) diff --git a/docs/role-logstash.md b/docs/role-logstash.md index d42f7da2..ae7e9bf3 100644 --- a/docs/role-logstash.md +++ b/docs/role-logstash.md @@ -1,133 +1,9 @@ -Ansible Role: Logstash -========= +# Logstash role -![Test Role Logstash](https://github.com/netways/ansible-collection-elasticstack/actions/workflows/test_role_logstash.yml/badge.svg) +The Logstash role documentation now lives with the role itself: -This role installs and configures [Logstash](https://www.elastic.co/products/logstash) on Linux systems. +**➜ [roles/logstash/README.md](../roles/logstash/README.md)** -It can optionally configure two types of Logstash pipelines: -* Pipeline configuration managed in an external git repository -* A default pipeline which will read from different Redis keys and write into Elasticsearch - -For details on how to configure pipelines please refer to our [docs about pipelines](./logstash-pipelines.md). - -Details about configured pipelines will be written into `pipelines.yml` as comments. Same goes for logging configuration in `log4j.options`. - -It will work with the standard Elastic Stack packages and Elastics OSS variant. - -Requirements ------------- - -* `community.general` collection - -You will need these packages / libraries installed. Some very basic packages like `openssl` get handled by the collection if needed. The following list contains packages and libraries which only apply to special cases or need for you to decide on the installation method. - -* `passlib` Python library if you do not disable password hashing for logstash user. It should be installed with pip on the Ansible controller. - -You need to have the Elastic Repos configured on your system. You can use our [role](./role-repos.md) - -If you want to use the default pipeline configuration you need to have `git` available. - -You need to have `curl` installed. We are using `curl` instead of the `uri` module because we got better results. Feel free to file a pull request if you find a working solution. (And please remove this line with it) - -If you want to use the default pipeline (or other pipelines communicating via Redis) you might want to install Redis first (e.g. by using an [Ansible Role for Redis](https://galaxy.ansible.com/geerlingguy/redis) - -Role Variables --------------- - -* *elasticstack_version*: Version number of Logstash to install (e.g. `7.10.1`). Only set if you don't want the latest. (default: none). For OSS version see `elasticstack_variant` below. -* *logstash_enable*: Start and enable Logstash service (default: `true`) -* *logstash_config_backup*: Keep backups of all changed configuration (default: `no`) -* *logstash_manage_yaml*: Manage and overwrite `logstash.yml` (default: `true`) -* *logstash_manage_logging*: Manage log4j configuration (default: `false`) -* *logstash_plugins*: List of plugins to install (default: none) -* *logstash_certs_dir*: Path to certificates. Will be used to build paths of several files. (Default: `/etc/logstash/certs`) -* *logstash_heap*: JVM heap size in GB (default: `1`). Elastic recommends 4-8 GB for typical ingestion and staying below 50-75% of physical memory. Keep it low when Logstash shares a host with Elasticsearch so they do not compete for memory. - -If `logstash.yml` is managed, the following settings apply. - -* *logstash_config_autoreload*: Enable autoreload of Logstash configuration (default: `true`) -* *logstash_config_path_data*: Logstash data directory (default: `/var/lib/logstash`) -* *logstash_config_path_logs*: Logstash log directory (default: `/var/log/logstash`) - -Aside from `logstash.yml` we can manage Logstashs pipelines. - -* *logstash_manage_pipelines*: Manage `pipelines.yml` (default: `true`) -* *logstash_no_pipelines*: Don't manage pipelines at all (default: `false`) -* *logstash_pipelines*: List of pipelines with optional URL to repo (see [pipelines documentation](file:///roles/logstash/docs/pipelines.md) for details) -* *logstash_global_ecs*: Set ECS compatibilty mode (default: none. Possible values: `disabled` or `v1`) -* *logstash_elasticsearch_output*: Enable default pipeline to Elasticsearch (default: `true`) -* *logstash_ident*: Add a field identifying the node that processed an event (default: `true`) -* *logstash_ident_field_name*: Name of the identifying the instance (default: `"[netways][instance]"`) -* *logstash_beats_input*: Enable default pipeline with `beats` input (default: `true`) -* *logstash_beats_input_congestion*: Optional congestion threshold for the beats input pipeline -* *logstash_beats_timeout*: Optional timeout for client connections. (Example: `60s`) -* *logstash_beats_tls*: Activate TLS for the beats input pipeline (default: none but `true` with full stack setup if not set) -* *logstash_tls_key_passphrase*: Passphrase for Logstash certificates (default: `LogstashChangeMe`) -* *elasticstack_ca_pass*: Password for Elasticsearch CA (default: `PleaseChangeMe`) -* *logstash_cert_validity_period*: number of days that the generated certificates are valid (default: 1095). -* *logstash_cert_expiration_buffer*: Ansible will renew the Logstash certificate if its validity is shorter than this value, which should be number of days. (default: 30) -* *logstash_cert_will_expire_soon*: Set it to true to renew logstash certificate (default: `false`), Or run the playbook with `--tags renew_logstash_cert` to do that. -* *logstash_elasticsearch*: Address of Elasticsearch instance for default output (default: list of Elasticsearch nodes from `elasticsearch` role or `localhost` when used standalone) -* *logstash_security*: Enable X-Security (No default set, but will be activated when in full stack mode) -* *logstash_create_user*: Enables creation `logstash_user_name` (Default: `true`) -* *logstash_user_name*: Name of the user to connect to Elasticsearch (Default: `logstash_writer`) -* *logstash_user_email*: email-address that is linked with the logstash_user_name (Default: `""`) -* *logstash_user_fullname*: fullname that is linked with the logstash_user_name (Default: `Internal Logstash User`) -* *logstash_user_password*: Password of `logstash_user_name` in Elasticsearch. It must be at least 6 characters long (default: `password`) -* *logstash_create_role*: Enables creation `logstash_role_name` (Default: `true`) -* *logstash_role_name*: Name of the logstash role that is getting created (Default: `logstash_writer`) -* *logstash_role_cluster_privileges*: Cluster privileges the role has access to (default: `"manage_index_templates", "monitor", "manage_ilm"`) -* *logstash_role_indicies_names*: Indices the role has access to (default: `"ecs-logstash*", "logstash*", "logs*"`) -* *logstash_role_indicies_privileges*: Index permissions the role has on `logstash_role_indicies_names` (default: `"write", "create", "delete", "create_index", "manage", "manage_ilm"`) -* *logstash_reset_writer_role*: Reset user and role with every run: (default: `true`) -* *logstash_validate_after_inactivity*: How long should logstash wait, before starting a new connection and leave the old one with elasticsearch, when the connection with elasticsearch get lost: (Default: `300`). -* *logstash_queue_type*: What kind of queue should Logstash use per default: (Default: `persisted`, alternative: `memory`) -* *logstash_queue_max_bytes*: The total capacity of ansible-forwarder queue in number of bytes: (Default: `2gb`) -* *logstash_sniffing*: Enable sniffing (Default: `false`). -* *logstash_sniffing_delay*: How long to wait, in seconds, between sniffing attempts (Default: `not set`). -* *logstash_sniffing_path*: HTTP Path to be used for the sniffing requests (Default: `not set`). -* *logstash_legacy_monitoring*: Enables legacy monitoring - ignored when `elasticstack_full_stack` is not set. (default: `true`) -* *logstash_redis_password*: If set this will use this password when connecting our simple inputs and outputs to Redis. (default: not set) - -* *logstash_mermaid*: Print overview over Logstash pipelines in Mermaid syntax. (default: `true`) -* *logstash_mermaid_logstash*: Place Mermaid syntax into `/etc/logstash/pipelines.mermaid` on Logstash hosts. (default: `true`) -* *logstash_mermaid_local*: Place Mermaid syntax into temporary file on control node. (default: `false`) -* *logstash_mermaid_extra*: You can add extra Mermaid syntax to the output by adding it to this variable. YAML-multiline is supported. (default: none) - -The following variables configure Log4j for Logstash. All default to `true` as this is the default after the installation. - -* *logstash_logging_console*: Log to console - syslog when run via systemd -* *logstash_logging_file*: Log to logfile -* *logstash_logging_slow_console*: Log slowlog to console - syslog when run via systemd -* *logstash_logging_slow_file*: Log slowlog to logfile - -The following variables configure extra fields in your events that help with identifying which pipelines have been passed or which pipeline is used how much. - -* *logstash_pipeline_identifier*: Activate this feature (default: `true`) -* *logstash_pipeline_identifier_field_name*: Name of the field to add (default: `"[netways][pipeline]"`) -* *logstash_pipeline_identifier_defaults*: Use identifiers in default pipelines, too (default: `false`) This could lead to the defaults dominating all statistics with virtually no value, but if you want to see, them, you can. - -The following variables are identical over all our elastic related roles, hence the different naming scheme. - -*elasticstack_release*: Major release version of Elastic stack to configure. (default: `7`) -*elasticstack_variant*: Variant of the stack to install. Valid values: `elastic` or `oss`. (default: `elastic`) - -The following variables only apply if you use this role together with our Elasticsearch and Kibana roles. - -* *elasticstack_full_stack*: Use `ansible-role-elasticsearch` as well (default: `false`) -* *elasticstack_ca_dir*: Directory where the CA and certificates lie on the main Elasticsearch host (default: `/opt/es-ca`) -* *elasticstack_elasticsearch_http_port*: Port of Elasticsearch to send events to (Default: `9200`) -* *elasticstack_initial_passwords*: File where initial passwords are stored on the main Elasticsearch host (default: `/usr/share/elasticsearch/initial_passwords`) - -## Usage - -``` -- name: Install Logstash - hosts: logstash-host - collections: - - netways.elasticstack - roles: - - repos - - logstash -``` +It covers requirements, usage, the full variable reference (generated from +`roles/logstash/meta/argument_specs.yml`) and the +[pipelines documentation](../roles/logstash/docs/pipelines.md). diff --git a/galaxy.yml b/galaxy.yml index e719ca01..9c34f7b0 100644 --- a/galaxy.yml +++ b/galaxy.yml @@ -37,4 +37,5 @@ build_ignore: - .mailmap - '*.tar.gz' - 'venv*' + - .docsmith # Minimum Ansible version required: 2.14, maximum supported: 2.20 From d91753b714cdd9653feb0911848b94a57c6e2541 Mon Sep 17 00:00:00 2001 From: Afeef Ghannam Date: Sat, 27 Jun 2026 17:44:30 +0200 Subject: [PATCH 07/28] docs: explain the argument_specs/README workflow for contributors --- README.md | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/README.md b/README.md index 7106f822..f4252d72 100644 --- a/README.md +++ b/README.md @@ -226,6 +226,25 @@ Every kind of contribution is very welcome. Open [issues](https://github.com/NET For now we open pull requests against `main`. We are planning to introduce dedicated branches to support older versions without breaking changes. Since we don't need them for now, please check back with this section because when we decided on how to proceed, you will find the information here. For now `main` always has the newest changes and if you want a stable version, please use the newest release. +### Documentation for role variables + +Role variables are documented from each role's `meta/argument_specs.yml`, which is +the single source of truth. When your pull request changes a role's variables: + +1. Update that role's `meta/argument_specs.yml` (type, default, description). +2. Regenerate the README variable table — please do **not** edit it by hand. The table is + produced by [ansible-docsmith](https://github.com/foundata/ansible-docsmith) + (install with `pip install ansible-docsmith` if you don't have it): + + ``` + ansible-docsmith generate roles/ --no-defaults --template-readme .docsmith/readme.md.j2 + ``` + +3. Commit the regenerated `README.md` together with your change. + +The `Test Documentation` workflow checks that each README matches its +`argument_specs.yml` and fails the pull request if they drift apart. + ## Testing Besides real tests that the developer should do before creating a PR, we built molecule scenarios to test the complete stack. From fff1a7b61b486596489a9d99b2babf39032934e6 Mon Sep 17 00:00:00 2001 From: Afeef Ghannam Date: Mon, 29 Jun 2026 13:14:25 +0200 Subject: [PATCH 08/28] Change workflow conditions --- .github/workflows/test_role_logstash.yml | 1 + .github/workflows/test_roles_pr.yml | 2 ++ 2 files changed, 3 insertions(+) diff --git a/.github/workflows/test_role_logstash.yml b/.github/workflows/test_role_logstash.yml index 0bab065b..57756040 100644 --- a/.github/workflows/test_role_logstash.yml +++ b/.github/workflows/test_role_logstash.yml @@ -15,6 +15,7 @@ on: pull_request: paths: - 'roles/logstash/**' + - '!roles/logstash/**/*.md' - '.github/workflows/test_role_logstash.yml' - 'molecule/logstash_**' diff --git a/.github/workflows/test_roles_pr.yml b/.github/workflows/test_roles_pr.yml index 8ec712b4..483c9bcb 100644 --- a/.github/workflows/test_roles_pr.yml +++ b/.github/workflows/test_roles_pr.yml @@ -15,6 +15,8 @@ on: pull_request: paths: - 'roles/**' + - '!roles/**/*.md' + - '!roles/**/meta/argument_specs.yml' - 'molecule/elasticstack_default/**' - 'requirements-test.txt' - '.github/workflows/test_roles_pr.yml' From 6366799f162566ebdc66d38e2b62baf000b466ae Mon Sep 17 00:00:00 2001 From: Afeef Ghannam Date: Mon, 29 Jun 2026 13:28:03 +0200 Subject: [PATCH 09/28] Remove extra lines --- .github/workflows/test_docs.yml | 2 +- docs/role-logstash.md | 4 ---- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/.github/workflows/test_docs.yml b/.github/workflows/test_docs.yml index 414f5268..c17d4104 100644 --- a/.github/workflows/test_docs.yml +++ b/.github/workflows/test_docs.yml @@ -32,7 +32,7 @@ jobs: with: python-version: "3.11" - # Pin the version: ansible-docsmith is a young, single-maintainer tool, so we + # Pin the version: ansible-docsmith is a young tool, so we # upgrade deliberately rather than picking up changes on every CI run. - name: Install ansible-docsmith run: python3 -m pip install ansible-docsmith==2.0.2 diff --git a/docs/role-logstash.md b/docs/role-logstash.md index ae7e9bf3..15d1fb7f 100644 --- a/docs/role-logstash.md +++ b/docs/role-logstash.md @@ -3,7 +3,3 @@ The Logstash role documentation now lives with the role itself: **➜ [roles/logstash/README.md](../roles/logstash/README.md)** - -It covers requirements, usage, the full variable reference (generated from -`roles/logstash/meta/argument_specs.yml`) and the -[pipelines documentation](../roles/logstash/docs/pipelines.md). From 2bc7d35f62fe093b0d8b1a65182ad199a6fda12e Mon Sep 17 00:00:00 2001 From: Afeef Ghannam Date: Mon, 29 Jun 2026 16:31:58 +0200 Subject: [PATCH 10/28] Uodate the spec file --- roles/logstash/README.md | 2 -- roles/logstash/meta/argument_specs.yml | 11 +++++++++++ 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/roles/logstash/README.md b/roles/logstash/README.md index fa932001..151dd270 100644 --- a/roles/logstash/README.md +++ b/roles/logstash/README.md @@ -1,7 +1,5 @@ # Ansible Role: Logstash -![Test Role Logstash](https://github.com/netways/ansible-collection-elasticstack/actions/workflows/test_role_logstash.yml/badge.svg) - Installs and configures [Logstash](https://www.elastic.co/products/logstash) on Linux systems. The role can manage `logstash.yml`, the log4j2 logging, the JVM heap, TLS, the Elasticsearch writer role/user, and pipelines — both the built-in diff --git a/roles/logstash/meta/argument_specs.yml b/roles/logstash/meta/argument_specs.yml index 52e5719b..cfaf9264 100644 --- a/roles/logstash/meta/argument_specs.yml +++ b/roles/logstash/meta/argument_specs.yml @@ -181,6 +181,17 @@ argument_specs: key: type: str description: Redis key to write to. + condition: + type: str + description: >- + Optional Logstash conditional. The output only receives events + matching it. With exclusive set, the conditions are chained with + else if. + congestion: + type: int + description: >- + Optional congestion threshold. The output stops once the target + Redis key holds more items than this value. logstash_elasticsearch_output: type: bool From ca952c211bba24251fd554fae1d78c3ca5614934 Mon Sep 17 00:00:00 2001 From: Afeef Ghannam Date: Mon, 29 Jun 2026 16:37:09 +0200 Subject: [PATCH 11/28] Fix pipeline doc to list format, merge duplicate into role doc --- docs/logstash-pipelines.md | 238 ------------------------------- roles/logstash/docs/pipelines.md | 62 ++++---- 2 files changed, 35 insertions(+), 265 deletions(-) delete mode 100644 docs/logstash-pipelines.md diff --git a/docs/logstash-pipelines.md b/docs/logstash-pipelines.md deleted file mode 100644 index 496d1ce2..00000000 --- a/docs/logstash-pipelines.md +++ /dev/null @@ -1,238 +0,0 @@ -# Pipelines # - -## Keeping an overview ## - -It can be quite difficult to stay on top of your pipeline configuration because they tend to become very complex. - -This collection will leave some comments about how pipelines are interconnected within the `/etc/logstash/pipelines.yml` configuration file. - -If you set `logstash_mermaid` to `true` (which is the default), then you will also get a new file in `/etc/logstash/pipelines.mermaid`. You can paste it into a Mermaid editor in your documentation tool or in an [online Mermaid editor](https://mermaid.live/). The same content will be available on your control node in a temporary file. You can even add arbitrary code to reflect manually managed pipelines by using `logstash_mermaid_extra`. - -## Git managed ## - -If you have pipeline code managed in (and available via) Git repositories, you can use this role to check them out and integrate them into `pipelines.yml`. - -``` -logstash_pipelines: - syslog: - name: syslog - source: https://github.com/widhalmt/syslog-logstash-pipeline.git -``` - -You can add a `version` attribute to your pipeline. It defaults to `main`. You can use every string, [Ansibles git](https://docs.ansible.com/ansible/latest/collections/ansible/builtin/git_module.html) module accepts. - -## Input and Output ## - -### Basic configuration ### - -To have a single Redis input and output to your files, use this. - -``` -logstash_pipelines: - syslog: - name: syslog - source: https://github.com/netways/syslog-logstash-pipeline.git - exclusive: false - input: - - name: default - key: syslog-input - output: - - name: default - key: syslog-output -``` - -This will result in your pipeline checking out the configuration on GitHub and adding these two extras. In extra files, just shown in one place to safe space. - -``` -input { - redis { - host => "localhost" - data_type => "list" - key => "syslog-input" - } -} -output { - redis { - host => "localhost" - data_type => "list" - key => "syslog-output" - } -} -``` - -### Multiple inputs ### - -Just give more inputs with `name` and `key`. Every key will be read. - -### More complex configuration ### - -If you want a bit more control over which outputs are used, the role offers more sophisticated configuration. - -If you have several outputs that all have conditions, like just send some messages to a development system or only alerts to a monitoring system. - -``` -logstash_pipelines: - syslog: - name: syslog - source: https://github.com/netways/syslog-logstash-pipeline.git - exclusive: false - input: - - name: default - key: input - output: - - name: special - key: myspecial - condition: '[program] == "special"' - - name: special2 - key: myspecial2 - condition: '[program] == "special2"' - - name: default - key: forwarder -``` -This will give you the following configuration: - -``` -input { - -# default output - redis { - host => "localhost" - data_type => "list" - key => "input" - } - -} - -output { - -# special output -if [program] == "special"{ - redis { - host => "localhost" - data_type => "list" - key => "myspecial" - } -} - -# special2 output -if [program] == "special2"{ - redis { - host => "localhost" - data_type => "list" - key => "myspecial2" - } -} - -# default output - redis { - host => "localhost" - data_type => "list" - key => "forwarder" - } - -} -``` - -Note that the `default` output get's **every** event, the other two outputs only get those where the condition is met. - -You can combine several outputs with `else`. That's helpful when you want to split events. Like syslog messages depending on which program logged an event. Just change `exclusive` to `true`. - -``` -logstash_pipelines: - syslog: - name: syslog - source: https://github.com/netways/syslog-logstash-pipeline.git - exclusive: true - input: - - name: default - key: input - output: - - name: special - key: myspecial - condition: '[program] == "special"' - - name: special2 - key: myspecial2 - condition: '[program] == "special2"' - - name: default - key: forwarder -``` - -This will give you the following Logstash configuration. - -``` -input { - -# default output - redis { - host => "localhost" - data_type => "list" - key => "input" - } - -} - -output { - -# special output -if [program] == "special" { - redis { - host => "localhost" - data_type => "list" - key => "myspecial" - } -} -# special2 output -else if [program] == "special2" { - redis { - host => "localhost" - data_type => "list" - key => "myspecial2" - } -} - -# default output -else { - redis { - host => "localhost" - data_type => "list" - key => "forwarder" - } - -} - -} -``` - -Here the `default` output only receives the events that haven't already been sent to one of the others. - -## Extra configuration ## - -### Congestion threshold ### - -Every Output can have a `congestion:` option with a numerical value. If the Redis key already holds more items than the value says, the output will stop. - -### Unsafe shutdown ### - -If you need unsafe Logstash shutdowns, e.g. for testing, you can set `logstash_pipeline_unsafe_shutdown` to `true`. The variable doesn't have a default so Logstash falls back to its internal default of `false`. - -## Caveats ## - -There are still some minor issues you need to keep in mind: - -* The default output in an `exclusive: true` setup must be the last in the YAML configuration. There's no sorting, the role simply expects the default to be the last one. -* The configuration *should* work but will make no sense if you have `exclusive: true` but two or more outputs without `condition`. - -## Custom pipelines ## - -If you have other ways of putting pipeline code into the correct directories, you can just skip the `source` option. - -``` -logstash_pipelines: - syslog: - name: syslog -``` -**You have to make sure the code is available or Logstash will constantly log errors!** - -This will create the directories and integrate all `*.conf` files within via `pipelines.yml`. - -**If you add a source later, the role will delete the directory and recreate it with it's own code. So make sure you have a backup!" diff --git a/roles/logstash/docs/pipelines.md b/roles/logstash/docs/pipelines.md index a41548e7..30ad89ba 100644 --- a/roles/logstash/docs/pipelines.md +++ b/roles/logstash/docs/pipelines.md @@ -1,18 +1,28 @@ # Pipelines # +## Keeping an overview ## + +It can be quite difficult to stay on top of your pipeline configuration because they tend to become very complex. + +This collection will leave some comments about how pipelines are interconnected within the `/etc/logstash/pipelines.yml` configuration file. + +If you set `logstash_mermaid` to `true` (which is the default), then you will also get a new file in `/etc/logstash/pipelines.mermaid`. You can paste it into a Mermaid editor in your documentation tool or in an [online Mermaid editor](https://mermaid.live/). The same content will be available on your control node in a temporary file. You can even add arbitrary code to reflect manually managed pipelines by using `logstash_mermaid_extra`. + ## Git managed ## +`logstash_pipelines` is a **list**. Each entry needs a `name`; everything else is optional. + If you have pipeline code managed in (and available via) Git repositories, you can use this role to check them out and integrate them into `pipelines.yml`. -``` +```yaml logstash_pipelines: - syslog: - name: syslog - source: https://github.com/widhalmt/syslog-logstash-pipeline.git + - name: syslog + source: https://github.com/netways/syslog-logstash-pipeline.git ``` -You can add a `version` attribute to your pipeline. It defaults to `main`. You can use every string, [Ansibles git](https://docs.ansible.com/ansible/latest/collections/ansible/builtin/git_module.html) module accepts. -You can also determine the place for logstash to store the data of each pipeline while processing it by using `queue_type` attribute. It defaults to `memory`, you can use `persisted` to write the data temporarily to the hard disk. Moreover you can limit the maximal size of data in every pipeline throgh `queue_max_bytes` attribute, which is default `1gb`. +You can add a `version` attribute to your pipeline. It defaults to `main`. You can use every string [Ansible's git](https://docs.ansible.com/ansible/latest/collections/ansible/builtin/git_module.html) module accepts. + +You can also determine where Logstash stores the data of each pipeline while processing it by using the `queue_type` attribute. It defaults to `memory`; use `persisted` to write the data temporarily to disk. You can limit the maximum amount of data per pipeline with `queue_max_bytes`, which defaults to `1gb`. ## Input and Output ## @@ -20,13 +30,12 @@ You can also determine the place for logstash to store the data of each pipeline To have a single Redis input and output to your files, use this. -``` +```yaml logstash_pipelines: - syslog: - name: syslog - source: https://github.com/widhalmt/syslog-logstash-pipeline.git - queue.type: memory - queue.max_bytes: 1gb + - name: syslog + source: https://github.com/netways/syslog-logstash-pipeline.git + queue_type: memory + queue_max_bytes: 1gb exclusive: false input: - name: default @@ -65,13 +74,10 @@ If you want a bit more control over which outputs are used, the role offers more If you have several outputs that all have conditions, like just send some messages to a development system or only alerts to a monitoring system. -``` +```yaml logstash_pipelines: - syslog: - name: syslog - source: https://github.com/widhalmt/syslog-logstash-pipeline.git - queue.type: memory - queue.max_bytes: 1gb + - name: syslog + source: https://github.com/netways/syslog-logstash-pipeline.git exclusive: false input: - name: default @@ -134,11 +140,10 @@ Note that the `default` output get's **every** event, the other two outputs only You can combine several outputs with `else`. That's helpful when you want to split events. Like syslog messages depending on which program logged an event. Just change `exclusive` to `true`. -``` +```yaml logstash_pipelines: - syslog: - name: syslog - source: https://github.com/widhalmt/syslog-logstash-pipeline.git + - name: syslog + source: https://github.com/netways/syslog-logstash-pipeline.git exclusive: true input: - name: default @@ -206,7 +211,11 @@ Here the `default` output only receives the events that haven't already been sen ### Congestion threshold ### -Every Output can have a `congestion:` option with a numerical value. If the Redis key already holds more items than the value says, the output will stop. +Every output can have a `congestion` option with a numerical value. If the Redis key already holds more items than the value says, the output will stop. + +### Unsafe shutdown ### + +If you need unsafe Logstash shutdowns, e.g. for testing, you can set `logstash_pipeline_unsafe_shutdown` to `true`. The variable doesn't have a default so Logstash falls back to its internal default of `false`. ## Caveats ## @@ -219,13 +228,12 @@ There are still some minor issues you need to keep in mind: If you have other ways of putting pipeline code into the correct directories, you can just skip the `source` option. -``` +```yaml logstash_pipelines: - syslog: - name: syslog + - name: syslog ``` **You have to make sure the code is available or Logstash will constantly log errors!** This will create the directories and integrate all `*.conf` files within via `pipelines.yml`. -**If you add a source later, the role will delete the directory and recreate it with it's own code. So make sure you have a backup!" +**If you add a source later, the role will delete the directory and recreate it with its own code. So make sure you have a backup!** From 99e5948bdeee27cda3127c14d4d804c96ac90cc8 Mon Sep 17 00:00:00 2001 From: Afeef Ghannam Date: Mon, 29 Jun 2026 16:41:27 +0200 Subject: [PATCH 12/28] Correct some variable description --- roles/logstash/README.md | 8 ++++---- roles/logstash/meta/argument_specs.yml | 10 +++++----- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/roles/logstash/README.md b/roles/logstash/README.md index 151dd270..2c8902eb 100644 --- a/roles/logstash/README.md +++ b/roles/logstash/README.md @@ -93,10 +93,10 @@ The following variables can be configured for this role: | `logstash_user_password` | `str` | `"password"` | — | Password of the Logstash user. Must be at least 6 characters long. | | `logstash_user_email` | `str` | `""` | — | Email address linked to the Logstash user. | | `logstash_user_fullname` | `str` | `"Internal Logstash User"` | — | Full name linked to the Logstash user. | -| `logstash_logging_console` | `bool` | `true` | — | Log to the console (syslog when run via systemd). | -| `logstash_logging_file` | `bool` | `true` | — | Log to the log file. | -| `logstash_logging_slow_console` | `bool` | `true` | — | Log the slowlog to the console (syslog when run via systemd). | -| `logstash_logging_slow_file` | `bool` | `true` | — | Log the slowlog to the log file. | +| `logstash_logging_console` | `bool` | `true` | — | Log to the console (syslog when run via systemd). Only effective when logstash_manage_logging is enabled. | +| `logstash_logging_file` | `bool` | `true` | — | Log to the log file. Only effective when logstash_manage_logging is enabled. | +| `logstash_logging_slow_console` | `bool` | `true` | — | Log the slowlog to the console (syslog when run via systemd). Only effective when logstash_manage_logging is enabled. | +| `logstash_logging_slow_file` | `bool` | `true` | — | Log the slowlog to the log file. Only effective when logstash_manage_logging is enabled. | | `logstash_ident` | `bool` | `true` | — | Add a field identifying the node that processed an event. | | `logstash_ident_field_name` | `str` | `"[netways][instance]"` | — | Name of the field that identifies the instance. | | `logstash_pipeline_identifier` | `bool` | `true` | — | Add a field identifying which pipeline processed an event. | diff --git a/roles/logstash/meta/argument_specs.yml b/roles/logstash/meta/argument_specs.yml index cfaf9264..5621e500 100644 --- a/roles/logstash/meta/argument_specs.yml +++ b/roles/logstash/meta/argument_specs.yml @@ -390,26 +390,26 @@ argument_specs: default: Internal Logstash User description: Full name linked to the Logstash user. - # ----- log4j2 logging ----- + # ----- log4j2 logging (only effective when logstash_manage_logging is enabled) ----- logstash_logging_console: type: bool default: true - description: Log to the console (syslog when run via systemd). + description: Log to the console (syslog when run via systemd). Only effective when logstash_manage_logging is enabled. logstash_logging_file: type: bool default: true - description: Log to the log file. + description: Log to the log file. Only effective when logstash_manage_logging is enabled. logstash_logging_slow_console: type: bool default: true - description: Log the slowlog to the console (syslog when run via systemd). + description: Log the slowlog to the console (syslog when run via systemd). Only effective when logstash_manage_logging is enabled. logstash_logging_slow_file: type: bool default: true - description: Log the slowlog to the log file. + description: Log the slowlog to the log file. Only effective when logstash_manage_logging is enabled. # ----- Identifying fields ----- logstash_ident: From 542c722ed2b9d5d6dfb593d95e4899aa77c9ca44 Mon Sep 17 00:00:00 2001 From: Afeef Ghannam Date: Mon, 29 Jun 2026 16:44:08 +0200 Subject: [PATCH 13/28] Add Tags section in docs --- roles/logstash/README.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/roles/logstash/README.md b/roles/logstash/README.md index 2c8902eb..999dbaec 100644 --- a/roles/logstash/README.md +++ b/roles/logstash/README.md @@ -35,6 +35,15 @@ You also need the Elastic repositories configured — use the [`repos`](../repos For how to configure pipelines (built-in Redis/Beats and external git repositories), see the [pipelines documentation](docs/pipelines.md). +## Tags + +Run only parts of the role with `--tags`: + +* `configuration` (alias `logstash_configuration`) — only (re)write the configuration, skip installation. +* `certificates` — only generate and distribute the TLS certificates. +* `renew_logstash_cert` / `renew_ca` — force renewal of the Logstash certificate. +* `mermaid` — only regenerate the pipeline overview (`pipelines.mermaid`). + ## Role variables From 9c3f88401bf9faa7bdb874c88b9beceeb5822dd4 Mon Sep 17 00:00:00 2001 From: Afeef Ghannam Date: Mon, 29 Jun 2026 17:38:53 +0200 Subject: [PATCH 14/28] Remove deprecated variable --- roles/logstash/README.md | 1 - roles/logstash/defaults/main.yml | 1 - roles/logstash/meta/argument_specs.yml | 5 ----- 3 files changed, 7 deletions(-) diff --git a/roles/logstash/README.md b/roles/logstash/README.md index 999dbaec..4c7971f1 100644 --- a/roles/logstash/README.md +++ b/roles/logstash/README.md @@ -96,7 +96,6 @@ The following variables can be configured for this role: | `logstash_role_cluster_privileges` | `list` of `str` | `['manage_index_templates', 'monitor', 'manage_ilm']` | — | Cluster privileges granted to the Logstash writer role. | | `logstash_role_indicies_names` | `list` of `str` | `['ecs-logstash*', 'logstash*', 'logs*']` | — | Index patterns the Logstash writer role may access. | | `logstash_role_indicies_privileges` | `list` of `str` | `['write', 'create', 'delete', 'create_index', 'manage', 'manage_ilm']` | — | Index privileges the Logstash writer role holds on its index patterns. | -| `logstash_reset_writer_role` | `bool` | `true` | — | Reset the writer role and user on every run. | | `logstash_create_user` | `bool` | `true` | — | Create the Logstash writer user in Elasticsearch. | | `logstash_user_name` | `str` | `"logstash_writer"` | — | Name of the Logstash user connecting to Elasticsearch. | | `logstash_user_password` | `str` | `"password"` | — | Password of the Logstash user. Must be at least 6 characters long. | diff --git a/roles/logstash/defaults/main.yml b/roles/logstash/defaults/main.yml index 477fd8fe..f8e297f5 100644 --- a/roles/logstash/defaults/main.yml +++ b/roles/logstash/defaults/main.yml @@ -69,7 +69,6 @@ logstash_user_name: logstash_writer logstash_user_password: password logstash_user_email: "" logstash_user_fullname: "Internal Logstash User" -logstash_reset_writer_role: true # logstash security logstash_tls_key_passphrase: LogstashChangeMe diff --git a/roles/logstash/meta/argument_specs.yml b/roles/logstash/meta/argument_specs.yml index 5621e500..4e54e66b 100644 --- a/roles/logstash/meta/argument_specs.yml +++ b/roles/logstash/meta/argument_specs.yml @@ -360,11 +360,6 @@ argument_specs: - manage_ilm description: Index privileges the Logstash writer role holds on its index patterns. - logstash_reset_writer_role: - type: bool - default: true - description: Reset the writer role and user on every run. - logstash_create_user: type: bool default: true From 0f067a2d0db63a01489b9df2537a21275ae003c7 Mon Sep 17 00:00:00 2001 From: Afeef Ghannam Date: Mon, 29 Jun 2026 18:19:32 +0200 Subject: [PATCH 15/28] drop unused logstash_queue_type --- roles/logstash/README.md | 1 - roles/logstash/defaults/main.yml | 1 - roles/logstash/meta/argument_specs.yml | 8 -------- 3 files changed, 10 deletions(-) diff --git a/roles/logstash/README.md b/roles/logstash/README.md index 4c7971f1..4b718a70 100644 --- a/roles/logstash/README.md +++ b/roles/logstash/README.md @@ -78,7 +78,6 @@ The following variables can be configured for this role: | `logstash_input_queue_max_bytes` | `str` | `"1gb"` | — | Maximum queue size for the default Beats input pipeline. | | `logstash_forwarder_queue_type` | `str` | `"memory"` | `memory`, `persisted` | Queue type for the default Elasticsearch forwarder pipeline. | | `logstash_forwarder_queue_max_bytes` | `str` | `"1gb"` | — | Maximum queue size for the default Elasticsearch forwarder pipeline. | -| `logstash_queue_type` | `str` | `"persisted"` | `persisted`, `memory` | Default queue type Logstash uses for pipelines. | | `logstash_redis_password` | `str` | N/A | — | Password used when the simple inputs/outputs connect to Redis. Unset by default. | | `logstash_elasticsearch` | `list` of `str` | N/A | — | Elasticsearch hosts for the default output. Defaults to the nodes from the elasticsearch group, or localhost when used standalone. | | `logstash_validate_after_inactivity` | `str` | `"300"` | — | Seconds Logstash waits before validating a previously idle connection to Elasticsearch. | diff --git a/roles/logstash/defaults/main.yml b/roles/logstash/defaults/main.yml index f8e297f5..d114438d 100644 --- a/roles/logstash/defaults/main.yml +++ b/roles/logstash/defaults/main.yml @@ -18,7 +18,6 @@ logstash_config_path_logs: /var/log/logstash # pipeline configuration # logstash_manage_pipelines: true -logstash_queue_type: persisted # this will deactivate all pipeline management logstash_no_pipelines: false diff --git a/roles/logstash/meta/argument_specs.yml b/roles/logstash/meta/argument_specs.yml index 4e54e66b..6655e113 100644 --- a/roles/logstash/meta/argument_specs.yml +++ b/roles/logstash/meta/argument_specs.yml @@ -243,14 +243,6 @@ argument_specs: default: 1gb description: Maximum queue size for the default Elasticsearch forwarder pipeline. - logstash_queue_type: - type: str - choices: - - persisted - - memory - default: persisted - description: Default queue type Logstash uses for pipelines. - logstash_redis_password: type: str description: Password used when the simple inputs/outputs connect to Redis. Unset by default. From 2949a78efa5064c881cdf9e3a7b3324e7b500138 Mon Sep 17 00:00:00 2001 From: Afeef Ghannam Date: Mon, 29 Jun 2026 18:42:42 +0200 Subject: [PATCH 16/28] show pipeline sub-options in README table, clarify persistent queues --- .docsmith/readme.md.j2 | 24 +++++++++++++++--------- roles/logstash/README.md | 22 +++++++++++++++++----- roles/logstash/meta/argument_specs.yml | 6 +++--- 3 files changed, 35 insertions(+), 17 deletions(-) diff --git a/.docsmith/readme.md.j2 b/.docsmith/readme.md.j2 index 22a9f4d6..86054973 100644 --- a/.docsmith/readme.md.j2 +++ b/.docsmith/readme.md.j2 @@ -1,17 +1,23 @@ -{# Compact README variable reference for ansible-docsmith. - Shared by all roles in this collection. Renders ONE table with the full - description in each cell (no truncation, no per-variable detail sections, - no table of contents). Pass it with: --template-readme .docsmith/readme.md.j2 #} +{# Compact README variable reference for ansible-docsmith. Shared by all roles. + Renders ONE table; nested sub-options (e.g. logstash_pipelines fields) appear as + indented rows. No per-variable detail sections, no table of contents. + Use with: --template-readme .docsmith/readme.md.j2 #} +{% macro row(name, spec, depth) -%} +{% set pad = '    ' * depth %} +{% set mark = '↳ ' if depth > 0 else '' %} +| {{ pad }}{{ mark }}`{{ name | ansible_escape }}` | `{{ spec.type }}`{% if spec.elements %} of `{{ spec.elements }}`{% endif %} | {% if spec.options %}…{% else %}{{ spec.default | format_default }}{% endif %} | {% if spec.choices %}{{ spec.choices | map('string') | map('code_escape') | join(', ') }}{% else %}—{% endif %} | {{ spec.description | format_table_description(name, 0) }} | +{% for sn, ss in (spec.options or {}).items() %} +{{ row(sn, ss, depth + 1) }} +{%- endfor %} +{%- endmacro %} ## Role variables {% if has_options %} -The following variables can be configured for this role: - | Variable | Type | Default | Choices | Description | |----------|------|---------|---------|-------------| -{% for var_name, var_spec in options.items() %} -| `{{ var_name | ansible_escape }}` | `{{ var_spec.type }}`{% if var_spec.elements %} of `{{ var_spec.elements }}`{% endif %} | {{ var_spec.default | format_default }} | {% if var_spec.choices %}{{ var_spec.choices | map('string') | map('code_escape') | join(', ') }}{% else %}—{% endif %} | {{ var_spec.description | format_table_description(var_name, 0) }} | -{% endfor %} +{% for name, spec in options.items() %} +{{ row(name, spec, 0) }} +{%- endfor %} {% else %} No variables are defined for this role. {% endif %} diff --git a/roles/logstash/README.md b/roles/logstash/README.md index 4b718a70..64dc6cf4 100644 --- a/roles/logstash/README.md +++ b/roles/logstash/README.md @@ -47,8 +47,6 @@ Run only parts of the role with `--tags`: ## Role variables -The following variables can be configured for this role: - | Variable | Type | Default | Choices | Description | |----------|------|---------|---------|-------------| | `logstash_enable` | `bool` | `true` | — | Start and enable the Logstash service. | @@ -68,15 +66,29 @@ The following variables can be configured for this role: | `logstash_legacy_monitoring` | `bool` | `true` | — | Enable legacy X-Pack monitoring. Ignored unless elasticstack_full_stack is set and only effective on Elastic Stack releases lower than 8. | | `logstash_manage_pipelines` | `bool` | `true` | — | Manage pipelines.yml. | | `logstash_no_pipelines` | `bool` | `false` | — | Disable all pipeline management entirely. | -| `logstash_pipelines` | `list` of `dict` | `[{'name': 'default', 'exclusive': False, 'queue_type': 'memory', 'queue_max_bytes': '1gb', 'input': [{'name': 'default', 'key': 'input'}], 'output': [{'name': 'default', 'key': 'forwarder'}]}]` | — | List of pipelines to configure.

Each entry needs a name. A pipeline either points to an external git repository (source/version) or defines simple input/output keys that connect to Redis. See the pipelines documentation for details. | +| `logstash_pipelines` | `list` of `dict` | … | — | List of pipelines to configure.

Each entry needs a name. A pipeline either points to an external git repository (source/version) or defines simple input/output keys that connect to Redis. See the pipelines documentation for details. | +|     ↳ `name` | `str` | N/A | — | Unique name of the pipeline. Becomes the pipeline directory and pipeline.id. | +|     ↳ `exclusive` | `bool` | N/A | — | Mark this pipeline as the only one allowed to handle its events. | +|     ↳ `source` | `str` | N/A | — | URL of a git repository holding the pipeline configuration. | +|     ↳ `version` | `str` | N/A | — | Git branch/tag/commit to check out from source (default "main"). | +|     ↳ `queue_type` | `str` | N/A | `memory`, `persisted` | Queue type for this pipeline (default "memory"). Use "persisted" for an on-disk persistent queue. | +|     ↳ `queue_max_bytes` | `str` | N/A | — | Maximum queue size for this pipeline (default "1gb"). | +|     ↳ `input` | `list` of `dict` | … | — | Simple Redis inputs for this pipeline. | +|         ↳ `name` | `str` | N/A | — | Name of the input. | +|         ↳ `key` | `str` | N/A | — | Redis key to read from. | +|     ↳ `output` | `list` of `dict` | … | — | Simple Redis outputs for this pipeline. | +|         ↳ `name` | `str` | N/A | — | Name of the output. | +|         ↳ `key` | `str` | N/A | — | Redis key to write to. | +|         ↳ `condition` | `str` | N/A | — | Optional Logstash conditional. The output only receives events matching it. With exclusive set, the conditions are chained with else if. | +|         ↳ `congestion` | `int` | N/A | — | Optional congestion threshold. The output stops once the target Redis key holds more items than this value. | | `logstash_elasticsearch_output` | `bool` | `true` | — | Create the default pipeline that forwards events to Elasticsearch. | | `logstash_beats_input` | `bool` | `true` | — | Create the default pipeline with a Beats input. | | `logstash_beats_input_congestion` | `int` | N/A | — | Congestion threshold (seconds) for the Beats input pipeline. Unset by default. | | `logstash_beats_timeout` | `str` | N/A | — | Timeout for idle client connections on the Beats input (e.g. "60s"). Unset by default. | | `logstash_beats_tls` | `bool` | N/A | — | Activate TLS on the Beats input pipeline. Unset by default, but enabled automatically in a full stack setup unless overridden. | -| `logstash_input_queue_type` | `str` | `"memory"` | `memory`, `persisted` | Queue type for the default Beats input pipeline. | +| `logstash_input_queue_type` | `str` | `"memory"` | `memory`, `persisted` | Queue type for the default Beats input pipeline. Use "persisted" for an on-disk persistent queue. | | `logstash_input_queue_max_bytes` | `str` | `"1gb"` | — | Maximum queue size for the default Beats input pipeline. | -| `logstash_forwarder_queue_type` | `str` | `"memory"` | `memory`, `persisted` | Queue type for the default Elasticsearch forwarder pipeline. | +| `logstash_forwarder_queue_type` | `str` | `"memory"` | `memory`, `persisted` | Queue type for the default Elasticsearch forwarder pipeline. Use "persisted" for an on-disk persistent queue. | | `logstash_forwarder_queue_max_bytes` | `str` | `"1gb"` | — | Maximum queue size for the default Elasticsearch forwarder pipeline. | | `logstash_redis_password` | `str` | N/A | — | Password used when the simple inputs/outputs connect to Redis. Unset by default. | | `logstash_elasticsearch` | `list` of `str` | N/A | — | Elasticsearch hosts for the default output. Defaults to the nodes from the elasticsearch group, or localhost when used standalone. | diff --git a/roles/logstash/meta/argument_specs.yml b/roles/logstash/meta/argument_specs.yml index 6655e113..a55e8778 100644 --- a/roles/logstash/meta/argument_specs.yml +++ b/roles/logstash/meta/argument_specs.yml @@ -155,7 +155,7 @@ argument_specs: choices: - memory - persisted - description: Queue type for this pipeline (default "memory"). + description: Queue type for this pipeline (default "memory"). Use "persisted" for an on-disk persistent queue. queue_max_bytes: type: str description: Maximum queue size for this pipeline (default "1gb"). @@ -223,7 +223,7 @@ argument_specs: - memory - persisted default: memory - description: Queue type for the default Beats input pipeline. + description: Queue type for the default Beats input pipeline. Use "persisted" for an on-disk persistent queue. logstash_input_queue_max_bytes: type: str @@ -236,7 +236,7 @@ argument_specs: - memory - persisted default: memory - description: Queue type for the default Elasticsearch forwarder pipeline. + description: Queue type for the default Elasticsearch forwarder pipeline. Use "persisted" for an on-disk persistent queue. logstash_forwarder_queue_max_bytes: type: str From 680807ebaee1168e8ee0f64b382ec26029821b60 Mon Sep 17 00:00:00 2001 From: Afeef Ghannam Date: Tue, 30 Jun 2026 10:07:52 +0200 Subject: [PATCH 17/28] correct logstash_beats_input_congestion description (items, not seconds) --- roles/logstash/README.md | 2 +- roles/logstash/meta/argument_specs.yml | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/roles/logstash/README.md b/roles/logstash/README.md index 64dc6cf4..0ff9284f 100644 --- a/roles/logstash/README.md +++ b/roles/logstash/README.md @@ -83,7 +83,7 @@ Run only parts of the role with `--tags`: |         ↳ `congestion` | `int` | N/A | — | Optional congestion threshold. The output stops once the target Redis key holds more items than this value. | | `logstash_elasticsearch_output` | `bool` | `true` | — | Create the default pipeline that forwards events to Elasticsearch. | | `logstash_beats_input` | `bool` | `true` | — | Create the default pipeline with a Beats input. | -| `logstash_beats_input_congestion` | `int` | N/A | — | Congestion threshold (seconds) for the Beats input pipeline. Unset by default. | +| `logstash_beats_input_congestion` | `int` | N/A | — | Congestion threshold for the default Beats input pipeline: it stops writing to Redis once the buffer key holds more items than this value. Unset by default. | | `logstash_beats_timeout` | `str` | N/A | — | Timeout for idle client connections on the Beats input (e.g. "60s"). Unset by default. | | `logstash_beats_tls` | `bool` | N/A | — | Activate TLS on the Beats input pipeline. Unset by default, but enabled automatically in a full stack setup unless overridden. | | `logstash_input_queue_type` | `str` | `"memory"` | `memory`, `persisted` | Queue type for the default Beats input pipeline. Use "persisted" for an on-disk persistent queue. | diff --git a/roles/logstash/meta/argument_specs.yml b/roles/logstash/meta/argument_specs.yml index a55e8778..6020bc23 100644 --- a/roles/logstash/meta/argument_specs.yml +++ b/roles/logstash/meta/argument_specs.yml @@ -205,7 +205,10 @@ argument_specs: logstash_beats_input_congestion: type: int - description: Congestion threshold (seconds) for the Beats input pipeline. Unset by default. + description: >- + Congestion threshold for the default Beats input pipeline: it stops + writing to Redis once the buffer key holds more items than this value. + Unset by default. logstash_beats_timeout: type: str From d369515137ee2edc102070b53c720ee57e0fa6d6 Mon Sep 17 00:00:00 2001 From: Afeef Ghannam Date: Tue, 30 Jun 2026 10:16:32 +0200 Subject: [PATCH 18/28] Point in-config repo links to the collection instead of the old standalone role --- roles/logstash/templates/log4j2.properties.j2 | 2 +- roles/logstash/templates/pipelines.mermaid.j2 | 2 +- roles/logstash/templates/pipelines.yml.j2 | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/roles/logstash/templates/log4j2.properties.j2 b/roles/logstash/templates/log4j2.properties.j2 index 78a025e9..10a908e0 100644 --- a/roles/logstash/templates/log4j2.properties.j2 +++ b/roles/logstash/templates/log4j2.properties.j2 @@ -1,5 +1,5 @@ # Managed by Ansible Role -# https://github.com/netways/ansible-role-logstash +# https://github.com/netways/ansible-collection-elasticstack # # Logging to logfile: {% if logstash_logging_file | bool %}true{% else %}false{% endif %} diff --git a/roles/logstash/templates/pipelines.mermaid.j2 b/roles/logstash/templates/pipelines.mermaid.j2 index adc30895..9832234d 100644 --- a/roles/logstash/templates/pipelines.mermaid.j2 +++ b/roles/logstash/templates/pipelines.mermaid.j2 @@ -1,5 +1,5 @@ # Managed via Ansible role -# https://github.com/netways/ansible-role-logstash +# https://github.com/netways/ansible-collection-elasticstack # Use the following code with your favorite Mermaid editor # Or paste into: https://mermaid.live/ diff --git a/roles/logstash/templates/pipelines.yml.j2 b/roles/logstash/templates/pipelines.yml.j2 index 901088e6..e4c00c10 100644 --- a/roles/logstash/templates/pipelines.yml.j2 +++ b/roles/logstash/templates/pipelines.yml.j2 @@ -1,7 +1,7 @@ --- # Managed via Ansible role -# https://github.com/netways/ansible-role-logstash +# https://github.com/netways/ansible-collection-elasticstack {% if logstash_beats_input_congestion is defined %} # global congestion threshold: {{ logstash_beats_input_congestion }} {% endif %} @@ -39,7 +39,7 @@ {% if logstash_pipelines is defined %} ### Autoconfigured pipelines ### -# See https://github.com/netways/ansible-role-logstash/blob/master/docs/pipelines.md +# See https://github.com/netways/ansible-collection-elasticstack/blob/main/roles/logstash/docs/pipelines.md # for details # Hint: They can be mixed up with manual code. When source or Redis is missing it's # likely that there is code from another source in place. From 515a4855c4c1bf7deb59a018c5cb0557a250e4f5 Mon Sep 17 00:00:00 2001 From: Afeef Ghannam Date: Tue, 30 Jun 2026 10:48:04 +0200 Subject: [PATCH 19/28] clarify what the role does in the intro and main spec description --- RELEASE.md | 18 ++++++++++++++ roles/logstash/README.md | 6 +++-- roles/logstash/RELEASE.md | 33 -------------------------- roles/logstash/meta/argument_specs.yml | 9 ++++--- 4 files changed, 28 insertions(+), 38 deletions(-) delete mode 100644 roles/logstash/RELEASE.md diff --git a/RELEASE.md b/RELEASE.md index 09f2d4ec..048f040c 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -8,6 +8,10 @@ Remove the note above saying that this is a work and progress before the first release. And remove this section. +### Lint + +Make sure to have one last pull request to remove as much lint as possible. Same goes for deprecation warnings, linter exceptions etc. + ### Authors Update the [.mailmap](.mailmap) and [AUTHORS](AUTHORS) files: @@ -15,3 +19,17 @@ Update the [.mailmap](.mailmap) and [AUTHORS](AUTHORS) files: ``` git log --use-mailmap | grep '^Author:' | cut -f2- -d' ' | sort | uniq > AUTHORS ``` + +## Tag + +Tag the version. Don't use `v` in the name. Use semantic versioning. + +e.g. + +``` +git tag -as 7.6.4 +``` + +## Release + +Make the tag a release on GitHub. diff --git a/roles/logstash/README.md b/roles/logstash/README.md index 0ff9284f..953f3838 100644 --- a/roles/logstash/README.md +++ b/roles/logstash/README.md @@ -2,8 +2,10 @@ Installs and configures [Logstash](https://www.elastic.co/products/logstash) on Linux systems. The role can manage `logstash.yml`, the log4j2 logging, the JVM -heap, TLS, the Elasticsearch writer role/user, and pipelines — both the built-in -Redis/Beats pipelines and pipelines pulled from external git repositories. +heap, TLS, and the Elasticsearch writer role/user. For pipelines it can create +two default pipelines — a Beats input and an Elasticsearch forwarder, both using +Redis — and manage your own pipelines, whose configuration can be checked out +from external git repositories. It works with the standard Elastic Stack packages and with Elastic's OSS variant. diff --git a/roles/logstash/RELEASE.md b/roles/logstash/RELEASE.md deleted file mode 100644 index 17c8792b..00000000 --- a/roles/logstash/RELEASE.md +++ /dev/null @@ -1,33 +0,0 @@ -# Release Workflow - -## Preparations - -### Lint - -Make sure to have one last pull request to remove as much lint as possible. Same goes for deprecation warnings, linter exceptions etc. - -### Authors - -Update the [.mailmap](.mailmap) and [AUTHORS](AUTHORS) files: - -``` -git log --use-mailmap | grep '^Author:' | cut -f2- -d' ' | sort | uniq > AUTHORS -``` - -## Tag - -Tag the version. Don't use `v` in the name. User semantic versioning. - -e.g. - -``` -git tag -as 7.6.4 -``` - -## Release - -Make the tag a release on GitHub - -## Ansible Galaxy - -Do *not* push this role to Ansible Galaxy. Versioning helps with ongoing projects but we will sum up this role and all related ones in an Ansible collection and push that to GitHub. diff --git a/roles/logstash/meta/argument_specs.yml b/roles/logstash/meta/argument_specs.yml index 6020bc23..95ac3641 100644 --- a/roles/logstash/meta/argument_specs.yml +++ b/roles/logstash/meta/argument_specs.yml @@ -13,9 +13,12 @@ argument_specs: description: - Installs and configures Logstash on Linux systems. - >- - Can manage logstash.yml, log4j2 logging, pipelines (default Redis/Beats - pipelines and pipelines from external git repositories), TLS and the - Elasticsearch writer role/user, and the JVM heap size. + Can manage logstash.yml, log4j2 logging, the JVM heap, TLS and the + Elasticsearch writer role/user. + - >- + Creates two default pipelines, a Beats input and an Elasticsearch + forwarder (both using Redis), and manages user-defined pipelines whose + configuration can be checked out from external git repositories. author: - NETWAYS GmbH From 20928463d459454dc76cfe95d4b7252a7cccf418 Mon Sep 17 00:00:00 2001 From: Afeef Ghannam Date: Tue, 30 Jun 2026 13:53:33 +0200 Subject: [PATCH 20/28] Test the pipeline --- roles/logstash/meta/argument_specs.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/roles/logstash/meta/argument_specs.yml b/roles/logstash/meta/argument_specs.yml index 95ac3641..95e71012 100644 --- a/roles/logstash/meta/argument_specs.yml +++ b/roles/logstash/meta/argument_specs.yml @@ -432,6 +432,11 @@ argument_specs: Also add pipeline identifiers in the default pipelines. This can let the defaults dominate statistics with little value. + test: + type: str + default: false + description: test + # ----- Mermaid pipeline overview ----- logstash_mermaid: type: bool From 98c37b1adc03955e4cdb1f7ef4df4714056fabd5 Mon Sep 17 00:00:00 2001 From: Afeef Ghannam Date: Tue, 30 Jun 2026 14:00:03 +0200 Subject: [PATCH 21/28] Another test --- roles/logstash/meta/argument_specs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/logstash/meta/argument_specs.yml b/roles/logstash/meta/argument_specs.yml index 95e71012..d4b32c86 100644 --- a/roles/logstash/meta/argument_specs.yml +++ b/roles/logstash/meta/argument_specs.yml @@ -434,7 +434,7 @@ argument_specs: test: type: str - default: false + default: hi description: test # ----- Mermaid pipeline overview ----- From 7845f73b7d30bed895e7ac007393ee60b47dd85a Mon Sep 17 00:00:00 2001 From: Afeef Ghannam Date: Tue, 30 Jun 2026 14:01:10 +0200 Subject: [PATCH 22/28] another test --- roles/logstash/defaults/main.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/roles/logstash/defaults/main.yml b/roles/logstash/defaults/main.yml index d114438d..2bbc9517 100644 --- a/roles/logstash/defaults/main.yml +++ b/roles/logstash/defaults/main.yml @@ -96,3 +96,5 @@ logstash_mermaid_logstash: true logstash_freshstart: changed: false + +test: hallo From 5d83ea27df40cbc3c1ee7bb693c8332bc6f65d1b Mon Sep 17 00:00:00 2001 From: Afeef Ghannam Date: Tue, 30 Jun 2026 14:03:48 +0200 Subject: [PATCH 23/28] Another test --- roles/logstash/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/logstash/defaults/main.yml b/roles/logstash/defaults/main.yml index 2bbc9517..43d21565 100644 --- a/roles/logstash/defaults/main.yml +++ b/roles/logstash/defaults/main.yml @@ -97,4 +97,4 @@ logstash_mermaid_logstash: true logstash_freshstart: changed: false -test: hallo +test: hi From c79f33538d74096947f6b60f4ad03e46f399c826 Mon Sep 17 00:00:00 2001 From: Afeef Ghannam Date: Tue, 30 Jun 2026 15:07:10 +0200 Subject: [PATCH 24/28] Removes tests --- .github/workflows/test_docs.yml | 2 +- roles/logstash/defaults/main.yml | 2 -- roles/logstash/meta/argument_specs.yml | 5 ----- 3 files changed, 1 insertion(+), 8 deletions(-) diff --git a/.github/workflows/test_docs.yml b/.github/workflows/test_docs.yml index c17d4104..efd58f54 100644 --- a/.github/workflows/test_docs.yml +++ b/.github/workflows/test_docs.yml @@ -59,7 +59,7 @@ jobs: # README freshness: regenerate every role README and fail if anything changed, # i.e. the committed README no longer matches argument_specs.yml. - - name: Check that every README is up to date + - name: Check that every variable in README is up to date run: | for role in roles/*/; do [ -f "${role}meta/argument_specs.yml" ] || continue diff --git a/roles/logstash/defaults/main.yml b/roles/logstash/defaults/main.yml index 43d21565..d114438d 100644 --- a/roles/logstash/defaults/main.yml +++ b/roles/logstash/defaults/main.yml @@ -96,5 +96,3 @@ logstash_mermaid_logstash: true logstash_freshstart: changed: false - -test: hi diff --git a/roles/logstash/meta/argument_specs.yml b/roles/logstash/meta/argument_specs.yml index d4b32c86..95ac3641 100644 --- a/roles/logstash/meta/argument_specs.yml +++ b/roles/logstash/meta/argument_specs.yml @@ -432,11 +432,6 @@ argument_specs: Also add pipeline identifiers in the default pipelines. This can let the defaults dominate statistics with little value. - test: - type: str - default: hi - description: test - # ----- Mermaid pipeline overview ----- logstash_mermaid: type: bool From 952af14d7f0ae969799d16ae6e7379b2b095f64f Mon Sep 17 00:00:00 2001 From: Afeef Ghannam Date: Wed, 1 Jul 2026 16:46:08 +0200 Subject: [PATCH 25/28] add meta/argument_specs.yml for the repos role --- roles/repos/meta/argument_specs.yml | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 roles/repos/meta/argument_specs.yml diff --git a/roles/repos/meta/argument_specs.yml b/roles/repos/meta/argument_specs.yml new file mode 100644 index 00000000..e0b9a8f8 --- /dev/null +++ b/roles/repos/meta/argument_specs.yml @@ -0,0 +1,20 @@ +--- +# Role argument specification for the repos role. +# This is the single source of truth for the role's variables: Ansible validates +# the supplied values against it at role start, and ansible-docsmith renders the +# README variable reference from it. +# +# This role has no role-specific variables. It is configured entirely through the +# shared elasticstack_* variables, which are documented centrally with the +# elasticstack role. + +argument_specs: + main: + short_description: Configure the Elastic package repositories + description: + - >- + Adds the Elastic package repositories (apt, yum or zypper) to the target + host so the other roles can install Elastic Stack packages. + - Supports the standard Elastic and the OSS variant. + author: + - NETWAYS GmbH From 2b8ff93282572c646a3c4323b4e2f5167a83fb9f Mon Sep 17 00:00:00 2001 From: Afeef Ghannam Date: Wed, 1 Jul 2026 16:48:38 +0200 Subject: [PATCH 26/28] Add README --- roles/repos/README.md | 42 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 roles/repos/README.md diff --git a/roles/repos/README.md b/roles/repos/README.md new file mode 100644 index 00000000..fcbeb99f --- /dev/null +++ b/roles/repos/README.md @@ -0,0 +1,42 @@ +# Ansible Role: Repos + +Adds the Elastic package repositories to the target host so the other roles can +install Elastic Stack packages. It supports apt (Debian/Ubuntu), yum (RedHat) and +zypper (SUSE), for both the standard Elastic and the OSS variant. + +Its main use is together with the other roles of this collection, which expect +the Elastic repositories to be present. + +## Requirements + +* The `community.general` collection — only needed on SUSE hosts, where the role + uses the `zypper` modules. +* Network access from the target host to the Elastic repository URL. + +The role installs the required signing tools (`gpg`/`gnupg`) itself. + +## Example + +```yaml +- name: Configure Elastic repositories + hosts: all + collections: + - netways.elasticstack + roles: + - repos +``` + + +## Role variables + +No variables are defined for this role. + + + +## Shared variables + +This role is configured through the collection-wide `elasticstack_*` variables — +mainly `elasticstack_release`, `elasticstack_variant`, `elasticstack_enable_repos`, +`elasticstack_repo_url`, `elasticstack_repo_key` and `elasticstack_rpm_workaround`. +They are documented centrally with the +[elasticsearch role](../../docs/role-elasticsearch.md). From a988200198e5f2fb4820c6050ca68773fe46a55d Mon Sep 17 00:00:00 2001 From: Afeef Ghannam Date: Wed, 1 Jul 2026 16:53:32 +0200 Subject: [PATCH 27/28] point docs to the role README --- README.md | 2 +- docs/getting-started.md | 2 +- docs/role-repos.md | 44 +++-------------------------------------- 3 files changed, 5 insertions(+), 43 deletions(-) diff --git a/README.md b/README.md index f4252d72..4615e139 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,7 @@ Every role is documented with all variables, please refer to the documentation f * [Elasticsearch](docs/role-elasticsearch.md) * [Kibana](docs/role-kibana.md) * [Logstash](roles/logstash/README.md) -* [Repos](docs/role-repos.md) +* [Repos](roles/repos/README.md) ## Modules documentation diff --git a/docs/getting-started.md b/docs/getting-started.md index dc49fb40..b43d83c1 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -51,7 +51,7 @@ Roles * [Elasticsearch](role-elasticsearch.md) * [Kibana](role-kibana.md) * [Logstash](../roles/logstash/README.md) -* [Repos](role-repos.md) +* [Repos](../roles/repos/README.md) Variables diff --git a/docs/role-repos.md b/docs/role-repos.md index a47e62ce..bce0c25a 100644 --- a/docs/role-repos.md +++ b/docs/role-repos.md @@ -1,43 +1,5 @@ -Elastic Repos -========= +# Repos role -![Test Role repos](https://github.com/netways/ansible-collection-elasticstack/actions/workflows/test_role_repos.yml/badge.svg) +The Repos role documentation now lives with the role itself: -The role adds Elastic repositories to the package manager. It's main use is in connection with other roles that provide installation and configuration of the Elastic Stack. - -Requirements ------------- - -* You need `gpg` to be installed because packages / repositories are digitally signed and verified. -* Debian and Ubuntu hosts need to have `apt-transport-https` installed to deal with Elastics repositories. -* Ubuntu hosts also need to have `gpg-agent` installed. -* For SuSE hosts you need the Ansible collection `community.general` on your Ansible controller. - -Role Variables --------------- - -* *elasticstack_release*: Major release version of Elastic stack to configure. (default: `7`). `7` and `8` are supported. -* *elasticstack_variant*: Variant of the stack to install. Valid values: `elastic` or `oss`. (default: `elastic`). -* *elasticstack_enable_repos*: Enable repositories after creating them. (default: `true`) Only works on RPM based distributions! - -Please note that no `oss` versions are available for Elastic Stack later than `7`. This role will fail if you try to install them. - -Usage --------- - -Upgrades -======== - -If you want to be able to update your operating system without worrying about accidentally upgrading Elastic Stack, set `elasticstack_enable_repos` to `false`. The roles in this collection will enable the repository in case they need it. Keep in mind that this will only work on rpm based distributions. - -Example playbook -================ - -``` - - hosts: all - become: yes - collections: - - netways.elasticstack - roles: - - repos -``` +**➜ [roles/repos/README.md](../roles/repos/README.md)** From 185f406398ff1780b220a69233f714a520220c2f Mon Sep 17 00:00:00 2001 From: Afeef Ghannam Date: Wed, 1 Jul 2026 16:55:49 +0200 Subject: [PATCH 28/28] skip the repos role molecule for docs-only changes --- .github/workflows/test_role_repos.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test_role_repos.yml b/.github/workflows/test_role_repos.yml index c9747027..50527099 100644 --- a/.github/workflows/test_role_repos.yml +++ b/.github/workflows/test_role_repos.yml @@ -14,6 +14,7 @@ on: pull_request: paths: - 'roles/repos/**' + - '!roles/repos/**/*.md' - '.github/workflows/test_role_repos.yml' - 'molecule/repos_**'