diff --git a/.docsmith/readme.md.j2 b/.docsmith/readme.md.j2 new file mode 100644 index 00000000..86054973 --- /dev/null +++ b/.docsmith/readme.md.j2 @@ -0,0 +1,23 @@ +{# Compact README variable reference for ansible-docsmith. Shared by all roles. + Renders ONE table; nested sub-options (e.g. logstash_pipelines fields) appear as + indented rows. No per-variable detail sections, no table of contents. + Use with: --template-readme .docsmith/readme.md.j2 #} +{% macro row(name, spec, depth) -%} +{% set pad = '    ' * depth %} +{% set mark = '↳ ' if depth > 0 else '' %} +| {{ pad }}{{ mark }}`{{ name | ansible_escape }}` | `{{ spec.type }}`{% if spec.elements %} of `{{ spec.elements }}`{% endif %} | {% if spec.options %}…{% else %}{{ spec.default | format_default }}{% endif %} | {% if spec.choices %}{{ spec.choices | map('string') | map('code_escape') | join(', ') }}{% else %}—{% endif %} | {{ spec.description | format_table_description(name, 0) }} | +{% for sn, ss in (spec.options or {}).items() %} +{{ row(sn, ss, depth + 1) }} +{%- endfor %} +{%- endmacro %} +## Role variables + +{% if has_options %} +| Variable | Type | Default | Choices | Description | +|----------|------|---------|---------|-------------| +{% for name, spec in options.items() %} +{{ row(name, spec, 0) }} +{%- endfor %} +{% else %} +No variables are defined for this role. +{% endif %} diff --git a/.github/workflows/test_docs.yml b/.github/workflows/test_docs.yml new file mode 100644 index 00000000..efd58f54 --- /dev/null +++ b/.github/workflows/test_docs.yml @@ -0,0 +1,73 @@ +--- +name: Test Documentation +on: + workflow_dispatch: + pull_request: + paths: + - 'roles/**/meta/argument_specs.yml' + - 'roles/**/defaults/**' + - 'roles/**/README.md' + - '.docsmith/**' + - '.github/workflows/test_docs.yml' + +# Cancel outdated pipeline runs when a new push occurs in the pull request. +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + docs: + # For non-PR events (workflow_dispatch, etc.), github.event.pull_request.draft is null. + # GitHub Actions coerces null == false to true, so the job runs for all non-draft events. + if: github.event.pull_request.draft == false + runs-on: ubuntu-latest + env: + NO_COLOR: "1" # make ansible-docsmith print plain text so grep is reliable + steps: + - name: Check out the codebase. + uses: actions/checkout@v6 + + - name: Set up Python 3.11 + uses: actions/setup-python@v6 + with: + python-version: "3.11" + + # Pin the version: ansible-docsmith is a young tool, so we + # upgrade deliberately rather than picking up changes on every CI run. + - name: Install ansible-docsmith + run: python3 -m pip install ansible-docsmith==2.0.2 + + # Consistency between meta/argument_specs.yml and defaults/main.yml. + # validate exits 1 on ERROR by itself; here we additionally treat any + # WARNING (e.g. a default that differs between spec and defaults) as a failure. + - name: Validate argument_specs (warnings = errors) + run: | + set -uo pipefail + status=0 + for role in roles/*/; do + [ -f "${role}meta/argument_specs.yml" ] || continue + echo "::group::validate ${role}" + out=$(ansible-docsmith validate "${role}" 2>&1) && rc=0 || rc=$? + echo "$out" + echo "::endgroup::" + if [ "$rc" -ne 0 ] || grep -q "Warnings:" <<<"$out"; then + echo "::error::ansible-docsmith reported errors or warnings for ${role}" + status=1 + fi + done + exit $status + + # README freshness: regenerate every role README and fail if anything changed, + # i.e. the committed README no longer matches argument_specs.yml. + - name: Check that every variable in README is up to date + run: | + for role in roles/*/; do + [ -f "${role}meta/argument_specs.yml" ] || continue + ansible-docsmith generate "${role}" --no-defaults \ + --template-readme .docsmith/readme.md.j2 + done + git diff --exit-code -- 'roles/*/README.md' || { + echo "::error::A role README is out of date. Run locally:"; + echo " ansible-docsmith generate --no-defaults --template-readme .docsmith/readme.md.j2"; + exit 1; + } diff --git a/.github/workflows/test_role_logstash.yml b/.github/workflows/test_role_logstash.yml index 0bab065b..57756040 100644 --- a/.github/workflows/test_role_logstash.yml +++ b/.github/workflows/test_role_logstash.yml @@ -15,6 +15,7 @@ on: pull_request: paths: - 'roles/logstash/**' + - '!roles/logstash/**/*.md' - '.github/workflows/test_role_logstash.yml' - 'molecule/logstash_**' diff --git a/.github/workflows/test_role_repos.yml b/.github/workflows/test_role_repos.yml index c9747027..50527099 100644 --- a/.github/workflows/test_role_repos.yml +++ b/.github/workflows/test_role_repos.yml @@ -14,6 +14,7 @@ on: pull_request: paths: - 'roles/repos/**' + - '!roles/repos/**/*.md' - '.github/workflows/test_role_repos.yml' - 'molecule/repos_**' diff --git a/.github/workflows/test_roles_pr.yml b/.github/workflows/test_roles_pr.yml index 8ec712b4..483c9bcb 100644 --- a/.github/workflows/test_roles_pr.yml +++ b/.github/workflows/test_roles_pr.yml @@ -15,6 +15,8 @@ on: pull_request: paths: - 'roles/**' + - '!roles/**/*.md' + - '!roles/**/meta/argument_specs.yml' - 'molecule/elasticstack_default/**' - 'requirements-test.txt' - '.github/workflows/test_roles_pr.yml' diff --git a/README.md b/README.md index 99b4389f..4615e139 100644 --- a/README.md +++ b/README.md @@ -15,8 +15,8 @@ Every role is documented with all variables, please refer to the documentation f * [Beats](docs/role-beats.md) * [Elasticsearch](docs/role-elasticsearch.md) * [Kibana](docs/role-kibana.md) -* [Logstash](docs/role-logstash.md) -* [Repos](docs/role-repos.md) +* [Logstash](roles/logstash/README.md) +* [Repos](roles/repos/README.md) ## Modules documentation @@ -226,6 +226,25 @@ Every kind of contribution is very welcome. Open [issues](https://github.com/NET For now we open pull requests against `main`. We are planning to introduce dedicated branches to support older versions without breaking changes. Since we don't need them for now, please check back with this section because when we decided on how to proceed, you will find the information here. For now `main` always has the newest changes and if you want a stable version, please use the newest release. +### Documentation for role variables + +Role variables are documented from each role's `meta/argument_specs.yml`, which is +the single source of truth. When your pull request changes a role's variables: + +1. Update that role's `meta/argument_specs.yml` (type, default, description). +2. Regenerate the README variable table — please do **not** edit it by hand. The table is + produced by [ansible-docsmith](https://github.com/foundata/ansible-docsmith) + (install with `pip install ansible-docsmith` if you don't have it): + + ``` + ansible-docsmith generate roles/ --no-defaults --template-readme .docsmith/readme.md.j2 + ``` + +3. Commit the regenerated `README.md` together with your change. + +The `Test Documentation` workflow checks that each README matches its +`argument_specs.yml` and fails the pull request if they drift apart. + ## Testing Besides real tests that the developer should do before creating a PR, we built molecule scenarios to test the complete stack. diff --git a/RELEASE.md b/RELEASE.md index 09f2d4ec..048f040c 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -8,6 +8,10 @@ Remove the note above saying that this is a work and progress before the first release. And remove this section. +### Lint + +Make sure to have one last pull request to remove as much lint as possible. Same goes for deprecation warnings, linter exceptions etc. + ### Authors Update the [.mailmap](.mailmap) and [AUTHORS](AUTHORS) files: @@ -15,3 +19,17 @@ Update the [.mailmap](.mailmap) and [AUTHORS](AUTHORS) files: ``` git log --use-mailmap | grep '^Author:' | cut -f2- -d' ' | sort | uniq > AUTHORS ``` + +## Tag + +Tag the version. Don't use `v` in the name. Use semantic versioning. + +e.g. + +``` +git tag -as 7.6.4 +``` + +## Release + +Make the tag a release on GitHub. diff --git a/docs/getting-started.md b/docs/getting-started.md index 79569b50..b43d83c1 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -50,8 +50,8 @@ Roles * [Beats](role-beats.md) * [Elasticsearch](role-elasticsearch.md) * [Kibana](role-kibana.md) -* [Logstash](role-logstash.md) -* [Repos](role-repos.md) +* [Logstash](../roles/logstash/README.md) +* [Repos](../roles/repos/README.md) Variables diff --git a/docs/logstash-pipelines.md b/docs/logstash-pipelines.md deleted file mode 100644 index 496d1ce2..00000000 --- a/docs/logstash-pipelines.md +++ /dev/null @@ -1,238 +0,0 @@ -# Pipelines # - -## Keeping an overview ## - -It can be quite difficult to stay on top of your pipeline configuration because they tend to become very complex. - -This collection will leave some comments about how pipelines are interconnected within the `/etc/logstash/pipelines.yml` configuration file. - -If you set `logstash_mermaid` to `true` (which is the default), then you will also get a new file in `/etc/logstash/pipelines.mermaid`. You can paste it into a Mermaid editor in your documentation tool or in an [online Mermaid editor](https://mermaid.live/). The same content will be available on your control node in a temporary file. You can even add arbitrary code to reflect manually managed pipelines by using `logstash_mermaid_extra`. - -## Git managed ## - -If you have pipeline code managed in (and available via) Git repositories, you can use this role to check them out and integrate them into `pipelines.yml`. - -``` -logstash_pipelines: - syslog: - name: syslog - source: https://github.com/widhalmt/syslog-logstash-pipeline.git -``` - -You can add a `version` attribute to your pipeline. It defaults to `main`. You can use every string, [Ansibles git](https://docs.ansible.com/ansible/latest/collections/ansible/builtin/git_module.html) module accepts. - -## Input and Output ## - -### Basic configuration ### - -To have a single Redis input and output to your files, use this. - -``` -logstash_pipelines: - syslog: - name: syslog - source: https://github.com/netways/syslog-logstash-pipeline.git - exclusive: false - input: - - name: default - key: syslog-input - output: - - name: default - key: syslog-output -``` - -This will result in your pipeline checking out the configuration on GitHub and adding these two extras. In extra files, just shown in one place to safe space. - -``` -input { - redis { - host => "localhost" - data_type => "list" - key => "syslog-input" - } -} -output { - redis { - host => "localhost" - data_type => "list" - key => "syslog-output" - } -} -``` - -### Multiple inputs ### - -Just give more inputs with `name` and `key`. Every key will be read. - -### More complex configuration ### - -If you want a bit more control over which outputs are used, the role offers more sophisticated configuration. - -If you have several outputs that all have conditions, like just send some messages to a development system or only alerts to a monitoring system. - -``` -logstash_pipelines: - syslog: - name: syslog - source: https://github.com/netways/syslog-logstash-pipeline.git - exclusive: false - input: - - name: default - key: input - output: - - name: special - key: myspecial - condition: '[program] == "special"' - - name: special2 - key: myspecial2 - condition: '[program] == "special2"' - - name: default - key: forwarder -``` -This will give you the following configuration: - -``` -input { - -# default output - redis { - host => "localhost" - data_type => "list" - key => "input" - } - -} - -output { - -# special output -if [program] == "special"{ - redis { - host => "localhost" - data_type => "list" - key => "myspecial" - } -} - -# special2 output -if [program] == "special2"{ - redis { - host => "localhost" - data_type => "list" - key => "myspecial2" - } -} - -# default output - redis { - host => "localhost" - data_type => "list" - key => "forwarder" - } - -} -``` - -Note that the `default` output get's **every** event, the other two outputs only get those where the condition is met. - -You can combine several outputs with `else`. That's helpful when you want to split events. Like syslog messages depending on which program logged an event. Just change `exclusive` to `true`. - -``` -logstash_pipelines: - syslog: - name: syslog - source: https://github.com/netways/syslog-logstash-pipeline.git - exclusive: true - input: - - name: default - key: input - output: - - name: special - key: myspecial - condition: '[program] == "special"' - - name: special2 - key: myspecial2 - condition: '[program] == "special2"' - - name: default - key: forwarder -``` - -This will give you the following Logstash configuration. - -``` -input { - -# default output - redis { - host => "localhost" - data_type => "list" - key => "input" - } - -} - -output { - -# special output -if [program] == "special" { - redis { - host => "localhost" - data_type => "list" - key => "myspecial" - } -} -# special2 output -else if [program] == "special2" { - redis { - host => "localhost" - data_type => "list" - key => "myspecial2" - } -} - -# default output -else { - redis { - host => "localhost" - data_type => "list" - key => "forwarder" - } - -} - -} -``` - -Here the `default` output only receives the events that haven't already been sent to one of the others. - -## Extra configuration ## - -### Congestion threshold ### - -Every Output can have a `congestion:` option with a numerical value. If the Redis key already holds more items than the value says, the output will stop. - -### Unsafe shutdown ### - -If you need unsafe Logstash shutdowns, e.g. for testing, you can set `logstash_pipeline_unsafe_shutdown` to `true`. The variable doesn't have a default so Logstash falls back to its internal default of `false`. - -## Caveats ## - -There are still some minor issues you need to keep in mind: - -* The default output in an `exclusive: true` setup must be the last in the YAML configuration. There's no sorting, the role simply expects the default to be the last one. -* The configuration *should* work but will make no sense if you have `exclusive: true` but two or more outputs without `condition`. - -## Custom pipelines ## - -If you have other ways of putting pipeline code into the correct directories, you can just skip the `source` option. - -``` -logstash_pipelines: - syslog: - name: syslog -``` -**You have to make sure the code is available or Logstash will constantly log errors!** - -This will create the directories and integrate all `*.conf` files within via `pipelines.yml`. - -**If you add a source later, the role will delete the directory and recreate it with it's own code. So make sure you have a backup!" diff --git a/docs/role-logstash.md b/docs/role-logstash.md index d42f7da2..15d1fb7f 100644 --- a/docs/role-logstash.md +++ b/docs/role-logstash.md @@ -1,133 +1,5 @@ -Ansible Role: Logstash -========= +# Logstash role -![Test Role Logstash](https://github.com/netways/ansible-collection-elasticstack/actions/workflows/test_role_logstash.yml/badge.svg) +The Logstash role documentation now lives with the role itself: -This role installs and configures [Logstash](https://www.elastic.co/products/logstash) on Linux systems. - -It can optionally configure two types of Logstash pipelines: -* Pipeline configuration managed in an external git repository -* A default pipeline which will read from different Redis keys and write into Elasticsearch - -For details on how to configure pipelines please refer to our [docs about pipelines](./logstash-pipelines.md). - -Details about configured pipelines will be written into `pipelines.yml` as comments. Same goes for logging configuration in `log4j.options`. - -It will work with the standard Elastic Stack packages and Elastics OSS variant. - -Requirements ------------- - -* `community.general` collection - -You will need these packages / libraries installed. Some very basic packages like `openssl` get handled by the collection if needed. The following list contains packages and libraries which only apply to special cases or need for you to decide on the installation method. - -* `passlib` Python library if you do not disable password hashing for logstash user. It should be installed with pip on the Ansible controller. - -You need to have the Elastic Repos configured on your system. You can use our [role](./role-repos.md) - -If you want to use the default pipeline configuration you need to have `git` available. - -You need to have `curl` installed. We are using `curl` instead of the `uri` module because we got better results. Feel free to file a pull request if you find a working solution. (And please remove this line with it) - -If you want to use the default pipeline (or other pipelines communicating via Redis) you might want to install Redis first (e.g. by using an [Ansible Role for Redis](https://galaxy.ansible.com/geerlingguy/redis) - -Role Variables --------------- - -* *elasticstack_version*: Version number of Logstash to install (e.g. `7.10.1`). Only set if you don't want the latest. (default: none). For OSS version see `elasticstack_variant` below. -* *logstash_enable*: Start and enable Logstash service (default: `true`) -* *logstash_config_backup*: Keep backups of all changed configuration (default: `no`) -* *logstash_manage_yaml*: Manage and overwrite `logstash.yml` (default: `true`) -* *logstash_manage_logging*: Manage log4j configuration (default: `false`) -* *logstash_plugins*: List of plugins to install (default: none) -* *logstash_certs_dir*: Path to certificates. Will be used to build paths of several files. (Default: `/etc/logstash/certs`) -* *logstash_heap*: JVM heap size in GB (default: `1`). Elastic recommends 4-8 GB for typical ingestion and staying below 50-75% of physical memory. Keep it low when Logstash shares a host with Elasticsearch so they do not compete for memory. - -If `logstash.yml` is managed, the following settings apply. - -* *logstash_config_autoreload*: Enable autoreload of Logstash configuration (default: `true`) -* *logstash_config_path_data*: Logstash data directory (default: `/var/lib/logstash`) -* *logstash_config_path_logs*: Logstash log directory (default: `/var/log/logstash`) - -Aside from `logstash.yml` we can manage Logstashs pipelines. - -* *logstash_manage_pipelines*: Manage `pipelines.yml` (default: `true`) -* *logstash_no_pipelines*: Don't manage pipelines at all (default: `false`) -* *logstash_pipelines*: List of pipelines with optional URL to repo (see [pipelines documentation](file:///roles/logstash/docs/pipelines.md) for details) -* *logstash_global_ecs*: Set ECS compatibilty mode (default: none. Possible values: `disabled` or `v1`) -* *logstash_elasticsearch_output*: Enable default pipeline to Elasticsearch (default: `true`) -* *logstash_ident*: Add a field identifying the node that processed an event (default: `true`) -* *logstash_ident_field_name*: Name of the identifying the instance (default: `"[netways][instance]"`) -* *logstash_beats_input*: Enable default pipeline with `beats` input (default: `true`) -* *logstash_beats_input_congestion*: Optional congestion threshold for the beats input pipeline -* *logstash_beats_timeout*: Optional timeout for client connections. (Example: `60s`) -* *logstash_beats_tls*: Activate TLS for the beats input pipeline (default: none but `true` with full stack setup if not set) -* *logstash_tls_key_passphrase*: Passphrase for Logstash certificates (default: `LogstashChangeMe`) -* *elasticstack_ca_pass*: Password for Elasticsearch CA (default: `PleaseChangeMe`) -* *logstash_cert_validity_period*: number of days that the generated certificates are valid (default: 1095). -* *logstash_cert_expiration_buffer*: Ansible will renew the Logstash certificate if its validity is shorter than this value, which should be number of days. (default: 30) -* *logstash_cert_will_expire_soon*: Set it to true to renew logstash certificate (default: `false`), Or run the playbook with `--tags renew_logstash_cert` to do that. -* *logstash_elasticsearch*: Address of Elasticsearch instance for default output (default: list of Elasticsearch nodes from `elasticsearch` role or `localhost` when used standalone) -* *logstash_security*: Enable X-Security (No default set, but will be activated when in full stack mode) -* *logstash_create_user*: Enables creation `logstash_user_name` (Default: `true`) -* *logstash_user_name*: Name of the user to connect to Elasticsearch (Default: `logstash_writer`) -* *logstash_user_email*: email-address that is linked with the logstash_user_name (Default: `""`) -* *logstash_user_fullname*: fullname that is linked with the logstash_user_name (Default: `Internal Logstash User`) -* *logstash_user_password*: Password of `logstash_user_name` in Elasticsearch. It must be at least 6 characters long (default: `password`) -* *logstash_create_role*: Enables creation `logstash_role_name` (Default: `true`) -* *logstash_role_name*: Name of the logstash role that is getting created (Default: `logstash_writer`) -* *logstash_role_cluster_privileges*: Cluster privileges the role has access to (default: `"manage_index_templates", "monitor", "manage_ilm"`) -* *logstash_role_indicies_names*: Indices the role has access to (default: `"ecs-logstash*", "logstash*", "logs*"`) -* *logstash_role_indicies_privileges*: Index permissions the role has on `logstash_role_indicies_names` (default: `"write", "create", "delete", "create_index", "manage", "manage_ilm"`) -* *logstash_reset_writer_role*: Reset user and role with every run: (default: `true`) -* *logstash_validate_after_inactivity*: How long should logstash wait, before starting a new connection and leave the old one with elasticsearch, when the connection with elasticsearch get lost: (Default: `300`). -* *logstash_queue_type*: What kind of queue should Logstash use per default: (Default: `persisted`, alternative: `memory`) -* *logstash_queue_max_bytes*: The total capacity of ansible-forwarder queue in number of bytes: (Default: `2gb`) -* *logstash_sniffing*: Enable sniffing (Default: `false`). -* *logstash_sniffing_delay*: How long to wait, in seconds, between sniffing attempts (Default: `not set`). -* *logstash_sniffing_path*: HTTP Path to be used for the sniffing requests (Default: `not set`). -* *logstash_legacy_monitoring*: Enables legacy monitoring - ignored when `elasticstack_full_stack` is not set. (default: `true`) -* *logstash_redis_password*: If set this will use this password when connecting our simple inputs and outputs to Redis. (default: not set) - -* *logstash_mermaid*: Print overview over Logstash pipelines in Mermaid syntax. (default: `true`) -* *logstash_mermaid_logstash*: Place Mermaid syntax into `/etc/logstash/pipelines.mermaid` on Logstash hosts. (default: `true`) -* *logstash_mermaid_local*: Place Mermaid syntax into temporary file on control node. (default: `false`) -* *logstash_mermaid_extra*: You can add extra Mermaid syntax to the output by adding it to this variable. YAML-multiline is supported. (default: none) - -The following variables configure Log4j for Logstash. All default to `true` as this is the default after the installation. - -* *logstash_logging_console*: Log to console - syslog when run via systemd -* *logstash_logging_file*: Log to logfile -* *logstash_logging_slow_console*: Log slowlog to console - syslog when run via systemd -* *logstash_logging_slow_file*: Log slowlog to logfile - -The following variables configure extra fields in your events that help with identifying which pipelines have been passed or which pipeline is used how much. - -* *logstash_pipeline_identifier*: Activate this feature (default: `true`) -* *logstash_pipeline_identifier_field_name*: Name of the field to add (default: `"[netways][pipeline]"`) -* *logstash_pipeline_identifier_defaults*: Use identifiers in default pipelines, too (default: `false`) This could lead to the defaults dominating all statistics with virtually no value, but if you want to see, them, you can. - -The following variables are identical over all our elastic related roles, hence the different naming scheme. - -*elasticstack_release*: Major release version of Elastic stack to configure. (default: `7`) -*elasticstack_variant*: Variant of the stack to install. Valid values: `elastic` or `oss`. (default: `elastic`) - -The following variables only apply if you use this role together with our Elasticsearch and Kibana roles. - -* *elasticstack_full_stack*: Use `ansible-role-elasticsearch` as well (default: `false`) -* *elasticstack_ca_dir*: Directory where the CA and certificates lie on the main Elasticsearch host (default: `/opt/es-ca`) -* *elasticstack_elasticsearch_http_port*: Port of Elasticsearch to send events to (Default: `9200`) -* *elasticstack_initial_passwords*: File where initial passwords are stored on the main Elasticsearch host (default: `/usr/share/elasticsearch/initial_passwords`) - -## Usage - -``` -- name: Install Logstash - hosts: logstash-host - collections: - - netways.elasticstack - roles: - - repos - - logstash -``` +**➜ [roles/logstash/README.md](../roles/logstash/README.md)** diff --git a/docs/role-repos.md b/docs/role-repos.md index a47e62ce..bce0c25a 100644 --- a/docs/role-repos.md +++ b/docs/role-repos.md @@ -1,43 +1,5 @@ -Elastic Repos -========= +# Repos role -![Test Role repos](https://github.com/netways/ansible-collection-elasticstack/actions/workflows/test_role_repos.yml/badge.svg) +The Repos role documentation now lives with the role itself: -The role adds Elastic repositories to the package manager. It's main use is in connection with other roles that provide installation and configuration of the Elastic Stack. - -Requirements ------------- - -* You need `gpg` to be installed because packages / repositories are digitally signed and verified. -* Debian and Ubuntu hosts need to have `apt-transport-https` installed to deal with Elastics repositories. -* Ubuntu hosts also need to have `gpg-agent` installed. -* For SuSE hosts you need the Ansible collection `community.general` on your Ansible controller. - -Role Variables --------------- - -* *elasticstack_release*: Major release version of Elastic stack to configure. (default: `7`). `7` and `8` are supported. -* *elasticstack_variant*: Variant of the stack to install. Valid values: `elastic` or `oss`. (default: `elastic`). -* *elasticstack_enable_repos*: Enable repositories after creating them. (default: `true`) Only works on RPM based distributions! - -Please note that no `oss` versions are available for Elastic Stack later than `7`. This role will fail if you try to install them. - -Usage --------- - -Upgrades -======== - -If you want to be able to update your operating system without worrying about accidentally upgrading Elastic Stack, set `elasticstack_enable_repos` to `false`. The roles in this collection will enable the repository in case they need it. Keep in mind that this will only work on rpm based distributions. - -Example playbook -================ - -``` - - hosts: all - become: yes - collections: - - netways.elasticstack - roles: - - repos -``` +**➜ [roles/repos/README.md](../roles/repos/README.md)** diff --git a/galaxy.yml b/galaxy.yml index e719ca01..9c34f7b0 100644 --- a/galaxy.yml +++ b/galaxy.yml @@ -37,4 +37,5 @@ build_ignore: - .mailmap - '*.tar.gz' - 'venv*' + - .docsmith # Minimum Ansible version required: 2.14, maximum supported: 2.20 diff --git a/roles/logstash/README.md b/roles/logstash/README.md new file mode 100644 index 00000000..953f3838 --- /dev/null +++ b/roles/logstash/README.md @@ -0,0 +1,139 @@ +# Ansible Role: Logstash + +Installs and configures [Logstash](https://www.elastic.co/products/logstash) on +Linux systems. The role can manage `logstash.yml`, the log4j2 logging, the JVM +heap, TLS, and the Elasticsearch writer role/user. For pipelines it can create +two default pipelines — a Beats input and an Elasticsearch forwarder, both using +Redis — and manage your own pipelines, whose configuration can be checked out +from external git repositories. + +It works with the standard Elastic Stack packages and with Elastic's OSS variant. + +## Requirements + +* The `community.general` collection. +* `passlib` (Python, on the controller) unless you disable password hashing for the Logstash user. +* `curl` on the target host. +* `git` on the target host if you use git-based pipelines. +* Redis if you use the default pipeline or other pipelines that talk to Redis + (e.g. via the [`geerlingguy.redis`](https://galaxy.ansible.com/geerlingguy/redis) role). + +You also need the Elastic repositories configured — use the [`repos`](../repos) role. + +## Example + +```yaml +- name: Install Logstash + hosts: logstash + collections: + - netways.elasticstack + roles: + - repos + - logstash +``` + +## Pipelines + +For how to configure pipelines (built-in Redis/Beats and external git +repositories), see the [pipelines documentation](docs/pipelines.md). + +## Tags + +Run only parts of the role with `--tags`: + +* `configuration` (alias `logstash_configuration`) — only (re)write the configuration, skip installation. +* `certificates` — only generate and distribute the TLS certificates. +* `renew_logstash_cert` / `renew_ca` — force renewal of the Logstash certificate. +* `mermaid` — only regenerate the pipeline overview (`pipelines.mermaid`). + + +## Role variables + +| Variable | Type | Default | Choices | Description | +|----------|------|---------|---------|-------------| +| `logstash_enable` | `bool` | `true` | — | Start and enable the Logstash service. | +| `logstash_config_backup` | `bool` | `false` | — | Keep backups of every configuration file the role changes. | +| `logstash_manage_yaml` | `bool` | `true` | — | Manage and overwrite logstash.yml. | +| `logstash_manage_logging` | `bool` | `false` | — | Manage the log4j2 logging configuration. | +| `logstash_heap` | `str` | `"1"` | — | JVM heap size in GB. Sets both -Xms and -Xmx via an LS_JAVA_OPTS systemd drop-in.

Elastic recommends 4-8 GB for typical ingestion and staying below 50-75% of physical memory. Keep it low when Logstash shares a host with Elasticsearch so they do not compete for memory. | +| `logstash_plugins` | `list` of `str` | N/A | — | List of Logstash plugins to install. Unset by default. | +| `logstash_config_autoreload` | `bool` | `true` | — | Enable automatic reload of the Logstash configuration. | +| `logstash_config_autoreload_interval` | `str` | N/A | — | Interval between configuration reload checks (e.g. "3s"). Only used when logstash_config_autoreload is enabled. Unset by default. | +| `logstash_config_path_data` | `str` | `"/var/lib/logstash"` | — | Logstash data directory (path.data). | +| `logstash_config_path_logs` | `str` | `"/var/log/logstash"` | — | Logstash log directory (path.logs). | +| `logstash_http_host` | `str` | N/A | — | Bind address of the Logstash monitoring API (http.host). Unset by default. | +| `logstash_http_port` | `str` | N/A | — | Port (or port range, e.g. "9600-9700") of the monitoring API (http.port). Unset by default. | +| `logstash_global_ecs` | `str` | N/A | `disabled`, `v1` | Set the global ECS compatibility mode (pipeline.ecs_compatibility). Unset by default. | +| `logstash_pipeline_unsafe_shutdown` | `bool` | N/A | — | Force Logstash to exit during shutdown even if there are in-flight events. Unset by default. | +| `logstash_legacy_monitoring` | `bool` | `true` | — | Enable legacy X-Pack monitoring. Ignored unless elasticstack_full_stack is set and only effective on Elastic Stack releases lower than 8. | +| `logstash_manage_pipelines` | `bool` | `true` | — | Manage pipelines.yml. | +| `logstash_no_pipelines` | `bool` | `false` | — | Disable all pipeline management entirely. | +| `logstash_pipelines` | `list` of `dict` | … | — | List of pipelines to configure.

Each entry needs a name. A pipeline either points to an external git repository (source/version) or defines simple input/output keys that connect to Redis. See the pipelines documentation for details. | +|     ↳ `name` | `str` | N/A | — | Unique name of the pipeline. Becomes the pipeline directory and pipeline.id. | +|     ↳ `exclusive` | `bool` | N/A | — | Mark this pipeline as the only one allowed to handle its events. | +|     ↳ `source` | `str` | N/A | — | URL of a git repository holding the pipeline configuration. | +|     ↳ `version` | `str` | N/A | — | Git branch/tag/commit to check out from source (default "main"). | +|     ↳ `queue_type` | `str` | N/A | `memory`, `persisted` | Queue type for this pipeline (default "memory"). Use "persisted" for an on-disk persistent queue. | +|     ↳ `queue_max_bytes` | `str` | N/A | — | Maximum queue size for this pipeline (default "1gb"). | +|     ↳ `input` | `list` of `dict` | … | — | Simple Redis inputs for this pipeline. | +|         ↳ `name` | `str` | N/A | — | Name of the input. | +|         ↳ `key` | `str` | N/A | — | Redis key to read from. | +|     ↳ `output` | `list` of `dict` | … | — | Simple Redis outputs for this pipeline. | +|         ↳ `name` | `str` | N/A | — | Name of the output. | +|         ↳ `key` | `str` | N/A | — | Redis key to write to. | +|         ↳ `condition` | `str` | N/A | — | Optional Logstash conditional. The output only receives events matching it. With exclusive set, the conditions are chained with else if. | +|         ↳ `congestion` | `int` | N/A | — | Optional congestion threshold. The output stops once the target Redis key holds more items than this value. | +| `logstash_elasticsearch_output` | `bool` | `true` | — | Create the default pipeline that forwards events to Elasticsearch. | +| `logstash_beats_input` | `bool` | `true` | — | Create the default pipeline with a Beats input. | +| `logstash_beats_input_congestion` | `int` | N/A | — | Congestion threshold for the default Beats input pipeline: it stops writing to Redis once the buffer key holds more items than this value. Unset by default. | +| `logstash_beats_timeout` | `str` | N/A | — | Timeout for idle client connections on the Beats input (e.g. "60s"). Unset by default. | +| `logstash_beats_tls` | `bool` | N/A | — | Activate TLS on the Beats input pipeline. Unset by default, but enabled automatically in a full stack setup unless overridden. | +| `logstash_input_queue_type` | `str` | `"memory"` | `memory`, `persisted` | Queue type for the default Beats input pipeline. Use "persisted" for an on-disk persistent queue. | +| `logstash_input_queue_max_bytes` | `str` | `"1gb"` | — | Maximum queue size for the default Beats input pipeline. | +| `logstash_forwarder_queue_type` | `str` | `"memory"` | `memory`, `persisted` | Queue type for the default Elasticsearch forwarder pipeline. Use "persisted" for an on-disk persistent queue. | +| `logstash_forwarder_queue_max_bytes` | `str` | `"1gb"` | — | Maximum queue size for the default Elasticsearch forwarder pipeline. | +| `logstash_redis_password` | `str` | N/A | — | Password used when the simple inputs/outputs connect to Redis. Unset by default. | +| `logstash_elasticsearch` | `list` of `str` | N/A | — | Elasticsearch hosts for the default output. Defaults to the nodes from the elasticsearch group, or localhost when used standalone. | +| `logstash_validate_after_inactivity` | `str` | `"300"` | — | Seconds Logstash waits before validating a previously idle connection to Elasticsearch. | +| `logstash_sniffing` | `bool` | `false` | — | Enable sniffing for additional Elasticsearch nodes. | +| `logstash_sniffing_delay` | `str` | N/A | — | Seconds to wait between sniffing attempts. Unset by default. | +| `logstash_sniffing_path` | `str` | N/A | — | HTTP path used for the sniffing requests. Unset by default. | +| `logstash_security` | `bool` | N/A | — | Enable X-Pack security (TLS) for Logstash. No default; enabled automatically in full stack mode with the elastic variant. | +| `logstash_tls_key_passphrase` | `str` | `"LogstashChangeMe"` | — | Passphrase for the generated Logstash certificates. | +| `logstash_certs_dir` | `str` | `"/etc/logstash/certs"` | — | Directory holding the Logstash certificates. Used to build several file paths. | +| `logstash_cert_validity_period` | `int` | `1095` | — | Number of days the generated certificates are valid. | +| `logstash_cert_expiration_buffer` | `int` | `30` | — | Renew the certificate when its remaining validity (in days) drops below this value. | +| `logstash_cert_will_expire_soon` | `bool` | `false` | — | Set to true to force renewal of the Logstash certificate. Alternatively run the playbook with the renew_logstash_cert tag. | +| `logstash_create_role` | `bool` | `true` | — | Create the Logstash writer role in Elasticsearch. | +| `logstash_role_name` | `str` | `"logstash_writer"` | — | Name of the Logstash writer role. | +| `logstash_role_cluster_privileges` | `list` of `str` | `['manage_index_templates', 'monitor', 'manage_ilm']` | — | Cluster privileges granted to the Logstash writer role. | +| `logstash_role_indicies_names` | `list` of `str` | `['ecs-logstash*', 'logstash*', 'logs*']` | — | Index patterns the Logstash writer role may access. | +| `logstash_role_indicies_privileges` | `list` of `str` | `['write', 'create', 'delete', 'create_index', 'manage', 'manage_ilm']` | — | Index privileges the Logstash writer role holds on its index patterns. | +| `logstash_create_user` | `bool` | `true` | — | Create the Logstash writer user in Elasticsearch. | +| `logstash_user_name` | `str` | `"logstash_writer"` | — | Name of the Logstash user connecting to Elasticsearch. | +| `logstash_user_password` | `str` | `"password"` | — | Password of the Logstash user. Must be at least 6 characters long. | +| `logstash_user_email` | `str` | `""` | — | Email address linked to the Logstash user. | +| `logstash_user_fullname` | `str` | `"Internal Logstash User"` | — | Full name linked to the Logstash user. | +| `logstash_logging_console` | `bool` | `true` | — | Log to the console (syslog when run via systemd). Only effective when logstash_manage_logging is enabled. | +| `logstash_logging_file` | `bool` | `true` | — | Log to the log file. Only effective when logstash_manage_logging is enabled. | +| `logstash_logging_slow_console` | `bool` | `true` | — | Log the slowlog to the console (syslog when run via systemd). Only effective when logstash_manage_logging is enabled. | +| `logstash_logging_slow_file` | `bool` | `true` | — | Log the slowlog to the log file. Only effective when logstash_manage_logging is enabled. | +| `logstash_ident` | `bool` | `true` | — | Add a field identifying the node that processed an event. | +| `logstash_ident_field_name` | `str` | `"[netways][instance]"` | — | Name of the field that identifies the instance. | +| `logstash_pipeline_identifier` | `bool` | `true` | — | Add a field identifying which pipeline processed an event. | +| `logstash_pipeline_identifier_field_name` | `str` | `"[netways][pipeline]"` | — | Name of the pipeline identifier field. | +| `logstash_pipeline_identifier_defaults` | `bool` | `false` | — | Also add pipeline identifiers in the default pipelines. This can let the defaults dominate statistics with little value. | +| `logstash_mermaid` | `bool` | `true` | — | Produce an overview of the Logstash pipelines in Mermaid syntax. | +| `logstash_mermaid_logstash` | `bool` | `true` | — | Place the Mermaid syntax into /etc/logstash/pipelines.mermaid on the Logstash hosts. | +| `logstash_mermaid_local` | `bool` | `false` | — | Place the Mermaid syntax into a temporary file on the control node. | +| `logstash_mermaid_extra` | `str` | N/A | — | Extra Mermaid syntax to append to the output (YAML multiline supported). Unset by default. | +| `logstash_freshstart` | `dict` | `{'changed': False}` | — | Internal state used by the role to detect a fresh install. Do not set manually. | + + + +## Shared variables + +This role also uses the collection-wide `elasticstack_*` variables (e.g. +`elasticstack_full_stack`, `elasticstack_ca_host`, `elasticstack_ca_pass`, +`elasticstack_release`, `elasticstack_variant`). They are documented centrally +with the [elasticstack role](../../docs/role-elasticsearch.md). diff --git a/roles/logstash/RELEASE.md b/roles/logstash/RELEASE.md deleted file mode 100644 index 17c8792b..00000000 --- a/roles/logstash/RELEASE.md +++ /dev/null @@ -1,33 +0,0 @@ -# Release Workflow - -## Preparations - -### Lint - -Make sure to have one last pull request to remove as much lint as possible. Same goes for deprecation warnings, linter exceptions etc. - -### Authors - -Update the [.mailmap](.mailmap) and [AUTHORS](AUTHORS) files: - -``` -git log --use-mailmap | grep '^Author:' | cut -f2- -d' ' | sort | uniq > AUTHORS -``` - -## Tag - -Tag the version. Don't use `v` in the name. User semantic versioning. - -e.g. - -``` -git tag -as 7.6.4 -``` - -## Release - -Make the tag a release on GitHub - -## Ansible Galaxy - -Do *not* push this role to Ansible Galaxy. Versioning helps with ongoing projects but we will sum up this role and all related ones in an Ansible collection and push that to GitHub. diff --git a/roles/logstash/defaults/main.yml b/roles/logstash/defaults/main.yml index 007a4d15..d114438d 100644 --- a/roles/logstash/defaults/main.yml +++ b/roles/logstash/defaults/main.yml @@ -1,7 +1,7 @@ --- # defaults file for logstash logstash_enable: true -logstash_config_backup: no +logstash_config_backup: false logstash_manage_yaml: true logstash_manage_logging: false @@ -18,7 +18,6 @@ logstash_config_path_logs: /var/log/logstash # pipeline configuration # logstash_manage_pipelines: true -logstash_queue_type: persisted # this will deactivate all pipeline management logstash_no_pipelines: false @@ -69,7 +68,6 @@ logstash_user_name: logstash_writer logstash_user_password: password logstash_user_email: "" logstash_user_fullname: "Internal Logstash User" -logstash_reset_writer_role: true # logstash security logstash_tls_key_passphrase: LogstashChangeMe diff --git a/roles/logstash/docs/pipelines.md b/roles/logstash/docs/pipelines.md index a41548e7..30ad89ba 100644 --- a/roles/logstash/docs/pipelines.md +++ b/roles/logstash/docs/pipelines.md @@ -1,18 +1,28 @@ # Pipelines # +## Keeping an overview ## + +It can be quite difficult to stay on top of your pipeline configuration because they tend to become very complex. + +This collection will leave some comments about how pipelines are interconnected within the `/etc/logstash/pipelines.yml` configuration file. + +If you set `logstash_mermaid` to `true` (which is the default), then you will also get a new file in `/etc/logstash/pipelines.mermaid`. You can paste it into a Mermaid editor in your documentation tool or in an [online Mermaid editor](https://mermaid.live/). The same content will be available on your control node in a temporary file. You can even add arbitrary code to reflect manually managed pipelines by using `logstash_mermaid_extra`. + ## Git managed ## +`logstash_pipelines` is a **list**. Each entry needs a `name`; everything else is optional. + If you have pipeline code managed in (and available via) Git repositories, you can use this role to check them out and integrate them into `pipelines.yml`. -``` +```yaml logstash_pipelines: - syslog: - name: syslog - source: https://github.com/widhalmt/syslog-logstash-pipeline.git + - name: syslog + source: https://github.com/netways/syslog-logstash-pipeline.git ``` -You can add a `version` attribute to your pipeline. It defaults to `main`. You can use every string, [Ansibles git](https://docs.ansible.com/ansible/latest/collections/ansible/builtin/git_module.html) module accepts. -You can also determine the place for logstash to store the data of each pipeline while processing it by using `queue_type` attribute. It defaults to `memory`, you can use `persisted` to write the data temporarily to the hard disk. Moreover you can limit the maximal size of data in every pipeline throgh `queue_max_bytes` attribute, which is default `1gb`. +You can add a `version` attribute to your pipeline. It defaults to `main`. You can use every string [Ansible's git](https://docs.ansible.com/ansible/latest/collections/ansible/builtin/git_module.html) module accepts. + +You can also determine where Logstash stores the data of each pipeline while processing it by using the `queue_type` attribute. It defaults to `memory`; use `persisted` to write the data temporarily to disk. You can limit the maximum amount of data per pipeline with `queue_max_bytes`, which defaults to `1gb`. ## Input and Output ## @@ -20,13 +30,12 @@ You can also determine the place for logstash to store the data of each pipeline To have a single Redis input and output to your files, use this. -``` +```yaml logstash_pipelines: - syslog: - name: syslog - source: https://github.com/widhalmt/syslog-logstash-pipeline.git - queue.type: memory - queue.max_bytes: 1gb + - name: syslog + source: https://github.com/netways/syslog-logstash-pipeline.git + queue_type: memory + queue_max_bytes: 1gb exclusive: false input: - name: default @@ -65,13 +74,10 @@ If you want a bit more control over which outputs are used, the role offers more If you have several outputs that all have conditions, like just send some messages to a development system or only alerts to a monitoring system. -``` +```yaml logstash_pipelines: - syslog: - name: syslog - source: https://github.com/widhalmt/syslog-logstash-pipeline.git - queue.type: memory - queue.max_bytes: 1gb + - name: syslog + source: https://github.com/netways/syslog-logstash-pipeline.git exclusive: false input: - name: default @@ -134,11 +140,10 @@ Note that the `default` output get's **every** event, the other two outputs only You can combine several outputs with `else`. That's helpful when you want to split events. Like syslog messages depending on which program logged an event. Just change `exclusive` to `true`. -``` +```yaml logstash_pipelines: - syslog: - name: syslog - source: https://github.com/widhalmt/syslog-logstash-pipeline.git + - name: syslog + source: https://github.com/netways/syslog-logstash-pipeline.git exclusive: true input: - name: default @@ -206,7 +211,11 @@ Here the `default` output only receives the events that haven't already been sen ### Congestion threshold ### -Every Output can have a `congestion:` option with a numerical value. If the Redis key already holds more items than the value says, the output will stop. +Every output can have a `congestion` option with a numerical value. If the Redis key already holds more items than the value says, the output will stop. + +### Unsafe shutdown ### + +If you need unsafe Logstash shutdowns, e.g. for testing, you can set `logstash_pipeline_unsafe_shutdown` to `true`. The variable doesn't have a default so Logstash falls back to its internal default of `false`. ## Caveats ## @@ -219,13 +228,12 @@ There are still some minor issues you need to keep in mind: If you have other ways of putting pipeline code into the correct directories, you can just skip the `source` option. -``` +```yaml logstash_pipelines: - syslog: - name: syslog + - name: syslog ``` **You have to make sure the code is available or Logstash will constantly log errors!** This will create the directories and integrate all `*.conf` files within via `pipelines.yml`. -**If you add a source later, the role will delete the directory and recreate it with it's own code. So make sure you have a backup!" +**If you add a source later, the role will delete the directory and recreate it with its own code. So make sure you have a backup!** diff --git a/roles/logstash/meta/argument_specs.yml b/roles/logstash/meta/argument_specs.yml new file mode 100644 index 00000000..95ac3641 --- /dev/null +++ b/roles/logstash/meta/argument_specs.yml @@ -0,0 +1,460 @@ +--- +# Role argument specification for the logstash role. +# This is the single source of truth for the role's variables: Ansible validates +# the supplied values against it at role start, and ansible-docsmith renders the +# README variable reference and the comments in defaults/main.yml from it. +# +# Only logstash_* variables live here. The shared elasticstack_* variables are +# documented centrally with the elasticstack role. + +argument_specs: + main: + short_description: Install and configure Logstash + description: + - Installs and configures Logstash on Linux systems. + - >- + Can manage logstash.yml, log4j2 logging, the JVM heap, TLS and the + Elasticsearch writer role/user. + - >- + Creates two default pipelines, a Beats input and an Elasticsearch + forwarder (both using Redis), and manages user-defined pipelines whose + configuration can be checked out from external git repositories. + author: + - NETWAYS GmbH + + options: + + # ----- Service & general ----- + logstash_enable: + type: bool + default: true + description: Start and enable the Logstash service. + + logstash_config_backup: + type: bool + default: false + description: Keep backups of every configuration file the role changes. + + logstash_manage_yaml: + type: bool + default: true + description: Manage and overwrite logstash.yml. + + logstash_manage_logging: + type: bool + default: false + description: Manage the log4j2 logging configuration. + + logstash_heap: + type: str + default: "1" + description: + - JVM heap size in GB. Sets both -Xms and -Xmx via an LS_JAVA_OPTS systemd drop-in. + - >- + Elastic recommends 4-8 GB for typical ingestion and staying below 50-75% + of physical memory. Keep it low when Logstash shares a host with + Elasticsearch so they do not compete for memory. + + logstash_plugins: + type: list + elements: str + description: List of Logstash plugins to install. Unset by default. + + # ----- logstash.yml settings ----- + logstash_config_autoreload: + type: bool + default: true + description: Enable automatic reload of the Logstash configuration. + + logstash_config_autoreload_interval: + type: str + description: >- + Interval between configuration reload checks (e.g. "3s"). Only used + when logstash_config_autoreload is enabled. Unset by default. + + logstash_config_path_data: + type: str + default: /var/lib/logstash + description: Logstash data directory (path.data). + + logstash_config_path_logs: + type: str + default: /var/log/logstash + description: Logstash log directory (path.logs). + + logstash_http_host: + type: str + description: Bind address of the Logstash monitoring API (http.host). Unset by default. + + logstash_http_port: + type: str + description: Port (or port range, e.g. "9600-9700") of the monitoring API (http.port). Unset by default. + + logstash_global_ecs: + type: str + choices: + - disabled + - v1 + description: Set the global ECS compatibility mode (pipeline.ecs_compatibility). Unset by default. + + logstash_pipeline_unsafe_shutdown: + type: bool + description: Force Logstash to exit during shutdown even if there are in-flight events. Unset by default. + + logstash_legacy_monitoring: + type: bool + default: true + description: >- + Enable legacy X-Pack monitoring. Ignored unless elasticstack_full_stack + is set and only effective on Elastic Stack releases lower than 8. + + # ----- Pipelines ----- + logstash_manage_pipelines: + type: bool + default: true + description: Manage pipelines.yml. + + logstash_no_pipelines: + type: bool + default: false + description: Disable all pipeline management entirely. + + logstash_pipelines: + type: list + elements: dict + description: + - List of pipelines to configure. + - >- + Each entry needs a name. A pipeline either points to an external git + repository (source/version) or defines simple input/output keys that + connect to Redis. See the pipelines documentation for details. + default: + - name: default + exclusive: false + queue_type: memory + queue_max_bytes: 1gb + input: + - name: default + key: input + output: + - name: default + key: forwarder + options: + name: + type: str + required: true + description: Unique name of the pipeline. Becomes the pipeline directory and pipeline.id. + exclusive: + type: bool + description: Mark this pipeline as the only one allowed to handle its events. + source: + type: str + description: URL of a git repository holding the pipeline configuration. + version: + type: str + description: Git branch/tag/commit to check out from source (default "main"). + queue_type: + type: str + choices: + - memory + - persisted + description: Queue type for this pipeline (default "memory"). Use "persisted" for an on-disk persistent queue. + queue_max_bytes: + type: str + description: Maximum queue size for this pipeline (default "1gb"). + input: + type: list + elements: dict + description: Simple Redis inputs for this pipeline. + options: + name: + type: str + description: Name of the input. + key: + type: str + description: Redis key to read from. + output: + type: list + elements: dict + description: Simple Redis outputs for this pipeline. + options: + name: + type: str + description: Name of the output. + key: + type: str + description: Redis key to write to. + condition: + type: str + description: >- + Optional Logstash conditional. The output only receives events + matching it. With exclusive set, the conditions are chained with + else if. + congestion: + type: int + description: >- + Optional congestion threshold. The output stops once the target + Redis key holds more items than this value. + + logstash_elasticsearch_output: + type: bool + default: true + description: Create the default pipeline that forwards events to Elasticsearch. + + logstash_beats_input: + type: bool + default: true + description: Create the default pipeline with a Beats input. + + logstash_beats_input_congestion: + type: int + description: >- + Congestion threshold for the default Beats input pipeline: it stops + writing to Redis once the buffer key holds more items than this value. + Unset by default. + + logstash_beats_timeout: + type: str + description: Timeout for idle client connections on the Beats input (e.g. "60s"). Unset by default. + + logstash_beats_tls: + type: bool + description: >- + Activate TLS on the Beats input pipeline. Unset by default, but enabled + automatically in a full stack setup unless overridden. + + logstash_input_queue_type: + type: str + choices: + - memory + - persisted + default: memory + description: Queue type for the default Beats input pipeline. Use "persisted" for an on-disk persistent queue. + + logstash_input_queue_max_bytes: + type: str + default: 1gb + description: Maximum queue size for the default Beats input pipeline. + + logstash_forwarder_queue_type: + type: str + choices: + - memory + - persisted + default: memory + description: Queue type for the default Elasticsearch forwarder pipeline. Use "persisted" for an on-disk persistent queue. + + logstash_forwarder_queue_max_bytes: + type: str + default: 1gb + description: Maximum queue size for the default Elasticsearch forwarder pipeline. + + logstash_redis_password: + type: str + description: Password used when the simple inputs/outputs connect to Redis. Unset by default. + + # ----- Elasticsearch connection / output ----- + logstash_elasticsearch: + type: list + elements: str + description: >- + Elasticsearch hosts for the default output. Defaults to the nodes from + the elasticsearch group, or localhost when used standalone. + + logstash_validate_after_inactivity: + type: str + default: "300" + description: >- + Seconds Logstash waits before validating a previously idle connection to + Elasticsearch. + + logstash_sniffing: + type: bool + default: false + description: Enable sniffing for additional Elasticsearch nodes. + + logstash_sniffing_delay: + type: str + description: Seconds to wait between sniffing attempts. Unset by default. + + logstash_sniffing_path: + type: str + description: HTTP path used for the sniffing requests. Unset by default. + + # ----- Security: certificates ----- + logstash_security: + type: bool + description: >- + Enable X-Pack security (TLS) for Logstash. No default; enabled + automatically in full stack mode with the elastic variant. + + logstash_tls_key_passphrase: + type: str + default: LogstashChangeMe + description: Passphrase for the generated Logstash certificates. + + logstash_certs_dir: + type: str + default: /etc/logstash/certs + description: Directory holding the Logstash certificates. Used to build several file paths. + + logstash_cert_validity_period: + type: int + default: 1095 + description: Number of days the generated certificates are valid. + + logstash_cert_expiration_buffer: + type: int + default: 30 + description: >- + Renew the certificate when its remaining validity (in days) drops below + this value. + + logstash_cert_will_expire_soon: + type: bool + default: false + description: >- + Set to true to force renewal of the Logstash certificate. Alternatively + run the playbook with the renew_logstash_cert tag. + + # ----- Security: Elasticsearch role & user ----- + logstash_create_role: + type: bool + default: true + description: Create the Logstash writer role in Elasticsearch. + + logstash_role_name: + type: str + default: logstash_writer + description: Name of the Logstash writer role. + + logstash_role_cluster_privileges: + type: list + elements: str + default: + - manage_index_templates + - monitor + - manage_ilm + description: Cluster privileges granted to the Logstash writer role. + + logstash_role_indicies_names: + type: list + elements: str + default: + - "ecs-logstash*" + - "logstash*" + - "logs*" + description: Index patterns the Logstash writer role may access. + + logstash_role_indicies_privileges: + type: list + elements: str + default: + - write + - create + - delete + - create_index + - manage + - manage_ilm + description: Index privileges the Logstash writer role holds on its index patterns. + + logstash_create_user: + type: bool + default: true + description: Create the Logstash writer user in Elasticsearch. + + logstash_user_name: + type: str + default: logstash_writer + description: Name of the Logstash user connecting to Elasticsearch. + + logstash_user_password: + type: str + default: password + description: Password of the Logstash user. Must be at least 6 characters long. + + logstash_user_email: + type: str + default: "" + description: Email address linked to the Logstash user. + + logstash_user_fullname: + type: str + default: Internal Logstash User + description: Full name linked to the Logstash user. + + # ----- log4j2 logging (only effective when logstash_manage_logging is enabled) ----- + logstash_logging_console: + type: bool + default: true + description: Log to the console (syslog when run via systemd). Only effective when logstash_manage_logging is enabled. + + logstash_logging_file: + type: bool + default: true + description: Log to the log file. Only effective when logstash_manage_logging is enabled. + + logstash_logging_slow_console: + type: bool + default: true + description: Log the slowlog to the console (syslog when run via systemd). Only effective when logstash_manage_logging is enabled. + + logstash_logging_slow_file: + type: bool + default: true + description: Log the slowlog to the log file. Only effective when logstash_manage_logging is enabled. + + # ----- Identifying fields ----- + logstash_ident: + type: bool + default: true + description: Add a field identifying the node that processed an event. + + logstash_ident_field_name: + type: str + default: "[netways][instance]" + description: Name of the field that identifies the instance. + + logstash_pipeline_identifier: + type: bool + default: true + description: Add a field identifying which pipeline processed an event. + + logstash_pipeline_identifier_field_name: + type: str + default: "[netways][pipeline]" + description: Name of the pipeline identifier field. + + logstash_pipeline_identifier_defaults: + type: bool + default: false + description: >- + Also add pipeline identifiers in the default pipelines. This can let the + defaults dominate statistics with little value. + + # ----- Mermaid pipeline overview ----- + logstash_mermaid: + type: bool + default: true + description: Produce an overview of the Logstash pipelines in Mermaid syntax. + + logstash_mermaid_logstash: + type: bool + default: true + description: Place the Mermaid syntax into /etc/logstash/pipelines.mermaid on the Logstash hosts. + + logstash_mermaid_local: + type: bool + default: false + description: Place the Mermaid syntax into a temporary file on the control node. + + logstash_mermaid_extra: + type: str + description: Extra Mermaid syntax to append to the output (YAML multiline supported). Unset by default. + + # ----- Internal ----- + logstash_freshstart: + type: dict + default: + changed: false + description: Internal state used by the role to detect a fresh install. Do not set manually. diff --git a/roles/logstash/templates/log4j2.properties.j2 b/roles/logstash/templates/log4j2.properties.j2 index 78a025e9..10a908e0 100644 --- a/roles/logstash/templates/log4j2.properties.j2 +++ b/roles/logstash/templates/log4j2.properties.j2 @@ -1,5 +1,5 @@ # Managed by Ansible Role -# https://github.com/netways/ansible-role-logstash +# https://github.com/netways/ansible-collection-elasticstack # # Logging to logfile: {% if logstash_logging_file | bool %}true{% else %}false{% endif %} diff --git a/roles/logstash/templates/pipelines.mermaid.j2 b/roles/logstash/templates/pipelines.mermaid.j2 index adc30895..9832234d 100644 --- a/roles/logstash/templates/pipelines.mermaid.j2 +++ b/roles/logstash/templates/pipelines.mermaid.j2 @@ -1,5 +1,5 @@ # Managed via Ansible role -# https://github.com/netways/ansible-role-logstash +# https://github.com/netways/ansible-collection-elasticstack # Use the following code with your favorite Mermaid editor # Or paste into: https://mermaid.live/ diff --git a/roles/logstash/templates/pipelines.yml.j2 b/roles/logstash/templates/pipelines.yml.j2 index 901088e6..e4c00c10 100644 --- a/roles/logstash/templates/pipelines.yml.j2 +++ b/roles/logstash/templates/pipelines.yml.j2 @@ -1,7 +1,7 @@ --- # Managed via Ansible role -# https://github.com/netways/ansible-role-logstash +# https://github.com/netways/ansible-collection-elasticstack {% if logstash_beats_input_congestion is defined %} # global congestion threshold: {{ logstash_beats_input_congestion }} {% endif %} @@ -39,7 +39,7 @@ {% if logstash_pipelines is defined %} ### Autoconfigured pipelines ### -# See https://github.com/netways/ansible-role-logstash/blob/master/docs/pipelines.md +# See https://github.com/netways/ansible-collection-elasticstack/blob/main/roles/logstash/docs/pipelines.md # for details # Hint: They can be mixed up with manual code. When source or Redis is missing it's # likely that there is code from another source in place. diff --git a/roles/repos/README.md b/roles/repos/README.md new file mode 100644 index 00000000..fcbeb99f --- /dev/null +++ b/roles/repos/README.md @@ -0,0 +1,42 @@ +# Ansible Role: Repos + +Adds the Elastic package repositories to the target host so the other roles can +install Elastic Stack packages. It supports apt (Debian/Ubuntu), yum (RedHat) and +zypper (SUSE), for both the standard Elastic and the OSS variant. + +Its main use is together with the other roles of this collection, which expect +the Elastic repositories to be present. + +## Requirements + +* The `community.general` collection — only needed on SUSE hosts, where the role + uses the `zypper` modules. +* Network access from the target host to the Elastic repository URL. + +The role installs the required signing tools (`gpg`/`gnupg`) itself. + +## Example + +```yaml +- name: Configure Elastic repositories + hosts: all + collections: + - netways.elasticstack + roles: + - repos +``` + + +## Role variables + +No variables are defined for this role. + + + +## Shared variables + +This role is configured through the collection-wide `elasticstack_*` variables — +mainly `elasticstack_release`, `elasticstack_variant`, `elasticstack_enable_repos`, +`elasticstack_repo_url`, `elasticstack_repo_key` and `elasticstack_rpm_workaround`. +They are documented centrally with the +[elasticsearch role](../../docs/role-elasticsearch.md). diff --git a/roles/repos/meta/argument_specs.yml b/roles/repos/meta/argument_specs.yml new file mode 100644 index 00000000..e0b9a8f8 --- /dev/null +++ b/roles/repos/meta/argument_specs.yml @@ -0,0 +1,20 @@ +--- +# Role argument specification for the repos role. +# This is the single source of truth for the role's variables: Ansible validates +# the supplied values against it at role start, and ansible-docsmith renders the +# README variable reference from it. +# +# This role has no role-specific variables. It is configured entirely through the +# shared elasticstack_* variables, which are documented centrally with the +# elasticstack role. + +argument_specs: + main: + short_description: Configure the Elastic package repositories + description: + - >- + Adds the Elastic package repositories (apt, yum or zypper) to the target + host so the other roles can install Elastic Stack packages. + - Supports the standard Elastic and the OSS variant. + author: + - NETWAYS GmbH