From 38f8dff3342435723c35d7e06a41f32429094cf4 Mon Sep 17 00:00:00 2001 From: Christoph Breit Date: Mon, 27 May 2024 09:46:40 +0200 Subject: [PATCH 01/18] replace user and role creation with the corresponding modules and add new variables to docs --- docs/role-logstash.md | 2 + roles/logstash/defaults/main.yml | 2 + roles/logstash/tasks/logstash-security.yml | 107 ++++++++------------- 3 files changed, 43 insertions(+), 68 deletions(-) diff --git a/docs/role-logstash.md b/docs/role-logstash.md index 8e33259a..af8fe285 100644 --- a/docs/role-logstash.md +++ b/docs/role-logstash.md @@ -69,7 +69,9 @@ Aside from `logstash.yml` we can manage Logstashs pipelines. * *logstash_cert_will_expire_soon*: Set it to true to renew logstash certificate (default: `false`), Or run the playbook with `--tags renew_logstash_cert` to do that. * *logstash_elasticsearch*: Address of Elasticsearch instance for default output (default: list of Elasticsearch nodes from `elasticsearch` role or `localhost` when used standalone) * *logstash_security*: Enable X-Security (No default set, but will be activated when in full stack mode) +* *logstash_role_name*: Name of the logstash role that is getting created (Default: `logstash_writer`) * *logstash_user*: Name of the user to connect to Elasticsearch (Default: `logstash_writer`) +* *logstash_email*: email-address that is linked with the logstash_user (Default: `new@user.de`) * *logstash_password_hash*: Generate and use a hash from your `logstash_password` (default: `true`) * *logstash_password_hash_algorithm*: Password hashing algorithms. Value must be same as `xpack.security.authc.password_hashing.algorithm` (default: `bcrypt`) * *logstash_password_salt_length*: base64 encoded Salt character lenght. This value must be integer and must be compatible to the selected password hashing algorithms (default: `22`) diff --git a/roles/logstash/defaults/main.yml b/roles/logstash/defaults/main.yml index 2a2a7690..63a9a2c4 100644 --- a/roles/logstash/defaults/main.yml +++ b/roles/logstash/defaults/main.yml @@ -48,8 +48,10 @@ logstash_sniffing: false logstash_password_hash: true logstash_password_hash_algorithm: bcrypt logstash_password_salt_length: 22 +logstash_role_name: logstash_writer logstash_user: logstash_writer logstash_password: password +logstash_email: new@user.de logstash_password_hash_salt_length: 22 logstash_password_hash_salt_seed: SeedChangeMe logstash_user_indices: '"ecs-logstash*", "logstash*", "logs*"' diff --git a/roles/logstash/tasks/logstash-security.yml b/roles/logstash/tasks/logstash-security.yml index e83b6c07..ee73665e 100644 --- a/roles/logstash/tasks/logstash-security.yml +++ b/roles/logstash/tasks/logstash-security.yml @@ -378,71 +378,42 @@ security_api_base_url: "https://{{ hostvars[elasticstack_ca].ansible_default_ipv4.address }}:{{ elasticstack_elasticsearch_http_port }}/_xpack/security/" when: elasticstack_release | int < 8 -- name: Check for logstash_writer role - ansible.builtin.uri: - url: "{{ security_api_base_url }}role/logstash_writer" - ca_path: "{{ elasticstack_ca_dir }}/ca.crt" - user: elastic - password: "{{ logstash_elasticstack_password.stdout }}" - register: check_logstash_writer_role_response - delegate_to: "{{ elasticstack_ca }}" - failed_when: false - changed_when: false - no_log: "{{ elasticstack_no_log }}" - run_once: true - -- name: Set logstash_writer_role_present - ansible.builtin.set_fact: - logstash_writer_role_present: true - when: check_logstash_writer_role_response.json.logstash_writer is defined - -- name: Put logstash_writer role into Elasticsearch if not present - ansible.builtin.uri: - url: "{{ security_api_base_url }}role/logstash_writer" - ca_path: "{{ elasticstack_ca_dir }}/ca.crt" - user: elastic - password: "{{ logstash_elasticstack_password.stdout }}" - method: PUT - headers: - Content-Type: application/json - body: "{{ lookup('template', 'logstash_writer_role.j2') }}" - body_format: json - register: put_logstash_writer_role_response - when: logstash_writer_role_present is not defined - delegate_to: "{{ elasticstack_ca }}" - failed_when: not put_logstash_writer_role_response.json.role.created | bool - run_once: true - -- name: Check for logstash_writer user - ansible.builtin.uri: - url: "{{ security_api_base_url }}user/{{ logstash_user }}" - ca_path: "{{ elasticstack_ca_dir }}/ca.crt" - user: elastic - password: "{{ logstash_elasticstack_password.stdout }}" - register: check_logstash_writer_user_response - delegate_to: "{{ elasticstack_ca }}" - failed_when: false - changed_when: false - run_once: true - -- name: Set logstash_writer_user_present - ansible.builtin.set_fact: - logstash_writer_user_present: true - when: check_logstash_writer_user_response.json.logstash_writer.username is defined and check_logstash_writer_user_response.json.logstash_writer.username == "logstash_writer" - -- name: Put logstash_writer user into Elasticsearch if not present - ansible.builtin.uri: - url: "{{ security_api_base_url }}user/{{ logstash_user }}" - ca_path: "{{ elasticstack_ca_dir }}/ca.crt" - user: elastic - password: "{{ logstash_elasticstack_password.stdout }}" - method: PUT - headers: - Content-Type: application/json - body: "{{ lookup('template', 'logstash_writer_user.j2') }}" - body_format: json - register: put_logstash_writer_user_response - when: logstash_writer_user_present is not defined - delegate_to: "{{ elasticstack_ca }}" - run_once: true - failed_when: not put_logstash_writer_user_response.json.created +- name: Create role + netways.elasticstack.elasticsearch_role: + name: "{{ logstash_role_name }}" + cluster: + - manage_index_templates + - monitor + - manage_ilm + indicies: + - names: + - logstash_user_indices + privileges: + - write + - create + - delete + - create_index + - manage + - manage_ilm + state: present + host: https://localhost:9200 + auth_user: elastic + auth_pass: "{{ logstash_elasticstack_password.stdout }}" + verify_certs: false + ca_certs: "{{ elasticstack_ca_dir }}/ca.crt" + + +- name: Create user + netways.elasticstack.elasticsearch_user: + name: "{{ logstash_user }}" + fullname: Internal Logstash User + password: "{{ logstash_password }}" + email: "{{ logstash_email }}" + roles: + - "{{ logstash_role_name }}" + enabled: true + state: present + host: https://localhost:9200 + auth_user: elastic + auth_pass: "{{ logstash_elasticstack_password.stdout }}" + verify_certs: false From e42504d046849a52699c654d481ff6cbc33a6d6d Mon Sep 17 00:00:00 2001 From: Christoph Breit Date: Mon, 27 May 2024 09:56:14 +0200 Subject: [PATCH 02/18] fix linting --- roles/logstash/tasks/logstash-security.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/roles/logstash/tasks/logstash-security.yml b/roles/logstash/tasks/logstash-security.yml index ee73665e..cccf336e 100644 --- a/roles/logstash/tasks/logstash-security.yml +++ b/roles/logstash/tasks/logstash-security.yml @@ -381,7 +381,7 @@ - name: Create role netways.elasticstack.elasticsearch_role: name: "{{ logstash_role_name }}" - cluster: + cluster: - manage_index_templates - monitor - manage_ilm @@ -402,7 +402,6 @@ verify_certs: false ca_certs: "{{ elasticstack_ca_dir }}/ca.crt" - - name: Create user netways.elasticstack.elasticsearch_user: name: "{{ logstash_user }}" From 2b6263b16b949a4006131161f973f173d4467107 Mon Sep 17 00:00:00 2001 From: Christoph Breit Date: Tue, 28 May 2024 14:23:38 +0200 Subject: [PATCH 03/18] add installation process for python module elasticsearch --- roles/logstash/defaults/main.yml | 2 +- roles/logstash/tasks/main.yml | 11 +++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/roles/logstash/defaults/main.yml b/roles/logstash/defaults/main.yml index 63a9a2c4..cc3688a1 100644 --- a/roles/logstash/defaults/main.yml +++ b/roles/logstash/defaults/main.yml @@ -51,7 +51,7 @@ logstash_password_salt_length: 22 logstash_role_name: logstash_writer logstash_user: logstash_writer logstash_password: password -logstash_email: new@user.de +logstash_email: "" logstash_password_hash_salt_length: 22 logstash_password_hash_salt_seed: SeedChangeMe logstash_user_indices: '"ecs-logstash*", "logstash*", "logs*"' diff --git a/roles/logstash/tasks/main.yml b/roles/logstash/tasks/main.yml index 00e4a70a..9de7f998 100644 --- a/roles/logstash/tasks/main.yml +++ b/roles/logstash/tasks/main.yml @@ -11,6 +11,17 @@ changed_when: false when: ansible_os_family == 'Debian' + +- name: Install pip Package + ansible.builtin.package: + name: "pip" + +- name: Install Elasticsearch Pyhon Module + ansible.builtin.pip: + name: + - elasticsearch + + - name: Prepare for whole stack roles if used when: - elasticstack_full_stack | bool From 2a59b15040c0ef73f6e3931a96cfde9b288ed43c Mon Sep 17 00:00:00 2001 From: Christoph Breit Date: Tue, 28 May 2024 14:26:08 +0200 Subject: [PATCH 04/18] fix linting --- roles/logstash/tasks/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/logstash/tasks/main.yml b/roles/logstash/tasks/main.yml index 9de7f998..68262803 100644 --- a/roles/logstash/tasks/main.yml +++ b/roles/logstash/tasks/main.yml @@ -18,7 +18,7 @@ - name: Install Elasticsearch Pyhon Module ansible.builtin.pip: - name: + name: - elasticsearch From 14996507eac4e7a5fdac115a7e077be8cabe7e15 Mon Sep 17 00:00:00 2001 From: Christoph Breit Date: Wed, 29 May 2024 12:02:16 +0200 Subject: [PATCH 05/18] make user and role creation more variable --- docs/role-logstash.md | 9 ++++--- roles/logstash/defaults/main.yml | 21 ++++++++++++--- roles/logstash/tasks/logstash-security.yml | 26 ++++++------------- roles/logstash/tasks/main.yml | 7 +++-- .../templates/logstash_writer_role.j2 | 9 ------- .../templates/logstash_writer_user.j2 | 12 --------- 6 files changed, 35 insertions(+), 49 deletions(-) delete mode 100644 roles/logstash/templates/logstash_writer_role.j2 delete mode 100644 roles/logstash/templates/logstash_writer_user.j2 diff --git a/docs/role-logstash.md b/docs/role-logstash.md index af8fe285..022d9741 100644 --- a/docs/role-logstash.md +++ b/docs/role-logstash.md @@ -71,13 +71,16 @@ Aside from `logstash.yml` we can manage Logstashs pipelines. * *logstash_security*: Enable X-Security (No default set, but will be activated when in full stack mode) * *logstash_role_name*: Name of the logstash role that is getting created (Default: `logstash_writer`) * *logstash_user*: Name of the user to connect to Elasticsearch (Default: `logstash_writer`) -* *logstash_email*: email-address that is linked with the logstash_user (Default: `new@user.de`) +* *logstash_user_email*: email-address that is linked with the logstash_user (Default: `""`) +* *logstash_user_fullname*: fullname that is linked with the logstash_user (Default: `Internal Logstash User`) * *logstash_password_hash*: Generate and use a hash from your `logstash_password` (default: `true`) * *logstash_password_hash_algorithm*: Password hashing algorithms. Value must be same as `xpack.security.authc.password_hashing.algorithm` (default: `bcrypt`) * *logstash_password_salt_length*: base64 encoded Salt character lenght. This value must be integer and must be compatible to the selected password hashing algorithms (default: `22`) * *logstash_password_hash_salt_seed*: A seed to generate random but idempotent salt on the elasticstack ca host. The salt will be used to create idempotent logstash hashed user password (default: `SeedChangeMe`) -* *logstash_password*: Password of Elasticsearch user. It must be at least 6 characters long (default: `password`) -* *logstash_user_indices*: Indices the user has access to (default: `'"ecs-logstash*", "logstash*", "logs*"'`) +* *logstash_user_password*: Password of Elasticsearch user. It must be at least 6 characters long (default: `password`) +* *logstash_role_cluster_privileges*: Cluster privileges the role has access to (default: `"manage_index_templates", "monitor", "manage_ilm"`) +* *logstash_role_indicies_names*: Indices the role has access to (default: `"ecs-logstash*", "logstash*", "logs*"`) +* *logstash_role_indicies_privileges*: Indices the role has access to (default: `"write", "create", "delete", "create_index", "manage", "manage_ilm"`) * *logstash_reset_writer_role*: Reset user and role with every run: (default: `true`) * *logstash_validate_after_inactivity*: How long should logstash wait, before starting a new connection and leave the old one with elasticsearch, when the connection with elasticsearch get lost: (Default: `300`). * *logstash_queue_type*: What kind of queue should Logstash use per default: (Default: `persisted`, alternative: `memory`) diff --git a/roles/logstash/defaults/main.yml b/roles/logstash/defaults/main.yml index cc3688a1..d747bbe8 100644 --- a/roles/logstash/defaults/main.yml +++ b/roles/logstash/defaults/main.yml @@ -45,16 +45,31 @@ logstash_forwarder_queue_max_bytes: 1gb logstash_sniffing: false # logstash security +logstasth_role_cluster: + - manage_index_templates + - monitor + - manage_ilm +logstash_role_indicies_names: + - "ecs-logstash*" + - "logstash*" + - "logs*" +logstash_role_indicies_privileges: + - write + - create + - delete + - create_index + - manage + - manage_ilm logstash_password_hash: true logstash_password_hash_algorithm: bcrypt logstash_password_salt_length: 22 logstash_role_name: logstash_writer logstash_user: logstash_writer -logstash_password: password -logstash_email: "" +logstash_user_password: password +logstash_user_email: "" +logstash_user_fullname: "Internal Logstash User" logstash_password_hash_salt_length: 22 logstash_password_hash_salt_seed: SeedChangeMe -logstash_user_indices: '"ecs-logstash*", "logstash*", "logs*"' logstash_reset_writer_role: true logstash_tls_key_passphrase: LogstashChangeMe diff --git a/roles/logstash/tasks/logstash-security.yml b/roles/logstash/tasks/logstash-security.yml index cccf336e..49307fb3 100644 --- a/roles/logstash/tasks/logstash-security.yml +++ b/roles/logstash/tasks/logstash-security.yml @@ -348,7 +348,7 @@ - name: Check the length of logstash user password ansible.builtin.fail: msg: logstash user password must be at least 6 characters long. - when: logstash_password | length < 6 + when: logstash_user_password | length < 6 - name: Set password hash salt as a fact ansible.builtin.set_fact: @@ -378,23 +378,13 @@ security_api_base_url: "https://{{ hostvars[elasticstack_ca].ansible_default_ipv4.address }}:{{ elasticstack_elasticsearch_http_port }}/_xpack/security/" when: elasticstack_release | int < 8 -- name: Create role +- name: Create logstash role {{ logstash_role_name }} netways.elasticstack.elasticsearch_role: name: "{{ logstash_role_name }}" - cluster: - - manage_index_templates - - monitor - - manage_ilm + cluster: "{{ logstasth_role_cluster }}" indicies: - - names: - - logstash_user_indices - privileges: - - write - - create - - delete - - create_index - - manage - - manage_ilm + - names: "{{ logstash_role_indicies_names }}" + privileges: "{{ logstash_role_indicies_privileges }}" state: present host: https://localhost:9200 auth_user: elastic @@ -402,12 +392,12 @@ verify_certs: false ca_certs: "{{ elasticstack_ca_dir }}/ca.crt" -- name: Create user +- name: Create logstash user {{ logstash_user }} netways.elasticstack.elasticsearch_user: name: "{{ logstash_user }}" - fullname: Internal Logstash User + fullname: "{{ logstash_user_fullname }}" password: "{{ logstash_password }}" - email: "{{ logstash_email }}" + email: "{{ logstash_user_email }}" roles: - "{{ logstash_role_name }}" enabled: true diff --git a/roles/logstash/tasks/main.yml b/roles/logstash/tasks/main.yml index 68262803..d34c5e8b 100644 --- a/roles/logstash/tasks/main.yml +++ b/roles/logstash/tasks/main.yml @@ -11,17 +11,16 @@ changed_when: false when: ansible_os_family == 'Debian' - -- name: Install pip Package +- name: Install Python Packages ansible.builtin.package: - name: "pip" + name: ["python3", "python3-pip"] + state: present - name: Install Elasticsearch Pyhon Module ansible.builtin.pip: name: - elasticsearch - - name: Prepare for whole stack roles if used when: - elasticstack_full_stack | bool diff --git a/roles/logstash/templates/logstash_writer_role.j2 b/roles/logstash/templates/logstash_writer_role.j2 deleted file mode 100644 index fc990cbe..00000000 --- a/roles/logstash/templates/logstash_writer_role.j2 +++ /dev/null @@ -1,9 +0,0 @@ -{ - "cluster": ["manage_index_templates", "monitor", "manage_ilm"], - "indices": [ - { - "names": [ {{ logstash_user_indices }} ], - "privileges": ["write","create","delete","create_index","manage","manage_ilm"] - } - ] -} diff --git a/roles/logstash/templates/logstash_writer_user.j2 b/roles/logstash/templates/logstash_writer_user.j2 deleted file mode 100644 index 5c21a745..00000000 --- a/roles/logstash/templates/logstash_writer_user.j2 +++ /dev/null @@ -1,12 +0,0 @@ -{ -{% if logstash_password_hash | bool %} -{# using a fixed salt is neccessary for idempotency, will be generated as a set fact. -rounds specifies the bcrypt version. The default version in Ansible module is 12. The acceptable one is 10 on elasticsearch 7. -On elasticsearch 8, the 12 and 10 versions will work, so we should use 10 until the support of 7 stops #} - "password_hash" : "{{ logstash_password | password_hash( hashtype=logstash_password_hash_algorithm, salt=logstash_password_hash_salt, ident='2a', rounds=10 ) }}", -{% else %} - "password" : "{{ logstash_password }}", -{% endif %} - "roles" : [ "logstash_writer"], - "full_name" : "Internal Logstash User" -} From 136232fe6d18b35de045c27c434069409b58c881 Mon Sep 17 00:00:00 2001 From: Christoph Breit Date: Wed, 29 May 2024 12:06:47 +0200 Subject: [PATCH 06/18] fix linting --- roles/logstash/defaults/main.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/roles/logstash/defaults/main.yml b/roles/logstash/defaults/main.yml index d747bbe8..60db55a7 100644 --- a/roles/logstash/defaults/main.yml +++ b/roles/logstash/defaults/main.yml @@ -45,13 +45,13 @@ logstash_forwarder_queue_max_bytes: 1gb logstash_sniffing: false # logstash security -logstasth_role_cluster: +logstasth_role_cluster_privileges: - manage_index_templates - monitor - manage_ilm logstash_role_indicies_names: - - "ecs-logstash*" - - "logstash*" + - "ecs-logstash*" + - "logstash*" - "logs*" logstash_role_indicies_privileges: - write From 0abc0d9c88ded95d7f3b912c95d71b7c4d479a7b Mon Sep 17 00:00:00 2001 From: Christoph Breit Date: Wed, 29 May 2024 12:18:02 +0200 Subject: [PATCH 07/18] fix typos --- roles/logstash/tasks/logstash-security.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/logstash/tasks/logstash-security.yml b/roles/logstash/tasks/logstash-security.yml index 49307fb3..df3481bb 100644 --- a/roles/logstash/tasks/logstash-security.yml +++ b/roles/logstash/tasks/logstash-security.yml @@ -381,7 +381,7 @@ - name: Create logstash role {{ logstash_role_name }} netways.elasticstack.elasticsearch_role: name: "{{ logstash_role_name }}" - cluster: "{{ logstasth_role_cluster }}" + cluster: "{{ logstash_role_cluster_privileges }}" indicies: - names: "{{ logstash_role_indicies_names }}" privileges: "{{ logstash_role_indicies_privileges }}" From 87256bdbecb6f4ef75bd0ddd3d80be6c3ab2d296 Mon Sep 17 00:00:00 2001 From: Christoph Breit Date: Wed, 29 May 2024 12:18:40 +0200 Subject: [PATCH 08/18] fix typos --- roles/logstash/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/logstash/defaults/main.yml b/roles/logstash/defaults/main.yml index 60db55a7..c3fc9c57 100644 --- a/roles/logstash/defaults/main.yml +++ b/roles/logstash/defaults/main.yml @@ -45,7 +45,7 @@ logstash_forwarder_queue_max_bytes: 1gb logstash_sniffing: false # logstash security -logstasth_role_cluster_privileges: +logstash_role_cluster_privileges: - manage_index_templates - monitor - manage_ilm From 84378a17ecf6fdc8a6164658dfc5e24a63fb5b60 Mon Sep 17 00:00:00 2001 From: Christoph Breit Date: Wed, 29 May 2024 12:25:45 +0200 Subject: [PATCH 09/18] fix typo --- roles/logstash/tasks/logstash-security.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/logstash/tasks/logstash-security.yml b/roles/logstash/tasks/logstash-security.yml index df3481bb..d5be98c0 100644 --- a/roles/logstash/tasks/logstash-security.yml +++ b/roles/logstash/tasks/logstash-security.yml @@ -396,7 +396,7 @@ netways.elasticstack.elasticsearch_user: name: "{{ logstash_user }}" fullname: "{{ logstash_user_fullname }}" - password: "{{ logstash_password }}" + password: "{{ logstash_user_password }}" email: "{{ logstash_user_email }}" roles: - "{{ logstash_role_name }}" From de7a8251b66de3a2c9146e6dad328ec4c304886f Mon Sep 17 00:00:00 2001 From: Christoph Breit Date: Wed, 29 May 2024 12:37:10 +0200 Subject: [PATCH 10/18] fix typos --- roles/logstash/templates/elasticsearch-output.conf.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/logstash/templates/elasticsearch-output.conf.j2 b/roles/logstash/templates/elasticsearch-output.conf.j2 index 1f93292c..7c9884e6 100644 --- a/roles/logstash/templates/elasticsearch-output.conf.j2 +++ b/roles/logstash/templates/elasticsearch-output.conf.j2 @@ -37,7 +37,7 @@ output { cacert => "{{ logstash_certs_dir }}/ca.crt" ssl => true user => "{{ logstash_user }}" - password => "{{ logstash_password }}" + password => "{{ logstash_user_password }}" {% endif %} } } From 8f64c53df23addfd1897ce3205c4b2fb422d0fb4 Mon Sep 17 00:00:00 2001 From: Christoph Breit Date: Wed, 29 May 2024 14:50:34 +0200 Subject: [PATCH 11/18] make host variable and remove logstash_password_hash variables --- docs/role-logstash.md | 4 ---- roles/logstash/defaults/main.yml | 5 ----- roles/logstash/tasks/logstash-security.yml | 19 ++----------------- 3 files changed, 2 insertions(+), 26 deletions(-) diff --git a/docs/role-logstash.md b/docs/role-logstash.md index 022d9741..1f2700e4 100644 --- a/docs/role-logstash.md +++ b/docs/role-logstash.md @@ -73,10 +73,6 @@ Aside from `logstash.yml` we can manage Logstashs pipelines. * *logstash_user*: Name of the user to connect to Elasticsearch (Default: `logstash_writer`) * *logstash_user_email*: email-address that is linked with the logstash_user (Default: `""`) * *logstash_user_fullname*: fullname that is linked with the logstash_user (Default: `Internal Logstash User`) -* *logstash_password_hash*: Generate and use a hash from your `logstash_password` (default: `true`) -* *logstash_password_hash_algorithm*: Password hashing algorithms. Value must be same as `xpack.security.authc.password_hashing.algorithm` (default: `bcrypt`) -* *logstash_password_salt_length*: base64 encoded Salt character lenght. This value must be integer and must be compatible to the selected password hashing algorithms (default: `22`) -* *logstash_password_hash_salt_seed*: A seed to generate random but idempotent salt on the elasticstack ca host. The salt will be used to create idempotent logstash hashed user password (default: `SeedChangeMe`) * *logstash_user_password*: Password of Elasticsearch user. It must be at least 6 characters long (default: `password`) * *logstash_role_cluster_privileges*: Cluster privileges the role has access to (default: `"manage_index_templates", "monitor", "manage_ilm"`) * *logstash_role_indicies_names*: Indices the role has access to (default: `"ecs-logstash*", "logstash*", "logs*"`) diff --git a/roles/logstash/defaults/main.yml b/roles/logstash/defaults/main.yml index c3fc9c57..ecab500e 100644 --- a/roles/logstash/defaults/main.yml +++ b/roles/logstash/defaults/main.yml @@ -60,16 +60,11 @@ logstash_role_indicies_privileges: - create_index - manage - manage_ilm -logstash_password_hash: true -logstash_password_hash_algorithm: bcrypt -logstash_password_salt_length: 22 logstash_role_name: logstash_writer logstash_user: logstash_writer logstash_user_password: password logstash_user_email: "" logstash_user_fullname: "Internal Logstash User" -logstash_password_hash_salt_length: 22 -logstash_password_hash_salt_seed: SeedChangeMe logstash_reset_writer_role: true logstash_tls_key_passphrase: LogstashChangeMe diff --git a/roles/logstash/tasks/logstash-security.yml b/roles/logstash/tasks/logstash-security.yml index d5be98c0..de91dc07 100644 --- a/roles/logstash/tasks/logstash-security.yml +++ b/roles/logstash/tasks/logstash-security.yml @@ -350,11 +350,6 @@ msg: logstash user password must be at least 6 characters long. when: logstash_user_password | length < 6 -- name: Set password hash salt as a fact - ansible.builtin.set_fact: - logstash_password_hash_salt: "{{ lookup('password', '/dev/null', chars=['ascii_lowercase', 'digits'], length=logstash_password_hash_salt_length, seed=logstash_password_hash_salt_seed) }}" - when: logstash_password_hash | bool and inventory_hostname == elasticstack_ca - - name: Fetch Elastic password # noqa: risky-shell-pipe ansible.builtin.shell: > if test -n "$(ps -p $$ | grep bash)"; then set -o pipefail; fi; @@ -368,16 +363,6 @@ - configuration - logstash_configuration -- name: Set elasticsearch security-api base url for elasticsearch > 7 - ansible.builtin.set_fact: - security_api_base_url: "https://{{ hostvars[elasticstack_ca].ansible_default_ipv4.address }}:{{ elasticstack_elasticsearch_http_port }}/_security/" - when: elasticstack_release | int > 7 - -- name: Set elasticsearch security-api base url for elasticsearch < 8 - ansible.builtin.set_fact: - security_api_base_url: "https://{{ hostvars[elasticstack_ca].ansible_default_ipv4.address }}:{{ elasticstack_elasticsearch_http_port }}/_xpack/security/" - when: elasticstack_release | int < 8 - - name: Create logstash role {{ logstash_role_name }} netways.elasticstack.elasticsearch_role: name: "{{ logstash_role_name }}" @@ -386,7 +371,7 @@ - names: "{{ logstash_role_indicies_names }}" privileges: "{{ logstash_role_indicies_privileges }}" state: present - host: https://localhost:9200 + host: "https://{{ hostvars[elasticstack_ca].ansible_default_ipv4.address }}:{{ elasticstack_elasticsearch_http_port }}" auth_user: elastic auth_pass: "{{ logstash_elasticstack_password.stdout }}" verify_certs: false @@ -402,7 +387,7 @@ - "{{ logstash_role_name }}" enabled: true state: present - host: https://localhost:9200 + host: "https://{{ hostvars[elasticstack_ca].ansible_default_ipv4.address }}:{{ elasticstack_elasticsearch_http_port }}" auth_user: elastic auth_pass: "{{ logstash_elasticstack_password.stdout }}" verify_certs: false From 2145d648c67af7660c9d6c0208acd7012d240bf1 Mon Sep 17 00:00:00 2001 From: Christoph Breit Date: Wed, 29 May 2024 15:30:09 +0200 Subject: [PATCH 12/18] raise ansible verbosity for debug --- molecule/elasticstack_default/molecule.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/molecule/elasticstack_default/molecule.yml b/molecule/elasticstack_default/molecule.yml index 122ee248..51536d2b 100644 --- a/molecule/elasticstack_default/molecule.yml +++ b/molecule/elasticstack_default/molecule.yml @@ -32,5 +32,7 @@ platforms: pre_build_image: true provisioner: name: ansible + env: + ANSIBLE_VERBOSITY: 3 verifier: name: ansible From 81cccf3ee6fad7fa79d8d6baa100fe357a2877e9 Mon Sep 17 00:00:00 2001 From: Christoph Breit Date: Wed, 29 May 2024 15:44:27 +0200 Subject: [PATCH 13/18] fix pipeline --- roles/logstash/tasks/logstash-security.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/logstash/tasks/logstash-security.yml b/roles/logstash/tasks/logstash-security.yml index de91dc07..d339bf6d 100644 --- a/roles/logstash/tasks/logstash-security.yml +++ b/roles/logstash/tasks/logstash-security.yml @@ -375,7 +375,7 @@ auth_user: elastic auth_pass: "{{ logstash_elasticstack_password.stdout }}" verify_certs: false - ca_certs: "{{ elasticstack_ca_dir }}/ca.crt" + ca_certs: "{{ logstash_certs_dir }}/ca.crt" - name: Create logstash user {{ logstash_user }} netways.elasticstack.elasticsearch_user: From 0a15b9f78d6994c1c007348249dbc7050dfe7a5b Mon Sep 17 00:00:00 2001 From: Tobias Bauriedel Date: Fri, 31 May 2024 11:41:17 +0200 Subject: [PATCH 14/18] add ca_certs for user creation --- roles/logstash/tasks/logstash-security.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/roles/logstash/tasks/logstash-security.yml b/roles/logstash/tasks/logstash-security.yml index d339bf6d..7acb3864 100644 --- a/roles/logstash/tasks/logstash-security.yml +++ b/roles/logstash/tasks/logstash-security.yml @@ -391,3 +391,4 @@ auth_user: elastic auth_pass: "{{ logstash_elasticstack_password.stdout }}" verify_certs: false + ca_certs: "{{ logstash_certs_dir }}/ca.crt" From 711e498b881ea22fd16e321f2d9747dce1c81785 Mon Sep 17 00:00:00 2001 From: Tobias Bauriedel Date: Fri, 31 May 2024 12:25:23 +0200 Subject: [PATCH 15/18] update README --- docs/role-logstash.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/role-logstash.md b/docs/role-logstash.md index 1f2700e4..3d220f4a 100644 --- a/docs/role-logstash.md +++ b/docs/role-logstash.md @@ -69,11 +69,11 @@ Aside from `logstash.yml` we can manage Logstashs pipelines. * *logstash_cert_will_expire_soon*: Set it to true to renew logstash certificate (default: `false`), Or run the playbook with `--tags renew_logstash_cert` to do that. * *logstash_elasticsearch*: Address of Elasticsearch instance for default output (default: list of Elasticsearch nodes from `elasticsearch` role or `localhost` when used standalone) * *logstash_security*: Enable X-Security (No default set, but will be activated when in full stack mode) -* *logstash_role_name*: Name of the logstash role that is getting created (Default: `logstash_writer`) * *logstash_user*: Name of the user to connect to Elasticsearch (Default: `logstash_writer`) * *logstash_user_email*: email-address that is linked with the logstash_user (Default: `""`) * *logstash_user_fullname*: fullname that is linked with the logstash_user (Default: `Internal Logstash User`) * *logstash_user_password*: Password of Elasticsearch user. It must be at least 6 characters long (default: `password`) +* *logstash_role_name*: Name of the logstash role that is getting created (Default: `logstash_writer`) * *logstash_role_cluster_privileges*: Cluster privileges the role has access to (default: `"manage_index_templates", "monitor", "manage_ilm"`) * *logstash_role_indicies_names*: Indices the role has access to (default: `"ecs-logstash*", "logstash*", "logs*"`) * *logstash_role_indicies_privileges*: Indices the role has access to (default: `"write", "create", "delete", "create_index", "manage", "manage_ilm"`) From 70ff3595a449dae59353d403d26e15c2c799f85c Mon Sep 17 00:00:00 2001 From: Tobias Bauriedel Date: Wed, 5 Jun 2024 15:03:14 +0200 Subject: [PATCH 16/18] Update roles/logstash/tasks/main.yml Co-authored-by: Thomas Widhalm --- roles/logstash/tasks/main.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/roles/logstash/tasks/main.yml b/roles/logstash/tasks/main.yml index d34c5e8b..9159ba5c 100644 --- a/roles/logstash/tasks/main.yml +++ b/roles/logstash/tasks/main.yml @@ -13,7 +13,9 @@ - name: Install Python Packages ansible.builtin.package: - name: ["python3", "python3-pip"] + name: + - python3 + - python3-pip state: present - name: Install Elasticsearch Pyhon Module From 87ac835fb42ccc59292260a2816548d2f3cc7938 Mon Sep 17 00:00:00 2001 From: Tobias Bauriedel Date: Wed, 5 Jun 2024 15:03:28 +0200 Subject: [PATCH 17/18] Update roles/logstash/tasks/logstash-security.yml Co-authored-by: Thomas Widhalm --- roles/logstash/tasks/logstash-security.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/logstash/tasks/logstash-security.yml b/roles/logstash/tasks/logstash-security.yml index 7acb3864..605c50aa 100644 --- a/roles/logstash/tasks/logstash-security.yml +++ b/roles/logstash/tasks/logstash-security.yml @@ -374,7 +374,7 @@ host: "https://{{ hostvars[elasticstack_ca].ansible_default_ipv4.address }}:{{ elasticstack_elasticsearch_http_port }}" auth_user: elastic auth_pass: "{{ logstash_elasticstack_password.stdout }}" - verify_certs: false + verify_certs: true ca_certs: "{{ logstash_certs_dir }}/ca.crt" - name: Create logstash user {{ logstash_user }} From 52796c9c8e501e94df1d2b142ccf787d4f646465 Mon Sep 17 00:00:00 2001 From: Tobias Bauriedel Date: Wed, 5 Jun 2024 15:17:48 +0200 Subject: [PATCH 18/18] Move installation of dependencies --- docs/role-logstash.md | 4 ++-- roles/elasticstack/tasks/packages.yml | 12 +++++++++++- roles/logstash/tasks/main.yml | 12 ------------ 3 files changed, 13 insertions(+), 15 deletions(-) diff --git a/docs/role-logstash.md b/docs/role-logstash.md index 3d220f4a..55187128 100644 --- a/docs/role-logstash.md +++ b/docs/role-logstash.md @@ -72,11 +72,11 @@ Aside from `logstash.yml` we can manage Logstashs pipelines. * *logstash_user*: Name of the user to connect to Elasticsearch (Default: `logstash_writer`) * *logstash_user_email*: email-address that is linked with the logstash_user (Default: `""`) * *logstash_user_fullname*: fullname that is linked with the logstash_user (Default: `Internal Logstash User`) -* *logstash_user_password*: Password of Elasticsearch user. It must be at least 6 characters long (default: `password`) +* *logstash_user_password*: Password of `logstash_user` in Elasticsearch. It must be at least 6 characters long (default: `password`) * *logstash_role_name*: Name of the logstash role that is getting created (Default: `logstash_writer`) * *logstash_role_cluster_privileges*: Cluster privileges the role has access to (default: `"manage_index_templates", "monitor", "manage_ilm"`) * *logstash_role_indicies_names*: Indices the role has access to (default: `"ecs-logstash*", "logstash*", "logs*"`) -* *logstash_role_indicies_privileges*: Indices the role has access to (default: `"write", "create", "delete", "create_index", "manage", "manage_ilm"`) +* *logstash_role_indicies_privileges*: Index permissions the role has on `logstash_role_indicies_names` (default: `"write", "create", "delete", "create_index", "manage", "manage_ilm"`) * *logstash_reset_writer_role*: Reset user and role with every run: (default: `true`) * *logstash_validate_after_inactivity*: How long should logstash wait, before starting a new connection and leave the old one with elasticsearch, when the connection with elasticsearch get lost: (Default: `300`). * *logstash_queue_type*: What kind of queue should Logstash use per default: (Default: `persisted`, alternative: `memory`) diff --git a/roles/elasticstack/tasks/packages.yml b/roles/elasticstack/tasks/packages.yml index 36a2f3f7..0b5b59c1 100644 --- a/roles/elasticstack/tasks/packages.yml +++ b/roles/elasticstack/tasks/packages.yml @@ -1,5 +1,4 @@ --- - - name: Update apt cache. ansible.builtin.apt: update_cache: yes @@ -20,3 +19,14 @@ - renew_beats_cert - renew_es_cert - renew_logstash_cert + +- name: Install packages for module dependencies + ansible.builtin.package: + name: + - python3 + - python3-pip + +- name: Install Elasticsearch Python Module + ansible.builtin.pip: + name: + - elasticsearch diff --git a/roles/logstash/tasks/main.yml b/roles/logstash/tasks/main.yml index 9159ba5c..00e4a70a 100644 --- a/roles/logstash/tasks/main.yml +++ b/roles/logstash/tasks/main.yml @@ -11,18 +11,6 @@ changed_when: false when: ansible_os_family == 'Debian' -- name: Install Python Packages - ansible.builtin.package: - name: - - python3 - - python3-pip - state: present - -- name: Install Elasticsearch Pyhon Module - ansible.builtin.pip: - name: - - elasticsearch - - name: Prepare for whole stack roles if used when: - elasticstack_full_stack | bool