From febe6b1be66d3fe97df3b42b375cf98b4469475e Mon Sep 17 00:00:00 2001 From: Isaac Prior Date: Thu, 16 Sep 2021 14:09:22 +0100 Subject: [PATCH] Manage Cloudkitty Elasticsearch index with template Improves support for elasticsearch as the cloudkitty storage backend. Enforces using an elasticsearch index alias with cloudkitty. Provides an elasticsearch index component template to manage replicas. Adds elasicsearch curator config to prune cloudkitty indices. Change-Id: I9208bdf9c1dfc541c1c1647cac6ad577d88b405c --- ansible/group_vars/all.yml | 7 +- ansible/roles/cloudkitty/defaults/main.yml | 17 +++- ansible/roles/cloudkitty/tasks/bootstrap.yml | 80 ++++++++++++++++--- .../cloudkitty/templates/cloudkitty.conf.j2 | 3 +- ...y_elasticsearch_component_template.json.j2 | 9 +++ .../cloudkitty_elasticsearch_reindex.json.j2 | 8 ++ ansible/roles/elasticsearch/defaults/main.yml | 26 +++++- .../elasticsearch-curator-actions.yml.j2 | 10 +++ ...search-alias-support-e6cc3fe1360aa4ae.yaml | 13 +++ 9 files changed, 158 insertions(+), 15 deletions(-) create mode 100644 ansible/roles/cloudkitty/templates/cloudkitty_elasticsearch_component_template.json.j2 create mode 100644 ansible/roles/cloudkitty/templates/cloudkitty_elasticsearch_reindex.json.j2 create mode 100644 releasenotes/notes/add-cloudkitty-elasticsearch-alias-support-e6cc3fe1360aa4ae.yaml diff --git a/ansible/group_vars/all.yml b/ansible/group_vars/all.yml index 0b1ca2dfe6..7cefd11082 100644 --- a/ansible/group_vars/all.yml +++ b/ansible/group_vars/all.yml @@ -953,13 +953,16 @@ cinder_backup_mount_options_nfs: "" ####################### # Valid option is gnocchi cloudkitty_collector_backend: "gnocchi" -# Valid options are 'sqlalchemy' or 'influxdb'. The default value is +# Valid options are 'sqlalchemy', 'elasticsearch' or 'influxdb'. The default value is # 'influxdb', which matches the default in Cloudkitty since the Stein release. # When the backend is "influxdb", we also enable Influxdb. -# Also, when using 'influxdb' as the backend, we trigger the configuration/use +# Also, when using 'influxdb' or 'elasticsearch' as the backend, we trigger the configuration/use # of Cloudkitty storage backend version 2. cloudkitty_storage_backend: "influxdb" +# Elasticsearch index alias name when cloudkitty_storage_backend is 'elasticsearch'. +cloudkitty_elasticsearch_alias_name: "cloudkitty" + ####################### # Designate options ####################### diff --git a/ansible/roles/cloudkitty/defaults/main.yml b/ansible/roles/cloudkitty/defaults/main.yml index 14ac9145c8..b1548b9023 100644 --- a/ansible/roles/cloudkitty/defaults/main.yml +++ b/ansible/roles/cloudkitty/defaults/main.yml @@ -152,8 +152,21 @@ cloudkitty_storage_backend: "influxdb" cloudkitty_influxdb_name: "cloudkitty" -# Set the elasticsearch index name. -cloudkitty_elasticsearch_index_name: "cloudkitty" +# Set the elasticsearch index alias name. +cloudkitty_elasticsearch_alias_name: "cloudkitty" + +# Set the elasticsearch index name, will be URI encoded. eg, . +cloudkitty_elasticsearch_index_name: "<{{ cloudkitty_elasticsearch_alias_name }}-{now/d}-000001>" + +# Set the name of the elasticsearch component template to be created. +cloudkitty_elasticsearch_component_template_name: "{{ cloudkitty_elasticsearch_alias_name }}_settings" + +# Set the elasticsearch component templates to include in the cloudkitty provided template. +cloudkitty_elasticsearch_component_templates: + - "{{ cloudkitty_elasticsearch_component_template_name }}" + +# Set the number of replicas for cloudkitty indices. +cloudkitty_elasticsearch_index_replicas: "{{ groups['elasticsearch'] | length -1 }}" # Set the elasticsearch host URL. cloudkitty_elasticsearch_url: "{{ internal_protocol }}://{{ elasticsearch_address }}:{{ elasticsearch_port }}" diff --git a/ansible/roles/cloudkitty/tasks/bootstrap.yml b/ansible/roles/cloudkitty/tasks/bootstrap.yml index 2ba2906564..b3fc429d7e 100644 --- a/ansible/roles/cloudkitty/tasks/bootstrap.yml +++ b/ansible/roles/cloudkitty/tasks/bootstrap.yml @@ -45,34 +45,96 @@ delegate_to: "{{ groups['cloudkitty-api'][0] }}" when: cloudkitty_storage_backend == 'influxdb' -- name: Checking if Cloudkitty elasticsearch index exists +- name: Creating Cloudkitty elasticsearch index settings component template become: true kolla_toolbox: module_name: uri module_args: - url: "{{ cloudkitty_elasticsearch_url }}/{{ cloudkitty_elasticsearch_index_name }}" + url: "{{ cloudkitty_elasticsearch_url }}/_component_template/{{ cloudkitty_elasticsearch_component_template_name }}" + method: PUT + status_code: 200 + return_content: yes + body: "{{ lookup('template', item) | to_json }}" + body_format: json + run_once: True + delegate_to: "{{ groups['cloudkitty-api'][0] }}" + when: cloudkitty_storage_backend == 'elasticsearch' + with_first_found: + - "{{ node_custom_config }}/cloudkitty/cloudkitty_elasticsearch_component_template.json" + - "templates/cloudkitty_elasticsearch_component_template.json.j2" + +- name: Checking if Cloudkitty elasticsearch index alias name exists + become: true + kolla_toolbox: + module_name: uri + module_args: + url: "{{ cloudkitty_elasticsearch_url }}/{{ cloudkitty_elasticsearch_alias_name }}" status_code: 200, 404 run_once: true delegate_to: "{{ groups['cloudkitty-api'][0] }}" register: cloudkitty_index when: cloudkitty_storage_backend == 'elasticsearch' -- name: Creating Cloudkitty elasticsearch index +- name: Checking if Cloudkitty elasticsearch index alias name is an alias become: true kolla_toolbox: module_name: uri module_args: - url: "{{ cloudkitty_elasticsearch_url }}/{{ cloudkitty_elasticsearch_index_name }}" - method: PUT + url: "{{ cloudkitty_elasticsearch_url }}/_alias/{{ cloudkitty_elasticsearch_alias_name }}" + method: HEAD + status_code: 200, 404 + run_once: true + delegate_to: "{{ groups['cloudkitty-api'][0] }}" + register: cloudkitty_index_alias + when: + - cloudkitty_index is not skipped + - cloudkitty_index.get('status') == 200 + +- name: Reindexing old Cloudkitty index to new index name + become: true + kolla_toolbox: + module_name: uri + module_args: + url: "{{ cloudkitty_elasticsearch_url }}/_reindex" + method: POST + status_code: 200 + body: "{{ lookup('template', 'cloudkitty_elasticsearch_reindex.json.j2') | to_json }}" + body_format: json + run_once: true + delegate_to: "{{ groups['cloudkitty-api'][0] }}" + register: cloudkitty_reindex + when: + - cloudkitty_index_alias is not skipped + - cloudkitty_index_alias.get('status') == 404 + +- name: Deleting old Cloudkitty index + become: true + kolla_toolbox: + module_name: uri + module_args: + url: "{{ cloudkitty_elasticsearch_url }}/{{ cloudkitty_elasticsearch_alias_name }}" + method: DELETE + status_code: 200 + run_once: true + delegate_to: "{{ groups['cloudkitty-api'][0] }}" + register: cloudkitty_delete + when: + - cloudkitty_reindex is not skipped + +- name: Adding alias to new Cloudkitty elasticsearch index + become: true + kolla_toolbox: + module_name: uri + module_args: + url: "{{ cloudkitty_elasticsearch_url }}/{{ cloudkitty_elasticsearch_index_name | urlencode | regex_replace('/','%2F') }}/_alias/{{ cloudkitty_elasticsearch_alias_name }}" + method: POST status_code: 200 return_content: yes - body: | - {} + body: '{ "is_write_index": true }' body_format: json run_once: True delegate_to: "{{ groups['cloudkitty-api'][0] }}" when: - - cloudkitty_storage_backend == 'elasticsearch' - - cloudkitty_index.get('status') != 200 + - cloudkitty_delete is not skipped - import_tasks: bootstrap_service.yml diff --git a/ansible/roles/cloudkitty/templates/cloudkitty.conf.j2 b/ansible/roles/cloudkitty/templates/cloudkitty.conf.j2 index efa54250e0..c6ea3ad2ba 100644 --- a/ansible/roles/cloudkitty/templates/cloudkitty.conf.j2 +++ b/ansible/roles/cloudkitty/templates/cloudkitty.conf.j2 @@ -126,7 +126,8 @@ cafile = {{ cloudkitty_influxdb_cafile }} {% if cloudkitty_storage_backend == 'elasticsearch' %} [storage_elasticsearch] host = {{ cloudkitty_elasticsearch_url }} -index_name = {{ cloudkitty_elasticsearch_index_name }} +index_name = {{ cloudkitty_elasticsearch_alias_name }} +component_templates = {{ cloudkitty_elasticsearch_component_templates | join(',') }} insecure = {{ cloudkitty_elasticsearch_insecure_connections }} {% if cloudkitty_elasticsearch_cafile is defined %} diff --git a/ansible/roles/cloudkitty/templates/cloudkitty_elasticsearch_component_template.json.j2 b/ansible/roles/cloudkitty/templates/cloudkitty_elasticsearch_component_template.json.j2 new file mode 100644 index 0000000000..44524c9d5f --- /dev/null +++ b/ansible/roles/cloudkitty/templates/cloudkitty_elasticsearch_component_template.json.j2 @@ -0,0 +1,9 @@ +{ + "template": { + "settings": { + "index" : { + "number_of_replicas": {{ cloudkitty_elasticsearch_index_replicas }} + } + } + } +} diff --git a/ansible/roles/cloudkitty/templates/cloudkitty_elasticsearch_reindex.json.j2 b/ansible/roles/cloudkitty/templates/cloudkitty_elasticsearch_reindex.json.j2 new file mode 100644 index 0000000000..fe35e1c241 --- /dev/null +++ b/ansible/roles/cloudkitty/templates/cloudkitty_elasticsearch_reindex.json.j2 @@ -0,0 +1,8 @@ +{ + "source": { + "index": "{{ cloudkitty_elasticsearch_alias_name }}" + }, + "dest": { + "index": "{{ cloudkitty_elasticsearch_index_name }}" + } +} diff --git a/ansible/roles/elasticsearch/defaults/main.yml b/ansible/roles/elasticsearch/defaults/main.yml index 10232d6997..4e742a0951 100644 --- a/ansible/roles/elasticsearch/defaults/main.yml +++ b/ansible/roles/elasticsearch/defaults/main.yml @@ -62,9 +62,15 @@ elasticsearch_curator_cron_schedule: "0 {{ elasticsearch_curator_instance_id }} # useful way of checking that Curator actions are working as expected. elasticsearch_curator_dry_run: false +# List of elasticsearch index names to be managed by Curator. +elasticsearch_curator_index_names: + - "{{ kibana_log_prefix }}" + - "{{ (enable_monasca | bool) | ternary('monasca', '') }}" + - "{{ (cloudkitty_storage_backend == 'elasticsearch') | ternary(cloudkitty_elasticsearch_alias_name, '') }}" + # Index prefix pattern. Any indices matching this regex will # be managed by Curator. -elasticsearch_curator_index_pattern: "^{{ '(monasca|' + kibana_log_prefix + ')' if enable_monasca|bool else kibana_log_prefix }}-.*" +elasticsearch_curator_index_pattern: "^({{ elasticsearch_curator_index_names | reject('eq', '') | unique | join('|') }})-.*" # Duration after which an index is staged for deletion. This is # implemented by closing the index. Whilst in this state the index @@ -75,6 +81,24 @@ elasticsearch_curator_soft_retention_period_days: 30 # Duration after which an index is permanently erased from the cluster. elasticsearch_curator_hard_retention_period_days: 60 +# Enable rollover of elasticsearch indices. +elasticsearch_curator_rollover: "{{ cloudkitty_storage_backend == 'elasticsearch' }}" + +# Dict containing elasticsearch rollover options. +# For help with options see curator docs: +# https://www.elastic.co/guide/en/elasticsearch/client/curator/5.8/rollover.html +elasticsearch_curator_rollover_options: + - name: "{{ cloudkitty_elasticsearch_alias_name }}" + conditions: + max_age: "{{ elasticsearch_curator_rollover_default_max_age }}" + max_docs: "{{ elasticsearch_curator_rollover_default_max_docs }}" + +# Elasticsearch index max age before rollover. +elasticsearch_curator_rollover_default_max_age: "1d" + +# Elasticsearch index max number of docs before rollover. +elasticsearch_curator_rollover_default_max_docs: "100000" + #################### # Keystone #################### diff --git a/ansible/roles/elasticsearch/templates/elasticsearch-curator-actions.yml.j2 b/ansible/roles/elasticsearch/templates/elasticsearch-curator-actions.yml.j2 index 0ef452874b..5efce92f54 100644 --- a/ansible/roles/elasticsearch/templates/elasticsearch-curator-actions.yml.j2 +++ b/ansible/roles/elasticsearch/templates/elasticsearch-curator-actions.yml.j2 @@ -31,3 +31,13 @@ actions: timestring: '%Y.%m.%d' unit: days unit_count: "{{ elasticsearch_curator_hard_retention_period_days }}" +{% if elasticsearch_curator_rollover | bool %} +{% for index in elasticsearch_curator_rollover_options %} + {{ 2 + loop.index }}: + action: rollover + description: >- + Rollover index associated with {{ index.name }} alias + options: + {{ index | to_nice_yaml | indent(6, false) }} +{% endfor %} +{% endif %} diff --git a/releasenotes/notes/add-cloudkitty-elasticsearch-alias-support-e6cc3fe1360aa4ae.yaml b/releasenotes/notes/add-cloudkitty-elasticsearch-alias-support-e6cc3fe1360aa4ae.yaml new file mode 100644 index 0000000000..5618a6246f --- /dev/null +++ b/releasenotes/notes/add-cloudkitty-elasticsearch-alias-support-e6cc3fe1360aa4ae.yaml @@ -0,0 +1,13 @@ +--- +features: + - | + Improves support for elasticsearch as the cloudkitty storage backend. + Enforces using an elasticsearch index alias with cloudkitty. + Provides an elasticsearch index component template to manage replicas. + Adds elasicsearch curator config to prune cloudkitty indices. +upgrade: + - | + Enforces using an elasticsearch index alias with cloudkitty. + If elasticsearch is used as the cloudkitty storage backend this will + reindex the data into a dated index and delete the old copy. The old index + name is used as an alias for the dated indices.