elasticsearch: add index mode and codec dashboard visualizations

packages/elasticsearch/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "1.22.0"
+  changes:
+    - description: Wire index mode and codec through the index_pivot transform, normalize missing values to standard/default in the monitoring_indices and index metrics ingest pipelines, and add dashboard visualizations for index mode and codec adoption
+      type: enhancement
+      link: https://github.com/elastic/beats/pull/49237
 - version: "1.21.1"
   changes:
     - description: Bugfix for `querylog` new data stream (docs and dashboard)

packages/elasticsearch/data_stream/index/elasticsearch/ingest_pipeline/default.yml

@@ -0,0 +1,23 @@
+---
+description: Pipeline for Elasticsearch stack monitoring index metrics
+processors:
+  - script:
+      source: |
+        if (ctx.elasticsearch?.index == null) {
+          ctx.elasticsearch = ctx.elasticsearch ?: [:];
+          ctx.elasticsearch.index = ctx.elasticsearch.index ?: [:];
+        }
+        def mode = ctx.elasticsearch.index.mode;
+        if (mode == null || mode == '' || mode == 'null') {
+          ctx.elasticsearch.index.mode = 'standard';
+        }
+        def codec = ctx.elasticsearch.index.codec;
+        if (codec == null || codec == '' || codec == 'null') {
+          ctx.elasticsearch.index.codec = 'default';
+        }
+      ignore_failure: true
+      tag: script_normalize_index_mode_codec
+on_failure:
+  - set:
+      field: error.message
+      value: "{{ _ingest.on_failure_message }}"

packages/elasticsearch/elasticsearch/ingest_pipeline/_dev/test/pipeline/test-common-config.yml

@@ -0,0 +1,2 @@
+dynamic_fields:
+  event.ingested: ".*"

packages/elasticsearch/elasticsearch/ingest_pipeline/_dev/test/pipeline/test-monitoring-indices-defaults.json

@@ -0,0 +1,30 @@
+{
+    "@timestamp": "2026-03-10T10:00:00.000Z",
+    "elasticsearch": {
+        "cluster": {
+            "name": "elasticsearch"
+        },
+        "index": {
+            "name": ".ds-metrics-elastic_agent.elastic_agent-default-2026.03.10-000001"
+        }
+    },
+    "end": {
+        "elasticsearch.index.tier_preference": "data_hot",
+        "elasticsearch.index.mode": "time_series",
+        "elasticsearch.index.creation_date": 1741600800000,
+        "elasticsearch.index.primaries.docs.count": 100,
+        "elasticsearch.index.primaries.docs.count_delta": 10,
+        "elasticsearch.index.primaries.store.total_data_set_size_in_bytes": 2048,
+        "elasticsearch.index.primaries.store.total_data_set_size_in_bytes_delta": 256,
+        "elasticsearch.index.total.store.size_in_bytes": 4096,
+        "elasticsearch.index.total.store.size_in_bytes_delta": 512,
+        "elasticsearch.index.total.search.query_total": 20,
+        "elasticsearch.index.total.search.query_total_delta": 5,
+        "elasticsearch.index.total.search.query_time_in_millis": 100,
+        "elasticsearch.index.total.search.query_time_in_millis_delta": 25,
+        "elasticsearch.index.total.indexing.index_total": 30,
+        "elasticsearch.index.total.indexing.index_total_delta": 8,
+        "elasticsearch.index.total.indexing.index_time_in_millis": 80,
+        "elasticsearch.index.total.indexing.index_time_in_millis_delta": 20
+    }
+}

packages/elasticsearch/elasticsearch/ingest_pipeline/_dev/test/pipeline/test-monitoring-indices-defaults.json-expected.json

@@ -0,0 +1,48 @@
+{
+    "@timestamp": "2026-03-10T10:00:00.000Z",
+    "elasticsearch": {
+        "cluster": {
+            "name": "elasticsearch"
+        },
+        "index": {
+            "age": 0,
+            "codec": "default",
+            "creation_date": "2026-03-10T10:00:00.000Z",
+            "datastream": "metrics-elastic_agent.elastic_agent-default",
+            "mode": "time_series",
+            "name": ".ds-metrics-elastic_agent.elastic_agent-default-2026.03.10-000001",
+            "primaries": {
+                "docs": {
+                    "count": 100,
+                    "count_delta": 10
+                },
+                "store": {
+                    "total_data_set_size_in_bytes": 2048,
+                    "total_data_set_size_in_bytes_delta": 256
+                }
+            },
+            "tier": "hot/content",
+            "total": {
+                "indexing": {
+                    "index_time_in_millis": 80,
+                    "index_time_in_millis_delta": 20,
+                    "index_total": 30,
+                    "index_total_delta": 8
+                },
+                "search": {
+                    "query_time_in_millis": 100,
+                    "query_time_in_millis_delta": 25,
+                    "query_total": 20,
+                    "query_total_delta": 5
+                },
+                "store": {
+                    "size_in_bytes": 4096,
+                    "size_in_bytes_delta": 512
+                }
+            }
+        }
+    },
+    "event": {
+        "ingested": "2026-03-10T10:00:00.000Z"
+    }
+}

packages/elasticsearch/elasticsearch/ingest_pipeline/_dev/test/pipeline/test-monitoring-indices-null-placeholders.json

@@ -0,0 +1,31 @@
+{
+    "@timestamp": "2026-03-10T10:00:00.000Z",
+    "elasticsearch": {
+        "cluster": {
+            "name": "elasticsearch"
+        },
+        "index": {
+            "name": "monitoring-indices"
+        }
+    },
+    "end": {
+        "elasticsearch.index.tier_preference": "data_hot",
+        "elasticsearch.index.mode": "null",
+        "elasticsearch.index.codec": "null",
+        "elasticsearch.index.creation_date": 1741600800000,
+        "elasticsearch.index.primaries.docs.count": 50,
+        "elasticsearch.index.primaries.docs.count_delta": 5,
+        "elasticsearch.index.primaries.store.total_data_set_size_in_bytes": 1024,
+        "elasticsearch.index.primaries.store.total_data_set_size_in_bytes_delta": 128,
+        "elasticsearch.index.total.store.size_in_bytes": 2048,
+        "elasticsearch.index.total.store.size_in_bytes_delta": 256,
+        "elasticsearch.index.total.search.query_total": 10,
+        "elasticsearch.index.total.search.query_total_delta": 2,
+        "elasticsearch.index.total.search.query_time_in_millis": 50,
+        "elasticsearch.index.total.search.query_time_in_millis_delta": 10,
+        "elasticsearch.index.total.indexing.index_total": 15,
+        "elasticsearch.index.total.indexing.index_total_delta": 3,
+        "elasticsearch.index.total.indexing.index_time_in_millis": 40,
+        "elasticsearch.index.total.indexing.index_time_in_millis_delta": 8
+    }
+}

packages/elasticsearch/elasticsearch/ingest_pipeline/_dev/test/pipeline/test-monitoring-indices-null-placeholders.json-expected.json

@@ -0,0 +1,47 @@
+{
+    "@timestamp": "2026-03-10T10:00:00.000Z",
+    "elasticsearch": {
+        "cluster": {
+            "name": "elasticsearch"
+        },
+        "index": {
+            "age": 0,
+            "codec": "default",
+            "creation_date": "2026-03-10T10:00:00.000Z",
+            "mode": "standard",
+            "name": "monitoring-indices",
+            "primaries": {
+                "docs": {
+                    "count": 50,
+                    "count_delta": 5
+                },
+                "store": {
+                    "total_data_set_size_in_bytes": 1024,
+                    "total_data_set_size_in_bytes_delta": 128
+                }
+            },
+            "tier": "hot/content",
+            "total": {
+                "indexing": {
+                    "index_time_in_millis": 40,
+                    "index_time_in_millis_delta": 8,
+                    "index_total": 15,
+                    "index_total_delta": 3
+                },
+                "search": {
+                    "query_time_in_millis": 50,
+                    "query_time_in_millis_delta": 10,
+                    "query_total": 10,
+                    "query_total_delta": 2
+                },
+                "store": {
+                    "size_in_bytes": 2048,
+                    "size_in_bytes_delta": 256
+                }
+            }
+        }
+    },
+    "event": {
+        "ingested": "2026-03-10T10:00:00.000Z"
+    }
+}

packages/elasticsearch/elasticsearch/ingest_pipeline/_dev/test/pipeline/test-monitoring-indices.json

@@ -0,0 +1,31 @@
+{
+    "@timestamp": "2026-03-10T10:00:00.000Z",
+    "elasticsearch": {
+        "cluster": {
+            "name": "elasticsearch"
+        },
+        "index": {
+            "name": ".ds-logs-default-2026.03.10-000001"
+        }
+    },
+    "end": {
+        "elasticsearch.index.tier_preference": "data_hot",
+        "elasticsearch.index.mode": "logsdb",
+        "elasticsearch.index.codec": "best_compression",
+        "elasticsearch.index.creation_date": 1741600800000,
+        "elasticsearch.index.primaries.docs.count": 100,
+        "elasticsearch.index.primaries.docs.count_delta": 10,
+        "elasticsearch.index.primaries.store.total_data_set_size_in_bytes": 2048,
+        "elasticsearch.index.primaries.store.total_data_set_size_in_bytes_delta": 256,
+        "elasticsearch.index.total.store.size_in_bytes": 4096,
+        "elasticsearch.index.total.store.size_in_bytes_delta": 512,
+        "elasticsearch.index.total.search.query_total": 20,
+        "elasticsearch.index.total.search.query_total_delta": 5,
+        "elasticsearch.index.total.search.query_time_in_millis": 100,
+        "elasticsearch.index.total.search.query_time_in_millis_delta": 25,
+        "elasticsearch.index.total.indexing.index_total": 30,
+        "elasticsearch.index.total.indexing.index_total_delta": 8,
+        "elasticsearch.index.total.indexing.index_time_in_millis": 80,
+        "elasticsearch.index.total.indexing.index_time_in_millis_delta": 20
+    }
+}

packages/elasticsearch/elasticsearch/ingest_pipeline/_dev/test/pipeline/test-monitoring-indices.json-expected.json

@@ -0,0 +1,48 @@
+{
+    "@timestamp": "2026-03-10T10:00:00.000Z",
+    "elasticsearch": {
+        "cluster": {
+            "name": "elasticsearch"
+        },
+        "index": {
+            "age": 0,
+            "codec": "best_compression",
+            "creation_date": "2026-03-10T10:00:00.000Z",
+            "datastream": "logs-default",
+            "mode": "logsdb",
+            "name": ".ds-logs-default-2026.03.10-000001",
+            "primaries": {
+                "docs": {
+                    "count": 100,
+                    "count_delta": 10
+                },
+                "store": {
+                    "total_data_set_size_in_bytes": 2048,
+                    "total_data_set_size_in_bytes_delta": 256
+                }
+            },
+            "tier": "hot/content",
+            "total": {
+                "indexing": {
+                    "index_time_in_millis": 80,
+                    "index_time_in_millis_delta": 20,
+                    "index_total": 30,
+                    "index_total_delta": 8
+                },
+                "search": {
+                    "query_time_in_millis": 100,
+                    "query_time_in_millis_delta": 25,
+                    "query_total": 20,
+                    "query_total_delta": 5
+                },
+                "store": {
+                    "size_in_bytes": 4096,
+                    "size_in_bytes_delta": 512
+                }
+            }
+        }
+    },
+    "event": {
+        "ingested": "2026-03-10T10:00:00.000Z"
+    }
+}

packages/elasticsearch/elasticsearch/ingest_pipeline/monitoring_indices.yml

@@ -42,6 +42,22 @@ processors:
   - dot_expander:
       field: "*"
       tag: dot_expander
+  - script:
+      source: |
+        if (ctx.elasticsearch?.index == null) {
+          ctx.elasticsearch = ctx.elasticsearch ?: [:];
+          ctx.elasticsearch.index = [:];
+        }
+        def mode = ctx.elasticsearch.index.mode;
+        if (mode == null || mode == '' || mode == 'null') {
+          ctx.elasticsearch.index.mode = 'standard';
+        }
+        def codec = ctx.elasticsearch.index.codec;
+        if (codec == null || codec == '' || codec == 'null') {
+          ctx.elasticsearch.index.codec = 'default';
+        }
+      ignore_failure: true
+      tag: script_normalize_index_mode_codec
   - date:
       field: elasticsearch.index.creation_date
       target_field: elasticsearch.index.creation_date

packages/elasticsearch/elasticsearch/transform/index_pivot/fields/fields.yml

@@ -24,6 +24,12 @@
         - name: tier
           type: keyword
           description: The data tier of the index (e.g., hot, warm, cold, frozen, or unknown).
+        - name: mode
+          type: keyword
+          description: Index mode from index settings (e.g. standard, time_series, logsdb).
+        - name: codec
+          type: keyword
+          description: Index codec from index settings (e.g. default, best_compression, none).
         - name: name
           type: keyword
           description: Name of the Elasticsearch index.

packages/elasticsearch/elasticsearch/transform/index_pivot/transform.yml

@@ -64,6 +64,8 @@ pivot:
         metrics:
           # We will need to activate this once it is exposed, the rest will work directly
           - field: "elasticsearch.index.tier_preference"
+          - field: "elasticsearch.index.mode"
+          - field: "elasticsearch.index.codec"
           - field: "elasticsearch.index.creation_date"
           - field: "elasticsearch.index.primaries.docs.count"
           - field: "elasticsearch.index.primaries.store.total_data_set_size_in_bytes"
@@ -131,5 +133,5 @@ sync:
     delay: 60s
     field: '@timestamp'
 _meta:
-  fleet_transform_version: 0.3.0
+  fleet_transform_version: 0.4.0
   run_as_kibana_system: false