Optimize the effective logic of num_most_recent_indicesconfiguration in the es plugin

conf/input.elasticsearch/elasticsearch.toml

@@ -100,3 +100,20 @@ cluster_info_interval = "5m"
 # tls_key = "/etc/categraf/key.pem"
 ## Use TLS but skip chain & host verification
 # insecure_skip_verify = true
+
+## Sets the number of most recent indices to return for indices that are configured with a date-stamped suffix.
+## Each 'indices_include' entry ending with a wildcard (*) or glob matching pattern will group together all indices that match it, and
+## sort them by the date or number after the wildcard. Metrics then are gathered for only the 'num_most_recent_indices' amount of most
+## recent indices.
+num_most_recent_indices = 0
+
+
+
+## must num_most_recent_indices Coordinated use
+## default ["(?P<date>(?:\\d{4}|\\d{2})[.-]?(?:\\d{2})[.-]?(?:\\d{2})?[.-]?(?:\\d{2})?)$","[\\.-._]\\d+(\\.\\d+){0,2}$"]
+## match //YYYY.MM.DD or YYYY-MM-DD or YYYYMMDD or YYYY-MM-DD-HH
+##       //YYYY.MM or YYYY-MM or YYYYMM or YYYYMMDDHH
+##      //YY.MM.DD or YY-MM-DD or YYMMDD or YY.MM.DD.HH
+##     //v1_001 v1_002 -->v1* v0.1 v0.2 -->v0* v5.2.3 v5.2.4 -->v5*
+## or self expansion
+dynamic_index_matcher_regexp = ["(?P<date>(?:\\d{4}|\\d{2})[.-]?(?:\\d{2})[.-]?(?:\\d{2})?[.-]?(?:\\d{2})?)$","[\\.-._]\\d+(\\.\\d+){0,2}$"]

inputs/elasticsearch/README.md

@@ -483,3 +483,20 @@ ES 7.x 支持基于角色的访问控制（RBACs）。`elasticsearch` 插件需
 | elasticsearch_slm_stats_snapshots_deleted_total          | counter | 按策略删除的快照数            |
 | elasticsearch_slm_stats_snapshot_deletion_failures_total | counter | 按策略快照删除失败次数          |
 | elasticsearch_slm_stats_operation_mode                   | gauge   | SLM操作模式（运行中，停止中，已停止） |
+
+
+#### `num_most_recent_indices = 0`
+
+
+| 设置日期类动态索引可取前"num_most_recent_indices"个最新index的指标数据
+| 可极大减少历史动态索引导致的大指标量级
+| 可与“indices_include”配置一起使用
+
+#### `dynamic_index_matcher_regexp` = ["(?P<date>(?:\\d{4}|\\d{2})[.-]?(?:\\d{2})[.-]?(?:\\d{2})?[.-]?(?:\\d{2})?)$","[\\.-._]\\d+(\\.\\d+){0,2}$"]
+| 与num_most_recent_indices 配合使用，用于指定动态索引的匹配逻辑，默认值：
+| ["(?P<date>(?:\\d{4}|\\d{2})[.-]?(?:\\d{2})[.-]?(?:\\d{2})?[.-]?(?:\\d{2})?)$","[\\.-._]\\d+(\\.\\d+){0,2}$"]
+| 支持匹配 //YYYY.MM.DD 或 YYYY-MM-DD 或 YYYYMMDD 或 YYYY-MM-DD-HH
+|       //YYYY.MM 或 YYYY-MM 或 YYYYMM 或YYYYMMDDHH
+|      //YY.MM.DD 或 YY-MM-DD 或 YYMMDD 或YYYY.MM.DD.HH
+|     //v1_001 v1_002 -->v1* v0.1 v0.2 -->v0* v5.2.3 v5.2.4 -->v5*
+| 也可自行扩展

inputs/elasticsearch/README_en.md

@@ -477,3 +477,19 @@ ES 7.x supports RBACs. The following security privileges are required for the `e
 | elasticsearch_slm_stats_snapshots_deleted_total                      | counter | Snapshots deleted by policy                                                                         |
 | elasticsearch_slm_stats_snapshot_deletion_failures_total             | counter | Snapshot deletion failures by policy                                                                |
 | elasticsearch_slm_stats_operation_mode                               | gauge   | SLM operation mode (Running, stopping, stopped)                                                     |
+
+
+#### `num_most_recent_indices = 0`
+|Set the indicator data for the latest index of the first "num_mast_decent_indice" in the date class dynamic index
+|It can greatly reduce the large scale of indicators caused by historical dynamic indexing
+|Can be used together with the 'indices_inclub' configuration
+
+
+#### `dynamic_index_matcher_regexp` = ["(?P<date>(?:\\d{4}|\\d{2})[.-]?(?:\\d{2})[.-]?(?:\\d{2})?[.-]?(?:\\d{2})?)$","[\\.-._]\\d+(\\.\\d+){0,2}$"]
+|Used in conjunction with num_mast_decent_indice to specify the matching logic for dynamic indexes, default value:
+| ["(?P<date>(?:\\d{4}|\\d{2})[.-]?(?:\\d{2})[.-]?(?:\\d{2})?[.-]?(?:\\d{2})?)$","[\\.-._]\\d+(\\.\\d+){0,2}$"]
+|Support matching with//YYYY.MM.DD or YYYY-MM-DD or YYYY-MMDD or YYYY-MM-DD-HH
+|//YYYY.MM or YYYY-MM or YYYYMM or YYYYMMDDHH
+|//YY.MM.DD or YY-MM-DD or YYMMDD or YYY.MM.DD.HH
+|     //v1_001 v1_002 -->v1* v0.1 v0.2 -->v0* v5.2.3 v5.2.4 -->v5*
+|It can also be expanded on its own

inputs/elasticsearch/collector/ilm_indices.go

@@ -15,16 +15,17 @@ package collector
 
 import (
 	"encoding/json"
-	"flashcat.cloud/categraf/pkg/filter"
 	"fmt"
 	"io"
 	"log"
 	"net/http"
 	"net/url"
 	"path"
-	"sort"
+	"slices"
 	"strings"
 
+	"flashcat.cloud/categraf/pkg/filter"
+
 	"github.com/prometheus/client_golang/prometheus"
 )
 
@@ -37,12 +38,13 @@ type ilmMetric struct {
 
 // Index Lifecycle Management information object
 type IlmIndiciesCollector struct {
-	client               *http.Client
-	url                  *url.URL
-	indicesIncluded      []string
-	numMostRecentIndices int
-	indexMatchers        map[string]filter.Filter
-	ilmMetric            ilmMetric
+	client                 *http.Client
+	url                    *url.URL
+	indicesIncluded        []string
+	numMostRecentIndices   int
+	maxIndicesIncludeCount int
+	indexMatchers          map[string]filter.Filter
+	ilmMetric              ilmMetric
 }
 
 type IlmResponse struct {
@@ -63,16 +65,14 @@ var (
 )
 
 // NewIlmIndicies defines Index Lifecycle Management Prometheus metrics
-func NewIlmIndicies(client *http.Client, url *url.URL, indicesIncluded []string, numMostRecentIndices int, indexMatchers map[string]filter.Filter) *IlmIndiciesCollector {
+func NewIlmIndicies(client *http.Client, url *url.URL, indicesIncluded []string, maxIndicesIncludeCount int) *IlmIndiciesCollector {
 	subsystem := "ilm_index"
 
 	return &IlmIndiciesCollector{
-		client:               client,
-		url:                  url,
-		indicesIncluded:      indicesIncluded,
-		numMostRecentIndices: numMostRecentIndices,
-		indexMatchers:        indexMatchers,
-
+		client:                 client,
+		url:                    url,
+		indicesIncluded:        indicesIncluded,
+		maxIndicesIncludeCount: maxIndicesIncludeCount,
 		ilmMetric: ilmMetric{
 			Type: prometheus.GaugeValue,
 			Desc: prometheus.NewDesc(
@@ -95,10 +95,11 @@ func (i *IlmIndiciesCollector) fetchAndDecodeIlm() (IlmResponse, error) {
 	var ir IlmResponse
 
 	u := *i.url
+
 	//add indices filter
-	if len(i.indicesIncluded) == 0 {
+	if len(i.indicesIncluded) == 0 || len(i.indicesIncluded) > i.maxIndicesIncludeCount {
 		u.Path = path.Join(u.Path, "/_all/_ilm/explain")
-	} else {
+	} else if len(i.indicesIncluded) <= i.maxIndicesIncludeCount {
 		u.Path = path.Join(u.Path, "/"+strings.Join(i.indicesIncluded, ",")+"/_ilm/explain")
 	}
 
@@ -128,6 +129,11 @@ func (i *IlmIndiciesCollector) fetchAndDecodeIlm() (IlmResponse, error) {
 		return ir, err
 	}
 
+	//filter
+	if len(i.indicesIncluded) > i.maxIndicesIncludeCount {
+		ir.Indices = i.filterMapByKeys(ir.Indices, i.indicesIncluded)
+	}
+
 	return ir, nil
 }
 
@@ -147,9 +153,6 @@ func (i *IlmIndiciesCollector) Collect(ch chan<- prometheus.Metric) {
 		return
 	}
 
-	//add config i.numMostRecentIndices process code
-	ilmResp = i.gatherIndividualIndicesStats(ilmResp)
-
 	for indexName, indexIlm := range ilmResp.Indices {
 		ch <- prometheus.MustNewConstMetric(
 			i.ilmMetric.Desc,
@@ -160,61 +163,13 @@ func (i *IlmIndiciesCollector) Collect(ch chan<- prometheus.Metric) {
 	}
 }
 
-func (i *IlmIndiciesCollector) gatherIndividualIndicesStats(resp IlmResponse) IlmResponse {
-	newIndicesMappings := make(map[string]IlmIndexResponse)
-
-	// Sort indices into buckets based on their configured prefix, if any matches.
-	categorizedIndexNames := i.categorizeIndices(resp)
-	for _, matchingIndices := range categorizedIndexNames {
-		// Establish the number of each category of indices to use. User can configure to use only the latest 'X' amount.
-		indicesCount := len(matchingIndices)
-		indicesToTrackCount := indicesCount
-
-		// Sort the indices if configured to do so.
-		if i.numMostRecentIndices > 0 {
-			if i.numMostRecentIndices < indicesToTrackCount {
-				indicesToTrackCount = i.numMostRecentIndices
-			}
-			sort.Strings(matchingIndices)
-		}
+func (i *IlmIndiciesCollector) filterMapByKeys(originalMap map[string]IlmIndexResponse, allowedKeys []string) map[string]IlmIndexResponse {
 
-		// Gather only the number of indexes that have been configured, in descending order (most recent, if date-stamped).
-		for i := indicesCount - 1; i >= indicesCount-indicesToTrackCount; i-- {
-			indexName := matchingIndices[i]
-			newIndicesMappings[indexName] = resp.Indices[indexName]
+	resultMap := make(map[string]IlmIndexResponse)
+	for key, value := range originalMap {
+		if slices.Contains(allowedKeys, key) {
+			resultMap[key] = value
 		}
 	}
-	//return new IlmResponse
-	var iml IlmResponse
-	iml.Indices = newIndicesMappings
-	return iml
-}
-
-func (i *IlmIndiciesCollector) categorizeIndices(resp IlmResponse) map[string][]string {
-	categorizedIndexNames := make(map[string][]string, len(resp.Indices))
-	// If all indices are configured to be gathered, bucket them all together.
-	if len(i.indicesIncluded) == 0 || i.indicesIncluded[0] == "_all" {
-		for indexName := range resp.Indices {
-			categorizedIndexNames["_all"] = append(categorizedIndexNames["_all"], indexName)
-		}
-
-		return categorizedIndexNames
-	}
-
-	// Bucket each returned index with its associated configured index (if any match).
-	for indexName := range resp.Indices {
-		match := indexName
-		for name, matcher := range i.indexMatchers {
-			// If a configured index matches one of the returned indexes, mark it as a match.
-			if matcher.Match(match) {
-				match = name
-				break
-			}
-		}
-
-		// Bucket all matching indices together for sorting.
-		categorizedIndexNames[match] = append(categorizedIndexNames[match], indexName)
-	}
-
-	return categorizedIndexNames
+	return resultMap
 }

inputs/elasticsearch/collector/ilm_indices_test.go

@@ -14,7 +14,6 @@
 package collector
 
 import (
-	"flashcat.cloud/categraf/pkg/filter"
 	"io"
 	"net/http"
 	"net/http/httptest"
@@ -99,9 +98,8 @@ elasticsearch_ilm_index_status{action="complete",index="facebook",phase="new",st
 			}
 
 			indicesIncluded := make([]string, 0)
-			var numMostRecentIndices int = 0
-			indexMatchers := make(map[string]filter.Filter)
-			c := NewIlmIndicies(http.DefaultClient, u, indicesIncluded, numMostRecentIndices, indexMatchers)
+			maxIndicesIncludeCount := 80
+			c := NewIlmIndicies(http.DefaultClient, u, indicesIncluded, maxIndicesIncludeCount)
 			if err != nil {
 				t.Fatal(err)
 			}