From 48c9998fa8578abb6c3f1f87c31bbc81013e065e Mon Sep 17 00:00:00 2001 From: Alexandr Yudin <57181751+u-veles-a@users.noreply.github.com> Date: Mon, 4 May 2026 12:59:06 +0000 Subject: [PATCH 01/37] WIP: fix test Signed-off-by: Alexandr Yudin <57181751+u-veles-a@users.noreply.github.com> --- .../storage/querier/merge_series_set_test.go | 70 +++++++++++++++++-- 1 file changed, 63 insertions(+), 7 deletions(-) diff --git a/pp/go/storage/querier/merge_series_set_test.go b/pp/go/storage/querier/merge_series_set_test.go index ae2008408..883968aa1 100644 --- a/pp/go/storage/querier/merge_series_set_test.go +++ b/pp/go/storage/querier/merge_series_set_test.go @@ -49,8 +49,24 @@ func (s *MergeShardSeriesSetSuite) TestHappyPath() { eseriesSets = eseriesSets[:0] aseriesSets = aseriesSets[:0] for i := 0; i < bm.numShards; i++ { - eseriesSets = append(eseriesSets, queryOpt(s.T(), head.lsses[i], head.dss[i], start, end, matcher)) - aseriesSets = append(aseriesSets, queryOpt(s.T(), head.lsses[i], head.dss[i], start, end, matcher)) + eseriesSets = append(eseriesSets, queryOpt( + s.T(), + head.lsses[i], + head.dss[i], + start, + end, + cppbridge.NoDownsampling, + matcher, + )) + aseriesSets = append(aseriesSets, queryOpt( + s.T(), + head.lsses[i], + head.dss[i], + start, + end, + cppbridge.NoDownsampling, + matcher, + )) } expectedSeriesSet := storage.NewMergeSeriesSet(eseriesSets, storage.ChainedSeriesMerge) @@ -96,8 +112,24 @@ func (s *MergeShardSeriesSetSuite) TestEmptySeriesSets() { aseriesSets = append(aseriesSets, &querier.SeriesSet{}) } - eseriesSets = append(eseriesSets, queryOpt(s.T(), head.lsses[i], head.dss[i], start, end, matcher)) - aseriesSets = append(aseriesSets, queryOpt(s.T(), head.lsses[i], head.dss[i], start, end, matcher)) + eseriesSets = append(eseriesSets, queryOpt( + s.T(), + head.lsses[i], + head.dss[i], + start, + end, + cppbridge.NoDownsampling, + matcher, + )) + aseriesSets = append(aseriesSets, queryOpt( + s.T(), + head.lsses[i], + head.dss[i], + start, + end, + cppbridge.NoDownsampling, + matcher, + )) } expectedSeriesSet := storage.NewMergeSeriesSet(eseriesSets, storage.ChainedSeriesMerge) @@ -138,11 +170,27 @@ func (s *MergeShardSeriesSetSuite) TestAcrossShardsSeriesSets() { eseriesSets = eseriesSets[:0] aseriesSets = aseriesSets[:0] for i := 0; i < bm.numShards; i++ { - eseriesSets = append(eseriesSets, queryOpt(s.T(), head.lsses[i], head.dss[i], start, end, matcher)) + eseriesSets = append(eseriesSets, queryOpt( + s.T(), + head.lsses[i], + head.dss[i], + start, + end, + cppbridge.NoDownsampling, + matcher, + )) } for i := bm.numShards - 1; i >= 0; i-- { - aseriesSets = append(aseriesSets, queryOpt(s.T(), head.lsses[i], head.dss[i], start, end, matcher)) + aseriesSets = append(aseriesSets, queryOpt( + s.T(), + head.lsses[i], + head.dss[i], + start, + end, + cppbridge.NoDownsampling, + matcher, + )) } expectedSeriesSet := storage.NewMergeSeriesSet(eseriesSets, storage.ChainedSeriesMerge) @@ -186,7 +234,15 @@ func BenchmarkMergeSeriesSet(b *testing.B) { b.StopTimer() seriesSets = seriesSets[:0] for i := 0; i < bm.numShards; i++ { - seriesSets = append(seriesSets, queryOpt(b, head.lsses[i], head.dss[i], start, end, matcher)) + seriesSets = append(seriesSets, queryOpt( + b, + head.lsses[i], + head.dss[i], + start, + end, + cppbridge.NoDownsampling, + matcher, + )) } b.StartTimer() From 994326c9835d5fa7c8e1c35ab0861c61dedda457 Mon Sep 17 00:00:00 2001 From: Alexandr Yudin <57181751+u-veles-a@users.noreply.github.com> Date: Tue, 5 May 2026 14:15:23 +0000 Subject: [PATCH 02/37] WIP: created methods for lss Signed-off-by: Alexandr Yudin <57181751+u-veles-a@users.noreply.github.com> --- pp/go/cppbridge/entrypoint.go | 6 ++-- pp/go/cppbridge/lss_snapshot.go | 3 +- pp/go/cppbridge/lss_snapshot_test.go | 14 +++++--- pp/go/cppbridge/primitives_lss.go | 12 ++++--- pp/go/cppbridge/primitives_lss_test.go | 6 ++-- pp/go/storage/head/shard/lss.go | 16 +++++++++ .../storage/querier/merge_series_set_test.go | 34 +++++++++++++++++++ 7 files changed, 75 insertions(+), 16 deletions(-) diff --git a/pp/go/cppbridge/entrypoint.go b/pp/go/cppbridge/entrypoint.go index bfa64d967..1411423ec 100644 --- a/pp/go/cppbridge/entrypoint.go +++ b/pp/go/cppbridge/entrypoint.go @@ -1573,7 +1573,7 @@ func primitivesLSSQueryLabelValues(lss uintptr, label_name string, matchers []mo return res.status, res.values } -func primitivesLSSGetLabelNameIDs(lss uintptr, names []string) []uint32 { +func primitivesLSSGetLabelNameIDs(lss uintptr, names []string, nameIDs []uint32) { args := struct { lss uintptr names []string @@ -1581,7 +1581,7 @@ func primitivesLSSGetLabelNameIDs(lss uintptr, names []string) []uint32 { res := struct { outIDs []uint32 - }{make([]uint32, len(names))} + }{nameIDs} testGC() fastcgo.UnsafeCall2( @@ -1589,8 +1589,6 @@ func primitivesLSSGetLabelNameIDs(lss uintptr, names []string) []uint32 { uintptr(unsafe.Pointer(&args)), uintptr(unsafe.Pointer(&res)), ) - - return res.outIDs } func primitivesLSSCreateSnapshotLSS(lss uintptr) uintptr { diff --git a/pp/go/cppbridge/lss_snapshot.go b/pp/go/cppbridge/lss_snapshot.go index 70e85805a..9ab1b716d 100644 --- a/pp/go/cppbridge/lss_snapshot.go +++ b/pp/go/cppbridge/lss_snapshot.go @@ -77,12 +77,13 @@ func (lss *LabelSetSnapshot) Query(selector uintptr) *LSSQueryResult { return result } +// SeriesGroups group series by label names. type SeriesGroups struct { Groups [][]uint32 } // GroupSeriesByLabelNames group series by label names -func (lss *LabelSetSnapshot) GroupSeriesByLabelNames(seriesIDs []uint32, labelNameIDs []uint32) *SeriesGroups { +func (lss *LabelSetSnapshot) GroupSeriesByLabelNames(seriesIDs, labelNameIDs []uint32) *SeriesGroups { result := &SeriesGroups{ Groups: primitivesGroupSeriesByLabelNames(lss.pointer, seriesIDs, labelNameIDs), } diff --git a/pp/go/cppbridge/lss_snapshot_test.go b/pp/go/cppbridge/lss_snapshot_test.go index 6aecd3356..360366ab6 100644 --- a/pp/go/cppbridge/lss_snapshot_test.go +++ b/pp/go/cppbridge/lss_snapshot_test.go @@ -52,13 +52,15 @@ func (s *LabelSetSnapshotSuite) TestGroupSeriesByLabelNames_ByJob() { idA0 := s.lss.FindOrEmplace(model.NewLabelSetBuilder().Set("__name__", "m").Set("job", "a").Set("instance", "i0").Build()).LabelSetID idA1 := s.lss.FindOrEmplace(model.NewLabelSetBuilder().Set("__name__", "m").Set("job", "a").Set("instance", "i1").Build()).LabelSetID idB := s.lss.FindOrEmplace(model.NewLabelSetBuilder().Set("__name__", "m").Set("job", "b").Set("instance", "i2").Build()).LabelSetID + names := []string{"job"} + nameIDs := make([]uint32, len(names)) - jobId := s.lss.GetLabelNameIDs([]string{"job"}) + s.lss.LabelNameToIDs(names, nameIDs) snap := s.lss.CreateLabelSetSnapshot() // Act - groupedSeries := snap.GroupSeriesByLabelNames([]uint32{idA0, idA1, idB}, jobId) + groupedSeries := snap.GroupSeriesByLabelNames([]uint32{idA0, idA1, idB}, nameIDs) // Assert s.Equal([][]uint32{{idA0, idA1}, {idB}}, groupedSeries.Groups) @@ -69,10 +71,12 @@ func (s *LabelSetSnapshotSuite) TestGroupSeriesByLabelNames_ByJobAndInstance() { idSame0 := s.lss.FindOrEmplace(model.NewLabelSetBuilder().Set("__name__", "m1").Set("job", "a").Set("instance", "i0").Build()).LabelSetID idOther := s.lss.FindOrEmplace(model.NewLabelSetBuilder().Set("__name__", "m2").Set("job", "a").Set("instance", "i1").Build()).LabelSetID idSame1 := s.lss.FindOrEmplace(model.NewLabelSetBuilder().Set("__name__", "m3").Set("job", "a").Set("instance", "i0").Build()).LabelSetID + names := []string{"job", "instance"} + nameIDs := make([]uint32, len(names)) - ids := s.lss.GetLabelNameIDs([]string{"job", "instance"}) - jobID := ids[0] - instanceID := ids[1] + s.lss.LabelNameToIDs(names, nameIDs) + jobID := nameIDs[0] + instanceID := nameIDs[1] snap := s.lss.CreateLabelSetSnapshot() diff --git a/pp/go/cppbridge/primitives_lss.go b/pp/go/cppbridge/primitives_lss.go index 745bef62d..a9373ea57 100644 --- a/pp/go/cppbridge/primitives_lss.go +++ b/pp/go/cppbridge/primitives_lss.go @@ -1,6 +1,7 @@ package cppbridge import ( + "fmt" "runtime" "unsafe" @@ -124,11 +125,14 @@ func (lss *LabelSetStorage) QueryLabelValues( return result } -// GetLabelNameIDs - returns label name ids -func (lss *LabelSetStorage) GetLabelNameIDs(names []string) []uint32 { - out := primitivesLSSGetLabelNameIDs(lss.pointer, names) +// LabelNameToIDs get label name ids from lss. +func (lss *LabelSetStorage) LabelNameToIDs(names []string, namesIDs []uint32) { + if len(names) != len(namesIDs) { + panic(fmt.Sprintf("names and namesIDs must have the same length: %d != %d", len(names), len(namesIDs))) + } + + primitivesLSSGetLabelNameIDs(lss.pointer, names, namesIDs) runtime.KeepAlive(lss) - return out } // GetLabelSets - returns copy of lss data. diff --git a/pp/go/cppbridge/primitives_lss_test.go b/pp/go/cppbridge/primitives_lss_test.go index feef51812..ae484f99c 100644 --- a/pp/go/cppbridge/primitives_lss_test.go +++ b/pp/go/cppbridge/primitives_lss_test.go @@ -362,12 +362,14 @@ func (s *QueryableLSSSuite) TestQueryLabelValues() { func (s *QueryableLSSSuite) TestGetLabelNameIDs() { // Arrange + names := []string{"lol", "foo", "nope", "lol"} + nameIDs := make([]uint32, len(names)) // Act - out := s.lss.GetLabelNameIDs([]string{"lol", "foo", "nope", "lol"}) + s.lss.LabelNameToIDs(names, nameIDs) // Assert - s.Equal([]uint32{0, 3, math.MaxUint32, 0}, out) + s.Equal([]uint32{0, 3, math.MaxUint32, 0}, nameIDs) } func (s *QueryableLSSSuite) testQueryLabelValuesImpl(testCase queryLabelValuesCase) { diff --git a/pp/go/storage/head/shard/lss.go b/pp/go/storage/head/shard/lss.go index d5ecbbf73..54b9dbabd 100644 --- a/pp/go/storage/head/shard/lss.go +++ b/pp/go/storage/head/shard/lss.go @@ -46,11 +46,27 @@ func (l *LSS) CopyAddedSeriesTo(destination *LSS) { destination.dstSrcLsIdsMapping = snapshot.CopyAddedSeries(bitsetSeries, destination.target) } +// GroupSeriesByLabelNames group series by label names. +func (l *LSS) GroupSeriesByLabelNames(seriesIDs, labelNameIDs []uint32) *cppbridge.SeriesGroups { + l.locker.RLock() + snapshot := l.getSnapshot() + l.locker.RUnlock() + + return snapshot.GroupSeriesByLabelNames(seriesIDs, labelNameIDs) +} + // Input returns input lss. func (l *LSS) Input() *cppbridge.LabelSetStorage { return l.input } +// LabelNameToIDs get label name ids from lss. +func (l *LSS) LabelNameToIDs(names []string, namesIDs []uint32) { + l.locker.RLock() + l.target.LabelNameToIDs(names, namesIDs) + l.locker.RUnlock() +} + // QueryLabelNames add to dedup all the unique label names present in lss in sorted order. func (l *LSS) QueryLabelNames( shardID uint16, diff --git a/pp/go/storage/querier/merge_series_set_test.go b/pp/go/storage/querier/merge_series_set_test.go index 883968aa1..6816b44a7 100644 --- a/pp/go/storage/querier/merge_series_set_test.go +++ b/pp/go/storage/querier/merge_series_set_test.go @@ -11,6 +11,7 @@ import ( "github.com/prometheus/prometheus/pp/go/storage/querier" "github.com/prometheus/prometheus/pp/go/storage/storagetest" "github.com/prometheus/prometheus/storage" + "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" ) @@ -286,8 +287,10 @@ func makeHead(numShards, numSeries, numSamples int) *testHead { func makeTimeSeries(numSeries, numSamples, shardID int) []storagetest.TimeSeries { timeSeries := make([]storagetest.TimeSeries, 0, numSeries) for j := range numSeries { + evenNumbered := j%2 == 0 ls := labels.FromStrings( "__name__", "metric", + "even_numbered", fmt.Sprintf("%t", evenNumbered), "foo", fmt.Sprintf("bar%d", j), "shard_id", fmt.Sprintf("id_%d", shardID), ) @@ -302,3 +305,34 @@ func makeTimeSeries(numSeries, numSamples, shardID int) []storagetest.TimeSeries return timeSeries } + +func TestAGGSS(t *testing.T) { + head := makeHead(2, 10, 5) + names := []string{ + "even_numbered", + "shard_id", + } + nameIDs := make([]uint32, len(names)) + + selector, snapshot, err := head.lsses[0].QuerySelector( + 0, + []model.LabelMatcher{{ + Name: "__name__", + Value: "metric", + MatcherType: model.MatcherTypeExactMatch, + }}, + ) + require.NoError(t, err) + + lssQueryResult := snapshot.Query(selector) + require.Equal(t, cppbridge.LSSQueryStatusMatch, lssQueryResult.Status()) + seriesIDs := lssQueryResult.IDs() + t.Log("seriesIDs", seriesIDs) + + t.Log("nameIDs 1", nameIDs) + head.lsses[0].LabelNameToIDs(names, nameIDs) + t.Log("nameIDs 2", nameIDs) + + seriesGroups := head.lsses[0].GroupSeriesByLabelNames(seriesIDs, nameIDs) + t.Log("seriesGroups", seriesGroups) +} From d926a614f322cc215184733911bc4751fe2a1597 Mon Sep 17 00:00:00 2001 From: Alexandr Yudin <57181751+u-veles-a@users.noreply.github.com> Date: Thu, 7 May 2026 12:32:31 +0000 Subject: [PATCH 03/37] WIP: fix test Signed-off-by: Alexandr Yudin <57181751+u-veles-a@users.noreply.github.com> --- .../storage/querier/merge_series_set_test.go | 58 +++++++++++++++++-- 1 file changed, 52 insertions(+), 6 deletions(-) diff --git a/pp/go/storage/querier/merge_series_set_test.go b/pp/go/storage/querier/merge_series_set_test.go index 339509c51..e0dd6d63e 100644 --- a/pp/go/storage/querier/merge_series_set_test.go +++ b/pp/go/storage/querier/merge_series_set_test.go @@ -77,8 +77,22 @@ func (s *MergeShardSeriesSetSuite) TestMergeShardSeriesSetScenarios() { esets := make([]storage.SeriesSet, 0, bm.numShards) asets := make([]storage.SeriesSet, 0, bm.numShards) for i := 0; i < bm.numShards; i++ { - esets = append(esets, queryOpt(s.T(), head.lsses[i], head.dss[i], start, end, matcher)) - asets = append(asets, queryOpt(s.T(), head.lsses[i], head.dss[i], start, end, matcher)) + esets = append(esets, queryOpt(s.T(), + head.lsses[i], + head.dss[i], + start, + end, + cppbridge.NoDownsampling, + matcher, + )) + asets = append(asets, queryOpt(s.T(), + head.lsses[i], + head.dss[i], + start, + end, + cppbridge.NoDownsampling, + matcher, + )) } assertMergeShardSeriesSetsEqual(s, esets, asets) }) @@ -91,8 +105,24 @@ func (s *MergeShardSeriesSetSuite) TestMergeShardSeriesSetScenarios() { esets = append(esets, &querier.SeriesSet{}) asets = append(asets, &querier.SeriesSet{}) } - esets = append(esets, queryOpt(s.T(), head.lsses[i], head.dss[i], start, end, matcher)) - asets = append(asets, queryOpt(s.T(), head.lsses[i], head.dss[i], start, end, matcher)) + esets = append(esets, queryOpt( + s.T(), + head.lsses[i], + head.dss[i], + start, + end, + cppbridge.NoDownsampling, + matcher, + )) + asets = append(asets, queryOpt( + s.T(), + head.lsses[i], + head.dss[i], + start, + end, + cppbridge.NoDownsampling, + matcher, + )) } assertMergeShardSeriesSetsEqual(s, esets, asets) }) @@ -101,10 +131,26 @@ func (s *MergeShardSeriesSetSuite) TestMergeShardSeriesSetScenarios() { esets := make([]storage.SeriesSet, 0, bm.numShards) asets := make([]storage.SeriesSet, 0, bm.numShards) for i := 0; i < bm.numShards; i++ { - esets = append(esets, queryOpt(s.T(), head.lsses[i], head.dss[i], start, end, matcher)) + esets = append(esets, queryOpt( + s.T(), + head.lsses[i], + head.dss[i], + start, + end, + cppbridge.NoDownsampling, + matcher, + )) } for i := bm.numShards - 1; i >= 0; i-- { - asets = append(asets, queryOpt(s.T(), head.lsses[i], head.dss[i], start, end, matcher)) + asets = append(asets, queryOpt( + s.T(), + head.lsses[i], + head.dss[i], + start, + end, + cppbridge.NoDownsampling, + matcher, + )) } assertMergeShardSeriesSetsEqual(s, esets, asets) }) From 58464d2edca0d0cbae6b71a2be1cc46dbee612db Mon Sep 17 00:00:00 2001 From: Alexandr Yudin <57181751+u-veles-a@users.noreply.github.com> Date: Fri, 8 May 2026 12:53:18 +0000 Subject: [PATCH 04/37] WIP: add AggSeriesSet Signed-off-by: Alexandr Yudin <57181751+u-veles-a@users.noreply.github.com> --- pp/go/cppbridge/entrypoint.h | 8 +- pp/go/cppbridge/head_test.go | 4 +- .../storage/head/poolprovider/poolprovider.go | 31 ++ pp/go/storage/head/transactionhead/head.go | 7 +- pp/go/storage/querier/agg_series_set.go | 369 ++++++++++++++++++ pp/go/storage/querier/interface.go | 9 + .../storage/querier/merge_series_set_test.go | 102 ++++- pp/go/storage/querier/querier.go | 105 ++++- pp/go/storage/querier/series.go | 3 +- .../decorator/window_function_iterator.h | 4 +- promql/engine.go | 4 + util/loopbackctx/loopbackctx.go | 24 ++ util/pool/slicepool.go | 68 ++++ util/pool/slicepool_test.go | 40 ++ 14 files changed, 754 insertions(+), 24 deletions(-) create mode 100644 pp/go/storage/querier/agg_series_set.go create mode 100644 util/loopbackctx/loopbackctx.go create mode 100644 util/pool/slicepool.go create mode 100644 util/pool/slicepool_test.go diff --git a/pp/go/cppbridge/entrypoint.h b/pp/go/cppbridge/entrypoint.h index 4a3172ac2..b83f93f83 100755 --- a/pp/go/cppbridge/entrypoint.h +++ b/pp/go/cppbridge/entrypoint.h @@ -1452,11 +1452,13 @@ void prompp_series_data_data_storage_allocated_memory(void* args, void* res); /** * @brief Queries data storage and serializes result (new serialization model). + * If args.downsamplingMs != 0 than DownsamplingIterator will be created regardless of the args.hints * * @param args { - * dataStorage uintptr // pointer to constructed data storage - * query DataStorageQuery // query - * downsamplingMs int64 // downsampling interval in milliseconds (0 - downsampling is disabled) + * dataStorage uintptr // pointer to constructed data storage + * query DataStorageQuery // query + * downsamplingMs int64 // downsampling interval in milliseconds (0 - downsampling is disabled) + * hints *storage.SelectHints // select hints * } * * @param res { diff --git a/pp/go/cppbridge/head_test.go b/pp/go/cppbridge/head_test.go index 52d0f86b6..5da2a1ef6 100644 --- a/pp/go/cppbridge/head_test.go +++ b/pp/go/cppbridge/head_test.go @@ -183,7 +183,7 @@ func (s *HeadSuite) TestInstantQuery() { // Arrange dataStorage := cppbridge.NewDataStorage() encoder := cppbridge.NewHeadEncoderWithDataStorage(dataStorage) - var series = []struct { + series := []struct { SeriesID uint32 cppbridge.Sample }{ @@ -320,7 +320,7 @@ func (s *DataStorageSerializedDataMultiSeriesIteratorSuite) TestSum() { Start: 0, End: 200, Step: 100, - Range: 100, + Range: 0, Func: "sum", }, []uint32{0, 1}, []uint32{0, 1}) diff --git a/pp/go/storage/head/poolprovider/poolprovider.go b/pp/go/storage/head/poolprovider/poolprovider.go index b913fc39f..5abc9e3cc 100644 --- a/pp/go/storage/head/poolprovider/poolprovider.go +++ b/pp/go/storage/head/poolprovider/poolprovider.go @@ -6,6 +6,7 @@ import ( "github.com/prometheus/prometheus/pp/go/cppbridge" "github.com/prometheus/prometheus/pp/go/storage/head/task" "github.com/prometheus/prometheus/storage" + "github.com/prometheus/prometheus/util/pool" "github.com/prometheus/prometheus/util/zeropool" ) @@ -40,9 +41,13 @@ type HeadPool[TGShard Shard] struct { chunkSeriesSetPool zeropool.Pool[[]storage.ChunkSeriesSet] serializedDataPool zeropool.Pool[[]*cppbridge.DataStorageSerializedData] errorsPool zeropool.Pool[[]error] + seriesGroupsPool zeropool.Pool[[]*cppbridge.SeriesGroups] + nameIDsPool pool.SlicePool[uint32] } // NewHeadPool init new [HeadPool], pools for reusable objects. +// +//revive:disable-next-line:function-length // this is constructor. func NewHeadPool[TGShard Shard](numberOfShards uint16) *HeadPool[TGShard] { return &HeadPool[TGShard]{ // used to reuse tasks @@ -92,6 +97,10 @@ func NewHeadPool[TGShard Shard](numberOfShards uint16) *HeadPool[TGShard] { errorsPool: zeropool.New(func() []error { return make([]error, numberOfShards) }), + seriesGroupsPool: zeropool.New(func() []*cppbridge.SeriesGroups { + return make([]*cppbridge.SeriesGroups, numberOfShards) + }), + nameIDsPool: pool.NewSlicePool[uint32]([]int{0, 1, 2, 3, 5}), } } @@ -225,3 +234,25 @@ func (hp *HeadPool[TGShard]) PutErrors(errs []error) { clear(errs) hp.errorsPool.Put(errs) } + +// GetSeriesGroups gets a slice of [cppbridge.SeriesGroups] from the pool. +func (hp *HeadPool[TGShard]) GetSeriesGroups() []*cppbridge.SeriesGroups { + return hp.seriesGroupsPool.Get() +} + +// PutSeriesGroups adds slice of [cppbridge.SeriesGroups] to the pool after resetting it. +func (hp *HeadPool[TGShard]) PutSeriesGroups(groups []*cppbridge.SeriesGroups) { + clear(groups) + hp.seriesGroupsPool.Put(groups) +} + +// GetNameIDs gets a slice of [uint32] from the pool. +func (hp *HeadPool[TGShard]) GetNameIDs(size int) []uint32 { + return hp.nameIDsPool.Get(size) +} + +// PutNameIDs adds slice of [uint32] to the pool after resetting it. +func (hp *HeadPool[TGShard]) PutNameIDs(nameIDs []uint32) { + clear(nameIDs) + hp.nameIDsPool.Put(nameIDs) +} diff --git a/pp/go/storage/head/transactionhead/head.go b/pp/go/storage/head/transactionhead/head.go index e0d9f7aa8..da30aebab 100644 --- a/pp/go/storage/head/transactionhead/head.go +++ b/pp/go/storage/head/transactionhead/head.go @@ -64,7 +64,7 @@ func NewHead[TShard Shard, TGShard Shard]( } // AcquireQuery implementation of the working [Head], no blocking. -func (*Head[TShard, TGShard]) AcquireQuery(ctx context.Context) (func(), error) { +func (*Head[TShard, TGShard]) AcquireQuery(context.Context) (func(), error) { return noopRelease, nil } @@ -98,6 +98,11 @@ func (*Head[TShard, TGShard]) Generation() uint64 { return 0 } +// ID returns the [Head] ID. +func (h *Head[TShard, TGShard]) ID() string { + return h.id +} + // NumberOfShards returns current number of shards in to [Head]. func (*Head[TShard, TGShard]) NumberOfShards() uint16 { return 1 diff --git a/pp/go/storage/querier/agg_series_set.go b/pp/go/storage/querier/agg_series_set.go new file mode 100644 index 000000000..3a770261b --- /dev/null +++ b/pp/go/storage/querier/agg_series_set.go @@ -0,0 +1,369 @@ +package querier + +import ( + "errors" + "fmt" + "slices" + "strconv" + + "github.com/prometheus/prometheus/model/histogram" + "github.com/prometheus/prometheus/model/labels" + "github.com/prometheus/prometheus/pp/go/cppbridge" + "github.com/prometheus/prometheus/storage" + "github.com/prometheus/prometheus/tsdb/chunkenc" + "github.com/prometheus/prometheus/util/annotations" +) + +// +// AggSeriesSet +// + +// AggSeriesSet contains a set of aggregated series. +type AggSeriesSet struct { + serializedData *cppbridge.DataStorageSerializedData + labelSetSnapshot *cppbridge.LabelSetSnapshot + seriesGroups *cppbridge.SeriesGroups + mint, maxt int64 + grouping []string + headID string + shardID uint16 + + // TODO: slice to + series []AggSeries + nextGroupIndex int +} + +// NewAggSeriesSet initializes a new [AggSeriesSet]. +func NewAggSeriesSet( + serializedData *cppbridge.DataStorageSerializedData, + labelSetSnapshot *cppbridge.LabelSetSnapshot, + seriesGroups *cppbridge.SeriesGroups, + mint, maxt int64, + grouping []string, + headID string, + shardID uint16, +) *AggSeriesSet { + return &AggSeriesSet{ + serializedData: serializedData, + labelSetSnapshot: labelSetSnapshot, + seriesGroups: seriesGroups, + mint: mint, + maxt: maxt, + grouping: grouping, + headID: headID, + shardID: shardID, + series: make([]AggSeries, 0, len(seriesGroups.Groups)), + } +} + +// At returns the current series. +// [storage.SeriesSet] interface implementation. +func (ss *AggSeriesSet) At() storage.Series { + return &ss.series[len(ss.series)-1] +} + +// Err returns the error of the [AggSeriesSet] - always nil. +// [storage.SeriesSet] interface implementation. +func (*AggSeriesSet) Err() error { + return nil +} + +// Next advances the iterator by one and returns false if there are no more values. +// [storage.SeriesSet] interface implementation. +func (ss *AggSeriesSet) Next() bool { + if ss.serializedData == nil { + return false + } + + if ss.nextGroupIndex >= len(ss.seriesGroups.Groups) { + return false + } + + builder := builderPool.Get().(*labels.ScratchBuilder) + builder.Reset() + ss.series = append(ss.series, NewAggSeries( + aggLabelSetCtor( + builder, + ss.labelSetSnapshot, + ss.grouping, + ss.headID, + ss.seriesGroups.Groups[ss.nextGroupIndex][0], // 0 is the first series ID + ss.shardID, + ), + ss.serializedData, + ss.seriesGroups.Groups[ss.nextGroupIndex], + ss.mint, + ss.maxt, + )) + builderPool.Put(builder) + ss.nextGroupIndex++ + + return true +} + +// Warnings returns the warnings of the [AggSeriesSet] - always nil. +// [storage.SeriesSet] interface implementation. +func (*AggSeriesSet) Warnings() annotations.Annotations { + return nil +} + +// +// AggSeries +// + +// AggSeries represents a time series with aggregated samples. +type AggSeries struct { + labelSet labels.Labels + serializedData *cppbridge.DataStorageSerializedData + seriesIDs []uint32 + mint, maxt int64 +} + +// NewAggSeries initializes a new [AggSeries]. +func NewAggSeries( + labelSet labels.Labels, + serializedData *cppbridge.DataStorageSerializedData, + seriesIDs []uint32, + mint, maxt int64, +) AggSeries { + return AggSeries{ + labelSet: labelSet, + serializedData: serializedData, + seriesIDs: seriesIDs, + mint: mint, + maxt: maxt, + } +} + +// Iterator returns an iterator that iterates over the aggregated samples of the [AggSeries]. +// [storage.Series] interface implementation. +func (s *AggSeries) Iterator(it chunkenc.Iterator) chunkenc.Iterator { + chunkIterator, ok := it.(*AggChunkIterator) + if !ok { + return NewAggChunkIterator( + s.serializedData, + s.seriesIDs, + s.mint, + s.maxt, + ) + } + + chunkIterator.reset(s.serializedData, s.seriesIDs, s.mint, s.maxt) + return chunkIterator +} + +// Labels returns the labels of the [AggSeries]. +// [storage.Series] interface implementation. +func (s *AggSeries) Labels() labels.Labels { + return s.labelSet +} + +// +// AggChunkIterator +// + +// AggChunkIterator iterates over the aggregated samples of a time series, that can only get the next value. +type AggChunkIterator struct { + serializedData *cppbridge.DataStorageSerializedData + chunkIterator cppbridge.DataStorageSerializedDataMultiSeriesIterator + mint int64 + maxt int64 + isInitialized bool +} + +// NewAggChunkIterator initializes a new [AggChunkIterator]. +func NewAggChunkIterator( + serializedData *cppbridge.DataStorageSerializedData, + seriesIDs []uint32, + mint, maxt int64, +) *AggChunkIterator { + it := &AggChunkIterator{ + serializedData: serializedData, + chunkIterator: cppbridge.NewDataStorageSerializedDataMultiSeriesIterator(serializedData, seriesIDs), + mint: mint, + maxt: maxt, + } + + if it.chunkIterator.Timestamp() < mint { + panic(fmt.Sprintf("NewAggChunkIterator: timestamp(%d) < mint(%d)", it.chunkIterator.Timestamp(), mint)) + // it.chunkIterator.Seek(mint) + } + + return it +} + +// At returns the current timestamp/value pair if the value is a float. +// [chunkenc.Iterator] interface implementation. +// +//nolint:gocritic // unnamedResult not need +func (it *AggChunkIterator) At() (int64, float64) { + return it.chunkIterator.Timestamp(), it.chunkIterator.Value() +} + +// AtFloatHistogram returns the current timestamp/value pair if the value is a histogram with floating-point counts. +// [chunkenc.Iterator] interface implementation. +func (*AggChunkIterator) AtFloatHistogram(*histogram.FloatHistogram) (int64, *histogram.FloatHistogram) { + return 0, nil +} + +// AtHistogram returns the current timestamp/value pair if the value is a histogram with integer counts. +// [chunkenc.Iterator] interface implementation. +func (*AggChunkIterator) AtHistogram(*histogram.Histogram) (int64, *histogram.Histogram) { + return 0, nil +} + +// AtT returns the current timestamp. +// [chunkenc.Iterator] interface implementation. +func (it *AggChunkIterator) AtT() int64 { + return it.chunkIterator.Timestamp() +} + +// Err returns the current error - always nil. +// [chunkenc.Iterator] interface implementation. +func (*AggChunkIterator) Err() error { + return nil +} + +// Next advances the iterator by one and returns the type of the value. +// [chunkenc.Iterator] interface implementation. +func (it *AggChunkIterator) Next() chunkenc.ValueType { + if it.nextValue() == chunkenc.ValNone { + return chunkenc.ValNone + } + + if it.AtT() > it.maxt { + return chunkenc.ValNone + } + + return chunkenc.ValFloat +} + +// Seek advances the iterator forward to the first sample with a timestamp equal or greater than t. +// [chunkenc.Iterator] interface implementation. +func (it *AggChunkIterator) Seek(t int64) chunkenc.ValueType { + // adjust lower limit. + if t < it.mint { + t = it.mint + } + + ts := it.AtT() + if !it.isInitialized || ts < t { + panic(fmt.Sprintf("Seek:timestamp(%d) < mint(%d)", ts, t)) + // it.chunkIterator.Seek(t) + // it.isInitialized = true + // if !it.chunkIterator.HasData() { + // return chunkenc.ValNone + // } + // ts = it.AtT() + } + + if ts > it.maxt { + return chunkenc.ValNone + } + + return chunkenc.ValFloat +} + +// nextValue advances the iterator by one and returns the type of the value. +func (it *AggChunkIterator) nextValue() chunkenc.ValueType { + if !it.isInitialized { + if !it.chunkIterator.HasData() { + return chunkenc.ValNone + } + + it.isInitialized = true + return chunkenc.ValFloat + } + + it.chunkIterator.Next() + if !it.chunkIterator.HasData() { + return chunkenc.ValNone + } + + return chunkenc.ValFloat +} + +// reset resets the iterator to the beginning of the serialized data. +func (it *AggChunkIterator) reset( + serializedData *cppbridge.DataStorageSerializedData, + seriesIDs []uint32, + mint, maxt int64, +) { + it.serializedData = serializedData + it.mint = mint + it.maxt = maxt + it.isInitialized = false + // it.chunkIterator.Reset(serializedData, chunkRef) + it.chunkIterator = cppbridge.NewDataStorageSerializedDataMultiSeriesIterator(serializedData, seriesIDs) + + if it.chunkIterator.Timestamp() < mint { + // it.chunkIterator.Seek(mint) + panic(fmt.Sprintf("reset: timestamp(%d) < mint(%d)", it.chunkIterator.Timestamp(), mint)) + } +} + +// +// aggLabelSetCtor +// + +const ( + // labelHeadID is the label name for the head ID. + labelHeadID = "__head_id" + + // labelShardID is the label name for the shard ID. + labelShardID = "__shard_id" +) + +// aggLabelSetCtor constructs the label set for an aggregated series. +func aggLabelSetCtor( + sb *labels.ScratchBuilder, + snapshot *cppbridge.LabelSetSnapshot, + grouping []string, + headID string, + seriesID uint32, + shardID uint16, +) labels.Labels { + sb.Add(labelHeadID, headID) + sb.Add(labelShardID, strconv.FormatUint(uint64(shardID), 10)) //revive:disable-line:add-constant it's base 10 + + if len(grouping) == 0 { + return sb.Labels() + } + + // grouping must be sorted + var sortedGrouping []string + if len(grouping) == 1 { + sortedGrouping = grouping + } else { + sortedGrouping = append(make([]string, 0, len(grouping)), grouping...) + slices.Sort(sortedGrouping) + } + + i := 0 + _ = snapshot.RangeLabelSet(seriesID, func(l cppbridge.Label) error { + if i >= len(sortedGrouping) { + // fast exit if the grouping labels is enough + return errors.New("grouping labels is enough") + } + + if l.Name > sortedGrouping[i] { + i++ + + if i >= len(sortedGrouping) { + // fast exit if the grouping labels is enough + return errors.New("grouping labels is enough") + } + } + + if l.Name == sortedGrouping[i] { + sb.Add(l.Name, l.Value) + i++ + } + + return nil + }) + + sb.Sort() + + return sb.Labels() +} diff --git a/pp/go/storage/querier/interface.go b/pp/go/storage/querier/interface.go index c822592ce..54452b813 100644 --- a/pp/go/storage/querier/interface.go +++ b/pp/go/storage/querier/interface.go @@ -66,6 +66,12 @@ type DataStorage interface { // LSS the minimum required [LSS] implementation. type LSS interface { + // GroupSeriesByLabelNames group series by label names. + GroupSeriesByLabelNames(seriesIDs, labelNameIDs []uint32) *cppbridge.SeriesGroups + + // LabelNameToIDs get label name ids from lss. + LabelNameToIDs(names []string, namesIDs []uint32) + // QueryLabelNames returns all the unique label names present in lss in sorted order. QueryLabelNames( shardID uint16, @@ -134,6 +140,9 @@ type Head[ // EnqueueOnShard the task to be executed on head on specific shard. EnqueueOnShard(t TTask, shardID uint16) + // ID returns the [Head] ID. + ID() string + // NumberOfShards returns current number of shards in to [Head]. NumberOfShards() uint16 diff --git a/pp/go/storage/querier/merge_series_set_test.go b/pp/go/storage/querier/merge_series_set_test.go index e0dd6d63e..2e5fb3da1 100644 --- a/pp/go/storage/querier/merge_series_set_test.go +++ b/pp/go/storage/querier/merge_series_set_test.go @@ -11,6 +11,7 @@ import ( "github.com/prometheus/prometheus/pp/go/storage/querier" "github.com/prometheus/prometheus/pp/go/storage/storagetest" "github.com/prometheus/prometheus/storage" + "github.com/prometheus/prometheus/tsdb/chunkenc" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" ) @@ -58,6 +59,7 @@ func TestMergeShardSeriesSetSuite(t *testing.T) { suite.Run(t, new(MergeShardSeriesSetSuite)) } +//revive:disable-next-line:cognitive-complexity // this is a test. func (s *MergeShardSeriesSetSuite) TestMergeShardSeriesSetScenarios() { var start int64 matcher := model.LabelMatcher{ @@ -259,16 +261,30 @@ func makeTimeSeries(numSeries, numSamples, shardID int) []storagetest.TimeSeries return timeSeries } +// +// +// + func TestAGGSS(t *testing.T) { - head := makeHead(2, 10, 5) - names := []string{ - "even_numbered", - "shard_id", + hints := &storage.SelectHints{ + Start: 0, + End: 6, + Step: 1, + Func: "sum", + Range: 1, + Grouping: []string{ + // "shard_id", + // "even_numbered", + "head_id", + }, + By: true, } - nameIDs := make([]uint32, len(names)) - selector, snapshot, err := head.lsses[0].QuerySelector( - 0, + shardID := uint16(0) + head := makeHead(2, 10, 5) + + selector, snapshot, err := head.lsses[shardID].QuerySelector( + shardID, []model.LabelMatcher{{ Name: "__name__", Value: "metric", @@ -282,10 +298,74 @@ func TestAGGSS(t *testing.T) { seriesIDs := lssQueryResult.IDs() t.Log("seriesIDs", seriesIDs) - t.Log("nameIDs 1", nameIDs) - head.lsses[0].LabelNameToIDs(names, nameIDs) - t.Log("nameIDs 2", nameIDs) + nameIDs := make([]uint32, len(hints.Grouping)) + t.Log("nameIDs empty", nameIDs) + head.lsses[0].LabelNameToIDs(hints.Grouping, nameIDs) + t.Log("nameIDs filled", nameIDs) seriesGroups := head.lsses[0].GroupSeriesByLabelNames(seriesIDs, nameIDs) - t.Log("seriesGroups", seriesGroups) + t.Log("seriesGroups", seriesGroups.Groups) + + // t.Log("createLabelSet", aggLabelSetCtor(snapshot, hints.Grouping, "headID", seriesGroups.Groups[0][0], shardID)) + // t.Log("createLabelSet", aggLabelSetCtor(snapshot, hints.Grouping, "headID", seriesGroups.Groups[1][0], shardID)) + + result := head.dss[shardID].Query( + cppbridge.DataStorageQuery{ + StartTimestampMs: hints.Start, + EndTimestampMs: hints.End, + LabelSetIDs: seriesIDs, + }, + cppbridge.NoDownsampling, + hints, + ) + + ss := querier.NewAggSeriesSet( + result.SerializedData, + snapshot, + seriesGroups, + hints.Start, + hints.End, + hints.Grouping, + "headID", + shardID, + ) + + var it chunkenc.Iterator + for ss.Next() { + s := ss.At() + t.Log(s.Labels()) + it = s.Iterator(it) + t.Log(it.Next()) + for it.Next() != chunkenc.ValNone { + ts, v := it.At() + t.Log(ts, v) + } + } +} + +type el struct { + a int + b string +} + +func (e *el) String() string { + return fmt.Sprintf("el{a: %d, b: %s}", e.a, e.b) +} + +type elem struct { + value el +} + +func (e *elem) Get() *el { + return &e.value +} + +func TestXxx(t *testing.T) { + e := elem{value: el{a: 1, b: "2"}} + el1 := e.Get() + t.Log(el1) + + e.value = el{a: 3, b: "4"} + el2 := e.Get() + t.Log(el1, el2) } diff --git a/pp/go/storage/querier/querier.go b/pp/go/storage/querier/querier.go index f16ae660f..f1a2c1f46 100644 --- a/pp/go/storage/querier/querier.go +++ b/pp/go/storage/querier/querier.go @@ -16,6 +16,7 @@ import ( "github.com/prometheus/prometheus/pp/go/util/locker" "github.com/prometheus/prometheus/storage" "github.com/prometheus/prometheus/util/annotations" + "github.com/prometheus/prometheus/util/loopbackctx" ) const ( @@ -28,6 +29,9 @@ const ( // lssLabelNamesQuerier name of task. lssLabelNamesQuerier = "lss_label_names_querier" + // lssGroupSeriesByLabelNames name of task. + lssGroupSeriesByLabelNames = "lss_group_series_by_label_names" + // dsQueryInstantQuerier name of task. dsQueryInstantQuerier = "data_storage_query_instant_querier" // dsQueryRangeQuerier name of task. @@ -37,6 +41,9 @@ const ( DefaultInstantQueryValueNotFoundTimestampValue int64 = 0 ) +// emptySeriesSet is an empty series set. +var emptySeriesSet = &SeriesSet{} + // // Querier // @@ -146,6 +153,7 @@ func (q *Querier[TTask, TDataStorage, TLSS, TShard, THead]) Select( if q.mint == q.maxt { return q.selectInstant(ctx, sortSeries, hints, matchers...) } + return q.selectRange(ctx, sortSeries, hints, matchers...) } @@ -163,7 +171,7 @@ func (q *Querier[TTask, TDataStorage, TLSS, TShard, THead]) selectInstant( release, err := q.head.AcquireQuery(ctx) if err != nil { if errors.Is(err, locker.ErrSemaphoreClosed) { - return &SeriesSet{} + return emptySeriesSet } logger.Warnf("[QUERIER]: select instant failed on the capture of the read lock query: %s", err) @@ -253,7 +261,7 @@ func (q *Querier[TTask, TDataStorage, TLSS, TShard, THead]) selectRange( release, err := q.head.AcquireQuery(ctx) if err != nil { if errors.Is(err, locker.ErrSemaphoreClosed) { - return &SeriesSet{} + return emptySeriesSet } logger.Warnf("[QUERIER]: select range failed on the capture of the read lock query: %s", err) @@ -280,10 +288,28 @@ func (q *Querier[TTask, TDataStorage, TLSS, TShard, THead]) selectRange( return storage.ErrSeriesSet(err) } + loopbackDuration := loopbackctx.LoopbackFromContext(ctx) + _ = loopbackDuration + shardedSerializedData := poolProvider.GetSerializedData() defer poolProvider.PutSerializedData(shardedSerializedData) queryDataStorage(dsQueryRangeQuerier, q.head, lssQueryResults, shardedSerializedData, q.mint, q.maxt, hints) + if isAggSeriesSet(hints) { + return q.makeAggSeriesSet(lssQueryResults, snapshots, shardedSerializedData, hints) + } + + return q.makeSeriesSet(lssQueryResults, snapshots, shardedSerializedData) +} + +// makeSeriesSet makes the series set. +func (q *Querier[TTask, TDataStorage, TLSS, TShard, THead]) makeSeriesSet( + lssQueryResults []*cppbridge.LSSQueryResult, + snapshots []*cppbridge.LabelSetSnapshot, + shardedSerializedData []*cppbridge.DataStorageSerializedData, +) storage.SeriesSet { + poolProvider := q.head.PoolProvider() + seriesSets := poolProvider.GetSeriesSet() defer poolProvider.PutSeriesSet(seriesSets) for shardID, serializedData := range shardedSerializedData { @@ -297,12 +323,85 @@ func (q *Querier[TTask, TDataStorage, TLSS, TShard, THead]) selectRange( ) continue } - seriesSets[shardID] = &SeriesSet{} + + seriesSets[shardID] = emptySeriesSet } return NewMergeShardSeriesSet(seriesSets) } +// makeAggSeriesSet queries the aggregated series set. +func (q *Querier[TTask, TDataStorage, TLSS, TShard, THead]) makeAggSeriesSet( + lssQueryResults []*cppbridge.LSSQueryResult, + snapshots []*cppbridge.LabelSetSnapshot, + shardedSerializedData []*cppbridge.DataStorageSerializedData, + hints *storage.SelectHints, +) storage.SeriesSet { + poolProvider := q.head.PoolProvider() + + seriesGroups := poolProvider.GetSeriesGroups() + defer poolProvider.PutSeriesGroups(seriesGroups) + t := q.head.CreateTask( + lssGroupSeriesByLabelNames, + func(shard TShard) error { + shardID := shard.ShardID() + lssQueryResult := lssQueryResults[shardID] + if lssQueryResult == nil { + return nil + } + + nameIDs := poolProvider.GetNameIDs(len(hints.Grouping)) + shard.LSS().LabelNameToIDs(hints.Grouping, nameIDs) + seriesGroups[shardID] = shard.LSS().GroupSeriesByLabelNames(lssQueryResults[shardID].IDs(), nameIDs) + poolProvider.PutNameIDs(nameIDs) + + return nil + }, + ) + defer q.head.PutTask(t) + q.head.Enqueue(t) + _ = t.Wait() + + seriesSets := poolProvider.GetSeriesSet() + defer poolProvider.PutSeriesSet(seriesSets) + for shardID, serializedData := range shardedSerializedData { + if serializedData == nil { + seriesSets[shardID] = emptySeriesSet + continue + } + + seriesSets[shardID] = NewAggSeriesSet( + serializedData, + snapshots[shardID], + seriesGroups[shardID], + q.mint, + q.maxt, + hints.Grouping, + q.head.ID(), + uint16(shardID), // #nosec G115 // no overflow + ) + } + + return NewMergeShardSeriesSet(seriesSets) +} + +// isAggSeriesSet checks if the series set is an aggregated series set. +func isAggSeriesSet(hints *storage.SelectHints) bool { + for _, group := range hints.Grouping { + if group == labelHeadID || group == labelShardID { + logger.Infof( + "[QUERIER]: isAggSeriesSet: head_id or shard_id is in the grouping, it will be ignored: %s", + group, + ) + return false + } + } + + return hints.Func == "sum" || + hints.Func == "max" || + hints.Func == "min" +} + // convertPrometheusMatchersToPPMatchers converts prometheus matchers to pp matchers. func convertPrometheusMatchersToPPMatchers(matchers ...*labels.Matcher) []model.LabelMatcher { promppMatchers := make([]model.LabelMatcher, len(matchers)) diff --git a/pp/go/storage/querier/series.go b/pp/go/storage/querier/series.go index 41bf9e894..c2abed521 100644 --- a/pp/go/storage/querier/series.go +++ b/pp/go/storage/querier/series.go @@ -182,8 +182,7 @@ type SeriesSet struct { labelSetSnapshot *cppbridge.LabelSetSnapshot serializedData *cppbridge.DataStorageSerializedData - lastIndexFromLSSQueryResult int - series []Series + series []Series } func NewSeriesSet( diff --git a/pp/series_data/decoder/decorator/window_function_iterator.h b/pp/series_data/decoder/decorator/window_function_iterator.h index 7bd2adf0b..7d5c47924 100644 --- a/pp/series_data/decoder/decorator/window_function_iterator.h +++ b/pp/series_data/decoder/decorator/window_function_iterator.h @@ -107,11 +107,11 @@ class WindowFunctionIterator { [[nodiscard]] PROMPP_ALWAYS_INLINE static Timestamp next_interval_boundary(Timestamp start, Timestamp step_ms, Timestamp range_ms) noexcept { if (range_ms <= step_ms) [[likely]] { - return start + range_ms; + return start + (range_ms == 0 ? step_ms : range_ms); } return start + range_ms - (range_ms - step_ms); } }; -} // namespace series_data::decoder::decorator \ No newline at end of file +} // namespace series_data::decoder::decorator diff --git a/promql/engine.go b/promql/engine.go index 95d3aa75d..f7d6c2aae 100644 --- a/promql/engine.go +++ b/promql/engine.go @@ -46,6 +46,7 @@ import ( "github.com/prometheus/prometheus/storage" "github.com/prometheus/prometheus/tsdb/chunkenc" "github.com/prometheus/prometheus/util/annotations" + "github.com/prometheus/prometheus/util/loopbackctx" "github.com/prometheus/prometheus/util/stats" "github.com/prometheus/prometheus/util/zeropool" ) @@ -714,6 +715,9 @@ func durationMilliseconds(d time.Duration) int64 { // execEvalStmt evaluates the expression of an evaluation statement for the given time range. func (ng *Engine) execEvalStmt(ctx context.Context, query *query, s *parser.EvalStmt) (parser.Value, annotations.Annotations, error) { + // add loopback duration to context for query evaluation from c++ + ctx = loopbackctx.ContextWithLoopback(ctx, s.LookbackDelta) + prepareSpanTimer, ctxPrepare := query.stats.GetSpanTimer(ctx, stats.QueryPreparationTime, ng.metrics.queryPrepareTime) mint, maxt := FindMinMaxTime(s) querier, err := query.queryable.Querier(mint, maxt) diff --git a/util/loopbackctx/loopbackctx.go b/util/loopbackctx/loopbackctx.go new file mode 100644 index 000000000..153e5e2d1 --- /dev/null +++ b/util/loopbackctx/loopbackctx.go @@ -0,0 +1,24 @@ +package loopbackctx + +import ( + "context" + "time" +) + +// loopbackParam key for loopback duration in context. +type loopbackParam struct{} + +// ContextWithLoopback append loopback duration to context. +func ContextWithLoopback(ctx context.Context, lookbackDelta time.Duration) context.Context { + return context.WithValue(ctx, loopbackParam{}, lookbackDelta) +} + +// LoopbackFromContext extract loopback duration from context. +func LoopbackFromContext(ctx context.Context) time.Duration { + lookbackDelta, ok := ctx.Value(loopbackParam{}).(time.Duration) + if !ok { + return time.Duration(0) + } + + return lookbackDelta +} diff --git a/util/pool/slicepool.go b/util/pool/slicepool.go new file mode 100644 index 000000000..7fe5439ce --- /dev/null +++ b/util/pool/slicepool.go @@ -0,0 +1,68 @@ +package pool + +import ( + "github.com/prometheus/prometheus/util/zeropool" +) + +// SlicePool is a pool of slices. +type SlicePool[T any] struct { + buckets []zeropool.Pool[[]T] + sizes []int +} + +// NewSlicePool creates a new [SlicePool]. +func NewSlicePool[T any](sizes []int) SlicePool[T] { + if len(sizes) == 0 { + panic("invalid sizes") + } + + for _, size := range sizes { + if size < 0 { + panic("invalid size") + } + } + + buckets := make([]zeropool.Pool[[]T], len(sizes)) + for i, size := range sizes { + buckets[i] = zeropool.New(func() []T { return make([]T, size) }) + } + + return SlicePool[T]{ + buckets: buckets, + sizes: sizes, + } +} + +// Get returns a new slice of the given size. +func (p *SlicePool[T]) Get(size int) []T { + if size < 0 { + panic("invalid size") + } + + for i, bktSize := range p.sizes { + if size > bktSize { + continue + } + + return p.buckets[i].Get()[:size] + } + + return make([]T, size) +} + +// Put adds a slice to the pool. +func (p *SlicePool[T]) Put(item []T) { + // If the item is larger than the largest size in the pool, don't put it back. + if cap(item) > p.sizes[len(p.sizes)-1] { + return + } + + for i, size := range p.sizes { + if cap(item) > size { + continue + } + + p.buckets[i].Put(item) + return + } +} diff --git a/util/pool/slicepool_test.go b/util/pool/slicepool_test.go new file mode 100644 index 000000000..eb7c7aec3 --- /dev/null +++ b/util/pool/slicepool_test.go @@ -0,0 +1,40 @@ +package pool_test + +import ( + "testing" + + "github.com/stretchr/testify/require" + + "github.com/prometheus/prometheus/util/pool" +) + +func TestSlicePool(t *testing.T) { + testPool := pool.NewSlicePool[int]([]int{0, 1, 2, 4}) + + cases := []struct { + size int + expectedCap int + }{ + { + size: 0, + expectedCap: 0, + }, + { + size: 2, + expectedCap: 2, + }, + { + size: 3, + expectedCap: 4, + }, + { + size: 5, + expectedCap: 5, + }, + } + for _, c := range cases { + ret := testPool.Get(c.size) + require.Equal(t, c.expectedCap, cap(ret)) + testPool.Put(ret) + } +} From 3ed9d23d5659f738bbdddd664b7c6d5274ca2626 Mon Sep 17 00:00:00 2001 From: Alexandr Yudin <57181751+u-veles-a@users.noreply.github.com> Date: Tue, 12 May 2026 08:19:15 +0000 Subject: [PATCH 05/37] WIP: add test, lookback Signed-off-by: Alexandr Yudin <57181751+u-veles-a@users.noreply.github.com> --- pp/go/storage/querier/agg_series_set.go | 20 +- pp/go/storage/querier/agg_series_set_test.go | 241 ++++++++++++++++++ .../storage/querier/merge_series_set_test.go | 111 -------- pp/go/storage/querier/querier.go | 4 - pp/go/storage/querier/series.go | 2 - promql/engine.go | 15 +- storage/interface.go | 6 +- util/loopbackctx/loopbackctx.go | 24 -- 8 files changed, 267 insertions(+), 156 deletions(-) create mode 100644 pp/go/storage/querier/agg_series_set_test.go delete mode 100644 util/loopbackctx/loopbackctx.go diff --git a/pp/go/storage/querier/agg_series_set.go b/pp/go/storage/querier/agg_series_set.go index 3a770261b..c409fa5f4 100644 --- a/pp/go/storage/querier/agg_series_set.go +++ b/pp/go/storage/querier/agg_series_set.go @@ -12,6 +12,7 @@ import ( "github.com/prometheus/prometheus/storage" "github.com/prometheus/prometheus/tsdb/chunkenc" "github.com/prometheus/prometheus/util/annotations" + "github.com/prometheus/prometheus/util/pool" ) // @@ -28,7 +29,6 @@ type AggSeriesSet struct { headID string shardID uint16 - // TODO: slice to series []AggSeries nextGroupIndex int } @@ -314,6 +314,14 @@ const ( labelShardID = "__shard_id" ) +var ( + // groupingPool is a pool of slices for sorted grouping. + groupingPool = pool.NewSlicePool[string]([]int{2, 3, 5}) + + // errGroupingLabelsIsEnough is the error returned when the grouping labels is enough. + errGroupingLabelsIsEnough = errors.New("grouping labels is enough") +) + // aggLabelSetCtor constructs the label set for an aggregated series. func aggLabelSetCtor( sb *labels.ScratchBuilder, @@ -335,7 +343,10 @@ func aggLabelSetCtor( if len(grouping) == 1 { sortedGrouping = grouping } else { - sortedGrouping = append(make([]string, 0, len(grouping)), grouping...) + sortedGrouping = groupingPool.Get(len(grouping)) + defer groupingPool.Put(sortedGrouping) + + copy(sortedGrouping, grouping) slices.Sort(sortedGrouping) } @@ -343,7 +354,7 @@ func aggLabelSetCtor( _ = snapshot.RangeLabelSet(seriesID, func(l cppbridge.Label) error { if i >= len(sortedGrouping) { // fast exit if the grouping labels is enough - return errors.New("grouping labels is enough") + return errGroupingLabelsIsEnough } if l.Name > sortedGrouping[i] { @@ -351,7 +362,7 @@ func aggLabelSetCtor( if i >= len(sortedGrouping) { // fast exit if the grouping labels is enough - return errors.New("grouping labels is enough") + return errGroupingLabelsIsEnough } } @@ -362,7 +373,6 @@ func aggLabelSetCtor( return nil }) - sb.Sort() return sb.Labels() diff --git a/pp/go/storage/querier/agg_series_set_test.go b/pp/go/storage/querier/agg_series_set_test.go new file mode 100644 index 000000000..6d186ce9a --- /dev/null +++ b/pp/go/storage/querier/agg_series_set_test.go @@ -0,0 +1,241 @@ +package querier_test + +import ( + "testing" + + "github.com/prometheus/prometheus/model/labels" + "github.com/prometheus/prometheus/pp/go/cppbridge" + "github.com/prometheus/prometheus/pp/go/model" + "github.com/prometheus/prometheus/pp/go/storage/head/shard" + "github.com/prometheus/prometheus/pp/go/storage/querier" + "github.com/prometheus/prometheus/pp/go/storage/storagetest" + "github.com/prometheus/prometheus/storage" + "github.com/prometheus/prometheus/tsdb/chunkenc" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" +) + +type AggSeriesSetSuite struct { + suite.Suite + + timeSeries []storagetest.TimeSeries + lss *shard.LSS + ds *shard.DataStorage +} + +func TestAggSeriesSetSuite(t *testing.T) { + suite.Run(t, new(AggSeriesSetSuite)) +} + +func (s *AggSeriesSetSuite) SetupTest() { + s.lss = shard.NewLSS() + s.ds = shard.NewDataStorage() + + s.timeSeries = []storagetest.TimeSeries{ + { + Labels: labels.FromStrings("__name__", "metric", "job", "test", "instance", "instance1"), + Samples: []cppbridge.Sample{ + {Timestamp: 10, Value: 1}, + }, + }, + { + Labels: labels.FromStrings("__name__", "metric", "job", "test", "instance", "instance1"), + Samples: []cppbridge.Sample{ + {Timestamp: 11, Value: 3}, + }, + }, + { + Labels: labels.FromStrings("__name__", "metric", "job", "test", "instance", "instance1"), + Samples: []cppbridge.Sample{ + {Timestamp: 12, Value: 5}, + }, + }, + { + Labels: labels.FromStrings("__name__", "metric", "job", "test", "instance", "instance1"), + Samples: []cppbridge.Sample{ + {Timestamp: 13, Value: 7}, + }, + }, + { + Labels: labels.FromStrings("__name__", "metric", "job", "test2", "instance", "instance2"), + Samples: []cppbridge.Sample{ + {Timestamp: 10, Value: 1}, + }, + }, + { + Labels: labels.FromStrings("__name__", "metric", "job", "test2", "instance", "instance2"), + Samples: []cppbridge.Sample{ + {Timestamp: 11, Value: 2}, + }, + }, + { + Labels: labels.FromStrings("__name__", "metric", "job", "test2", "instance", "instance2"), + Samples: []cppbridge.Sample{ + {Timestamp: 12, Value: 4}, + }, + }, + } +} + +func (s *AggSeriesSetSuite) query( + lss *shard.LSS, + ds *shard.DataStorage, + start, end, downsamplingMs int64, + hints *storage.SelectHints, + matchers ...model.LabelMatcher, +) *querier.AggSeriesSet { + selector, snapshot, err := lss.QuerySelector(0, matchers) + s.Require().NoError(err) + + if selector == 0 || snapshot == nil { + return &querier.AggSeriesSet{} + } + + lssQueryResult := snapshot.Query(selector) + if lssQueryResult.Status() == cppbridge.LSSQueryStatusNoMatch { + return &querier.AggSeriesSet{} + } + + dsQueryResult := ds.Query(cppbridge.DataStorageQuery{ + StartTimestampMs: start, + EndTimestampMs: end, + LabelSetIDs: lssQueryResult.IDs(), + }, downsamplingMs, hints) + + nameIDs := make([]uint32, len(hints.Grouping)) + lss.LabelNameToIDs(hints.Grouping, nameIDs) + seriesGroups := lss.GroupSeriesByLabelNames(lssQueryResult.IDs(), nameIDs) + + s.Require().Equal(cppbridge.DataStorageQueryStatusSuccess, dsQueryResult.Status) + return querier.NewAggSeriesSet( + dsQueryResult.SerializedData, + snapshot, + seriesGroups, + start, + end, + hints.Grouping, + "head_id", + 0, + ) +} + +func (s *AggSeriesSetSuite) TestQuerySum() { + // Arrange + matcher := model.LabelMatcher{ + Name: "__name__", + Value: "metric", + MatcherType: model.MatcherTypeExactMatch, + } + + var start int64 + var end int64 = 50 + hints := &storage.SelectHints{ + Start: 10, + End: 13, + Step: 1, + Func: "sum", + Range: 0, + } + + expected := []storagetest.TimeSeries{ + { + Labels: labels.FromStrings("__head_id", "head_id", "__shard_id", "0"), + Samples: []cppbridge.Sample{ + {Timestamp: 10, Value: 1}, + {Timestamp: 11, Value: 5}, + {Timestamp: 12, Value: 9}, + {Timestamp: 13, Value: 7}, + }, + }, + } + + storagetest.MustAppendTimeSeriesToLSSAndDataStorage(s.lss, s.ds, s.timeSeries...) + + // Act + seriesSet := s.query(s.lss, s.ds, start, end, cppbridge.NoDownsampling, hints, matcher) + + // Assert + s.Require().Equal(expected, storagetest.TimeSeriesFromSeriesSet(seriesSet, true)) +} + +// +// +// + +func TestAGGSS(t *testing.T) { + hints := &storage.SelectHints{ + Start: 0, + End: 6, + Step: 1, + Func: "sum", + Range: 1, + Grouping: []string{ + "shard_id", + "even_numbered", + "head_id", + }, + By: true, + } + + shardID := uint16(0) + head := makeHead(2, 10, 5) + + selector, snapshot, err := head.lsses[shardID].QuerySelector( + shardID, + []model.LabelMatcher{{ + Name: "__name__", + Value: "metric", + MatcherType: model.MatcherTypeExactMatch, + }}, + ) + require.NoError(t, err) + + lssQueryResult := snapshot.Query(selector) + require.Equal(t, cppbridge.LSSQueryStatusMatch, lssQueryResult.Status()) + seriesIDs := lssQueryResult.IDs() + t.Log("seriesIDs", seriesIDs) + + nameIDs := make([]uint32, len(hints.Grouping)) + t.Log("nameIDs empty", nameIDs) + head.lsses[0].LabelNameToIDs(hints.Grouping, nameIDs) + t.Log("nameIDs filled", nameIDs) + + seriesGroups := head.lsses[0].GroupSeriesByLabelNames(seriesIDs, nameIDs) + t.Log("seriesGroups", seriesGroups.Groups) + + // t.Log("createLabelSet", aggLabelSetCtor(snapshot, hints.Grouping, "headID", seriesGroups.Groups[0][0], shardID)) + // t.Log("createLabelSet", aggLabelSetCtor(snapshot, hints.Grouping, "headID", seriesGroups.Groups[1][0], shardID)) + + result := head.dss[shardID].Query( + cppbridge.DataStorageQuery{ + StartTimestampMs: hints.Start, + EndTimestampMs: hints.End, + LabelSetIDs: seriesIDs, + }, + cppbridge.NoDownsampling, + hints, + ) + + ss := querier.NewAggSeriesSet( + result.SerializedData, + snapshot, + seriesGroups, + hints.Start, + hints.End, + hints.Grouping, + "headID", + shardID, + ) + + var it chunkenc.Iterator + for ss.Next() { + s := ss.At() + t.Log(s.Labels()) + it = s.Iterator(it) + t.Log(it.Next()) + for it.Next() != chunkenc.ValNone { + ts, v := it.At() + t.Log(ts, v) + } + } +} diff --git a/pp/go/storage/querier/merge_series_set_test.go b/pp/go/storage/querier/merge_series_set_test.go index 2e5fb3da1..5652d069d 100644 --- a/pp/go/storage/querier/merge_series_set_test.go +++ b/pp/go/storage/querier/merge_series_set_test.go @@ -11,8 +11,6 @@ import ( "github.com/prometheus/prometheus/pp/go/storage/querier" "github.com/prometheus/prometheus/pp/go/storage/storagetest" "github.com/prometheus/prometheus/storage" - "github.com/prometheus/prometheus/tsdb/chunkenc" - "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" ) @@ -260,112 +258,3 @@ func makeTimeSeries(numSeries, numSamples, shardID int) []storagetest.TimeSeries return timeSeries } - -// -// -// - -func TestAGGSS(t *testing.T) { - hints := &storage.SelectHints{ - Start: 0, - End: 6, - Step: 1, - Func: "sum", - Range: 1, - Grouping: []string{ - // "shard_id", - // "even_numbered", - "head_id", - }, - By: true, - } - - shardID := uint16(0) - head := makeHead(2, 10, 5) - - selector, snapshot, err := head.lsses[shardID].QuerySelector( - shardID, - []model.LabelMatcher{{ - Name: "__name__", - Value: "metric", - MatcherType: model.MatcherTypeExactMatch, - }}, - ) - require.NoError(t, err) - - lssQueryResult := snapshot.Query(selector) - require.Equal(t, cppbridge.LSSQueryStatusMatch, lssQueryResult.Status()) - seriesIDs := lssQueryResult.IDs() - t.Log("seriesIDs", seriesIDs) - - nameIDs := make([]uint32, len(hints.Grouping)) - t.Log("nameIDs empty", nameIDs) - head.lsses[0].LabelNameToIDs(hints.Grouping, nameIDs) - t.Log("nameIDs filled", nameIDs) - - seriesGroups := head.lsses[0].GroupSeriesByLabelNames(seriesIDs, nameIDs) - t.Log("seriesGroups", seriesGroups.Groups) - - // t.Log("createLabelSet", aggLabelSetCtor(snapshot, hints.Grouping, "headID", seriesGroups.Groups[0][0], shardID)) - // t.Log("createLabelSet", aggLabelSetCtor(snapshot, hints.Grouping, "headID", seriesGroups.Groups[1][0], shardID)) - - result := head.dss[shardID].Query( - cppbridge.DataStorageQuery{ - StartTimestampMs: hints.Start, - EndTimestampMs: hints.End, - LabelSetIDs: seriesIDs, - }, - cppbridge.NoDownsampling, - hints, - ) - - ss := querier.NewAggSeriesSet( - result.SerializedData, - snapshot, - seriesGroups, - hints.Start, - hints.End, - hints.Grouping, - "headID", - shardID, - ) - - var it chunkenc.Iterator - for ss.Next() { - s := ss.At() - t.Log(s.Labels()) - it = s.Iterator(it) - t.Log(it.Next()) - for it.Next() != chunkenc.ValNone { - ts, v := it.At() - t.Log(ts, v) - } - } -} - -type el struct { - a int - b string -} - -func (e *el) String() string { - return fmt.Sprintf("el{a: %d, b: %s}", e.a, e.b) -} - -type elem struct { - value el -} - -func (e *elem) Get() *el { - return &e.value -} - -func TestXxx(t *testing.T) { - e := elem{value: el{a: 1, b: "2"}} - el1 := e.Get() - t.Log(el1) - - e.value = el{a: 3, b: "4"} - el2 := e.Get() - t.Log(el1, el2) -} diff --git a/pp/go/storage/querier/querier.go b/pp/go/storage/querier/querier.go index f1a2c1f46..03135fd15 100644 --- a/pp/go/storage/querier/querier.go +++ b/pp/go/storage/querier/querier.go @@ -16,7 +16,6 @@ import ( "github.com/prometheus/prometheus/pp/go/util/locker" "github.com/prometheus/prometheus/storage" "github.com/prometheus/prometheus/util/annotations" - "github.com/prometheus/prometheus/util/loopbackctx" ) const ( @@ -288,9 +287,6 @@ func (q *Querier[TTask, TDataStorage, TLSS, TShard, THead]) selectRange( return storage.ErrSeriesSet(err) } - loopbackDuration := loopbackctx.LoopbackFromContext(ctx) - _ = loopbackDuration - shardedSerializedData := poolProvider.GetSerializedData() defer poolProvider.PutSerializedData(shardedSerializedData) queryDataStorage(dsQueryRangeQuerier, q.head, lssQueryResults, shardedSerializedData, q.mint, q.maxt, hints) diff --git a/pp/go/storage/querier/series.go b/pp/go/storage/querier/series.go index c2abed521..3f4a7d2d4 100644 --- a/pp/go/storage/querier/series.go +++ b/pp/go/storage/querier/series.go @@ -178,7 +178,6 @@ func (s *Series) Iterator(it chunkenc.Iterator) chunkenc.Iterator { type SeriesSet struct { mint, maxt int64 - lssQueryResult *cppbridge.LSSQueryResult labelSetSnapshot *cppbridge.LabelSetSnapshot serializedData *cppbridge.DataStorageSerializedData @@ -194,7 +193,6 @@ func NewSeriesSet( return &SeriesSet{ mint: mint, maxt: maxt, - lssQueryResult: lssQueryResult, labelSetSnapshot: labelSetSnapshot, serializedData: serializedData, series: make([]Series, 0, lssQueryResult.Len()), diff --git a/promql/engine.go b/promql/engine.go index f7d6c2aae..cf5ee5bca 100644 --- a/promql/engine.go +++ b/promql/engine.go @@ -46,7 +46,6 @@ import ( "github.com/prometheus/prometheus/storage" "github.com/prometheus/prometheus/tsdb/chunkenc" "github.com/prometheus/prometheus/util/annotations" - "github.com/prometheus/prometheus/util/loopbackctx" "github.com/prometheus/prometheus/util/stats" "github.com/prometheus/prometheus/util/zeropool" ) @@ -715,9 +714,6 @@ func durationMilliseconds(d time.Duration) int64 { // execEvalStmt evaluates the expression of an evaluation statement for the given time range. func (ng *Engine) execEvalStmt(ctx context.Context, query *query, s *parser.EvalStmt) (parser.Value, annotations.Annotations, error) { - // add loopback duration to context for query evaluation from c++ - ctx = loopbackctx.ContextWithLoopback(ctx, s.LookbackDelta) - prepareSpanTimer, ctxPrepare := query.stats.GetSpanTimer(ctx, stats.QueryPreparationTime, ng.metrics.queryPrepareTime) mint, maxt := FindMinMaxTime(s) querier, err := query.queryable.Querier(mint, maxt) @@ -977,11 +973,12 @@ func (ng *Engine) populateSeries(ctx context.Context, querier storage.Querier, s interval = s.Interval } hints := &storage.SelectHints{ - Start: start, - End: end, - Step: durationMilliseconds(interval), - Range: durationMilliseconds(evalRange), - Func: extractFuncFromPath(path), + Start: start, + End: end, + Step: durationMilliseconds(interval), + Range: durationMilliseconds(evalRange), + Func: extractFuncFromPath(path), + LookbackDelta: durationMilliseconds(s.LookbackDelta), } evalRange = 0 hints.By, hints.Grouping = extractGroupsFromPath(path) diff --git a/storage/interface.go b/storage/interface.go index 2f125e590..1a849ccf8 100644 --- a/storage/interface.go +++ b/storage/interface.go @@ -197,7 +197,6 @@ type SelectHints struct { Func string // String representation of surrounding function or aggregation. Grouping []string // List of label names used in aggregation. - By bool // Indicate whether it is without or by. Range int64 // Range vector selector range in milliseconds. // ShardCount is the total number of shards that series should be split into @@ -214,10 +213,15 @@ type SelectHints struct { // Series are sharded by "labels stable hash" mod "ShardCount". ShardIndex uint64 + // LookbackDelta duration for the query in milliseconds. + LookbackDelta int64 + // DisableTrimming allows to disable trimming of matching series chunks based on query Start and End time. // When disabled, the result may contain samples outside the queried time range but Select() performances // may be improved. DisableTrimming bool + + By bool // Indicate whether it is without or by. } // LabelHints specifies hints passed for label reads. diff --git a/util/loopbackctx/loopbackctx.go b/util/loopbackctx/loopbackctx.go deleted file mode 100644 index 153e5e2d1..000000000 --- a/util/loopbackctx/loopbackctx.go +++ /dev/null @@ -1,24 +0,0 @@ -package loopbackctx - -import ( - "context" - "time" -) - -// loopbackParam key for loopback duration in context. -type loopbackParam struct{} - -// ContextWithLoopback append loopback duration to context. -func ContextWithLoopback(ctx context.Context, lookbackDelta time.Duration) context.Context { - return context.WithValue(ctx, loopbackParam{}, lookbackDelta) -} - -// LoopbackFromContext extract loopback duration from context. -func LoopbackFromContext(ctx context.Context) time.Duration { - lookbackDelta, ok := ctx.Value(loopbackParam{}).(time.Duration) - if !ok { - return time.Duration(0) - } - - return lookbackDelta -} From ec057cc4cfeae1ce6391d7e40949c446c52df5d3 Mon Sep 17 00:00:00 2001 From: Alexandr Yudin <57181751+u-veles-a@users.noreply.github.com> Date: Tue, 12 May 2026 11:07:17 +0000 Subject: [PATCH 06/37] WIP: add test Signed-off-by: Alexandr Yudin <57181751+u-veles-a@users.noreply.github.com> --- pp/go/storage/querier/agg_series_set.go | 2 + pp/go/storage/querier/agg_series_set_test.go | 203 +++++++++++++++---- 2 files changed, 168 insertions(+), 37 deletions(-) diff --git a/pp/go/storage/querier/agg_series_set.go b/pp/go/storage/querier/agg_series_set.go index c409fa5f4..b59b5ab46 100644 --- a/pp/go/storage/querier/agg_series_set.go +++ b/pp/go/storage/querier/agg_series_set.go @@ -20,6 +20,8 @@ import ( // // AggSeriesSet contains a set of aggregated series. +// If grouping is empty, it will return series with labels "__head_id" and "__shard_id". +// If grouping is not empty, it will return series with "__head_id" and "__shard_id" and the grouping labels. type AggSeriesSet struct { serializedData *cppbridge.DataStorageSerializedData labelSetSnapshot *cppbridge.LabelSetSnapshot diff --git a/pp/go/storage/querier/agg_series_set_test.go b/pp/go/storage/querier/agg_series_set_test.go index 6d186ce9a..0cebffac2 100644 --- a/pp/go/storage/querier/agg_series_set_test.go +++ b/pp/go/storage/querier/agg_series_set_test.go @@ -33,46 +33,32 @@ func (s *AggSeriesSetSuite) SetupTest() { s.timeSeries = []storagetest.TimeSeries{ { - Labels: labels.FromStrings("__name__", "metric", "job", "test", "instance", "instance1"), - Samples: []cppbridge.Sample{ - {Timestamp: 10, Value: 1}, - }, + Labels: labels.FromStrings("__name__", "metric", "job", "test", "instance", "instance1"), + Samples: []cppbridge.Sample{{Timestamp: 10, Value: 1}}, }, { - Labels: labels.FromStrings("__name__", "metric", "job", "test", "instance", "instance1"), - Samples: []cppbridge.Sample{ - {Timestamp: 11, Value: 3}, - }, + Labels: labels.FromStrings("__name__", "metric", "job", "test", "instance", "instance1"), + Samples: []cppbridge.Sample{{Timestamp: 11, Value: 3}}, }, { - Labels: labels.FromStrings("__name__", "metric", "job", "test", "instance", "instance1"), - Samples: []cppbridge.Sample{ - {Timestamp: 12, Value: 5}, - }, + Labels: labels.FromStrings("__name__", "metric", "job", "test", "instance", "instance1"), + Samples: []cppbridge.Sample{{Timestamp: 12, Value: 5}}, }, { - Labels: labels.FromStrings("__name__", "metric", "job", "test", "instance", "instance1"), - Samples: []cppbridge.Sample{ - {Timestamp: 13, Value: 7}, - }, + Labels: labels.FromStrings("__name__", "metric", "job", "test", "instance", "instance1"), + Samples: []cppbridge.Sample{{Timestamp: 13, Value: 7}}, }, { - Labels: labels.FromStrings("__name__", "metric", "job", "test2", "instance", "instance2"), - Samples: []cppbridge.Sample{ - {Timestamp: 10, Value: 1}, - }, + Labels: labels.FromStrings("__name__", "metric", "job", "test2", "instance", "instance2"), + Samples: []cppbridge.Sample{{Timestamp: 10, Value: 1}}, }, { - Labels: labels.FromStrings("__name__", "metric", "job", "test2", "instance", "instance2"), - Samples: []cppbridge.Sample{ - {Timestamp: 11, Value: 2}, - }, + Labels: labels.FromStrings("__name__", "metric", "job", "test2", "instance", "instance2"), + Samples: []cppbridge.Sample{{Timestamp: 11, Value: 2}}, }, { - Labels: labels.FromStrings("__name__", "metric", "job", "test2", "instance", "instance2"), - Samples: []cppbridge.Sample{ - {Timestamp: 12, Value: 4}, - }, + Labels: labels.FromStrings("__name__", "metric", "job", "test2", "instance", "instance2"), + Samples: []cppbridge.Sample{{Timestamp: 12, Value: 4}}, }, } } @@ -119,7 +105,7 @@ func (s *AggSeriesSetSuite) query( ) } -func (s *AggSeriesSetSuite) TestQuerySum() { +func (s *AggSeriesSetSuite) TestQueryWithoutGrouping() { // Arrange matcher := model.LabelMatcher{ Name: "__name__", @@ -139,13 +125,8 @@ func (s *AggSeriesSetSuite) TestQuerySum() { expected := []storagetest.TimeSeries{ { - Labels: labels.FromStrings("__head_id", "head_id", "__shard_id", "0"), - Samples: []cppbridge.Sample{ - {Timestamp: 10, Value: 1}, - {Timestamp: 11, Value: 5}, - {Timestamp: 12, Value: 9}, - {Timestamp: 13, Value: 7}, - }, + Labels: labels.FromStrings("__head_id", "head_id", "__shard_id", "0"), + Samples: []cppbridge.Sample{}, }, } @@ -155,7 +136,155 @@ func (s *AggSeriesSetSuite) TestQuerySum() { seriesSet := s.query(s.lss, s.ds, start, end, cppbridge.NoDownsampling, hints, matcher) // Assert - s.Require().Equal(expected, storagetest.TimeSeriesFromSeriesSet(seriesSet, true)) + actual := storagetest.TimeSeriesFromSeriesSet(seriesSet, true) + s.Require().Equal(len(expected), len(actual)) + s.Require().Equal(expected[0].Labels, actual[0].Labels) +} + +func (s *AggSeriesSetSuite) TestQueryGrouping_OneGroupingLabel() { + // Arrange + matcher := model.LabelMatcher{ + Name: "__name__", + Value: "metric", + MatcherType: model.MatcherTypeExactMatch, + } + + var start int64 + var end int64 = 50 + hints := &storage.SelectHints{ + Start: 10, + End: 13, + Step: 1, + Func: "sum", + Range: 0, + Grouping: []string{"job"}, + } + + expected := []storagetest.TimeSeries{ + { + Labels: labels.FromStrings("__head_id", "head_id", "__shard_id", "0", "job", "test"), + Samples: []cppbridge.Sample{}, + }, + { + Labels: labels.FromStrings("__head_id", "head_id", "__shard_id", "0", "job", "test2"), + Samples: []cppbridge.Sample{}, + }, + } + + storagetest.MustAppendTimeSeriesToLSSAndDataStorage(s.lss, s.ds, s.timeSeries...) + + // Act + seriesSet := s.query(s.lss, s.ds, start, end, cppbridge.NoDownsampling, hints, matcher) + + // Assert + actual := storagetest.TimeSeriesFromSeriesSet(seriesSet, true) + s.Require().Equal(len(expected), len(actual)) + s.Require().Equal(expected[0].Labels, actual[0].Labels) + s.Require().Equal(expected[1].Labels, actual[1].Labels) +} + +func (s *AggSeriesSetSuite) TestQueryGrouping_TwoGroupingLabels() { + // Arrange + matcher := model.LabelMatcher{ + Name: "__name__", + Value: "metric", + MatcherType: model.MatcherTypeExactMatch, + } + + var start int64 + var end int64 = 50 + hints := &storage.SelectHints{ + Start: 10, + End: 13, + Step: 1, + Func: "sum", + Range: 0, + Grouping: []string{"instance", "job"}, + } + + expected := []storagetest.TimeSeries{ + { + Labels: labels.FromStrings( + "__head_id", "head_id", + "__shard_id", "0", + "job", "test", + "instance", "instance1", + ), + Samples: []cppbridge.Sample{}, + }, + { + Labels: labels.FromStrings( + "__head_id", "head_id", + "__shard_id", "0", + "job", "test2", + "instance", "instance2", + ), + Samples: []cppbridge.Sample{}, + }, + } + + storagetest.MustAppendTimeSeriesToLSSAndDataStorage(s.lss, s.ds, s.timeSeries...) + + // Act + seriesSet := s.query(s.lss, s.ds, start, end, cppbridge.NoDownsampling, hints, matcher) + + // Assert + actual := storagetest.TimeSeriesFromSeriesSet(seriesSet, true) + s.Require().Equal(len(expected), len(actual)) + s.Require().Equal(expected[0].Labels, actual[0].Labels) + s.Require().Equal(expected[1].Labels, actual[1].Labels) +} + +func (s *AggSeriesSetSuite) TestQueryGrouping_TwoGroupingLabels_WithMissingGroupingLabel() { + // Arrange + matcher := model.LabelMatcher{ + Name: "__name__", + Value: "metric", + MatcherType: model.MatcherTypeExactMatch, + } + + var start int64 + var end int64 = 50 + hints := &storage.SelectHints{ + Start: 10, + End: 13, + Step: 1, + Func: "sum", + Range: 0, + Grouping: []string{"job", "instance", "head"}, + } + + expected := []storagetest.TimeSeries{ + { + Labels: labels.FromStrings( + "__head_id", "head_id", + "__shard_id", "0", + "job", "test", + "instance", "instance1", + ), + Samples: []cppbridge.Sample{}, + }, + { + Labels: labels.FromStrings( + "__head_id", "head_id", + "__shard_id", "0", + "job", "test2", + "instance", "instance2", + ), + Samples: []cppbridge.Sample{}, + }, + } + + storagetest.MustAppendTimeSeriesToLSSAndDataStorage(s.lss, s.ds, s.timeSeries...) + + // Act + seriesSet := s.query(s.lss, s.ds, start, end, cppbridge.NoDownsampling, hints, matcher) + + // Assert + actual := storagetest.TimeSeriesFromSeriesSet(seriesSet, true) + s.Require().Equal(len(expected), len(actual)) + s.Require().Equal(expected[0].Labels, actual[0].Labels) + s.Require().Equal(expected[1].Labels, actual[1].Labels) } // From 62c8c6fb09fd8b3d0947a8307cb81d8e04e33cd9 Mon Sep 17 00:00:00 2001 From: Alexandr Yudin <57181751+u-veles-a@users.noreply.github.com> Date: Tue, 12 May 2026 12:06:34 +0000 Subject: [PATCH 07/37] WIP: add PROMPP_FEATURES select_func_optimization Signed-off-by: Alexandr Yudin <57181751+u-veles-a@users.noreply.github.com> --- cmd/prometheus/main.go | 16 ++++++ pp/go/storage/querier/querier.go | 88 ++++++++++++++++++++++++++++---- 2 files changed, 94 insertions(+), 10 deletions(-) diff --git a/cmd/prometheus/main.go b/cmd/prometheus/main.go index 22e80a6df..374aa68ec 100644 --- a/cmd/prometheus/main.go +++ b/cmd/prometheus/main.go @@ -2238,6 +2238,22 @@ func readPromPPFeatures(logger log.Logger) { ) remotewriter.DefaultSampleAgeLimit = defaultSampleAgeLimit + + case "select_func_optimization": + opt, err := querier.ParseSelectFuncOpt(strings.TrimSpace(fvalue)) + if err != nil { + level.Error(logger).Log( + "msg", "[FEATURE] Error parsing select_func_optimization value", + "err", err, + ) + continue + } + + querier.SelectFuncOpt = opt + level.Info(logger).Log( + "msg", "[FEATURE] Select function optimization is set.", + "optimization", fvalue, + ) } } } diff --git a/pp/go/storage/querier/querier.go b/pp/go/storage/querier/querier.go index 03135fd15..e5aaebdf5 100644 --- a/pp/go/storage/querier/querier.go +++ b/pp/go/storage/querier/querier.go @@ -40,6 +40,44 @@ const ( DefaultInstantQueryValueNotFoundTimestampValue int64 = 0 ) +const ( + // NoneOpt is the option without any optimization. + NoneOpt uint8 = iota + + // AggrOpt is the option for aggregated functions optimization. + AggrOpt + + // CrossSeriesOpt is the option for cross series functions optimization. + CrossSeriesOpt + + // AllOpt is the option for aggregated and cross functions optimization. + AllOpt +) + +// ParseSelectFuncOpt parses the select func optimization option from string. +func ParseSelectFuncOpt(opt string) (uint8, error) { + switch opt { + case "none": + return NoneOpt, nil + case "aggr": + return AggrOpt, nil + case "cross": + return CrossSeriesOpt, nil + case "all": + return AllOpt, nil + default: + return 0, fmt.Errorf( + "invalid select func optimization option: '%s', valid options are: 'none', 'aggr', 'cross', 'all'", opt, + ) + } +} + +// SelectFuncOpt is the option for selecting functions optimization. +var SelectFuncOpt = NoneOpt + +// emptySelectHints is an empty select hints, it's used when no optimization is needed. +var emptySelectHints = &storage.SelectHints{} + // emptySeriesSet is an empty series set. var emptySeriesSet = &SeriesSet{} @@ -287,12 +325,13 @@ func (q *Querier[TTask, TDataStorage, TLSS, TShard, THead]) selectRange( return storage.ErrSeriesSet(err) } + hints = selectOpt(hints) shardedSerializedData := poolProvider.GetSerializedData() defer poolProvider.PutSerializedData(shardedSerializedData) queryDataStorage(dsQueryRangeQuerier, q.head, lssQueryResults, shardedSerializedData, q.mint, q.maxt, hints) - if isAggSeriesSet(hints) { - return q.makeAggSeriesSet(lssQueryResults, snapshots, shardedSerializedData, hints) + if isCrossSeriesSet(hints) { + return q.makeCrossSeriesSet(lssQueryResults, snapshots, shardedSerializedData, hints) } return q.makeSeriesSet(lssQueryResults, snapshots, shardedSerializedData) @@ -326,8 +365,8 @@ func (q *Querier[TTask, TDataStorage, TLSS, TShard, THead]) makeSeriesSet( return NewMergeShardSeriesSet(seriesSets) } -// makeAggSeriesSet queries the aggregated series set. -func (q *Querier[TTask, TDataStorage, TLSS, TShard, THead]) makeAggSeriesSet( +// makeCrossSeriesSet queries the cross series set. +func (q *Querier[TTask, TDataStorage, TLSS, TShard, THead]) makeCrossSeriesSet( lssQueryResults []*cppbridge.LSSQueryResult, snapshots []*cppbridge.LabelSetSnapshot, shardedSerializedData []*cppbridge.DataStorageSerializedData, @@ -381,21 +420,50 @@ func (q *Querier[TTask, TDataStorage, TLSS, TShard, THead]) makeAggSeriesSet( return NewMergeShardSeriesSet(seriesSets) } -// isAggSeriesSet checks if the series set is an aggregated series set. -func isAggSeriesSet(hints *storage.SelectHints) bool { +// selectOpt selects the optimization option. +func selectOpt(hints *storage.SelectHints) *storage.SelectHints { + switch SelectFuncOpt { + case NoneOpt: + return emptySelectHints + + case AggrOpt: + if isCrossSeriesFunc(hints.Func) { + return emptySelectHints + } + + return hints + + case CrossSeriesOpt: + if !isCrossSeriesFunc(hints.Func) { + return emptySelectHints + } + + return hints + } + + return hints +} + +// isCrossSeriesSet checks if the series set is an cross series set. +func isCrossSeriesSet(hints *storage.SelectHints) bool { for _, group := range hints.Grouping { if group == labelHeadID || group == labelShardID { logger.Infof( - "[QUERIER]: isAggSeriesSet: head_id or shard_id is in the grouping, it will be ignored: %s", + "[QUERIER]: isCrossSeriesSet: head_id or shard_id is in the grouping, it will be ignored: %s", group, ) return false } } - return hints.Func == "sum" || - hints.Func == "max" || - hints.Func == "min" + return isCrossSeriesFunc(hints.Func) +} + +// isCrossSeriesFunc checks if the function is a cross series function. +func isCrossSeriesFunc(funcName string) bool { + return funcName == "sum" || + funcName == "max" || + funcName == "min" } // convertPrometheusMatchersToPPMatchers converts prometheus matchers to pp matchers. From 74512853f23f602c32c25b2cca6983a9c240c27e Mon Sep 17 00:00:00 2001 From: Alexandr Yudin <57181751+u-veles-a@users.noreply.github.com> Date: Tue, 12 May 2026 12:21:26 +0000 Subject: [PATCH 08/37] WIP: fix naming Signed-off-by: Alexandr Yudin <57181751+u-veles-a@users.noreply.github.com> --- pp/go/storage/querier/querier.go | 40 ++++++++++++++++---------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/pp/go/storage/querier/querier.go b/pp/go/storage/querier/querier.go index e5aaebdf5..6165e9dd4 100644 --- a/pp/go/storage/querier/querier.go +++ b/pp/go/storage/querier/querier.go @@ -44,11 +44,11 @@ const ( // NoneOpt is the option without any optimization. NoneOpt uint8 = iota - // AggrOpt is the option for aggregated functions optimization. - AggrOpt + // AggrFuncOpt is the option for aggregated functions optimization. + AggrFuncOpt - // CrossSeriesOpt is the option for cross series functions optimization. - CrossSeriesOpt + // CrossSeriesFuncOpt is the option for cross series functions optimization. + CrossSeriesFuncOpt // AllOpt is the option for aggregated and cross functions optimization. AllOpt @@ -60,9 +60,9 @@ func ParseSelectFuncOpt(opt string) (uint8, error) { case "none": return NoneOpt, nil case "aggr": - return AggrOpt, nil + return AggrFuncOpt, nil case "cross": - return CrossSeriesOpt, nil + return CrossSeriesFuncOpt, nil case "all": return AllOpt, nil default: @@ -325,13 +325,13 @@ func (q *Querier[TTask, TDataStorage, TLSS, TShard, THead]) selectRange( return storage.ErrSeriesSet(err) } - hints = selectOpt(hints) + hints = selectFuncOpt(hints) shardedSerializedData := poolProvider.GetSerializedData() defer poolProvider.PutSerializedData(shardedSerializedData) queryDataStorage(dsQueryRangeQuerier, q.head, lssQueryResults, shardedSerializedData, q.mint, q.maxt, hints) - if isCrossSeriesSet(hints) { - return q.makeCrossSeriesSet(lssQueryResults, snapshots, shardedSerializedData, hints) + if isCrossSeriesFunc(hints.Func) && isAllowedGroupingForCrossSeriesFunc(hints.Grouping) { + return q.makeAggSeriesSet(lssQueryResults, snapshots, shardedSerializedData, hints) } return q.makeSeriesSet(lssQueryResults, snapshots, shardedSerializedData) @@ -365,8 +365,8 @@ func (q *Querier[TTask, TDataStorage, TLSS, TShard, THead]) makeSeriesSet( return NewMergeShardSeriesSet(seriesSets) } -// makeCrossSeriesSet queries the cross series set. -func (q *Querier[TTask, TDataStorage, TLSS, TShard, THead]) makeCrossSeriesSet( +// makeAggSeriesSet queries the aggregated cross series set. +func (q *Querier[TTask, TDataStorage, TLSS, TShard, THead]) makeAggSeriesSet( lssQueryResults []*cppbridge.LSSQueryResult, snapshots []*cppbridge.LabelSetSnapshot, shardedSerializedData []*cppbridge.DataStorageSerializedData, @@ -420,20 +420,20 @@ func (q *Querier[TTask, TDataStorage, TLSS, TShard, THead]) makeCrossSeriesSet( return NewMergeShardSeriesSet(seriesSets) } -// selectOpt selects the optimization option. -func selectOpt(hints *storage.SelectHints) *storage.SelectHints { +// selectFuncOpt selects the function optimization hints. +func selectFuncOpt(hints *storage.SelectHints) *storage.SelectHints { switch SelectFuncOpt { case NoneOpt: return emptySelectHints - case AggrOpt: + case AggrFuncOpt: if isCrossSeriesFunc(hints.Func) { return emptySelectHints } return hints - case CrossSeriesOpt: + case CrossSeriesFuncOpt: if !isCrossSeriesFunc(hints.Func) { return emptySelectHints } @@ -444,19 +444,19 @@ func selectOpt(hints *storage.SelectHints) *storage.SelectHints { return hints } -// isCrossSeriesSet checks if the series set is an cross series set. -func isCrossSeriesSet(hints *storage.SelectHints) bool { - for _, group := range hints.Grouping { +// isAllowedGroupingForCrossSeriesFunc checks if the series set is an cross series set. +func isAllowedGroupingForCrossSeriesFunc(grouping []string) bool { + for _, group := range grouping { if group == labelHeadID || group == labelShardID { logger.Infof( - "[QUERIER]: isCrossSeriesSet: head_id or shard_id is in the grouping, it will be ignored: %s", + "[QUERIER]: head_id or shard_id is in the grouping, it will be ignored: %s", group, ) return false } } - return isCrossSeriesFunc(hints.Func) + return true } // isCrossSeriesFunc checks if the function is a cross series function. From b92bfef9e944bd66b4d0059ca4db15a011166bd0 Mon Sep 17 00:00:00 2001 From: Alexandr Yudin <57181751+u-veles-a@users.noreply.github.com> Date: Wed, 13 May 2026 16:02:24 +0000 Subject: [PATCH 09/37] WIP: add work with stalenan_series_set, fix bugs Signed-off-by: Alexandr Yudin <57181751+u-veles-a@users.noreply.github.com> --- pp/entrypoint/series_data_data_storage.cpp | 22 +++-- pp/entrypoint/series_data_data_storage.h | 11 ++- pp/go/cppbridge/data_storage.go | 4 +- pp/go/cppbridge/entrypoint.go | 21 +++-- pp/go/cppbridge/entrypoint.h | 11 ++- pp/go/cppbridge/head_test.go | 44 ++++++++-- pp/go/storage/head/head/head.go | 8 +- .../storage/head/poolprovider/poolprovider.go | 46 ++++++++-- pp/go/storage/head/shard/data_storage.go | 4 +- pp/go/storage/querier/agg_series_set.go | 52 ++++++----- pp/go/storage/querier/agg_series_set_test.go | 86 ++++++++++++++++--- pp/go/storage/querier/interface.go | 4 + pp/go/storage/querier/merge_series_set.go | 32 +++++++ .../storage/querier/merge_series_set_test.go | 2 +- pp/go/storage/querier/querier.go | 64 ++++++++++++-- pp/go/storage/querier/stalenan_series_set.go | 10 --- pp/go/storage/storagetest/fixtures.go | 4 +- pp/prometheus/query.h | 11 ++- 18 files changed, 323 insertions(+), 113 deletions(-) diff --git a/pp/entrypoint/series_data_data_storage.cpp b/pp/entrypoint/series_data_data_storage.cpp index 3bf74e3cb..9b84a4b71 100644 --- a/pp/entrypoint/series_data_data_storage.cpp +++ b/pp/entrypoint/series_data_data_storage.cpp @@ -194,27 +194,31 @@ extern "C" void prompp_series_data_data_storage_query_final(void* args) { } } -extern "C" void prompp_series_data_data_storage_query_first_timestamps(void* args, void* res) { +extern "C" void prompp_series_data_data_storage_query_first_timestamps(void* args) { using PromPP::Primitives::Timestamp; using series_data::Decoder; struct Arguments { DataStoragePtr data_storage; + Timestamp not_found_timestamp_value; SliceView series_ids; - }; - - struct Result { Slice timestamps; }; const auto in = static_cast(args); - const auto out = static_cast(res); - assert(in->series_ids.size() == out->timestamps.size()); + assert(in->series_ids.size() == in->timestamps.size()); const auto& storage = *in->data_storage; - std::ranges::transform(in->series_ids, out->timestamps.begin(), - [&storage](uint32_t series_id) { return Decoder::get_series_min_timestamp(storage, series_id); }); + std::ranges::transform(in->series_ids, in->timestamps.begin(), [&storage, in](uint32_t series_id) { + if (storage.open_chunks.size() > series_id) [[likely]] { + if (!storage.open_chunks[series_id].is_empty()) [[likely]] { + return Decoder::get_series_min_timestamp(storage, series_id); + } + } + + return in->not_found_timestamp_value; + }); } extern "C" void prompp_series_data_data_storage_allocated_memory(void* args, void* res) { @@ -472,4 +476,4 @@ extern "C" void prompp_series_data_data_storage_loader_dtor(void* args) { }; static_cast(args)->~Arguments(); -} \ No newline at end of file +} diff --git a/pp/entrypoint/series_data_data_storage.h b/pp/entrypoint/series_data_data_storage.h index 044ebbc7b..fcbbee619 100644 --- a/pp/entrypoint/series_data_data_storage.h +++ b/pp/entrypoint/series_data_data_storage.h @@ -132,14 +132,13 @@ void prompp_series_data_data_storage_instant_query(void* args, void* res); * @brief Get the first sample timestamp per series * * @param args { - * dataStorage uintptr // pointer to constructed data storage - * seriesIds []uint32 // series ids - * } - * @param res { - * timestamps []int64 // same length as seriesIds; filled from storage + * dataStorage uintptr // pointer to constructed data storage + * notFoundTimestampValue int64 // timestamp value to return if series is not found in storage + * seriesIds []uint32 // series ids + * timestamps []int64 // same length as seriesIds; filled from storage * } */ -void prompp_series_data_data_storage_query_first_timestamps(void* args, void* res); +void prompp_series_data_data_storage_query_first_timestamps(void* args); /** * @brief finishes all Queriers after data load. diff --git a/pp/go/cppbridge/data_storage.go b/pp/go/cppbridge/data_storage.go index 159e901f6..0193c7792 100644 --- a/pp/go/cppbridge/data_storage.go +++ b/pp/go/cppbridge/data_storage.go @@ -121,8 +121,8 @@ func (ds *DataStorage) InstantQuery(targetTimestamp int64, labelSetIDs []uint32, } // QueryFirstTimestamps fills timestamps with the first sample timestamp (Prometheus ms) for each series in seriesIDs -func (ds *DataStorage) QueryFirstTimestamps(seriesIDs []uint32, timestamps []int64) { - seriesDataDataStorageQueryFirstTimestamps(ds.dataStorage, seriesIDs, timestamps) +func (ds *DataStorage) QueryFirstTimestamps(seriesIDs []uint32, timestamps []int64, notFoundTimestampValue int64) { + seriesDataDataStorageQueryFirstTimestamps(ds.dataStorage, notFoundTimestampValue, seriesIDs, timestamps) runtime.KeepAlive(ds) } diff --git a/pp/go/cppbridge/entrypoint.go b/pp/go/cppbridge/entrypoint.go index 1411423ec..5a3a0a46b 100644 --- a/pp/go/cppbridge/entrypoint.go +++ b/pp/go/cppbridge/entrypoint.go @@ -2122,20 +2122,23 @@ func seriesDataDataStorageInstantQuery(dataStorage uintptr, labelSetIDs []uint32 return res } -func seriesDataDataStorageQueryFirstTimestamps(dataStorage uintptr, seriesIDs []uint32, timestamps []int64) { +func seriesDataDataStorageQueryFirstTimestamps( + dataStorage uintptr, + notFoundTimestampValue int64, + seriesIDs []uint32, + timestamps []int64, +) { args := struct { - dataStorage uintptr - seriesIDs []uint32 - }{dataStorage, seriesIDs} - res := struct { - timestamps []int64 - }{timestamps} + dataStorage uintptr + notFoundTimestampValue int64 + seriesIDs []uint32 + timestamps []int64 + }{dataStorage, notFoundTimestampValue, seriesIDs, timestamps} testGC() - fastcgo.UnsafeCall2( + fastcgo.UnsafeCall1( C.prompp_series_data_data_storage_query_first_timestamps, uintptr(unsafe.Pointer(&args)), - uintptr(unsafe.Pointer(&res)), ) } diff --git a/pp/go/cppbridge/entrypoint.h b/pp/go/cppbridge/entrypoint.h index b83f93f83..0ab330ccf 100755 --- a/pp/go/cppbridge/entrypoint.h +++ b/pp/go/cppbridge/entrypoint.h @@ -1490,14 +1490,13 @@ void prompp_series_data_data_storage_instant_query(void* args, void* res); * @brief Get the first sample timestamp per series * * @param args { - * dataStorage uintptr // pointer to constructed data storage - * seriesIds []uint32 // series ids - * } - * @param res { - * timestamps []int64 // same length as seriesIds; filled from storage + * dataStorage uintptr // pointer to constructed data storage + * notFoundTimestampValue int64 // timestamp value to return if series is not found in storage + * seriesIds []uint32 // series ids + * timestamps []int64 // same length as seriesIds; filled from storage * } */ -void prompp_series_data_data_storage_query_first_timestamps(void* args, void* res); +void prompp_series_data_data_storage_query_first_timestamps(void* args); /** * @brief finishes all Queriers after data load. diff --git a/pp/go/cppbridge/head_test.go b/pp/go/cppbridge/head_test.go index 5da2a1ef6..e0032e929 100644 --- a/pp/go/cppbridge/head_test.go +++ b/pp/go/cppbridge/head_test.go @@ -7,7 +7,6 @@ import ( "github.com/prometheus/prometheus/pp/go/storage/querier" "github.com/prometheus/prometheus/storage" - "github.com/stretchr/testify/require" "github.com/prometheus/prometheus/pp/go/cppbridge" "github.com/prometheus/prometheus/pp/go/model" @@ -213,19 +212,19 @@ func (s *HeadSuite) TestInstantQuery() { result := dataStorage.InstantQuery(targetTimestamp, seriesIDs, uintptr(unsafe.Pointer(unsafe.SliceData(instantSeries)))) // Assert - require.Equal(s.T(), cppbridge.DataStorageQueryStatusSuccess, result.Status) + s.Require().Equal(cppbridge.DataStorageQueryStatusSuccess, result.Status) s.Equal(defaultTimestamp, instantSeries[0].Timestamp) - s.Equal(series[2].Sample, cppbridge.Sample{Timestamp: instantSeries[1].Timestamp, Value: instantSeries[1].Value}) - s.Equal(series[5].Sample, cppbridge.Sample{Timestamp: instantSeries[2].Timestamp, Value: instantSeries[2].Value}) - s.Equal(series[6].Sample, cppbridge.Sample{Timestamp: instantSeries[3].Timestamp, Value: instantSeries[3].Value}) + s.Equal(cppbridge.Sample{Timestamp: instantSeries[1].Timestamp, Value: instantSeries[1].Value}, series[2].Sample) + s.Equal(cppbridge.Sample{Timestamp: instantSeries[2].Timestamp, Value: instantSeries[2].Value}, series[5].Sample) + s.Equal(cppbridge.Sample{Timestamp: instantSeries[3].Timestamp, Value: instantSeries[3].Value}, series[6].Sample) } func (s *HeadSuite) TestQueryFirstTimestampsWithEmptySeriesIds() { // Arrange // Act - s.dataStorage.QueryFirstTimestamps(nil, nil) + s.dataStorage.QueryFirstTimestamps(nil, nil, 0) // Assert } @@ -242,7 +241,7 @@ func (s *HeadSuite) TestQueryFirstTimestamps() { // Act timestamps := make([]int64, 2) - s.dataStorage.QueryFirstTimestamps([]uint32{1, 0}, timestamps) + s.dataStorage.QueryFirstTimestamps([]uint32{1, 0}, timestamps, 0) // Assert s.Equal([]int64{2, 5}, timestamps) @@ -259,12 +258,41 @@ func (s *HeadSuite) TestQueryFirstTimestampsInFinalizedChunk() { // Act timestamps := make([]int64, 1) - s.dataStorage.QueryFirstTimestamps([]uint32{0}, timestamps) + s.dataStorage.QueryFirstTimestamps([]uint32{0}, timestamps, 0) // Assert s.Equal([]int64{5}, timestamps) } +func (s *HeadSuite) TestQueryFirstTimestampsRotatedLSS() { + // Arrange + s.lss.FindOrEmplace(model.NewLabelSetBuilder().Set("job", "1").Build()) + s.lss.FindOrEmplace(model.NewLabelSetBuilder().Set("job", "2").Build()) + + // Act + timestamps := make([]int64, 2) + s.dataStorage.QueryFirstTimestamps([]uint32{1, 0}, timestamps, -1) + + // Assert + s.Equal([]int64{-1, -1}, timestamps) +} + +func (s *HeadSuite) TestQueryFirstTimestampsRotatedLSSWithEmptySeries() { + // Arrange + s.lss.FindOrEmplace(model.NewLabelSetBuilder().Set("job", "1").Build()) + s.lss.FindOrEmplace(model.NewLabelSetBuilder().Set("job", "2").Build()) + + s.encoder.Encode(1, 5, 1.0) + s.encoder.Encode(1, 9, 1.0) + + // Act + timestamps := make([]int64, 2) + s.dataStorage.QueryFirstTimestamps([]uint32{1, 0}, timestamps, -1) + + // Assert + s.Equal([]int64{5, -1}, timestamps) +} + type DataStorageSerializedDataMultiSeriesIteratorSuite struct { suite.Suite lss *cppbridge.LabelSetStorage diff --git a/pp/go/storage/head/head/head.go b/pp/go/storage/head/head/head.go index dbadb80ce..57fe78d60 100644 --- a/pp/go/storage/head/head/head.go +++ b/pp/go/storage/head/head/head.go @@ -66,8 +66,6 @@ type Head[TShard Shard, TGShard Shard] struct { // for tasks metrics tasksCreated *prometheus.CounterVec tasksDone *prometheus.CounterVec - tasksLive *prometheus.CounterVec - tasksExecute *prometheus.CounterVec // pools for reusable objects headPool *poolprovider.HeadPool[TGShard] @@ -89,9 +87,9 @@ func NewHead[TShard Shard, TGShard Shard]( concurrency := calculateHeadConcurrency(numberOfShards) // current head workers concurrency for shardID := range numberOfShards { // append and query can create 2 tasks per request, so minimal length of channel is - // cap(querySemaphore)*2+cap(appendSemaphore)*2 = 2*concurrency*2+2*concurrency*2 = 8*concurrency - // add extra slots to channel for safety = x9 for back pressure - taskChs[shardID] = make(chan *task.Generic[TGShard], 9*concurrency) + // cap(querySemaphore)*4+cap(appendSemaphore)*2 = 2*concurrency*4+2*concurrency*2 = 12*concurrency + // add extra slots to channel for safety = x13 for back pressure + taskChs[shardID] = make(chan *task.Generic[TGShard], 13*concurrency) } factory := util.NewUnconflictRegisterer(registerer) diff --git a/pp/go/storage/head/poolprovider/poolprovider.go b/pp/go/storage/head/poolprovider/poolprovider.go index 5abc9e3cc..e98186d9e 100644 --- a/pp/go/storage/head/poolprovider/poolprovider.go +++ b/pp/go/storage/head/poolprovider/poolprovider.go @@ -34,15 +34,17 @@ type HeadPool[TGShard Shard] struct { shardedInnerSeriesPool sync.Pool statsPool zeropool.Pool[[]cppbridge.RelabelerStats] // use in querier - snapshotsPool zeropool.Pool[[]*cppbridge.LabelSetSnapshot] - lssQueryResultsPool zeropool.Pool[[]*cppbridge.LSSQueryResult] - selectorsPool zeropool.Pool[[]uintptr] - seriesSetPool zeropool.Pool[[]storage.SeriesSet] - chunkSeriesSetPool zeropool.Pool[[]storage.ChunkSeriesSet] - serializedDataPool zeropool.Pool[[]*cppbridge.DataStorageSerializedData] - errorsPool zeropool.Pool[[]error] - seriesGroupsPool zeropool.Pool[[]*cppbridge.SeriesGroups] - nameIDsPool pool.SlicePool[uint32] + snapshotsPool zeropool.Pool[[]*cppbridge.LabelSetSnapshot] + lssQueryResultsPool zeropool.Pool[[]*cppbridge.LSSQueryResult] + selectorsPool zeropool.Pool[[]uintptr] + seriesSetPool zeropool.Pool[[]storage.SeriesSet] + chunkSeriesSetPool zeropool.Pool[[]storage.ChunkSeriesSet] + serializedDataPool zeropool.Pool[[]*cppbridge.DataStorageSerializedData] + errorsPool zeropool.Pool[[]error] + sliceOfTimestampsPool zeropool.Pool[[][]int64] + timestampsPool pool.SlicePool[int64] + seriesGroupsPool zeropool.Pool[[]*cppbridge.SeriesGroups] + nameIDsPool pool.SlicePool[uint32] } // NewHeadPool init new [HeadPool], pools for reusable objects. @@ -97,6 +99,10 @@ func NewHeadPool[TGShard Shard](numberOfShards uint16) *HeadPool[TGShard] { errorsPool: zeropool.New(func() []error { return make([]error, numberOfShards) }), + sliceOfTimestampsPool: zeropool.New(func() [][]int64 { + return make([][]int64, numberOfShards) + }), + timestampsPool: pool.NewSlicePool[int64]([]int{2, 4, 8, 16, 32, 64, 128, 256, 512, 1024}), seriesGroupsPool: zeropool.New(func() []*cppbridge.SeriesGroups { return make([]*cppbridge.SeriesGroups, numberOfShards) }), @@ -235,6 +241,28 @@ func (hp *HeadPool[TGShard]) PutErrors(errs []error) { hp.errorsPool.Put(errs) } +// GetSliceOfTimestamps gets a slice of []int64 from the pool. +func (hp *HeadPool[TGShard]) GetSliceOfTimestamps() [][]int64 { + return hp.sliceOfTimestampsPool.Get() +} + +// PutSliceOfTimestamps adds slice of []int64 to the pool after resetting it. +func (hp *HeadPool[TGShard]) PutSliceOfTimestamps(ts [][]int64) { + clear(ts) + hp.sliceOfTimestampsPool.Put(ts) +} + +// GetTimestamps gets a slice of [int64] from the pool. +func (hp *HeadPool[TGShard]) GetTimestamps(size int) []int64 { + return hp.timestampsPool.Get(size) +} + +// PutTimestamps adds slice of [int64] to the pool after resetting it. +func (hp *HeadPool[TGShard]) PutTimestamps(ts []int64) { + clear(ts) + hp.timestampsPool.Put(ts) +} + // GetSeriesGroups gets a slice of [cppbridge.SeriesGroups] from the pool. func (hp *HeadPool[TGShard]) GetSeriesGroups() []*cppbridge.SeriesGroups { return hp.seriesGroupsPool.Get() diff --git a/pp/go/storage/head/shard/data_storage.go b/pp/go/storage/head/shard/data_storage.go index ddc681844..bf1bf706d 100644 --- a/pp/go/storage/head/shard/data_storage.go +++ b/pp/go/storage/head/shard/data_storage.go @@ -90,9 +90,9 @@ func (ds *DataStorage) QueryFinal(queriers []uintptr) { } // QueryFirstTimestamps fills timestamps with the first sample timestamp (Prometheus ms) for each series in seriesIDs. -func (ds *DataStorage) QueryFirstTimestamps(ids []uint32, timestamps []int64) { +func (ds *DataStorage) QueryFirstTimestamps(ids []uint32, timestamps []int64, notFoundTimestampValue int64) { ds.locker.RLock() - ds.dataStorage.QueryFirstTimestamps(ids, timestamps) + ds.dataStorage.QueryFirstTimestamps(ids, timestamps, notFoundTimestampValue) ds.locker.RUnlock() } diff --git a/pp/go/storage/querier/agg_series_set.go b/pp/go/storage/querier/agg_series_set.go index b59b5ab46..176294b74 100644 --- a/pp/go/storage/querier/agg_series_set.go +++ b/pp/go/storage/querier/agg_series_set.go @@ -93,7 +93,8 @@ func (ss *AggSeriesSet) Next() bool { ss.shardID, ), ss.serializedData, - ss.seriesGroups.Groups[ss.nextGroupIndex], + ss.seriesGroups, + ss.nextGroupIndex, ss.mint, ss.maxt, )) @@ -117,7 +118,8 @@ func (*AggSeriesSet) Warnings() annotations.Annotations { type AggSeries struct { labelSet labels.Labels serializedData *cppbridge.DataStorageSerializedData - seriesIDs []uint32 + seriesGroups *cppbridge.SeriesGroups + groupIndex int mint, maxt int64 } @@ -125,13 +127,15 @@ type AggSeries struct { func NewAggSeries( labelSet labels.Labels, serializedData *cppbridge.DataStorageSerializedData, - seriesIDs []uint32, + seriesGroups *cppbridge.SeriesGroups, + groupIndex int, mint, maxt int64, ) AggSeries { return AggSeries{ labelSet: labelSet, serializedData: serializedData, - seriesIDs: seriesIDs, + seriesGroups: seriesGroups, + groupIndex: groupIndex, mint: mint, maxt: maxt, } @@ -144,13 +148,14 @@ func (s *AggSeries) Iterator(it chunkenc.Iterator) chunkenc.Iterator { if !ok { return NewAggChunkIterator( s.serializedData, - s.seriesIDs, + s.seriesGroups, + s.groupIndex, s.mint, s.maxt, ) } - chunkIterator.reset(s.serializedData, s.seriesIDs, s.mint, s.maxt) + chunkIterator.reset(s.serializedData, s.seriesGroups, s.groupIndex, s.mint, s.maxt) return chunkIterator } @@ -167,6 +172,7 @@ func (s *AggSeries) Labels() labels.Labels { // AggChunkIterator iterates over the aggregated samples of a time series, that can only get the next value. type AggChunkIterator struct { serializedData *cppbridge.DataStorageSerializedData + seriesGroups *cppbridge.SeriesGroups chunkIterator cppbridge.DataStorageSerializedDataMultiSeriesIterator mint int64 maxt int64 @@ -176,19 +182,19 @@ type AggChunkIterator struct { // NewAggChunkIterator initializes a new [AggChunkIterator]. func NewAggChunkIterator( serializedData *cppbridge.DataStorageSerializedData, - seriesIDs []uint32, + seriesGroups *cppbridge.SeriesGroups, + groupIndex int, mint, maxt int64, ) *AggChunkIterator { it := &AggChunkIterator{ serializedData: serializedData, - chunkIterator: cppbridge.NewDataStorageSerializedDataMultiSeriesIterator(serializedData, seriesIDs), - mint: mint, - maxt: maxt, - } - - if it.chunkIterator.Timestamp() < mint { - panic(fmt.Sprintf("NewAggChunkIterator: timestamp(%d) < mint(%d)", it.chunkIterator.Timestamp(), mint)) - // it.chunkIterator.Seek(mint) + seriesGroups: seriesGroups, + chunkIterator: cppbridge.NewDataStorageSerializedDataMultiSeriesIterator( + serializedData, + seriesGroups.Groups[groupIndex], + ), + mint: mint, + maxt: maxt, } return it @@ -250,7 +256,7 @@ func (it *AggChunkIterator) Seek(t int64) chunkenc.ValueType { ts := it.AtT() if !it.isInitialized || ts < t { - panic(fmt.Sprintf("Seek:timestamp(%d) < mint(%d)", ts, t)) + panic(fmt.Sprintf("Seek: timestamp(%d) < mint(%d)", ts, t)) // it.chunkIterator.Seek(t) // it.isInitialized = true // if !it.chunkIterator.HasData() { @@ -288,20 +294,20 @@ func (it *AggChunkIterator) nextValue() chunkenc.ValueType { // reset resets the iterator to the beginning of the serialized data. func (it *AggChunkIterator) reset( serializedData *cppbridge.DataStorageSerializedData, - seriesIDs []uint32, + seriesGroups *cppbridge.SeriesGroups, + groupIndex int, mint, maxt int64, ) { it.serializedData = serializedData + it.seriesGroups = seriesGroups it.mint = mint it.maxt = maxt it.isInitialized = false // it.chunkIterator.Reset(serializedData, chunkRef) - it.chunkIterator = cppbridge.NewDataStorageSerializedDataMultiSeriesIterator(serializedData, seriesIDs) - - if it.chunkIterator.Timestamp() < mint { - // it.chunkIterator.Seek(mint) - panic(fmt.Sprintf("reset: timestamp(%d) < mint(%d)", it.chunkIterator.Timestamp(), mint)) - } + it.chunkIterator = cppbridge.NewDataStorageSerializedDataMultiSeriesIterator( + serializedData, + seriesGroups.Groups[groupIndex], + ) } // diff --git a/pp/go/storage/querier/agg_series_set_test.go b/pp/go/storage/querier/agg_series_set_test.go index 0cebffac2..9981c28c9 100644 --- a/pp/go/storage/querier/agg_series_set_test.go +++ b/pp/go/storage/querier/agg_series_set_test.go @@ -1,9 +1,11 @@ package querier_test import ( + "math" "testing" "github.com/prometheus/prometheus/model/labels" + "github.com/prometheus/prometheus/model/value" "github.com/prometheus/prometheus/pp/go/cppbridge" "github.com/prometheus/prometheus/pp/go/model" "github.com/prometheus/prometheus/pp/go/storage/head/shard" @@ -69,7 +71,7 @@ func (s *AggSeriesSetSuite) query( start, end, downsamplingMs int64, hints *storage.SelectHints, matchers ...model.LabelMatcher, -) *querier.AggSeriesSet { +) storage.SeriesSet { selector, snapshot, err := lss.QuerySelector(0, matchers) s.Require().NoError(err) @@ -82,6 +84,17 @@ func (s *AggSeriesSetSuite) query( return &querier.AggSeriesSet{} } + valueNotFoundTimestampValue := int64(0) + timestamps := make([]int64, lssQueryResult.Len()) + ds.QueryFirstTimestamps(lssQueryResult.IDs(), timestamps, 0) + + sNaNSS := querier.NewStaleNaNSeriesSet( + querier.NewStaleNaNSeriesSliceFromTimestamps(timestamps), + lssQueryResult, + snapshot, + valueNotFoundTimestampValue, + ) + dsQueryResult := ds.Query(cppbridge.DataStorageQuery{ StartTimestampMs: start, EndTimestampMs: end, @@ -93,7 +106,8 @@ func (s *AggSeriesSetSuite) query( seriesGroups := lss.GroupSeriesByLabelNames(lssQueryResult.IDs(), nameIDs) s.Require().Equal(cppbridge.DataStorageQueryStatusSuccess, dsQueryResult.Status) - return querier.NewAggSeriesSet( + + aggSS := querier.NewAggSeriesSet( dsQueryResult.SerializedData, snapshot, seriesGroups, @@ -103,6 +117,8 @@ func (s *AggSeriesSetSuite) query( "head_id", 0, ) + + return querier.NewMergeShardSeriesSet([]storage.SeriesSet{sNaNSS, aggSS}) } func (s *AggSeriesSetSuite) TestQueryWithoutGrouping() { @@ -128,6 +144,14 @@ func (s *AggSeriesSetSuite) TestQueryWithoutGrouping() { Labels: labels.FromStrings("__head_id", "head_id", "__shard_id", "0"), Samples: []cppbridge.Sample{}, }, + { + Labels: labels.FromStrings("__name__", "metric", "job", "test", "instance", "instance1"), + Samples: []cppbridge.Sample{{Timestamp: 10, Value: math.Float64frombits(value.StaleNaN)}}, + }, + { + Labels: labels.FromStrings("__name__", "metric", "job", "test2", "instance", "instance2"), + Samples: []cppbridge.Sample{{Timestamp: 10, Value: math.Float64frombits(value.StaleNaN)}}, + }, } storagetest.MustAppendTimeSeriesToLSSAndDataStorage(s.lss, s.ds, s.timeSeries...) @@ -139,6 +163,8 @@ func (s *AggSeriesSetSuite) TestQueryWithoutGrouping() { actual := storagetest.TimeSeriesFromSeriesSet(seriesSet, true) s.Require().Equal(len(expected), len(actual)) s.Require().Equal(expected[0].Labels, actual[0].Labels) + s.Require().Equal(expected[1].Labels, actual[1].Labels) + s.Require().Equal(expected[2].Labels, actual[2].Labels) } func (s *AggSeriesSetSuite) TestQueryGrouping_OneGroupingLabel() { @@ -169,6 +195,14 @@ func (s *AggSeriesSetSuite) TestQueryGrouping_OneGroupingLabel() { Labels: labels.FromStrings("__head_id", "head_id", "__shard_id", "0", "job", "test2"), Samples: []cppbridge.Sample{}, }, + { + Labels: labels.FromStrings("__name__", "metric", "job", "test", "instance", "instance1"), + Samples: []cppbridge.Sample{{Timestamp: 10, Value: math.Float64frombits(value.StaleNaN)}}, + }, + { + Labels: labels.FromStrings("__name__", "metric", "job", "test2", "instance", "instance2"), + Samples: []cppbridge.Sample{{Timestamp: 10, Value: math.Float64frombits(value.StaleNaN)}}, + }, } storagetest.MustAppendTimeSeriesToLSSAndDataStorage(s.lss, s.ds, s.timeSeries...) @@ -181,6 +215,8 @@ func (s *AggSeriesSetSuite) TestQueryGrouping_OneGroupingLabel() { s.Require().Equal(len(expected), len(actual)) s.Require().Equal(expected[0].Labels, actual[0].Labels) s.Require().Equal(expected[1].Labels, actual[1].Labels) + s.Require().Equal(expected[2].Labels, actual[2].Labels) + s.Require().Equal(expected[3].Labels, actual[3].Labels) } func (s *AggSeriesSetSuite) TestQueryGrouping_TwoGroupingLabels() { @@ -221,6 +257,14 @@ func (s *AggSeriesSetSuite) TestQueryGrouping_TwoGroupingLabels() { ), Samples: []cppbridge.Sample{}, }, + { + Labels: labels.FromStrings("__name__", "metric", "job", "test", "instance", "instance1"), + Samples: []cppbridge.Sample{{Timestamp: 10, Value: math.Float64frombits(value.StaleNaN)}}, + }, + { + Labels: labels.FromStrings("__name__", "metric", "job", "test2", "instance", "instance2"), + Samples: []cppbridge.Sample{{Timestamp: 10, Value: math.Float64frombits(value.StaleNaN)}}, + }, } storagetest.MustAppendTimeSeriesToLSSAndDataStorage(s.lss, s.ds, s.timeSeries...) @@ -233,6 +277,8 @@ func (s *AggSeriesSetSuite) TestQueryGrouping_TwoGroupingLabels() { s.Require().Equal(len(expected), len(actual)) s.Require().Equal(expected[0].Labels, actual[0].Labels) s.Require().Equal(expected[1].Labels, actual[1].Labels) + s.Require().Equal(expected[2].Labels, actual[2].Labels) + s.Require().Equal(expected[3].Labels, actual[3].Labels) } func (s *AggSeriesSetSuite) TestQueryGrouping_TwoGroupingLabels_WithMissingGroupingLabel() { @@ -273,6 +319,14 @@ func (s *AggSeriesSetSuite) TestQueryGrouping_TwoGroupingLabels_WithMissingGroup ), Samples: []cppbridge.Sample{}, }, + { + Labels: labels.FromStrings("__name__", "metric", "job", "test", "instance", "instance1"), + Samples: []cppbridge.Sample{{Timestamp: 10, Value: math.Float64frombits(value.StaleNaN)}}, + }, + { + Labels: labels.FromStrings("__name__", "metric", "job", "test2", "instance", "instance2"), + Samples: []cppbridge.Sample{{Timestamp: 10, Value: math.Float64frombits(value.StaleNaN)}}, + }, } storagetest.MustAppendTimeSeriesToLSSAndDataStorage(s.lss, s.ds, s.timeSeries...) @@ -288,7 +342,7 @@ func (s *AggSeriesSetSuite) TestQueryGrouping_TwoGroupingLabels_WithMissingGroup } // -// +// TODO DELETE // func TestAGGSS(t *testing.T) { @@ -329,11 +383,20 @@ func TestAGGSS(t *testing.T) { head.lsses[0].LabelNameToIDs(hints.Grouping, nameIDs) t.Log("nameIDs filled", nameIDs) - seriesGroups := head.lsses[0].GroupSeriesByLabelNames(seriesIDs, nameIDs) + seriesGroups := head.lsses[shardID].GroupSeriesByLabelNames(seriesIDs, nameIDs) t.Log("seriesGroups", seriesGroups.Groups) - // t.Log("createLabelSet", aggLabelSetCtor(snapshot, hints.Grouping, "headID", seriesGroups.Groups[0][0], shardID)) - // t.Log("createLabelSet", aggLabelSetCtor(snapshot, hints.Grouping, "headID", seriesGroups.Groups[1][0], shardID)) + valueNotFoundTimestampValue := int64(0) + timestamps := make([]int64, lssQueryResult.Len()) + head.dss[shardID].QueryFirstTimestamps(lssQueryResult.IDs(), timestamps, 0) + t.Log("timestamps", timestamps) + + sNaNSS := querier.NewStaleNaNSeriesSet( + querier.NewStaleNaNSeriesSliceFromTimestamps(timestamps), + lssQueryResult, + snapshot, + valueNotFoundTimestampValue, + ) result := head.dss[shardID].Query( cppbridge.DataStorageQuery{ @@ -345,7 +408,7 @@ func TestAGGSS(t *testing.T) { hints, ) - ss := querier.NewAggSeriesSet( + aggSS := querier.NewAggSeriesSet( result.SerializedData, snapshot, seriesGroups, @@ -356,15 +419,18 @@ func TestAGGSS(t *testing.T) { shardID, ) + ss := querier.NewMergeShardSeriesSet([]storage.SeriesSet{sNaNSS, aggSS}) + var it chunkenc.Iterator for ss.Next() { s := ss.At() - t.Log(s.Labels()) + t.Log("s.Labels()", s.Labels()) it = s.Iterator(it) - t.Log(it.Next()) + t.Log("it.Next()", it.Next()) + t.Log(it.At()) for it.Next() != chunkenc.ValNone { ts, v := it.At() - t.Log(ts, v) + t.Log("ts", ts, "v", v) } } } diff --git a/pp/go/storage/querier/interface.go b/pp/go/storage/querier/interface.go index 54452b813..03b5d3039 100644 --- a/pp/go/storage/querier/interface.go +++ b/pp/go/storage/querier/interface.go @@ -56,6 +56,10 @@ type DataStorage interface { hints *storage.SelectHints, ) cppbridge.DataStorageQueryResult + // QueryFirstTimestamps fills timestamps with the first sample + // timestamp (Prometheus ms) for each series in seriesIDs. + QueryFirstTimestamps(ids []uint32, timestamps []int64, notFoundTimestampValue int64) + // WithRLock calls fn on raw [cppbridge.DataStorage] with read lock. WithRLock(fn func(ds *cppbridge.DataStorage) error) error } diff --git a/pp/go/storage/querier/merge_series_set.go b/pp/go/storage/querier/merge_series_set.go index 6fe49f7dc..48695335a 100644 --- a/pp/go/storage/querier/merge_series_set.go +++ b/pp/go/storage/querier/merge_series_set.go @@ -40,6 +40,38 @@ func NewMergeShardSeriesSet(sets []storage.SeriesSet) storage.SeriesSet { return s } +// NewMergeManyShardSeriesSets merges many [storage.SeriesSet] together from different shards. +// The implementation assumes only our shard-local sets are passed in: +// - always no nil sets; +// - always no Err()/Warnings() - because shards should not have any errors and warnings; +// - always sorted output sets; +// - always no identical label sets across shards. +func NewMergeManyShardSeriesSets(ssets ...[]storage.SeriesSet) storage.SeriesSet { + if len(ssets) == 0 { + return emptySeriesSet + } + + length := 0 + for _, sets := range ssets { + length += len(sets) + } + + s := &mergeShardSeriesSet{ + heap: make(seriesSetHeap, 0, length), + } + + for _, sets := range ssets { + for _, set := range sets { + // shard series don't have errors and not nil, so we can safely call Next + if set.Next() { + heap.Push(&s.heap, set) + } + } + } + + return s +} + // At returns the current [storage.Series], implement [storage.SeriesSet] interface. func (s *mergeShardSeriesSet) At() storage.Series { return s.currentSet.At() diff --git a/pp/go/storage/querier/merge_series_set_test.go b/pp/go/storage/querier/merge_series_set_test.go index 5652d069d..7b4a2ff68 100644 --- a/pp/go/storage/querier/merge_series_set_test.go +++ b/pp/go/storage/querier/merge_series_set_test.go @@ -250,7 +250,7 @@ func makeTimeSeries(numSeries, numSamples, shardID int) []storagetest.TimeSeries samples := make([]cppbridge.Sample, 0, numSamples) for k := range numSamples { - samples = append(samples, cppbridge.Sample{Timestamp: int64(k), Value: float64(k)}) + samples = append(samples, cppbridge.Sample{Timestamp: int64(k + 1), Value: float64(k)}) } timeSeries = append(timeSeries, storagetest.TimeSeries{Labels: ls, Samples: samples}) diff --git a/pp/go/storage/querier/querier.go b/pp/go/storage/querier/querier.go index 6165e9dd4..afdff222f 100644 --- a/pp/go/storage/querier/querier.go +++ b/pp/go/storage/querier/querier.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "math" "sort" "time" "unsafe" @@ -35,9 +36,14 @@ const ( dsQueryInstantQuerier = "data_storage_query_instant_querier" // dsQueryRangeQuerier name of task. dsQueryRangeQuerier = "data_storage_query_range_querier" + // dsQueryFirstTimestampsQuerier name of task. + dsQueryFirstTimestampsQuerier = "data_storage_query_first_timestamps_querier" - // DefaultInstantQueryValueNotFoundTimestampValue default value for not found timestamp value. + // DefaultInstantQueryValueNotFoundTimestampValue default value for not found timestamp value for instant query. DefaultInstantQueryValueNotFoundTimestampValue int64 = 0 + + // DefaultNotFoundTimestampValue default value for not found timestamp value. + DefaultNotFoundTimestampValue int64 = math.MinInt64 ) const ( @@ -260,6 +266,7 @@ func (q *Querier[TTask, TDataStorage, TLSS, TShard, THead]) selectInstant( lssQueryResult.IDs(), uintptr(unsafe.Pointer(unsafe.SliceData(instantSeries))), // #nosec G103 // it's meant to be that way ) + if result.Status == cppbridge.DataStorageQueryStatusNeedDataLoad { loadAndQueryWaiter.Add(s, result.Querier) } @@ -373,10 +380,40 @@ func (q *Querier[TTask, TDataStorage, TLSS, TShard, THead]) makeAggSeriesSet( hints *storage.SelectHints, ) storage.SeriesSet { poolProvider := q.head.PoolProvider() + timestamps := poolProvider.GetSliceOfTimestamps() + defer poolProvider.PutSliceOfTimestamps(timestamps) + for i := range timestamps { + if lssQueryResults[i] == nil { + continue + } + + timestamps[i] = poolProvider.GetTimestamps(lssQueryResults[i].Len()) + defer poolProvider.PutTimestamps(timestamps[i]) + } + + tds := q.head.CreateTask( + dsQueryFirstTimestampsQuerier, + func(shard TShard) error { + shardID := shard.ShardID() + lssQueryResult := lssQueryResults[shardID] + if lssQueryResult == nil { + return nil + } + + shard.DataStorage().QueryFirstTimestamps( + lssQueryResult.IDs(), + timestamps[shardID], + DefaultNotFoundTimestampValue, + ) + + return nil + }, + ) + q.head.Enqueue(tds) seriesGroups := poolProvider.GetSeriesGroups() defer poolProvider.PutSeriesGroups(seriesGroups) - t := q.head.CreateTask( + tlss := q.head.CreateTask( lssGroupSeriesByLabelNames, func(shard TShard) error { shardID := shard.ShardID() @@ -387,24 +424,37 @@ func (q *Querier[TTask, TDataStorage, TLSS, TShard, THead]) makeAggSeriesSet( nameIDs := poolProvider.GetNameIDs(len(hints.Grouping)) shard.LSS().LabelNameToIDs(hints.Grouping, nameIDs) - seriesGroups[shardID] = shard.LSS().GroupSeriesByLabelNames(lssQueryResults[shardID].IDs(), nameIDs) + seriesGroups[shardID] = shard.LSS().GroupSeriesByLabelNames(lssQueryResult.IDs(), nameIDs) poolProvider.PutNameIDs(nameIDs) return nil }, ) - defer q.head.PutTask(t) - q.head.Enqueue(t) - _ = t.Wait() + q.head.Enqueue(tlss) + + _ = tds.Wait() + _ = tlss.Wait() + q.head.PutTask(tds) + q.head.PutTask(tlss) seriesSets := poolProvider.GetSeriesSet() defer poolProvider.PutSeriesSet(seriesSets) + sNaNSeriesSets := poolProvider.GetSeriesSet() + defer poolProvider.PutSeriesSet(sNaNSeriesSets) for shardID, serializedData := range shardedSerializedData { if serializedData == nil { + sNaNSeriesSets[shardID] = emptySeriesSet seriesSets[shardID] = emptySeriesSet continue } + sNaNSeriesSets[shardID] = NewStaleNaNSeriesSet( + NewStaleNaNSeriesSliceFromTimestamps(timestamps[shardID]), + lssQueryResults[shardID], + snapshots[shardID], + DefaultNotFoundTimestampValue, + ) + seriesSets[shardID] = NewAggSeriesSet( serializedData, snapshots[shardID], @@ -417,7 +467,7 @@ func (q *Querier[TTask, TDataStorage, TLSS, TShard, THead]) makeAggSeriesSet( ) } - return NewMergeShardSeriesSet(seriesSets) + return NewMergeManyShardSeriesSets(seriesSets, sNaNSeriesSets) } // selectFuncOpt selects the function optimization hints. diff --git a/pp/go/storage/querier/stalenan_series_set.go b/pp/go/storage/querier/stalenan_series_set.go index 19862ee8f..2ee26d725 100644 --- a/pp/go/storage/querier/stalenan_series_set.go +++ b/pp/go/storage/querier/stalenan_series_set.go @@ -15,16 +15,6 @@ import ( // floatStaleNaN is the float64 representation of the [value.StaleNaN] value. var floatStaleNaN = math.Float64frombits(value.StaleNaN) -// MakeTimestampsSliceWithDefault creates a slice with the default timestamp. -func MakeTimestampsSliceWithDefault(size int, defaultTimestamp int64) []int64 { - timestamps := make([]int64, size) - for i := range timestamps { - timestamps[i] = defaultTimestamp - } - - return timestamps -} - // NewStaleNaNSeriesSliceFromTimestamps creates [StaleNaNSeries] slice from timestamps. func NewStaleNaNSeriesSliceFromTimestamps(timestamps []int64) []StaleNaNSeries { seriesSlice := make([]StaleNaNSeries, len(timestamps)) diff --git a/pp/go/storage/storagetest/fixtures.go b/pp/go/storage/storagetest/fixtures.go index 2afcdb901..9788c3954 100644 --- a/pp/go/storage/storagetest/fixtures.go +++ b/pp/go/storage/storagetest/fixtures.go @@ -341,8 +341,8 @@ func StaleNaNQuery( return &querier.StaleNaNSeriesSet{}, nil } - timestamps := querier.MakeTimestampsSliceWithDefault(lssQueryResult.Len(), valueNotFoundTimestampValue) - ds.QueryFirstTimestamps(lssQueryResult.IDs(), timestamps) + timestamps := make([]int64, lssQueryResult.Len()) + ds.QueryFirstTimestamps(lssQueryResult.IDs(), timestamps, valueNotFoundTimestampValue) return querier.NewStaleNaNSeriesSet( querier.NewStaleNaNSeriesSliceFromTimestamps(timestamps), diff --git a/pp/prometheus/query.h b/pp/prometheus/query.h index 2e126cf72..76a51106b 100644 --- a/pp/prometheus/query.h +++ b/pp/prometheus/query.h @@ -16,13 +16,15 @@ struct GenericSelectHints { String func{}; Slice grouping; - bool by{}; int64_t range_ms{}; uint64_t shard_count{}; uint64_t shard_index{}; + int64_t lookback_delta_ms{}; + bool disable_trimming{}; + bool by{}; GenericSelectHints() = default; @@ -33,11 +35,12 @@ struct GenericSelectHints { limit(hints.limit), step_ms(hints.step_ms), func(static_cast(hints.func)), - by(hints.by), range_ms(hints.range_ms), shard_count(hints.shard_count), shard_index(hints.shard_index), - disable_trimming(hints.disable_trimming) { + lookback_delta_ms(hints.lookback_delta_ms), + disable_trimming(hints.disable_trimming), + by(hints.by) { if (!hints.grouping.empty()) [[unlikely]] { grouping.reserve(hints.grouping.size()); for (const auto& group : hints.grouping) { @@ -67,4 +70,4 @@ struct LabelValuesQuery { bool operator==(const LabelValuesQuery&) const noexcept = default; }; -} // namespace PromPP::Prometheus \ No newline at end of file +} // namespace PromPP::Prometheus From ea2abd9f24441d9874b70abc05528794e9cef77d Mon Sep 17 00:00:00 2001 From: Alexandr Yudin <57181751+u-veles-a@users.noreply.github.com> Date: Thu, 14 May 2026 09:59:27 +0000 Subject: [PATCH 10/37] WIP: add hints_by Signed-off-by: Alexandr Yudin <57181751+u-veles-a@users.noreply.github.com> --- pp/go/storage/querier/agg_series_set.go | 15 +++------ pp/go/storage/querier/agg_series_set_test.go | 34 ++++---------------- pp/go/storage/querier/querier.go | 18 +++++------ 3 files changed, 21 insertions(+), 46 deletions(-) diff --git a/pp/go/storage/querier/agg_series_set.go b/pp/go/storage/querier/agg_series_set.go index 176294b74..a89048b60 100644 --- a/pp/go/storage/querier/agg_series_set.go +++ b/pp/go/storage/querier/agg_series_set.go @@ -4,7 +4,6 @@ import ( "errors" "fmt" "slices" - "strconv" "github.com/prometheus/prometheus/model/histogram" "github.com/prometheus/prometheus/model/labels" @@ -20,8 +19,8 @@ import ( // // AggSeriesSet contains a set of aggregated series. -// If grouping is empty, it will return series with labels "__head_id" and "__shard_id". -// If grouping is not empty, it will return series with "__head_id" and "__shard_id" and the grouping labels. +// If grouping is empty, it will return series with labels "__head__shard_id". +// If grouping is not empty, it will return series with "__head__shard_id" and the grouping labels. type AggSeriesSet struct { serializedData *cppbridge.DataStorageSerializedData labelSetSnapshot *cppbridge.LabelSetSnapshot @@ -315,11 +314,8 @@ func (it *AggChunkIterator) reset( // const ( - // labelHeadID is the label name for the head ID. - labelHeadID = "__head_id" - - // labelShardID is the label name for the shard ID. - labelShardID = "__shard_id" + // labelHeadIDShardID is the label name for the head ID and shard ID. + labelHeadIDShardID = "__head__shard_id" ) var ( @@ -339,8 +335,7 @@ func aggLabelSetCtor( seriesID uint32, shardID uint16, ) labels.Labels { - sb.Add(labelHeadID, headID) - sb.Add(labelShardID, strconv.FormatUint(uint64(shardID), 10)) //revive:disable-line:add-constant it's base 10 + sb.Add(labelHeadIDShardID, fmt.Sprintf("%s__%d", headID, shardID)) if len(grouping) == 0 { return sb.Labels() diff --git a/pp/go/storage/querier/agg_series_set_test.go b/pp/go/storage/querier/agg_series_set_test.go index 9981c28c9..f0efa1ab5 100644 --- a/pp/go/storage/querier/agg_series_set_test.go +++ b/pp/go/storage/querier/agg_series_set_test.go @@ -141,7 +141,7 @@ func (s *AggSeriesSetSuite) TestQueryWithoutGrouping() { expected := []storagetest.TimeSeries{ { - Labels: labels.FromStrings("__head_id", "head_id", "__shard_id", "0"), + Labels: labels.FromStrings("__head__shard_id", "head_id__0"), Samples: []cppbridge.Sample{}, }, { @@ -188,11 +188,11 @@ func (s *AggSeriesSetSuite) TestQueryGrouping_OneGroupingLabel() { expected := []storagetest.TimeSeries{ { - Labels: labels.FromStrings("__head_id", "head_id", "__shard_id", "0", "job", "test"), + Labels: labels.FromStrings("__head__shard_id", "head_id__0", "job", "test"), Samples: []cppbridge.Sample{}, }, { - Labels: labels.FromStrings("__head_id", "head_id", "__shard_id", "0", "job", "test2"), + Labels: labels.FromStrings("__head__shard_id", "head_id__0", "job", "test2"), Samples: []cppbridge.Sample{}, }, { @@ -240,21 +240,11 @@ func (s *AggSeriesSetSuite) TestQueryGrouping_TwoGroupingLabels() { expected := []storagetest.TimeSeries{ { - Labels: labels.FromStrings( - "__head_id", "head_id", - "__shard_id", "0", - "job", "test", - "instance", "instance1", - ), + Labels: labels.FromStrings("__head__shard_id", "head_id__0", "job", "test", "instance", "instance1"), Samples: []cppbridge.Sample{}, }, { - Labels: labels.FromStrings( - "__head_id", "head_id", - "__shard_id", "0", - "job", "test2", - "instance", "instance2", - ), + Labels: labels.FromStrings("__head__shard_id", "head_id__0", "job", "test2", "instance", "instance2"), Samples: []cppbridge.Sample{}, }, { @@ -302,21 +292,11 @@ func (s *AggSeriesSetSuite) TestQueryGrouping_TwoGroupingLabels_WithMissingGroup expected := []storagetest.TimeSeries{ { - Labels: labels.FromStrings( - "__head_id", "head_id", - "__shard_id", "0", - "job", "test", - "instance", "instance1", - ), + Labels: labels.FromStrings("__head__shard_id", "head_id__0", "job", "test", "instance", "instance1"), Samples: []cppbridge.Sample{}, }, { - Labels: labels.FromStrings( - "__head_id", "head_id", - "__shard_id", "0", - "job", "test2", - "instance", "instance2", - ), + Labels: labels.FromStrings("__head__shard_id", "head_id__0", "job", "test2", "instance", "instance2"), Samples: []cppbridge.Sample{}, }, { diff --git a/pp/go/storage/querier/querier.go b/pp/go/storage/querier/querier.go index afdff222f..023e9d382 100644 --- a/pp/go/storage/querier/querier.go +++ b/pp/go/storage/querier/querier.go @@ -337,7 +337,7 @@ func (q *Querier[TTask, TDataStorage, TLSS, TShard, THead]) selectRange( defer poolProvider.PutSerializedData(shardedSerializedData) queryDataStorage(dsQueryRangeQuerier, q.head, lssQueryResults, shardedSerializedData, q.mint, q.maxt, hints) - if isCrossSeriesFunc(hints.Func) && isAllowedGroupingForCrossSeriesFunc(hints.Grouping) { + if isCrossSeriesFunc(hints) && isAllowedGroupingForCrossSeriesFunc(hints.Grouping) { return q.makeAggSeriesSet(lssQueryResults, snapshots, shardedSerializedData, hints) } @@ -477,14 +477,14 @@ func selectFuncOpt(hints *storage.SelectHints) *storage.SelectHints { return emptySelectHints case AggrFuncOpt: - if isCrossSeriesFunc(hints.Func) { + if isCrossSeriesFunc(hints) { return emptySelectHints } return hints case CrossSeriesFuncOpt: - if !isCrossSeriesFunc(hints.Func) { + if !isCrossSeriesFunc(hints) { return emptySelectHints } @@ -497,9 +497,9 @@ func selectFuncOpt(hints *storage.SelectHints) *storage.SelectHints { // isAllowedGroupingForCrossSeriesFunc checks if the series set is an cross series set. func isAllowedGroupingForCrossSeriesFunc(grouping []string) bool { for _, group := range grouping { - if group == labelHeadID || group == labelShardID { + if group == labelHeadIDShardID { logger.Infof( - "[QUERIER]: head_id or shard_id is in the grouping, it will be ignored: %s", + "[QUERIER]: __head__shard_id is in the grouping, it will be ignored: %s", group, ) return false @@ -510,10 +510,10 @@ func isAllowedGroupingForCrossSeriesFunc(grouping []string) bool { } // isCrossSeriesFunc checks if the function is a cross series function. -func isCrossSeriesFunc(funcName string) bool { - return funcName == "sum" || - funcName == "max" || - funcName == "min" +func isCrossSeriesFunc(hints *storage.SelectHints) bool { + return hints.Func == "sum" || + hints.Func == "max" || + hints.Func == "min" && hints.By } // convertPrometheusMatchersToPPMatchers converts prometheus matchers to pp matchers. From 63ae14a2aa09c67773b7b4d5aec76af595a3ec1d Mon Sep 17 00:00:00 2001 From: Alexandr Yudin <57181751+u-veles-a@users.noreply.github.com> Date: Thu, 14 May 2026 11:02:02 +0000 Subject: [PATCH 11/37] WIP: fix isCrossSeriesFunc Signed-off-by: Alexandr Yudin <57181751+u-veles-a@users.noreply.github.com> --- pp/go/storage/querier/querier.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pp/go/storage/querier/querier.go b/pp/go/storage/querier/querier.go index 023e9d382..28672b6ed 100644 --- a/pp/go/storage/querier/querier.go +++ b/pp/go/storage/querier/querier.go @@ -511,9 +511,9 @@ func isAllowedGroupingForCrossSeriesFunc(grouping []string) bool { // isCrossSeriesFunc checks if the function is a cross series function. func isCrossSeriesFunc(hints *storage.SelectHints) bool { - return hints.Func == "sum" || + return (hints.Func == "sum" || hints.Func == "max" || - hints.Func == "min" && hints.By + hints.Func == "min") && hints.By } // convertPrometheusMatchersToPPMatchers converts prometheus matchers to pp matchers. From cb7d11ad291fbeaa711bd2b4a9a29b665e2e0147 Mon Sep 17 00:00:00 2001 From: Alexandr Yudin <57181751+u-veles-a@users.noreply.github.com> Date: Mon, 18 May 2026 07:25:48 +0000 Subject: [PATCH 12/37] WIP: rebuild flags Signed-off-by: Alexandr Yudin <57181751+u-veles-a@users.noreply.github.com> --- cmd/prometheus/main.go | 4 +- pp/go/storage/querier/querier.go | 109 +++++++++++++++++++++++-------- 2 files changed, 84 insertions(+), 29 deletions(-) diff --git a/cmd/prometheus/main.go b/cmd/prometheus/main.go index 374aa68ec..433070340 100644 --- a/cmd/prometheus/main.go +++ b/cmd/prometheus/main.go @@ -2240,8 +2240,7 @@ func readPromPPFeatures(logger log.Logger) { remotewriter.DefaultSampleAgeLimit = defaultSampleAgeLimit case "select_func_optimization": - opt, err := querier.ParseSelectFuncOpt(strings.TrimSpace(fvalue)) - if err != nil { + if err := querier.SetSelectFuncOptimize(strings.TrimSpace(fvalue)); err != nil { level.Error(logger).Log( "msg", "[FEATURE] Error parsing select_func_optimization value", "err", err, @@ -2249,7 +2248,6 @@ func readPromPPFeatures(logger log.Logger) { continue } - querier.SelectFuncOpt = opt level.Info(logger).Log( "msg", "[FEATURE] Select function optimization is set.", "optimization", fvalue, diff --git a/pp/go/storage/querier/querier.go b/pp/go/storage/querier/querier.go index 28672b6ed..125ad0501 100644 --- a/pp/go/storage/querier/querier.go +++ b/pp/go/storage/querier/querier.go @@ -46,40 +46,55 @@ const ( DefaultNotFoundTimestampValue int64 = math.MinInt64 ) +// +// queryOptimizeType +// + +// queryOptimizeType is the type for query optimization. +type queryOptimizeType uint8 + const ( - // NoneOpt is the option without any optimization. - NoneOpt uint8 = iota + // aggrOptimizeType is the option for aggregated functions optimization. + aggrOptimizeType queryOptimizeType = 1 << iota - // AggrFuncOpt is the option for aggregated functions optimization. - AggrFuncOpt + // crossSeriesOptimizeType is the option for cross-series functions optimization. + crossSeriesOptimizeType - // CrossSeriesFuncOpt is the option for cross series functions optimization. - CrossSeriesFuncOpt + // noneOptimizeType is the option without any optimization. + noneOptimizeType queryOptimizeType = 0 - // AllOpt is the option for aggregated and cross functions optimization. - AllOpt + // allOptimizeType is the option for all functions optimization. + allOptimizeType queryOptimizeType = 3 ) -// ParseSelectFuncOpt parses the select func optimization option from string. -func ParseSelectFuncOpt(opt string) (uint8, error) { +// SetSelectFuncOptimize sets the select func optimization option by name. +func SetSelectFuncOptimize(opt string) error { switch opt { case "none": - return NoneOpt, nil + selectFuncOptimize = noneOptimizeType + return nil + case "aggr": - return AggrFuncOpt, nil + selectFuncOptimize = aggrOptimizeType + return nil + case "cross": - return CrossSeriesFuncOpt, nil + selectFuncOptimize = crossSeriesOptimizeType + return nil + case "all": - return AllOpt, nil + selectFuncOptimize = allOptimizeType + return nil + default: - return 0, fmt.Errorf( + return fmt.Errorf( "invalid select func optimization option: '%s', valid options are: 'none', 'aggr', 'cross', 'all'", opt, ) } } -// SelectFuncOpt is the option for selecting functions optimization. -var SelectFuncOpt = NoneOpt +// selectFuncOptimize is the option for selecting functions optimization. +var selectFuncOptimize = noneOptimizeType // emptySelectHints is an empty select hints, it's used when no optimization is needed. var emptySelectHints = &storage.SelectHints{} @@ -105,6 +120,7 @@ type Querier[ deduplicatorCtor deduplicatorCtor closer func() error metrics *Metrics + queryOptimize queryOptimizeType } // NewQuerier init new [Querier]. @@ -120,6 +136,41 @@ func NewQuerier[ mint, maxt int64, closer func() error, metrics *Metrics, +) *Querier[TTask, TDataStorage, TLSS, TShard, THead] { + return newQuerierWithSelectFuncOptimize(head, deduplicatorCtor, mint, maxt, closer, metrics, selectFuncOptimize) +} + +// NewQuerierWithOutSelectFuncOptimize init new [Querier] without select func optimization. +func NewQuerierWithOutSelectFuncOptimize[ + TTask Task, + TDataStorage DataStorage, + TLSS LSS, + TShard Shard[TDataStorage, TLSS], + THead Head[TTask, TDataStorage, TLSS, TShard], +]( + head THead, + deduplicatorCtor deduplicatorCtor, + mint, maxt int64, + closer func() error, + metrics *Metrics, +) *Querier[TTask, TDataStorage, TLSS, TShard, THead] { + return newQuerierWithSelectFuncOptimize(head, deduplicatorCtor, mint, maxt, closer, metrics, noneOptimizeType) +} + +// newQuerierWithSelectFuncOptimize init new [Querier] with select func optimization. +func newQuerierWithSelectFuncOptimize[ + TTask Task, + TDataStorage DataStorage, + TLSS LSS, + TShard Shard[TDataStorage, TLSS], + THead Head[TTask, TDataStorage, TLSS, TShard], +]( + head THead, + deduplicatorCtor deduplicatorCtor, + mint, maxt int64, + closer func() error, + metrics *Metrics, + queryOptimize queryOptimizeType, ) *Querier[TTask, TDataStorage, TLSS, TShard, THead] { return &Querier[TTask, TDataStorage, TLSS, TShard, THead]{ mint: mint, @@ -128,6 +179,7 @@ func NewQuerier[ deduplicatorCtor: deduplicatorCtor, closer: closer, metrics: metrics, + queryOptimize: queryOptimize, } } @@ -332,7 +384,7 @@ func (q *Querier[TTask, TDataStorage, TLSS, TShard, THead]) selectRange( return storage.ErrSeriesSet(err) } - hints = selectFuncOpt(hints) + hints = q.switchFuncOptimize(hints) shardedSerializedData := poolProvider.GetSerializedData() defer poolProvider.PutSerializedData(shardedSerializedData) queryDataStorage(dsQueryRangeQuerier, q.head, lssQueryResults, shardedSerializedData, q.mint, q.maxt, hints) @@ -470,28 +522,33 @@ func (q *Querier[TTask, TDataStorage, TLSS, TShard, THead]) makeAggSeriesSet( return NewMergeManyShardSeriesSets(seriesSets, sNaNSeriesSets) } -// selectFuncOpt selects the function optimization hints. -func selectFuncOpt(hints *storage.SelectHints) *storage.SelectHints { - switch SelectFuncOpt { - case NoneOpt: +// switchFuncOptimize switch the function optimization hints. +func (q *Querier[TTask, TDataStorage, TLSS, TShard, THead]) switchFuncOptimize( + hints *storage.SelectHints, +) *storage.SelectHints { + switch q.queryOptimize { + case noneOptimizeType: return emptySelectHints - case AggrFuncOpt: + case aggrOptimizeType: if isCrossSeriesFunc(hints) { return emptySelectHints } return hints - case CrossSeriesFuncOpt: + case crossSeriesOptimizeType: if !isCrossSeriesFunc(hints) { return emptySelectHints } return hints - } + case allOptimizeType: + return hints - return hints + default: + return emptySelectHints + } } // isAllowedGroupingForCrossSeriesFunc checks if the series set is an cross series set. From d7551750efec35758c7c6fbdb2131277b2df927e Mon Sep 17 00:00:00 2001 From: Alexandr Yudin <57181751+u-veles-a@users.noreply.github.com> Date: Mon, 18 May 2026 07:34:21 +0000 Subject: [PATCH 13/37] WIP: fix adapter Signed-off-by: Alexandr Yudin <57181751+u-veles-a@users.noreply.github.com> --- pp-pkg/storage/adapter.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/pp-pkg/storage/adapter.go b/pp-pkg/storage/adapter.go index 4ae0973ad..81d704145 100644 --- a/pp-pkg/storage/adapter.go +++ b/pp-pkg/storage/adapter.go @@ -311,7 +311,14 @@ func (ar *Adapter) Querier(mint, maxt int64) (storage.Querier, error) { queriers = append( queriers, - querier.NewQuerier(head, querier.NewNoOpShardedDeduplicator, mint, maxt, nil, ar.storageQuerierMetrics), + querier.NewQuerierWithOutSelectFuncOptimize( + head, + querier.NewNoOpShardedDeduplicator, + mint, + maxt, + nil, + ar.storageQuerierMetrics, + ), ) } From 01977e8ce6e06a1eab52155b3b39cec97805aa83 Mon Sep 17 00:00:00 2001 From: Alexandr Yudin <57181751+u-veles-a@users.noreply.github.com> Date: Mon, 18 May 2026 10:11:39 +0000 Subject: [PATCH 14/37] WIP: rebuild SwitchFuncOptimize Signed-off-by: Alexandr Yudin <57181751+u-veles-a@users.noreply.github.com> --- pp/go/storage/querier/querier.go | 82 +++++++++++++++++++------------- 1 file changed, 49 insertions(+), 33 deletions(-) diff --git a/pp/go/storage/querier/querier.go b/pp/go/storage/querier/querier.go index 125ad0501..2993fd7fc 100644 --- a/pp/go/storage/querier/querier.go +++ b/pp/go/storage/querier/querier.go @@ -54,17 +54,24 @@ const ( type queryOptimizeType uint8 const ( - // aggrOptimizeType is the option for aggregated functions optimization. - aggrOptimizeType queryOptimizeType = 1 << iota + // dropPointOptimizeType is the option for drop point functions optimization. + dropPointOptimizeType queryOptimizeType = 1 << iota + + // newPointOptimizeType is the option for new point functions optimization. + // Optimization creates a new point at the end of the window or step. + newPointOptimizeType // crossSeriesOptimizeType is the option for cross-series functions optimization. + // A new series is created. crossSeriesOptimizeType +) +const ( // noneOptimizeType is the option without any optimization. noneOptimizeType queryOptimizeType = 0 // allOptimizeType is the option for all functions optimization. - allOptimizeType queryOptimizeType = 3 + allOptimizeType queryOptimizeType = dropPointOptimizeType | newPointOptimizeType | crossSeriesOptimizeType ) // SetSelectFuncOptimize sets the select func optimization option by name. @@ -74,8 +81,12 @@ func SetSelectFuncOptimize(opt string) error { selectFuncOptimize = noneOptimizeType return nil - case "aggr": - selectFuncOptimize = aggrOptimizeType + case "drop_point": + selectFuncOptimize = dropPointOptimizeType + return nil + + case "new_point": + selectFuncOptimize = newPointOptimizeType return nil case "cross": @@ -88,7 +99,8 @@ func SetSelectFuncOptimize(opt string) error { default: return fmt.Errorf( - "invalid select func optimization option: '%s', valid options are: 'none', 'aggr', 'cross', 'all'", opt, + "invalid select func optimization option: '%s', valid options are: "+ + "'none', 'drop_point', 'new_point', 'cross', 'all'", opt, ) } } @@ -384,7 +396,7 @@ func (q *Querier[TTask, TDataStorage, TLSS, TShard, THead]) selectRange( return storage.ErrSeriesSet(err) } - hints = q.switchFuncOptimize(hints) + hints = SwitchFuncOptimize(hints, q.queryOptimize) shardedSerializedData := poolProvider.GetSerializedData() defer poolProvider.PutSerializedData(shardedSerializedData) queryDataStorage(dsQueryRangeQuerier, q.head, lssQueryResults, shardedSerializedData, q.mint, q.maxt, hints) @@ -522,33 +534,39 @@ func (q *Querier[TTask, TDataStorage, TLSS, TShard, THead]) makeAggSeriesSet( return NewMergeManyShardSeriesSets(seriesSets, sNaNSeriesSets) } -// switchFuncOptimize switch the function optimization hints. -func (q *Querier[TTask, TDataStorage, TLSS, TShard, THead]) switchFuncOptimize( - hints *storage.SelectHints, -) *storage.SelectHints { - switch q.queryOptimize { - case noneOptimizeType: - return emptySelectHints - - case aggrOptimizeType: - if isCrossSeriesFunc(hints) { - return emptySelectHints - } - +// SwitchFuncOptimize switch the function optimization hints. +func SwitchFuncOptimize(hints *storage.SelectHints, queryOptimize queryOptimizeType) *storage.SelectHints { + if funcOptimizeMap[hints.Func]&queryOptimize != 0 && isNotWithpout(hints) { return hints + } - case crossSeriesOptimizeType: - if !isCrossSeriesFunc(hints) { - return emptySelectHints - } + return emptySelectHints +} - return hints - case allOptimizeType: - return hints +// isNotWithpout checks if the hints is not without by. +func isNotWithpout(hints *storage.SelectHints) bool { + return hints.By || len(hints.Grouping) == 0 +} - default: - return emptySelectHints - } +// funcOptimizeMap is the map of the function to the query optimization type. +var funcOptimizeMap = map[string]queryOptimizeType{ + "sum": crossSeriesOptimizeType, + "max": crossSeriesOptimizeType, + "min": crossSeriesOptimizeType, + "group": crossSeriesOptimizeType, + "count": newPointOptimizeType, + "sum_over_time": newPointOptimizeType, + "max_over_time": dropPointOptimizeType, + "min_over_time": dropPointOptimizeType, + "count_over_time": newPointOptimizeType, + "last_over_time": dropPointOptimizeType, + "rate": dropPointOptimizeType, + "irate": dropPointOptimizeType, + "delta": dropPointOptimizeType, + "idelta": dropPointOptimizeType, + "increase": dropPointOptimizeType, + "resets": dropPointOptimizeType, + "changes": dropPointOptimizeType, } // isAllowedGroupingForCrossSeriesFunc checks if the series set is an cross series set. @@ -568,9 +586,7 @@ func isAllowedGroupingForCrossSeriesFunc(grouping []string) bool { // isCrossSeriesFunc checks if the function is a cross series function. func isCrossSeriesFunc(hints *storage.SelectHints) bool { - return (hints.Func == "sum" || - hints.Func == "max" || - hints.Func == "min") && hints.By + return funcOptimizeMap[hints.Func]&crossSeriesOptimizeType == crossSeriesOptimizeType && isNotWithpout(hints) } // convertPrometheusMatchersToPPMatchers converts prometheus matchers to pp matchers. From 97e5987bed31337e0556f983a47931f24301231e Mon Sep 17 00:00:00 2001 From: Alexandr Yudin <57181751+u-veles-a@users.noreply.github.com> Date: Mon, 18 May 2026 10:22:45 +0000 Subject: [PATCH 15/37] WIP: fix Signed-off-by: Alexandr Yudin <57181751+u-veles-a@users.noreply.github.com> --- pp/go/storage/querier/querier.go | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/pp/go/storage/querier/querier.go b/pp/go/storage/querier/querier.go index 2993fd7fc..bba2ec5f3 100644 --- a/pp/go/storage/querier/querier.go +++ b/pp/go/storage/querier/querier.go @@ -459,16 +459,12 @@ func (q *Querier[TTask, TDataStorage, TLSS, TShard, THead]) makeAggSeriesSet( dsQueryFirstTimestampsQuerier, func(shard TShard) error { shardID := shard.ShardID() - lssQueryResult := lssQueryResults[shardID] - if lssQueryResult == nil { + res := lssQueryResults[shardID] + if res == nil { return nil } - shard.DataStorage().QueryFirstTimestamps( - lssQueryResult.IDs(), - timestamps[shardID], - DefaultNotFoundTimestampValue, - ) + shard.DataStorage().QueryFirstTimestamps(res.IDs(), timestamps[shardID], DefaultNotFoundTimestampValue) return nil }, @@ -481,14 +477,14 @@ func (q *Querier[TTask, TDataStorage, TLSS, TShard, THead]) makeAggSeriesSet( lssGroupSeriesByLabelNames, func(shard TShard) error { shardID := shard.ShardID() - lssQueryResult := lssQueryResults[shardID] - if lssQueryResult == nil { + res := lssQueryResults[shardID] + if res == nil { return nil } nameIDs := poolProvider.GetNameIDs(len(hints.Grouping)) shard.LSS().LabelNameToIDs(hints.Grouping, nameIDs) - seriesGroups[shardID] = shard.LSS().GroupSeriesByLabelNames(lssQueryResult.IDs(), nameIDs) + seriesGroups[shardID] = shard.LSS().GroupSeriesByLabelNames(res.IDs(), nameIDs) poolProvider.PutNameIDs(nameIDs) return nil From 834b1586c2dac071b80db0f9a7380fbd2651d873 Mon Sep 17 00:00:00 2001 From: Alexandr Yudin <57181751+u-veles-a@users.noreply.github.com> Date: Mon, 18 May 2026 10:30:26 +0000 Subject: [PATCH 16/37] WIP: small fix Signed-off-by: Alexandr Yudin <57181751+u-veles-a@users.noreply.github.com> --- pp/go/storage/querier/querier.go | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/pp/go/storage/querier/querier.go b/pp/go/storage/querier/querier.go index bba2ec5f3..28ed401bf 100644 --- a/pp/go/storage/querier/querier.go +++ b/pp/go/storage/querier/querier.go @@ -149,7 +149,15 @@ func NewQuerier[ closer func() error, metrics *Metrics, ) *Querier[TTask, TDataStorage, TLSS, TShard, THead] { - return newQuerierWithSelectFuncOptimize(head, deduplicatorCtor, mint, maxt, closer, metrics, selectFuncOptimize) + return newQuerierWithSelectFuncOptimize( + head, + deduplicatorCtor, + mint, + maxt, + closer, + metrics, + selectFuncOptimize&dropPointOptimizeType, + ) } // NewQuerierWithOutSelectFuncOptimize init new [Querier] without select func optimization. From d8950edb2945a208c3bd816bbec133d450da3f7c Mon Sep 17 00:00:00 2001 From: Alexandr Yudin <57181751+u-veles-a@users.noreply.github.com> Date: Tue, 19 May 2026 10:47:40 +0000 Subject: [PATCH 17/37] WIP: rebuild SetSelectFuncOptimize Signed-off-by: Alexandr Yudin <57181751+u-veles-a@users.noreply.github.com> --- cmd/prometheus/main.go | 15 ++++++++++++++- pp/go/storage/querier/querier.go | 6 +++--- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/cmd/prometheus/main.go b/cmd/prometheus/main.go index 433070340..2d88e3978 100644 --- a/cmd/prometheus/main.go +++ b/cmd/prometheus/main.go @@ -2240,11 +2240,12 @@ func readPromPPFeatures(logger log.Logger) { remotewriter.DefaultSampleAgeLimit = defaultSampleAgeLimit case "select_func_optimization": - if err := querier.SetSelectFuncOptimize(strings.TrimSpace(fvalue)); err != nil { + if err := selectFuncOptimization(strings.TrimSpace(fvalue)); err != nil { level.Error(logger).Log( "msg", "[FEATURE] Error parsing select_func_optimization value", "err", err, ) + continue } @@ -2255,3 +2256,15 @@ func readPromPPFeatures(logger log.Logger) { } } } + +// selectFuncOptimization sets the select function optimization. +func selectFuncOptimization(fvalue string) error { + for opt := range strings.SplitSeq(fvalue, "|") { + if err := querier.SetSelectFuncOptimize(opt); err != nil { + _ = querier.SetSelectFuncOptimize("none") // reset to none if error + return err + } + } + + return nil +} diff --git a/pp/go/storage/querier/querier.go b/pp/go/storage/querier/querier.go index 28ed401bf..cf5a82caf 100644 --- a/pp/go/storage/querier/querier.go +++ b/pp/go/storage/querier/querier.go @@ -82,15 +82,15 @@ func SetSelectFuncOptimize(opt string) error { return nil case "drop_point": - selectFuncOptimize = dropPointOptimizeType + selectFuncOptimize |= dropPointOptimizeType return nil case "new_point": - selectFuncOptimize = newPointOptimizeType + selectFuncOptimize |= newPointOptimizeType return nil case "cross": - selectFuncOptimize = crossSeriesOptimizeType + selectFuncOptimize |= crossSeriesOptimizeType return nil case "all": From 584b51bef4fbf2e25c835746f2db29dd50f03796 Mon Sep 17 00:00:00 2001 From: Alexandr Yudin <57181751+u-veles-a@users.noreply.github.com> Date: Tue, 19 May 2026 11:28:16 +0000 Subject: [PATCH 18/37] WIP: rebuild flag select_func_optimization Signed-off-by: Alexandr Yudin <57181751+u-veles-a@users.noreply.github.com> --- cmd/prometheus/main.go | 2 +- pp/go/storage/querier/querier.go | 10 +++++++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/cmd/prometheus/main.go b/cmd/prometheus/main.go index 2d88e3978..279c01798 100644 --- a/cmd/prometheus/main.go +++ b/cmd/prometheus/main.go @@ -2261,7 +2261,7 @@ func readPromPPFeatures(logger log.Logger) { func selectFuncOptimization(fvalue string) error { for opt := range strings.SplitSeq(fvalue, "|") { if err := querier.SetSelectFuncOptimize(opt); err != nil { - _ = querier.SetSelectFuncOptimize("none") // reset to none if error + querier.SetDefaultOptimizeType() // reset to default if error return err } } diff --git a/pp/go/storage/querier/querier.go b/pp/go/storage/querier/querier.go index cf5a82caf..caf33ec5a 100644 --- a/pp/go/storage/querier/querier.go +++ b/pp/go/storage/querier/querier.go @@ -74,6 +74,9 @@ const ( allOptimizeType queryOptimizeType = dropPointOptimizeType | newPointOptimizeType | crossSeriesOptimizeType ) +// defaultOptimizeType is the default option for selecting functions optimization. +var defaultOptimizeType = noneOptimizeType + // SetSelectFuncOptimize sets the select func optimization option by name. func SetSelectFuncOptimize(opt string) error { switch opt { @@ -105,8 +108,13 @@ func SetSelectFuncOptimize(opt string) error { } } +// SetDefaultOptimizeType set the default option for selecting functions optimization. +func SetDefaultOptimizeType() { + selectFuncOptimize = defaultOptimizeType +} + // selectFuncOptimize is the option for selecting functions optimization. -var selectFuncOptimize = noneOptimizeType +var selectFuncOptimize = defaultOptimizeType // emptySelectHints is an empty select hints, it's used when no optimization is needed. var emptySelectHints = &storage.SelectHints{} From 5a7efe619610b0fca3c0542c89f66abf0bc351ea Mon Sep 17 00:00:00 2001 From: Alexandr Yudin <57181751+u-veles-a@users.noreply.github.com> Date: Tue, 19 May 2026 16:41:12 +0000 Subject: [PATCH 19/37] WIP: fix and test Signed-off-by: Alexandr Yudin <57181751+u-veles-a@users.noreply.github.com> --- pp/go/storage/querier/querier.go | 12 +- pp/go/storage/querier/querier_test.go | 597 ++++++++++++++++++++++++++ 2 files changed, 607 insertions(+), 2 deletions(-) diff --git a/pp/go/storage/querier/querier.go b/pp/go/storage/querier/querier.go index caf33ec5a..0abf16c06 100644 --- a/pp/go/storage/querier/querier.go +++ b/pp/go/storage/querier/querier.go @@ -164,7 +164,7 @@ func NewQuerier[ maxt, closer, metrics, - selectFuncOptimize&dropPointOptimizeType, + selectFuncOptimize, ) } @@ -182,7 +182,15 @@ func NewQuerierWithOutSelectFuncOptimize[ closer func() error, metrics *Metrics, ) *Querier[TTask, TDataStorage, TLSS, TShard, THead] { - return newQuerierWithSelectFuncOptimize(head, deduplicatorCtor, mint, maxt, closer, metrics, noneOptimizeType) + return newQuerierWithSelectFuncOptimize( + head, + deduplicatorCtor, + mint, + maxt, + closer, + metrics, + selectFuncOptimize&dropPointOptimizeType, + ) } // newQuerierWithSelectFuncOptimize init new [Querier] with select func optimization. diff --git a/pp/go/storage/querier/querier_test.go b/pp/go/storage/querier/querier_test.go index af077d588..c1dd2b559 100644 --- a/pp/go/storage/querier/querier_test.go +++ b/pp/go/storage/querier/querier_test.go @@ -41,16 +41,60 @@ type QuerierSuite struct { dataDir string context context.Context head *storage.Head + + timeSeries []storagetest.TimeSeries + hints *prom_storage.SelectHints + matcher *labels.Matcher } func TestQuerierSuite(t *testing.T) { suite.Run(t, new(QuerierSuite)) } +func (s *QuerierSuite) SetupSuite() { + s.timeSeries = []storagetest.TimeSeries{ + { + Labels: labels.FromStrings("__name__", "metric", "job", "test"), + Samples: []cppbridge.Sample{ + {Timestamp: 20, Value: 10}, + {Timestamp: 40, Value: 10}, + {Timestamp: 60, Value: 20}, + {Timestamp: 80, Value: 30}, + {Timestamp: 110, Value: 50}, + {Timestamp: 130, Value: 80}, + {Timestamp: 150, Value: 130}, + {Timestamp: 170, Value: 210}, + {Timestamp: 190, Value: 340}, + }, + }, + { + Labels: labels.FromStrings("__name__", "metric", "job", "test2"), + Samples: []cppbridge.Sample{ + {Timestamp: 10, Value: 1}, + {Timestamp: 30, Value: 1}, + {Timestamp: 50, Value: 2}, + {Timestamp: 70, Value: 3}, + {Timestamp: 90, Value: 5}, + {Timestamp: 100, Value: 8}, + {Timestamp: 120, Value: 13}, + {Timestamp: 140, Value: 21}, + {Timestamp: 160, Value: 34}, + {Timestamp: 180, Value: 55}, + }, + }, + } + s.matcher, _ = labels.NewMatcher(labels.MatchEqual, "__name__", "metric") +} + func (s *QuerierSuite) SetupTest() { s.dataDir = s.createDataDirectory() s.context = context.Background() s.head = s.mustCreateHead(1) + s.hints = &prom_storage.SelectHints{ + Start: 0, + End: 200, + Range: 100, + } } func (s *QuerierSuite) createDataDirectory() string { @@ -509,3 +553,556 @@ func (s *QuerierSuite) TestLabelValuesNoMatchesOnName() { s.Equal([]string{}, names) s.Len(anns.AsErrors(), 1) } + +func (s *QuerierSuite) TestRangeQuerySumOverTimeStep100() { + // Arrange + s.appendTimeSeries(s.timeSeries) + err := querier.SetSelectFuncOptimize("all") + s.Require().NoError(err) + defer querier.SetDefaultOptimizeType() + + q := querier.NewQuerier(s.head, querier.NewNoOpShardedDeduplicator, 0, s.hints.End, nil, nil) + defer func() { _ = q.Close() }() + + s.hints.Func = "sum_over_time" + s.hints.Step = 100 + + // Act + seriesSet := q.Select(s.context, false, s.hints, s.matcher) + + // Assert + s.Equal( + []storagetest.TimeSeries{ + { + Labels: labels.FromStrings("__name__", "metric", "job", "test"), + Samples: []cppbridge.Sample{ + {Timestamp: 99, Value: 70}, + {Timestamp: 199, Value: 810}, + }, + }, + { + Labels: labels.FromStrings("__name__", "metric", "job", "test2"), + Samples: []cppbridge.Sample{ + {Timestamp: 99, Value: 20}, + {Timestamp: 199, Value: 123}, + }, + }, + }, + storagetest.TimeSeriesFromSeriesSet(seriesSet, true), + ) +} + +func (s *QuerierSuite) TestRangeQuerySumOverTimeStep50() { + // Arrange + s.appendTimeSeries(s.timeSeries) + err := querier.SetSelectFuncOptimize("all") + s.Require().NoError(err) + defer querier.SetDefaultOptimizeType() + + q := querier.NewQuerier(s.head, querier.NewNoOpShardedDeduplicator, 0, 200, nil, nil) + defer func() { _ = q.Close() }() + s.hints.Func = "sum_over_time" + s.hints.Step = 50 + + // Act + seriesSet := q.Select(s.context, false, s.hints, s.matcher) + + // Assert + s.Equal( + []storagetest.TimeSeries{ + { + Labels: labels.FromStrings("__name__", "metric", "job", "test"), + Samples: []cppbridge.Sample{ + {Timestamp: 49, Value: 20}, + {Timestamp: 99, Value: 50}, + {Timestamp: 149, Value: 260}, + {Timestamp: 199, Value: 550}, + }, + }, + { + Labels: labels.FromStrings("__name__", "metric", "job", "test2"), + Samples: []cppbridge.Sample{ + {Timestamp: 49, Value: 4}, + {Timestamp: 99, Value: 16}, + {Timestamp: 149, Value: 34}, + {Timestamp: 199, Value: 89}, + }, + }, + }, + storagetest.TimeSeriesFromSeriesSet(seriesSet, true), + ) +} + +func (s *QuerierSuite) TestRangeQueryMinOverTimeStep100() { + // Arrange + s.appendTimeSeries(s.timeSeries) + err := querier.SetSelectFuncOptimize("all") + s.Require().NoError(err) + defer querier.SetDefaultOptimizeType() + + q := querier.NewQuerier(s.head, querier.NewNoOpShardedDeduplicator, 0, 200, nil, nil) + defer func() { _ = q.Close() }() + s.hints.Func = "min_over_time" + s.hints.Step = 100 + + // Act + seriesSet := q.Select(s.context, false, s.hints, s.matcher) + + // Assert + s.Equal( + []storagetest.TimeSeries{ + { + Labels: labels.FromStrings("__name__", "metric", "job", "test"), + Samples: []cppbridge.Sample{ + {Timestamp: 20, Value: 10}, + {Timestamp: 110, Value: 50}, + }, + }, + { + Labels: labels.FromStrings("__name__", "metric", "job", "test2"), + Samples: []cppbridge.Sample{ + {Timestamp: 10, Value: 1}, + {Timestamp: 120, Value: 13}, + }, + }, + }, + storagetest.TimeSeriesFromSeriesSet(seriesSet, true), + ) +} + +func (s *QuerierSuite) TestRangeQueryMinOverTimeStep50() { + // Arrange + s.appendTimeSeries(s.timeSeries) + err := querier.SetSelectFuncOptimize("all") + s.Require().NoError(err) + defer querier.SetDefaultOptimizeType() + + q := querier.NewQuerier(s.head, querier.NewNoOpShardedDeduplicator, 0, 200, nil, nil) + defer func() { _ = q.Close() }() + s.hints.Func = "min_over_time" + s.hints.Step = 50 + + // Act + seriesSet := q.Select(s.context, false, s.hints, s.matcher) + + // Assert + s.Equal( + []storagetest.TimeSeries{ + { + Labels: labels.FromStrings("__name__", "metric", "job", "test"), + Samples: []cppbridge.Sample{ + {Timestamp: 20, Value: 10}, + {Timestamp: 60, Value: 20}, + {Timestamp: 110, Value: 50}, + {Timestamp: 170, Value: 210}, + }, + }, + { + Labels: labels.FromStrings("__name__", "metric", "job", "test2"), + Samples: []cppbridge.Sample{ + {Timestamp: 10, Value: 1}, + {Timestamp: 70, Value: 3}, + {Timestamp: 120, Value: 13}, + {Timestamp: 160, Value: 34}, + }, + }, + }, + storagetest.TimeSeriesFromSeriesSet(seriesSet, true), + ) +} + +func (s *QuerierSuite) TestRangeQueryMaxOverTimeStep100() { + // Arrange + s.appendTimeSeries(s.timeSeries) + err := querier.SetSelectFuncOptimize("all") + s.Require().NoError(err) + defer querier.SetDefaultOptimizeType() + + q := querier.NewQuerier(s.head, querier.NewNoOpShardedDeduplicator, 0, 200, nil, nil) + defer func() { _ = q.Close() }() + s.hints.Func = "max_over_time" + s.hints.Step = 100 + + // Act + seriesSet := q.Select(s.context, false, s.hints, s.matcher) + + // Assert + s.Equal( + []storagetest.TimeSeries{ + { + Labels: labels.FromStrings("__name__", "metric", "job", "test"), + Samples: []cppbridge.Sample{ + {Timestamp: 80, Value: 30}, + {Timestamp: 190, Value: 340}, + }, + }, + { + Labels: labels.FromStrings("__name__", "metric", "job", "test2"), + Samples: []cppbridge.Sample{ + {Timestamp: 100, Value: 8}, + {Timestamp: 180, Value: 55}, + }, + }, + }, + storagetest.TimeSeriesFromSeriesSet(seriesSet, true), + ) +} + +func (s *QuerierSuite) TestRangeQueryMaxOverTimeStep50() { + // Arrange + s.appendTimeSeries(s.timeSeries) + err := querier.SetSelectFuncOptimize("all") + s.Require().NoError(err) + defer querier.SetDefaultOptimizeType() + + q := querier.NewQuerier(s.head, querier.NewNoOpShardedDeduplicator, 0, 200, nil, nil) + defer func() { _ = q.Close() }() + s.hints.Func = "max_over_time" + s.hints.Step = 50 + + // Act + seriesSet := q.Select(s.context, false, s.hints, s.matcher) + + // Assert + s.Equal( + []storagetest.TimeSeries{ + { + Labels: labels.FromStrings("__name__", "metric", "job", "test"), + Samples: []cppbridge.Sample{ + {Timestamp: 20, Value: 10}, + {Timestamp: 80, Value: 30}, + {Timestamp: 150, Value: 130}, + {Timestamp: 190, Value: 340}, + }, + }, + { + Labels: labels.FromStrings("__name__", "metric", "job", "test2"), + Samples: []cppbridge.Sample{ + {Timestamp: 50, Value: 2}, + {Timestamp: 100, Value: 8}, + {Timestamp: 140, Value: 21}, + {Timestamp: 180, Value: 55}, + }, + }, + }, + storagetest.TimeSeriesFromSeriesSet(seriesSet, true), + ) +} + +func (s *QuerierSuite) TestRangeQueryCountOverTimeStep100() { + // Arrange + s.appendTimeSeries(s.timeSeries) + err := querier.SetSelectFuncOptimize("all") + s.Require().NoError(err) + defer querier.SetDefaultOptimizeType() + + q := querier.NewQuerier(s.head, querier.NewNoOpShardedDeduplicator, 0, 200, nil, nil) + defer func() { _ = q.Close() }() + s.hints.Func = "count_over_time" + s.hints.Step = 100 + + // Act + seriesSet := q.Select(s.context, false, s.hints, s.matcher) + + // Assert + s.Equal( + []storagetest.TimeSeries{ + { + Labels: labels.FromStrings("__name__", "metric", "job", "test"), + Samples: []cppbridge.Sample{ + {Timestamp: 99, Value: 4}, + {Timestamp: 199, Value: 5}, + }, + }, + { + Labels: labels.FromStrings("__name__", "metric", "job", "test2"), + Samples: []cppbridge.Sample{ + {Timestamp: 99, Value: 6}, + {Timestamp: 199, Value: 4}, + }, + }, + }, + storagetest.TimeSeriesFromSeriesSet(seriesSet, true), + ) +} + +func (s *QuerierSuite) TestRangeQueryCountOverTimeStep50() { + // Arrange + s.appendTimeSeries(s.timeSeries) + err := querier.SetSelectFuncOptimize("all") + s.Require().NoError(err) + defer querier.SetDefaultOptimizeType() + + q := querier.NewQuerier(s.head, querier.NewNoOpShardedDeduplicator, 0, 200, nil, nil) + defer func() { _ = q.Close() }() + s.hints.Func = "count_over_time" + s.hints.Step = 50 + + // Act + seriesSet := q.Select(s.context, false, s.hints, s.matcher) + + // Assert + s.Equal( + []storagetest.TimeSeries{ + { + Labels: labels.FromStrings("__name__", "metric", "job", "test"), + Samples: []cppbridge.Sample{ + {Timestamp: 49, Value: 2}, + {Timestamp: 99, Value: 2}, + {Timestamp: 149, Value: 3}, + {Timestamp: 199, Value: 2}, + }, + }, + { + Labels: labels.FromStrings("__name__", "metric", "job", "test2"), + Samples: []cppbridge.Sample{ + {Timestamp: 49, Value: 3}, + {Timestamp: 99, Value: 3}, + {Timestamp: 149, Value: 2}, + {Timestamp: 199, Value: 2}, + }, + }, + }, + storagetest.TimeSeriesFromSeriesSet(seriesSet, true), + ) +} + +func (s *QuerierSuite) TestRangeQueryLastOverTimeStep100() { + // Arrange + s.appendTimeSeries(s.timeSeries) + err := querier.SetSelectFuncOptimize("all") + s.Require().NoError(err) + defer querier.SetDefaultOptimizeType() + + q := querier.NewQuerier(s.head, querier.NewNoOpShardedDeduplicator, 0, 200, nil, nil) + defer func() { _ = q.Close() }() + s.hints.Func = "last_over_time" + s.hints.Step = 100 + + // Act + seriesSet := q.Select(s.context, false, s.hints, s.matcher) + + // Assert + s.Equal( + []storagetest.TimeSeries{ + { + Labels: labels.FromStrings("__name__", "metric", "job", "test"), + Samples: []cppbridge.Sample{ + {Timestamp: 80, Value: 30}, + {Timestamp: 190, Value: 340}, + }, + }, + { + Labels: labels.FromStrings("__name__", "metric", "job", "test2"), + Samples: []cppbridge.Sample{ + {Timestamp: 100, Value: 8}, + {Timestamp: 180, Value: 55}, + }, + }, + }, + storagetest.TimeSeriesFromSeriesSet(seriesSet, true), + ) +} + +func (s *QuerierSuite) TestRangeQueryLastOverTimeStep50() { + // Arrange + s.appendTimeSeries(s.timeSeries) + err := querier.SetSelectFuncOptimize("all") + s.Require().NoError(err) + defer querier.SetDefaultOptimizeType() + + q := querier.NewQuerier(s.head, querier.NewNoOpShardedDeduplicator, 0, 200, nil, nil) + defer func() { _ = q.Close() }() + s.hints.Func = "last_over_time" + s.hints.Step = 50 + + // Act + seriesSet := q.Select(s.context, false, s.hints, s.matcher) + + // Assert + s.Equal( + []storagetest.TimeSeries{ + { + Labels: labels.FromStrings("__name__", "metric", "job", "test"), + Samples: []cppbridge.Sample{ + {Timestamp: 40, Value: 10}, + {Timestamp: 80, Value: 30}, + {Timestamp: 150, Value: 130}, + {Timestamp: 190, Value: 340}, + }, + }, + { + Labels: labels.FromStrings("__name__", "metric", "job", "test2"), + Samples: []cppbridge.Sample{ + {Timestamp: 50, Value: 2}, + {Timestamp: 100, Value: 8}, + {Timestamp: 140, Value: 21}, + {Timestamp: 180, Value: 55}, + }, + }, + }, + storagetest.TimeSeriesFromSeriesSet(seriesSet, true), + ) +} + +func (s *QuerierSuite) TestRangeQueryRateStep100() { + // Arrange + s.appendTimeSeries(s.timeSeries) + err := querier.SetSelectFuncOptimize("all") + s.Require().NoError(err) + defer querier.SetDefaultOptimizeType() + + q := querier.NewQuerier(s.head, querier.NewNoOpShardedDeduplicator, 0, 200, nil, nil) + defer func() { _ = q.Close() }() + s.hints.Func = "rate" + s.hints.Step = 100 + + // Act + seriesSet := q.Select(s.context, false, s.hints, s.matcher) + + // Assert + s.Equal( + []storagetest.TimeSeries{ + { + Labels: labels.FromStrings("__name__", "metric", "job", "test"), + Samples: []cppbridge.Sample{ + {Timestamp: 20, Value: 10}, + {Timestamp: 80, Value: 30}, + {Timestamp: 110, Value: 50}, + {Timestamp: 190, Value: 340}, + }, + }, + { + Labels: labels.FromStrings("__name__", "metric", "job", "test2"), + Samples: []cppbridge.Sample{ + {Timestamp: 10, Value: 1}, + {Timestamp: 100, Value: 8}, + {Timestamp: 120, Value: 13}, + {Timestamp: 180, Value: 55}, + }, + }, + }, + storagetest.TimeSeriesFromSeriesSet(seriesSet, true), + ) +} + +func (s *QuerierSuite) TestRangeQueryRateStep50() { + // Arrange + s.appendTimeSeries(s.timeSeries) + err := querier.SetSelectFuncOptimize("all") + s.Require().NoError(err) + defer querier.SetDefaultOptimizeType() + + q := querier.NewQuerier(s.head, querier.NewNoOpShardedDeduplicator, 0, 200, nil, nil) + defer func() { _ = q.Close() }() + s.hints.Func = "rate" + s.hints.Step = 50 + + // Act + seriesSet := q.Select(s.context, false, s.hints, s.matcher) + + // Assert + s.Equal( + []storagetest.TimeSeries{ + { + Labels: labels.FromStrings("__name__", "metric", "job", "test"), + Samples: []cppbridge.Sample{ + {Timestamp: 20, Value: 10}, + {Timestamp: 40, Value: 10}, + {Timestamp: 60, Value: 20}, + {Timestamp: 80, Value: 30}, + {Timestamp: 110, Value: 50}, + {Timestamp: 150, Value: 130}, + {Timestamp: 170, Value: 210}, + {Timestamp: 190, Value: 340}, + }, + }, + { + Labels: labels.FromStrings("__name__", "metric", "job", "test2"), + Samples: []cppbridge.Sample{ + {Timestamp: 10, Value: 1}, + {Timestamp: 50, Value: 2}, + {Timestamp: 70, Value: 3}, + {Timestamp: 100, Value: 8}, + {Timestamp: 120, Value: 13}, + {Timestamp: 140, Value: 21}, + {Timestamp: 160, Value: 34}, + {Timestamp: 180, Value: 55}, + }, + }, + }, + storagetest.TimeSeriesFromSeriesSet(seriesSet, true), + ) +} + +func (s *QuerierSuite) TestRangeQueryIRateStep100() { + // Arrange + s.appendTimeSeries(s.timeSeries) + err := querier.SetSelectFuncOptimize("all") + s.Require().NoError(err) + defer querier.SetDefaultOptimizeType() + + q := querier.NewQuerier(s.head, querier.NewNoOpShardedDeduplicator, 0, 200, nil, nil) + defer func() { _ = q.Close() }() + s.hints.Func = "irate" + s.hints.Step = 100 + + // Act + seriesSet := q.Select(s.context, false, s.hints, s.matcher) + + // Assert + s.Equal( + []storagetest.TimeSeries{ + { + Labels: labels.FromStrings("__name__", "metric", "job", "test"), + Samples: []cppbridge.Sample{ + {Timestamp: 60, Value: 20}, + {Timestamp: 80, Value: 30}, + {Timestamp: 170, Value: 210}, + {Timestamp: 190, Value: 340}, + }, + }, + { + Labels: labels.FromStrings("__name__", "metric", "job", "test2"), + Samples: []cppbridge.Sample{ + {Timestamp: 90, Value: 5}, + {Timestamp: 100, Value: 8}, + {Timestamp: 160, Value: 34}, + {Timestamp: 180, Value: 55}, + }, + }, + }, + storagetest.TimeSeriesFromSeriesSet(seriesSet, true), + ) +} + +func (s *QuerierSuite) TestRangeQueryIRateStep10() { + // Arrange + s.appendTimeSeries(s.timeSeries) + err := querier.SetSelectFuncOptimize("all") + s.Require().NoError(err) + defer querier.SetDefaultOptimizeType() + + q := querier.NewQuerier(s.head, querier.NewNoOpShardedDeduplicator, 0, 200, nil, nil) + defer func() { _ = q.Close() }() + s.hints.Func = "irate" + s.hints.Step = 10 + + // Act + seriesSet := q.Select(s.context, false, s.hints, s.matcher) + + // Assert + s.Equal( + []storagetest.TimeSeries{ + { + Labels: labels.FromStrings("__name__", "metric", "job", "test"), + Samples: nil, + }, + { + Labels: labels.FromStrings("__name__", "metric", "job", "test2"), + Samples: nil, + }, + }, + storagetest.TimeSeriesFromSeriesSet(seriesSet, true), + ) +} From 2f1ba7ec8d1176ec9f1396257ece4b896574caf6 Mon Sep 17 00:00:00 2001 From: Alexandr Yudin <57181751+u-veles-a@users.noreply.github.com> Date: Thu, 28 May 2026 07:55:48 +0000 Subject: [PATCH 20/37] WIP: add cross test Signed-off-by: Alexandr Yudin <57181751+u-veles-a@users.noreply.github.com> --- .../storage/querier/querier_optimize_test.go | 39 +- pp/go/storage/querier/querier_test.go | 553 ------------------ 2 files changed, 23 insertions(+), 569 deletions(-) diff --git a/pp/go/storage/querier/querier_optimize_test.go b/pp/go/storage/querier/querier_optimize_test.go index c16fe4222..f61741b48 100644 --- a/pp/go/storage/querier/querier_optimize_test.go +++ b/pp/go/storage/querier/querier_optimize_test.go @@ -455,17 +455,20 @@ func (s *QuerierOptimizeSuite) SetupSuite() { s.queryOpts = promql.NewPrometheusQueryOpts(false, s.lookbackDelta) s.queryFuncs = []queryFunc{ - // {name: "rate", needRange: true}, // - - // {name: "irate", needRange: true}, // - - // {name: "delta", needRange: true}, // - - // {name: "idelta", needRange: true}, // - - // {name: "increase", needRange: true}, // - - {name: "min_over_time", needRange: true}, // + - {name: "max_over_time", needRange: true}, // + - {name: "last_over_time", needRange: true}, // + - // {name: "sum_over_time", needRange: true}, // - - // {name: "resets", needRange: true}, // - - {name: "changes", needRange: true}, // + + // // {name: "rate", needRange: true}, // - + // // {name: "irate", needRange: true}, // - + // // {name: "delta", needRange: true}, // - + // // {name: "idelta", needRange: true}, // - + // // {name: "increase", needRange: true}, // - + // {name: "min_over_time", needRange: true}, // + + // {name: "max_over_time", needRange: true}, // + + // {name: "last_over_time", needRange: true}, // + + // // {name: "sum_over_time", needRange: true}, // - + // // {name: "resets", needRange: true}, // - + // {name: "changes", needRange: true}, // + + {name: "min", needRange: false}, // + + {name: "max", needRange: false}, // + + // {name: "sum", needRange: false}, // + } q, err := s.Querier(s.start.UnixMilli(), s.end.UnixMilli()) @@ -699,17 +702,21 @@ func (s *MatrixQuerierOptimizeSuiteSuite) rangeArgs(fn func( fn(qFunc, metricName, step, subq, mod, o) } } + + if !qFunc.needRange { + break // skip subQuery + } } } } } } -// rangeArgsWithStep runs the given function for all combinations of +// rangeArgsWithoutStep runs the given function for all combinations of // query functions, metric names, subqueries, modifiers and offsets. // //revive:disable-next-line:cognitive-complexity // matrix test -func (s *MatrixQuerierOptimizeSuiteSuite) rangeArgsWithStep(fn func( +func (s *MatrixQuerierOptimizeSuiteSuite) rangeArgsWithoutStep(fn func( qFunc queryFunc, metricName string, subq subQuery, @@ -775,7 +782,7 @@ func (s *MatrixQuerierOptimizeSuiteSuite) TestQueryRangeWithStep() { func (s *MatrixQuerierOptimizeSuiteSuite) TestQueryInstantStart() { ctx := s.T().Context() - s.rangeArgsWithStep(func(qFunc queryFunc, metricName string, subq subQuery, mod modifier, o offset) { + s.rangeArgsWithoutStep(func(qFunc queryFunc, metricName string, subq subQuery, mod modifier, o offset) { query := qFunc.toQueryString(metricName, subq, mod, o) s.Run(query, func() { res, err := queryInstant(ctx, "none", s.queryEngine, s, s.queryOpts, query, s.start) @@ -794,7 +801,7 @@ func (s *MatrixQuerierOptimizeSuiteSuite) TestQueryInstantStart() { func (s *MatrixQuerierOptimizeSuiteSuite) TestQueryInstantMiddle() { ctx := s.T().Context() - s.rangeArgsWithStep(func(qFunc queryFunc, metricName string, subq subQuery, mod modifier, o offset) { + s.rangeArgsWithoutStep(func(qFunc queryFunc, metricName string, subq subQuery, mod modifier, o offset) { query := qFunc.toQueryString(metricName, subq, mod, o) s.Run(query, func() { res, err := queryInstant(ctx, "none", s.queryEngine, s, s.queryOpts, query, s.start.Add(s.step*90)) @@ -813,7 +820,7 @@ func (s *MatrixQuerierOptimizeSuiteSuite) TestQueryInstantMiddle() { func (s *MatrixQuerierOptimizeSuiteSuite) TestQueryInstantEnd() { ctx := s.T().Context() - s.rangeArgsWithStep(func(qFunc queryFunc, metricName string, subq subQuery, mod modifier, o offset) { + s.rangeArgsWithoutStep(func(qFunc queryFunc, metricName string, subq subQuery, mod modifier, o offset) { query := qFunc.toQueryString(metricName, subq, mod, o) s.Run(query, func() { res, err := queryInstant(ctx, "none", s.queryEngine, s, s.queryOpts, query, s.end) diff --git a/pp/go/storage/querier/querier_test.go b/pp/go/storage/querier/querier_test.go index 89c977035..5a43e606a 100644 --- a/pp/go/storage/querier/querier_test.go +++ b/pp/go/storage/querier/querier_test.go @@ -554,556 +554,3 @@ func (s *QuerierSuite) TestLabelValuesNoMatchesOnName() { s.Equal([]string{}, names) s.Len(anns.AsErrors(), 1) } - -func (s *QuerierSuite) TestRangeQuerySumOverTimeStep100() { - // Arrange - s.appendTimeSeries(s.timeSeries) - err := querier.SetSelectFuncOptimize("all") - s.Require().NoError(err) - defer querier.SetDefaultOptimizeType() - - q := querier.NewQuerier(s.head, querier.NewNoOpShardedDeduplicator, 0, s.hints.End, nil, nil) - defer func() { _ = q.Close() }() - - s.hints.Func = "sum_over_time" - s.hints.Step = 100 - - // Act - seriesSet := q.Select(s.context, false, s.hints, s.matcher) - - // Assert - s.Equal( - []storagetest.TimeSeries{ - { - Labels: labels.FromStrings("__name__", "metric", "job", "test"), - Samples: []cppbridge.Sample{ - {Timestamp: 99, Value: 70}, - {Timestamp: 199, Value: 810}, - }, - }, - { - Labels: labels.FromStrings("__name__", "metric", "job", "test2"), - Samples: []cppbridge.Sample{ - {Timestamp: 99, Value: 20}, - {Timestamp: 199, Value: 123}, - }, - }, - }, - storagetest.TimeSeriesFromSeriesSet(seriesSet, true), - ) -} - -func (s *QuerierSuite) TestRangeQuerySumOverTimeStep50() { - // Arrange - s.appendTimeSeries(s.timeSeries) - err := querier.SetSelectFuncOptimize("all") - s.Require().NoError(err) - defer querier.SetDefaultOptimizeType() - - q := querier.NewQuerier(s.head, querier.NewNoOpShardedDeduplicator, 0, 200, nil, nil) - defer func() { _ = q.Close() }() - s.hints.Func = "sum_over_time" - s.hints.Step = 50 - - // Act - seriesSet := q.Select(s.context, false, s.hints, s.matcher) - - // Assert - s.Equal( - []storagetest.TimeSeries{ - { - Labels: labels.FromStrings("__name__", "metric", "job", "test"), - Samples: []cppbridge.Sample{ - {Timestamp: 49, Value: 20}, - {Timestamp: 99, Value: 50}, - {Timestamp: 149, Value: 260}, - {Timestamp: 199, Value: 550}, - }, - }, - { - Labels: labels.FromStrings("__name__", "metric", "job", "test2"), - Samples: []cppbridge.Sample{ - {Timestamp: 49, Value: 4}, - {Timestamp: 99, Value: 16}, - {Timestamp: 149, Value: 34}, - {Timestamp: 199, Value: 89}, - }, - }, - }, - storagetest.TimeSeriesFromSeriesSet(seriesSet, true), - ) -} - -func (s *QuerierSuite) TestRangeQueryMinOverTimeStep100() { - // Arrange - s.appendTimeSeries(s.timeSeries) - err := querier.SetSelectFuncOptimize("all") - s.Require().NoError(err) - defer querier.SetDefaultOptimizeType() - - q := querier.NewQuerier(s.head, querier.NewNoOpShardedDeduplicator, 0, 200, nil, nil) - defer func() { _ = q.Close() }() - s.hints.Func = "min_over_time" - s.hints.Step = 100 - - // Act - seriesSet := q.Select(s.context, false, s.hints, s.matcher) - - // Assert - s.Equal( - []storagetest.TimeSeries{ - { - Labels: labels.FromStrings("__name__", "metric", "job", "test"), - Samples: []cppbridge.Sample{ - {Timestamp: 20, Value: 10}, - {Timestamp: 110, Value: 50}, - }, - }, - { - Labels: labels.FromStrings("__name__", "metric", "job", "test2"), - Samples: []cppbridge.Sample{ - {Timestamp: 10, Value: 1}, - {Timestamp: 120, Value: 13}, - }, - }, - }, - storagetest.TimeSeriesFromSeriesSet(seriesSet, true), - ) -} - -func (s *QuerierSuite) TestRangeQueryMinOverTimeStep50() { - // Arrange - s.appendTimeSeries(s.timeSeries) - err := querier.SetSelectFuncOptimize("all") - s.Require().NoError(err) - defer querier.SetDefaultOptimizeType() - - q := querier.NewQuerier(s.head, querier.NewNoOpShardedDeduplicator, 0, 200, nil, nil) - defer func() { _ = q.Close() }() - s.hints.Func = "min_over_time" - s.hints.Step = 50 - - // Act - seriesSet := q.Select(s.context, false, s.hints, s.matcher) - - // Assert - s.Equal( - []storagetest.TimeSeries{ - { - Labels: labels.FromStrings("__name__", "metric", "job", "test"), - Samples: []cppbridge.Sample{ - {Timestamp: 20, Value: 10}, - {Timestamp: 60, Value: 20}, - {Timestamp: 110, Value: 50}, - {Timestamp: 170, Value: 210}, - }, - }, - { - Labels: labels.FromStrings("__name__", "metric", "job", "test2"), - Samples: []cppbridge.Sample{ - {Timestamp: 10, Value: 1}, - {Timestamp: 70, Value: 3}, - {Timestamp: 120, Value: 13}, - {Timestamp: 160, Value: 34}, - }, - }, - }, - storagetest.TimeSeriesFromSeriesSet(seriesSet, true), - ) -} - -func (s *QuerierSuite) TestRangeQueryMaxOverTimeStep100() { - // Arrange - s.appendTimeSeries(s.timeSeries) - err := querier.SetSelectFuncOptimize("all") - s.Require().NoError(err) - defer querier.SetDefaultOptimizeType() - - q := querier.NewQuerier(s.head, querier.NewNoOpShardedDeduplicator, 0, 200, nil, nil) - defer func() { _ = q.Close() }() - s.hints.Func = "max_over_time" - s.hints.Step = 100 - - // Act - seriesSet := q.Select(s.context, false, s.hints, s.matcher) - - // Assert - s.Equal( - []storagetest.TimeSeries{ - { - Labels: labels.FromStrings("__name__", "metric", "job", "test"), - Samples: []cppbridge.Sample{ - {Timestamp: 80, Value: 30}, - {Timestamp: 190, Value: 340}, - }, - }, - { - Labels: labels.FromStrings("__name__", "metric", "job", "test2"), - Samples: []cppbridge.Sample{ - {Timestamp: 100, Value: 8}, - {Timestamp: 180, Value: 55}, - }, - }, - }, - storagetest.TimeSeriesFromSeriesSet(seriesSet, true), - ) -} - -func (s *QuerierSuite) TestRangeQueryMaxOverTimeStep50() { - // Arrange - s.appendTimeSeries(s.timeSeries) - err := querier.SetSelectFuncOptimize("all") - s.Require().NoError(err) - defer querier.SetDefaultOptimizeType() - - q := querier.NewQuerier(s.head, querier.NewNoOpShardedDeduplicator, 0, 200, nil, nil) - defer func() { _ = q.Close() }() - s.hints.Func = "max_over_time" - s.hints.Step = 50 - - // Act - seriesSet := q.Select(s.context, false, s.hints, s.matcher) - - // Assert - s.Equal( - []storagetest.TimeSeries{ - { - Labels: labels.FromStrings("__name__", "metric", "job", "test"), - Samples: []cppbridge.Sample{ - {Timestamp: 20, Value: 10}, - {Timestamp: 80, Value: 30}, - {Timestamp: 150, Value: 130}, - {Timestamp: 190, Value: 340}, - }, - }, - { - Labels: labels.FromStrings("__name__", "metric", "job", "test2"), - Samples: []cppbridge.Sample{ - {Timestamp: 50, Value: 2}, - {Timestamp: 100, Value: 8}, - {Timestamp: 140, Value: 21}, - {Timestamp: 180, Value: 55}, - }, - }, - }, - storagetest.TimeSeriesFromSeriesSet(seriesSet, true), - ) -} - -func (s *QuerierSuite) TestRangeQueryCountOverTimeStep100() { - // Arrange - s.appendTimeSeries(s.timeSeries) - err := querier.SetSelectFuncOptimize("all") - s.Require().NoError(err) - defer querier.SetDefaultOptimizeType() - - q := querier.NewQuerier(s.head, querier.NewNoOpShardedDeduplicator, 0, 200, nil, nil) - defer func() { _ = q.Close() }() - s.hints.Func = "count_over_time" - s.hints.Step = 100 - - // Act - seriesSet := q.Select(s.context, false, s.hints, s.matcher) - - // Assert - s.Equal( - []storagetest.TimeSeries{ - { - Labels: labels.FromStrings("__name__", "metric", "job", "test"), - Samples: []cppbridge.Sample{ - {Timestamp: 99, Value: 4}, - {Timestamp: 199, Value: 5}, - }, - }, - { - Labels: labels.FromStrings("__name__", "metric", "job", "test2"), - Samples: []cppbridge.Sample{ - {Timestamp: 99, Value: 6}, - {Timestamp: 199, Value: 4}, - }, - }, - }, - storagetest.TimeSeriesFromSeriesSet(seriesSet, true), - ) -} - -func (s *QuerierSuite) TestRangeQueryCountOverTimeStep50() { - // Arrange - s.appendTimeSeries(s.timeSeries) - err := querier.SetSelectFuncOptimize("all") - s.Require().NoError(err) - defer querier.SetDefaultOptimizeType() - - q := querier.NewQuerier(s.head, querier.NewNoOpShardedDeduplicator, 0, 200, nil, nil) - defer func() { _ = q.Close() }() - s.hints.Func = "count_over_time" - s.hints.Step = 50 - - // Act - seriesSet := q.Select(s.context, false, s.hints, s.matcher) - - // Assert - s.Equal( - []storagetest.TimeSeries{ - { - Labels: labels.FromStrings("__name__", "metric", "job", "test"), - Samples: []cppbridge.Sample{ - {Timestamp: 49, Value: 2}, - {Timestamp: 99, Value: 2}, - {Timestamp: 149, Value: 3}, - {Timestamp: 199, Value: 2}, - }, - }, - { - Labels: labels.FromStrings("__name__", "metric", "job", "test2"), - Samples: []cppbridge.Sample{ - {Timestamp: 49, Value: 3}, - {Timestamp: 99, Value: 3}, - {Timestamp: 149, Value: 2}, - {Timestamp: 199, Value: 2}, - }, - }, - }, - storagetest.TimeSeriesFromSeriesSet(seriesSet, true), - ) -} - -func (s *QuerierSuite) TestRangeQueryLastOverTimeStep100() { - // Arrange - s.appendTimeSeries(s.timeSeries) - err := querier.SetSelectFuncOptimize("all") - s.Require().NoError(err) - defer querier.SetDefaultOptimizeType() - - q := querier.NewQuerier(s.head, querier.NewNoOpShardedDeduplicator, 0, 200, nil, nil) - defer func() { _ = q.Close() }() - s.hints.Func = "last_over_time" - s.hints.Step = 100 - - // Act - seriesSet := q.Select(s.context, false, s.hints, s.matcher) - - // Assert - s.Equal( - []storagetest.TimeSeries{ - { - Labels: labels.FromStrings("__name__", "metric", "job", "test"), - Samples: []cppbridge.Sample{ - {Timestamp: 80, Value: 30}, - {Timestamp: 190, Value: 340}, - }, - }, - { - Labels: labels.FromStrings("__name__", "metric", "job", "test2"), - Samples: []cppbridge.Sample{ - {Timestamp: 100, Value: 8}, - {Timestamp: 180, Value: 55}, - }, - }, - }, - storagetest.TimeSeriesFromSeriesSet(seriesSet, true), - ) -} - -func (s *QuerierSuite) TestRangeQueryLastOverTimeStep50() { - // Arrange - s.appendTimeSeries(s.timeSeries) - err := querier.SetSelectFuncOptimize("all") - s.Require().NoError(err) - defer querier.SetDefaultOptimizeType() - - q := querier.NewQuerier(s.head, querier.NewNoOpShardedDeduplicator, 0, 200, nil, nil) - defer func() { _ = q.Close() }() - s.hints.Func = "last_over_time" - s.hints.Step = 50 - - // Act - seriesSet := q.Select(s.context, false, s.hints, s.matcher) - - // Assert - s.Equal( - []storagetest.TimeSeries{ - { - Labels: labels.FromStrings("__name__", "metric", "job", "test"), - Samples: []cppbridge.Sample{ - {Timestamp: 40, Value: 10}, - {Timestamp: 80, Value: 30}, - {Timestamp: 150, Value: 130}, - {Timestamp: 190, Value: 340}, - }, - }, - { - Labels: labels.FromStrings("__name__", "metric", "job", "test2"), - Samples: []cppbridge.Sample{ - {Timestamp: 50, Value: 2}, - {Timestamp: 100, Value: 8}, - {Timestamp: 140, Value: 21}, - {Timestamp: 180, Value: 55}, - }, - }, - }, - storagetest.TimeSeriesFromSeriesSet(seriesSet, true), - ) -} - -func (s *QuerierSuite) TestRangeQueryRateStep100() { - // Arrange - s.appendTimeSeries(s.timeSeries) - err := querier.SetSelectFuncOptimize("all") - s.Require().NoError(err) - defer querier.SetDefaultOptimizeType() - - q := querier.NewQuerier(s.head, querier.NewNoOpShardedDeduplicator, 0, 200, nil, nil) - defer func() { _ = q.Close() }() - s.hints.Func = "rate" - s.hints.Step = 100 - - // Act - seriesSet := q.Select(s.context, false, s.hints, s.matcher) - - // Assert - s.Equal( - []storagetest.TimeSeries{ - { - Labels: labels.FromStrings("__name__", "metric", "job", "test"), - Samples: []cppbridge.Sample{ - {Timestamp: 20, Value: 10}, - {Timestamp: 80, Value: 30}, - {Timestamp: 110, Value: 50}, - {Timestamp: 190, Value: 340}, - }, - }, - { - Labels: labels.FromStrings("__name__", "metric", "job", "test2"), - Samples: []cppbridge.Sample{ - {Timestamp: 10, Value: 1}, - {Timestamp: 100, Value: 8}, - {Timestamp: 120, Value: 13}, - {Timestamp: 180, Value: 55}, - }, - }, - }, - storagetest.TimeSeriesFromSeriesSet(seriesSet, true), - ) -} - -func (s *QuerierSuite) TestRangeQueryRateStep50() { - // Arrange - s.appendTimeSeries(s.timeSeries) - err := querier.SetSelectFuncOptimize("all") - s.Require().NoError(err) - defer querier.SetDefaultOptimizeType() - - q := querier.NewQuerier(s.head, querier.NewNoOpShardedDeduplicator, 0, 200, nil, nil) - defer func() { _ = q.Close() }() - s.hints.Func = "rate" - s.hints.Step = 50 - - // Act - seriesSet := q.Select(s.context, false, s.hints, s.matcher) - - // Assert - s.Equal( - []storagetest.TimeSeries{ - { - Labels: labels.FromStrings("__name__", "metric", "job", "test"), - Samples: []cppbridge.Sample{ - {Timestamp: 20, Value: 10}, - {Timestamp: 40, Value: 10}, - {Timestamp: 60, Value: 20}, - {Timestamp: 80, Value: 30}, - {Timestamp: 110, Value: 50}, - {Timestamp: 150, Value: 130}, - {Timestamp: 170, Value: 210}, - {Timestamp: 190, Value: 340}, - }, - }, - { - Labels: labels.FromStrings("__name__", "metric", "job", "test2"), - Samples: []cppbridge.Sample{ - {Timestamp: 10, Value: 1}, - {Timestamp: 50, Value: 2}, - {Timestamp: 70, Value: 3}, - {Timestamp: 100, Value: 8}, - {Timestamp: 120, Value: 13}, - {Timestamp: 140, Value: 21}, - {Timestamp: 160, Value: 34}, - {Timestamp: 180, Value: 55}, - }, - }, - }, - storagetest.TimeSeriesFromSeriesSet(seriesSet, true), - ) -} - -func (s *QuerierSuite) TestRangeQueryIRateStep100() { - // Arrange - s.appendTimeSeries(s.timeSeries) - err := querier.SetSelectFuncOptimize("all") - s.Require().NoError(err) - defer querier.SetDefaultOptimizeType() - - q := querier.NewQuerier(s.head, querier.NewNoOpShardedDeduplicator, 0, 200, nil, nil) - defer func() { _ = q.Close() }() - s.hints.Func = "irate" - s.hints.Step = 100 - - // Act - seriesSet := q.Select(s.context, false, s.hints, s.matcher) - - // Assert - s.Equal( - []storagetest.TimeSeries{ - { - Labels: labels.FromStrings("__name__", "metric", "job", "test"), - Samples: []cppbridge.Sample{ - {Timestamp: 60, Value: 20}, - {Timestamp: 80, Value: 30}, - {Timestamp: 170, Value: 210}, - {Timestamp: 190, Value: 340}, - }, - }, - { - Labels: labels.FromStrings("__name__", "metric", "job", "test2"), - Samples: []cppbridge.Sample{ - {Timestamp: 90, Value: 5}, - {Timestamp: 100, Value: 8}, - {Timestamp: 160, Value: 34}, - {Timestamp: 180, Value: 55}, - }, - }, - }, - storagetest.TimeSeriesFromSeriesSet(seriesSet, true), - ) -} - -func (s *QuerierSuite) TestRangeQueryIRateStep10() { - // Arrange - s.appendTimeSeries(s.timeSeries) - err := querier.SetSelectFuncOptimize("all") - s.Require().NoError(err) - defer querier.SetDefaultOptimizeType() - - q := querier.NewQuerier(s.head, querier.NewNoOpShardedDeduplicator, 0, 200, nil, nil) - defer func() { _ = q.Close() }() - s.hints.Func = "irate" - s.hints.Step = 10 - - // Act - seriesSet := q.Select(s.context, false, s.hints, s.matcher) - - // Assert - s.Equal( - []storagetest.TimeSeries{ - { - Labels: labels.FromStrings("__name__", "metric", "job", "test"), - Samples: nil, - }, - { - Labels: labels.FromStrings("__name__", "metric", "job", "test2"), - Samples: nil, - }, - }, - storagetest.TimeSeriesFromSeriesSet(seriesSet, true), - ) -} From 8bf7cf88c6c519521fd6c4d531ea7e1414be2489 Mon Sep 17 00:00:00 2001 From: Alexandr Yudin <57181751+u-veles-a@users.noreply.github.com> Date: Fri, 29 May 2026 10:31:29 +0000 Subject: [PATCH 21/37] WIP: add result equal Signed-off-by: Alexandr Yudin <57181751+u-veles-a@users.noreply.github.com> --- pp/go/storage/querier/agg_series_set.go | 29 +- pp/go/storage/querier/querier.go | 1 + .../querier/querier_optimize_quick_test.go | 60 +++- .../storage/querier/querier_optimize_test.go | 268 +++++++++++++++++- .../querier_optimize_variables_test.go | 2 +- .../decorator/lookback_delta_iterator.h | 6 +- 6 files changed, 332 insertions(+), 34 deletions(-) diff --git a/pp/go/storage/querier/agg_series_set.go b/pp/go/storage/querier/agg_series_set.go index a89048b60..1c4138f9d 100644 --- a/pp/go/storage/querier/agg_series_set.go +++ b/pp/go/storage/querier/agg_series_set.go @@ -161,6 +161,7 @@ func (s *AggSeries) Iterator(it chunkenc.Iterator) chunkenc.Iterator { // Labels returns the labels of the [AggSeries]. // [storage.Series] interface implementation. func (s *AggSeries) Labels() labels.Labels { + fmt.Println("Labels", s.labelSet) return s.labelSet } @@ -204,6 +205,7 @@ func NewAggChunkIterator( // //nolint:gocritic // unnamedResult not need func (it *AggChunkIterator) At() (int64, float64) { + fmt.Println("At", it.chunkIterator.Timestamp(), it.chunkIterator.Value()) return it.chunkIterator.Timestamp(), it.chunkIterator.Value() } @@ -235,39 +237,36 @@ func (*AggChunkIterator) Err() error { // [chunkenc.Iterator] interface implementation. func (it *AggChunkIterator) Next() chunkenc.ValueType { if it.nextValue() == chunkenc.ValNone { + fmt.Println("Next: ValNone") return chunkenc.ValNone } if it.AtT() > it.maxt { + fmt.Println("Next: AtT() > maxt") return chunkenc.ValNone } + fmt.Println("Next: ValFloat") + return chunkenc.ValFloat } // Seek advances the iterator forward to the first sample with a timestamp equal or greater than t. // [chunkenc.Iterator] interface implementation. func (it *AggChunkIterator) Seek(t int64) chunkenc.ValueType { - // adjust lower limit. - if t < it.mint { - t = it.mint - } - - ts := it.AtT() - if !it.isInitialized || ts < t { - panic(fmt.Sprintf("Seek: timestamp(%d) < mint(%d)", ts, t)) - // it.chunkIterator.Seek(t) - // it.isInitialized = true - // if !it.chunkIterator.HasData() { - // return chunkenc.ValNone - // } - // ts = it.AtT() + it.isInitialized = true + if t > it.AtT() { + fmt.Println("Seek: t > AtT()") + return it.Next() } - if ts > it.maxt { + if it.AtT() > it.maxt { + fmt.Println("Seek: AtT() > maxt") return chunkenc.ValNone } + fmt.Println("Seek: ValFloat") + return chunkenc.ValFloat } diff --git a/pp/go/storage/querier/querier.go b/pp/go/storage/querier/querier.go index 05eeaa667..018632149 100644 --- a/pp/go/storage/querier/querier.go +++ b/pp/go/storage/querier/querier.go @@ -413,6 +413,7 @@ func (q *Querier[TTask, TDataStorage, TLSS, TShard, THead]) selectRange( } hints = SwitchFuncOptimize(hints, q.queryOptimize) + fmt.Println("hints", hints) shardedSerializedData := poolProvider.GetSerializedData() defer poolProvider.PutSerializedData(shardedSerializedData) queryDataStorage(dsQueryRangeQuerier, q.head, lssQueryResults, shardedSerializedData, q.mint, q.maxt, hints) diff --git a/pp/go/storage/querier/querier_optimize_quick_test.go b/pp/go/storage/querier/querier_optimize_quick_test.go index 194cb7ec7..957924966 100644 --- a/pp/go/storage/querier/querier_optimize_quick_test.go +++ b/pp/go/storage/querier/querier_optimize_quick_test.go @@ -41,6 +41,27 @@ func (qp QueryParam) Generate(rd *rand.Rand, _ int) reflect.Value { return reflect.ValueOf(qp) } +// Format formats the query parameter to string. +func (qp QueryParam) Format(f fmt.State, _ rune) { + _, _ = fmt.Fprintf( + f, + "{start: %d, end: %d, step: %d}", + qp.Start.UnixMilli(), + qp.End.UnixMilli(), + qp.Step.Milliseconds(), + ) +} + +// String converts the query parameter to string. +func (qp *QueryParam) String() string { + return fmt.Sprintf( + "{start: %d, end: %d, step: %d}", + qp.Start.UnixMilli(), + qp.End.UnixMilli(), + qp.Step.Milliseconds(), + ) +} + // gen generates a random query parameter. func (qp *QueryParam) gen(rd *rand.Rand) { qp.Start = time.UnixMilli(defaultStartMs - defaultJitterMs + rd.Int63n(2*defaultJitterMs)) @@ -80,6 +101,27 @@ func (sqp SubQueryParams) Generate(rd *rand.Rand, _ int) reflect.Value { return reflect.ValueOf(sqp) } +// Format formats the subquery parameter to string. +func (sqp SubQueryParams) Format(f fmt.State, _ rune) { + _, _ = fmt.Fprintf( + f, + "{query_param: %s, subQueryStep: %d, subQueryRange: %d}", + sqp.QueryParam.String(), + sqp.SubQueryStep.Milliseconds(), + sqp.SubQueryRange.Milliseconds(), + ) +} + +// String converts the subquery parameter to string. +func (sqp *SubQueryParams) String() string { + return fmt.Sprintf( + "{query_param: %s, subQueryStep: %d, subQueryRange: %d}", + sqp.QueryParam.String(), + sqp.SubQueryStep.Milliseconds(), + sqp.SubQueryRange.Milliseconds(), + ) +} + // subGen generates a random subquery parameter. func (sqp *SubQueryParams) subGen(rd *rand.Rand) { sqp.gen(rd) @@ -193,7 +235,7 @@ func (s *QuickQuerierOptimizeSuite) SetupSuite() { s.quickQE = promql.NewEngine(promql.EngineOpts{ Logger: log.NewNopLogger(), MaxSamples: 500000, - Timeout: 10 * time.Second, + Timeout: 100 * time.Second, LookbackDelta: s.lookbackDelta, NoStepSubqueryIntervalFn: func(int64) int64 { return s.lookbackDelta.Milliseconds() }, EnableAtModifier: true, @@ -222,7 +264,7 @@ func (s *QuickQuerierOptimizeSuite) TestQueryRangeQuickQueryParam() { s.Require().NoError(err) defer resOpt.qry.Close() - return s.Equal(res.res, resOpt.res) + return s.True(resultEqual(res.res, resOpt.res, query)) } s.Require().NoError(quick.Check(f, nil)) @@ -251,7 +293,7 @@ func (s *QuickQuerierOptimizeSuite) TestQueryRangeQuickSubQueryParams() { s.Require().NoError(err) defer resOpt.qry.Close() - return s.Equal(res.res, resOpt.res) + return s.True(resultEqual(res.res, resOpt.res, query)) } s.Require().NoError(quick.Check(f, nil)) @@ -280,7 +322,7 @@ func (s *QuickQuerierOptimizeSuite) TestQueryRangeQuickModifierQueryParams() { s.Require().NoError(err) defer resOpt.qry.Close() - return s.Equal(res.res, resOpt.res) + return s.True(resultEqual(res.res, resOpt.res, query)) } s.Require().NoError(quick.Check(f, nil)) @@ -309,7 +351,7 @@ func (s *QuickQuerierOptimizeSuite) TestQueryRangeQuickOffsetQueryParams() { s.Require().NoError(err) defer resOpt.qry.Close() - return s.Equal(res.res, resOpt.res) + return s.True(resultEqual(res.res, resOpt.res, query)) } s.Require().NoError(quick.Check(f, nil)) @@ -338,7 +380,7 @@ func (s *QuickQuerierOptimizeSuite) TestQueryInstantQuickQueryParam() { s.Require().NoError(err) defer resOpt.qry.Close() - return s.Equal(res.res, resOpt.res) + return s.True(resultEqual(res.res, resOpt.res, query)) } s.Require().NoError(quick.Check(f, nil)) @@ -367,7 +409,7 @@ func (s *QuickQuerierOptimizeSuite) TestQueryInstantQuickSubQueryParams() { s.Require().NoError(err) defer resOpt.qry.Close() - return s.Equal(res.res, resOpt.res) + return s.True(resultEqual(res.res, resOpt.res, query)) } s.Require().NoError(quick.Check(f, nil)) @@ -396,7 +438,7 @@ func (s *QuickQuerierOptimizeSuite) TestQueryInstantQuickModifierQueryParams() { s.Require().NoError(err) defer resOpt.qry.Close() - return s.Equal(res.res, resOpt.res) + return s.True(resultEqual(res.res, resOpt.res, query)) } s.Require().NoError(quick.Check(f, nil)) @@ -425,7 +467,7 @@ func (s *QuickQuerierOptimizeSuite) TestQueryInstantQuickOffsetQueryParams() { s.Require().NoError(err) defer resOpt.qry.Close() - return s.Equal(res.res, resOpt.res) + return s.True(resultEqual(res.res, resOpt.res, query)) } s.Require().NoError(quick.Check(f, nil)) diff --git a/pp/go/storage/querier/querier_optimize_test.go b/pp/go/storage/querier/querier_optimize_test.go index 4a0ac5f14..92dc9175f 100644 --- a/pp/go/storage/querier/querier_optimize_test.go +++ b/pp/go/storage/querier/querier_optimize_test.go @@ -3,9 +3,11 @@ package querier_test import ( "context" "fmt" + "maps" "math" "os" "path/filepath" + "strings" "testing" "time" @@ -23,6 +25,7 @@ import ( "github.com/prometheus/prometheus/pp/go/storage/querier" "github.com/prometheus/prometheus/pp/go/storage/storagetest" "github.com/prometheus/prometheus/promql" + "github.com/prometheus/prometheus/promql/parser" prom_storage "github.com/prometheus/prometheus/storage" ) @@ -459,9 +462,9 @@ func (s *QuerierOptimizeSuite) SetupSuite() { // // {name: "sum_over_time", needRange: true}, // - // // {name: "resets", needRange: true}, // - // {name: "changes", needRange: true}, // + - {name: "min", needRange: false}, // + - {name: "max", needRange: false}, // + - // {name: "sum", needRange: false}, // + + // {name: "min", needRange: false}, // + + // {name: "max", needRange: false}, // + + {name: "sum", needRange: false}, // + } q, err := s.Querier(s.start.UnixMilli(), s.end.UnixMilli()) @@ -606,6 +609,10 @@ func (s *QuerierOptimizeSuite) fillHead() { valueCounter++ } + fmt.Println("timeSeries 5:", timeSeries[5].Samples[len(timeSeries[5].Samples)-1]) + fmt.Println("timeSeries 6:", timeSeries[6].Samples[len(timeSeries[6].Samples)-1]) + fmt.Println("timeSeries 7:", timeSeries[7].Samples[len(timeSeries[7].Samples)-1]) + s.appendTimeSeries(timeSeries) } @@ -712,6 +719,10 @@ func (s *MatrixQuerierOptimizeSuiteSuite) rangeArgsWithoutStep(fn func( fn(qFunc, metricName, subq, mod, o) } } + + if !qFunc.needRange { + break // skip subQuery + } } } } @@ -731,7 +742,7 @@ func (s *MatrixQuerierOptimizeSuiteSuite) TestQueryRange() { s.Require().NoError(err) defer resOpt.qry.Close() - s.Require().Equal(res.res, resOpt.res) + s.Require().True(resultEqual(res.res, resOpt.res, query)) }) }) } @@ -750,7 +761,7 @@ func (s *MatrixQuerierOptimizeSuiteSuite) TestQueryInstantStart() { s.Require().NoError(err) defer resOpt.qry.Close() - s.Require().Equal(res.res, resOpt.res) + s.Require().True(resultEqual(res.res, resOpt.res, query)) }) }) } @@ -769,7 +780,7 @@ func (s *MatrixQuerierOptimizeSuiteSuite) TestQueryInstantMiddle() { s.Require().NoError(err) defer resOpt.qry.Close() - s.Require().Equal(res.res, resOpt.res) + s.Require().True(resultEqual(res.res, resOpt.res, query)) }) }) } @@ -788,7 +799,7 @@ func (s *MatrixQuerierOptimizeSuiteSuite) TestQueryInstantEnd() { s.Require().NoError(err) defer resOpt.qry.Close() - s.Require().Equal(res.res, resOpt.res) + s.Require().True(resultEqual(res.res, resOpt.res, query)) }) }) } @@ -808,7 +819,248 @@ func (s *MatrixQuerierOptimizeSuiteSuite) TestQueryRangeSingle() { s.Require().NoError(err) defer resOpt.qry.Close() - s.Require().Equal(res.res, resOpt.res) + s.Require().True(resultEqual(res.res, resOpt.res, query)) }) } } + +func (s *MatrixQuerierOptimizeSuiteSuite) TestQueryRangeSingle2() { + ctx := s.T().Context() + query := "sum(counter_metric)" + start := time.UnixMilli(1779290900885) + end := time.UnixMilli(1779298553283) + step := time.Duration(7424188 * 1e6) + // for _, metricName := range s.metricNames { + // query := fmt.Sprintf(queryF, metricName) + s.Run(query, func() { + res, err := queryRange(ctx, "none", s.queryEngine, s, s.queryOpts, query, start, end, step) + s.Require().NoError(err) + defer res.qry.Close() + + resOpt, err := queryRange(ctx, "all", s.queryEngine, s, s.queryOpts, query, start, end, step) + s.Require().NoError(err) + defer resOpt.qry.Close() + + s.Require().True(resultEqual(res.res, resOpt.res, query)) + }) + // } +} + +// +// resultEqual +// + +// defaultEpsilon is the default epsilon for comparing two values. +var defaultEpsilon = 0.0000000000001 + +// resultEqual compares two results. +// +//nolint:gocritic // unnamedResult // comporator +func resultEqual(exp, act *promql.Result, query string) (bool, string) { + if exp == nil && act == nil { + return true, "" + } + + if exp == nil || act == nil { + return false, fmt.Sprintf("one of the results is nil: %s", query) + } + + if exp.Err != act.Err { + return false, fmt.Sprintf("expected error %v, got %v: %s", exp.Err, act.Err, query) + } + + if !maps.Equal(exp.Warnings, act.Warnings) { + return false, fmt.Sprintf("expected warnings %v, got %v: %s", exp.Warnings, act.Warnings, query) + } + + if eq, result := valueEqual(exp.Value, act.Value); !eq { + return false, fmt.Sprintf("query: %s\n%s", query, result) + } + + return true, "" +} + +// valueEqual compares two values. +// +//nolint:gocritic // unnamedResult // comporator +func valueEqual(exp, act parser.Value) (bool, string) { + if exp == nil && act == nil { + return true, "" + } + + if exp == nil || act == nil { + return false, "one of the values is nil" + } + + if exp.Type() != act.Type() { + return false, fmt.Sprintf("value type: expected %s, got %s", exp.Type(), act.Type()) + } + + switch expType := exp.(type) { + case promql.Scalar: + return scalarEqual(expType, act.(promql.Scalar)) + + case promql.Vector: + return vectorEqual(expType, act.(promql.Vector)) + + case promql.Matrix: + return matrixEqual(expType, act.(promql.Matrix)) + + default: + return false, fmt.Sprintf("expected scalar, vector or matrix, got %T", exp) + } +} + +// scalarEqual compares two scalars. +// +//nolint:gocritic // unnamedResult // comporator +func scalarEqual(exp, act promql.Scalar) (bool, string) { + if exp.T != act.T || !inEpsilon(exp.V, act.V, defaultEpsilon) { + return false, fmt.Sprintf("scalar: %s != %s", exp, act) + } + + return true, "" +} + +// vectorEqual compares two vectors. +// +//nolint:gocritic // unnamedResult // comporator +func vectorEqual(exp, act promql.Vector) (bool, string) { + if len(exp) != len(act) { + return false, fmt.Sprintf("vector: length: %d != %d", len(exp), len(act)) + } + + msg := strings.Builder{} + _, _ = msg.WriteString("vector:\n") + isEqual := true + + for i, v := range exp { + if eq, result := sampleEqual(v, act[i]); !eq { + _, _ = msg.WriteString(result) + isEqual = false + } + } + + if isEqual { + msg.Reset() + } + + return isEqual, msg.String() +} + +// sampleEqual compares two samples. +// +//nolint:gocritic // unnamedResult // comporator +func sampleEqual(exp, act promql.Sample) (bool, string) { + if !labels.Equal(exp.Metric, act.Metric) { + return false, fmt.Sprintf("labels: %s != %s\n", exp.Metric, act.Metric) + } + + msg := strings.Builder{} + _, _ = fmt.Fprintf(&msg, "labels: %s\n", exp.Metric) + isEqual := true + + if exp.T != act.T || !inEpsilon(exp.F, act.F, defaultEpsilon) { + _, _ = fmt.Fprintf( + &msg, + "floats:\n %s != %s\n", + promql.FPoint{T: exp.T, F: exp.F}, + promql.FPoint{T: act.T, F: act.F}, + ) + isEqual = false + } + + if isEqual { + msg.Reset() + } + + return isEqual, msg.String() +} + +// matrixEqual compares two matrices. +// +//nolint:gocritic // unnamedResult // comporator +func matrixEqual(exp, act promql.Matrix) (bool, string) { + if len(exp) != len(act) { + return false, fmt.Sprintf("matrix: length: %d != %d", len(exp), len(act)) + } + + msg := strings.Builder{} + _, _ = msg.WriteString("matrix:\n") + isEqual := true + + for i, v := range exp { + if eq, result := seriesEqual(v, act[i]); !eq { + _, _ = msg.WriteString(result) + isEqual = false + } + } + + if isEqual { + msg.Reset() + } + + return isEqual, msg.String() +} + +// seriesEqual compares two series. +// +//nolint:gocritic // unnamedResult // comporator +func seriesEqual(exp, act promql.Series) (bool, string) { + if !labels.Equal(exp.Metric, act.Metric) { + return false, fmt.Sprintf("labels: %s != %s\n", exp.Metric, act.Metric) + } + + msg := strings.Builder{} + isEqual := true + _, _ = fmt.Fprintf(&msg, "labels: %s\n", exp.Metric) + + if len(exp.Floats) != len(act.Floats) { + _, _ = fmt.Fprintf(&msg, "floats: length: %d != %d\n", len(exp.Floats), len(act.Floats)) + _, _ = fmt.Fprintf(&msg, " exp: %s\n", exp.Floats) + _, _ = fmt.Fprintf(&msg, " act: %s\n", act.Floats) + return false, msg.String() + } + + _, _ = msg.WriteString("floats:\n") + + for i, v := range exp.Floats { + if v.T != act.Floats[i].T || !inEpsilon(v.F, act.Floats[i].F, defaultEpsilon) { + // if v.T != act.Floats[i].T || v.F != act.Floats[i].F { + _, _ = fmt.Fprintf(&msg, " %s != %s\n", v, act.Floats[i]) + isEqual = false + } + } + + if isEqual { + msg.Reset() + } + + return isEqual, msg.String() +} + +// inEpsilon checks if two values are within epsilon. +func inEpsilon(expected, actual, epsilon float64) bool { + if math.IsNaN(expected) && math.IsNaN(actual) { + return true + } + + if math.IsNaN(expected) || math.IsNaN(actual) { + return false + } + + if expected == 0 && actual == 0 { + return true + } + + if expected == 0 || actual == 0 { + return false + } + + return calcRelative(expected, actual) <= epsilon +} + +// calcRelative calculates the relative between two values. +func calcRelative(expected, actual float64) float64 { + return math.Abs(expected-actual) / math.Abs(expected) +} diff --git a/pp/go/storage/querier/querier_optimize_variables_test.go b/pp/go/storage/querier/querier_optimize_variables_test.go index c4aa40641..58163a62d 100644 --- a/pp/go/storage/querier/querier_optimize_variables_test.go +++ b/pp/go/storage/querier/querier_optimize_variables_test.go @@ -73,7 +73,7 @@ func (s *MatrixQuerierOptimizeSuiteSuite) TestQueryRangeWithStep() { s.Require().NoError(err) defer resOpt.qry.Close() - s.Require().Equal(res.res, resOpt.res) + s.Require().True(resultEqual(res.res, resOpt.res, query)) }) }) } diff --git a/pp/series_data/decoder/decorator/lookback_delta_iterator.h b/pp/series_data/decoder/decorator/lookback_delta_iterator.h index c09940eb9..9c5e818c0 100644 --- a/pp/series_data/decoder/decorator/lookback_delta_iterator.h +++ b/pp/series_data/decoder/decorator/lookback_delta_iterator.h @@ -1,5 +1,6 @@ #pragma once +#include #include "primitives/primitives.h" #include "series_data/decoder/universal_decode_iterator.h" @@ -45,6 +46,8 @@ class LookbackDeltaIterator { PromPP::Primitives::Timestamp lookback_delta_; PROMPP_ALWAYS_INLINE void update_sample() noexcept { + std::cout << "update_sample: " << iterator_->timestamp << " v: " << iterator_->value << std::endl; + if (iterator_ != DecodeIteratorSentinel{}) [[likely]] { if (BareBones::Encoding::Gorilla::isstalenan(iterator_->value)) [[unlikely]] { sample_ = kInvalidSample; @@ -57,8 +60,9 @@ class LookbackDeltaIterator { } [[nodiscard]] PROMPP_ALWAYS_INLINE bool sample_in_lookback_delta_interval() const noexcept { + std::cout << "sample_in_lookback_delta_interval: " << sample_.timestamp << " > " << iterator_.interval().max << " - " << lookback_delta_ << std::endl; return sample_.timestamp > iterator_.interval().max - lookback_delta_; } }; -} // namespace series_data::decoder::decorator \ No newline at end of file +} // namespace series_data::decoder::decorator From 6a7126c717afaac6593b9390660f16842bd014b5 Mon Sep 17 00:00:00 2001 From: Alexandr Yudin <57181751+u-veles-a@users.noreply.github.com> Date: Fri, 29 May 2026 13:24:17 +0000 Subject: [PATCH 22/37] WIP: clearing Signed-off-by: Alexandr Yudin <57181751+u-veles-a@users.noreply.github.com> --- pp/go/storage/querier/agg_series_set.go | 10 --- pp/go/storage/querier/querier.go | 1 - .../storage/querier/querier_optimize_test.go | 64 ++++++------------- .../decorator/lookback_delta_iterator.h | 3 - 4 files changed, 19 insertions(+), 59 deletions(-) diff --git a/pp/go/storage/querier/agg_series_set.go b/pp/go/storage/querier/agg_series_set.go index 1c4138f9d..30ac3ea5d 100644 --- a/pp/go/storage/querier/agg_series_set.go +++ b/pp/go/storage/querier/agg_series_set.go @@ -161,7 +161,6 @@ func (s *AggSeries) Iterator(it chunkenc.Iterator) chunkenc.Iterator { // Labels returns the labels of the [AggSeries]. // [storage.Series] interface implementation. func (s *AggSeries) Labels() labels.Labels { - fmt.Println("Labels", s.labelSet) return s.labelSet } @@ -205,7 +204,6 @@ func NewAggChunkIterator( // //nolint:gocritic // unnamedResult not need func (it *AggChunkIterator) At() (int64, float64) { - fmt.Println("At", it.chunkIterator.Timestamp(), it.chunkIterator.Value()) return it.chunkIterator.Timestamp(), it.chunkIterator.Value() } @@ -237,17 +235,13 @@ func (*AggChunkIterator) Err() error { // [chunkenc.Iterator] interface implementation. func (it *AggChunkIterator) Next() chunkenc.ValueType { if it.nextValue() == chunkenc.ValNone { - fmt.Println("Next: ValNone") return chunkenc.ValNone } if it.AtT() > it.maxt { - fmt.Println("Next: AtT() > maxt") return chunkenc.ValNone } - fmt.Println("Next: ValFloat") - return chunkenc.ValFloat } @@ -256,17 +250,13 @@ func (it *AggChunkIterator) Next() chunkenc.ValueType { func (it *AggChunkIterator) Seek(t int64) chunkenc.ValueType { it.isInitialized = true if t > it.AtT() { - fmt.Println("Seek: t > AtT()") return it.Next() } if it.AtT() > it.maxt { - fmt.Println("Seek: AtT() > maxt") return chunkenc.ValNone } - fmt.Println("Seek: ValFloat") - return chunkenc.ValFloat } diff --git a/pp/go/storage/querier/querier.go b/pp/go/storage/querier/querier.go index ba46c460d..b914b5f1d 100644 --- a/pp/go/storage/querier/querier.go +++ b/pp/go/storage/querier/querier.go @@ -413,7 +413,6 @@ func (q *Querier[TTask, TDataStorage, TLSS, TShard, THead]) selectRange( } hints = SwitchFuncOptimize(hints, q.queryOptimize) - fmt.Println("hints", hints) shardedSerializedData := poolProvider.GetSerializedData() defer poolProvider.PutSerializedData(shardedSerializedData) queryDataStorage(dsQueryRangeQuerier, q.head, lssQueryResults, shardedSerializedData, q.mint, q.maxt, hints) diff --git a/pp/go/storage/querier/querier_optimize_test.go b/pp/go/storage/querier/querier_optimize_test.go index 7bb2b1060..96e88f9c1 100644 --- a/pp/go/storage/querier/querier_optimize_test.go +++ b/pp/go/storage/querier/querier_optimize_test.go @@ -471,20 +471,21 @@ func (s *QuerierOptimizeSuite) SetupSuite() { s.queryOpts = promql.NewPrometheusQueryOpts(false, s.lookbackDelta) s.queryFuncs = []queryFunc{ - // // {name: "rate", needRange: true}, // - - // // {name: "irate", needRange: true}, // - - // // {name: "delta", needRange: true}, // - - // // {name: "idelta", needRange: true}, // - - // // {name: "increase", needRange: true}, // - - // {name: "min_over_time", needRange: true}, // + - // {name: "max_over_time", needRange: true}, // + - // {name: "last_over_time", needRange: true}, // + - // // {name: "sum_over_time", needRange: true}, // - - // // {name: "resets", needRange: true}, // - - // {name: "changes", needRange: true}, // + - // {name: "min", needRange: false}, // + - // {name: "max", needRange: false}, // + - {name: "sum", needRange: false}, // + + {name: "min_over_time", needRange: true}, // + + {name: "max_over_time", needRange: true}, // + + {name: "last_over_time", needRange: true}, // + + {name: "changes", needRange: true}, // + + {name: "min", needRange: false}, // + + {name: "max", needRange: false}, // + + {name: "sum", needRange: false}, // + + + // {name: "rate", needRange: true}, // - + // {name: "irate", needRange: true}, // - + // {name: "delta", needRange: true}, // - + // {name: "idelta", needRange: true}, // - + // {name: "increase", needRange: true}, // - + // {name: "sum_over_time", needRange: true}, // - + // {name: "resets", needRange: true}, // - } q, err := s.Querier(s.start.UnixMilli(), s.end.UnixMilli()) @@ -629,10 +630,6 @@ func (s *QuerierOptimizeSuite) fillHead() { valueCounter++ } - fmt.Println("timeSeries 5:", timeSeries[5].Samples[len(timeSeries[5].Samples)-1]) - fmt.Println("timeSeries 6:", timeSeries[6].Samples[len(timeSeries[6].Samples)-1]) - fmt.Println("timeSeries 7:", timeSeries[7].Samples[len(timeSeries[7].Samples)-1]) - s.appendTimeSeries(timeSeries) } @@ -844,28 +841,6 @@ func (s *MatrixQuerierOptimizeSuiteSuite) TestQueryRangeSingle() { } } -func (s *MatrixQuerierOptimizeSuiteSuite) TestQueryRangeSingle2() { - ctx := s.T().Context() - query := "sum(counter_metric)" - start := time.UnixMilli(1779290900885) - end := time.UnixMilli(1779298553283) - step := time.Duration(7424188 * 1e6) - // for _, metricName := range s.metricNames { - // query := fmt.Sprintf(queryF, metricName) - s.Run(query, func() { - res, err := queryRange(ctx, "none", s.queryEngine, s, s.queryOpts, query, start, end, step) - s.Require().NoError(err) - defer res.qry.Close() - - resOpt, err := queryRange(ctx, "all", s.queryEngine, s, s.queryOpts, query, start, end, step) - s.Require().NoError(err) - defer resOpt.qry.Close() - - s.Require().True(resultEqual(res.res, resOpt.res, query)) - }) - // } -} - // // resultEqual // @@ -882,15 +857,15 @@ func resultEqual(exp, act *promql.Result, query string) (bool, string) { } if exp == nil || act == nil { - return false, fmt.Sprintf("one of the results is nil: %s", query) + return false, fmt.Sprintf("query: %s\none of the results is nil", query) } if exp.Err != act.Err { - return false, fmt.Sprintf("expected error %v, got %v: %s", exp.Err, act.Err, query) + return false, fmt.Sprintf("query: %s\nerror: %v, got %v", query, exp.Err, act.Err) } if !maps.Equal(exp.Warnings, act.Warnings) { - return false, fmt.Sprintf("expected warnings %v, got %v: %s", exp.Warnings, act.Warnings, query) + return false, fmt.Sprintf("query: %s\nwarnings: %v, got %v", query, exp.Warnings, act.Warnings) } if eq, result := valueEqual(exp.Value, act.Value); !eq { @@ -1046,8 +1021,7 @@ func seriesEqual(exp, act promql.Series) (bool, string) { for i, v := range exp.Floats { if v.T != act.Floats[i].T || !inEpsilon(v.F, act.Floats[i].F, defaultEpsilon) { - // if v.T != act.Floats[i].T || v.F != act.Floats[i].F { - _, _ = fmt.Fprintf(&msg, " %s != %s\n", v, act.Floats[i]) + _, _ = fmt.Fprintf(&msg, " %s != %s\n", v, act.Floats[i]) isEqual = false } } diff --git a/pp/series_data/decoder/decorator/lookback_delta_iterator.h b/pp/series_data/decoder/decorator/lookback_delta_iterator.h index 28f1a0c39..d9bbe9eb7 100644 --- a/pp/series_data/decoder/decorator/lookback_delta_iterator.h +++ b/pp/series_data/decoder/decorator/lookback_delta_iterator.h @@ -46,8 +46,6 @@ class LookbackDeltaIterator { PromPP::Primitives::Timestamp lookback_delta_; PROMPP_ALWAYS_INLINE void update_sample() noexcept { - std::cout << "update_sample: " << iterator_->timestamp << " v: " << iterator_->value << std::endl; - if (iterator_ != DecodeIteratorSentinel{}) [[likely]] { if (BareBones::Encoding::Gorilla::isstalenan(iterator_->value)) [[unlikely]] { sample_ = kInvalidSample; @@ -63,7 +61,6 @@ class LookbackDeltaIterator { } [[nodiscard]] PROMPP_ALWAYS_INLINE bool sample_in_lookback_delta_interval() const noexcept { - std::cout << "sample_in_lookback_delta_interval: " << sample_.timestamp << " > " << iterator_.interval().max << " - " << lookback_delta_ << std::endl; return sample_.timestamp > iterator_.interval().max - lookback_delta_; } }; From d81d41db18d0ca3b7b6ff57afa22038b0974cd69 Mon Sep 17 00:00:00 2001 From: Alexandr Yudin <57181751+u-veles-a@users.noreply.github.com> Date: Fri, 29 May 2026 14:30:49 +0000 Subject: [PATCH 23/37] WIP: refactoring test Signed-off-by: Alexandr Yudin <57181751+u-veles-a@users.noreply.github.com> --- pp/go/storage/appender/appender_test.go | 22 +++--- pp/go/storage/block/writer_test.go | 4 +- .../storage/head/services/persistener_test.go | 16 ++--- pp/go/storage/loader_test.go | 2 +- .../querier/querier_optimize_quick_test.go | 9 ++- .../storage/querier/querier_optimize_test.go | 71 +++++++++---------- pp/go/storage/querier/querier_test.go | 2 +- pp/go/storage/storagetest/fixtures.go | 18 ++--- 8 files changed, 75 insertions(+), 69 deletions(-) diff --git a/pp/go/storage/appender/appender_test.go b/pp/go/storage/appender/appender_test.go index 5e7bfbade..90ecdf94a 100644 --- a/pp/go/storage/appender/appender_test.go +++ b/pp/go/storage/appender/appender_test.go @@ -143,7 +143,7 @@ func (s *AppenderSuite) TestDropInvalidSeries() { // Act stats, err := s.appender.Append( context.Background(), - storagetest.NewIncomingData(&s.Suite, []model.TimeSeries{ + storagetest.NewIncomingData(s.Require().NoError, []model.TimeSeries{ { LabelSet: model.NewLabelSetBuilder().Set("name", "metric1").Build(), Timestamp: 1, @@ -165,7 +165,7 @@ func (s *AppenderSuite) TestAppendMultipleSamplesInOneSeries() { // Act stats, err := s.appender.Append( context.Background(), - storagetest.NewIncomingData(&s.Suite, []model.TimeSeries{ + storagetest.NewIncomingData(s.Require().NoError, []model.TimeSeries{ { LabelSet: model.NewLabelSetBuilder().Set("__name__", "metric1").Build(), Timestamp: 1, @@ -213,7 +213,7 @@ func (s *AppenderSuite) TestSeriesPerShardTransfer() { // Act stats, err := s.appender.Append( context.Background(), - storagetest.NewIncomingData(&s.Suite, []model.TimeSeries{ + storagetest.NewIncomingData(s.Require().NoError, []model.TimeSeries{ { LabelSet: model.NewLabelSetBuilder().Set("__name__", "metric1").Build(), Timestamp: 1, @@ -261,7 +261,7 @@ func (s *AppenderSuite) TestShardedRelabeledSeriesFullNotEmpty() { // Act stats, err := s.appender.Append( context.Background(), - storagetest.NewIncomingData(&s.Suite, []model.TimeSeries{ + storagetest.NewIncomingData(s.Require().NoError, []model.TimeSeries{ { LabelSet: model.NewLabelSetBuilder().Set("__name__", "metric1").Build(), Timestamp: 1, @@ -330,7 +330,7 @@ func (s *AppenderSuite) TestTrackStaleness() { // Act stats, err := s.appender.Append( context.Background(), - storagetest.NewIncomingData(&s.Suite, []model.TimeSeries{ + storagetest.NewIncomingData(s.Require().NoError, []model.TimeSeries{ { LabelSet: model.NewLabelSetBuilder().Set("__name__", "metric1").Build(), Timestamp: 1, @@ -377,7 +377,7 @@ func (s *AppenderSuite) TestTrackStalenessWithoutHonorTimestamps() { // Act stats, err := s.appender.Append( context.Background(), - storagetest.NewIncomingData(&s.Suite, []model.TimeSeries{ + storagetest.NewIncomingData(s.Require().NoError, []model.TimeSeries{ { LabelSet: model.NewLabelSetBuilder().Set("__name__", "metric1").Build(), Timestamp: 1, @@ -420,7 +420,7 @@ func (s *AppenderSuite) TestWithoutCommitToWal() { // Act _, err := s.appender.Append( context.Background(), - storagetest.NewIncomingData(&s.Suite, []model.TimeSeries{ + storagetest.NewIncomingData(s.Require().NoError, []model.TimeSeries{ { LabelSet: model.NewLabelSetBuilder().Set("__name__", "metric1").Build(), Timestamp: 1, @@ -442,7 +442,7 @@ func (s *AppenderSuite) TestWithCommitToWal() { // Act _, err := s.appender.Append( context.Background(), - storagetest.NewIncomingData(&s.Suite, []model.TimeSeries{ + storagetest.NewIncomingData(s.Require().NoError, []model.TimeSeries{ { LabelSet: model.NewLabelSetBuilder().Set("__name__", "metric1").Build(), Timestamp: 1, @@ -464,7 +464,7 @@ func (s *AppenderSuite) TestWithCommitToWalByLimitExhausted() { // Act _, err := s.appender.Append( context.Background(), - storagetest.NewIncomingData(&s.Suite, []model.TimeSeries{ + storagetest.NewIncomingData(s.Require().NoError, []model.TimeSeries{ { LabelSet: model.NewLabelSetBuilder().Set("__name__", "metric1").Build(), Timestamp: 1, @@ -496,7 +496,7 @@ func (s *AppenderSuite) TestWithCommitToWalByLimitExhausted() { // Act _, _, _ = s.appender.Append( context.Background(), - storagetest.NewIncomingData(&s.Suite, []model.TimeSeries{ + storagetest.NewIncomingData(s.Require().NoError, []model.TimeSeries{ { LabelSet: model.NewLabelSetBuilder().Set("__name__", "metric1").Build(), Timestamp: 1, @@ -508,7 +508,7 @@ func (s *AppenderSuite) TestWithCommitToWalByLimitExhausted() { _, stats, err := s.appender.Append( context.Background(), - storagetest.NewIncomingData(&s.Suite, []model.TimeSeries{ + storagetest.NewIncomingData(s.Require().NoError, []model.TimeSeries{ { LabelSet: model.NewLabelSetBuilder().Set("__name__", "metric1").Build(), Timestamp: 1, diff --git a/pp/go/storage/block/writer_test.go b/pp/go/storage/block/writer_test.go index ce1a07092..76524792f 100644 --- a/pp/go/storage/block/writer_test.go +++ b/pp/go/storage/block/writer_test.go @@ -111,7 +111,7 @@ func (s *WriterSuite) shard() *shard.Shard { func (s *WriterSuite) fillHead() { ts := time.UnixMilli(1753805651969) - storagetest.MustAppendTimeSeries(&s.Suite, s.head, []storagetest.TimeSeries{ + storagetest.MustAppendTimeSeries(s.T().Context(), s.Require().NoError, s.head, []storagetest.TimeSeries{ { Labels: labels.FromStrings("__name__", "value1"), Samples: []cppbridge.Sample{ @@ -283,7 +283,7 @@ func (s *WriterSuite) TestWriteWithDataUnloadingInBatches() { func (s *WriterSuite) TestSkipEmptyBlock() { // Arrange - storagetest.MustAppendTimeSeries(&s.Suite, s.head, []storagetest.TimeSeries{ + storagetest.MustAppendTimeSeries(s.T().Context(), s.Require().NoError, s.head, []storagetest.TimeSeries{ { Labels: labels.FromStrings("__name__", "value1"), Samples: []cppbridge.Sample{ diff --git a/pp/go/storage/head/services/persistener_test.go b/pp/go/storage/head/services/persistener_test.go index a1fc2b1d1..306d5c780 100644 --- a/pp/go/storage/head/services/persistener_test.go +++ b/pp/go/storage/head/services/persistener_test.go @@ -147,7 +147,7 @@ func (s *PersistenerSuite) TestNoPersistWritableHead() { func (s *PersistenerSuite) TestNoPersistPersistedHead() { // Arrange head := s.mustCreateHead() - storagetest.MustAppendTimeSeries(&s.Suite, head, []storagetest.TimeSeries{ + storagetest.MustAppendTimeSeries(s.T().Context(), s.Require().NoError, head, []storagetest.TimeSeries{ { Labels: labels.FromStrings("__name__", "value1"), Samples: []cppbridge.Sample{ @@ -172,7 +172,7 @@ func (s *PersistenerSuite) TestNoPersistPersistedHead() { func (s *PersistenerSuite) TestOutdatedPersistedHead() { // Arrange head := s.mustCreateHead() - storagetest.MustAppendTimeSeries(&s.Suite, head, []storagetest.TimeSeries{ + storagetest.MustAppendTimeSeries(s.T().Context(), s.Require().NoError, head, []storagetest.TimeSeries{ { Labels: labels.FromStrings("__name__", "value1"), Samples: []cppbridge.Sample{ @@ -199,7 +199,7 @@ func (s *PersistenerSuite) TestOutdatedHead() { s.clock.Advance(tsdbRetentionPeriod) head := s.mustCreateHead() - storagetest.MustAppendTimeSeries(&s.Suite, head, []storagetest.TimeSeries{ + storagetest.MustAppendTimeSeries(s.T().Context(), s.Require().NoError, head, []storagetest.TimeSeries{ { Labels: labels.FromStrings("__name__", "value1"), Samples: []cppbridge.Sample{ @@ -231,7 +231,7 @@ func (s *PersistenerSuite) TestPersistHeadSuccess() { } head := s.mustCreateHead() - storagetest.MustAppendTimeSeries(&s.Suite, head, []storagetest.TimeSeries{ + storagetest.MustAppendTimeSeries(s.T().Context(), s.Require().NoError, head, []storagetest.TimeSeries{ { Labels: labels.FromStrings("__name__", "value1"), Samples: []cppbridge.Sample{ @@ -271,7 +271,7 @@ func (s *PersistenerSuite) TestPersistHeadErrorOnBlockWriterForSecondShard() { } head := s.mustCreateHead() - storagetest.MustAppendTimeSeries(&s.Suite, head, []storagetest.TimeSeries{ + storagetest.MustAppendTimeSeries(s.T().Context(), s.Require().NoError, head, []storagetest.TimeSeries{ { Labels: labels.FromStrings("__name__", "value1"), Samples: []cppbridge.Sample{ @@ -341,7 +341,7 @@ func (s *PersistenerServiceSuite) TestRemoveOutdatedHeadFromKeeper() { // Arrange s.clock.Advance(tsdbRetentionPeriod) head := s.mustCreateHead() - storagetest.MustAppendTimeSeries(&s.Suite, head, []storagetest.TimeSeries{ + storagetest.MustAppendTimeSeries(s.T().Context(), s.Require().NoError, head, []storagetest.TimeSeries{ { Labels: labels.FromStrings("__name__", "value1"), Samples: []cppbridge.Sample{ @@ -364,7 +364,7 @@ func (s *PersistenerServiceSuite) TestRemoveOutdatedHeadFromKeeper() { func (s *PersistenerServiceSuite) TestLoadHeadsInKeeper() { // Arrange head := s.mustCreateHead() - storagetest.MustAppendTimeSeries(&s.Suite, head, []storagetest.TimeSeries{ + storagetest.MustAppendTimeSeries(s.T().Context(), s.Require().NoError, head, []storagetest.TimeSeries{ { Labels: labels.FromStrings("__name__", "value1"), Samples: []cppbridge.Sample{ @@ -387,7 +387,7 @@ func (s *PersistenerServiceSuite) TestLoadHeadsInKeeper() { func (s *PersistenerServiceSuite) TestHeadAlreadyExistsInKeeper() { // Arrange head := s.mustCreateHead() - storagetest.MustAppendTimeSeries(&s.Suite, head, []storagetest.TimeSeries{ + storagetest.MustAppendTimeSeries(s.T().Context(), s.Require().NoError, head, []storagetest.TimeSeries{ { Labels: labels.FromStrings("__name__", "value1"), Samples: []cppbridge.Sample{ diff --git a/pp/go/storage/loader_test.go b/pp/go/storage/loader_test.go index a69ea9cc5..2c526e09d 100644 --- a/pp/go/storage/loader_test.go +++ b/pp/go/storage/loader_test.go @@ -133,7 +133,7 @@ func (s *HeadLoadSuite) lockFileForCreation(fileName string) { } func (s *HeadLoadSuite) appendTimeSeries(head *storage.Head, timeSeries []storagetest.TimeSeries) { - storagetest.MustAppendTimeSeries(&s.Suite, head, timeSeries) + storagetest.MustAppendTimeSeries(s.T().Context(), s.Require().NoError, head, timeSeries) } func (*HeadLoadSuite) shards(head *storage.Head) (result []*shard.Shard) { diff --git a/pp/go/storage/querier/querier_optimize_quick_test.go b/pp/go/storage/querier/querier_optimize_quick_test.go index 957924966..1b4ef4abf 100644 --- a/pp/go/storage/querier/querier_optimize_quick_test.go +++ b/pp/go/storage/querier/querier_optimize_quick_test.go @@ -220,7 +220,8 @@ func (oqp *OffsetQueryParams) offsetGen(rd *rand.Rand) { // type QuickQuerierOptimizeSuite struct { - QuerierOptimizeSuite + suite.Suite + querierOptimize quickQE *promql.Engine } @@ -230,7 +231,7 @@ func TestQuickQuerierOptimizeSuite(t *testing.T) { } func (s *QuickQuerierOptimizeSuite) SetupSuite() { - s.QuerierOptimizeSuite.SetupSuite() + s.querierOptimize.setup(s.T().Context(), s.T().TempDir(), s.Require().NoError) s.quickQE = promql.NewEngine(promql.EngineOpts{ Logger: log.NewNopLogger(), @@ -243,6 +244,10 @@ func (s *QuickQuerierOptimizeSuite) SetupSuite() { }) } +func (s *QuickQuerierOptimizeSuite) TearDownSuite() { + s.Suite.Require().NoError(s.querierOptimize.close()) +} + func (s *QuickQuerierOptimizeSuite) TestQueryRangeQuickQueryParam() { ctx := s.T().Context() diff --git a/pp/go/storage/querier/querier_optimize_test.go b/pp/go/storage/querier/querier_optimize_test.go index 96e88f9c1..ce7fee083 100644 --- a/pp/go/storage/querier/querier_optimize_test.go +++ b/pp/go/storage/querier/querier_optimize_test.go @@ -440,11 +440,12 @@ func queryInstant( } // -// QuerierOptimizeSuite +// querierOptimize // -type QuerierOptimizeSuite struct { - suite.Suite +// querierOptimize is the querier optimizer for testing. +type querierOptimize struct { + noErrorFunc storagetest.NoErrorFunc dataDir string head *storage.Head @@ -458,14 +459,18 @@ type QuerierOptimizeSuite struct { queryFuncs []queryFunc } -func (s *QuerierOptimizeSuite) SetupSuite() { +// setup sets up the querier optimizer. +func (s *querierOptimize) setup(ctx context.Context, baseDir string, noErrorFunc storagetest.NoErrorFunc) { + s.noErrorFunc = noErrorFunc s.start = time.UnixMilli(defaultStartMs) s.step = defaultStep s.end = s.start.Add(s.step * defaultCountOfSteps) // 480 steps - s.dataDir = s.createDataDirectory() + s.dataDir = filepath.Join(baseDir, "data") + s.noErrorFunc(os.MkdirAll(s.dataDir, os.ModeDir)) + s.head = s.mustCreateHead(0) - s.fillHead() + s.fillHead(ctx) s.lookbackDelta = 5 * time.Minute s.queryOpts = promql.NewPrometheusQueryOpts(false, s.lookbackDelta) @@ -489,28 +494,24 @@ func (s *QuerierOptimizeSuite) SetupSuite() { } q, err := s.Querier(s.start.UnixMilli(), s.end.UnixMilli()) - s.Require().NoError(err) + s.noErrorFunc(err) - names, _, err := q.LabelValues(s.T().Context(), "__name__", &prom_storage.LabelHints{}) - s.Require().NoError(err) + names, _, err := q.LabelValues(ctx, "__name__", &prom_storage.LabelHints{}) + s.noErrorFunc(err) s.metricNames = querier.DeduplicateAndSortStringSlices(names) - s.Require().NoError(q.Close()) -} - -func (s *QuerierOptimizeSuite) TearDownSuite() { - s.Require().NoError(s.head.Close()) + s.noErrorFunc(q.Close()) } -func (s *QuerierOptimizeSuite) createDataDirectory() string { - dataDir := filepath.Join(s.T().TempDir(), "data") - s.Require().NoError(os.MkdirAll(dataDir, os.ModeDir)) - return dataDir +// close closes the querier optimizer. +func (s *querierOptimize) close() error { + return s.head.Close() } -func (s *QuerierOptimizeSuite) mustCreateCatalog() *catalog.Catalog { +// mustCreateHead creates a new head. +func (s *querierOptimize) mustCreateHead(unloadDataStorageInterval time.Duration) *storage.Head { l, err := catalog.NewFileLogV2(filepath.Join(s.dataDir, "catalog.log")) - s.Require().NoError(err) + s.noErrorFunc(err) c, err := catalog.New( clockwork.NewFakeClock(), @@ -519,28 +520,21 @@ func (s *QuerierOptimizeSuite) mustCreateCatalog() *catalog.Catalog { catalog.DefaultMaxLogFileSize, nil, ) - s.Require().NoError(err) - - return c -} + s.noErrorFunc(err) -func (s *QuerierOptimizeSuite) mustCreateHead(unloadDataStorageInterval time.Duration) *storage.Head { h, err := storage.NewBuilder( - s.mustCreateCatalog(), + c, s.dataDir, maxSegmentSize, prometheus.DefaultRegisterer, unloadDataStorageInterval, ).Build(0, numberOfShards) - s.Require().NoError(err) + s.noErrorFunc(err) return h } -func (s *QuerierOptimizeSuite) appendTimeSeries(timeSeries []storagetest.TimeSeries) { - storagetest.MustAppendTimeSeries(&s.Suite, s.head, timeSeries) -} - -func (s *QuerierOptimizeSuite) fillHead() { +// fillHead fills the head with the given time series. +func (s *querierOptimize) fillHead(ctx context.Context) { countOfSamples := (s.end.UnixMilli()-s.start.UnixMilli())/s.step.Milliseconds() + 1 timeSeries := []storagetest.TimeSeries{ { @@ -630,11 +624,11 @@ func (s *QuerierOptimizeSuite) fillHead() { valueCounter++ } - s.appendTimeSeries(timeSeries) + storagetest.MustAppendTimeSeries(ctx, s.noErrorFunc, s.head, timeSeries) } // Querier implements the [prom_storage.Queryable] interface. -func (s *QuerierOptimizeSuite) Querier(mint, maxt int64) (prom_storage.Querier, error) { +func (s *querierOptimize) Querier(mint, maxt int64) (prom_storage.Querier, error) { return querier.NewQuerier(s.head, querier.NewNoOpShardedDeduplicator, mint, maxt, nil, nil), nil } @@ -643,7 +637,8 @@ func (s *QuerierOptimizeSuite) Querier(mint, maxt int64) (prom_storage.Querier, // type MatrixQuerierOptimizeSuiteSuite struct { - QuerierOptimizeSuite + suite.Suite + querierOptimize queryEngine *promql.Engine steps []time.Duration @@ -657,7 +652,7 @@ func TestMatrixQuerierOptimizeSuiteSuite(t *testing.T) { } func (s *MatrixQuerierOptimizeSuiteSuite) SetupSuite() { - s.QuerierOptimizeSuite.SetupSuite() + s.querierOptimize.setup(s.T().Context(), s.T().TempDir(), s.Require().NoError) s.queryEngine = promql.NewEngine(promql.EngineOpts{ Logger: log.NewNopLogger(), @@ -684,6 +679,10 @@ func (s *MatrixQuerierOptimizeSuiteSuite) SetupSuite() { s.Require().NoError(q.Close()) } +func (s *MatrixQuerierOptimizeSuiteSuite) TearDownSuite() { + s.Suite.Require().NoError(s.querierOptimize.close()) +} + // rangeArgs runs the given function for all combinations of // query functions, metric names, steps, subqueries, modifiers and offsets. // diff --git a/pp/go/storage/querier/querier_test.go b/pp/go/storage/querier/querier_test.go index 5a43e606a..30e08e3bd 100644 --- a/pp/go/storage/querier/querier_test.go +++ b/pp/go/storage/querier/querier_test.go @@ -133,7 +133,7 @@ func (s *QuerierSuite) mustCreateHead(unloadDataStorageInterval time.Duration) * } func (s *QuerierSuite) appendTimeSeries(timeSeries []storagetest.TimeSeries) { - storagetest.MustAppendTimeSeries(&s.Suite, s.head, timeSeries) + storagetest.MustAppendTimeSeries(s.T().Context(), s.Require().NoError, s.head, timeSeries) } func (s *QuerierSuite) TestRangeQuery() { diff --git a/pp/go/storage/storagetest/fixtures.go b/pp/go/storage/storagetest/fixtures.go index 9788c3954..844f575e7 100644 --- a/pp/go/storage/storagetest/fixtures.go +++ b/pp/go/storage/storagetest/fixtures.go @@ -28,7 +28,6 @@ import ( "github.com/prometheus/prometheus/pp/go/storage/querier" promstorage "github.com/prometheus/prometheus/storage" "github.com/prometheus/prometheus/tsdb/chunkenc" - "github.com/stretchr/testify/suite" ) // TimeSeries test data. @@ -74,30 +73,33 @@ func (tsd *timeSeriesDataSlice) Destroy() { tsd.timeSeries = nil } +// NoErrorFunc is a function that can be used to assert that an error is nil. +type NoErrorFunc func(err error, msgAndArgs ...any) + // MustAppendTimeSeries add time series to head. -func MustAppendTimeSeries(s *suite.Suite, head *storage.Head, timeSeries []TimeSeries) { +func MustAppendTimeSeries(ctx context.Context, noErrorFunc NoErrorFunc, head *storage.Head, timeSeries []TimeSeries) { headAppender := appender.New(head, services.CFViaRange) statelessRelabeler, err := cppbridge.NewStatelessRelabeler([]*cppbridge.RelabelConfig{}) - s.Require().NoError(err) + noErrorFunc(err) state := cppbridge.NewStateV2WithoutLock() state.SetStatelessRelabeler(statelessRelabeler) for i := range timeSeries { _, err = headAppender.Append( - context.Background(), - NewIncomingData(s, timeSeries[i].toModelTimeSeries()), + ctx, + NewIncomingData(noErrorFunc, timeSeries[i].toModelTimeSeries()), state, true) - s.NoError(err) + noErrorFunc(err) } } -func NewIncomingData(s *suite.Suite, timeSeries []model.TimeSeries) *appender.IncomingData { +func NewIncomingData(noErrorFunc NoErrorFunc, timeSeries []model.TimeSeries) *appender.IncomingData { tsd := timeSeriesDataSlice{timeSeries: timeSeries} hx, err := (cppbridge.HashdexFactory{}).GoModel(tsd.TimeSeries(), cppbridge.DefaultWALHashdexLimits()) - s.Require().NoError(err) + noErrorFunc(err) return &appender.IncomingData{Hashdex: hx, Data: &tsd} } From b3042e70d0af7898fe13d547bd09d6108a566a34 Mon Sep 17 00:00:00 2001 From: Alexandr Yudin <57181751+u-veles-a@users.noreply.github.com> Date: Fri, 29 May 2026 16:39:18 +0000 Subject: [PATCH 24/37] WIP: add BenchmarkRangeQuery Signed-off-by: Alexandr Yudin <57181751+u-veles-a@users.noreply.github.com> --- .../storage/querier/querier_optimize_test.go | 74 +++++++++++++++++++ 1 file changed, 74 insertions(+) diff --git a/pp/go/storage/querier/querier_optimize_test.go b/pp/go/storage/querier/querier_optimize_test.go index ce7fee083..9e43023d8 100644 --- a/pp/go/storage/querier/querier_optimize_test.go +++ b/pp/go/storage/querier/querier_optimize_test.go @@ -7,6 +7,7 @@ import ( "math" "os" "path/filepath" + "strconv" "strings" "testing" "time" @@ -15,6 +16,7 @@ import ( "github.com/jonboulle/clockwork" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/common/model" + "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" "github.com/prometheus/prometheus/model/labels" @@ -627,6 +629,32 @@ func (s *querierOptimize) fillHead(ctx context.Context) { storagetest.MustAppendTimeSeries(ctx, s.noErrorFunc, s.head, timeSeries) } +// fillHeadWithCounter fills the head with the given number of counter metrics. +func (s *querierOptimize) fillHeadWithCounter(ctx context.Context, counter int) { + countOfSamples := (s.end.UnixMilli()-s.start.UnixMilli())/s.step.Milliseconds() + 1 + timeSeries := make([]storagetest.TimeSeries, 0, counter) + for i := range counter { + timeSeries = append(timeSeries, storagetest.TimeSeries{ + Labels: labels.FromStrings("__name__", "counter_metric", "value", "inc", "counter", strconv.Itoa(i)), + Samples: make([]cppbridge.Sample, 0, countOfSamples), + }) + } + + valueCounter := 1 + for ts := s.start; !ts.After(s.end); ts = ts.Add(s.step) { + tsMilli := ts.UnixMilli() + for i := range counter { + timeSeries[i].Samples = append(timeSeries[i].Samples, + cppbridge.Sample{Timestamp: tsMilli, Value: float64(valueCounter)}, + ) + } + + valueCounter++ + } + + storagetest.MustAppendTimeSeries(ctx, s.noErrorFunc, s.head, timeSeries) +} + // Querier implements the [prom_storage.Queryable] interface. func (s *querierOptimize) Querier(mint, maxt int64) (prom_storage.Querier, error) { return querier.NewQuerier(s.head, querier.NewNoOpShardedDeduplicator, mint, maxt, nil, nil), nil @@ -1057,3 +1085,49 @@ func inEpsilon(expected, actual, epsilon float64) bool { func calcRelative(expected, actual float64) float64 { return math.Abs(expected-actual) / math.Abs(expected) } + +// +// Benchmark +// + +func BenchmarkRangeQuery(b *testing.B) { + ctx := b.Context() + qo := &querierOptimize{} + qo.setup(ctx, b.TempDir(), func(err error, msgAndArgs ...any) { require.NoError(b, err, msgAndArgs) }) + qo.fillHeadWithCounter(ctx, 50) + defer qo.close() + + queryEngine := promql.NewEngine(promql.EngineOpts{ + Logger: log.NewNopLogger(), + MaxSamples: 100000, + Timeout: 10 * time.Second, + LookbackDelta: qo.lookbackDelta, + NoStepSubqueryIntervalFn: func(int64) int64 { return qo.lookbackDelta.Milliseconds() }, + EnableAtModifier: true, + EnableNegativeOffset: true, + }) + + query := "sum(counter_metric)" + // query := "max_over_time(counter_metric[3600s])" + + // step := qo.step + step := qo.step * 4 + + b.Run("none", func(b *testing.B) { + b.ResetTimer() + for b.Loop() { + res, err := queryRange(ctx, "none", queryEngine, qo, qo.queryOpts, query, qo.start, qo.end, step) + require.NoError(b, err) + res.qry.Close() + } + }) + + b.Run("all", func(b *testing.B) { + b.ResetTimer() + for b.Loop() { + res, err := queryRange(ctx, "all", queryEngine, qo, qo.queryOpts, query, qo.start, qo.end, step) + require.NoError(b, err) + res.qry.Close() + } + }) +} From bca8982e15692318e00030fa315f353148c906b2 Mon Sep 17 00:00:00 2001 From: Alexandr Yudin <57181751+u-veles-a@users.noreply.github.com> Date: Tue, 2 Jun 2026 08:58:44 +0000 Subject: [PATCH 25/37] WIP: add benchmark, opt Signed-off-by: Alexandr Yudin <57181751+u-veles-a@users.noreply.github.com> --- pp/go/storage/querier/querier.go | 41 +++++++- .../storage/querier/querier_optimize_test.go | 99 +++++++++++++++---- .../querier_optimize_variables_test.go | 2 +- 3 files changed, 119 insertions(+), 23 deletions(-) diff --git a/pp/go/storage/querier/querier.go b/pp/go/storage/querier/querier.go index b914b5f1d..cb8705caa 100644 --- a/pp/go/storage/querier/querier.go +++ b/pp/go/storage/querier/querier.go @@ -74,6 +74,9 @@ const ( allOptimizeType queryOptimizeType = dropPointOptimizeType | newPointOptimizeType | crossSeriesOptimizeType ) +// DefaultCountOfSeriesToOptimize is the default count of series to optimize. +const DefaultCountOfSeriesToOptimize = 6 + // defaultOptimizeType is the default option for selecting functions optimization. var defaultOptimizeType = noneOptimizeType @@ -412,12 +415,12 @@ func (q *Querier[TTask, TDataStorage, TLSS, TShard, THead]) selectRange( return storage.ErrSeriesSet(err) } - hints = SwitchFuncOptimize(hints, q.queryOptimize) + hints = SwitchFuncOptimize(hints, isPossibleToOptimize(lssQueryResults, hints), q.queryOptimize) shardedSerializedData := poolProvider.GetSerializedData() defer poolProvider.PutSerializedData(shardedSerializedData) queryDataStorage(dsQueryRangeQuerier, q.head, lssQueryResults, shardedSerializedData, q.mint, q.maxt, hints) - if isCrossSeriesFunc(hints) && isAllowedGroupingForCrossSeriesFunc(hints.Grouping) { + if isCrossSeriesFunc(hints) { return q.makeAggSeriesSet(lssQueryResults, snapshots, shardedSerializedData, hints) } @@ -547,7 +550,11 @@ func (q *Querier[TTask, TDataStorage, TLSS, TShard, THead]) makeAggSeriesSet( } // SwitchFuncOptimize switch the function optimization hints. -func SwitchFuncOptimize(hints *storage.SelectHints, queryOptimize queryOptimizeType) *storage.SelectHints { +func SwitchFuncOptimize( + hints *storage.SelectHints, + possibleToOptimize func() bool, + queryOptimize queryOptimizeType, +) *storage.SelectHints { if hints == nil { return emptySelectHints } @@ -556,13 +563,37 @@ func SwitchFuncOptimize(hints *storage.SelectHints, queryOptimize queryOptimizeT return emptySelectHints } - if funcOptimizeMap[hints.Func]&queryOptimize != 0 && isNotWithpout(hints) { + if funcOptimizeMap[hints.Func]&queryOptimize != 0 && + isNotWithpout(hints) && + isAllowedGroupingForCrossSeriesFunc(hints.Grouping) && + possibleToOptimize() { return hints } return emptySelectHints } +// isPossibleToOptimize checks if the query possible to optimization. +func isPossibleToOptimize(lssQueryResults []*cppbridge.LSSQueryResult, hints *storage.SelectHints) func() bool { + return func() bool { + countOfSeries := 0 + for _, lssQueryResult := range lssQueryResults { + if lssQueryResult == nil { + continue + } + + countOfSeries += lssQueryResult.Len() + } + + if hints.Step == 0 && isCrossSeriesFunc(hints) { + //revive:disable-next-line:add-constant // x3 we need to optimize the query for the crossseries function + return countOfSeries >= DefaultCountOfSeriesToOptimize*3 + } + + return countOfSeries >= DefaultCountOfSeriesToOptimize + } +} + // isNotWithpout checks if the hints is not without by. func isNotWithpout(hints *storage.SelectHints) bool { return hints.By || len(hints.Grouping) == 0 @@ -612,7 +643,7 @@ func isAllowedGroupingForCrossSeriesFunc(grouping []string) bool { // isCrossSeriesFunc checks if the function is a cross series function. func isCrossSeriesFunc(hints *storage.SelectHints) bool { - return funcOptimizeMap[hints.Func]&crossSeriesOptimizeType == crossSeriesOptimizeType && isNotWithpout(hints) + return funcOptimizeMap[hints.Func]&crossSeriesOptimizeType == crossSeriesOptimizeType } // convertPrometheusMatchersToPPMatchers converts prometheus matchers to pp matchers. diff --git a/pp/go/storage/querier/querier_optimize_test.go b/pp/go/storage/querier/querier_optimize_test.go index 9e43023d8..dd77ebc7c 100644 --- a/pp/go/storage/querier/querier_optimize_test.go +++ b/pp/go/storage/querier/querier_optimize_test.go @@ -37,12 +37,18 @@ import ( type SwitchFuncOptimizeSuite struct { suite.Suite + + isPossibleToOptimize func() bool } func TestSwitchFuncOptimizeSuite(t *testing.T) { suite.Run(t, new(SwitchFuncOptimizeSuite)) } +func (s *SwitchFuncOptimizeSuite) SetupSuite() { + s.isPossibleToOptimize = func() bool { return true } +} + func (s *SwitchFuncOptimizeSuite) TestNone() { tests := []struct { hints *prom_storage.SelectHints @@ -83,7 +89,7 @@ func (s *SwitchFuncOptimizeSuite) TestNone() { } for _, test := range tests { - result := querier.SwitchFuncOptimize(test.hints, 0) + result := querier.SwitchFuncOptimize(test.hints, s.isPossibleToOptimize, 0) s.Require().Equal(test.expected, result) } } @@ -128,7 +134,7 @@ func (s *SwitchFuncOptimizeSuite) TestDropPoint() { } for _, test := range tests { - result := querier.SwitchFuncOptimize(test.hints, 1) + result := querier.SwitchFuncOptimize(test.hints, s.isPossibleToOptimize, 1) s.Require().Equal(test.expected, result) } } @@ -173,7 +179,7 @@ func (s *SwitchFuncOptimizeSuite) TestNewPoint() { } for _, test := range tests { - result := querier.SwitchFuncOptimize(test.hints, 2) + result := querier.SwitchFuncOptimize(test.hints, s.isPossibleToOptimize, 2) s.Require().Equal(test.expected, result) } } @@ -218,7 +224,7 @@ func (s *SwitchFuncOptimizeSuite) TestCrossSeries() { } for _, test := range tests { - result := querier.SwitchFuncOptimize(test.hints, 4) + result := querier.SwitchFuncOptimize(test.hints, s.isPossibleToOptimize, 4) s.Require().Equal(test.expected, result) } } @@ -263,7 +269,7 @@ func (s *SwitchFuncOptimizeSuite) TestAll() { } for _, test := range tests { - result := querier.SwitchFuncOptimize(test.hints, 7) + result := querier.SwitchFuncOptimize(test.hints, s.isPossibleToOptimize, 7) s.Require().Equal(test.expected, result) } } @@ -277,7 +283,7 @@ const ( defaultStartMs = 1779290789000 // defaultStep is the default step. - defaultStep = 15 * time.Second + defaultStep = 30 * time.Second ) // @@ -473,18 +479,22 @@ func (s *querierOptimize) setup(ctx context.Context, baseDir string, noErrorFunc s.head = s.mustCreateHead(0) s.fillHead(ctx) + s.fillHeadWithCounter(ctx, querier.DefaultCountOfSeriesToOptimize) s.lookbackDelta = 5 * time.Minute s.queryOpts = promql.NewPrometheusQueryOpts(false, s.lookbackDelta) s.queryFuncs = []queryFunc{ - {name: "min_over_time", needRange: true}, // + - {name: "max_over_time", needRange: true}, // + - {name: "last_over_time", needRange: true}, // + - {name: "changes", needRange: true}, // + - {name: "min", needRange: false}, // + - {name: "max", needRange: false}, // + - {name: "sum", needRange: false}, // + + {name: "min_over_time", needRange: true}, // + + {name: "max_over_time", needRange: true}, // + + {name: "last_over_time", needRange: true}, // + + {name: "changes", needRange: true}, // + + {name: "min", needRange: false}, // + + {name: "min by(value) ", needRange: false}, // + + {name: "max", needRange: false}, // + + {name: "max by(value) ", needRange: false}, // + + {name: "sum", needRange: false}, // + + {name: "sum by(value) ", needRange: false}, // + // {name: "rate", needRange: true}, // - // {name: "irate", needRange: true}, // - @@ -632,7 +642,7 @@ func (s *querierOptimize) fillHead(ctx context.Context) { // fillHeadWithCounter fills the head with the given number of counter metrics. func (s *querierOptimize) fillHeadWithCounter(ctx context.Context, counter int) { countOfSamples := (s.end.UnixMilli()-s.start.UnixMilli())/s.step.Milliseconds() + 1 - timeSeries := make([]storagetest.TimeSeries, 0, counter) + timeSeries := make([]storagetest.TimeSeries, 0, counter*2) for i := range counter { timeSeries = append(timeSeries, storagetest.TimeSeries{ Labels: labels.FromStrings("__name__", "counter_metric", "value", "inc", "counter", strconv.Itoa(i)), @@ -640,6 +650,13 @@ func (s *querierOptimize) fillHeadWithCounter(ctx context.Context, counter int) }) } + for i := range counter { + timeSeries = append(timeSeries, storagetest.TimeSeries{ + Labels: labels.FromStrings("__name__", "sin_cos_metric", "value", "sin_cos", "counter", strconv.Itoa(i)), + Samples: make([]cppbridge.Sample, 0, countOfSamples), + }) + } + valueCounter := 1 for ts := s.start; !ts.After(s.end); ts = ts.Add(s.step) { tsMilli := ts.UnixMilli() @@ -647,6 +664,10 @@ func (s *querierOptimize) fillHeadWithCounter(ctx context.Context, counter int) timeSeries[i].Samples = append(timeSeries[i].Samples, cppbridge.Sample{Timestamp: tsMilli, Value: float64(valueCounter)}, ) + + timeSeries[i+counter].Samples = append(timeSeries[i+counter].Samples, + cppbridge.Sample{Timestamp: tsMilli, Value: math.Sin(float64(i))*10 + math.Cos(float64(i))*10}, + ) } valueCounter++ @@ -1094,7 +1115,7 @@ func BenchmarkRangeQuery(b *testing.B) { ctx := b.Context() qo := &querierOptimize{} qo.setup(ctx, b.TempDir(), func(err error, msgAndArgs ...any) { require.NoError(b, err, msgAndArgs) }) - qo.fillHeadWithCounter(ctx, 50) + qo.fillHeadWithCounter(ctx, 3) defer qo.close() queryEngine := promql.NewEngine(promql.EngineOpts{ @@ -1110,8 +1131,8 @@ func BenchmarkRangeQuery(b *testing.B) { query := "sum(counter_metric)" // query := "max_over_time(counter_metric[3600s])" - // step := qo.step - step := qo.step * 4 + step := qo.step + // step := qo.step * 4 b.Run("none", func(b *testing.B) { b.ResetTimer() @@ -1131,3 +1152,47 @@ func BenchmarkRangeQuery(b *testing.B) { } }) } + +func BenchmarkInstantQuery(b *testing.B) { + ctx := b.Context() + qo := &querierOptimize{} + qo.setup(ctx, b.TempDir(), func(err error, msgAndArgs ...any) { require.NoError(b, err, msgAndArgs) }) + qo.fillHeadWithCounter(ctx, 15) + defer qo.close() + + queryEngine := promql.NewEngine(promql.EngineOpts{ + Logger: log.NewNopLogger(), + MaxSamples: 100000, + Timeout: 10 * time.Second, + LookbackDelta: qo.lookbackDelta, + NoStepSubqueryIntervalFn: func(int64) int64 { return qo.lookbackDelta.Milliseconds() }, + EnableAtModifier: true, + EnableNegativeOffset: true, + }) + + query := "sum(counter_metric)" + // query := "max_over_time(counter_metric[3600s])" + + ttt := 4800 * time.Second + mid := qo.start.Add(ttt) + // mid := qo.start + // mid := qo.end + + b.Run("none", func(b *testing.B) { + b.ResetTimer() + for b.Loop() { + res, err := queryInstant(ctx, "none", queryEngine, qo, qo.queryOpts, query, mid) + require.NoError(b, err) + res.qry.Close() + } + }) + + b.Run("all", func(b *testing.B) { + b.ResetTimer() + for b.Loop() { + res, err := queryInstant(ctx, "all", queryEngine, qo, qo.queryOpts, query, mid) + require.NoError(b, err) + res.qry.Close() + } + }) +} diff --git a/pp/go/storage/querier/querier_optimize_variables_test.go b/pp/go/storage/querier/querier_optimize_variables_test.go index 58163a62d..5da789160 100644 --- a/pp/go/storage/querier/querier_optimize_variables_test.go +++ b/pp/go/storage/querier/querier_optimize_variables_test.go @@ -11,7 +11,7 @@ import ( const ( // defaultCountOfSteps is the default count of steps. - defaultCountOfSteps = 480 + defaultCountOfSteps = 240 ) // defaultSteps is the default steps. From e8882ed0397574ed9e6dd135e38b68ccf99111e8 Mon Sep 17 00:00:00 2001 From: Alexandr Yudin <57181751+u-veles-a@users.noreply.github.com> Date: Tue, 2 Jun 2026 15:38:47 +0000 Subject: [PATCH 26/37] WIP: fix iterator dtor Signed-off-by: Alexandr Yudin <57181751+u-veles-a@users.noreply.github.com> --- pp/go/storage/querier/agg_series_set.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/pp/go/storage/querier/agg_series_set.go b/pp/go/storage/querier/agg_series_set.go index 30ac3ea5d..d7c115784 100644 --- a/pp/go/storage/querier/agg_series_set.go +++ b/pp/go/storage/querier/agg_series_set.go @@ -3,6 +3,7 @@ package querier import ( "errors" "fmt" + "runtime" "slices" "github.com/prometheus/prometheus/model/histogram" @@ -196,6 +197,12 @@ func NewAggChunkIterator( maxt: maxt, } + runtime.SetFinalizer(it, func(iter *AggChunkIterator) { + iter.chunkIterator.Close() + iter.serializedData = nil + iter.seriesGroups = nil + }) + return it } @@ -292,6 +299,7 @@ func (it *AggChunkIterator) reset( it.maxt = maxt it.isInitialized = false // it.chunkIterator.Reset(serializedData, chunkRef) + it.chunkIterator.Close() it.chunkIterator = cppbridge.NewDataStorageSerializedDataMultiSeriesIterator( serializedData, seriesGroups.Groups[groupIndex], From 16725ba653dcd59aedce64b7ae43c83eb378a079 Mon Sep 17 00:00:00 2001 From: Alexandr Yudin <57181751+u-veles-a@users.noreply.github.com> Date: Thu, 4 Jun 2026 07:38:30 +0000 Subject: [PATCH 27/37] WIP: add iterator reset Signed-off-by: Alexandr Yudin <57181751+u-veles-a@users.noreply.github.com> --- pp/go/storage/querier/agg_series_set.go | 7 +------ pp/go/storage/querier/querier_optimize_test.go | 10 ++++++---- 2 files changed, 7 insertions(+), 10 deletions(-) diff --git a/pp/go/storage/querier/agg_series_set.go b/pp/go/storage/querier/agg_series_set.go index d7c115784..df4c5d700 100644 --- a/pp/go/storage/querier/agg_series_set.go +++ b/pp/go/storage/querier/agg_series_set.go @@ -298,12 +298,7 @@ func (it *AggChunkIterator) reset( it.mint = mint it.maxt = maxt it.isInitialized = false - // it.chunkIterator.Reset(serializedData, chunkRef) - it.chunkIterator.Close() - it.chunkIterator = cppbridge.NewDataStorageSerializedDataMultiSeriesIterator( - serializedData, - seriesGroups.Groups[groupIndex], - ) + it.chunkIterator.Reset(serializedData, seriesGroups.Groups[groupIndex]) } // diff --git a/pp/go/storage/querier/querier_optimize_test.go b/pp/go/storage/querier/querier_optimize_test.go index dd77ebc7c..4d811e1a5 100644 --- a/pp/go/storage/querier/querier_optimize_test.go +++ b/pp/go/storage/querier/querier_optimize_test.go @@ -1129,6 +1129,7 @@ func BenchmarkRangeQuery(b *testing.B) { }) query := "sum(counter_metric)" + // query := "sum by(value) (counter_metric)" // query := "max_over_time(counter_metric[3600s])" step := qo.step @@ -1157,7 +1158,7 @@ func BenchmarkInstantQuery(b *testing.B) { ctx := b.Context() qo := &querierOptimize{} qo.setup(ctx, b.TempDir(), func(err error, msgAndArgs ...any) { require.NoError(b, err, msgAndArgs) }) - qo.fillHeadWithCounter(ctx, 15) + qo.fillHeadWithCounter(ctx, 3) defer qo.close() queryEngine := promql.NewEngine(promql.EngineOpts{ @@ -1170,10 +1171,11 @@ func BenchmarkInstantQuery(b *testing.B) { EnableNegativeOffset: true, }) - query := "sum(counter_metric)" - // query := "max_over_time(counter_metric[3600s])" + // query := "sum(counter_metric)" + // query := "sum by(value) (counter_metric)" + query := "max_over_time(counter_metric[3600s])" - ttt := 4800 * time.Second + ttt := 3700 * time.Second mid := qo.start.Add(ttt) // mid := qo.start // mid := qo.end From d8893bea4b59d3381fe0f6ad8d25f1bfe73ff5ee Mon Sep 17 00:00:00 2001 From: Alexandr Yudin <57181751+u-veles-a@users.noreply.github.com> Date: Thu, 4 Jun 2026 11:32:22 +0000 Subject: [PATCH 28/37] WIP: add scrape interval Signed-off-by: Alexandr Yudin <57181751+u-veles-a@users.noreply.github.com> --- pp-pkg/storage/adapter.go | 26 +++++--- pp-pkg/storage/batch_storage.go | 16 ++++- pp/go/storage/querier/querier.go | 44 ++++++++----- .../storage/querier/querier_optimize_test.go | 2 +- pp/go/storage/querier/querier_test.go | 63 +++++-------------- 5 files changed, 75 insertions(+), 76 deletions(-) diff --git a/pp-pkg/storage/adapter.go b/pp-pkg/storage/adapter.go index 81d704145..cd0d471ca 100644 --- a/pp-pkg/storage/adapter.go +++ b/pp-pkg/storage/adapter.go @@ -36,6 +36,7 @@ type Adapter struct { hashdexLimits atomic.Value // stores cppbridge.WALHashdexLimits transparentState *cppbridge.StateV2 mergeOutOfOrderChunks func() + scrapeInterval atomic.Int64 // stat activeQuerierMetrics *querier.Metrics @@ -132,7 +133,10 @@ func (ar *Adapter) AppendSnappyProtobuf( state *cppbridge.StateV2, commitToWal bool, ) error { - hx, err := cppbridge.NewWALSnappyProtobufHashdex(compressedData.Bytes(), ar.hashdexLimits.Load().(cppbridge.WALHashdexLimits)) + hx, err := cppbridge.NewWALSnappyProtobufHashdex( + compressedData.Bytes(), + ar.hashdexLimits.Load().(cppbridge.WALHashdexLimits), + ) compressedData.Destroy() if err != nil { return err @@ -247,11 +251,12 @@ func (ar *Adapter) ChunkQuerier(mint, maxt int64) (storage.ChunkQuerier, error) // Label limit fields follow the same 0 = no limit semantics as scrape config. func (ar *Adapter) ApplyConfig(cfg *config.Config) error { limits := cppbridge.WALHashdexLimits{ - MaxLabelNamesPerTimeseries: uint32(cfg.GlobalConfig.LabelLimit), - MaxLabelNameLength: uint32(cfg.GlobalConfig.LabelNameLengthLimit), - MaxLabelValueLength: uint32(cfg.GlobalConfig.LabelValueLengthLimit), + MaxLabelNamesPerTimeseries: uint32(cfg.GlobalConfig.LabelLimit), // #nosec G115 // no overflow + MaxLabelNameLength: uint32(cfg.GlobalConfig.LabelNameLengthLimit), // #nosec G115 // no overflow + MaxLabelValueLength: uint32(cfg.GlobalConfig.LabelValueLengthLimit), // #nosec G115 // no overflow } ar.hashdexLimits.Store(limits) + ar.scrapeInterval.Store(int64(cfg.GlobalConfig.ScrapeInterval)) return nil } @@ -269,7 +274,7 @@ func (ar *Adapter) HeadQuerier(mint, maxt int64) (storage.Querier, error) { querier.NewNoOpShardedDeduplicator, mint, maxt, - nil, + ar.scrapeInterval.Load(), ar.activeQuerierMetrics, ), nil } @@ -296,7 +301,14 @@ func (ar *Adapter) Querier(mint, maxt int64) (storage.Querier, error) { ahead := ar.proxy.Get() queriers = append( queriers, - querier.NewQuerier(ahead, querier.NewNoOpShardedDeduplicator, mint, maxt, nil, ar.activeQuerierMetrics), + querier.NewQuerier( + ahead, + querier.NewNoOpShardedDeduplicator, + mint, + maxt, + ar.scrapeInterval.Load(), + ar.activeQuerierMetrics, + ), ) for _, head := range ar.proxy.Heads() { @@ -316,7 +328,7 @@ func (ar *Adapter) Querier(mint, maxt int64) (storage.Querier, error) { querier.NewNoOpShardedDeduplicator, mint, maxt, - nil, + ar.scrapeInterval.Load(), ar.storageQuerierMetrics, ), ) diff --git a/pp-pkg/storage/batch_storage.go b/pp-pkg/storage/batch_storage.go index 5794fce1b..4099c653e 100644 --- a/pp-pkg/storage/batch_storage.go +++ b/pp-pkg/storage/batch_storage.go @@ -84,11 +84,23 @@ func (bs *BatchStorage) Commit(ctx context.Context) error { // CommitWithState adds aggregated series from [pp_storage.TransactionHead] to the [Head] with [cppbridge.StateV2]. func (bs *BatchStorage) CommitWithState(ctx context.Context, state *cppbridge.StateV2) error { s := bs.transactionHead.Shards()[0] - _, err := bs.adapter.AppendGoHeadHashdex(ctx, cppbridge.NewGoHeadHashdex(s.LSS().Target(), s.DataStorage().Raw()), state, false) + _, err := bs.adapter.AppendGoHeadHashdex( + ctx, + cppbridge.NewGoHeadHashdex(s.LSS().Target(), s.DataStorage().Raw()), + state, + false, + ) return err } // Querier calls f() with the given parameters. Returns a [querier.Querier]. func (bs *BatchStorage) Querier(mint, maxt int64) (storage.Querier, error) { - return querier.NewQuerier(bs.transactionHead, querier.NewNoOpShardedDeduplicator, mint, maxt, nil, nil), nil + return querier.NewQuerier( + bs.transactionHead, + querier.NewNoOpShardedDeduplicator, + mint, + maxt, + bs.adapter.scrapeInterval.Load(), + nil, + ), nil } diff --git a/pp/go/storage/querier/querier.go b/pp/go/storage/querier/querier.go index cb8705caa..5566278c4 100644 --- a/pp/go/storage/querier/querier.go +++ b/pp/go/storage/querier/querier.go @@ -139,9 +139,9 @@ type Querier[ ] struct { mint int64 maxt int64 + scrapeInterval int64 head THead deduplicatorCtor deduplicatorCtor - closer func() error metrics *Metrics queryOptimize queryOptimizeType } @@ -156,11 +156,18 @@ func NewQuerier[ ]( head THead, deduplicatorCtor deduplicatorCtor, - mint, maxt int64, - closer func() error, + mint, maxt, scrapeInterval int64, metrics *Metrics, ) *Querier[TTask, TDataStorage, TLSS, TShard, THead] { - return newQuerierWithSelectFuncOptimize(head, deduplicatorCtor, mint, maxt, closer, metrics, selectFuncOptimize) + return newQuerierWithSelectFuncOptimize( + head, + deduplicatorCtor, + mint, + maxt, + scrapeInterval, + metrics, + selectFuncOptimize, + ) } // NewQuerierWithOutSelectFuncOptimize init new [Querier] without select func optimization. @@ -173,8 +180,7 @@ func NewQuerierWithOutSelectFuncOptimize[ ]( head THead, deduplicatorCtor deduplicatorCtor, - mint, maxt int64, - closer func() error, + mint, maxt, scrapeInterval int64, metrics *Metrics, ) *Querier[TTask, TDataStorage, TLSS, TShard, THead] { return newQuerierWithSelectFuncOptimize( @@ -182,7 +188,7 @@ func NewQuerierWithOutSelectFuncOptimize[ deduplicatorCtor, mint, maxt, - closer, + scrapeInterval, metrics, selectFuncOptimize&dropPointOptimizeType, ) @@ -198,17 +204,16 @@ func newQuerierWithSelectFuncOptimize[ ]( head THead, deduplicatorCtor deduplicatorCtor, - mint, maxt int64, - closer func() error, + mint, maxt, scrapeInterval int64, metrics *Metrics, queryOptimize queryOptimizeType, ) *Querier[TTask, TDataStorage, TLSS, TShard, THead] { return &Querier[TTask, TDataStorage, TLSS, TShard, THead]{ mint: mint, maxt: maxt, + scrapeInterval: scrapeInterval, head: head, deduplicatorCtor: deduplicatorCtor, - closer: closer, metrics: metrics, queryOptimize: queryOptimize, } @@ -217,11 +222,7 @@ func newQuerierWithSelectFuncOptimize[ // Close [Querier] if need. // //revive:disable-next-line:confusing-naming // other type of querier. -func (q *Querier[TTask, TDataStorage, TLSS, TShard, THead]) Close() error { - if q.closer != nil { - return q.closer() - } - +func (*Querier[TTask, TDataStorage, TLSS, TShard, THead]) Close() error { return nil } @@ -415,7 +416,7 @@ func (q *Querier[TTask, TDataStorage, TLSS, TShard, THead]) selectRange( return storage.ErrSeriesSet(err) } - hints = SwitchFuncOptimize(hints, isPossibleToOptimize(lssQueryResults, hints), q.queryOptimize) + hints = SwitchFuncOptimize(hints, isPossibleToOptimize(lssQueryResults, hints, q.scrapeInterval), q.queryOptimize) shardedSerializedData := poolProvider.GetSerializedData() defer poolProvider.PutSerializedData(shardedSerializedData) queryDataStorage(dsQueryRangeQuerier, q.head, lssQueryResults, shardedSerializedData, q.mint, q.maxt, hints) @@ -574,7 +575,11 @@ func SwitchFuncOptimize( } // isPossibleToOptimize checks if the query possible to optimization. -func isPossibleToOptimize(lssQueryResults []*cppbridge.LSSQueryResult, hints *storage.SelectHints) func() bool { +func isPossibleToOptimize( + lssQueryResults []*cppbridge.LSSQueryResult, + hints *storage.SelectHints, + scrapeInterval int64, +) func() bool { return func() bool { countOfSeries := 0 for _, lssQueryResult := range lssQueryResults { @@ -585,6 +590,11 @@ func isPossibleToOptimize(lssQueryResults []*cppbridge.LSSQueryResult, hints *st countOfSeries += lssQueryResult.Len() } + //revive:disable-next-line:add-constant // x2 scrape interval are required to enable optimization + if hints.Step*1e6 >= scrapeInterval*2 { + return true + } + if hints.Step == 0 && isCrossSeriesFunc(hints) { //revive:disable-next-line:add-constant // x3 we need to optimize the query for the crossseries function return countOfSeries >= DefaultCountOfSeriesToOptimize*3 diff --git a/pp/go/storage/querier/querier_optimize_test.go b/pp/go/storage/querier/querier_optimize_test.go index 4d811e1a5..fe1bacc07 100644 --- a/pp/go/storage/querier/querier_optimize_test.go +++ b/pp/go/storage/querier/querier_optimize_test.go @@ -678,7 +678,7 @@ func (s *querierOptimize) fillHeadWithCounter(ctx context.Context, counter int) // Querier implements the [prom_storage.Queryable] interface. func (s *querierOptimize) Querier(mint, maxt int64) (prom_storage.Querier, error) { - return querier.NewQuerier(s.head, querier.NewNoOpShardedDeduplicator, mint, maxt, nil, nil), nil + return querier.NewQuerier(s.head, querier.NewNoOpShardedDeduplicator, mint, maxt, int64(s.step), nil), nil } // diff --git a/pp/go/storage/querier/querier_test.go b/pp/go/storage/querier/querier_test.go index 30e08e3bd..50801dc0a 100644 --- a/pp/go/storage/querier/querier_test.go +++ b/pp/go/storage/querier/querier_test.go @@ -43,50 +43,14 @@ type QuerierSuite struct { context context.Context head *storage.Head - timeSeries []storagetest.TimeSeries - hints *prom_storage.SelectHints - matcher *labels.Matcher + hints *prom_storage.SelectHints + scrapeInterval int64 } func TestQuerierSuite(t *testing.T) { suite.Run(t, new(QuerierSuite)) } -func (s *QuerierSuite) SetupSuite() { - s.timeSeries = []storagetest.TimeSeries{ - { - Labels: labels.FromStrings("__name__", "metric", "job", "test"), - Samples: []cppbridge.Sample{ - {Timestamp: 20, Value: 10}, - {Timestamp: 40, Value: 10}, - {Timestamp: 60, Value: 20}, - {Timestamp: 80, Value: 30}, - {Timestamp: 110, Value: 50}, - {Timestamp: 130, Value: 80}, - {Timestamp: 150, Value: 130}, - {Timestamp: 170, Value: 210}, - {Timestamp: 190, Value: 340}, - }, - }, - { - Labels: labels.FromStrings("__name__", "metric", "job", "test2"), - Samples: []cppbridge.Sample{ - {Timestamp: 10, Value: 1}, - {Timestamp: 30, Value: 1}, - {Timestamp: 50, Value: 2}, - {Timestamp: 70, Value: 3}, - {Timestamp: 90, Value: 5}, - {Timestamp: 100, Value: 8}, - {Timestamp: 120, Value: 13}, - {Timestamp: 140, Value: 21}, - {Timestamp: 160, Value: 34}, - {Timestamp: 180, Value: 55}, - }, - }, - } - s.matcher, _ = labels.NewMatcher(labels.MatchEqual, "__name__", "metric") -} - func (s *QuerierSuite) SetupTest() { s.dataDir = s.createDataDirectory() s.context = context.Background() @@ -96,6 +60,7 @@ func (s *QuerierSuite) SetupTest() { End: 200, Range: 100, } + s.scrapeInterval = 1 } func (s *QuerierSuite) createDataDirectory() string { @@ -159,7 +124,7 @@ func (s *QuerierSuite) TestRangeQuery() { *shard.LSS, *shard.PerGoroutineShard, *storage.Head, - ](s.head, querier.NewNoOpShardedDeduplicator, 0, 2, nil, nil) + ](s.head, querier.NewNoOpShardedDeduplicator, 0, 2, s.scrapeInterval, nil) defer func() { _ = q.Close() }() matcher, _ := labels.NewMatcher(labels.MatchEqual, "__name__", "metric") @@ -187,7 +152,7 @@ func (s *QuerierSuite) TestRangeQueryWithoutMatching() { *shard.LSS, *shard.PerGoroutineShard, *storage.Head, - ](s.head, querier.NewNoOpShardedDeduplicator, 0, 2, nil, nil) + ](s.head, querier.NewNoOpShardedDeduplicator, 0, 2, s.scrapeInterval, nil) defer func() { _ = q.Close() }() matcher, _ := labels.NewMatcher(labels.MatchEqual, "__name__", "unknown_metric") @@ -240,7 +205,7 @@ func (s *QuerierSuite) TestRangeQueryWithDataStorageLoading() { *shard.LSS, *shard.PerGoroutineShard, *storage.Head, - ](s.head, querier.NewNoOpShardedDeduplicator, 0, 3, nil, nil) + ](s.head, querier.NewNoOpShardedDeduplicator, 0, 3, s.scrapeInterval, nil) defer func() { _ = q.Close() }() matcher, _ := labels.NewMatcher(labels.MatchEqual, "__name__", "metric") @@ -278,7 +243,7 @@ func (s *QuerierSuite) TestInstantQuery() { *shard.LSS, *shard.PerGoroutineShard, *storage.Head, - ](s.head, querier.NewNoOpShardedDeduplicator, 0, 0, nil, nil) + ](s.head, querier.NewNoOpShardedDeduplicator, 0, 0, s.scrapeInterval, nil) defer func() { _ = q.Close() }() matcher, _ := labels.NewMatcher(labels.MatchEqual, "__name__", "metric") @@ -331,7 +296,7 @@ func (s *QuerierSuite) TestInstantQueryWithDataStorageLoading() { *shard.LSS, *shard.PerGoroutineShard, *storage.Head, - ](s.head, querier.NewNoOpShardedDeduplicator, 0, 0, nil, nil) + ](s.head, querier.NewNoOpShardedDeduplicator, 0, 0, s.scrapeInterval, nil) defer func() { _ = q.Close() }() matcher, _ := labels.NewMatcher(labels.MatchEqual, "__name__", "metric") @@ -375,7 +340,7 @@ func (s *QuerierSuite) TestLabelNames() { } s.appendTimeSeries(timeSeries) - q := querier.NewQuerier(s.head, querier.NewNoOpShardedDeduplicator, 0, 2, nil, nil) + q := querier.NewQuerier(s.head, querier.NewNoOpShardedDeduplicator, 0, 2, s.scrapeInterval, nil) defer func() { _ = q.Close() }() matcher, err := labels.NewMatcher(labels.MatchEqual, "__name__", "metric0") s.Require().NoError(err) @@ -408,7 +373,7 @@ func (s *QuerierSuite) TestLabelNamesWithLimit() { } s.appendTimeSeries(timeSeries) - q := querier.NewQuerier(s.head, querier.NewNoOpShardedDeduplicator, 0, 2, nil, nil) + q := querier.NewQuerier(s.head, querier.NewNoOpShardedDeduplicator, 0, 2, s.scrapeInterval, nil) defer func() { _ = q.Close() }() matcher, err := labels.NewMatcher(labels.MatchEqual, "__name__", "metric0") s.Require().NoError(err) @@ -441,7 +406,7 @@ func (s *QuerierSuite) TestLabelNamesNoMatches() { } s.appendTimeSeries(timeSeries) - q := querier.NewQuerier(s.head, querier.NewNoOpShardedDeduplicator, 0, 2, nil, nil) + q := querier.NewQuerier(s.head, querier.NewNoOpShardedDeduplicator, 0, 2, s.scrapeInterval, nil) defer func() { _ = q.Close() }() matcher, err := labels.NewMatcher(labels.MatchEqual, "__name__", "metric3") s.Require().NoError(err) @@ -474,7 +439,7 @@ func (s *QuerierSuite) TestLabelValues() { } s.appendTimeSeries(timeSeries) - q := querier.NewQuerier(s.head, querier.NewNoOpShardedDeduplicator, 0, 2, nil, nil) + q := querier.NewQuerier(s.head, querier.NewNoOpShardedDeduplicator, 0, 2, s.scrapeInterval, nil) defer func() { _ = q.Close() }() matcher, err := labels.NewMatcher(labels.MatchRegexp, "__name__", "metric.*") s.Require().NoError(err) @@ -507,7 +472,7 @@ func (s *QuerierSuite) TestLabelValuesNoMatches() { } s.appendTimeSeries(timeSeries) - q := querier.NewQuerier(s.head, querier.NewNoOpShardedDeduplicator, 0, 2, nil, nil) + q := querier.NewQuerier(s.head, querier.NewNoOpShardedDeduplicator, 0, 2, s.scrapeInterval, nil) defer func() { _ = q.Close() }() matcher, err := labels.NewMatcher(labels.MatchEqual, "__name__", "metric2") s.Require().NoError(err) @@ -540,7 +505,7 @@ func (s *QuerierSuite) TestLabelValuesNoMatchesOnName() { } s.appendTimeSeries(timeSeries) - q := querier.NewQuerier(s.head, querier.NewNoOpShardedDeduplicator, 0, 2, nil, nil) + q := querier.NewQuerier(s.head, querier.NewNoOpShardedDeduplicator, 0, 2, s.scrapeInterval, nil) defer func() { _ = q.Close() }() matcher, err := labels.NewMatcher(labels.MatchRegexp, "__name__", "metric.*") s.Require().NoError(err) From da4ced67c82e3594e884d21cfd9f08fe16b89cca Mon Sep 17 00:00:00 2001 From: Alexandr Yudin <57181751+u-veles-a@users.noreply.github.com> Date: Fri, 5 Jun 2026 15:25:09 +0000 Subject: [PATCH 29/37] renaming Signed-off-by: Alexandr Yudin <57181751+u-veles-a@users.noreply.github.com> --- ...{agg_series_set.go => cross_series_set.go} | 100 +++++++++--------- ...s_set_test.go => cross_series_set_test.go} | 26 ++--- pp/go/storage/querier/querier.go | 8 +- 3 files changed, 67 insertions(+), 67 deletions(-) rename pp/go/storage/querier/{agg_series_set.go => cross_series_set.go} (76%) rename pp/go/storage/querier/{agg_series_set_test.go => cross_series_set_test.go} (94%) diff --git a/pp/go/storage/querier/agg_series_set.go b/pp/go/storage/querier/cross_series_set.go similarity index 76% rename from pp/go/storage/querier/agg_series_set.go rename to pp/go/storage/querier/cross_series_set.go index df4c5d700..9faefb398 100644 --- a/pp/go/storage/querier/agg_series_set.go +++ b/pp/go/storage/querier/cross_series_set.go @@ -16,13 +16,13 @@ import ( ) // -// AggSeriesSet +// CrossSeriesSet // -// AggSeriesSet contains a set of aggregated series. +// CrossSeriesSet contains a set of cross series. // If grouping is empty, it will return series with labels "__head__shard_id". // If grouping is not empty, it will return series with "__head__shard_id" and the grouping labels. -type AggSeriesSet struct { +type CrossSeriesSet struct { serializedData *cppbridge.DataStorageSerializedData labelSetSnapshot *cppbridge.LabelSetSnapshot seriesGroups *cppbridge.SeriesGroups @@ -31,12 +31,12 @@ type AggSeriesSet struct { headID string shardID uint16 - series []AggSeries + series []CrossSeries nextGroupIndex int } -// NewAggSeriesSet initializes a new [AggSeriesSet]. -func NewAggSeriesSet( +// NewCrossSeriesSet initializes a new [CrossSeriesSet]. +func NewCrossSeriesSet( serializedData *cppbridge.DataStorageSerializedData, labelSetSnapshot *cppbridge.LabelSetSnapshot, seriesGroups *cppbridge.SeriesGroups, @@ -44,8 +44,8 @@ func NewAggSeriesSet( grouping []string, headID string, shardID uint16, -) *AggSeriesSet { - return &AggSeriesSet{ +) *CrossSeriesSet { + return &CrossSeriesSet{ serializedData: serializedData, labelSetSnapshot: labelSetSnapshot, seriesGroups: seriesGroups, @@ -54,25 +54,25 @@ func NewAggSeriesSet( grouping: grouping, headID: headID, shardID: shardID, - series: make([]AggSeries, 0, len(seriesGroups.Groups)), + series: make([]CrossSeries, 0, len(seriesGroups.Groups)), } } // At returns the current series. // [storage.SeriesSet] interface implementation. -func (ss *AggSeriesSet) At() storage.Series { +func (ss *CrossSeriesSet) At() storage.Series { return &ss.series[len(ss.series)-1] } -// Err returns the error of the [AggSeriesSet] - always nil. +// Err returns the error of the [CrossSeriesSet] - always nil. // [storage.SeriesSet] interface implementation. -func (*AggSeriesSet) Err() error { +func (*CrossSeriesSet) Err() error { return nil } // Next advances the iterator by one and returns false if there are no more values. // [storage.SeriesSet] interface implementation. -func (ss *AggSeriesSet) Next() bool { +func (ss *CrossSeriesSet) Next() bool { if ss.serializedData == nil { return false } @@ -83,8 +83,8 @@ func (ss *AggSeriesSet) Next() bool { builder := builderPool.Get().(*labels.ScratchBuilder) builder.Reset() - ss.series = append(ss.series, NewAggSeries( - aggLabelSetCtor( + ss.series = append(ss.series, NewCrossSeries( + crossLabelSetCtor( builder, ss.labelSetSnapshot, ss.grouping, @@ -104,18 +104,18 @@ func (ss *AggSeriesSet) Next() bool { return true } -// Warnings returns the warnings of the [AggSeriesSet] - always nil. +// Warnings returns the warnings of the [CrossSeriesSet] - always nil. // [storage.SeriesSet] interface implementation. -func (*AggSeriesSet) Warnings() annotations.Annotations { +func (*CrossSeriesSet) Warnings() annotations.Annotations { return nil } // -// AggSeries +// CrossSeries // -// AggSeries represents a time series with aggregated samples. -type AggSeries struct { +// CrossSeries represents a time series with cross samples. +type CrossSeries struct { labelSet labels.Labels serializedData *cppbridge.DataStorageSerializedData seriesGroups *cppbridge.SeriesGroups @@ -123,15 +123,15 @@ type AggSeries struct { mint, maxt int64 } -// NewAggSeries initializes a new [AggSeries]. -func NewAggSeries( +// NewCrossSeries initializes a new [CrossSeries]. +func NewCrossSeries( labelSet labels.Labels, serializedData *cppbridge.DataStorageSerializedData, seriesGroups *cppbridge.SeriesGroups, groupIndex int, mint, maxt int64, -) AggSeries { - return AggSeries{ +) CrossSeries { + return CrossSeries{ labelSet: labelSet, serializedData: serializedData, seriesGroups: seriesGroups, @@ -141,12 +141,12 @@ func NewAggSeries( } } -// Iterator returns an iterator that iterates over the aggregated samples of the [AggSeries]. +// Iterator returns an iterator that iterates over the cross samples of the [CrossSeries]. // [storage.Series] interface implementation. -func (s *AggSeries) Iterator(it chunkenc.Iterator) chunkenc.Iterator { - chunkIterator, ok := it.(*AggChunkIterator) +func (s *CrossSeries) Iterator(it chunkenc.Iterator) chunkenc.Iterator { + chunkIterator, ok := it.(*CrossChunkIterator) if !ok { - return NewAggChunkIterator( + return NewCrossChunkIterator( s.serializedData, s.seriesGroups, s.groupIndex, @@ -159,18 +159,18 @@ func (s *AggSeries) Iterator(it chunkenc.Iterator) chunkenc.Iterator { return chunkIterator } -// Labels returns the labels of the [AggSeries]. +// Labels returns the labels of the [CrossSeries]. // [storage.Series] interface implementation. -func (s *AggSeries) Labels() labels.Labels { +func (s *CrossSeries) Labels() labels.Labels { return s.labelSet } // -// AggChunkIterator +// CrossChunkIterator // -// AggChunkIterator iterates over the aggregated samples of a time series, that can only get the next value. -type AggChunkIterator struct { +// CrossChunkIterator iterates over the cross samples of a time series, that can only get the next value. +type CrossChunkIterator struct { serializedData *cppbridge.DataStorageSerializedData seriesGroups *cppbridge.SeriesGroups chunkIterator cppbridge.DataStorageSerializedDataMultiSeriesIterator @@ -179,14 +179,14 @@ type AggChunkIterator struct { isInitialized bool } -// NewAggChunkIterator initializes a new [AggChunkIterator]. -func NewAggChunkIterator( +// NewCrossChunkIterator initializes a new [CrossChunkIterator]. +func NewCrossChunkIterator( serializedData *cppbridge.DataStorageSerializedData, seriesGroups *cppbridge.SeriesGroups, groupIndex int, mint, maxt int64, -) *AggChunkIterator { - it := &AggChunkIterator{ +) *CrossChunkIterator { + it := &CrossChunkIterator{ serializedData: serializedData, seriesGroups: seriesGroups, chunkIterator: cppbridge.NewDataStorageSerializedDataMultiSeriesIterator( @@ -197,7 +197,7 @@ func NewAggChunkIterator( maxt: maxt, } - runtime.SetFinalizer(it, func(iter *AggChunkIterator) { + runtime.SetFinalizer(it, func(iter *CrossChunkIterator) { iter.chunkIterator.Close() iter.serializedData = nil iter.seriesGroups = nil @@ -210,37 +210,37 @@ func NewAggChunkIterator( // [chunkenc.Iterator] interface implementation. // //nolint:gocritic // unnamedResult not need -func (it *AggChunkIterator) At() (int64, float64) { +func (it *CrossChunkIterator) At() (int64, float64) { return it.chunkIterator.Timestamp(), it.chunkIterator.Value() } // AtFloatHistogram returns the current timestamp/value pair if the value is a histogram with floating-point counts. // [chunkenc.Iterator] interface implementation. -func (*AggChunkIterator) AtFloatHistogram(*histogram.FloatHistogram) (int64, *histogram.FloatHistogram) { +func (*CrossChunkIterator) AtFloatHistogram(*histogram.FloatHistogram) (int64, *histogram.FloatHistogram) { return 0, nil } // AtHistogram returns the current timestamp/value pair if the value is a histogram with integer counts. // [chunkenc.Iterator] interface implementation. -func (*AggChunkIterator) AtHistogram(*histogram.Histogram) (int64, *histogram.Histogram) { +func (*CrossChunkIterator) AtHistogram(*histogram.Histogram) (int64, *histogram.Histogram) { return 0, nil } // AtT returns the current timestamp. // [chunkenc.Iterator] interface implementation. -func (it *AggChunkIterator) AtT() int64 { +func (it *CrossChunkIterator) AtT() int64 { return it.chunkIterator.Timestamp() } // Err returns the current error - always nil. // [chunkenc.Iterator] interface implementation. -func (*AggChunkIterator) Err() error { +func (*CrossChunkIterator) Err() error { return nil } // Next advances the iterator by one and returns the type of the value. // [chunkenc.Iterator] interface implementation. -func (it *AggChunkIterator) Next() chunkenc.ValueType { +func (it *CrossChunkIterator) Next() chunkenc.ValueType { if it.nextValue() == chunkenc.ValNone { return chunkenc.ValNone } @@ -254,7 +254,7 @@ func (it *AggChunkIterator) Next() chunkenc.ValueType { // Seek advances the iterator forward to the first sample with a timestamp equal or greater than t. // [chunkenc.Iterator] interface implementation. -func (it *AggChunkIterator) Seek(t int64) chunkenc.ValueType { +func (it *CrossChunkIterator) Seek(t int64) chunkenc.ValueType { it.isInitialized = true if t > it.AtT() { return it.Next() @@ -268,7 +268,7 @@ func (it *AggChunkIterator) Seek(t int64) chunkenc.ValueType { } // nextValue advances the iterator by one and returns the type of the value. -func (it *AggChunkIterator) nextValue() chunkenc.ValueType { +func (it *CrossChunkIterator) nextValue() chunkenc.ValueType { if !it.isInitialized { if !it.chunkIterator.HasData() { return chunkenc.ValNone @@ -287,7 +287,7 @@ func (it *AggChunkIterator) nextValue() chunkenc.ValueType { } // reset resets the iterator to the beginning of the serialized data. -func (it *AggChunkIterator) reset( +func (it *CrossChunkIterator) reset( serializedData *cppbridge.DataStorageSerializedData, seriesGroups *cppbridge.SeriesGroups, groupIndex int, @@ -302,7 +302,7 @@ func (it *AggChunkIterator) reset( } // -// aggLabelSetCtor +// crossLabelSetCtor // const ( @@ -318,8 +318,8 @@ var ( errGroupingLabelsIsEnough = errors.New("grouping labels is enough") ) -// aggLabelSetCtor constructs the label set for an aggregated series. -func aggLabelSetCtor( +// crossLabelSetCtor constructs the label set for a cross series. +func crossLabelSetCtor( sb *labels.ScratchBuilder, snapshot *cppbridge.LabelSetSnapshot, grouping []string, diff --git a/pp/go/storage/querier/agg_series_set_test.go b/pp/go/storage/querier/cross_series_set_test.go similarity index 94% rename from pp/go/storage/querier/agg_series_set_test.go rename to pp/go/storage/querier/cross_series_set_test.go index f0efa1ab5..8a6e416d1 100644 --- a/pp/go/storage/querier/agg_series_set_test.go +++ b/pp/go/storage/querier/cross_series_set_test.go @@ -17,7 +17,7 @@ import ( "github.com/stretchr/testify/suite" ) -type AggSeriesSetSuite struct { +type CrossSeriesSetSuite struct { suite.Suite timeSeries []storagetest.TimeSeries @@ -25,11 +25,11 @@ type AggSeriesSetSuite struct { ds *shard.DataStorage } -func TestAggSeriesSetSuite(t *testing.T) { - suite.Run(t, new(AggSeriesSetSuite)) +func TestCrossSeriesSetSuite(t *testing.T) { + suite.Run(t, new(CrossSeriesSetSuite)) } -func (s *AggSeriesSetSuite) SetupTest() { +func (s *CrossSeriesSetSuite) SetupTest() { s.lss = shard.NewLSS() s.ds = shard.NewDataStorage() @@ -65,7 +65,7 @@ func (s *AggSeriesSetSuite) SetupTest() { } } -func (s *AggSeriesSetSuite) query( +func (s *CrossSeriesSetSuite) query( lss *shard.LSS, ds *shard.DataStorage, start, end, downsamplingMs int64, @@ -76,12 +76,12 @@ func (s *AggSeriesSetSuite) query( s.Require().NoError(err) if selector == 0 || snapshot == nil { - return &querier.AggSeriesSet{} + return &querier.CrossSeriesSet{} } lssQueryResult := snapshot.Query(selector) if lssQueryResult.Status() == cppbridge.LSSQueryStatusNoMatch { - return &querier.AggSeriesSet{} + return &querier.CrossSeriesSet{} } valueNotFoundTimestampValue := int64(0) @@ -107,7 +107,7 @@ func (s *AggSeriesSetSuite) query( s.Require().Equal(cppbridge.DataStorageQueryStatusSuccess, dsQueryResult.Status) - aggSS := querier.NewAggSeriesSet( + aggSS := querier.NewCrossSeriesSet( dsQueryResult.SerializedData, snapshot, seriesGroups, @@ -121,7 +121,7 @@ func (s *AggSeriesSetSuite) query( return querier.NewMergeShardSeriesSet([]storage.SeriesSet{sNaNSS, aggSS}) } -func (s *AggSeriesSetSuite) TestQueryWithoutGrouping() { +func (s *CrossSeriesSetSuite) TestQueryWithoutGrouping() { // Arrange matcher := model.LabelMatcher{ Name: "__name__", @@ -167,7 +167,7 @@ func (s *AggSeriesSetSuite) TestQueryWithoutGrouping() { s.Require().Equal(expected[2].Labels, actual[2].Labels) } -func (s *AggSeriesSetSuite) TestQueryGrouping_OneGroupingLabel() { +func (s *CrossSeriesSetSuite) TestQueryGrouping_OneGroupingLabel() { // Arrange matcher := model.LabelMatcher{ Name: "__name__", @@ -219,7 +219,7 @@ func (s *AggSeriesSetSuite) TestQueryGrouping_OneGroupingLabel() { s.Require().Equal(expected[3].Labels, actual[3].Labels) } -func (s *AggSeriesSetSuite) TestQueryGrouping_TwoGroupingLabels() { +func (s *CrossSeriesSetSuite) TestQueryGrouping_TwoGroupingLabels() { // Arrange matcher := model.LabelMatcher{ Name: "__name__", @@ -271,7 +271,7 @@ func (s *AggSeriesSetSuite) TestQueryGrouping_TwoGroupingLabels() { s.Require().Equal(expected[3].Labels, actual[3].Labels) } -func (s *AggSeriesSetSuite) TestQueryGrouping_TwoGroupingLabels_WithMissingGroupingLabel() { +func (s *CrossSeriesSetSuite) TestQueryGrouping_TwoGroupingLabels_WithMissingGroupingLabel() { // Arrange matcher := model.LabelMatcher{ Name: "__name__", @@ -388,7 +388,7 @@ func TestAGGSS(t *testing.T) { hints, ) - aggSS := querier.NewAggSeriesSet( + aggSS := querier.NewCrossSeriesSet( result.SerializedData, snapshot, seriesGroups, diff --git a/pp/go/storage/querier/querier.go b/pp/go/storage/querier/querier.go index 5566278c4..22a38bed5 100644 --- a/pp/go/storage/querier/querier.go +++ b/pp/go/storage/querier/querier.go @@ -422,7 +422,7 @@ func (q *Querier[TTask, TDataStorage, TLSS, TShard, THead]) selectRange( queryDataStorage(dsQueryRangeQuerier, q.head, lssQueryResults, shardedSerializedData, q.mint, q.maxt, hints) if isCrossSeriesFunc(hints) { - return q.makeAggSeriesSet(lssQueryResults, snapshots, shardedSerializedData, hints) + return q.makeCrossSeriesSet(lssQueryResults, snapshots, shardedSerializedData, hints) } return q.makeSeriesSet(lssQueryResults, snapshots, shardedSerializedData) @@ -456,8 +456,8 @@ func (q *Querier[TTask, TDataStorage, TLSS, TShard, THead]) makeSeriesSet( return NewMergeShardSeriesSet(seriesSets) } -// makeAggSeriesSet queries the aggregated cross series set. -func (q *Querier[TTask, TDataStorage, TLSS, TShard, THead]) makeAggSeriesSet( +// makeCrossSeriesSet queries the cross series set. +func (q *Querier[TTask, TDataStorage, TLSS, TShard, THead]) makeCrossSeriesSet( lssQueryResults []*cppbridge.LSSQueryResult, snapshots []*cppbridge.LabelSetSnapshot, shardedSerializedData []*cppbridge.DataStorageSerializedData, @@ -535,7 +535,7 @@ func (q *Querier[TTask, TDataStorage, TLSS, TShard, THead]) makeAggSeriesSet( DefaultNotFoundTimestampValue, ) - seriesSets[shardID] = NewAggSeriesSet( + seriesSets[shardID] = NewCrossSeriesSet( serializedData, snapshots[shardID], seriesGroups[shardID], From b1a219955c0ae2ce5e4c8edd63f2a3794f207fe0 Mon Sep 17 00:00:00 2001 From: Alexandr Yudin <57181751+u-veles-a@users.noreply.github.com> Date: Thu, 11 Jun 2026 16:41:43 +0000 Subject: [PATCH 30/37] WIP: calc aggr points Signed-off-by: Alexandr Yudin <57181751+u-veles-a@users.noreply.github.com> --- pp/go/storage/querier/aggr_series_set.go | 31 +- pp/go/storage/querier/cross_series_set.go | 31 +- pp/go/storage/querier/querier.go | 78 +++- .../storage/querier/querier_optimize_test.go | 412 ++++++++++++++++-- .../querier_optimize_variables_test.go | 32 +- pp/go/storage/querier/series_test.go | 103 +++++ 6 files changed, 573 insertions(+), 114 deletions(-) diff --git a/pp/go/storage/querier/aggr_series_set.go b/pp/go/storage/querier/aggr_series_set.go index 0884d8c26..a4c7f4f09 100644 --- a/pp/go/storage/querier/aggr_series_set.go +++ b/pp/go/storage/querier/aggr_series_set.go @@ -203,8 +203,16 @@ func (*AggrChunkIterator) Err() error { // Next advances the iterator by one and returns the type of the value. // [chunkenc.Iterator] interface implementation. func (it *AggrChunkIterator) Next() chunkenc.ValueType { - if it.nextValue() == chunkenc.ValNone { - return chunkenc.ValNone + if !it.isInitialized { + if !it.chunkIterator.HasData() { + return chunkenc.ValNone + } + it.isInitialized = true + } else { + it.chunkIterator.Next() + if !it.chunkIterator.HasData() { + return chunkenc.ValNone + } } if it.AtT() > it.maxt { @@ -229,25 +237,6 @@ func (it *AggrChunkIterator) Seek(t int64) chunkenc.ValueType { return chunkenc.ValFloat } -// nextValue advances the iterator by one and returns the type of the value. -func (it *AggrChunkIterator) nextValue() chunkenc.ValueType { - if !it.isInitialized { - if !it.chunkIterator.HasData() { - return chunkenc.ValNone - } - - it.isInitialized = true - return chunkenc.ValFloat - } - - it.chunkIterator.Next() - if !it.chunkIterator.HasData() { - return chunkenc.ValNone - } - - return chunkenc.ValFloat -} - // reset resets the iterator to the beginning of the serialized data. func (it *AggrChunkIterator) reset( serializedData *cppbridge.DataStorageSerializedData, diff --git a/pp/go/storage/querier/cross_series_set.go b/pp/go/storage/querier/cross_series_set.go index 9faefb398..dd939bb9c 100644 --- a/pp/go/storage/querier/cross_series_set.go +++ b/pp/go/storage/querier/cross_series_set.go @@ -241,8 +241,16 @@ func (*CrossChunkIterator) Err() error { // Next advances the iterator by one and returns the type of the value. // [chunkenc.Iterator] interface implementation. func (it *CrossChunkIterator) Next() chunkenc.ValueType { - if it.nextValue() == chunkenc.ValNone { - return chunkenc.ValNone + if !it.isInitialized { + if !it.chunkIterator.HasData() { + return chunkenc.ValNone + } + it.isInitialized = true + } else { + it.chunkIterator.Next() + if !it.chunkIterator.HasData() { + return chunkenc.ValNone + } } if it.AtT() > it.maxt { @@ -267,25 +275,6 @@ func (it *CrossChunkIterator) Seek(t int64) chunkenc.ValueType { return chunkenc.ValFloat } -// nextValue advances the iterator by one and returns the type of the value. -func (it *CrossChunkIterator) nextValue() chunkenc.ValueType { - if !it.isInitialized { - if !it.chunkIterator.HasData() { - return chunkenc.ValNone - } - - it.isInitialized = true - return chunkenc.ValFloat - } - - it.chunkIterator.Next() - if !it.chunkIterator.HasData() { - return chunkenc.ValNone - } - - return chunkenc.ValFloat -} - // reset resets the iterator to the beginning of the serialized data. func (it *CrossChunkIterator) reset( serializedData *cppbridge.DataStorageSerializedData, diff --git a/pp/go/storage/querier/querier.go b/pp/go/storage/querier/querier.go index 3a704a517..705ccb888 100644 --- a/pp/go/storage/querier/querier.go +++ b/pp/go/storage/querier/querier.go @@ -75,7 +75,7 @@ const ( ) // DefaultCountOfSeriesToOptimize is the default count of series to optimize. -const DefaultCountOfSeriesToOptimize = 6 +const DefaultCountOfSeriesToOptimize = 11 // defaultOptimizeType is the default option for selecting functions optimization. var defaultOptimizeType = noneOptimizeType @@ -622,18 +622,78 @@ func isPossibleToOptimize( countOfSeries += lssQueryResult.Len() } - //revive:disable-next-line:add-constant // x2 scrape interval are required to enable optimization - if hints.Step*1e6 >= scrapeInterval*2 { - return true + if isCrossSeriesFunc(hints) { + return isPossibleToOptimizeCrossSeriesFunc(hints, scrapeInterval, countOfSeries) } - if hints.Step == 0 && isCrossSeriesFunc(hints) { - //revive:disable-next-line:add-constant // x3 we need to optimize the query for the crossseries function - return countOfSeries >= DefaultCountOfSeriesToOptimize*3 - } + return true + } +} + +// isPossibleToOptimizeCrossSeriesFunc checks if the cross series function is possible to optimize. +func isPossibleToOptimizeCrossSeriesFunc( + hints *storage.SelectHints, + scrapeInterval int64, + countOfSeries int, +) bool { + // for cross series functions for instant query, we don't need to optimize the query + if hints.Step == 0 { + return false + } - return countOfSeries >= DefaultCountOfSeriesToOptimize + if countOfSeries < DefaultCountOfSeriesToOptimize { + return false } + + if len(hints.Grouping) == 0 { + return true + } + + hintStep := hints.Step * 1e6 //revive:disable-line:add-constant // ms to ns + + // grouping by + if hintStep == scrapeInterval && countOfSeries > DefaultCountOfSeriesToOptimize+1 { + return true + } + + //revive:disable-next-line:add-constant // x2 scrape interval are required to enable optimization + if hintStep >= scrapeInterval*2 && countOfSeries >= DefaultCountOfSeriesToOptimize { + return true + } + + //revive:disable-next-line:add-constant // x3 scrape interval are required to enable optimization + if hintStep >= scrapeInterval*3 && countOfSeries > DefaultCountOfSeriesToOptimize { + return true + } + + return false +} + +func isPossibleToOptimizeAggregationFunc( + hints *storage.SelectHints, + scrapeInterval, shift int64, +) bool { + hintStep := hints.Step * 1e6 //revive:disable-line:add-constant // ms to ns + if hintStep <= scrapeInterval { + return false + } + + // instant query, shift is x5 scrape interval and range is x5 scrape interval + if hints.Step == 0 { + return hintStep >= scrapeInterval*5 && hints.Range >= scrapeInterval*5 + } + + hintRange := hints.Range * 1e6 //revive:disable-line:add-constant // ms to ns + if hintRange < scrapeInterval { + return false + } + + //revive:disable-next-line:add-constant // check if the range is even + if hintRange/scrapeInterval%2 != 0 { + return false + } + + return true } // isNotWithpout checks if the hints is not without by. diff --git a/pp/go/storage/querier/querier_optimize_test.go b/pp/go/storage/querier/querier_optimize_test.go index fe1bacc07..4a58abc87 100644 --- a/pp/go/storage/querier/querier_optimize_test.go +++ b/pp/go/storage/querier/querier_optimize_test.go @@ -479,7 +479,7 @@ func (s *querierOptimize) setup(ctx context.Context, baseDir string, noErrorFunc s.head = s.mustCreateHead(0) s.fillHead(ctx) - s.fillHeadWithCounter(ctx, querier.DefaultCountOfSeriesToOptimize) + s.fillHeadWithCounter(ctx, 0, querier.DefaultCountOfSeriesToOptimize) s.lookbackDelta = 5 * time.Minute s.queryOpts = promql.NewPrometheusQueryOpts(false, s.lookbackDelta) @@ -640,17 +640,17 @@ func (s *querierOptimize) fillHead(ctx context.Context) { } // fillHeadWithCounter fills the head with the given number of counter metrics. -func (s *querierOptimize) fillHeadWithCounter(ctx context.Context, counter int) { +func (s *querierOptimize) fillHeadWithCounter(ctx context.Context, start, counter int) { countOfSamples := (s.end.UnixMilli()-s.start.UnixMilli())/s.step.Milliseconds() + 1 timeSeries := make([]storagetest.TimeSeries, 0, counter*2) - for i := range counter { + for i := start; i < start+counter; i++ { timeSeries = append(timeSeries, storagetest.TimeSeries{ Labels: labels.FromStrings("__name__", "counter_metric", "value", "inc", "counter", strconv.Itoa(i)), Samples: make([]cppbridge.Sample, 0, countOfSamples), }) } - for i := range counter { + for i := start; i < start+counter; i++ { timeSeries = append(timeSeries, storagetest.TimeSeries{ Labels: labels.FromStrings("__name__", "sin_cos_metric", "value", "sin_cos", "counter", strconv.Itoa(i)), Samples: make([]cppbridge.Sample, 0, countOfSamples), @@ -889,6 +889,28 @@ func (s *MatrixQuerierOptimizeSuiteSuite) TestQueryRangeSingle() { } } +func (s *MatrixQuerierOptimizeSuiteSuite) TestQueryRangeSingle2() { + ctx := s.T().Context() + query := "max_over_time(counter_metric[75s])" + start := s.start + step := 60 * time.Second + + s.Run(query, func() { + res, err := queryRange(ctx, "none", s.queryEngine, s, s.queryOpts, query, start, s.end, step) + s.Require().NoError(err) + defer res.qry.Close() + + resOpt, err := queryRange(ctx, "all", s.queryEngine, s, s.queryOpts, query, start, s.end, step) + s.Require().NoError(err) + defer resOpt.qry.Close() + + fmt.Println("res", res.res) + fmt.Println("resOpt", resOpt.res) + + s.Require().True(resultEqual(res.res, resOpt.res, query)) + }) +} + // // resultEqual // @@ -1111,11 +1133,10 @@ func calcRelative(expected, actual float64) float64 { // Benchmark // -func BenchmarkRangeQuery(b *testing.B) { +func BenchmarkRangeQuerySum(b *testing.B) { ctx := b.Context() qo := &querierOptimize{} qo.setup(ctx, b.TempDir(), func(err error, msgAndArgs ...any) { require.NoError(b, err, msgAndArgs) }) - qo.fillHeadWithCounter(ctx, 3) defer qo.close() queryEngine := promql.NewEngine(promql.EngineOpts{ @@ -1129,36 +1150,181 @@ func BenchmarkRangeQuery(b *testing.B) { }) query := "sum(counter_metric)" - // query := "sum by(value) (counter_metric)" - // query := "max_over_time(counter_metric[3600s])" - - step := qo.step - // step := qo.step * 4 - - b.Run("none", func(b *testing.B) { - b.ResetTimer() - for b.Loop() { - res, err := queryRange(ctx, "none", queryEngine, qo, qo.queryOpts, query, qo.start, qo.end, step) - require.NoError(b, err) - res.qry.Close() + steps := []time.Duration{ + qo.step, + qo.step * 2, + qo.step * 3, + } + + series := 6 + qo.fillHeadWithCounter(ctx, 0, series) + + // 3 - default series for counter_metric + for i := series + 3; i < 14; i++ { + qo.fillHeadWithCounter(ctx, i, 1) + for _, step := range steps { + b.Run(fmt.Sprintf("opt_none_series_%d_step_%s_range_0s", i, step), func(b *testing.B) { + b.ResetTimer() + for b.Loop() { + res, err := queryRange(ctx, "none", queryEngine, qo, qo.queryOpts, query, qo.start, qo.end, step) + require.NoError(b, err) + res.qry.Close() + } + }) + + b.Run(fmt.Sprintf("opt_all_series_%d_step_%s_range_0s", i, step), func(b *testing.B) { + b.ResetTimer() + for b.Loop() { + res, err := queryRange(ctx, "all", queryEngine, qo, qo.queryOpts, query, qo.start, qo.end, step) + require.NoError(b, err) + res.qry.Close() + } + }) } + } +} + +func BenchmarkRangeQuerySumBy(b *testing.B) { + ctx := b.Context() + qo := &querierOptimize{} + qo.setup(ctx, b.TempDir(), func(err error, msgAndArgs ...any) { require.NoError(b, err, msgAndArgs) }) + defer qo.close() + + queryEngine := promql.NewEngine(promql.EngineOpts{ + Logger: log.NewNopLogger(), + MaxSamples: 100000, + Timeout: 10 * time.Second, + LookbackDelta: qo.lookbackDelta, + NoStepSubqueryIntervalFn: func(int64) int64 { return qo.lookbackDelta.Milliseconds() }, + EnableAtModifier: true, + EnableNegativeOffset: true, }) - b.Run("all", func(b *testing.B) { - b.ResetTimer() - for b.Loop() { - res, err := queryRange(ctx, "all", queryEngine, qo, qo.queryOpts, query, qo.start, qo.end, step) - require.NoError(b, err) - res.qry.Close() + query := "sum by(value) (counter_metric)" + steps := []time.Duration{ + qo.step, + qo.step * 2, + qo.step * 3, + } + + series := 7 + qo.fillHeadWithCounter(ctx, 0, series) + + // 3 - default series for counter_metric + for i := series + 3; i < 20; i++ { + qo.fillHeadWithCounter(ctx, i, 1) + for _, step := range steps { + b.Run(fmt.Sprintf("opt_none_series_%d_step_%s_range_0s", i, step), func(b *testing.B) { + b.ResetTimer() + for b.Loop() { + res, err := queryRange(ctx, "none", queryEngine, qo, qo.queryOpts, query, qo.start, qo.end, step) + require.NoError(b, err) + res.qry.Close() + } + }) + + b.Run(fmt.Sprintf("opt_all_series_%d_step_%s_range_0s", i, step), func(b *testing.B) { + b.ResetTimer() + for b.Loop() { + res, err := queryRange(ctx, "all", queryEngine, qo, qo.queryOpts, query, qo.start, qo.end, step) + require.NoError(b, err) + res.qry.Close() + } + }) } + } +} + +//revive:disable-next-line:cognitive-complexity // this is a benchmark +func BenchmarkRangeQueryOverTime(b *testing.B) { + ctx := b.Context() + qo := &querierOptimize{} + qo.setup(ctx, b.TempDir(), func(err error, msgAndArgs ...any) { require.NoError(b, err, msgAndArgs) }) + defer qo.close() + + queryEngine := promql.NewEngine(promql.EngineOpts{ + Logger: log.NewNopLogger(), + MaxSamples: 100000, + Timeout: 10 * time.Second, + LookbackDelta: qo.lookbackDelta, + NoStepSubqueryIntervalFn: func(int64) int64 { return qo.lookbackDelta.Milliseconds() }, + EnableAtModifier: true, + EnableNegativeOffset: true, }) + + queryf := "max_over_time(counter_metric[%s])" + steps := []time.Duration{ + qo.step, + qo.step * 2, + qo.step * 25 / 10, + qo.step * 3, + qo.step * 35 / 10, + qo.step * 4, + qo.step * 9, + qo.step * 10, + } + ranges := []time.Duration{ + qo.step / 2, + qo.step, + qo.step * 15 / 10, + qo.step * 2, + qo.step * 25 / 10, + qo.step * 3, + qo.step * 35 / 10, + qo.step * 4, + qo.step * 45 / 10, + qo.step * 5, + qo.step * 55 / 10, + qo.step * 6, + qo.step * 65 / 10, + qo.step * 7, + qo.step * 75 / 10, + qo.step * 8, + qo.step * 85 / 10, + qo.step * 9, + qo.step * 95 / 10, + qo.step * 10, + qo.step * 20, + qo.step * 30, + qo.step * 40, + qo.step * 60, + } + + series := 6 + qo.fillHeadWithCounter(ctx, 0, series) + + // 3 - default series for counter_metric + for i := series + 3; i < 12; i++ { + qo.fillHeadWithCounter(ctx, i, 1) + for _, step := range steps { + for _, r := range ranges { + query := fmt.Sprintf(queryf, r) + b.Run(fmt.Sprintf("opt_none_series_%d_step_%s_range_%s", i, step, r), func(b *testing.B) { + b.ResetTimer() + for b.Loop() { + res, err := queryRange(ctx, "none", queryEngine, qo, qo.queryOpts, query, qo.start, qo.end, step) + require.NoError(b, err) + res.qry.Close() + } + }) + + b.Run(fmt.Sprintf("opt_all_series_%d_step_%s_range_%s", i, step, r), func(b *testing.B) { + b.ResetTimer() + for b.Loop() { + res, err := queryRange(ctx, "all", queryEngine, qo, qo.queryOpts, query, qo.start, qo.end, step) + require.NoError(b, err) + res.qry.Close() + } + }) + } + } + } } -func BenchmarkInstantQuery(b *testing.B) { +func BenchmarkInstantQuerySum(b *testing.B) { ctx := b.Context() qo := &querierOptimize{} qo.setup(ctx, b.TempDir(), func(err error, msgAndArgs ...any) { require.NoError(b, err, msgAndArgs) }) - qo.fillHeadWithCounter(ctx, 3) defer qo.close() queryEngine := promql.NewEngine(promql.EngineOpts{ @@ -1171,30 +1337,182 @@ func BenchmarkInstantQuery(b *testing.B) { EnableNegativeOffset: true, }) - // query := "sum(counter_metric)" - // query := "sum by(value) (counter_metric)" - query := "max_over_time(counter_metric[3600s])" - - ttt := 3700 * time.Second - mid := qo.start.Add(ttt) - // mid := qo.start - // mid := qo.end - - b.Run("none", func(b *testing.B) { - b.ResetTimer() - for b.Loop() { - res, err := queryInstant(ctx, "none", queryEngine, qo, qo.queryOpts, query, mid) - require.NoError(b, err) - res.qry.Close() + query := "sum(counter_metric)" + shifts := []time.Duration{ + qo.step * 4, // 240s + qo.step * 60, // 1800s + qo.step * 125, // 3750s + } + + series := 10 + qo.fillHeadWithCounter(ctx, 0, series) + + // 3 - default series for counter_metric + for i := series + 3; i < 18; i++ { + qo.fillHeadWithCounter(ctx, i, 1) + for _, shift := range shifts { + b.Run(fmt.Sprintf("opt_none_series_%d_shift_%s_range_0s", i, shift), func(b *testing.B) { + b.ResetTimer() + for b.Loop() { + res, err := queryInstant(ctx, "none", queryEngine, qo, qo.queryOpts, query, qo.start.Add(shift)) + require.NoError(b, err) + res.qry.Close() + } + }) + + b.Run(fmt.Sprintf("opt_all_series_%d_shift_%s_range_0s", i, shift), func(b *testing.B) { + b.ResetTimer() + for b.Loop() { + res, err := queryInstant(ctx, "all", queryEngine, qo, qo.queryOpts, query, qo.start.Add(shift)) + require.NoError(b, err) + res.qry.Close() + } + }) } + } +} + +func BenchmarkInstantQuerySumBy(b *testing.B) { + ctx := b.Context() + qo := &querierOptimize{} + qo.setup(ctx, b.TempDir(), func(err error, msgAndArgs ...any) { require.NoError(b, err, msgAndArgs) }) + defer qo.close() + + queryEngine := promql.NewEngine(promql.EngineOpts{ + Logger: log.NewNopLogger(), + MaxSamples: 100000, + Timeout: 10 * time.Second, + LookbackDelta: qo.lookbackDelta, + NoStepSubqueryIntervalFn: func(int64) int64 { return qo.lookbackDelta.Milliseconds() }, + EnableAtModifier: true, + EnableNegativeOffset: true, }) - b.Run("all", func(b *testing.B) { - b.ResetTimer() - for b.Loop() { - res, err := queryInstant(ctx, "all", queryEngine, qo, qo.queryOpts, query, mid) - require.NoError(b, err) - res.qry.Close() + // query := "sum(counter_metric)" + query := "sum by(value) (counter_metric)" + shifts := []time.Duration{ + qo.step * 4, // 240s + qo.step * 60, // 1800s + qo.step * 125, // 3750s + } + + series := 6 + qo.fillHeadWithCounter(ctx, 0, series) + + // 3 - default series for counter_metric + for i := series + 3; i < 14; i++ { + qo.fillHeadWithCounter(ctx, i, 1) + for _, shift := range shifts { + b.Run(fmt.Sprintf("opt_none_series_%d_shift_%s_range_0s", i, shift), func(b *testing.B) { + b.ResetTimer() + for b.Loop() { + res, err := queryInstant(ctx, "none", queryEngine, qo, qo.queryOpts, query, qo.start.Add(shift)) + require.NoError(b, err) + res.qry.Close() + } + }) + + b.Run(fmt.Sprintf("opt_all_series_%d_shift_%s_range_0s", i, shift), func(b *testing.B) { + b.ResetTimer() + for b.Loop() { + res, err := queryInstant(ctx, "all", queryEngine, qo, qo.queryOpts, query, qo.start.Add(shift)) + require.NoError(b, err) + res.qry.Close() + } + }) } + } +} + +//revive:disable-next-line:cognitive-complexity // this is a benchmark +func BenchmarkInstantQueryOverTime(b *testing.B) { + ctx := b.Context() + qo := &querierOptimize{} + qo.setup(ctx, b.TempDir(), func(err error, msgAndArgs ...any) { require.NoError(b, err, msgAndArgs) }) + defer qo.close() + + queryEngine := promql.NewEngine(promql.EngineOpts{ + Logger: log.NewNopLogger(), + MaxSamples: 100000, + Timeout: 10 * time.Second, + LookbackDelta: qo.lookbackDelta, + NoStepSubqueryIntervalFn: func(int64) int64 { return qo.lookbackDelta.Milliseconds() }, + EnableAtModifier: true, + EnableNegativeOffset: true, }) + + queryf := "max_over_time(counter_metric[%s])" + shifts := []time.Duration{ + qo.step, + qo.step * 2, + qo.step * 3, + qo.step * 4, + qo.step * 5, + qo.step * 6, + qo.step * 7, + qo.step * 8, + qo.step * 9, + qo.step * 10, + qo.step * 11, + qo.step * 12, + qo.step * 62, + // qo.step * 125, + } + ranges := []time.Duration{ + qo.step / 2, + qo.step, + qo.step * 15 / 10, + qo.step * 2, + qo.step * 25 / 10, + qo.step * 3, + qo.step * 35 / 10, + qo.step * 4, + qo.step * 45 / 10, + qo.step * 5, + qo.step * 55 / 10, + qo.step * 6, + qo.step * 65 / 10, + qo.step * 7, + qo.step * 75 / 10, + qo.step * 8, + qo.step * 85 / 10, + qo.step * 9, + qo.step * 95 / 10, + qo.step * 10, + qo.step * 20, + qo.step * 30, + qo.step * 40, + qo.step * 60, + } + + series := 6 + qo.fillHeadWithCounter(ctx, 0, series) + + // 3 - default series for counter_metric + for i := series + 3; i < 11; i++ { + qo.fillHeadWithCounter(ctx, i, 1) + for _, shift := range shifts { + for _, r := range ranges { + query := fmt.Sprintf(queryf, r) + + b.Run(fmt.Sprintf("opt_none_series_%d_shift_%s_range_%s", i, shift, r), func(b *testing.B) { + b.ResetTimer() + for b.Loop() { + res, err := queryInstant(ctx, "none", queryEngine, qo, qo.queryOpts, query, qo.start.Add(shift)) + require.NoError(b, err) + res.qry.Close() + } + }) + + b.Run(fmt.Sprintf("opt_all_series_%d_shift_%s_range_%s", i, shift, r), func(b *testing.B) { + b.ResetTimer() + for b.Loop() { + res, err := queryInstant(ctx, "all", queryEngine, qo, qo.queryOpts, query, qo.start.Add(shift)) + require.NoError(b, err) + res.qry.Close() + } + }) + } + } + } } diff --git a/pp/go/storage/querier/querier_optimize_variables_test.go b/pp/go/storage/querier/querier_optimize_variables_test.go index 5da789160..3429cd2d6 100644 --- a/pp/go/storage/querier/querier_optimize_variables_test.go +++ b/pp/go/storage/querier/querier_optimize_variables_test.go @@ -16,26 +16,26 @@ const ( // defaultSteps is the default steps. var defaultSteps = []time.Duration{ - defaultStep - time.Second, // [14s] - defaultStep, // [15s] - (defaultStep - time.Second) * 2, // [29s] - defaultStep * 2, // [30s] - (defaultStep - time.Second) * 4, // [59s] - defaultStep * 4, // [60s] - (defaultStep - time.Second) * 5, // [70s] + defaultStep - time.Second, + defaultStep, + (defaultStep - time.Second) * 2, + defaultStep * 2, + (defaultStep - time.Second) * 4, + defaultStep * 4, + (defaultStep - time.Second) * 5, } // defaultSubQueries is the default subqueries. var defaultSubQueries = []subQuery{ - {window: model.Duration(defaultStep), step: 0}, // [15s] - {window: model.Duration(defaultStep * 4), step: 0}, // [60s] - {window: model.Duration(defaultStep*4 - time.Second), step: 0}, // [59s] - {window: model.Duration(defaultStep*4 + time.Second), step: 0}, // [61s] - {window: model.Duration(defaultStep * 4), step: 0, defaultStep: true}, // [60s:] - {window: model.Duration(defaultStep * 16), step: model.Duration(defaultStep * 4)}, // [240s:60s] - {window: model.Duration(defaultStep*16 - time.Second), step: 0}, // [239s] - {window: model.Duration(defaultStep * 16), step: 0}, // [240s] - {window: model.Duration(defaultStep*16 + time.Second), step: 0}, // [241s] + {window: model.Duration(defaultStep), step: 0}, + {window: model.Duration(defaultStep * 4), step: 0}, + {window: model.Duration(defaultStep*4 - time.Second), step: 0}, + {window: model.Duration(defaultStep*4 + time.Second), step: 0}, + {window: model.Duration(defaultStep * 4), step: 0, defaultStep: true}, + {window: model.Duration(defaultStep * 16), step: model.Duration(defaultStep * 4)}, + {window: model.Duration(defaultStep*16 - time.Second), step: 0}, + {window: model.Duration(defaultStep * 16), step: 0}, + {window: model.Duration(defaultStep*16 + time.Second), step: 0}, } // defaultModifiers is the default modifiers. diff --git a/pp/go/storage/querier/series_test.go b/pp/go/storage/querier/series_test.go index 4d29a4fa1..0a56d5ec1 100644 --- a/pp/go/storage/querier/series_test.go +++ b/pp/go/storage/querier/series_test.go @@ -1,6 +1,7 @@ package querier_test import ( + "fmt" "math" "testing" @@ -745,3 +746,105 @@ func (s *SeriesSetTestSuite) TestChangesFunc() { s.Equal(int64(250), actual[0].Samples[3].Timestamp) s.Equal(value.StaleNaN, math.Float64bits(actual[0].Samples[3].Value)) } + +func (s *SeriesSetTestSuite) TestMaxOverTimeFunc2() { + // Arrange + samples := []cppbridge.Sample{} + + timeData := int64(600) + endData := int64(1540) + stepData := int64(30) + val := 1.0 + for ; timeData < endData; timeData += stepData { + samples = append(samples, cppbridge.Sample{Timestamp: timeData, Value: val}) + val++ + } + + storagetest.MustAppendTimeSeriesToLSSAndDataStorage(s.lss, s.ds, []storagetest.TimeSeries{ + { + Labels: labels.FromStrings("__name__", "metric", "job", "test"), + Samples: samples, + }, + }...) + + // + + step := int64(70) + for i := 15; i <= 600; i += 1 { + hints := &storage.SelectHints{ + Start: 900 - int64(i) + 1, + End: 1500, + Func: "max_over_time", + Step: step, + Range: int64(i), + LookbackDelta: 120, + } + + // Act + seriesSet := s.queryAggr(s.lss, s.ds, 0, hints.End, cppbridge.NoDownsampling, hints, model.LabelMatcher{ + Name: "__name__", + Value: "metric", + MatcherType: model.MatcherTypeExactMatch, + }) + + ts := storagetest.TimeSeriesFromSeriesSet(seriesSet, true) + calc(hints, stepData, int64(len(ts[0].Samples))) + // fmt.Println( + // "step", step, + // "range", i, + // "ts", len(ts[0].Samples), + // "calc", calc(hints, stepData, int64(len(ts[0].Samples))), + // // "samples", ts[0].Samples, + + // "\n============", + // ) + } +} + +func calc(hints *storage.SelectHints, interval, samples int64) bool { + // maxPoints := (hints.End-hints.Start)/interval + 1 + stepPoints := (hints.End-hints.Start)/hints.Step + 1 + // rangePoints := (hints.End-hints.Start)/hints.Range + 1 + // fmt.Println( + // " === maxPoints", maxPoints, + // "stepPoints", stepPoints, + // "rangePoints", rangePoints, + // "deltaTS", (hints.End - hints.Start), + // "hints.End", hints.End, + // "hints.Start", hints.Start, + // "samples", samples, + // ) + if hints.Step >= hints.Range { + // fmt.Println(" === 1 ", (hints.End-hints.Start)/hints.Step, maxPoints) + return true + } + + k := hints.Range % hints.Step + fmt.Println(" === k", k, calc2(float64(samples-stepPoints), float64(stepPoints))) + // fmt.Println(" === k1", hints.Range%interval, (hints.End-hints.Start)/(hints.Step-k)+1) + // fmt.Println(" === expected", (hints.End-hints.Start)/(samples-1)) + if k == 0 || k > interval { + return true + } + + return false +} + +func TestXxx(t *testing.T) { + t.Log(75 % 70) + t.Log(floorDiv(300, 70)) + // srape interval + // step/scrape_interval (*2 if range > step and range % step > 0) +} + +func floorDiv(a, b int64) int64 { + quotient := a / b + if a%b < 0 { + quotient-- + } + return quotient +} + +func calc2(a, b float64) float64 { + return math.Round((a/b)*10) / 10 +} From 14e5317fcc8ae4ac8a08c2e7dff10b4d475e71e5 Mon Sep 17 00:00:00 2001 From: Alexandr Yudin <57181751+u-veles-a@users.noreply.github.com> Date: Tue, 23 Jun 2026 06:55:49 +0000 Subject: [PATCH 31/37] WIP: added a selector for optimizations Signed-off-by: Alexandr Yudin <57181751+u-veles-a@users.noreply.github.com> --- pp-pkg/storage/adapter.go | 25 +- pp-pkg/storage/batch_storage.go | 8 + pp/go/cppbridge/data_storage.go | 33 ++- pp/go/storage/head/shard/data_storage.go | 9 + pp/go/storage/head/shard/shard.go | 5 + pp/go/storage/querier/querier.go | 83 +++--- .../querier/querier_optimize_quick_test.go | 21 +- .../storage/querier/querier_optimize_test.go | 250 +++++++++++++----- pp/go/storage/querier/querier_test.go | 22 +- pp/go/storage/querier/series_test.go | 103 -------- 10 files changed, 345 insertions(+), 214 deletions(-) diff --git a/pp-pkg/storage/adapter.go b/pp-pkg/storage/adapter.go index cd0d471ca..874500f2a 100644 --- a/pp-pkg/storage/adapter.go +++ b/pp-pkg/storage/adapter.go @@ -21,6 +21,9 @@ import ( "github.com/prometheus/prometheus/storage" ) +// defaultCacheCheckIntervalMs is the default cache check interval in milliseconds. +const defaultCacheCheckIntervalMs = int64(5*time.Minute) / 1e6 // ns to ms + // // Adapter // @@ -256,7 +259,7 @@ func (ar *Adapter) ApplyConfig(cfg *config.Config) error { MaxLabelValueLength: uint32(cfg.GlobalConfig.LabelValueLengthLimit), // #nosec G115 // no overflow } ar.hashdexLimits.Store(limits) - ar.scrapeInterval.Store(int64(cfg.GlobalConfig.ScrapeInterval)) + ar.scrapeInterval.Store(int64(cfg.GlobalConfig.ScrapeInterval / 1e6)) // ns to ms return nil } @@ -269,12 +272,15 @@ func (ar *Adapter) Close() error { // HeadQuerier returns [storage.Querier] from active head. func (ar *Adapter) HeadQuerier(mint, maxt int64) (storage.Querier, error) { + ahead := ar.proxy.Get() + aTimeInterval := headTimeIntervalWithValidateCache(ahead, defaultCacheCheckIntervalMs) return querier.NewQuerier( - ar.proxy.Get(), + ahead, querier.NewNoOpShardedDeduplicator, mint, maxt, ar.scrapeInterval.Load(), + aTimeInterval.MinT, ar.activeQuerierMetrics, ), nil } @@ -299,6 +305,7 @@ func (ar *Adapter) MergeOutOfOrderChunks() { func (ar *Adapter) Querier(mint, maxt int64) (storage.Querier, error) { queriers := make([]storage.Querier, 0, 1) //revive:disable-line:add-constant // the best way ahead := ar.proxy.Get() + aTimeInterval := headTimeIntervalWithValidateCache(ahead, defaultCacheCheckIntervalMs) queriers = append( queriers, querier.NewQuerier( @@ -307,6 +314,7 @@ func (ar *Adapter) Querier(mint, maxt int64) (storage.Querier, error) { mint, maxt, ar.scrapeInterval.Load(), + aTimeInterval.MinT, ar.activeQuerierMetrics, ), ) @@ -329,6 +337,7 @@ func (ar *Adapter) Querier(mint, maxt int64) (storage.Querier, error) { mint, maxt, ar.scrapeInterval.Load(), + aTimeInterval.MinT, ar.storageQuerierMetrics, ), ) @@ -354,3 +363,15 @@ func headTimeInterval(head *pp_storage.Head) cppbridge.TimeInterval { return timeInterval } + +// headTimeIntervalWithValidateCache returns [cppbridge.TimeInterval] from [pp_storage.Head] with validate cache. +func headTimeIntervalWithValidateCache(head *pp_storage.Head, cacheCheckIntervalMs int64) cppbridge.TimeInterval { + timeInterval := cppbridge.NewInvalidTimeInterval() + for _, shard := range head.Shards() { + interval := shard.TimeIntervalWithValidateCache(cacheCheckIntervalMs) + timeInterval.MinT = min(interval.MinT, timeInterval.MinT) + timeInterval.MaxT = max(interval.MaxT, timeInterval.MaxT) + } + + return timeInterval +} diff --git a/pp-pkg/storage/batch_storage.go b/pp-pkg/storage/batch_storage.go index 4099c653e..d5a06df7e 100644 --- a/pp-pkg/storage/batch_storage.go +++ b/pp-pkg/storage/batch_storage.go @@ -95,12 +95,20 @@ func (bs *BatchStorage) CommitWithState(ctx context.Context, state *cppbridge.St // Querier calls f() with the given parameters. Returns a [querier.Querier]. func (bs *BatchStorage) Querier(mint, maxt int64) (storage.Querier, error) { + aTimeInterval := cppbridge.NewInvalidTimeInterval() + for _, shard := range bs.transactionHead.Shards() { + interval := shard.TimeIntervalWithValidateCache(defaultCacheCheckIntervalMs) + aTimeInterval.MinT = min(interval.MinT, aTimeInterval.MinT) + aTimeInterval.MaxT = max(interval.MaxT, aTimeInterval.MaxT) + } + return querier.NewQuerier( bs.transactionHead, querier.NewNoOpShardedDeduplicator, mint, maxt, bs.adapter.scrapeInterval.Load(), + aTimeInterval.MinT, nil, ), nil } diff --git a/pp/go/cppbridge/data_storage.go b/pp/go/cppbridge/data_storage.go index 0193c7792..c85d80d8c 100644 --- a/pp/go/cppbridge/data_storage.go +++ b/pp/go/cppbridge/data_storage.go @@ -2,23 +2,25 @@ package cppbridge import ( "runtime" + "sync" "sync/atomic" + "time" "unsafe" ) // DataStorage is Go wrapper around series_data::Data_storage. type DataStorage struct { - dataStorage uintptr - gcDestroyDetector *uint64 - timeInterval atomic.Pointer[TimeInterval] + dataStorage uintptr + timeInterval atomic.Pointer[TimeInterval] + lastUpdateTime atomic.Int64 + m sync.Mutex } // NewDataStorage - constructor. func NewDataStorage() *DataStorage { ds := &DataStorage{ - dataStorage: seriesDataDataStorageCtor(), - gcDestroyDetector: &gcDestroyDetector, - timeInterval: atomic.Pointer[TimeInterval]{}, + dataStorage: seriesDataDataStorageCtor(), + timeInterval: atomic.Pointer[TimeInterval]{}, } ds.timeInterval.Store(newInvalidTimeIntervalPtr()) @@ -45,6 +47,25 @@ func (ds *DataStorage) TimeInterval(invalidateCache bool) TimeInterval { return *ds.timeInterval.Load() } +// TimeIntervalWithValidateCache gets time interval from [DataStorage] with validate cache. +func (ds *DataStorage) TimeIntervalWithValidateCache(cacheCheckIntervalMs int64) TimeInterval { + now := time.Now().UnixMilli() + if now-ds.lastUpdateTime.Load() > cacheCheckIntervalMs { + // slow path + ds.m.Lock() + if now-ds.lastUpdateTime.Load() > cacheCheckIntervalMs { + timeInterval := seriesDataDataStorageTimeInterval(ds.dataStorage) + ds.timeInterval.Store(&timeInterval) + ds.lastUpdateTime.Store(now) + } + ds.m.Unlock() + + runtime.KeepAlive(ds) + } + + return *ds.timeInterval.Load() +} + func (ds *DataStorage) GetQueriedSeriesBitset() []byte { size := seriesDataDataStorageQueriedSeriesBitsetSize(ds.dataStorage) bitset := seriesDataDataStorageQueriedSeriesBitset(ds.dataStorage, make([]byte, 0, size)) diff --git a/pp/go/storage/head/shard/data_storage.go b/pp/go/storage/head/shard/data_storage.go index bf1bf706d..baadf6df0 100644 --- a/pp/go/storage/head/shard/data_storage.go +++ b/pp/go/storage/head/shard/data_storage.go @@ -135,6 +135,15 @@ func (ds *DataStorage) TimeInterval(invalidateCache bool) cppbridge.TimeInterval return result } +// TimeIntervalWithValidateCache gets time interval from [DataStorage] with validate cache. +func (ds *DataStorage) TimeIntervalWithValidateCache(cacheCheckIntervalMs int64) cppbridge.TimeInterval { + ds.locker.RLock() + result := ds.dataStorage.TimeIntervalWithValidateCache(cacheCheckIntervalMs) + ds.locker.RUnlock() + + return result +} + // CreateUnusedSeriesDataUnloader create unused series data unloader func (ds *DataStorage) CreateUnusedSeriesDataUnloader() *cppbridge.UnusedSeriesDataUnloader { return ds.dataStorage.CreateUnusedSeriesDataUnloader() diff --git a/pp/go/storage/head/shard/shard.go b/pp/go/storage/head/shard/shard.go index ac8cdf71e..8f03eedcf 100644 --- a/pp/go/storage/head/shard/shard.go +++ b/pp/go/storage/head/shard/shard.go @@ -186,6 +186,11 @@ func (s *Shard) TimeInterval(invalidateCache bool) cppbridge.TimeInterval { return s.dataStorage.TimeInterval(invalidateCache) } +// TimeIntervalWithValidateCache gets time interval from [DataStorage] with validate cache. +func (s *Shard) TimeIntervalWithValidateCache(cacheCheckIntervalMs int64) cppbridge.TimeInterval { + return s.dataStorage.TimeIntervalWithValidateCache(cacheCheckIntervalMs) +} + // UnloadedDataStorage get unloaded data storage func (s *Shard) UnloadedDataStorage() *UnloadedDataStorage { return s.unloadedDataStorage diff --git a/pp/go/storage/querier/querier.go b/pp/go/storage/querier/querier.go index 705ccb888..c16900c1f 100644 --- a/pp/go/storage/querier/querier.go +++ b/pp/go/storage/querier/querier.go @@ -77,6 +77,9 @@ const ( // DefaultCountOfSeriesToOptimize is the default count of series to optimize. const DefaultCountOfSeriesToOptimize = 11 +// optimizeAggregationRatio is the ratio to optimize the aggregation function. +const optimizeAggregationRatio = 3.3 + // defaultOptimizeType is the default option for selecting functions optimization. var defaultOptimizeType = noneOptimizeType @@ -125,6 +128,13 @@ var emptySelectHints = &storage.SelectHints{} // emptySeriesSet is an empty series set. var emptySeriesSet = &SeriesSet{} +// IsPossibleToOptimize is the function to check if the query possible to optimization. +var IsPossibleToOptimize func( + lssQueryResults []*cppbridge.LSSQueryResult, + hints *storage.SelectHints, + scrapeIntervalMS, shiftMS int64, +) func() bool = DefaultIsPossibleToOptimize + // // Querier // @@ -139,7 +149,8 @@ type Querier[ ] struct { mint int64 maxt int64 - scrapeInterval int64 + scrapeIntervalMS int64 + headMinTSMS int64 head THead deduplicatorCtor deduplicatorCtor metrics *Metrics @@ -156,7 +167,7 @@ func NewQuerier[ ]( head THead, deduplicatorCtor deduplicatorCtor, - mint, maxt, scrapeInterval int64, + mint, maxt, scrapeIntervalMS, headMinTSMS int64, metrics *Metrics, ) *Querier[TTask, TDataStorage, TLSS, TShard, THead] { return newQuerierWithSelectFuncOptimize( @@ -164,7 +175,8 @@ func NewQuerier[ deduplicatorCtor, mint, maxt, - scrapeInterval, + scrapeIntervalMS, + headMinTSMS, metrics, selectFuncOptimize, ) @@ -180,7 +192,7 @@ func NewQuerierWithOutSelectFuncOptimize[ ]( head THead, deduplicatorCtor deduplicatorCtor, - mint, maxt, scrapeInterval int64, + mint, maxt, scrapeIntervalMS, headMinTSMS int64, metrics *Metrics, ) *Querier[TTask, TDataStorage, TLSS, TShard, THead] { return newQuerierWithSelectFuncOptimize( @@ -188,7 +200,8 @@ func NewQuerierWithOutSelectFuncOptimize[ deduplicatorCtor, mint, maxt, - scrapeInterval, + scrapeIntervalMS, + headMinTSMS, metrics, selectFuncOptimize&dropPointOptimizeType, ) @@ -204,14 +217,15 @@ func newQuerierWithSelectFuncOptimize[ ]( head THead, deduplicatorCtor deduplicatorCtor, - mint, maxt, scrapeInterval int64, + mint, maxt, scrapeIntervalMS, headMinTSMS int64, metrics *Metrics, queryOptimize queryOptimizeType, ) *Querier[TTask, TDataStorage, TLSS, TShard, THead] { return &Querier[TTask, TDataStorage, TLSS, TShard, THead]{ mint: mint, maxt: maxt, - scrapeInterval: scrapeInterval, + scrapeIntervalMS: scrapeIntervalMS, + headMinTSMS: headMinTSMS, head: head, deduplicatorCtor: deduplicatorCtor, metrics: metrics, @@ -416,7 +430,11 @@ func (q *Querier[TTask, TDataStorage, TLSS, TShard, THead]) selectRange( return storage.ErrSeriesSet(err) } - hints = SwitchFuncOptimize(hints, isPossibleToOptimize(lssQueryResults, hints, q.scrapeInterval), q.queryOptimize) + hints = SwitchFuncOptimize( + hints, + IsPossibleToOptimize(lssQueryResults, hints, q.scrapeIntervalMS, q.headMinTSMS), + q.queryOptimize, + ) shardedSerializedData := poolProvider.GetSerializedData() defer poolProvider.PutSerializedData(shardedSerializedData) queryDataStorage(dsQueryRangeQuerier, q.head, lssQueryResults, shardedSerializedData, q.mint, q.maxt, hints) @@ -606,11 +624,11 @@ func SwitchFuncOptimize( return emptySelectHints } -// isPossibleToOptimize checks if the query possible to optimization. -func isPossibleToOptimize( +// DefaultIsPossibleToOptimize checks if the query possible to optimization. +func DefaultIsPossibleToOptimize( lssQueryResults []*cppbridge.LSSQueryResult, hints *storage.SelectHints, - scrapeInterval int64, + scrapeIntervalMS, headMinTSMS int64, ) func() bool { return func() bool { countOfSeries := 0 @@ -623,17 +641,21 @@ func isPossibleToOptimize( } if isCrossSeriesFunc(hints) { - return isPossibleToOptimizeCrossSeriesFunc(hints, scrapeInterval, countOfSeries) + return isPossibleToOptimizeCrossSeriesFunc(hints, scrapeIntervalMS, countOfSeries) } - return true + if isAggregationSeriesFunc(hints) { + return isPossibleToOptimizeAggregationFunc(hints, scrapeIntervalMS, headMinTSMS) + } + + return false } } // isPossibleToOptimizeCrossSeriesFunc checks if the cross series function is possible to optimize. func isPossibleToOptimizeCrossSeriesFunc( hints *storage.SelectHints, - scrapeInterval int64, + scrapeIntervalMS int64, countOfSeries int, ) bool { // for cross series functions for instant query, we don't need to optimize the query @@ -649,51 +671,52 @@ func isPossibleToOptimizeCrossSeriesFunc( return true } - hintStep := hints.Step * 1e6 //revive:disable-line:add-constant // ms to ns - // grouping by - if hintStep == scrapeInterval && countOfSeries > DefaultCountOfSeriesToOptimize+1 { + if hints.Step == scrapeIntervalMS && countOfSeries > DefaultCountOfSeriesToOptimize+1 { return true } //revive:disable-next-line:add-constant // x2 scrape interval are required to enable optimization - if hintStep >= scrapeInterval*2 && countOfSeries >= DefaultCountOfSeriesToOptimize { + if hints.Step >= scrapeIntervalMS*2 && countOfSeries >= DefaultCountOfSeriesToOptimize { return true } //revive:disable-next-line:add-constant // x3 scrape interval are required to enable optimization - if hintStep >= scrapeInterval*3 && countOfSeries > DefaultCountOfSeriesToOptimize { + if hints.Step >= scrapeIntervalMS*3 && countOfSeries > DefaultCountOfSeriesToOptimize { return true } return false } +// isPossibleToOptimizeAggregationFunc checks if the aggregation function is possible to optimize. func isPossibleToOptimizeAggregationFunc( hints *storage.SelectHints, - scrapeInterval, shift int64, + scrapeIntervalMS, headMinTSMS int64, ) bool { - hintStep := hints.Step * 1e6 //revive:disable-line:add-constant // ms to ns - if hintStep <= scrapeInterval { + if hints.Step <= scrapeIntervalMS { return false } - // instant query, shift is x5 scrape interval and range is x5 scrape interval if hints.Step == 0 { - return hintStep >= scrapeInterval*5 && hints.Range >= scrapeInterval*5 + // instant query, shift is x5 scrape interval and range is x5 scrape interval + //revive:disable-next-line:add-constant // x5 scrape interval + return hints.Start-headMinTSMS >= scrapeIntervalMS*5 && hints.Range >= scrapeIntervalMS*5 } - hintRange := hints.Range * 1e6 //revive:disable-line:add-constant // ms to ns - if hintRange < scrapeInterval { + if hints.Range < scrapeIntervalMS { return false } - //revive:disable-next-line:add-constant // check if the range is even - if hintRange/scrapeInterval%2 != 0 { - return false + if hints.Step > hints.Range { + return true } - return true + if float64(hints.Step)/float64(scrapeIntervalMS) >= optimizeAggregationRatio { + return true + } + + return false } // isNotWithpout checks if the hints is not without by. diff --git a/pp/go/storage/querier/querier_optimize_quick_test.go b/pp/go/storage/querier/querier_optimize_quick_test.go index 1b4ef4abf..2ce51351f 100644 --- a/pp/go/storage/querier/querier_optimize_quick_test.go +++ b/pp/go/storage/querier/querier_optimize_quick_test.go @@ -12,7 +12,10 @@ import ( "github.com/go-kit/log" "github.com/prometheus/common/model" + "github.com/prometheus/prometheus/pp/go/cppbridge" + "github.com/prometheus/prometheus/pp/go/storage/querier" "github.com/prometheus/prometheus/promql" + prom_storage "github.com/prometheus/prometheus/storage" "github.com/stretchr/testify/suite" ) @@ -231,7 +234,23 @@ func TestQuickQuerierOptimizeSuite(t *testing.T) { } func (s *QuickQuerierOptimizeSuite) SetupSuite() { - s.querierOptimize.setup(s.T().Context(), s.T().TempDir(), s.Require().NoError) + s.querierOptimize.setup( + s.T().Context(), + s.T().TempDir(), + s.Require().NoError, + time.UnixMilli(defaultStartMs), + defaultStep, + defaultCountOfSteps, + ) + querier.IsPossibleToOptimize = func( + []*cppbridge.LSSQueryResult, + *prom_storage.SelectHints, + int64, int64, + ) func() bool { + return func() bool { + return true + } + } s.quickQE = promql.NewEngine(promql.EngineOpts{ Logger: log.NewNopLogger(), diff --git a/pp/go/storage/querier/querier_optimize_test.go b/pp/go/storage/querier/querier_optimize_test.go index 4a58abc87..aea91d29d 100644 --- a/pp/go/storage/querier/querier_optimize_test.go +++ b/pp/go/storage/querier/querier_optimize_test.go @@ -468,11 +468,18 @@ type querierOptimize struct { } // setup sets up the querier optimizer. -func (s *querierOptimize) setup(ctx context.Context, baseDir string, noErrorFunc storagetest.NoErrorFunc) { +func (s *querierOptimize) setup( + ctx context.Context, + baseDir string, + noErrorFunc storagetest.NoErrorFunc, + start time.Time, + step time.Duration, + countOfSteps int, +) { s.noErrorFunc = noErrorFunc - s.start = time.UnixMilli(defaultStartMs) - s.step = defaultStep - s.end = s.start.Add(s.step * defaultCountOfSteps) // 480 steps + s.start = start + s.step = step + s.end = s.start.Add(s.step * time.Duration(countOfSteps)) s.dataDir = filepath.Join(baseDir, "data") s.noErrorFunc(os.MkdirAll(s.dataDir, os.ModeDir)) @@ -678,7 +685,15 @@ func (s *querierOptimize) fillHeadWithCounter(ctx context.Context, start, counte // Querier implements the [prom_storage.Queryable] interface. func (s *querierOptimize) Querier(mint, maxt int64) (prom_storage.Querier, error) { - return querier.NewQuerier(s.head, querier.NewNoOpShardedDeduplicator, mint, maxt, int64(s.step), nil), nil + return querier.NewQuerier( + s.head, + querier.NewNoOpShardedDeduplicator, + mint, + maxt, + s.step.Milliseconds(), + s.start.UnixMilli(), + nil, + ), nil } // @@ -701,7 +716,23 @@ func TestMatrixQuerierOptimizeSuiteSuite(t *testing.T) { } func (s *MatrixQuerierOptimizeSuiteSuite) SetupSuite() { - s.querierOptimize.setup(s.T().Context(), s.T().TempDir(), s.Require().NoError) + s.querierOptimize.setup( + s.T().Context(), + s.T().TempDir(), + s.Require().NoError, + time.UnixMilli(defaultStartMs), + defaultStep, + defaultCountOfSteps, + ) + querier.IsPossibleToOptimize = func( + []*cppbridge.LSSQueryResult, + *prom_storage.SelectHints, + int64, int64, + ) func() bool { + return func() bool { + return true + } + } s.queryEngine = promql.NewEngine(promql.EngineOpts{ Logger: log.NewNopLogger(), @@ -889,28 +920,6 @@ func (s *MatrixQuerierOptimizeSuiteSuite) TestQueryRangeSingle() { } } -func (s *MatrixQuerierOptimizeSuiteSuite) TestQueryRangeSingle2() { - ctx := s.T().Context() - query := "max_over_time(counter_metric[75s])" - start := s.start - step := 60 * time.Second - - s.Run(query, func() { - res, err := queryRange(ctx, "none", s.queryEngine, s, s.queryOpts, query, start, s.end, step) - s.Require().NoError(err) - defer res.qry.Close() - - resOpt, err := queryRange(ctx, "all", s.queryEngine, s, s.queryOpts, query, start, s.end, step) - s.Require().NoError(err) - defer resOpt.qry.Close() - - fmt.Println("res", res.res) - fmt.Println("resOpt", resOpt.res) - - s.Require().True(resultEqual(res.res, resOpt.res, query)) - }) -} - // // resultEqual // @@ -1136,8 +1145,24 @@ func calcRelative(expected, actual float64) float64 { func BenchmarkRangeQuerySum(b *testing.B) { ctx := b.Context() qo := &querierOptimize{} - qo.setup(ctx, b.TempDir(), func(err error, msgAndArgs ...any) { require.NoError(b, err, msgAndArgs) }) + qo.setup( + ctx, + b.TempDir(), + func(err error, msgAndArgs ...any) { require.NoError(b, err, msgAndArgs) }, + time.UnixMilli(defaultStartMs), + defaultStep, + defaultCountOfSteps, + ) defer qo.close() + querier.IsPossibleToOptimize = func( + []*cppbridge.LSSQueryResult, + *prom_storage.SelectHints, + int64, int64, + ) func() bool { + return func() bool { + return true + } + } queryEngine := promql.NewEngine(promql.EngineOpts{ Logger: log.NewNopLogger(), @@ -1187,8 +1212,24 @@ func BenchmarkRangeQuerySum(b *testing.B) { func BenchmarkRangeQuerySumBy(b *testing.B) { ctx := b.Context() qo := &querierOptimize{} - qo.setup(ctx, b.TempDir(), func(err error, msgAndArgs ...any) { require.NoError(b, err, msgAndArgs) }) + qo.setup( + ctx, + b.TempDir(), + func(err error, msgAndArgs ...any) { require.NoError(b, err, msgAndArgs) }, + time.UnixMilli(defaultStartMs), + defaultStep, + defaultCountOfSteps, + ) defer qo.close() + querier.IsPossibleToOptimize = func( + []*cppbridge.LSSQueryResult, + *prom_storage.SelectHints, + int64, int64, + ) func() bool { + return func() bool { + return true + } + } queryEngine := promql.NewEngine(promql.EngineOpts{ Logger: log.NewNopLogger(), @@ -1239,8 +1280,24 @@ func BenchmarkRangeQuerySumBy(b *testing.B) { func BenchmarkRangeQueryOverTime(b *testing.B) { ctx := b.Context() qo := &querierOptimize{} - qo.setup(ctx, b.TempDir(), func(err error, msgAndArgs ...any) { require.NoError(b, err, msgAndArgs) }) + qo.setup( + ctx, + b.TempDir(), + func(err error, msgAndArgs ...any) { require.NoError(b, err, msgAndArgs) }, + time.UnixMilli(defaultStartMs), + defaultStep, + defaultCountOfSteps, + ) defer qo.close() + // querier.IsPossibleToOptimize = func( + // []*cppbridge.LSSQueryResult, + // *prom_storage.SelectHints, + // int64, int64, + // ) func() bool { + // return func() bool { + // return true + // } + // } queryEngine := promql.NewEngine(promql.EngineOpts{ Logger: log.NewNopLogger(), @@ -1254,47 +1311,70 @@ func BenchmarkRangeQueryOverTime(b *testing.B) { queryf := "max_over_time(counter_metric[%s])" steps := []time.Duration{ - qo.step, - qo.step * 2, qo.step * 25 / 10, - qo.step * 3, + qo.step * 26 / 10, + qo.step * 27 / 10, + qo.step * 28 / 10, + qo.step * 29 / 10, + qo.step * 30 / 10, + qo.step * 31 / 10, + qo.step * 32 / 10, + qo.step * 33 / 10, + qo.step * 34 / 10, qo.step * 35 / 10, - qo.step * 4, - qo.step * 9, - qo.step * 10, } ranges := []time.Duration{ - qo.step / 2, - qo.step, - qo.step * 15 / 10, - qo.step * 2, - qo.step * 25 / 10, - qo.step * 3, + qo.step * 27 / 10, + qo.step * 28 / 10, + qo.step * 29 / 10, + qo.step * 30 / 10, + qo.step * 31 / 10, + qo.step * 32 / 10, + qo.step * 33 / 10, + qo.step * 34 / 10, qo.step * 35 / 10, - qo.step * 4, + qo.step * 36 / 10, + qo.step * 37 / 10, + qo.step * 38 / 10, + qo.step * 39 / 10, + qo.step * 40 / 10, + qo.step * 41 / 10, + qo.step * 42 / 10, + qo.step * 43 / 10, + qo.step * 44 / 10, qo.step * 45 / 10, - qo.step * 5, + qo.step * 46 / 10, + qo.step * 47 / 10, + qo.step * 48 / 10, + qo.step * 49 / 10, + qo.step * 50 / 10, + qo.step * 51 / 10, + qo.step * 52 / 10, + qo.step * 53 / 10, + qo.step * 54 / 10, qo.step * 55 / 10, - qo.step * 6, + qo.step * 56 / 10, + qo.step * 57 / 10, + qo.step * 58 / 10, + qo.step * 59 / 10, + qo.step * 60 / 10, + qo.step * 61 / 10, + qo.step * 62 / 10, + qo.step * 63 / 10, + qo.step * 64 / 10, qo.step * 65 / 10, - qo.step * 7, - qo.step * 75 / 10, - qo.step * 8, - qo.step * 85 / 10, - qo.step * 9, - qo.step * 95 / 10, - qo.step * 10, - qo.step * 20, - qo.step * 30, - qo.step * 40, - qo.step * 60, + qo.step * 66 / 10, + qo.step * 67 / 10, + qo.step * 68 / 10, + qo.step * 69 / 10, + qo.step * 70 / 10, } series := 6 qo.fillHeadWithCounter(ctx, 0, series) // 3 - default series for counter_metric - for i := series + 3; i < 12; i++ { + for i := series + 3; i < 10; i++ { qo.fillHeadWithCounter(ctx, i, 1) for _, step := range steps { for _, r := range ranges { @@ -1324,8 +1404,24 @@ func BenchmarkRangeQueryOverTime(b *testing.B) { func BenchmarkInstantQuerySum(b *testing.B) { ctx := b.Context() qo := &querierOptimize{} - qo.setup(ctx, b.TempDir(), func(err error, msgAndArgs ...any) { require.NoError(b, err, msgAndArgs) }) + qo.setup( + ctx, + b.TempDir(), + func(err error, msgAndArgs ...any) { require.NoError(b, err, msgAndArgs) }, + time.UnixMilli(defaultStartMs), + defaultStep, + defaultCountOfSteps, + ) defer qo.close() + querier.IsPossibleToOptimize = func( + []*cppbridge.LSSQueryResult, + *prom_storage.SelectHints, + int64, int64, + ) func() bool { + return func() bool { + return true + } + } queryEngine := promql.NewEngine(promql.EngineOpts{ Logger: log.NewNopLogger(), @@ -1375,8 +1471,24 @@ func BenchmarkInstantQuerySum(b *testing.B) { func BenchmarkInstantQuerySumBy(b *testing.B) { ctx := b.Context() qo := &querierOptimize{} - qo.setup(ctx, b.TempDir(), func(err error, msgAndArgs ...any) { require.NoError(b, err, msgAndArgs) }) + qo.setup( + ctx, + b.TempDir(), + func(err error, msgAndArgs ...any) { require.NoError(b, err, msgAndArgs) }, + time.UnixMilli(defaultStartMs), + defaultStep, + defaultCountOfSteps, + ) defer qo.close() + querier.IsPossibleToOptimize = func( + []*cppbridge.LSSQueryResult, + *prom_storage.SelectHints, + int64, int64, + ) func() bool { + return func() bool { + return true + } + } queryEngine := promql.NewEngine(promql.EngineOpts{ Logger: log.NewNopLogger(), @@ -1428,8 +1540,24 @@ func BenchmarkInstantQuerySumBy(b *testing.B) { func BenchmarkInstantQueryOverTime(b *testing.B) { ctx := b.Context() qo := &querierOptimize{} - qo.setup(ctx, b.TempDir(), func(err error, msgAndArgs ...any) { require.NoError(b, err, msgAndArgs) }) + qo.setup( + ctx, + b.TempDir(), + func(err error, msgAndArgs ...any) { require.NoError(b, err, msgAndArgs) }, + time.UnixMilli(defaultStartMs), + defaultStep, + defaultCountOfSteps, + ) defer qo.close() + querier.IsPossibleToOptimize = func( + []*cppbridge.LSSQueryResult, + *prom_storage.SelectHints, + int64, int64, + ) func() bool { + return func() bool { + return true + } + } queryEngine := promql.NewEngine(promql.EngineOpts{ Logger: log.NewNopLogger(), diff --git a/pp/go/storage/querier/querier_test.go b/pp/go/storage/querier/querier_test.go index 50801dc0a..a29615c88 100644 --- a/pp/go/storage/querier/querier_test.go +++ b/pp/go/storage/querier/querier_test.go @@ -124,7 +124,7 @@ func (s *QuerierSuite) TestRangeQuery() { *shard.LSS, *shard.PerGoroutineShard, *storage.Head, - ](s.head, querier.NewNoOpShardedDeduplicator, 0, 2, s.scrapeInterval, nil) + ](s.head, querier.NewNoOpShardedDeduplicator, 0, 2, s.scrapeInterval, 0, nil) defer func() { _ = q.Close() }() matcher, _ := labels.NewMatcher(labels.MatchEqual, "__name__", "metric") @@ -152,7 +152,7 @@ func (s *QuerierSuite) TestRangeQueryWithoutMatching() { *shard.LSS, *shard.PerGoroutineShard, *storage.Head, - ](s.head, querier.NewNoOpShardedDeduplicator, 0, 2, s.scrapeInterval, nil) + ](s.head, querier.NewNoOpShardedDeduplicator, 0, 2, s.scrapeInterval, 0, nil) defer func() { _ = q.Close() }() matcher, _ := labels.NewMatcher(labels.MatchEqual, "__name__", "unknown_metric") @@ -205,7 +205,7 @@ func (s *QuerierSuite) TestRangeQueryWithDataStorageLoading() { *shard.LSS, *shard.PerGoroutineShard, *storage.Head, - ](s.head, querier.NewNoOpShardedDeduplicator, 0, 3, s.scrapeInterval, nil) + ](s.head, querier.NewNoOpShardedDeduplicator, 0, 3, s.scrapeInterval, 0, nil) defer func() { _ = q.Close() }() matcher, _ := labels.NewMatcher(labels.MatchEqual, "__name__", "metric") @@ -243,7 +243,7 @@ func (s *QuerierSuite) TestInstantQuery() { *shard.LSS, *shard.PerGoroutineShard, *storage.Head, - ](s.head, querier.NewNoOpShardedDeduplicator, 0, 0, s.scrapeInterval, nil) + ](s.head, querier.NewNoOpShardedDeduplicator, 0, 0, s.scrapeInterval, 0, nil) defer func() { _ = q.Close() }() matcher, _ := labels.NewMatcher(labels.MatchEqual, "__name__", "metric") @@ -296,7 +296,7 @@ func (s *QuerierSuite) TestInstantQueryWithDataStorageLoading() { *shard.LSS, *shard.PerGoroutineShard, *storage.Head, - ](s.head, querier.NewNoOpShardedDeduplicator, 0, 0, s.scrapeInterval, nil) + ](s.head, querier.NewNoOpShardedDeduplicator, 0, 0, s.scrapeInterval, 0, nil) defer func() { _ = q.Close() }() matcher, _ := labels.NewMatcher(labels.MatchEqual, "__name__", "metric") @@ -340,7 +340,7 @@ func (s *QuerierSuite) TestLabelNames() { } s.appendTimeSeries(timeSeries) - q := querier.NewQuerier(s.head, querier.NewNoOpShardedDeduplicator, 0, 2, s.scrapeInterval, nil) + q := querier.NewQuerier(s.head, querier.NewNoOpShardedDeduplicator, 0, 2, s.scrapeInterval, 0, nil) defer func() { _ = q.Close() }() matcher, err := labels.NewMatcher(labels.MatchEqual, "__name__", "metric0") s.Require().NoError(err) @@ -373,7 +373,7 @@ func (s *QuerierSuite) TestLabelNamesWithLimit() { } s.appendTimeSeries(timeSeries) - q := querier.NewQuerier(s.head, querier.NewNoOpShardedDeduplicator, 0, 2, s.scrapeInterval, nil) + q := querier.NewQuerier(s.head, querier.NewNoOpShardedDeduplicator, 0, 2, s.scrapeInterval, 0, nil) defer func() { _ = q.Close() }() matcher, err := labels.NewMatcher(labels.MatchEqual, "__name__", "metric0") s.Require().NoError(err) @@ -406,7 +406,7 @@ func (s *QuerierSuite) TestLabelNamesNoMatches() { } s.appendTimeSeries(timeSeries) - q := querier.NewQuerier(s.head, querier.NewNoOpShardedDeduplicator, 0, 2, s.scrapeInterval, nil) + q := querier.NewQuerier(s.head, querier.NewNoOpShardedDeduplicator, 0, 2, s.scrapeInterval, 0, nil) defer func() { _ = q.Close() }() matcher, err := labels.NewMatcher(labels.MatchEqual, "__name__", "metric3") s.Require().NoError(err) @@ -439,7 +439,7 @@ func (s *QuerierSuite) TestLabelValues() { } s.appendTimeSeries(timeSeries) - q := querier.NewQuerier(s.head, querier.NewNoOpShardedDeduplicator, 0, 2, s.scrapeInterval, nil) + q := querier.NewQuerier(s.head, querier.NewNoOpShardedDeduplicator, 0, 2, s.scrapeInterval, 0, nil) defer func() { _ = q.Close() }() matcher, err := labels.NewMatcher(labels.MatchRegexp, "__name__", "metric.*") s.Require().NoError(err) @@ -472,7 +472,7 @@ func (s *QuerierSuite) TestLabelValuesNoMatches() { } s.appendTimeSeries(timeSeries) - q := querier.NewQuerier(s.head, querier.NewNoOpShardedDeduplicator, 0, 2, s.scrapeInterval, nil) + q := querier.NewQuerier(s.head, querier.NewNoOpShardedDeduplicator, 0, 2, s.scrapeInterval, 0, nil) defer func() { _ = q.Close() }() matcher, err := labels.NewMatcher(labels.MatchEqual, "__name__", "metric2") s.Require().NoError(err) @@ -505,7 +505,7 @@ func (s *QuerierSuite) TestLabelValuesNoMatchesOnName() { } s.appendTimeSeries(timeSeries) - q := querier.NewQuerier(s.head, querier.NewNoOpShardedDeduplicator, 0, 2, s.scrapeInterval, nil) + q := querier.NewQuerier(s.head, querier.NewNoOpShardedDeduplicator, 0, 2, s.scrapeInterval, 0, nil) defer func() { _ = q.Close() }() matcher, err := labels.NewMatcher(labels.MatchRegexp, "__name__", "metric.*") s.Require().NoError(err) diff --git a/pp/go/storage/querier/series_test.go b/pp/go/storage/querier/series_test.go index 0a56d5ec1..4d29a4fa1 100644 --- a/pp/go/storage/querier/series_test.go +++ b/pp/go/storage/querier/series_test.go @@ -1,7 +1,6 @@ package querier_test import ( - "fmt" "math" "testing" @@ -746,105 +745,3 @@ func (s *SeriesSetTestSuite) TestChangesFunc() { s.Equal(int64(250), actual[0].Samples[3].Timestamp) s.Equal(value.StaleNaN, math.Float64bits(actual[0].Samples[3].Value)) } - -func (s *SeriesSetTestSuite) TestMaxOverTimeFunc2() { - // Arrange - samples := []cppbridge.Sample{} - - timeData := int64(600) - endData := int64(1540) - stepData := int64(30) - val := 1.0 - for ; timeData < endData; timeData += stepData { - samples = append(samples, cppbridge.Sample{Timestamp: timeData, Value: val}) - val++ - } - - storagetest.MustAppendTimeSeriesToLSSAndDataStorage(s.lss, s.ds, []storagetest.TimeSeries{ - { - Labels: labels.FromStrings("__name__", "metric", "job", "test"), - Samples: samples, - }, - }...) - - // - - step := int64(70) - for i := 15; i <= 600; i += 1 { - hints := &storage.SelectHints{ - Start: 900 - int64(i) + 1, - End: 1500, - Func: "max_over_time", - Step: step, - Range: int64(i), - LookbackDelta: 120, - } - - // Act - seriesSet := s.queryAggr(s.lss, s.ds, 0, hints.End, cppbridge.NoDownsampling, hints, model.LabelMatcher{ - Name: "__name__", - Value: "metric", - MatcherType: model.MatcherTypeExactMatch, - }) - - ts := storagetest.TimeSeriesFromSeriesSet(seriesSet, true) - calc(hints, stepData, int64(len(ts[0].Samples))) - // fmt.Println( - // "step", step, - // "range", i, - // "ts", len(ts[0].Samples), - // "calc", calc(hints, stepData, int64(len(ts[0].Samples))), - // // "samples", ts[0].Samples, - - // "\n============", - // ) - } -} - -func calc(hints *storage.SelectHints, interval, samples int64) bool { - // maxPoints := (hints.End-hints.Start)/interval + 1 - stepPoints := (hints.End-hints.Start)/hints.Step + 1 - // rangePoints := (hints.End-hints.Start)/hints.Range + 1 - // fmt.Println( - // " === maxPoints", maxPoints, - // "stepPoints", stepPoints, - // "rangePoints", rangePoints, - // "deltaTS", (hints.End - hints.Start), - // "hints.End", hints.End, - // "hints.Start", hints.Start, - // "samples", samples, - // ) - if hints.Step >= hints.Range { - // fmt.Println(" === 1 ", (hints.End-hints.Start)/hints.Step, maxPoints) - return true - } - - k := hints.Range % hints.Step - fmt.Println(" === k", k, calc2(float64(samples-stepPoints), float64(stepPoints))) - // fmt.Println(" === k1", hints.Range%interval, (hints.End-hints.Start)/(hints.Step-k)+1) - // fmt.Println(" === expected", (hints.End-hints.Start)/(samples-1)) - if k == 0 || k > interval { - return true - } - - return false -} - -func TestXxx(t *testing.T) { - t.Log(75 % 70) - t.Log(floorDiv(300, 70)) - // srape interval - // step/scrape_interval (*2 if range > step and range % step > 0) -} - -func floorDiv(a, b int64) int64 { - quotient := a / b - if a%b < 0 { - quotient-- - } - return quotient -} - -func calc2(a, b float64) float64 { - return math.Round((a/b)*10) / 10 -} From af89ae6065e01d0959c5649a6917fac3d91df46c Mon Sep 17 00:00:00 2001 From: Alexandr Yudin <57181751+u-veles-a@users.noreply.github.com> Date: Tue, 23 Jun 2026 07:24:45 +0000 Subject: [PATCH 32/37] add test for AggrSeriesSet Signed-off-by: Alexandr Yudin <57181751+u-veles-a@users.noreply.github.com> --- pp/go/storage/querier/aggr_series_set_test.go | 230 ++++++++++++++++++ 1 file changed, 230 insertions(+) create mode 100644 pp/go/storage/querier/aggr_series_set_test.go diff --git a/pp/go/storage/querier/aggr_series_set_test.go b/pp/go/storage/querier/aggr_series_set_test.go new file mode 100644 index 000000000..a5ddf2718 --- /dev/null +++ b/pp/go/storage/querier/aggr_series_set_test.go @@ -0,0 +1,230 @@ +package querier_test + +import ( + "testing" + + "github.com/stretchr/testify/suite" + + "github.com/prometheus/prometheus/model/labels" + "github.com/prometheus/prometheus/pp/go/cppbridge" + "github.com/prometheus/prometheus/pp/go/model" + "github.com/prometheus/prometheus/pp/go/storage/head/shard" + "github.com/prometheus/prometheus/pp/go/storage/querier" + "github.com/prometheus/prometheus/pp/go/storage/storagetest" + "github.com/prometheus/prometheus/storage" +) + +type AggrSeriesSetSuite struct { + suite.Suite + + timeSeries []storagetest.TimeSeries + lss *shard.LSS + ds *shard.DataStorage +} + +func TestAggrSeriesSetSuite(t *testing.T) { + suite.Run(t, new(AggrSeriesSetSuite)) +} + +func (s *AggrSeriesSetSuite) SetupTest() { + s.lss = shard.NewLSS() + s.ds = shard.NewDataStorage() + + s.timeSeries = []storagetest.TimeSeries{ + { + Labels: labels.FromStrings("__name__", "metric", "job", "test", "instance", "instance1"), + Samples: []cppbridge.Sample{{Timestamp: 10, Value: 1}}, + }, + { + Labels: labels.FromStrings("__name__", "metric", "job", "test", "instance", "instance1"), + Samples: []cppbridge.Sample{{Timestamp: 11, Value: 5}}, + }, + { + Labels: labels.FromStrings("__name__", "metric", "job", "test", "instance", "instance1"), + Samples: []cppbridge.Sample{{Timestamp: 12, Value: 3}}, + }, + { + Labels: labels.FromStrings("__name__", "metric", "job", "test", "instance", "instance1"), + Samples: []cppbridge.Sample{{Timestamp: 13, Value: 7}}, + }, + { + Labels: labels.FromStrings("__name__", "metric", "job", "test2", "instance", "instance2"), + Samples: []cppbridge.Sample{{Timestamp: 10, Value: 1}}, + }, + { + Labels: labels.FromStrings("__name__", "metric", "job", "test2", "instance", "instance2"), + Samples: []cppbridge.Sample{{Timestamp: 11, Value: 4}}, + }, + { + Labels: labels.FromStrings("__name__", "metric", "job", "test2", "instance", "instance2"), + Samples: []cppbridge.Sample{{Timestamp: 12, Value: 2}}, + }, + } +} + +func (s *AggrSeriesSetSuite) query( + lss *shard.LSS, + ds *shard.DataStorage, + start, end, downsamplingMs int64, + hints *storage.SelectHints, + matchers ...model.LabelMatcher, +) storage.SeriesSet { + selector, snapshot, err := lss.QuerySelector(0, matchers) + s.Require().NoError(err) + + if selector == 0 || snapshot == nil { + return &querier.AggrSeriesSet{} + } + + lssQueryResult := snapshot.Query(selector) + if lssQueryResult.Status() == cppbridge.LSSQueryStatusNoMatch { + return &querier.AggrSeriesSet{} + } + + dsQueryResult := ds.Query(cppbridge.DataStorageQuery{ + StartTimestampMs: start, + EndTimestampMs: end, + LabelSetIDs: lssQueryResult.IDs(), + }, downsamplingMs, hints) + + s.Require().Equal(cppbridge.DataStorageQueryStatusSuccess, dsQueryResult.Status) + + aggSS := querier.NewAggrSeriesSet( + snapshot, + dsQueryResult.SerializedData, + lssQueryResult, + start, + end, + ) + + return aggSS +} + +func (s *AggrSeriesSetSuite) TestMaxOverTimeFunc() { + // Arrange + matcher := model.LabelMatcher{ + Name: "__name__", + Value: "metric", + MatcherType: model.MatcherTypeExactMatch, + } + + var start int64 + var end int64 = 50 + hints := &storage.SelectHints{ + Start: 10, + End: 13, + Step: 3, + Func: "max_over_time", + Range: 3, + } + + expected := []storagetest.TimeSeries{ + { + Labels: labels.FromStrings("__name__", "metric", "instance", "instance1", "job", "test"), + Samples: []cppbridge.Sample{{Timestamp: 11, Value: 5}}, + }, + { + Labels: labels.FromStrings("__name__", "metric", "instance", "instance2", "job", "test2"), + Samples: []cppbridge.Sample{{Timestamp: 11, Value: 4}}, + }, + } + + storagetest.MustAppendTimeSeriesToLSSAndDataStorage(s.lss, s.ds, s.timeSeries...) + + // Act + seriesSet := s.query(s.lss, s.ds, start, end, cppbridge.NoDownsampling, hints, matcher) + + // Assert + actual := storagetest.TimeSeriesFromSeriesSet(seriesSet, true) + s.Require().Equal(len(expected), len(actual)) + for i := range expected { + s.Require().Equal(expected[i].Labels, actual[i].Labels) + s.Require().Equal(expected[i].Samples, actual[i].Samples) + } +} + +func (s *AggrSeriesSetSuite) TestMinOverTimeFunc() { + // Arrange + matcher := model.LabelMatcher{ + Name: "__name__", + Value: "metric", + MatcherType: model.MatcherTypeExactMatch, + } + + var start int64 + var end int64 = 50 + hints := &storage.SelectHints{ + Start: 10, + End: 13, + Step: 3, + Func: "min_over_time", + Range: 3, + } + + expected := []storagetest.TimeSeries{ + { + Labels: labels.FromStrings("__name__", "metric", "instance", "instance1", "job", "test"), + Samples: []cppbridge.Sample{{Timestamp: 10, Value: 1}}, + }, + { + Labels: labels.FromStrings("__name__", "metric", "instance", "instance2", "job", "test2"), + Samples: []cppbridge.Sample{{Timestamp: 10, Value: 1}}, + }, + } + + storagetest.MustAppendTimeSeriesToLSSAndDataStorage(s.lss, s.ds, s.timeSeries...) + + // Act + seriesSet := s.query(s.lss, s.ds, start, end, cppbridge.NoDownsampling, hints, matcher) + + // Assert + actual := storagetest.TimeSeriesFromSeriesSet(seriesSet, true) + s.Require().Equal(len(expected), len(actual)) + for i := range expected { + s.Require().Equal(expected[i].Labels, actual[i].Labels) + s.Require().Equal(expected[i].Samples, actual[i].Samples) + } +} + +func (s *AggrSeriesSetSuite) TestLastOverTimeFunc() { + // Arrange + matcher := model.LabelMatcher{ + Name: "__name__", + Value: "metric", + MatcherType: model.MatcherTypeExactMatch, + } + + var start int64 + var end int64 = 50 + hints := &storage.SelectHints{ + Start: 10, + End: 13, + Step: 3, + Func: "last_over_time", + Range: 3, + } + + expected := []storagetest.TimeSeries{ + { + Labels: labels.FromStrings("__name__", "metric", "instance", "instance1", "job", "test"), + Samples: []cppbridge.Sample{{Timestamp: 12, Value: 3}}, + }, + { + Labels: labels.FromStrings("__name__", "metric", "instance", "instance2", "job", "test2"), + Samples: []cppbridge.Sample{{Timestamp: 12, Value: 2}}, + }, + } + + storagetest.MustAppendTimeSeriesToLSSAndDataStorage(s.lss, s.ds, s.timeSeries...) + + // Act + seriesSet := s.query(s.lss, s.ds, start, end, cppbridge.NoDownsampling, hints, matcher) + + // Assert + actual := storagetest.TimeSeriesFromSeriesSet(seriesSet, true) + s.Require().Equal(len(expected), len(actual)) + for i := range expected { + s.Require().Equal(expected[i].Labels, actual[i].Labels) + s.Require().Equal(expected[i].Samples, actual[i].Samples) + } +} From d71279456d8ae9ad26e4eaba19a26f92cb6b3bf4 Mon Sep 17 00:00:00 2001 From: Alexandr Yudin <57181751+u-veles-a@users.noreply.github.com> Date: Tue, 23 Jun 2026 07:26:42 +0000 Subject: [PATCH 33/37] small fix Signed-off-by: Alexandr Yudin <57181751+u-veles-a@users.noreply.github.com> --- pp/go/storage/querier/cross_series_set_test.go | 5 +++-- pp/go/storage/querier/querier_optimize_quick_test.go | 3 ++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/pp/go/storage/querier/cross_series_set_test.go b/pp/go/storage/querier/cross_series_set_test.go index 8a6e416d1..e5b4dd158 100644 --- a/pp/go/storage/querier/cross_series_set_test.go +++ b/pp/go/storage/querier/cross_series_set_test.go @@ -4,6 +4,9 @@ import ( "math" "testing" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" + "github.com/prometheus/prometheus/model/labels" "github.com/prometheus/prometheus/model/value" "github.com/prometheus/prometheus/pp/go/cppbridge" @@ -13,8 +16,6 @@ import ( "github.com/prometheus/prometheus/pp/go/storage/storagetest" "github.com/prometheus/prometheus/storage" "github.com/prometheus/prometheus/tsdb/chunkenc" - "github.com/stretchr/testify/require" - "github.com/stretchr/testify/suite" ) type CrossSeriesSetSuite struct { diff --git a/pp/go/storage/querier/querier_optimize_quick_test.go b/pp/go/storage/querier/querier_optimize_quick_test.go index 2ce51351f..e5a6bed63 100644 --- a/pp/go/storage/querier/querier_optimize_quick_test.go +++ b/pp/go/storage/querier/querier_optimize_quick_test.go @@ -12,11 +12,12 @@ import ( "github.com/go-kit/log" "github.com/prometheus/common/model" + "github.com/stretchr/testify/suite" + "github.com/prometheus/prometheus/pp/go/cppbridge" "github.com/prometheus/prometheus/pp/go/storage/querier" "github.com/prometheus/prometheus/promql" prom_storage "github.com/prometheus/prometheus/storage" - "github.com/stretchr/testify/suite" ) // From db92b4fb4595173dd9ef401f7edd966acb29da6b Mon Sep 17 00:00:00 2001 From: Alexandr Yudin <57181751+u-veles-a@users.noreply.github.com> Date: Tue, 23 Jun 2026 08:01:38 +0000 Subject: [PATCH 34/37] add metrics Signed-off-by: Alexandr Yudin <57181751+u-veles-a@users.noreply.github.com> --- pp/go/storage/querier/metrics.go | 11 +++++++++++ pp/go/storage/querier/querier.go | 24 +++++++++++++----------- 2 files changed, 24 insertions(+), 11 deletions(-) diff --git a/pp/go/storage/querier/metrics.go b/pp/go/storage/querier/metrics.go index 71e968890..4e65b93c4 100644 --- a/pp/go/storage/querier/metrics.go +++ b/pp/go/storage/querier/metrics.go @@ -17,9 +17,12 @@ type Metrics struct { LabelNamesDuration prometheus.Histogram LabelValuesDuration prometheus.Histogram SelectDuration *prometheus.HistogramVec + OptimizationType *prometheus.CounterVec } // NewMetrics init new [Metrics]. +// +//revive:disable-next-line:function-length // metrics constructor. func NewMetrics(registerer prometheus.Registerer, source string) *Metrics { factory := util.NewUnconflictRegisterer(registerer) return &Metrics{ @@ -63,5 +66,13 @@ func NewMetrics(registerer prometheus.Registerer, source string) *Metrics { }, []string{"query_type"}, ), + OptimizationType: factory.NewCounterVec( + prometheus.CounterOpts{ + Name: "prompp_querier_query_optimization_type_count", + Help: "Optimization type count", + ConstLabels: prometheus.Labels{"source": source}, + }, + []string{"optimization_type"}, + ), } } diff --git a/pp/go/storage/querier/querier.go b/pp/go/storage/querier/querier.go index c16900c1f..c210aff84 100644 --- a/pp/go/storage/querier/querier.go +++ b/pp/go/storage/querier/querier.go @@ -9,7 +9,6 @@ import ( "time" "unsafe" - "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/prometheus/model/labels" "github.com/prometheus/prometheus/pp/go/cppbridge" "github.com/prometheus/prometheus/pp/go/logger" @@ -170,6 +169,10 @@ func NewQuerier[ mint, maxt, scrapeIntervalMS, headMinTSMS int64, metrics *Metrics, ) *Querier[TTask, TDataStorage, TLSS, TShard, THead] { + if metrics == nil { + metrics = NewMetrics(nil, "queryable_unknown") + } + return newQuerierWithSelectFuncOptimize( head, deduplicatorCtor, @@ -195,6 +198,10 @@ func NewQuerierWithOutSelectFuncOptimize[ mint, maxt, scrapeIntervalMS, headMinTSMS int64, metrics *Metrics, ) *Querier[TTask, TDataStorage, TLSS, TShard, THead] { + if metrics == nil { + metrics = NewMetrics(nil, "queryable_unknown") + } + return newQuerierWithSelectFuncOptimize( head, deduplicatorCtor, @@ -321,11 +328,7 @@ func (q *Querier[TTask, TDataStorage, TLSS, TShard, THead]) selectInstant( defer release() defer func() { - if q.metrics != nil { - q.metrics.SelectDuration.With( - prometheus.Labels{"query_type": "instant"}, - ).Observe(float64(time.Since(start).Microseconds())) - } + q.metrics.SelectDuration.WithLabelValues("instant").Observe(float64(time.Since(start).Microseconds())) }() poolProvider := q.head.PoolProvider() @@ -412,11 +415,7 @@ func (q *Querier[TTask, TDataStorage, TLSS, TShard, THead]) selectRange( defer release() defer func() { - if q.metrics != nil { - q.metrics.SelectDuration.With( - prometheus.Labels{"query_type": "range"}, - ).Observe(float64(time.Since(start).Microseconds())) - } + q.metrics.SelectDuration.WithLabelValues("range").Observe(float64(time.Since(start).Microseconds())) }() poolProvider := q.head.PoolProvider() @@ -440,13 +439,16 @@ func (q *Querier[TTask, TDataStorage, TLSS, TShard, THead]) selectRange( queryDataStorage(dsQueryRangeQuerier, q.head, lssQueryResults, shardedSerializedData, q.mint, q.maxt, hints) if isCrossSeriesFunc(hints) { + q.metrics.OptimizationType.WithLabelValues("cross_series").Inc() return q.makeCrossSeriesSet(lssQueryResults, snapshots, shardedSerializedData, hints) } if isAggregationSeriesFunc(hints) { + q.metrics.OptimizationType.WithLabelValues("aggregation").Inc() return q.makeAggrSeriesSet(lssQueryResults, snapshots, shardedSerializedData) } + q.metrics.OptimizationType.WithLabelValues("none").Inc() return q.makeSeriesSet(lssQueryResults, snapshots, shardedSerializedData) } From bccfdac5a31499014641292d8f0f832db899a655 Mon Sep 17 00:00:00 2001 From: Alexandr Yudin <57181751+u-veles-a@users.noreply.github.com> Date: Tue, 23 Jun 2026 11:09:42 +0000 Subject: [PATCH 35/37] fix sort vector Signed-off-by: Alexandr Yudin <57181751+u-veles-a@users.noreply.github.com> --- promql/engine.go | 1 + 1 file changed, 1 insertion(+) diff --git a/promql/engine.go b/promql/engine.go index d323d2323..50d23d5d1 100644 --- a/promql/engine.go +++ b/promql/engine.go @@ -769,6 +769,7 @@ func (ng *Engine) execEvalStmt(ctx context.Context, query *query, s *parser.Eval query.matrix = mat switch s.Expr.Type() { case parser.ValueTypeVector: + ng.sortMatrixResult(ctx, query, mat) // Convert matrix with one value per series into vector. vector := make(Vector, len(mat)) for i, s := range mat { From 5d24d806669a14134309c0d2bf7d17a7e5a5711b Mon Sep 17 00:00:00 2001 From: Alexandr Yudin <57181751+u-veles-a@users.noreply.github.com> Date: Tue, 23 Jun 2026 13:15:46 +0000 Subject: [PATCH 36/37] tuning asan test Signed-off-by: Alexandr Yudin <57181751+u-veles-a@users.noreply.github.com> --- .../storage/querier/querier_optimize_test.go | 29 +++++++------------ .../querier_optimize_variables_asan_test.go | 8 ++--- 2 files changed, 14 insertions(+), 23 deletions(-) diff --git a/pp/go/storage/querier/querier_optimize_test.go b/pp/go/storage/querier/querier_optimize_test.go index aea91d29d..b8a6178eb 100644 --- a/pp/go/storage/querier/querier_optimize_test.go +++ b/pp/go/storage/querier/querier_optimize_test.go @@ -486,7 +486,6 @@ func (s *querierOptimize) setup( s.head = s.mustCreateHead(0) s.fillHead(ctx) - s.fillHeadWithCounter(ctx, 0, querier.DefaultCountOfSeriesToOptimize) s.lookbackDelta = 5 * time.Minute s.queryOpts = promql.NewPrometheusQueryOpts(false, s.lookbackDelta) @@ -724,6 +723,7 @@ func (s *MatrixQuerierOptimizeSuiteSuite) SetupSuite() { defaultStep, defaultCountOfSteps, ) + querier.IsPossibleToOptimize = func( []*cppbridge.LSSQueryResult, *prom_storage.SelectHints, @@ -748,15 +748,6 @@ func (s *MatrixQuerierOptimizeSuiteSuite) SetupSuite() { s.subQueries = defaultSubQueries s.modifiers = defaultModifiers s.offsets = defaultOffsets - - q, err := s.Querier(s.start.UnixMilli(), s.end.UnixMilli()) - s.Require().NoError(err) - - names, _, err := q.LabelValues(s.T().Context(), "__name__", &prom_storage.LabelHints{}) - s.Require().NoError(err) - - s.metricNames = querier.DeduplicateAndSortStringSlices(names) - s.Require().NoError(q.Close()) } func (s *MatrixQuerierOptimizeSuiteSuite) TearDownSuite() { @@ -1289,15 +1280,15 @@ func BenchmarkRangeQueryOverTime(b *testing.B) { defaultCountOfSteps, ) defer qo.close() - // querier.IsPossibleToOptimize = func( - // []*cppbridge.LSSQueryResult, - // *prom_storage.SelectHints, - // int64, int64, - // ) func() bool { - // return func() bool { - // return true - // } - // } + querier.IsPossibleToOptimize = func( + []*cppbridge.LSSQueryResult, + *prom_storage.SelectHints, + int64, int64, + ) func() bool { + return func() bool { + return true + } + } queryEngine := promql.NewEngine(promql.EngineOpts{ Logger: log.NewNopLogger(), diff --git a/pp/go/storage/querier/querier_optimize_variables_asan_test.go b/pp/go/storage/querier/querier_optimize_variables_asan_test.go index 0e30c607d..bea4a4f0a 100644 --- a/pp/go/storage/querier/querier_optimize_variables_asan_test.go +++ b/pp/go/storage/querier/querier_optimize_variables_asan_test.go @@ -15,14 +15,14 @@ const ( // defaultSteps is the default steps. var defaultSteps = []time.Duration{ - defaultStep - time.Second, // [14s] - defaultStep * 2, // [30s] + defaultStep - time.Second, + defaultStep * 2, } // defaultSubQueries is the default subqueries. var defaultSubQueries = []subQuery{ - {window: model.Duration(defaultStep), step: 0}, // [15s] - {window: model.Duration(defaultStep*4 + time.Second), step: 0}, // [61s] + {window: model.Duration(defaultStep), step: 0}, + {window: model.Duration(defaultStep*4 + time.Second), step: 0}, } // defaultModifiers is the default modifiers. From daf445c08c1b3670b1e158e5b49e7cfd1386cbc4 Mon Sep 17 00:00:00 2001 From: Alexandr Yudin <57181751+u-veles-a@users.noreply.github.com> Date: Tue, 23 Jun 2026 15:30:20 +0000 Subject: [PATCH 37/37] tuning and fix test Signed-off-by: Alexandr Yudin <57181751+u-veles-a@users.noreply.github.com> --- pp/go/storage/querier/querier_optimize_test.go | 9 +++++++++ .../querier/querier_optimize_variables_asan_test.go | 1 - promql/engine.go | 1 - 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/pp/go/storage/querier/querier_optimize_test.go b/pp/go/storage/querier/querier_optimize_test.go index b8a6178eb..58139c98d 100644 --- a/pp/go/storage/querier/querier_optimize_test.go +++ b/pp/go/storage/querier/querier_optimize_test.go @@ -7,6 +7,7 @@ import ( "math" "os" "path/filepath" + "slices" "strconv" "strings" "testing" @@ -999,6 +1000,14 @@ func vectorEqual(exp, act promql.Vector) (bool, string) { _, _ = msg.WriteString("vector:\n") isEqual := true + // we are sorting, because the sorting is broken on instant requests + slices.SortFunc(exp, func(a, b promql.Sample) int { + return labels.Compare(a.Metric, b.Metric) + }) + slices.SortFunc(act, func(a, b promql.Sample) int { + return labels.Compare(a.Metric, b.Metric) + }) + for i, v := range exp { if eq, result := sampleEqual(v, act[i]); !eq { _, _ = msg.WriteString(result) diff --git a/pp/go/storage/querier/querier_optimize_variables_asan_test.go b/pp/go/storage/querier/querier_optimize_variables_asan_test.go index bea4a4f0a..1e16b90c8 100644 --- a/pp/go/storage/querier/querier_optimize_variables_asan_test.go +++ b/pp/go/storage/querier/querier_optimize_variables_asan_test.go @@ -15,7 +15,6 @@ const ( // defaultSteps is the default steps. var defaultSteps = []time.Duration{ - defaultStep - time.Second, defaultStep * 2, } diff --git a/promql/engine.go b/promql/engine.go index 50d23d5d1..d323d2323 100644 --- a/promql/engine.go +++ b/promql/engine.go @@ -769,7 +769,6 @@ func (ng *Engine) execEvalStmt(ctx context.Context, query *query, s *parser.Eval query.matrix = mat switch s.Expr.Type() { case parser.ValueTypeVector: - ng.sortMatrixResult(ctx, query, mat) // Convert matrix with one value per series into vector. vector := make(Vector, len(mat)) for i, s := range mat {