From afd201a86cdce5ae6fc30b361612aa5284410922 Mon Sep 17 00:00:00 2001 From: Ben Ye Date: Thu, 25 Jun 2026 07:35:45 +0000 Subject: [PATCH 1/3] fix(ingester): select histogram decoder by type, not count value The ingester Push path chose the float vs integer native-histogram decoder by checking `hp.GetCountFloat() > 0`. A float histogram with a count of 0 (e.g. a staleness marker or an empty histogram) still has the CountFloat oneof set, so `IsFloatHistogram()` reports true while the value check is false. Such samples were routed to `HistogramProtoToHistogram`, which panics with "HistogramProtoToHistogram called with a float histogram", crashing the whole ingester process in the synchronous Push handler. Select the decoder by the proto type via `IsFloatHistogram()`, matching the discriminator already used in `util/validation` and upstream Prometheus. Add a regression test covering count-0 float histograms (staleness marker and empty histogram). Signed-off-by: Ben Ye --- pkg/ingester/ingester.go | 9 +++++- pkg/ingester/ingester_test.go | 59 +++++++++++++++++++++++++++++++++++ 2 files changed, 67 insertions(+), 1 deletion(-) diff --git a/pkg/ingester/ingester.go b/pkg/ingester/ingester.go index 81780e4a01..bb9062fc40 100644 --- a/pkg/ingester/ingester.go +++ b/pkg/ingester/ingester.go @@ -1591,7 +1591,14 @@ func (i *Ingester) Push(ctx context.Context, req *cortexpb.WriteRequest) (*corte fh *histogram.FloatHistogram ) - if hp.GetCountFloat() > 0 { + // Choose the decoder based on the histogram's proto type (the + // CountInt/CountFloat oneof), not the count value. A float + // histogram with a count of 0 (e.g. a staleness marker or an + // empty histogram) still has the CountFloat oneof set, so a + // value-based check (hp.GetCountFloat() > 0) would misroute it + // to the integer decoder, which panics. This mirrors the + // discriminator used everywhere else (e.g. util/validation). + if hp.Histogram.IsFloatHistogram() { fh = cortexpb.FloatHistogramProtoToFloatHistogram(hp.Histogram) } else { h = cortexpb.HistogramProtoToHistogram(hp.Histogram) diff --git a/pkg/ingester/ingester_test.go b/pkg/ingester/ingester_test.go index fe396e6568..1f33a0ebc9 100644 --- a/pkg/ingester/ingester_test.go +++ b/pkg/ingester/ingester_test.go @@ -30,6 +30,7 @@ import ( "github.com/prometheus/common/promslog" "github.com/prometheus/prometheus/model/histogram" "github.com/prometheus/prometheus/model/labels" + "github.com/prometheus/prometheus/model/value" "github.com/prometheus/prometheus/storage" "github.com/prometheus/prometheus/tsdb" "github.com/prometheus/prometheus/tsdb/chunkenc" @@ -2635,6 +2636,64 @@ func TestIngester_PushNativeHistogramErrors(t *testing.T) { } } +func TestIngester_Push_FloatHistogramWithZeroCount(t *testing.T) { + // Regression test: a float histogram with a count of 0 (e.g. a staleness + // marker or an empty histogram) has the CountFloat oneof set, so it IS a + // float histogram. A value-based discriminator (hp.GetCountFloat() > 0) + // misroutes it to the integer decoder cortexpb.HistogramProtoToHistogram, + // which panics with "HistogramProtoToHistogram called with a float + // histogram" and crashes the ingester. The decoder must be selected by the + // proto type via IsFloatHistogram() instead. + metricLabelAdapters := []cortexpb.LabelAdapter{{Name: labels.MetricName, Value: "test"}} + metricLabels := cortexpb.FromLabelAdaptersToLabels(metricLabelAdapters) + userID := "test" + + for _, tc := range []struct { + name string + histogram cortexpb.WrappedHistogram + }{ + { + name: "staleness marker float histogram with zero count", + histogram: cortexpb.WrapHistogram(cortexpb.FloatHistogramToHistogramProto(10, &histogram.FloatHistogram{Sum: math.Float64frombits(value.StaleNaN)})), + }, + { + name: "empty float histogram with zero count", + histogram: cortexpb.WrapHistogram(cortexpb.FloatHistogramToHistogramProto(10, &histogram.FloatHistogram{})), + }, + } { + t.Run(tc.name, func(t *testing.T) { + registry := prometheus.NewRegistry() + + // Create a mocked ingester + cfg := defaultIngesterTestConfig(t) + cfg.LifecyclerConfig.JoinAfter = 0 + + limits := defaultLimitsTestConfig() + limits.EnableNativeHistograms = true + i, err := prepareIngesterWithBlocksStorageAndLimits(t, cfg, limits, nil, "", registry) + require.NoError(t, err) + require.NoError(t, services.StartAndAwaitRunning(context.Background(), i)) + defer services.StopAndAwaitTerminated(context.Background(), i) //nolint:errcheck + + ctx := user.InjectOrgID(context.Background(), userID) + + // Wait until the ingester is ACTIVE + test.Poll(t, 100*time.Millisecond, ring.ACTIVE, func() any { + return i.lifecycler.GetState() + }) + + req := cortexpb.ToWriteRequest([]labels.Labels{metricLabels}, nil, nil, []cortexpb.WrappedHistogram{tc.histogram}, cortexpb.API) + // Before the fix this panics inside Push; after the fix it must be + // ingested without error. + _, err = i.Push(ctx, req) + require.NoError(t, err) + + require.Equal(t, float64(0), testutil.ToFloat64(i.metrics.ingestedHistogramsFail)) + require.Equal(t, float64(1), testutil.ToFloat64(i.metrics.ingestedHistograms)) + }) + } +} + func TestIngester_Push_ShouldCorrectlyTrackMetricsInMultiTenantScenario(t *testing.T) { metricLabelAdapters := []cortexpb.LabelAdapter{{Name: labels.MetricName, Value: "test"}} metricLabels := cortexpb.FromLabelAdaptersToLabels(metricLabelAdapters) From c8b10c66aa4e418b7125e9b369c09fa47e6c20db Mon Sep 17 00:00:00 2001 From: Ben Ye Date: Thu, 25 Jun 2026 07:36:47 +0000 Subject: [PATCH 2/3] docs: add CHANGELOG entry for ingester float histogram panic fix (#7645) Signed-off-by: Ben Ye --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2b12895372..897a1a8bf4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -70,6 +70,7 @@ * [BUGFIX] Ruler: Register xfunctions (xincrease, xrate, xdelta) in the global parser before loading rule files. #7621 * [BUGFIX] Security: Reject empty entries in `-distributor.sign-write-requests-keys` caused by stray or trailing commas (e.g. `newkey,`). Previously these were silently accepted and produced an empty signing key, which downgraded HMAC stream-push authentication to a forgeable signature. Misconfigured flags now fail at process startup; audit your configs before upgrading. #7587 * [BUGFIX] Querier: Fix panic due to request tracker truncating multi-byte UTF-8 character #7640 +* [BUGFIX] Ingester: Fix panic (`HistogramProtoToHistogram called with a float histogram`) when ingesting a float native histogram with a zero count (e.g. a staleness marker or empty histogram). The decoder is now selected by histogram type via `IsFloatHistogram()` instead of by count value. #7645 ## 1.21.0 2026-04-24 From ee813b3179c2d684c96743f5add8d613cf43d8bc Mon Sep 17 00:00:00 2001 From: Ben Ye Date: Thu, 25 Jun 2026 07:42:03 +0000 Subject: [PATCH 3/3] fix(ingester): simplify embedded selector to satisfy staticcheck QF1008 Signed-off-by: Ben Ye --- pkg/ingester/ingester.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/ingester/ingester.go b/pkg/ingester/ingester.go index bb9062fc40..06d9f13747 100644 --- a/pkg/ingester/ingester.go +++ b/pkg/ingester/ingester.go @@ -1598,7 +1598,7 @@ func (i *Ingester) Push(ctx context.Context, req *cortexpb.WriteRequest) (*corte // value-based check (hp.GetCountFloat() > 0) would misroute it // to the integer decoder, which panics. This mirrors the // discriminator used everywhere else (e.g. util/validation). - if hp.Histogram.IsFloatHistogram() { + if hp.IsFloatHistogram() { fh = cortexpb.FloatHistogramProtoToFloatHistogram(hp.Histogram) } else { h = cortexpb.HistogramProtoToHistogram(hp.Histogram)