Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,7 @@
* [BUGFIX] Ruler: Register xfunctions (xincrease, xrate, xdelta) in the global parser before loading rule files. #7621
* [BUGFIX] Security: Reject empty entries in `-distributor.sign-write-requests-keys` caused by stray or trailing commas (e.g. `newkey,`). Previously these were silently accepted and produced an empty signing key, which downgraded HMAC stream-push authentication to a forgeable signature. Misconfigured flags now fail at process startup; audit your configs before upgrading. #7587
* [BUGFIX] Querier: Fix panic due to request tracker truncating multi-byte UTF-8 character #7640
* [BUGFIX] Ingester: Fix panic (`HistogramProtoToHistogram called with a float histogram`) when ingesting a float native histogram with a zero count (e.g. a staleness marker or empty histogram). The decoder is now selected by histogram type via `IsFloatHistogram()` instead of by count value. #7645

## 1.21.0 2026-04-24

Expand Down
9 changes: 8 additions & 1 deletion pkg/ingester/ingester.go
Original file line number Diff line number Diff line change
Expand Up @@ -1591,7 +1591,14 @@ func (i *Ingester) Push(ctx context.Context, req *cortexpb.WriteRequest) (*corte
fh *histogram.FloatHistogram
)

if hp.GetCountFloat() > 0 {
// Choose the decoder based on the histogram's proto type (the
// CountInt/CountFloat oneof), not the count value. A float
// histogram with a count of 0 (e.g. a staleness marker or an
// empty histogram) still has the CountFloat oneof set, so a
// value-based check (hp.GetCountFloat() > 0) would misroute it
// to the integer decoder, which panics. This mirrors the
// discriminator used everywhere else (e.g. util/validation).
if hp.IsFloatHistogram() {
fh = cortexpb.FloatHistogramProtoToFloatHistogram(hp.Histogram)
} else {
h = cortexpb.HistogramProtoToHistogram(hp.Histogram)
Expand Down
59 changes: 59 additions & 0 deletions pkg/ingester/ingester_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ import (
"github.com/prometheus/common/promslog"
"github.com/prometheus/prometheus/model/histogram"
"github.com/prometheus/prometheus/model/labels"
"github.com/prometheus/prometheus/model/value"
"github.com/prometheus/prometheus/storage"
"github.com/prometheus/prometheus/tsdb"
"github.com/prometheus/prometheus/tsdb/chunkenc"
Expand Down Expand Up @@ -2635,6 +2636,64 @@ func TestIngester_PushNativeHistogramErrors(t *testing.T) {
}
}

func TestIngester_Push_FloatHistogramWithZeroCount(t *testing.T) {
// Regression test: a float histogram with a count of 0 (e.g. a staleness
// marker or an empty histogram) has the CountFloat oneof set, so it IS a
// float histogram. A value-based discriminator (hp.GetCountFloat() > 0)
// misroutes it to the integer decoder cortexpb.HistogramProtoToHistogram,
// which panics with "HistogramProtoToHistogram called with a float
// histogram" and crashes the ingester. The decoder must be selected by the
// proto type via IsFloatHistogram() instead.
metricLabelAdapters := []cortexpb.LabelAdapter{{Name: labels.MetricName, Value: "test"}}
metricLabels := cortexpb.FromLabelAdaptersToLabels(metricLabelAdapters)
userID := "test"

for _, tc := range []struct {
name string
histogram cortexpb.WrappedHistogram
}{
{
name: "staleness marker float histogram with zero count",
histogram: cortexpb.WrapHistogram(cortexpb.FloatHistogramToHistogramProto(10, &histogram.FloatHistogram{Sum: math.Float64frombits(value.StaleNaN)})),
},
{
name: "empty float histogram with zero count",
histogram: cortexpb.WrapHistogram(cortexpb.FloatHistogramToHistogramProto(10, &histogram.FloatHistogram{})),
},
} {
t.Run(tc.name, func(t *testing.T) {
registry := prometheus.NewRegistry()

// Create a mocked ingester
cfg := defaultIngesterTestConfig(t)
cfg.LifecyclerConfig.JoinAfter = 0

limits := defaultLimitsTestConfig()
limits.EnableNativeHistograms = true
i, err := prepareIngesterWithBlocksStorageAndLimits(t, cfg, limits, nil, "", registry)
require.NoError(t, err)
require.NoError(t, services.StartAndAwaitRunning(context.Background(), i))
defer services.StopAndAwaitTerminated(context.Background(), i) //nolint:errcheck

ctx := user.InjectOrgID(context.Background(), userID)

// Wait until the ingester is ACTIVE
test.Poll(t, 100*time.Millisecond, ring.ACTIVE, func() any {
return i.lifecycler.GetState()
})

req := cortexpb.ToWriteRequest([]labels.Labels{metricLabels}, nil, nil, []cortexpb.WrappedHistogram{tc.histogram}, cortexpb.API)
// Before the fix this panics inside Push; after the fix it must be
// ingested without error.
_, err = i.Push(ctx, req)
require.NoError(t, err)

require.Equal(t, float64(0), testutil.ToFloat64(i.metrics.ingestedHistogramsFail))
require.Equal(t, float64(1), testutil.ToFloat64(i.metrics.ingestedHistograms))
})
}
}

func TestIngester_Push_ShouldCorrectlyTrackMetricsInMultiTenantScenario(t *testing.T) {
metricLabelAdapters := []cortexpb.LabelAdapter{{Name: labels.MetricName, Value: "test"}}
metricLabels := cortexpb.FromLabelAdaptersToLabels(metricLabelAdapters)
Expand Down
Loading