From e86ac81eb2506d86b9bc336dfc6e8fe40b62e965 Mon Sep 17 00:00:00 2001 From: Zaq? Question Date: Tue, 7 Jul 2026 17:04:45 -0700 Subject: [PATCH] feat: optionally export sampled traces to a GCS bucket Adds a new GCSExport config group that, when enabled, writes every span of every kept trace to a GCS bucket as gzipped JSON Lines objects under time-partitioned paths (prefix/YYYY/MM/DD/HH/host-ts-seq.jsonl.gz), in addition to normal Honeycomb delivery. - transmit: new GCSTransmission with a non-blocking bounded queue, size/interval-based batching, and up to 4 concurrent uploads; a slow or unavailable GCS drops spans from the export (gcs_export_dropped) and never blocks the collector send path. Auth is via Application Default Credentials; uploads use a DoesNotExist precondition so retries are idempotent. - collect: kept spans are teed to the exporter at all three send sites (normal send, late-arriving spans, stress-relief keeps); dry-run "would have dropped" traces and non-trace events are not exported. - config: GCSExport group (Enabled, Bucket, KeyPrefix, FlushInterval, MaxBatchSize, QueueSize) with metadata, env var / CLI flag for the bucket, and regenerated docs/templates. - main: exporter is injected as "gcsExport", with a noop when disabled. New metrics: gcs_export_events, gcs_export_dropped, gcs_export_batches, gcs_export_errors, gcs_export_batch_bytes, gcs_export_queue_length. Adds cloud.google.com/go/storage (grpc pinned at v1.80.0). Co-Authored-By: Claude Fable 5 --- app/app_test.go | 1 + cmd/refinery/main.go | 8 + collect/collect.go | 18 ++ collect/collect_test.go | 1 + config.md | 74 ++++++- config/cmdenv.go | 1 + config/config.go | 3 + config/config_serializer.go | 1 + config/file_config.go | 19 ++ config/metadata/configMeta.yaml | 88 ++++++++ config/mock.go | 8 + config_complete.yaml | 80 ++++++- go.mod | 59 +++-- go.sum | 136 +++++++++--- metrics.md | 9 +- refinery_config.md | 71 ++++++ route/route_test.go | 1 + rules.md | 30 ++- tools/convert/configDataNames.txt | 16 +- tools/convert/metricsMeta.yaml | 28 +++ tools/convert/minimal_config.yaml | 11 +- tools/convert/templates/configV2.tmpl | 80 ++++++- transmit/gcs_transmit.go | 307 ++++++++++++++++++++++++++ transmit/gcs_transmit_test.go | 256 +++++++++++++++++++++ 24 files changed, 1247 insertions(+), 59 deletions(-) create mode 100644 transmit/gcs_transmit.go create mode 100644 transmit/gcs_transmit_test.go diff --git a/app/app_test.go b/app/app_test.go index cc4bd18788..c9671cb4bc 100644 --- a/app/app_test.go +++ b/app/app_test.go @@ -368,6 +368,7 @@ func newStartedApp( &inject.Object{Value: http.DefaultTransport, Name: "upstreamTransport"}, &inject.Object{Value: upstreamTransmission, Name: "upstreamTransmission"}, &inject.Object{Value: peerTransmissionWrapper, Name: "peerTransmission"}, + &inject.Object{Value: &transmit.NoopTransmission{}, Name: "gcsExport"}, &inject.Object{Value: shrdr}, &inject.Object{Value: noop.NewTracerProvider().Tracer("test"), Name: "tracer"}, &inject.Object{Value: collector}, diff --git a/cmd/refinery/main.go b/cmd/refinery/main.go index efe4e82110..2642224de2 100644 --- a/cmd/refinery/main.go +++ b/cmd/refinery/main.go @@ -186,6 +186,13 @@ func main() { nil, // No custom headers for peer-to-peer traffic ) + // the GCS exporter is optional; when it's disabled we inject a noop so + // that the collector's dependency is always satisfied + var gcsExport transmit.Transmission = &transmit.NoopTransmission{} + if c.GetGCSExportConfig().Enabled { + gcsExport = transmit.NewGCSTransmission() + } + // we need to include all the metrics types so we can inject them in case they're needed // but we only want to instantiate the ones that are enabled with non-null values var promMetrics metrics.MetricsBackend = &metrics.NullMetrics{} @@ -241,6 +248,7 @@ func main() { {Value: peerTransport, Name: "peerTransport"}, {Value: upstreamTransmission, Name: "upstreamTransmission"}, {Value: peerTransmission, Name: "peerTransmission"}, + {Value: gcsExport, Name: "gcsExport"}, {Value: shrdr}, {Value: collector}, {Value: promMetrics, Name: "promMetrics"}, diff --git a/collect/collect.go b/collect/collect.go index 0ce0be596b..829a091d36 100644 --- a/collect/collect.go +++ b/collect/collect.go @@ -88,6 +88,7 @@ type InMemCollector struct { Transmission transmit.Transmission `inject:"upstreamTransmission"` PeerTransmission transmit.Transmission `inject:"peerTransmission"` + GCSExport transmit.Transmission `inject:"gcsExport"` PubSub pubsub.PubSub `inject:""` Metrics metrics.Metrics `inject:"metrics"` SamplerFactory *sample.SamplerFactory `inject:""` @@ -491,10 +492,20 @@ func (i *InMemCollector) ProcessSpanImmediately(sp *types.Span) (processed bool, i.addAdditionalAttributes(sp) mergeTraceAndSpanSampleRates(sp, rate, i.Config.GetIsDryRun()) i.Transmission.EnqueueSpan(sp) + i.exportSpan(sp) return true, true } +// exportSpan sends a kept span to the optional GCS exporter, if one is +// configured. It must only be called for spans of traces that were actually +// kept by the sampler (not for dry-run "would have dropped" sends). +func (i *InMemCollector) exportSpan(sp *types.Span) { + if i.GCSExport != nil { + i.GCSExport.EnqueueSpan(sp) + } +} + // dealWithSentTrace handles a span that has arrived after the sampling decision // on the trace has already been made, and it obeys that decision by either // sending the span immediately or dropping it. @@ -557,6 +568,7 @@ func (i *InMemCollector) dealWithSentTrace(ctx context.Context, tr cache.TraceSe i.Metrics.Increment(TraceSendLateSpan) i.addAdditionalAttributes(sp) i.Transmission.EnqueueSpan(sp) + i.exportSpan(sp) return } i.Metrics.Increment("events_dropped") @@ -834,6 +846,12 @@ func (i *InMemCollector) sendTraces() { sp.APIKey = t.APIKey i.Transmission.EnqueueSpan(sp) + // only export spans of traces the sampler actually kept; in dry + // run mode, traces that would have been dropped still reach here + // with shouldSend=false and are not exported. + if t.shouldSend { + i.exportSpan(sp) + } } span.End() } diff --git a/collect/collect_test.go b/collect/collect_test.go index a83de54535..ed33bdf7e0 100644 --- a/collect/collect_test.go +++ b/collect/collect_test.go @@ -882,6 +882,7 @@ func TestDependencyInjection(t *testing.T) { &inject.Object{Value: &sharder.SingleServerSharder{}}, &inject.Object{Value: &transmit.MockTransmission{}, Name: "upstreamTransmission"}, &inject.Object{Value: &transmit.MockTransmission{}, Name: "peerTransmission"}, + &inject.Object{Value: &transmit.NoopTransmission{}, Name: "gcsExport"}, &inject.Object{Value: &pubsub.LocalPubSub{}}, &inject.Object{Value: &metrics.NullMetrics{}, Name: "metrics"}, &inject.Object{Value: &sample.SamplerFactory{}}, diff --git a/config.md b/config.md index 23178b357d..a82a5b1b6e 100644 --- a/config.md +++ b/config.md @@ -3,7 +3,7 @@ # Honeycomb Refinery Configuration Documentation This is the documentation for the configuration file for Honeycomb's Refinery. -It was automatically generated on 2026-04-09 at 22:21:32 UTC. +It was automatically generated on 2026-07-07 at 21:27:51 UTC. ## The Config file @@ -48,6 +48,7 @@ The remainder of this document describes the sections within the file and the fi - [gRPC Server Parameters](#grpc-server-parameters) - [Sample Cache](#sample-cache) - [Stress Relief](#stress-relief) +- [GCS Export](#gcs-export) ## General Configuration `General` contains general configuration options that apply to the entire Refinery process. @@ -1315,3 +1316,74 @@ This setting helps to prevent oscillations. - Type: `duration` - Default: `10s` +## GCS Export + +`GCSExport` contains configuration for optionally exporting sampled (kept) trace spans to a Google Cloud Storage bucket in addition to sending them to Honeycomb. +Spans are buffered in memory and written as gzipped JSON Lines objects, batched into time-partitioned paths of the form `KeyPrefix/YYYY/MM/DD/HH/hostname-timestamp.jsonl.gz`. +Authentication uses Google Application Default Credentials. + +### `Enabled` + +Enabled controls whether kept trace spans are also exported to a GCS bucket. + +If `true`, every span belonging to a trace that Refinery decides to keep is also written to the configured GCS bucket. +Spans of dropped traces are never exported. + +- Not eligible for live reload. +- Type: `bool` + +### `Bucket` + +Bucket is the name of the GCS bucket to which sampled traces are exported. + +The bucket must already exist and the credentials available via Application Default Credentials must have permission to create objects in it. +Required when `Enabled` is `true`. + +- Not eligible for live reload. +- Type: `string` +- Example: `my-trace-archive` +- Environment variable: `REFINERY_GCS_EXPORT_BUCKET` + +### `KeyPrefix` + +KeyPrefix is the object name prefix under which exported objects are written. + +Object names are formed by appending time-partitioned path elements to this prefix. +Leave empty to write at the root of the bucket. + +- Not eligible for live reload. +- Type: `string` +- Example: `refinery/traces` + +### `FlushInterval` + +FlushInterval is the maximum time a batch of spans is buffered before being written to GCS. + +A batch is flushed to GCS when it reaches `MaxBatchSize` or when this interval has elapsed since the batch was started, whichever comes first. + +- Not eligible for live reload. +- Type: `duration` +- Default: `60s` + +### `MaxBatchSize` + +MaxBatchSize is the maximum uncompressed size of a batch before it is written to GCS. + +When the serialized (uncompressed) size of the current batch exceeds this value, it is compressed and written to GCS immediately. +The objects stored in GCS are gzip-compressed and thus considerably smaller than this value. + +- Not eligible for live reload. +- Type: `memorysize` +- Default: `100MB` + +### `QueueSize` + +QueueSize is the number of spans that can be queued for export before spans are dropped. + +Spans are handed to the exporter through a fixed-size queue so that a slow or unavailable GCS never blocks sending traces to Honeycomb. +If the queue is full, new spans are dropped from the export (they are still sent to Honeycomb) and the `gcs_export_dropped` metric is incremented. + +- Not eligible for live reload. +- Type: `int` +- Default: `100000` + diff --git a/config/cmdenv.go b/config/cmdenv.go index 4daa63a031..63553fd226 100644 --- a/config/cmdenv.go +++ b/config/cmdenv.go @@ -48,6 +48,7 @@ type CmdEnv struct { QueryAuthToken string `long:"query-auth-token" env:"REFINERY_QUERY_AUTH_TOKEN" description:"Token for debug/management queries. Setting this value via a flag may expose credentials - it is recommended to use the env var or a configuration file."` AvailableMemory MemorySize `long:"available-memory" env:"REFINERY_AVAILABLE_MEMORY" description:"The maximum memory available for Refinery to use (ex: 4GiB)."` SendKey string `long:"send-key" env:"REFINERY_SEND_KEY" description:"The Honeycomb API key that Refinery can use to send data to Honeycomb."` + GCSExportBucket string `long:"gcs-export-bucket" env:"REFINERY_GCS_EXPORT_BUCKET" description:"Name of the GCS bucket to export sampled traces to (when GCSExport is enabled)."` Debug bool `short:"d" long:"debug" description:"Runs debug service (on the first open port between localhost:6060 and :6069 by default)"` Version bool `short:"v" long:"version" description:"Print version number and exit"` InterfaceNames bool `long:"interface-names" description:"Print system's network interface names and exit."` diff --git a/config/config.go b/config/config.go index a709cf1b41..a6b60ca28d 100644 --- a/config/config.go +++ b/config/config.go @@ -122,6 +122,9 @@ type Config interface { GetOTelTracingConfig() OTelTracingConfig + // GetGCSExportConfig returns the config specific to GCSExport + GetGCSExportConfig() GCSExportConfig + GetUseIPV6Identifier() bool GetRedisIdentifier() string diff --git a/config/config_serializer.go b/config/config_serializer.go index e02cd1dec9..e0aa54efb8 100644 --- a/config/config_serializer.go +++ b/config/config_serializer.go @@ -75,6 +75,7 @@ func populateConfigContents(cfg Config) configContents { GRPCServerParameters: cfg.GetGRPCConfig(), SampleCache: cfg.GetSampleCacheConfig(), StressRelief: cfg.GetStressReliefConfig(), + GCSExport: cfg.GetGCSExportConfig(), } } diff --git a/config/file_config.go b/config/file_config.go index e2490d8bc7..a79548a105 100644 --- a/config/file_config.go +++ b/config/file_config.go @@ -67,6 +67,7 @@ type configContents struct { GRPCServerParameters GRPCServerParameters `yaml:"GRPCServerParameters"` SampleCache SampleCacheConfig `yaml:"SampleCache"` StressRelief StressReliefConfig `yaml:"StressRelief"` + GCSExport GCSExportConfig `yaml:"GCSExport"` } type GeneralConfig struct { @@ -287,6 +288,17 @@ type OTelMetricsConfig struct { AdditionalAttributes map[string]string `yaml:"AdditionalAttributes" default:"{}" cmdenv:"OTelMetricsAdditionalAttributes"` } +// GCSExportConfig configures the optional export of sampled (kept) trace +// spans to a Google Cloud Storage bucket as gzipped JSON Lines objects. +type GCSExportConfig struct { + Enabled bool `yaml:"Enabled" default:"false"` + Bucket string `yaml:"Bucket" cmdenv:"GCSExportBucket"` + KeyPrefix string `yaml:"KeyPrefix"` + FlushInterval Duration `yaml:"FlushInterval" default:"60s"` + MaxBatchSize MemorySize `yaml:"MaxBatchSize" default:"100MB"` + QueueSize int `yaml:"QueueSize" default:"100000"` +} + type OTelTracingConfig struct { Enabled bool `yaml:"Enabled" default:"false"` APIHost string `yaml:"APIHost" default:"https://api.honeycomb.io" cmdenv:"TelemetryEndpoint"` @@ -1027,6 +1039,13 @@ func (f *fileConfig) GetOTelMetricsConfig() OTelMetricsConfig { return f.mainConfig.OTelMetrics } +func (f *fileConfig) GetGCSExportConfig() GCSExportConfig { + f.mux.RLock() + defer f.mux.RUnlock() + + return f.mainConfig.GCSExport +} + func (f *fileConfig) GetDebugServiceAddr() string { f.mux.RLock() defer f.mux.RUnlock() diff --git a/config/metadata/configMeta.yaml b/config/metadata/configMeta.yaml index 32c4ff1aed..f814352edc 100644 --- a/config/metadata/configMeta.yaml +++ b/config/metadata/configMeta.yaml @@ -2123,3 +2123,91 @@ groups: If this duration is `0`, then Refinery will not start in stressed mode, which will provide faster startup at the possible cost of startup instability. + + - name: GCSExport + title: "GCS Export" + description: > + contains configuration for optionally exporting sampled (kept) trace + spans to a Google Cloud Storage bucket in addition to sending them to + Honeycomb. Spans are buffered in memory and written as gzipped JSON + Lines objects, batched into time-partitioned paths of the form + `KeyPrefix/YYYY/MM/DD/HH/hostname-timestamp.jsonl.gz`. Authentication + uses Google Application Default Credentials. + fields: + - name: Enabled + type: bool + valuetype: nondefault + firstversion: v3.3 + default: false + reload: false + summary: controls whether kept trace spans are also exported to a GCS bucket. + description: > + If `true`, every span belonging to a trace that Refinery decides to + keep is also written to the configured GCS bucket. Spans of dropped + traces are never exported. + + - name: Bucket + type: string + valuetype: nondefault + firstversion: v3.3 + default: "" + example: "my-trace-archive" + envvar: REFINERY_GCS_EXPORT_BUCKET + commandline: gcs-export-bucket + reload: false + summary: is the name of the GCS bucket to which sampled traces are exported. + description: > + The bucket must already exist and the credentials available via + Application Default Credentials must have permission to create + objects in it. Required when `Enabled` is `true`. + + - name: KeyPrefix + type: string + valuetype: nondefault + firstversion: v3.3 + default: "" + example: "refinery/traces" + reload: false + summary: is the object name prefix under which exported objects are written. + description: > + Object names are formed by appending time-partitioned path elements + to this prefix. Leave empty to write at the root of the bucket. + + - name: FlushInterval + type: duration + valuetype: nondefault + firstversion: v3.3 + default: 60s + reload: false + summary: is the maximum time a batch of spans is buffered before being written to GCS. + description: > + A batch is flushed to GCS when it reaches `MaxBatchSize` or when + this interval has elapsed since the batch was started, whichever + comes first. + + - name: MaxBatchSize + type: memorysize + valuetype: memorysize + firstversion: v3.3 + default: 100MB + reload: false + summary: is the maximum uncompressed size of a batch before it is written to GCS. + description: > + When the serialized (uncompressed) size of the current batch exceeds + this value, it is compressed and written to GCS immediately. The + objects stored in GCS are gzip-compressed and thus considerably + smaller than this value. + + - name: QueueSize + type: int + valuetype: nondefault + firstversion: v3.3 + default: 100000 + reload: false + summary: is the number of spans that can be queued for export before spans are dropped. + description: > + Spans are handed to the exporter through a fixed-size queue so that + a slow or unavailable GCS never blocks sending traces to Honeycomb. + If the queue is full, new spans are dropped from the export (they + are still sent to Honeycomb) and the `gcs_export_dropped` metric is + incremented. diff --git a/config/mock.go b/config/mock.go index 58660a281f..eea9933861 100644 --- a/config/mock.go +++ b/config/mock.go @@ -36,6 +36,7 @@ type MockConfig struct { GetOpAmpConfigVal OpAMPConfig GetOTelMetricsConfigVal OTelMetricsConfig GetOTelTracingConfigVal OTelTracingConfig + GetGCSExportConfigVal GCSExportConfig IdentifierInterfaceName string UseIPV6Identifier bool RedisIdentifier string @@ -251,6 +252,13 @@ func (m *MockConfig) GetOTelTracingConfig() OTelTracingConfig { return m.GetOTelTracingConfigVal } +func (m *MockConfig) GetGCSExportConfig() GCSExportConfig { + m.Mux.RLock() + defer m.Mux.RUnlock() + + return m.GetGCSExportConfigVal +} + // GetSamplerConfigForDestName returns the sampler config for the given dataset/environment. // If Samplers map is populated, it will look up the dataset-specific config. // Falls back to GetSamplerTypeVal if Samplers is not set (backwards compatible). diff --git a/config_complete.yaml b/config_complete.yaml index 21e1a24751..0c58cfa1cd 100644 --- a/config_complete.yaml +++ b/config_complete.yaml @@ -2,7 +2,7 @@ ## Honeycomb Refinery Configuration ## ###################################### # -# created on 2026-04-09 at 22:21:32 UTC from ../../config.yaml using a template generated on 2026-04-09 at 22:21:28 UTC +# created on 2026-07-07 at 21:27:51 UTC from ../../config.yaml using a template generated on 2026-07-07 at 21:27:49 UTC # This file contains a configuration for the Honeycomb Refinery. It is in YAML # format, organized into named groups, each of which contains a set of @@ -1431,6 +1431,84 @@ StressRelief: ## Eligible for live reload. # MinimumActivationDuration: 10s +################ +## GCS Export ## +################ +GCSExport: + ## GCSExport contains configuration for optionally exporting sampled + ## (kept) trace spans to a Google Cloud Storage bucket in addition to + ## sending them to Honeycomb. Spans are buffered in memory and written as + ## gzipped JSON Lines objects, batched into time-partitioned paths of the + ## form `KeyPrefix/YYYY/MM/DD/HH/hostname-timestamp.jsonl.gz`. + ## Authentication uses Google Application Default Credentials. + ## + #### + ## Enabled controls whether kept trace spans are also exported to a GCS + ## bucket. + ## + ## If `true`, every span belonging to a trace that Refinery decides to + ## keep is also written to the configured GCS bucket. Spans of dropped + ## traces are never exported. + ## + ## Not eligible for live reload. + # Enabled: false + + ## Bucket is the name of the GCS bucket to which sampled traces are + ## exported. + ## + ## The bucket must already exist and the credentials available via + ## Application Default Credentials must have permission to create objects + ## in it. Required when `Enabled` is `true`. + ## + ## Not eligible for live reload. + # Bucket: "" + + ## KeyPrefix is the object name prefix under which exported objects are + ## written. + ## + ## Object names are formed by appending time-partitioned path elements to + ## this prefix. Leave empty to write at the root of the bucket. + ## + ## Not eligible for live reload. + # KeyPrefix: "" + + ## FlushInterval is the maximum time a batch of spans is buffered before + ## being written to GCS. + ## + ## A batch is flushed to GCS when it reaches `MaxBatchSize` or when this + ## interval has elapsed since the batch was started, whichever comes + ## first. + ## + ## Accepts a duration string with units, like "60s". + ## default: 60s + ## Not eligible for live reload. + # FlushInterval: 60s + + ## MaxBatchSize is the maximum uncompressed size of a batch before it is + ## written to GCS. + ## + ## When the serialized (uncompressed) size of the current batch exceeds + ## this value, it is compressed and written to GCS immediately. The + ## objects stored in GCS are gzip-compressed and thus considerably + ## smaller than this value. + ## + ## default: 100MB + ## Not eligible for live reload. + # MaxBatchSize: "" + + ## QueueSize is the number of spans that can be queued for export before + ## spans are dropped. + ## + ## Spans are handed to the exporter through a fixed-size queue so that a + ## slow or unavailable GCS never blocks sending traces to Honeycomb. If + ## the queue is full, new spans are dropped from the export (they are + ## still sent to Honeycomb) and the `gcs_export_dropped` metric is + ## incremented. + ## + ## default: 100000 + ## Not eligible for live reload. + # QueueSize: 100_000 + ################################################### ## Config values removed by the config converter ## diff --git a/go.mod b/go.mod index eb1c581408..d0c6ae679c 100644 --- a/go.mod +++ b/go.mod @@ -1,11 +1,12 @@ module github.com/honeycombio/refinery -go 1.25.0 +go 1.25.8 require ( + cloud.google.com/go/storage v1.62.3 github.com/agnivade/levenshtein v1.2.1 github.com/creasty/defaults v1.8.0 - github.com/davecgh/go-spew v1.1.1 + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc github.com/dgryski/go-wyhash v0.0.0-20191203203029-c4841ae36371 github.com/facebookgo/inject v0.0.0-20180706035515-f23751cae28b github.com/facebookgo/startstop v0.0.0-20161013234910-bc158412526d @@ -39,14 +40,14 @@ require ( github.com/vmihailenco/msgpack/v5 v5.4.1 go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.68.0 go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.68.0 - go.opentelemetry.io/otel v1.43.0 + go.opentelemetry.io/otel v1.44.0 go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.43.0 go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.43.0 go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.43.0 - go.opentelemetry.io/otel/metric v1.43.0 - go.opentelemetry.io/otel/sdk v1.43.0 - go.opentelemetry.io/otel/sdk/metric v1.43.0 - go.opentelemetry.io/otel/trace v1.43.0 + go.opentelemetry.io/otel/metric v1.44.0 + go.opentelemetry.io/otel/sdk v1.44.0 + go.opentelemetry.io/otel/sdk/metric v1.44.0 + go.opentelemetry.io/otel/trace v1.44.0 go.opentelemetry.io/proto/otlp v1.10.0 golang.org/x/exp v0.0.0-20250531010427-b6e5de432a8b google.golang.org/grpc v1.80.0 @@ -55,10 +56,21 @@ require ( ) require ( + cel.dev/expr v0.25.1 // indirect + cloud.google.com/go v0.123.0 // indirect + cloud.google.com/go/auth v0.20.0 // indirect + cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect + cloud.google.com/go/compute/metadata v0.9.0 // indirect + cloud.google.com/go/iam v1.11.0 // indirect + cloud.google.com/go/monitoring v1.29.0 // indirect dario.cat/mergo v1.0.2 // indirect github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c // indirect + github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.31.0 // indirect + github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.56.0 // indirect + github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.57.0 // indirect github.com/Microsoft/go-winio v0.6.2 // indirect github.com/cenkalti/backoff/v5 v5.0.3 // indirect + github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2 // indirect github.com/containerd/errdefs v1.0.0 // indirect github.com/containerd/errdefs/pkg v0.3.0 // indirect github.com/containerd/log v0.1.0 // indirect @@ -68,10 +80,16 @@ require ( github.com/docker/go-connections v0.6.0 // indirect github.com/docker/go-units v0.5.0 // indirect github.com/ebitengine/purego v0.10.0 // indirect + github.com/envoyproxy/go-control-plane/envoy v1.37.0 // indirect + github.com/envoyproxy/protoc-gen-validate v1.3.3 // indirect + github.com/go-jose/go-jose/v4 v4.1.4 // indirect github.com/go-ole/go-ole v1.2.6 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/google/go-licenses/v2 v2.0.1 // indirect github.com/google/licenseclassifier/v2 v2.0.0 // indirect + github.com/google/s2a-go v0.1.9 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.3.16 // indirect + github.com/googleapis/gax-go/v2 v2.22.0 // indirect github.com/gorilla/websocket v1.5.3 // indirect github.com/hashicorp/go-version v1.9.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect @@ -94,11 +112,13 @@ require ( github.com/opencontainers/image-spec v1.1.1 // indirect github.com/otiai10/copy v1.10.0 // indirect github.com/philhofer/fwd v1.2.0 // indirect + github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect github.com/sergi/go-diff v1.2.0 // indirect github.com/shirou/gopsutil/v4 v4.26.3 // indirect github.com/spf13/cobra v1.7.0 // indirect github.com/spf13/pflag v1.0.5 // indirect + github.com/spiffe/go-spiffe/v2 v2.6.0 // indirect github.com/stretchr/objx v0.5.3 // indirect github.com/testcontainers/testcontainers-go v0.42.0 // indirect github.com/tklauser/go-sysconf v0.3.16 // indirect @@ -108,12 +128,17 @@ require ( go.opentelemetry.io/auto/sdk v1.2.1 // indirect go.opentelemetry.io/collector/featuregate v1.57.0 // indirect go.opentelemetry.io/collector/pdata/xpdata v0.151.0 // indirect + go.opentelemetry.io/contrib/detectors/gcp v1.42.0 // indirect go.opentelemetry.io/proto/otlp/collector/profiles/v1development v0.2.0 // indirect go.opentelemetry.io/proto/otlp/profiles/v1development v0.2.0 // indirect go.uber.org/atomic v1.11.0 // indirect go.yaml.in/yaml/v2 v2.4.2 // indirect - golang.org/x/crypto v0.49.0 // indirect - golang.org/x/sync v0.20.0 // indirect + golang.org/x/crypto v0.53.0 // indirect + golang.org/x/oauth2 v0.36.0 // indirect + golang.org/x/sync v0.21.0 // indirect + golang.org/x/time v0.15.0 // indirect + google.golang.org/api v0.279.0 // indirect + google.golang.org/genproto v0.0.0-20260519071638-aa98bba5eb94 // indirect gopkg.in/alexcesaro/statsd.v2 v2.0.0 // indirect k8s.io/klog/v2 v2.90.1 // indirect ) @@ -135,7 +160,7 @@ require ( github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect - github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/prometheus/client_model v0.6.2 // indirect github.com/prometheus/common v0.66.1 // indirect github.com/prometheus/procfs v0.16.1 // indirect @@ -145,13 +170,13 @@ require ( github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect go.opentelemetry.io/collector/pdata v1.57.0 go.uber.org/multierr v1.11.0 // indirect - golang.org/x/mod v0.35.0 - golang.org/x/net v0.52.0 // indirect - golang.org/x/sys v0.42.0 // indirect - golang.org/x/text v0.35.0 // indirect - golang.org/x/tools v0.43.0 - google.golang.org/genproto/googleapis/api v0.0.0-20260401024825-9d38bb4040a9 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20260406210006-6f92a3bedf2d // indirect + golang.org/x/mod v0.36.0 + golang.org/x/net v0.56.0 // indirect + golang.org/x/sys v0.46.0 // indirect + golang.org/x/text v0.38.0 // indirect + golang.org/x/tools v0.45.0 + google.golang.org/genproto/googleapis/api v0.0.0-20260615183401-62b3387ff324 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260615183401-62b3387ff324 // indirect ) tool github.com/google/go-licenses/v2 diff --git a/go.sum b/go.sum index ebf3dece30..f74dae8fab 100644 --- a/go.sum +++ b/go.sum @@ -1,4 +1,26 @@ +cel.dev/expr v0.25.1 h1:1KrZg61W6TWSxuNZ37Xy49ps13NUovb66QLprthtwi4= +cel.dev/expr v0.25.1/go.mod h1:hrXvqGP6G6gyx8UAHSHJ5RGk//1Oj5nXQ2NI02Nrsg4= cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.123.0 h1:2NAUJwPR47q+E35uaJeYoNhuNEM9kM8SjgRgdeOJUSE= +cloud.google.com/go v0.123.0/go.mod h1:xBoMV08QcqUGuPW65Qfm1o9Y4zKZBpGS+7bImXLTAZU= +cloud.google.com/go/auth v0.20.0 h1:kXTssoVb4azsVDoUiF8KvxAqrsQcQtB53DcSgta74CA= +cloud.google.com/go/auth v0.20.0/go.mod h1:942/yi/itH1SsmpyrbnTMDgGfdy2BUqIKyd0cyYLc5Q= +cloud.google.com/go/auth/oauth2adapt v0.2.8 h1:keo8NaayQZ6wimpNSmW5OPc283g65QNIiLpZnkHRbnc= +cloud.google.com/go/auth/oauth2adapt v0.2.8/go.mod h1:XQ9y31RkqZCcwJWNSx2Xvric3RrU88hAYYbjDWYDL+c= +cloud.google.com/go/compute/metadata v0.9.0 h1:pDUj4QMoPejqq20dK0Pg2N4yG9zIkYGdBtwLoEkH9Zs= +cloud.google.com/go/compute/metadata v0.9.0/go.mod h1:E0bWwX5wTnLPedCKqk3pJmVgCBSM6qQI1yTBdEb3C10= +cloud.google.com/go/iam v1.11.0 h1:KieQ9Pb+LLPak1O3Rv3GgCxhnmkYf7Xyh0P5HfF1jFM= +cloud.google.com/go/iam v1.11.0/go.mod h1:KP+nKGugNJW4LcLx1uEZcq1ok5sQHFaQehQNl4QDgV4= +cloud.google.com/go/logging v1.18.0 h1:KhzZq+1cSkPH9YUaKLLhLtQxIHitVayBmk0sGfoM9+k= +cloud.google.com/go/logging v1.18.0/go.mod h1:ZGKnpBaURITh+g/uom2VhbiFoFWvejcrHPDhxFtU/gI= +cloud.google.com/go/longrunning v1.0.0 h1:lwzWEYD8+NkYV7dhexOz6kmlvajZA70+bW/xMhRVVdY= +cloud.google.com/go/longrunning v1.0.0/go.mod h1:8nqFBPOO1U/XkhWl0I19AMZEphrHi73VNABIpKYaTwM= +cloud.google.com/go/monitoring v1.29.0 h1:AHhDsFaSax1/4k+qlIDX/SDGe6hggnfXJ9dkgD9qBPY= +cloud.google.com/go/monitoring v1.29.0/go.mod h1:72NOVjJXHY/HBfoLT0+qlCZBT059+9VXLeAnL2PeeVM= +cloud.google.com/go/storage v1.62.3 h1:SZq1t23NCI+e96dH77Dg3PEfsNNEjqO8zE5AnD8gVD0= +cloud.google.com/go/storage v1.62.3/go.mod h1:cpYz/kRVZ+UQAF1uHeea10/9ewcRbxGoGNKsS9daSXA= +cloud.google.com/go/trace v1.16.0 h1:GmQovzFc5F0CNfl0VLgL64aoTtu7xsM0YajW2GlG9+E= +cloud.google.com/go/trace v1.16.0/go.mod h1:r+bdAn16dKLSV1G2D5v3e58IlQlizfxWrUfjx7kM7X0= dario.cat/mergo v1.0.2 h1:85+piFYR1tMbRrLcDwR18y4UKJ3aH1Tbzi24VRW1TK8= dario.cat/mergo v1.0.2/go.mod h1:E/hbnu0NxMFBjpMIE34DRGLWqDy0g5FuKDhCb31ngxA= github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6 h1:He8afgbRMd7mFxO99hRNu+6tazq8nFF9lIwo9JFroBk= @@ -8,6 +30,14 @@ github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c/go.mod h1:xomTg6 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/DataDog/zstd v1.5.7 h1:ybO8RBeh29qrxIhCA9E8gKY6xfONU9T6G6aP9DTKfLE= github.com/DataDog/zstd v1.5.7/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.31.0 h1:DHa2U07rk8syqvCge0QIGMCE1WxGj9njT44GH7zNJLQ= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.31.0/go.mod h1:P4WPRUkOhJC13W//jWpyfJNDAIpvRbAUIYLX/4jtlE0= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.56.0 h1:O2sXMyJh8b7devAGdE+163xtRurt0RVpB6DIzX5vGfg= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.56.0/go.mod h1:hEpiGU18xf70qb3jbTcIggWAiEfX/cOIVc2OTe4OegA= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.56.0 h1:ZIT85vKP7LBS84XJ0WdJ3dPOX3iz4j3c0+lpajGQMyo= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.56.0/go.mod h1:rqP9UEhOXv9WhQ7Gjz+G5y/pf8+BJZW5/Ts0AhE0PwE= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.57.0 h1:RoO5+d7uCmDqovLrHCr2/BuViUXvdcrNxyNM1pN9dDQ= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.57.0/go.mod h1:YqwkQPrWSC7+byyc1VlKbWLBF5JsW5IoL6xUkemYSXk= github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= github.com/agnivade/levenshtein v1.2.1 h1:EHBY3UOn1gwdy/VbFwgo4cxecRznFk7fKWN1KOX7eoM= @@ -29,6 +59,8 @@ github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UF github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2 h1:aBangftG7EVZoUb69Os8IaYg++6uMOdKK83QtkkvJik= +github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2/go.mod h1:qwXFYgsP6T7XnJtbKlf1HP8AjxZZyzxMmc+Lq5GjlU4= github.com/containerd/errdefs v1.0.0 h1:tg5yIfIlQIrxYtu9ajqY42W3lpS19XqdxRQeEwYG8PI= github.com/containerd/errdefs v1.0.0/go.mod h1:+YBYIdtsnF4Iw6nWZhJcqGSg/dwvV7tyJ/kCkyJ2k+M= github.com/containerd/errdefs/pkg v0.3.0 h1:9IKJ06FvyNlexW690DXuQNx2KA2cUJXx151Xdx3ZPPE= @@ -45,8 +77,9 @@ github.com/creack/pty v1.1.24/go.mod h1:08sCNb52WyoAwi2QDyzUCTgcvVFhUzewun7wtTfv github.com/creasty/defaults v1.8.0 h1:z27FJxCAa0JKt3utc0sCImAEb+spPucmKoOdLHvHYKk= github.com/creasty/defaults v1.8.0/go.mod h1:iGzKe6pbEHnpMPtfDXZEr0NVxWnPTjb1bbDy08fPzYM= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dgryski/go-metro v0.0.0-20200812162917-85c65e2d0165/go.mod h1:c9O8+fpSOX1DM8cPNSkX/qsBWdkD4yd2dpciOWQjpBw= github.com/dgryski/go-metro v0.0.0-20250106013310-edb8663e5e33 h1:ucRHb6/lvW/+mTEIGbvhcYU3S8+uSNkuMjx/qZFfhtM= github.com/dgryski/go-metro v0.0.0-20250106013310-edb8663e5e33/go.mod h1:c9O8+fpSOX1DM8cPNSkX/qsBWdkD4yd2dpciOWQjpBw= @@ -65,7 +98,15 @@ github.com/ebitengine/purego v0.10.0/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLA github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/go-control-plane v0.14.0 h1:hbG2kr4RuFj222B6+7T83thSPqLjwBIfQawTkC++2HA= +github.com/envoyproxy/go-control-plane v0.14.0/go.mod h1:NcS5X47pLl/hfqxU70yPwL9ZMkUlwlKxtAohpi2wBEU= +github.com/envoyproxy/go-control-plane/envoy v1.37.0 h1:u3riX6BoYRfF4Dr7dwSOroNfdSbEPe9Yyl09/B6wBrQ= +github.com/envoyproxy/go-control-plane/envoy v1.37.0/go.mod h1:DReE9MMrmecPy+YvQOAOHNYMALuowAnbjjEMkkWOi6A= +github.com/envoyproxy/go-control-plane/ratelimit v0.1.0 h1:/G9QYbddjL25KvtKTv3an9lx6VBE2cnb8wp1vEGNYGI= +github.com/envoyproxy/go-control-plane/ratelimit v0.1.0/go.mod h1:Wk+tMFAFbCXaJPzVVHnPgRKdUdwW/KdbRt94AzgRee4= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/envoyproxy/protoc-gen-validate v1.3.3 h1:MVQghNeW+LZcmXe7SY1V36Z+WFMDjpqGAGacLe2T0ds= +github.com/envoyproxy/protoc-gen-validate v1.3.3/go.mod h1:TsndJ/ngyIdQRhMcVVGDDHINPLWB7C82oDArY51KfB0= github.com/facebookgo/clock v0.0.0-20150410010913-600d898af40a h1:yDWHCSQ40h88yih2JAcL6Ls/kVkSE8GFACTGVnMPruw= github.com/facebookgo/clock v0.0.0-20150410010913-600d898af40a/go.mod h1:7Ga40egUymuWXxAe151lTNnCv97MddSOVsjpPPkityA= github.com/facebookgo/ensure v0.0.0-20200202191622-63f1cf65ac4c h1:8ISkoahWXwZR41ois5lSJBSVw4D0OV19Ht/JSTzvSv0= @@ -86,6 +127,8 @@ github.com/facebookgo/subset v0.0.0-20200203212716-c811ad88dec4 h1:7HZCaLC5+BZpm github.com/facebookgo/subset v0.0.0-20200203212716-c811ad88dec4/go.mod h1:5tD+neXqOorC30/tWg0LCSkrqj/AR6gu8yY8/fpw1q0= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/go-jose/go-jose/v4 v4.1.4 h1:moDMcTHmvE6Groj34emNPLs/qtYXRVcd6S7NHbHz3kA= +github.com/go-jose/go-jose/v4 v4.1.4/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08= github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= @@ -130,11 +173,17 @@ github.com/google/go-replayers/httpreplay v1.2.0/go.mod h1:WahEFFZZ7a1P4VM1qEeHy github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/licenseclassifier/v2 v2.0.0 h1:1Y57HHILNf4m0ABuMVb6xk4vAJYEUO0gDxNpog0pyeA= github.com/google/licenseclassifier/v2 v2.0.0/go.mod h1:cOjbdH0kyC9R22sdQbYsFkto4NGCAc+ZSwbeThazEtM= -github.com/google/martian/v3 v3.3.2 h1:IqNFLAmvJOgVlpdEBiQbDc2EwKW77amAycfTuWKdfvw= -github.com/google/martian/v3 v3.3.2/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk= +github.com/google/martian/v3 v3.3.3 h1:DIhPTQrbPkgs2yJYdXU/eNACCG5DVQjySNRNlflZ9Fc= +github.com/google/martian/v3 v3.3.3/go.mod h1:iEPrYcgCF7jA9OtScMFQyAlZZ4YXTKEtJ1E6RWzmBA0= +github.com/google/s2a-go v0.1.9 h1:LGD7gtMgezd8a/Xak7mEWL0PjoTQFvpRudN895yqKW0= +github.com/google/s2a-go v0.1.9/go.mod h1:YA0Ei2ZQL3acow2O62kdp9UlnvMmU7kA6Eutn0dXayM= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/enterprise-certificate-proxy v0.3.16 h1:F/VPrx0YPBdksZJQdCAp0WUsqnNmZpUZszzfYt0M5Dw= +github.com/googleapis/enterprise-certificate-proxy v0.3.16/go.mod h1:9Yb0eAkH/Xqhvv3zbeKf/+wMJqCeocWc6KIhDvEAuYE= +github.com/googleapis/gax-go/v2 v2.22.0 h1:PjIWBpgGIVKGoCXuiCoP64altEJCj3/Ei+kSU5vlZD4= +github.com/googleapis/gax-go/v2 v2.22.0/go.mod h1:irWBbALSr0Sk3qlqb9SyJ1h68WjgeFuiOzI4Rqw5+aY= github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY= github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ= github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg= @@ -236,8 +285,11 @@ github.com/philhofer/fwd v1.2.0 h1:e6DnBTl7vGY+Gz322/ASL4Gyp1FspeMvx1RNDoToZuM= github.com/philhofer/fwd v1.2.0/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 h1:GFCKgmp0tecUJ0sJuv4pzYCqS9+RGSn52M3FUwPs+uo= +github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 h1:o4JXh1EVt9k/+g42oCprj/FisM4qX9L3sZB3upGN2ZU= github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o= @@ -271,6 +323,8 @@ github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I= github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spiffe/go-spiffe/v2 v2.6.0 h1:l+DolpxNWYgruGQVV0xsfeya3CsC7m8iBzDnMpsbLuo= +github.com/spiffe/go-spiffe/v2 v2.6.0/go.mod h1:gm2SeUoMZEtpnzPNs2Csc0D/gX33k1xIx7lEzqblHEs= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= @@ -324,26 +378,32 @@ go.opentelemetry.io/collector/pdata/pprofile v0.151.0 h1:hsU0+DpkvhJh3xL1Y8CX2vA go.opentelemetry.io/collector/pdata/pprofile v0.151.0/go.mod h1:5zfGTQqRuaKyh2SRaZi4SV4nSD8TzY1kYoOjniOD3uk= go.opentelemetry.io/collector/pdata/xpdata v0.151.0 h1:trsLPS6jCkwVwJyKxbPqQerAiMpKkQrQLEGIEcyC6yM= go.opentelemetry.io/collector/pdata/xpdata v0.151.0/go.mod h1:0vID3D52DGVoypLa8S7izv41ElTBEgtAbc0HmB4KF60= +go.opentelemetry.io/contrib/detectors/gcp v1.42.0 h1:kpt2PEJuOuqYkPcktfJqWWDjTEd/FNgrxcniL7kQrXQ= +go.opentelemetry.io/contrib/detectors/gcp v1.42.0/go.mod h1:W9zQ439utxymRrXsUOzZbFX4JhLxXU4+ZnCt8GG7yA8= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.68.0 h1:0Qx7VGBacMm9ZENQ7TnNObTYI4ShC+lHI16seduaxZo= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.68.0/go.mod h1:Sje3i3MjSPKTSPvVWCaL8ugBzJwik3u4smCjUeuupqg= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.68.0 h1:CqXxU8VOmDefoh0+ztfGaymYbhdB/tT3zs79QaZTNGY= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.68.0/go.mod h1:BuhAPThV8PBHBvg8ZzZ/Ok3idOdhWIodywz2xEcRbJo= -go.opentelemetry.io/otel v1.43.0 h1:mYIM03dnh5zfN7HautFE4ieIig9amkNANT+xcVxAj9I= -go.opentelemetry.io/otel v1.43.0/go.mod h1:JuG+u74mvjvcm8vj8pI5XiHy1zDeoCS2LB1spIq7Ay0= +go.opentelemetry.io/otel v1.44.0 h1:JjwHmHpA4iZ3wBxluu2fbbE7j4kqlE8jXyAyPXH7HqU= +go.opentelemetry.io/otel v1.44.0/go.mod h1:BMgjTHL9WPRlRjL2oZCBTL4whCGtXch2H4BhOPIAyYc= go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.43.0 h1:w1K+pCJoPpQifuVpsKamUdn9U0zM3xUziVOqsGksUrY= go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.43.0/go.mod h1:HBy4BjzgVE8139ieRI75oXm3EcDN+6GhD88JT1Kjvxg= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.43.0 h1:88Y4s2C8oTui1LGM6bTWkw0ICGcOLCAI5l6zsD1j20k= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.43.0/go.mod h1:Vl1/iaggsuRlrHf/hfPJPvVag77kKyvrLeD10kpMl+A= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.43.0 h1:3iZJKlCZufyRzPzlQhUIWVmfltrXuGyfjREgGP3UUjc= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.43.0/go.mod h1:/G+nUPfhq2e+qiXMGxMwumDrP5jtzU+mWN7/sjT2rak= -go.opentelemetry.io/otel/metric v1.43.0 h1:d7638QeInOnuwOONPp4JAOGfbCEpYb+K6DVWvdxGzgM= -go.opentelemetry.io/otel/metric v1.43.0/go.mod h1:RDnPtIxvqlgO8GRW18W6Z/4P462ldprJtfxHxyKd2PY= -go.opentelemetry.io/otel/sdk v1.43.0 h1:pi5mE86i5rTeLXqoF/hhiBtUNcrAGHLKQdhg4h4V9Dg= -go.opentelemetry.io/otel/sdk v1.43.0/go.mod h1:P+IkVU3iWukmiit/Yf9AWvpyRDlUeBaRg6Y+C58QHzg= -go.opentelemetry.io/otel/sdk/metric v1.43.0 h1:S88dyqXjJkuBNLeMcVPRFXpRw2fuwdvfCGLEo89fDkw= -go.opentelemetry.io/otel/sdk/metric v1.43.0/go.mod h1:C/RJtwSEJ5hzTiUz5pXF1kILHStzb9zFlIEe85bhj6A= -go.opentelemetry.io/otel/trace v1.43.0 h1:BkNrHpup+4k4w+ZZ86CZoHHEkohws8AY+WTX09nk+3A= -go.opentelemetry.io/otel/trace v1.43.0/go.mod h1:/QJhyVBUUswCphDVxq+8mld+AvhXZLhe+8WVFxiFff0= +go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.43.0 h1:TC+BewnDpeiAmcscXbGMfxkO+mwYUwE/VySwvw88PfA= +go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.43.0/go.mod h1:J/ZyF4vfPwsSr9xJSPyQ4LqtcTPULFR64KwTikGLe+A= +go.opentelemetry.io/otel/metric v1.44.0 h1:1w0gILTcHdr3YI+ixLyjemwrVnsMURbTZFrSYCdDdmc= +go.opentelemetry.io/otel/metric v1.44.0/go.mod h1:8O7hanEPBNgEMmybD3s2VBKcgWOCsA6tzHBPODAiquo= +go.opentelemetry.io/otel/metric/x v0.66.0 h1:YkCrx1zLOChi9ZcZ6euupOcsgzbVlec7D/xoEU1+cTA= +go.opentelemetry.io/otel/metric/x v0.66.0/go.mod h1:d1+BDj9t96do0/1LoU1ayfCv79ZgNE41qbhBvnMOBZk= +go.opentelemetry.io/otel/sdk v1.44.0 h1:nHYwb9lK+fJPU/dnT6s7W7Z8itMWyqrnVfbheVYrZ58= +go.opentelemetry.io/otel/sdk v1.44.0/go.mod h1:Osuydd3Se74nqjAKxid74N5eC+jfEqfTegHRnq58oK0= +go.opentelemetry.io/otel/sdk/metric v1.44.0 h1:3LlKgI+VjbVsjNRFZJZAJ30WjXC5VkNRks6si09iEfI= +go.opentelemetry.io/otel/sdk/metric v1.44.0/go.mod h1:5B5pMARnXxKhltooO4xUuCBorl65a4EpnTalObqOigA= +go.opentelemetry.io/otel/trace v1.44.0 h1:jxF5CsGYCe74MCRx2X4g7WsY/VBKRqqpNvXlX/6gtIk= +go.opentelemetry.io/otel/trace v1.44.0/go.mod h1:oLl1jrMQAVo6v3GAggN+1VH9VIz9iUSvW53sW1Q8PIE= go.opentelemetry.io/proto/otlp v1.10.0 h1:IQRWgT5srOCYfiWnpqUYz9CVmbO8bFmKcwYxpuCSL2g= go.opentelemetry.io/proto/otlp v1.10.0/go.mod h1:/CV4QoCR/S9yaPj8utp3lvQPoqMtxXdzn7ozvvozVqk= go.opentelemetry.io/proto/otlp/collector/profiles/v1development v0.2.0 h1:40vBjolEOioNBl8zPj1wxqlA7kJ82RxR4HnUv7W8zRI= @@ -366,30 +426,32 @@ go.yaml.in/yaml/v2 v2.4.2 h1:DzmwEr2rDGHl7lsFgAHxmNz/1NlQ7xLIrlN2h5d1eGI= go.yaml.in/yaml/v2 v2.4.2/go.mod h1:081UH+NErpNdqlCXm3TtEran0rJZGxAYx9hb/ELlsPU= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.49.0 h1:+Ng2ULVvLHnJ/ZFEq4KdcDd/cfjrrjjNSXNzxg0Y4U4= -golang.org/x/crypto v0.49.0/go.mod h1:ErX4dUh2UM+CFYiXZRTcMpEcN8b/1gxEuv3nODoYtCA= +golang.org/x/crypto v0.53.0 h1:QZ4Muo8THX6CizN2vPPd5fBGHyogrdK9fG4wLPFUsto= +golang.org/x/crypto v0.53.0/go.mod h1:DNLU434OwVakk9PzuwV8w62mAJpRJL3vsgcfp4Qnsio= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20250531010427-b6e5de432a8b h1:QoALfVG9rhQ/M7vYDScfPdWjGL9dlsVVM5VGh7aKoAA= golang.org/x/exp v0.0.0-20250531010427-b6e5de432a8b/go.mod h1:U6Lno4MTRCDY+Ba7aCcauB9T60gsv5s4ralQzP72ZoQ= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/mod v0.35.0 h1:Ww1D637e6Pg+Zb2KrWfHQUnH2dQRLBQyAtpr/haaJeM= -golang.org/x/mod v0.35.0/go.mod h1:+GwiRhIInF8wPm+4AoT6L0FA1QWAad3OMdTRx4tFYlU= +golang.org/x/mod v0.36.0 h1:JJjpVx6myfUsUdAzZuOSTTmRE0PfZeNWzzvKrP7amb4= +golang.org/x/mod v0.36.0/go.mod h1:moc6ELqsWcOw5Ef3xVprK5ul/MvtVvkIXLziUOICjUQ= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.52.0 h1:He/TN1l0e4mmR3QqHMT2Xab3Aj3L9qjbhRm78/6jrW0= -golang.org/x/net v0.52.0/go.mod h1:R1MAz7uMZxVMualyPXb+VaqGSa3LIaUqk0eEt3w36Sw= +golang.org/x/net v0.56.0 h1:Rw8j/hFzGvJUZwNBXnAtf5sVDVt+65SK2C7IxCxZt5o= +golang.org/x/net v0.56.0/go.mod h1:D3Ku6r+V6JROoZK144D2XfMHFcMq/0zSfLelVTCFKec= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs= +golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= -golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/sync v0.21.0 h1:HLII4xRRTtCRkxYp4HNFF0Js/Og6q2i++KXbg0gHCwM= +golang.org/x/sync v0.21.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -397,33 +459,39 @@ golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.42.0 h1:omrd2nAlyT5ESRdCLYdm3+fMfNFE/+Rf4bDIQImRJeo= -golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= -golang.org/x/term v0.41.0 h1:QCgPso/Q3RTJx2Th4bDLqML4W6iJiaXFq2/ftQF13YU= -golang.org/x/term v0.41.0/go.mod h1:3pfBgksrReYfZ5lvYM0kSO0LIkAl4Yl2bXOkKP7Ec2A= +golang.org/x/sys v0.46.0 h1:noSf2Fq6F8DBgS+LysIkx7rIExoNHJsxOAtPp4rthXw= +golang.org/x/sys v0.46.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/term v0.44.0 h1:0rLvDRCtNj0gZkyIXhCyOb2OAzEhLVqc4B+hrsBhrmc= +golang.org/x/term v0.44.0/go.mod h1:7ze4MdzUzLXpSAoFP1H0bOI9aXDqveSvatT5vKcFh2Y= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.35.0 h1:JOVx6vVDFokkpaq1AEptVzLTpDe9KGpj5tR4/X+ybL8= -golang.org/x/text v0.35.0/go.mod h1:khi/HExzZJ2pGnjenulevKNX1W67CUy0AsXcNubPGCA= +golang.org/x/text v0.38.0 h1:sXmwo9DwP3OK9EZ7PqAdaooSGozfl/3a6/xJcbzPRhE= +golang.org/x/text v0.38.0/go.mod h1:YXZt3QhHUKYT53r2lLKFIVi6Ao1jdzrTR/KQ09qyxF4= +golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= +golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.43.0 h1:12BdW9CeB3Z+J/I/wj34VMl8X+fEXBxVR90JeMX5E7s= -golang.org/x/tools v0.43.0/go.mod h1:uHkMso649BX2cZK6+RpuIPXS3ho2hZo4FVwfoy1vIk0= +golang.org/x/tools v0.45.0 h1:18qN3FAooORvApf5XjCXgsuayZOEtXf6JK18I3+ONa8= +golang.org/x/tools v0.45.0/go.mod h1:LuUGqqaXcXMEFEruIVJVm5mgDD8vww/z/SR1gQ4uE/0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= +google.golang.org/api v0.279.0 h1:hsx2M2OaRcaKtVYK6vXEUnQvdjnend7ZYES+lYaot74= +google.golang.org/api v0.279.0/go.mod h1:B9TqLBwJqVjp1mtt7WeoQwWRwvu/400y5lETOql+giQ= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= -google.golang.org/genproto/googleapis/api v0.0.0-20260401024825-9d38bb4040a9 h1:VPWxll4HlMw1Vs/qXtN7BvhZqsS9cdAittCNvVENElA= -google.golang.org/genproto/googleapis/api v0.0.0-20260401024825-9d38bb4040a9/go.mod h1:7QBABkRtR8z+TEnmXTqIqwJLlzrZKVfAUm7tY3yGv0M= -google.golang.org/genproto/googleapis/rpc v0.0.0-20260406210006-6f92a3bedf2d h1:wT2n40TBqFY6wiwazVK9/iTWbsQrgk5ZfCSVFLO9LQA= -google.golang.org/genproto/googleapis/rpc v0.0.0-20260406210006-6f92a3bedf2d/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= +google.golang.org/genproto v0.0.0-20260519071638-aa98bba5eb94 h1:YJjbgu+dkp5kUJLfpMyCLfBIWZb/FcJyuLeo1gVBOuo= +google.golang.org/genproto v0.0.0-20260519071638-aa98bba5eb94/go.mod h1:RRHjglSYABVCWpQ7USCpdfhcd9t4PkajvVwyynZizTc= +google.golang.org/genproto/googleapis/api v0.0.0-20260615183401-62b3387ff324 h1:g0RAkxK/smSu/iRwC/KIX1mwUoVJtk2OjbgaeS4DmUM= +google.golang.org/genproto/googleapis/api v0.0.0-20260615183401-62b3387ff324/go.mod h1:Z4WJ5pJOYWFWcHEQUelD5QaZDknIQkpIL/+fyJOT9+A= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260615183401-62b3387ff324 h1:9HZDLIdYBJXAnaFOr9WHrKVycfpY+75s9HGadC0305A= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260615183401-62b3387ff324/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= diff --git a/metrics.md b/metrics.md index 2b39f3f9f8..a6c8b2f275 100644 --- a/metrics.md +++ b/metrics.md @@ -3,7 +3,7 @@ # Honeycomb Refinery Metrics Documentation This document contains the description of various metrics used in Refinery. -It was automatically generated on 2026-04-09 at 22:21:31 UTC. +It was automatically generated on 2026-07-07 at 21:27:50 UTC. Note: This document does not include metrics defined in the dynsampler-go dependency, as those metrics are generated dynamically at runtime. As a result, certain metrics may be missing or incomplete in this document, but they will still be available during execution with their full names. @@ -81,6 +81,7 @@ Metrics in this table don't contain their expected prefixes. This is because the | Name | Type | Unit | Description | |------|------|------|-------------| +| unique_dynsampler_count | Gauge | Dimensionless | Number of unique dynsampler-go samplers created | | _num_dropped | Counter | Dimensionless | Number of traces dropped by configured sampler | | _num_kept | Counter | Dimensionless | Number of traces kept by configured sampler | | _sample_rate | Histogram | Dimensionless | Sample rate for traces | @@ -98,6 +99,12 @@ Metrics in this table don't contain their expected prefixes. This is because the | _messages_sent | Counter | Dimensionless | number of messages sent to destination | | _response_decode_errors | Counter | Dimensionless | number of errors encountered while decoding responses from destination | | _stale_dispatch_time | Histogram | Microseconds | The time spent per iteration of the stale batch dispatch loop | +| gcs_export_events | Counter | Dimensionless | number of events written to the GCS export | +| gcs_export_dropped | Counter | Dimensionless | number of events dropped from the GCS export because the queue was full | +| gcs_export_batches | Counter | Dimensionless | number of batch objects written to GCS | +| gcs_export_errors | Counter | Dimensionless | number of errors encountered while writing batches to GCS | +| gcs_export_batch_bytes | Histogram | Bytes | compressed size of batch objects written to GCS | +| gcs_export_queue_length | Gauge | Dimensionless | number of events waiting to be batched for the GCS export | | _router_proxied | Counter | Dimensionless | the number of events proxied to another refinery | | _router_event | Counter | Dimensionless | the number of events received | | _router_event_bytes | Histogram | Bytes | the number of bytes per event received | diff --git a/refinery_config.md b/refinery_config.md index 47d67bd338..87b0e1329c 100644 --- a/refinery_config.md +++ b/refinery_config.md @@ -1302,3 +1302,74 @@ This setting helps to prevent oscillations. - Type: `duration` - Default: `10s` +## GCS Export + +`GCSExport` contains configuration for optionally exporting sampled (kept) trace spans to a Google Cloud Storage bucket in addition to sending them to Honeycomb. +Spans are buffered in memory and written as gzipped JSON Lines objects, batched into time-partitioned paths of the form `KeyPrefix/YYYY/MM/DD/HH/hostname-timestamp.jsonl.gz`. +Authentication uses Google Application Default Credentials. + +### `Enabled` + +`Enabled` controls whether kept trace spans are also exported to a GCS bucket. + +If `true`, every span belonging to a trace that Refinery decides to keep is also written to the configured GCS bucket. +Spans of dropped traces are never exported. + +- Not eligible for live reload. +- Type: `bool` + +### `Bucket` + +`Bucket` is the name of the GCS bucket to which sampled traces are exported. + +The bucket must already exist and the credentials available via Application Default Credentials must have permission to create objects in it. +Required when `Enabled` is `true`. + +- Not eligible for live reload. +- Type: `string` +- Example: `my-trace-archive` +- Environment variable: `REFINERY_GCS_EXPORT_BUCKET` + +### `KeyPrefix` + +`KeyPrefix` is the object name prefix under which exported objects are written. + +Object names are formed by appending time-partitioned path elements to this prefix. +Leave empty to write at the root of the bucket. + +- Not eligible for live reload. +- Type: `string` +- Example: `refinery/traces` + +### `FlushInterval` + +`FlushInterval` is the maximum time a batch of spans is buffered before being written to GCS. + +A batch is flushed to GCS when it reaches `MaxBatchSize` or when this interval has elapsed since the batch was started, whichever comes first. + +- Not eligible for live reload. +- Type: `duration` +- Default: `60s` + +### `MaxBatchSize` + +`MaxBatchSize` is the maximum uncompressed size of a batch before it is written to GCS. + +When the serialized (uncompressed) size of the current batch exceeds this value, it is compressed and written to GCS immediately. +The objects stored in GCS are gzip-compressed and thus considerably smaller than this value. + +- Not eligible for live reload. +- Type: `memorysize` +- Default: `100MB` + +### `QueueSize` + +`QueueSize` is the number of spans that can be queued for export before spans are dropped. + +Spans are handed to the exporter through a fixed-size queue so that a slow or unavailable GCS never blocks sending traces to Honeycomb. +If the queue is full, new spans are dropped from the export (they are still sent to Honeycomb) and the `gcs_export_dropped` metric is incremented. + +- Not eligible for live reload. +- Type: `int` +- Default: `100000` + diff --git a/route/route_test.go b/route/route_test.go index 1e47db3beb..fb0f446bba 100644 --- a/route/route_test.go +++ b/route/route_test.go @@ -722,6 +722,7 @@ func TestDependencyInjection(t *testing.T) { &inject.Object{Value: http.DefaultTransport, Name: "upstreamTransport"}, &inject.Object{Value: &transmit.MockTransmission{}, Name: "upstreamTransmission"}, &inject.Object{Value: &transmit.MockTransmission{}, Name: "peerTransmission"}, + &inject.Object{Value: &transmit.NoopTransmission{}, Name: "gcsExport"}, &inject.Object{Value: &pubsub.LocalPubSub{}}, &inject.Object{Value: &sharder.MockSharder{}}, &inject.Object{Value: &collect.InMemCollector{}}, diff --git a/rules.md b/rules.md index a401c882d1..5d22e46b57 100644 --- a/rules.md +++ b/rules.md @@ -3,7 +3,7 @@ # Honeycomb Refinery Rules Documentation This is the documentation for the rules configuration for Honeycomb's Refinery. -It was automatically generated on 2026-04-09 at 22:21:32 UTC. +It was automatically generated on 2026-07-07 at 21:27:51 UTC. ## The Rules file @@ -55,6 +55,7 @@ The remainder of this document describes the samplers that can be used within th - [Rules for Rules-based Samplers](#rules-for-rules-based-samplers) - [Conditions for the Rules in Rules-based Samplers](#conditions-for-the-rules-in-rules-based-samplers) - [Total Throughput Sampler](#total-throughput-sampler) +- [Custom Span Count Configuration](#custom-span-count-configuration) --- ## Deterministic Sampler @@ -715,3 +716,30 @@ If your traces are consistent lengths and changes in trace length is a useful in Type: `bool` +--- +## Custom Span Count Configuration + +### Name: `SpanCounters` + +Defines a single custom span counter. +Each counter has a Key that names the field written to the root span, and an optional list of Conditions that must all match for a span to be counted. +Spans are counted when all of the entry's Conditions match. +If Conditions is empty, every span in the trace is counted. +The counter value is written to the root span under the key specified by `Key`. +If no root span exists when the trace is sent, the counter is written to the first non-annotation span instead. + +### `Key` + +The name of the field that will be added to the root span. +Must not be empty. + +Type: `string` + +### `Conditions` + +All conditions must match for a span to be counted. +If empty, every span in the trace is counted. +Uses the same condition format as rules-based sampler conditions. + +Type: `objectarray` + diff --git a/tools/convert/configDataNames.txt b/tools/convert/configDataNames.txt index 06cdaaa88e..171e207fee 100644 --- a/tools/convert/configDataNames.txt +++ b/tools/convert/configDataNames.txt @@ -1,5 +1,5 @@ # Names of groups and fields in the new config file format. -# Automatically generated on 2026-04-09 at 22:21:29 UTC. +# Automatically generated on 2026-07-07 at 21:27:50 UTC. General: - ConfigurationVersion @@ -294,3 +294,17 @@ StressRelief: - MinimumStartupDuration (originally StressRelief.MinimumStartupDuration) + +GCSExport: + - Enabled + + - Bucket + + - KeyPrefix + + - FlushInterval + + - MaxBatchSize + + - QueueSize + diff --git a/tools/convert/metricsMeta.yaml b/tools/convert/metricsMeta.yaml index 032c6d9025..0d3271f139 100644 --- a/tools/convert/metricsMeta.yaml +++ b/tools/convert/metricsMeta.yaml @@ -244,6 +244,10 @@ complete: unit: Dimensionless description: The hash of the current rules configuration hasprefix: + - name: unique_dynsampler_count + type: Gauge + unit: Dimensionless + description: Number of unique dynsampler-go samplers created - name: _num_dropped type: Counter unit: Dimensionless @@ -312,6 +316,30 @@ hasprefix: type: Histogram unit: Microseconds description: The time spent per iteration of the stale batch dispatch loop + - name: gcs_export_events + type: Counter + unit: Dimensionless + description: number of events written to the GCS export + - name: gcs_export_dropped + type: Counter + unit: Dimensionless + description: number of events dropped from the GCS export because the queue was full + - name: gcs_export_batches + type: Counter + unit: Dimensionless + description: number of batch objects written to GCS + - name: gcs_export_errors + type: Counter + unit: Dimensionless + description: number of errors encountered while writing batches to GCS + - name: gcs_export_batch_bytes + type: Histogram + unit: Bytes + description: compressed size of batch objects written to GCS + - name: gcs_export_queue_length + type: Gauge + unit: Dimensionless + description: number of events waiting to be batched for the GCS export - name: _router_proxied type: Counter unit: Dimensionless diff --git a/tools/convert/minimal_config.yaml b/tools/convert/minimal_config.yaml index db18ce37f7..3d29c02364 100644 --- a/tools/convert/minimal_config.yaml +++ b/tools/convert/minimal_config.yaml @@ -1,5 +1,5 @@ # sample uncommented config file containing all possible fields -# automatically generated on 2026-04-09 at 22:21:30 UTC +# automatically generated on 2026-07-07 at 21:27:50 UTC General: ConfigurationVersion: 2 MinRefineryVersion: "v2.0" @@ -147,4 +147,11 @@ StressRelief: ActivationLevel: 90 DeactivationLevel: 75 SamplingRate: 100 - MinimumActivationDuration: 10s \ No newline at end of file + MinimumActivationDuration: 10s +GCSExport: + Enabled: false + Bucket: "my-trace-archive" + KeyPrefix: "refinery/traces" + FlushInterval: 60s + MaxBatchSize: 100MB + QueueSize: 100_000 \ No newline at end of file diff --git a/tools/convert/templates/configV2.tmpl b/tools/convert/templates/configV2.tmpl index 4546348b92..fca415e1a4 100644 --- a/tools/convert/templates/configV2.tmpl +++ b/tools/convert/templates/configV2.tmpl @@ -2,7 +2,7 @@ ## Honeycomb Refinery Configuration ## ###################################### # -# created {{ now }} from {{ .Input }} using a template generated on 2026-04-09 at 22:21:28 UTC +# created {{ now }} from {{ .Input }} using a template generated on 2026-07-07 at 21:27:49 UTC # This file contains a configuration for the Honeycomb Refinery. It is in YAML # format, organized into named groups, each of which contains a set of @@ -1414,6 +1414,84 @@ StressRelief: ## Eligible for live reload. {{ nonDefaultOnly .Data "MinimumActivationDuration" "StressRelief.MinimumActivationDuration" "10s" }} +################ +## GCS Export ## +################ +GCSExport: + ## GCSExport contains configuration for optionally exporting sampled + ## (kept) trace spans to a Google Cloud Storage bucket in addition to + ## sending them to Honeycomb. Spans are buffered in memory and written as + ## gzipped JSON Lines objects, batched into time-partitioned paths of the + ## form `KeyPrefix/YYYY/MM/DD/HH/hostname-timestamp.jsonl.gz`. + ## Authentication uses Google Application Default Credentials. + ## + #### + ## Enabled controls whether kept trace spans are also exported to a GCS + ## bucket. + ## + ## If `true`, every span belonging to a trace that Refinery decides to + ## keep is also written to the configured GCS bucket. Spans of dropped + ## traces are never exported. + ## + ## Not eligible for live reload. + {{ nonDefaultOnly .Data "Enabled" "Enabled" false }} + + ## Bucket is the name of the GCS bucket to which sampled traces are + ## exported. + ## + ## The bucket must already exist and the credentials available via + ## Application Default Credentials must have permission to create objects + ## in it. Required when `Enabled` is `true`. + ## + ## Not eligible for live reload. + {{ nonDefaultOnly .Data "Bucket" "Bucket" "" }} + + ## KeyPrefix is the object name prefix under which exported objects are + ## written. + ## + ## Object names are formed by appending time-partitioned path elements to + ## this prefix. Leave empty to write at the root of the bucket. + ## + ## Not eligible for live reload. + {{ nonDefaultOnly .Data "KeyPrefix" "KeyPrefix" "" }} + + ## FlushInterval is the maximum time a batch of spans is buffered before + ## being written to GCS. + ## + ## A batch is flushed to GCS when it reaches `MaxBatchSize` or when this + ## interval has elapsed since the batch was started, whichever comes + ## first. + ## + ## Accepts a duration string with units, like "60s". + ## default: 60s + ## Not eligible for live reload. + {{ nonDefaultOnly .Data "FlushInterval" "FlushInterval" "60s" }} + + ## MaxBatchSize is the maximum uncompressed size of a batch before it is + ## written to GCS. + ## + ## When the serialized (uncompressed) size of the current batch exceeds + ## this value, it is compressed and written to GCS immediately. The + ## objects stored in GCS are gzip-compressed and thus considerably + ## smaller than this value. + ## + ## default: 100MB + ## Not eligible for live reload. + {{ memorysize .Data "MaxBatchSize" "MaxBatchSize" "" }} + + ## QueueSize is the number of spans that can be queued for export before + ## spans are dropped. + ## + ## Spans are handed to the exporter through a fixed-size queue so that a + ## slow or unavailable GCS never blocks sending traces to Honeycomb. If + ## the queue is full, new spans are dropped from the export (they are + ## still sent to Honeycomb) and the `gcs_export_dropped` metric is + ## incremented. + ## + ## default: 100000 + ## Not eligible for live reload. + {{ nonDefaultOnly .Data "QueueSize" "QueueSize" 100000 }} + ################################################### ## Config values removed by the config converter ## diff --git a/transmit/gcs_transmit.go b/transmit/gcs_transmit.go new file mode 100644 index 0000000000..f354635bc8 --- /dev/null +++ b/transmit/gcs_transmit.go @@ -0,0 +1,307 @@ +package transmit + +import ( + "bytes" + "compress/gzip" + "context" + "encoding/json" + "fmt" + "os" + "strings" + "sync" + "time" + + "cloud.google.com/go/storage" + "github.com/jonboulle/clockwork" + "github.com/sourcegraph/conc/pool" + + "github.com/honeycombio/refinery/config" + "github.com/honeycombio/refinery/logger" + "github.com/honeycombio/refinery/metrics" + "github.com/honeycombio/refinery/types" +) + +const ( + counterGCSExportEvents = "gcs_export_events" + counterGCSExportDropped = "gcs_export_dropped" + counterGCSExportBatches = "gcs_export_batches" + counterGCSExportErrors = "gcs_export_errors" + histogramGCSExportBatchBytes = "gcs_export_batch_bytes" + gaugeGCSExportQueueLength = "gcs_export_queue_length" + + // number of batch uploads that may be in flight at once; when they're all + // busy the flusher blocks, the queue fills, and new spans are dropped from + // the export rather than blocking the collector. + maxConcurrentGCSUploads = 4 + + // how long a single object upload may take before it is abandoned + gcsUploadTimeout = 60 * time.Second +) + +var gcsTransmissionMetrics = []metrics.Metadata{ + {Name: counterGCSExportEvents, Type: metrics.Counter, Unit: metrics.Dimensionless, Description: "number of events written to the GCS export"}, + {Name: counterGCSExportDropped, Type: metrics.Counter, Unit: metrics.Dimensionless, Description: "number of events dropped from the GCS export because the queue was full"}, + {Name: counterGCSExportBatches, Type: metrics.Counter, Unit: metrics.Dimensionless, Description: "number of batch objects written to GCS"}, + {Name: counterGCSExportErrors, Type: metrics.Counter, Unit: metrics.Dimensionless, Description: "number of errors encountered while writing batches to GCS"}, + {Name: histogramGCSExportBatchBytes, Type: metrics.Histogram, Unit: metrics.Bytes, Description: "compressed size of batch objects written to GCS"}, + {Name: gaugeGCSExportQueueLength, Type: metrics.Gauge, Unit: metrics.Dimensionless, Description: "number of events waiting to be batched for the GCS export"}, +} + +// gcsExportRecord is the shape of one line in an exported JSON Lines object. +type gcsExportRecord struct { + Time time.Time `json:"time"` + TraceID string `json:"trace_id,omitempty"` + Dataset string `json:"dataset"` + Environment string `json:"environment,omitempty"` + SampleRate uint `json:"sample_rate"` + Data types.Payload `json:"data"` +} + +// gcsUploader writes a finished (compressed) batch to an object. It exists so +// tests can capture uploads without a real GCS client. +type gcsUploader func(ctx context.Context, objectName string, data []byte) error + +// gcsExportItem carries an event through the export queue along with its +// trace ID, which lives on the Span rather than the Event. +type gcsExportItem struct { + ev *types.Event + traceID string +} + +// GCSTransmission implements Transmission by batching events into gzipped +// JSON Lines objects and writing them to a GCS bucket under time-partitioned +// object names. Enqueueing never blocks: if the internal queue is full +// (because GCS is slow or unavailable), events are dropped from the export +// and counted in gcs_export_dropped. +type GCSTransmission struct { + Config config.Config `inject:""` + Logger logger.Logger `inject:""` + Metrics metrics.Metrics `inject:"metrics"` + Clock clockwork.Clock `inject:""` + + // uploadFn may be set before Start to override how batches are written + // (used in tests). If nil, a real GCS client is created at Start. + uploadFn gcsUploader + + client *storage.Client + bucket string + keyPrefix string + flushEvery time.Duration + maxBatch int + + hostname string + seq uint64 + events chan gcsExportItem + done chan struct{} + flushWG sync.WaitGroup + uploadPool *pool.Pool +} + +func NewGCSTransmission() *GCSTransmission { + return &GCSTransmission{} +} + +func (g *GCSTransmission) Start() error { + cfg := g.Config.GetGCSExportConfig() + if cfg.Bucket == "" { + return fmt.Errorf("GCSExport is enabled but no Bucket is configured") + } + g.bucket = cfg.Bucket + g.keyPrefix = strings.Trim(cfg.KeyPrefix, "/") + g.flushEvery = time.Duration(cfg.FlushInterval) + if g.flushEvery <= 0 { + g.flushEvery = time.Minute + } + g.maxBatch = int(cfg.MaxBatchSize) + + if g.Clock == nil { + g.Clock = clockwork.NewRealClock() + } + + for _, m := range gcsTransmissionMetrics { + g.Metrics.Register(m) + } + + g.hostname, _ = os.Hostname() + if g.hostname == "" { + g.hostname = "unknown" + } + + if g.uploadFn == nil { + client, err := storage.NewClient(context.Background()) + if err != nil { + return fmt.Errorf("failed to create GCS client for trace export: %w", err) + } + g.client = client + g.uploadFn = g.uploadObject + } + + g.events = make(chan gcsExportItem, cfg.QueueSize) + g.done = make(chan struct{}) + g.uploadPool = pool.New().WithMaxGoroutines(maxConcurrentGCSUploads) + g.flushWG.Add(1) + go g.flushLoop() + + g.Logger.Info().WithString("bucket", g.bucket).Logf("started GCS trace export") + return nil +} + +func (g *GCSTransmission) Stop() error { + close(g.done) + g.flushWG.Wait() + g.uploadPool.Wait() + if g.client != nil { + g.client.Close() + } + return nil +} + +// EnqueueEvent queues an event for export to GCS. It never blocks; when the +// queue is full the event is dropped from the export. +func (g *GCSTransmission) EnqueueEvent(ev *types.Event) { + g.enqueue(gcsExportItem{ev: ev, traceID: ev.Data.MetaTraceID}) +} + +func (g *GCSTransmission) EnqueueSpan(sp *types.Span) { + g.enqueue(gcsExportItem{ev: sp.Event, traceID: sp.TraceID}) +} + +func (g *GCSTransmission) enqueue(item gcsExportItem) { + select { + case g.events <- item: + default: + g.Metrics.Increment(counterGCSExportDropped) + } +} + +// flushLoop drains the event queue, serializing events into a gzipped JSON +// Lines buffer, and hands the buffer off for upload when it exceeds the +// configured batch size or the flush interval elapses. +func (g *GCSTransmission) flushLoop() { + defer g.flushWG.Done() + + ticker := g.Clock.NewTicker(g.flushEvery) + defer ticker.Stop() + + var buf bytes.Buffer + gz := gzip.NewWriter(&buf) + var uncompressed, count int + + flush := func() { + if count == 0 { + return + } + if err := gz.Close(); err != nil { + g.Metrics.Increment(counterGCSExportErrors) + g.Logger.Error().WithString("error", err.Error()).Logf("failed to finish compressing GCS export batch") + } else { + g.dispatchUpload(buf.Bytes(), count) + } + buf = bytes.Buffer{} + gz = gzip.NewWriter(&buf) + uncompressed, count = 0, 0 + } + + add := func(item gcsExportItem) { + ev := item.ev + g.Metrics.Gauge(gaugeGCSExportQueueLength, float64(len(g.events))) + line, err := json.Marshal(gcsExportRecord{ + Time: ev.Timestamp, + TraceID: item.traceID, + Dataset: ev.Dataset, + Environment: ev.Environment, + SampleRate: ev.SampleRate, + Data: ev.Data, + }) + if err != nil { + g.Metrics.Increment(counterGCSExportErrors) + g.Logger.Error().WithString("error", err.Error()).Logf("failed to serialize event for GCS export") + return + } + gz.Write(line) + gz.Write([]byte{'\n'}) + uncompressed += len(line) + 1 + count++ + g.Metrics.Increment(counterGCSExportEvents) + if uncompressed >= g.maxBatch { + flush() + } + } + + for { + select { + case item := <-g.events: + add(item) + case <-ticker.Chan(): + flush() + case <-g.done: + // drain whatever is still queued, then do a final flush + for { + select { + case item := <-g.events: + add(item) + default: + flush() + return + } + } + } + } +} + +// dispatchUpload hands a finished batch to the upload pool. If all upload +// slots are busy this blocks, which in turn causes the queue to fill and +// events to be dropped from the export — never blocking the collector. +func (g *GCSTransmission) dispatchUpload(data []byte, count int) { + name := g.objectName(g.Clock.Now().UTC()) + g.uploadPool.Go(func() { + ctx, cancel := context.WithTimeout(context.Background(), gcsUploadTimeout) + defer cancel() + if err := g.uploadFn(ctx, name, data); err != nil { + g.Metrics.Increment(counterGCSExportErrors) + g.Logger.Error(). + WithString("error", err.Error()). + WithString("object", name). + WithField("num_events", count). + Logf("failed to write trace export batch to GCS") + return + } + g.Metrics.Increment(counterGCSExportBatches) + g.Metrics.Histogram(histogramGCSExportBatchBytes, float64(len(data))) + }) +} + +// objectName returns a time-partitioned object name like +// prefix/2026/07/07/15/hostname-1783100000-000001.jsonl.gz +func (g *GCSTransmission) objectName(t time.Time) string { + g.seq++ + name := fmt.Sprintf("%04d/%02d/%02d/%02d/%s-%d-%06d.jsonl.gz", + t.Year(), t.Month(), t.Day(), t.Hour(), g.hostname, t.Unix(), g.seq) + if g.keyPrefix != "" { + name = g.keyPrefix + "/" + name + } + return name +} + +// uploadObject writes one batch to GCS. Object names are unique, so we use a +// does-not-exist precondition, which also makes retries idempotent. +func (g *GCSTransmission) uploadObject(ctx context.Context, objectName string, data []byte) error { + w := g.client.Bucket(g.bucket). + Object(objectName). + If(storage.Conditions{DoesNotExist: true}). + NewWriter(ctx) + w.ContentType = "application/gzip" + if _, err := w.Write(data); err != nil { + w.Close() + return err + } + return w.Close() +} + +// NoopTransmission implements Transmission and discards everything it +// receives. It is injected in place of optional transmissions (like the GCS +// export) when they are disabled. +type NoopTransmission struct{} + +func (n *NoopTransmission) EnqueueEvent(ev *types.Event) {} +func (n *NoopTransmission) EnqueueSpan(sp *types.Span) {} diff --git a/transmit/gcs_transmit_test.go b/transmit/gcs_transmit_test.go new file mode 100644 index 0000000000..debb647cd7 --- /dev/null +++ b/transmit/gcs_transmit_test.go @@ -0,0 +1,256 @@ +package transmit + +import ( + "bufio" + "compress/gzip" + "bytes" + "context" + "encoding/json" + "strings" + "sync" + "testing" + "time" + + "github.com/jonboulle/clockwork" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/honeycombio/refinery/config" + "github.com/honeycombio/refinery/logger" + "github.com/honeycombio/refinery/metrics" + "github.com/honeycombio/refinery/types" +) + +// capturingUploader records uploaded objects for inspection. +type capturingUploader struct { + mutex sync.Mutex + objects map[string][]byte +} + +func newCapturingUploader() *capturingUploader { + return &capturingUploader{objects: make(map[string][]byte)} +} + +func (c *capturingUploader) upload(ctx context.Context, name string, data []byte) error { + c.mutex.Lock() + defer c.mutex.Unlock() + c.objects[name] = data + return nil +} + +func (c *capturingUploader) len() int { + c.mutex.Lock() + defer c.mutex.Unlock() + return len(c.objects) +} + +func (c *capturingUploader) all() map[string][]byte { + c.mutex.Lock() + defer c.mutex.Unlock() + out := make(map[string][]byte, len(c.objects)) + for k, v := range c.objects { + out[k] = v + } + return out +} + +func newTestGCSTransmission(cfg config.GCSExportConfig, clock clockwork.Clock) (*GCSTransmission, *capturingUploader) { + uploader := newCapturingUploader() + g := &GCSTransmission{ + Config: &config.MockConfig{GetGCSExportConfigVal: cfg}, + Logger: &logger.NullLogger{}, + Metrics: &metrics.NullMetrics{}, + Clock: clock, + uploadFn: uploader.upload, + } + return g, uploader +} + +func testEvent(cfg config.Config, traceID string) *types.Event { + data := types.NewPayload(cfg, map[string]any{ + "service.name": "test-service", + "trace.trace_id": traceID, + }) + return &types.Event{ + Dataset: "test-dataset", + Environment: "test-env", + SampleRate: 7, + Timestamp: time.Date(2026, 7, 7, 12, 0, 0, 0, time.UTC), + Data: data, + } +} + +func decodeRecords(t *testing.T, data []byte) []gcsExportRecord { + t.Helper() + gz, err := gzip.NewReader(bytes.NewReader(data)) + require.NoError(t, err) + defer gz.Close() + + var records []gcsExportRecord + scanner := bufio.NewScanner(gz) + scanner.Buffer(make([]byte, 1024*1024), 1024*1024) + for scanner.Scan() { + var rec gcsExportRecord + require.NoError(t, json.Unmarshal(scanner.Bytes(), &rec)) + records = append(records, rec) + } + require.NoError(t, scanner.Err()) + return records +} + +func TestGCSTransmissionRequiresBucket(t *testing.T) { + g, _ := newTestGCSTransmission(config.GCSExportConfig{Enabled: true}, clockwork.NewFakeClock()) + err := g.Start() + assert.Error(t, err) +} + +func TestGCSTransmissionFlushesOnStop(t *testing.T) { + cfg := config.GCSExportConfig{ + Enabled: true, + Bucket: "test-bucket", + KeyPrefix: "refinery/traces", + FlushInterval: config.Duration(time.Minute), + MaxBatchSize: config.MemorySize(100 * 1024 * 1024), + QueueSize: 100, + } + g, uploader := newTestGCSTransmission(cfg, clockwork.NewFakeClock()) + require.NoError(t, g.Start()) + + mockCfg := &config.MockConfig{} + for i := 0; i < 5; i++ { + g.EnqueueSpan(&types.Span{Event: testEvent(mockCfg, "trace-1"), TraceID: "trace-1"}) + } + require.NoError(t, g.Stop()) + + require.Equal(t, 1, uploader.len()) + for name, data := range uploader.all() { + // prefix and time-partitioned path from the event flush time + assert.True(t, strings.HasPrefix(name, "refinery/traces/"), "object name %q should start with key prefix", name) + assert.True(t, strings.HasSuffix(name, ".jsonl.gz"), "object name %q should end in .jsonl.gz", name) + + records := decodeRecords(t, data) + require.Len(t, records, 5) + for _, rec := range records { + assert.Equal(t, "test-dataset", rec.Dataset) + assert.Equal(t, "test-env", rec.Environment) + assert.Equal(t, uint(7), rec.SampleRate) + assert.Equal(t, "trace-1", rec.TraceID) + assert.Equal(t, "test-service", rec.Data.Get("service.name")) + } + } +} + +func TestGCSTransmissionFlushesOnBatchSize(t *testing.T) { + cfg := config.GCSExportConfig{ + Enabled: true, + Bucket: "test-bucket", + FlushInterval: config.Duration(time.Minute), + // tiny batch size: every event should produce its own object + MaxBatchSize: config.MemorySize(1), + QueueSize: 100, + } + g, uploader := newTestGCSTransmission(cfg, clockwork.NewFakeClock()) + require.NoError(t, g.Start()) + + mockCfg := &config.MockConfig{} + for i := 0; i < 3; i++ { + g.EnqueueEvent(testEvent(mockCfg, "trace-2")) + } + + assert.Eventually(t, func() bool { + return uploader.len() == 3 + }, 2*time.Second, 10*time.Millisecond) + + require.NoError(t, g.Stop()) + + // object names must be unique and unprefixed + for name := range uploader.all() { + assert.False(t, strings.HasPrefix(name, "/"), "object name %q should not start with a slash", name) + records := decodeRecords(t, data(t, uploader, name)) + assert.Len(t, records, 1) + } +} + +// data fetches one object's bytes from the uploader. +func data(t *testing.T, u *capturingUploader, name string) []byte { + t.Helper() + u.mutex.Lock() + defer u.mutex.Unlock() + d, ok := u.objects[name] + require.True(t, ok) + return d +} + +func TestGCSTransmissionFlushesOnInterval(t *testing.T) { + clock := clockwork.NewFakeClock() + cfg := config.GCSExportConfig{ + Enabled: true, + Bucket: "test-bucket", + FlushInterval: config.Duration(time.Minute), + MaxBatchSize: config.MemorySize(100 * 1024 * 1024), + QueueSize: 100, + } + g, uploader := newTestGCSTransmission(cfg, clock) + require.NoError(t, g.Start()) + + mockCfg := &config.MockConfig{} + g.EnqueueEvent(testEvent(mockCfg, "trace-3")) + + // wait for the event to be consumed by the flush loop before advancing + assert.Eventually(t, func() bool { + return len(g.events) == 0 + }, 2*time.Second, 10*time.Millisecond) + + clock.Advance(time.Minute + time.Second) + + assert.Eventually(t, func() bool { + return uploader.len() == 1 + }, 2*time.Second, 10*time.Millisecond) + + require.NoError(t, g.Stop()) +} + +func TestGCSTransmissionDropsWhenQueueFull(t *testing.T) { + cfg := config.GCSExportConfig{ + Enabled: true, + Bucket: "test-bucket", + FlushInterval: config.Duration(time.Minute), + MaxBatchSize: config.MemorySize(100 * 1024 * 1024), + QueueSize: 1, + } + uploader := newCapturingUploader() + mockMetrics := &metrics.MockMetrics{} + mockMetrics.Start() + g := &GCSTransmission{ + Config: &config.MockConfig{GetGCSExportConfigVal: cfg}, + Logger: &logger.NullLogger{}, + Metrics: mockMetrics, + Clock: clockwork.NewFakeClock(), + uploadFn: uploader.upload, + } + require.NoError(t, g.Start()) + // stop the flush loop so nothing drains the queue, then overfill it + require.NoError(t, g.Stop()) + + mockCfg := &config.MockConfig{} + for i := 0; i < 10; i++ { + g.EnqueueEvent(testEvent(mockCfg, "trace-4")) + } + + dropped, ok := mockMetrics.Get(counterGCSExportDropped) + require.True(t, ok) + // queue holds 1 (plus possibly one held by the exited loop); the rest dropped + assert.GreaterOrEqual(t, dropped, float64(8)) +} + +func TestGCSObjectNamePartitioning(t *testing.T) { + g := &GCSTransmission{keyPrefix: "some/prefix", hostname: "host-1"} + ts := time.Date(2026, 7, 7, 15, 4, 5, 0, time.UTC) + name := g.objectName(ts) + assert.True(t, strings.HasPrefix(name, "some/prefix/2026/07/07/15/host-1-"), "got %q", name) + assert.True(t, strings.HasSuffix(name, "-000001.jsonl.gz"), "got %q", name) + + // sequence increments for uniqueness + name2 := g.objectName(ts) + assert.NotEqual(t, name, name2) +}