From b7017544bad9a2c8e44d7f639639501176cb00c1 Mon Sep 17 00:00:00 2001 From: Oleksii Sholik Date: Thu, 2 Jul 2026 18:15:43 +0200 Subject: [PATCH 01/10] feat(telemetry): add EmptyResponseSampler decision function Pure function deciding the SampleRate weight for a shape-GET root span so the spans of empty/up-to-date responses can be tail-dropped. Keep-on-error wins first (5xx -> SampleRate=1), empty non-SSE 2xx responses get the SampleRate=0 drop sentinel when enabled, everything else is left unchanged. Co-Authored-By: Claude Opus 4.8 --- .../telemetry/empty_response_sampler.ex | 52 +++++++++++++++++++ .../telemetry/empty_response_sampler_test.exs | 34 ++++++++++++ 2 files changed, 86 insertions(+) create mode 100644 packages/sync-service/lib/electric/telemetry/empty_response_sampler.ex create mode 100644 packages/sync-service/test/electric/telemetry/empty_response_sampler_test.exs diff --git a/packages/sync-service/lib/electric/telemetry/empty_response_sampler.ex b/packages/sync-service/lib/electric/telemetry/empty_response_sampler.ex new file mode 100644 index 0000000000..87eb5d3ddb --- /dev/null +++ b/packages/sync-service/lib/electric/telemetry/empty_response_sampler.ex @@ -0,0 +1,52 @@ +defmodule Electric.Telemetry.EmptyResponseSampler do + @moduledoc """ + Decides the `SampleRate` weight for a shape-GET root span, tail-dropping the spans of + empty/up-to-date responses to cut trace volume. + + The vast majority of shape-GET responses are empty (up-to-date, incl. long-poll + timeouts) and carry no useful trace detail, yet they dominate exported span volume. + When the drop is enabled, such spans are stamped with `SampleRate = 0`, which + `Electric.Telemetry.OpenTelemetry.EmptyResponseDropProcessor` recognises as a sentinel + to drop the span before it is queued for export. + + The decision is a pure function so it can be unit-tested in isolation. Ordering mirrors + the worker's `traceSampleRate` (see stratovolt#1611): + + 1. Error responses (`status >= 500`) are kept and stamped with `SampleRate = 1` — + keep-on-error wins and is checked first. + 2. Empty, non-SSE 2xx responses are dropped (`SampleRate = 0`) when the drop is + enabled. + 3. Everything else is left unchanged (`:unchanged`), preserving whatever base + `SampleRate` the upstream rate hint produced. + + Ratio-based sampling of empties is expected to live on the cloud worker side and reach + the origin via the tracestate rate hint, so the origin only needs an all-or-nothing + toggle rather than its own ratio knob. + """ + + @drop_sample_rate 0 + @error_sample_rate 1 + + @doc """ + The effective `SampleRate` for a shape-GET root span, or `:unchanged` to leave the base + rate untouched. + + * `status` - the final HTTP status of the response. + * `is_empty_response?` - whether the response was empty/up-to-date. + * `is_sse_response?` - whether the response used server-sent events (excluded from the + drop). + * `drop_enabled?` - whether the empty-response drop is enabled. + + Returns `0` (the drop sentinel) for a dropped empty response, `1` for a kept error + response, or `:unchanged` otherwise. + """ + @spec sample_rate(integer() | nil, boolean(), boolean(), boolean()) :: + non_neg_integer() | :unchanged + def sample_rate(status, is_empty_response?, is_sse_response?, drop_enabled?) do + cond do + is_integer(status) and status >= 500 -> @error_sample_rate + drop_enabled? and is_empty_response? and not is_sse_response? -> @drop_sample_rate + true -> :unchanged + end + end +end diff --git a/packages/sync-service/test/electric/telemetry/empty_response_sampler_test.exs b/packages/sync-service/test/electric/telemetry/empty_response_sampler_test.exs new file mode 100644 index 0000000000..d407c1bfea --- /dev/null +++ b/packages/sync-service/test/electric/telemetry/empty_response_sampler_test.exs @@ -0,0 +1,34 @@ +defmodule Electric.Telemetry.EmptyResponseSamplerTest do + use ExUnit.Case, async: true + + alias Electric.Telemetry.EmptyResponseSampler + + describe "sample_rate/4 with the drop enabled" do + test "empty 2xx responses are dropped (SampleRate = 0)" do + assert 0 = EmptyResponseSampler.sample_rate(200, true, false, true) + end + + test "error responses (>= 500) are kept and stamped with SampleRate = 1, even when empty" do + assert 1 = EmptyResponseSampler.sample_rate(500, true, false, true) + assert 1 = EmptyResponseSampler.sample_rate(503, false, false, true) + end + + test "SSE empty responses are left unchanged (never dropped)" do + assert :unchanged = EmptyResponseSampler.sample_rate(200, true, true, true) + end + + test "non-empty 2xx responses are left unchanged" do + assert :unchanged = EmptyResponseSampler.sample_rate(200, false, false, true) + end + end + + describe "sample_rate/4 with the drop disabled" do + test "empty 2xx responses are left unchanged (not dropped)" do + assert :unchanged = EmptyResponseSampler.sample_rate(200, true, false, false) + end + + test "error responses are still kept with SampleRate = 1 (checked before the toggle)" do + assert 1 = EmptyResponseSampler.sample_rate(500, true, false, false) + end + end +end From 5bbecb73bae0393349c70771d02a7ed86af634e4 Mon Sep 17 00:00:00 2001 From: Oleksii Sholik Date: Thu, 2 Jul 2026 18:15:43 +0200 Subject: [PATCH 02/10] feat(telemetry): drop spans stamped with the SampleRate=0 sentinel Add an OTel span processor whose on_end/2 returns :dropped for spans carrying SampleRate=0, and register it ahead of otel_batch_processor. The SDK folds on_end with a short-circuiting andalso, so a non-true return before the batch processor prevents the span from ever being queued for export. Only empty-response spans carry the sentinel; all others pass through unchanged. Co-Authored-By: Claude Opus 4.8 --- .../telemetry/open_telemetry/config.ex | 10 +++- .../empty_response_drop_processor.ex | 53 +++++++++++++++++++ .../empty_response_drop_processor_test.exs | 50 +++++++++++++++++ 3 files changed, 112 insertions(+), 1 deletion(-) create mode 100644 packages/sync-service/lib/electric/telemetry/open_telemetry/empty_response_drop_processor.ex create mode 100644 packages/sync-service/test/electric/telemetry/open_telemetry/empty_response_drop_processor_test.exs diff --git a/packages/sync-service/lib/electric/telemetry/open_telemetry/config.ex b/packages/sync-service/lib/electric/telemetry/open_telemetry/config.ex index 5afd0e2e0f..480ae2c218 100644 --- a/packages/sync-service/lib/electric/telemetry/open_telemetry/config.ex +++ b/packages/sync-service/lib/electric/telemetry/open_telemetry/config.ex @@ -36,6 +36,14 @@ defmodule Electric.Telemetry.OpenTelemetry.Config do Electric.Telemetry.OpenTelemetry.ResourceDetector ], resource: otel_resource, - processors: [otel_batch_processor, otel_simple_processor] |> Enum.reject(&is_nil/1) + processors: + [ + # Runs first so its `:dropped` return short-circuits the SDK's `andalso` fold + # before the batch processor queues the span for export. + {Electric.Telemetry.OpenTelemetry.EmptyResponseDropProcessor, %{}}, + otel_batch_processor, + otel_simple_processor + ] + |> Enum.reject(&is_nil/1) end end diff --git a/packages/sync-service/lib/electric/telemetry/open_telemetry/empty_response_drop_processor.ex b/packages/sync-service/lib/electric/telemetry/open_telemetry/empty_response_drop_processor.ex new file mode 100644 index 0000000000..343ce296d4 --- /dev/null +++ b/packages/sync-service/lib/electric/telemetry/open_telemetry/empty_response_drop_processor.ex @@ -0,0 +1,53 @@ +# This processor reads the SDK-internal `#span{}` record, whose definition lives in the +# `opentelemetry` (SDK) application. That app is only a dependency when building for the +# telemetry target, so the module is only compiled there. +if Electric.telemetry_enabled?() do + defmodule Electric.Telemetry.OpenTelemetry.EmptyResponseDropProcessor do + @moduledoc """ + An OTel span processor that tail-drops spans stamped with the `SampleRate = 0` + sentinel by `Electric.Telemetry.EmptyResponseSampler`. + + The SDK folds `on_end/2` over the processor list with a short-circuiting `andalso` + (`Bool andalso P:on_end(Span, Config) =:= true`), so a processor returning anything + other than `true` before `otel_batch_processor` prevents the batch processor from ever + seeing the span, and it is never queued for export. This processor is therefore + registered first (see `Electric.Telemetry.OpenTelemetry.Config`). + + Only empty/up-to-date shape-GET response spans carry `SampleRate = 0`; every other span + passes through unchanged. + """ + + @behaviour :otel_span_processor + + require Record + + Record.defrecordp( + :span, + Record.extract(:span, from_lib: "opentelemetry/include/otel_span.hrl") + ) + + @sample_rate_attr "SampleRate" + @drop_sample_rate 0 + + @impl true + def on_start(_ctx, span, _config), do: span + + @impl true + def on_end(span, _config) do + case sample_rate(span) do + @drop_sample_rate -> :dropped + _ -> true + end + end + + @impl true + def force_flush(_config), do: :ok + + defp sample_rate(span) do + case span(span, :attributes) do + :undefined -> nil + attributes -> attributes |> :otel_attributes.map() |> Map.get(@sample_rate_attr) + end + end + end +end diff --git a/packages/sync-service/test/electric/telemetry/open_telemetry/empty_response_drop_processor_test.exs b/packages/sync-service/test/electric/telemetry/open_telemetry/empty_response_drop_processor_test.exs new file mode 100644 index 0000000000..e8e6f0043a --- /dev/null +++ b/packages/sync-service/test/electric/telemetry/open_telemetry/empty_response_drop_processor_test.exs @@ -0,0 +1,50 @@ +# The processor reads the SDK-internal `#span{}` record, only available on the telemetry +# target (see the module for details). +if Electric.telemetry_enabled?() do + defmodule Electric.Telemetry.OpenTelemetry.EmptyResponseDropProcessorTest do + use ExUnit.Case, async: true + + alias Electric.Telemetry.OpenTelemetry.EmptyResponseDropProcessor, as: Processor + + require Record + + Record.defrecordp( + :span, + Record.extract(:span, from_lib: "opentelemetry/include/otel_span.hrl") + ) + + defp span_with_attributes(attrs) do + span(attributes: :otel_attributes.new(attrs, 128, :infinity)) + end + + describe "on_end/2" do + test "drops a span stamped with SampleRate = 0" do + span = span_with_attributes(%{"SampleRate" => 0}) + assert Processor.on_end(span, %{}) == :dropped + end + + test "keeps a span with a non-zero SampleRate" do + span = span_with_attributes(%{"SampleRate" => 20}) + assert Processor.on_end(span, %{}) == true + end + + test "keeps a span with no SampleRate attribute" do + span = span_with_attributes(%{"other" => "attr"}) + assert Processor.on_end(span, %{}) == true + end + + test "keeps a span with no attributes at all" do + assert Processor.on_end(span(), %{}) == true + end + end + + test "on_start/3 returns the span unchanged" do + span = span_with_attributes(%{"SampleRate" => 0}) + assert Processor.on_start(:otel_ctx.new(), span, %{}) == span + end + + test "force_flush/1 returns :ok" do + assert Processor.force_flush(%{}) == :ok + end + end +end From 5017d00c6603d6ca620b1e5192d3e0972c42b15f Mon Sep 17 00:00:00 2001 From: Oleksii Sholik Date: Thu, 2 Jul 2026 18:15:43 +0200 Subject: [PATCH 03/10] feat(telemetry): tail-drop empty shape-GET response spans Wire the empty-response drop into the request-end SampleRate stamp: empty, non-SSE 2xx shape-GET responses are stamped with the SampleRate=0 sentinel so the drop processor skips them at export time. Gated by the new ELECTRIC_DROP_EMPTY_RESPONSE_SPANS env var (default true); set it to false to export those spans normally. Empty/up-to-date responses are ~95% of shape-GET root spans in production, so dropping them substantially cuts exported trace volume. Error (5xx) and SSE responses are never dropped. Ratio-based keep-sampling is deferred to the cloud worker, which will propagate its rate to the origin via tracestate. Fixes #4664 Co-Authored-By: Claude Opus 4.8 --- .changeset/drop-empty-response-spans.md | 5 +++ packages/sync-service/config/runtime.exs | 1 + packages/sync-service/lib/electric/config.ex | 4 +++ .../lib/electric/plug/serve_shape_plug.ex | 34 ++++++++++++++++++- 4 files changed, 43 insertions(+), 1 deletion(-) create mode 100644 .changeset/drop-empty-response-spans.md diff --git a/.changeset/drop-empty-response-spans.md b/.changeset/drop-empty-response-spans.md new file mode 100644 index 0000000000..a79f1bc81b --- /dev/null +++ b/.changeset/drop-empty-response-spans.md @@ -0,0 +1,5 @@ +--- +"@core/sync-service": patch +--- + +Tail-drop the OpenTelemetry spans of empty/up-to-date shape-GET responses at export time to cut trace volume. Enabled by default; set `ELECTRIC_DROP_EMPTY_RESPONSE_SPANS=false` to export those spans normally. Error (5xx) and SSE responses are never dropped. diff --git a/packages/sync-service/config/runtime.exs b/packages/sync-service/config/runtime.exs index 6c73b12631..4e1fa09863 100644 --- a/packages/sync-service/config/runtime.exs +++ b/packages/sync-service/config/runtime.exs @@ -235,6 +235,7 @@ config :electric, otel_export_period: otel_export_period, otel_sampling_ratio: env!("ELECTRIC_OTEL_SAMPLING_RATIO", :float, nil), metrics_sampling_ratio: env!("ELECTRIC_METRICS_SAMPLING_RATIO", :float, nil), + drop_empty_response_spans?: env!("ELECTRIC_DROP_EMPTY_RESPONSE_SPANS", :boolean, nil), telemetry_top_process_limit: env!("ELECTRIC_TELEMETRY_TOP_PROCESS_LIMIT", &Electric.Config.parse_top_process_limit!/1, nil) || env!( diff --git a/packages/sync-service/lib/electric/config.ex b/packages/sync-service/lib/electric/config.ex index c6d77a830c..3631770e32 100644 --- a/packages/sync-service/lib/electric/config.ex +++ b/packages/sync-service/lib/electric/config.ex @@ -84,6 +84,10 @@ defmodule Electric.Config do telemetry_url: URI.new!("https://checkpoint.electric-sql.com"), otel_sampling_ratio: 0.01, metrics_sampling_ratio: 1, + # When true (default), the OTel spans of empty/up-to-date shape-GET responses are + # tail-dropped at export time to cut trace volume. Setting it false disables the drop + # so empties are exported normally. See Electric.Telemetry.EmptyResponseSampler. + drop_empty_response_spans?: true, ## Memory # After this duration of inactivity, consumer processes will hibernate # to allow garbage collection diff --git a/packages/sync-service/lib/electric/plug/serve_shape_plug.ex b/packages/sync-service/lib/electric/plug/serve_shape_plug.ex index 34da1c016c..c639e05ada 100644 --- a/packages/sync-service/lib/electric/plug/serve_shape_plug.ex +++ b/packages/sync-service/lib/electric/plug/serve_shape_plug.ex @@ -36,6 +36,7 @@ defmodule Electric.Plug.ServeShapePlug do alias Electric.ShapeCache alias Electric.ShapeCache.ShapeStatus alias Electric.Shapes.Api + alias Electric.Telemetry.EmptyResponseSampler alias Electric.Telemetry.OpenTelemetry alias Electric.Utils alias Plug.Conn @@ -504,17 +505,48 @@ defmodule Electric.Plug.ServeShapePlug do # the parent-based sampler left no recording span, so `add_span_attributes` is a no-op # and nothing is stamped or exported. # + # Empty/up-to-date responses are additionally tail-dropped: when the drop is enabled + # they are stamped with `SampleRate = 0`, which the drop processor recognises as a + # sentinel to skip export (see Electric.Telemetry.EmptyResponseSampler). + # # Called both at span start (status not yet known: the rate hint is stamped as-is) and # at emit time, when the final attribute values overwrite the initial ones. defp add_span_attrs_from_conn(conn) do conn |> open_telemetry_attrs() - |> Map.merge(TraceContextPlug.sample_rate_attrs(conn, conn.status)) + |> Map.merge(sample_rate_attrs(conn)) |> OpenTelemetry.add_span_attributes() conn end + # The `SampleRate` span attribute, combining the upstream rate hint with the + # empty-response tail-drop decision. The drop overrides the base rate with the + # `SampleRate = 0` sentinel for empty, non-SSE responses when enabled. + defp sample_rate_attrs(conn) do + base_attrs = TraceContextPlug.sample_rate_attrs(conn, conn.status) + trace_attrs = response_trace_attrs(conn) + + decision = + EmptyResponseSampler.sample_rate( + conn.status, + trace_attrs[:ot_is_empty_response] || false, + trace_attrs[:ot_is_sse_response] || false, + Electric.Config.get_env(:drop_empty_response_spans?) + ) + + case decision do + :unchanged -> base_attrs + sample_rate -> Map.put(base_attrs, TraceContextPlug.sample_rate_attr(), sample_rate) + end + end + + defp response_trace_attrs(%Conn{assigns: assigns}) do + request = Map.get(assigns, :request, %{}) |> bare_map() + response = (Map.get(assigns, :response) || Map.get(request, :response) || %{}) |> bare_map() + Map.get(response, :trace_attrs, %{}) + end + defp open_telemetry_attrs(%Conn{assigns: assigns} = conn) do request = Map.get(assigns, :request, %{}) |> bare_map() params = Map.get(request, :params, %{}) |> bare_map() From e6827c9cd90dedd58e2e31307bef9b898bb7810c Mon Sep 17 00:00:00 2001 From: Oleksii Sholik Date: Thu, 2 Jul 2026 18:37:42 +0200 Subject: [PATCH 04/10] test(integration): verify empty-response spans are dropped Extend the otel-export lux suite: an already-up-to-date request returns an empty no-change response, and the suite asserts a data Plug_shape_get root span is exported (shape_req.is_empty_response=false) while the empty one is never exported (no Plug_shape_get span with is_empty_response=true reaches the collector). Exercises the drop end-to-end through the real OTel export pipeline. Co-Authored-By: Claude Opus 4.8 --- integration-tests/tests/otel-export.lux | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/integration-tests/tests/otel-export.lux b/integration-tests/tests/otel-export.lux index b325c34912..62b051896c 100644 --- a/integration-tests/tests/otel-export.lux +++ b/integration-tests/tests/otel-export.lux @@ -58,8 +58,19 @@ [invoke curl_shape "http://localhost:3000/v1/shape?table=items&handle=$handle&offset=$offset"] ??HTTP/1.1 200 OK + ?electric-offset: ([\w\d_]+) + [global latest_offset=$1] ??"value":{"val":"3"} + # Request the same shape again at the now-current offset. It is already + # up-to-date, so Electric returns an empty (no-change) 200 response. Its + # Plug_shape_get root span carries shape_req.is_empty_response=true and must be + # tail-dropped (ELECTRIC_DROP_EMPTY_RESPONSE_SPANS defaults to true), so it must + # never reach the collector. Verified below. + [invoke curl_shape "http://localhost:3000/v1/shape?table=items&handle=$handle&offset=$latest_offset"] + ??HTTP/1.1 200 OK + ??"control":"up-to-date" + [sleep 10] [shell otel_collector_startup] @@ -124,6 +135,16 @@ # (ELECTRIC_OTEL_SAMPLING_RATIO=1.0); it is sampled at 1% by default. [invoke grep_otel_collector_output "Span pg_txn\.replication_client\.transaction_received .*total_processing_time=[0-9]+"] + # A data (non-empty) shape-GET response is exported as a Plug_shape_get root span + # flagged shape_req.is_empty_response=false. + [invoke grep_otel_collector_output "Span Plug_shape_get .*shape_req\.is_empty_response=false"] + + # The empty/up-to-date shape-GET response above was tail-dropped: no Plug_shape_get + # span carrying shape_req.is_empty_response=true is ever exported to the collector. + !docker logs $otel_collector_container_name 2>&1 | python3 $(realpath ../support_files/normalize-otel-output.py) | grep -c "Span Plug_shape_get .*shape_req\.is_empty_response=true"; echo EMPTY_SPAN_COUNT_DONE + ?^0$ + ??EMPTY_SPAN_COUNT_DONE + ### [cleanup] From 9055f4f1825a862ade0c276e80957cfe7b541c8b Mon Sep 17 00:00:00 2001 From: Oleksii Sholik Date: Fri, 3 Jul 2026 11:44:32 +0200 Subject: [PATCH 05/10] refactor(telemetry): dedup request/response resolution in ServeShapePlug Extract request_and_response/1 so open_telemetry_attrs/1 and response_trace_attrs/1 share a single resolver for the request/response structs instead of each re-deriving them from the conn assigns, keeping the two in sync. Co-Authored-By: Claude Opus 4.8 --- .../lib/electric/plug/serve_shape_plug.ex | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/packages/sync-service/lib/electric/plug/serve_shape_plug.ex b/packages/sync-service/lib/electric/plug/serve_shape_plug.ex index c639e05ada..7bbadaf590 100644 --- a/packages/sync-service/lib/electric/plug/serve_shape_plug.ex +++ b/packages/sync-service/lib/electric/plug/serve_shape_plug.ex @@ -541,16 +541,23 @@ defmodule Electric.Plug.ServeShapePlug do end end - defp response_trace_attrs(%Conn{assigns: assigns}) do + # Resolves the request and response structs from the conn assigns as bare maps. + # The response may live directly on the conn or nested under the request + # (its emit-time home), depending on where in the pipeline this runs. + defp request_and_response(assigns) do request = Map.get(assigns, :request, %{}) |> bare_map() response = (Map.get(assigns, :response) || Map.get(request, :response) || %{}) |> bare_map() + {request, response} + end + + defp response_trace_attrs(%Conn{assigns: assigns}) do + {_request, response} = request_and_response(assigns) Map.get(response, :trace_attrs, %{}) end defp open_telemetry_attrs(%Conn{assigns: assigns} = conn) do - request = Map.get(assigns, :request, %{}) |> bare_map() + {request, response} = request_and_response(assigns) params = Map.get(request, :params, %{}) |> bare_map() - response = (Map.get(assigns, :response) || Map.get(request, :response) || %{}) |> bare_map() attrs = Map.get(response, :trace_attrs, %{}) maybe_up_to_date = Map.get(response, :up_to_date, false) From 8f94fb9c2c520159f6196ea81eac30e80060a635 Mon Sep 17 00:00:00 2001 From: Oleksii Sholik Date: Fri, 3 Jul 2026 11:55:49 +0200 Subject: [PATCH 06/10] test(integration): trigger the empty-response path via a live long-poll The empty-response assertion was exercising the wrong path: ot_is_empty_response is set only by no_change_response, which is reached only when a live long-poll times out with no new data. A plain non-live request at the current offset returns is_empty_response=false, so the drop was never triggered and the assertion passed vacuously. Use a live request that long-polls until timeout, and assert its body is the up-to-date control only (no data element) so the drop check is meaningful. Also replace the bare "0" count match, which flaked on lux prompt-gluing (SH-PROMPT:0), with the suite's `test ... && echo TOKEN` / `??TOKEN` idiom. Co-Authored-By: Claude Opus 4.8 --- integration-tests/tests/otel-export.lux | 27 +++++++++++++++---------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/integration-tests/tests/otel-export.lux b/integration-tests/tests/otel-export.lux index 62b051896c..04a012ce8c 100644 --- a/integration-tests/tests/otel-export.lux +++ b/integration-tests/tests/otel-export.lux @@ -62,14 +62,20 @@ [global latest_offset=$1] ??"value":{"val":"3"} - # Request the same shape again at the now-current offset. It is already - # up-to-date, so Electric returns an empty (no-change) 200 response. Its - # Plug_shape_get root span carries shape_req.is_empty_response=true and must be - # tail-dropped (ELECTRIC_DROP_EMPTY_RESPONSE_SPANS defaults to true), so it must - # never reach the collector. Verified below. - [invoke curl_shape "http://localhost:3000/v1/shape?table=items&handle=$handle&offset=$latest_offset"] + # A live request at the current offset receives no new data, so it long-polls + # until it times out and returns an empty (no-change) up-to-date 200 response + # (this is the only path that sets ot_is_empty_response). Its Plug_shape_get root + # span carries shape_req.is_empty_response=true and must be tail-dropped + # (ELECTRIC_DROP_EMPTY_RESPONSE_SPANS defaults to true), so it must never reach + # the collector. Verified below. The long-poll timeout is 20s by default, so allow + # ample time for the response. + [timeout 60] + [invoke curl_shape "http://localhost:3000/v1/shape?table=items&handle=$handle&offset=$latest_offset&live"] ??HTTP/1.1 200 OK - ??"control":"up-to-date" + # Body is the up-to-date control only (no data element) — confirms the empty + # no-change path was taken, so the drop assertion below is not vacuous. + ??[{"headers":{"control":"up-to-date" + [timeout] [sleep 10] @@ -139,11 +145,10 @@ # flagged shape_req.is_empty_response=false. [invoke grep_otel_collector_output "Span Plug_shape_get .*shape_req\.is_empty_response=false"] - # The empty/up-to-date shape-GET response above was tail-dropped: no Plug_shape_get + # The empty/up-to-date long-poll response above was tail-dropped: no Plug_shape_get # span carrying shape_req.is_empty_response=true is ever exported to the collector. - !docker logs $otel_collector_container_name 2>&1 | python3 $(realpath ../support_files/normalize-otel-output.py) | grep -c "Span Plug_shape_get .*shape_req\.is_empty_response=true"; echo EMPTY_SPAN_COUNT_DONE - ?^0$ - ??EMPTY_SPAN_COUNT_DONE + !test $(docker logs $otel_collector_container_name 2>&1 | python3 $(realpath ../support_files/normalize-otel-output.py) | grep -c "Span Plug_shape_get .*shape_req\.is_empty_response=true") -eq 0 && echo EMPTY_RESPONSE_SPAN_DROPPED + ??EMPTY_RESPONSE_SPAN_DROPPED ### From d1bed4d69327dc79193d217d9136c404e7c7c150 Mon Sep 17 00:00:00 2001 From: Oleksii Sholik Date: Fri, 3 Jul 2026 17:15:02 +0200 Subject: [PATCH 07/10] feat(telemetry): default the empty-response span drop to off Flip ELECTRIC_DROP_EMPTY_RESPONSE_SPANS to default false so the drop ships dormant and is opt-in per environment: merging the code changes nothing until an operator sets it true. The otel-export integration test now enables it explicitly to exercise the drop. Co-Authored-By: Claude Opus 4.8 --- .changeset/drop-empty-response-spans.md | 2 +- integration-tests/tests/otel-export.lux | 4 ++-- packages/sync-service/lib/electric/config.ex | 8 ++++---- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.changeset/drop-empty-response-spans.md b/.changeset/drop-empty-response-spans.md index a79f1bc81b..a1b4890dfb 100644 --- a/.changeset/drop-empty-response-spans.md +++ b/.changeset/drop-empty-response-spans.md @@ -2,4 +2,4 @@ "@core/sync-service": patch --- -Tail-drop the OpenTelemetry spans of empty/up-to-date shape-GET responses at export time to cut trace volume. Enabled by default; set `ELECTRIC_DROP_EMPTY_RESPONSE_SPANS=false` to export those spans normally. Error (5xx) and SSE responses are never dropped. +Tail-drop the OpenTelemetry spans of empty/up-to-date shape-GET responses at export time to cut trace volume. Disabled by default; set `ELECTRIC_DROP_EMPTY_RESPONSE_SPANS=true` to enable the drop. Error (5xx) and SSE responses are never dropped. diff --git a/integration-tests/tests/otel-export.lux b/integration-tests/tests/otel-export.lux index 04a012ce8c..7fef3f4704 100644 --- a/integration-tests/tests/otel-export.lux +++ b/integration-tests/tests/otel-export.lux @@ -10,7 +10,7 @@ [invoke setup_pg] -[invoke setup_electric_with_env "ELECTRIC_OTLP_ENDPOINT=http://localhost:4318 ELECTRIC_SYSTEM_METRICS_POLL_INTERVAL=1s ELECTRIC_STACK_TELEMETRY_INIT_DELAY=1s ELECTRIC_OTEL_EXPORT_PERIOD=2s ELECTRIC_OTEL_SAMPLING_RATIO=1.0 DO_NOT_START_CONN_MAN_PING=1 ELECTRIC_LOG_LEVEL=info OTEL_RESOURCE_ATTRIBUTES=custom.attr=electric.val"] +[invoke setup_electric_with_env "ELECTRIC_OTLP_ENDPOINT=http://localhost:4318 ELECTRIC_SYSTEM_METRICS_POLL_INTERVAL=1s ELECTRIC_STACK_TELEMETRY_INIT_DELAY=1s ELECTRIC_OTEL_EXPORT_PERIOD=2s ELECTRIC_OTEL_SAMPLING_RATIO=1.0 ELECTRIC_DROP_EMPTY_RESPONSE_SPANS=true DO_NOT_START_CONN_MAN_PING=1 ELECTRIC_LOG_LEVEL=info OTEL_RESOURCE_ATTRIBUTES=custom.attr=electric.val"] # Spawn a process containing off-heap binary references and ensure it's in the top 5 by memory footprint. [shell electric] @@ -66,7 +66,7 @@ # until it times out and returns an empty (no-change) up-to-date 200 response # (this is the only path that sets ot_is_empty_response). Its Plug_shape_get root # span carries shape_req.is_empty_response=true and must be tail-dropped - # (ELECTRIC_DROP_EMPTY_RESPONSE_SPANS defaults to true), so it must never reach + # (ELECTRIC_DROP_EMPTY_RESPONSE_SPANS=true is set above), so it must never reach # the collector. Verified below. The long-poll timeout is 20s by default, so allow # ample time for the response. [timeout 60] diff --git a/packages/sync-service/lib/electric/config.ex b/packages/sync-service/lib/electric/config.ex index 3631770e32..70fee25429 100644 --- a/packages/sync-service/lib/electric/config.ex +++ b/packages/sync-service/lib/electric/config.ex @@ -84,10 +84,10 @@ defmodule Electric.Config do telemetry_url: URI.new!("https://checkpoint.electric-sql.com"), otel_sampling_ratio: 0.01, metrics_sampling_ratio: 1, - # When true (default), the OTel spans of empty/up-to-date shape-GET responses are - # tail-dropped at export time to cut trace volume. Setting it false disables the drop - # so empties are exported normally. See Electric.Telemetry.EmptyResponseSampler. - drop_empty_response_spans?: true, + # When true, the OTel spans of empty/up-to-date shape-GET responses are tail-dropped + # at export time to cut trace volume. Defaults to false (empties exported normally); + # set it true to enable the drop. See Electric.Telemetry.EmptyResponseSampler. + drop_empty_response_spans?: false, ## Memory # After this duration of inactivity, consumer processes will hibernate # to allow garbage collection From db9c771bc7c79045c867ce17f4933a6829868caf Mon Sep 17 00:00:00 2001 From: Oleksii Sholik Date: Fri, 3 Jul 2026 17:34:51 +0200 Subject: [PATCH 08/10] feat(config): make the long-poll timeout configurable Expose the existing long_poll_timeout config value via ELECTRIC_LONG_POLL_TIMEOUT so it can be tuned per environment, e.g. shortened in integration tests to trigger an empty long-poll response quickly. Co-Authored-By: Claude Opus 4.8 --- packages/sync-service/config/runtime.exs | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/sync-service/config/runtime.exs b/packages/sync-service/config/runtime.exs index 4e1fa09863..cd0e4c3ecd 100644 --- a/packages/sync-service/config/runtime.exs +++ b/packages/sync-service/config/runtime.exs @@ -226,6 +226,7 @@ config :electric, env!("ELECTRIC_EXPERIMENTAL_MAX_SHAPES", :integer, nil), consumer_partitions: env!("ELECTRIC_CONSUMER_PARTITIONS", :integer, nil), max_concurrent_requests: max_concurrent_requests, + long_poll_timeout: env!("ELECTRIC_LONG_POLL_TIMEOUT", :integer, nil), # Used in telemetry instance_id: instance_id, call_home_telemetry?: env!("ELECTRIC_USAGE_REPORTING", :boolean, config_env() == :prod), From 5a37855c28c67a920c831746bc3a2b04a860e63e Mon Sep 17 00:00:00 2001 From: Oleksii Sholik Date: Fri, 3 Jul 2026 17:34:51 +0200 Subject: [PATCH 09/10] docs(telemetry): trim comments on the empty-response span drop Address review feedback: remove over-explanatory module/inline comments, drop references to the private worker repo and to SDK-internal fold mechanics, and stop restating default values in comments. Simplify the otel-export lux test to trigger the empty long-poll response via a short ELECTRIC_LONG_POLL_TIMEOUT instead of waiting out the default, and tighten its comments. Co-Authored-By: Claude Opus 4.8 --- integration-tests/tests/otel-export.lux | 21 +++++-------------- packages/sync-service/lib/electric/config.ex | 3 +-- .../lib/electric/plug/serve_shape_plug.ex | 3 --- .../telemetry/empty_response_sampler.ex | 9 +------- .../telemetry/open_telemetry/config.ex | 4 ++-- .../empty_response_drop_processor.ex | 9 ++++---- 6 files changed, 13 insertions(+), 36 deletions(-) diff --git a/integration-tests/tests/otel-export.lux b/integration-tests/tests/otel-export.lux index 7fef3f4704..ec5f9990f3 100644 --- a/integration-tests/tests/otel-export.lux +++ b/integration-tests/tests/otel-export.lux @@ -10,7 +10,7 @@ [invoke setup_pg] -[invoke setup_electric_with_env "ELECTRIC_OTLP_ENDPOINT=http://localhost:4318 ELECTRIC_SYSTEM_METRICS_POLL_INTERVAL=1s ELECTRIC_STACK_TELEMETRY_INIT_DELAY=1s ELECTRIC_OTEL_EXPORT_PERIOD=2s ELECTRIC_OTEL_SAMPLING_RATIO=1.0 ELECTRIC_DROP_EMPTY_RESPONSE_SPANS=true DO_NOT_START_CONN_MAN_PING=1 ELECTRIC_LOG_LEVEL=info OTEL_RESOURCE_ATTRIBUTES=custom.attr=electric.val"] +[invoke setup_electric_with_env "ELECTRIC_OTLP_ENDPOINT=http://localhost:4318 ELECTRIC_SYSTEM_METRICS_POLL_INTERVAL=1s ELECTRIC_STACK_TELEMETRY_INIT_DELAY=1s ELECTRIC_OTEL_EXPORT_PERIOD=2s ELECTRIC_OTEL_SAMPLING_RATIO=1.0 ELECTRIC_DROP_EMPTY_RESPONSE_SPANS=true ELECTRIC_LONG_POLL_TIMEOUT=1000 DO_NOT_START_CONN_MAN_PING=1 ELECTRIC_LOG_LEVEL=info OTEL_RESOURCE_ATTRIBUTES=custom.attr=electric.val"] # Spawn a process containing off-heap binary references and ensure it's in the top 5 by memory footprint. [shell electric] @@ -62,20 +62,11 @@ [global latest_offset=$1] ??"value":{"val":"3"} - # A live request at the current offset receives no new data, so it long-polls - # until it times out and returns an empty (no-change) up-to-date 200 response - # (this is the only path that sets ot_is_empty_response). Its Plug_shape_get root - # span carries shape_req.is_empty_response=true and must be tail-dropped - # (ELECTRIC_DROP_EMPTY_RESPONSE_SPANS=true is set above), so it must never reach - # the collector. Verified below. The long-poll timeout is 20s by default, so allow - # ample time for the response. - [timeout 60] + # A live request with no new data long-polls until it times out and returns an + # empty up-to-date response — the only path that sets is_empty_response. [invoke curl_shape "http://localhost:3000/v1/shape?table=items&handle=$handle&offset=$latest_offset&live"] ??HTTP/1.1 200 OK - # Body is the up-to-date control only (no data element) — confirms the empty - # no-change path was taken, so the drop assertion below is not vacuous. ??[{"headers":{"control":"up-to-date" - [timeout] [sleep 10] @@ -141,12 +132,10 @@ # (ELECTRIC_OTEL_SAMPLING_RATIO=1.0); it is sampled at 1% by default. [invoke grep_otel_collector_output "Span pg_txn\.replication_client\.transaction_received .*total_processing_time=[0-9]+"] - # A data (non-empty) shape-GET response is exported as a Plug_shape_get root span - # flagged shape_req.is_empty_response=false. + # Verify that a span is exported for a non-empty shape response. [invoke grep_otel_collector_output "Span Plug_shape_get .*shape_req\.is_empty_response=false"] - # The empty/up-to-date long-poll response above was tail-dropped: no Plug_shape_get - # span carrying shape_req.is_empty_response=true is ever exported to the collector. + # Verify that no span is exported for an empty shape response. !test $(docker logs $otel_collector_container_name 2>&1 | python3 $(realpath ../support_files/normalize-otel-output.py) | grep -c "Span Plug_shape_get .*shape_req\.is_empty_response=true") -eq 0 && echo EMPTY_RESPONSE_SPAN_DROPPED ??EMPTY_RESPONSE_SPAN_DROPPED diff --git a/packages/sync-service/lib/electric/config.ex b/packages/sync-service/lib/electric/config.ex index 70fee25429..2966bc351f 100644 --- a/packages/sync-service/lib/electric/config.ex +++ b/packages/sync-service/lib/electric/config.ex @@ -85,8 +85,7 @@ defmodule Electric.Config do otel_sampling_ratio: 0.01, metrics_sampling_ratio: 1, # When true, the OTel spans of empty/up-to-date shape-GET responses are tail-dropped - # at export time to cut trace volume. Defaults to false (empties exported normally); - # set it true to enable the drop. See Electric.Telemetry.EmptyResponseSampler. + # at export time to cut trace volume. See Electric.Telemetry.EmptyResponseSampler. drop_empty_response_spans?: false, ## Memory # After this duration of inactivity, consumer processes will hibernate diff --git a/packages/sync-service/lib/electric/plug/serve_shape_plug.ex b/packages/sync-service/lib/electric/plug/serve_shape_plug.ex index 7bbadaf590..86bf0f41dc 100644 --- a/packages/sync-service/lib/electric/plug/serve_shape_plug.ex +++ b/packages/sync-service/lib/electric/plug/serve_shape_plug.ex @@ -520,9 +520,6 @@ defmodule Electric.Plug.ServeShapePlug do conn end - # The `SampleRate` span attribute, combining the upstream rate hint with the - # empty-response tail-drop decision. The drop overrides the base rate with the - # `SampleRate = 0` sentinel for empty, non-SSE responses when enabled. defp sample_rate_attrs(conn) do base_attrs = TraceContextPlug.sample_rate_attrs(conn, conn.status) trace_attrs = response_trace_attrs(conn) diff --git a/packages/sync-service/lib/electric/telemetry/empty_response_sampler.ex b/packages/sync-service/lib/electric/telemetry/empty_response_sampler.ex index 87eb5d3ddb..2dae82821e 100644 --- a/packages/sync-service/lib/electric/telemetry/empty_response_sampler.ex +++ b/packages/sync-service/lib/electric/telemetry/empty_response_sampler.ex @@ -3,14 +3,11 @@ defmodule Electric.Telemetry.EmptyResponseSampler do Decides the `SampleRate` weight for a shape-GET root span, tail-dropping the spans of empty/up-to-date responses to cut trace volume. - The vast majority of shape-GET responses are empty (up-to-date, incl. long-poll - timeouts) and carry no useful trace detail, yet they dominate exported span volume. When the drop is enabled, such spans are stamped with `SampleRate = 0`, which `Electric.Telemetry.OpenTelemetry.EmptyResponseDropProcessor` recognises as a sentinel to drop the span before it is queued for export. - The decision is a pure function so it can be unit-tested in isolation. Ordering mirrors - the worker's `traceSampleRate` (see stratovolt#1611): + The decision, in order: 1. Error responses (`status >= 500`) are kept and stamped with `SampleRate = 1` — keep-on-error wins and is checked first. @@ -18,10 +15,6 @@ defmodule Electric.Telemetry.EmptyResponseSampler do enabled. 3. Everything else is left unchanged (`:unchanged`), preserving whatever base `SampleRate` the upstream rate hint produced. - - Ratio-based sampling of empties is expected to live on the cloud worker side and reach - the origin via the tracestate rate hint, so the origin only needs an all-or-nothing - toggle rather than its own ratio knob. """ @drop_sample_rate 0 diff --git a/packages/sync-service/lib/electric/telemetry/open_telemetry/config.ex b/packages/sync-service/lib/electric/telemetry/open_telemetry/config.ex index 480ae2c218..df2645b1ec 100644 --- a/packages/sync-service/lib/electric/telemetry/open_telemetry/config.ex +++ b/packages/sync-service/lib/electric/telemetry/open_telemetry/config.ex @@ -38,8 +38,8 @@ defmodule Electric.Telemetry.OpenTelemetry.Config do resource: otel_resource, processors: [ - # Runs first so its `:dropped` return short-circuits the SDK's `andalso` fold - # before the batch processor queues the span for export. + # Must run before the exporting processors so that a span it drops is never + # queued for export. {Electric.Telemetry.OpenTelemetry.EmptyResponseDropProcessor, %{}}, otel_batch_processor, otel_simple_processor diff --git a/packages/sync-service/lib/electric/telemetry/open_telemetry/empty_response_drop_processor.ex b/packages/sync-service/lib/electric/telemetry/open_telemetry/empty_response_drop_processor.ex index 343ce296d4..15c3bef1d7 100644 --- a/packages/sync-service/lib/electric/telemetry/open_telemetry/empty_response_drop_processor.ex +++ b/packages/sync-service/lib/electric/telemetry/open_telemetry/empty_response_drop_processor.ex @@ -7,11 +7,10 @@ if Electric.telemetry_enabled?() do An OTel span processor that tail-drops spans stamped with the `SampleRate = 0` sentinel by `Electric.Telemetry.EmptyResponseSampler`. - The SDK folds `on_end/2` over the processor list with a short-circuiting `andalso` - (`Bool andalso P:on_end(Span, Config) =:= true`), so a processor returning anything - other than `true` before `otel_batch_processor` prevents the batch processor from ever - seeing the span, and it is never queued for export. This processor is therefore - registered first (see `Electric.Telemetry.OpenTelemetry.Config`). + `on_end/2` returns `:dropped` for such spans. The SDK stops running the remaining + processors for a span as soon as one declines it, so registering this processor + ahead of the exporting processors (see `Electric.Telemetry.OpenTelemetry.Config`) + keeps a dropped span from ever being queued for export. Only empty/up-to-date shape-GET response spans carry `SampleRate = 0`; every other span passes through unchanged. From 79aebf8a9f5d8d8ee7234093465daf5e3dd2fee7 Mon Sep 17 00:00:00 2001 From: Oleksii Sholik Date: Fri, 3 Jul 2026 18:13:50 +0200 Subject: [PATCH 10/10] feat(telemetry): tag the serve_shape request metric with the empty flag Add `empty` to the [:electric, :plug, :serve_shape] event metadata and to the electric.plug.serve_shape.requests.count metric's tags. This keeps a precise count of empty vs non-empty responses via metrics even when their spans are dropped (ELECTRIC_DROP_EMPTY_RESPONSE_SPANS=true), since the metric path is independent of span sampling. The otel-export test asserts the dropped empty response is still counted with empty=true in the exported metric. Co-Authored-By: Claude Opus 4.8 --- integration-tests/tests/otel-export.lux | 4 ++++ .../lib/electric/telemetry/stack_telemetry.ex | 2 +- packages/sync-service/lib/electric/plug/serve_shape_plug.ex | 4 +++- 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/integration-tests/tests/otel-export.lux b/integration-tests/tests/otel-export.lux index ec5f9990f3..5868243cc5 100644 --- a/integration-tests/tests/otel-export.lux +++ b/integration-tests/tests/otel-export.lux @@ -139,6 +139,10 @@ !test $(docker logs $otel_collector_container_name 2>&1 | python3 $(realpath ../support_files/normalize-otel-output.py) | grep -c "Span Plug_shape_get .*shape_req\.is_empty_response=true") -eq 0 && echo EMPTY_RESPONSE_SPAN_DROPPED ??EMPTY_RESPONSE_SPAN_DROPPED + # Even though its span was dropped, the empty response is still counted by the + # serve_shape request metric, tagged empty=true. + [invoke grep_otel_collector_output "Metric electric\.plug\.serve_shape\.requests\.count .*empty=true"] + ### [cleanup] diff --git a/packages/electric-telemetry/lib/electric/telemetry/stack_telemetry.ex b/packages/electric-telemetry/lib/electric/telemetry/stack_telemetry.ex index e8d6086110..bdd08a6465 100644 --- a/packages/electric-telemetry/lib/electric/telemetry/stack_telemetry.ex +++ b/packages/electric-telemetry/lib/electric/telemetry/stack_telemetry.ex @@ -94,7 +94,7 @@ defmodule ElectricTelemetry.StackTelemetry do counter("electric.plug.serve_shape.requests.count", event_name: [:electric, :plug, :serve_shape], measurement: :count, - tags: [:status, :known_error, :live] + tags: [:status, :known_error, :live, :empty] ), distribution("electric.shape.response_size.bytes", unit: :byte, diff --git a/packages/sync-service/lib/electric/plug/serve_shape_plug.ex b/packages/sync-service/lib/electric/plug/serve_shape_plug.ex index 86bf0f41dc..60eb91d8c8 100644 --- a/packages/sync-service/lib/electric/plug/serve_shape_plug.ex +++ b/packages/sync-service/lib/electric/plug/serve_shape_plug.ex @@ -444,6 +444,7 @@ defmodule Electric.Plug.ServeShapePlug do bytes_sent = assigns[:streaming_bytes_sent] || 0 is_live = get_live_mode(assigns) stack_id = get_in(conn.assigns, [:config, :stack_id]) + is_empty = response_trace_attrs(conn)[:ot_is_empty_response] || false OpenTelemetry.execute( [:electric, :plug, :serve_shape], @@ -459,7 +460,8 @@ defmodule Electric.Plug.ServeShapePlug do client_ip: conn.remote_ip, status: conn.status, stack_id: stack_id, - known_error: Api.Response.conn_has_known_error?(conn) + known_error: Api.Response.conn_has_known_error?(conn), + empty: is_empty } )