From ec8b33a244a6898317132116131d768841ad98d7 Mon Sep 17 00:00:00 2001 From: Oleksii Sholik Date: Thu, 18 Jun 2026 19:23:29 +0200 Subject: [PATCH 1/5] feat(telemetry): emit per-individual-table ETS memory/size metrics Wire the already-present EtsTables.top_tables/1 to a new [:ets, :table] telemetry event, emitting ets.table.memory and ets.table.size tagged by table_name + table_type for the top N tables by memory. N is the new top_ets_individual_count option (default 10). Complements the existing per-table_type ets.memory.total aggregate, which is left untouched. Co-Authored-By: Claude Opus 4.8 (1M context) --- .changeset/ets-per-table-memory-metrics.md | 5 ++ .../telemetry/application_telemetry.ex | 16 ++++++ .../lib/electric/telemetry/opts.ex | 1 + .../telemetry/application_telemetry_test.exs | 52 +++++++++++++++++++ 4 files changed, 74 insertions(+) create mode 100644 .changeset/ets-per-table-memory-metrics.md diff --git a/.changeset/ets-per-table-memory-metrics.md b/.changeset/ets-per-table-memory-metrics.md new file mode 100644 index 0000000000..5865859434 --- /dev/null +++ b/.changeset/ets-per-table-memory-metrics.md @@ -0,0 +1,5 @@ +--- +'@core/electric-telemetry': patch +--- + +Emit per-individual-table ETS memory/size telemetry (`ets.table.memory` and `ets.table.size`, tagged by `table_name` and `table_type`) for the top N tables by memory, controlled by the new `top_ets_individual_count` option (default 10). Complements the existing per-`table_type` `ets.memory.total` aggregate. diff --git a/packages/electric-telemetry/lib/electric/telemetry/application_telemetry.ex b/packages/electric-telemetry/lib/electric/telemetry/application_telemetry.ex index d2479ec687..682c82820d 100644 --- a/packages/electric-telemetry/lib/electric/telemetry/application_telemetry.ex +++ b/packages/electric-telemetry/lib/electric/telemetry/application_telemetry.ex @@ -82,6 +82,7 @@ defmodule ElectricTelemetry.ApplicationTelemetry do :process_memory, :process_bin_memory, :ets_memory, + :ets_table_memory, :get_system_load_average, :get_system_memory_usage ], @@ -98,6 +99,8 @@ defmodule ElectricTelemetry.ApplicationTelemetry do last_value("process.bin_memory.max_ref_count", tags: [:process_type]), last_value("process.bin_memory.avg_ref_count", tags: [:process_type]), last_value("ets.memory.total", tags: [:table_type], unit: :byte), + last_value("ets.table.memory", tags: [:table_name, :table_type], unit: :byte), + last_value("ets.table.size", tags: [:table_name, :table_type]), last_value("system.cpu.core_count"), last_value("system.cpu.utilization.total"), last_value("system.load_percent.avg1"), @@ -214,6 +217,19 @@ defmodule ElectricTelemetry.ApplicationTelemetry do end end + def ets_table_memory(%{ + intervals_and_thresholds: %{top_ets_individual_count: individual_count} + }) do + for %{name: name, type: type, memory: memory, size: size} <- + ElectricTelemetry.EtsTables.top_tables(individual_count) do + :telemetry.execute( + [:ets, :table], + %{memory: memory, size: size}, + %{table_name: to_string(name), table_type: to_string(type)} + ) + end + end + def cpu_utilization(_) do case :cpu_sup.util([:per_cpu]) do {:error, reason} -> diff --git a/packages/electric-telemetry/lib/electric/telemetry/opts.ex b/packages/electric-telemetry/lib/electric/telemetry/opts.ex index fedd966578..992bb4626f 100644 --- a/packages/electric-telemetry/lib/electric/telemetry/opts.ex +++ b/packages/electric-telemetry/lib/electric/telemetry/opts.ex @@ -38,6 +38,7 @@ defmodule ElectricTelemetry.Opts do default: {:count, 5} ], top_ets_table_count: [type: :integer, default: 10], + top_ets_individual_count: [type: :integer, default: 10], # Garbage collection should run almost instantly since each process has its own heap that # is garbage collected independently of others. 50ms might be too generous. long_gc_threshold: [type: :integer, default: 50], diff --git a/packages/electric-telemetry/test/electric/telemetry/application_telemetry_test.exs b/packages/electric-telemetry/test/electric/telemetry/application_telemetry_test.exs index 5a4eba5657..74eb708b87 100644 --- a/packages/electric-telemetry/test/electric/telemetry/application_telemetry_test.exs +++ b/packages/electric-telemetry/test/electric/telemetry/application_telemetry_test.exs @@ -72,4 +72,56 @@ defmodule ElectricTelemetry.ApplicationTelemetryTest do end end end + + describe "ets_table_memory/1" do + test "emits a [:ets, :table] event per top table with memory/size and name/type tags" do + table = :ets.new(:"ApplicationTelemetryTest:ets_table_memory", [:public, :named_table]) + for i <- 1..200, do: :ets.insert(table, {i, :binary.copy(<<0>>, 1000)}) + + ref = make_ref() + test_pid = self() + handler_id = {__MODULE__, :ets_table, ref} + + :telemetry.attach( + handler_id, + [:ets, :table], + fn _event, measurements, metadata, _config -> + send(test_pid, {ref, measurements, metadata}) + end, + nil + ) + + on_exit(fn -> :telemetry.detach(handler_id) end) + + ApplicationTelemetry.ets_table_memory(%{ + intervals_and_thresholds: %{top_ets_individual_count: 100} + }) + + events = collect_events(ref, []) + + assert events != [] + + for {measurements, metadata} <- events do + assert %{memory: memory, size: size} = measurements + assert is_integer(memory) and memory > 0 + assert is_integer(size) and size >= 0 + assert %{table_name: name, table_type: type} = metadata + assert is_binary(name) + assert is_binary(type) + end + + # Our named test table should be among the emitted tables. + assert Enum.any?(events, fn {_measurements, metadata} -> + metadata.table_name == "ApplicationTelemetryTest:ets_table_memory" + end) + end + end + + defp collect_events(ref, acc) do + receive do + {^ref, measurements, metadata} -> collect_events(ref, [{measurements, metadata} | acc]) + after + 0 -> Enum.reverse(acc) + end + end end From 89f49b7ac2a4130ec2ff43ebc305f5272cbc7943 Mon Sep 17 00:00:00 2001 From: Oleksii Sholik Date: Fri, 19 Jun 2026 11:28:04 +0200 Subject: [PATCH 2/5] refactor(telemetry): drop redundant to_string/1 on ETS tag values MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every metrics reporter already stringifies tag values itself — OTel (OtlpUtils.to_kv_value has an is_atom clause), Prometheus (exporter escape/1 starts with to_string/1), and StatsD (both standard and datadog formatters to_string the value). So the to_string/1 at the telemetry call site was a no-op. Pass the raw atom/binary tag values through for ets.memory.total and ets.table.{memory,size}. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../lib/electric/telemetry/application_telemetry.ex | 4 ++-- .../test/electric/telemetry/application_telemetry_test.exs | 6 ++++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/packages/electric-telemetry/lib/electric/telemetry/application_telemetry.ex b/packages/electric-telemetry/lib/electric/telemetry/application_telemetry.ex index 682c82820d..dbe857404f 100644 --- a/packages/electric-telemetry/lib/electric/telemetry/application_telemetry.ex +++ b/packages/electric-telemetry/lib/electric/telemetry/application_telemetry.ex @@ -213,7 +213,7 @@ defmodule ElectricTelemetry.ApplicationTelemetry do def ets_memory(%{intervals_and_thresholds: %{top_ets_table_count: ets_table_count}}) do for %{type: type, memory: memory} <- ElectricTelemetry.EtsTables.top_by_type(ets_table_count) do - :telemetry.execute([:ets, :memory], %{total: memory}, %{table_type: to_string(type)}) + :telemetry.execute([:ets, :memory], %{total: memory}, %{table_type: type}) end end @@ -225,7 +225,7 @@ defmodule ElectricTelemetry.ApplicationTelemetry do :telemetry.execute( [:ets, :table], %{memory: memory, size: size}, - %{table_name: to_string(name), table_type: to_string(type)} + %{table_name: name, table_type: type} ) end end diff --git a/packages/electric-telemetry/test/electric/telemetry/application_telemetry_test.exs b/packages/electric-telemetry/test/electric/telemetry/application_telemetry_test.exs index 74eb708b87..cc9cecc764 100644 --- a/packages/electric-telemetry/test/electric/telemetry/application_telemetry_test.exs +++ b/packages/electric-telemetry/test/electric/telemetry/application_telemetry_test.exs @@ -105,14 +105,16 @@ defmodule ElectricTelemetry.ApplicationTelemetryTest do assert %{memory: memory, size: size} = measurements assert is_integer(memory) and memory > 0 assert is_integer(size) and size >= 0 + # Tag values are left as-is (atom name, string type); every reporter + # stringifies tag values itself, so no to_string/1 at the call site. assert %{table_name: name, table_type: type} = metadata - assert is_binary(name) + assert is_atom(name) assert is_binary(type) end # Our named test table should be among the emitted tables. assert Enum.any?(events, fn {_measurements, metadata} -> - metadata.table_name == "ApplicationTelemetryTest:ets_table_memory" + metadata.table_name == :"ApplicationTelemetryTest:ets_table_memory" end) end end From f644c8fce1030e55ac5ee60caef5347f997c5424 Mon Sep 17 00:00:00 2001 From: Oleksii Sholik Date: Fri, 19 Jun 2026 12:09:47 +0200 Subject: [PATCH 3/5] refactor(telemetry): drop redundant to_string/1 on process tag values Same as the ETS tag cleanup: every reporter that carries these tagged metrics (OTel, Prometheus, StatsD) stringifies tag values itself, and the CallHomeReporter doesn't carry the process.* metrics at all. So the to_string/1 on process_type in process_memory/1 and process_bin_memory/1 was a no-op. Pass map.type through raw. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../lib/electric/telemetry/application_telemetry.ex | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/electric-telemetry/lib/electric/telemetry/application_telemetry.ex b/packages/electric-telemetry/lib/electric/telemetry/application_telemetry.ex index dbe857404f..517e26917c 100644 --- a/packages/electric-telemetry/lib/electric/telemetry/application_telemetry.ex +++ b/packages/electric-telemetry/lib/electric/telemetry/application_telemetry.ex @@ -189,7 +189,7 @@ defmodule ElectricTelemetry.ApplicationTelemetry do :telemetry.execute( [:process, :memory], %{total: map.proc_mem}, - %{process_type: to_string(map.type)} + %{process_type: map.type} ) end end @@ -205,7 +205,7 @@ defmodule ElectricTelemetry.ApplicationTelemetry do max_ref_count: map.max_ref_count, avg_ref_count: map.avg_ref_count }, - %{process_type: to_string(map.type)} + %{process_type: map.type} ) end end From e653bf57003ce9b0e274b6e8659c7fcaf9c028c3 Mon Sep 17 00:00:00 2001 From: Oleksii Sholik Date: Fri, 19 Jun 2026 12:20:14 +0200 Subject: [PATCH 4/5] fix(telemetry): keep per-table ETS gauges off the Prometheus registry ets.table.memory/size are tagged by the raw table name (per-shape/stack ids), so the top-N set rotates over time. OTel clears series between exports and StatsD is per-scrape, so a rotating high-cardinality set is fine there (and is the intended trade-off for Honeycomb). But TelemetryMetricsPrometheus.Core has no series TTL, so those series would accumulate in the local registry indefinitely and report stale frozen last_values for tables that drop out of the top N. Exclude ets.table.* from the Prometheus reporter's metric list (the bounded ets.memory.total by table_type still goes to Prometheus). Add a routing test asserting ets.table.* reach OTel + StatsD but not Prometheus. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../telemetry/application_telemetry.ex | 20 ++++++++++-- .../telemetry/application_telemetry_test.exs | 31 +++++++++++++++++++ 2 files changed, 48 insertions(+), 3 deletions(-) diff --git a/packages/electric-telemetry/lib/electric/telemetry/application_telemetry.ex b/packages/electric-telemetry/lib/electric/telemetry/application_telemetry.ex index 517e26917c..8f3add8a34 100644 --- a/packages/electric-telemetry/lib/electric/telemetry/application_telemetry.ex +++ b/packages/electric-telemetry/lib/electric/telemetry/application_telemetry.ex @@ -42,9 +42,18 @@ defmodule ElectricTelemetry.ApplicationTelemetry do defp exporter_child_specs(opts) do metrics = metrics(opts) - # Metrics that should reach Prometheus only, e.g. stack-level metrics that the other - # reporters already export per-stack. Appending them here avoids double-reporting. - prometheus_metrics = metrics ++ Map.get(opts, :additional_prometheus_metrics, []) + # `ets.table.*` are high-cardinality per-table gauges tagged by the raw table name + # (which embeds per-shape/stack ids). That's the intended trade-off for OTel/Honeycomb + # and is harmless for StatsD (series are cleared between OTel exports and sent per-scrape + # over StatsD), but `TelemetryMetricsPrometheus.Core` keeps every series in its registry + # with no TTL — so a rotating top-N set would accumulate stale series indefinitely. Keep + # these off the Prometheus `/metrics` path; its `ets.memory.total` (by `table_type`) stays. + # + # `additional_prometheus_metrics` are, conversely, Prometheus-only extras (stack-level + # metrics the other reporters already export per-stack) appended to avoid double-reporting. + prometheus_metrics = + Enum.reject(metrics, &prometheus_excluded?/1) ++ + Map.get(opts, :additional_prometheus_metrics, []) [ Reporters.CallHomeReporter.child_spec( @@ -57,6 +66,11 @@ defmodule ElectricTelemetry.ApplicationTelemetry do ] end + # Per-table ETS gauges are excluded from the (TTL-less) Prometheus registry; see + # `exporter_child_specs/1`. They still flow to OTel and StatsD. + defp prometheus_excluded?(%{name: [:ets, :table | _]}), do: true + defp prometheus_excluded?(_), do: false + @impl ElectricTelemetry.Poller def builtin_periodic_measurements(telemetry_opts) do [ diff --git a/packages/electric-telemetry/test/electric/telemetry/application_telemetry_test.exs b/packages/electric-telemetry/test/electric/telemetry/application_telemetry_test.exs index cc9cecc764..33dc997ad3 100644 --- a/packages/electric-telemetry/test/electric/telemetry/application_telemetry_test.exs +++ b/packages/electric-telemetry/test/electric/telemetry/application_telemetry_test.exs @@ -35,6 +35,37 @@ defmodule ElectricTelemetry.ApplicationTelemetryTest do end end + describe "ets.table.* metric routing" do + setup do + {:ok, opts} = + ElectricTelemetry.validate_options( + instance_id: "test-instance", + version: "1.0.0", + reporters: [prometheus?: true, statsd_host: "localhost", otel_metrics?: true] + ) + + {:ok, {_flags, children}} = ApplicationTelemetry.init(opts) + %{children: children} + end + + test "reach OTel and StatsD but are kept off the Prometheus registry", %{children: children} do + ets_table? = &(&1.name in [[:ets, :table, :memory], [:ets, :table, :size]]) + + assert Enum.any?(reporter_metrics(children, OtelMetricExporter), ets_table?) + assert Enum.any?(reporter_metrics(children, TelemetryMetricsStatsd), ets_table?) + refute Enum.any?(reporter_metrics(children, :prometheus_metrics), ets_table?) + end + + test "the bounded ets.memory.total (by table_type) still reaches Prometheus", %{ + children: children + } do + assert Enum.any?( + reporter_metrics(children, :prometheus_metrics), + &(&1.name == [:ets, :memory, :total]) + ) + end + end + # `Supervisor.init/2` normalises child specs into maps, nesting the reporter's start args # (which carry `:metrics`) inside `:start`. Reporters are identified by their child spec id. defp reporter_metrics(children, id) do From a733372fed189f19f0ab8ce106277fade9bcbb82 Mon Sep 17 00:00:00 2001 From: Oleksii Sholik Date: Fri, 19 Jun 2026 12:48:09 +0200 Subject: [PATCH 5/5] docs(telemetry): clarify OTel vs StatsD wording in prometheus-exclusion comment --- .../lib/electric/telemetry/application_telemetry.ex | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/packages/electric-telemetry/lib/electric/telemetry/application_telemetry.ex b/packages/electric-telemetry/lib/electric/telemetry/application_telemetry.ex index 8f3add8a34..51b5d0f465 100644 --- a/packages/electric-telemetry/lib/electric/telemetry/application_telemetry.ex +++ b/packages/electric-telemetry/lib/electric/telemetry/application_telemetry.ex @@ -43,9 +43,10 @@ defmodule ElectricTelemetry.ApplicationTelemetry do metrics = metrics(opts) # `ets.table.*` are high-cardinality per-table gauges tagged by the raw table name - # (which embeds per-shape/stack ids). That's the intended trade-off for OTel/Honeycomb - # and is harmless for StatsD (series are cleared between OTel exports and sent per-scrape - # over StatsD), but `TelemetryMetricsPrometheus.Core` keeps every series in its registry + # (which embeds per-shape/stack ids). That rotating top-N set is the intended trade-off + # for OTel/Honeycomb (the exporter clears its series between exports) and is harmless for + # StatsD (each value is pushed as it's emitted, so nothing accumulates in-process). But + # `TelemetryMetricsPrometheus.Core` keeps every series in its registry # with no TTL — so a rotating top-N set would accumulate stale series indefinitely. Keep # these off the Prometheus `/metrics` path; its `ets.memory.total` (by `table_type`) stays. #