From 0a83552bfc5e94da038b490fb2b387e7a0b5c2bc Mon Sep 17 00:00:00 2001 From: Tomasz Kopacki Date: Fri, 6 Feb 2026 14:53:44 +0100 Subject: [PATCH 01/10] Apply specific variant types for ChainIndexingMetrics --- packages/ponder-sdk/src/client.test.ts | 2 +- .../src/deserialize/indexing-metrics.mock.ts | 77 ++++--- .../src/deserialize/indexing-metrics.ts | 216 +++++++++++++----- packages/ponder-sdk/src/deserialize/utils.ts | 37 +++ packages/ponder-sdk/src/indexing-metrics.ts | 131 ++++++++--- 5 files changed, 346 insertions(+), 117 deletions(-) create mode 100644 packages/ponder-sdk/src/deserialize/utils.ts diff --git a/packages/ponder-sdk/src/client.test.ts b/packages/ponder-sdk/src/client.test.ts index dd074c3fd..d869117d0 100644 --- a/packages/ponder-sdk/src/client.test.ts +++ b/packages/ponder-sdk/src/client.test.ts @@ -189,7 +189,7 @@ describe("Ponder Client", () => { // Act & Assert await expect(ponderClient.metrics()).rejects.toThrowError( - /Invalid serialized Ponder Indexing Metrics.*Chain Indexing Metrics cannot have both `indexingCompleted` and `indexingRealtime` as `true`/, + /Invalid serialized Ponder Indexing Metrics.*'ponder_sync_is_complete' and 'ponder_sync_is_realtime' metrics cannot both be 1 at the same time for chain 10/, ); }); }); diff --git a/packages/ponder-sdk/src/deserialize/indexing-metrics.mock.ts b/packages/ponder-sdk/src/deserialize/indexing-metrics.mock.ts index 7e1d0a25e..eaca64a4d 100644 --- a/packages/ponder-sdk/src/deserialize/indexing-metrics.mock.ts +++ b/packages/ponder-sdk/src/deserialize/indexing-metrics.mock.ts @@ -1,4 +1,6 @@ import { + type ChainIndexingMetricsRealtime, + ChainIndexingMetricTypes, PonderAppCommands, type PonderIndexingMetrics, PonderIndexingOrderings, @@ -28,6 +30,15 @@ ponder_sync_block_timestamp{chain="534352"} 1770136654 ponder_sync_block_timestamp{chain="42161"} 1770136654 ponder_sync_block_timestamp{chain="59144"} 1770136654 +# HELP ponder_historical_completed_indexing_seconds Number of seconds that have been completed +# TYPE ponder_historical_completed_indexing_seconds gauge +ponder_historical_completed_indexing_seconds{chain="10"} 34242 +ponder_historical_completed_indexing_seconds{chain="1"} 23124 +ponder_historical_completed_indexing_seconds{chain="8453"} 53253 +ponder_historical_completed_indexing_seconds{chain="534352"} 32503 +ponder_historical_completed_indexing_seconds{chain="42161"} 76864 +ponder_historical_completed_indexing_seconds{chain="59144"} 34235 + # HELP ponder_sync_is_realtime Boolean (0 or 1) indicating if the sync is realtime mode # TYPE ponder_sync_is_realtime gauge ponder_sync_is_realtime{chain="42161"} 1 @@ -55,56 +66,44 @@ ponder_historical_total_blocks{chain="59144"} 21873991 [ 10, { - backfillSyncBlocksTotal: 36827849, - indexingCompleted: false, - indexingRealtime: true, - latestSyncedBlock: { number: 147268938, timestamp: 1770136653 }, - }, + type: ChainIndexingMetricTypes.Realtime, + latestKnownBlock: { number: 147268938, timestamp: 1770136653 }, + } satisfies ChainIndexingMetricsRealtime, ], [ 1, { - backfillSyncBlocksTotal: 21042285, - indexingCompleted: false, - indexingRealtime: true, - latestSyncedBlock: { number: 24377568, timestamp: 1770136655 }, - }, + type: ChainIndexingMetricTypes.Realtime, + latestKnownBlock: { number: 24377568, timestamp: 1770136655 }, + } satisfies ChainIndexingMetricsRealtime, ], [ 8453, { - backfillSyncBlocksTotal: 24103899, - indexingCompleted: false, - indexingRealtime: true, - latestSyncedBlock: { number: 41673653, timestamp: 1770136653 }, - }, + type: ChainIndexingMetricTypes.Realtime, + latestKnownBlock: { number: 41673653, timestamp: 1770136653 }, + } satisfies ChainIndexingMetricsRealtime, ], [ 534352, { - backfillSyncBlocksTotal: 12693186, - indexingCompleted: false, - indexingRealtime: true, - latestSyncedBlock: { number: 29373405, timestamp: 1770136654 }, - }, + type: ChainIndexingMetricTypes.Realtime, + latestKnownBlock: { number: 29373405, timestamp: 1770136654 }, + } satisfies ChainIndexingMetricsRealtime, ], [ 42161, { - backfillSyncBlocksTotal: 78607197, - indexingCompleted: false, - indexingRealtime: true, - latestSyncedBlock: { number: 428248999, timestamp: 1770136654 }, - }, + type: ChainIndexingMetricTypes.Realtime, + latestKnownBlock: { number: 428248999, timestamp: 1770136654 }, + } satisfies ChainIndexingMetricsRealtime, ], [ 59144, { - backfillSyncBlocksTotal: 21873991, - indexingCompleted: false, - indexingRealtime: true, - latestSyncedBlock: { number: 28584906, timestamp: 1770136654 }, - }, + type: ChainIndexingMetricTypes.Realtime, + latestKnownBlock: { number: 28584906, timestamp: 1770136654 }, + } satisfies ChainIndexingMetricsRealtime, ], ]), } satisfies PonderIndexingMetrics, @@ -134,6 +133,15 @@ ponder_sync_block_timestamp{chain="scroll"} 1770136654 ponder_sync_block_timestamp{chain="arbitrum"} 1770136654 ponder_sync_block_timestamp{chain="linea"} 1770136654 +# HELP ponder_historical_completed_indexing_seconds Number of seconds that have been completed +# TYPE ponder_historical_completed_indexing_seconds gauge +ponder_historical_completed_indexing_seconds{chain="optimism"} 34242 +ponder_historical_completed_indexing_seconds{chain="mainnet"} 23124 +ponder_historical_completed_indexing_seconds{chain="base"} 53253 +ponder_historical_completed_indexing_seconds{chain="scroll"} 32503 +ponder_historical_completed_indexing_seconds{chain="arbitrum"} 76864 +ponder_historical_completed_indexing_seconds{chain="linea"} 34235 + # HELP ponder_sync_is_realtime Boolean (0 or 1) indicating if the sync is realtime mode # TYPE ponder_sync_is_realtime gauge ponder_sync_is_realtime{chain="arbitrum"} 1 @@ -206,6 +214,15 @@ ponder_sync_block_timestamp{chain="534352"} 1770136654 ponder_sync_block_timestamp{chain="42161"} 1770136654 ponder_sync_block_timestamp{chain="59144"} 1770136654 +# HELP ponder_historical_completed_indexing_seconds Number of seconds that have been completed +# TYPE ponder_historical_completed_indexing_seconds gauge +ponder_historical_completed_indexing_seconds{chain="10"} 34242 +ponder_historical_completed_indexing_seconds{chain="1"} 23124 +ponder_historical_completed_indexing_seconds{chain="8453"} 53253 +ponder_historical_completed_indexing_seconds{chain="534352"} 32503 +ponder_historical_completed_indexing_seconds{chain="42161"} 76864 +ponder_historical_completed_indexing_seconds{chain="59144"} 34235 + # HELP ponder_sync_is_realtime Boolean (0 or 1) indicating if the sync is realtime mode # TYPE ponder_sync_is_realtime gauge ponder_sync_is_realtime{chain="42161"} 1 diff --git a/packages/ponder-sdk/src/deserialize/indexing-metrics.ts b/packages/ponder-sdk/src/deserialize/indexing-metrics.ts index b4a1c5365..7f9c4f2d0 100644 --- a/packages/ponder-sdk/src/deserialize/indexing-metrics.ts +++ b/packages/ponder-sdk/src/deserialize/indexing-metrics.ts @@ -11,42 +11,50 @@ import type { ParsePayload } from "zod/v4/core"; import { type BlockRef, schemaBlockRef } from "../blocks"; import { + type ChainIndexingMetrics, + type ChainIndexingMetricsBackfill, + type ChainIndexingMetricsCompleted, + type ChainIndexingMetricsQueued, + type ChainIndexingMetricsRealtime, + ChainIndexingMetricTypes, PonderAppCommands, + type PonderApplicationSettings, type PonderIndexingMetrics, PonderIndexingOrderings, } from "../indexing-metrics"; import { schemaPositiveInteger } from "../numbers"; import { schemaChainIdString } from "./chains"; import { deserializePrometheusMetrics, type PrometheusMetrics } from "./prometheus-metrics-text"; +import type { DeepPartial } from "./utils"; -function invariant_indexingCompletedAndRealtimeAreNotBothTrue( - ctx: ParsePayload, -) { - const data = ctx.value; +const schemaSerializedChainIndexingMetricsQueued = z.object({ + type: z.literal(ChainIndexingMetricTypes.Queued), +}); - if (data.indexingCompleted && data.indexingRealtime) { - ctx.issues.push({ - code: "custom", - input: ctx.value, - message: - "Chain Indexing Metrics cannot have both `indexingCompleted` and `indexingRealtime` as `true`.", - }); - } -} +const schemaSerializedChainIndexingMetricsBackfill = z.object({ + type: z.literal(ChainIndexingMetricTypes.Backfill), + backfillTotalBlocks: schemaPositiveInteger, +}); + +const schemaSerializedChainIndexingMetricsRealtime = z.object({ + type: z.literal(ChainIndexingMetricTypes.Realtime), + latestKnownBlock: schemaBlockRef, +}); + +const schemaSerializedChainIndexingMetricsCompleted = z.object({ + type: z.literal(ChainIndexingMetricTypes.Completed), + targetBlock: schemaBlockRef, +}); /** * Schema describing the chain indexing metrics. */ -const schemaSerializedChainIndexingMetrics = z - .object({ - backfillSyncBlocksTotal: schemaPositiveInteger, - latestSyncedBlock: schemaBlockRef, - indexingCompleted: z.boolean(), - indexingRealtime: z.boolean(), - }) - .check(invariant_indexingCompletedAndRealtimeAreNotBothTrue); - -type SerializedChainIndexingMetrics = z.infer; +const schemaSerializedChainIndexingMetrics = z.discriminatedUnion("type", [ + schemaSerializedChainIndexingMetricsQueued, + schemaSerializedChainIndexingMetricsBackfill, + schemaSerializedChainIndexingMetricsRealtime, + schemaSerializedChainIndexingMetricsCompleted, +]); /** * Schema describing the chains indexing metrics. @@ -56,6 +64,82 @@ const schemaSerializedChainsIndexingMetrics = z.map( schemaSerializedChainIndexingMetrics, ); +/** + * Build unvalidated (and perhaps partial) Chain Indexing Metrics + * + * @param maybeChainId A string maybe representing a chain ID. + * @param prometheusMetrics valid Prometheus Metrics from Ponder app. + * @returns Unvalidated (possibly incomplete) Chain Indexing Metrics + * to be validated by {@link schemaSerializedChainIndexingMetrics}. + */ +function buildUnvalidatedChainIndexingMetrics( + maybeChainId: string, + prometheusMetrics: PrometheusMetrics, +): DeepPartial { + const ponderHistoricalCompletedIndexingSeconds = prometheusMetrics.getValue( + "ponder_historical_completed_indexing_seconds", + { + chain: maybeChainId, + }, + ); + + // If no time has been recorded for historical completed indexing, + // we can assume the chain is still queued to be indexed. + if (ponderHistoricalCompletedIndexingSeconds === 0) { + return { + type: ChainIndexingMetricTypes.Queued, + } satisfies DeepPartial; + } + + const ponderSyncIsComplete = prometheusMetrics.getValue("ponder_sync_is_complete", { + chain: maybeChainId, + }); + + const ponderSyncIsRealtime = prometheusMetrics.getValue("ponder_sync_is_realtime", { + chain: maybeChainId, + }); + + const latestSyncedBlockNumber = prometheusMetrics.getValue("ponder_sync_block", { + chain: maybeChainId, + }); + + const latestSyncedBlockTimestamp = prometheusMetrics.getValue("ponder_sync_block_timestamp", { + chain: maybeChainId, + }); + + const latestSyncedBlock = { + number: latestSyncedBlockNumber, + timestamp: latestSyncedBlockTimestamp, + } satisfies Partial; + + // The `ponder_sync_is_complete` metric is set to `1` if, and only if, + // the indexing has been completed for the chain. + if (ponderSyncIsComplete === 1) { + return { + type: ChainIndexingMetricTypes.Completed, + targetBlock: latestSyncedBlock, + } satisfies DeepPartial; + } + + // The `ponder_sync_is_realtime` metric is set to `1` if, and only if, + // the indexing is currently in real-time for the chain. + if (ponderSyncIsRealtime === 1) { + return { + type: ChainIndexingMetricTypes.Realtime, + latestKnownBlock: latestSyncedBlock, + } satisfies DeepPartial; + } + + const backfillTotalBlocks = prometheusMetrics.getValue("ponder_historical_total_blocks", { + chain: maybeChainId, + }); + + return { + type: ChainIndexingMetricTypes.Backfill, + backfillTotalBlocks, + } satisfies DeepPartial; +} + function invariant_includesAtLeastOneIndexedChain( ctx: ParsePayload, ) { @@ -96,12 +180,15 @@ function invariant_includesRequiredMetrics(ctx: ParsePayload) "ponder_settings_info", "ponder_sync_block", "ponder_sync_block_timestamp", + "ponder_historical_completed_indexing_seconds", "ponder_historical_total_blocks", "ponder_sync_is_complete", "ponder_sync_is_realtime", ]; + // Validate metrics presence invariants. for (const requiredMetricName of requiredMetricNames) { + // Invariant: Required metric must be present in the Prometheus metrics. if (!metricNames.includes(requiredMetricName)) { ctx.issues.push({ code: "custom", @@ -110,6 +197,47 @@ function invariant_includesRequiredMetrics(ctx: ParsePayload) }); } } + + const chainReferences = prometheusMetrics.getLabels("ponder_sync_block", "chain"); + + // Validate per-chain invariants. + for (const chainReference of chainReferences) { + const ponderHistoricalCompletedIndexingSeconds = prometheusMetrics.getValue( + "ponder_historical_completed_indexing_seconds", + { chain: chainReference }, + ); + + // Invariant: historical completed indexing seconds must be a non-negative integer. + if ( + typeof ponderHistoricalCompletedIndexingSeconds !== "number" || + !Number.isInteger(ponderHistoricalCompletedIndexingSeconds) || + ponderHistoricalCompletedIndexingSeconds < 0 + ) { + ctx.issues.push({ + code: "custom", + input: ctx.value, + message: `'ponder_historical_completed_indexing_seconds' metric for '${chainReference}' chain must be a non-negative integer. Received: ${ponderHistoricalCompletedIndexingSeconds}`, + }); + } + + const ponderSyncIsComplete = prometheusMetrics.getValue("ponder_sync_is_complete", { + chain: chainReference, + }); + + const ponderSyncIsRealtime = prometheusMetrics.getValue("ponder_sync_is_realtime", { + chain: chainReference, + }); + + // Invariant: `ponder_sync_is_complete` and `ponder_sync_is_realtime` cannot + // both be `1` at the same time. + if (ponderSyncIsComplete === 1 && ponderSyncIsRealtime === 1) { + ctx.issues.push({ + code: "custom", + input: ctx.value, + message: `'ponder_sync_is_complete' and 'ponder_sync_is_realtime' metrics cannot both be 1 at the same time for chain ${chainReference}`, + }); + } + } } /** @@ -135,46 +263,24 @@ type SerializedPonderIndexingMetrics = z.infer { const appSettings = { command: prometheusMetrics.getLabel("ponder_settings_info", "command"), ordering: prometheusMetrics.getLabel("ponder_settings_info", "ordering"), - }; + } as DeepPartial; const chainReferences = prometheusMetrics.getLabels("ponder_sync_block", "chain"); - - const chains = new Map(); + const chains = new Map>(); for (const maybeChainId of chainReferences) { - const latestSyncedBlock = { - number: prometheusMetrics.getValue("ponder_sync_block", { - chain: maybeChainId, - }), - timestamp: prometheusMetrics.getValue("ponder_sync_block_timestamp", { - chain: maybeChainId, - }), - } satisfies Partial; - - const backfillSyncBlocksTotal = prometheusMetrics.getValue("ponder_historical_total_blocks", { - chain: maybeChainId, - }); + const chainIndexingMetrics = buildUnvalidatedChainIndexingMetrics( + maybeChainId, + prometheusMetrics, + ); - const indexingCompleted = - prometheusMetrics.getValue("ponder_sync_is_complete", { - chain: maybeChainId, - }) === 1; - - const indexingRealtime = - prometheusMetrics.getValue("ponder_sync_is_realtime", { - chain: maybeChainId, - }) === 1; - - chains.set(maybeChainId, { - latestSyncedBlock, - backfillSyncBlocksTotal, - indexingCompleted, - indexingRealtime, - }); + chains.set(maybeChainId, chainIndexingMetrics); } const unvalidatedPonderIndexingMetrics = { diff --git a/packages/ponder-sdk/src/deserialize/utils.ts b/packages/ponder-sdk/src/deserialize/utils.ts new file mode 100644 index 000000000..8e0b44c45 --- /dev/null +++ b/packages/ponder-sdk/src/deserialize/utils.ts @@ -0,0 +1,37 @@ +/** + * A utility type that makes all properties of a type optional recursively, + * including nested objects and arrays. + * + * @example + * ```typescript + * type Config = { + * a: string; + * b: { + * x: number; + * y: { z: boolean }; + * }; + * c: { id: string }[]; + * } + * + * type PartialConfig = DeepPartial; + * // Results in: + * // { + * // a?: string; + * // b?: { + * // x?: number; + * // y?: { z?: boolean }; + * // }; + * // c?: { id?: string }[]; + * // } + * + * // Usage: + * const update: PartialConfig = { b: { y: { z: true } } }; + * ``` + */ +export type DeepPartial = { + [P in keyof T]?: T[P] extends (infer U)[] + ? DeepPartial[] + : T[P] extends object + ? DeepPartial + : T[P]; +}; diff --git a/packages/ponder-sdk/src/indexing-metrics.ts b/packages/ponder-sdk/src/indexing-metrics.ts index 08b390077..8a6593660 100644 --- a/packages/ponder-sdk/src/indexing-metrics.ts +++ b/packages/ponder-sdk/src/indexing-metrics.ts @@ -45,57 +45,126 @@ export interface PonderApplicationSettings { } /** - * Chain Indexing Metrics + * Chain Indexing Metric Types * - * Represents the indexing metrics for a specific chain indexed by a Ponder app. + * Represents the different types of indexing states for a chain indexed by + * a Ponder app. + */ +export const ChainIndexingMetricTypes = { + Queued: "queued", + Backfill: "backfill", + Completed: "completed", + Realtime: "realtime", +} as const; + +export type ChainIndexingMetricType = + (typeof ChainIndexingMetricTypes)[keyof typeof ChainIndexingMetricTypes]; + +/** + * Chain Indexing Metrics Queued + * + * Represents the indexing metrics for a chain that has not started + * indexing yet, and is queued to be indexed by a Ponder app. + */ +export interface ChainIndexingMetricsQueued { + type: typeof ChainIndexingMetricTypes.Queued; +} + +/** + * Chain Indexing Metrics Backfill * - * Guarantees: - * - `indexingCompleted` and `indexingRealtime` cannot both be `true` - * at the same time. All other combinations are valid. + * Represents the indexing metrics for a chain that is currently in + * the backfill phase of indexing by a Ponder app. */ -export interface ChainIndexingMetrics { +export interface ChainIndexingMetricsBackfill { + type: typeof ChainIndexingMetricTypes.Backfill; /** - * Number of blocks required to be synced to complete - * the backfill phase of indexing. + * Number of blocks required to be indexed during backfill. * * This value is calculated by Ponder at the time - * the backfill starts. It corresponds to the number of blocks between: - * - the first block to be indexed (specified in Ponder config), and - * - the last block to be indexed during backfill. - * The last block to be indexed during backfill is one of: - * - The specified end block (if any) in the Ponder config, or - * - The latest known block at the time the backfill started. + * the backfill starts. + * + * If Ponder config specifies a "config end block" for the chain, + * the `backfillTotalBlocks` will be the number of blocks + * between the "config start block" and the specified "config end block". + * For example: + * ``` + * backfillTotalBlocks = configEndBlock - configStartBlock + 1 + * ``` + * + * If Ponder config does not specify the "config end block" for the chain, + * the `backfillTotalBlocks` will be the number of blocks + * between the "config start block" and the "latest known block" + * for the chain at the time the backfill starts. + * The "latest known block" is the "highest" block that has been + * discovered by RPCs and stored in the RPC cache as of the time + * the metric value was captured. + * + * Each restart of the Ponder app will result in a new value based on + * the current "latest known block" for the chain at that time. + * For example: + * ``` + * backfillTotalBlocks = latestKnownBlock - configStartBlock + 1 + * ``` * * Guarantees: * - Is a positive integer. */ - backfillSyncBlocksTotal: number; + backfillTotalBlocks: number; +} - /** - * Latest synced block - * - * Corresponds to the latest block stored in the RPC cache for the chain. - */ - latestSyncedBlock: BlockRef; +/** + * Chain Indexing Metrics Realtime + * + * Represents the indexing metrics for a chain that is currently in + * the real-time indexing phase by a Ponder app. It means that + * the backfill phase transitioned to completed phase, as there was + * no "config end block" specified for the chain. + * + * The indexing continues in real-time, with no "target end block". + * The "latest known block" is continuously updated as new blocks are + * discovered by RPCs and stored in the RPC cache. + */ +export interface ChainIndexingMetricsRealtime { + type: typeof ChainIndexingMetricTypes.Realtime; /** - * Is indexing completed for the chain? - * - * This will be true when the backfill has been completed, - * and a specified end block for the chain has been reached. + * A {@link BlockRef} to the "highest" block that has been discovered by RPCs + * and stored in the RPC cache as of the time the metric value was captured. */ - indexingCompleted: boolean; + latestKnownBlock: BlockRef; +} + +/** + * Chain Indexing Metrics Completed + * + * Represents the indexing metrics for a chain that has completed indexing by + * a Ponder app. It means that the backfill phase transitioned to completed phase. + * No more blocks are required to be indexed for the chain at this point. + */ +export interface ChainIndexingMetricsCompleted { + type: typeof ChainIndexingMetricTypes.Completed; /** - * Is indexing following in real-time for the chain? + * Target block * - * This will be true when the backfill has been completed, - * and there was no specified end block for the chain, - * so indexing continues in real-time. + * A {@link BlockRef} to the block that was the target of the indexing for the chain. + * There are no more blocks required to be indexed for the chain after this block. */ - indexingRealtime: boolean; + targetBlock: BlockRef; } +/** + * Chain Indexing Metrics + * + * Represents the indexing metrics for a specific chain indexed by a Ponder app. + */ +export type ChainIndexingMetrics = + | ChainIndexingMetricsQueued + | ChainIndexingMetricsBackfill + | ChainIndexingMetricsCompleted + | ChainIndexingMetricsRealtime; + /** * Ponder Indexing Metrics * From 045a3f5de8bd633fcc7502620e2a4c2873a6c626 Mon Sep 17 00:00:00 2001 From: Tomasz Kopacki Date: Fri, 6 Feb 2026 15:16:15 +0100 Subject: [PATCH 02/10] Fix typos --- packages/ponder-sdk/src/deserialize/indexing-metrics.ts | 2 +- packages/ponder-sdk/src/indexing-metrics.ts | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/ponder-sdk/src/deserialize/indexing-metrics.ts b/packages/ponder-sdk/src/deserialize/indexing-metrics.ts index 7f9c4f2d0..786554803 100644 --- a/packages/ponder-sdk/src/deserialize/indexing-metrics.ts +++ b/packages/ponder-sdk/src/deserialize/indexing-metrics.ts @@ -122,7 +122,7 @@ function buildUnvalidatedChainIndexingMetrics( } // The `ponder_sync_is_realtime` metric is set to `1` if, and only if, - // the indexing is currently in real-time for the chain. + // the indexing is currently in realtime for the chain. if (ponderSyncIsRealtime === 1) { return { type: ChainIndexingMetricTypes.Realtime, diff --git a/packages/ponder-sdk/src/indexing-metrics.ts b/packages/ponder-sdk/src/indexing-metrics.ts index 8a6593660..f73551a6b 100644 --- a/packages/ponder-sdk/src/indexing-metrics.ts +++ b/packages/ponder-sdk/src/indexing-metrics.ts @@ -117,11 +117,11 @@ export interface ChainIndexingMetricsBackfill { * Chain Indexing Metrics Realtime * * Represents the indexing metrics for a chain that is currently in - * the real-time indexing phase by a Ponder app. It means that - * the backfill phase transitioned to completed phase, as there was + * the realtime indexing phase by a Ponder app. It means that + * the backfill phase transitioned to realtime phase, as there was * no "config end block" specified for the chain. * - * The indexing continues in real-time, with no "target end block". + * The indexing continues in realtime, with no "target end block". * The "latest known block" is continuously updated as new blocks are * discovered by RPCs and stored in the RPC cache. */ From 83c0cf98f09c3474c8dcea602e2acf2179733cb0 Mon Sep 17 00:00:00 2001 From: Tomasz Kopacki Date: Fri, 6 Feb 2026 17:23:05 +0100 Subject: [PATCH 03/10] Improve `PonderIndexingStatus` data model --- .../src/deserialize/indexing-status.ts | 7 +++-- packages/ponder-sdk/src/indexing-status.ts | 26 ++++++++++++++++--- 2 files changed, 25 insertions(+), 8 deletions(-) diff --git a/packages/ponder-sdk/src/deserialize/indexing-status.ts b/packages/ponder-sdk/src/deserialize/indexing-status.ts index 9f998c160..fa85060e2 100644 --- a/packages/ponder-sdk/src/deserialize/indexing-status.ts +++ b/packages/ponder-sdk/src/deserialize/indexing-status.ts @@ -9,11 +9,10 @@ import { prettifyError, z } from "zod/v4"; import type { ParsePayload } from "zod/v4/core"; -import type { BlockRef } from "../blocks"; import { schemaBlockRef } from "../blocks"; import type { ChainId } from "../chains"; import { schemaChainId } from "../chains"; -import type { PonderIndexingStatus } from "../indexing-status"; +import type { ChainIndexingStatus, PonderIndexingStatus } from "../indexing-status"; const schemaSerializedChainName = z.string(); @@ -54,10 +53,10 @@ export type SerializedPonderIndexingStatus = z.infer(); + const chains = new Map(); for (const [, chainData] of Object.entries(data)) { - chains.set(chainData.id, chainData.block); + chains.set(chainData.id, { checkpointBlock: chainData.block }); } return { diff --git a/packages/ponder-sdk/src/indexing-status.ts b/packages/ponder-sdk/src/indexing-status.ts index 8bd444b4d..d59327869 100644 --- a/packages/ponder-sdk/src/indexing-status.ts +++ b/packages/ponder-sdk/src/indexing-status.ts @@ -1,6 +1,27 @@ import type { BlockRef } from "./blocks"; import type { ChainId } from "./chains"; +/** + * Chain Indexing Status + * + * Represents the indexing status for a specific chain in a Ponder app. + */ +export interface ChainIndexingStatus { + /** + * Checkpoint Block + * + * During omnichain indexing, a Ponder app indexes the chain and + * keeps track of a checkpoint block for each indexed chain. + * + * @see https://ponder.sh/docs/api-reference/ponder/database#checkpoint-table + * + * The `checkpointBlock` is a reference to either: + * - the first block to be indexed for the chain (if indexing is queued), or + * - the last indexed block for the chain (if indexing is in progress). + */ + checkpointBlock: BlockRef; +} + /** * Ponder Indexing Status * @@ -12,9 +33,6 @@ export interface PonderIndexingStatus { * * Guarantees: * - Includes entry for at least one indexed chain. - * - BlockRef corresponds to either: - * - The first block to be indexed (when chain indexing is currently queued). - * - The last indexed block (when chain indexing is currently in progress). */ - chains: Map; + chains: Map; } From 22494363ade1c7e070bd4940d62af1fbd7c85100 Mon Sep 17 00:00:00 2001 From: Tomasz Kopacki Date: Sat, 7 Feb 2026 07:49:26 +0100 Subject: [PATCH 04/10] Update `ChainIndexingMetricsQueued` data model Add `backfillTotalBlocks` field to `ChainIndexingMetricsQueued` The `ponder_historical_total_blocks` metric is calculated for all indexed chains during Ponder app initialization, including queued chains. When indexing multiple chains, typically one chain is in backfill phase while others are queued. Since `ponder_historical_total_blocks` can be useful in both phases, both `ChainIndexingMetricsQueued` and `ChainIndexingMetricsBackfill` now include the `backfillTotalBlocks` field to reference this metric. --- .../src/deserialize/indexing-metrics.ts | 10 ++- packages/ponder-sdk/src/indexing-metrics.ts | 77 +++++++++++-------- 2 files changed, 51 insertions(+), 36 deletions(-) diff --git a/packages/ponder-sdk/src/deserialize/indexing-metrics.ts b/packages/ponder-sdk/src/deserialize/indexing-metrics.ts index 786554803..8136ae0a7 100644 --- a/packages/ponder-sdk/src/deserialize/indexing-metrics.ts +++ b/packages/ponder-sdk/src/deserialize/indexing-metrics.ts @@ -29,6 +29,7 @@ import type { DeepPartial } from "./utils"; const schemaSerializedChainIndexingMetricsQueued = z.object({ type: z.literal(ChainIndexingMetricTypes.Queued), + backfillTotalBlocks: schemaPositiveInteger, }); const schemaSerializedChainIndexingMetricsBackfill = z.object({ @@ -76,6 +77,10 @@ function buildUnvalidatedChainIndexingMetrics( maybeChainId: string, prometheusMetrics: PrometheusMetrics, ): DeepPartial { + const backfillTotalBlocks = prometheusMetrics.getValue("ponder_historical_total_blocks", { + chain: maybeChainId, + }); + const ponderHistoricalCompletedIndexingSeconds = prometheusMetrics.getValue( "ponder_historical_completed_indexing_seconds", { @@ -88,6 +93,7 @@ function buildUnvalidatedChainIndexingMetrics( if (ponderHistoricalCompletedIndexingSeconds === 0) { return { type: ChainIndexingMetricTypes.Queued, + backfillTotalBlocks, } satisfies DeepPartial; } @@ -130,10 +136,6 @@ function buildUnvalidatedChainIndexingMetrics( } satisfies DeepPartial; } - const backfillTotalBlocks = prometheusMetrics.getValue("ponder_historical_total_blocks", { - chain: maybeChainId, - }); - return { type: ChainIndexingMetricTypes.Backfill, backfillTotalBlocks, diff --git a/packages/ponder-sdk/src/indexing-metrics.ts b/packages/ponder-sdk/src/indexing-metrics.ts index f73551a6b..0cc9cdb3c 100644 --- a/packages/ponder-sdk/src/indexing-metrics.ts +++ b/packages/ponder-sdk/src/indexing-metrics.ts @@ -60,14 +60,55 @@ export const ChainIndexingMetricTypes = { export type ChainIndexingMetricType = (typeof ChainIndexingMetricTypes)[keyof typeof ChainIndexingMetricTypes]; +/** + * Number of blocks required to be indexed during backfill. + * + * References `ponder_historical_total_blocks` Ponder metric.. + * + * This value is calculated at the time the Ponder app starts, + * even for each indexed chain. + * + * If Ponder config specifies a "config end block" for the chain, + * the `ponder_historical_total_blocks` will be the number of blocks + * between the "config start block" and the specified "config end block". + * For example: + * ``` + * ponder_historical_total_blocks = configEndBlock - configStartBlock + 1 + * ``` + * + * If Ponder config does not specify the "config end block" for the chain, + * the `ponder_historical_total_blocks` will be the number of blocks + * between the "config start block" and the "latest known block" + * for the chain at the time the backfill starts. + * The "latest known block" is the "highest" block that has been + * discovered by RPCs and stored in the RPC cache as of the time + * the metric value was captured. + * + * Each restart of the Ponder app will result in a new value based on + * the current "latest known block" for the chain at that time. + * For example: + * ``` + * ponder_historical_total_blocks = latestKnownBlock - configStartBlock + 1 + * ``` + * + * Guaranteed to be a positive integer. + */ +export type BackfillTotalBlocks = number; + /** * Chain Indexing Metrics Queued * * Represents the indexing metrics for a chain that has not started - * indexing yet, and is queued to be indexed by a Ponder app. + * indexing yet, and is queued to transition to backfill phase, + * where it will be indexed by a Ponder app */ export interface ChainIndexingMetricsQueued { type: typeof ChainIndexingMetricTypes.Queued; + + /** + * Total number of blocks to be indexed for the chain during backfill phase. + */ + backfillTotalBlocks: BackfillTotalBlocks; } /** @@ -78,39 +119,11 @@ export interface ChainIndexingMetricsQueued { */ export interface ChainIndexingMetricsBackfill { type: typeof ChainIndexingMetricTypes.Backfill; + /** - * Number of blocks required to be indexed during backfill. - * - * This value is calculated by Ponder at the time - * the backfill starts. - * - * If Ponder config specifies a "config end block" for the chain, - * the `backfillTotalBlocks` will be the number of blocks - * between the "config start block" and the specified "config end block". - * For example: - * ``` - * backfillTotalBlocks = configEndBlock - configStartBlock + 1 - * ``` - * - * If Ponder config does not specify the "config end block" for the chain, - * the `backfillTotalBlocks` will be the number of blocks - * between the "config start block" and the "latest known block" - * for the chain at the time the backfill starts. - * The "latest known block" is the "highest" block that has been - * discovered by RPCs and stored in the RPC cache as of the time - * the metric value was captured. - * - * Each restart of the Ponder app will result in a new value based on - * the current "latest known block" for the chain at that time. - * For example: - * ``` - * backfillTotalBlocks = latestKnownBlock - configStartBlock + 1 - * ``` - * - * Guarantees: - * - Is a positive integer. + * Total number of blocks to be indexed for the chain during backfill phase. */ - backfillTotalBlocks: number; + backfillTotalBlocks: BackfillTotalBlocks; } /** From 5e56cc8624668a721c66ae7772e97a8293801c31 Mon Sep 17 00:00:00 2001 From: Tomasz Kopacki Date: Sat, 7 Feb 2026 07:57:20 +0100 Subject: [PATCH 05/10] Fix typo --- packages/ponder-sdk/src/indexing-metrics.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/ponder-sdk/src/indexing-metrics.ts b/packages/ponder-sdk/src/indexing-metrics.ts index 0cc9cdb3c..821ce817a 100644 --- a/packages/ponder-sdk/src/indexing-metrics.ts +++ b/packages/ponder-sdk/src/indexing-metrics.ts @@ -63,7 +63,7 @@ export type ChainIndexingMetricType = /** * Number of blocks required to be indexed during backfill. * - * References `ponder_historical_total_blocks` Ponder metric.. + * References `ponder_historical_total_blocks` Ponder metric. * * This value is calculated at the time the Ponder app starts, * even for each indexed chain. From 886d4eb4b51104527cfc3d3c6a1ff5ad3b126172 Mon Sep 17 00:00:00 2001 From: Tomasz Kopacki Date: Mon, 9 Feb 2026 13:18:53 +0100 Subject: [PATCH 06/10] Apply PR feedback --- .../src/deserialize/indexing-metrics.mock.ts | 26 +++--- .../src/deserialize/indexing-metrics.ts | 28 +++--- packages/ponder-sdk/src/indexing-metrics.ts | 86 ++++++------------- packages/ponder-sdk/src/indexing-status.ts | 9 +- 4 files changed, 60 insertions(+), 89 deletions(-) diff --git a/packages/ponder-sdk/src/deserialize/indexing-metrics.mock.ts b/packages/ponder-sdk/src/deserialize/indexing-metrics.mock.ts index eaca64a4d..79b4f091b 100644 --- a/packages/ponder-sdk/src/deserialize/indexing-metrics.mock.ts +++ b/packages/ponder-sdk/src/deserialize/indexing-metrics.mock.ts @@ -1,6 +1,6 @@ import { type ChainIndexingMetricsRealtime, - ChainIndexingMetricTypes, + ChainIndexingStates, PonderAppCommands, type PonderIndexingMetrics, PonderIndexingOrderings, @@ -66,43 +66,43 @@ ponder_historical_total_blocks{chain="59144"} 21873991 [ 10, { - type: ChainIndexingMetricTypes.Realtime, - latestKnownBlock: { number: 147268938, timestamp: 1770136653 }, + state: ChainIndexingStates.Realtime, + latestSyncedBlock: { number: 147268938, timestamp: 1770136653 }, } satisfies ChainIndexingMetricsRealtime, ], [ 1, { - type: ChainIndexingMetricTypes.Realtime, - latestKnownBlock: { number: 24377568, timestamp: 1770136655 }, + state: ChainIndexingStates.Realtime, + latestSyncedBlock: { number: 24377568, timestamp: 1770136655 }, } satisfies ChainIndexingMetricsRealtime, ], [ 8453, { - type: ChainIndexingMetricTypes.Realtime, - latestKnownBlock: { number: 41673653, timestamp: 1770136653 }, + state: ChainIndexingStates.Realtime, + latestSyncedBlock: { number: 41673653, timestamp: 1770136653 }, } satisfies ChainIndexingMetricsRealtime, ], [ 534352, { - type: ChainIndexingMetricTypes.Realtime, - latestKnownBlock: { number: 29373405, timestamp: 1770136654 }, + state: ChainIndexingStates.Realtime, + latestSyncedBlock: { number: 29373405, timestamp: 1770136654 }, } satisfies ChainIndexingMetricsRealtime, ], [ 42161, { - type: ChainIndexingMetricTypes.Realtime, - latestKnownBlock: { number: 428248999, timestamp: 1770136654 }, + state: ChainIndexingStates.Realtime, + latestSyncedBlock: { number: 428248999, timestamp: 1770136654 }, } satisfies ChainIndexingMetricsRealtime, ], [ 59144, { - type: ChainIndexingMetricTypes.Realtime, - latestKnownBlock: { number: 28584906, timestamp: 1770136654 }, + state: ChainIndexingStates.Realtime, + latestSyncedBlock: { number: 28584906, timestamp: 1770136654 }, } satisfies ChainIndexingMetricsRealtime, ], ]), diff --git a/packages/ponder-sdk/src/deserialize/indexing-metrics.ts b/packages/ponder-sdk/src/deserialize/indexing-metrics.ts index 8136ae0a7..e898fd414 100644 --- a/packages/ponder-sdk/src/deserialize/indexing-metrics.ts +++ b/packages/ponder-sdk/src/deserialize/indexing-metrics.ts @@ -16,7 +16,7 @@ import { type ChainIndexingMetricsCompleted, type ChainIndexingMetricsQueued, type ChainIndexingMetricsRealtime, - ChainIndexingMetricTypes, + ChainIndexingStates, PonderAppCommands, type PonderApplicationSettings, type PonderIndexingMetrics, @@ -28,29 +28,29 @@ import { deserializePrometheusMetrics, type PrometheusMetrics } from "./promethe import type { DeepPartial } from "./utils"; const schemaSerializedChainIndexingMetricsQueued = z.object({ - type: z.literal(ChainIndexingMetricTypes.Queued), + state: z.literal(ChainIndexingStates.Queued), backfillTotalBlocks: schemaPositiveInteger, }); const schemaSerializedChainIndexingMetricsBackfill = z.object({ - type: z.literal(ChainIndexingMetricTypes.Backfill), + state: z.literal(ChainIndexingStates.Backfill), backfillTotalBlocks: schemaPositiveInteger, }); const schemaSerializedChainIndexingMetricsRealtime = z.object({ - type: z.literal(ChainIndexingMetricTypes.Realtime), - latestKnownBlock: schemaBlockRef, + state: z.literal(ChainIndexingStates.Realtime), + latestSyncedBlock: schemaBlockRef, }); const schemaSerializedChainIndexingMetricsCompleted = z.object({ - type: z.literal(ChainIndexingMetricTypes.Completed), - targetBlock: schemaBlockRef, + state: z.literal(ChainIndexingStates.Completed), + finalIndexedBlock: schemaBlockRef, }); /** * Schema describing the chain indexing metrics. */ -const schemaSerializedChainIndexingMetrics = z.discriminatedUnion("type", [ +const schemaSerializedChainIndexingMetrics = z.discriminatedUnion("state", [ schemaSerializedChainIndexingMetricsQueued, schemaSerializedChainIndexingMetricsBackfill, schemaSerializedChainIndexingMetricsRealtime, @@ -92,7 +92,7 @@ function buildUnvalidatedChainIndexingMetrics( // we can assume the chain is still queued to be indexed. if (ponderHistoricalCompletedIndexingSeconds === 0) { return { - type: ChainIndexingMetricTypes.Queued, + state: ChainIndexingStates.Queued, backfillTotalBlocks, } satisfies DeepPartial; } @@ -122,8 +122,8 @@ function buildUnvalidatedChainIndexingMetrics( // the indexing has been completed for the chain. if (ponderSyncIsComplete === 1) { return { - type: ChainIndexingMetricTypes.Completed, - targetBlock: latestSyncedBlock, + state: ChainIndexingStates.Completed, + finalIndexedBlock: latestSyncedBlock, } satisfies DeepPartial; } @@ -131,13 +131,13 @@ function buildUnvalidatedChainIndexingMetrics( // the indexing is currently in realtime for the chain. if (ponderSyncIsRealtime === 1) { return { - type: ChainIndexingMetricTypes.Realtime, - latestKnownBlock: latestSyncedBlock, + state: ChainIndexingStates.Realtime, + latestSyncedBlock, } satisfies DeepPartial; } return { - type: ChainIndexingMetricTypes.Backfill, + state: ChainIndexingStates.Backfill, backfillTotalBlocks, } satisfies DeepPartial; } diff --git a/packages/ponder-sdk/src/indexing-metrics.ts b/packages/ponder-sdk/src/indexing-metrics.ts index 821ce817a..7ac90d480 100644 --- a/packages/ponder-sdk/src/indexing-metrics.ts +++ b/packages/ponder-sdk/src/indexing-metrics.ts @@ -45,55 +45,18 @@ export interface PonderApplicationSettings { } /** - * Chain Indexing Metric Types + * Chain Indexing States * - * Represents the different types of indexing states for a chain indexed by - * a Ponder app. + * Represents the indexing state of a chain indexed by a Ponder app. */ -export const ChainIndexingMetricTypes = { +export const ChainIndexingStates = { Queued: "queued", Backfill: "backfill", Completed: "completed", Realtime: "realtime", } as const; -export type ChainIndexingMetricType = - (typeof ChainIndexingMetricTypes)[keyof typeof ChainIndexingMetricTypes]; - -/** - * Number of blocks required to be indexed during backfill. - * - * References `ponder_historical_total_blocks` Ponder metric. - * - * This value is calculated at the time the Ponder app starts, - * even for each indexed chain. - * - * If Ponder config specifies a "config end block" for the chain, - * the `ponder_historical_total_blocks` will be the number of blocks - * between the "config start block" and the specified "config end block". - * For example: - * ``` - * ponder_historical_total_blocks = configEndBlock - configStartBlock + 1 - * ``` - * - * If Ponder config does not specify the "config end block" for the chain, - * the `ponder_historical_total_blocks` will be the number of blocks - * between the "config start block" and the "latest known block" - * for the chain at the time the backfill starts. - * The "latest known block" is the "highest" block that has been - * discovered by RPCs and stored in the RPC cache as of the time - * the metric value was captured. - * - * Each restart of the Ponder app will result in a new value based on - * the current "latest known block" for the chain at that time. - * For example: - * ``` - * ponder_historical_total_blocks = latestKnownBlock - configStartBlock + 1 - * ``` - * - * Guaranteed to be a positive integer. - */ -export type BackfillTotalBlocks = number; +export type ChainIndexingState = (typeof ChainIndexingStates)[keyof typeof ChainIndexingStates]; /** * Chain Indexing Metrics Queued @@ -103,12 +66,15 @@ export type BackfillTotalBlocks = number; * where it will be indexed by a Ponder app */ export interface ChainIndexingMetricsQueued { - type: typeof ChainIndexingMetricTypes.Queued; + state: typeof ChainIndexingStates.Queued; /** - * Total number of blocks to be indexed for the chain during backfill phase. + * + * Total count of blocks required to be indexed during backfill. + * + * Guaranteed to be a positive integer. */ - backfillTotalBlocks: BackfillTotalBlocks; + backfillTotalBlocks: number; } /** @@ -118,12 +84,15 @@ export interface ChainIndexingMetricsQueued { * the backfill phase of indexing by a Ponder app. */ export interface ChainIndexingMetricsBackfill { - type: typeof ChainIndexingMetricTypes.Backfill; + state: typeof ChainIndexingStates.Backfill; /** - * Total number of blocks to be indexed for the chain during backfill phase. + * + * Total count of blocks required to be indexed during backfill. + * + * Guaranteed to be a positive integer. */ - backfillTotalBlocks: BackfillTotalBlocks; + backfillTotalBlocks: number; } /** @@ -135,36 +104,37 @@ export interface ChainIndexingMetricsBackfill { * no "config end block" specified for the chain. * * The indexing continues in realtime, with no "target end block". - * The "latest known block" is continuously updated as new blocks are + * The "latest synced block" is continuously updated as new blocks are * discovered by RPCs and stored in the RPC cache. */ export interface ChainIndexingMetricsRealtime { - type: typeof ChainIndexingMetricTypes.Realtime; + state: typeof ChainIndexingStates.Realtime; /** * A {@link BlockRef} to the "highest" block that has been discovered by RPCs * and stored in the RPC cache as of the time the metric value was captured. */ - latestKnownBlock: BlockRef; + latestSyncedBlock: BlockRef; } /** * Chain Indexing Metrics Completed * - * Represents the indexing metrics for a chain that has completed indexing by - * a Ponder app. It means that the backfill phase transitioned to completed phase. - * No more blocks are required to be indexed for the chain at this point. + * Represents the indexing metrics for a chain configured to only index + * a finite range of blocks where all blocks in that finite range + * have been indexed. */ export interface ChainIndexingMetricsCompleted { - type: typeof ChainIndexingMetricTypes.Completed; + state: typeof ChainIndexingStates.Completed; /** - * Target block + * Final indexed block * - * A {@link BlockRef} to the block that was the target of the indexing for the chain. - * There are no more blocks required to be indexed for the chain after this block. + * A {@link BlockRef} to the final block that was the finite target + * for indexing the chain. No more blocks will be indexed for the chain + * after this block. */ - targetBlock: BlockRef; + finalIndexedBlock: BlockRef; } /** diff --git a/packages/ponder-sdk/src/indexing-status.ts b/packages/ponder-sdk/src/indexing-status.ts index d59327869..d528c227c 100644 --- a/packages/ponder-sdk/src/indexing-status.ts +++ b/packages/ponder-sdk/src/indexing-status.ts @@ -10,14 +10,15 @@ export interface ChainIndexingStatus { /** * Checkpoint Block * - * During omnichain indexing, a Ponder app indexes the chain and + * During indexing, a Ponder app indexes the chain and * keeps track of a checkpoint block for each indexed chain. * * @see https://ponder.sh/docs/api-reference/ponder/database#checkpoint-table * * The `checkpointBlock` is a reference to either: * - the first block to be indexed for the chain (if indexing is queued), or - * - the last indexed block for the chain (if indexing is in progress). + * - the last indexed block for the chain (if one or more blocks + * have been indexed for the chain). */ checkpointBlock: BlockRef; } @@ -25,11 +26,11 @@ export interface ChainIndexingStatus { /** * Ponder Indexing Status * - * Represents the chain indexing status in a Ponder application. + * Represents the indexing status of each chain in a Ponder app. */ export interface PonderIndexingStatus { /** - * Map of indexed chain IDs to their block reference. + * Map of indexed chain IDs to their chain indexing status. * * Guarantees: * - Includes entry for at least one indexed chain. From 1523a03e5ca2f72400940f7efafe0b7776b21ff3 Mon Sep 17 00:00:00 2001 From: Tomasz Kopacki Date: Mon, 9 Feb 2026 14:06:46 +0100 Subject: [PATCH 07/10] Update indexing metrics data model --- .../src/deserialize/indexing-metrics.ts | 2 ++ packages/ponder-sdk/src/indexing-metrics.ts | 21 ++++++++++++++++++- 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/packages/ponder-sdk/src/deserialize/indexing-metrics.ts b/packages/ponder-sdk/src/deserialize/indexing-metrics.ts index e898fd414..0e1849a22 100644 --- a/packages/ponder-sdk/src/deserialize/indexing-metrics.ts +++ b/packages/ponder-sdk/src/deserialize/indexing-metrics.ts @@ -30,11 +30,13 @@ import type { DeepPartial } from "./utils"; const schemaSerializedChainIndexingMetricsQueued = z.object({ state: z.literal(ChainIndexingStates.Queued), backfillTotalBlocks: schemaPositiveInteger, + latestSyncedBlock: schemaBlockRef, }); const schemaSerializedChainIndexingMetricsBackfill = z.object({ state: z.literal(ChainIndexingStates.Backfill), backfillTotalBlocks: schemaPositiveInteger, + latestSyncedBlock: schemaBlockRef, }); const schemaSerializedChainIndexingMetricsRealtime = z.object({ diff --git a/packages/ponder-sdk/src/indexing-metrics.ts b/packages/ponder-sdk/src/indexing-metrics.ts index 7ac90d480..c3e8e2d0b 100644 --- a/packages/ponder-sdk/src/indexing-metrics.ts +++ b/packages/ponder-sdk/src/indexing-metrics.ts @@ -63,15 +63,25 @@ export type ChainIndexingState = (typeof ChainIndexingStates)[keyof typeof Chain * * Represents the indexing metrics for a chain that has not started * indexing yet, and is queued to transition to backfill phase, - * where it will be indexed by a Ponder app + * where it will be indexed by a Ponder app. While indexing is queued, + * the Ponder app may be discovering blocks for the chain via RPCs. */ export interface ChainIndexingMetricsQueued { state: typeof ChainIndexingStates.Queued; + /** + * A {@link BlockRef} to the "highest" block that has been discovered by RPCs + * and stored in the RPC cache as of the time the metric value was captured. + */ + latestSyncedBlock: BlockRef; + /** * * Total count of blocks required to be indexed during backfill. * + * Each time a Ponder app restarts, if it enters queued mode, + * a new value will be calculated based on the current state of the chain. + * * Guaranteed to be a positive integer. */ backfillTotalBlocks: number; @@ -86,10 +96,19 @@ export interface ChainIndexingMetricsQueued { export interface ChainIndexingMetricsBackfill { state: typeof ChainIndexingStates.Backfill; + /** + * A {@link BlockRef} to the "highest" block that has been discovered by RPCs + * and stored in the RPC cache as of the time the metric value was captured. + */ + latestSyncedBlock: BlockRef; + /** * * Total count of blocks required to be indexed during backfill. * + * Each time a Ponder app restarts, if it enters backfill mode, + * a new value will be calculated based on the current state of the chain. + * * Guaranteed to be a positive integer. */ backfillTotalBlocks: number; From 4ea2b106e3adf0d7ba4731260c247846d963b7d6 Mon Sep 17 00:00:00 2001 From: Tomasz Kopacki Date: Mon, 9 Feb 2026 20:56:39 +0100 Subject: [PATCH 08/10] Merge "queued" and "backfill" metrics statuses into "Historical" status This change supports the current ENSIndexer goals, for example, fetching a block ref on application startup from RPC for each chain that is in either "queued" or "backfill" status. --- .../src/deserialize/indexing-metrics.mock.ts | 27 -------- .../src/deserialize/indexing-metrics.ts | 67 ++++--------------- packages/ponder-sdk/src/indexing-metrics.ts | 57 ++++------------ 3 files changed, 26 insertions(+), 125 deletions(-) diff --git a/packages/ponder-sdk/src/deserialize/indexing-metrics.mock.ts b/packages/ponder-sdk/src/deserialize/indexing-metrics.mock.ts index 79b4f091b..48441dae2 100644 --- a/packages/ponder-sdk/src/deserialize/indexing-metrics.mock.ts +++ b/packages/ponder-sdk/src/deserialize/indexing-metrics.mock.ts @@ -30,15 +30,6 @@ ponder_sync_block_timestamp{chain="534352"} 1770136654 ponder_sync_block_timestamp{chain="42161"} 1770136654 ponder_sync_block_timestamp{chain="59144"} 1770136654 -# HELP ponder_historical_completed_indexing_seconds Number of seconds that have been completed -# TYPE ponder_historical_completed_indexing_seconds gauge -ponder_historical_completed_indexing_seconds{chain="10"} 34242 -ponder_historical_completed_indexing_seconds{chain="1"} 23124 -ponder_historical_completed_indexing_seconds{chain="8453"} 53253 -ponder_historical_completed_indexing_seconds{chain="534352"} 32503 -ponder_historical_completed_indexing_seconds{chain="42161"} 76864 -ponder_historical_completed_indexing_seconds{chain="59144"} 34235 - # HELP ponder_sync_is_realtime Boolean (0 or 1) indicating if the sync is realtime mode # TYPE ponder_sync_is_realtime gauge ponder_sync_is_realtime{chain="42161"} 1 @@ -133,15 +124,6 @@ ponder_sync_block_timestamp{chain="scroll"} 1770136654 ponder_sync_block_timestamp{chain="arbitrum"} 1770136654 ponder_sync_block_timestamp{chain="linea"} 1770136654 -# HELP ponder_historical_completed_indexing_seconds Number of seconds that have been completed -# TYPE ponder_historical_completed_indexing_seconds gauge -ponder_historical_completed_indexing_seconds{chain="optimism"} 34242 -ponder_historical_completed_indexing_seconds{chain="mainnet"} 23124 -ponder_historical_completed_indexing_seconds{chain="base"} 53253 -ponder_historical_completed_indexing_seconds{chain="scroll"} 32503 -ponder_historical_completed_indexing_seconds{chain="arbitrum"} 76864 -ponder_historical_completed_indexing_seconds{chain="linea"} 34235 - # HELP ponder_sync_is_realtime Boolean (0 or 1) indicating if the sync is realtime mode # TYPE ponder_sync_is_realtime gauge ponder_sync_is_realtime{chain="arbitrum"} 1 @@ -214,15 +196,6 @@ ponder_sync_block_timestamp{chain="534352"} 1770136654 ponder_sync_block_timestamp{chain="42161"} 1770136654 ponder_sync_block_timestamp{chain="59144"} 1770136654 -# HELP ponder_historical_completed_indexing_seconds Number of seconds that have been completed -# TYPE ponder_historical_completed_indexing_seconds gauge -ponder_historical_completed_indexing_seconds{chain="10"} 34242 -ponder_historical_completed_indexing_seconds{chain="1"} 23124 -ponder_historical_completed_indexing_seconds{chain="8453"} 53253 -ponder_historical_completed_indexing_seconds{chain="534352"} 32503 -ponder_historical_completed_indexing_seconds{chain="42161"} 76864 -ponder_historical_completed_indexing_seconds{chain="59144"} 34235 - # HELP ponder_sync_is_realtime Boolean (0 or 1) indicating if the sync is realtime mode # TYPE ponder_sync_is_realtime gauge ponder_sync_is_realtime{chain="42161"} 1 diff --git a/packages/ponder-sdk/src/deserialize/indexing-metrics.ts b/packages/ponder-sdk/src/deserialize/indexing-metrics.ts index 0e1849a22..57affc0f3 100644 --- a/packages/ponder-sdk/src/deserialize/indexing-metrics.ts +++ b/packages/ponder-sdk/src/deserialize/indexing-metrics.ts @@ -12,9 +12,8 @@ import type { ParsePayload } from "zod/v4/core"; import { type BlockRef, schemaBlockRef } from "../blocks"; import { type ChainIndexingMetrics, - type ChainIndexingMetricsBackfill, type ChainIndexingMetricsCompleted, - type ChainIndexingMetricsQueued, + type ChainIndexingMetricsHistorical, type ChainIndexingMetricsRealtime, ChainIndexingStates, PonderAppCommands, @@ -27,16 +26,10 @@ import { schemaChainIdString } from "./chains"; import { deserializePrometheusMetrics, type PrometheusMetrics } from "./prometheus-metrics-text"; import type { DeepPartial } from "./utils"; -const schemaSerializedChainIndexingMetricsQueued = z.object({ - state: z.literal(ChainIndexingStates.Queued), - backfillTotalBlocks: schemaPositiveInteger, - latestSyncedBlock: schemaBlockRef, -}); - -const schemaSerializedChainIndexingMetricsBackfill = z.object({ - state: z.literal(ChainIndexingStates.Backfill), - backfillTotalBlocks: schemaPositiveInteger, +const schemaSerializedChainIndexingMetricsHistorical = z.object({ + state: z.literal(ChainIndexingStates.Historical), latestSyncedBlock: schemaBlockRef, + historicalTotalBlocks: schemaPositiveInteger, }); const schemaSerializedChainIndexingMetricsRealtime = z.object({ @@ -53,8 +46,7 @@ const schemaSerializedChainIndexingMetricsCompleted = z.object({ * Schema describing the chain indexing metrics. */ const schemaSerializedChainIndexingMetrics = z.discriminatedUnion("state", [ - schemaSerializedChainIndexingMetricsQueued, - schemaSerializedChainIndexingMetricsBackfill, + schemaSerializedChainIndexingMetricsHistorical, schemaSerializedChainIndexingMetricsRealtime, schemaSerializedChainIndexingMetricsCompleted, ]); @@ -79,26 +71,6 @@ function buildUnvalidatedChainIndexingMetrics( maybeChainId: string, prometheusMetrics: PrometheusMetrics, ): DeepPartial { - const backfillTotalBlocks = prometheusMetrics.getValue("ponder_historical_total_blocks", { - chain: maybeChainId, - }); - - const ponderHistoricalCompletedIndexingSeconds = prometheusMetrics.getValue( - "ponder_historical_completed_indexing_seconds", - { - chain: maybeChainId, - }, - ); - - // If no time has been recorded for historical completed indexing, - // we can assume the chain is still queued to be indexed. - if (ponderHistoricalCompletedIndexingSeconds === 0) { - return { - state: ChainIndexingStates.Queued, - backfillTotalBlocks, - } satisfies DeepPartial; - } - const ponderSyncIsComplete = prometheusMetrics.getValue("ponder_sync_is_complete", { chain: maybeChainId, }); @@ -138,10 +110,14 @@ function buildUnvalidatedChainIndexingMetrics( } satisfies DeepPartial; } + const historicalTotalBlocks = prometheusMetrics.getValue("ponder_historical_total_blocks", { + chain: maybeChainId, + }); + return { - state: ChainIndexingStates.Backfill, - backfillTotalBlocks, - } satisfies DeepPartial; + state: ChainIndexingStates.Historical, + historicalTotalBlocks, + } satisfies DeepPartial; } function invariant_includesAtLeastOneIndexedChain( @@ -184,7 +160,6 @@ function invariant_includesRequiredMetrics(ctx: ParsePayload) "ponder_settings_info", "ponder_sync_block", "ponder_sync_block_timestamp", - "ponder_historical_completed_indexing_seconds", "ponder_historical_total_blocks", "ponder_sync_is_complete", "ponder_sync_is_realtime", @@ -206,24 +181,6 @@ function invariant_includesRequiredMetrics(ctx: ParsePayload) // Validate per-chain invariants. for (const chainReference of chainReferences) { - const ponderHistoricalCompletedIndexingSeconds = prometheusMetrics.getValue( - "ponder_historical_completed_indexing_seconds", - { chain: chainReference }, - ); - - // Invariant: historical completed indexing seconds must be a non-negative integer. - if ( - typeof ponderHistoricalCompletedIndexingSeconds !== "number" || - !Number.isInteger(ponderHistoricalCompletedIndexingSeconds) || - ponderHistoricalCompletedIndexingSeconds < 0 - ) { - ctx.issues.push({ - code: "custom", - input: ctx.value, - message: `'ponder_historical_completed_indexing_seconds' metric for '${chainReference}' chain must be a non-negative integer. Received: ${ponderHistoricalCompletedIndexingSeconds}`, - }); - } - const ponderSyncIsComplete = prometheusMetrics.getValue("ponder_sync_is_complete", { chain: chainReference, }); diff --git a/packages/ponder-sdk/src/indexing-metrics.ts b/packages/ponder-sdk/src/indexing-metrics.ts index c3e8e2d0b..9e8d0046e 100644 --- a/packages/ponder-sdk/src/indexing-metrics.ts +++ b/packages/ponder-sdk/src/indexing-metrics.ts @@ -50,8 +50,7 @@ export interface PonderApplicationSettings { * Represents the indexing state of a chain indexed by a Ponder app. */ export const ChainIndexingStates = { - Queued: "queued", - Backfill: "backfill", + Historical: "historical", Completed: "completed", Realtime: "realtime", } as const; @@ -59,15 +58,13 @@ export const ChainIndexingStates = { export type ChainIndexingState = (typeof ChainIndexingStates)[keyof typeof ChainIndexingStates]; /** - * Chain Indexing Metrics Queued + * Chain Indexing Metrics Historical * - * Represents the indexing metrics for a chain that has not started - * indexing yet, and is queued to transition to backfill phase, - * where it will be indexed by a Ponder app. While indexing is queued, - * the Ponder app may be discovering blocks for the chain via RPCs. + * Represents the indexing metrics for a chain that is currently queued for + * indexing or in the backfill phase by a Ponder app. */ -export interface ChainIndexingMetricsQueued { - state: typeof ChainIndexingStates.Queued; +export interface ChainIndexingMetricsHistorical { + state: typeof ChainIndexingStates.Historical; /** * A {@link BlockRef} to the "highest" block that has been discovered by RPCs @@ -76,42 +73,17 @@ export interface ChainIndexingMetricsQueued { latestSyncedBlock: BlockRef; /** + * Total count of historical blocks. * - * Total count of blocks required to be indexed during backfill. - * - * Each time a Ponder app restarts, if it enters queued mode, - * a new value will be calculated based on the current state of the chain. - * - * Guaranteed to be a positive integer. - */ - backfillTotalBlocks: number; -} - -/** - * Chain Indexing Metrics Backfill - * - * Represents the indexing metrics for a chain that is currently in - * the backfill phase of indexing by a Ponder app. - */ -export interface ChainIndexingMetricsBackfill { - state: typeof ChainIndexingStates.Backfill; - - /** - * A {@link BlockRef} to the "highest" block that has been discovered by RPCs - * and stored in the RPC cache as of the time the metric value was captured. - */ - latestSyncedBlock: BlockRef; - - /** - * - * Total count of blocks required to be indexed during backfill. - * - * Each time a Ponder app restarts, if it enters backfill mode, - * a new value will be calculated based on the current state of the chain. + * Each time a Ponder app restarts, if the historical blocks have + * not been fully indexed yet, for example, the chain is queued for + * indexing or in the backfill phase, the count of historical blocks is + * reset and starts increasing again as more historical blocks are + * discovered by RPCs and stored in the RPC cache. * * Guaranteed to be a positive integer. */ - backfillTotalBlocks: number; + historicalTotalBlocks: number; } /** @@ -162,8 +134,7 @@ export interface ChainIndexingMetricsCompleted { * Represents the indexing metrics for a specific chain indexed by a Ponder app. */ export type ChainIndexingMetrics = - | ChainIndexingMetricsQueued - | ChainIndexingMetricsBackfill + | ChainIndexingMetricsHistorical | ChainIndexingMetricsCompleted | ChainIndexingMetricsRealtime; From 8feeada95e0403bcbfa0649af9c5d64dc2b009b4 Mon Sep 17 00:00:00 2001 From: Tomasz Kopacki Date: Tue, 10 Feb 2026 18:25:37 +0100 Subject: [PATCH 09/10] Build Ponder SDK valiated objects from `Unvalidated` results --- packages/ponder-sdk/src/chains.ts | 5 + packages/ponder-sdk/src/client.test.ts | 26 +- .../src/deserialize/indexing-metrics.mock.ts | 4 +- .../src/deserialize/indexing-metrics.ts | 316 ++++++++++-------- packages/ponder-sdk/src/deserialize/utils.ts | 31 ++ 5 files changed, 226 insertions(+), 156 deletions(-) diff --git a/packages/ponder-sdk/src/chains.ts b/packages/ponder-sdk/src/chains.ts index 943a07fba..67aa08d51 100644 --- a/packages/ponder-sdk/src/chains.ts +++ b/packages/ponder-sdk/src/chains.ts @@ -14,3 +14,8 @@ export const schemaChainId = schemaPositiveInteger; * Chain id standards are organized by the Ethereum Community @ https://github.com/ethereum-lists/chains **/ export type ChainId = z.infer; + +/** + * String representation of a valid Chain ID. + */ +export type ChainIdString = string; diff --git a/packages/ponder-sdk/src/client.test.ts b/packages/ponder-sdk/src/client.test.ts index d869117d0..3b4bd73d4 100644 --- a/packages/ponder-sdk/src/client.test.ts +++ b/packages/ponder-sdk/src/client.test.ts @@ -124,12 +124,6 @@ describe("Ponder Client", () => { expect(errorMessage).toContain( "Missing required Prometheus metric: ponder_historical_total_blocks", ); - expect(errorMessage).toContain( - "Missing required Prometheus metric: ponder_sync_is_complete", - ); - expect(errorMessage).toContain( - "Missing required Prometheus metric: ponder_sync_is_realtime", - ); } }); @@ -151,11 +145,21 @@ describe("Ponder Client", () => { const errorMessage = error instanceof Error ? error.message : "unknown error"; // Assert expect(errorMessage).toContain("Invalid serialized Ponder Indexing Metrics"); - expect(errorMessage).toContain("'optimism' must be a string representing a chain ID"); - expect(errorMessage).toContain("'mainnet' must be a string representing a chain ID"); - expect(errorMessage).toContain("'base' must be a string representing a chain ID"); - expect(errorMessage).toContain("'scroll' must be a string representing a chain ID"); - expect(errorMessage).toContain("'linea' must be a string representing a chain ID"); + expect(errorMessage).toContain( + "metric must be a string representing a valid ChainId, but got: 'optimism'", + ); + expect(errorMessage).toContain( + "metric must be a string representing a valid ChainId, but got: 'mainnet'", + ); + expect(errorMessage).toContain( + "metric must be a string representing a valid ChainId, but got: 'base'", + ); + expect(errorMessage).toContain( + "metric must be a string representing a valid ChainId, but got: 'scroll'", + ); + expect(errorMessage).toContain( + "metric must be a string representing a valid ChainId, but got: 'linea'", + ); } }); diff --git a/packages/ponder-sdk/src/deserialize/indexing-metrics.mock.ts b/packages/ponder-sdk/src/deserialize/indexing-metrics.mock.ts index 48441dae2..64dff3b20 100644 --- a/packages/ponder-sdk/src/deserialize/indexing-metrics.mock.ts +++ b/packages/ponder-sdk/src/deserialize/indexing-metrics.mock.ts @@ -205,6 +205,8 @@ ponder_sync_is_realtime{chain="1"} 1 ponder_sync_is_realtime{chain="59144"} 1 ponder_sync_is_realtime{chain="8453"} 1 +# HELP ponder_sync_is_complete Boolean (0 or 1) indicating if the sync has synced all blocks +# TYPE ponder_sync_is_complete gauge ponder_sync_is_complete{chain="42161"} 1 ponder_sync_is_complete{chain="534352"} 1 ponder_sync_is_complete{chain="10"} 1 @@ -212,8 +214,6 @@ ponder_sync_is_complete{chain="1"} 1 ponder_sync_is_complete{chain="59144"} 1 ponder_sync_is_complete{chain="8453"} 1 -# HELP ponder_sync_is_complete Boolean (0 or 1) indicating if the sync has synced all blocks -# TYPE ponder_sync_is_complete gauge # HELP ponder_historical_total_blocks Number of blocks required for the historical sync # TYPE ponder_historical_total_blocks gauge diff --git a/packages/ponder-sdk/src/deserialize/indexing-metrics.ts b/packages/ponder-sdk/src/deserialize/indexing-metrics.ts index 57affc0f3..5ef3ada3d 100644 --- a/packages/ponder-sdk/src/deserialize/indexing-metrics.ts +++ b/packages/ponder-sdk/src/deserialize/indexing-metrics.ts @@ -10,34 +10,36 @@ import { prettifyError, z } from "zod/v4"; import type { ParsePayload } from "zod/v4/core"; import { type BlockRef, schemaBlockRef } from "../blocks"; +import { type ChainId, type ChainIdString, schemaChainId } from "../chains"; import { type ChainIndexingMetrics, type ChainIndexingMetricsCompleted, type ChainIndexingMetricsHistorical, type ChainIndexingMetricsRealtime, ChainIndexingStates, + type PonderAppCommand, PonderAppCommands, - type PonderApplicationSettings, type PonderIndexingMetrics, + type PonderIndexingOrdering, PonderIndexingOrderings, } from "../indexing-metrics"; import { schemaPositiveInteger } from "../numbers"; import { schemaChainIdString } from "./chains"; import { deserializePrometheusMetrics, type PrometheusMetrics } from "./prometheus-metrics-text"; -import type { DeepPartial } from "./utils"; +import type { Unvalidated } from "./utils"; -const schemaSerializedChainIndexingMetricsHistorical = z.object({ +const schemaChainIndexingMetricsHistorical = z.object({ state: z.literal(ChainIndexingStates.Historical), latestSyncedBlock: schemaBlockRef, historicalTotalBlocks: schemaPositiveInteger, }); -const schemaSerializedChainIndexingMetricsRealtime = z.object({ +const schemaChainIndexingMetricsRealtime = z.object({ state: z.literal(ChainIndexingStates.Realtime), latestSyncedBlock: schemaBlockRef, }); -const schemaSerializedChainIndexingMetricsCompleted = z.object({ +const schemaChainIndexingMetricsCompleted = z.object({ state: z.literal(ChainIndexingStates.Completed), finalIndexedBlock: schemaBlockRef, }); @@ -45,129 +47,55 @@ const schemaSerializedChainIndexingMetricsCompleted = z.object({ /** * Schema describing the chain indexing metrics. */ -const schemaSerializedChainIndexingMetrics = z.discriminatedUnion("state", [ - schemaSerializedChainIndexingMetricsHistorical, - schemaSerializedChainIndexingMetricsRealtime, - schemaSerializedChainIndexingMetricsCompleted, +const schemaChainIndexingMetrics = z.discriminatedUnion("state", [ + schemaChainIndexingMetricsHistorical, + schemaChainIndexingMetricsRealtime, + schemaChainIndexingMetricsCompleted, ]); /** * Schema describing the chains indexing metrics. */ -const schemaSerializedChainsIndexingMetrics = z.map( - schemaChainIdString, - schemaSerializedChainIndexingMetrics, -); +const schemaChainsIndexingMetrics = z.map(schemaChainId, schemaChainIndexingMetrics); -/** - * Build unvalidated (and perhaps partial) Chain Indexing Metrics - * - * @param maybeChainId A string maybe representing a chain ID. - * @param prometheusMetrics valid Prometheus Metrics from Ponder app. - * @returns Unvalidated (possibly incomplete) Chain Indexing Metrics - * to be validated by {@link schemaSerializedChainIndexingMetrics}. - */ -function buildUnvalidatedChainIndexingMetrics( - maybeChainId: string, - prometheusMetrics: PrometheusMetrics, -): DeepPartial { - const ponderSyncIsComplete = prometheusMetrics.getValue("ponder_sync_is_complete", { - chain: maybeChainId, - }); - - const ponderSyncIsRealtime = prometheusMetrics.getValue("ponder_sync_is_realtime", { - chain: maybeChainId, - }); - - const latestSyncedBlockNumber = prometheusMetrics.getValue("ponder_sync_block", { - chain: maybeChainId, - }); - - const latestSyncedBlockTimestamp = prometheusMetrics.getValue("ponder_sync_block_timestamp", { - chain: maybeChainId, - }); - - const latestSyncedBlock = { - number: latestSyncedBlockNumber, - timestamp: latestSyncedBlockTimestamp, - } satisfies Partial; - - // The `ponder_sync_is_complete` metric is set to `1` if, and only if, - // the indexing has been completed for the chain. - if (ponderSyncIsComplete === 1) { - return { - state: ChainIndexingStates.Completed, - finalIndexedBlock: latestSyncedBlock, - } satisfies DeepPartial; - } - - // The `ponder_sync_is_realtime` metric is set to `1` if, and only if, - // the indexing is currently in realtime for the chain. - if (ponderSyncIsRealtime === 1) { - return { - state: ChainIndexingStates.Realtime, - latestSyncedBlock, - } satisfies DeepPartial; - } - - const historicalTotalBlocks = prometheusMetrics.getValue("ponder_historical_total_blocks", { - chain: maybeChainId, - }); - - return { - state: ChainIndexingStates.Historical, - historicalTotalBlocks, - } satisfies DeepPartial; -} - -function invariant_includesAtLeastOneIndexedChain( - ctx: ParsePayload, +function invariant_indexingCompletedAndRealtimeAreNotBothTrue( + ctx: ParsePayload, ) { - const { chains } = ctx.value; + const prometheusMetrics = ctx.value; + const chainReferences = prometheusMetrics.getLabels("ponder_sync_block", "chain"); - if (chains.size === 0) { - ctx.issues.push({ - code: "custom", - input: ctx.value, - message: "Ponder Indexing Metrics must include at least one indexed chain.", + for (const maybeChainId of chainReferences) { + const ponderSyncIsComplete = prometheusMetrics.getValue("ponder_sync_is_complete", { + chain: maybeChainId, }); - } -} -/** - * Schema representing settings of a Ponder app. - */ -const schemaSerializedApplicationSettings = z.object({ - command: z.enum(PonderAppCommands), - ordering: z.enum(PonderIndexingOrderings), -}); + const ponderSyncIsRealtime = prometheusMetrics.getValue("ponder_sync_is_realtime", { + chain: maybeChainId, + }); -/** - * Schema describing the Ponder Indexing Metrics. - */ -const schemaPonderIndexingMetrics = z - .object({ - appSettings: schemaSerializedApplicationSettings, - chains: schemaSerializedChainsIndexingMetrics, - }) - .check(invariant_includesAtLeastOneIndexedChain); + if (ponderSyncIsComplete === 1 && ponderSyncIsRealtime === 1) { + ctx.issues.push({ + code: "custom", + input: ctx.value, + message: `'ponder_sync_is_complete' and 'ponder_sync_is_realtime' metrics cannot both be 1 at the same time for chain ${maybeChainId}`, + }); + } + } +} function invariant_includesRequiredMetrics(ctx: ParsePayload) { const prometheusMetrics = ctx.value; const metricNames = prometheusMetrics.getMetricNames(); - const requiredMetricNames = [ - "ponder_settings_info", + const requiredChainMetricNames = [ "ponder_sync_block", "ponder_sync_block_timestamp", "ponder_historical_total_blocks", - "ponder_sync_is_complete", - "ponder_sync_is_realtime", ]; + const requiredMetricNames = ["ponder_settings_info", ...requiredChainMetricNames]; - // Validate metrics presence invariants. + // Invariant: All required metrics must be present in the Prometheus metrics text. for (const requiredMetricName of requiredMetricNames) { - // Invariant: Required metric must be present in the Prometheus metrics. if (!metricNames.includes(requiredMetricName)) { ctx.issues.push({ code: "custom", @@ -177,27 +105,30 @@ function invariant_includesRequiredMetrics(ctx: ParsePayload) } } - const chainReferences = prometheusMetrics.getLabels("ponder_sync_block", "chain"); - - // Validate per-chain invariants. - for (const chainReference of chainReferences) { - const ponderSyncIsComplete = prometheusMetrics.getValue("ponder_sync_is_complete", { - chain: chainReference, - }); - - const ponderSyncIsRealtime = prometheusMetrics.getValue("ponder_sync_is_realtime", { - chain: chainReference, - }); + // Invariant: All required chain metrics must include a 'chain' label. + for (const requiredChainMetricName of requiredChainMetricNames) { + const metricLabels = prometheusMetrics.getLabels(requiredChainMetricName, "chain"); - // Invariant: `ponder_sync_is_complete` and `ponder_sync_is_realtime` cannot - // both be `1` at the same time. - if (ponderSyncIsComplete === 1 && ponderSyncIsRealtime === 1) { + if (metricLabels.length === 0) { ctx.issues.push({ code: "custom", input: ctx.value, - message: `'ponder_sync_is_complete' and 'ponder_sync_is_realtime' metrics cannot both be 1 at the same time for chain ${chainReference}`, + message: `At least one '${requiredChainMetricName}' metric must include a 'chain' label.`, }); } + + // Invariant: All values in the 'chain' label of required chain metrics must be valid ChainId strings. + for (const maybeChainId of metricLabels) { + const result = schemaChainIdString.safeParse(maybeChainId); + + if (!result.success) { + ctx.issues.push({ + code: "custom", + input: ctx.value, + message: `Value in 'chain' label of '${requiredChainMetricName}' metric must be a string representing a valid ChainId, but got: '${maybeChainId}'`, + }); + } + } } } @@ -209,58 +140,157 @@ const schemaSerializedPonderIndexingMetrics = z.coerce .nonempty({ error: `Ponder Indexing Metrics must be a non-empty string.` }) .transform(deserializePrometheusMetrics) // deserialize Prometheus metrics text into PrometheusMetrics instance .check(invariant_includesRequiredMetrics) - .transform(buildUnvalidatedPonderIndexingMetrics) - .pipe(schemaPonderIndexingMetrics); + .check(invariant_indexingCompletedAndRealtimeAreNotBothTrue); + +function invariant_includesAtLeastOneIndexedChain(ctx: ParsePayload) { + const { chains } = ctx.value; + + if (chains.size === 0) { + ctx.issues.push({ + code: "custom", + input: ctx.value, + message: "Ponder Indexing Metrics must include at least one indexed chain.", + }); + } +} + +/** + * Schema representing settings of a Ponder app. + */ +const schemaApplicationSettings = z.object({ + command: z.enum(PonderAppCommands), + ordering: z.enum(PonderIndexingOrderings), +}); /** - * Serialized Ponder Indexing Metrics. + * Schema describing Ponder Indexing Metrics. */ -type SerializedPonderIndexingMetrics = z.infer; +const schemaPonderIndexingMetrics = z + .object({ + appSettings: schemaApplicationSettings, + chains: schemaChainsIndexingMetrics, + }) + .check(invariant_includesAtLeastOneIndexedChain); /** - * Build unvalidated (and perhaps partial) Ponder Indexing Metrics + * Build unvalidated Chain Indexing Metrics * + * @param chainId Chain ID * @param prometheusMetrics valid Prometheus Metrics from Ponder app. - * @returns Unvalidated (possibly incomplete) Ponder Indexing Metrics - * to be validated with {@link schemaSerializedPonderIndexingMetrics}. + * @returns Unvalidated Chain Indexing Metrics + * to be validated by {@link schemaChainIndexingMetrics}. + */ +function buildUnvalidatedChainIndexingMetrics( + chainIdString: ChainIdString, + prometheusMetrics: PrometheusMetrics, +): Unvalidated { + const ponderSyncIsComplete = prometheusMetrics.getValue("ponder_sync_is_complete", { + chain: chainIdString, + }); + + const ponderSyncIsRealtime = prometheusMetrics.getValue("ponder_sync_is_realtime", { + chain: chainIdString, + }); + + const latestSyncedBlockNumber = prometheusMetrics.getValue("ponder_sync_block", { + chain: chainIdString, + }); + + const latestSyncedBlockTimestamp = prometheusMetrics.getValue("ponder_sync_block_timestamp", { + chain: chainIdString, + }); + + const latestSyncedBlock = { + number: latestSyncedBlockNumber, + timestamp: latestSyncedBlockTimestamp, + } satisfies Unvalidated; + + // The `ponder_sync_is_complete` metric is set to `1` if, and only if, + // the indexing has been completed for the chain. + if (ponderSyncIsComplete === 1) { + return { + state: ChainIndexingStates.Completed, + finalIndexedBlock: latestSyncedBlock, + } satisfies Unvalidated; + } + + // The `ponder_sync_is_realtime` metric is set to `1` if, and only if, + // the indexing is currently in realtime for the chain. + if (ponderSyncIsRealtime === 1) { + return { + state: ChainIndexingStates.Realtime, + latestSyncedBlock, + } satisfies Unvalidated; + } + + const historicalTotalBlocks = prometheusMetrics.getValue("ponder_historical_total_blocks", { + chain: chainIdString, + }); + + return { + state: ChainIndexingStates.Historical, + historicalTotalBlocks, + latestSyncedBlock, + } satisfies Unvalidated; +} + +/** + * Build unvalidated Ponder Indexing Metrics + * + * @param prometheusMetrics valid Prometheus Metrics from Ponder app. + * @returns Unvalidated Ponder Indexing Metrics + * to be validated with {@link schemaPonderIndexingMetrics}. */ function buildUnvalidatedPonderIndexingMetrics( prometheusMetrics: PrometheusMetrics, -): DeepPartial { +): Unvalidated { const appSettings = { - command: prometheusMetrics.getLabel("ponder_settings_info", "command"), - ordering: prometheusMetrics.getLabel("ponder_settings_info", "ordering"), - } as DeepPartial; + command: prometheusMetrics.getLabel( + "ponder_settings_info", + "command", + ) as Unvalidated, + ordering: prometheusMetrics.getLabel( + "ponder_settings_info", + "ordering", + ) as Unvalidated, + }; - const chainReferences = prometheusMetrics.getLabels("ponder_sync_block", "chain"); - const chains = new Map>(); + const chainReferences = prometheusMetrics.getLabels( + "ponder_sync_block", + "chain", + ) satisfies ChainIdString[]; - for (const maybeChainId of chainReferences) { + const chains = new Map, Unvalidated>(); + + for (const chainIdString of chainReferences) { const chainIndexingMetrics = buildUnvalidatedChainIndexingMetrics( - maybeChainId, + chainIdString, prometheusMetrics, ); - chains.set(maybeChainId, chainIndexingMetrics); + const chainId = Number(chainIdString) satisfies Unvalidated; + + chains.set(chainId, chainIndexingMetrics); } - const unvalidatedPonderIndexingMetrics = { + return { appSettings, chains, }; - - return unvalidatedPonderIndexingMetrics; } /** * Deserialize and validate a Serialized Ponder Indexing Metrics. * - * @param data Maybe a string representing Ponder Indexing Metrics. + * @param ponderMetricsText Raw text maybe including Prometheus metrics. * @returns Deserialized and validated Ponder Indexing Metrics. * @throws Error if data cannot be deserialized into a valid Ponder Indexing Metrics. */ -export function deserializePonderIndexingMetrics(data: string | unknown): PonderIndexingMetrics { - const validation = schemaSerializedPonderIndexingMetrics.safeParse(data); +export function deserializePonderIndexingMetrics(ponderMetricsText: string): PonderIndexingMetrics { + const validation = schemaSerializedPonderIndexingMetrics + .transform(buildUnvalidatedPonderIndexingMetrics) + .pipe(schemaPonderIndexingMetrics) + .safeParse(ponderMetricsText); if (!validation.success) { throw new Error( diff --git a/packages/ponder-sdk/src/deserialize/utils.ts b/packages/ponder-sdk/src/deserialize/utils.ts index 8e0b44c45..b99f20a6e 100644 --- a/packages/ponder-sdk/src/deserialize/utils.ts +++ b/packages/ponder-sdk/src/deserialize/utils.ts @@ -35,3 +35,34 @@ export type DeepPartial = { ? DeepPartial : T[P]; }; + +/** + * Helper type to represent an unvalidated version of a business layer type `T`, + * where all properties are optional. + * + * This is useful for building a validated object `T` from partial input, + * where the input may be missing required fields or have fields that + * are not yet validated. + * + * For example, transforming serialized representation of type `T` into + * an unvalidated version of `T` that can be later validated against + * defined business rules and constraints. + * + * ```ts + * function buildUnvalidatedValue(serialized: SerializedChainId): Unvalidated { + * // transform serialized chainId into unvalidated number (e.g. parseInt) + * return parseInt(serialized, 10); + * } + * + * // Later, we can validate the unvalidated value against our business rules + * function validateChainId(unvalidatedChainId: Unvalidated): ChainId { + * if (typeof unvalidatedChainId !== "number" || unvalidatedChainId <= 0) { + * throw new Error("Invalid ChainId"); + * } + * + * return unvalidatedChainId as ChainId; + * } + * + * ``` + */ +export type Unvalidated = DeepPartial; From 586ce130b670b37e57816beb33bed597ccd2ae0b Mon Sep 17 00:00:00 2001 From: Tomek Kopacki Date: Wed, 11 Feb 2026 11:23:04 +0100 Subject: [PATCH 10/10] Apply suggestion from @tk-o --- packages/ponder-sdk/src/indexing-metrics.ts | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/packages/ponder-sdk/src/indexing-metrics.ts b/packages/ponder-sdk/src/indexing-metrics.ts index 9e8d0046e..24d0dc31f 100644 --- a/packages/ponder-sdk/src/indexing-metrics.ts +++ b/packages/ponder-sdk/src/indexing-metrics.ts @@ -75,11 +75,13 @@ export interface ChainIndexingMetricsHistorical { /** * Total count of historical blocks. * - * Each time a Ponder app restarts, if the historical blocks have - * not been fully indexed yet, for example, the chain is queued for - * indexing or in the backfill phase, the count of historical blocks is - * reset and starts increasing again as more historical blocks are - * discovered by RPCs and stored in the RPC cache. + * The count of historical blocks is only reset when a Ponder app + * restarts. If historical blocks have not been fully indexed yet + * (for example, the chain is queued for indexing or in the backfill + * phase), the count will increase as more historical blocks are + * discovered by RPCs and stored in the RPC cache, potentially exceeding + * the count from before the restart. Between restarts, this count + * remains unchanged. * * Guaranteed to be a positive integer. */