From 35e3f427537a8bb585f2125378e6415ba850bafb Mon Sep 17 00:00:00 2001 From: Hugo Dias Date: Tue, 2 Jun 2026 17:31:23 +0100 Subject: [PATCH 01/13] feat: improve retry and poll logic - updates to iso-web 3.0.0 with retry and poll separate logic - most SP endpoints retry on error - findOnProviders uses a HEAD request on the retrieval url - response schema validation is now handled by iso-web - retry and poll options exposed in all methods closes #738 --- examples/cli/src/commands/upload-dataset.ts | 2 +- packages/synapse-core/src/piece/download.ts | 38 +++++++-- .../src/piece/resolve-piece-url.ts | 44 +++++++---- packages/synapse-core/src/sp/add-pieces.ts | 67 ++++++++++------ .../src/sp/create-dataset-add-pieces.ts | 50 +++++++++--- .../synapse-core/src/sp/create-dataset.ts | 77 ++++++++++-------- packages/synapse-core/src/sp/find-piece.ts | 26 ++++--- packages/synapse-core/src/sp/get-data-set.ts | 22 ++++-- packages/synapse-core/src/sp/ping.ts | 5 +- packages/synapse-core/src/sp/pull-pieces.ts | 78 +++++++++++-------- .../src/sp/schedule-piece-deletion.ts | 20 ++++- .../synapse-core/src/sp/upload-streaming.ts | 29 ++++--- packages/synapse-core/src/sp/upload.ts | 23 ++++-- packages/synapse-core/src/utils/constants.ts | 9 ++- packages/synapse-core/test/pull.test.ts | 1 + .../test/resolve-piece-url.test.ts | 44 ++++++----- packages/synapse-core/test/sp.test.ts | 22 +++++- packages/synapse-sdk/src/storage/context.ts | 2 +- packages/synapse-sdk/src/test/storage.test.ts | 12 ++- packages/synapse-sdk/src/test/synapse.test.ts | 29 ++----- pnpm-workspace.yaml | 2 +- 21 files changed, 391 insertions(+), 211 deletions(-) diff --git a/examples/cli/src/commands/upload-dataset.ts b/examples/cli/src/commands/upload-dataset.ts index 15217561b..d9d806856 100644 --- a/examples/cli/src/commands/upload-dataset.ts +++ b/examples/cli/src/commands/upload-dataset.ts @@ -55,7 +55,7 @@ export const uploadDataset: Command = command( await SP.findPiece({ pieceCid, serviceURL: provider.pdp.serviceURL, - retry: true, + poll: true, }) const rsp = await SP.createDataSetAndAddPieces(client, { diff --git a/packages/synapse-core/src/piece/download.ts b/packages/synapse-core/src/piece/download.ts index 7fb131901..6dbbb885b 100644 --- a/packages/synapse-core/src/piece/download.ts +++ b/packages/synapse-core/src/piece/download.ts @@ -1,6 +1,7 @@ -import { type AbortError, HttpError, type NetworkError, request, type TimeoutError } from 'iso-web/http' +import { HttpError, type Errors as HttpErrors, request } from 'iso-web/http' import { DownloadPieceError } from '../errors/pdp.ts' import { InvalidPieceCIDError } from '../errors/piece.ts' +import { RETRY_CONSTANTS } from '../utils/constants.ts' import { transformStream } from './calculate.ts' import { tryFrom } from './parse.ts' import type { PieceCID } from './piece-cid.ts' @@ -8,9 +9,15 @@ import type { PieceCID } from './piece-cid.ts' export namespace download { export type OptionsType = { url: string + /** The number of retries. Defaults to 2. */ + retryCount?: number + /** The delay with exponential backoff between retries in milliseconds. Defaults to 250ms. */ + retryDelay?: number + /** The signal to abort the request. */ + signal?: AbortSignal } export type ReturnType = Uint8Array - export type ErrorType = DownloadPieceError | TimeoutError | NetworkError | AbortError + export type ErrorType = DownloadPieceError | HttpErrors } /** @@ -21,7 +28,14 @@ export namespace download { * @throws Errors {@link download.ErrorType} */ export async function download(options: download.OptionsType): Promise { - const response = await request.get(options.url) + const response = await request.get(options.url, { + timeout: false, + retry: { + retries: options.retryCount, + minTimeout: options.retryDelay ?? RETRY_CONSTANTS.RETRY_DELAY, + }, + signal: options.signal, + }) if (response.error) { if (HttpError.is(response.error)) { throw new DownloadPieceError(await response.error.response.text()) @@ -35,9 +49,15 @@ export namespace downloadAndValidate { export type OptionsType = { url: string expectedPieceCid: string | PieceCID + /** The number of retries. Defaults to 2. */ + retryCount?: number + /** The delay with exponential backoff between retries in milliseconds. Defaults to 250ms. */ + retryDelay?: number + /** The signal to abort the request. */ + signal?: AbortSignal } export type ReturnType = Uint8Array - export type ErrorType = DownloadPieceError | TimeoutError | NetworkError | AbortError | InvalidPieceCIDError + export type ErrorType = DownloadPieceError | HttpErrors | InvalidPieceCIDError } /** @@ -71,7 +91,15 @@ export async function downloadAndValidate(options: downloadAndValidate.OptionsTy throw new InvalidPieceCIDError(expectedPieceCid) } - const rsp = await request.get(url) + const rsp = await request.get(url, { + timeout: false, + retry: { + retries: options.retryCount, + minTimeout: options.retryDelay ?? RETRY_CONSTANTS.RETRY_DELAY, + }, + signal: options.signal, + }) + if (rsp.error) { if (HttpError.is(rsp.error)) { throw new DownloadPieceError(await rsp.error.response.text()) diff --git a/packages/synapse-core/src/piece/resolve-piece-url.ts b/packages/synapse-core/src/piece/resolve-piece-url.ts index a30f8da38..e9dadf7fb 100644 --- a/packages/synapse-core/src/piece/resolve-piece-url.ts +++ b/packages/synapse-core/src/piece/resolve-piece-url.ts @@ -3,7 +3,6 @@ import pLocate from 'p-locate' import pSome from 'p-some' import type { Address, Chain, Client, Transport } from 'viem' import { asChain } from '../chains.ts' -import { findPiece } from '../sp/find-piece.ts' import type { PDPProvider } from '../sp-registry/types.ts' import { createPieceUrlPDP } from '../utils/piece-url.ts' import { getPdpDataSets } from '../warm-storage/get-pdp-data-sets.ts' @@ -132,13 +131,17 @@ export async function chainResolver(options: resolvePieceUrl.ResolverFnOptionsTy }, new Map()) const providers = [...providersById.values()] - const result = await findPieceOnProviders(providers, pieceCid, signal) + const result = await findPieceOnProviders( + providers.map((p) => p.pdp.serviceURL), + pieceCid, + signal + ) if (result == null) { throw new Error('No provider found') } return createPieceUrlPDP({ cid: pieceCid.toString(), - serviceURL: result.pdp.serviceURL, + serviceURL: result, }) } @@ -160,14 +163,18 @@ export async function chainResolver(options: resolvePieceUrl.ResolverFnOptionsTy export function providersResolver(providers: PDPProvider[]): resolvePieceUrl.ResolverFnType { return async (options: resolvePieceUrl.ResolverFnOptionsType) => { const { pieceCid, signal } = options - const result = await findPieceOnProviders(providers, pieceCid, signal) + const result = await findPieceOnProviders( + providers.map((p) => p.pdp.serviceURL), + pieceCid, + signal + ) if (result == null) { throw new Error('No provider found') } return createPieceUrlPDP({ cid: pieceCid.toString(), - serviceURL: result.pdp.serviceURL, + serviceURL: result, }) } } @@ -175,28 +182,35 @@ export function providersResolver(providers: PDPProvider[]): resolvePieceUrl.Res /** * Find the piece on the providers * - * @param providers - {@link PDPProvider[]} + * @param serviceURLs - {@link string[]} * @param pieceCid - {@link PieceCID} * @param signal - {@link AbortSignal} * @returns The piece URL */ -export async function findPieceOnProviders(providers: PDPProvider[], pieceCid: PieceCID, signal?: AbortSignal) { +export async function findPieceOnProviders(serviceURLs: string[], pieceCid: PieceCID, signal?: AbortSignal) { const controller = new AbortController() const _signal = signal ? AbortSignal.any([controller.signal, signal]) : controller.signal + async function headPiece(serviceURL: string) { + const result = await request.head(new URL(`piece/${pieceCid.toString()}`, serviceURL), { + signal: _signal, + retry: true, + }) + if (result.error) { + throw result.error + } + return serviceURL + } + const result = await pLocate( - providers.map((p) => - findPiece({ - serviceURL: p.pdp.serviceURL, - pieceCid, - signal: _signal, - }).then( + serviceURLs.map((p) => + headPiece(p).then( () => p, - () => null + () => undefined ) ), (p) => { - if (p !== null) { + if (p != null) { controller.abort() return true } diff --git a/packages/synapse-core/src/sp/add-pieces.ts b/packages/synapse-core/src/sp/add-pieces.ts index 734673f01..1e7311103 100644 --- a/packages/synapse-core/src/sp/add-pieces.ts +++ b/packages/synapse-core/src/sp/add-pieces.ts @@ -21,6 +21,10 @@ export namespace addPiecesApiRequest { pieces: PieceCID[] /** The extra data for the add pieces. {@link TypedData.signAddPieces} */ extraData: Hex + /** The number of retries. Defaults to 2. */ + retryCount?: number + /** The delay with exponential backoff between retries in milliseconds. Defaults to 250ms. */ + retryDelay?: number } export type OutputType = { /** The transaction hash. */ @@ -52,17 +56,19 @@ export async function addPiecesApiRequest( ): Promise { const { serviceURL, dataSetId, pieces, extraData } = options const response = await request.post(new URL(`pdp/data-sets/${dataSetId}/pieces`, serviceURL), { - headers: { - 'Content-Type': 'application/json', - }, - body: JSON.stringify({ + json: { pieces: pieces.map((piece) => ({ pieceCid: piece.toString(), subPieces: [{ subPieceCid: piece.toString() }], })), extraData: extraData, - }), - timeout: RETRY_CONSTANTS.MAX_RETRY_TIME, + }, + timeout: RETRY_CONSTANTS.TIMEOUT, + retry: { + methods: ['post'], + retries: options.retryCount, + minTimeout: options.retryDelay ?? RETRY_CONSTANTS.RETRY_DELAY, + }, }) if (response.error) { @@ -101,6 +107,10 @@ export namespace addPieces { nonce?: bigint /** Pre-built signed extraData. When provided, skips internal EIP-712 signing. */ extraData?: Hex + /** The number of retries. Defaults to 2. */ + retryCount?: number + /** The delay with exponential backoff between retries in milliseconds. Defaults to 250ms. */ + retryDelay?: number } export type OutputType = addPiecesApiRequest.OutputType @@ -154,6 +164,8 @@ export async function addPieces( dataSetId: options.dataSetId, pieces: options.pieces.map((piece) => piece.pieceCid), extraData, + retryCount: options.retryCount, + retryDelay: options.retryDelay, }) } @@ -199,7 +211,13 @@ export namespace waitForAddPieces { statusUrl: string /** The timeout in milliseconds. Defaults to 5 minutes. */ timeout?: number - /** The polling interval in milliseconds. Defaults to 4 seconds. */ + /** The number of retries. Defaults to 2. */ + retryCount?: number + /** The delay with exponential backoff between retries in milliseconds. Defaults to 250ms. */ + retryDelay?: number + /** Whether to poll the request. Defaults to false. */ + poll?: boolean + /** The poll interval in milliseconds. Defaults to 4 second. */ pollInterval?: number } export type OutputType = AddPiecesOutput @@ -221,22 +239,22 @@ export namespace waitForAddPieces { * @throws Errors {@link waitForAddPieces.ErrorType} */ export async function waitForAddPieces(options: waitForAddPieces.OptionsType): Promise { - const response = await request.json.get(options.statusUrl, { - async onResponse(response) { - if (response.ok) { - const data = (await response.clone().json()) as AddPiecesResponse - if (data.piecesAdded === false) { - throw new Error('Still pending') - } - } - }, + const response = await request.json.get(options.statusUrl, { retry: { - shouldRetry: (ctx) => ctx.error.message === 'Still pending', - retries: RETRY_CONSTANTS.RETRIES, - factor: RETRY_CONSTANTS.FACTOR, - minTimeout: options.pollInterval ?? RETRY_CONSTANTS.DELAY_TIME, + retries: options.retryCount, + minTimeout: options.retryDelay ?? RETRY_CONSTANTS.RETRY_DELAY, + }, + poll: { + limit: RETRY_CONSTANTS.POLL_LIMIT, + interval: options.pollInterval ?? RETRY_CONSTANTS.POLL_INTERVAL, + statusCodes: [202, 200], // 202 is processing, 200 is success + shouldPoll: async (ctx) => { + const data = (await ctx.response.clone().json()) as AddPiecesResponse + return data.piecesAdded === false + }, }, - timeout: options.timeout ?? RETRY_CONSTANTS.MAX_RETRY_TIME, + timeout: options.timeout ?? RETRY_CONSTANTS.TIMEOUT, + schema, }) if (response.error) { if (HttpError.is(response.error)) { @@ -244,9 +262,8 @@ export async function waitForAddPieces(options: waitForAddPieces.OptionsType): P } throw response.error } - const data = schema.parse(response.result) - if (data.txStatus === 'rejected') { - throw new WaitForAddPiecesRejectedError(data) + if (response.result.txStatus === 'rejected') { + throw new WaitForAddPiecesRejectedError(response.result) } - return data + return response.result } diff --git a/packages/synapse-core/src/sp/create-dataset-add-pieces.ts b/packages/synapse-core/src/sp/create-dataset-add-pieces.ts index e8d25e3b3..2d7793537 100644 --- a/packages/synapse-core/src/sp/create-dataset-add-pieces.ts +++ b/packages/synapse-core/src/sp/create-dataset-add-pieces.ts @@ -1,4 +1,4 @@ -import { type AbortError, HttpError, type NetworkError, request, type TimeoutError } from 'iso-web/http' +import { HttpError, type Errors as HttpErrors, request } from 'iso-web/http' import type { ToString } from 'multiformats' import { type Account, type Address, type Chain, type Client, type Hex, isHex, type Transport } from 'viem' import { asChain } from '../chains.ts' @@ -26,6 +26,10 @@ export namespace createDataSetAndAddPiecesApiRequest { extraData: Hex /** The pieces to add. */ pieces: PieceCID[] + /** The number of retries. Defaults to 2. */ + retryCount?: number + /** The delay with exponential backoff between retries in milliseconds. Defaults to 250ms. */ + retryDelay?: number } export type OutputType = { /** The transaction hash. */ @@ -33,7 +37,7 @@ export namespace createDataSetAndAddPiecesApiRequest { /** The status URL. */ statusUrl: string } - export type ErrorType = CreateDataSetError | LocationHeaderError | TimeoutError | NetworkError | AbortError + export type ErrorType = CreateDataSetError | LocationHeaderError | HttpErrors export type RequestBody = { recordKeeper: Address extraData: Hex @@ -58,18 +62,20 @@ export async function createDataSetAndAddPiecesApiRequest( ): Promise { // Send the create data set message to the PDP const response = await request.post(new URL(`pdp/data-sets/create-and-add`, options.serviceURL), { - body: JSON.stringify({ + json: { recordKeeper: options.recordKeeper, extraData: options.extraData, pieces: options.pieces.map((piece) => ({ pieceCid: piece.toString(), subPieces: [{ subPieceCid: piece.toString() }], })), - }), - headers: { - 'Content-Type': 'application/json', }, - timeout: RETRY_CONSTANTS.MAX_RETRY_TIME, + timeout: RETRY_CONSTANTS.TIMEOUT, + retry: { + methods: ['post'], + retries: options.retryCount, + minTimeout: options.retryDelay ?? RETRY_CONSTANTS.RETRY_DELAY, + }, }) if (response.error) { @@ -114,6 +120,10 @@ export type CreateDataSetAndAddPiecesOptions = { cdn?: boolean /** The address of the record keeper to use for the signature. If not provided, the default is the Warm Storage contract address. */ recordKeeper?: Address + /** The number of retries. Defaults to 2. */ + retryCount?: number + /** The delay with exponential backoff between retries in milliseconds. Defaults to 250ms. */ + retryDelay?: number } export namespace createDataSetAndAddPieces { @@ -156,6 +166,8 @@ export async function createDataSetAndAddPieces( recordKeeper: options.recordKeeper ?? chain.contracts.fwss.address, extraData, pieces: options.pieces.map((piece) => piece.pieceCid), + retryCount: options.retryCount, + retryDelay: options.retryDelay, }) } @@ -165,7 +177,13 @@ export namespace waitForCreateDataSetAddPieces { statusUrl: string /** The timeout in milliseconds. Defaults to 5 minutes. */ timeout?: number - /** The polling interval in milliseconds. Defaults to 4 seconds. */ + /** The number of retries. Defaults to 2. */ + retryCount?: number + /** The delay with exponential backoff between retries in milliseconds. Defaults to 250ms. */ + retryDelay?: number + /** Whether to poll the request. Defaults to false. */ + poll?: boolean + /** The poll interval in milliseconds. Defaults to 4 second. */ pollInterval?: number } export type ReturnType = { @@ -178,9 +196,7 @@ export namespace waitForCreateDataSetAddPieces { | WaitForCreateDataSetRejectedError | WaitForAddPiecesError | WaitForAddPiecesRejectedError - | TimeoutError - | NetworkError - | AbortError + | HttpErrors } /** @@ -196,12 +212,22 @@ export async function waitForCreateDataSetAddPieces( options: waitForCreateDataSetAddPieces.OptionsType ): Promise { const origin = new URL(options.statusUrl).origin - const createdDataset = await waitForCreateDataSet({ statusUrl: options.statusUrl }) + const createdDataset = await waitForCreateDataSet({ + statusUrl: options.statusUrl, + retryCount: options.retryCount, + retryDelay: options.retryDelay, + poll: options.poll, + pollInterval: options.pollInterval, + }) const addedPieces = await waitForAddPieces({ statusUrl: new URL( `/pdp/data-sets/${createdDataset.dataSetId}/pieces/added/${createdDataset.createMessageHash}`, origin ).toString(), + retryCount: options.retryCount, + retryDelay: options.retryDelay, + poll: options.poll, + pollInterval: options.pollInterval, }) return { hash: createdDataset.createMessageHash, diff --git a/packages/synapse-core/src/sp/create-dataset.ts b/packages/synapse-core/src/sp/create-dataset.ts index 4bac8cc70..a21690174 100644 --- a/packages/synapse-core/src/sp/create-dataset.ts +++ b/packages/synapse-core/src/sp/create-dataset.ts @@ -1,4 +1,4 @@ -import { HttpError, request } from 'iso-web/http' +import { HttpError, type Errors as HttpErrors, request } from 'iso-web/http' import { type Account, type Address, @@ -18,7 +18,6 @@ import { signCreateDataSet } from '../typed-data/sign-create-dataset.ts' import { RETRY_CONSTANTS } from '../utils/constants.ts' import { datasetMetadataObjectToEntry, type MetadataObject } from '../utils/metadata.ts' import { zHex, zNumberToBigInt } from '../utils/schemas.ts' -import type { AbortError, NetworkError, TimeoutError } from './index.ts' export namespace createDataSetApiRequest { /** @@ -31,6 +30,10 @@ export namespace createDataSetApiRequest { recordKeeper: Address /** The extra data for the create data set. */ extraData: Hex + /** The number of retries. Defaults to 2. */ + retryCount?: number + /** The delay with exponential backoff between retries in milliseconds. Defaults to 250ms. */ + retryDelay?: number } export type OutputType = { @@ -38,7 +41,7 @@ export namespace createDataSetApiRequest { statusUrl: string } - export type ErrorType = CreateDataSetError | LocationHeaderError | TimeoutError | NetworkError | AbortError + export type ErrorType = CreateDataSetError | LocationHeaderError | HttpErrors export type RequestBody = { recordKeeper: Address @@ -60,14 +63,16 @@ export async function createDataSetApiRequest( ): Promise { // Send the create data set message to the PDP const response = await request.post(new URL(`pdp/data-sets`, options.serviceURL), { - body: JSON.stringify({ + json: { recordKeeper: options.recordKeeper, extraData: options.extraData, - }), - headers: { - 'Content-Type': 'application/json', }, - timeout: RETRY_CONSTANTS.MAX_RETRY_TIME, + timeout: RETRY_CONSTANTS.TIMEOUT, + retry: { + methods: ['post'], + retries: options.retryCount, + minTimeout: options.retryDelay ?? RETRY_CONSTANTS.RETRY_DELAY, + }, }) if (response.error) { @@ -108,6 +113,10 @@ export namespace createDataSet { clientDataSetId?: bigint /** The address of the record keeper to use for the signature. If not provided, the default is the Warm Storage contract address. */ recordKeeper?: Address + /** The number of retries. Defaults to 2. */ + retryCount?: number + /** The delay with exponential backoff between retries in milliseconds. Defaults to 250ms. */ + retryDelay?: number } export type ReturnType = createDataSetApiRequest.OutputType export type ErrorType = @@ -142,6 +151,8 @@ export async function createDataSet(client: Client, o serviceURL: options.serviceURL, recordKeeper: options.recordKeeper ?? chain.contracts.fwss.address, extraData, + retryCount: options.retryCount, + retryDelay: options.retryDelay, }) } @@ -192,16 +203,17 @@ export namespace waitForCreateDataSet { statusUrl: string /** The timeout in milliseconds. Defaults to 5 minutes. */ timeout?: number - /** The polling interval in milliseconds. Defaults to 4 seconds. */ + /** The number of retries. Defaults to 2. */ + retryCount?: number + /** The delay with exponential backoff between retries in milliseconds. Defaults to 250ms. */ + retryDelay?: number + /** Whether to poll the request. Defaults to false. */ + poll?: boolean + /** The poll interval in milliseconds. Defaults to 4 second. */ pollInterval?: number } export type ReturnType = CreateDataSetSuccess - export type ErrorType = - | WaitForCreateDataSetError - | WaitForCreateDataSetRejectedError - | TimeoutError - | NetworkError - | AbortError + export type ErrorType = WaitForCreateDataSetError | WaitForCreateDataSetRejectedError | HttpErrors } /** @@ -216,23 +228,23 @@ export namespace waitForCreateDataSet { export async function waitForCreateDataSet( options: waitForCreateDataSet.OptionsType ): Promise { - const response = await request.json.get(options.statusUrl, { - async onResponse(response) { - if (response.ok) { - const data = (await response.clone().json()) as CreateDataSetResponse - if (data.dataSetCreated === false) { - throw new Error('Still pending') - } - } - }, + const response = await request.json.get(options.statusUrl, { retry: { - shouldRetry: (ctx) => ctx.error.message === 'Still pending', - retries: RETRY_CONSTANTS.RETRIES, - factor: RETRY_CONSTANTS.FACTOR, - minTimeout: options.pollInterval ?? RETRY_CONSTANTS.DELAY_TIME, + retries: options.retryCount, + minTimeout: options.retryDelay ?? RETRY_CONSTANTS.RETRY_DELAY, + }, + poll: { + limit: RETRY_CONSTANTS.POLL_LIMIT, + interval: options.pollInterval ?? RETRY_CONSTANTS.POLL_INTERVAL, + statusCodes: [202, 200], // 202 is processing, 200 is success + shouldPoll: async (ctx) => { + const data = (await ctx.response.clone().json()) as CreateDataSetResponse + return data.dataSetCreated === false + }, }, - timeout: options.timeout ?? RETRY_CONSTANTS.MAX_RETRY_TIME, + timeout: options.timeout ?? RETRY_CONSTANTS.TIMEOUT, + schema, }) if (response.error) { if (HttpError.is(response.error)) { @@ -241,9 +253,8 @@ export async function waitForCreateDataSet( throw response.error } - const data = schema.parse(response.result) - if (data.txStatus === 'rejected') { - throw new WaitForCreateDataSetRejectedError(data) + if (response.result.txStatus === 'rejected') { + throw new WaitForCreateDataSetRejectedError(response.result) } - return data + return response.result } diff --git a/packages/synapse-core/src/sp/find-piece.ts b/packages/synapse-core/src/sp/find-piece.ts index c8401c6e4..784bc2d6f 100644 --- a/packages/synapse-core/src/sp/find-piece.ts +++ b/packages/synapse-core/src/sp/find-piece.ts @@ -12,10 +12,14 @@ export namespace findPiece { pieceCid: PieceCID /** The signal to abort the request. */ signal?: AbortSignal - /** Whether to retry the request. Defaults to false. */ - retry?: boolean /** The timeout in milliseconds. Defaults to 5 minutes. */ timeout?: number + /** The number of retries. Defaults to 2. */ + retryCount?: number + /** The delay with exponential backoff between retries in milliseconds. Defaults to 250ms. */ + retryDelay?: number + /** Whether to poll the request. Defaults to false. */ + poll?: boolean /** The poll interval in milliseconds. Defaults to 1 second. */ pollInterval?: number } @@ -34,18 +38,20 @@ export namespace findPiece { export async function findPiece(options: findPiece.OptionsType): Promise { const { pieceCid, serviceURL } = options const params = new URLSearchParams({ pieceCid: pieceCid.toString() }) - const retry = options.retry ?? false const response = await request.json.get<{ pieceCid: string }>(new URL(`pdp/piece?${params.toString()}`, serviceURL), { signal: options.signal, - retry: retry + timeout: options.timeout ?? RETRY_CONSTANTS.TIMEOUT, + retry: { + retries: options.retryCount, + minTimeout: options.retryDelay ?? RETRY_CONSTANTS.RETRY_DELAY, + }, + poll: options.poll ? { - statusCodes: [202, 404], - retries: RETRY_CONSTANTS.RETRIES, - factor: RETRY_CONSTANTS.FACTOR, - minTimeout: options.pollInterval ?? 1000, + limit: RETRY_CONSTANTS.POLL_LIMIT, + interval: options.pollInterval ?? 1000, + statusCodes: [202, 404], // 202 is processing, 404 is not found } - : undefined, - timeout: options.timeout ?? RETRY_CONSTANTS.MAX_RETRY_TIME, + : false, }) if (response.error) { diff --git a/packages/synapse-core/src/sp/get-data-set.ts b/packages/synapse-core/src/sp/get-data-set.ts index 1a3eb1504..f90f2d963 100644 --- a/packages/synapse-core/src/sp/get-data-set.ts +++ b/packages/synapse-core/src/sp/get-data-set.ts @@ -1,6 +1,7 @@ -import { type AbortError, HttpError, type NetworkError, request, type TimeoutError } from 'iso-web/http' +import { HttpError, type Errors as HttpErrors, request } from 'iso-web/http' import * as z from 'zod' import { GetDataSetError } from '../errors/pdp.ts' +import { RETRY_CONSTANTS } from '../utils/constants.ts' import { zNumberToBigInt, zStringToCid } from '../utils/schemas.ts' const PieceSchema = z.object({ @@ -27,9 +28,13 @@ export namespace getDataSet { serviceURL: string /** The ID of the data set. */ dataSetId: bigint + /** The number of retries. Defaults to 2. */ + retryCount?: number + /** The delay with exponential backoff between retries in milliseconds. Defaults to 250ms. */ + retryDelay?: number } export type OutputType = DataSet - export type ErrorType = GetDataSetError | TimeoutError | NetworkError | AbortError + export type ErrorType = GetDataSetError | HttpErrors } /** @@ -43,9 +48,14 @@ export namespace getDataSet { * @throws Errors {@link getDataSet.ErrorType} */ export async function getDataSet(options: getDataSet.OptionsType): Promise { - const response = await request.json.get( - new URL(`pdp/data-sets/${options.dataSetId}`, options.serviceURL) - ) + const response = await request.json.get(new URL(`pdp/data-sets/${options.dataSetId}`, options.serviceURL), { + timeout: RETRY_CONSTANTS.TIMEOUT, + retry: { + retries: options.retryCount, + minTimeout: options.retryDelay ?? RETRY_CONSTANTS.RETRY_DELAY, + }, + schema: DataSetSchema, + }) if (response.error) { if (HttpError.is(response.error)) { throw new GetDataSetError(await response.error.response.text()) @@ -53,5 +63,5 @@ export async function getDataSet(options: getDataSet.OptionsType): Promise { const response = await request.post(new URL('pdp/piece/pull', options.serviceURL), { - body: buildPullRequestBody(options), - headers: { - 'Content-Type': 'application/json', + json: buildPullRequestBody(options), + timeout: RETRY_CONSTANTS.TIMEOUT, + retry: { + methods: ['post'], + retries: options.retryCount, + minTimeout: options.retryDelay ?? RETRY_CONSTANTS.RETRY_DELAY, }, - timeout: RETRY_CONSTANTS.MAX_RETRY_TIME, signal: options.signal, }) @@ -147,7 +153,13 @@ export namespace waitForPullPiecesApiRequest { onStatus?: (response: pullPiecesApiRequest.ReturnType) => void /** The timeout in milliseconds. Defaults to 5 minutes. */ timeout?: number - /** The polling interval in milliseconds. Defaults to 4 seconds. */ + /** The number of retries. Defaults to 2. */ + retryCount?: number + /** The delay with exponential backoff between retries in milliseconds. Defaults to 250ms. */ + retryDelay?: number + /** Whether to poll the request. Defaults to false. */ + poll?: boolean + /** The poll interval in milliseconds. Defaults to 4 second. */ pollInterval?: number } @@ -170,35 +182,28 @@ export async function waitForPullPiecesApiRequest( options: waitForPullPiecesApiRequest.OptionsType ): Promise { const url = new URL('pdp/piece/pull', options.serviceURL) - const body = buildPullRequestBody(options) - const headers = { 'Content-Type': 'application/json' } const response = await request.post(url, { - body, - headers, - async onResponse(response) { - if (response.ok) { - const data = (await response.clone().json()) as pullPiecesApiRequest.ReturnType - + json: buildPullRequestBody(options), + retry: { + methods: ['post'], + retries: options.retryCount, + minTimeout: options.retryDelay ?? RETRY_CONSTANTS.RETRY_DELAY, + }, + poll: { + limit: RETRY_CONSTANTS.POLL_LIMIT, + interval: options.pollInterval ?? RETRY_CONSTANTS.POLL_INTERVAL, + statusCodes: [202, 200], // 202 is processing, 200 is success + shouldPoll: async (ctx) => { + const data = (await ctx.response.clone().json()) as pullPiecesApiRequest.ReturnType // Invoke status callback if provided if (options.onStatus) { options.onStatus(data) } - - // Stop polling when complete or failed - if (data.status === 'complete' || data.status === 'failed') { - return response - } - throw new Error('Pull not complete') - } - }, - retry: { - shouldRetry: (ctx) => ctx.error.message === 'Pull not complete', - retries: RETRY_CONSTANTS.RETRIES, - factor: RETRY_CONSTANTS.FACTOR, - minTimeout: options.pollInterval ?? RETRY_CONSTANTS.DELAY_TIME, + return data.status !== 'complete' && data.status !== 'failed' + }, }, - timeout: options.timeout ?? RETRY_CONSTANTS.MAX_RETRY_TIME, + timeout: options.timeout ?? RETRY_CONSTANTS.TIMEOUT, signal: options.signal, }) @@ -391,7 +396,13 @@ export namespace waitForPullPieces { onStatus?: (response: pullPieces.ReturnType) => void /** The timeout in milliseconds. Defaults to 5 minutes. */ timeout?: number - /** The polling interval in milliseconds. Defaults to 4 seconds. */ + /** The number of retries. Defaults to 2. */ + retryCount?: number + /** The delay with exponential backoff between retries in milliseconds. Defaults to 250ms. */ + retryDelay?: number + /** Whether to poll the request. Defaults to false. */ + poll?: boolean + /** The poll interval in milliseconds. Defaults to 4 second. */ pollInterval?: number } @@ -421,5 +432,8 @@ export async function waitForPullPieces( onStatus: options.onStatus, timeout: options.timeout, pollInterval: options.pollInterval, + poll: options.poll, + retryCount: options.retryCount, + retryDelay: options.retryDelay, }) } diff --git a/packages/synapse-core/src/sp/schedule-piece-deletion.ts b/packages/synapse-core/src/sp/schedule-piece-deletion.ts index 1375ca3e9..9bafef0f3 100644 --- a/packages/synapse-core/src/sp/schedule-piece-deletion.ts +++ b/packages/synapse-core/src/sp/schedule-piece-deletion.ts @@ -1,4 +1,4 @@ -import { type AbortError, HttpError, type NetworkError, request, type TimeoutError } from 'iso-web/http' +import { HttpError, type Errors as HttpErrors, request } from 'iso-web/http' import type { Account, Chain, Client, Hex, Transport } from 'viem' import { DeletePieceError } from '../errors/pdp.ts' import { signSchedulePieceRemovals } from '../typed-data/sign-schedule-piece-removals.ts' @@ -10,11 +10,15 @@ export namespace deletePiece { dataSetId: bigint pieceId: bigint extraData: Hex + /** The number of retries. Defaults to 2. */ + retryCount?: number + /** The delay with exponential backoff between retries in milliseconds. Defaults to 250ms. */ + retryDelay?: number } export type OutputType = { hash: Hex } - export type ErrorType = DeletePieceError | TimeoutError | NetworkError | AbortError + export type ErrorType = DeletePieceError | HttpErrors } /** @@ -32,7 +36,11 @@ export async function deletePiece(options: deletePiece.OptionsType): Promise { // Create upload session (POST /pdp/piece/uploads) const createResponse = await request.post(new URL('pdp/piece/uploads', options.serviceURL), { - timeout: RETRY_CONSTANTS.MAX_RETRY_TIME, + timeout: RETRY_CONSTANTS.TIMEOUT, + retry: { + methods: ['post'], + retries: options.retryCount, + minTimeout: options.retryDelay ?? RETRY_CONSTANTS.RETRY_DELAY, + }, signal: options.signal, }) @@ -169,7 +178,7 @@ export async function uploadPieceStreaming( timeout: false, // No timeout for streaming upload signal: options.signal, ...fetchOptions, - } as Parameters[1] & { duplex?: 'half' }) + }) if (uploadResponse.error) { if (HttpError.is(uploadResponse.error)) { @@ -185,17 +194,17 @@ export async function uploadPieceStreaming( // Get PieceCID (either provided or calculated) and finalize. const pieceCid = await pieceCidPromise - const finalizeBody = JSON.stringify({ - pieceCid: pieceCid.toString(), - }) - // POST /pdp/piece/uploads/{uuid} with PieceCID const finalizeResponse = await request.post(new URL(`pdp/piece/uploads/${uploadUuid}`, options.serviceURL), { - body: finalizeBody, - headers: { - 'Content-Type': 'application/json', + json: { + pieceCid: pieceCid.toString(), + }, + timeout: RETRY_CONSTANTS.TIMEOUT, + retry: { + methods: ['post'], + retries: options.retryCount, + minTimeout: options.retryDelay ?? RETRY_CONSTANTS.RETRY_DELAY, }, - timeout: RETRY_CONSTANTS.MAX_RETRY_TIME, signal: options.signal, }) diff --git a/packages/synapse-core/src/sp/upload.ts b/packages/synapse-core/src/sp/upload.ts index 15c870f8c..576d4e4f6 100644 --- a/packages/synapse-core/src/sp/upload.ts +++ b/packages/synapse-core/src/sp/upload.ts @@ -19,6 +19,10 @@ export namespace uploadPiece { data: Uint8Array /** The piece CID to upload. */ pieceCid: PieceCID + /** The number of retries. Defaults to 2. */ + retryCount?: number + /** The delay with exponential backoff between retries in milliseconds. Defaults to 250ms. */ + retryDelay?: number } export type ErrorType = InvalidUploadSizeError | LocationHeaderError | TimeoutError | NetworkError | AbortError } @@ -42,13 +46,15 @@ export async function uploadPiece(options: uploadPiece.OptionsType): Promise, options: await findPiece({ pieceCid, serviceURL, - retry: true, + poll: true, }) options.onEvent?.('pieceParked', { pieceCid, url, dataSet }) diff --git a/packages/synapse-core/src/utils/constants.ts b/packages/synapse-core/src/utils/constants.ts index d321e93bf..08d4d80cf 100644 --- a/packages/synapse-core/src/utils/constants.ts +++ b/packages/synapse-core/src/utils/constants.ts @@ -141,10 +141,11 @@ export const CDN_FIXED_LOCKUP = { export const USDFC_SYBIL_FEE = 100_000_000_000_000_000n // 0.1 USDFC export const RETRY_CONSTANTS = { - RETRIES: Infinity, - FACTOR: 1, - DELAY_TIME: 4000, // 4 seconds in milliseconds between retries - MAX_RETRY_TIME: 1000 * 60 * 5, // 5 minutes in milliseconds + POLL_INTERVAL: 4000, // 4 seconds in milliseconds between retries + POLL_LIMIT: Infinity, + RETRY_DELAY: 250, // 250ms in milliseconds between retries + /** The timeout in milliseconds. Defaults to 5 minutes. */ + TIMEOUT: 1000 * 60 * 5, } as const /** diff --git a/packages/synapse-core/test/pull.test.ts b/packages/synapse-core/test/pull.test.ts index b0b4939ac..46dd4e4b9 100644 --- a/packages/synapse-core/test/pull.test.ts +++ b/packages/synapse-core/test/pull.test.ts @@ -28,6 +28,7 @@ describe('Pull', () => { recordKeeper: TEST_RECORD_KEEPER, extraData: TEST_EXTRA_DATA, pieces: [{ pieceCid: TEST_PIECE_CID, sourceUrl: TEST_SOURCE_URL }], + retryDelay: 10, }) before(async () => { diff --git a/packages/synapse-core/test/resolve-piece-url.test.ts b/packages/synapse-core/test/resolve-piece-url.test.ts index 45ad07d9e..a15bafb88 100644 --- a/packages/synapse-core/test/resolve-piece-url.test.ts +++ b/packages/synapse-core/test/resolve-piece-url.test.ts @@ -109,9 +109,8 @@ describe('resolve-piece-url', () => { server.use( JSONRPC(presets.basic), http.head(filbeamUrl, () => HttpResponse.text('not found', { status: 404 })), - http.get('https://pdp.example.com/pdp/piece', ({ request }) => { - const url = new URL(request.url) - return HttpResponse.json({ pieceCid: url.searchParams.get('pieceCid') }, { status: 200 }) + http.head(`https://pdp.example.com/piece/${pieceCidString}`, () => { + return new HttpResponse(null, { status: 200 }) }) ) @@ -191,26 +190,35 @@ describe('resolve-piece-url', () => { ] server.use( - http.get('https://missing.example.com/pdp/piece', () => HttpResponse.text('not found', { status: 404 })), - http.get('https://pdp.example.com/pdp/piece', ({ request }) => { - const url = new URL(request.url) - return HttpResponse.json({ pieceCid: url.searchParams.get('pieceCid') }, { status: 200 }) + http.head(`https://missing.example.com/piece/${pieceCidString}`, () => + HttpResponse.text('not found', { status: 404 }) + ), + http.head(`https://pdp.example.com/piece/${pieceCidString}`, () => { + return new HttpResponse(null, { status: 200 }) }) ) - const result = await findPieceOnProviders(providers, pieceCid) + const result = await findPieceOnProviders( + providers.map((p) => p.pdp.serviceURL), + pieceCid + ) assert.ok(result) - assert.equal(result?.id, 2n) + assert.equal(result, providers[1].pdp.serviceURL) }) it('returns undefined when no provider has the piece', async () => { const providers: PDPProvider[] = [createProvider('https://missing.example.com/')] server.use( - http.get('https://missing.example.com/pdp/piece', () => HttpResponse.text('not found', { status: 404 })) + http.head(`https://missing.example.com/piece/${pieceCidString}`, () => + HttpResponse.text('not found', { status: 404 }) + ) ) - const result = await findPieceOnProviders(providers, pieceCid) + const result = await findPieceOnProviders( + providers.map((p) => p.pdp.serviceURL), + pieceCid + ) assert.equal(result, undefined) }) }) @@ -219,9 +227,8 @@ describe('resolve-piece-url', () => { it('returns serviceURL when a provider contains the piece', async () => { const providers: PDPProvider[] = [createProvider('https://pdp.example.com/', 5n)] server.use( - http.get('https://pdp.example.com/pdp/piece', ({ request }) => { - const url = new URL(request.url) - return HttpResponse.json({ pieceCid: url.searchParams.get('pieceCid') }, { status: 200 }) + http.head(`https://pdp.example.com/piece/${pieceCidString}`, () => { + return new HttpResponse(null, { status: 200 }) }) ) @@ -237,7 +244,9 @@ describe('resolve-piece-url', () => { it('throws when no provider has the piece', async () => { const providers: PDPProvider[] = [createProvider('https://missing.example.com/')] server.use( - http.get('https://missing.example.com/pdp/piece', () => HttpResponse.text('not found', { status: 404 })) + http.head(`https://missing.example.com/piece/${pieceCidString}`, () => + HttpResponse.text('not found', { status: 404 }) + ) ) const resolver = providersResolver(providers) @@ -256,9 +265,8 @@ describe('resolve-piece-url', () => { it('resolves piece URL from on-chain provider list', async () => { server.use( JSONRPC(presets.basic), - http.get('https://pdp.example.com/pdp/piece', ({ request }) => { - const url = new URL(request.url) - return HttpResponse.json({ pieceCid: url.searchParams.get('pieceCid') }, { status: 200 }) + http.head(`https://pdp.example.com/piece/${pieceCidString}`, () => { + return new HttpResponse(null, { status: 200 }) }) ) diff --git a/packages/synapse-core/test/sp.test.ts b/packages/synapse-core/test/sp.test.ts index 24c0e9c2d..bfd99835f 100644 --- a/packages/synapse-core/test/sp.test.ts +++ b/packages/synapse-core/test/sp.test.ts @@ -214,6 +214,8 @@ describe('SP', () => { clientDataSetId: 0n, payee: ADDRESSES.client1, }), + retryCount: 1, + retryDelay: 10, }) assert.fail('Should have thrown error for CreateDataSetError error') } catch (e) { @@ -255,6 +257,8 @@ invariant failure: insufficient funds to cover lockup after function execution` clientDataSetId: 0n, payee: ADDRESSES.client1, }), + retryCount: 1, + retryDelay: 10, }) assert.fail('Should have thrown error for CreateDataSetError error') } catch (error) { @@ -296,6 +300,8 @@ InvalidSignature(address expected, address actual) clientDataSetId: 0n, payee: ADDRESSES.client1, }), + retryCount: 1, + retryDelay: 10, }) assert.fail('Should have thrown error for CreateDataSetError error') } catch (error) { @@ -402,6 +408,7 @@ InvalidSignature(address expected, address actual) try { await waitForCreateDataSet({ statusUrl: `http://pdp.local/pdp/data-sets/created/${mockTxHash}`, + retryDelay: 10, }) assert.fail('Should have thrown error for server error') } catch (error) { @@ -541,6 +548,7 @@ InvalidSignature(address expected, address actual) dataSetId: 1n, pieces: [pieceCid], extraData, + retryDelay: 10, }) assert.fail('Should have thrown error for server error') } catch (error) { @@ -752,6 +760,7 @@ InvalidSignature(address expected, address actual) try { await waitForAddPieces({ statusUrl: `http://pdp.local/pdp/data-sets/1/pieces/added/${mockTxHash}`, + retryDelay: 10, }) assert.fail('Should have thrown error for server error') } catch (error) { @@ -849,6 +858,7 @@ InvalidSignature(address expected, address actual) dataSetId: 1n, pieceId: 2n, extraData, + retryDelay: 10, }) assert.fail('Should have thrown error for server error') } catch (error) { @@ -883,8 +893,8 @@ InvalidSignature(address expected, address actual) await findPiece({ serviceURL: 'https://pdp.example.com', pieceCid, - retry: true, - timeout: 50, + poll: true, + timeout: 100, }) assert.fail('Should have thrown error for not found') } catch (error) { @@ -909,6 +919,7 @@ InvalidSignature(address expected, address actual) await findPiece({ serviceURL: 'https://pdp.example.com', pieceCid, + retryDelay: 10, }) assert.fail('Should have thrown error for server error') } catch (error) { @@ -936,7 +947,7 @@ InvalidSignature(address expected, address actual) const result = await findPiece({ serviceURL: 'http://pdp.local', pieceCid, - retry: true, + poll: true, pollInterval: 10, }) assert.strictEqual(result.toString(), mockPieceCidStr) @@ -1058,6 +1069,7 @@ InvalidSignature(address expected, address actual) serviceURL: 'http://pdp.local', data: testData, pieceCid, + retryDelay: 10, }) assert.fail('Should have thrown error for POST error') } catch (error) { @@ -1082,6 +1094,7 @@ InvalidSignature(address expected, address actual) serviceURL: 'https://pdp.example.com', data: testData, pieceCid, + retryDelay: 10, }) assert.fail('Should have thrown error for PUT error') } catch (error) { @@ -1187,6 +1200,7 @@ InvalidSignature(address expected, address actual) serviceURL: 'http://pdp.local', data: testData, pieceCid, + retryDelay: 10, }) assert.fail('Should have thrown error for session creation failure') } catch (error) { @@ -1332,6 +1346,7 @@ InvalidSignature(address expected, address actual) serviceURL: 'https://pdp.example.com', data: testData, pieceCid, + retryDelay: 10, }) assert.fail('Should have thrown error for finalize failure') } catch (error) { @@ -1441,6 +1456,7 @@ InvalidSignature(address expected, address actual) await getDataSet({ serviceURL: 'http://pdp.local', dataSetId: 292n, + retryDelay: 10, }) assert.fail('Should have thrown error for server error') } catch (error) { diff --git a/packages/synapse-sdk/src/storage/context.ts b/packages/synapse-sdk/src/storage/context.ts index 6a80beff8..bc84782ea 100644 --- a/packages/synapse-sdk/src/storage/context.ts +++ b/packages/synapse-sdk/src/storage/context.ts @@ -680,7 +680,7 @@ export class StorageContext { await SP.findPiece({ serviceURL: this._pdpEndpoint, pieceCid: uploadResult.pieceCid, - retry: true, + poll: true, signal: options?.signal, }) } catch (error) { diff --git a/packages/synapse-sdk/src/test/storage.test.ts b/packages/synapse-sdk/src/test/storage.test.ts index 1757e3337..3110e2fe6 100644 --- a/packages/synapse-sdk/src/test/storage.test.ts +++ b/packages/synapse-sdk/src/test/storage.test.ts @@ -827,7 +827,9 @@ describe('StorageService', () => { status: 404, }) }), - Mocks.pdp.findPieceHandler(testPieceCID, true, pdpOptions), + http.head('https://pdp.example.com/piece/:pieceCid', async () => { + return new HttpResponse(null, { status: 200 }) + }), http.get('https://pdp.example.com/piece/:pieceCid', async () => { return HttpResponse.arrayBuffer(testData.buffer) }) @@ -849,7 +851,9 @@ describe('StorageService', () => { ...Mocks.presets.basic, }), Mocks.PING(), - Mocks.pdp.findPieceHandler(testPieceCID, true, pdpOptions), + http.head('https://pdp.example.com/piece/:pieceCid', async () => { + return new HttpResponse(null, { status: 200 }) + }), http.get('https://pdp.example.com/piece/:pieceCid', async () => { return HttpResponse.error() }) @@ -875,7 +879,9 @@ describe('StorageService', () => { ...Mocks.presets.basic, }), Mocks.PING(), - Mocks.pdp.findPieceHandler(testPieceCID, true, pdpOptions), + http.head('https://pdp.example.com/piece/:pieceCid', async () => { + return new HttpResponse(null, { status: 200 }) + }), http.get('https://pdp.example.com/piece/:pieceCid', async () => { return HttpResponse.arrayBuffer(testData.buffer) }) diff --git a/packages/synapse-sdk/src/test/synapse.test.ts b/packages/synapse-sdk/src/test/synapse.test.ts index 9829f81b6..b5788c271 100644 --- a/packages/synapse-sdk/src/test/synapse.test.ts +++ b/packages/synapse-sdk/src/test/synapse.test.ts @@ -194,13 +194,8 @@ describe('Synapse', () => { const testData = new TextEncoder().encode('test data') server.use( Mocks.JSONRPC(Mocks.presets.basic), - http.get('https://pdp.example.com/pdp/piece', async ({ request }) => { - const url = new URL(request.url) - const pieceCid = url.searchParams.get('pieceCid') - - return HttpResponse.json({ - pieceCid, - }) + http.head('https://pdp.example.com/piece/:pieceCid', async () => { + return new HttpResponse(null, { status: 200 }) }), http.get('https://pdp.example.com/piece/:pieceCid', async () => { return HttpResponse.arrayBuffer(testData.buffer) @@ -230,13 +225,8 @@ describe('Synapse', () => { deferred.resolve(params) return HttpResponse.arrayBuffer(testData.buffer) }), - http.get('https://pdp.example.com/pdp/piece', async ({ request }) => { - const url = new URL(request.url) - const pieceCid = url.searchParams.get('pieceCid') - - return HttpResponse.json({ - pieceCid, - }) + http.head('https://pdp.example.com/piece/:pieceCid', async () => { + return new HttpResponse(null, { status: 200 }) }), http.get('https://pdp.example.com/piece/:pieceCid', async () => { return HttpResponse.arrayBuffer(testData.buffer) @@ -281,13 +271,8 @@ describe('Synapse', () => { }, }, }), - http.get('https://pdp.example.com/pdp/piece', async ({ request }) => { - const url = new URL(request.url) - const pieceCid = url.searchParams.get('pieceCid') - - return HttpResponse.json({ - pieceCid, - }) + http.head('https://pdp.example.com/piece/:pieceCid', async () => { + return new HttpResponse(null, { status: 200 }) }), http.get('https://pdp.example.com/piece/:pieceCid', async () => { return HttpResponse.arrayBuffer(testData.buffer) @@ -308,7 +293,7 @@ describe('Synapse', () => { it('should handle download errors', async () => { server.use( Mocks.JSONRPC(Mocks.presets.basic), - http.get('https://pdp.example.com/pdp/piece', async () => { + http.head('https://pdp.example.com/piece/:pieceCid', async () => { return HttpResponse.error() }) ) diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 2fee726cb..44f2afcb9 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -28,7 +28,7 @@ catalog: viem: ^2.52.0 wagmi: ^3.0.2 zod: ^4.3.5 - iso-web: 2.2.1 + iso-web: ^3.0.0 minimumReleaseAge: 10080 From fb00e1a5958421006f13aa2e0ed8d07ca74f5974 Mon Sep 17 00:00:00 2001 From: Hugo Dias Date: Tue, 2 Jun 2026 17:41:58 +0100 Subject: [PATCH 02/13] chore: fix docs --- docs/src/content/docs/developer-guides/synapse-core.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/src/content/docs/developer-guides/synapse-core.mdx b/docs/src/content/docs/developer-guides/synapse-core.mdx index 21ef19f1d..ee6ab462d 100644 --- a/docs/src/content/docs/developer-guides/synapse-core.mdx +++ b/docs/src/content/docs/developer-guides/synapse-core.mdx @@ -170,7 +170,7 @@ await sp.uploadPiece({ await sp.findPiece({ pieceCid, serviceURL: provider.pdp.serviceURL, - retry: true, + poll: true, }) console.log(`Piece ${pieceCid.toString()} uploaded to provider ${provider.pdp.serviceURL}`) From 61227f0961bd489d2d6129a108ac196febbcdff4 Mon Sep 17 00:00:00 2001 From: Hugo Dias Date: Wed, 3 Jun 2026 08:42:20 +0100 Subject: [PATCH 03/13] chore: pr comments --- packages/synapse-core/src/piece/download.ts | 6 +++--- packages/synapse-core/src/piece/resolve-piece-url.ts | 9 ++------- packages/synapse-core/src/sp/add-pieces.ts | 2 -- .../synapse-core/src/sp/create-dataset-add-pieces.ts | 10 +++------- packages/synapse-core/src/sp/create-dataset.ts | 8 +++----- packages/synapse-core/src/sp/get-data-set.ts | 4 ++-- packages/synapse-core/src/sp/ping.ts | 5 ++++- packages/synapse-core/src/sp/pull-pieces.ts | 9 ++------- .../synapse-core/src/sp/schedule-piece-deletion.ts | 4 ++-- packages/synapse-core/src/utils/constants.ts | 9 ++++++--- pnpm-workspace.yaml | 2 +- 11 files changed, 28 insertions(+), 40 deletions(-) diff --git a/packages/synapse-core/src/piece/download.ts b/packages/synapse-core/src/piece/download.ts index 6dbbb885b..dbfd1a30c 100644 --- a/packages/synapse-core/src/piece/download.ts +++ b/packages/synapse-core/src/piece/download.ts @@ -1,4 +1,4 @@ -import { HttpError, type Errors as HttpErrors, request } from 'iso-web/http' +import { HttpError, type RequestErrors, request } from 'iso-web/http' import { DownloadPieceError } from '../errors/pdp.ts' import { InvalidPieceCIDError } from '../errors/piece.ts' import { RETRY_CONSTANTS } from '../utils/constants.ts' @@ -17,7 +17,7 @@ export namespace download { signal?: AbortSignal } export type ReturnType = Uint8Array - export type ErrorType = DownloadPieceError | HttpErrors + export type ErrorType = DownloadPieceError | RequestErrors } /** @@ -57,7 +57,7 @@ export namespace downloadAndValidate { signal?: AbortSignal } export type ReturnType = Uint8Array - export type ErrorType = DownloadPieceError | HttpErrors | InvalidPieceCIDError + export type ErrorType = DownloadPieceError | RequestErrors | InvalidPieceCIDError } /** diff --git a/packages/synapse-core/src/piece/resolve-piece-url.ts b/packages/synapse-core/src/piece/resolve-piece-url.ts index e9dadf7fb..ba9ba8704 100644 --- a/packages/synapse-core/src/piece/resolve-piece-url.ts +++ b/packages/synapse-core/src/piece/resolve-piece-url.ts @@ -185,7 +185,7 @@ export function providersResolver(providers: PDPProvider[]): resolvePieceUrl.Res * @param serviceURLs - {@link string[]} * @param pieceCid - {@link PieceCID} * @param signal - {@link AbortSignal} - * @returns The piece URL + * @returns The Service URL */ export async function findPieceOnProviders(serviceURLs: string[], pieceCid: PieceCID, signal?: AbortSignal) { const controller = new AbortController() @@ -203,12 +203,7 @@ export async function findPieceOnProviders(serviceURLs: string[], pieceCid: Piec } const result = await pLocate( - serviceURLs.map((p) => - headPiece(p).then( - () => p, - () => undefined - ) - ), + serviceURLs.map((p) => headPiece(p).catch(() => undefined)), (p) => { if (p != null) { controller.abort() diff --git a/packages/synapse-core/src/sp/add-pieces.ts b/packages/synapse-core/src/sp/add-pieces.ts index 1e7311103..4bfac3185 100644 --- a/packages/synapse-core/src/sp/add-pieces.ts +++ b/packages/synapse-core/src/sp/add-pieces.ts @@ -215,8 +215,6 @@ export namespace waitForAddPieces { retryCount?: number /** The delay with exponential backoff between retries in milliseconds. Defaults to 250ms. */ retryDelay?: number - /** Whether to poll the request. Defaults to false. */ - poll?: boolean /** The poll interval in milliseconds. Defaults to 4 second. */ pollInterval?: number } diff --git a/packages/synapse-core/src/sp/create-dataset-add-pieces.ts b/packages/synapse-core/src/sp/create-dataset-add-pieces.ts index 2d7793537..9a9bfcde8 100644 --- a/packages/synapse-core/src/sp/create-dataset-add-pieces.ts +++ b/packages/synapse-core/src/sp/create-dataset-add-pieces.ts @@ -1,4 +1,4 @@ -import { HttpError, type Errors as HttpErrors, request } from 'iso-web/http' +import { HttpError, type RequestErrors, type RequestJsonErrors, request } from 'iso-web/http' import type { ToString } from 'multiformats' import { type Account, type Address, type Chain, type Client, type Hex, isHex, type Transport } from 'viem' import { asChain } from '../chains.ts' @@ -37,7 +37,7 @@ export namespace createDataSetAndAddPiecesApiRequest { /** The status URL. */ statusUrl: string } - export type ErrorType = CreateDataSetError | LocationHeaderError | HttpErrors + export type ErrorType = CreateDataSetError | LocationHeaderError | RequestErrors export type RequestBody = { recordKeeper: Address extraData: Hex @@ -181,8 +181,6 @@ export namespace waitForCreateDataSetAddPieces { retryCount?: number /** The delay with exponential backoff between retries in milliseconds. Defaults to 250ms. */ retryDelay?: number - /** Whether to poll the request. Defaults to false. */ - poll?: boolean /** The poll interval in milliseconds. Defaults to 4 second. */ pollInterval?: number } @@ -196,7 +194,7 @@ export namespace waitForCreateDataSetAddPieces { | WaitForCreateDataSetRejectedError | WaitForAddPiecesError | WaitForAddPiecesRejectedError - | HttpErrors + | RequestJsonErrors } /** @@ -216,7 +214,6 @@ export async function waitForCreateDataSetAddPieces( statusUrl: options.statusUrl, retryCount: options.retryCount, retryDelay: options.retryDelay, - poll: options.poll, pollInterval: options.pollInterval, }) const addedPieces = await waitForAddPieces({ @@ -226,7 +223,6 @@ export async function waitForCreateDataSetAddPieces( ).toString(), retryCount: options.retryCount, retryDelay: options.retryDelay, - poll: options.poll, pollInterval: options.pollInterval, }) return { diff --git a/packages/synapse-core/src/sp/create-dataset.ts b/packages/synapse-core/src/sp/create-dataset.ts index a21690174..9a87e7390 100644 --- a/packages/synapse-core/src/sp/create-dataset.ts +++ b/packages/synapse-core/src/sp/create-dataset.ts @@ -1,4 +1,4 @@ -import { HttpError, type Errors as HttpErrors, request } from 'iso-web/http' +import { HttpError, type RequestErrors, type RequestJsonErrors, request } from 'iso-web/http' import { type Account, type Address, @@ -41,7 +41,7 @@ export namespace createDataSetApiRequest { statusUrl: string } - export type ErrorType = CreateDataSetError | LocationHeaderError | HttpErrors + export type ErrorType = CreateDataSetError | LocationHeaderError | RequestErrors export type RequestBody = { recordKeeper: Address @@ -207,13 +207,11 @@ export namespace waitForCreateDataSet { retryCount?: number /** The delay with exponential backoff between retries in milliseconds. Defaults to 250ms. */ retryDelay?: number - /** Whether to poll the request. Defaults to false. */ - poll?: boolean /** The poll interval in milliseconds. Defaults to 4 second. */ pollInterval?: number } export type ReturnType = CreateDataSetSuccess - export type ErrorType = WaitForCreateDataSetError | WaitForCreateDataSetRejectedError | HttpErrors + export type ErrorType = WaitForCreateDataSetError | WaitForCreateDataSetRejectedError | RequestJsonErrors } /** diff --git a/packages/synapse-core/src/sp/get-data-set.ts b/packages/synapse-core/src/sp/get-data-set.ts index f90f2d963..2dd35fd84 100644 --- a/packages/synapse-core/src/sp/get-data-set.ts +++ b/packages/synapse-core/src/sp/get-data-set.ts @@ -1,4 +1,4 @@ -import { HttpError, type Errors as HttpErrors, request } from 'iso-web/http' +import { HttpError, type RequestJsonErrors, request } from 'iso-web/http' import * as z from 'zod' import { GetDataSetError } from '../errors/pdp.ts' import { RETRY_CONSTANTS } from '../utils/constants.ts' @@ -34,7 +34,7 @@ export namespace getDataSet { retryDelay?: number } export type OutputType = DataSet - export type ErrorType = GetDataSetError | HttpErrors + export type ErrorType = GetDataSetError | RequestJsonErrors } /** diff --git a/packages/synapse-core/src/sp/ping.ts b/packages/synapse-core/src/sp/ping.ts index 75594a004..a835cf439 100644 --- a/packages/synapse-core/src/sp/ping.ts +++ b/packages/synapse-core/src/sp/ping.ts @@ -1,4 +1,5 @@ import { request } from 'iso-web/http' +import { RETRY_CONSTANTS } from '../utils/constants.ts' /** * Ping the PDP API. @@ -11,7 +12,9 @@ import { request } from 'iso-web/http' */ export async function ping(serviceURL: string) { const response = await request.get(new URL(`pdp/ping`, serviceURL), { - retry: true, + retry: { + minTimeout: RETRY_CONSTANTS.RETRY_DELAY, + }, timeout: 1000, }) if (response.error) { diff --git a/packages/synapse-core/src/sp/pull-pieces.ts b/packages/synapse-core/src/sp/pull-pieces.ts index a726814a1..9e9cc980a 100644 --- a/packages/synapse-core/src/sp/pull-pieces.ts +++ b/packages/synapse-core/src/sp/pull-pieces.ts @@ -1,4 +1,4 @@ -import { HttpError, type Errors as HttpErrors, request } from 'iso-web/http' +import { HttpError, type RequestErrors, request } from 'iso-web/http' import type { Account, Address, Chain, Client, Hex, Transport } from 'viem' import { asChain } from '../chains.ts' import { PullError } from '../errors/pull.ts' @@ -79,7 +79,7 @@ export namespace pullPiecesApiRequest { pieces: PullPieceStatus[] } - export type ErrorType = PullError | HttpErrors + export type ErrorType = PullError | RequestErrors export type RequestBody = { extraData: Hex @@ -157,8 +157,6 @@ export namespace waitForPullPiecesApiRequest { retryCount?: number /** The delay with exponential backoff between retries in milliseconds. Defaults to 250ms. */ retryDelay?: number - /** Whether to poll the request. Defaults to false. */ - poll?: boolean /** The poll interval in milliseconds. Defaults to 4 second. */ pollInterval?: number } @@ -400,8 +398,6 @@ export namespace waitForPullPieces { retryCount?: number /** The delay with exponential backoff between retries in milliseconds. Defaults to 250ms. */ retryDelay?: number - /** Whether to poll the request. Defaults to false. */ - poll?: boolean /** The poll interval in milliseconds. Defaults to 4 second. */ pollInterval?: number } @@ -432,7 +428,6 @@ export async function waitForPullPieces( onStatus: options.onStatus, timeout: options.timeout, pollInterval: options.pollInterval, - poll: options.poll, retryCount: options.retryCount, retryDelay: options.retryDelay, }) diff --git a/packages/synapse-core/src/sp/schedule-piece-deletion.ts b/packages/synapse-core/src/sp/schedule-piece-deletion.ts index 9bafef0f3..89ad232e6 100644 --- a/packages/synapse-core/src/sp/schedule-piece-deletion.ts +++ b/packages/synapse-core/src/sp/schedule-piece-deletion.ts @@ -1,4 +1,4 @@ -import { HttpError, type Errors as HttpErrors, request } from 'iso-web/http' +import { HttpError, type RequestJsonErrors, request } from 'iso-web/http' import type { Account, Chain, Client, Hex, Transport } from 'viem' import { DeletePieceError } from '../errors/pdp.ts' import { signSchedulePieceRemovals } from '../typed-data/sign-schedule-piece-removals.ts' @@ -18,7 +18,7 @@ export namespace deletePiece { export type OutputType = { hash: Hex } - export type ErrorType = DeletePieceError | HttpErrors + export type ErrorType = DeletePieceError | RequestJsonErrors } /** diff --git a/packages/synapse-core/src/utils/constants.ts b/packages/synapse-core/src/utils/constants.ts index 08d4d80cf..997aeefdb 100644 --- a/packages/synapse-core/src/utils/constants.ts +++ b/packages/synapse-core/src/utils/constants.ts @@ -141,10 +141,13 @@ export const CDN_FIXED_LOCKUP = { export const USDFC_SYBIL_FEE = 100_000_000_000_000_000n // 0.1 USDFC export const RETRY_CONSTANTS = { - POLL_INTERVAL: 4000, // 4 seconds in milliseconds between retries + /** The interval in milliseconds between polls. 4 seconds is the default interval between polls. */ + POLL_INTERVAL: 4000, + /** The limit of polls. */ POLL_LIMIT: Infinity, - RETRY_DELAY: 250, // 250ms in milliseconds between retries - /** The timeout in milliseconds. Defaults to 5 minutes. */ + /** The delay in milliseconds between retries. 250ms is the default delay between retries. */ + RETRY_DELAY: 250, + /** The timeout in milliseconds. 5 minutes is the default timeout. */ TIMEOUT: 1000 * 60 * 5, } as const diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 44f2afcb9..9c86cf9bf 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -28,7 +28,7 @@ catalog: viem: ^2.52.0 wagmi: ^3.0.2 zod: ^4.3.5 - iso-web: ^3.0.0 + iso-web: ^3.1.0 minimumReleaseAge: 10080 From dbf0e39ca3310bcf8172d492b8848b809bfef4b5 Mon Sep 17 00:00:00 2001 From: Hugo Dias Date: Wed, 3 Jun 2026 08:51:37 +0100 Subject: [PATCH 04/13] docs: update retry delay documentation to reference RETRY_CONSTANTS --- packages/synapse-core/src/piece/download.ts | 4 ++-- packages/synapse-core/src/sp/add-pieces.ts | 8 ++++---- .../synapse-core/src/sp/create-dataset-add-pieces.ts | 8 ++++---- packages/synapse-core/src/sp/create-dataset.ts | 8 ++++---- packages/synapse-core/src/sp/find-piece.ts | 2 +- packages/synapse-core/src/sp/get-data-set.ts | 2 +- packages/synapse-core/src/sp/pull-pieces.ts | 10 +++++----- .../synapse-core/src/sp/schedule-piece-deletion.ts | 4 ++-- packages/synapse-core/src/sp/upload-streaming.ts | 2 +- packages/synapse-core/src/sp/upload.ts | 2 +- 10 files changed, 25 insertions(+), 25 deletions(-) diff --git a/packages/synapse-core/src/piece/download.ts b/packages/synapse-core/src/piece/download.ts index dbfd1a30c..bb1610820 100644 --- a/packages/synapse-core/src/piece/download.ts +++ b/packages/synapse-core/src/piece/download.ts @@ -11,7 +11,7 @@ export namespace download { url: string /** The number of retries. Defaults to 2. */ retryCount?: number - /** The delay with exponential backoff between retries in milliseconds. Defaults to 250ms. */ + /** The delay with exponential backoff between retries in milliseconds. Defaults to {@link RETRY_CONSTANTS.RETRY_DELAY}. */ retryDelay?: number /** The signal to abort the request. */ signal?: AbortSignal @@ -51,7 +51,7 @@ export namespace downloadAndValidate { expectedPieceCid: string | PieceCID /** The number of retries. Defaults to 2. */ retryCount?: number - /** The delay with exponential backoff between retries in milliseconds. Defaults to 250ms. */ + /** The delay with exponential backoff between retries in milliseconds. Defaults to {@link RETRY_CONSTANTS.RETRY_DELAY}. */ retryDelay?: number /** The signal to abort the request. */ signal?: AbortSignal diff --git a/packages/synapse-core/src/sp/add-pieces.ts b/packages/synapse-core/src/sp/add-pieces.ts index 4bfac3185..7e15da659 100644 --- a/packages/synapse-core/src/sp/add-pieces.ts +++ b/packages/synapse-core/src/sp/add-pieces.ts @@ -23,7 +23,7 @@ export namespace addPiecesApiRequest { extraData: Hex /** The number of retries. Defaults to 2. */ retryCount?: number - /** The delay with exponential backoff between retries in milliseconds. Defaults to 250ms. */ + /** The delay with exponential backoff between retries in milliseconds. Defaults to {@link RETRY_CONSTANTS.RETRY_DELAY}. */ retryDelay?: number } export type OutputType = { @@ -109,7 +109,7 @@ export namespace addPieces { extraData?: Hex /** The number of retries. Defaults to 2. */ retryCount?: number - /** The delay with exponential backoff between retries in milliseconds. Defaults to 250ms. */ + /** The delay with exponential backoff between retries in milliseconds. Defaults to {@link RETRY_CONSTANTS.RETRY_DELAY}. */ retryDelay?: number } @@ -213,9 +213,9 @@ export namespace waitForAddPieces { timeout?: number /** The number of retries. Defaults to 2. */ retryCount?: number - /** The delay with exponential backoff between retries in milliseconds. Defaults to 250ms. */ + /** The delay with exponential backoff between retries in milliseconds. Defaults to {@link RETRY_CONSTANTS.RETRY_DELAY}. */ retryDelay?: number - /** The poll interval in milliseconds. Defaults to 4 second. */ + /** The poll interval in milliseconds. Defaults to {@link RETRY_CONSTANTS.POLL_INTERVAL}. */ pollInterval?: number } export type OutputType = AddPiecesOutput diff --git a/packages/synapse-core/src/sp/create-dataset-add-pieces.ts b/packages/synapse-core/src/sp/create-dataset-add-pieces.ts index 9a9bfcde8..cf774fe89 100644 --- a/packages/synapse-core/src/sp/create-dataset-add-pieces.ts +++ b/packages/synapse-core/src/sp/create-dataset-add-pieces.ts @@ -28,7 +28,7 @@ export namespace createDataSetAndAddPiecesApiRequest { pieces: PieceCID[] /** The number of retries. Defaults to 2. */ retryCount?: number - /** The delay with exponential backoff between retries in milliseconds. Defaults to 250ms. */ + /** The delay with exponential backoff between retries in milliseconds. Defaults to {@link RETRY_CONSTANTS.RETRY_DELAY}. */ retryDelay?: number } export type OutputType = { @@ -122,7 +122,7 @@ export type CreateDataSetAndAddPiecesOptions = { recordKeeper?: Address /** The number of retries. Defaults to 2. */ retryCount?: number - /** The delay with exponential backoff between retries in milliseconds. Defaults to 250ms. */ + /** The delay with exponential backoff between retries in milliseconds. Defaults to {@link RETRY_CONSTANTS.RETRY_DELAY}. */ retryDelay?: number } @@ -179,9 +179,9 @@ export namespace waitForCreateDataSetAddPieces { timeout?: number /** The number of retries. Defaults to 2. */ retryCount?: number - /** The delay with exponential backoff between retries in milliseconds. Defaults to 250ms. */ + /** The delay with exponential backoff between retries in milliseconds. Defaults to {@link RETRY_CONSTANTS.RETRY_DELAY}. */ retryDelay?: number - /** The poll interval in milliseconds. Defaults to 4 second. */ + /** The poll interval in milliseconds. Defaults to {@link RETRY_CONSTANTS.POLL_INTERVAL}. */ pollInterval?: number } export type ReturnType = { diff --git a/packages/synapse-core/src/sp/create-dataset.ts b/packages/synapse-core/src/sp/create-dataset.ts index 9a87e7390..931909d3d 100644 --- a/packages/synapse-core/src/sp/create-dataset.ts +++ b/packages/synapse-core/src/sp/create-dataset.ts @@ -32,7 +32,7 @@ export namespace createDataSetApiRequest { extraData: Hex /** The number of retries. Defaults to 2. */ retryCount?: number - /** The delay with exponential backoff between retries in milliseconds. Defaults to 250ms. */ + /** The delay with exponential backoff between retries in milliseconds. Defaults to {@link RETRY_CONSTANTS.RETRY_DELAY}. */ retryDelay?: number } @@ -115,7 +115,7 @@ export namespace createDataSet { recordKeeper?: Address /** The number of retries. Defaults to 2. */ retryCount?: number - /** The delay with exponential backoff between retries in milliseconds. Defaults to 250ms. */ + /** The delay with exponential backoff between retries in milliseconds. Defaults to {@link RETRY_CONSTANTS.RETRY_DELAY}. */ retryDelay?: number } export type ReturnType = createDataSetApiRequest.OutputType @@ -205,9 +205,9 @@ export namespace waitForCreateDataSet { timeout?: number /** The number of retries. Defaults to 2. */ retryCount?: number - /** The delay with exponential backoff between retries in milliseconds. Defaults to 250ms. */ + /** The delay with exponential backoff between retries in milliseconds. Defaults to {@link RETRY_CONSTANTS.RETRY_DELAY}. */ retryDelay?: number - /** The poll interval in milliseconds. Defaults to 4 second. */ + /** The poll interval in milliseconds. Defaults to {@link RETRY_CONSTANTS.POLL_INTERVAL}. */ pollInterval?: number } export type ReturnType = CreateDataSetSuccess diff --git a/packages/synapse-core/src/sp/find-piece.ts b/packages/synapse-core/src/sp/find-piece.ts index 784bc2d6f..cd8d82104 100644 --- a/packages/synapse-core/src/sp/find-piece.ts +++ b/packages/synapse-core/src/sp/find-piece.ts @@ -16,7 +16,7 @@ export namespace findPiece { timeout?: number /** The number of retries. Defaults to 2. */ retryCount?: number - /** The delay with exponential backoff between retries in milliseconds. Defaults to 250ms. */ + /** The delay with exponential backoff between retries in milliseconds. Defaults to {@link RETRY_CONSTANTS.RETRY_DELAY}. */ retryDelay?: number /** Whether to poll the request. Defaults to false. */ poll?: boolean diff --git a/packages/synapse-core/src/sp/get-data-set.ts b/packages/synapse-core/src/sp/get-data-set.ts index 2dd35fd84..30c9f251f 100644 --- a/packages/synapse-core/src/sp/get-data-set.ts +++ b/packages/synapse-core/src/sp/get-data-set.ts @@ -30,7 +30,7 @@ export namespace getDataSet { dataSetId: bigint /** The number of retries. Defaults to 2. */ retryCount?: number - /** The delay with exponential backoff between retries in milliseconds. Defaults to 250ms. */ + /** The delay with exponential backoff between retries in milliseconds. Defaults to {@link RETRY_CONSTANTS.RETRY_DELAY}. */ retryDelay?: number } export type OutputType = DataSet diff --git a/packages/synapse-core/src/sp/pull-pieces.ts b/packages/synapse-core/src/sp/pull-pieces.ts index 9e9cc980a..4baa4651f 100644 --- a/packages/synapse-core/src/sp/pull-pieces.ts +++ b/packages/synapse-core/src/sp/pull-pieces.ts @@ -68,7 +68,7 @@ export namespace pullPiecesApiRequest { signal?: AbortSignal /** The number of retries. Defaults to 2. */ retryCount?: number - /** The delay with exponential backoff between retries in milliseconds. Defaults to 250ms. */ + /** The delay with exponential backoff between retries in milliseconds. Defaults to {@link RETRY_CONSTANTS.RETRY_DELAY}. */ retryDelay?: number } @@ -155,9 +155,9 @@ export namespace waitForPullPiecesApiRequest { timeout?: number /** The number of retries. Defaults to 2. */ retryCount?: number - /** The delay with exponential backoff between retries in milliseconds. Defaults to 250ms. */ + /** The delay with exponential backoff between retries in milliseconds. Defaults to {@link RETRY_CONSTANTS.RETRY_DELAY}. */ retryDelay?: number - /** The poll interval in milliseconds. Defaults to 4 second. */ + /** The poll interval in milliseconds. Defaults to {@link RETRY_CONSTANTS.POLL_INTERVAL}. */ pollInterval?: number } @@ -396,9 +396,9 @@ export namespace waitForPullPieces { timeout?: number /** The number of retries. Defaults to 2. */ retryCount?: number - /** The delay with exponential backoff between retries in milliseconds. Defaults to 250ms. */ + /** The delay with exponential backoff between retries in milliseconds. Defaults to {@link RETRY_CONSTANTS.RETRY_DELAY}. */ retryDelay?: number - /** The poll interval in milliseconds. Defaults to 4 second. */ + /** The poll interval in milliseconds. Defaults to {@link RETRY_CONSTANTS.POLL_INTERVAL}. */ pollInterval?: number } diff --git a/packages/synapse-core/src/sp/schedule-piece-deletion.ts b/packages/synapse-core/src/sp/schedule-piece-deletion.ts index 89ad232e6..17be36293 100644 --- a/packages/synapse-core/src/sp/schedule-piece-deletion.ts +++ b/packages/synapse-core/src/sp/schedule-piece-deletion.ts @@ -12,7 +12,7 @@ export namespace deletePiece { extraData: Hex /** The number of retries. Defaults to 2. */ retryCount?: number - /** The delay with exponential backoff between retries in milliseconds. Defaults to 250ms. */ + /** The delay with exponential backoff between retries in milliseconds. Defaults to {@link RETRY_CONSTANTS.RETRY_DELAY}. */ retryDelay?: number } export type OutputType = { @@ -66,7 +66,7 @@ export namespace schedulePieceDeletion { serviceURL: string /** The number of retries. Defaults to 2. */ retryCount?: number - /** The delay with exponential backoff between retries in milliseconds. Defaults to 250ms. */ + /** The delay with exponential backoff between retries in milliseconds. Defaults to {@link RETRY_CONSTANTS.RETRY_DELAY}. */ retryDelay?: number } export type OutputType = deletePiece.OutputType diff --git a/packages/synapse-core/src/sp/upload-streaming.ts b/packages/synapse-core/src/sp/upload-streaming.ts index 7d8325b50..1b52302b0 100644 --- a/packages/synapse-core/src/sp/upload-streaming.ts +++ b/packages/synapse-core/src/sp/upload-streaming.ts @@ -23,7 +23,7 @@ export namespace uploadPieceStreaming { signal?: AbortSignal /** The number of retries. Defaults to 2. */ retryCount?: number - /** The delay with exponential backoff between retries in milliseconds. Defaults to 250ms. */ + /** The delay with exponential backoff between retries in milliseconds. Defaults to {@link RETRY_CONSTANTS.RETRY_DELAY}. */ retryDelay?: number } export type OutputType = { diff --git a/packages/synapse-core/src/sp/upload.ts b/packages/synapse-core/src/sp/upload.ts index 576d4e4f6..afc2fce1e 100644 --- a/packages/synapse-core/src/sp/upload.ts +++ b/packages/synapse-core/src/sp/upload.ts @@ -21,7 +21,7 @@ export namespace uploadPiece { pieceCid: PieceCID /** The number of retries. Defaults to 2. */ retryCount?: number - /** The delay with exponential backoff between retries in milliseconds. Defaults to 250ms. */ + /** The delay with exponential backoff between retries in milliseconds. Defaults to {@link RETRY_CONSTANTS.RETRY_DELAY}. */ retryDelay?: number } export type ErrorType = InvalidUploadSizeError | LocationHeaderError | TimeoutError | NetworkError | AbortError From 09a57b4549af0a59b68b0f179c3e8f6e28125733 Mon Sep 17 00:00:00 2001 From: Hugo Dias Date: Wed, 3 Jun 2026 12:15:35 +0100 Subject: [PATCH 05/13] fix: retry only on HTTP 429 status code for POSTs --- packages/synapse-core/src/sp/add-pieces.ts | 1 + packages/synapse-core/src/sp/create-dataset-add-pieces.ts | 1 + packages/synapse-core/src/sp/create-dataset.ts | 1 + packages/synapse-core/src/sp/schedule-piece-deletion.ts | 2 ++ 4 files changed, 5 insertions(+) diff --git a/packages/synapse-core/src/sp/add-pieces.ts b/packages/synapse-core/src/sp/add-pieces.ts index 7e15da659..df196e297 100644 --- a/packages/synapse-core/src/sp/add-pieces.ts +++ b/packages/synapse-core/src/sp/add-pieces.ts @@ -66,6 +66,7 @@ export async function addPiecesApiRequest( timeout: RETRY_CONSTANTS.TIMEOUT, retry: { methods: ['post'], + statusCodes: [429], retries: options.retryCount, minTimeout: options.retryDelay ?? RETRY_CONSTANTS.RETRY_DELAY, }, diff --git a/packages/synapse-core/src/sp/create-dataset-add-pieces.ts b/packages/synapse-core/src/sp/create-dataset-add-pieces.ts index cf774fe89..8dd580571 100644 --- a/packages/synapse-core/src/sp/create-dataset-add-pieces.ts +++ b/packages/synapse-core/src/sp/create-dataset-add-pieces.ts @@ -73,6 +73,7 @@ export async function createDataSetAndAddPiecesApiRequest( timeout: RETRY_CONSTANTS.TIMEOUT, retry: { methods: ['post'], + statusCodes: [429], retries: options.retryCount, minTimeout: options.retryDelay ?? RETRY_CONSTANTS.RETRY_DELAY, }, diff --git a/packages/synapse-core/src/sp/create-dataset.ts b/packages/synapse-core/src/sp/create-dataset.ts index 931909d3d..1b6da1e0b 100644 --- a/packages/synapse-core/src/sp/create-dataset.ts +++ b/packages/synapse-core/src/sp/create-dataset.ts @@ -70,6 +70,7 @@ export async function createDataSetApiRequest( timeout: RETRY_CONSTANTS.TIMEOUT, retry: { methods: ['post'], + statusCodes: [429], retries: options.retryCount, minTimeout: options.retryDelay ?? RETRY_CONSTANTS.RETRY_DELAY, }, diff --git a/packages/synapse-core/src/sp/schedule-piece-deletion.ts b/packages/synapse-core/src/sp/schedule-piece-deletion.ts index 17be36293..998b4db96 100644 --- a/packages/synapse-core/src/sp/schedule-piece-deletion.ts +++ b/packages/synapse-core/src/sp/schedule-piece-deletion.ts @@ -38,6 +38,8 @@ export async function deletePiece(options: deletePiece.OptionsType): Promise Date: Wed, 3 Jun 2026 12:26:02 +0100 Subject: [PATCH 06/13] chore: trust issues --- pnpm-workspace.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 9c86cf9bf..1cfe1b0dc 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -64,3 +64,4 @@ trustPolicyExclude: # Last 4.x (Dec 2024) predates chokidar adopting trusted publishing at 5.0.0. # Permanent while anything depends on chokidar@^4. - chokidar@4.0.3 + - tinyclip@0.1.13 From 861d0e8e6df014b620cb95e52dc8fefcf0a2b912 Mon Sep 17 00:00:00 2001 From: Hugo Dias Date: Wed, 3 Jun 2026 12:42:04 +0100 Subject: [PATCH 07/13] fix: dont retry on POST network errors --- packages/synapse-core/src/sp/add-pieces.ts | 3 ++- packages/synapse-core/src/sp/create-dataset-add-pieces.ts | 3 ++- packages/synapse-core/src/sp/create-dataset.ts | 3 ++- packages/synapse-core/src/sp/schedule-piece-deletion.ts | 3 ++- 4 files changed, 8 insertions(+), 4 deletions(-) diff --git a/packages/synapse-core/src/sp/add-pieces.ts b/packages/synapse-core/src/sp/add-pieces.ts index df196e297..5fd90ddda 100644 --- a/packages/synapse-core/src/sp/add-pieces.ts +++ b/packages/synapse-core/src/sp/add-pieces.ts @@ -1,4 +1,4 @@ -import { type AbortError, HttpError, type NetworkError, request, type TimeoutError } from 'iso-web/http' +import { type AbortError, HttpError, NetworkError, request, type TimeoutError } from 'iso-web/http' import type { ToString } from 'multiformats' import { type Account, type Chain, type Client, type Hex, isHex, type Transport } from 'viem' import * as z from 'zod' @@ -69,6 +69,7 @@ export async function addPiecesApiRequest( statusCodes: [429], retries: options.retryCount, minTimeout: options.retryDelay ?? RETRY_CONSTANTS.RETRY_DELAY, + shouldRetry: (ctx) => !NetworkError.is(ctx.error), }, }) diff --git a/packages/synapse-core/src/sp/create-dataset-add-pieces.ts b/packages/synapse-core/src/sp/create-dataset-add-pieces.ts index 8dd580571..18464fcc3 100644 --- a/packages/synapse-core/src/sp/create-dataset-add-pieces.ts +++ b/packages/synapse-core/src/sp/create-dataset-add-pieces.ts @@ -1,4 +1,4 @@ -import { HttpError, type RequestErrors, type RequestJsonErrors, request } from 'iso-web/http' +import { HttpError, NetworkError, type RequestErrors, type RequestJsonErrors, request } from 'iso-web/http' import type { ToString } from 'multiformats' import { type Account, type Address, type Chain, type Client, type Hex, isHex, type Transport } from 'viem' import { asChain } from '../chains.ts' @@ -76,6 +76,7 @@ export async function createDataSetAndAddPiecesApiRequest( statusCodes: [429], retries: options.retryCount, minTimeout: options.retryDelay ?? RETRY_CONSTANTS.RETRY_DELAY, + shouldRetry: (ctx) => !NetworkError.is(ctx.error), }, }) diff --git a/packages/synapse-core/src/sp/create-dataset.ts b/packages/synapse-core/src/sp/create-dataset.ts index 1b6da1e0b..516d37ca8 100644 --- a/packages/synapse-core/src/sp/create-dataset.ts +++ b/packages/synapse-core/src/sp/create-dataset.ts @@ -1,4 +1,4 @@ -import { HttpError, type RequestErrors, type RequestJsonErrors, request } from 'iso-web/http' +import { HttpError, NetworkError, type RequestErrors, type RequestJsonErrors, request } from 'iso-web/http' import { type Account, type Address, @@ -73,6 +73,7 @@ export async function createDataSetApiRequest( statusCodes: [429], retries: options.retryCount, minTimeout: options.retryDelay ?? RETRY_CONSTANTS.RETRY_DELAY, + shouldRetry: (ctx) => !NetworkError.is(ctx.error), }, }) diff --git a/packages/synapse-core/src/sp/schedule-piece-deletion.ts b/packages/synapse-core/src/sp/schedule-piece-deletion.ts index 998b4db96..771a681f7 100644 --- a/packages/synapse-core/src/sp/schedule-piece-deletion.ts +++ b/packages/synapse-core/src/sp/schedule-piece-deletion.ts @@ -1,4 +1,4 @@ -import { HttpError, type RequestJsonErrors, request } from 'iso-web/http' +import { HttpError, NetworkError, type RequestJsonErrors, request } from 'iso-web/http' import type { Account, Chain, Client, Hex, Transport } from 'viem' import { DeletePieceError } from '../errors/pdp.ts' import { signSchedulePieceRemovals } from '../typed-data/sign-schedule-piece-removals.ts' @@ -42,6 +42,7 @@ export async function deletePiece(options: deletePiece.OptionsType): Promise !NetworkError.is(ctx.error), }, } ) From d7b734be5ae7c3592db608d48b1fda0d92610ad5 Mon Sep 17 00:00:00 2001 From: Hugo Dias Date: Wed, 3 Jun 2026 16:00:10 +0100 Subject: [PATCH 08/13] fix: update retry logic to only retry on HTTP 429 errors for POST requests --- packages/synapse-core/src/sp/add-pieces.ts | 5 +- .../src/sp/create-dataset-add-pieces.ts | 5 +- .../synapse-core/src/sp/create-dataset.ts | 5 +- .../src/sp/schedule-piece-deletion.ts | 5 +- packages/synapse-core/test/piece-url.test.ts | 2 +- packages/synapse-core/test/rand.test.ts | 2 +- packages/synapse-core/test/sp.test.ts | 70 +++++++++++++++++++ 7 files changed, 80 insertions(+), 14 deletions(-) diff --git a/packages/synapse-core/src/sp/add-pieces.ts b/packages/synapse-core/src/sp/add-pieces.ts index 5fd90ddda..5d13edb04 100644 --- a/packages/synapse-core/src/sp/add-pieces.ts +++ b/packages/synapse-core/src/sp/add-pieces.ts @@ -1,4 +1,4 @@ -import { type AbortError, HttpError, NetworkError, request, type TimeoutError } from 'iso-web/http' +import { type AbortError, HttpError, type NetworkError, request, type TimeoutError } from 'iso-web/http' import type { ToString } from 'multiformats' import { type Account, type Chain, type Client, type Hex, isHex, type Transport } from 'viem' import * as z from 'zod' @@ -66,10 +66,9 @@ export async function addPiecesApiRequest( timeout: RETRY_CONSTANTS.TIMEOUT, retry: { methods: ['post'], - statusCodes: [429], retries: options.retryCount, minTimeout: options.retryDelay ?? RETRY_CONSTANTS.RETRY_DELAY, - shouldRetry: (ctx) => !NetworkError.is(ctx.error), + shouldRetry: (ctx) => HttpError.is(ctx.error) && ctx.error.code === 429, }, }) diff --git a/packages/synapse-core/src/sp/create-dataset-add-pieces.ts b/packages/synapse-core/src/sp/create-dataset-add-pieces.ts index 18464fcc3..d65f44a8e 100644 --- a/packages/synapse-core/src/sp/create-dataset-add-pieces.ts +++ b/packages/synapse-core/src/sp/create-dataset-add-pieces.ts @@ -1,4 +1,4 @@ -import { HttpError, NetworkError, type RequestErrors, type RequestJsonErrors, request } from 'iso-web/http' +import { HttpError, type RequestErrors, type RequestJsonErrors, request } from 'iso-web/http' import type { ToString } from 'multiformats' import { type Account, type Address, type Chain, type Client, type Hex, isHex, type Transport } from 'viem' import { asChain } from '../chains.ts' @@ -73,10 +73,9 @@ export async function createDataSetAndAddPiecesApiRequest( timeout: RETRY_CONSTANTS.TIMEOUT, retry: { methods: ['post'], - statusCodes: [429], retries: options.retryCount, minTimeout: options.retryDelay ?? RETRY_CONSTANTS.RETRY_DELAY, - shouldRetry: (ctx) => !NetworkError.is(ctx.error), + shouldRetry: (ctx) => HttpError.is(ctx.error) && ctx.error.code === 429, }, }) diff --git a/packages/synapse-core/src/sp/create-dataset.ts b/packages/synapse-core/src/sp/create-dataset.ts index 516d37ca8..fa9a0d881 100644 --- a/packages/synapse-core/src/sp/create-dataset.ts +++ b/packages/synapse-core/src/sp/create-dataset.ts @@ -1,4 +1,4 @@ -import { HttpError, NetworkError, type RequestErrors, type RequestJsonErrors, request } from 'iso-web/http' +import { HttpError, type RequestErrors, type RequestJsonErrors, request } from 'iso-web/http' import { type Account, type Address, @@ -70,10 +70,9 @@ export async function createDataSetApiRequest( timeout: RETRY_CONSTANTS.TIMEOUT, retry: { methods: ['post'], - statusCodes: [429], retries: options.retryCount, minTimeout: options.retryDelay ?? RETRY_CONSTANTS.RETRY_DELAY, - shouldRetry: (ctx) => !NetworkError.is(ctx.error), + shouldRetry: (ctx) => HttpError.is(ctx.error) && ctx.error.code === 429, }, }) diff --git a/packages/synapse-core/src/sp/schedule-piece-deletion.ts b/packages/synapse-core/src/sp/schedule-piece-deletion.ts index 771a681f7..2f0d8f497 100644 --- a/packages/synapse-core/src/sp/schedule-piece-deletion.ts +++ b/packages/synapse-core/src/sp/schedule-piece-deletion.ts @@ -1,4 +1,4 @@ -import { HttpError, NetworkError, type RequestJsonErrors, request } from 'iso-web/http' +import { HttpError, type RequestJsonErrors, request } from 'iso-web/http' import type { Account, Chain, Client, Hex, Transport } from 'viem' import { DeletePieceError } from '../errors/pdp.ts' import { signSchedulePieceRemovals } from '../typed-data/sign-schedule-piece-removals.ts' @@ -39,10 +39,9 @@ export async function deletePiece(options: deletePiece.OptionsType): Promise !NetworkError.is(ctx.error), + shouldRetry: (ctx) => HttpError.is(ctx.error) && ctx.error.code === 429, }, } ) diff --git a/packages/synapse-core/test/piece-url.test.ts b/packages/synapse-core/test/piece-url.test.ts index c5b13851c..24a3cd65a 100644 --- a/packages/synapse-core/test/piece-url.test.ts +++ b/packages/synapse-core/test/piece-url.test.ts @@ -1,5 +1,5 @@ -import { calibration, devnet, mainnet } from '@filoz/synapse-core/chains' import { assert } from 'chai' +import { calibration, devnet, mainnet } from '../src/chains.ts' import { createPieceUrl, createPieceUrlPDP } from '../src/utils/piece-url.ts' describe('createPieceUrl', () => { diff --git a/packages/synapse-core/test/rand.test.ts b/packages/synapse-core/test/rand.test.ts index 7f45de0e3..3f930c020 100644 --- a/packages/synapse-core/test/rand.test.ts +++ b/packages/synapse-core/test/rand.test.ts @@ -1,5 +1,5 @@ -import { fallbackRandIndex, fallbackRandU256, randIndex, randU256 } from '@filoz/synapse-core/utils' import { assert } from 'chai' +import { fallbackRandIndex, fallbackRandU256, randIndex, randU256 } from '../src/utils/rand.ts' const randIndexMethods = [randIndex, fallbackRandIndex] randIndexMethods.forEach((randIndexMethod) => { diff --git a/packages/synapse-core/test/sp.test.ts b/packages/synapse-core/test/sp.test.ts index bfd99835f..368c6dbc1 100644 --- a/packages/synapse-core/test/sp.test.ts +++ b/packages/synapse-core/test/sp.test.ts @@ -35,6 +35,7 @@ import { deletePiece, findPiece, getDataSet, + NetworkError, TimeoutError, uploadPiece, waitForAddPieces, @@ -558,6 +559,75 @@ InvalidSignature(address expected, address actual) } }) + it('should not retry on network errors', async () => { + const pieceCid = Piece.from(validPieceCid) + let callCount = 0 + server.use( + http.post('http://pdp.local/pdp/data-sets/:id/pieces', () => { + callCount++ + return HttpResponse.error() + }) + ) + + const extraData = await TypedData.signAddPieces(client, { + clientDataSetId: 0n, + pieces: [{ pieceCid }], + }) + + try { + await addPiecesApiRequest({ + serviceURL: 'http://pdp.local', + dataSetId: 1n, + pieces: [pieceCid], + extraData, + retryDelay: 10, + }) + assert.fail('Should have thrown error for network error') + } catch (error) { + assert.strictEqual(callCount, 1) + assert.instanceOf(error, NetworkError) + assert.include(error.message, 'Network request failed') + } + }) + + it.only('should retry on 429 errors', async () => { + const pieceCid = Piece.from(validPieceCid) + let callCount = 0 + server.use( + http.post('http://pdp.local/pdp/data-sets/:id/pieces', () => { + callCount++ + return new HttpResponse(null, { + status: 429, + headers: { + 'Retry-After': '0.01', + }, + statusText: 'Too Many Requests', + }) + }) + ) + + const extraData = await TypedData.signAddPieces(client, { + clientDataSetId: 0n, + pieces: [{ pieceCid }], + }) + + try { + await addPiecesApiRequest({ + serviceURL: 'http://pdp.local', + dataSetId: 1n, + pieces: [pieceCid], + extraData, + retryCount: 2, + retryDelay: 10, + }) + assert.fail('Should have thrown error for 429 error') + } catch (error) { + assert.strictEqual(callCount, 3) + assert.instanceOf(error, AddPiecesError) + assert.include(error.message, 'Failed to add pieces.') + } + }) + it('should handle multiple pieces', async () => { const mockTxHash = '0xabcdef1234567890abcdef1234567890abcdef1234567890abcdef1234567890' const pieceCid1 = Piece.from(validPieceCid) From 54dfeb15d31f90b61352569476b5815342b71b1a Mon Sep 17 00:00:00 2001 From: Hugo Dias Date: Wed, 3 Jun 2026 16:05:03 +0100 Subject: [PATCH 09/13] test: remove exclusive focus on retry test for 429 errors --- packages/synapse-core/test/sp.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/synapse-core/test/sp.test.ts b/packages/synapse-core/test/sp.test.ts index 368c6dbc1..21a78e607 100644 --- a/packages/synapse-core/test/sp.test.ts +++ b/packages/synapse-core/test/sp.test.ts @@ -590,7 +590,7 @@ InvalidSignature(address expected, address actual) } }) - it.only('should retry on 429 errors', async () => { + it('should retry on 429 errors', async () => { const pieceCid = Piece.from(validPieceCid) let callCount = 0 server.use( From 37e8ffaff921eae5cb84db17f16c6156591f3b4f Mon Sep 17 00:00:00 2001 From: Hugo Dias Date: Thu, 4 Jun 2026 12:50:54 +0100 Subject: [PATCH 10/13] fix: remove POST and DELETE method retries from API request configurations --- packages/synapse-core/src/sp/add-pieces.ts | 1 - packages/synapse-core/src/sp/create-dataset-add-pieces.ts | 1 - packages/synapse-core/src/sp/create-dataset.ts | 1 - packages/synapse-core/src/sp/schedule-piece-deletion.ts | 1 - pnpm-workspace.yaml | 3 +-- 5 files changed, 1 insertion(+), 6 deletions(-) diff --git a/packages/synapse-core/src/sp/add-pieces.ts b/packages/synapse-core/src/sp/add-pieces.ts index 5d13edb04..523589ba0 100644 --- a/packages/synapse-core/src/sp/add-pieces.ts +++ b/packages/synapse-core/src/sp/add-pieces.ts @@ -65,7 +65,6 @@ export async function addPiecesApiRequest( }, timeout: RETRY_CONSTANTS.TIMEOUT, retry: { - methods: ['post'], retries: options.retryCount, minTimeout: options.retryDelay ?? RETRY_CONSTANTS.RETRY_DELAY, shouldRetry: (ctx) => HttpError.is(ctx.error) && ctx.error.code === 429, diff --git a/packages/synapse-core/src/sp/create-dataset-add-pieces.ts b/packages/synapse-core/src/sp/create-dataset-add-pieces.ts index d65f44a8e..6ad0ae591 100644 --- a/packages/synapse-core/src/sp/create-dataset-add-pieces.ts +++ b/packages/synapse-core/src/sp/create-dataset-add-pieces.ts @@ -72,7 +72,6 @@ export async function createDataSetAndAddPiecesApiRequest( }, timeout: RETRY_CONSTANTS.TIMEOUT, retry: { - methods: ['post'], retries: options.retryCount, minTimeout: options.retryDelay ?? RETRY_CONSTANTS.RETRY_DELAY, shouldRetry: (ctx) => HttpError.is(ctx.error) && ctx.error.code === 429, diff --git a/packages/synapse-core/src/sp/create-dataset.ts b/packages/synapse-core/src/sp/create-dataset.ts index fa9a0d881..61397cf26 100644 --- a/packages/synapse-core/src/sp/create-dataset.ts +++ b/packages/synapse-core/src/sp/create-dataset.ts @@ -69,7 +69,6 @@ export async function createDataSetApiRequest( }, timeout: RETRY_CONSTANTS.TIMEOUT, retry: { - methods: ['post'], retries: options.retryCount, minTimeout: options.retryDelay ?? RETRY_CONSTANTS.RETRY_DELAY, shouldRetry: (ctx) => HttpError.is(ctx.error) && ctx.error.code === 429, diff --git a/packages/synapse-core/src/sp/schedule-piece-deletion.ts b/packages/synapse-core/src/sp/schedule-piece-deletion.ts index 2f0d8f497..da3eda33f 100644 --- a/packages/synapse-core/src/sp/schedule-piece-deletion.ts +++ b/packages/synapse-core/src/sp/schedule-piece-deletion.ts @@ -38,7 +38,6 @@ export async function deletePiece(options: deletePiece.OptionsType): Promise HttpError.is(ctx.error) && ctx.error.code === 429, diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 1cfe1b0dc..c036b30d0 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -28,7 +28,7 @@ catalog: viem: ^2.52.0 wagmi: ^3.0.2 zod: ^4.3.5 - iso-web: ^3.1.0 + iso-web: ^3.1.2 minimumReleaseAge: 10080 @@ -64,4 +64,3 @@ trustPolicyExclude: # Last 4.x (Dec 2024) predates chokidar adopting trusted publishing at 5.0.0. # Permanent while anything depends on chokidar@^4. - chokidar@4.0.3 - - tinyclip@0.1.13 From 98ff33c6b47e0921c8cd21e3cd4e71505c3f2594 Mon Sep 17 00:00:00 2001 From: Hugo Dias Date: Thu, 4 Jun 2026 13:03:16 +0100 Subject: [PATCH 11/13] chore: update astro dependency and add it to minimum release age exclusions --- docs/package.json | 2 +- pnpm-workspace.yaml | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/package.json b/docs/package.json index 4e13a4737..339126ea8 100644 --- a/docs/package.json +++ b/docs/package.json @@ -25,7 +25,7 @@ "@hugomrdias/docs": "^0.2.0", "@types/react": "catalog:", "@types/react-dom": "catalog:", - "astro": "^6.1.6", + "astro": "^6.4.2", "astro-mermaid": "^2.0.1", "expressive-code-twoslash": "^0.6.1", "mermaid": "^11.12.2", diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index c036b30d0..17cdd752e 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -44,6 +44,7 @@ minimumReleaseAgeExclude: # Remove after 2026-06-08 when both age out. - viem@2.52.0 - ox@0.14.27 + - astro trustPolicy: no-downgrade From 0020e21d4e408701d782f06f7b1e7828a1821c73 Mon Sep 17 00:00:00 2001 From: Hugo Dias Date: Thu, 4 Jun 2026 14:24:51 +0100 Subject: [PATCH 12/13] fix: enhance retry logic to include 404 errors and adjust polling status codes --- packages/synapse-core/src/sp/find-piece.ts | 3 ++- packages/synapse-core/test/sp.test.ts | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/packages/synapse-core/src/sp/find-piece.ts b/packages/synapse-core/src/sp/find-piece.ts index cd8d82104..e4203e678 100644 --- a/packages/synapse-core/src/sp/find-piece.ts +++ b/packages/synapse-core/src/sp/find-piece.ts @@ -44,12 +44,13 @@ export async function findPiece(options: findPiece.OptionsType): Promise HttpError.is(ctx.error) && ctx.error.code === 404, }, poll: options.poll ? { limit: RETRY_CONSTANTS.POLL_LIMIT, interval: options.pollInterval ?? 1000, - statusCodes: [202, 404], // 202 is processing, 404 is not found + statusCodes: [202], // 202 is processing } : false, }) diff --git a/packages/synapse-core/test/sp.test.ts b/packages/synapse-core/test/sp.test.ts index 21a78e607..26119649d 100644 --- a/packages/synapse-core/test/sp.test.ts +++ b/packages/synapse-core/test/sp.test.ts @@ -964,7 +964,7 @@ InvalidSignature(address expected, address actual) serviceURL: 'https://pdp.example.com', pieceCid, poll: true, - timeout: 100, + timeout: 10, }) assert.fail('Should have thrown error for not found') } catch (error) { From 85c9090b3cfac57546652a663544c6845930e3a5 Mon Sep 17 00:00:00 2001 From: Hugo Dias Date: Thu, 4 Jun 2026 14:26:45 +0100 Subject: [PATCH 13/13] chore: update astro dependency to version 6.4.2 and adjust minimum release age exclusions --- pnpm-workspace.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 17cdd752e..73a0e2890 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -44,7 +44,9 @@ minimumReleaseAgeExclude: # Remove after 2026-06-08 when both age out. - viem@2.52.0 - ox@0.14.27 - - astro + # astro 6.4.2 fixes MDX/Starlight compatibility after 6.4.0. + # Remove after 2026-06-05 when it ages out. + - astro@6.4.2 trustPolicy: no-downgrade