diff --git a/docs/package.json b/docs/package.json index 4e13a473..339126ea 100644 --- a/docs/package.json +++ b/docs/package.json @@ -25,7 +25,7 @@ "@hugomrdias/docs": "^0.2.0", "@types/react": "catalog:", "@types/react-dom": "catalog:", - "astro": "^6.1.6", + "astro": "^6.4.2", "astro-mermaid": "^2.0.1", "expressive-code-twoslash": "^0.6.1", "mermaid": "^11.12.2", diff --git a/docs/src/content/docs/developer-guides/synapse-core.mdx b/docs/src/content/docs/developer-guides/synapse-core.mdx index 21ef19f1..ee6ab462 100644 --- a/docs/src/content/docs/developer-guides/synapse-core.mdx +++ b/docs/src/content/docs/developer-guides/synapse-core.mdx @@ -170,7 +170,7 @@ await sp.uploadPiece({ await sp.findPiece({ pieceCid, serviceURL: provider.pdp.serviceURL, - retry: true, + poll: true, }) console.log(`Piece ${pieceCid.toString()} uploaded to provider ${provider.pdp.serviceURL}`) diff --git a/examples/cli/src/commands/upload-dataset.ts b/examples/cli/src/commands/upload-dataset.ts index 15217561..d9d80685 100644 --- a/examples/cli/src/commands/upload-dataset.ts +++ b/examples/cli/src/commands/upload-dataset.ts @@ -55,7 +55,7 @@ export const uploadDataset: Command = command( await SP.findPiece({ pieceCid, serviceURL: provider.pdp.serviceURL, - retry: true, + poll: true, }) const rsp = await SP.createDataSetAndAddPieces(client, { diff --git a/packages/synapse-core/src/piece/download.ts b/packages/synapse-core/src/piece/download.ts index 7fb13190..bb161082 100644 --- a/packages/synapse-core/src/piece/download.ts +++ b/packages/synapse-core/src/piece/download.ts @@ -1,6 +1,7 @@ -import { type AbortError, HttpError, type NetworkError, request, type TimeoutError } from 'iso-web/http' +import { HttpError, type RequestErrors, request } from 'iso-web/http' import { DownloadPieceError } from '../errors/pdp.ts' import { InvalidPieceCIDError } from '../errors/piece.ts' +import { RETRY_CONSTANTS } from '../utils/constants.ts' import { transformStream } from './calculate.ts' import { tryFrom } from './parse.ts' import type { PieceCID } from './piece-cid.ts' @@ -8,9 +9,15 @@ import type { PieceCID } from './piece-cid.ts' export namespace download { export type OptionsType = { url: string + /** The number of retries. Defaults to 2. */ + retryCount?: number + /** The delay with exponential backoff between retries in milliseconds. Defaults to {@link RETRY_CONSTANTS.RETRY_DELAY}. */ + retryDelay?: number + /** The signal to abort the request. */ + signal?: AbortSignal } export type ReturnType = Uint8Array - export type ErrorType = DownloadPieceError | TimeoutError | NetworkError | AbortError + export type ErrorType = DownloadPieceError | RequestErrors } /** @@ -21,7 +28,14 @@ export namespace download { * @throws Errors {@link download.ErrorType} */ export async function download(options: download.OptionsType): Promise { - const response = await request.get(options.url) + const response = await request.get(options.url, { + timeout: false, + retry: { + retries: options.retryCount, + minTimeout: options.retryDelay ?? RETRY_CONSTANTS.RETRY_DELAY, + }, + signal: options.signal, + }) if (response.error) { if (HttpError.is(response.error)) { throw new DownloadPieceError(await response.error.response.text()) @@ -35,9 +49,15 @@ export namespace downloadAndValidate { export type OptionsType = { url: string expectedPieceCid: string | PieceCID + /** The number of retries. Defaults to 2. */ + retryCount?: number + /** The delay with exponential backoff between retries in milliseconds. Defaults to {@link RETRY_CONSTANTS.RETRY_DELAY}. */ + retryDelay?: number + /** The signal to abort the request. */ + signal?: AbortSignal } export type ReturnType = Uint8Array - export type ErrorType = DownloadPieceError | TimeoutError | NetworkError | AbortError | InvalidPieceCIDError + export type ErrorType = DownloadPieceError | RequestErrors | InvalidPieceCIDError } /** @@ -71,7 +91,15 @@ export async function downloadAndValidate(options: downloadAndValidate.OptionsTy throw new InvalidPieceCIDError(expectedPieceCid) } - const rsp = await request.get(url) + const rsp = await request.get(url, { + timeout: false, + retry: { + retries: options.retryCount, + minTimeout: options.retryDelay ?? RETRY_CONSTANTS.RETRY_DELAY, + }, + signal: options.signal, + }) + if (rsp.error) { if (HttpError.is(rsp.error)) { throw new DownloadPieceError(await rsp.error.response.text()) diff --git a/packages/synapse-core/src/piece/resolve-piece-url.ts b/packages/synapse-core/src/piece/resolve-piece-url.ts index a30f8da3..ba9ba870 100644 --- a/packages/synapse-core/src/piece/resolve-piece-url.ts +++ b/packages/synapse-core/src/piece/resolve-piece-url.ts @@ -3,7 +3,6 @@ import pLocate from 'p-locate' import pSome from 'p-some' import type { Address, Chain, Client, Transport } from 'viem' import { asChain } from '../chains.ts' -import { findPiece } from '../sp/find-piece.ts' import type { PDPProvider } from '../sp-registry/types.ts' import { createPieceUrlPDP } from '../utils/piece-url.ts' import { getPdpDataSets } from '../warm-storage/get-pdp-data-sets.ts' @@ -132,13 +131,17 @@ export async function chainResolver(options: resolvePieceUrl.ResolverFnOptionsTy }, new Map()) const providers = [...providersById.values()] - const result = await findPieceOnProviders(providers, pieceCid, signal) + const result = await findPieceOnProviders( + providers.map((p) => p.pdp.serviceURL), + pieceCid, + signal + ) if (result == null) { throw new Error('No provider found') } return createPieceUrlPDP({ cid: pieceCid.toString(), - serviceURL: result.pdp.serviceURL, + serviceURL: result, }) } @@ -160,14 +163,18 @@ export async function chainResolver(options: resolvePieceUrl.ResolverFnOptionsTy export function providersResolver(providers: PDPProvider[]): resolvePieceUrl.ResolverFnType { return async (options: resolvePieceUrl.ResolverFnOptionsType) => { const { pieceCid, signal } = options - const result = await findPieceOnProviders(providers, pieceCid, signal) + const result = await findPieceOnProviders( + providers.map((p) => p.pdp.serviceURL), + pieceCid, + signal + ) if (result == null) { throw new Error('No provider found') } return createPieceUrlPDP({ cid: pieceCid.toString(), - serviceURL: result.pdp.serviceURL, + serviceURL: result, }) } } @@ -175,28 +182,30 @@ export function providersResolver(providers: PDPProvider[]): resolvePieceUrl.Res /** * Find the piece on the providers * - * @param providers - {@link PDPProvider[]} + * @param serviceURLs - {@link string[]} * @param pieceCid - {@link PieceCID} * @param signal - {@link AbortSignal} - * @returns The piece URL + * @returns The Service URL */ -export async function findPieceOnProviders(providers: PDPProvider[], pieceCid: PieceCID, signal?: AbortSignal) { +export async function findPieceOnProviders(serviceURLs: string[], pieceCid: PieceCID, signal?: AbortSignal) { const controller = new AbortController() const _signal = signal ? AbortSignal.any([controller.signal, signal]) : controller.signal + async function headPiece(serviceURL: string) { + const result = await request.head(new URL(`piece/${pieceCid.toString()}`, serviceURL), { + signal: _signal, + retry: true, + }) + if (result.error) { + throw result.error + } + return serviceURL + } + const result = await pLocate( - providers.map((p) => - findPiece({ - serviceURL: p.pdp.serviceURL, - pieceCid, - signal: _signal, - }).then( - () => p, - () => null - ) - ), + serviceURLs.map((p) => headPiece(p).catch(() => undefined)), (p) => { - if (p !== null) { + if (p != null) { controller.abort() return true } diff --git a/packages/synapse-core/src/sp/add-pieces.ts b/packages/synapse-core/src/sp/add-pieces.ts index 734673f0..523589ba 100644 --- a/packages/synapse-core/src/sp/add-pieces.ts +++ b/packages/synapse-core/src/sp/add-pieces.ts @@ -21,6 +21,10 @@ export namespace addPiecesApiRequest { pieces: PieceCID[] /** The extra data for the add pieces. {@link TypedData.signAddPieces} */ extraData: Hex + /** The number of retries. Defaults to 2. */ + retryCount?: number + /** The delay with exponential backoff between retries in milliseconds. Defaults to {@link RETRY_CONSTANTS.RETRY_DELAY}. */ + retryDelay?: number } export type OutputType = { /** The transaction hash. */ @@ -52,17 +56,19 @@ export async function addPiecesApiRequest( ): Promise { const { serviceURL, dataSetId, pieces, extraData } = options const response = await request.post(new URL(`pdp/data-sets/${dataSetId}/pieces`, serviceURL), { - headers: { - 'Content-Type': 'application/json', - }, - body: JSON.stringify({ + json: { pieces: pieces.map((piece) => ({ pieceCid: piece.toString(), subPieces: [{ subPieceCid: piece.toString() }], })), extraData: extraData, - }), - timeout: RETRY_CONSTANTS.MAX_RETRY_TIME, + }, + timeout: RETRY_CONSTANTS.TIMEOUT, + retry: { + retries: options.retryCount, + minTimeout: options.retryDelay ?? RETRY_CONSTANTS.RETRY_DELAY, + shouldRetry: (ctx) => HttpError.is(ctx.error) && ctx.error.code === 429, + }, }) if (response.error) { @@ -101,6 +107,10 @@ export namespace addPieces { nonce?: bigint /** Pre-built signed extraData. When provided, skips internal EIP-712 signing. */ extraData?: Hex + /** The number of retries. Defaults to 2. */ + retryCount?: number + /** The delay with exponential backoff between retries in milliseconds. Defaults to {@link RETRY_CONSTANTS.RETRY_DELAY}. */ + retryDelay?: number } export type OutputType = addPiecesApiRequest.OutputType @@ -154,6 +164,8 @@ export async function addPieces( dataSetId: options.dataSetId, pieces: options.pieces.map((piece) => piece.pieceCid), extraData, + retryCount: options.retryCount, + retryDelay: options.retryDelay, }) } @@ -199,7 +211,11 @@ export namespace waitForAddPieces { statusUrl: string /** The timeout in milliseconds. Defaults to 5 minutes. */ timeout?: number - /** The polling interval in milliseconds. Defaults to 4 seconds. */ + /** The number of retries. Defaults to 2. */ + retryCount?: number + /** The delay with exponential backoff between retries in milliseconds. Defaults to {@link RETRY_CONSTANTS.RETRY_DELAY}. */ + retryDelay?: number + /** The poll interval in milliseconds. Defaults to {@link RETRY_CONSTANTS.POLL_INTERVAL}. */ pollInterval?: number } export type OutputType = AddPiecesOutput @@ -221,22 +237,22 @@ export namespace waitForAddPieces { * @throws Errors {@link waitForAddPieces.ErrorType} */ export async function waitForAddPieces(options: waitForAddPieces.OptionsType): Promise { - const response = await request.json.get(options.statusUrl, { - async onResponse(response) { - if (response.ok) { - const data = (await response.clone().json()) as AddPiecesResponse - if (data.piecesAdded === false) { - throw new Error('Still pending') - } - } - }, + const response = await request.json.get(options.statusUrl, { retry: { - shouldRetry: (ctx) => ctx.error.message === 'Still pending', - retries: RETRY_CONSTANTS.RETRIES, - factor: RETRY_CONSTANTS.FACTOR, - minTimeout: options.pollInterval ?? RETRY_CONSTANTS.DELAY_TIME, + retries: options.retryCount, + minTimeout: options.retryDelay ?? RETRY_CONSTANTS.RETRY_DELAY, + }, + poll: { + limit: RETRY_CONSTANTS.POLL_LIMIT, + interval: options.pollInterval ?? RETRY_CONSTANTS.POLL_INTERVAL, + statusCodes: [202, 200], // 202 is processing, 200 is success + shouldPoll: async (ctx) => { + const data = (await ctx.response.clone().json()) as AddPiecesResponse + return data.piecesAdded === false + }, }, - timeout: options.timeout ?? RETRY_CONSTANTS.MAX_RETRY_TIME, + timeout: options.timeout ?? RETRY_CONSTANTS.TIMEOUT, + schema, }) if (response.error) { if (HttpError.is(response.error)) { @@ -244,9 +260,8 @@ export async function waitForAddPieces(options: waitForAddPieces.OptionsType): P } throw response.error } - const data = schema.parse(response.result) - if (data.txStatus === 'rejected') { - throw new WaitForAddPiecesRejectedError(data) + if (response.result.txStatus === 'rejected') { + throw new WaitForAddPiecesRejectedError(response.result) } - return data + return response.result } diff --git a/packages/synapse-core/src/sp/create-dataset-add-pieces.ts b/packages/synapse-core/src/sp/create-dataset-add-pieces.ts index e8d25e3b..6ad0ae59 100644 --- a/packages/synapse-core/src/sp/create-dataset-add-pieces.ts +++ b/packages/synapse-core/src/sp/create-dataset-add-pieces.ts @@ -1,4 +1,4 @@ -import { type AbortError, HttpError, type NetworkError, request, type TimeoutError } from 'iso-web/http' +import { HttpError, type RequestErrors, type RequestJsonErrors, request } from 'iso-web/http' import type { ToString } from 'multiformats' import { type Account, type Address, type Chain, type Client, type Hex, isHex, type Transport } from 'viem' import { asChain } from '../chains.ts' @@ -26,6 +26,10 @@ export namespace createDataSetAndAddPiecesApiRequest { extraData: Hex /** The pieces to add. */ pieces: PieceCID[] + /** The number of retries. Defaults to 2. */ + retryCount?: number + /** The delay with exponential backoff between retries in milliseconds. Defaults to {@link RETRY_CONSTANTS.RETRY_DELAY}. */ + retryDelay?: number } export type OutputType = { /** The transaction hash. */ @@ -33,7 +37,7 @@ export namespace createDataSetAndAddPiecesApiRequest { /** The status URL. */ statusUrl: string } - export type ErrorType = CreateDataSetError | LocationHeaderError | TimeoutError | NetworkError | AbortError + export type ErrorType = CreateDataSetError | LocationHeaderError | RequestErrors export type RequestBody = { recordKeeper: Address extraData: Hex @@ -58,18 +62,20 @@ export async function createDataSetAndAddPiecesApiRequest( ): Promise { // Send the create data set message to the PDP const response = await request.post(new URL(`pdp/data-sets/create-and-add`, options.serviceURL), { - body: JSON.stringify({ + json: { recordKeeper: options.recordKeeper, extraData: options.extraData, pieces: options.pieces.map((piece) => ({ pieceCid: piece.toString(), subPieces: [{ subPieceCid: piece.toString() }], })), - }), - headers: { - 'Content-Type': 'application/json', }, - timeout: RETRY_CONSTANTS.MAX_RETRY_TIME, + timeout: RETRY_CONSTANTS.TIMEOUT, + retry: { + retries: options.retryCount, + minTimeout: options.retryDelay ?? RETRY_CONSTANTS.RETRY_DELAY, + shouldRetry: (ctx) => HttpError.is(ctx.error) && ctx.error.code === 429, + }, }) if (response.error) { @@ -114,6 +120,10 @@ export type CreateDataSetAndAddPiecesOptions = { cdn?: boolean /** The address of the record keeper to use for the signature. If not provided, the default is the Warm Storage contract address. */ recordKeeper?: Address + /** The number of retries. Defaults to 2. */ + retryCount?: number + /** The delay with exponential backoff between retries in milliseconds. Defaults to {@link RETRY_CONSTANTS.RETRY_DELAY}. */ + retryDelay?: number } export namespace createDataSetAndAddPieces { @@ -156,6 +166,8 @@ export async function createDataSetAndAddPieces( recordKeeper: options.recordKeeper ?? chain.contracts.fwss.address, extraData, pieces: options.pieces.map((piece) => piece.pieceCid), + retryCount: options.retryCount, + retryDelay: options.retryDelay, }) } @@ -165,7 +177,11 @@ export namespace waitForCreateDataSetAddPieces { statusUrl: string /** The timeout in milliseconds. Defaults to 5 minutes. */ timeout?: number - /** The polling interval in milliseconds. Defaults to 4 seconds. */ + /** The number of retries. Defaults to 2. */ + retryCount?: number + /** The delay with exponential backoff between retries in milliseconds. Defaults to {@link RETRY_CONSTANTS.RETRY_DELAY}. */ + retryDelay?: number + /** The poll interval in milliseconds. Defaults to {@link RETRY_CONSTANTS.POLL_INTERVAL}. */ pollInterval?: number } export type ReturnType = { @@ -178,9 +194,7 @@ export namespace waitForCreateDataSetAddPieces { | WaitForCreateDataSetRejectedError | WaitForAddPiecesError | WaitForAddPiecesRejectedError - | TimeoutError - | NetworkError - | AbortError + | RequestJsonErrors } /** @@ -196,12 +210,20 @@ export async function waitForCreateDataSetAddPieces( options: waitForCreateDataSetAddPieces.OptionsType ): Promise { const origin = new URL(options.statusUrl).origin - const createdDataset = await waitForCreateDataSet({ statusUrl: options.statusUrl }) + const createdDataset = await waitForCreateDataSet({ + statusUrl: options.statusUrl, + retryCount: options.retryCount, + retryDelay: options.retryDelay, + pollInterval: options.pollInterval, + }) const addedPieces = await waitForAddPieces({ statusUrl: new URL( `/pdp/data-sets/${createdDataset.dataSetId}/pieces/added/${createdDataset.createMessageHash}`, origin ).toString(), + retryCount: options.retryCount, + retryDelay: options.retryDelay, + pollInterval: options.pollInterval, }) return { hash: createdDataset.createMessageHash, diff --git a/packages/synapse-core/src/sp/create-dataset.ts b/packages/synapse-core/src/sp/create-dataset.ts index 4bac8cc7..61397cf2 100644 --- a/packages/synapse-core/src/sp/create-dataset.ts +++ b/packages/synapse-core/src/sp/create-dataset.ts @@ -1,4 +1,4 @@ -import { HttpError, request } from 'iso-web/http' +import { HttpError, type RequestErrors, type RequestJsonErrors, request } from 'iso-web/http' import { type Account, type Address, @@ -18,7 +18,6 @@ import { signCreateDataSet } from '../typed-data/sign-create-dataset.ts' import { RETRY_CONSTANTS } from '../utils/constants.ts' import { datasetMetadataObjectToEntry, type MetadataObject } from '../utils/metadata.ts' import { zHex, zNumberToBigInt } from '../utils/schemas.ts' -import type { AbortError, NetworkError, TimeoutError } from './index.ts' export namespace createDataSetApiRequest { /** @@ -31,6 +30,10 @@ export namespace createDataSetApiRequest { recordKeeper: Address /** The extra data for the create data set. */ extraData: Hex + /** The number of retries. Defaults to 2. */ + retryCount?: number + /** The delay with exponential backoff between retries in milliseconds. Defaults to {@link RETRY_CONSTANTS.RETRY_DELAY}. */ + retryDelay?: number } export type OutputType = { @@ -38,7 +41,7 @@ export namespace createDataSetApiRequest { statusUrl: string } - export type ErrorType = CreateDataSetError | LocationHeaderError | TimeoutError | NetworkError | AbortError + export type ErrorType = CreateDataSetError | LocationHeaderError | RequestErrors export type RequestBody = { recordKeeper: Address @@ -60,14 +63,16 @@ export async function createDataSetApiRequest( ): Promise { // Send the create data set message to the PDP const response = await request.post(new URL(`pdp/data-sets`, options.serviceURL), { - body: JSON.stringify({ + json: { recordKeeper: options.recordKeeper, extraData: options.extraData, - }), - headers: { - 'Content-Type': 'application/json', }, - timeout: RETRY_CONSTANTS.MAX_RETRY_TIME, + timeout: RETRY_CONSTANTS.TIMEOUT, + retry: { + retries: options.retryCount, + minTimeout: options.retryDelay ?? RETRY_CONSTANTS.RETRY_DELAY, + shouldRetry: (ctx) => HttpError.is(ctx.error) && ctx.error.code === 429, + }, }) if (response.error) { @@ -108,6 +113,10 @@ export namespace createDataSet { clientDataSetId?: bigint /** The address of the record keeper to use for the signature. If not provided, the default is the Warm Storage contract address. */ recordKeeper?: Address + /** The number of retries. Defaults to 2. */ + retryCount?: number + /** The delay with exponential backoff between retries in milliseconds. Defaults to {@link RETRY_CONSTANTS.RETRY_DELAY}. */ + retryDelay?: number } export type ReturnType = createDataSetApiRequest.OutputType export type ErrorType = @@ -142,6 +151,8 @@ export async function createDataSet(client: Client, o serviceURL: options.serviceURL, recordKeeper: options.recordKeeper ?? chain.contracts.fwss.address, extraData, + retryCount: options.retryCount, + retryDelay: options.retryDelay, }) } @@ -192,16 +203,15 @@ export namespace waitForCreateDataSet { statusUrl: string /** The timeout in milliseconds. Defaults to 5 minutes. */ timeout?: number - /** The polling interval in milliseconds. Defaults to 4 seconds. */ + /** The number of retries. Defaults to 2. */ + retryCount?: number + /** The delay with exponential backoff between retries in milliseconds. Defaults to {@link RETRY_CONSTANTS.RETRY_DELAY}. */ + retryDelay?: number + /** The poll interval in milliseconds. Defaults to {@link RETRY_CONSTANTS.POLL_INTERVAL}. */ pollInterval?: number } export type ReturnType = CreateDataSetSuccess - export type ErrorType = - | WaitForCreateDataSetError - | WaitForCreateDataSetRejectedError - | TimeoutError - | NetworkError - | AbortError + export type ErrorType = WaitForCreateDataSetError | WaitForCreateDataSetRejectedError | RequestJsonErrors } /** @@ -216,23 +226,23 @@ export namespace waitForCreateDataSet { export async function waitForCreateDataSet( options: waitForCreateDataSet.OptionsType ): Promise { - const response = await request.json.get(options.statusUrl, { - async onResponse(response) { - if (response.ok) { - const data = (await response.clone().json()) as CreateDataSetResponse - if (data.dataSetCreated === false) { - throw new Error('Still pending') - } - } - }, + const response = await request.json.get(options.statusUrl, { retry: { - shouldRetry: (ctx) => ctx.error.message === 'Still pending', - retries: RETRY_CONSTANTS.RETRIES, - factor: RETRY_CONSTANTS.FACTOR, - minTimeout: options.pollInterval ?? RETRY_CONSTANTS.DELAY_TIME, + retries: options.retryCount, + minTimeout: options.retryDelay ?? RETRY_CONSTANTS.RETRY_DELAY, + }, + poll: { + limit: RETRY_CONSTANTS.POLL_LIMIT, + interval: options.pollInterval ?? RETRY_CONSTANTS.POLL_INTERVAL, + statusCodes: [202, 200], // 202 is processing, 200 is success + shouldPoll: async (ctx) => { + const data = (await ctx.response.clone().json()) as CreateDataSetResponse + return data.dataSetCreated === false + }, }, - timeout: options.timeout ?? RETRY_CONSTANTS.MAX_RETRY_TIME, + timeout: options.timeout ?? RETRY_CONSTANTS.TIMEOUT, + schema, }) if (response.error) { if (HttpError.is(response.error)) { @@ -241,9 +251,8 @@ export async function waitForCreateDataSet( throw response.error } - const data = schema.parse(response.result) - if (data.txStatus === 'rejected') { - throw new WaitForCreateDataSetRejectedError(data) + if (response.result.txStatus === 'rejected') { + throw new WaitForCreateDataSetRejectedError(response.result) } - return data + return response.result } diff --git a/packages/synapse-core/src/sp/find-piece.ts b/packages/synapse-core/src/sp/find-piece.ts index c8401c6e..e4203e67 100644 --- a/packages/synapse-core/src/sp/find-piece.ts +++ b/packages/synapse-core/src/sp/find-piece.ts @@ -12,10 +12,14 @@ export namespace findPiece { pieceCid: PieceCID /** The signal to abort the request. */ signal?: AbortSignal - /** Whether to retry the request. Defaults to false. */ - retry?: boolean /** The timeout in milliseconds. Defaults to 5 minutes. */ timeout?: number + /** The number of retries. Defaults to 2. */ + retryCount?: number + /** The delay with exponential backoff between retries in milliseconds. Defaults to {@link RETRY_CONSTANTS.RETRY_DELAY}. */ + retryDelay?: number + /** Whether to poll the request. Defaults to false. */ + poll?: boolean /** The poll interval in milliseconds. Defaults to 1 second. */ pollInterval?: number } @@ -34,18 +38,21 @@ export namespace findPiece { export async function findPiece(options: findPiece.OptionsType): Promise { const { pieceCid, serviceURL } = options const params = new URLSearchParams({ pieceCid: pieceCid.toString() }) - const retry = options.retry ?? false const response = await request.json.get<{ pieceCid: string }>(new URL(`pdp/piece?${params.toString()}`, serviceURL), { signal: options.signal, - retry: retry + timeout: options.timeout ?? RETRY_CONSTANTS.TIMEOUT, + retry: { + retries: options.retryCount, + minTimeout: options.retryDelay ?? RETRY_CONSTANTS.RETRY_DELAY, + shouldRetry: (ctx) => HttpError.is(ctx.error) && ctx.error.code === 404, + }, + poll: options.poll ? { - statusCodes: [202, 404], - retries: RETRY_CONSTANTS.RETRIES, - factor: RETRY_CONSTANTS.FACTOR, - minTimeout: options.pollInterval ?? 1000, + limit: RETRY_CONSTANTS.POLL_LIMIT, + interval: options.pollInterval ?? 1000, + statusCodes: [202], // 202 is processing } - : undefined, - timeout: options.timeout ?? RETRY_CONSTANTS.MAX_RETRY_TIME, + : false, }) if (response.error) { diff --git a/packages/synapse-core/src/sp/get-data-set.ts b/packages/synapse-core/src/sp/get-data-set.ts index 1a3eb150..30c9f251 100644 --- a/packages/synapse-core/src/sp/get-data-set.ts +++ b/packages/synapse-core/src/sp/get-data-set.ts @@ -1,6 +1,7 @@ -import { type AbortError, HttpError, type NetworkError, request, type TimeoutError } from 'iso-web/http' +import { HttpError, type RequestJsonErrors, request } from 'iso-web/http' import * as z from 'zod' import { GetDataSetError } from '../errors/pdp.ts' +import { RETRY_CONSTANTS } from '../utils/constants.ts' import { zNumberToBigInt, zStringToCid } from '../utils/schemas.ts' const PieceSchema = z.object({ @@ -27,9 +28,13 @@ export namespace getDataSet { serviceURL: string /** The ID of the data set. */ dataSetId: bigint + /** The number of retries. Defaults to 2. */ + retryCount?: number + /** The delay with exponential backoff between retries in milliseconds. Defaults to {@link RETRY_CONSTANTS.RETRY_DELAY}. */ + retryDelay?: number } export type OutputType = DataSet - export type ErrorType = GetDataSetError | TimeoutError | NetworkError | AbortError + export type ErrorType = GetDataSetError | RequestJsonErrors } /** @@ -43,9 +48,14 @@ export namespace getDataSet { * @throws Errors {@link getDataSet.ErrorType} */ export async function getDataSet(options: getDataSet.OptionsType): Promise { - const response = await request.json.get( - new URL(`pdp/data-sets/${options.dataSetId}`, options.serviceURL) - ) + const response = await request.json.get(new URL(`pdp/data-sets/${options.dataSetId}`, options.serviceURL), { + timeout: RETRY_CONSTANTS.TIMEOUT, + retry: { + retries: options.retryCount, + minTimeout: options.retryDelay ?? RETRY_CONSTANTS.RETRY_DELAY, + }, + schema: DataSetSchema, + }) if (response.error) { if (HttpError.is(response.error)) { throw new GetDataSetError(await response.error.response.text()) @@ -53,5 +63,5 @@ export async function getDataSet(options: getDataSet.OptionsType): Promise { const response = await request.post(new URL('pdp/piece/pull', options.serviceURL), { - body: buildPullRequestBody(options), - headers: { - 'Content-Type': 'application/json', + json: buildPullRequestBody(options), + timeout: RETRY_CONSTANTS.TIMEOUT, + retry: { + methods: ['post'], + retries: options.retryCount, + minTimeout: options.retryDelay ?? RETRY_CONSTANTS.RETRY_DELAY, }, - timeout: RETRY_CONSTANTS.MAX_RETRY_TIME, signal: options.signal, }) @@ -147,7 +153,11 @@ export namespace waitForPullPiecesApiRequest { onStatus?: (response: pullPiecesApiRequest.ReturnType) => void /** The timeout in milliseconds. Defaults to 5 minutes. */ timeout?: number - /** The polling interval in milliseconds. Defaults to 4 seconds. */ + /** The number of retries. Defaults to 2. */ + retryCount?: number + /** The delay with exponential backoff between retries in milliseconds. Defaults to {@link RETRY_CONSTANTS.RETRY_DELAY}. */ + retryDelay?: number + /** The poll interval in milliseconds. Defaults to {@link RETRY_CONSTANTS.POLL_INTERVAL}. */ pollInterval?: number } @@ -170,35 +180,28 @@ export async function waitForPullPiecesApiRequest( options: waitForPullPiecesApiRequest.OptionsType ): Promise { const url = new URL('pdp/piece/pull', options.serviceURL) - const body = buildPullRequestBody(options) - const headers = { 'Content-Type': 'application/json' } const response = await request.post(url, { - body, - headers, - async onResponse(response) { - if (response.ok) { - const data = (await response.clone().json()) as pullPiecesApiRequest.ReturnType - + json: buildPullRequestBody(options), + retry: { + methods: ['post'], + retries: options.retryCount, + minTimeout: options.retryDelay ?? RETRY_CONSTANTS.RETRY_DELAY, + }, + poll: { + limit: RETRY_CONSTANTS.POLL_LIMIT, + interval: options.pollInterval ?? RETRY_CONSTANTS.POLL_INTERVAL, + statusCodes: [202, 200], // 202 is processing, 200 is success + shouldPoll: async (ctx) => { + const data = (await ctx.response.clone().json()) as pullPiecesApiRequest.ReturnType // Invoke status callback if provided if (options.onStatus) { options.onStatus(data) } - - // Stop polling when complete or failed - if (data.status === 'complete' || data.status === 'failed') { - return response - } - throw new Error('Pull not complete') - } - }, - retry: { - shouldRetry: (ctx) => ctx.error.message === 'Pull not complete', - retries: RETRY_CONSTANTS.RETRIES, - factor: RETRY_CONSTANTS.FACTOR, - minTimeout: options.pollInterval ?? RETRY_CONSTANTS.DELAY_TIME, + return data.status !== 'complete' && data.status !== 'failed' + }, }, - timeout: options.timeout ?? RETRY_CONSTANTS.MAX_RETRY_TIME, + timeout: options.timeout ?? RETRY_CONSTANTS.TIMEOUT, signal: options.signal, }) @@ -391,7 +394,11 @@ export namespace waitForPullPieces { onStatus?: (response: pullPieces.ReturnType) => void /** The timeout in milliseconds. Defaults to 5 minutes. */ timeout?: number - /** The polling interval in milliseconds. Defaults to 4 seconds. */ + /** The number of retries. Defaults to 2. */ + retryCount?: number + /** The delay with exponential backoff between retries in milliseconds. Defaults to {@link RETRY_CONSTANTS.RETRY_DELAY}. */ + retryDelay?: number + /** The poll interval in milliseconds. Defaults to {@link RETRY_CONSTANTS.POLL_INTERVAL}. */ pollInterval?: number } @@ -421,5 +428,7 @@ export async function waitForPullPieces( onStatus: options.onStatus, timeout: options.timeout, pollInterval: options.pollInterval, + retryCount: options.retryCount, + retryDelay: options.retryDelay, }) } diff --git a/packages/synapse-core/src/sp/schedule-piece-deletion.ts b/packages/synapse-core/src/sp/schedule-piece-deletion.ts index 1375ca3e..da3eda33 100644 --- a/packages/synapse-core/src/sp/schedule-piece-deletion.ts +++ b/packages/synapse-core/src/sp/schedule-piece-deletion.ts @@ -1,4 +1,4 @@ -import { type AbortError, HttpError, type NetworkError, request, type TimeoutError } from 'iso-web/http' +import { HttpError, type RequestJsonErrors, request } from 'iso-web/http' import type { Account, Chain, Client, Hex, Transport } from 'viem' import { DeletePieceError } from '../errors/pdp.ts' import { signSchedulePieceRemovals } from '../typed-data/sign-schedule-piece-removals.ts' @@ -10,11 +10,15 @@ export namespace deletePiece { dataSetId: bigint pieceId: bigint extraData: Hex + /** The number of retries. Defaults to 2. */ + retryCount?: number + /** The delay with exponential backoff between retries in milliseconds. Defaults to {@link RETRY_CONSTANTS.RETRY_DELAY}. */ + retryDelay?: number } export type OutputType = { hash: Hex } - export type ErrorType = DeletePieceError | TimeoutError | NetworkError | AbortError + export type ErrorType = DeletePieceError | RequestJsonErrors } /** @@ -32,7 +36,12 @@ export async function deletePiece(options: deletePiece.OptionsType): Promise HttpError.is(ctx.error) && ctx.error.code === 429, + }, } ) @@ -56,6 +65,10 @@ export namespace schedulePieceDeletion { clientDataSetId: bigint /** The service URL of the PDP API. */ serviceURL: string + /** The number of retries. Defaults to 2. */ + retryCount?: number + /** The delay with exponential backoff between retries in milliseconds. Defaults to {@link RETRY_CONSTANTS.RETRY_DELAY}. */ + retryDelay?: number } export type OutputType = deletePiece.OutputType export type ErrorType = deletePiece.ErrorType @@ -107,5 +120,7 @@ export async function schedulePieceDeletion( clientDataSetId: options.clientDataSetId, pieceIds: [options.pieceId], }), + retryCount: options.retryCount, + retryDelay: options.retryDelay, }) } diff --git a/packages/synapse-core/src/sp/upload-streaming.ts b/packages/synapse-core/src/sp/upload-streaming.ts index cdd6b0ec..1b52302b 100644 --- a/packages/synapse-core/src/sp/upload-streaming.ts +++ b/packages/synapse-core/src/sp/upload-streaming.ts @@ -21,6 +21,10 @@ export namespace uploadPieceStreaming { pieceCid?: PieceCID /** The signal to abort the request. */ signal?: AbortSignal + /** The number of retries. Defaults to 2. */ + retryCount?: number + /** The delay with exponential backoff between retries in milliseconds. Defaults to {@link RETRY_CONSTANTS.RETRY_DELAY}. */ + retryDelay?: number } export type OutputType = { pieceCid: PieceCID @@ -47,7 +51,12 @@ export async function uploadPieceStreaming( ): Promise { // Create upload session (POST /pdp/piece/uploads) const createResponse = await request.post(new URL('pdp/piece/uploads', options.serviceURL), { - timeout: RETRY_CONSTANTS.MAX_RETRY_TIME, + timeout: RETRY_CONSTANTS.TIMEOUT, + retry: { + methods: ['post'], + retries: options.retryCount, + minTimeout: options.retryDelay ?? RETRY_CONSTANTS.RETRY_DELAY, + }, signal: options.signal, }) @@ -169,7 +178,7 @@ export async function uploadPieceStreaming( timeout: false, // No timeout for streaming upload signal: options.signal, ...fetchOptions, - } as Parameters[1] & { duplex?: 'half' }) + }) if (uploadResponse.error) { if (HttpError.is(uploadResponse.error)) { @@ -185,17 +194,17 @@ export async function uploadPieceStreaming( // Get PieceCID (either provided or calculated) and finalize. const pieceCid = await pieceCidPromise - const finalizeBody = JSON.stringify({ - pieceCid: pieceCid.toString(), - }) - // POST /pdp/piece/uploads/{uuid} with PieceCID const finalizeResponse = await request.post(new URL(`pdp/piece/uploads/${uploadUuid}`, options.serviceURL), { - body: finalizeBody, - headers: { - 'Content-Type': 'application/json', + json: { + pieceCid: pieceCid.toString(), + }, + timeout: RETRY_CONSTANTS.TIMEOUT, + retry: { + methods: ['post'], + retries: options.retryCount, + minTimeout: options.retryDelay ?? RETRY_CONSTANTS.RETRY_DELAY, }, - timeout: RETRY_CONSTANTS.MAX_RETRY_TIME, signal: options.signal, }) diff --git a/packages/synapse-core/src/sp/upload.ts b/packages/synapse-core/src/sp/upload.ts index 15c870f8..afc2fce1 100644 --- a/packages/synapse-core/src/sp/upload.ts +++ b/packages/synapse-core/src/sp/upload.ts @@ -19,6 +19,10 @@ export namespace uploadPiece { data: Uint8Array /** The piece CID to upload. */ pieceCid: PieceCID + /** The number of retries. Defaults to 2. */ + retryCount?: number + /** The delay with exponential backoff between retries in milliseconds. Defaults to {@link RETRY_CONSTANTS.RETRY_DELAY}. */ + retryDelay?: number } export type ErrorType = InvalidUploadSizeError | LocationHeaderError | TimeoutError | NetworkError | AbortError } @@ -42,13 +46,15 @@ export async function uploadPiece(options: uploadPiece.OptionsType): Promise, options: await findPiece({ pieceCid, serviceURL, - retry: true, + poll: true, }) options.onEvent?.('pieceParked', { pieceCid, url, dataSet }) diff --git a/packages/synapse-core/src/utils/constants.ts b/packages/synapse-core/src/utils/constants.ts index d321e93b..997aeefd 100644 --- a/packages/synapse-core/src/utils/constants.ts +++ b/packages/synapse-core/src/utils/constants.ts @@ -141,10 +141,14 @@ export const CDN_FIXED_LOCKUP = { export const USDFC_SYBIL_FEE = 100_000_000_000_000_000n // 0.1 USDFC export const RETRY_CONSTANTS = { - RETRIES: Infinity, - FACTOR: 1, - DELAY_TIME: 4000, // 4 seconds in milliseconds between retries - MAX_RETRY_TIME: 1000 * 60 * 5, // 5 minutes in milliseconds + /** The interval in milliseconds between polls. 4 seconds is the default interval between polls. */ + POLL_INTERVAL: 4000, + /** The limit of polls. */ + POLL_LIMIT: Infinity, + /** The delay in milliseconds between retries. 250ms is the default delay between retries. */ + RETRY_DELAY: 250, + /** The timeout in milliseconds. 5 minutes is the default timeout. */ + TIMEOUT: 1000 * 60 * 5, } as const /** diff --git a/packages/synapse-core/test/piece-url.test.ts b/packages/synapse-core/test/piece-url.test.ts index c5b13851..24a3cd65 100644 --- a/packages/synapse-core/test/piece-url.test.ts +++ b/packages/synapse-core/test/piece-url.test.ts @@ -1,5 +1,5 @@ -import { calibration, devnet, mainnet } from '@filoz/synapse-core/chains' import { assert } from 'chai' +import { calibration, devnet, mainnet } from '../src/chains.ts' import { createPieceUrl, createPieceUrlPDP } from '../src/utils/piece-url.ts' describe('createPieceUrl', () => { diff --git a/packages/synapse-core/test/pull.test.ts b/packages/synapse-core/test/pull.test.ts index b0b4939a..46dd4e4b 100644 --- a/packages/synapse-core/test/pull.test.ts +++ b/packages/synapse-core/test/pull.test.ts @@ -28,6 +28,7 @@ describe('Pull', () => { recordKeeper: TEST_RECORD_KEEPER, extraData: TEST_EXTRA_DATA, pieces: [{ pieceCid: TEST_PIECE_CID, sourceUrl: TEST_SOURCE_URL }], + retryDelay: 10, }) before(async () => { diff --git a/packages/synapse-core/test/rand.test.ts b/packages/synapse-core/test/rand.test.ts index 7f45de0e..3f930c02 100644 --- a/packages/synapse-core/test/rand.test.ts +++ b/packages/synapse-core/test/rand.test.ts @@ -1,5 +1,5 @@ -import { fallbackRandIndex, fallbackRandU256, randIndex, randU256 } from '@filoz/synapse-core/utils' import { assert } from 'chai' +import { fallbackRandIndex, fallbackRandU256, randIndex, randU256 } from '../src/utils/rand.ts' const randIndexMethods = [randIndex, fallbackRandIndex] randIndexMethods.forEach((randIndexMethod) => { diff --git a/packages/synapse-core/test/resolve-piece-url.test.ts b/packages/synapse-core/test/resolve-piece-url.test.ts index 45ad07d9..a15bafb8 100644 --- a/packages/synapse-core/test/resolve-piece-url.test.ts +++ b/packages/synapse-core/test/resolve-piece-url.test.ts @@ -109,9 +109,8 @@ describe('resolve-piece-url', () => { server.use( JSONRPC(presets.basic), http.head(filbeamUrl, () => HttpResponse.text('not found', { status: 404 })), - http.get('https://pdp.example.com/pdp/piece', ({ request }) => { - const url = new URL(request.url) - return HttpResponse.json({ pieceCid: url.searchParams.get('pieceCid') }, { status: 200 }) + http.head(`https://pdp.example.com/piece/${pieceCidString}`, () => { + return new HttpResponse(null, { status: 200 }) }) ) @@ -191,26 +190,35 @@ describe('resolve-piece-url', () => { ] server.use( - http.get('https://missing.example.com/pdp/piece', () => HttpResponse.text('not found', { status: 404 })), - http.get('https://pdp.example.com/pdp/piece', ({ request }) => { - const url = new URL(request.url) - return HttpResponse.json({ pieceCid: url.searchParams.get('pieceCid') }, { status: 200 }) + http.head(`https://missing.example.com/piece/${pieceCidString}`, () => + HttpResponse.text('not found', { status: 404 }) + ), + http.head(`https://pdp.example.com/piece/${pieceCidString}`, () => { + return new HttpResponse(null, { status: 200 }) }) ) - const result = await findPieceOnProviders(providers, pieceCid) + const result = await findPieceOnProviders( + providers.map((p) => p.pdp.serviceURL), + pieceCid + ) assert.ok(result) - assert.equal(result?.id, 2n) + assert.equal(result, providers[1].pdp.serviceURL) }) it('returns undefined when no provider has the piece', async () => { const providers: PDPProvider[] = [createProvider('https://missing.example.com/')] server.use( - http.get('https://missing.example.com/pdp/piece', () => HttpResponse.text('not found', { status: 404 })) + http.head(`https://missing.example.com/piece/${pieceCidString}`, () => + HttpResponse.text('not found', { status: 404 }) + ) ) - const result = await findPieceOnProviders(providers, pieceCid) + const result = await findPieceOnProviders( + providers.map((p) => p.pdp.serviceURL), + pieceCid + ) assert.equal(result, undefined) }) }) @@ -219,9 +227,8 @@ describe('resolve-piece-url', () => { it('returns serviceURL when a provider contains the piece', async () => { const providers: PDPProvider[] = [createProvider('https://pdp.example.com/', 5n)] server.use( - http.get('https://pdp.example.com/pdp/piece', ({ request }) => { - const url = new URL(request.url) - return HttpResponse.json({ pieceCid: url.searchParams.get('pieceCid') }, { status: 200 }) + http.head(`https://pdp.example.com/piece/${pieceCidString}`, () => { + return new HttpResponse(null, { status: 200 }) }) ) @@ -237,7 +244,9 @@ describe('resolve-piece-url', () => { it('throws when no provider has the piece', async () => { const providers: PDPProvider[] = [createProvider('https://missing.example.com/')] server.use( - http.get('https://missing.example.com/pdp/piece', () => HttpResponse.text('not found', { status: 404 })) + http.head(`https://missing.example.com/piece/${pieceCidString}`, () => + HttpResponse.text('not found', { status: 404 }) + ) ) const resolver = providersResolver(providers) @@ -256,9 +265,8 @@ describe('resolve-piece-url', () => { it('resolves piece URL from on-chain provider list', async () => { server.use( JSONRPC(presets.basic), - http.get('https://pdp.example.com/pdp/piece', ({ request }) => { - const url = new URL(request.url) - return HttpResponse.json({ pieceCid: url.searchParams.get('pieceCid') }, { status: 200 }) + http.head(`https://pdp.example.com/piece/${pieceCidString}`, () => { + return new HttpResponse(null, { status: 200 }) }) ) diff --git a/packages/synapse-core/test/sp.test.ts b/packages/synapse-core/test/sp.test.ts index 24c0e9c2..26119649 100644 --- a/packages/synapse-core/test/sp.test.ts +++ b/packages/synapse-core/test/sp.test.ts @@ -35,6 +35,7 @@ import { deletePiece, findPiece, getDataSet, + NetworkError, TimeoutError, uploadPiece, waitForAddPieces, @@ -214,6 +215,8 @@ describe('SP', () => { clientDataSetId: 0n, payee: ADDRESSES.client1, }), + retryCount: 1, + retryDelay: 10, }) assert.fail('Should have thrown error for CreateDataSetError error') } catch (e) { @@ -255,6 +258,8 @@ invariant failure: insufficient funds to cover lockup after function execution` clientDataSetId: 0n, payee: ADDRESSES.client1, }), + retryCount: 1, + retryDelay: 10, }) assert.fail('Should have thrown error for CreateDataSetError error') } catch (error) { @@ -296,6 +301,8 @@ InvalidSignature(address expected, address actual) clientDataSetId: 0n, payee: ADDRESSES.client1, }), + retryCount: 1, + retryDelay: 10, }) assert.fail('Should have thrown error for CreateDataSetError error') } catch (error) { @@ -402,6 +409,7 @@ InvalidSignature(address expected, address actual) try { await waitForCreateDataSet({ statusUrl: `http://pdp.local/pdp/data-sets/created/${mockTxHash}`, + retryDelay: 10, }) assert.fail('Should have thrown error for server error') } catch (error) { @@ -541,6 +549,7 @@ InvalidSignature(address expected, address actual) dataSetId: 1n, pieces: [pieceCid], extraData, + retryDelay: 10, }) assert.fail('Should have thrown error for server error') } catch (error) { @@ -550,6 +559,75 @@ InvalidSignature(address expected, address actual) } }) + it('should not retry on network errors', async () => { + const pieceCid = Piece.from(validPieceCid) + let callCount = 0 + server.use( + http.post('http://pdp.local/pdp/data-sets/:id/pieces', () => { + callCount++ + return HttpResponse.error() + }) + ) + + const extraData = await TypedData.signAddPieces(client, { + clientDataSetId: 0n, + pieces: [{ pieceCid }], + }) + + try { + await addPiecesApiRequest({ + serviceURL: 'http://pdp.local', + dataSetId: 1n, + pieces: [pieceCid], + extraData, + retryDelay: 10, + }) + assert.fail('Should have thrown error for network error') + } catch (error) { + assert.strictEqual(callCount, 1) + assert.instanceOf(error, NetworkError) + assert.include(error.message, 'Network request failed') + } + }) + + it('should retry on 429 errors', async () => { + const pieceCid = Piece.from(validPieceCid) + let callCount = 0 + server.use( + http.post('http://pdp.local/pdp/data-sets/:id/pieces', () => { + callCount++ + return new HttpResponse(null, { + status: 429, + headers: { + 'Retry-After': '0.01', + }, + statusText: 'Too Many Requests', + }) + }) + ) + + const extraData = await TypedData.signAddPieces(client, { + clientDataSetId: 0n, + pieces: [{ pieceCid }], + }) + + try { + await addPiecesApiRequest({ + serviceURL: 'http://pdp.local', + dataSetId: 1n, + pieces: [pieceCid], + extraData, + retryCount: 2, + retryDelay: 10, + }) + assert.fail('Should have thrown error for 429 error') + } catch (error) { + assert.strictEqual(callCount, 3) + assert.instanceOf(error, AddPiecesError) + assert.include(error.message, 'Failed to add pieces.') + } + }) + it('should handle multiple pieces', async () => { const mockTxHash = '0xabcdef1234567890abcdef1234567890abcdef1234567890abcdef1234567890' const pieceCid1 = Piece.from(validPieceCid) @@ -752,6 +830,7 @@ InvalidSignature(address expected, address actual) try { await waitForAddPieces({ statusUrl: `http://pdp.local/pdp/data-sets/1/pieces/added/${mockTxHash}`, + retryDelay: 10, }) assert.fail('Should have thrown error for server error') } catch (error) { @@ -849,6 +928,7 @@ InvalidSignature(address expected, address actual) dataSetId: 1n, pieceId: 2n, extraData, + retryDelay: 10, }) assert.fail('Should have thrown error for server error') } catch (error) { @@ -883,8 +963,8 @@ InvalidSignature(address expected, address actual) await findPiece({ serviceURL: 'https://pdp.example.com', pieceCid, - retry: true, - timeout: 50, + poll: true, + timeout: 10, }) assert.fail('Should have thrown error for not found') } catch (error) { @@ -909,6 +989,7 @@ InvalidSignature(address expected, address actual) await findPiece({ serviceURL: 'https://pdp.example.com', pieceCid, + retryDelay: 10, }) assert.fail('Should have thrown error for server error') } catch (error) { @@ -936,7 +1017,7 @@ InvalidSignature(address expected, address actual) const result = await findPiece({ serviceURL: 'http://pdp.local', pieceCid, - retry: true, + poll: true, pollInterval: 10, }) assert.strictEqual(result.toString(), mockPieceCidStr) @@ -1058,6 +1139,7 @@ InvalidSignature(address expected, address actual) serviceURL: 'http://pdp.local', data: testData, pieceCid, + retryDelay: 10, }) assert.fail('Should have thrown error for POST error') } catch (error) { @@ -1082,6 +1164,7 @@ InvalidSignature(address expected, address actual) serviceURL: 'https://pdp.example.com', data: testData, pieceCid, + retryDelay: 10, }) assert.fail('Should have thrown error for PUT error') } catch (error) { @@ -1187,6 +1270,7 @@ InvalidSignature(address expected, address actual) serviceURL: 'http://pdp.local', data: testData, pieceCid, + retryDelay: 10, }) assert.fail('Should have thrown error for session creation failure') } catch (error) { @@ -1332,6 +1416,7 @@ InvalidSignature(address expected, address actual) serviceURL: 'https://pdp.example.com', data: testData, pieceCid, + retryDelay: 10, }) assert.fail('Should have thrown error for finalize failure') } catch (error) { @@ -1441,6 +1526,7 @@ InvalidSignature(address expected, address actual) await getDataSet({ serviceURL: 'http://pdp.local', dataSetId: 292n, + retryDelay: 10, }) assert.fail('Should have thrown error for server error') } catch (error) { diff --git a/packages/synapse-sdk/src/storage/context.ts b/packages/synapse-sdk/src/storage/context.ts index 6a80beff..bc84782e 100644 --- a/packages/synapse-sdk/src/storage/context.ts +++ b/packages/synapse-sdk/src/storage/context.ts @@ -680,7 +680,7 @@ export class StorageContext { await SP.findPiece({ serviceURL: this._pdpEndpoint, pieceCid: uploadResult.pieceCid, - retry: true, + poll: true, signal: options?.signal, }) } catch (error) { diff --git a/packages/synapse-sdk/src/test/storage.test.ts b/packages/synapse-sdk/src/test/storage.test.ts index 1757e333..3110e2fe 100644 --- a/packages/synapse-sdk/src/test/storage.test.ts +++ b/packages/synapse-sdk/src/test/storage.test.ts @@ -827,7 +827,9 @@ describe('StorageService', () => { status: 404, }) }), - Mocks.pdp.findPieceHandler(testPieceCID, true, pdpOptions), + http.head('https://pdp.example.com/piece/:pieceCid', async () => { + return new HttpResponse(null, { status: 200 }) + }), http.get('https://pdp.example.com/piece/:pieceCid', async () => { return HttpResponse.arrayBuffer(testData.buffer) }) @@ -849,7 +851,9 @@ describe('StorageService', () => { ...Mocks.presets.basic, }), Mocks.PING(), - Mocks.pdp.findPieceHandler(testPieceCID, true, pdpOptions), + http.head('https://pdp.example.com/piece/:pieceCid', async () => { + return new HttpResponse(null, { status: 200 }) + }), http.get('https://pdp.example.com/piece/:pieceCid', async () => { return HttpResponse.error() }) @@ -875,7 +879,9 @@ describe('StorageService', () => { ...Mocks.presets.basic, }), Mocks.PING(), - Mocks.pdp.findPieceHandler(testPieceCID, true, pdpOptions), + http.head('https://pdp.example.com/piece/:pieceCid', async () => { + return new HttpResponse(null, { status: 200 }) + }), http.get('https://pdp.example.com/piece/:pieceCid', async () => { return HttpResponse.arrayBuffer(testData.buffer) }) diff --git a/packages/synapse-sdk/src/test/synapse.test.ts b/packages/synapse-sdk/src/test/synapse.test.ts index 9829f81b..b5788c27 100644 --- a/packages/synapse-sdk/src/test/synapse.test.ts +++ b/packages/synapse-sdk/src/test/synapse.test.ts @@ -194,13 +194,8 @@ describe('Synapse', () => { const testData = new TextEncoder().encode('test data') server.use( Mocks.JSONRPC(Mocks.presets.basic), - http.get('https://pdp.example.com/pdp/piece', async ({ request }) => { - const url = new URL(request.url) - const pieceCid = url.searchParams.get('pieceCid') - - return HttpResponse.json({ - pieceCid, - }) + http.head('https://pdp.example.com/piece/:pieceCid', async () => { + return new HttpResponse(null, { status: 200 }) }), http.get('https://pdp.example.com/piece/:pieceCid', async () => { return HttpResponse.arrayBuffer(testData.buffer) @@ -230,13 +225,8 @@ describe('Synapse', () => { deferred.resolve(params) return HttpResponse.arrayBuffer(testData.buffer) }), - http.get('https://pdp.example.com/pdp/piece', async ({ request }) => { - const url = new URL(request.url) - const pieceCid = url.searchParams.get('pieceCid') - - return HttpResponse.json({ - pieceCid, - }) + http.head('https://pdp.example.com/piece/:pieceCid', async () => { + return new HttpResponse(null, { status: 200 }) }), http.get('https://pdp.example.com/piece/:pieceCid', async () => { return HttpResponse.arrayBuffer(testData.buffer) @@ -281,13 +271,8 @@ describe('Synapse', () => { }, }, }), - http.get('https://pdp.example.com/pdp/piece', async ({ request }) => { - const url = new URL(request.url) - const pieceCid = url.searchParams.get('pieceCid') - - return HttpResponse.json({ - pieceCid, - }) + http.head('https://pdp.example.com/piece/:pieceCid', async () => { + return new HttpResponse(null, { status: 200 }) }), http.get('https://pdp.example.com/piece/:pieceCid', async () => { return HttpResponse.arrayBuffer(testData.buffer) @@ -308,7 +293,7 @@ describe('Synapse', () => { it('should handle download errors', async () => { server.use( Mocks.JSONRPC(Mocks.presets.basic), - http.get('https://pdp.example.com/pdp/piece', async () => { + http.head('https://pdp.example.com/piece/:pieceCid', async () => { return HttpResponse.error() }) ) diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 2fee726c..73a0e289 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -28,7 +28,7 @@ catalog: viem: ^2.52.0 wagmi: ^3.0.2 zod: ^4.3.5 - iso-web: 2.2.1 + iso-web: ^3.1.2 minimumReleaseAge: 10080 @@ -44,6 +44,9 @@ minimumReleaseAgeExclude: # Remove after 2026-06-08 when both age out. - viem@2.52.0 - ox@0.14.27 + # astro 6.4.2 fixes MDX/Starlight compatibility after 6.4.0. + # Remove after 2026-06-05 when it ages out. + - astro@6.4.2 trustPolicy: no-downgrade