From 482ca6775a59e2ca53fe0dec62a18856bfb33d3c Mon Sep 17 00:00:00 2001 From: Rod Vagg Date: Wed, 3 Jun 2026 16:23:24 +1000 Subject: [PATCH 1/2] feat: cap addPieces batches at 40, tighten piece metadata limits Enforce a 40-piece ceiling per addPieces/createDataSetAndAddPieces via validateAddPiecesBatch, wired into addPieces, createDataSetAndAddPieces, commit, and presignForCommit so oversized batches fail early with TooManyPiecesError rather than reverting on-chain at the FVM event-size limit. Sync piece metadata caps to the contract: MAX_KEYS_PER_PIECE 5 -> 3, MAX_VALUE_LENGTH 128 -> 96. Ref: https://github.com/filecoin-project/curio/pull/1272 Ref: https://github.com/FilOzone/filecoin-services/issues/496 Ref: https://www.notion.so/filecoindev/addPieces-Batch-Limits-and-Proposed-Changes-36ddc41950c180d39f45f0da8982090c --- AGENTS.md | 2 +- .../storage/storage-operations.mdx | 2 +- .../synapse-core/src/errors/warm-storage.ts | 11 +++++ packages/synapse-core/src/sp/add-pieces.ts | 25 ++++++++-- .../src/sp/create-dataset-add-pieces.ts | 3 +- packages/synapse-core/src/utils/constants.ts | 8 ++++ packages/synapse-core/src/utils/metadata.ts | 4 +- packages/synapse-core/test/add-pieces.test.ts | 20 ++++++++ packages/synapse-core/test/metadata.test.ts | 4 +- packages/synapse-sdk/src/storage/context.ts | 8 +++- packages/synapse-sdk/src/test/storage.test.ts | 48 +++++++++++++++++++ 11 files changed, 122 insertions(+), 13 deletions(-) create mode 100644 packages/synapse-core/test/add-pieces.test.ts diff --git a/AGENTS.md b/AGENTS.md index 0021077d1..66e979658 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -172,7 +172,7 @@ Ref: FRC-0069 **Internal**: `MetadataEntry[]` (alphabetically sorted for EIP-712) -**Validation**: Data sets max 10 keys, pieces max 5 keys, keys max 32 chars, values max 128 chars. Validated early in PDPServer. +**Validation**: Data sets max 10 keys, pieces max 3 keys, keys max 32 chars, values max 96 chars. Validated early in PDPServer. **Security**: Uses `Object.create(null)` for prototype-safe objects from contracts. diff --git a/docs/src/content/docs/developer-guides/storage/storage-operations.mdx b/docs/src/content/docs/developer-guides/storage/storage-operations.mdx index 469eac4ae..01eea5d24 100644 --- a/docs/src/content/docs/developer-guides/storage/storage-operations.mdx +++ b/docs/src/content/docs/developer-guides/storage/storage-operations.mdx @@ -18,7 +18,7 @@ This page covers data set management, retrieval, and lifecycle operations. For u **Metadata**: Optional key-value pairs for organization: - **Data Set Metadata**: Max 10 keys (e.g., `project`, `environment`) -- **Piece Metadata**: Max 5 keys per piece (e.g., `filename`, `contentType`) +- **Piece Metadata**: Max 3 keys per piece (e.g., `filename`, `contentType`) **Copies and Durability**: By default, `upload()` stores your data with 2 independent providers. Each provider maintains its own data set with separate PDP proofs and payment rails. If one provider goes down, your data is still available from the other. diff --git a/packages/synapse-core/src/errors/warm-storage.ts b/packages/synapse-core/src/errors/warm-storage.ts index 1fcd69653..a1c821b6c 100644 --- a/packages/synapse-core/src/errors/warm-storage.ts +++ b/packages/synapse-core/src/errors/warm-storage.ts @@ -21,3 +21,14 @@ export class AtLeastOnePieceRequiredError extends SynapseError { return isSynapseError(value) && value.name === 'AtLeastOnePieceRequiredError' } } + +export class TooManyPiecesError extends SynapseError { + override name: 'TooManyPiecesError' = 'TooManyPiecesError' + constructor(count: number, max: number) { + super(`Too many pieces: ${count}, max ${max} per addPieces call. Split into smaller batches.`) + } + + static override is(value: unknown): value is TooManyPiecesError { + return isSynapseError(value) && value.name === 'TooManyPiecesError' + } +} diff --git a/packages/synapse-core/src/sp/add-pieces.ts b/packages/synapse-core/src/sp/add-pieces.ts index d116c4802..9277ae8f5 100644 --- a/packages/synapse-core/src/sp/add-pieces.ts +++ b/packages/synapse-core/src/sp/add-pieces.ts @@ -4,10 +4,10 @@ import { type Account, type Chain, type Client, type Hex, isHex, type Transport import * as z from 'zod' import { AddPiecesError, LocationHeaderError } from '../errors/index.ts' import { WaitForAddPiecesError, WaitForAddPiecesRejectedError } from '../errors/pdp.ts' -import { AtLeastOnePieceRequiredError } from '../errors/warm-storage.ts' +import { AtLeastOnePieceRequiredError, TooManyPiecesError } from '../errors/warm-storage.ts' import type { PieceCID } from '../piece/piece-cid.ts' import { signAddPieces } from '../typed-data/sign-add-pieces.ts' -import { RETRY_CONSTANTS } from '../utils/constants.ts' +import { RETRY_CONSTANTS, SIZE_CONSTANTS } from '../utils/constants.ts' import { type MetadataObject, pieceMetadataObjectToEntry } from '../utils/metadata.ts' import { zHex, zNumberToBigInt } from '../utils/schemas.ts' @@ -107,6 +107,23 @@ export namespace addPieces { export type ErrorType = addPiecesApiRequest.ErrorType | signAddPieces.ErrorType } +/** + * Validate the piece count for an addPieces (or createDataSetAndAddPieces) batch, + * failing early instead of reverting on-chain. + * + * @param pieceCount - Number of pieces in the batch + * @throws AtLeastOnePieceRequiredError when empty + * @throws TooManyPiecesError when above {@link SIZE_CONSTANTS.MAX_ADD_PIECES_BATCH_SIZE} + */ +export function validateAddPiecesBatch(pieceCount: number): void { + if (pieceCount === 0) { + throw new AtLeastOnePieceRequiredError() + } + if (pieceCount > SIZE_CONSTANTS.MAX_ADD_PIECES_BATCH_SIZE) { + throw new TooManyPiecesError(pieceCount, SIZE_CONSTANTS.MAX_ADD_PIECES_BATCH_SIZE) + } +} + /** * Add pieces to a data set * @@ -121,9 +138,7 @@ export async function addPieces( client: Client, options: addPieces.OptionsType ): Promise { - if (options.pieces.length === 0) { - throw new AtLeastOnePieceRequiredError() - } + validateAddPiecesBatch(options.pieces.length) const extraData = options.extraData ?? (await signAddPieces(client, { diff --git a/packages/synapse-core/src/sp/create-dataset-add-pieces.ts b/packages/synapse-core/src/sp/create-dataset-add-pieces.ts index 051630a3a..e8d25e3b3 100644 --- a/packages/synapse-core/src/sp/create-dataset-add-pieces.ts +++ b/packages/synapse-core/src/sp/create-dataset-add-pieces.ts @@ -13,7 +13,7 @@ import type { PieceCID } from '../piece/piece-cid.ts' import { signCreateDataSetAndAddPieces } from '../typed-data/sign-create-dataset-add-pieces.ts' import { RETRY_CONSTANTS } from '../utils/constants.ts' import { datasetMetadataObjectToEntry, type MetadataObject, pieceMetadataObjectToEntry } from '../utils/metadata.ts' -import { waitForAddPieces } from './add-pieces.ts' +import { validateAddPiecesBatch, waitForAddPieces } from './add-pieces.ts' import { waitForCreateDataSet } from './create-dataset.ts' export namespace createDataSetAndAddPiecesApiRequest { @@ -134,6 +134,7 @@ export async function createDataSetAndAddPieces( client: Client, options: CreateDataSetAndAddPiecesOptions ): Promise { + validateAddPiecesBatch(options.pieces.length) const chain = asChain(client.chain) const extraData = options.extraData ?? diff --git a/packages/synapse-core/src/utils/constants.ts b/packages/synapse-core/src/utils/constants.ts index 6d5341d8e..d321e93bf 100644 --- a/packages/synapse-core/src/utils/constants.ts +++ b/packages/synapse-core/src/utils/constants.ts @@ -91,6 +91,14 @@ export const SIZE_CONSTANTS = { */ DEFAULT_UPLOAD_BATCH_SIZE: 32, + /** + * Maximum pieces per addPieces (or createDataSetAndAddPieces) call. + * + * On-chain limitations fail batch sizes above 41; we constrain to 40 here to + * catch those failures early and surface informative errors. + */ + MAX_ADD_PIECES_BATCH_SIZE: 40, + /** * Bytes per leaf in the PDP merkle tree. * The FWSS contract converts leaf counts to bytes via `totalBytes = leafCount * BYTES_PER_LEAF`. diff --git a/packages/synapse-core/src/utils/metadata.ts b/packages/synapse-core/src/utils/metadata.ts index 3395e711c..86374dc0d 100644 --- a/packages/synapse-core/src/utils/metadata.ts +++ b/packages/synapse-core/src/utils/metadata.ts @@ -16,9 +16,9 @@ export type MetadataObject = Record // Metadata size and count limits from the contract export const METADATA_LIMITS = { MAX_KEY_LENGTH: 32, - MAX_VALUE_LENGTH: 128, + MAX_VALUE_LENGTH: 96, MAX_KEYS_PER_DATASET: 10, - MAX_KEYS_PER_PIECE: 5, + MAX_KEYS_PER_PIECE: 3, } export function metadataArrayToObject(metadataArray: MetadataArray): Record { diff --git a/packages/synapse-core/test/add-pieces.test.ts b/packages/synapse-core/test/add-pieces.test.ts new file mode 100644 index 000000000..6f7bcff7c --- /dev/null +++ b/packages/synapse-core/test/add-pieces.test.ts @@ -0,0 +1,20 @@ +import assert from 'assert' +import { AtLeastOnePieceRequiredError, TooManyPiecesError } from '../src/errors/warm-storage.ts' +import { validateAddPiecesBatch } from '../src/sp/add-pieces.ts' +import { SIZE_CONSTANTS } from '../src/utils/constants.ts' + +describe('validateAddPiecesBatch', () => { + const max = SIZE_CONSTANTS.MAX_ADD_PIECES_BATCH_SIZE + + it('should throw when empty', () => { + assert.throws(() => validateAddPiecesBatch(0), AtLeastOnePieceRequiredError) + }) + + it('should accept a count at the maximum', () => { + assert.doesNotThrow(() => validateAddPiecesBatch(max)) + }) + + it('should throw when above the maximum', () => { + assert.throws(() => validateAddPiecesBatch(max + 1), TooManyPiecesError) + }) +}) diff --git a/packages/synapse-core/test/metadata.test.ts b/packages/synapse-core/test/metadata.test.ts index 064d8ecef..a6853379d 100644 --- a/packages/synapse-core/test/metadata.test.ts +++ b/packages/synapse-core/test/metadata.test.ts @@ -235,9 +235,9 @@ describe('Metadata Utils', () => { describe('METADATA_LIMITS', () => { it('should have expected limit values', () => { assert.equal(METADATA_LIMITS.MAX_KEY_LENGTH, 32) - assert.equal(METADATA_LIMITS.MAX_VALUE_LENGTH, 128) + assert.equal(METADATA_LIMITS.MAX_VALUE_LENGTH, 96) assert.equal(METADATA_LIMITS.MAX_KEYS_PER_DATASET, 10) - assert.equal(METADATA_LIMITS.MAX_KEYS_PER_PIECE, 5) + assert.equal(METADATA_LIMITS.MAX_KEYS_PER_PIECE, 3) }) }) }) diff --git a/packages/synapse-sdk/src/storage/context.ts b/packages/synapse-sdk/src/storage/context.ts index 289d83b36..6a80beff8 100644 --- a/packages/synapse-sdk/src/storage/context.ts +++ b/packages/synapse-sdk/src/storage/context.ts @@ -703,6 +703,7 @@ export class StorageContext { * @returns Signed extraData hex to pass to pull() or commit() */ async presignForCommit(pieces: Array<{ pieceCid: PieceCID; pieceMetadata?: MetadataObject }>): Promise { + SP.validateAddPiecesBatch(pieces.length) const signingPieces = pieces.map((p) => ({ pieceCid: p.pieceCid, metadata: pieceMetadataObjectToEntry(p.pieceMetadata), @@ -738,6 +739,10 @@ export class StorageContext { async pull(options: PullOptions): Promise { const { pieces, from, signal, onProgress, extraData } = options + // The SP estimateGas-validates the eventual addPieces, so an oversized batch + // fails there too; reject early for a clear error on non-presigned paths. + SP.validateAddPiecesBatch(pieces.length) + const getSourceUrl = (pieceCid: PieceCID): string => { if (typeof from === 'string') { return createPieceUrlPDP({ cid: pieceCid.toString(), serviceURL: from }) @@ -820,7 +825,8 @@ export class StorageContext { async commit(options: CommitOptions): Promise { const { pieces, extraData } = options - // Validate metadata early + // Validate batch size and metadata early, before any chain reads or signing + SP.validateAddPiecesBatch(pieces.length) for (const piece of pieces) { if (piece.pieceMetadata) { pieceMetadataObjectToEntry(piece.pieceMetadata) diff --git a/packages/synapse-sdk/src/test/storage.test.ts b/packages/synapse-sdk/src/test/storage.test.ts index 7b3d61975..1757e3337 100644 --- a/packages/synapse-sdk/src/test/storage.test.ts +++ b/packages/synapse-sdk/src/test/storage.test.ts @@ -1,4 +1,5 @@ import { type Chain, calibration } from '@filoz/synapse-core/chains' +import { TooManyPiecesError } from '@filoz/synapse-core/errors' import * as Mocks from '@filoz/synapse-core/mocks' import * as Piece from '@filoz/synapse-core/piece' import { calculate, calculate as calculatePieceCID } from '@filoz/synapse-core/piece' @@ -892,6 +893,53 @@ describe('StorageService', () => { }) }) + describe('addPieces batch limit', () => { + const tooMany = SIZE_CONSTANTS.MAX_ADD_PIECES_BATCH_SIZE + 1 + + async function makeContext() { + server.use(Mocks.JSONRPC({ ...Mocks.presets.basic }), Mocks.PING()) + const synapse = new Synapse({ client, source: null }) + const warmStorageService = new WarmStorageService({ client }) + return StorageContext.create({ synapse, warmStorageService }) + } + + it('presignForCommit rejects batches above the limit', async () => { + const service = await makeContext() + const pieceCid = await calculate(new Uint8Array(127).fill(1)) + const pieces = Array.from({ length: tooMany }, () => ({ pieceCid })) + try { + await service.presignForCommit(pieces) + assert.fail('Should have thrown') + } catch (error) { + assert.instanceOf(error, TooManyPiecesError) + } + }) + + it('commit rejects batches above the limit', async () => { + const service = await makeContext() + const pieceCid = await calculate(new Uint8Array(127).fill(1)) + const pieces = Array.from({ length: tooMany }, () => ({ pieceCid })) + try { + await service.commit({ pieces }) + assert.fail('Should have thrown') + } catch (error) { + assert.instanceOf(error, TooManyPiecesError) + } + }) + + it('pull rejects batches above the limit', async () => { + const service = await makeContext() + const pieceCid = await calculate(new Uint8Array(127).fill(1)) + const pieces = Array.from({ length: tooMany }, () => pieceCid) + try { + await service.pull({ pieces, from: 'https://pdp.example.com' }) + assert.fail('Should have thrown') + } catch (error) { + assert.instanceOf(error, TooManyPiecesError) + } + }) + }) + describe('upload', () => { it('should handle errors in batch processing gracefully', async () => { server.use( From bcb6af16ba9b8406998ce3d173cf6b9e2998b815 Mon Sep 17 00:00:00 2001 From: Rod Vagg Date: Wed, 3 Jun 2026 16:52:05 +1000 Subject: [PATCH 2/2] fixup! feat: cap addPieces batches at 40, tighten piece metadata limits --- packages/synapse-core/src/errors/warm-storage.ts | 2 +- packages/synapse-core/src/sp/add-pieces.ts | 4 ++-- packages/synapse-core/test/add-pieces.test.ts | 6 ++++++ 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/packages/synapse-core/src/errors/warm-storage.ts b/packages/synapse-core/src/errors/warm-storage.ts index a1c821b6c..572c6f528 100644 --- a/packages/synapse-core/src/errors/warm-storage.ts +++ b/packages/synapse-core/src/errors/warm-storage.ts @@ -25,7 +25,7 @@ export class AtLeastOnePieceRequiredError extends SynapseError { export class TooManyPiecesError extends SynapseError { override name: 'TooManyPiecesError' = 'TooManyPiecesError' constructor(count: number, max: number) { - super(`Too many pieces: ${count}, max ${max} per addPieces call. Split into smaller batches.`) + super(`Too many pieces: ${count}, max ${max} per batch. Split into smaller batches.`) } static override is(value: unknown): value is TooManyPiecesError { diff --git a/packages/synapse-core/src/sp/add-pieces.ts b/packages/synapse-core/src/sp/add-pieces.ts index 9277ae8f5..734673f01 100644 --- a/packages/synapse-core/src/sp/add-pieces.ts +++ b/packages/synapse-core/src/sp/add-pieces.ts @@ -112,11 +112,11 @@ export namespace addPieces { * failing early instead of reverting on-chain. * * @param pieceCount - Number of pieces in the batch - * @throws AtLeastOnePieceRequiredError when empty + * @throws AtLeastOnePieceRequiredError when not a positive integer * @throws TooManyPiecesError when above {@link SIZE_CONSTANTS.MAX_ADD_PIECES_BATCH_SIZE} */ export function validateAddPiecesBatch(pieceCount: number): void { - if (pieceCount === 0) { + if (!Number.isInteger(pieceCount) || pieceCount < 1) { throw new AtLeastOnePieceRequiredError() } if (pieceCount > SIZE_CONSTANTS.MAX_ADD_PIECES_BATCH_SIZE) { diff --git a/packages/synapse-core/test/add-pieces.test.ts b/packages/synapse-core/test/add-pieces.test.ts index 6f7bcff7c..98309c361 100644 --- a/packages/synapse-core/test/add-pieces.test.ts +++ b/packages/synapse-core/test/add-pieces.test.ts @@ -10,6 +10,12 @@ describe('validateAddPiecesBatch', () => { assert.throws(() => validateAddPiecesBatch(0), AtLeastOnePieceRequiredError) }) + it('should throw for non-positive or non-integer counts', () => { + for (const bad of [-1, 1.5, Number.NaN, Number.POSITIVE_INFINITY]) { + assert.throws(() => validateAddPiecesBatch(bad), AtLeastOnePieceRequiredError) + } + }) + it('should accept a count at the maximum', () => { assert.doesNotThrow(() => validateAddPiecesBatch(max)) })