diff --git a/AGENTS.md b/AGENTS.md index 0021077d..66e97965 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -172,7 +172,7 @@ Ref: FRC-0069 **Internal**: `MetadataEntry[]` (alphabetically sorted for EIP-712) -**Validation**: Data sets max 10 keys, pieces max 5 keys, keys max 32 chars, values max 128 chars. Validated early in PDPServer. +**Validation**: Data sets max 10 keys, pieces max 3 keys, keys max 32 chars, values max 96 chars. Validated early in PDPServer. **Security**: Uses `Object.create(null)` for prototype-safe objects from contracts. diff --git a/docs/src/content/docs/developer-guides/storage/storage-operations.mdx b/docs/src/content/docs/developer-guides/storage/storage-operations.mdx index 469eac4a..01eea5d2 100644 --- a/docs/src/content/docs/developer-guides/storage/storage-operations.mdx +++ b/docs/src/content/docs/developer-guides/storage/storage-operations.mdx @@ -18,7 +18,7 @@ This page covers data set management, retrieval, and lifecycle operations. For u **Metadata**: Optional key-value pairs for organization: - **Data Set Metadata**: Max 10 keys (e.g., `project`, `environment`) -- **Piece Metadata**: Max 5 keys per piece (e.g., `filename`, `contentType`) +- **Piece Metadata**: Max 3 keys per piece (e.g., `filename`, `contentType`) **Copies and Durability**: By default, `upload()` stores your data with 2 independent providers. Each provider maintains its own data set with separate PDP proofs and payment rails. If one provider goes down, your data is still available from the other. diff --git a/packages/synapse-core/src/errors/warm-storage.ts b/packages/synapse-core/src/errors/warm-storage.ts index 1fcd6965..572c6f52 100644 --- a/packages/synapse-core/src/errors/warm-storage.ts +++ b/packages/synapse-core/src/errors/warm-storage.ts @@ -21,3 +21,14 @@ export class AtLeastOnePieceRequiredError extends SynapseError { return isSynapseError(value) && value.name === 'AtLeastOnePieceRequiredError' } } + +export class TooManyPiecesError extends SynapseError { + override name: 'TooManyPiecesError' = 'TooManyPiecesError' + constructor(count: number, max: number) { + super(`Too many pieces: ${count}, max ${max} per batch. Split into smaller batches.`) + } + + static override is(value: unknown): value is TooManyPiecesError { + return isSynapseError(value) && value.name === 'TooManyPiecesError' + } +} diff --git a/packages/synapse-core/src/sp/add-pieces.ts b/packages/synapse-core/src/sp/add-pieces.ts index d116c480..734673f0 100644 --- a/packages/synapse-core/src/sp/add-pieces.ts +++ b/packages/synapse-core/src/sp/add-pieces.ts @@ -4,10 +4,10 @@ import { type Account, type Chain, type Client, type Hex, isHex, type Transport import * as z from 'zod' import { AddPiecesError, LocationHeaderError } from '../errors/index.ts' import { WaitForAddPiecesError, WaitForAddPiecesRejectedError } from '../errors/pdp.ts' -import { AtLeastOnePieceRequiredError } from '../errors/warm-storage.ts' +import { AtLeastOnePieceRequiredError, TooManyPiecesError } from '../errors/warm-storage.ts' import type { PieceCID } from '../piece/piece-cid.ts' import { signAddPieces } from '../typed-data/sign-add-pieces.ts' -import { RETRY_CONSTANTS } from '../utils/constants.ts' +import { RETRY_CONSTANTS, SIZE_CONSTANTS } from '../utils/constants.ts' import { type MetadataObject, pieceMetadataObjectToEntry } from '../utils/metadata.ts' import { zHex, zNumberToBigInt } from '../utils/schemas.ts' @@ -107,6 +107,23 @@ export namespace addPieces { export type ErrorType = addPiecesApiRequest.ErrorType | signAddPieces.ErrorType } +/** + * Validate the piece count for an addPieces (or createDataSetAndAddPieces) batch, + * failing early instead of reverting on-chain. + * + * @param pieceCount - Number of pieces in the batch + * @throws AtLeastOnePieceRequiredError when not a positive integer + * @throws TooManyPiecesError when above {@link SIZE_CONSTANTS.MAX_ADD_PIECES_BATCH_SIZE} + */ +export function validateAddPiecesBatch(pieceCount: number): void { + if (!Number.isInteger(pieceCount) || pieceCount < 1) { + throw new AtLeastOnePieceRequiredError() + } + if (pieceCount > SIZE_CONSTANTS.MAX_ADD_PIECES_BATCH_SIZE) { + throw new TooManyPiecesError(pieceCount, SIZE_CONSTANTS.MAX_ADD_PIECES_BATCH_SIZE) + } +} + /** * Add pieces to a data set * @@ -121,9 +138,7 @@ export async function addPieces( client: Client, options: addPieces.OptionsType ): Promise { - if (options.pieces.length === 0) { - throw new AtLeastOnePieceRequiredError() - } + validateAddPiecesBatch(options.pieces.length) const extraData = options.extraData ?? (await signAddPieces(client, { diff --git a/packages/synapse-core/src/sp/create-dataset-add-pieces.ts b/packages/synapse-core/src/sp/create-dataset-add-pieces.ts index 051630a3..e8d25e3b 100644 --- a/packages/synapse-core/src/sp/create-dataset-add-pieces.ts +++ b/packages/synapse-core/src/sp/create-dataset-add-pieces.ts @@ -13,7 +13,7 @@ import type { PieceCID } from '../piece/piece-cid.ts' import { signCreateDataSetAndAddPieces } from '../typed-data/sign-create-dataset-add-pieces.ts' import { RETRY_CONSTANTS } from '../utils/constants.ts' import { datasetMetadataObjectToEntry, type MetadataObject, pieceMetadataObjectToEntry } from '../utils/metadata.ts' -import { waitForAddPieces } from './add-pieces.ts' +import { validateAddPiecesBatch, waitForAddPieces } from './add-pieces.ts' import { waitForCreateDataSet } from './create-dataset.ts' export namespace createDataSetAndAddPiecesApiRequest { @@ -134,6 +134,7 @@ export async function createDataSetAndAddPieces( client: Client, options: CreateDataSetAndAddPiecesOptions ): Promise { + validateAddPiecesBatch(options.pieces.length) const chain = asChain(client.chain) const extraData = options.extraData ?? diff --git a/packages/synapse-core/src/utils/constants.ts b/packages/synapse-core/src/utils/constants.ts index 6d5341d8..d321e93b 100644 --- a/packages/synapse-core/src/utils/constants.ts +++ b/packages/synapse-core/src/utils/constants.ts @@ -91,6 +91,14 @@ export const SIZE_CONSTANTS = { */ DEFAULT_UPLOAD_BATCH_SIZE: 32, + /** + * Maximum pieces per addPieces (or createDataSetAndAddPieces) call. + * + * On-chain limitations fail batch sizes above 41; we constrain to 40 here to + * catch those failures early and surface informative errors. + */ + MAX_ADD_PIECES_BATCH_SIZE: 40, + /** * Bytes per leaf in the PDP merkle tree. * The FWSS contract converts leaf counts to bytes via `totalBytes = leafCount * BYTES_PER_LEAF`. diff --git a/packages/synapse-core/src/utils/metadata.ts b/packages/synapse-core/src/utils/metadata.ts index 3395e711..86374dc0 100644 --- a/packages/synapse-core/src/utils/metadata.ts +++ b/packages/synapse-core/src/utils/metadata.ts @@ -16,9 +16,9 @@ export type MetadataObject = Record // Metadata size and count limits from the contract export const METADATA_LIMITS = { MAX_KEY_LENGTH: 32, - MAX_VALUE_LENGTH: 128, + MAX_VALUE_LENGTH: 96, MAX_KEYS_PER_DATASET: 10, - MAX_KEYS_PER_PIECE: 5, + MAX_KEYS_PER_PIECE: 3, } export function metadataArrayToObject(metadataArray: MetadataArray): Record { diff --git a/packages/synapse-core/test/add-pieces.test.ts b/packages/synapse-core/test/add-pieces.test.ts new file mode 100644 index 00000000..98309c36 --- /dev/null +++ b/packages/synapse-core/test/add-pieces.test.ts @@ -0,0 +1,26 @@ +import assert from 'assert' +import { AtLeastOnePieceRequiredError, TooManyPiecesError } from '../src/errors/warm-storage.ts' +import { validateAddPiecesBatch } from '../src/sp/add-pieces.ts' +import { SIZE_CONSTANTS } from '../src/utils/constants.ts' + +describe('validateAddPiecesBatch', () => { + const max = SIZE_CONSTANTS.MAX_ADD_PIECES_BATCH_SIZE + + it('should throw when empty', () => { + assert.throws(() => validateAddPiecesBatch(0), AtLeastOnePieceRequiredError) + }) + + it('should throw for non-positive or non-integer counts', () => { + for (const bad of [-1, 1.5, Number.NaN, Number.POSITIVE_INFINITY]) { + assert.throws(() => validateAddPiecesBatch(bad), AtLeastOnePieceRequiredError) + } + }) + + it('should accept a count at the maximum', () => { + assert.doesNotThrow(() => validateAddPiecesBatch(max)) + }) + + it('should throw when above the maximum', () => { + assert.throws(() => validateAddPiecesBatch(max + 1), TooManyPiecesError) + }) +}) diff --git a/packages/synapse-core/test/metadata.test.ts b/packages/synapse-core/test/metadata.test.ts index 064d8ece..a6853379 100644 --- a/packages/synapse-core/test/metadata.test.ts +++ b/packages/synapse-core/test/metadata.test.ts @@ -235,9 +235,9 @@ describe('Metadata Utils', () => { describe('METADATA_LIMITS', () => { it('should have expected limit values', () => { assert.equal(METADATA_LIMITS.MAX_KEY_LENGTH, 32) - assert.equal(METADATA_LIMITS.MAX_VALUE_LENGTH, 128) + assert.equal(METADATA_LIMITS.MAX_VALUE_LENGTH, 96) assert.equal(METADATA_LIMITS.MAX_KEYS_PER_DATASET, 10) - assert.equal(METADATA_LIMITS.MAX_KEYS_PER_PIECE, 5) + assert.equal(METADATA_LIMITS.MAX_KEYS_PER_PIECE, 3) }) }) }) diff --git a/packages/synapse-sdk/src/storage/context.ts b/packages/synapse-sdk/src/storage/context.ts index 289d83b3..6a80beff 100644 --- a/packages/synapse-sdk/src/storage/context.ts +++ b/packages/synapse-sdk/src/storage/context.ts @@ -703,6 +703,7 @@ export class StorageContext { * @returns Signed extraData hex to pass to pull() or commit() */ async presignForCommit(pieces: Array<{ pieceCid: PieceCID; pieceMetadata?: MetadataObject }>): Promise { + SP.validateAddPiecesBatch(pieces.length) const signingPieces = pieces.map((p) => ({ pieceCid: p.pieceCid, metadata: pieceMetadataObjectToEntry(p.pieceMetadata), @@ -738,6 +739,10 @@ export class StorageContext { async pull(options: PullOptions): Promise { const { pieces, from, signal, onProgress, extraData } = options + // The SP estimateGas-validates the eventual addPieces, so an oversized batch + // fails there too; reject early for a clear error on non-presigned paths. + SP.validateAddPiecesBatch(pieces.length) + const getSourceUrl = (pieceCid: PieceCID): string => { if (typeof from === 'string') { return createPieceUrlPDP({ cid: pieceCid.toString(), serviceURL: from }) @@ -820,7 +825,8 @@ export class StorageContext { async commit(options: CommitOptions): Promise { const { pieces, extraData } = options - // Validate metadata early + // Validate batch size and metadata early, before any chain reads or signing + SP.validateAddPiecesBatch(pieces.length) for (const piece of pieces) { if (piece.pieceMetadata) { pieceMetadataObjectToEntry(piece.pieceMetadata) diff --git a/packages/synapse-sdk/src/test/storage.test.ts b/packages/synapse-sdk/src/test/storage.test.ts index 7b3d6197..1757e333 100644 --- a/packages/synapse-sdk/src/test/storage.test.ts +++ b/packages/synapse-sdk/src/test/storage.test.ts @@ -1,4 +1,5 @@ import { type Chain, calibration } from '@filoz/synapse-core/chains' +import { TooManyPiecesError } from '@filoz/synapse-core/errors' import * as Mocks from '@filoz/synapse-core/mocks' import * as Piece from '@filoz/synapse-core/piece' import { calculate, calculate as calculatePieceCID } from '@filoz/synapse-core/piece' @@ -892,6 +893,53 @@ describe('StorageService', () => { }) }) + describe('addPieces batch limit', () => { + const tooMany = SIZE_CONSTANTS.MAX_ADD_PIECES_BATCH_SIZE + 1 + + async function makeContext() { + server.use(Mocks.JSONRPC({ ...Mocks.presets.basic }), Mocks.PING()) + const synapse = new Synapse({ client, source: null }) + const warmStorageService = new WarmStorageService({ client }) + return StorageContext.create({ synapse, warmStorageService }) + } + + it('presignForCommit rejects batches above the limit', async () => { + const service = await makeContext() + const pieceCid = await calculate(new Uint8Array(127).fill(1)) + const pieces = Array.from({ length: tooMany }, () => ({ pieceCid })) + try { + await service.presignForCommit(pieces) + assert.fail('Should have thrown') + } catch (error) { + assert.instanceOf(error, TooManyPiecesError) + } + }) + + it('commit rejects batches above the limit', async () => { + const service = await makeContext() + const pieceCid = await calculate(new Uint8Array(127).fill(1)) + const pieces = Array.from({ length: tooMany }, () => ({ pieceCid })) + try { + await service.commit({ pieces }) + assert.fail('Should have thrown') + } catch (error) { + assert.instanceOf(error, TooManyPiecesError) + } + }) + + it('pull rejects batches above the limit', async () => { + const service = await makeContext() + const pieceCid = await calculate(new Uint8Array(127).fill(1)) + const pieces = Array.from({ length: tooMany }, () => pieceCid) + try { + await service.pull({ pieces, from: 'https://pdp.example.com' }) + assert.fail('Should have thrown') + } catch (error) { + assert.instanceOf(error, TooManyPiecesError) + } + }) + }) + describe('upload', () => { it('should handle errors in batch processing gracefully', async () => { server.use(