From a4a3465e7a8dad106694dfbb49c464b41d26a9ed Mon Sep 17 00:00:00 2001 From: "${Mr.DJA}" <42304709+iMrDJAi@users.noreply.github.com> Date: Sat, 8 Nov 2025 22:14:22 +0100 Subject: [PATCH 01/10] =?UTF-8?q?=E2=9C=A8=20Added=20onKey()=20and=20onVal?= =?UTF-8?q?ue()=20callbacks?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/CBORDecoderStream.ts | 5 +- src/Decoder.ts | 198 +++++++++++++++++++++------ src/Encoder.ts | 40 +++++- src/decodeAsyncIterable.ts | 265 ++++++++++++++++++++++++++++++------- src/decodeIterable.ts | 240 ++++++++++++++++++++++++++------- src/index.ts | 2 +- src/options.ts | 44 ++++++ 7 files changed, 649 insertions(+), 145 deletions(-) diff --git a/src/CBORDecoderStream.ts b/src/CBORDecoderStream.ts index dfafe0e..75beeba 100644 --- a/src/CBORDecoderStream.ts +++ b/src/CBORDecoderStream.ts @@ -1,10 +1,9 @@ -import { Decoder } from "./decodeAsyncIterable.js" -import { DecodeOptions } from "./options.js" +import { Decoder, type AsyncDecodeOptions } from "./decodeAsyncIterable.js" import { CBORValue } from "./types.js" /** Decode a Web Streams API ReadableStream */ export class CBORDecoderStream extends TransformStream { - constructor(options: DecodeOptions = {}) { + constructor(options: AsyncDecodeOptions = {}) { let readableController: ReadableStreamDefaultController const readable = new ReadableStream({ diff --git a/src/Decoder.ts b/src/Decoder.ts index bdadd2d..9f5d933 100644 --- a/src/Decoder.ts +++ b/src/Decoder.ts @@ -1,6 +1,6 @@ import { getFloat16 } from "fp16" -import type { CBORValue } from "./types.js" +import type { CBORValue, CBORArray, CBORMap } from "./types.js" import type { DecodeOptions, FloatSize } from "./options.js" import { UnsafeIntegerError, maxSafeInteger, minSafeInteger } from "./utils.js" @@ -8,8 +8,21 @@ export class Decoder { public readonly allowUndefined: boolean public readonly minFloatSize: (typeof FloatSize)[keyof typeof FloatSize] + private readonly decoder = new TextDecoder() + private readonly onKey?: (decodeKey: () => string, length: number) => string|void + private readonly onValue?: ( + decodeValue: () => CBORValue, + length: number, + type: string, + keyPath: (string|number)[] + ) => CBORValue|void + #offset: number #view: DataView + #env: { + isKey: boolean + keyPath: (string|number)[] + } public constructor( private readonly data: Uint8Array, @@ -17,14 +30,25 @@ export class Decoder { ) { this.#offset = 0 this.#view = new DataView(data.buffer, data.byteOffset, data.byteLength) + this.#env = { isKey: false, keyPath: [] } this.allowUndefined = options.allowUndefined ?? true this.minFloatSize = options.minFloatSize ?? 16 + this.onKey = options.onKey + this.onValue = options.onValue } public getOffset(): number { return this.#offset } + private pushKey(key: string|number) { + this.#env.keyPath.push(key) + } + + private popKey() { + this.#env.keyPath.pop() + } + private constant = (size: number, f: () => T) => () => { @@ -49,7 +73,7 @@ export class Decoder { } private decodeString(length: number): string { - const value = new TextDecoder().decode(this.data.subarray(this.#offset, this.#offset + length)) + const value = this.decoder.decode(this.data.subarray(this.#offset, this.#offset + length)) this.#offset += length return value } @@ -57,19 +81,20 @@ export class Decoder { private getArgument(additionalInformation: number): { value: number uint64?: bigint + size: number } { if (additionalInformation < 24) { - return { value: additionalInformation } + return { value: additionalInformation, size: 1 } } else if (additionalInformation === 24) { - return { value: this.uint8() } + return { value: this.uint8(), size: 1 } } else if (additionalInformation === 25) { - return { value: this.uint16() } + return { value: this.uint16(), size: 2 } } else if (additionalInformation === 26) { - return { value: this.uint32() } + return { value: this.uint32(), size: 4 } } else if (additionalInformation === 27) { const uint64 = this.uint64() const value = maxSafeInteger < uint64 ? Infinity : Number(uint64) - return { value, uint64 } + return { value, uint64, size: 8 } } else if (additionalInformation === 31) { throw new Error("microcbor does not support decoding indefinite-length items") } else { @@ -81,77 +106,132 @@ export class Decoder { const initialByte = this.uint8() const majorType = initialByte >> 5 const additionalInformation = initialByte & 0x1f + const { isKey, keyPath } = this.#env if (majorType === 0) { - const { value, uint64 } = this.getArgument(additionalInformation) + const { value, uint64, size } = this.getArgument(additionalInformation) if (uint64 !== undefined && maxSafeInteger < uint64) { throw new UnsafeIntegerError("cannot decode integers greater than 2^53-1", uint64) - } else { - return value } + const val = this.onValue?.(() => value, size, "number", keyPath) + return val === undefined ? value : val } else if (majorType === 1) { - const { value, uint64 } = this.getArgument(additionalInformation) + const { value, uint64, size } = this.getArgument(additionalInformation) if (uint64 !== undefined && -1n - uint64 < minSafeInteger) { throw new UnsafeIntegerError("cannot decode integers less than -2^53+1", -1n - uint64) - } else { - return -1 - value } + const val = this.onValue?.(() => (-1 - value), size, "number", keyPath) + return val === undefined ? (-1 - value) : val } else if (majorType === 2) { const { value: length } = this.getArgument(additionalInformation) - return this.decodeBytes(length) + let value: CBORValue + const callback = () => ( + value = (value === undefined ? this.decodeBytes(length) : value) as Uint8Array + ) + const val = this.onValue?.(callback, length, "Uint8Array", keyPath) + if (val !== undefined) { + if (value === undefined) this.#offset += length + return val + } + return callback() } else if (majorType === 3) { const { value: length } = this.getArgument(additionalInformation) - return this.decodeString(length) + let value: CBORValue, val + const callback = () => ( + value = (value === undefined ? this.decodeString(length) : value) as string + ) + if (isKey) val = this.onKey?.(callback, length) + else val = this.onValue?.(callback, length, "string", keyPath) + if (val !== undefined) { + if (value === undefined) this.#offset += length + return val + } + return callback() } else if (majorType === 4) { const { value: length } = this.getArgument(additionalInformation) - const value = new Array(length) - for (let i = 0; i < length; i++) { - value[i] = this.decodeValue() + let value: CBORValue + const callback = () => { + if (value !== undefined) return value as CBORArray + value = new Array(length) + for (let i = 0; i < length; i++) { + this.pushKey(i) + value[i] = this.decodeValue() + this.popKey() + } + return value } - return value + const val = this.onValue?.(callback, length, "array", keyPath) + if (val !== undefined) { + if (value !== undefined) return val + for (let i = 0; i < length; i++) this.skipValue() + return val + } + return callback() } else if (majorType === 5) { const { value: length } = this.getArgument(additionalInformation) - const value: Record = {} - for (let i = 0; i < length; i++) { - const key = this.decodeValue() - if (typeof key !== "string") { - throw new Error("microcbor only supports string keys in objects") + let value: CBORValue|void + const callback = () => { + if (value !== undefined) return value as CBORMap + value = {} + for (let i = 0; i < length; i++) { + this.#env.isKey = true + const key = this.decodeValue() + this.#env.isKey = false + if (typeof key !== "string") { + throw new Error("microcbor only supports string keys in objects") + } + this.pushKey(key) + value[key] = this.decodeValue() + this.popKey() } - value[key] = this.decodeValue() + return value } - return value + const val = this.onValue?.(callback, length, "object", keyPath) + if (val !== undefined) { + if (value !== undefined) return val + for (let i = 0; i < length * 2; i++) this.skipValue() + return val + } + return callback() } else if (majorType === 6) { throw new Error("microcbor does not support tagged data items") } else if (majorType === 7) { + let val switch (additionalInformation) { case 20: - return false + val = this.onValue?.(() => false, 1, "boolean", keyPath) + return val === undefined ? false : val case 21: - return true + val = this.onValue?.(() => true, 1, "boolean", keyPath) + return val === undefined ? true : val case 22: - return null + val = this.onValue?.(() => null, 1, "null", keyPath) + return val === undefined ? null : val case 23: - if (this.allowUndefined) { - return undefined - } else { - throw new TypeError("`undefined` not allowed") - } + if (!this.allowUndefined) throw new TypeError("`undefined` not allowed") + return this.onValue?.(() => undefined, 1, "undefined", keyPath) as CBORValue case 24: throw new Error("microcbor does not support decoding unassigned simple values") case 25: if (this.minFloatSize <= 16) { - return this.float16() + const value = this.float16() + val = this.onValue?.(() => value, 2, "number", keyPath) + return val === undefined ? value : val } else { throw new Error("cannot decode float16 type - below provided minFloatSize") } case 26: if (this.minFloatSize <= 32) { - return this.float32() + const value = this.float32() + val = this.onValue?.(() => value, 4, "number", keyPath) + return val === undefined ? value : val } else { throw new Error("cannot decode float32 type - below provided minFloatSize") } case 27: - return this.float64() + const value = this.float64() + val = this.onValue?.(() => value, 8, "number", keyPath) + return val === undefined ? value : val case 31: throw new Error("microcbor does not support decoding indefinite-length items") default: @@ -161,6 +241,50 @@ export class Decoder { throw new Error("invalid major type") } } + + private skipValue() { + const initialByte = this.uint8() + const majorType = initialByte >> 5 + const additionalInformation = initialByte & 0x1f + + if (majorType === 0 || majorType === 1) { + this.getArgument(additionalInformation) + } else if (majorType === 2 || majorType === 3) { + const { value: length } = this.getArgument(additionalInformation) + this.#offset += length + } else if (majorType === 4) { + const { value: length } = this.getArgument(additionalInformation) + for (let i = 0; i < length; i++) this.skipValue() + } else if (majorType === 5) { + const { value: length } = this.getArgument(additionalInformation) + for (let i = 0; i < length * 2; i++) this.skipValue() + } else if (majorType === 6) { + throw new Error("microcbor does not support tagged data items") + } else if (majorType === 7) { + switch (additionalInformation) { + case 20: case 21: case 22: + break + case 23: + if (!this.allowUndefined) throw new TypeError("`undefined` not allowed") + break + case 24: + throw new Error("microcbor does not support decoding unassigned simple values") + case 25: + this.#offset += 2 + break + case 26: + this.#offset += 4 + break + case 27: + this.#offset += 8 + break + case 31: + throw new Error("microcbor does not support decoding indefinite-length items") + default: + throw new Error("invalid simple value") + } + } + } } /** Decode a single CBOR value */ diff --git a/src/Encoder.ts b/src/Encoder.ts index 08d3ab8..3fbd156 100644 --- a/src/Encoder.ts +++ b/src/Encoder.ts @@ -13,11 +13,16 @@ export class Encoder { public readonly chunkSize: number public readonly minFloatSize: (typeof FloatSize)[keyof typeof FloatSize] + private readonly onKey?: (key: string) => string|void + private readonly onValue?: (value: unknown, keyPath: (string|number)[]) => CBORValue|void private readonly encoder = new TextEncoder() private readonly buffer: ArrayBuffer private readonly view: DataView private readonly array: Uint8Array private offset: number + #env: { + keyPath: (string|number)[] + } constructor(options: EncodeOptions = {}) { this.allowUndefined = options.allowUndefined ?? true @@ -25,18 +30,29 @@ export class Encoder { this.chunkRecycling = options.chunkRecycling ?? false this.chunkSize = options.chunkSize ?? Encoder.defaultChunkSize assert(this.chunkSize >= 8, "expected chunkSize >= 8") + this.onKey = options.onKey + this.onValue = options.onValue this.buffer = new ArrayBuffer(this.chunkSize) this.view = new DataView(this.buffer) this.array = new Uint8Array(this.buffer, 0, this.chunkSize) this.offset = 0 this.#closed = false + this.#env = { keyPath: [] } } public get closed() { return this.#closed } + private pushKey(key: string|number) { + this.#env.keyPath.push(key) + } + + private popKey() { + this.#env.keyPath.pop() + } + #flush(): Uint8Array { if (this.chunkRecycling) { const chunk = new Uint8Array(this.buffer, 0, this.offset) @@ -214,19 +230,28 @@ export class Encoder { yield* this.encodeBytes(value) } else if (Array.isArray(value)) { yield* this.encodeTypeAndArgument(4, value.length) - for (const element of value) { - yield* this.encodeValue(element) + for (let i = 0; i < value.length; i++) { + this.pushKey(i) + const val = this.onValue?.(value[i], this.#env.keyPath) + yield* this.encodeValue(val === undefined ? value[i] : val) + this.popKey() } } else { const entries = Object.entries(value) - .map<[Uint8Array, CBORValue]>(([key, value]) => [this.encoder.encode(key), value]) + .map<[Uint8Array, CBORValue, string]>(([ogKey, value]) => { + let key = this.onKey?.(ogKey) + return [this.encoder.encode(key === undefined ? ogKey : key + ''), value, ogKey] + }) .sort(Encoder.compareEntries) yield* this.encodeTypeAndArgument(5, entries.length) - for (const [key, value] of entries) { + for (const [key, value, ogKey] of entries) { yield* this.encodeTypeAndArgument(3, key.byteLength) yield* this.writeBytes(key) - yield* this.encodeValue(value) + this.pushKey(ogKey) + const val = this.onValue?.(value, this.#env.keyPath) + yield* this.encodeValue(val === undefined ? value : val) + this.popKey() } } } @@ -249,7 +274,10 @@ export class Encoder { // with a longer length, since strings are encoded with a length // prefix (either in the additionalInformation bits, if < 24, or // in the next serveral bytes, but in all cases the order holds). - private static compareEntries([a]: [key: Uint8Array, value: CBORValue], [b]: [key: Uint8Array, value: CBORValue]) { + private static compareEntries( + [a]: [key: Uint8Array, value: CBORValue, ogKey: string], + [b]: [key: Uint8Array, value: CBORValue, ogKey: string] + ) { if (a.byteLength < b.byteLength) return -1 if (b.byteLength < a.byteLength) return 1 diff --git a/src/decodeAsyncIterable.ts b/src/decodeAsyncIterable.ts index 55e95a4..dbb663b 100644 --- a/src/decodeAsyncIterable.ts +++ b/src/decodeAsyncIterable.ts @@ -1,11 +1,43 @@ import { getFloat16 } from "fp16" -import type { CBORValue } from "./types.js" +import type { CBORValue, CBORArray, CBORMap } from "./types.js" import { UnsafeIntegerError, maxSafeInteger, minSafeInteger } from "./utils.js" import { DecodeOptions, FloatSize } from "./options.js" -export interface AsyncDecodeOptions extends DecodeOptions { +type Awaitable = T | PromiseLike + +export interface AsyncDecodeOptions extends Omit { + /** + * Function to remap/validate object keys while decoding + * (async version that works with AsyncIterable and streams) + * @param decodeKey Function to decode original object key (async) + * @param length Key length to validate pre-decoding + * @throws Error if length/key is invalid + * @returns An optional replacement key string + */ + onKey?: ( + decodeKey: () => Awaitable, + length: number + ) => Promise + + /** + * Function to validate/transform/replace values while decoding + * (async version that works with AsyncIterable and streams) + * @param decodeValue Function to decode value (async) + * @param length Value length/size to validate pre-decoding + * @param type Value type (e.g. 'number', 'string', 'Uint8Array'...) + * @param keyPath Array of keys describing the access path to this value + * @throws Error if length/value is invalid + * @returns An optional replacement value to use + */ + onValue?: ( + decodeValue: () => Awaitable, + length: number, + type: string, + keyPath: (string|number)[] + ) => Promise + onFree?: (chunk: Uint8Array) => void } @@ -15,17 +47,34 @@ export class Decoder implements AsyncIterableIt private offset = 0 private byteLength = 0 + private readonly decoder = new TextDecoder() private readonly chunks: Uint8Array[] = [] private readonly constantBuffer = new ArrayBuffer(8) private readonly constantView = new DataView(this.constantBuffer) private readonly iter: AsyncIterator private readonly onFree?: (chunk: Uint8Array) => void + private readonly onKey?: ( + decodeKey: () => Awaitable, + length: number + ) => Promise + private readonly onValue?: ( + decodeValue: () => Awaitable, + length: number, + type: string, + keyPath: (string|number)[] + ) => Promise + private env: { + isKey: boolean + keyPath: (string|number)[] + } = { isKey: false, keyPath: [] } public constructor(source: AsyncIterable, options: AsyncDecodeOptions = {}) { this.onFree = options.onFree this.allowUndefined = options.allowUndefined ?? true this.minFloatSize = options.minFloatSize ?? 16 this.iter = source[Symbol.asyncIterator]() + this.onKey = options.onKey + this.onValue = options.onValue } [Symbol.asyncIterator] = () => this @@ -42,27 +91,27 @@ export class Decoder implements AsyncIterableIt } } - private fill(target: Uint8Array) { - if (this.byteLength < target.byteLength) { + private advance(length: number, target?: Uint8Array) { + if (this.byteLength < length) { throw new Error("internal error - please file a bug report!") } let byteLength = 0 let deleteCount = 0 - for (let i = 0; byteLength < target.byteLength; i++) { + for (let i = 0; byteLength < length; i++) { const chunk = this.chunks[i] - const capacity = target.byteLength - byteLength - const length = chunk.byteLength - this.offset - if (length <= capacity) { + const capacity = length - byteLength + const available = chunk.byteLength - this.offset + if (available <= capacity) { // copy the entire remainder of the chunk - target.set(chunk.subarray(this.offset), byteLength) - byteLength += length + target?.set(chunk.subarray(this.offset), byteLength) + byteLength += available deleteCount += 1 this.offset = 0 - this.byteLength -= length + this.byteLength -= available } else { // fill the remainder of the target - target.set(chunk.subarray(this.offset, this.offset + capacity), byteLength) + target?.set(chunk.subarray(this.offset, this.offset + capacity), byteLength) byteLength += capacity // equivalent to break this.offset += capacity @@ -79,6 +128,18 @@ export class Decoder implements AsyncIterableIt this.chunks.splice(0, deleteCount) } + private fill(target: Uint8Array) { + this.advance(target.byteLength, target) + } + + private pushKey(key: string|number) { + this.env.keyPath.push(key) + } + + private popKey() { + this.env.keyPath.pop() + } + private constant = (size: number, f: (view: DataView) => T) => { return async () => { await this.allocate(size) @@ -107,25 +168,26 @@ export class Decoder implements AsyncIterableIt await this.allocate(length) const data = new Uint8Array(length) this.fill(data) - return new TextDecoder().decode(data) + return this.decoder.decode(data) } private async getArgument(additionalInformation: number): Promise<{ value: number uint64?: bigint + size: number }> { if (additionalInformation < 24) { - return { value: additionalInformation } + return { value: additionalInformation, size: 1 } } else if (additionalInformation === 24) { - return { value: await this.uint8() } + return { value: await this.uint8(), size: 1 } } else if (additionalInformation === 25) { - return { value: await this.uint16() } + return { value: await this.uint16(), size: 2 } } else if (additionalInformation === 26) { - return { value: await this.uint32() } + return { value: await this.uint32(), size: 4 } } else if (additionalInformation === 27) { const uint64 = await this.uint64() const value = maxSafeInteger < uint64 ? Infinity : Number(uint64) - return { value, uint64 } + return { value, uint64, size: 8 } } else if (additionalInformation === 31) { throw new Error("microcbor does not support decoding indefinite-length items") } else { @@ -152,77 +214,136 @@ export class Decoder implements AsyncIterableIt const initialByte = await this.uint8() const majorType = initialByte >> 5 const additionalInformation = initialByte & 0x1f + const { isKey, keyPath } = this.env if (majorType === 0) { - const { value, uint64 } = await this.getArgument(additionalInformation) + const { value, uint64, size } = await this.getArgument(additionalInformation) if (uint64 !== undefined && maxSafeInteger < uint64) { throw new UnsafeIntegerError("cannot decode integers greater than 2^53-1", uint64) - } else { - return value } + const val = await this.onValue?.(() => value, size, "number", keyPath) + return val === undefined ? value : val } else if (majorType === 1) { - const { value, uint64 } = await this.getArgument(additionalInformation) + const { value, uint64, size } = await this.getArgument(additionalInformation) if (uint64 !== undefined && -1n - uint64 < minSafeInteger) { throw new UnsafeIntegerError("cannot decode integers less than -2^53+1", -1n - uint64) - } else { - return -1 - value } + const val = await this.onValue?.(() => (-1 - value), size, "number", keyPath) + return val === undefined ? (-1 - value) : val } else if (majorType === 2) { const { value: length } = await this.getArgument(additionalInformation) - return await this.decodeBytes(length) + let value: CBORValue + const callback = async () => ( + value = (value === undefined ? await this.decodeBytes(length) : value) as Uint8Array + ) + const val = await this.onValue?.(callback, length, "Uint8Array", keyPath) + if (val !== undefined) { + if (value !== undefined) return val + await this.allocate(length) + this.advance(length) + return val + } + return callback() } else if (majorType === 3) { const { value: length } = await this.getArgument(additionalInformation) - return await this.decodeString(length) + let value: CBORValue, val + const callback = async () => ( + value = (value === undefined ? await this.decodeString(length) : value) as string + ) + if (isKey) val = await this.onKey?.(callback, length) + else val = await this.onValue?.(callback, length, "string", keyPath) + if (val !== undefined) { + if (value !== undefined) return val + await this.allocate(length) + this.advance(length) + return val + } + return callback() } else if (majorType === 4) { const { value: length } = await this.getArgument(additionalInformation) - const value = new Array(length) - for (let i = 0; i < length; i++) { - value[i] = await this.decodeValue() + let value: CBORValue + const callback = async () => { + if (value !== undefined) return value as CBORArray + value = new Array(length) + for (let i = 0; i < length; i++) { + this.pushKey(i) + value[i] = await this.decodeValue() + this.popKey() + } + return value + } + const val = await this.onValue?.(callback, length, "array", keyPath) + if (val !== undefined) { + if (value !== undefined) return val + for (let i = 0; i < length; i++) await this.skipValue() + return val } - return value + return callback() } else if (majorType === 5) { const { value: length } = await this.getArgument(additionalInformation) - const value: Record = {} - for (let i = 0; i < length; i++) { - const key = await this.decodeValue() - if (typeof key !== "string") { - throw new Error("microcbor only supports string keys in objects") + let value: CBORValue|void + const callback = async () => { + if (value !== undefined) return value as CBORMap + value = {} + for (let i = 0; i < length; i++) { + this.env.isKey = true + const key = await this.decodeValue() + this.env.isKey = false + if (typeof key !== "string") { + throw new Error("microcbor only supports string keys in objects") + } + this.pushKey(key) + value[key] = await this.decodeValue() + this.popKey() } - value[key] = await this.decodeValue() + return value + } + const val = await this.onValue?.(callback, length, "object", keyPath) + if (val !== undefined) { + if (value !== undefined) return val + for (let i = 0; i < length * 2; i++) await this.skipValue() + return val } - return value + return callback() } else if (majorType === 6) { throw new Error("microcbor does not support tagged data items") } else if (majorType === 7) { + let val switch (additionalInformation) { case 20: - return false + val = await this.onValue?.(() => false, 1, "boolean", keyPath) + return val === undefined ? false : val case 21: - return true + val = await this.onValue?.(() => true, 1, "boolean", keyPath) + return val === undefined ? true : val case 22: - return null + val = await this.onValue?.(() => null, 1, "null", keyPath) + return val === undefined ? null : val case 23: - if (this.allowUndefined) { - return undefined - } else { - throw new TypeError("`undefined` not allowed") - } + if (!this.allowUndefined) throw new TypeError("`undefined` not allowed") + return await this.onValue?.(() => undefined, 1, "undefined", keyPath) as CBORValue case 24: throw new Error("microcbor does not support decoding unassigned simple values") case 25: if (this.minFloatSize <= 16) { - return this.float16() + const value = await this.float16() + val = await this.onValue?.(() => value, 2, "number", keyPath) + return val === undefined ? value : val } else { throw new Error("cannot decode float16 type - below provided minFloatSize") } case 26: if (this.minFloatSize <= 32) { - return this.float32() + const value = await this.float32() + val = await this.onValue?.(() => value, 4, "number", keyPath) + return val === undefined ? value : val } else { throw new Error("cannot decode float32 type - below provided minFloatSize") } case 27: - return await this.float64() + const value = await this.float64() + val = await this.onValue?.(() => value, 8, "number", keyPath) + return val === undefined ? value : val case 31: throw new Error("microcbor does not support decoding indefinite-length items") default: @@ -232,6 +353,54 @@ export class Decoder implements AsyncIterableIt throw new Error("invalid major type") } } + + private async skipValue() { + const initialByte = await this.uint8() + const majorType = initialByte >> 5 + const additionalInformation = initialByte & 0x1f + + if (majorType === 0 || majorType === 1) { + await this.getArgument(additionalInformation) + } else if (majorType === 2 || majorType === 3) { + const { value: length } = await this.getArgument(additionalInformation) + await this.allocate(length) + this.advance(length) + } else if (majorType === 4) { + const { value: length } = await this.getArgument(additionalInformation) + for (let i = 0; i < length; i++) await this.skipValue() + } else if (majorType === 5) { + const { value: length } = await this.getArgument(additionalInformation) + for (let i = 0; i < length * 2; i++) await this.skipValue() + } else if (majorType === 6) { + throw new Error("microcbor does not support tagged data items") + } else if (majorType === 7) { + switch (additionalInformation) { + case 20: case 21: case 22: + break + case 23: + if (!this.allowUndefined) throw new TypeError("`undefined` not allowed") + break + case 24: + throw new Error("microcbor does not support decoding unassigned simple values") + case 25: + await this.allocate(2) + this.advance(2) + break + case 26: + await this.allocate(4) + this.advance(4) + break + case 27: + await this.allocate(8) + this.advance(8) + break + case 31: + throw new Error("microcbor does not support decoding indefinite-length items") + default: + throw new Error("invalid simple value") + } + } + } } /** Decode an async iterable of Uint8Array chunks into an async iterable of CBOR values */ diff --git a/src/decodeIterable.ts b/src/decodeIterable.ts index 0099d1e..c677eed 100644 --- a/src/decodeIterable.ts +++ b/src/decodeIterable.ts @@ -1,6 +1,6 @@ import { getFloat16 } from "fp16" -import type { CBORValue } from "./types.js" +import type { CBORValue, CBORArray, CBORMap } from "./types.js" import type { DecodeOptions, FloatSize } from "./options.js" import { UnsafeIntegerError, maxSafeInteger, minSafeInteger } from "./utils.js" @@ -10,15 +10,29 @@ export class Decoder implements IterableIterato private offset = 0 private byteLength = 0 + private readonly decoder = new TextDecoder() private readonly chunks: Uint8Array[] = [] private readonly constantBuffer = new ArrayBuffer(8) private readonly constantView = new DataView(this.constantBuffer) private readonly iter: Iterator + private readonly onKey?: (decodeKey: () => string, length: number) => string|void + private readonly onValue?: ( + decodeValue: () => CBORValue, + length: number, + type: string, + keyPath: (string|number)[] + ) => CBORValue|void + private env: { + isKey: boolean + keyPath: (string|number)[] + } = { isKey: false, keyPath: [] } public constructor(source: Iterable, options: DecodeOptions = {}) { this.allowUndefined = options.allowUndefined ?? true this.minFloatSize = options.minFloatSize ?? 16 this.iter = source[Symbol.iterator]() + this.onKey = options.onKey + this.onValue = options.onValue } [Symbol.iterator] = () => this @@ -35,27 +49,27 @@ export class Decoder implements IterableIterato } } - private fill(target: Uint8Array) { - if (this.byteLength < target.byteLength) { + private advance(length: number, target?: Uint8Array) { + if (this.byteLength < length) { throw new Error("internal error - please file a bug report!") } let byteLength = 0 let deleteCount = 0 - for (let i = 0; byteLength < target.byteLength; i++) { + for (let i = 0; byteLength < length; i++) { const chunk = this.chunks[i] - const capacity = target.byteLength - byteLength - const length = chunk.byteLength - this.offset - if (length <= capacity) { + const capacity = length - byteLength + const available = chunk.byteLength - this.offset + if (available <= capacity) { // copy the entire remainder of the chunk - target.set(chunk.subarray(this.offset), byteLength) - byteLength += length + target?.set(chunk.subarray(this.offset), byteLength) + byteLength += available deleteCount += 1 this.offset = 0 - this.byteLength -= length + this.byteLength -= available } else { // fill the remainder of the target - target.set(chunk.subarray(this.offset, this.offset + capacity), byteLength) + target?.set(chunk.subarray(this.offset, this.offset + capacity), byteLength) byteLength += capacity // equivalent to break this.offset += capacity @@ -66,6 +80,18 @@ export class Decoder implements IterableIterato this.chunks.splice(0, deleteCount) } + private fill(target: Uint8Array) { + this.advance(target.byteLength, target) + } + + private pushKey(key: string|number) { + this.env.keyPath.push(key) + } + + private popKey() { + this.env.keyPath.pop() + } + private constant = (size: number, f: (view: DataView) => T) => { return () => { this.allocate(size) @@ -94,22 +120,26 @@ export class Decoder implements IterableIterato this.allocate(length) const data = new Uint8Array(length) this.fill(data) - return new TextDecoder().decode(data) + return this.decoder.decode(data) } - private getArgument(additionalInformation: number): { value: number; uint64?: bigint } { + private getArgument(additionalInformation: number): { + value: number + uint64?: bigint + size: number + } { if (additionalInformation < 24) { - return { value: additionalInformation } + return { value: additionalInformation, size: 1 } } else if (additionalInformation === 24) { - return { value: this.uint8() } + return { value: this.uint8(), size: 1 } } else if (additionalInformation === 25) { - return { value: this.uint16() } + return { value: this.uint16(), size: 2 } } else if (additionalInformation === 26) { - return { value: this.uint32() } + return { value: this.uint32(), size: 4 } } else if (additionalInformation === 27) { const uint64 = this.uint64() const value = maxSafeInteger < uint64 ? Infinity : Number(uint64) - return { value, uint64 } + return { value, uint64, size: 8 } } else if (additionalInformation === 31) { throw new Error("microcbor does not support decoding indefinite-length items") } else { @@ -136,77 +166,136 @@ export class Decoder implements IterableIterato const initialByte = this.uint8() const majorType = initialByte >> 5 const additionalInformation = initialByte & 0x1f + const { isKey, keyPath } = this.env if (majorType === 0) { - const { value, uint64 } = this.getArgument(additionalInformation) + const { value, uint64, size } = this.getArgument(additionalInformation) if (uint64 !== undefined && maxSafeInteger < uint64) { throw new UnsafeIntegerError("cannot decode integers greater than 2^53-1", uint64) - } else { - return value } + const val = this.onValue?.(() => value, size, "number", keyPath) + return val === undefined ? value : val } else if (majorType === 1) { - const { value, uint64 } = this.getArgument(additionalInformation) + const { value, uint64, size } = this.getArgument(additionalInformation) if (uint64 !== undefined && -1n - uint64 < minSafeInteger) { throw new UnsafeIntegerError("cannot decode integers less than -2^53+1", -1n - uint64) - } else { - return -1 - value } + const val = this.onValue?.(() => (-1 - value), size, "number", keyPath) + return val === undefined ? (-1 - value) : val } else if (majorType === 2) { const { value: length } = this.getArgument(additionalInformation) - return this.decodeBytes(length) + let value: CBORValue + const callback = () => ( + value = (value === undefined ? this.decodeBytes(length) : value) as Uint8Array + ) + const val = this.onValue?.(callback, length, "Uint8Array", keyPath) + if (val !== undefined) { + if (value !== undefined) return val + this.allocate(length) + this.advance(length) + return val + } + return callback() } else if (majorType === 3) { const { value: length } = this.getArgument(additionalInformation) - return this.decodeString(length) + let value: CBORValue, val + const callback = () => ( + value = (value === undefined ? this.decodeString(length) : value) as string + ) + if (isKey) val = this.onKey?.(callback, length) + else val = this.onValue?.(callback, length, "string", keyPath) + if (val !== undefined) { + if (value !== undefined) return val + this.allocate(length) + this.advance(length) + return val + } + return callback() } else if (majorType === 4) { const { value: length } = this.getArgument(additionalInformation) - const value = new Array(length) - for (let i = 0; i < length; i++) { - value[i] = this.decodeValue() + let value: CBORValue + const callback = () => { + if (value !== undefined) return value as CBORArray + value = new Array(length) + for (let i = 0; i < length; i++) { + this.pushKey(i) + value[i] = this.decodeValue() + this.popKey() + } + return value } - return value + const val = this.onValue?.(callback, length, "array", keyPath) + if (val !== undefined) { + if (value !== undefined) return val + for (let i = 0; i < length; i++) this.skipValue() + return val + } + return callback() } else if (majorType === 5) { const { value: length } = this.getArgument(additionalInformation) - const value: Record = {} - for (let i = 0; i < length; i++) { - const key = this.decodeValue() - if (typeof key !== "string") { - throw new Error("microcbor only supports string keys in objects") + let value: CBORValue|void + const callback = () => { + if (value !== undefined) return value as CBORMap + value = {} + for (let i = 0; i < length; i++) { + this.env.isKey = true + const key = this.decodeValue() + this.env.isKey = false + if (typeof key !== "string") { + throw new Error("microcbor only supports string keys in objects") + } + this.pushKey(key) + value[key] = this.decodeValue() + this.popKey() } - value[key] = this.decodeValue() + return value } - return value + const val = this.onValue?.(callback, length, "object", keyPath) + if (val !== undefined) { + if (value !== undefined) return val + for (let i = 0; i < length * 2; i++) this.skipValue() + return val + } + return callback() } else if (majorType === 6) { throw new Error("microcbor does not support tagged data items") } else if (majorType === 7) { + let val switch (additionalInformation) { case 20: - return false + val = this.onValue?.(() => false, 1, "boolean", keyPath) + return val === undefined ? false : val case 21: - return true + val = this.onValue?.(() => true, 1, "boolean", keyPath) + return val === undefined ? true : val case 22: - return null + val = this.onValue?.(() => null, 1, "null", keyPath) + return val === undefined ? null : val case 23: - if (this.allowUndefined) { - return undefined - } else { - throw new TypeError("`undefined` not allowed") - } + if (!this.allowUndefined) throw new TypeError("`undefined` not allowed") + return this.onValue?.(() => undefined, 1, "undefined", keyPath) as CBORValue case 24: throw new Error("microcbor does not support decoding unassigned simple values") case 25: if (this.minFloatSize <= 16) { - return this.float16() + const value = this.float16() + val = this.onValue?.(() => value, 2, "number", keyPath) + return val === undefined ? value : val } else { throw new Error("cannot decode float16 type - below provided minFloatSize") } case 26: if (this.minFloatSize <= 32) { - return this.float32() + const value = this.float32() + val = this.onValue?.(() => value, 4, "number", keyPath) + return val === undefined ? value : val } else { throw new Error("cannot decode float32 type - below provided minFloatSize") } case 27: - return this.float64() + const value = this.float64() + val = this.onValue?.(() => value, 8, "number", keyPath) + return val === undefined ? value : val case 31: throw new Error("microcbor does not support decoding indefinite-length items") default: @@ -216,9 +305,60 @@ export class Decoder implements IterableIterato throw new Error("invalid major type") } } + + private skipValue() { + const initialByte = this.uint8() + const majorType = initialByte >> 5 + const additionalInformation = initialByte & 0x1f + + if (majorType === 0 || majorType === 1) { + this.getArgument(additionalInformation) + } else if (majorType === 2 || majorType === 3) { + const { value: length } = this.getArgument(additionalInformation) + this.allocate(length) + this.advance(length) + } else if (majorType === 4) { + const { value: length } = this.getArgument(additionalInformation) + for (let i = 0; i < length; i++) this.skipValue() + } else if (majorType === 5) { + const { value: length } = this.getArgument(additionalInformation) + for (let i = 0; i < length * 2; i++) this.skipValue() + } else if (majorType === 6) { + throw new Error("microcbor does not support tagged data items") + } else if (majorType === 7) { + switch (additionalInformation) { + case 20: case 21: case 22: + break + case 23: + if (!this.allowUndefined) throw new TypeError("`undefined` not allowed") + break + case 24: + throw new Error("microcbor does not support decoding unassigned simple values") + case 25: + this.allocate(2) + this.advance(2) + break + case 26: + this.allocate(4) + this.advance(4) + break + case 27: + this.allocate(8) + this.advance(8) + break + case 31: + throw new Error("microcbor does not support decoding indefinite-length items") + default: + throw new Error("invalid simple value") + } + } + } } /** Decode an iterable of Uint8Array chunks into an iterable of CBOR values */ -export function* decodeIterable(source: Iterable): IterableIterator { - yield* new Decoder(source) +export function* decodeIterable( + source: Iterable, + options?: DecodeOptions +): IterableIterator { + yield* new Decoder(source, options) } diff --git a/src/index.ts b/src/index.ts index df79e68..746c757 100644 --- a/src/index.ts +++ b/src/index.ts @@ -5,7 +5,7 @@ export { Decoder, decode } from "./Decoder.js" export { encodeIterable } from "./encodeIterable.js" export { decodeIterable } from "./decodeIterable.js" export { encodeAsyncIterable } from "./encodeAsyncIterable.js" -export { decodeAsyncIterable } from "./decodeAsyncIterable.js" +export { decodeAsyncIterable, type AsyncDecodeOptions } from "./decodeAsyncIterable.js" export { CBORDecoderStream } from "./CBORDecoderStream.js" export { CBOREncoderStream } from "./CBOREncoderStream.js" export { UnsafeIntegerError } from "./utils.js" diff --git a/src/options.ts b/src/options.ts index 40d5571..1d935d5 100644 --- a/src/options.ts +++ b/src/options.ts @@ -1,3 +1,5 @@ +import type { CBORValue } from "./types.js" + export const FloatSize = { f16: 16, f32: 32, @@ -32,6 +34,23 @@ export interface EncodeOptions { * @default 16 */ minFloatSize?: (typeof FloatSize)[keyof typeof FloatSize] + + /** + * Function to remap/validate object keys while encoding + * @param key Original object key + * @throws Error if key is invalid + * @returns An optional replacement key string + */ + onKey?: (key: string) => string|void + + /** + * Function to validate/transform/replace values while encoding + * @param value Value to validate/transform/replace + * @param keyPath Array of keys describing the access path to this value + * @throws Error if value is invalid + * @returns An optional replacement value to use + */ + onValue?: (value: unknown, keyPath: (string|number)[]) => CBORValue|void } export interface DecodeOptions { @@ -46,4 +65,29 @@ export interface DecodeOptions { * @default 16 */ minFloatSize?: (typeof FloatSize)[keyof typeof FloatSize] + + /** + * Function to remap/validate object keys while decoding + * @param decodeKey Function to decode original object key + * @param length Key length to validate pre-decoding + * @throws Error if length/key is invalid + * @returns An optional replacement key string + */ + onKey?: (decodeKey: () => string, length: number) => string|void + + /** + * Function to validate/transform/replace values while decoding + * @param decodeValue Function to decode value + * @param length Value length/size to validate pre-decoding + * @param type Value type (e.g. 'number', 'string', 'Uint8Array'...) + * @param keyPath Array of keys describing the access path to this value + * @throws Error if length/value is invalid + * @returns An optional replacement value to use + */ + onValue?: ( + decodeValue: () => CBORValue, + length: number, + type: string, + keyPath: (string|number)[] + ) => CBORValue|void } From 29090aa68138ac034eaf4f7bb491981c94c8b642 Mon Sep 17 00:00:00 2001 From: "${Mr.DJA}" <42304709+iMrDJAi@users.noreply.github.com> Date: Sat, 8 Nov 2025 22:14:54 +0100 Subject: [PATCH 02/10] =?UTF-8?q?=F0=9F=90=9B=20Allow=20flush()=20to=20pro?= =?UTF-8?q?cess=20the=20last=20chunk=20in=20CBORDecoderStream?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/CBORDecoderStream.ts | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/CBORDecoderStream.ts b/src/CBORDecoderStream.ts index 75beeba..687d699 100644 --- a/src/CBORDecoderStream.ts +++ b/src/CBORDecoderStream.ts @@ -5,6 +5,7 @@ import { CBORValue } from "./types.js" export class CBORDecoderStream extends TransformStream { constructor(options: AsyncDecodeOptions = {}) { let readableController: ReadableStreamDefaultController + let pipePromise: Promise const readable = new ReadableStream({ start(controller) { @@ -29,7 +30,7 @@ export class CBORDecoderStream extends Transfor super({ start(controller) { - pipe(controller).catch((err) => controller.error(err)) + pipePromise = pipe(controller).catch((err) => controller.error(err)) }, transform(chunk) { @@ -39,8 +40,10 @@ export class CBORDecoderStream extends Transfor }) }, - flush() { + async flush() { readableController.close() + // Wait for pipe to complete before finishing flush + await pipePromise }, }) } From ff31ba500d56f4ba7666cfa5f788e1f65334005b Mon Sep 17 00:00:00 2001 From: "${Mr.DJA}" <42304709+iMrDJAi@users.noreply.github.com> Date: Fri, 14 Nov 2025 22:14:59 +0100 Subject: [PATCH 03/10] =?UTF-8?q?=F0=9F=90=9B=20Allow=20onValue()=20callba?= =?UTF-8?q?ck=20to=20catch=20individual=20values=20in=20Encoder?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/Encoder.ts | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/src/Encoder.ts b/src/Encoder.ts index 3fbd156..b65df90 100644 --- a/src/Encoder.ts +++ b/src/Encoder.ts @@ -210,6 +210,11 @@ export class Encoder { return } + if (this.onValue) { + const val = this.onValue(value, this.#env.keyPath) + if (val !== undefined) value = val + } + if (value === false) { yield* this.uint8(0xf4) } else if (value === true) { @@ -232,15 +237,14 @@ export class Encoder { yield* this.encodeTypeAndArgument(4, value.length) for (let i = 0; i < value.length; i++) { this.pushKey(i) - const val = this.onValue?.(value[i], this.#env.keyPath) - yield* this.encodeValue(val === undefined ? value[i] : val) + yield* this.encodeValue(value[i]) this.popKey() } } else { const entries = Object.entries(value) .map<[Uint8Array, CBORValue, string]>(([ogKey, value]) => { let key = this.onKey?.(ogKey) - return [this.encoder.encode(key === undefined ? ogKey : key + ''), value, ogKey] + return [this.encoder.encode(key === undefined ? ogKey : key + ""), value, ogKey] }) .sort(Encoder.compareEntries) @@ -249,8 +253,7 @@ export class Encoder { yield* this.encodeTypeAndArgument(3, key.byteLength) yield* this.writeBytes(key) this.pushKey(ogKey) - const val = this.onValue?.(value, this.#env.keyPath) - yield* this.encodeValue(val === undefined ? value : val) + yield* this.encodeValue(value) this.popKey() } } From 6b401bf122dffebb7a670aa6d84f63aa18a4cb53 Mon Sep 17 00:00:00 2001 From: "${Mr.DJA}" <42304709+iMrDJAi@users.noreply.github.com> Date: Sat, 15 Nov 2025 00:38:14 +0100 Subject: [PATCH 04/10] =?UTF-8?q?=F0=9F=8F=B7=EF=B8=8F=20Implemented=20sma?= =?UTF-8?q?rt=20type=20checking=20that=20enforces=20the=20use=20of=20`onVa?= =?UTF-8?q?lue()`=20when=20dealing=20with=20non-CBORValues?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/CBORDecoderStream.ts | 16 +++++++++------ src/CBOREncoderStream.ts | 14 ++++++++----- src/Decoder.ts | 27 +++++++++++++++++-------- src/Encoder.ts | 23 +++++++++++++-------- src/decodeAsyncIterable.ts | 41 +++++++++++++++++++++----------------- src/decodeIterable.ts | 26 +++++++++++++++--------- src/encodeAsyncIterable.ts | 16 ++++++++++++--- src/encodeIterable.ts | 16 ++++++++++++--- src/options.ts | 8 ++++---- src/utils.ts | 18 +++++++++++++++++ 10 files changed, 141 insertions(+), 64 deletions(-) diff --git a/src/CBORDecoderStream.ts b/src/CBORDecoderStream.ts index 687d699..9252c92 100644 --- a/src/CBORDecoderStream.ts +++ b/src/CBORDecoderStream.ts @@ -1,9 +1,13 @@ import { Decoder, type AsyncDecodeOptions } from "./decodeAsyncIterable.js" -import { CBORValue } from "./types.js" +import type { WithRequired, Flatten, NoInfer } from "./utils.js" +import type { CBORValue } from "./types.js" /** Decode a Web Streams API ReadableStream */ -export class CBORDecoderStream extends TransformStream { - constructor(options: AsyncDecodeOptions = {}) { +export class CBORDecoderStream extends TransformStream { + constructor(...[options = {}]: T extends CBORValue + ? []|[AsyncDecodeOptions] + : [WithRequired>>, "onValue">] + ) { let readableController: ReadableStreamDefaultController let pipePromise: Promise @@ -18,13 +22,13 @@ export class CBORDecoderStream extends Transfor const chunks = new WeakMap void }>() async function pipe(controller: TransformStreamDefaultController) { - const decoder = new Decoder(readable.values(), { + const decoder = new Decoder(readable.values(), { ...options, onFree: (chunk) => chunks.get(chunk)?.resolve(), - }) + } as AsyncDecodeOptions) for await (const value of decoder) { - controller.enqueue(value) + controller.enqueue(value as T) } } diff --git a/src/CBOREncoderStream.ts b/src/CBOREncoderStream.ts index 4944e2f..5318cb1 100644 --- a/src/CBOREncoderStream.ts +++ b/src/CBOREncoderStream.ts @@ -1,14 +1,18 @@ import { Encoder } from "./Encoder.js" -import { CBORValue } from "./types.js" -import { EncodeOptions } from "./options.js" +import type { CBORValue } from "./types.js" +import type { EncodeOptions } from "./options.js" +import type { Flatten, WithRequired, NoInfer } from "./utils.js" /** * Encode a Web Streams API ReadableStream. * options.chunkRecycling has no effect here. */ -export class CBOREncoderStream extends TransformStream { - constructor(options: EncodeOptions = {}) { - const encoder = new Encoder({ ...options, chunkRecycling: false }) +export class CBOREncoderStream extends TransformStream { + constructor(...[options = {}]: T extends CBORValue + ? []|[EncodeOptions] + : [WithRequired>>, "onValue">] + ) { + const encoder = new Encoder({ ...options, chunkRecycling: false } as EncodeOptions) super({ transform(value: CBORValue, controller: TransformStreamDefaultController) { diff --git a/src/Decoder.ts b/src/Decoder.ts index 9f5d933..6a85246 100644 --- a/src/Decoder.ts +++ b/src/Decoder.ts @@ -2,9 +2,10 @@ import { getFloat16 } from "fp16" import type { CBORValue, CBORArray, CBORMap } from "./types.js" import type { DecodeOptions, FloatSize } from "./options.js" +import type { WithRequired, Flatten, NoInfer } from "./utils.js" import { UnsafeIntegerError, maxSafeInteger, minSafeInteger } from "./utils.js" -export class Decoder { +export class Decoder { public readonly allowUndefined: boolean public readonly minFloatSize: (typeof FloatSize)[keyof typeof FloatSize] @@ -17,6 +18,7 @@ export class Decoder { keyPath: (string|number)[] ) => CBORValue|void + private data: Uint8Array #offset: number #view: DataView #env: { @@ -24,17 +26,18 @@ export class Decoder { keyPath: (string|number)[] } - public constructor( - private readonly data: Uint8Array, - options: DecodeOptions = {}, + public constructor(...[data, options = {}]: T extends CBORValue + ? ([Uint8Array]|[Uint8Array, DecodeOptions]) + : [Uint8Array, WithRequired>>, "onValue">] ) { + this.data = data this.#offset = 0 this.#view = new DataView(data.buffer, data.byteOffset, data.byteLength) this.#env = { isKey: false, keyPath: [] } this.allowUndefined = options.allowUndefined ?? true this.minFloatSize = options.minFloatSize ?? 16 this.onKey = options.onKey - this.onValue = options.onValue + this.onValue = (options as DecodeOptions).onValue } public getOffset(): number { @@ -102,6 +105,7 @@ export class Decoder { } } + public decodeValue(): R public decodeValue(): CBORValue { const initialByte = this.uint8() const majorType = initialByte >> 5 @@ -287,7 +291,14 @@ export class Decoder { } } -/** Decode a single CBOR value */ -export function decode(data: Uint8Array, options: DecodeOptions = {}): T { - return new Decoder(data, options).decodeValue() as T +/** + * Decode a single CBOR value + * @param data Data to decode + * @param options Decode options + */ +export function decode(...[data, options]: T extends CBORValue + ? ([Uint8Array]|[Uint8Array, DecodeOptions]) + : [Uint8Array, WithRequired>>, "onValue">] +) { + return new Decoder(data, options as DecodeOptions).decodeValue() as T } diff --git a/src/Encoder.ts b/src/Encoder.ts index b65df90..41330b5 100644 --- a/src/Encoder.ts +++ b/src/Encoder.ts @@ -1,10 +1,10 @@ import { Precision, getFloat16Precision, getFloat32Precision, setFloat16 } from "fp16" import type { CBORValue } from "./types.js" -import { EncodeOptions, FloatSize } from "./options.js" -import { assert } from "./utils.js" +import { FloatSize, type EncodeOptions } from "./options.js" +import { assert, type WithRequired, type Flatten, type NoInfer } from "./utils.js" -export class Encoder { +export class Encoder { public static defaultChunkSize = 4096 #closed: boolean @@ -14,7 +14,7 @@ export class Encoder { public readonly minFloatSize: (typeof FloatSize)[keyof typeof FloatSize] private readonly onKey?: (key: string) => string|void - private readonly onValue?: (value: unknown, keyPath: (string|number)[]) => CBORValue|void + private readonly onValue?: (value: CBORValue, keyPath: (string|number)[]) => CBORValue|void private readonly encoder = new TextEncoder() private readonly buffer: ArrayBuffer private readonly view: DataView @@ -24,14 +24,17 @@ export class Encoder { keyPath: (string|number)[] } - constructor(options: EncodeOptions = {}) { + constructor(...[options = {}]: T extends CBORValue + ? []|[EncodeOptions] + : [WithRequired>>, "onValue">] + ) { this.allowUndefined = options.allowUndefined ?? true this.minFloatSize = options.minFloatSize ?? 16 this.chunkRecycling = options.chunkRecycling ?? false this.chunkSize = options.chunkSize ?? Encoder.defaultChunkSize assert(this.chunkSize >= 8, "expected chunkSize >= 8") this.onKey = options.onKey - this.onValue = options.onValue + this.onValue = (options as EncodeOptions).onValue this.buffer = new ArrayBuffer(this.chunkSize) this.view = new DataView(this.buffer) @@ -205,13 +208,13 @@ export class Encoder { } } - public *encodeValue(value: CBORValue): Iterable { + public *encodeValue(value: T|CBORValue): Iterable { if (this.#closed) { return } if (this.onValue) { - const val = this.onValue(value, this.#env.keyPath) + const val = this.onValue(value as CBORValue, this.#env.keyPath) if (val !== undefined) value = val } @@ -309,8 +312,12 @@ export class Encoder { /** * Encode a single CBOR value. + * @param value Value to encode + * @param options Encode options * options.chunkRecycling has no effect here. */ +export function encode(value: T, options?: EncodeOptions): Uint8Array +export function encode(value: T, options: WithRequired>>, "onValue">): Uint8Array export function encode(value: CBORValue, options: EncodeOptions = {}): Uint8Array { const encoder = new Encoder({ ...options, chunkRecycling: false }) diff --git a/src/decodeAsyncIterable.ts b/src/decodeAsyncIterable.ts index dbb663b..edd9b24 100644 --- a/src/decodeAsyncIterable.ts +++ b/src/decodeAsyncIterable.ts @@ -1,13 +1,11 @@ import { getFloat16 } from "fp16" import type { CBORValue, CBORArray, CBORMap } from "./types.js" - +import type { DecodeOptions, FloatSize } from "./options.js" +import type { WithRequired, Flatten, NoInfer, Awaitable } from "./utils.js" import { UnsafeIntegerError, maxSafeInteger, minSafeInteger } from "./utils.js" -import { DecodeOptions, FloatSize } from "./options.js" - -type Awaitable = T | PromiseLike -export interface AsyncDecodeOptions extends Omit { +export interface AsyncDecodeOptions extends Omit { /** * Function to remap/validate object keys while decoding * (async version that works with AsyncIterable and streams) @@ -19,7 +17,7 @@ export interface AsyncDecodeOptions extends Omit Awaitable, length: number - ) => Promise + ) => Awaitable /** * Function to validate/transform/replace values while decoding @@ -36,12 +34,12 @@ export interface AsyncDecodeOptions extends Omit Promise + ) => Awaitable onFree?: (chunk: Uint8Array) => void } -export class Decoder implements AsyncIterableIterator { +export class Decoder implements AsyncIterableIterator { public readonly allowUndefined: boolean public readonly minFloatSize: (typeof FloatSize)[keyof typeof FloatSize] @@ -56,25 +54,28 @@ export class Decoder implements AsyncIterableIt private readonly onKey?: ( decodeKey: () => Awaitable, length: number - ) => Promise + ) => Awaitable private readonly onValue?: ( decodeValue: () => Awaitable, length: number, type: string, keyPath: (string|number)[] - ) => Promise + ) => Awaitable private env: { isKey: boolean keyPath: (string|number)[] } = { isKey: false, keyPath: [] } - public constructor(source: AsyncIterable, options: AsyncDecodeOptions = {}) { + public constructor(...[source, options = {}]: T extends CBORValue + ? ([AsyncIterable]|[AsyncIterable, AsyncDecodeOptions]) + : [AsyncIterable, WithRequired>>, "onValue">] + ) { this.onFree = options.onFree this.allowUndefined = options.allowUndefined ?? true this.minFloatSize = options.minFloatSize ?? 16 this.iter = source[Symbol.asyncIterator]() this.onKey = options.onKey - this.onValue = options.onValue + this.onValue = (options as AsyncDecodeOptions).onValue } [Symbol.asyncIterator] = () => this @@ -403,10 +404,14 @@ export class Decoder implements AsyncIterableIt } } -/** Decode an async iterable of Uint8Array chunks into an async iterable of CBOR values */ -export async function* decodeAsyncIterable( - source: AsyncIterable, - options: AsyncDecodeOptions = {}, -): AsyncIterableIterator { - yield* new Decoder(source, options) +/** + * Decode an async iterable of Uint8Array chunks into an async iterable of CBOR values + * @param source Async iterable of Uint8Array chunks + * @param options Decode options + */ +export async function* decodeAsyncIterable(...args: T extends CBORValue + ? ([AsyncIterable]|[AsyncIterable, AsyncDecodeOptions]) + : [AsyncIterable, WithRequired>>, "onValue">] +): AsyncIterableIterator { + yield* new Decoder(...args) } diff --git a/src/decodeIterable.ts b/src/decodeIterable.ts index c677eed..ead34f2 100644 --- a/src/decodeIterable.ts +++ b/src/decodeIterable.ts @@ -2,9 +2,10 @@ import { getFloat16 } from "fp16" import type { CBORValue, CBORArray, CBORMap } from "./types.js" import type { DecodeOptions, FloatSize } from "./options.js" +import type { WithRequired, Flatten, NoInfer } from "./utils.js" import { UnsafeIntegerError, maxSafeInteger, minSafeInteger } from "./utils.js" -export class Decoder implements IterableIterator { +export class Decoder implements IterableIterator { public readonly allowUndefined: boolean public readonly minFloatSize: (typeof FloatSize)[keyof typeof FloatSize] @@ -27,12 +28,15 @@ export class Decoder implements IterableIterato keyPath: (string|number)[] } = { isKey: false, keyPath: [] } - public constructor(source: Iterable, options: DecodeOptions = {}) { + public constructor(...[source, options = {}]: T extends CBORValue + ? ([Iterable]|[Iterable, DecodeOptions]) + : [Iterable, WithRequired>>, "onValue">] + ) { this.allowUndefined = options.allowUndefined ?? true this.minFloatSize = options.minFloatSize ?? 16 this.iter = source[Symbol.iterator]() this.onKey = options.onKey - this.onValue = options.onValue + this.onValue = (options as DecodeOptions).onValue } [Symbol.iterator] = () => this @@ -355,10 +359,14 @@ export class Decoder implements IterableIterato } } -/** Decode an iterable of Uint8Array chunks into an iterable of CBOR values */ -export function* decodeIterable( - source: Iterable, - options?: DecodeOptions -): IterableIterator { - yield* new Decoder(source, options) +/** + * Decode an iterable of Uint8Array chunks into an iterable of CBOR values + * @param source Iterable of Uint8Array chunks + * @param options Decode options + */ +export function* decodeIterable(...args: T extends CBORValue + ? ([Iterable]|[Iterable, DecodeOptions]) + : [Iterable, WithRequired>>, "onValue">] +): IterableIterator { + yield* new Decoder(...args) } diff --git a/src/encodeAsyncIterable.ts b/src/encodeAsyncIterable.ts index 7fe5736..1fc02d5 100644 --- a/src/encodeAsyncIterable.ts +++ b/src/encodeAsyncIterable.ts @@ -1,9 +1,19 @@ -import type { CBORValue } from "./types.js" - import { Encoder } from "./Encoder.js" -import { EncodeOptions } from "./options.js" +import type { CBORValue } from "./types.js" +import type { EncodeOptions } from "./options.js" +import type { Flatten, WithRequired, NoInfer } from "./utils.js" /** Encode an async iterable of CBOR values into an async iterable of Uint8Array chunks */ +export function encodeAsyncIterable( + source: AsyncIterable, + options?: EncodeOptions +): AsyncIterableIterator + +export function encodeAsyncIterable( + source: AsyncIterable, + options: WithRequired>>, "onValue"> +): AsyncIterableIterator + export async function* encodeAsyncIterable( source: AsyncIterable, options: EncodeOptions = {}, diff --git a/src/encodeIterable.ts b/src/encodeIterable.ts index e602ff0..22fba8b 100644 --- a/src/encodeIterable.ts +++ b/src/encodeIterable.ts @@ -1,9 +1,19 @@ -import type { CBORValue } from "./types.js" - import { Encoder } from "./Encoder.js" -import { EncodeOptions } from "./options.js" +import type { CBORValue } from "./types.js" +import type { EncodeOptions } from "./options.js" +import type { Flatten, WithRequired, NoInfer } from "./utils.js" /** Encode an iterable of CBOR values into an iterable of Uint8Array chunks */ +export function encodeIterable( + source: Iterable, + options?: EncodeOptions +): IterableIterator + +export function encodeIterable( + source: Iterable, + options: WithRequired>>, "onValue"> +): IterableIterator + export function* encodeIterable( source: Iterable, options: EncodeOptions = {}, diff --git a/src/options.ts b/src/options.ts index 1d935d5..75c4988 100644 --- a/src/options.ts +++ b/src/options.ts @@ -6,7 +6,7 @@ export const FloatSize = { f64: 64, } -export interface EncodeOptions { +export interface EncodeOptions { /** * Allow `undefined` * @default true @@ -50,10 +50,10 @@ export interface EncodeOptions { * @throws Error if value is invalid * @returns An optional replacement value to use */ - onValue?: (value: unknown, keyPath: (string|number)[]) => CBORValue|void + onValue?: (value: T, keyPath: (string|number)[]) => CBORValue|void } -export interface DecodeOptions { +export interface DecodeOptions { /** * Allow `undefined` * @default true @@ -89,5 +89,5 @@ export interface DecodeOptions { length: number, type: string, keyPath: (string|number)[] - ) => CBORValue|void + ) => T|void } diff --git a/src/utils.ts b/src/utils.ts index 1491a5a..9b48e09 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -90,3 +90,21 @@ export function getByteLength(string: string): number { return bytes } + +export type WithRequired = T & { [P in K]-?: T[P] }; + +export type DeepValueUnion = + T extends readonly (infer E)[] + ? DeepValueUnion + : T extends Record + ? { [K in keyof T]: DeepValueUnion }[keyof T] + : T + +export type Flatten = + | DeepValueUnion + | Flatten[] + | { [K: string]: Flatten } + +export type NoInfer = [T][T extends any ? 0 : never] + +export type Awaitable = T | PromiseLike From ffe69e18a5db6bf2d0b41a7eeb6893f46b11442a Mon Sep 17 00:00:00 2001 From: "${Mr.DJA}" <42304709+iMrDJAi@users.noreply.github.com> Date: Wed, 7 Jan 2026 15:10:22 +0100 Subject: [PATCH 05/10] =?UTF-8?q?=F0=9F=90=9B=20Added=20duplicate=20object?= =?UTF-8?q?=20key=20check=20in=20decodeValue()?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/Decoder.ts | 3 +++ src/decodeAsyncIterable.ts | 3 +++ src/decodeIterable.ts | 3 +++ 3 files changed, 9 insertions(+) diff --git a/src/Decoder.ts b/src/Decoder.ts index 6a85246..728ad7b 100644 --- a/src/Decoder.ts +++ b/src/Decoder.ts @@ -184,6 +184,9 @@ export class Decoder { if (typeof key !== "string") { throw new Error("microcbor only supports string keys in objects") } + if (key in value) { + throw new Error("duplicate object key") + } this.pushKey(key) value[key] = this.decodeValue() this.popKey() diff --git a/src/decodeAsyncIterable.ts b/src/decodeAsyncIterable.ts index edd9b24..47c1362 100644 --- a/src/decodeAsyncIterable.ts +++ b/src/decodeAsyncIterable.ts @@ -293,6 +293,9 @@ export class Decoder implements AsyncIterableIterator { if (typeof key !== "string") { throw new Error("microcbor only supports string keys in objects") } + if (key in value) { + throw new Error("duplicate object key") + } this.pushKey(key) value[key] = await this.decodeValue() this.popKey() diff --git a/src/decodeIterable.ts b/src/decodeIterable.ts index ead34f2..cc7e163 100644 --- a/src/decodeIterable.ts +++ b/src/decodeIterable.ts @@ -248,6 +248,9 @@ export class Decoder implements IterableIterator { if (typeof key !== "string") { throw new Error("microcbor only supports string keys in objects") } + if (key in value) { + throw new Error("duplicate object key") + } this.pushKey(key) value[key] = this.decodeValue() this.popKey() From 8be3d2372930a5f5e5ff852c8fc4a339fea4c702 Mon Sep 17 00:00:00 2001 From: "${Mr.DJA}" <42304709+iMrDJAi@users.noreply.github.com> Date: Sat, 31 Jan 2026 15:28:23 +0100 Subject: [PATCH 06/10] =?UTF-8?q?=E2=9C=A8=20Added=20backpressure=20to=20C?= =?UTF-8?q?BOREncoderStream?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/CBOREncoderStream.ts | 25 +++++++++++-------- src/utils.ts | 54 +++++++++++++++++++++++++++++++++++++++- 2 files changed, 67 insertions(+), 12 deletions(-) diff --git a/src/CBOREncoderStream.ts b/src/CBOREncoderStream.ts index 5318cb1..a6e84bf 100644 --- a/src/CBOREncoderStream.ts +++ b/src/CBOREncoderStream.ts @@ -1,4 +1,5 @@ import { Encoder } from "./Encoder.js" +import { createTransformWithBackpressure } from "./utils.js" import type { CBORValue } from "./types.js" import type { EncodeOptions } from "./options.js" import type { Flatten, WithRequired, NoInfer } from "./utils.js" @@ -7,27 +8,29 @@ import type { Flatten, WithRequired, NoInfer } from "./utils.js" * Encode a Web Streams API ReadableStream. * options.chunkRecycling has no effect here. */ -export class CBOREncoderStream extends TransformStream { +export class CBOREncoderStream { + readable!: ReadableStream + writable!: WritableStream + constructor(...[options = {}]: T extends CBORValue ? []|[EncodeOptions] : [WithRequired>>, "onValue">] ) { const encoder = new Encoder({ ...options, chunkRecycling: false } as EncodeOptions) - super({ - transform(value: CBORValue, controller: TransformStreamDefaultController) { + return createTransformWithBackpressure( + async (value, enqueue) => { // Encode the incoming value and push all resulting chunks - for (const chunk of encoder.encodeValue(value)) { - controller.enqueue(chunk) + for (const chunk of encoder.encodeValue(value as CBORValue)) { + await enqueue(chunk) } }, - - flush(controller: TransformStreamDefaultController) { - // Push any remaining chunks when the stream is closing + async (enqueue) => { + // Flush any remaining chunks for (const chunk of encoder.flush()) { - controller.enqueue(chunk) + await enqueue(chunk) } - }, - }) + } + ) } } diff --git a/src/utils.ts b/src/utils.ts index 9b48e09..070a36e 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -91,10 +91,62 @@ export function getByteLength(string: string): number { return bytes } +export function createTransformWithBackpressure( + transform: (chunk: I, enqueue: (out: O) => Promise) => Awaitable, + flush?: (enqueue: (out: O) => Promise) => Awaitable +): ReadableWritablePair { + let readableController: ReadableStreamDefaultController + let pullResolve: (() => void) | null = null + let closed = false + + const enqueue = async (out: O) => { + if (closed) throw new Error('Stream closed: cannot enqueue') + readableController.enqueue(out) + await new Promise(res => { + pullResolve = () => { + pullResolve = null + res() + } + }) + } + + const readable = new ReadableStream({ + start(controller) { + readableController = controller + }, + pull() { + pullResolve?.() + }, + cancel() { + closed = true + pullResolve?.() + } + }, { highWaterMark: 1 }) + + const writable = new WritableStream({ + write(chunk) { + return transform(chunk, enqueue) + }, + async close() { + pullResolve?.() + if (flush) await flush(enqueue) + closed = true + readableController.close() + }, + abort(e) { + closed = true + pullResolve?.() + readableController.error(e) + } + }, { highWaterMark: 1 }) + + return { readable, writable } +} + export type WithRequired = T & { [P in K]-?: T[P] }; export type DeepValueUnion = - T extends readonly (infer E)[] + T extends readonly (infer E)[] ? DeepValueUnion : T extends Record ? { [K in keyof T]: DeepValueUnion }[keyof T] From c026c61d66ba96d3d66f8d9348e3a75c17aa8600 Mon Sep 17 00:00:00 2001 From: "${Mr.DJA}" <42304709+iMrDJAi@users.noreply.github.com> Date: Sun, 8 Feb 2026 13:11:32 +0100 Subject: [PATCH 07/10] =?UTF-8?q?=F0=9F=90=9B=20Yield=20execution=20before?= =?UTF-8?q?=20`transform()`=20to=20prevent=20blocking=20event=20loop?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/utils.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/utils.ts b/src/utils.ts index 070a36e..3d2593a 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -124,7 +124,8 @@ export function createTransformWithBackpressure( }, { highWaterMark: 1 }) const writable = new WritableStream({ - write(chunk) { + async write(chunk) { + await new Promise(res => setImmediate(res)) // yield return transform(chunk, enqueue) }, async close() { From b1b51c2213e1ebfb328007a633673e450869788b Mon Sep 17 00:00:00 2001 From: "${Mr.DJA}" <42304709+iMrDJAi@users.noreply.github.com> Date: Sun, 15 Mar 2026 11:38:04 +0100 Subject: [PATCH 08/10] =?UTF-8?q?=E2=9C=A8=20Added=20backpressure=20to=20C?= =?UTF-8?q?BORDecoderStream?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/CBORDecoderStream.ts | 75 ++++++++++++++++++-------------------- src/decodeAsyncIterable.ts | 15 +++----- src/utils.ts | 2 +- 3 files changed, 43 insertions(+), 49 deletions(-) diff --git a/src/CBORDecoderStream.ts b/src/CBORDecoderStream.ts index 9252c92..57e8a3a 100644 --- a/src/CBORDecoderStream.ts +++ b/src/CBORDecoderStream.ts @@ -1,54 +1,51 @@ import { Decoder, type AsyncDecodeOptions } from "./decodeAsyncIterable.js" +import { createTransformWithBackpressure } from "./utils.js" import type { WithRequired, Flatten, NoInfer } from "./utils.js" import type { CBORValue } from "./types.js" /** Decode a Web Streams API ReadableStream */ -export class CBORDecoderStream extends TransformStream { +export class CBORDecoderStream { + readable!: ReadableStream + writable!: WritableStream + constructor(...[options = {}]: T extends CBORValue ? []|[AsyncDecodeOptions] : [WithRequired>>, "onValue">] ) { - let readableController: ReadableStreamDefaultController - let pipePromise: Promise - - const readable = new ReadableStream({ - start(controller) { - readableController = controller + let transformResolve: (() => void) | null = null + const { writable, readable: readable_ } = new TransformStream( + { + transform(chunk, controller) { + console.log('transform', chunk.length, controller.desiredSize) + return new Promise((resolve) => { + transformResolve = () => { + transformResolve = null + resolve() + } + controller.enqueue(chunk) + }) + } }, - }) - - // We need to track whick chunks have been "processed" and only resolve each - // .transform() promise once all data from each chunk has been enqueued. - const chunks = new WeakMap void }>() - - async function pipe(controller: TransformStreamDefaultController) { - const decoder = new Decoder(readable.values(), { - ...options, - onFree: (chunk) => chunks.get(chunk)?.resolve(), - } as AsyncDecodeOptions) - - for await (const value of decoder) { - controller.enqueue(value as T) + { highWaterMark: 1 }, + { highWaterMark: 1 } + ) + + const { readable, writable: writable_ } = createTransformWithBackpressure( + async function pipe(_, enqueue) { + const decoder = new Decoder(readable_.values(), { + ...options, + onPull: () => transformResolve?.() + } as AsyncDecodeOptions) + for await (const value of decoder) { + await enqueue(value as T) + } + writer.close() // Close stream } - } + ) - super({ - start(controller) { - pipePromise = pipe(controller).catch((err) => controller.error(err)) - }, - - transform(chunk) { - return new Promise((resolve) => { - chunks.set(chunk, { resolve }) - readableController.enqueue(chunk) - }) - }, + const writer = writable_.getWriter() + writer.write(undefined) // Jump-start stream - async flush() { - readableController.close() - // Wait for pipe to complete before finishing flush - await pipePromise - }, - }) + return { readable, writable } } } diff --git a/src/decodeAsyncIterable.ts b/src/decodeAsyncIterable.ts index 47c1362..bd3b94b 100644 --- a/src/decodeAsyncIterable.ts +++ b/src/decodeAsyncIterable.ts @@ -36,7 +36,8 @@ export interface AsyncDecodeOptions extends Omit Awaitable - onFree?: (chunk: Uint8Array) => void + /** Callback function when the decoder requires more data */ + onPull?: () => void } export class Decoder implements AsyncIterableIterator { @@ -50,7 +51,7 @@ export class Decoder implements AsyncIterableIterator { private readonly constantBuffer = new ArrayBuffer(8) private readonly constantView = new DataView(this.constantBuffer) private readonly iter: AsyncIterator - private readonly onFree?: (chunk: Uint8Array) => void + private readonly onPull?: () => void private readonly onKey?: ( decodeKey: () => Awaitable, length: number @@ -70,7 +71,7 @@ export class Decoder implements AsyncIterableIterator { ? ([AsyncIterable]|[AsyncIterable, AsyncDecodeOptions]) : [AsyncIterable, WithRequired>>, "onValue">] ) { - this.onFree = options.onFree + this.onPull = options.onPull this.allowUndefined = options.allowUndefined ?? true this.minFloatSize = options.minFloatSize ?? 16 this.iter = source[Symbol.asyncIterator]() @@ -82,6 +83,7 @@ export class Decoder implements AsyncIterableIterator { private async allocate(size: number) { while (this.byteLength < size) { + this.onPull?.() const { done, value } = await this.iter.next() if (done) { throw new Error("stream ended prematurely") @@ -120,12 +122,6 @@ export class Decoder implements AsyncIterableIterator { } } - if (this.onFree !== undefined) { - for (let i = 0; i < deleteCount; i++) { - this.onFree(this.chunks[i]) - } - } - this.chunks.splice(0, deleteCount) } @@ -198,6 +194,7 @@ export class Decoder implements AsyncIterableIterator { public async next(): Promise<{ done: true; value: undefined } | { done: false; value: T }> { while (this.byteLength === 0) { + this.onPull?.() const { done, value } = await this.iter.next() if (done) { return { done: true, value: undefined } diff --git a/src/utils.ts b/src/utils.ts index 3d2593a..09efd8b 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -100,7 +100,7 @@ export function createTransformWithBackpressure( let closed = false const enqueue = async (out: O) => { - if (closed) throw new Error('Stream closed: cannot enqueue') + if (closed) throw new Error('cannot enqueue - stream closed') readableController.enqueue(out) await new Promise(res => { pullResolve = () => { From afa97d89f6933de938efc686ccec4ecf285fa091 Mon Sep 17 00:00:00 2001 From: "${Mr.DJA}" <42304709+iMrDJAi@users.noreply.github.com> Date: Sun, 15 Mar 2026 14:31:14 +0100 Subject: [PATCH 09/10] =?UTF-8?q?=F0=9F=94=87Supressed=20logs?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/CBORDecoderStream.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/src/CBORDecoderStream.ts b/src/CBORDecoderStream.ts index 57e8a3a..8acce84 100644 --- a/src/CBORDecoderStream.ts +++ b/src/CBORDecoderStream.ts @@ -16,7 +16,6 @@ export class CBORDecoderStream { const { writable, readable: readable_ } = new TransformStream( { transform(chunk, controller) { - console.log('transform', chunk.length, controller.desiredSize) return new Promise((resolve) => { transformResolve = () => { transformResolve = null From 983817e134ff718216c814cadfad7475f8fda0e2 Mon Sep 17 00:00:00 2001 From: "${Mr.DJA}" <42304709+iMrDJAi@users.noreply.github.com> Date: Sun, 15 Mar 2026 17:19:49 +0100 Subject: [PATCH 10/10] =?UTF-8?q?=F0=9F=93=9D=20Updated=20README.md?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 321 ++++++++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 285 insertions(+), 36 deletions(-) diff --git a/README.md b/README.md index 2d5a00f..084dfb1 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,8 @@ microcbor is a minimal JavaScript [CBOR](https://cbor.io/) implementation featur - small footprint - fast performance - `Iterable` and `AsyncIterable` streaming APIs with "chunk recycling" encoding option -- [Web Streams API](https://developer.mozilla.org/en-US/docs/Web/API/Streams_API)-compatible [TransformStream](https://developer.mozilla.org/en-US/docs/Web/API/TransformStream) classes +- [Web Streams API](https://developer.mozilla.org/en-US/docs/Web/API/Streams_API)-compatible [TransformStream](https://developer.mozilla.org/en-US/docs/Web/API/TransformStream) classes with proper backpressure +- key mapping and value validation with `onKey()`, `onValue()` callbacks. microcbor follows the [deterministic CBOR encoding requirements](https://www.rfc-editor.org/rfc/rfc8949.html#core-det) - all floating-point numbers are serialized in the smallest possible size without losing precision, and object entries are always sorted by key in byte-wise utf-8 lexicographic order. `NaN` is always serialized as `0xf97e00`. **microcbor doesn't support tags, bigints, typed arrays, non-string keys, or indefinite-length collections.** @@ -29,8 +30,10 @@ This library is TypeScript-native, ESM-only, and has just **one dependency** [jo - [`encodeAsyncIterable`](#encodeasynciterable) - [`CBOREncoderStream`](#cborencoderstream) - [Decoding](#decoding) + - [`DecodeOptions`](#decodeoptions) - [`decode`](#decode) - [`decodeIterable`](#decodeiterable) + - [`AsyncDecodeOptions`](#asyncdecodeoptions) - [`decodeAsyncIterable`](#decodeasynciterable) - [`CBORDecoderStream`](#cbordecoderstream) - [Value mapping](#value-mapping) @@ -41,9 +44,13 @@ This library is TypeScript-native, ESM-only, and has just **one dependency** [jo ## Install -``` +```bash npm i microcbor ``` +or +```bash +bun add microcbor +``` ## Usage @@ -89,7 +96,7 @@ interface CBORMap { #### `EncodeOptions` ```ts -export interface EncodeOptions { +export interface EncodeOptions { /** * Allow `undefined` * @default true @@ -107,16 +114,33 @@ export interface EncodeOptions { chunkRecycling?: boolean /** - * Maximum chunk size. + * Maximum chunk size * @default 4096 */ chunkSize?: number /** - * Minimum bitsize for floating-point numbers: 16, 32, or 64. + * Minimum bitsize for floating-point numbers: 16, 32, or 64 * @default 16 */ minFloatSize?: (typeof FloatSize)[keyof typeof FloatSize] + + /** + * Function to remap/validate object keys while encoding + * @param key Original object key + * @throws Error if key is invalid + * @returns An optional replacement key string + */ + onKey?: (key: string) => string|void + + /** + * Function to validate/transform/replace values while encoding + * @param value Value to validate/transform/replace + * @param keyPath Array of keys describing the access path to this value + * @throws Error if value is invalid + * @returns An optional replacement value to use + */ + onValue?: (value: T, keyPath: (string|number)[]) => CBORValue|void } ``` @@ -127,7 +151,7 @@ export interface EncodeOptions { * Calculate the byte length that a value will encode into * without actually allocating anything. */ -declare function encodingLength( +export function encodingLength( value: CBORValue, options?: EncodeOptions, ): number @@ -138,31 +162,46 @@ declare function encodingLength( ```ts /** * Encode a single CBOR value. + * @param value Value to encode + * @param options Encode options * options.chunkRecycling has no effect here. */ -export function encode(value: CBORValue, options?: EncodeOptions): Uint8Array +export function encode( + value: T, + options?: EncodeOptions +): Uint8Array +export function encode( + value: T, + options: WithRequired>>, "onValue"> +): Uint8Array ``` #### `encodeIterable` ```ts /** Encode an iterable of CBOR values into an iterable of Uint8Array chunks */ -export function* encodeIterable( - source: Iterable, - options?: EncodeOptions, +export function* encodeIterable( + source: Iterable, + options?: EncodeOptions +): IterableIterator +export function* encodeIterable( + source: Iterable, + options: WithRequired>>, "onValue"> ): IterableIterator - ``` #### `encodeAsyncIterable` ```ts /** Encode an async iterable of CBOR values into an async iterable of Uint8Array chunks */ -export async function* encodeAsyncIterable( - source: AsyncIterable, - options?: EncodeOptions, +export async function* encodeAsyncIterable( + source: AsyncIterable, + options?: EncodeOptions +): AsyncIterableIterator +export async function* encodeAsyncIterable( + source: AsyncIterable, + options: WithRequired>>, "onValue"> ): AsyncIterableIterator - ``` #### `CBOREncoderStream` @@ -172,8 +211,14 @@ export async function* encodeAsyncIterable( * Encode a Web Streams API ReadableStream. * options.chunkRecycling has no effect here. */ -export class CBOREncoderStream extends TransformStream { - public constructor(options?: EncodeOptions) +export class CBOREncoderStream { + readable: ReadableStream + writable: WritableStream + + public constructor(...[options]: T extends CBORValue + ? []|[EncodeOptions] + : [WithRequired>>, "onValue">] + ) } ``` @@ -182,7 +227,7 @@ export class CBOREncoderStream extends TransformStream { #### `DecodeOptions` ```ts -export interface DecodeOptions { +export interface DecodeOptions { /** * Allow `undefined` * @default true @@ -194,47 +239,127 @@ export interface DecodeOptions { * @default 16 */ minFloatSize?: (typeof FloatSize)[keyof typeof FloatSize] + + /** + * Function to remap/validate object keys while decoding + * @param decodeKey Function to decode original object key + * @param length Key length to validate pre-decoding + * @throws Error if length/key is invalid + * @returns An optional replacement key string + */ + onKey?: (decodeKey: () => string, length: number) => string|void + + /** + * Function to validate/transform/replace values while decoding + * @param decodeValue Function to decode value + * @param length Value length/size to validate pre-decoding + * @param type Value type (e.g. 'number', 'string', 'Uint8Array'...) + * @param keyPath Array of keys describing the access path to this value + * @throws Error if length/value is invalid + * @returns An optional replacement value to use + */ + onValue?: ( + decodeValue: () => CBORValue, + length: number, + type: string, + keyPath: (string|number)[] + ) => T|void } ``` #### `decode` ```ts -/** Decode a single CBOR value. */ -export function decode( - data: Uint8Array, - options?: DecodeOptions, +/* + * Decode a single CBOR value + * @param data Data to decode + * @param options Decode options + */ +export function decode(...[data, options]: T extends CBORValue + ? ([Uint8Array]|[Uint8Array, DecodeOptions]) + : [Uint8Array, WithRequired>>, "onValue">] ): T ``` #### `decodeIterable` ```ts -/** Decode an iterable of Uint8Array chunks into an iterable of CBOR values */ -export function* decodeIterable( - source: Iterable, - options?: DecodeOptions, +/** + * Decode an iterable of Uint8Array chunks into an iterable of CBOR values + * @param source Iterable of Uint8Array chunks + * @param options Decode options + */ +export function* decodeIterable(...[source, options]: T extends CBORValue + ? ([Iterable]|[Iterable, DecodeOptions]) + : [Iterable, WithRequired>>, "onValue">] ): IterableIterator ``` +#### `AsyncDecodeOptions` + +```ts +export interface AsyncDecodeOptions extends Omit { + /** + * Function to remap/validate object keys while decoding + * (async version that works with AsyncIterable and streams) + * @param decodeKey Function to decode original object key (async) + * @param length Key length to validate pre-decoding + * @throws Error if length/key is invalid + * @returns An optional replacement key string + */ + onKey?: ( + decodeKey: () => Awaitable, + length: number + ) => Awaitable + + /** + * Function to validate/transform/replace values while decoding + * (async version that works with AsyncIterable and streams) + * @param decodeValue Function to decode value (async) + * @param length Value length/size to validate pre-decoding + * @param type Value type (e.g. 'number', 'string', 'Uint8Array'...) + * @param keyPath Array of keys describing the access path to this value + * @throws Error if length/value is invalid + * @returns An optional replacement value to use + */ + onValue?: ( + decodeValue: () => Awaitable, + length: number, + type: string, + keyPath: (string|number)[] + ) => Awaitable + + /** Callback function when the decoder requires more data */ + onPull?: () => void +} +``` + #### `decodeAsyncIterable` ```ts -/** Decode an async iterable of Uint8Array chunks into an async iterable of CBOR values */ -export async function* decodeAsyncIterable( - source: AsyncIterable, - options?: DecodeOptions, -): AsyncIterable +/** + * Decode an async iterable of Uint8Array chunks into an async iterable of CBOR values + * @param source Async iterable of Uint8Array chunks + * @param options Decode options + */ +export async function* decodeAsyncIterable(...[source, options]: T extends CBORValue + ? ([AsyncIterable]|[AsyncIterable, AsyncDecodeOptions]) + : [AsyncIterable, WithRequired>>, "onValue">] +): AsyncIterableIterator ``` #### `CBORDecoderStream` ```ts -/** Decode a Web Streams API ReadableStream. */ -export class CBORDecoderStream< - T extends CBORValue = CBORValue, -> extends TransformStream { - public constructor() +/** Decode a Web Streams API ReadableStream */ +export class CBORDecoderStream { + readable: ReadableStream + writable: WritableStream + + constructor(...[options]: T extends CBORValue + ? []|[AsyncDecodeOptions] + : [WithRequired>>, "onValue">] + ) } ``` @@ -265,6 +390,130 @@ declare class UnsafeIntegerError extends RangeError { | `7` (null) | `null` | | | `7` (undefined) | `undefined` | | +## Key mapping + +`onKey()` provides a way to remap object keys during encoding/decoding, allowing to reduce payload size or add a layer of obfuscation. It can also be used to check for abnormal key lengths, aborting the operation early and saving resources. + +Example: + +```ts +import { encode, decode } from "microcbor" + +const keys = ["firstName", "lastName", "emailAddress", ...] + +const encoded = encode( + { firstName: "John", lastName: "Doe", emailAddress: "john@example.com" }, + { onKey: (key) => keys.indexOf(key) + "" } +) + +const decoded = decode(encoded, { + onKey: (decodeKey, length) => { + if (length > 30) throw new Error("Key length is too long") + return keys[+decodeKey()] + } +}) +``` + +## Value validation and transformation + +`onValue()` enables the validation of values and their lengths during decoding, allowing to catch invalid data early in the process. Here is a very basic example on how it could be done using Zod: + +```ts +import { encode, decode, type CBORMap } from "microcbor" +import { z, type ZodObject, type ZodOptional, type ZodString } from "zod" + +const UserSchema = z.object({ + id: z.number().int().positive(), + email: z.email().min(5).max(19), + profile: z.object({ + age: z.number().int().min(0).max(150).optional(), + createdAt: z.number().positive() + }) +}) + +type User = z.infer + +const user: User = { + id: 1234, + email: "dummy@example.com", + profile: { + age: 30, + createdAt: Date.now() + } +} + +const encoded = encode(user) +console.log(encoded) + +const decoded = decode(encoded, { + onValue: (decodeValue, length, type, keyPath) => { + let zodVal: unknown = UserSchema + for (const key of keyPath) { + zodVal = (zodVal as ZodObject)?.shape?.[key] + while ((zodVal as ZodOptional)?.unwrap) + zodVal = (zodVal as ZodOptional).unwrap() + if (!zodVal) throw new Error("Unknown property") + } + + switch(type) { + case "string": { + const zodStr = zodVal as ZodString + if (zodStr.type !== type) throw new Error("Incorrect value type") + const { minimum, maximum } = zodStr._zod.bag + if (minimum !== undefined && length < minimum) throw new Error("String too short") + if (maximum !== undefined && length > maximum) throw new Error("String too long") + zodStr.parse(decodeValue() as string) + break + } + case "object": { + const zodObj = zodVal as ZodObject + const keys = Object.entries(zodObj.shape) + .filter(([_, val]) => !val.isOptional()) + .map(([key]) => key) + if (length < keys.length) throw new Error("Missing properties") + const object = decodeValue() as CBORMap + for (const key of keys) + if (!(key in object)) throw new Error("Missing properties") + break + } + default: + (zodVal as ZodObject).parse(decodeValue()) + } + } +}) +console.log(decoded) +``` + +`onValue()` can also be used to transform values on the fly. Here is an example for converting between Dates and timestamps: + +```ts +import { encode, decode } from "microcbor" + +const user = { + id: 1234, + email: "dummy@example.com", + profile: { + age: 30, + createdAt: new Date() + } +} + +const encoded = encode(user, { + onValue: (value, _keyPath) => { + if (value instanceof Date) return +value + } +}) +console.log(encoded) + +const decoded = decode(encoded, { + onValue: (decodeValue, _length, type, keyPath) => { + if (type === "number" && keyPath.join(".") === "profile.createdAt") + return new Date(+decodeValue()) + } +}) +console.log(decoded) +``` + ## Testing Tests use [AVA](https://github.com/avajs/ava) and live in the [test](./test/) directory. Tests use [node-cbor](https://github.com/hildjj/node-cbor/) to validate encoding results. More tests are always welcome!