diff --git a/benchmark/fs/bench-filehandle-pull-vs-webstream.js b/benchmark/fs/bench-filehandle-pull-vs-webstream.js new file mode 100644 index 00000000000000..5c81fe53b8bc50 --- /dev/null +++ b/benchmark/fs/bench-filehandle-pull-vs-webstream.js @@ -0,0 +1,196 @@ +// Compare FileHandle.createReadStream() vs readableWebStream() vs pull() +// reading a large file through two transforms: uppercase then gzip compress. +'use strict'; + +const common = require('../common.js'); +const fs = require('fs'); +const zlib = require('zlib'); +const { Transform, Writable, pipeline } = require('stream'); + +const tmpdir = require('../../test/common/tmpdir'); +tmpdir.refresh(); +const filename = tmpdir.resolve(`.removeme-benchmark-garbage-${process.pid}`); + +const bench = common.createBenchmark(main, { + api: ['classic', 'webstream', 'pull'], + filesize: [1024 * 1024, 16 * 1024 * 1024, 64 * 1024 * 1024], + n: [5], +}); + +function main({ api, filesize, n }) { + // Create the fixture file with repeating lowercase ASCII + const chunk = Buffer.alloc(Math.min(filesize, 64 * 1024), 'abcdefghij'); + const fd = fs.openSync(filename, 'w'); + let remaining = filesize; + while (remaining > 0) { + const toWrite = Math.min(remaining, chunk.length); + fs.writeSync(fd, chunk, 0, toWrite); + remaining -= toWrite; + } + fs.closeSync(fd); + + if (api === 'classic') { + benchClassic(n, filesize).then(() => cleanup()); + } else if (api === 'webstream') { + benchWebStream(n, filesize).then(() => cleanup()); + } else { + benchPull(n, filesize).then(() => cleanup()); + } +} + +function cleanup() { + try { fs.unlinkSync(filename); } catch { /* ignore */ } +} + +// --------------------------------------------------------------------------- +// Classic streams path: createReadStream -> Transform (upper) -> createGzip +// --------------------------------------------------------------------------- +async function benchClassic(n, filesize) { + // Warm up + await runClassic(); + + bench.start(); + let totalBytes = 0; + for (let i = 0; i < n; i++) { + totalBytes += await runClassic(); + } + bench.end(totalBytes / (1024 * 1024)); +} + +function runClassic() { + return new Promise((resolve, reject) => { + const rs = fs.createReadStream(filename); + + // Transform 1: uppercase + const upper = new Transform({ + transform(chunk, encoding, callback) { + const buf = Buffer.allocUnsafe(chunk.length); + for (let i = 0; i < chunk.length; i++) { + const b = chunk[i]; + buf[i] = (b >= 0x61 && b <= 0x7a) ? b - 0x20 : b; + } + callback(null, buf); + }, + }); + + // Transform 2: gzip + const gz = zlib.createGzip(); + + // Sink: count compressed bytes + let totalBytes = 0; + const sink = new Writable({ + write(chunk, encoding, callback) { + totalBytes += chunk.length; + callback(); + }, + }); + + pipeline(rs, upper, gz, sink, (err) => { + if (err) reject(err); + else resolve(totalBytes); + }); + }); +} + +// --------------------------------------------------------------------------- +// WebStream path: readableWebStream -> TransformStream (upper) -> CompressionStream +// --------------------------------------------------------------------------- +async function benchWebStream(n, filesize) { + // Warm up + await runWebStream(); + + bench.start(); + let totalBytes = 0; + for (let i = 0; i < n; i++) { + totalBytes += await runWebStream(); + } + bench.end(totalBytes / (1024 * 1024)); +} + +async function runWebStream() { + const fh = await fs.promises.open(filename, 'r'); + try { + const rs = fh.readableWebStream(); + + // Transform 1: uppercase + const upper = new TransformStream({ + transform(chunk, controller) { + const buf = new Uint8Array(chunk.length); + for (let i = 0; i < chunk.length; i++) { + const b = chunk[i]; + // a-z (0x61-0x7a) -> A-Z (0x41-0x5a) + buf[i] = (b >= 0x61 && b <= 0x7a) ? b - 0x20 : b; + } + controller.enqueue(buf); + }, + }); + + // Transform 2: gzip via CompressionStream + const compress = new CompressionStream('gzip'); + + const output = rs.pipeThrough(upper).pipeThrough(compress); + const reader = output.getReader(); + + let totalBytes = 0; + while (true) { + const { done, value } = await reader.read(); + if (done) break; + totalBytes += value.byteLength; + } + return totalBytes; + } finally { + await fh.close(); + } +} + +// --------------------------------------------------------------------------- +// New streams path: pull() with uppercase transform + gzip transform +// --------------------------------------------------------------------------- +async function benchPull(n, filesize) { + const { pull, compressGzip } = require('stream/new'); + + // Warm up + await runPull(pull, compressGzip); + + bench.start(); + let totalBytes = 0; + for (let i = 0; i < n; i++) { + totalBytes += await runPull(pull, compressGzip); + } + bench.end(totalBytes / (1024 * 1024)); +} + +async function runPull(pull, compressGzip) { + const fh = await fs.promises.open(filename, 'r'); + try { + // Stateless transform: uppercase each chunk in the batch + const upper = (chunks) => { + if (chunks === null) return null; + const out = new Array(chunks.length); + for (let j = 0; j < chunks.length; j++) { + const src = chunks[j]; + const buf = new Uint8Array(src.length); + for (let i = 0; i < src.length; i++) { + const b = src[i]; + buf[i] = (b >= 0x61 && b <= 0x7a) ? b - 0x20 : b; + } + out[j] = buf; + } + return out; + }; + + const readable = fh.pull(upper, compressGzip()); + + // Count bytes symmetrically with the classic path (no final + // concatenation into a single buffer). + let totalBytes = 0; + for await (const chunks of readable) { + for (let i = 0; i < chunks.length; i++) { + totalBytes += chunks[i].byteLength; + } + } + return totalBytes; + } finally { + await fh.close(); + } +} diff --git a/doc/api/fs.md b/doc/api/fs.md index 66d29bc80fbf18..81c9f4be4b0c67 100644 --- a/doc/api/fs.md +++ b/doc/api/fs.md @@ -377,6 +377,61 @@ added: v10.0.0 * Type: {number} The numeric file descriptor managed by the {FileHandle} object. +#### `filehandle.pull([...transforms][, options])` + + + +> Stability: 1 - Experimental + +* `...transforms` {Function|Object} Optional transforms to apply via + [`stream/new pull()`][]. +* `options` {Object} + * `signal` {AbortSignal} + * `autoClose` {boolean} Close the file handle when the stream ends. + **Default:** `false`. +* Returns: {AsyncIterable\} + +Return the file contents as an async iterable using the +[`node:stream/new`][] pull model. Reads are performed in 64 KB chunks. +If transforms are provided, they are applied via [`stream/new pull()`][]. + +The file handle is locked while the iterable is being consumed and unlocked +when iteration completes. + +```mjs +import { open } from 'node:fs/promises'; +import { text, compressGzip } from 'node:stream/new'; + +const fh = await open('input.txt', 'r'); + +// Read as text +console.log(await text(fh.pull({ autoClose: true }))); + +// Read with compression +const fh2 = await open('input.txt', 'r'); +const compressed = fh2.pull(compressGzip(), { autoClose: true }); +``` + +```cjs +const { open } = require('node:fs/promises'); +const { text, compressGzip } = require('node:stream/new'); + +async function run() { + const fh = await open('input.txt', 'r'); + + // Read as text + console.log(await text(fh.pull({ autoClose: true }))); + + // Read with compression + const fh2 = await open('input.txt', 'r'); + const compressed = fh2.pull(compressGzip(), { autoClose: true }); +} + +run().catch(console.error); +``` + #### `filehandle.read(buffer, offset, length, position)` + +> Stability: 1 - Experimental + +* `options` {Object} + * `autoClose` {boolean} Close the file handle when the writer ends. + **Default:** `false`. + * `start` {number} Byte offset to start writing at. **Default:** current + position (append). +* Returns: {Object} + * `write(chunk)` {Function} Returns {Promise\}. + * `writev(chunks)` {Function} Returns {Promise\}. Uses scatter/gather + I/O via a single `writev()` syscall. + * `end()` {Function} Returns {Promise\} total bytes written. + * `abort(reason)` {Function} Returns {Promise\}. + +Return a [`node:stream/new`][] writer backed by this file handle. + +The writer supports `Symbol.asyncDispose`, so it can be used with +`await using`. + +```mjs +import { open } from 'node:fs/promises'; +import { from, pipeTo, compressGzip } from 'node:stream/new'; + +const fh = await open('output.gz', 'w'); +const w = fh.writer({ autoClose: true }); +await pipeTo(from('Hello!'), compressGzip(), w); +await w.end(); +``` + +```cjs +const { open } = require('node:fs/promises'); +const { from, pipeTo, compressGzip } = require('node:stream/new'); + +async function run() { + const fh = await open('output.gz', 'w'); + const w = fh.writer({ autoClose: true }); + await pipeTo(from('Hello!'), compressGzip(), w); + await w.end(); +} + +run().catch(console.error); +``` + #### `filehandle[Symbol.asyncDispose]()` + +> Stability: 1 - Experimental + + + +The `node:stream/new` module provides a new streaming API built on iterables +rather than the event-driven `Readable`/`Writable`/`Transform` class hierarchy. + +Streams are represented as `AsyncIterable` (async) or +`Iterable` (sync). There are no base classes to extend -- any +object implementing the iterable protocol can participate. Transforms are plain +functions or objects with a `transform` method. + +Data flows in **batches** (`Uint8Array[]` per iteration) to amortize the cost +of async operations. + +```mjs +import { from, pull, text, compressGzip, decompressGzip } from 'node:stream/new'; + +// Compress and decompress a string +const compressed = pull(from('Hello, world!'), compressGzip()); +const result = await text(pull(compressed, decompressGzip())); +console.log(result); // 'Hello, world!' +``` + +```cjs +const { from, pull, text, compressGzip, decompressGzip } = require('node:stream/new'); + +async function run() { + // Compress and decompress a string + const compressed = pull(from('Hello, world!'), compressGzip()); + const result = await text(pull(compressed, decompressGzip())); + console.log(result); // 'Hello, world!' +} + +run().catch(console.error); +``` + +```mjs +import { open } from 'node:fs/promises'; +import { text, compressGzip, decompressGzip, pipeTo } from 'node:stream/new'; + +// Read a file, compress, write to another file +const src = await open('input.txt', 'r'); +const dst = await open('output.gz', 'w'); +await pipeTo(src.pull(), compressGzip(), dst.writer({ autoClose: true })); +await src.close(); + +// Read it back +const gz = await open('output.gz', 'r'); +console.log(await text(gz.pull(decompressGzip(), { autoClose: true }))); +``` + +```cjs +const { open } = require('node:fs/promises'); +const { text, compressGzip, decompressGzip, pipeTo } = require('node:stream/new'); + +async function run() { + // Read a file, compress, write to another file + const src = await open('input.txt', 'r'); + const dst = await open('output.gz', 'w'); + await pipeTo(src.pull(), compressGzip(), dst.writer({ autoClose: true })); + await src.close(); + + // Read it back + const gz = await open('output.gz', 'r'); + console.log(await text(gz.pull(decompressGzip(), { autoClose: true }))); +} + +run().catch(console.error); +``` + +## Concepts + +### Byte streams + +All data in the new streams API is represented as `Uint8Array` bytes. Strings +are automatically UTF-8 encoded when passed to `from()`, `push()`, or +`pipeTo()`. This removes ambiguity around encodings and enables zero-copy +transfers between streams and native code. + +### Batching + +Each iteration yields a **batch** -- an array of `Uint8Array` chunks +(`Uint8Array[]`). Batching amortizes the cost of `await` and Promise creation +across multiple chunks. A consumer that processes one chunk at a time can +simply iterate the inner array: + +```mjs +for await (const batch of source) { + for (const chunk of batch) { + handle(chunk); + } +} +``` + +```cjs +async function run() { + for await (const batch of source) { + for (const chunk of batch) { + handle(chunk); + } + } +} +``` + +### Transforms + +Transforms come in two forms: + +* **Stateless** -- a function `(chunks) => result` called once per batch. + Receives `Uint8Array[]` (or `null` as the flush signal). Returns + `Uint8Array[]`, `null`, or an iterable of chunks. + +* **Stateful** -- an object `{ transform(source) }` where `transform` is a + generator (sync or async) that receives the entire upstream iterable and + yields output. This form is used for compression, encryption, and any + transform that needs to buffer across batches. + +The flush signal (`null`) is sent after the source ends, giving transforms +a chance to emit trailing data (e.g., compression footers). + +```js +// Stateless: uppercase transform +const upper = (chunks) => { + if (chunks === null) return null; // flush + return chunks.map((c) => new TextEncoder().encode( + new TextDecoder().decode(c).toUpperCase(), + )); +}; + +// Stateful: line splitter +const lines = { + transform: async function*(source) { + let partial = ''; + for await (const chunks of source) { + if (chunks === null) { + if (partial) yield [new TextEncoder().encode(partial)]; + continue; + } + for (const chunk of chunks) { + const str = partial + new TextDecoder().decode(chunk); + const parts = str.split('\n'); + partial = parts.pop(); + for (const line of parts) { + yield [new TextEncoder().encode(`${line}\n`)]; + } + } + } + }, +}; +``` + +### Pull vs. push + +The API supports two models: + +* **Pull** -- data flows on demand. `pull()` and `pullSync()` create lazy + pipelines that only read from the source when the consumer iterates. + +* **Push** -- data is written explicitly. `push()` creates a writer/readable + pair with backpressure. The writer pushes data in; the readable is consumed + as an async iterable. + +### Writers + +A writer is any object with a `write(chunk)` method. Writers optionally +support `writev(chunks)` for batch writes (mapped to scatter/gather I/O where +available), `end()` to signal completion, and `abort(reason)` to signal +failure. + +## `require('node:stream/new')` + +All functions are available both as named exports and as properties of the +`Stream` namespace object: + +```mjs +// Named exports +import { from, pull, bytes, Stream } from 'node:stream/new'; + +// Namespace access +Stream.from('hello'); +``` + +```cjs +// Named exports +const { from, pull, bytes, Stream } = require('node:stream/new'); + +// Namespace access +Stream.from('hello'); +``` + +## Sources + +### `from(input)` + + + +* `input` {string|ArrayBuffer|ArrayBufferView|Iterable|AsyncIterable} +* Returns: {AsyncIterable\} + +Create an async byte stream from the given input. Strings are UTF-8 encoded. +`ArrayBuffer` and `ArrayBufferView` values are wrapped as `Uint8Array`. Arrays +and iterables are recursively flattened and normalized. + +Objects implementing `Symbol.for('Stream.toAsyncStreamable')` or +`Symbol.for('Stream.toStreamable')` are converted via those protocols. + +```mjs +import { Buffer } from 'node:buffer'; +import { from, text } from 'node:stream/new'; + +console.log(await text(from('hello'))); // 'hello' +console.log(await text(from(Buffer.from('hello')))); // 'hello' +``` + +```cjs +const { from, text } = require('node:stream/new'); + +async function run() { + console.log(await text(from('hello'))); // 'hello' + console.log(await text(from(Buffer.from('hello')))); // 'hello' +} + +run().catch(console.error); +``` + +### `fromSync(input)` + + + +* `input` {string|ArrayBuffer|ArrayBufferView|Iterable} +* Returns: {Iterable\} + +Synchronous version of [`from()`][]. Returns a sync iterable. Cannot accept +async iterables or promises. + +```mjs +import { fromSync, textSync } from 'node:stream/new'; + +console.log(textSync(fromSync('hello'))); // 'hello' +``` + +```cjs +const { fromSync, textSync } = require('node:stream/new'); + +console.log(textSync(fromSync('hello'))); // 'hello' +``` + +## Pipelines + +### `pipeTo(source[, ...transforms], writer[, options])` + + + +* `source` {AsyncIterable|Iterable} The data source. +* `...transforms` {Function|Object} Zero or more transforms to apply. +* `writer` {Object} Destination with `write(chunk)` method. +* `options` {Object} + * `signal` {AbortSignal} Abort the pipeline. + * `preventClose` {boolean} If `true`, do not call `writer.end()` when + the source ends. **Default:** `false`. + * `preventAbort` {boolean} If `true`, do not call `writer.abort()` on + error. **Default:** `false`. +* Returns: {Promise\} Total bytes written. + +Pipe a source through transforms into a writer. If the writer has a +`writev(chunks)` method, entire batches are passed in a single call (enabling +scatter/gather I/O). + +```mjs +import { from, pipeTo, compressGzip } from 'node:stream/new'; +import { open } from 'node:fs/promises'; + +const fh = await open('output.gz', 'w'); +const totalBytes = await pipeTo( + from('Hello, world!'), + compressGzip(), + fh.writer({ autoClose: true }), +); +``` + +```cjs +const { from, pipeTo, compressGzip } = require('node:stream/new'); +const { open } = require('node:fs/promises'); + +async function run() { + const fh = await open('output.gz', 'w'); + const totalBytes = await pipeTo( + from('Hello, world!'), + compressGzip(), + fh.writer({ autoClose: true }), + ); +} + +run().catch(console.error); +``` + +### `pipeToSync(source[, ...transforms], writer[, options])` + + + +* `source` {Iterable} The sync data source. +* `...transforms` {Function|Object} Zero or more sync transforms. +* `writer` {Object} Destination with `write(chunk)` method. +* `options` {Object} + * `preventClose` {boolean} **Default:** `false`. + * `preventAbort` {boolean} **Default:** `false`. +* Returns: {number} Total bytes written. + +Synchronous version of [`pipeTo()`][]. + +### `pull(source[, ...transforms][, options])` + + + +* `source` {AsyncIterable|Iterable} The data source. +* `...transforms` {Function|Object} Zero or more transforms to apply. +* `options` {Object} + * `signal` {AbortSignal} Abort the pipeline. +* Returns: {AsyncIterable\} + +Create a lazy async pipeline. Data is not read from `source` until the +returned iterable is consumed. Transforms are applied in order. + +```mjs +import { from, pull, text } from 'node:stream/new'; + +const upper = (chunks) => { + if (chunks === null) return null; + return chunks.map((c) => + new TextEncoder().encode(new TextDecoder().decode(c).toUpperCase()), + ); +}; + +const result = pull(from('hello'), upper); +console.log(await text(result)); // 'HELLO' +``` + +```cjs +const { from, pull, text } = require('node:stream/new'); + +const upper = (chunks) => { + if (chunks === null) return null; + return chunks.map((c) => + new TextEncoder().encode(new TextDecoder().decode(c).toUpperCase()), + ); +}; + +async function run() { + const result = pull(from('hello'), upper); + console.log(await text(result)); // 'HELLO' +} + +run().catch(console.error); +``` + +Using an `AbortSignal`: + +```mjs +import { pull } from 'node:stream/new'; + +const ac = new AbortController(); +const result = pull(source, transform, { signal: ac.signal }); +ac.abort(); // Pipeline throws AbortError on next iteration +``` + +```cjs +const { pull } = require('node:stream/new'); + +const ac = new AbortController(); +const result = pull(source, transform, { signal: ac.signal }); +ac.abort(); // Pipeline throws AbortError on next iteration +``` + +### `pullSync(source[, ...transforms])` + + + +* `source` {Iterable} The sync data source. +* `...transforms` {Function|Object} Zero or more sync transforms. +* Returns: {Iterable\} + +Synchronous version of [`pull()`][]. All transforms must be synchronous. + +## Push streams + +### `push([...transforms][, options])` + + + +* `...transforms` {Function|Object} Optional transforms applied to the + readable side. +* `options` {Object} + * `highWaterMark` {number} Maximum number of buffered slots before + backpressure is applied. **Default:** `1`. + * `backpressure` {string} Backpressure policy: `'strict'`, `'block'`, + `'drop-oldest'`, or `'drop-newest'`. **Default:** `'strict'`. + * `signal` {AbortSignal} Abort the stream. +* Returns: {Object} + * `writer` {PushWriter} The writer side. + * `readable` {AsyncIterable\} The readable side. + +Create a push stream with backpressure. The writer pushes data in; the +readable side is consumed as an async iterable. + +```mjs +import { push, text } from 'node:stream/new'; + +const { writer, readable } = push(); +writer.write('hello'); +writer.write(' world'); +writer.end(); + +console.log(await text(readable)); // 'hello world' +``` + +```cjs +const { push, text } = require('node:stream/new'); + +async function run() { + const { writer, readable } = push(); + writer.write('hello'); + writer.write(' world'); + writer.end(); + + console.log(await text(readable)); // 'hello world' +} + +run().catch(console.error); +``` + +#### Writer + +The writer returned by `push()` has the following methods: + +##### `writer.abort(reason)` + +* `reason` {Error} +* Returns: {Promise\} + +Abort the stream with an error. + +##### `writer.desiredSize` + +* {number|null} + +The number of buffer slots available before the high water mark is reached. +Returns `null` if the writer is closed or the consumer has disconnected. + +##### `writer.end()` + +* Returns: {Promise\} Total bytes written. + +Signal that no more data will be written. + +##### `writer.write(chunk)` + +* `chunk` {Uint8Array|string} +* Returns: {Promise\} + +Write a chunk. The promise resolves when buffer space is available. + +##### `writer.writeSync(chunk)` + +* `chunk` {Uint8Array|string} +* Returns: {boolean} `true` if the write was accepted, `false` if the + buffer is full. + +Synchronous write. Does not block; returns `false` if backpressure is active. + +##### `writer.writev(chunks)` + +* `chunks` {Uint8Array\[]|string\[]} +* Returns: {Promise\} + +Write multiple chunks as a single batch. + +##### `writer.writevSync(chunks)` + +* `chunks` {Uint8Array\[]|string\[]} +* Returns: {boolean} + +Synchronous batch write. + +## Consumers + +### `array(source[, options])` + + + +* `source` {AsyncIterable\|Iterable\} +* `options` {Object} + * `signal` {AbortSignal} + * `limit` {number} +* Returns: {Promise\} + +Collect all chunks as an array of `Uint8Array` values (without concatenating). + +### `arrayBuffer(source[, options])` + + + +* `source` {AsyncIterable\|Iterable\} +* `options` {Object} + * `signal` {AbortSignal} + * `limit` {number} +* Returns: {Promise\} + +Collect all bytes into an `ArrayBuffer`. + +### `arrayBufferSync(source[, options])` + + + +* `source` {Iterable\} +* `options` {Object} + * `limit` {number} +* Returns: {ArrayBuffer} + +Synchronous version of [`arrayBuffer()`][]. + +### `arraySync(source[, options])` + + + +* `source` {Iterable\} +* `options` {Object} + * `limit` {number} +* Returns: {Uint8Array\[]} + +Synchronous version of [`array()`][]. + +### `bytes(source[, options])` + + + +* `source` {AsyncIterable\|Iterable\} +* `options` {Object} + * `signal` {AbortSignal} + * `limit` {number} Maximum bytes to collect. Throws if exceeded. +* Returns: {Promise\} + +Collect all bytes from a stream into a single `Uint8Array`. + +```mjs +import { from, bytes } from 'node:stream/new'; + +const data = await bytes(from('hello')); +console.log(data); // Uint8Array(5) [ 104, 101, 108, 108, 111 ] +``` + +```cjs +const { from, bytes } = require('node:stream/new'); + +async function run() { + const data = await bytes(from('hello')); + console.log(data); // Uint8Array(5) [ 104, 101, 108, 108, 111 ] +} + +run().catch(console.error); +``` + +### `bytesSync(source[, options])` + + + +* `source` {Iterable\} +* `options` {Object} + * `limit` {number} +* Returns: {Uint8Array} + +Synchronous version of [`bytes()`][]. + +### `text(source[, options])` + + + +* `source` {AsyncIterable\|Iterable\} +* `options` {Object} + * `encoding` {string} Text encoding. **Default:** `'utf-8'`. + * `signal` {AbortSignal} + * `limit` {number} +* Returns: {Promise\} + +Collect all bytes and decode as text. + +```mjs +import { from, text } from 'node:stream/new'; + +console.log(await text(from('hello'))); // 'hello' +``` + +```cjs +const { from, text } = require('node:stream/new'); + +async function run() { + console.log(await text(from('hello'))); // 'hello' +} + +run().catch(console.error); +``` + +### `textSync(source[, options])` + + + +* `source` {Iterable\} +* `options` {Object} + * `encoding` {string} **Default:** `'utf-8'`. + * `limit` {number} +* Returns: {string} + +Synchronous version of [`text()`][]. + +## Utilities + +### `merge(...sources[, options])` + + + +* `...sources` {AsyncIterable\} Two or more async iterables. +* `options` {Object} + * `signal` {AbortSignal} +* Returns: {AsyncIterable\} + +Merge multiple async iterables by yielding batches in temporal order +(whichever source produces data first). All sources are consumed +concurrently. + +```mjs +import { from, merge, text } from 'node:stream/new'; + +const merged = merge(from('hello '), from('world')); +console.log(await text(merged)); // Order depends on timing +``` + +```cjs +const { from, merge, text } = require('node:stream/new'); + +async function run() { + const merged = merge(from('hello '), from('world')); + console.log(await text(merged)); // Order depends on timing +} + +run().catch(console.error); +``` + +### `ondrain(drainable)` + + + +* `drainable` {Object} An object implementing the drainable protocol. +* Returns: {Promise\|null} + +Wait for a drainable writer's backpressure to clear. Returns a promise that +resolves to `true` when the writer can accept more data, or `null` if the +object does not implement the drainable protocol. + +```mjs +import { push, ondrain } from 'node:stream/new'; + +const { writer, readable } = push({ highWaterMark: 2 }); +writer.writeSync('a'); +writer.writeSync('b'); + +// Buffer is full -- wait for drain +const canWrite = await ondrain(writer); +``` + +```cjs +const { push, ondrain } = require('node:stream/new'); + +async function run() { + const { writer, readable } = push({ highWaterMark: 2 }); + writer.writeSync('a'); + writer.writeSync('b'); + + // Buffer is full -- wait for drain + const canWrite = await ondrain(writer); +} + +run().catch(console.error); +``` + +### `tap(callback)` + + + +* `callback` {Function} `(chunks) => void` Called with each batch. +* Returns: {Function} A stateless transform. + +Create a pass-through transform that observes batches without modifying them. +Useful for logging, metrics, or debugging. + +```mjs +import { from, pull, text, tap } from 'node:stream/new'; + +const result = pull( + from('hello'), + tap((chunks) => console.log('Batch size:', chunks.length)), +); +console.log(await text(result)); +``` + +```cjs +const { from, pull, text, tap } = require('node:stream/new'); + +async function run() { + const result = pull( + from('hello'), + tap((chunks) => console.log('Batch size:', chunks.length)), + ); + console.log(await text(result)); +} + +run().catch(console.error); +``` + +### `tapSync(callback)` + + + +* `callback` {Function} +* Returns: {Function} + +Synchronous version of [`tap()`][]. + +## Multi-consumer + +### `broadcast([options])` + + + +* `options` {Object} + * `highWaterMark` {number} Buffer size in slots. **Default:** `16`. + * `backpressure` {string} `'strict'` or `'block'`. **Default:** `'strict'`. + * `signal` {AbortSignal} +* Returns: {Object} + * `writer` {BroadcastWriter} + * `broadcast` {Broadcast} + +Create a push-model multi-consumer broadcast channel. A single writer pushes +data to multiple consumers. Each consumer has an independent cursor into a +shared buffer. + +```mjs +import { broadcast, text } from 'node:stream/new'; + +const { writer, broadcast: bc } = broadcast(); + +const c1 = bc.push(); // Consumer 1 +const c2 = bc.push(); // Consumer 2 + +writer.write('hello'); +writer.end(); + +console.log(await text(c1)); // 'hello' +console.log(await text(c2)); // 'hello' +``` + +```cjs +const { broadcast, text } = require('node:stream/new'); + +async function run() { + const { writer, broadcast: bc } = broadcast(); + + const c1 = bc.push(); // Consumer 1 + const c2 = bc.push(); // Consumer 2 + + writer.write('hello'); + writer.end(); + + console.log(await text(c1)); // 'hello' + console.log(await text(c2)); // 'hello' +} + +run().catch(console.error); +``` + +#### `broadcast.cancel([reason])` + +* `reason` {Error} + +Cancel the broadcast. All consumers receive an error. + +#### `broadcast.push([...transforms][, options])` + +* `...transforms` {Function|Object} +* `options` {Object} + * `signal` {AbortSignal} +* Returns: {AsyncIterable\} + +Create a new consumer. Each consumer receives all data written to the +broadcast from the point of subscription onward. Optional transforms are +applied to this consumer's view of the data. + +#### `broadcast[Symbol.dispose]()` + +Alias for `broadcast.cancel()`. + +### `Broadcast.from(input[, options])` + + + +* `input` {AsyncIterable|Iterable|Broadcastable} +* `options` {Object} Same as `broadcast()`. +* Returns: {Object} `{ writer, broadcast }` + +Create a broadcast from an existing source. The source is consumed +automatically and pushed to all subscribers. + +### `share(source[, options])` + + + +* `source` {AsyncIterable} The source to share. +* `options` {Object} + * `highWaterMark` {number} Buffer size. **Default:** `16`. + * `backpressure` {string} `'strict'`, `'block'`, or `'drop-oldest'`. + **Default:** `'strict'`. +* Returns: {Share} + +Create a pull-model multi-consumer shared stream. Unlike `broadcast()`, the +source is only read when a consumer pulls. Multiple consumers share a single +buffer. + +```mjs +import { from, share, text } from 'node:stream/new'; + +const shared = share(from('hello')); + +const c1 = shared.pull(); +const c2 = shared.pull(); + +console.log(await text(c1)); // 'hello' +console.log(await text(c2)); // 'hello' +``` + +```cjs +const { from, share, text } = require('node:stream/new'); + +async function run() { + const shared = share(from('hello')); + + const c1 = shared.pull(); + const c2 = shared.pull(); + + console.log(await text(c1)); // 'hello' + console.log(await text(c2)); // 'hello' +} + +run().catch(console.error); +``` + +#### `share.cancel([reason])` + +* `reason` {Error} + +Cancel the share. All consumers receive an error. + +#### `share.pull([...transforms][, options])` + +* `...transforms` {Function|Object} +* `options` {Object} + * `signal` {AbortSignal} +* Returns: {AsyncIterable\} + +Create a new consumer of the shared source. + +#### `share[Symbol.dispose]()` + +Alias for `share.cancel()`. + +### `Share.from(input[, options])` + + + +* `input` {AsyncIterable|Shareable} +* `options` {Object} Same as `share()`. +* Returns: {Share} + +Create a share from an existing source. + +### `shareSync(source[, options])` + + + +* `source` {Iterable} The sync source to share. +* `options` {Object} + * `highWaterMark` {number} **Default:** `16`. + * `backpressure` {string} **Default:** `'strict'`. +* Returns: {SyncShare} + +Synchronous version of [`share()`][]. + +### `SyncShare.fromSync(input[, options])` + + + +* `input` {Iterable|SyncShareable} +* `options` {Object} +* Returns: {SyncShare} + +## Compression and decompression + +These transforms use the built-in zlib, Brotli, and Zstd compression +available in Node.js. Compression work is performed asynchronously, +overlapping with upstream I/O for maximum throughput. + +All compression transforms are stateful (they return `{ transform }` objects) +and can be passed to `pull()`, `pipeTo()`, or `push()`. + +### `compressBrotli([options])` + + + +* `options` {Object} + * `chunkSize` {number} **Default:** `16384`. + * `params` {Object} Key-value object where keys and values are + `zlib.constants` entries. The most important compressor parameters are: + * `BROTLI_PARAM_MODE` -- `BROTLI_MODE_GENERIC` (default), + `BROTLI_MODE_TEXT`, or `BROTLI_MODE_FONT`. + * `BROTLI_PARAM_QUALITY` -- ranges from `BROTLI_MIN_QUALITY` to + `BROTLI_MAX_QUALITY`. **Default:** `BROTLI_DEFAULT_QUALITY`. + * `BROTLI_PARAM_SIZE_HINT` -- expected input size. **Default:** `0` + (unknown). + * `BROTLI_PARAM_LGWIN` -- window size (log2). Ranges from + `BROTLI_MIN_WINDOW_BITS` to `BROTLI_MAX_WINDOW_BITS`. + * `BROTLI_PARAM_LGBLOCK` -- input block size (log2). + See the [Brotli compressor options][] in the zlib documentation for the + full list. + * `dictionary` {Buffer|TypedArray|DataView} +* Returns: {Object} A stateful transform. + +Create a Brotli compression transform. Output is compatible with +`zlib.brotliDecompress()` and `decompressBrotli()`. + +### `compressDeflate([options])` + + + +* `options` {Object} + * `chunkSize` {number} Output buffer size. **Default:** `16384`. + * `level` {number} Compression level (`0`-`9`). **Default:** `Z_DEFAULT_COMPRESSION`. + * `windowBits` {number} **Default:** `Z_DEFAULT_WINDOWBITS`. + * `memLevel` {number} **Default:** `Z_DEFAULT_MEMLEVEL`. + * `strategy` {number} **Default:** `Z_DEFAULT_STRATEGY`. + * `dictionary` {Buffer|TypedArray|DataView} +* Returns: {Object} A stateful transform. + +Create a deflate compression transform. Output is compatible with +`zlib.inflate()` and `decompressDeflate()`. + +### `compressGzip([options])` + + + +* `options` {Object} + * `chunkSize` {number} Output buffer size. **Default:** `16384`. + * `level` {number} Compression level (`0`-`9`). **Default:** `Z_DEFAULT_COMPRESSION`. + * `windowBits` {number} **Default:** `Z_DEFAULT_WINDOWBITS`. + * `memLevel` {number} **Default:** `Z_DEFAULT_MEMLEVEL`. + * `strategy` {number} **Default:** `Z_DEFAULT_STRATEGY`. + * `dictionary` {Buffer|TypedArray|DataView} +* Returns: {Object} A stateful transform. + +Create a gzip compression transform. Output is compatible with `zlib.gunzip()` +and `decompressGzip()`. + +```mjs +import { from, pull, bytes, text, compressGzip, decompressGzip } from 'node:stream/new'; + +const compressed = await bytes(pull(from('hello'), compressGzip())); +const original = await text(pull(from(compressed), decompressGzip())); +console.log(original); // 'hello' +``` + +```cjs +const { from, pull, bytes, text, compressGzip, decompressGzip } = require('node:stream/new'); + +async function run() { + const compressed = await bytes(pull(from('hello'), compressGzip())); + const original = await text(pull(from(compressed), decompressGzip())); + console.log(original); // 'hello' +} + +run().catch(console.error); +``` + +### `compressZstd([options])` + + + +* `options` {Object} + * `chunkSize` {number} **Default:** `16384`. + * `params` {Object} Key-value object where keys and values are + `zlib.constants` entries. The most important compressor parameters are: + * `ZSTD_c_compressionLevel` -- **Default:** `ZSTD_CLEVEL_DEFAULT` (3). + * `ZSTD_c_checksumFlag` -- generate a checksum. **Default:** `0`. + * `ZSTD_c_strategy` -- compression strategy. Values include + `ZSTD_fast`, `ZSTD_dfast`, `ZSTD_greedy`, `ZSTD_lazy`, + `ZSTD_lazy2`, `ZSTD_btlazy2`, `ZSTD_btopt`, `ZSTD_btultra`, + `ZSTD_btultra2`. + See the [Zstd compressor options][] in the zlib documentation for the + full list. + * `pledgedSrcSize` {number} Expected uncompressed size (optional hint). + * `dictionary` {Buffer|TypedArray|DataView} +* Returns: {Object} A stateful transform. + +Create a Zstandard compression transform. Output is compatible with +`zlib.zstdDecompress()` and `decompressZstd()`. + +### `decompressBrotli([options])` + + + +* `options` {Object} + * `chunkSize` {number} **Default:** `16384`. + * `params` {Object} Key-value object where keys and values are + `zlib.constants` entries. Available decompressor parameters: + * `BROTLI_DECODER_PARAM_DISABLE_RING_BUFFER_REALLOCATION` -- boolean + flag affecting internal memory allocation. + * `BROTLI_DECODER_PARAM_LARGE_WINDOW` -- boolean flag enabling "Large + Window Brotli" mode (not compatible with [RFC 7932][]). + See the [Brotli decompressor options][] in the zlib documentation for + details. + * `dictionary` {Buffer|TypedArray|DataView} +* Returns: {Object} A stateful transform. + +Create a Brotli decompression transform. + +### `decompressDeflate([options])` + + + +* `options` {Object} + * `chunkSize` {number} Output buffer size. **Default:** `16384`. + * `level` {number} Compression level (`0`-`9`). **Default:** `Z_DEFAULT_COMPRESSION`. + * `windowBits` {number} **Default:** `Z_DEFAULT_WINDOWBITS`. + * `memLevel` {number} **Default:** `Z_DEFAULT_MEMLEVEL`. + * `strategy` {number} **Default:** `Z_DEFAULT_STRATEGY`. + * `dictionary` {Buffer|TypedArray|DataView} +* Returns: {Object} A stateful transform. + +Create a deflate decompression transform. + +### `decompressGzip([options])` + + + +* `options` {Object} + * `chunkSize` {number} Output buffer size. **Default:** `16384`. + * `level` {number} Compression level (`0`-`9`). **Default:** `Z_DEFAULT_COMPRESSION`. + * `windowBits` {number} **Default:** `Z_DEFAULT_WINDOWBITS`. + * `memLevel` {number} **Default:** `Z_DEFAULT_MEMLEVEL`. + * `strategy` {number} **Default:** `Z_DEFAULT_STRATEGY`. + * `dictionary` {Buffer|TypedArray|DataView} +* Returns: {Object} A stateful transform. + +Create a gzip decompression transform. + +### `decompressZstd([options])` + + + +* `options` {Object} + * `chunkSize` {number} **Default:** `16384`. + * `params` {Object} Key-value object where keys and values are + `zlib.constants` entries. Available decompressor parameters: + * `ZSTD_d_windowLogMax` -- maximum window size (log2) the decompressor + will allocate. Limits memory usage against malicious input. + See the [Zstd decompressor options][] in the zlib documentation for + details. + * `dictionary` {Buffer|TypedArray|DataView} +* Returns: {Object} A stateful transform. + +Create a Zstandard decompression transform. + +## Protocol symbols + +These well-known symbols allow third-party objects to participate in the +streaming protocol without importing from `node:stream/new` directly. + +### `Stream.broadcastProtocol` + +* Value: `Symbol.for('Stream.broadcastProtocol')` + +Implement to make an object usable with `Broadcast.from()`. + +### `Stream.drainableProtocol` + +* Value: `Symbol.for('Stream.drainableProtocol')` + +Implement to make a writer compatible with `ondrain()`. The method should +return a promise that resolves when backpressure clears, or `null` if no +backpressure. + +### `Stream.shareProtocol` + +* Value: `Symbol.for('Stream.shareProtocol')` + +Implement to make an object usable with `Share.from()`. + +### `Stream.shareSyncProtocol` + +* Value: `Symbol.for('Stream.shareSyncProtocol')` + +Implement to make an object usable with `SyncShare.fromSync()`. + +### `Stream.toAsyncStreamable` + +* Value: `Symbol.for('Stream.toAsyncStreamable')` + +Async version of `toStreamable`. The method may return a promise. + +### `Stream.toStreamable` + +* Value: `Symbol.for('Stream.toStreamable')` + +Implement this symbol as a method that returns a sync-streamable value +(string, `Uint8Array`, `Iterable`, etc.). Used by `from()` and `fromSync()`. + +```js +const obj = { + [Symbol.for('Stream.toStreamable')]() { + return 'hello from custom object'; + }, +}; +// from(obj) and fromSync(obj) will UTF-8 encode the returned string. +``` + +[Brotli compressor options]: zlib.md#compressor-options +[Brotli decompressor options]: zlib.md#decompressor-options +[RFC 7932]: https://www.rfc-editor.org/rfc/rfc7932 +[Zstd compressor options]: zlib.md#compressor-options-1 +[Zstd decompressor options]: zlib.md#decompressor-options-1 +[`array()`]: #arraysource-options +[`arrayBuffer()`]: #arraybuffersource-options +[`bytes()`]: #bytessource-options +[`from()`]: #frominput +[`pipeTo()`]: #pipetosource-transforms-writer-options +[`pull()`]: #pullsource-transforms-options +[`share()`]: #sharesource-options +[`tap()`]: #tapcallback +[`text()`]: #textsource-options diff --git a/lib/internal/fs/promises.js b/lib/internal/fs/promises.js index 2f95c4b79e17fd..d258e85e631dc7 100644 --- a/lib/internal/fs/promises.js +++ b/lib/internal/fs/promises.js @@ -16,6 +16,7 @@ const { SafePromisePrototypeFinally, Symbol, SymbolAsyncDispose, + SymbolAsyncIterator, Uint8Array, } = primordials; @@ -40,6 +41,7 @@ const { ERR_INVALID_ARG_VALUE, ERR_INVALID_STATE, ERR_METHOD_NOT_IMPLEMENTED, + ERR_OPERATION_FAILED, }, } = require('internal/errors'); const { isArrayBufferView } = require('internal/util/types'); @@ -139,6 +141,17 @@ const lazyReadableStream = getLazy(() => require('internal/webstreams/readablestream').ReadableStream, ); +// Lazy loaded to avoid circular dependency with new streams. +let newStreamsPull; +let newStreamsParsePullArgs; +function lazyNewStreams() { + if (newStreamsPull === undefined) { + newStreamsPull = require('internal/streams/new/pull').pull; + newStreamsParsePullArgs = + require('internal/streams/new/utils').parsePullArgs; + } +} + // By the time the C++ land creates an error for a promise rejection (likely from a // libuv callback), there is already no JS frames on the stack. So we need to // wait until V8 resumes execution back to JS land before we have enough information @@ -341,6 +354,249 @@ class FileHandle extends EventEmitter { return readable; } + /** + * Return the file contents as an AsyncIterable using the + * new streams pull model. Optional transforms and options (including + * AbortSignal) may be provided as trailing arguments, mirroring the + * Stream.pull() signature. + * @param {...(Function|object)} args - Optional transforms and/or options + * @returns {AsyncIterable} + */ + pull(...args) { + if (this[kFd] === -1) + throw new ERR_INVALID_STATE('The FileHandle is closed'); + if (this[kClosePromise]) + throw new ERR_INVALID_STATE('The FileHandle is closing'); + if (this[kLocked]) + throw new ERR_INVALID_STATE('The FileHandle is locked'); + this[kLocked] = true; + + lazyNewStreams(); + const { transforms, options } = newStreamsParsePullArgs(args); + + const handle = this; + const fd = this[kFd]; + const autoClose = options?.autoClose ?? false; + const signal = options?.signal; + + const source = { + async *[SymbolAsyncIterator]() { + handle[kRef](); + const readSize = 65536; + try { + if (signal) { + // Signal-aware path + while (true) { + if (signal.aborted) { + throw signal.reason ?? + lazyDOMException('The operation was aborted', + 'AbortError'); + } + // Allocate a fresh buffer each iteration. At 64 KiB this + // bypasses the slab pool, so there is no reuse benefit. + // Yielding the buffer directly avoids the per-chunk copy + // that was needed when a single buffer was reused. + const buf = Buffer.allocUnsafe(readSize); + let bytesRead; + try { + bytesRead = + (await binding.read(fd, buf, 0, + readSize, -1, kUsePromises)) || 0; + } catch (err) { + ErrorCaptureStackTrace(err, handleErrorFromBinding); + throw err; + } + if (bytesRead === 0) break; + yield [bytesRead < readSize ? buf.subarray(0, bytesRead) : buf]; + } + } else { + // Fast path - no signal check per iteration + while (true) { + const buf = Buffer.allocUnsafe(readSize); + let bytesRead; + try { + bytesRead = + (await binding.read(fd, buf, 0, + readSize, -1, kUsePromises)) || 0; + } catch (err) { + ErrorCaptureStackTrace(err, handleErrorFromBinding); + throw err; + } + if (bytesRead === 0) break; + yield [bytesRead < readSize ? buf.subarray(0, bytesRead) : buf]; + } + } + } finally { + handle[kLocked] = false; + handle[kUnref](); + if (autoClose) { + await handle.close(); + } + } + }, + }; + + // If transforms provided, wrap with pull pipeline + if (transforms.length > 0) { + const pullArgs = [...transforms]; + if (options) { + ArrayPrototypePush(pullArgs, options); + } + return newStreamsPull(source, ...pullArgs); + } + return source; + } + + /** + * Return a new-streams Writer backed by this file handle. + * The writer uses direct binding.writeBuffer / binding.writeBuffers + * calls, bypassing the FileHandle.write() validation chain. + * + * Supports writev() for batch writes (single syscall per batch). + * Handles EAGAIN with retry (up to 5 attempts), matching WriteStream. + * @param {{ + * autoClose?: boolean; + * start?: number; + * }} [options] + * @returns {{ write, writev, end, abort }} + */ + writer(options) { + if (this[kFd] === -1) + throw new ERR_INVALID_STATE('The FileHandle is closed'); + if (this[kClosePromise]) + throw new ERR_INVALID_STATE('The FileHandle is closing'); + if (this[kLocked]) + throw new ERR_INVALID_STATE('The FileHandle is locked'); + this[kLocked] = true; + + const handle = this; + const fd = this[kFd]; + const autoClose = options?.autoClose ?? false; + let pos = options?.start ?? -1; + let totalBytesWritten = 0; + let closed = false; + + if (pos !== -1) { + validateInteger(pos, 'options.start', 0); + } + + handle[kRef](); + + // Write a single buffer with EAGAIN retry (up to 5 retries). + async function writeAll(buf, offset, length, position) { + let retries = 0; + while (length > 0) { + const bytesWritten = (await PromisePrototypeThen( + binding.writeBuffer(fd, buf, offset, length, position, + kUsePromises), + undefined, + handleErrorFromBinding, + )) || 0; + + if (bytesWritten === 0) { + if (++retries > 5) { + throw new ERR_OPERATION_FAILED('write failed after retries'); + } + } else { + retries = 0; + } + + totalBytesWritten += bytesWritten; + offset += bytesWritten; + length -= bytesWritten; + if (position >= 0) position += bytesWritten; + } + } + + // Writev with EAGAIN retry. On partial write, concatenates remaining + // buffers and falls back to writeAll (same approach as WriteStream). + async function writevAll(buffers, position) { + let totalSize = 0; + for (let i = 0; i < buffers.length; i++) { + totalSize += buffers[i].byteLength; + } + + let retries = 0; + while (totalSize > 0) { + const bytesWritten = (await PromisePrototypeThen( + binding.writeBuffers(fd, buffers, position, kUsePromises), + undefined, + handleErrorFromBinding, + )) || 0; + + if (bytesWritten === 0) { + if (++retries > 5) { + throw new ERR_OPERATION_FAILED('writev failed after retries'); + } + } else { + retries = 0; + } + + totalBytesWritten += bytesWritten; + totalSize -= bytesWritten; + if (position >= 0) position += bytesWritten; + + if (totalSize > 0) { + // Partial write - concatenate remaining and use writeAll. + const remaining = Buffer.concat(buffers); + const wrote = bytesWritten; + await writeAll(remaining, wrote, remaining.length - wrote, + position); + return; + } + } + } + + async function cleanup() { + if (closed) return; + closed = true; + handle[kLocked] = false; + handle[kUnref](); + if (autoClose) { + await handle.close(); + } + } + + return { + write(chunk) { + if (closed) { + return PromiseReject( + new ERR_INVALID_STATE('The writer is closed')); + } + const position = pos; + if (pos >= 0) pos += chunk.byteLength; + return writeAll(chunk, 0, chunk.byteLength, position); + }, + + writev(chunks) { + if (closed) { + return PromiseReject( + new ERR_INVALID_STATE('The writer is closed')); + } + const position = pos; + if (pos >= 0) { + for (let i = 0; i < chunks.length; i++) { + pos += chunks[i].byteLength; + } + } + return writevAll(chunks, position); + }, + + async end() { + await cleanup(); + return totalBytesWritten; + }, + + async abort(reason) { + await cleanup(); + }, + + async [SymbolAsyncDispose]() { + await cleanup(); + }, + }; + } + /** * @typedef {import('./streams').ReadStream * } ReadStream diff --git a/lib/internal/streams/new/broadcast.js b/lib/internal/streams/new/broadcast.js new file mode 100644 index 00000000000000..3c4a02bae6b6c0 --- /dev/null +++ b/lib/internal/streams/new/broadcast.js @@ -0,0 +1,587 @@ +'use strict'; + +// New Streams API - Broadcast +// +// Push-model multi-consumer streaming. A single writer can push data to +// multiple consumers. Each consumer has an independent cursor into a +// shared buffer. + +const { + ArrayIsArray, + ArrayPrototypeMap, + ArrayPrototypePush, + ArrayPrototypeShift, + ArrayPrototypeSlice, + ArrayPrototypeSplice, + Error, + MathMax, + Promise, + PromiseResolve, + SafeSet, + String, + SymbolAsyncIterator, + SymbolDispose, +} = primordials; + +const { TextEncoder } = require('internal/encoding'); + +const { + codes: { + ERR_INVALID_ARG_TYPE, + ERR_INVALID_STATE, + }, +} = require('internal/errors'); + +const { + broadcastProtocol, + drainableProtocol, +} = require('internal/streams/new/types'); + +const { + isAsyncIterable, + isSyncIterable, +} = require('internal/streams/new/from'); + +const { + pull: pullWithTransforms, +} = require('internal/streams/new/pull'); + +const encoder = new TextEncoder(); + +// ============================================================================= +// Argument Parsing +// ============================================================================= + +function isPushStreamOptions(value) { + return ( + value !== null && + typeof value === 'object' && + !('transform' in value) && + !('write' in value) + ); +} + +function parsePushArgs(args) { + if (args.length === 0) { + return { transforms: [], options: undefined }; + } + const last = args[args.length - 1]; + if (isPushStreamOptions(last)) { + return { + transforms: ArrayPrototypeSlice(args, 0, -1), + options: last, + }; + } + return { transforms: args, options: undefined }; +} + +// ============================================================================= +// Broadcast Implementation +// ============================================================================= + +class BroadcastImpl { + constructor(options) { + this._buffer = []; + this._bufferStart = 0; + this._consumers = new SafeSet(); + this._ended = false; + this._error = null; + this._cancelled = false; + this._options = options; + this._onBufferDrained = null; + } + + get consumerCount() { + return this._consumers.size; + } + + get bufferSize() { + return this._buffer.length; + } + + push(...args) { + const { transforms, options } = parsePushArgs(args); + const rawConsumer = this._createRawConsumer(); + + if (transforms.length > 0) { + if (options?.signal) { + return pullWithTransforms( + rawConsumer, ...transforms, { signal: options.signal }); + } + return pullWithTransforms(rawConsumer, ...transforms); + } + return rawConsumer; + } + + _createRawConsumer() { + const state = { + cursor: this._bufferStart + this._buffer.length, + resolve: null, + reject: null, + detached: false, + }; + + this._consumers.add(state); + const self = this; + + return { + [SymbolAsyncIterator]() { + return { + async next() { + if (state.detached) { + return { __proto__: null, done: true, value: undefined }; + } + + const bufferIndex = state.cursor - self._bufferStart; + if (bufferIndex < self._buffer.length) { + const chunk = self._buffer[bufferIndex]; + state.cursor++; + self._tryTrimBuffer(); + return { __proto__: null, done: false, value: chunk }; + } + + if (self._error) { + state.detached = true; + self._consumers.delete(state); + throw self._error; + } + + if (self._ended || self._cancelled) { + state.detached = true; + self._consumers.delete(state); + return { __proto__: null, done: true, value: undefined }; + } + + return new Promise((resolve, reject) => { + state.resolve = resolve; + state.reject = reject; + }); + }, + + async return() { + state.detached = true; + state.resolve = null; + state.reject = null; + self._consumers.delete(state); + self._tryTrimBuffer(); + return { __proto__: null, done: true, value: undefined }; + }, + + async throw() { + state.detached = true; + state.resolve = null; + state.reject = null; + self._consumers.delete(state); + self._tryTrimBuffer(); + return { __proto__: null, done: true, value: undefined }; + }, + }; + }, + }; + } + + cancel(reason) { + if (this._cancelled) return; + this._cancelled = true; + + if (reason) { + this._error = reason; + } + + for (const consumer of this._consumers) { + if (consumer.resolve) { + if (reason) { + consumer.reject?.(reason); + } else { + consumer.resolve({ done: true, value: undefined }); + } + consumer.resolve = null; + consumer.reject = null; + } + consumer.detached = true; + } + this._consumers.clear(); + } + + [SymbolDispose]() { + this.cancel(); + } + + // Internal methods called by Writer + + _write(chunk) { + if (this._ended || this._cancelled) return false; + + if (this._buffer.length >= this._options.highWaterMark) { + switch (this._options.backpressure) { + case 'strict': + case 'block': + return false; + case 'drop-oldest': + ArrayPrototypeShift(this._buffer); + this._bufferStart++; + for (const consumer of this._consumers) { + if (consumer.cursor < this._bufferStart) { + consumer.cursor = this._bufferStart; + } + } + break; + case 'drop-newest': + return true; + } + } + + ArrayPrototypePush(this._buffer, chunk); + this._notifyConsumers(); + return true; + } + + _end() { + if (this._ended) return; + this._ended = true; + + for (const consumer of this._consumers) { + if (consumer.resolve) { + const bufferIndex = consumer.cursor - this._bufferStart; + if (bufferIndex < this._buffer.length) { + const chunk = this._buffer[bufferIndex]; + consumer.cursor++; + consumer.resolve({ done: false, value: chunk }); + } else { + consumer.resolve({ done: true, value: undefined }); + } + consumer.resolve = null; + consumer.reject = null; + } + } + } + + _abort(reason) { + if (this._ended || this._error) return; + this._error = reason; + this._ended = true; + + for (const consumer of this._consumers) { + if (consumer.reject) { + consumer.reject(reason); + consumer.resolve = null; + consumer.reject = null; + } + } + } + + _getDesiredSize() { + if (this._ended || this._cancelled) return null; + return MathMax(0, this._options.highWaterMark - this._buffer.length); + } + + _canWrite() { + if (this._ended || this._cancelled) return false; + if ((this._options.backpressure === 'strict' || + this._options.backpressure === 'block') && + this._buffer.length >= this._options.highWaterMark) { + return false; + } + return true; + } + + _getMinCursor() { + let min = Infinity; + for (const consumer of this._consumers) { + if (consumer.cursor < min) { + min = consumer.cursor; + } + } + return min === Infinity ? + this._bufferStart + this._buffer.length : min; + } + + _tryTrimBuffer() { + const minCursor = this._getMinCursor(); + const trimCount = minCursor - this._bufferStart; + if (trimCount > 0) { + ArrayPrototypeSplice(this._buffer, 0, trimCount); + this._bufferStart = minCursor; + + if (this._onBufferDrained && + this._buffer.length < this._options.highWaterMark) { + this._onBufferDrained(); + } + } + } + + _notifyConsumers() { + for (const consumer of this._consumers) { + if (consumer.resolve) { + const bufferIndex = consumer.cursor - this._bufferStart; + if (bufferIndex < this._buffer.length) { + const chunk = this._buffer[bufferIndex]; + consumer.cursor++; + const resolve = consumer.resolve; + consumer.resolve = null; + consumer.reject = null; + resolve({ done: false, value: chunk }); + this._tryTrimBuffer(); + } + } + } + } +} + +// ============================================================================= +// BroadcastWriter +// ============================================================================= + +class BroadcastWriter { + constructor(broadcastImpl) { + this._broadcast = broadcastImpl; + this._totalBytes = 0; + this._closed = false; + this._aborted = false; + this._pendingWrites = []; + this._pendingDrains = []; + + this._broadcast._onBufferDrained = () => { + this._resolvePendingWrites(); + this._resolvePendingDrains(true); + }; + } + + [drainableProtocol]() { + const desired = this.desiredSize; + if (desired === null) return null; + if (desired > 0) return PromiseResolve(true); + return new Promise((resolve, reject) => { + ArrayPrototypePush(this._pendingDrains, { resolve, reject }); + }); + } + + get desiredSize() { + if (this._closed || this._aborted) return null; + return this._broadcast._getDesiredSize(); + } + + async write(chunk) { + return this.writev([chunk]); + } + + async writev(chunks) { + if (this._closed || this._aborted) { + throw new ERR_INVALID_STATE('Writer is closed'); + } + + const converted = ArrayPrototypeMap(chunks, (c) => + (typeof c === 'string' ? encoder.encode(c) : c)); + + if (this._broadcast._write(converted)) { + for (let i = 0; i < converted.length; i++) { + this._totalBytes += converted[i].byteLength; + } + return; + } + + const policy = this._broadcast._options?.backpressure ?? 'strict'; + const highWaterMark = this._broadcast._options?.highWaterMark ?? 16; + + if (policy === 'strict') { + if (this._pendingWrites.length >= highWaterMark) { + throw new ERR_INVALID_STATE( + 'Backpressure violation: too many pending writes. ' + + 'Await each write() call to respect backpressure.'); + } + return new Promise((resolve, reject) => { + ArrayPrototypePush(this._pendingWrites, + { chunk: converted, resolve, reject }); + }); + } + + // 'block' policy + return new Promise((resolve, reject) => { + ArrayPrototypePush(this._pendingWrites, + { chunk: converted, resolve, reject }); + }); + } + + writeSync(chunk) { + if (this._closed || this._aborted) return false; + if (!this._broadcast._canWrite()) return false; + const converted = + typeof chunk === 'string' ? encoder.encode(chunk) : chunk; + if (this._broadcast._write([converted])) { + this._totalBytes += converted.byteLength; + return true; + } + return false; + } + + writevSync(chunks) { + if (this._closed || this._aborted) return false; + if (!this._broadcast._canWrite()) return false; + const converted = ArrayPrototypeMap(chunks, (c) => + (typeof c === 'string' ? encoder.encode(c) : c)); + if (this._broadcast._write(converted)) { + for (let i = 0; i < converted.length; i++) { + this._totalBytes += converted[i].byteLength; + } + return true; + } + return false; + } + + async end() { + if (this._closed) return this._totalBytes; + this._closed = true; + this._broadcast._end(); + this._resolvePendingDrains(false); + return this._totalBytes; + } + + endSync() { + if (this._closed) return this._totalBytes; + this._closed = true; + this._broadcast._end(); + this._resolvePendingDrains(false); + return this._totalBytes; + } + + async abort(reason) { + if (this._aborted) return; + this._aborted = true; + this._closed = true; + const error = reason ?? new ERR_INVALID_STATE('Aborted'); + this._rejectPendingWrites(error); + this._rejectPendingDrains(error); + this._broadcast._abort(error); + } + + abortSync(reason) { + if (this._aborted) return true; + this._aborted = true; + this._closed = true; + const error = reason ?? new ERR_INVALID_STATE('Aborted'); + this._rejectPendingWrites(error); + this._rejectPendingDrains(error); + this._broadcast._abort(error); + return true; + } + + _resolvePendingWrites() { + while (this._pendingWrites.length > 0 && this._broadcast._canWrite()) { + const pending = ArrayPrototypeShift(this._pendingWrites); + if (this._broadcast._write(pending.chunk)) { + for (let i = 0; i < pending.chunk.length; i++) { + this._totalBytes += pending.chunk[i].byteLength; + } + pending.resolve(); + } else { + this._pendingWrites.unshift(pending); + break; + } + } + } + + _rejectPendingWrites(error) { + const writes = this._pendingWrites; + this._pendingWrites = []; + for (let i = 0; i < writes.length; i++) { + writes[i].reject(error); + } + } + + _resolvePendingDrains(canWrite) { + const drains = this._pendingDrains; + this._pendingDrains = []; + for (let i = 0; i < drains.length; i++) { + drains[i].resolve(canWrite); + } + } + + _rejectPendingDrains(error) { + const drains = this._pendingDrains; + this._pendingDrains = []; + for (let i = 0; i < drains.length; i++) { + drains[i].reject(error); + } + } +} + +// ============================================================================= +// Public API +// ============================================================================= + +/** + * Create a broadcast channel for push-model multi-consumer streaming. + * @param {{ highWaterMark?: number, backpressure?: string, signal?: AbortSignal }} [options] + * @returns {{ writer: Writer, broadcast: Broadcast }} + */ +function broadcast(options) { + const opts = { + highWaterMark: options?.highWaterMark ?? 16, + backpressure: options?.backpressure ?? 'strict', + signal: options?.signal, + }; + + const broadcastImpl = new BroadcastImpl(opts); + const writer = new BroadcastWriter(broadcastImpl); + + if (opts.signal) { + if (opts.signal.aborted) { + broadcastImpl.cancel(); + } else { + opts.signal.addEventListener('abort', () => { + broadcastImpl.cancel(); + }, { once: true }); + } + } + + return { writer, broadcast: broadcastImpl }; +} + +function isBroadcastable(value) { + return ( + value !== null && + typeof value === 'object' && + broadcastProtocol in value && + typeof value[broadcastProtocol] === 'function' + ); +} + +const Broadcast = { + from(input, options) { + if (isBroadcastable(input)) { + const bc = input[broadcastProtocol](options); + return { writer: {}, broadcast: bc }; + } + + const result = broadcast(options); + + (async () => { + try { + if (isAsyncIterable(input)) { + for await (const chunks of input) { + if (ArrayIsArray(chunks)) { + await result.writer.writev(chunks); + } + } + } else if (isSyncIterable(input)) { + for (const chunks of input) { + if (ArrayIsArray(chunks)) { + await result.writer.writev(chunks); + } + } + } + await result.writer.end(); + } catch (error) { + await result.writer.abort( + error instanceof Error ? error : new ERR_INVALID_ARG_TYPE('error', 'Error', String(error))); + } + })(); + + return result; + }, +}; + +module.exports = { + broadcast, + Broadcast, +}; diff --git a/lib/internal/streams/new/consumers.js b/lib/internal/streams/new/consumers.js new file mode 100644 index 00000000000000..b4ee67c88c779f --- /dev/null +++ b/lib/internal/streams/new/consumers.js @@ -0,0 +1,505 @@ +'use strict'; + +// New Streams API - Consumers & Utilities +// +// bytes(), text(), arrayBuffer() - collect entire stream +// tap(), tapSync() - observe without modifying +// merge() - temporal combining of sources +// ondrain() - backpressure drain utility + +const { + ArrayPrototypeFilter, + ArrayPrototypeMap, + ArrayPrototypePush, + ArrayPrototypeSlice, + SafePromiseAllReturnVoid, + SafePromiseRace, + SymbolAsyncIterator, +} = primordials; + +const { + codes: { + ERR_INVALID_ARG_TYPE, + ERR_OUT_OF_RANGE, + }, +} = require('internal/errors'); +const { TextDecoder } = require('internal/encoding'); +const { lazyDOMException } = require('internal/util'); + +const { + isAsyncIterable, + isSyncIterable, +} = require('internal/streams/new/from'); + +const { + concatBytes, +} = require('internal/streams/new/utils'); + +const { + drainableProtocol, +} = require('internal/streams/new/types'); + +// ============================================================================= +// Type Guards +// ============================================================================= + +function isMergeOptions(value) { + return ( + value !== null && + typeof value === 'object' && + !isAsyncIterable(value) && + !isSyncIterable(value) + ); +} + +// ============================================================================= +// Sync Consumers +// ============================================================================= + +/** + * Collect all bytes from a sync source. + * @param {Iterable} source + * @param {{ limit?: number }} [options] + * @returns {Uint8Array} + */ +function bytesSync(source, options) { + const limit = options?.limit; + const chunks = []; + let totalBytes = 0; + + for (const batch of source) { + for (let i = 0; i < batch.length; i++) { + const chunk = batch[i]; + if (limit !== undefined) { + totalBytes += chunk.byteLength; + if (totalBytes > limit) { + throw new ERR_OUT_OF_RANGE('totalBytes', `<= ${limit}`, totalBytes); + } + } + ArrayPrototypePush(chunks, chunk); + } + } + + return concatBytes(chunks); +} + +/** + * Collect and decode text from a sync source. + * @param {Iterable} source + * @param {{ encoding?: string, limit?: number }} [options] + * @returns {string} + */ +function textSync(source, options) { + const data = bytesSync(source, options); + const decoder = new TextDecoder(options?.encoding ?? 'utf-8', { + fatal: true, + ignoreBOM: true, + }); + return decoder.decode(data); +} + +/** + * Collect bytes as ArrayBuffer from a sync source. + * @param {Iterable} source + * @param {{ limit?: number }} [options] + * @returns {ArrayBuffer} + */ +function arrayBufferSync(source, options) { + const data = bytesSync(source, options); + if (data.byteOffset === 0 && data.byteLength === data.buffer.byteLength) { + return data.buffer; + } + return data.buffer.slice(data.byteOffset, + data.byteOffset + data.byteLength); +} + +/** + * Collect all chunks as an array from a sync source. + * @param {Iterable} source + * @param {{ limit?: number }} [options] + * @returns {Uint8Array[]} + */ +function arraySync(source, options) { + const limit = options?.limit; + const chunks = []; + let totalBytes = 0; + + for (const batch of source) { + for (let i = 0; i < batch.length; i++) { + const chunk = batch[i]; + if (limit !== undefined) { + totalBytes += chunk.byteLength; + if (totalBytes > limit) { + throw new ERR_OUT_OF_RANGE('totalBytes', `<= ${limit}`, totalBytes); + } + } + ArrayPrototypePush(chunks, chunk); + } + } + + return chunks; +} + +// ============================================================================= +// Async Consumers +// ============================================================================= + +/** + * Collect all bytes from an async or sync source. + * @param {AsyncIterable|Iterable} source + * @param {{ signal?: AbortSignal, limit?: number }} [options] + * @returns {Promise} + */ +async function bytes(source, options) { + const signal = options?.signal; + const limit = options?.limit; + + if (signal?.aborted) { + throw signal.reason ?? lazyDOMException('Aborted', 'AbortError'); + } + + const chunks = []; + + // Fast path: no signal and no limit + if (!signal && limit === undefined) { + if (isAsyncIterable(source)) { + for await (const batch of source) { + for (let i = 0; i < batch.length; i++) { + ArrayPrototypePush(chunks, batch[i]); + } + } + } else if (isSyncIterable(source)) { + for (const batch of source) { + for (let i = 0; i < batch.length; i++) { + ArrayPrototypePush(chunks, batch[i]); + } + } + } else { + throw new ERR_INVALID_ARG_TYPE('source', ['AsyncIterable', 'Iterable'], source); + } + return concatBytes(chunks); + } + + // Slow path: with signal or limit checks + let totalBytes = 0; + + if (isAsyncIterable(source)) { + for await (const batch of source) { + if (signal?.aborted) { + throw signal.reason ?? lazyDOMException('Aborted', 'AbortError'); + } + for (let i = 0; i < batch.length; i++) { + const chunk = batch[i]; + if (limit !== undefined) { + totalBytes += chunk.byteLength; + if (totalBytes > limit) { + throw new ERR_OUT_OF_RANGE('totalBytes', `<= ${limit}`, totalBytes); + } + } + ArrayPrototypePush(chunks, chunk); + } + } + } else if (isSyncIterable(source)) { + for (const batch of source) { + if (signal?.aborted) { + throw signal.reason ?? lazyDOMException('Aborted', 'AbortError'); + } + for (let i = 0; i < batch.length; i++) { + const chunk = batch[i]; + if (limit !== undefined) { + totalBytes += chunk.byteLength; + if (totalBytes > limit) { + throw new ERR_OUT_OF_RANGE('totalBytes', `<= ${limit}`, totalBytes); + } + } + ArrayPrototypePush(chunks, chunk); + } + } + } else { + throw new ERR_INVALID_ARG_TYPE('source', ['AsyncIterable', 'Iterable'], source); + } + + return concatBytes(chunks); +} + +/** + * Collect and decode text from an async or sync source. + * @param {AsyncIterable|Iterable} source + * @param {{ encoding?: string, signal?: AbortSignal, limit?: number }} [options] + * @returns {Promise} + */ +async function text(source, options) { + const data = await bytes(source, options); + const decoder = new TextDecoder(options?.encoding ?? 'utf-8', { + fatal: true, + ignoreBOM: true, + }); + return decoder.decode(data); +} + +/** + * Collect bytes as ArrayBuffer from an async or sync source. + * @param {AsyncIterable|Iterable} source + * @param {{ signal?: AbortSignal, limit?: number }} [options] + * @returns {Promise} + */ +async function arrayBuffer(source, options) { + const data = await bytes(source, options); + if (data.byteOffset === 0 && data.byteLength === data.buffer.byteLength) { + return data.buffer; + } + return data.buffer.slice(data.byteOffset, + data.byteOffset + data.byteLength); +} + +/** + * Collect all chunks as an array from an async or sync source. + * @param {AsyncIterable|Iterable} source + * @param {{ signal?: AbortSignal, limit?: number }} [options] + * @returns {Promise} + */ +async function array(source, options) { + const signal = options?.signal; + const limit = options?.limit; + + if (signal?.aborted) { + throw signal.reason ?? lazyDOMException('Aborted', 'AbortError'); + } + + const chunks = []; + + // Fast path: no signal and no limit + if (!signal && limit === undefined) { + if (isAsyncIterable(source)) { + for await (const batch of source) { + for (let i = 0; i < batch.length; i++) { + ArrayPrototypePush(chunks, batch[i]); + } + } + } else if (isSyncIterable(source)) { + for (const batch of source) { + for (let i = 0; i < batch.length; i++) { + ArrayPrototypePush(chunks, batch[i]); + } + } + } else { + throw new ERR_INVALID_ARG_TYPE('source', ['AsyncIterable', 'Iterable'], source); + } + return chunks; + } + + // Slow path + let totalBytes = 0; + + if (isAsyncIterable(source)) { + for await (const batch of source) { + if (signal?.aborted) { + throw signal.reason ?? lazyDOMException('Aborted', 'AbortError'); + } + for (let i = 0; i < batch.length; i++) { + const chunk = batch[i]; + if (limit !== undefined) { + totalBytes += chunk.byteLength; + if (totalBytes > limit) { + throw new ERR_OUT_OF_RANGE('totalBytes', `<= ${limit}`, totalBytes); + } + } + ArrayPrototypePush(chunks, chunk); + } + } + } else if (isSyncIterable(source)) { + for (const batch of source) { + if (signal?.aborted) { + throw signal.reason ?? lazyDOMException('Aborted', 'AbortError'); + } + for (let i = 0; i < batch.length; i++) { + const chunk = batch[i]; + if (limit !== undefined) { + totalBytes += chunk.byteLength; + if (totalBytes > limit) { + throw new ERR_OUT_OF_RANGE('totalBytes', `<= ${limit}`, totalBytes); + } + } + ArrayPrototypePush(chunks, chunk); + } + } + } else { + throw new ERR_INVALID_ARG_TYPE('source', ['AsyncIterable', 'Iterable'], source); + } + + return chunks; +} + +// ============================================================================= +// Tap Utilities +// ============================================================================= + +/** + * Create a pass-through transform that observes chunks without modifying them. + * @param {Function} callback + * @returns {Function} + */ +function tap(callback) { + return async (chunks) => { + await callback(chunks); + return chunks; + }; +} + +/** + * Create a sync pass-through transform that observes chunks. + * @param {Function} callback + * @returns {Function} + */ +function tapSync(callback) { + return (chunks) => { + callback(chunks); + return chunks; + }; +} + +// ============================================================================= +// Drain Utility +// ============================================================================= + +/** + * Wait for a drainable object's backpressure to clear. + * @param {object} drainable + * @returns {Promise|null} + */ +function ondrain(drainable) { + if ( + drainable === null || + drainable === undefined || + typeof drainable !== 'object' + ) { + return null; + } + + if ( + !(drainableProtocol in drainable) || + typeof drainable[drainableProtocol] !== 'function' + ) { + return null; + } + + try { + return drainable[drainableProtocol](); + } catch { + return null; + } +} + +// ============================================================================= +// Merge Utility +// ============================================================================= + +/** + * Merge multiple async iterables by yielding values in temporal order. + * @param {...(AsyncIterable|object)} args + * @returns {AsyncIterable} + */ +function merge(...args) { + let sources; + let options; + + if (args.length > 0 && isMergeOptions(args[args.length - 1])) { + options = args[args.length - 1]; + sources = ArrayPrototypeSlice(args, 0, -1); + } else { + sources = args; + } + + return { + async *[SymbolAsyncIterator]() { + const signal = options?.signal; + + if (signal?.aborted) { + throw signal.reason ?? lazyDOMException('Aborted', 'AbortError'); + } + + if (sources.length === 0) return; + + if (sources.length === 1) { + for await (const batch of sources[0]) { + if (signal?.aborted) { + throw signal.reason ?? lazyDOMException('Aborted', 'AbortError'); + } + yield batch; + } + return; + } + + // Multiple sources - race them + const states = ArrayPrototypeMap(sources, (source) => ({ + iterator: source[SymbolAsyncIterator](), + done: false, + pending: null, + })); + + const startIterator = (state, index) => { + if (!state.done && !state.pending) { + state.pending = state.iterator.next().then( + (result) => ({ index, result })); + } + }; + + // Start all + for (let i = 0; i < states.length; i++) { + startIterator(states[i], i); + } + + try { + while (true) { + if (signal?.aborted) { + throw signal.reason ?? lazyDOMException('Aborted', 'AbortError'); + } + + const pending = ArrayPrototypeFilter( + ArrayPrototypeMap(states, + (state) => state.pending), + (p) => p !== null); + + if (pending.length === 0) break; + + const { index, result } = await SafePromiseRace(pending); + + states[index].pending = null; + + if (result.done) { + states[index].done = true; + } else { + yield result.value; + startIterator(states[index], index); + } + } + } finally { + // Clean up: return all iterators + await SafePromiseAllReturnVoid(states, async (state) => { + if (!state.done && state.iterator.return) { + try { + await state.iterator.return(); + } catch { + // Ignore return errors + } + } + }); + } + }, + }; +} + +module.exports = { + bytes, + bytesSync, + text, + textSync, + arrayBuffer, + arrayBufferSync, + array, + arraySync, + tap, + tapSync, + merge, + ondrain, +}; diff --git a/lib/internal/streams/new/duplex.js b/lib/internal/streams/new/duplex.js new file mode 100644 index 00000000000000..272bf0a816dca2 --- /dev/null +++ b/lib/internal/streams/new/duplex.js @@ -0,0 +1,79 @@ +'use strict'; + +// New Streams API - Duplex Channel +// +// Creates a pair of connected channels where data written to one +// channel's writer appears in the other channel's readable. + +const { + SymbolAsyncDispose, +} = primordials; + +const { + push, +} = require('internal/streams/new/push'); + +/** + * Create a pair of connected duplex channels for bidirectional communication. + * @param {{ highWaterMark?: number, backpressure?: string, signal?: AbortSignal, + * a?: object, b?: object }} [options] + * @returns {[DuplexChannel, DuplexChannel]} + */ +function duplex(options) { + const { highWaterMark, backpressure, signal, a, b } = options ?? {}; + + // Channel A writes to B's readable (A->B direction) + const { writer: aWriter, readable: bReadable } = push({ + highWaterMark: a?.highWaterMark ?? highWaterMark, + backpressure: a?.backpressure ?? backpressure, + signal, + }); + + // Channel B writes to A's readable (B->A direction) + const { writer: bWriter, readable: aReadable } = push({ + highWaterMark: b?.highWaterMark ?? highWaterMark, + backpressure: b?.backpressure ?? backpressure, + signal, + }); + + let aWriterRef = aWriter; + let bWriterRef = bWriter; + + const channelA = { + get writer() { return aWriter; }, + readable: aReadable, + async close() { + if (aWriterRef === null) return; + const writer = aWriterRef; + aWriterRef = null; + if (!writer.endSync()) { + await writer.end(); + } + }, + [SymbolAsyncDispose]() { + return this.close(); + }, + }; + + const channelB = { + get writer() { return bWriter; }, + readable: bReadable, + async close() { + if (bWriterRef === null) return; + const writer = bWriterRef; + bWriterRef = null; + if (!writer.endSync()) { + await writer.end(); + } + }, + [SymbolAsyncDispose]() { + return this.close(); + }, + }; + + return [channelA, channelB]; +} + +module.exports = { + duplex, +}; diff --git a/lib/internal/streams/new/from.js b/lib/internal/streams/new/from.js new file mode 100644 index 00000000000000..552ac800e33d0e --- /dev/null +++ b/lib/internal/streams/new/from.js @@ -0,0 +1,575 @@ +'use strict'; + +// New Streams API - from() and fromSync() +// +// Creates normalized byte stream iterables from various input types. +// Handles recursive flattening of nested iterables and protocol conversions. + +const { + ArrayBuffer, + ArrayBufferIsView, + ArrayIsArray, + ArrayPrototypeEvery, + ArrayPrototypePush, + ArrayPrototypeSlice, + ObjectPrototypeToString, + Promise, + SymbolAsyncIterator, + SymbolIterator, + SymbolToPrimitive, + Uint8Array, +} = primordials; + +const { + codes: { + ERR_INVALID_ARG_TYPE, + }, +} = require('internal/errors'); +const { TextEncoder } = require('internal/encoding'); + +const { + toStreamable, + toAsyncStreamable, +} = require('internal/streams/new/types'); + +// Shared TextEncoder instance for string conversion. +const encoder = new TextEncoder(); + +// ============================================================================= +// Type Guards and Detection +// ============================================================================= + +/** + * Check if value is a primitive chunk (string, ArrayBuffer, or ArrayBufferView). + * @returns {boolean} + */ +function isPrimitiveChunk(value) { + if (typeof value === 'string') return true; + if (value instanceof ArrayBuffer) return true; + if (ArrayBufferIsView(value)) return true; + return false; +} + +/** + * Check if value implements ToStreamable protocol. + * @returns {boolean} + */ +function isToStreamable(value) { + return ( + value !== null && + typeof value === 'object' && + toStreamable in value && + typeof value[toStreamable] === 'function' + ); +} + +/** + * Check if value implements ToAsyncStreamable protocol. + * @returns {boolean} + */ +function isToAsyncStreamable(value) { + return ( + value !== null && + typeof value === 'object' && + toAsyncStreamable in value && + typeof value[toAsyncStreamable] === 'function' + ); +} + +/** + * Check if value is a sync iterable (has Symbol.iterator). + * @returns {boolean} + */ +function isSyncIterable(value) { + return ( + value !== null && + typeof value === 'object' && + SymbolIterator in value && + typeof value[SymbolIterator] === 'function' + ); +} + +/** + * Check if value is an async iterable (has Symbol.asyncIterator). + * @returns {boolean} + */ +function isAsyncIterable(value) { + return ( + value !== null && + typeof value === 'object' && + SymbolAsyncIterator in value && + typeof value[SymbolAsyncIterator] === 'function' + ); +} + +/** + * Check if object has a custom toString() (not Object.prototype.toString). + * @returns {boolean} + */ +function hasCustomToString(obj) { + const toString = obj.toString; + return typeof toString === 'function' && + toString !== ObjectPrototypeToString; +} + +/** + * Check if object has Symbol.toPrimitive. + * @returns {boolean} + */ +function hasToPrimitive(obj) { + return ( + SymbolToPrimitive in obj && + typeof obj[SymbolToPrimitive] === 'function' + ); +} + +// ============================================================================= +// Primitive Conversion +// ============================================================================= + +/** + * Convert a primitive chunk to Uint8Array. + * - string: UTF-8 encoded + * - ArrayBuffer: wrapped as Uint8Array view (no copy) + * - ArrayBufferView: converted to Uint8Array view of same memory + * @param {string|ArrayBuffer|ArrayBufferView} chunk + * @returns {Uint8Array} + */ +function primitiveToUint8Array(chunk) { + if (typeof chunk === 'string') { + return encoder.encode(chunk); + } + if (chunk instanceof ArrayBuffer) { + return new Uint8Array(chunk); + } + if (chunk instanceof Uint8Array) { + return chunk; + } + // Other ArrayBufferView types (Int8Array, DataView, etc.) + return new Uint8Array(chunk.buffer, chunk.byteOffset, chunk.byteLength); +} + +/** + * Try to coerce an object to string using custom methods. + * Returns null if object has no custom string coercion. + * @returns {string|null} + */ +function tryStringCoercion(obj) { + // Check for Symbol.toPrimitive first + if (hasToPrimitive(obj)) { + const toPrimitive = obj[SymbolToPrimitive]; + const result = toPrimitive.call(obj, 'string'); + if (typeof result === 'string') { + return result; + } + // toPrimitive returned non-string, fall through to toString + } + + // Check for custom toString + if (hasCustomToString(obj)) { + const result = obj.toString(); + return result; + } + + return null; +} + +// ============================================================================= +// Sync Normalization (for fromSync and sync contexts) +// ============================================================================= + +/** + * Normalize a sync streamable yield value to Uint8Array chunks. + * Recursively flattens arrays, iterables, and protocol conversions. + * @yields {Uint8Array} + */ +function* normalizeSyncValue(value) { + // Handle primitives + if (isPrimitiveChunk(value)) { + yield primitiveToUint8Array(value); + return; + } + + // Handle ToStreamable protocol + if (isToStreamable(value)) { + const result = value[toStreamable](); + yield* normalizeSyncValue(result); + return; + } + + // Handle arrays (which are also iterable, but check first for efficiency) + if (ArrayIsArray(value)) { + for (let i = 0; i < value.length; i++) { + yield* normalizeSyncValue(value[i]); + } + return; + } + + // Handle other sync iterables + if (isSyncIterable(value)) { + for (const item of value) { + yield* normalizeSyncValue(item); + } + return; + } + + // Try string coercion for objects with custom toString/toPrimitive + if (typeof value === 'object' && value !== null) { + const str = tryStringCoercion(value); + if (str !== null) { + yield encoder.encode(str); + return; + } + } + + // Reject: no valid conversion + throw new ERR_INVALID_ARG_TYPE( + 'value', + ['string', 'ArrayBuffer', 'ArrayBufferView', 'Iterable'], + value, + ); +} + +/** + * Check if value is already a Uint8Array[] batch (fast path). + * @returns {boolean} + */ +function isUint8ArrayBatch(value) { + if (!ArrayIsArray(value)) return false; + if (value.length === 0) return true; + // Check first element - if it's a Uint8Array, assume the rest are too + return value[0] instanceof Uint8Array; +} + +/** + * Normalize a sync streamable source, yielding batches of Uint8Array. + * @param {Iterable} source + * @yields {Uint8Array[]} + */ +function* normalizeSyncSource(source) { + for (const value of source) { + // Fast path 1: value is already a Uint8Array[] batch + if (isUint8ArrayBatch(value)) { + if (value.length > 0) { + yield value; + } + continue; + } + // Fast path 2: value is a single Uint8Array (very common) + if (value instanceof Uint8Array) { + yield [value]; + continue; + } + // Slow path: normalize the value + const batch = []; + for (const chunk of normalizeSyncValue(value)) { + ArrayPrototypePush(batch, chunk); + } + if (batch.length > 0) { + yield batch; + } + } +} + +// ============================================================================= +// Async Normalization (for from and async contexts) +// ============================================================================= + +/** + * Normalize an async streamable yield value to Uint8Array chunks. + * Recursively flattens arrays, iterables, async iterables, promises, + * and protocol conversions. + * @yields {Uint8Array} + */ +async function* normalizeAsyncValue(value) { + // Handle promises first + if (value instanceof Promise) { + const resolved = await value; + yield* normalizeAsyncValue(resolved); + return; + } + + // Handle primitives + if (isPrimitiveChunk(value)) { + yield primitiveToUint8Array(value); + return; + } + + // Handle ToAsyncStreamable protocol (check before ToStreamable) + if (isToAsyncStreamable(value)) { + const result = value[toAsyncStreamable](); + if (result instanceof Promise) { + yield* normalizeAsyncValue(await result); + } else { + yield* normalizeAsyncValue(result); + } + return; + } + + // Handle ToStreamable protocol + if (isToStreamable(value)) { + const result = value[toStreamable](); + yield* normalizeAsyncValue(result); + return; + } + + // Handle arrays (which are also iterable, but check first for efficiency) + if (ArrayIsArray(value)) { + for (let i = 0; i < value.length; i++) { + yield* normalizeAsyncValue(value[i]); + } + return; + } + + // Handle async iterables (check before sync iterables since some objects + // have both) + if (isAsyncIterable(value)) { + for await (const item of value) { + yield* normalizeAsyncValue(item); + } + return; + } + + // Handle sync iterables + if (isSyncIterable(value)) { + for (const item of value) { + yield* normalizeAsyncValue(item); + } + return; + } + + // Try string coercion for objects with custom toString/toPrimitive + if (typeof value === 'object' && value !== null) { + const str = tryStringCoercion(value); + if (str !== null) { + yield encoder.encode(str); + return; + } + } + + // Reject: no valid conversion + throw new ERR_INVALID_ARG_TYPE( + 'value', + ['string', 'ArrayBuffer', 'ArrayBufferView', 'Iterable', 'AsyncIterable'], + value, + ); +} + +/** + * Normalize an async streamable source, yielding batches of Uint8Array. + * @param {AsyncIterable|Iterable} source + * @yields {Uint8Array[]} + */ +async function* normalizeAsyncSource(source) { + // Prefer async iteration if available + if (isAsyncIterable(source)) { + for await (const value of source) { + // Fast path 1: value is already a Uint8Array[] batch + if (isUint8ArrayBatch(value)) { + if (value.length > 0) { + yield value; + } + continue; + } + // Fast path 2: value is a single Uint8Array (very common) + if (value instanceof Uint8Array) { + yield [value]; + continue; + } + // Slow path: normalize the value + const batch = []; + for await (const chunk of normalizeAsyncValue(value)) { + ArrayPrototypePush(batch, chunk); + } + if (batch.length > 0) { + yield batch; + } + } + return; + } + + // Fall back to sync iteration - batch all sync values together + if (isSyncIterable(source)) { + const batch = []; + + for (const value of source) { + // Fast path 1: value is already a Uint8Array[] batch + if (isUint8ArrayBatch(value)) { + // Flush any accumulated batch first + if (batch.length > 0) { + yield ArrayPrototypeSlice(batch); + batch.length = 0; + } + if (value.length > 0) { + yield value; + } + continue; + } + // Fast path 2: value is a single Uint8Array (very common) + if (value instanceof Uint8Array) { + ArrayPrototypePush(batch, value); + continue; + } + // Slow path: normalize the value - must flush and yield individually + if (batch.length > 0) { + yield ArrayPrototypeSlice(batch); + batch.length = 0; + } + const asyncBatch = []; + for await (const chunk of normalizeAsyncValue(value)) { + ArrayPrototypePush(asyncBatch, chunk); + } + if (asyncBatch.length > 0) { + yield asyncBatch; + } + } + + // Yield any remaining batched values + if (batch.length > 0) { + yield batch; + } + return; + } + + throw new ERR_INVALID_ARG_TYPE( + 'source', + ['Iterable', 'AsyncIterable'], + source, + ); +} + +// ============================================================================= +// Public API: from() and fromSync() +// ============================================================================= + +/** + * Create a SyncByteStreamReadable from a ByteInput or SyncStreamable. + * @param {string|ArrayBuffer|ArrayBufferView|Iterable} input + * @returns {Iterable} + */ +function fromSync(input) { + // Check for primitives first (ByteInput) + if (isPrimitiveChunk(input)) { + const chunk = primitiveToUint8Array(input); + return { + *[SymbolIterator]() { + yield [chunk]; + }, + }; + } + + // Fast path: Uint8Array[] - yield as a single batch + if (ArrayIsArray(input)) { + if (input.length === 0) { + return { + *[SymbolIterator]() { + // Empty - yield nothing + }, + }; + } + // Check if it's an array of Uint8Array (common case) + if (input[0] instanceof Uint8Array) { + const allUint8 = ArrayPrototypeEvery(input, + (item) => item instanceof Uint8Array); + if (allUint8) { + const batch = input; + return { + *[SymbolIterator]() { + yield batch; + }, + }; + } + } + } + + // Must be a SyncStreamable + if (!isSyncIterable(input)) { + throw new ERR_INVALID_ARG_TYPE( + 'input', + ['string', 'ArrayBuffer', 'ArrayBufferView', 'Iterable'], + input, + ); + } + + return { + *[SymbolIterator]() { + yield* normalizeSyncSource(input); + }, + }; +} + +/** + * Create a ByteStreamReadable from a ByteInput or Streamable. + * @param {string|ArrayBuffer|ArrayBufferView|Iterable|AsyncIterable} input + * @returns {AsyncIterable} + */ +function from(input) { + // Check for primitives first (ByteInput) + if (isPrimitiveChunk(input)) { + const chunk = primitiveToUint8Array(input); + return { + async *[SymbolAsyncIterator]() { + yield [chunk]; + }, + }; + } + + // Fast path: Uint8Array[] - yield as a single batch + if (ArrayIsArray(input)) { + if (input.length === 0) { + return { + async *[SymbolAsyncIterator]() { + // Empty - yield nothing + }, + }; + } + if (input[0] instanceof Uint8Array) { + const allUint8 = ArrayPrototypeEvery(input, + (item) => item instanceof Uint8Array); + if (allUint8) { + const batch = input; + return { + async *[SymbolAsyncIterator]() { + yield batch; + }, + }; + } + } + } + + // Must be a Streamable (sync or async iterable) + if (!isSyncIterable(input) && !isAsyncIterable(input)) { + throw new ERR_INVALID_ARG_TYPE( + 'input', + ['string', 'ArrayBuffer', 'ArrayBufferView', 'Iterable', 'AsyncIterable'], + input, + ); + } + + return { + async *[SymbolAsyncIterator]() { + yield* normalizeAsyncSource(input); + }, + }; +} + +// ============================================================================= +// Exports +// ============================================================================= + +module.exports = { + from, + fromSync, + // Internal helpers used by pull, pipeTo, etc. + normalizeSyncValue, + normalizeSyncSource, + normalizeAsyncValue, + normalizeAsyncSource, + isPrimitiveChunk, + isToStreamable, + isToAsyncStreamable, + isSyncIterable, + isAsyncIterable, + isUint8ArrayBatch, + primitiveToUint8Array, +}; diff --git a/lib/internal/streams/new/pull.js b/lib/internal/streams/new/pull.js new file mode 100644 index 00000000000000..8f80ee19585784 --- /dev/null +++ b/lib/internal/streams/new/pull.js @@ -0,0 +1,710 @@ +'use strict'; + +// New Streams API - Pull Pipeline +// +// pull(), pullSync(), pipeTo(), pipeToSync() +// Pull-through pipelines with transforms. Data flows on-demand from source +// through transforms to consumer. + +const { + ArrayIsArray, + ArrayPrototypePush, + ArrayPrototypeSlice, + Error, + Promise, + SafePromiseAllReturnVoid, + String, + SymbolAsyncIterator, + SymbolIterator, + Uint8Array, +} = primordials; + +const { + codes: { + ERR_INVALID_ARG_TYPE, + ERR_INVALID_ARG_VALUE, + ERR_OPERATION_FAILED, + }, +} = require('internal/errors'); +const { TextEncoder } = require('internal/encoding'); +const { lazyDOMException } = require('internal/util'); + +const { + normalizeAsyncSource, + normalizeSyncSource, + isSyncIterable, + isAsyncIterable, + isUint8ArrayBatch, +} = require('internal/streams/new/from'); + +const { + isPullOptions, + parsePullArgs, +} = require('internal/streams/new/utils'); + +// Shared TextEncoder instance for string conversion. +const encoder = new TextEncoder(); + +// ============================================================================= +// Type Guards and Helpers +// ============================================================================= + +/** + * Check if a value is a TransformObject (has transform property). + * @returns {boolean} + */ +function isTransformObject(value) { + return ( + value !== null && + typeof value === 'object' && + 'transform' in value && + typeof value.transform === 'function' + ); +} + +/** + * Check if a value is a Writer (has write method). + * @returns {boolean} + */ +function isWriter(value) { + return ( + value !== null && + typeof value === 'object' && + 'write' in value && + typeof value.write === 'function' + ); +} + +/** + * Parse variadic arguments for pipeTo/pipeToSync. + * Returns { transforms, writer, options } + * @returns {object} + */ +function parsePipeToArgs(args) { + if (args.length === 0) { + throw new ERR_INVALID_ARG_VALUE('args', args, 'pipeTo requires a writer argument'); + } + + let options; + let writerIndex = args.length - 1; + + // Check if last arg is options + const last = args[args.length - 1]; + if (isPullOptions(last) && !isWriter(last)) { + options = last; + writerIndex = args.length - 2; + } + + if (writerIndex < 0) { + throw new ERR_INVALID_ARG_VALUE('args', args, 'pipeTo requires a writer argument'); + } + + const writer = args[writerIndex]; + if (!isWriter(writer)) { + throw new ERR_INVALID_ARG_TYPE('writer', 'object with a write method', writer); + } + + return { + transforms: ArrayPrototypeSlice(args, 0, writerIndex), + writer, + options, + }; +} + +// ============================================================================= +// Transform Output Flattening +// ============================================================================= + +/** + * Flatten transform yield to Uint8Array chunks (sync). + * @yields {Uint8Array} + */ +function* flattenTransformYieldSync(value) { + if (value instanceof Uint8Array) { + yield value; + return; + } + if (typeof value === 'string') { + yield encoder.encode(value); + return; + } + // Must be Iterable + if (isSyncIterable(value)) { + for (const item of value) { + yield* flattenTransformYieldSync(item); + } + return; + } + throw new ERR_INVALID_ARG_TYPE('value', ['Uint8Array', 'string', 'Iterable'], value); +} + +/** + * Flatten transform yield to Uint8Array chunks (async). + * @yields {Uint8Array} + */ +async function* flattenTransformYieldAsync(value) { + if (value instanceof Uint8Array) { + yield value; + return; + } + if (typeof value === 'string') { + yield encoder.encode(value); + return; + } + // Check for async iterable first + if (isAsyncIterable(value)) { + for await (const item of value) { + yield* flattenTransformYieldAsync(item); + } + return; + } + // Must be sync Iterable + if (isSyncIterable(value)) { + for (const item of value) { + yield* flattenTransformYieldAsync(item); + } + return; + } + throw new ERR_INVALID_ARG_TYPE('value', ['Uint8Array', 'string', 'Iterable', 'AsyncIterable'], value); +} + +/** + * Process transform result (sync). + * @yields {Uint8Array[]} + */ +function* processTransformResultSync(result) { + if (result === null) { + return; + } + if (ArrayIsArray(result) && result.length > 0 && + result[0] instanceof Uint8Array) { + // Fast path: Uint8Array[] + if (result.length > 0) { + yield result; + } + return; + } + // Iterable or Generator + if (isSyncIterable(result)) { + const batch = []; + for (const item of result) { + for (const chunk of flattenTransformYieldSync(item)) { + ArrayPrototypePush(batch, chunk); + } + } + if (batch.length > 0) { + yield batch; + } + return; + } + throw new ERR_INVALID_ARG_TYPE('result', ['Array', 'Iterable'], result); +} + +/** + * Process transform result (async). + * @yields {Uint8Array[]} + */ +async function* processTransformResultAsync(result) { + // Handle Promise + if (result instanceof Promise) { + const resolved = await result; + yield* processTransformResultAsync(resolved); + return; + } + if (result === null) { + return; + } + if (ArrayIsArray(result) && + (result.length === 0 || result[0] instanceof Uint8Array)) { + // Fast path: Uint8Array[] + if (result.length > 0) { + yield result; + } + return; + } + // Check for async iterable/generator first + if (isAsyncIterable(result)) { + const batch = []; + for await (const item of result) { + // Fast path: item is already Uint8Array + if (item instanceof Uint8Array) { + ArrayPrototypePush(batch, item); + continue; + } + // Slow path: flatten the item + for await (const chunk of flattenTransformYieldAsync(item)) { + ArrayPrototypePush(batch, chunk); + } + } + if (batch.length > 0) { + yield batch; + } + return; + } + // Sync Iterable or Generator + if (isSyncIterable(result)) { + const batch = []; + for (const item of result) { + // Fast path: item is already Uint8Array + if (item instanceof Uint8Array) { + ArrayPrototypePush(batch, item); + continue; + } + // Slow path: flatten the item + for await (const chunk of flattenTransformYieldAsync(item)) { + ArrayPrototypePush(batch, chunk); + } + } + if (batch.length > 0) { + yield batch; + } + return; + } + throw new ERR_INVALID_ARG_TYPE('result', ['Array', 'Iterable', 'AsyncIterable'], result); +} + +// ============================================================================= +// Sync Pipeline Implementation +// ============================================================================= + +/** + * Apply a single stateless sync transform to a source. + * @yields {Uint8Array[]} + */ +function* applyStatelessSyncTransform(source, transform) { + for (const chunks of source) { + const result = transform(chunks); + yield* processTransformResultSync(result); + } +} + +/** + * Apply a single stateful sync transform to a source. + * @yields {Uint8Array[]} + */ +function* applyStatefulSyncTransform(source, transform) { + const output = transform(source); + const batch = []; + for (const item of output) { + for (const chunk of flattenTransformYieldSync(item)) { + ArrayPrototypePush(batch, chunk); + } + } + if (batch.length > 0) { + yield batch; + } +} + +/** + * Wrap sync source to add null flush signal at end. + * @yields {Uint8Array[]} + */ +function* withFlushSignalSync(source) { + for (const batch of source) { + yield batch; + } + yield null; // Flush signal +} + +/** + * Create a sync pipeline from source through transforms. + * @yields {Uint8Array[]} + */ +function* createSyncPipeline(source, transforms) { + // Normalize source + let current = withFlushSignalSync(normalizeSyncSource(source)); + + // Apply transforms - Object = stateful, function = stateless + for (let i = 0; i < transforms.length; i++) { + const transform = transforms[i]; + if (isTransformObject(transform)) { + current = applyStatefulSyncTransform(current, transform.transform); + } else { + current = applyStatelessSyncTransform(current, transform); + } + } + + // Yield results (filter out null from final output) + for (const batch of current) { + if (batch !== null) { + yield batch; + } + } +} + +// ============================================================================= +// Async Pipeline Implementation +// ============================================================================= + +/** + * Apply a single stateless async transform to a source. + * @yields {Uint8Array[]} + */ +async function* applyStatelessAsyncTransform(source, transform) { + for await (const chunks of source) { + const result = transform(chunks); + // Fast path: result is already Uint8Array[] (common case) + if (result === null) continue; + if (isUint8ArrayBatch(result)) { + if (result.length > 0) { + yield result; + } + continue; + } + // Handle Promise of Uint8Array[] + if (result instanceof Promise) { + const resolved = await result; + if (resolved === null) continue; + if (isUint8ArrayBatch(resolved)) { + if (resolved.length > 0) { + yield resolved; + } + continue; + } + // Fall through to slow path + yield* processTransformResultAsync(resolved); + continue; + } + // Fast path: sync generator/iterable - collect all yielded items + if (isSyncIterable(result) && !isAsyncIterable(result)) { + const batch = []; + for (const item of result) { + if (isUint8ArrayBatch(item)) { + for (let i = 0; i < item.length; i++) { + ArrayPrototypePush(batch, item[i]); + } + } else if (item instanceof Uint8Array) { + ArrayPrototypePush(batch, item); + } else if (item !== null && item !== undefined) { + for await (const chunk of flattenTransformYieldAsync(item)) { + ArrayPrototypePush(batch, chunk); + } + } + } + if (batch.length > 0) { + yield batch; + } + continue; + } + // Slow path for other types + yield* processTransformResultAsync(result); + } +} + +/** + * Apply a single stateful async transform to a source. + * @yields {Uint8Array[]} + */ +async function* applyStatefulAsyncTransform(source, transform) { + const output = transform(source); + for await (const item of output) { + // Fast path: item is already a Uint8Array[] batch (e.g. compression transforms) + if (isUint8ArrayBatch(item)) { + if (item.length > 0) { + yield item; + } + continue; + } + // Fast path: single Uint8Array + if (item instanceof Uint8Array) { + yield [item]; + continue; + } + // Slow path: flatten arbitrary transform yield + const batch = []; + for await (const chunk of flattenTransformYieldAsync(item)) { + ArrayPrototypePush(batch, chunk); + } + if (batch.length > 0) { + yield batch; + } + } +} + +/** + * Wrap async source to add null flush signal at end. + * @yields {Uint8Array[]} + */ +async function* withFlushSignalAsync(source) { + for await (const batch of source) { + yield batch; + } + yield null; // Flush signal +} + +/** + * Convert sync iterable to async iterable. + * @yields {Uint8Array[]} + */ +async function* syncToAsync(source) { + for (const item of source) { + yield item; + } +} + +/** + * Create an async pipeline from source through transforms. + * @yields {Uint8Array[]} + */ +async function* createAsyncPipeline(source, transforms, signal) { + // Check for abort + if (signal?.aborted) { + throw signal.reason ?? lazyDOMException('Aborted', 'AbortError'); + } + + // Normalize source to async + let normalized; + if (isAsyncIterable(source)) { + normalized = normalizeAsyncSource(source); + } else if (isSyncIterable(source)) { + normalized = syncToAsync(normalizeSyncSource(source)); + } else { + throw new ERR_INVALID_ARG_TYPE('source', ['Iterable', 'AsyncIterable'], source); + } + + // Fast path: no transforms, just yield normalized source directly + if (transforms.length === 0) { + for await (const batch of normalized) { + if (signal?.aborted) { + throw signal.reason ?? lazyDOMException('Aborted', 'AbortError'); + } + yield batch; + } + return; + } + + // Add flush signal + let current = withFlushSignalAsync(normalized); + + // Track stateful transforms for abort handling + const statefulTransforms = []; + + try { + // Apply transforms - Object = stateful, function = stateless + for (let i = 0; i < transforms.length; i++) { + const transform = transforms[i]; + if (isTransformObject(transform)) { + ArrayPrototypePush(statefulTransforms, transform); + current = applyStatefulAsyncTransform(current, transform.transform); + } else { + current = applyStatelessAsyncTransform(current, transform); + } + } + + // Yield results (filter out null from final output) + for await (const batch of current) { + // Check for abort on each iteration + if (signal?.aborted) { + throw signal.reason ?? lazyDOMException('Aborted', 'AbortError'); + } + if (batch !== null) { + yield batch; + } + } + } catch (error) { + // Abort all stateful transforms + for (let i = 0; i < statefulTransforms.length; i++) { + const transformObj = statefulTransforms[i]; + if (transformObj.abort) { + try { + await transformObj.abort( + error instanceof Error ? error : new ERR_OPERATION_FAILED(String(error))); + } catch { + // Ignore abort errors + } + } + } + throw error; + } +} + +// ============================================================================= +// Public API: pull() and pullSync() +// ============================================================================= + +/** + * Create a sync pull-through pipeline with transforms. + * @param {Iterable} source - The sync streamable source + * @param {...Function} transforms - Variadic transforms + * @returns {Iterable} + */ +function pullSync(source, ...transforms) { + return { + *[SymbolIterator]() { + yield* createSyncPipeline(source, transforms); + }, + }; +} + +/** + * Create an async pull-through pipeline with transforms. + * @param {Iterable|AsyncIterable} source - The streamable source + * @param {...(Function|object)} args - Transforms, with optional PullOptions + * as last argument + * @returns {AsyncIterable} + */ +function pull(source, ...args) { + const { transforms, options } = parsePullArgs(args); + + return { + async *[SymbolAsyncIterator]() { + yield* createAsyncPipeline(source, transforms, options?.signal); + }, + }; +} + +// ============================================================================= +// Public API: pipeTo() and pipeToSync() +// ============================================================================= + +/** + * Write a sync source through transforms to a sync writer. + * @param {Iterable} source + * @param {...(Function|object)} args - Transforms, writer, and optional options + * @returns {number} Total bytes written + */ +function pipeToSync(source, ...args) { + const { transforms, writer, options } = parsePipeToArgs(args); + + // Handle transform-writer + const finalTransforms = ArrayPrototypeSlice(transforms); + if (isTransformObject(writer)) { + ArrayPrototypePush(finalTransforms, writer); + } + + // Create pipeline + const pipeline = finalTransforms.length > 0 ? + createSyncPipeline( + { [SymbolIterator]: () => source[SymbolIterator]() }, + finalTransforms) : + source; + + let totalBytes = 0; + + try { + for (const batch of pipeline) { + for (let i = 0; i < batch.length; i++) { + const chunk = batch[i]; + writer.write(chunk); + totalBytes += chunk.byteLength; + } + } + + if (!options?.preventClose) { + writer.end(); + } + } catch (error) { + if (!options?.preventAbort) { + writer.abort(error instanceof Error ? error : new ERR_OPERATION_FAILED(String(error))); + } + throw error; + } + + return totalBytes; +} + +/** + * Write an async source through transforms to a writer. + * @param {AsyncIterable|Iterable} source + * @param {...(Function|object)} args - Transforms, writer, and optional options + * @returns {Promise} Total bytes written + */ +async function pipeTo(source, ...args) { + const { transforms, writer, options } = parsePipeToArgs(args); + + // Handle transform-writer + const finalTransforms = ArrayPrototypeSlice(transforms); + if (isTransformObject(writer)) { + ArrayPrototypePush(finalTransforms, writer); + } + + const signal = options?.signal; + + // Check for abort + if (signal?.aborted) { + throw signal.reason ?? lazyDOMException('Aborted', 'AbortError'); + } + + let totalBytes = 0; + const hasWritev = typeof writer.writev === 'function'; + + // Helper to write a batch efficiently + const writeBatch = async (batch) => { + if (hasWritev && batch.length > 1) { + await writer.writev(batch); + for (let i = 0; i < batch.length; i++) { + totalBytes += batch[i].byteLength; + } + } else { + const promises = []; + for (let i = 0; i < batch.length; i++) { + const chunk = batch[i]; + const result = writer.write(chunk); + if (result !== undefined) { + ArrayPrototypePush(promises, result); + } + totalBytes += chunk.byteLength; + } + if (promises.length > 0) { + await SafePromiseAllReturnVoid(promises); + } + } + }; + + try { + // Fast path: no transforms - iterate directly + if (finalTransforms.length === 0) { + if (isAsyncIterable(source)) { + for await (const batch of source) { + if (signal?.aborted) { + throw signal.reason ?? + lazyDOMException('Aborted', 'AbortError'); + } + await writeBatch(batch); + } + } else { + for (const batch of source) { + if (signal?.aborted) { + throw signal.reason ?? + lazyDOMException('Aborted', 'AbortError'); + } + await writeBatch(batch); + } + } + } else { + // Slow path: has transforms - need pipeline + const streamableSource = isAsyncIterable(source) ? + { [SymbolAsyncIterator]: () => source[SymbolAsyncIterator]() } : + { [SymbolIterator]: () => source[SymbolIterator]() }; + + const pipeline = createAsyncPipeline( + streamableSource, finalTransforms, signal); + + for await (const batch of pipeline) { + if (signal?.aborted) { + throw signal.reason ?? lazyDOMException('Aborted', 'AbortError'); + } + await writeBatch(batch); + } + } + + if (!options?.preventClose) { + await writer.end(); + } + } catch (error) { + if (!options?.preventAbort) { + await writer.abort( + error instanceof Error ? error : new ERR_OPERATION_FAILED(String(error))); + } + throw error; + } + + return totalBytes; +} + +module.exports = { + pull, + pullSync, + pipeTo, + pipeToSync, +}; diff --git a/lib/internal/streams/new/push.js b/lib/internal/streams/new/push.js new file mode 100644 index 00000000000000..0260cf10ba2600 --- /dev/null +++ b/lib/internal/streams/new/push.js @@ -0,0 +1,519 @@ +'use strict'; + +// New Streams API - Push Stream Implementation +// +// Creates a bonded pair of writer and async iterable for push-based streaming +// with built-in backpressure. + +const { + ArrayPrototypePush, + ArrayPrototypeShift, + ArrayPrototypeSlice, + Error, + MathMax, + Promise, + PromiseResolve, + SymbolAsyncIterator, +} = primordials; + +const { + codes: { + ERR_INVALID_STATE, + }, +} = require('internal/errors'); +const { lazyDOMException } = require('internal/util'); + +const { + drainableProtocol, +} = require('internal/streams/new/types'); + +const { + toUint8Array, +} = require('internal/streams/new/utils'); + +const { + pull: pullWithTransforms, +} = require('internal/streams/new/pull'); + +// ============================================================================= +// PushQueue - Internal Queue with Chunk-Based Backpressure +// ============================================================================= + +class PushQueue { + constructor(options = {}) { + /** Buffered chunks (each slot is from one write/writev call) */ + this._slots = []; + /** Pending writes waiting for buffer space */ + this._pendingWrites = []; + /** Pending reads waiting for data */ + this._pendingReads = []; + /** Pending drains waiting for backpressure to clear */ + this._pendingDrains = []; + /** Writer state: 'open' | 'closed' | 'errored' */ + this._writerState = 'open'; + /** Consumer state: 'active' | 'returned' | 'thrown' */ + this._consumerState = 'active'; + /** Error that closed the stream */ + this._error = null; + /** Total bytes written */ + this._bytesWritten = 0; + + /** Configuration */ + this._highWaterMark = options.highWaterMark ?? 1; + this._backpressure = options.backpressure ?? 'strict'; + this._signal = options.signal; + this._abortHandler = undefined; + + if (this._signal) { + if (this._signal.aborted) { + this.abort(this._signal.reason instanceof Error ? + this._signal.reason : + lazyDOMException('Aborted', 'AbortError')); + } else { + this._abortHandler = () => { + this.abort(this._signal.reason instanceof Error ? + this._signal.reason : + lazyDOMException('Aborted', 'AbortError')); + }; + this._signal.addEventListener('abort', this._abortHandler, + { once: true }); + } + } + } + + // =========================================================================== + // Writer Methods + // =========================================================================== + + /** + * Get slots available before hitting highWaterMark. + * Returns null if writer is closed/errored or consumer has terminated. + * @returns {number | null} + */ + get desiredSize() { + if (this._writerState !== 'open' || this._consumerState !== 'active') { + return null; + } + return MathMax(0, this._highWaterMark - this._slots.length); + } + + /** + * Check if a sync write would be accepted. + * @returns {boolean} + */ + canWriteSync() { + if (this._writerState !== 'open') return false; + if (this._consumerState !== 'active') return false; + if ((this._backpressure === 'strict' || + this._backpressure === 'block') && + this._slots.length >= this._highWaterMark) { + return false; + } + return true; + } + + /** + * Write chunks synchronously if possible. + * Returns true if write completed, false if buffer is full. + * @returns {boolean} + */ + writeSync(chunks) { + if (this._writerState !== 'open') return false; + if (this._consumerState !== 'active') return false; + + if (this._slots.length >= this._highWaterMark) { + switch (this._backpressure) { + case 'strict': + case 'block': + return false; + case 'drop-oldest': + if (this._slots.length > 0) { + ArrayPrototypeShift(this._slots); + } + break; + case 'drop-newest': + // Discard this write, but return true + for (let i = 0; i < chunks.length; i++) { + this._bytesWritten += chunks[i].byteLength; + } + return true; + } + } + + ArrayPrototypePush(this._slots, chunks); + for (let i = 0; i < chunks.length; i++) { + this._bytesWritten += chunks[i].byteLength; + } + + this._resolvePendingReads(); + return true; + } + + /** + * Write chunks asynchronously. + */ + async writeAsync(chunks) { + if (this._writerState !== 'open') { + throw new ERR_INVALID_STATE('Writer is closed'); + } + if (this._consumerState !== 'active') { + throw this._consumerState === 'thrown' && this._error ? + this._error : + new ERR_INVALID_STATE('Stream closed by consumer'); + } + + // Try sync first + if (this.writeSync(chunks)) { + return; + } + + // Buffer is full + switch (this._backpressure) { + case 'strict': + if (this._pendingWrites.length >= this._highWaterMark) { + throw new ERR_INVALID_STATE( + 'Backpressure violation: too many pending writes. ' + + 'Await each write() call to respect backpressure.'); + } + return new Promise((resolve, reject) => { + ArrayPrototypePush(this._pendingWrites, + { chunks, resolve, reject }); + }); + case 'block': + return new Promise((resolve, reject) => { + ArrayPrototypePush(this._pendingWrites, + { chunks, resolve, reject }); + }); + default: + throw new ERR_INVALID_STATE( + 'Unexpected: writeSync should have handled non-strict policy'); + } + } + + /** + * Signal end of stream. Returns total bytes written. + * @returns {number} + */ + end() { + if (this._writerState !== 'open') { + return this._bytesWritten; + } + + this._writerState = 'closed'; + this._cleanup(); + this._resolvePendingReads(); + this._rejectPendingWrites(new ERR_INVALID_STATE('Writer closed')); + this._resolvePendingDrains(false); + return this._bytesWritten; + } + + /** + * Signal error/abort. + */ + abort(reason) { + if (this._writerState === 'errored') return; + + this._writerState = 'errored'; + this._error = reason ?? new ERR_INVALID_STATE('Aborted'); + this._cleanup(); + this._rejectPendingReads(this._error); + this._rejectPendingWrites(this._error); + this._rejectPendingDrains(this._error); + } + + get totalBytesWritten() { + return this._bytesWritten; + } + + /** + * Wait for backpressure to clear (desiredSize > 0). + */ + waitForDrain() { + return new Promise((resolve, reject) => { + ArrayPrototypePush(this._pendingDrains, { resolve, reject }); + }); + } + + // =========================================================================== + // Consumer Methods + // =========================================================================== + + async read() { + // If there's data in the buffer, return it immediately + if (this._slots.length > 0) { + const result = this._drain(); + this._resolvePendingWrites(); + return { __proto__: null, value: result, done: false }; + } + + if (this._writerState === 'closed') { + return { __proto__: null, value: undefined, done: true }; + } + + if (this._writerState === 'errored' && this._error) { + throw this._error; + } + + return new Promise((resolve, reject) => { + ArrayPrototypePush(this._pendingReads, { resolve, reject }); + }); + } + + consumerReturn() { + if (this._consumerState !== 'active') return; + this._consumerState = 'returned'; + this._cleanup(); + this._rejectPendingWrites(new ERR_INVALID_STATE('Stream closed by consumer')); + } + + consumerThrow(error) { + if (this._consumerState !== 'active') return; + this._consumerState = 'thrown'; + this._error = error; + this._cleanup(); + this._rejectPendingWrites(error); + } + + // =========================================================================== + // Private Methods + // =========================================================================== + + _drain() { + const result = []; + for (let i = 0; i < this._slots.length; i++) { + const slot = this._slots[i]; + for (let j = 0; j < slot.length; j++) { + ArrayPrototypePush(result, slot[j]); + } + } + this._slots = []; + return result; + } + + _resolvePendingReads() { + while (this._pendingReads.length > 0) { + if (this._slots.length > 0) { + const pending = ArrayPrototypeShift(this._pendingReads); + const result = this._drain(); + this._resolvePendingWrites(); + pending.resolve({ value: result, done: false }); + } else if (this._writerState === 'closed') { + const pending = ArrayPrototypeShift(this._pendingReads); + pending.resolve({ value: undefined, done: true }); + } else if (this._writerState === 'errored' && this._error) { + const pending = ArrayPrototypeShift(this._pendingReads); + pending.reject(this._error); + } else { + break; + } + } + } + + _resolvePendingWrites() { + while (this._pendingWrites.length > 0 && + this._slots.length < this._highWaterMark) { + const pending = ArrayPrototypeShift(this._pendingWrites); + ArrayPrototypePush(this._slots, pending.chunks); + for (let i = 0; i < pending.chunks.length; i++) { + this._bytesWritten += pending.chunks[i].byteLength; + } + pending.resolve(); + } + + if (this._slots.length < this._highWaterMark) { + this._resolvePendingDrains(true); + } + } + + _resolvePendingDrains(canWrite) { + const drains = this._pendingDrains; + this._pendingDrains = []; + for (let i = 0; i < drains.length; i++) { + drains[i].resolve(canWrite); + } + } + + _rejectPendingDrains(error) { + const drains = this._pendingDrains; + this._pendingDrains = []; + for (let i = 0; i < drains.length; i++) { + drains[i].reject(error); + } + } + + _rejectPendingReads(error) { + const reads = this._pendingReads; + this._pendingReads = []; + for (let i = 0; i < reads.length; i++) { + reads[i].reject(error); + } + } + + _rejectPendingWrites(error) { + const writes = this._pendingWrites; + this._pendingWrites = []; + for (let i = 0; i < writes.length; i++) { + writes[i].reject(error); + } + } + + _cleanup() { + if (this._signal && this._abortHandler) { + this._signal.removeEventListener('abort', this._abortHandler); + this._abortHandler = undefined; + } + } +} + +// ============================================================================= +// PushWriter Implementation +// ============================================================================= + +class PushWriter { + constructor(queue) { + this._queue = queue; + } + + [drainableProtocol]() { + const desired = this.desiredSize; + if (desired === null) return null; + if (desired > 0) return PromiseResolve(true); + return this._queue.waitForDrain(); + } + + get desiredSize() { + return this._queue.desiredSize; + } + + async write(chunk) { + const bytes = toUint8Array(chunk); + await this._queue.writeAsync([bytes]); + } + + async writev(chunks) { + const bytes = []; + for (let i = 0; i < chunks.length; i++) { + ArrayPrototypePush(bytes, toUint8Array(chunks[i])); + } + await this._queue.writeAsync(bytes); + } + + writeSync(chunk) { + if (!this._queue.canWriteSync()) return false; + const bytes = toUint8Array(chunk); + return this._queue.writeSync([bytes]); + } + + writevSync(chunks) { + if (!this._queue.canWriteSync()) return false; + const bytes = []; + for (let i = 0; i < chunks.length; i++) { + ArrayPrototypePush(bytes, toUint8Array(chunks[i])); + } + return this._queue.writeSync(bytes); + } + + async end() { + return this._queue.end(); + } + + endSync() { + return this._queue.end(); + } + + async abort(reason) { + this._queue.abort(reason); + } + + abortSync(reason) { + this._queue.abort(reason); + return true; + } +} + +// ============================================================================= +// Readable Implementation +// ============================================================================= + +function createReadable(queue) { + return { + [SymbolAsyncIterator]() { + return { + async next() { + return queue.read(); + }, + async return() { + queue.consumerReturn(); + return { __proto__: null, value: undefined, done: true }; + }, + async throw(error) { + queue.consumerThrow(error); + return { __proto__: null, value: undefined, done: true }; + }, + }; + }, + }; +} + +// ============================================================================= +// Stream.push() Factory +// ============================================================================= + +function isOptions(arg) { + return ( + typeof arg === 'object' && + arg !== null && + !('transform' in arg) + ); +} + +function parseArgs(args) { + if (args.length === 0) { + return { transforms: [], options: {} }; + } + + const last = args[args.length - 1]; + if (isOptions(last)) { + return { + transforms: ArrayPrototypeSlice(args, 0, -1), + options: last, + }; + } + + return { + transforms: args, + options: {}, + }; +} + +/** + * Create a push stream with optional transforms. + * @param {...(Function|object)} args - Transforms, then options (optional) + * @returns {{ writer: Writer, readable: AsyncIterable }} + */ +function push(...args) { + const { transforms, options } = parseArgs(args); + + const queue = new PushQueue(options); + const writer = new PushWriter(queue); + const rawReadable = createReadable(queue); + + // Apply transforms lazily if provided + let readable; + if (transforms.length > 0) { + if (options.signal) { + readable = pullWithTransforms( + rawReadable, ...transforms, { signal: options.signal }); + } else { + readable = pullWithTransforms(rawReadable, ...transforms); + } + } else { + readable = rawReadable; + } + + return { writer, readable }; +} + +module.exports = { + push, +}; diff --git a/lib/internal/streams/new/share.js b/lib/internal/streams/new/share.js new file mode 100644 index 00000000000000..f7f4353393a36e --- /dev/null +++ b/lib/internal/streams/new/share.js @@ -0,0 +1,636 @@ +'use strict'; + +// New Streams API - Share +// +// Pull-model multi-consumer streaming. Shares a single source among +// multiple consumers with explicit buffering. + +const { + ArrayPrototypePush, + ArrayPrototypeShift, + ArrayPrototypeSplice, + Error, + Promise, + PromiseResolve, + SafeSet, + String, + SymbolAsyncIterator, + SymbolDispose, + SymbolIterator, +} = primordials; + +const { + shareProtocol, + shareSyncProtocol, +} = require('internal/streams/new/types'); + +const { + isAsyncIterable, + isSyncIterable, +} = require('internal/streams/new/from'); + +const { + pull: pullWithTransforms, + pullSync: pullSyncWithTransforms, +} = require('internal/streams/new/pull'); + +const { + parsePullArgs, +} = require('internal/streams/new/utils'); + +const { + codes: { + ERR_INVALID_ARG_TYPE, + ERR_OPERATION_FAILED, + ERR_OUT_OF_RANGE, + }, +} = require('internal/errors'); + +// ============================================================================= +// Async Share Implementation +// ============================================================================= + +class ShareImpl { + constructor(source, options) { + this._source = source; + this._options = options; + this._buffer = []; + this._bufferStart = 0; + this._consumers = new SafeSet(); + this._sourceIterator = null; + this._sourceExhausted = false; + this._sourceError = null; + this._cancelled = false; + this._pulling = false; + this._pullWaiters = []; + } + + get consumerCount() { + return this._consumers.size; + } + + get bufferSize() { + return this._buffer.length; + } + + pull(...args) { + const { transforms, options } = parsePullArgs(args); + const rawConsumer = this._createRawConsumer(); + + if (transforms.length > 0) { + if (options) { + return pullWithTransforms(rawConsumer, ...transforms, options); + } + return pullWithTransforms(rawConsumer, ...transforms); + } + return rawConsumer; + } + + _createRawConsumer() { + const state = { + cursor: this._bufferStart, + resolve: null, + reject: null, + detached: false, + }; + + this._consumers.add(state); + const self = this; + + return { + [SymbolAsyncIterator]() { + return { + async next() { + if (self._sourceError) { + state.detached = true; + self._consumers.delete(state); + throw self._sourceError; + } + + if (state.detached) { + return { __proto__: null, done: true, value: undefined }; + } + + if (self._cancelled) { + state.detached = true; + self._consumers.delete(state); + return { __proto__: null, done: true, value: undefined }; + } + + // Check if data is available in buffer + const bufferIndex = state.cursor - self._bufferStart; + if (bufferIndex < self._buffer.length) { + const chunk = self._buffer[bufferIndex]; + state.cursor++; + self._tryTrimBuffer(); + return { __proto__: null, done: false, value: chunk }; + } + + if (self._sourceExhausted) { + state.detached = true; + self._consumers.delete(state); + return { __proto__: null, done: true, value: undefined }; + } + + // Need to pull from source - check buffer limit + const canPull = await self._waitForBufferSpace(state); + if (!canPull) { + state.detached = true; + self._consumers.delete(state); + if (self._sourceError) throw self._sourceError; + return { __proto__: null, done: true, value: undefined }; + } + + await self._pullFromSource(); + + if (self._sourceError) { + state.detached = true; + self._consumers.delete(state); + throw self._sourceError; + } + + const newBufferIndex = state.cursor - self._bufferStart; + if (newBufferIndex < self._buffer.length) { + const chunk = self._buffer[newBufferIndex]; + state.cursor++; + self._tryTrimBuffer(); + return { __proto__: null, done: false, value: chunk }; + } + + if (self._sourceExhausted) { + state.detached = true; + self._consumers.delete(state); + return { __proto__: null, done: true, value: undefined }; + } + + return { __proto__: null, done: true, value: undefined }; + }, + + async return() { + state.detached = true; + state.resolve = null; + state.reject = null; + self._consumers.delete(state); + self._tryTrimBuffer(); + return { __proto__: null, done: true, value: undefined }; + }, + + async throw() { + state.detached = true; + state.resolve = null; + state.reject = null; + self._consumers.delete(state); + self._tryTrimBuffer(); + return { __proto__: null, done: true, value: undefined }; + }, + }; + }, + }; + } + + cancel(reason) { + if (this._cancelled) return; + this._cancelled = true; + + if (reason) { + this._sourceError = reason; + } + + if (this._sourceIterator?.return) { + this._sourceIterator.return().catch(() => {}); + } + + for (const consumer of this._consumers) { + if (consumer.resolve) { + if (reason) { + consumer.reject?.(reason); + } else { + consumer.resolve({ done: true, value: undefined }); + } + consumer.resolve = null; + consumer.reject = null; + } + consumer.detached = true; + } + this._consumers.clear(); + + for (let i = 0; i < this._pullWaiters.length; i++) { + this._pullWaiters[i](); + } + this._pullWaiters = []; + } + + [SymbolDispose]() { + this.cancel(); + } + + // Internal methods + + async _waitForBufferSpace(_state) { + while (this._buffer.length >= this._options.highWaterMark) { + if (this._cancelled || this._sourceError || this._sourceExhausted) { + return !this._cancelled; + } + + switch (this._options.backpressure) { + case 'strict': + throw new ERR_OUT_OF_RANGE( + 'buffer size', `<= ${this._options.highWaterMark}`, + this._buffer.length); + case 'block': + await new Promise((resolve) => { + ArrayPrototypePush(this._pullWaiters, resolve); + }); + break; + case 'drop-oldest': + ArrayPrototypeShift(this._buffer); + this._bufferStart++; + for (const consumer of this._consumers) { + if (consumer.cursor < this._bufferStart) { + consumer.cursor = this._bufferStart; + } + } + return true; + case 'drop-newest': + return true; + } + } + return true; + } + + _pullFromSource() { + if (this._sourceExhausted || this._cancelled) { + return PromiseResolve(); + } + + if (this._pulling) { + return new Promise((resolve) => { + ArrayPrototypePush(this._pullWaiters, resolve); + }); + } + + this._pulling = true; + + return (async () => { + try { + if (!this._sourceIterator) { + if (isAsyncIterable(this._source)) { + this._sourceIterator = + this._source[SymbolAsyncIterator](); + } else if (isSyncIterable(this._source)) { + const syncIterator = + this._source[SymbolIterator](); + this._sourceIterator = { + async next() { + return syncIterator.next(); + }, + async return() { + return syncIterator.return?.() ?? + { done: true, value: undefined }; + }, + }; + } else { + throw new ERR_INVALID_ARG_TYPE( + 'source', ['AsyncIterable', 'Iterable'], this._source); + } + } + + const result = await this._sourceIterator.next(); + + if (result.done) { + this._sourceExhausted = true; + } else { + ArrayPrototypePush(this._buffer, result.value); + } + } catch (error) { + this._sourceError = + error instanceof Error ? error : new ERR_OPERATION_FAILED(String(error)); + this._sourceExhausted = true; + } finally { + this._pulling = false; + for (let i = 0; i < this._pullWaiters.length; i++) { + this._pullWaiters[i](); + } + this._pullWaiters = []; + } + })(); + } + + _getMinCursor() { + let min = Infinity; + for (const consumer of this._consumers) { + if (consumer.cursor < min) { + min = consumer.cursor; + } + } + return min === Infinity ? + this._bufferStart + this._buffer.length : min; + } + + _tryTrimBuffer() { + const minCursor = this._getMinCursor(); + const trimCount = minCursor - this._bufferStart; + if (trimCount > 0) { + ArrayPrototypeSplice(this._buffer, 0, trimCount); + this._bufferStart = minCursor; + for (let i = 0; i < this._pullWaiters.length; i++) { + this._pullWaiters[i](); + } + this._pullWaiters = []; + } + } +} + +// ============================================================================= +// Sync Share Implementation +// ============================================================================= + +class SyncShareImpl { + constructor(source, options) { + this._source = source; + this._options = options; + this._buffer = []; + this._bufferStart = 0; + this._consumers = new SafeSet(); + this._sourceIterator = null; + this._sourceExhausted = false; + this._sourceError = null; + this._cancelled = false; + } + + get consumerCount() { + return this._consumers.size; + } + + get bufferSize() { + return this._buffer.length; + } + + pull(...transforms) { + const rawConsumer = this._createRawConsumer(); + + if (transforms.length > 0) { + return pullSyncWithTransforms(rawConsumer, ...transforms); + } + return rawConsumer; + } + + _createRawConsumer() { + const state = { + cursor: this._bufferStart, + detached: false, + }; + + this._consumers.add(state); + const self = this; + + return { + [SymbolIterator]() { + return { + next() { + if (state.detached) { + return { done: true, value: undefined }; + } + if (self._sourceError) { + state.detached = true; + self._consumers.delete(state); + throw self._sourceError; + } + if (self._cancelled) { + state.detached = true; + self._consumers.delete(state); + return { done: true, value: undefined }; + } + + const bufferIndex = state.cursor - self._bufferStart; + if (bufferIndex < self._buffer.length) { + const chunk = self._buffer[bufferIndex]; + state.cursor++; + self._tryTrimBuffer(); + return { done: false, value: chunk }; + } + + if (self._sourceExhausted) { + state.detached = true; + self._consumers.delete(state); + return { done: true, value: undefined }; + } + + // Check buffer limit + if (self._buffer.length >= self._options.highWaterMark) { + switch (self._options.backpressure) { + case 'strict': + throw new ERR_OUT_OF_RANGE( + 'buffer size', `<= ${self._options.highWaterMark}`, + self._buffer.length); + case 'block': + throw new ERR_OUT_OF_RANGE( + 'buffer size', `<= ${self._options.highWaterMark}`, + self._buffer.length); + case 'drop-oldest': + ArrayPrototypeShift(self._buffer); + self._bufferStart++; + for (const consumer of self._consumers) { + if (consumer.cursor < self._bufferStart) { + consumer.cursor = self._bufferStart; + } + } + break; + case 'drop-newest': + state.detached = true; + self._consumers.delete(state); + return { done: true, value: undefined }; + } + } + + self._pullFromSource(); + + if (self._sourceError) { + state.detached = true; + self._consumers.delete(state); + throw self._sourceError; + } + + const newBufferIndex = state.cursor - self._bufferStart; + if (newBufferIndex < self._buffer.length) { + const chunk = self._buffer[newBufferIndex]; + state.cursor++; + self._tryTrimBuffer(); + return { done: false, value: chunk }; + } + + if (self._sourceExhausted) { + state.detached = true; + self._consumers.delete(state); + return { done: true, value: undefined }; + } + + return { done: true, value: undefined }; + }, + + return() { + state.detached = true; + self._consumers.delete(state); + self._tryTrimBuffer(); + return { done: true, value: undefined }; + }, + + throw() { + state.detached = true; + self._consumers.delete(state); + self._tryTrimBuffer(); + return { done: true, value: undefined }; + }, + }; + }, + }; + } + + cancel(reason) { + if (this._cancelled) return; + this._cancelled = true; + + if (reason) { + this._sourceError = reason; + } + + if (this._sourceIterator?.return) { + this._sourceIterator.return(); + } + + for (const consumer of this._consumers) { + consumer.detached = true; + } + this._consumers.clear(); + } + + [SymbolDispose]() { + this.cancel(); + } + + _pullFromSource() { + if (this._sourceExhausted || this._cancelled) return; + + try { + this._sourceIterator ||= this._source[SymbolIterator](); + + const result = this._sourceIterator.next(); + + if (result.done) { + this._sourceExhausted = true; + } else { + ArrayPrototypePush(this._buffer, result.value); + } + } catch (error) { + this._sourceError = + error instanceof Error ? error : new ERR_OPERATION_FAILED(String(error)); + this._sourceExhausted = true; + } + } + + _getMinCursor() { + let min = Infinity; + for (const consumer of this._consumers) { + if (consumer.cursor < min) { + min = consumer.cursor; + } + } + return min === Infinity ? + this._bufferStart + this._buffer.length : min; + } + + _tryTrimBuffer() { + const minCursor = this._getMinCursor(); + const trimCount = minCursor - this._bufferStart; + if (trimCount > 0) { + ArrayPrototypeSplice(this._buffer, 0, trimCount); + this._bufferStart = minCursor; + } + } +} + +// ============================================================================= +// Public API +// ============================================================================= + +function share(source, options) { + const opts = { + highWaterMark: options?.highWaterMark ?? 16, + backpressure: options?.backpressure ?? 'strict', + signal: options?.signal, + }; + + const shareImpl = new ShareImpl(source, opts); + + if (opts.signal) { + if (opts.signal.aborted) { + shareImpl.cancel(); + } else { + opts.signal.addEventListener('abort', () => { + shareImpl.cancel(); + }, { once: true }); + } + } + + return shareImpl; +} + +function shareSync(source, options) { + const opts = { + highWaterMark: options?.highWaterMark ?? 16, + backpressure: options?.backpressure ?? 'strict', + }; + + return new SyncShareImpl(source, opts); +} + +function isShareable(value) { + return ( + value !== null && + typeof value === 'object' && + shareProtocol in value && + typeof value[shareProtocol] === 'function' + ); +} + +function isSyncShareable(value) { + return ( + value !== null && + typeof value === 'object' && + shareSyncProtocol in value && + typeof value[shareSyncProtocol] === 'function' + ); +} + +const Share = { + from(input, options) { + if (isShareable(input)) { + return input[shareProtocol](options); + } + if (isAsyncIterable(input) || isSyncIterable(input)) { + return share(input, options); + } + throw new ERR_INVALID_ARG_TYPE( + 'input', ['Shareable', 'AsyncIterable', 'Iterable'], input); + }, +}; + +const SyncShare = { + fromSync(input, options) { + if (isSyncShareable(input)) { + return input[shareSyncProtocol](options); + } + if (isSyncIterable(input)) { + return shareSync(input, options); + } + throw new ERR_INVALID_ARG_TYPE( + 'input', ['SyncShareable', 'Iterable'], input); + }, +}; + +module.exports = { + share, + shareSync, + Share, + SyncShare, +}; diff --git a/lib/internal/streams/new/transform.js b/lib/internal/streams/new/transform.js new file mode 100644 index 00000000000000..c9bf3ad845866b --- /dev/null +++ b/lib/internal/streams/new/transform.js @@ -0,0 +1,430 @@ +'use strict'; + +// New Streams API - Compression / Decompression Transforms +// +// Creates bare native zlib handles via internalBinding('zlib'), bypassing +// the stream.Transform / ZlibBase / EventEmitter machinery entirely. +// Compression runs on the libuv threadpool via handle.write() (async) so +// I/O and upstream transforms can overlap with compression work. +// Each factory returns a transform descriptor that can be passed to pull(). + +const { + ArrayPrototypePush, + ArrayPrototypeSplice, + MathMax, + NumberIsNaN, + ObjectEntries, + ObjectKeys, + Promise, + SymbolAsyncIterator, + Uint32Array, +} = primordials; + +const { Buffer } = require('buffer'); +const { + genericNodeError, +} = require('internal/errors'); +const binding = internalBinding('zlib'); +const constants = internalBinding('constants').zlib; + +const { + // Zlib modes + DEFLATE, INFLATE, GZIP, GUNZIP, + BROTLI_ENCODE, BROTLI_DECODE, + ZSTD_COMPRESS, ZSTD_DECOMPRESS, + // Zlib flush + Z_NO_FLUSH, Z_FINISH, + // Zlib defaults + Z_DEFAULT_WINDOWBITS, Z_DEFAULT_COMPRESSION, + Z_DEFAULT_MEMLEVEL, Z_DEFAULT_STRATEGY, Z_DEFAULT_CHUNK, + // Brotli flush + BROTLI_OPERATION_PROCESS, BROTLI_OPERATION_FINISH, + // Zstd flush + ZSTD_e_continue, ZSTD_e_end, +} = constants; + +// --------------------------------------------------------------------------- +// Batch high water mark - yield output in chunks of approximately this size. +// --------------------------------------------------------------------------- +const BATCH_HWM = 64 * 1024; + +// Pre-allocated empty buffer for flush/finalize calls. +const kEmpty = Buffer.alloc(0); + +// --------------------------------------------------------------------------- +// Brotli / Zstd parameter arrays (computed once, reused per init call). +// Mirrors the pattern in lib/zlib.js. +// --------------------------------------------------------------------------- +const kMaxBrotliParam = MathMax( + ...ObjectEntries(constants) + .map(({ 0: key, 1: value }) => + (key.startsWith('BROTLI_PARAM_') ? value : 0)), +); +const brotliInitParamsArray = new Uint32Array(kMaxBrotliParam + 1); + +const kMaxZstdCParam = MathMax(...ObjectKeys(constants).map( + (key) => (key.startsWith('ZSTD_c_') ? constants[key] : 0)), +); +const zstdInitCParamsArray = new Uint32Array(kMaxZstdCParam + 1); + +const kMaxZstdDParam = MathMax(...ObjectKeys(constants).map( + (key) => (key.startsWith('ZSTD_d_') ? constants[key] : 0)), +); +const zstdInitDParamsArray = new Uint32Array(kMaxZstdDParam + 1); + +// --------------------------------------------------------------------------- +// Handle creation - bare native handles, no Transform/EventEmitter overhead. +// +// Each factory accepts a processCallback (called from the threadpool +// completion path in C++) and an onError handler. +// --------------------------------------------------------------------------- + +/** + * Create a bare Zlib handle (gzip, gunzip, deflate, inflate). + * @returns {{ handle: object, writeState: Uint32Array, chunkSize: number }} + */ +function createZlibHandle(mode, options, processCallback, onError) { + const handle = new binding.Zlib(mode); + const writeState = new Uint32Array(2); + const chunkSize = options?.chunkSize ?? Z_DEFAULT_CHUNK; + + handle.onerror = onError; + handle.init( + options?.windowBits ?? Z_DEFAULT_WINDOWBITS, + options?.level ?? Z_DEFAULT_COMPRESSION, + options?.memLevel ?? Z_DEFAULT_MEMLEVEL, + options?.strategy ?? Z_DEFAULT_STRATEGY, + writeState, + processCallback, + options?.dictionary, + ); + + return { handle, writeState, chunkSize }; +} + +/** + * Create a bare Brotli handle. + * @returns {{ handle: object, writeState: Uint32Array, chunkSize: number }} + */ +function createBrotliHandle(mode, options, processCallback, onError) { + const handle = mode === BROTLI_ENCODE ? + new binding.BrotliEncoder(mode) : new binding.BrotliDecoder(mode); + const writeState = new Uint32Array(2); + const chunkSize = options?.chunkSize ?? Z_DEFAULT_CHUNK; + + brotliInitParamsArray.fill(-1); + if (options?.params) { + const params = options.params; + const keys = ObjectKeys(params); + for (let i = 0; i < keys.length; i++) { + const key = +keys[i]; + if (!NumberIsNaN(key) && key >= 0 && key <= kMaxBrotliParam) { + brotliInitParamsArray[key] = params[keys[i]]; + } + } + } + + handle.onerror = onError; + handle.init( + brotliInitParamsArray, + writeState, + processCallback, + options?.dictionary, + ); + + return { handle, writeState, chunkSize }; +} + +/** + * Create a bare Zstd handle. + * @returns {{ handle: object, writeState: Uint32Array, chunkSize: number }} + */ +function createZstdHandle(mode, options, processCallback, onError) { + const isCompress = mode === ZSTD_COMPRESS; + const handle = isCompress ? + new binding.ZstdCompress() : new binding.ZstdDecompress(); + const writeState = new Uint32Array(2); + const chunkSize = options?.chunkSize ?? Z_DEFAULT_CHUNK; + + const initArray = isCompress ? zstdInitCParamsArray : zstdInitDParamsArray; + const maxParam = isCompress ? kMaxZstdCParam : kMaxZstdDParam; + + initArray.fill(-1); + if (options?.params) { + const params = options.params; + const keys = ObjectKeys(params); + for (let i = 0; i < keys.length; i++) { + const key = +keys[i]; + if (!NumberIsNaN(key) && key >= 0 && key <= maxParam) { + initArray[key] = params[keys[i]]; + } + } + } + + handle.onerror = onError; + handle.init( + initArray, + options?.pledgedSrcSize, + writeState, + processCallback, + options?.dictionary, + ); + + return { handle, writeState, chunkSize }; +} + +// --------------------------------------------------------------------------- +// Core: makeZlibTransform +// +// Uses async handle.write() so compression runs on the libuv threadpool. +// The generator manually iterates the source with pre-reading: the next +// upstream read+transform is started before awaiting the current compression, +// so I/O and upstream work overlap with threadpool compression. +// --------------------------------------------------------------------------- +function makeZlibTransform(createHandleFn, processFlag, finishFlag) { + return { + transform: async function*(source) { + // ---- Per-invocation state shared with the write callback ---- + let outBuf; + let outOffset = 0; + let chunkSize; + const pending = []; + let pendingBytes = 0; + + // Current write operation state (read by the callback for looping). + let resolveWrite, rejectWrite; + let writeInput, writeFlush; + let writeInOff, writeAvailIn, writeAvailOutBefore; + + // processCallback: called by C++ AfterThreadPoolWork when compression + // on the threadpool completes. Collects output, loops if the engine + // has more output to produce (availOut === 0), then resolves the + // promise when all output for this input chunk is collected. + function onWriteComplete() { + const availOut = writeState[0]; + const availInAfter = writeState[1]; + const have = writeAvailOutBefore - availOut; + + if (have > 0) { + ArrayPrototypePush(pending, + outBuf.slice(outOffset, outOffset + have)); + pendingBytes += have; + outOffset += have; + } + + // Reallocate output buffer if exhausted. + if (availOut === 0 || outOffset >= chunkSize) { + outBuf = Buffer.allocUnsafe(chunkSize); + outOffset = 0; + } + + if (availOut === 0) { + // Engine has more output - loop on the threadpool. + const consumed = writeAvailIn - availInAfter; + writeInOff += consumed; + writeAvailIn = availInAfter; + writeAvailOutBefore = chunkSize - outOffset; + + handle.write(writeFlush, + writeInput, writeInOff, writeAvailIn, + outBuf, outOffset, writeAvailOutBefore); + return; // Will call onWriteComplete again. + } + + // All input consumed and output collected. + handle.buffer = null; + const resolve = resolveWrite; + resolveWrite = undefined; + rejectWrite = undefined; + resolve(); + } + + // onError: called by C++ when the engine encounters an error. + // Fires instead of onWriteComplete - reject the promise. + function onError(message, errno, code) { + const error = genericNodeError(message, { errno, code }); + error.errno = errno; + error.code = code; + const reject = rejectWrite; + resolveWrite = undefined; + rejectWrite = undefined; + if (reject) reject(error); + } + + // ---- Create the handle with our callbacks ---- + const result = createHandleFn(onWriteComplete, onError); + const handle = result.handle; + const writeState = result.writeState; + chunkSize = result.chunkSize; + outBuf = Buffer.allocUnsafe(chunkSize); + + // Dispatch input to the threadpool and return a promise. + function processInputAsync(input, flushFlag) { + return new Promise((resolve, reject) => { + resolveWrite = resolve; + rejectWrite = reject; + writeInput = input; + writeFlush = flushFlag; + writeInOff = 0; + writeAvailIn = input.byteLength; + writeAvailOutBefore = chunkSize - outOffset; + + // Keep input alive while the threadpool references it. + handle.buffer = input; + + handle.write(flushFlag, + input, 0, input.byteLength, + outBuf, outOffset, writeAvailOutBefore); + }); + } + + function drainBatch() { + if (pendingBytes <= BATCH_HWM) { + const batch = ArrayPrototypeSplice(pending, 0, pending.length); + pendingBytes = 0; + return batch; + } + const batch = []; + let batchBytes = 0; + while (pending.length > 0 && batchBytes < BATCH_HWM) { + const buf = pending.shift(); + ArrayPrototypePush(batch, buf); + batchBytes += buf.byteLength; + pendingBytes -= buf.byteLength; + } + return batch; + } + + let finalized = false; + + try { + // Manually iterate the source so we can pre-read: calling + // iter.next() starts the upstream read + transform on libuv + // before we await the current compression on the threadpool. + const iter = source[SymbolAsyncIterator](); + let nextResult = iter.next(); + + while (true) { + const { value: chunks, done } = await nextResult; + if (done) break; + + if (chunks === null) { + // Flush signal - finalize the engine. + if (!finalized) { + finalized = true; + await processInputAsync(kEmpty, finishFlag); + while (pending.length > 0) { + yield drainBatch(); + } + } + nextResult = iter.next(); + continue; + } + + // Pre-read: start upstream I/O + transform for the NEXT batch + // while we compress the current batch on the threadpool. + nextResult = iter.next(); + + for (let i = 0; i < chunks.length; i++) { + await processInputAsync(chunks[i], processFlag); + } + + if (pendingBytes >= BATCH_HWM) { + while (pending.length > 0 && pendingBytes >= BATCH_HWM) { + yield drainBatch(); + } + } + if (pending.length > 0) { + yield drainBatch(); + } + } + + // Source ended - finalize if not already done by a null signal. + if (!finalized) { + finalized = true; + await processInputAsync(kEmpty, finishFlag); + while (pending.length > 0) { + yield drainBatch(); + } + } + } finally { + handle.close(); + } + }, + }; +} + +// --------------------------------------------------------------------------- +// Compression factories +// --------------------------------------------------------------------------- + +function compressGzip(options) { + return makeZlibTransform( + (cb, onErr) => createZlibHandle(GZIP, options, cb, onErr), + Z_NO_FLUSH, Z_FINISH, + ); +} + +function compressDeflate(options) { + return makeZlibTransform( + (cb, onErr) => createZlibHandle(DEFLATE, options, cb, onErr), + Z_NO_FLUSH, Z_FINISH, + ); +} + +function compressBrotli(options) { + return makeZlibTransform( + (cb, onErr) => createBrotliHandle(BROTLI_ENCODE, options, cb, onErr), + BROTLI_OPERATION_PROCESS, BROTLI_OPERATION_FINISH, + ); +} + +function compressZstd(options) { + return makeZlibTransform( + (cb, onErr) => createZstdHandle(ZSTD_COMPRESS, options, cb, onErr), + ZSTD_e_continue, ZSTD_e_end, + ); +} + +// --------------------------------------------------------------------------- +// Decompression factories +// --------------------------------------------------------------------------- + +function decompressGzip(options) { + return makeZlibTransform( + (cb, onErr) => createZlibHandle(GUNZIP, options, cb, onErr), + Z_NO_FLUSH, Z_FINISH, + ); +} + +function decompressDeflate(options) { + return makeZlibTransform( + (cb, onErr) => createZlibHandle(INFLATE, options, cb, onErr), + Z_NO_FLUSH, Z_FINISH, + ); +} + +function decompressBrotli(options) { + return makeZlibTransform( + (cb, onErr) => createBrotliHandle(BROTLI_DECODE, options, cb, onErr), + BROTLI_OPERATION_PROCESS, BROTLI_OPERATION_FINISH, + ); +} + +function decompressZstd(options) { + return makeZlibTransform( + (cb, onErr) => createZstdHandle(ZSTD_DECOMPRESS, options, cb, onErr), + ZSTD_e_continue, ZSTD_e_end, + ); +} + +module.exports = { + compressGzip, + compressDeflate, + compressBrotli, + compressZstd, + decompressGzip, + decompressDeflate, + decompressBrotli, + decompressZstd, +}; diff --git a/lib/internal/streams/new/types.js b/lib/internal/streams/new/types.js new file mode 100644 index 00000000000000..ce4b385f9dfbb0 --- /dev/null +++ b/lib/internal/streams/new/types.js @@ -0,0 +1,59 @@ +'use strict'; + +// New Streams API - Protocol Symbols +// +// These symbols allow objects to participate in streaming. +// Using Symbol.for() allows third-party code to implement protocols +// without importing these symbols directly. + +const { + SymbolFor, +} = primordials; + +/** + * Symbol for sync value-to-streamable conversion protocol. + * Objects implementing this can be written to streams or yielded + * from generators. Works in both sync and async contexts. + * + * Third-party: [Symbol.for('Stream.toStreamable')]() { ... } + */ +const toStreamable = SymbolFor('Stream.toStreamable'); + +/** + * Symbol for async value-to-streamable conversion protocol. + * Objects implementing this can be written to async streams. + * Works in async contexts only. + * + * Third-party: [Symbol.for('Stream.toAsyncStreamable')]() { ... } + */ +const toAsyncStreamable = SymbolFor('Stream.toAsyncStreamable'); + +/** + * Symbol for Broadcastable protocol - object can provide a Broadcast. + */ +const broadcastProtocol = SymbolFor('Stream.broadcastProtocol'); + +/** + * Symbol for Shareable protocol - object can provide a Share. + */ +const shareProtocol = SymbolFor('Stream.shareProtocol'); + +/** + * Symbol for SyncShareable protocol - object can provide a SyncShare. + */ +const shareSyncProtocol = SymbolFor('Stream.shareSyncProtocol'); + +/** + * Symbol for Drainable protocol - object can signal when backpressure + * clears. Used to bridge event-driven sources that need drain notification. + */ +const drainableProtocol = SymbolFor('Stream.drainableProtocol'); + +module.exports = { + toStreamable, + toAsyncStreamable, + broadcastProtocol, + shareProtocol, + shareSyncProtocol, + drainableProtocol, +}; diff --git a/lib/internal/streams/new/utils.js b/lib/internal/streams/new/utils.js new file mode 100644 index 00000000000000..57ca93b85cd09c --- /dev/null +++ b/lib/internal/streams/new/utils.js @@ -0,0 +1,106 @@ +'use strict'; + +// New Streams API - Utility Functions + +const { + ArrayPrototypeSlice, + TypedArrayPrototypeSet, + Uint8Array, +} = primordials; + +const { TextEncoder } = require('internal/encoding'); + +// Shared TextEncoder instance for string conversion. +const encoder = new TextEncoder(); + +/** + * Convert a chunk (string or Uint8Array) to Uint8Array. + * Strings are UTF-8 encoded. + * @param {Uint8Array|string} chunk + * @returns {Uint8Array} + */ +function toUint8Array(chunk) { + if (typeof chunk === 'string') { + return encoder.encode(chunk); + } + return chunk; +} + +/** + * Calculate total byte length of an array of chunks. + * @param {Uint8Array[]} chunks + * @returns {number} + */ +function totalByteLength(chunks) { + let total = 0; + for (let i = 0; i < chunks.length; i++) { + total += chunks[i].byteLength; + } + return total; +} + +/** + * Concatenate multiple Uint8Arrays into a single Uint8Array. + * @param {Uint8Array[]} chunks + * @returns {Uint8Array} + */ +function concatBytes(chunks) { + if (chunks.length === 0) { + return new Uint8Array(0); + } + if (chunks.length === 1) { + return chunks[0]; + } + + const total = totalByteLength(chunks); + const result = new Uint8Array(total); + let offset = 0; + for (let i = 0; i < chunks.length; i++) { + TypedArrayPrototypeSet(result, chunks[i], offset); + offset += chunks[i].byteLength; + } + return result; +} + +/** + * Check if a value is PullOptions (object without transform or write property). + * @param {unknown} value + * @returns {boolean} + */ +function isPullOptions(value) { + return ( + value !== null && + typeof value === 'object' && + !('transform' in value) && + !('write' in value) + ); +} + +/** + * Parse variadic arguments for pull/pullSync. + * Returns { transforms, options } + * @param {Array} args + * @returns {{ transforms: Array, options: object|undefined }} + */ +function parsePullArgs(args) { + if (args.length === 0) { + return { transforms: [], options: undefined }; + } + + const last = args[args.length - 1]; + if (isPullOptions(last)) { + return { + transforms: ArrayPrototypeSlice(args, 0, -1), + options: last, + }; + } + + return { transforms: args, options: undefined }; +} + +module.exports = { + toUint8Array, + concatBytes, + isPullOptions, + parsePullArgs, +}; diff --git a/lib/stream/new.js b/lib/stream/new.js new file mode 100644 index 00000000000000..27596be8ba0604 --- /dev/null +++ b/lib/stream/new.js @@ -0,0 +1,208 @@ +'use strict'; + +// Public entry point for the new streams API. +// Usage: require('stream/new') or require('node:stream/new') + +const { + ObjectFreeze, +} = primordials; + +// Protocol symbols +const { + toStreamable, + toAsyncStreamable, + broadcastProtocol, + shareProtocol, + shareSyncProtocol, + drainableProtocol, +} = require('internal/streams/new/types'); + +// Factories +const { push } = require('internal/streams/new/push'); +const { duplex } = require('internal/streams/new/duplex'); +const { from, fromSync } = require('internal/streams/new/from'); + +// Pipelines +const { + pull, + pullSync, + pipeTo, + pipeToSync, +} = require('internal/streams/new/pull'); + +// Consumers +const { + bytes, + bytesSync, + text, + textSync, + arrayBuffer, + arrayBufferSync, + array, + arraySync, + tap, + tapSync, + merge, + ondrain, +} = require('internal/streams/new/consumers'); + +// Transforms +const { + compressGzip, + compressDeflate, + compressBrotli, + compressZstd, + decompressGzip, + decompressDeflate, + decompressBrotli, + decompressZstd, +} = require('internal/streams/new/transform'); + +// Multi-consumer +const { broadcast, Broadcast } = require('internal/streams/new/broadcast'); +const { + share, + shareSync, + Share, + SyncShare, +} = require('internal/streams/new/share'); + +/** + * Stream namespace - unified access to all stream functions. + * @example + * const { Stream } = require('stream/new'); + * + * const { writer, readable } = Stream.push(); + * await writer.write("hello"); + * await writer.end(); + * + * const output = Stream.pull(readable, transform1, transform2); + * const data = await Stream.bytes(output); + */ +const Stream = ObjectFreeze({ + // Factories + push, + duplex, + from, + fromSync, + + // Pipelines + pull, + pullSync, + + // Pipe to destination + pipeTo, + pipeToSync, + + // Consumers (async) + bytes, + text, + arrayBuffer, + array, + + // Consumers (sync) + bytesSync, + textSync, + arrayBufferSync, + arraySync, + + // Combining + merge, + + // Multi-consumer (push model) + broadcast, + + // Multi-consumer (pull model) + share, + shareSync, + + // Utilities + tap, + tapSync, + + // Drain utility for event source integration + ondrain, + + // Compression / decompression transforms + compressGzip, + compressDeflate, + compressBrotli, + compressZstd, + decompressGzip, + decompressDeflate, + decompressBrotli, + decompressZstd, + + // Protocol symbols + toStreamable, + toAsyncStreamable, + broadcastProtocol, + shareProtocol, + shareSyncProtocol, + drainableProtocol, +}); + +module.exports = { + // The Stream namespace + Stream, + + // Also export everything individually for destructured imports + + // Protocol symbols + toStreamable, + toAsyncStreamable, + broadcastProtocol, + shareProtocol, + shareSyncProtocol, + drainableProtocol, + + // Factories + push, + duplex, + from, + fromSync, + + // Pipelines + pull, + pullSync, + pipeTo, + pipeToSync, + + // Consumers (async) + bytes, + text, + arrayBuffer, + array, + + // Consumers (sync) + bytesSync, + textSync, + arrayBufferSync, + arraySync, + + // Combining + merge, + + // Multi-consumer + broadcast, + Broadcast, + share, + shareSync, + Share, + SyncShare, + + // Utilities + tap, + tapSync, + ondrain, + + // Compression / decompression transforms + compressGzip, + compressDeflate, + compressBrotli, + compressZstd, + decompressGzip, + decompressDeflate, + decompressBrotli, + decompressZstd, +}; diff --git a/test/parallel/test-fs-promises-file-handle-pull.js b/test/parallel/test-fs-promises-file-handle-pull.js new file mode 100644 index 00000000000000..6b206ee3fae2ab --- /dev/null +++ b/test/parallel/test-fs-promises-file-handle-pull.js @@ -0,0 +1,254 @@ +'use strict'; + +const common = require('../common'); +const assert = require('assert'); +const fs = require('fs'); +const { open } = fs.promises; +const path = require('path'); +const tmpdir = require('../common/tmpdir'); +const { text, bytes } = require('stream/new'); + +tmpdir.refresh(); + +const tmpDir = tmpdir.path; + +// ============================================================================= +// Basic pull() +// ============================================================================= + +async function testBasicPull() { + const filePath = path.join(tmpDir, 'pull-basic.txt'); + fs.writeFileSync(filePath, 'hello from file'); + + const fh = await open(filePath, 'r'); + try { + const readable = fh.pull(); + const data = await text(readable); + assert.strictEqual(data, 'hello from file'); + } finally { + await fh.close(); + } +} + +async function testPullBinary() { + const filePath = path.join(tmpDir, 'pull-binary.bin'); + const buf = Buffer.alloc(256); + for (let i = 0; i < 256; i++) buf[i] = i; + fs.writeFileSync(filePath, buf); + + const fh = await open(filePath, 'r'); + try { + const readable = fh.pull(); + const data = await bytes(readable); + assert.strictEqual(data.byteLength, 256); + for (let i = 0; i < 256; i++) { + assert.strictEqual(data[i], i); + } + } finally { + await fh.close(); + } +} + +async function testPullEmptyFile() { + const filePath = path.join(tmpDir, 'pull-empty.txt'); + fs.writeFileSync(filePath, ''); + + const fh = await open(filePath, 'r'); + try { + const readable = fh.pull(); + const data = await bytes(readable); + assert.strictEqual(data.byteLength, 0); + } finally { + await fh.close(); + } +} + +// ============================================================================= +// Large file (multi-chunk) +// ============================================================================= + +async function testPullLargeFile() { + const filePath = path.join(tmpDir, 'pull-large.bin'); + // Write 64KB — enough for multiple 16KB read chunks + const size = 64 * 1024; + const buf = Buffer.alloc(size, 0x42); + fs.writeFileSync(filePath, buf); + + const fh = await open(filePath, 'r'); + try { + const readable = fh.pull(); + const data = await bytes(readable); + assert.strictEqual(data.byteLength, size); + // Verify content + for (let i = 0; i < data.byteLength; i++) { + assert.strictEqual(data[i], 0x42); + } + } finally { + await fh.close(); + } +} + +// ============================================================================= +// With transforms +// ============================================================================= + +async function testPullWithTransform() { + const filePath = path.join(tmpDir, 'pull-transform.txt'); + fs.writeFileSync(filePath, 'hello'); + + const fh = await open(filePath, 'r'); + try { + const upper = (chunks) => { + if (chunks === null) return null; + return chunks.map((c) => { + const str = new TextDecoder().decode(c); + return new TextEncoder().encode(str.toUpperCase()); + }); + }; + + const readable = fh.pull(upper); + const data = await text(readable); + assert.strictEqual(data, 'HELLO'); + } finally { + await fh.close(); + } +} + +// ============================================================================= +// autoClose option +// ============================================================================= + +async function testPullAutoClose() { + const filePath = path.join(tmpDir, 'pull-autoclose.txt'); + fs.writeFileSync(filePath, 'auto close data'); + + const fh = await open(filePath, 'r'); + const readable = fh.pull({ autoClose: true }); + const data = await text(readable); + assert.strictEqual(data, 'auto close data'); + + // After consuming with autoClose, the file handle should be closed + // Trying to read again should throw + await assert.rejects( + async () => { + await fh.stat(); + }, + (err) => err.code === 'ERR_INVALID_STATE' || err.code === 'EBADF', + ); +} + +// ============================================================================= +// Locking +// ============================================================================= + +async function testPullLocking() { + const filePath = path.join(tmpDir, 'pull-lock.txt'); + fs.writeFileSync(filePath, 'lock data'); + + const fh = await open(filePath, 'r'); + try { + // First pull locks the handle + const readable = fh.pull(); + + // Second pull while locked should throw + assert.throws( + () => fh.pull(), + { code: 'ERR_INVALID_STATE' }, + ); + + // Consume the first stream to unlock + await text(readable); + + // Now it should be usable again + const readable2 = fh.pull(); + const data = await text(readable2); + assert.strictEqual(data, ''); // Already read to end + } finally { + await fh.close(); + } +} + +// ============================================================================= +// Closed handle +// ============================================================================= + +async function testPullClosedHandle() { + const filePath = path.join(tmpDir, 'pull-closed.txt'); + fs.writeFileSync(filePath, 'data'); + + const fh = await open(filePath, 'r'); + await fh.close(); + + assert.throws( + () => fh.pull(), + { code: 'ERR_INVALID_STATE' }, + ); +} + +// ============================================================================= +// AbortSignal +// ============================================================================= + +async function testPullAbortSignal() { + const filePath = path.join(tmpDir, 'pull-abort.txt'); + // Write enough data that we can abort mid-stream + fs.writeFileSync(filePath, 'a'.repeat(1024)); + + const ac = new AbortController(); + const fh = await open(filePath, 'r'); + try { + ac.abort(); + const readable = fh.pull({ signal: ac.signal }); + + await assert.rejects( + async () => { + // eslint-disable-next-line no-unused-vars + for await (const _ of readable) { + assert.fail('Should not reach here'); + } + }, + (err) => err.name === 'AbortError', + ); + } finally { + await fh.close(); + } +} + +// ============================================================================= +// Iterate batches directly +// ============================================================================= + +async function testPullIterateBatches() { + const filePath = path.join(tmpDir, 'pull-batches.txt'); + fs.writeFileSync(filePath, 'batch data'); + + const fh = await open(filePath, 'r'); + try { + const readable = fh.pull(); + const batches = []; + for await (const batch of readable) { + batches.push(batch); + // Each batch should be an array of Uint8Array + assert.ok(Array.isArray(batch)); + for (const chunk of batch) { + assert.ok(chunk instanceof Uint8Array); + } + } + assert.ok(batches.length > 0); + } finally { + await fh.close(); + } +} + +Promise.all([ + testBasicPull(), + testPullBinary(), + testPullEmptyFile(), + testPullLargeFile(), + testPullWithTransform(), + testPullAutoClose(), + testPullLocking(), + testPullClosedHandle(), + testPullAbortSignal(), + testPullIterateBatches(), +]).then(common.mustCall()); diff --git a/test/parallel/test-fs-promises-file-handle-writer.js b/test/parallel/test-fs-promises-file-handle-writer.js new file mode 100644 index 00000000000000..5a0e2914a3e52c --- /dev/null +++ b/test/parallel/test-fs-promises-file-handle-writer.js @@ -0,0 +1,473 @@ +'use strict'; + +const common = require('../common'); +const assert = require('assert'); +const fs = require('fs'); +const { open } = fs.promises; +const path = require('path'); +const tmpdir = require('../common/tmpdir'); +const { + pipeTo, text, + compressGzip, decompressGzip, +} = require('stream/new'); + +tmpdir.refresh(); + +const tmpDir = tmpdir.path; + +// ============================================================================= +// Basic write() +// ============================================================================= + +async function testBasicWrite() { + const filePath = path.join(tmpDir, 'writer-basic.txt'); + const fh = await open(filePath, 'w'); + const w = fh.writer(); + await w.write(Buffer.from('Hello ')); + await w.write(Buffer.from('World!')); + const totalBytes = await w.end(); + await fh.close(); + + assert.strictEqual(totalBytes, 12); + assert.strictEqual(fs.readFileSync(filePath, 'utf8'), 'Hello World!'); +} + +// ============================================================================= +// Basic writev() +// ============================================================================= + +async function testBasicWritev() { + const filePath = path.join(tmpDir, 'writer-writev.txt'); + const fh = await open(filePath, 'w'); + const w = fh.writer(); + await w.writev([ + Buffer.from('aaa'), + Buffer.from('bbb'), + Buffer.from('ccc'), + ]); + const totalBytes = await w.end(); + await fh.close(); + + assert.strictEqual(totalBytes, 9); + assert.strictEqual(fs.readFileSync(filePath, 'utf8'), 'aaabbbccc'); +} + +// ============================================================================= +// Mixed write() and writev() +// ============================================================================= + +async function testMixedWriteAndWritev() { + const filePath = path.join(tmpDir, 'writer-mixed.txt'); + const fh = await open(filePath, 'w'); + const w = fh.writer(); + await w.write(Buffer.from('head-')); + await w.writev([Buffer.from('mid1-'), Buffer.from('mid2-')]); + await w.write(Buffer.from('tail')); + const totalBytes = await w.end(); + await fh.close(); + + assert.strictEqual(totalBytes, 19); + assert.strictEqual(fs.readFileSync(filePath, 'utf8'), 'head-mid1-mid2-tail'); +} + +// ============================================================================= +// end() returns totalBytesWritten +// ============================================================================= + +async function testEndReturnsTotalBytes() { + const filePath = path.join(tmpDir, 'writer-totalbytes.txt'); + const fh = await open(filePath, 'w'); + const w = fh.writer(); + + // Write some data in various sizes + const sizes = [100, 200, 300, 400, 500]; + let expected = 0; + for (const size of sizes) { + await w.write(Buffer.alloc(size, 0x41)); + expected += size; + } + const totalBytes = await w.end(); + await fh.close(); + + assert.strictEqual(totalBytes, expected); + assert.strictEqual(totalBytes, 1500); + assert.strictEqual(fs.statSync(filePath).size, 1500); +} + +// ============================================================================= +// autoClose: true — handle closed after end() +// ============================================================================= + +async function testAutoCloseOnEnd() { + const filePath = path.join(tmpDir, 'writer-autoclose-end.txt'); + const fh = await open(filePath, 'w'); + const w = fh.writer({ autoClose: true }); + await w.write(Buffer.from('auto close test')); + await w.end(); + + // Handle should be closed + await assert.rejects(fh.stat(), { code: 'EBADF' }); + assert.strictEqual(fs.readFileSync(filePath, 'utf8'), 'auto close test'); +} + +// ============================================================================= +// autoClose: true — handle closed after abort() +// ============================================================================= + +async function testAutoCloseOnAbort() { + const filePath = path.join(tmpDir, 'writer-autoclose-abort.txt'); + const fh = await open(filePath, 'w'); + const w = fh.writer({ autoClose: true }); + await w.write(Buffer.from('partial')); + await w.abort(new Error('test abort')); + + // Handle should be closed + await assert.rejects(fh.stat(), { code: 'EBADF' }); + // Partial data should still be on disk (abort doesn't truncate) + assert.strictEqual(fs.readFileSync(filePath, 'utf8'), 'partial'); +} + +// ============================================================================= +// start option — write at specified offset +// ============================================================================= + +async function testStartOption() { + const filePath = path.join(tmpDir, 'writer-start.txt'); + // Pre-fill with 10 A's + fs.writeFileSync(filePath, 'AAAAAAAAAA'); + + const fh = await open(filePath, 'r+'); + const w = fh.writer({ start: 3 }); + await w.write(Buffer.from('BBB')); + await w.end(); + await fh.close(); + + assert.strictEqual(fs.readFileSync(filePath, 'utf8'), 'AAABBBAAAA'); +} + +// ============================================================================= +// start option — sequential writes advance position +// ============================================================================= + +async function testStartSequentialPosition() { + const filePath = path.join(tmpDir, 'writer-start-seq.txt'); + fs.writeFileSync(filePath, 'XXXXXXXXXX'); + + const fh = await open(filePath, 'r+'); + const w = fh.writer({ start: 2 }); + await w.write(Buffer.from('AA')); + await w.write(Buffer.from('BB')); + await w.writev([Buffer.from('C'), Buffer.from('D')]); + await w.end(); + await fh.close(); + + assert.strictEqual(fs.readFileSync(filePath, 'utf8'), 'XXAABBCDXX'); +} + +// ============================================================================= +// Locked state — can't create second writer while active +// ============================================================================= + +async function testLockedState() { + const filePath = path.join(tmpDir, 'writer-locked.txt'); + const fh = await open(filePath, 'w'); + const w = fh.writer(); + + assert.throws(() => fh.writer(), { + name: 'Error', + message: /locked/, + }); + + // Also can't pull while writer is active + assert.throws(() => fh.pull(), { + name: 'Error', + message: /locked/, + }); + + await w.end(); + await fh.close(); +} + +// ============================================================================= +// Unlock after end — handle reusable +// ============================================================================= + +async function testUnlockAfterEnd() { + const filePath = path.join(tmpDir, 'writer-unlock.txt'); + const fh = await open(filePath, 'w'); + + const w1 = fh.writer(); + await w1.write(Buffer.from('first')); + await w1.end(); + + // Should work — handle is unlocked + const w2 = fh.writer(); + await w2.write(Buffer.from(' second')); + await w2.end(); + await fh.close(); + + assert.strictEqual(fs.readFileSync(filePath, 'utf8'), 'first second'); +} + +// ============================================================================= +// Unlock after abort — handle reusable +// ============================================================================= + +async function testUnlockAfterAbort() { + const filePath = path.join(tmpDir, 'writer-unlock-abort.txt'); + const fh = await open(filePath, 'w'); + + const w1 = fh.writer(); + await w1.write(Buffer.from('aborted')); + await w1.abort(new Error('test')); + + // Should work — handle is unlocked + const w2 = fh.writer(); + await w2.write(Buffer.from('recovered')); + await w2.end(); + await fh.close(); + + // 'recovered' is appended after 'aborted' at current file offset + const content = fs.readFileSync(filePath, 'utf8'); + assert.ok(content.startsWith('aborted')); + assert.ok(content.includes('recovered')); +} + +// ============================================================================= +// Write after end/abort rejects +// ============================================================================= + +async function testWriteAfterEndRejects() { + const filePath = path.join(tmpDir, 'writer-closed.txt'); + const fh = await open(filePath, 'w'); + const w = fh.writer(); + await w.write(Buffer.from('data')); + await w.end(); + + await assert.rejects(w.write(Buffer.from('more')), { + name: 'Error', + message: /closed/, + }); + await assert.rejects(w.writev([Buffer.from('more')]), { + name: 'Error', + message: /closed/, + }); + + await fh.close(); +} + +// ============================================================================= +// Closed handle — writer() throws +// ============================================================================= + +async function testClosedHandle() { + const filePath = path.join(tmpDir, 'writer-closed-handle.txt'); + const fh = await open(filePath, 'w'); + await fh.close(); + + assert.throws(() => fh.writer(), { + name: 'Error', + message: /closed/, + }); +} + +// ============================================================================= +// pipeTo() integration — pipe source through writer +// ============================================================================= + +async function testPipeToIntegration() { + const srcPath = path.join(tmpDir, 'writer-pipeto-src.txt'); + const dstPath = path.join(tmpDir, 'writer-pipeto-dst.txt'); + const data = 'The quick brown fox jumps over the lazy dog.\n'.repeat(500); + fs.writeFileSync(srcPath, data); + + const rfh = await open(srcPath, 'r'); + const wfh = await open(dstPath, 'w'); + const w = wfh.writer(); + + const totalBytes = await pipeTo(rfh.pull(), w); + + await rfh.close(); + await wfh.close(); + + assert.strictEqual(totalBytes, Buffer.byteLength(data)); + assert.strictEqual(fs.readFileSync(dstPath, 'utf8'), data); +} + +// ============================================================================= +// pipeTo() with transforms — uppercase through writer +// ============================================================================= + +async function testPipeToWithTransform() { + const srcPath = path.join(tmpDir, 'writer-transform-src.txt'); + const dstPath = path.join(tmpDir, 'writer-transform-dst.txt'); + const data = 'hello world from transforms test\n'.repeat(200); + fs.writeFileSync(srcPath, data); + + function uppercase(chunks) { + if (chunks === null) return null; + const out = new Array(chunks.length); + for (let i = 0; i < chunks.length; i++) { + const src = chunks[i]; + const buf = Buffer.allocUnsafe(src.length); + for (let j = 0; j < src.length; j++) { + const b = src[j]; + buf[j] = (b >= 0x61 && b <= 0x7a) ? b - 0x20 : b; + } + out[i] = buf; + } + return out; + } + + const rfh = await open(srcPath, 'r'); + const wfh = await open(dstPath, 'w'); + const w = wfh.writer(); + + await pipeTo(rfh.pull(), uppercase, w); + + await rfh.close(); + await wfh.close(); + + assert.strictEqual(fs.readFileSync(dstPath, 'utf8'), data.toUpperCase()); +} + +// ============================================================================= +// Round-trip: pull → compress → writer, pull → decompress → verify +// ============================================================================= + +async function testCompressRoundTrip() { + const srcPath = path.join(tmpDir, 'writer-rt-src.txt'); + const gzPath = path.join(tmpDir, 'writer-rt.gz'); + const original = 'Round trip compression test data. '.repeat(2000); + fs.writeFileSync(srcPath, original); + + // Compress: pull → gzip → writer + { + const rfh = await open(srcPath, 'r'); + const wfh = await open(gzPath, 'w'); + const w = wfh.writer({ autoClose: true }); + await pipeTo(rfh.pull(), compressGzip(), w); + await rfh.close(); + } + + // Verify compressed file is smaller + const compressedSize = fs.statSync(gzPath).size; + assert.ok(compressedSize < Buffer.byteLength(original), + `Compressed ${compressedSize} should be < original ${Buffer.byteLength(original)}`); + + // Decompress: pull → gunzip → text → verify + { + const rfh = await open(gzPath, 'r'); + const result = await text(rfh.pull(decompressGzip())); + await rfh.close(); + assert.strictEqual(result, original); + } +} + +// ============================================================================= +// Large file write — write 1MB in 64KB chunks +// ============================================================================= + +async function testLargeFileWrite() { + const filePath = path.join(tmpDir, 'writer-large.bin'); + const fh = await open(filePath, 'w'); + const w = fh.writer(); + + const chunkSize = 65536; + const totalSize = 1024 * 1024; // 1MB + const chunk = Buffer.alloc(chunkSize, 0x42); + let written = 0; + + while (written < totalSize) { + await w.write(chunk); + written += chunkSize; + } + + const totalBytes = await w.end(); + await fh.close(); + + assert.strictEqual(totalBytes, totalSize); + assert.strictEqual(fs.statSync(filePath).size, totalSize); + + // Verify content + const data = fs.readFileSync(filePath); + for (let i = 0; i < data.length; i++) { + if (data[i] !== 0x42) { + assert.fail(`Byte at offset ${i} is ${data[i]}, expected 0x42`); + } + } +} + +// ============================================================================= +// Symbol.asyncDispose — await using +// ============================================================================= + +async function testAsyncDispose() { + const filePath = path.join(tmpDir, 'writer-async-dispose.txt'); + { + await using fh = await open(filePath, 'w'); + await using w = fh.writer({ autoClose: true }); + await w.write(Buffer.from('async dispose')); + } + // Both writer and file handle should be cleaned up + assert.strictEqual(fs.readFileSync(filePath, 'utf8'), 'async dispose'); + + // Verify the handle is actually closed by trying to open a new one + // (if the old one were still open with a write lock on some OSes, + // this could fail — but it should succeed). + const fh2 = await open(filePath, 'r'); + await fh2.close(); +} + +// ============================================================================= +// Symbol.asyncDispose — cleanup on error (await using unwinds) +// ============================================================================= + +async function testAsyncDisposeOnError() { + const filePath = path.join(tmpDir, 'writer-dispose-error.txt'); + const fh = await open(filePath, 'w'); + + try { + await using w = fh.writer(); + await w.write(Buffer.from('before error')); + throw new Error('intentional'); + } catch (e) { + assert.strictEqual(e.message, 'intentional'); + } + + // If asyncDispose ran, the handle should be unlocked and reusable + const w2 = fh.writer(); + await w2.write(Buffer.from('after error')); + await w2.end(); + await fh.close(); + + const content = fs.readFileSync(filePath, 'utf8'); + assert.ok(content.includes('after error'), + `Expected 'after error' in ${JSON.stringify(content)}`); +} + +// ============================================================================= +// Run all tests +// ============================================================================= + +Promise.all([ + testBasicWrite(), + testBasicWritev(), + testMixedWriteAndWritev(), + testEndReturnsTotalBytes(), + testAutoCloseOnEnd(), + testAutoCloseOnAbort(), + testStartOption(), + testStartSequentialPosition(), + testLockedState(), + testUnlockAfterEnd(), + testUnlockAfterAbort(), + testWriteAfterEndRejects(), + testClosedHandle(), + testPipeToIntegration(), + testPipeToWithTransform(), + testCompressRoundTrip(), + testLargeFileWrite(), + testAsyncDispose(), + testAsyncDisposeOnError(), +]).then(common.mustCall()); diff --git a/test/parallel/test-stream-new-broadcast.js b/test/parallel/test-stream-new-broadcast.js new file mode 100644 index 00000000000000..6ea0b3858bb64d --- /dev/null +++ b/test/parallel/test-stream-new-broadcast.js @@ -0,0 +1,276 @@ +'use strict'; + +const common = require('../common'); +const assert = require('assert'); +const { broadcast, Broadcast, from, text } = require('stream/new'); + +// ============================================================================= +// Basic broadcast +// ============================================================================= + +async function testBasicBroadcast() { + const { writer, broadcast: bc } = broadcast(); + + // Create two consumers + const consumer1 = bc.push(); + const consumer2 = bc.push(); + + assert.strictEqual(bc.consumerCount, 2); + + await writer.write('hello'); + await writer.end(); + + const [data1, data2] = await Promise.all([ + text(consumer1), + text(consumer2), + ]); + + assert.strictEqual(data1, 'hello'); + assert.strictEqual(data2, 'hello'); +} + +async function testMultipleWrites() { + const { writer, broadcast: bc } = broadcast({ highWaterMark: 10 }); + + const consumer = bc.push(); + + await writer.write('a'); + await writer.write('b'); + await writer.write('c'); + await writer.end(); + + const data = await text(consumer); + assert.strictEqual(data, 'abc'); +} + +async function testConsumerCount() { + const { broadcast: bc } = broadcast(); + + assert.strictEqual(bc.consumerCount, 0); + + const c1 = bc.push(); + assert.strictEqual(bc.consumerCount, 1); + + bc.push(); + assert.strictEqual(bc.consumerCount, 2); + + // Consume c1 to completion (it returns immediately since no data has been + // pushed and we haven't ended yet — but we'll cancel to detach) + bc.cancel(); + + // After cancel, consumers are detached + const batches = []; + for await (const batch of c1) { + batches.push(batch); + } + assert.strictEqual(batches.length, 0); +} + +// ============================================================================= +// Writer methods +// ============================================================================= + +async function testWriteSync() { + const { writer, broadcast: bc } = broadcast({ highWaterMark: 2 }); + const consumer = bc.push(); + + assert.strictEqual(writer.writeSync('a'), true); + assert.strictEqual(writer.writeSync('b'), true); + // Buffer full (highWaterMark=2, strict policy) + assert.strictEqual(writer.writeSync('c'), false); + + writer.endSync(); + + const data = await text(consumer); + assert.strictEqual(data, 'ab'); +} + +async function testWritevSync() { + const { writer, broadcast: bc } = broadcast({ highWaterMark: 10 }); + const consumer = bc.push(); + + assert.strictEqual(writer.writevSync(['hello', ' ', 'world']), true); + writer.endSync(); + + const data = await text(consumer); + assert.strictEqual(data, 'hello world'); +} + +async function testWriterEnd() { + const { writer, broadcast: bc } = broadcast(); + const consumer = bc.push(); + + await writer.write('data'); + const totalBytes = await writer.end(); + assert.ok(totalBytes > 0); + + const data = await text(consumer); + assert.strictEqual(data, 'data'); +} + +async function testWriterAbort() { + const { writer, broadcast: bc } = broadcast(); + const consumer = bc.push(); + + await writer.abort(new Error('test error')); + + await assert.rejects( + async () => { + // eslint-disable-next-line no-unused-vars + for await (const _ of consumer) { + assert.fail('Should not reach here'); + } + }, + { message: 'test error' }, + ); +} + +// ============================================================================= +// Backpressure policies +// ============================================================================= + +async function testDropOldest() { + const { writer, broadcast: bc } = broadcast({ + highWaterMark: 2, + backpressure: 'drop-oldest', + }); + const consumer = bc.push(); + + writer.writeSync('first'); + writer.writeSync('second'); + // This should drop 'first' + writer.writeSync('third'); + writer.endSync(); + + const data = await text(consumer); + assert.strictEqual(data, 'secondthird'); +} + +async function testDropNewest() { + const { writer, broadcast: bc } = broadcast({ + highWaterMark: 1, + backpressure: 'drop-newest', + }); + const consumer = bc.push(); + + writer.writeSync('kept'); + // This should be silently dropped + writer.writeSync('dropped'); + writer.endSync(); + + const data = await text(consumer); + assert.strictEqual(data, 'kept'); +} + +// ============================================================================= +// Cancel +// ============================================================================= + +async function testCancelWithoutReason() { + const { broadcast: bc } = broadcast(); + const consumer = bc.push(); + + bc.cancel(); + + const batches = []; + for await (const batch of consumer) { + batches.push(batch); + } + assert.strictEqual(batches.length, 0); +} + +async function testCancelWithReason() { + const { broadcast: bc } = broadcast(); + + // Start a consumer that is waiting for data (promise pending) + const consumer = bc.push(); + const resultPromise = text(consumer).catch((err) => err); + + // Give the consumer time to enter the waiting state + await new Promise((resolve) => setImmediate(resolve)); + + bc.cancel(new Error('cancelled')); + + const result = await resultPromise; + assert.ok(result instanceof Error); + assert.strictEqual(result.message, 'cancelled'); +} + +// ============================================================================= +// Broadcast.from +// ============================================================================= + +async function testBroadcastFromAsyncIterable() { + const source = from('broadcast-from'); + const { broadcast: bc } = Broadcast.from(source); + const consumer = bc.push(); + + const data = await text(consumer); + assert.strictEqual(data, 'broadcast-from'); +} + +async function testBroadcastFromMultipleConsumers() { + const source = from('shared-data'); + const { broadcast: bc } = Broadcast.from(source); + + const c1 = bc.push(); + const c2 = bc.push(); + + const [data1, data2] = await Promise.all([ + text(c1), + text(c2), + ]); + + assert.strictEqual(data1, 'shared-data'); + assert.strictEqual(data2, 'shared-data'); +} + +// ============================================================================= +// AbortSignal +// ============================================================================= + +async function testAbortSignal() { + const ac = new AbortController(); + const { broadcast: bc } = broadcast({ signal: ac.signal }); + const consumer = bc.push(); + + ac.abort(); + + const batches = []; + for await (const batch of consumer) { + batches.push(batch); + } + assert.strictEqual(batches.length, 0); +} + +async function testAlreadyAbortedSignal() { + const ac = new AbortController(); + ac.abort(); + + const { broadcast: bc } = broadcast({ signal: ac.signal }); + const consumer = bc.push(); + + const batches = []; + for await (const batch of consumer) { + batches.push(batch); + } + assert.strictEqual(batches.length, 0); +} + +Promise.all([ + testBasicBroadcast(), + testMultipleWrites(), + testConsumerCount(), + testWriteSync(), + testWritevSync(), + testWriterEnd(), + testWriterAbort(), + testDropOldest(), + testDropNewest(), + testCancelWithoutReason(), + testCancelWithReason(), + testBroadcastFromAsyncIterable(), + testBroadcastFromMultipleConsumers(), + testAbortSignal(), + testAlreadyAbortedSignal(), +]).then(common.mustCall()); diff --git a/test/parallel/test-stream-new-consumers.js b/test/parallel/test-stream-new-consumers.js new file mode 100644 index 00000000000000..68561ecf18d834 --- /dev/null +++ b/test/parallel/test-stream-new-consumers.js @@ -0,0 +1,319 @@ +'use strict'; + +const common = require('../common'); +const assert = require('assert'); +const { + from, + fromSync, + push, + bytes, + bytesSync, + text, + textSync, + arrayBuffer, + arrayBufferSync, + array, + arraySync, + tap, + tapSync, + merge, +} = require('stream/new'); + +// ============================================================================= +// bytesSync / bytes +// ============================================================================= + +async function testBytesSyncBasic() { + const source = fromSync('hello'); + const data = bytesSync(source); + assert.deepStrictEqual(data, new TextEncoder().encode('hello')); +} + +async function testBytesSyncLimit() { + const source = fromSync('hello world'); + assert.throws( + () => bytesSync(source, { limit: 3 }), + { name: 'RangeError' }, + ); +} + +async function testBytesAsync() { + const source = from('hello-async'); + const data = await bytes(source); + assert.deepStrictEqual(data, new TextEncoder().encode('hello-async')); +} + +async function testBytesAsyncLimit() { + const source = from('hello world'); + await assert.rejects( + () => bytes(source, { limit: 3 }), + { name: 'RangeError' }, + ); +} + +async function testBytesAsyncAbort() { + const ac = new AbortController(); + ac.abort(); + const source = from('data'); + await assert.rejects( + () => bytes(source, { signal: ac.signal }), + (err) => err.name === 'AbortError', + ); +} + +async function testBytesEmpty() { + const source = from([]); + const data = await bytes(source); + assert.strictEqual(data.byteLength, 0); +} + +// ============================================================================= +// textSync / text +// ============================================================================= + +async function testTextSyncBasic() { + const source = fromSync('hello text'); + const data = textSync(source); + assert.strictEqual(data, 'hello text'); +} + +async function testTextAsync() { + const source = from('hello async text'); + const data = await text(source); + assert.strictEqual(data, 'hello async text'); +} + +async function testTextEncoding() { + // Default encoding is utf-8 + const source = from('café'); + const data = await text(source); + assert.strictEqual(data, 'café'); +} + +// ============================================================================= +// arrayBufferSync / arrayBuffer +// ============================================================================= + +async function testArrayBufferSyncBasic() { + const source = fromSync(new Uint8Array([1, 2, 3])); + const ab = arrayBufferSync(source); + assert.ok(ab instanceof ArrayBuffer); + assert.strictEqual(ab.byteLength, 3); + const view = new Uint8Array(ab); + assert.deepStrictEqual(view, new Uint8Array([1, 2, 3])); +} + +async function testArrayBufferAsync() { + const source = from(new Uint8Array([10, 20, 30])); + const ab = await arrayBuffer(source); + assert.ok(ab instanceof ArrayBuffer); + assert.strictEqual(ab.byteLength, 3); + const view = new Uint8Array(ab); + assert.deepStrictEqual(view, new Uint8Array([10, 20, 30])); +} + +// ============================================================================= +// arraySync / array +// ============================================================================= + +async function testArraySyncBasic() { + function* gen() { + yield new Uint8Array([1]); + yield new Uint8Array([2]); + yield new Uint8Array([3]); + } + const source = fromSync(gen()); + const chunks = arraySync(source); + assert.strictEqual(chunks.length, 3); + assert.deepStrictEqual(chunks[0], new Uint8Array([1])); + assert.deepStrictEqual(chunks[1], new Uint8Array([2])); + assert.deepStrictEqual(chunks[2], new Uint8Array([3])); +} + +async function testArraySyncLimit() { + function* gen() { + yield new Uint8Array(100); + yield new Uint8Array(100); + } + const source = fromSync(gen()); + assert.throws( + () => arraySync(source, { limit: 50 }), + { name: 'RangeError' }, + ); +} + +async function testArrayAsync() { + async function* gen() { + yield [new Uint8Array([1])]; + yield [new Uint8Array([2])]; + } + const chunks = await array(gen()); + assert.strictEqual(chunks.length, 2); + assert.deepStrictEqual(chunks[0], new Uint8Array([1])); + assert.deepStrictEqual(chunks[1], new Uint8Array([2])); +} + +async function testArrayAsyncLimit() { + async function* gen() { + yield [new Uint8Array(100)]; + yield [new Uint8Array(100)]; + } + await assert.rejects( + () => array(gen(), { limit: 50 }), + { name: 'RangeError' }, + ); +} + +// ============================================================================= +// tap / tapSync +// ============================================================================= + +async function testTapSync() { + const observed = []; + const observer = tapSync((chunks) => { + if (chunks !== null) { + observed.push(chunks.length); + } + }); + + // tapSync returns a function transform + assert.strictEqual(typeof observer, 'function'); + + // Test that it passes data through unchanged + const input = [new Uint8Array([1]), new Uint8Array([2])]; + const result = observer(input); + assert.deepStrictEqual(result, input); + assert.deepStrictEqual(observed, [2]); + + // null (flush) passes through + const flushResult = observer(null); + assert.strictEqual(flushResult, null); +} + +async function testTapAsync() { + const observed = []; + const observer = tap(async (chunks) => { + if (chunks !== null) { + observed.push(chunks.length); + } + }); + + assert.strictEqual(typeof observer, 'function'); + + const input = [new Uint8Array([1])]; + const result = await observer(input); + assert.deepStrictEqual(result, input); + assert.deepStrictEqual(observed, [1]); +} + +async function testTapInPipeline() { + const { writer, readable } = push(); + const seen = []; + + const observer = tap(async (chunks) => { + if (chunks !== null) { + for (const chunk of chunks) { + seen.push(new TextDecoder().decode(chunk)); + } + } + }); + + writer.write('hello'); + writer.end(); + + // Use pull with tap as a transform + const { pull } = require('stream/new'); + const result = pull(readable, observer); + const data = await text(result); + + assert.strictEqual(data, 'hello'); + assert.strictEqual(seen.length, 1); + assert.strictEqual(seen[0], 'hello'); +} + +// ============================================================================= +// merge +// ============================================================================= + +async function testMergeTwoSources() { + const { writer: w1, readable: r1 } = push(); + const { writer: w2, readable: r2 } = push(); + + w1.write('from-a'); + w1.end(); + w2.write('from-b'); + w2.end(); + + const merged = merge(r1, r2); + const chunks = []; + for await (const batch of merged) { + for (const chunk of batch) { + chunks.push(new TextDecoder().decode(chunk)); + } + } + + // Both sources should be present (order is temporal, not guaranteed) + assert.strictEqual(chunks.length, 2); + assert.ok(chunks.includes('from-a')); + assert.ok(chunks.includes('from-b')); +} + +async function testMergeSingleSource() { + const source = from('only-one'); + const merged = merge(source); + + const data = await text(merged); + assert.strictEqual(data, 'only-one'); +} + +async function testMergeEmpty() { + const merged = merge(); + const batches = []; + for await (const batch of merged) { + batches.push(batch); + } + assert.strictEqual(batches.length, 0); +} + +async function testMergeWithAbortSignal() { + const ac = new AbortController(); + ac.abort(); + + const source = from('data'); + const merged = merge(source, { signal: ac.signal }); + + await assert.rejects( + async () => { + // eslint-disable-next-line no-unused-vars + for await (const _ of merged) { + assert.fail('Should not reach here'); + } + }, + (err) => err.name === 'AbortError', + ); +} + +Promise.all([ + testBytesSyncBasic(), + testBytesSyncLimit(), + testBytesAsync(), + testBytesAsyncLimit(), + testBytesAsyncAbort(), + testBytesEmpty(), + testTextSyncBasic(), + testTextAsync(), + testTextEncoding(), + testArrayBufferSyncBasic(), + testArrayBufferAsync(), + testArraySyncBasic(), + testArraySyncLimit(), + testArrayAsync(), + testArrayAsyncLimit(), + testTapSync(), + testTapAsync(), + testTapInPipeline(), + testMergeTwoSources(), + testMergeSingleSource(), + testMergeEmpty(), + testMergeWithAbortSignal(), +]).then(common.mustCall()); diff --git a/test/parallel/test-stream-new-duplex.js b/test/parallel/test-stream-new-duplex.js new file mode 100644 index 00000000000000..7692cb53e360d2 --- /dev/null +++ b/test/parallel/test-stream-new-duplex.js @@ -0,0 +1,152 @@ +'use strict'; + +const common = require('../common'); +const assert = require('assert'); +const { duplex, text, bytes } = require('stream/new'); + +// ============================================================================= +// Basic duplex +// ============================================================================= + +async function testBasicDuplex() { + const [channelA, channelB] = duplex(); + + // A writes, B reads + await channelA.writer.write('hello from A'); + await channelA.close(); + + const dataAtB = await text(channelB.readable); + assert.strictEqual(dataAtB, 'hello from A'); +} + +async function testBidirectional() { + const [channelA, channelB] = duplex(); + + // A writes to B, B writes to A concurrently + const writeA = (async () => { + await channelA.writer.write('A to B'); + await channelA.close(); + })(); + + const writeB = (async () => { + await channelB.writer.write('B to A'); + await channelB.close(); + })(); + + const readAtB = text(channelB.readable); + const readAtA = text(channelA.readable); + + await Promise.all([writeA, writeB]); + + const [dataAtA, dataAtB] = await Promise.all([readAtA, readAtB]); + + assert.strictEqual(dataAtB, 'A to B'); + assert.strictEqual(dataAtA, 'B to A'); +} + +async function testMultipleWrites() { + const [channelA, channelB] = duplex({ highWaterMark: 10 }); + + await channelA.writer.write('one'); + await channelA.writer.write('two'); + await channelA.writer.write('three'); + await channelA.close(); + + const data = await text(channelB.readable); + assert.strictEqual(data, 'onetwothree'); +} + +async function testChannelClose() { + const [channelA, channelB] = duplex(); + + await channelA.close(); + + // Should be able to close twice without error + await channelA.close(); + + // B's readable should end (A -> B direction is closed) + const batches = []; + for await (const batch of channelB.readable) { + batches.push(batch); + } + assert.strictEqual(batches.length, 0); +} + +async function testWithOptions() { + const [channelA, channelB] = duplex({ + highWaterMark: 2, + backpressure: 'strict', + }); + + await channelA.writer.write('msg'); + await channelA.close(); + + const data = await text(channelB.readable); + assert.strictEqual(data, 'msg'); +} + +async function testPerChannelOptions() { + const [channelA, channelB] = duplex({ + a: { highWaterMark: 1 }, + b: { highWaterMark: 4 }, + }); + + // Channel A -> B direction uses A's options + // Channel B -> A direction uses B's options + await channelA.writer.write('from-a'); + await channelA.close(); + + await channelB.writer.write('from-b'); + await channelB.close(); + + const [dataAtA, dataAtB] = await Promise.all([ + text(channelA.readable), + text(channelB.readable), + ]); + + assert.strictEqual(dataAtB, 'from-a'); + assert.strictEqual(dataAtA, 'from-b'); +} + +async function testAbortSignal() { + const ac = new AbortController(); + const [channelA] = duplex({ signal: ac.signal }); + + ac.abort(); + + // Both directions should error + await assert.rejects( + async () => { + // eslint-disable-next-line no-unused-vars + for await (const _ of channelA.readable) { + assert.fail('Should not reach here'); + } + }, + (err) => err.name === 'AbortError', + ); +} + +async function testEmptyDuplex() { + const [channelA, channelB] = duplex(); + + // Close without writing + await channelA.close(); + await channelB.close(); + + const dataAtA = await bytes(channelA.readable); + const dataAtB = await bytes(channelB.readable); + + assert.strictEqual(dataAtA.byteLength, 0); + assert.strictEqual(dataAtB.byteLength, 0); +} + +Promise.all([ + testBasicDuplex(), + testBidirectional(), + testMultipleWrites(), + testChannelClose(), + testWithOptions(), + testPerChannelOptions(), + testAbortSignal(), + testEmptyDuplex(), +]).then(common.mustCall()); diff --git a/test/parallel/test-stream-new-from.js b/test/parallel/test-stream-new-from.js new file mode 100644 index 00000000000000..e4925128cd26a2 --- /dev/null +++ b/test/parallel/test-stream-new-from.js @@ -0,0 +1,223 @@ +'use strict'; + +const common = require('../common'); +const assert = require('assert'); +const { from, fromSync, Stream } = require('stream/new'); + +// ============================================================================= +// fromSync() tests +// ============================================================================= + +async function testFromSyncString() { + // String input should be UTF-8 encoded + const readable = fromSync('hello'); + const batches = []; + for (const batch of readable) { + batches.push(batch); + } + assert.strictEqual(batches.length, 1); + assert.strictEqual(batches[0].length, 1); + assert.deepStrictEqual(batches[0][0], + new TextEncoder().encode('hello')); +} + +async function testFromSyncUint8Array() { + const input = new Uint8Array([1, 2, 3]); + const readable = fromSync(input); + const batches = []; + for (const batch of readable) { + batches.push(batch); + } + assert.strictEqual(batches.length, 1); + assert.strictEqual(batches[0].length, 1); + assert.deepStrictEqual(batches[0][0], input); +} + +async function testFromSyncArrayBuffer() { + const ab = new ArrayBuffer(4); + new Uint8Array(ab).set([10, 20, 30, 40]); + const readable = fromSync(ab); + const batches = []; + for (const batch of readable) { + batches.push(batch); + } + assert.strictEqual(batches.length, 1); + assert.deepStrictEqual(batches[0][0], new Uint8Array([10, 20, 30, 40])); +} + +async function testFromSyncUint8ArrayArray() { + // Array of Uint8Array should yield as a single batch + const chunks = [new Uint8Array([1]), new Uint8Array([2])]; + const readable = fromSync(chunks); + const batches = []; + for (const batch of readable) { + batches.push(batch); + } + assert.strictEqual(batches.length, 1); + assert.strictEqual(batches[0].length, 2); +} + +async function testFromSyncGenerator() { + function* gen() { + yield new Uint8Array([1, 2]); + yield new Uint8Array([3, 4]); + } + const readable = fromSync(gen()); + const batches = []; + for (const batch of readable) { + batches.push(batch); + } + assert.strictEqual(batches.length, 2); + assert.deepStrictEqual(batches[0][0], new Uint8Array([1, 2])); + assert.deepStrictEqual(batches[1][0], new Uint8Array([3, 4])); +} + +async function testFromSyncNestedIterables() { + // Nested arrays and strings should be flattened + function* gen() { + yield ['hello', ' ', 'world']; + } + const readable = fromSync(gen()); + const batches = []; + for (const batch of readable) { + batches.push(batch); + } + assert.strictEqual(batches.length, 1); + assert.strictEqual(batches[0].length, 3); +} + +async function testFromSyncToStreamableProtocol() { + const sym = Symbol.for('Stream.toStreamable'); + const obj = { + [sym]() { + return 'protocol-data'; + }, + }; + function* gen() { + yield obj; + } + const readable = fromSync(gen()); + const batches = []; + for (const batch of readable) { + batches.push(batch); + } + assert.strictEqual(batches.length, 1); + assert.deepStrictEqual(batches[0][0], + new TextEncoder().encode('protocol-data')); +} + +async function testFromSyncRejectsNonStreamable() { + assert.throws( + () => fromSync(12345), + { name: 'TypeError' }, + ); +} + +// ============================================================================= +// from() tests (async) +// ============================================================================= + +async function testFromString() { + const readable = from('hello-async'); + const batches = []; + for await (const batch of readable) { + batches.push(batch); + } + assert.strictEqual(batches.length, 1); + assert.deepStrictEqual(batches[0][0], + new TextEncoder().encode('hello-async')); +} + +async function testFromAsyncGenerator() { + async function* gen() { + yield new Uint8Array([10, 20]); + yield new Uint8Array([30, 40]); + } + const readable = from(gen()); + const batches = []; + for await (const batch of readable) { + batches.push(batch); + } + assert.strictEqual(batches.length, 2); + assert.deepStrictEqual(batches[0][0], new Uint8Array([10, 20])); + assert.deepStrictEqual(batches[1][0], new Uint8Array([30, 40])); +} + +async function testFromSyncIterableAsAsync() { + // Sync iterable passed to from() should work + function* gen() { + yield new Uint8Array([1]); + yield new Uint8Array([2]); + } + const readable = from(gen()); + const batches = []; + for await (const batch of readable) { + batches.push(batch); + } + // Sync iterables get batched together + assert.ok(batches.length >= 1); +} + +async function testFromToAsyncStreamableProtocol() { + const sym = Symbol.for('Stream.toAsyncStreamable'); + const obj = { + [sym]() { + return 'async-protocol-data'; + }, + }; + async function* gen() { + yield obj; + } + const readable = from(gen()); + const batches = []; + for await (const batch of readable) { + batches.push(batch); + } + assert.strictEqual(batches.length, 1); + assert.deepStrictEqual(batches[0][0], + new TextEncoder().encode('async-protocol-data')); +} + +async function testFromRejectsNonStreamable() { + assert.throws( + () => from(12345), + { name: 'TypeError' }, + ); +} + +async function testFromEmptyArray() { + const readable = from([]); + const batches = []; + for await (const batch of readable) { + batches.push(batch); + } + assert.strictEqual(batches.length, 0); +} + +// Also accessible via Stream namespace +async function testStreamNamespace() { + const readable = Stream.from('via-namespace'); + const batches = []; + for await (const batch of readable) { + batches.push(batch); + } + assert.strictEqual(batches.length, 1); +} + +Promise.all([ + testFromSyncString(), + testFromSyncUint8Array(), + testFromSyncArrayBuffer(), + testFromSyncUint8ArrayArray(), + testFromSyncGenerator(), + testFromSyncNestedIterables(), + testFromSyncToStreamableProtocol(), + testFromSyncRejectsNonStreamable(), + testFromString(), + testFromAsyncGenerator(), + testFromSyncIterableAsAsync(), + testFromToAsyncStreamableProtocol(), + testFromRejectsNonStreamable(), + testFromEmptyArray(), + testStreamNamespace(), +]).then(common.mustCall()); diff --git a/test/parallel/test-stream-new-namespace.js b/test/parallel/test-stream-new-namespace.js new file mode 100644 index 00000000000000..cacdead5a19ea0 --- /dev/null +++ b/test/parallel/test-stream-new-namespace.js @@ -0,0 +1,209 @@ +'use strict'; + +const common = require('../common'); +const assert = require('assert'); +const streamNew = require('stream/new'); + +// ============================================================================= +// Stream namespace object +// ============================================================================= + +async function testStreamNamespaceExists() { + assert.ok(streamNew.Stream); + assert.strictEqual(typeof streamNew.Stream, 'object'); +} + +async function testStreamNamespaceFrozen() { + assert.ok(Object.isFrozen(streamNew.Stream)); +} + +async function testStreamNamespaceFactories() { + const { Stream } = streamNew; + + assert.strictEqual(typeof Stream.push, 'function'); + assert.strictEqual(typeof Stream.duplex, 'function'); + assert.strictEqual(typeof Stream.from, 'function'); + assert.strictEqual(typeof Stream.fromSync, 'function'); +} + +async function testStreamNamespacePipelines() { + const { Stream } = streamNew; + + assert.strictEqual(typeof Stream.pull, 'function'); + assert.strictEqual(typeof Stream.pullSync, 'function'); + assert.strictEqual(typeof Stream.pipeTo, 'function'); + assert.strictEqual(typeof Stream.pipeToSync, 'function'); +} + +async function testStreamNamespaceAsyncConsumers() { + const { Stream } = streamNew; + + assert.strictEqual(typeof Stream.bytes, 'function'); + assert.strictEqual(typeof Stream.text, 'function'); + assert.strictEqual(typeof Stream.arrayBuffer, 'function'); + assert.strictEqual(typeof Stream.array, 'function'); +} + +async function testStreamNamespaceSyncConsumers() { + const { Stream } = streamNew; + + assert.strictEqual(typeof Stream.bytesSync, 'function'); + assert.strictEqual(typeof Stream.textSync, 'function'); + assert.strictEqual(typeof Stream.arrayBufferSync, 'function'); + assert.strictEqual(typeof Stream.arraySync, 'function'); +} + +async function testStreamNamespaceCombining() { + const { Stream } = streamNew; + + assert.strictEqual(typeof Stream.merge, 'function'); + assert.strictEqual(typeof Stream.broadcast, 'function'); + assert.strictEqual(typeof Stream.share, 'function'); + assert.strictEqual(typeof Stream.shareSync, 'function'); +} + +async function testStreamNamespaceUtilities() { + const { Stream } = streamNew; + + assert.strictEqual(typeof Stream.tap, 'function'); + assert.strictEqual(typeof Stream.tapSync, 'function'); + assert.strictEqual(typeof Stream.ondrain, 'function'); +} + +async function testStreamNamespaceProtocols() { + const { Stream } = streamNew; + + assert.strictEqual(typeof Stream.toStreamable, 'symbol'); + assert.strictEqual(typeof Stream.toAsyncStreamable, 'symbol'); + assert.strictEqual(typeof Stream.broadcastProtocol, 'symbol'); + assert.strictEqual(typeof Stream.shareProtocol, 'symbol'); + assert.strictEqual(typeof Stream.shareSyncProtocol, 'symbol'); + assert.strictEqual(typeof Stream.drainableProtocol, 'symbol'); +} + +// ============================================================================= +// Individual exports (destructured imports) +// ============================================================================= + +async function testIndividualExports() { + // Factories + assert.strictEqual(typeof streamNew.push, 'function'); + assert.strictEqual(typeof streamNew.duplex, 'function'); + assert.strictEqual(typeof streamNew.from, 'function'); + assert.strictEqual(typeof streamNew.fromSync, 'function'); + + // Pipelines + assert.strictEqual(typeof streamNew.pull, 'function'); + assert.strictEqual(typeof streamNew.pullSync, 'function'); + assert.strictEqual(typeof streamNew.pipeTo, 'function'); + assert.strictEqual(typeof streamNew.pipeToSync, 'function'); + + // Consumers + assert.strictEqual(typeof streamNew.bytes, 'function'); + assert.strictEqual(typeof streamNew.bytesSync, 'function'); + assert.strictEqual(typeof streamNew.text, 'function'); + assert.strictEqual(typeof streamNew.textSync, 'function'); + assert.strictEqual(typeof streamNew.arrayBuffer, 'function'); + assert.strictEqual(typeof streamNew.arrayBufferSync, 'function'); + assert.strictEqual(typeof streamNew.array, 'function'); + assert.strictEqual(typeof streamNew.arraySync, 'function'); + + // Combining + assert.strictEqual(typeof streamNew.merge, 'function'); + assert.strictEqual(typeof streamNew.broadcast, 'function'); + assert.strictEqual(typeof streamNew.share, 'function'); + assert.strictEqual(typeof streamNew.shareSync, 'function'); + + // Utilities + assert.strictEqual(typeof streamNew.tap, 'function'); + assert.strictEqual(typeof streamNew.tapSync, 'function'); + assert.strictEqual(typeof streamNew.ondrain, 'function'); + + // Protocol symbols + assert.strictEqual(typeof streamNew.toStreamable, 'symbol'); + assert.strictEqual(typeof streamNew.toAsyncStreamable, 'symbol'); + assert.strictEqual(typeof streamNew.broadcastProtocol, 'symbol'); + assert.strictEqual(typeof streamNew.shareProtocol, 'symbol'); + assert.strictEqual(typeof streamNew.shareSyncProtocol, 'symbol'); + assert.strictEqual(typeof streamNew.drainableProtocol, 'symbol'); +} + +async function testMultiConsumerExports() { + // Broadcast and Share constructors/factories + assert.ok(streamNew.Broadcast); + assert.strictEqual(typeof streamNew.Broadcast.from, 'function'); + assert.ok(streamNew.Share); + assert.strictEqual(typeof streamNew.Share.from, 'function'); + assert.ok(streamNew.SyncShare); + assert.strictEqual(typeof streamNew.SyncShare.fromSync, 'function'); +} + +// ============================================================================= +// Cross-check: namespace matches individual exports +// ============================================================================= + +async function testNamespaceMatchesExports() { + const { Stream } = streamNew; + + // Every function on Stream should also be available as a direct export + assert.strictEqual(Stream.push, streamNew.push); + assert.strictEqual(Stream.duplex, streamNew.duplex); + assert.strictEqual(Stream.from, streamNew.from); + assert.strictEqual(Stream.fromSync, streamNew.fromSync); + assert.strictEqual(Stream.pull, streamNew.pull); + assert.strictEqual(Stream.pullSync, streamNew.pullSync); + assert.strictEqual(Stream.pipeTo, streamNew.pipeTo); + assert.strictEqual(Stream.pipeToSync, streamNew.pipeToSync); + assert.strictEqual(Stream.bytes, streamNew.bytes); + assert.strictEqual(Stream.text, streamNew.text); + assert.strictEqual(Stream.arrayBuffer, streamNew.arrayBuffer); + assert.strictEqual(Stream.array, streamNew.array); + assert.strictEqual(Stream.bytesSync, streamNew.bytesSync); + assert.strictEqual(Stream.textSync, streamNew.textSync); + assert.strictEqual(Stream.arrayBufferSync, streamNew.arrayBufferSync); + assert.strictEqual(Stream.arraySync, streamNew.arraySync); + assert.strictEqual(Stream.merge, streamNew.merge); + assert.strictEqual(Stream.broadcast, streamNew.broadcast); + assert.strictEqual(Stream.share, streamNew.share); + assert.strictEqual(Stream.shareSync, streamNew.shareSync); + assert.strictEqual(Stream.tap, streamNew.tap); + assert.strictEqual(Stream.tapSync, streamNew.tapSync); + assert.strictEqual(Stream.ondrain, streamNew.ondrain); + + // Protocol symbols + assert.strictEqual(Stream.toStreamable, streamNew.toStreamable); + assert.strictEqual(Stream.toAsyncStreamable, streamNew.toAsyncStreamable); + assert.strictEqual(Stream.broadcastProtocol, streamNew.broadcastProtocol); + assert.strictEqual(Stream.shareProtocol, streamNew.shareProtocol); + assert.strictEqual(Stream.shareSyncProtocol, streamNew.shareSyncProtocol); + assert.strictEqual(Stream.drainableProtocol, streamNew.drainableProtocol); +} + +// ============================================================================= +// Require paths +// ============================================================================= + +async function testRequirePaths() { + // Both require('stream/new') and require('node:stream/new') should work + const fromPlain = require('stream/new'); + const fromNode = require('node:stream/new'); + + assert.strictEqual(fromPlain.Stream, fromNode.Stream); + assert.strictEqual(fromPlain.push, fromNode.push); +} + +Promise.all([ + testStreamNamespaceExists(), + testStreamNamespaceFrozen(), + testStreamNamespaceFactories(), + testStreamNamespacePipelines(), + testStreamNamespaceAsyncConsumers(), + testStreamNamespaceSyncConsumers(), + testStreamNamespaceCombining(), + testStreamNamespaceUtilities(), + testStreamNamespaceProtocols(), + testIndividualExports(), + testMultiConsumerExports(), + testNamespaceMatchesExports(), + testRequirePaths(), +]).then(common.mustCall()); diff --git a/test/parallel/test-stream-new-pull.js b/test/parallel/test-stream-new-pull.js new file mode 100644 index 00000000000000..34db9b96c74e5c --- /dev/null +++ b/test/parallel/test-stream-new-pull.js @@ -0,0 +1,213 @@ +'use strict'; + +const common = require('../common'); +const assert = require('assert'); +const { pull, pullSync, pipeTo, pipeToSync, from, fromSync, bytesSync, + text } = require('stream/new'); + +// ============================================================================= +// pullSync() tests +// ============================================================================= + +async function testPullSyncIdentity() { + // No transforms - just pass through + const source = fromSync('hello'); + const result = pullSync(source); + const data = bytesSync(result); + assert.deepStrictEqual(data, new TextEncoder().encode('hello')); +} + +async function testPullSyncStatelessTransform() { + const source = fromSync('abc'); + const upper = (chunks) => { + if (chunks === null) return null; + return chunks.map((c) => { + const str = new TextDecoder().decode(c); + return new TextEncoder().encode(str.toUpperCase()); + }); + }; + const result = pullSync(source, upper); + const data = bytesSync(result); + assert.deepStrictEqual(data, new TextEncoder().encode('ABC')); +} + +async function testPullSyncStatefulTransform() { + const source = fromSync('data'); + const stateful = { + transform: function*(source) { + for (const chunks of source) { + if (chunks === null) { + // Flush: emit trailer + yield new TextEncoder().encode('-END'); + continue; + } + for (const chunk of chunks) { + yield chunk; + } + } + }, + }; + const result = pullSync(source, stateful); + const data = new TextDecoder().decode(bytesSync(result)); + assert.strictEqual(data, 'data-END'); +} + +async function testPullSyncChainedTransforms() { + const source = fromSync('hello'); + const addExcl = (chunks) => { + if (chunks === null) return null; + return [...chunks, new TextEncoder().encode('!')]; + }; + const addQ = (chunks) => { + if (chunks === null) return null; + return [...chunks, new TextEncoder().encode('?')]; + }; + const result = pullSync(source, addExcl, addQ); + const data = new TextDecoder().decode(bytesSync(result)); + assert.strictEqual(data, 'hello!?'); +} + +// ============================================================================= +// pull() tests (async) +// ============================================================================= + +async function testPullIdentity() { + const source = from('hello-async'); + const result = pull(source); + const data = await text(result); + assert.strictEqual(data, 'hello-async'); +} + +async function testPullStatelessTransform() { + const source = from('abc'); + const upper = (chunks) => { + if (chunks === null) return null; + return chunks.map((c) => { + const str = new TextDecoder().decode(c); + return new TextEncoder().encode(str.toUpperCase()); + }); + }; + const result = pull(source, upper); + const data = await text(result); + assert.strictEqual(data, 'ABC'); +} + +async function testPullStatefulTransform() { + const source = from('data'); + const stateful = { + transform: async function*(source) { + for await (const chunks of source) { + if (chunks === null) { + yield new TextEncoder().encode('-ASYNC-END'); + continue; + } + for (const chunk of chunks) { + yield chunk; + } + } + }, + }; + const result = pull(source, stateful); + const data = await text(result); + assert.strictEqual(data, 'data-ASYNC-END'); +} + +async function testPullWithAbortSignal() { + const ac = new AbortController(); + ac.abort(); + + async function* gen() { + yield [new Uint8Array([1])]; + } + + const result = pull(gen(), { signal: ac.signal }); + await assert.rejects( + async () => { + // eslint-disable-next-line no-unused-vars + for await (const _ of result) { + assert.fail('Should not reach here'); + } + }, + (err) => err.name === 'AbortError', + ); +} + +async function testPullChainedTransforms() { + const source = from('hello'); + const transforms = [ + (chunks) => { + if (chunks === null) return null; + return [...chunks, new TextEncoder().encode('!')]; + }, + (chunks) => { + if (chunks === null) return null; + return [...chunks, new TextEncoder().encode('?')]; + }, + ]; + const result = pull(source, ...transforms); + const data = await text(result); + assert.strictEqual(data, 'hello!?'); +} + +// ============================================================================= +// pipeTo() / pipeToSync() tests +// ============================================================================= + +async function testPipeToSync() { + const source = fromSync('pipe-data'); + const written = []; + const writer = { + write(chunk) { written.push(chunk); }, + end() { return written.length; }, + abort() {}, + }; + + const totalBytes = pipeToSync(source, writer); + assert.ok(totalBytes > 0); + assert.ok(written.length > 0); + const result = new TextDecoder().decode( + new Uint8Array(written.reduce((acc, c) => [...acc, ...c], []))); + assert.strictEqual(result, 'pipe-data'); +} + +async function testPipeTo() { + const source = from('async-pipe-data'); + const written = []; + const writer = { + async write(chunk) { written.push(chunk); }, + async end() { return written.length; }, + async abort() {}, + }; + + const totalBytes = await pipeTo(source, writer); + assert.ok(totalBytes > 0); + assert.ok(written.length > 0); +} + +async function testPipeToPreventClose() { + const source = from('data'); + let endCalled = false; + const writer = { + async write() {}, + async end() { endCalled = true; }, + async abort() {}, + }; + + await pipeTo(source, writer, { preventClose: true }); + assert.strictEqual(endCalled, false); +} + +Promise.all([ + testPullSyncIdentity(), + testPullSyncStatelessTransform(), + testPullSyncStatefulTransform(), + testPullSyncChainedTransforms(), + testPullIdentity(), + testPullStatelessTransform(), + testPullStatefulTransform(), + testPullWithAbortSignal(), + testPullChainedTransforms(), + testPipeToSync(), + testPipeTo(), + testPipeToPreventClose(), +]).then(common.mustCall()); diff --git a/test/parallel/test-stream-new-push.js b/test/parallel/test-stream-new-push.js new file mode 100644 index 00000000000000..2c0a2550ed510d --- /dev/null +++ b/test/parallel/test-stream-new-push.js @@ -0,0 +1,220 @@ +'use strict'; + +const common = require('../common'); +const assert = require('assert'); +const { push, text, ondrain } = require('stream/new'); + +async function testBasicWriteRead() { + const { writer, readable } = push(); + + writer.write('hello'); + writer.end(); + + const data = await text(readable); + assert.strictEqual(data, 'hello'); +} + +async function testMultipleWrites() { + const { writer, readable } = push({ highWaterMark: 10 }); + + writer.write('a'); + writer.write('b'); + writer.write('c'); + writer.end(); + + const data = await text(readable); + assert.strictEqual(data, 'abc'); +} + +async function testDesiredSize() { + const { writer } = push({ highWaterMark: 3 }); + + assert.strictEqual(writer.desiredSize, 3); + writer.writeSync('a'); + assert.strictEqual(writer.desiredSize, 2); + writer.writeSync('b'); + assert.strictEqual(writer.desiredSize, 1); + writer.writeSync('c'); + assert.strictEqual(writer.desiredSize, 0); + + writer.end(); + assert.strictEqual(writer.desiredSize, null); +} + +async function testStrictBackpressure() { + const { writer, readable } = push({ + highWaterMark: 1, + backpressure: 'strict', + }); + + // First write should succeed synchronously + assert.strictEqual(writer.writeSync('a'), true); + // Second write should fail synchronously (buffer full) + assert.strictEqual(writer.writeSync('b'), false); + + // Consume to free space, then end + const resultPromise = text(readable); + writer.end(); + const data = await resultPromise; + assert.strictEqual(data, 'a'); +} + +async function testDropOldest() { + const { writer, readable } = push({ + highWaterMark: 2, + backpressure: 'drop-oldest', + }); + + writer.writeSync('first'); + writer.writeSync('second'); + // This should drop 'first' + writer.writeSync('third'); + writer.end(); + + const batches = []; + for await (const batch of readable) { + batches.push(batch); + } + // Should have 'second' and 'third' + const allBytes = []; + for (const batch of batches) { + for (const chunk of batch) { + allBytes.push(...chunk); + } + } + const result = new TextDecoder().decode(new Uint8Array(allBytes)); + assert.strictEqual(result, 'secondthird'); +} + +async function testDropNewest() { + const { writer, readable } = push({ + highWaterMark: 1, + backpressure: 'drop-newest', + }); + + writer.writeSync('kept'); + // This is silently dropped + writer.writeSync('dropped'); + writer.end(); + + const data = await text(readable); + assert.strictEqual(data, 'kept'); +} + +async function testWriterEnd() { + const { writer, readable } = push(); + + const totalBytes = writer.endSync(); + assert.strictEqual(totalBytes, 0); + + const batches = []; + for await (const batch of readable) { + batches.push(batch); + } + assert.strictEqual(batches.length, 0); +} + +async function testWriterAbort() { + const { writer, readable } = push(); + + writer.abort(new Error('test abort')); + + await assert.rejects( + async () => { + // eslint-disable-next-line no-unused-vars + for await (const _ of readable) { + assert.fail('Should not reach here'); + } + }, + { message: 'test abort' }, + ); +} + +async function testConsumerBreak() { + const { writer, readable } = push({ highWaterMark: 10 }); + + writer.writeSync('a'); + writer.writeSync('b'); + writer.writeSync('c'); + + // Break after first batch + // eslint-disable-next-line no-unused-vars + for await (const _ of readable) { + break; + } + + // Writer should now see null desiredSize + assert.strictEqual(writer.desiredSize, null); +} + +async function testAbortSignal() { + const ac = new AbortController(); + const { readable } = push({ signal: ac.signal }); + + ac.abort(); + + await assert.rejects( + async () => { + // eslint-disable-next-line no-unused-vars + for await (const _ of readable) { + assert.fail('Should not reach here'); + } + }, + (err) => err.name === 'AbortError', + ); +} + +async function testOndrain() { + const { writer } = push({ highWaterMark: 1 }); + + // With space available, ondrain resolves immediately + const drainResult = ondrain(writer); + assert.ok(drainResult instanceof Promise); + const result = await drainResult; + assert.strictEqual(result, true); + + // After close, ondrain returns null + writer.end(); + assert.strictEqual(ondrain(writer), null); +} + +async function testOndainNonDrainable() { + // Non-drainable objects return null + assert.strictEqual(ondrain(null), null); + assert.strictEqual(ondrain({}), null); + assert.strictEqual(ondrain('string'), null); +} + +async function testPushWithTransforms() { + const upper = (chunks) => { + if (chunks === null) return null; + return chunks.map((c) => { + const str = new TextDecoder().decode(c); + return new TextEncoder().encode(str.toUpperCase()); + }); + }; + + const { writer, readable } = push(upper); + + writer.write('hello'); + writer.end(); + + const data = await text(readable); + assert.strictEqual(data, 'HELLO'); +} + +Promise.all([ + testBasicWriteRead(), + testMultipleWrites(), + testDesiredSize(), + testStrictBackpressure(), + testDropOldest(), + testDropNewest(), + testWriterEnd(), + testWriterAbort(), + testConsumerBreak(), + testAbortSignal(), + testOndrain(), + testOndainNonDrainable(), + testPushWithTransforms(), +]).then(common.mustCall()); diff --git a/test/parallel/test-stream-new-share.js b/test/parallel/test-stream-new-share.js new file mode 100644 index 00000000000000..a97ae62deaf3ad --- /dev/null +++ b/test/parallel/test-stream-new-share.js @@ -0,0 +1,240 @@ +'use strict'; + +const common = require('../common'); +const assert = require('assert'); +const { + share, + shareSync, + Share, + SyncShare, + from, + fromSync, + text, + textSync, + +} = require('stream/new'); + +// ============================================================================= +// Async share() +// ============================================================================= + +async function testBasicShare() { + const source = from('hello shared'); + const shared = share(source); + + const consumer = shared.pull(); + const data = await text(consumer); + assert.strictEqual(data, 'hello shared'); +} + +async function testShareMultipleConsumers() { + async function* gen() { + yield [new TextEncoder().encode('chunk1')]; + yield [new TextEncoder().encode('chunk2')]; + yield [new TextEncoder().encode('chunk3')]; + } + + const shared = share(gen(), { highWaterMark: 16 }); + + const c1 = shared.pull(); + const c2 = shared.pull(); + + assert.strictEqual(shared.consumerCount, 2); + + const [data1, data2] = await Promise.all([ + text(c1), + text(c2), + ]); + + assert.strictEqual(data1, 'chunk1chunk2chunk3'); + assert.strictEqual(data2, 'chunk1chunk2chunk3'); +} + +async function testShareConsumerCount() { + const source = from('data'); + const shared = share(source); + + assert.strictEqual(shared.consumerCount, 0); + + const c1 = shared.pull(); + assert.strictEqual(shared.consumerCount, 1); + + const c2 = shared.pull(); + assert.strictEqual(shared.consumerCount, 2); + + // Cancel detaches all consumers + shared.cancel(); + + // Both should complete immediately + const [data1, data2] = await Promise.all([ + text(c1), + text(c2), + ]); + assert.strictEqual(data1, ''); + assert.strictEqual(data2, ''); +} + +async function testShareCancel() { + const source = from('data'); + const shared = share(source); + const consumer = shared.pull(); + + shared.cancel(); + + const batches = []; + for await (const batch of consumer) { + batches.push(batch); + } + assert.strictEqual(batches.length, 0); +} + +async function testShareCancelWithReason() { + const source = from('data'); + const shared = share(source); + const consumer = shared.pull(); + + shared.cancel(new Error('share cancelled')); + + await assert.rejects( + async () => { + // eslint-disable-next-line no-unused-vars + for await (const _ of consumer) { + assert.fail('Should not reach here'); + } + }, + { message: 'share cancelled' }, + ); +} + +async function testShareAbortSignal() { + const ac = new AbortController(); + const source = from('data'); + const shared = share(source, { signal: ac.signal }); + const consumer = shared.pull(); + + ac.abort(); + + const batches = []; + for await (const batch of consumer) { + batches.push(batch); + } + assert.strictEqual(batches.length, 0); +} + +async function testShareAlreadyAborted() { + const ac = new AbortController(); + ac.abort(); + + const source = from('data'); + const shared = share(source, { signal: ac.signal }); + const consumer = shared.pull(); + + const batches = []; + for await (const batch of consumer) { + batches.push(batch); + } + assert.strictEqual(batches.length, 0); +} + +// ============================================================================= +// Share.from +// ============================================================================= + +async function testShareFrom() { + const source = from('share-from'); + const shared = Share.from(source); + const consumer = shared.pull(); + + const data = await text(consumer); + assert.strictEqual(data, 'share-from'); +} + +async function testShareFromRejectsNonStreamable() { + assert.throws( + () => Share.from(12345), + { name: 'TypeError' }, + ); +} + +// ============================================================================= +// Sync share +// ============================================================================= + +async function testShareSyncBasic() { + const source = fromSync('sync shared'); + const shared = shareSync(source); + + const consumer = shared.pull(); + const data = textSync(consumer); + assert.strictEqual(data, 'sync shared'); +} + +async function testShareSyncMultipleConsumers() { + function* gen() { + yield [new TextEncoder().encode('a')]; + yield [new TextEncoder().encode('b')]; + yield [new TextEncoder().encode('c')]; + } + + const shared = shareSync(gen(), { highWaterMark: 16 }); + + const c1 = shared.pull(); + const c2 = shared.pull(); + + const data1 = textSync(c1); + const data2 = textSync(c2); + + assert.strictEqual(data1, 'abc'); + assert.strictEqual(data2, 'abc'); +} + +async function testShareSyncCancel() { + const source = fromSync('data'); + const shared = shareSync(source); + const consumer = shared.pull(); + + shared.cancel(); + + const batches = []; + for (const batch of consumer) { + batches.push(batch); + } + assert.strictEqual(batches.length, 0); +} + +// ============================================================================= +// SyncShare.fromSync +// ============================================================================= + +async function testSyncShareFromSync() { + const source = fromSync('sync-share-from'); + const shared = SyncShare.fromSync(source); + const consumer = shared.pull(); + + const data = textSync(consumer); + assert.strictEqual(data, 'sync-share-from'); +} + +async function testSyncShareFromRejectsNonStreamable() { + assert.throws( + () => SyncShare.fromSync(12345), + { name: 'TypeError' }, + ); +} + +Promise.all([ + testBasicShare(), + testShareMultipleConsumers(), + testShareConsumerCount(), + testShareCancel(), + testShareCancelWithReason(), + testShareAbortSignal(), + testShareAlreadyAborted(), + testShareFrom(), + testShareFromRejectsNonStreamable(), + testShareSyncBasic(), + testShareSyncMultipleConsumers(), + testShareSyncCancel(), + testSyncShareFromSync(), + testSyncShareFromRejectsNonStreamable(), +]).then(common.mustCall()); diff --git a/test/parallel/test-stream-new-transform.js b/test/parallel/test-stream-new-transform.js new file mode 100644 index 00000000000000..fb027dcff5497d --- /dev/null +++ b/test/parallel/test-stream-new-transform.js @@ -0,0 +1,395 @@ +'use strict'; + +const common = require('../common'); +const assert = require('assert'); +const { + from, + pull, + bytes, + text, + compressGzip, + compressDeflate, + compressBrotli, + compressZstd, + decompressGzip, + decompressDeflate, + decompressBrotli, + decompressZstd, +} = require('stream/new'); + +// ============================================================================= +// Helper: compress then decompress, verify round-trip equality +// ============================================================================= + +async function roundTrip(input, compress, decompress) { + const source = from(input); + const compressed = pull(source, compress); + const decompressed = pull(compressed, decompress); + return text(decompressed); +} + +async function roundTripBytes(inputBuf, compress, decompress) { + const source = from(inputBuf); + const compressed = pull(source, compress); + const decompressed = pull(compressed, decompress); + return bytes(decompressed); +} + +// ============================================================================= +// Gzip round-trip tests +// ============================================================================= + +async function testGzipRoundTrip() { + const input = 'Hello, gzip compression!'; + const result = await roundTrip(input, compressGzip(), decompressGzip()); + assert.strictEqual(result, input); +} + +async function testGzipLargeData() { + // 100KB of repeated text - exercises multi-chunk path + const input = 'gzip large data test. '.repeat(5000); + const result = await roundTrip(input, compressGzip(), decompressGzip()); + assert.strictEqual(result, input); +} + +async function testGzipActuallyCompresses() { + const input = 'Repeated data compresses well. '.repeat(1000); + const inputBuf = Buffer.from(input); + const source = from(inputBuf); + const compressed = await bytes(pull(source, compressGzip())); + assert.ok(compressed.byteLength < inputBuf.byteLength, + `Compressed ${compressed.byteLength} should be < original ${inputBuf.byteLength}`); +} + +// ============================================================================= +// Deflate round-trip tests +// ============================================================================= + +async function testDeflateRoundTrip() { + const input = 'Hello, deflate compression!'; + const result = await roundTrip(input, compressDeflate(), decompressDeflate()); + assert.strictEqual(result, input); +} + +async function testDeflateLargeData() { + const input = 'deflate large data test. '.repeat(5000); + const result = await roundTrip(input, compressDeflate(), decompressDeflate()); + assert.strictEqual(result, input); +} + +async function testDeflateActuallyCompresses() { + const input = 'Repeated data compresses well. '.repeat(1000); + const inputBuf = Buffer.from(input); + const source = from(inputBuf); + const compressed = await bytes(pull(source, compressDeflate())); + assert.ok(compressed.byteLength < inputBuf.byteLength, + `Compressed ${compressed.byteLength} should be < original ${inputBuf.byteLength}`); +} + +// ============================================================================= +// Brotli round-trip tests +// ============================================================================= + +async function testBrotliRoundTrip() { + const input = 'Hello, brotli compression!'; + const result = await roundTrip(input, compressBrotli(), decompressBrotli()); + assert.strictEqual(result, input); +} + +async function testBrotliLargeData() { + const input = 'brotli large data test. '.repeat(5000); + const result = await roundTrip(input, compressBrotli(), decompressBrotli()); + assert.strictEqual(result, input); +} + +async function testBrotliActuallyCompresses() { + const input = 'Repeated data compresses well. '.repeat(1000); + const inputBuf = Buffer.from(input); + const source = from(inputBuf); + const compressed = await bytes(pull(source, compressBrotli())); + assert.ok(compressed.byteLength < inputBuf.byteLength, + `Compressed ${compressed.byteLength} should be < original ${inputBuf.byteLength}`); +} + +// ============================================================================= +// Zstd round-trip tests +// ============================================================================= + +async function testZstdRoundTrip() { + const input = 'Hello, zstd compression!'; + const result = await roundTrip(input, compressZstd(), decompressZstd()); + assert.strictEqual(result, input); +} + +async function testZstdLargeData() { + const input = 'zstd large data test. '.repeat(5000); + const result = await roundTrip(input, compressZstd(), decompressZstd()); + assert.strictEqual(result, input); +} + +async function testZstdActuallyCompresses() { + const input = 'Repeated data compresses well. '.repeat(1000); + const inputBuf = Buffer.from(input); + const source = from(inputBuf); + const compressed = await bytes(pull(source, compressZstd())); + assert.ok(compressed.byteLength < inputBuf.byteLength, + `Compressed ${compressed.byteLength} should be < original ${inputBuf.byteLength}`); +} + +// ============================================================================= +// Binary data round-trip - verify no corruption on non-text data +// ============================================================================= + +async function testBinaryRoundTripGzip() { + const input = Buffer.alloc(1024); + for (let i = 0; i < input.length; i++) input[i] = i & 0xFF; + const result = await roundTripBytes(input, compressGzip(), decompressGzip()); + assert.strictEqual(result.byteLength, input.byteLength); + assert.deepStrictEqual(Buffer.from(result), input); +} + +async function testBinaryRoundTripDeflate() { + const input = Buffer.alloc(1024); + for (let i = 0; i < input.length; i++) input[i] = i & 0xFF; + const result = await roundTripBytes(input, compressDeflate(), + decompressDeflate()); + assert.strictEqual(result.byteLength, input.byteLength); + assert.deepStrictEqual(Buffer.from(result), input); +} + +async function testBinaryRoundTripBrotli() { + const input = Buffer.alloc(1024); + for (let i = 0; i < input.length; i++) input[i] = i & 0xFF; + const result = await roundTripBytes(input, compressBrotli(), + decompressBrotli()); + assert.strictEqual(result.byteLength, input.byteLength); + assert.deepStrictEqual(Buffer.from(result), input); +} + +async function testBinaryRoundTripZstd() { + const input = Buffer.alloc(1024); + for (let i = 0; i < input.length; i++) input[i] = i & 0xFF; + const result = await roundTripBytes(input, compressZstd(), + decompressZstd()); + assert.strictEqual(result.byteLength, input.byteLength); + assert.deepStrictEqual(Buffer.from(result), input); +} + +// ============================================================================= +// Empty input +// ============================================================================= + +async function testEmptyInputGzip() { + const result = await roundTrip('', compressGzip(), decompressGzip()); + assert.strictEqual(result, ''); +} + +async function testEmptyInputDeflate() { + const result = await roundTrip('', compressDeflate(), decompressDeflate()); + assert.strictEqual(result, ''); +} + +async function testEmptyInputBrotli() { + const result = await roundTrip('', compressBrotli(), decompressBrotli()); + assert.strictEqual(result, ''); +} + +async function testEmptyInputZstd() { + const result = await roundTrip('', compressZstd(), decompressZstd()); + assert.strictEqual(result, ''); +} + +// ============================================================================= +// Chained transforms - compress with one, then another, decompress in reverse +// ============================================================================= + +async function testChainedGzipDeflate() { + const input = 'Double compression test data. '.repeat(100); + const source = from(input); + // Compress: gzip then deflate + const compressed = pull(pull(source, compressGzip()), compressDeflate()); + // Decompress: deflate then gzip (reverse order) + const decompressed = pull(pull(compressed, decompressDeflate()), + decompressGzip()); + const result = await text(decompressed); + assert.strictEqual(result, input); +} + +// ============================================================================= +// Transform protocol: verify each factory returns a proper transform object +// ============================================================================= + +async function testTransformProtocol() { + const factories = [ + compressGzip, compressDeflate, compressBrotli, compressZstd, + decompressGzip, decompressDeflate, decompressBrotli, decompressZstd, + ]; + + for (const factory of factories) { + const t = factory(); + assert.strictEqual(typeof t.transform, 'function', + `${factory.name}() should have a transform function`); + } +} + +// ============================================================================= +// Cross-compatibility: verify gzip/deflate output is compatible with zlib +// ============================================================================= + +async function testGzipCompatWithZlib() { + const zlib = require('zlib'); + const { promisify } = require('util'); + const gunzip = promisify(zlib.gunzip); + + const input = 'Cross-compat test with node:zlib. '.repeat(100); + const source = from(input); + const compressed = await bytes(pull(source, compressGzip())); + + // Decompress with standard zlib + const decompressed = await gunzip(compressed); + assert.strictEqual(decompressed.toString(), input); +} + +async function testDeflateCompatWithZlib() { + const zlib = require('zlib'); + const { promisify } = require('util'); + const inflate = promisify(zlib.inflate); + + const input = 'Cross-compat deflate test. '.repeat(100); + const source = from(input); + const compressed = await bytes(pull(source, compressDeflate())); + + // Decompress with standard zlib + const decompressed = await inflate(compressed); + assert.strictEqual(decompressed.toString(), input); +} + +async function testBrotliCompatWithZlib() { + const zlib = require('zlib'); + const { promisify } = require('util'); + const brotliDecompress = promisify(zlib.brotliDecompress); + + const input = 'Cross-compat brotli test. '.repeat(100); + const source = from(input); + const compressed = await bytes(pull(source, compressBrotli())); + + const decompressed = await brotliDecompress(compressed); + assert.strictEqual(decompressed.toString(), input); +} + +async function testZstdCompatWithZlib() { + const zlib = require('zlib'); + const { promisify } = require('util'); + const zstdDecompress = promisify(zlib.zstdDecompress); + + const input = 'Cross-compat zstd test. '.repeat(100); + const source = from(input); + const compressed = await bytes(pull(source, compressZstd())); + + const decompressed = await zstdDecompress(compressed); + assert.strictEqual(decompressed.toString(), input); +} + +// ============================================================================= +// Reverse compat: compress with zlib, decompress with new streams +// ============================================================================= + +async function testZlibGzipToNewStreams() { + const zlib = require('zlib'); + const { promisify } = require('util'); + const gzip = promisify(zlib.gzip); + + const input = 'Reverse compat gzip test. '.repeat(100); + const compressed = await gzip(input); + const result = await text(pull(from(compressed), decompressGzip())); + assert.strictEqual(result, input); +} + +async function testZlibDeflateToNewStreams() { + const zlib = require('zlib'); + const { promisify } = require('util'); + const deflate = promisify(zlib.deflate); + + const input = 'Reverse compat deflate test. '.repeat(100); + const compressed = await deflate(input); + const result = await text(pull(from(compressed), decompressDeflate())); + assert.strictEqual(result, input); +} + +async function testZlibBrotliToNewStreams() { + const zlib = require('zlib'); + const { promisify } = require('util'); + const brotliCompress = promisify(zlib.brotliCompress); + + const input = 'Reverse compat brotli test. '.repeat(100); + const compressed = await brotliCompress(input); + const result = await text(pull(from(compressed), decompressBrotli())); + assert.strictEqual(result, input); +} + +async function testZlibZstdToNewStreams() { + const zlib = require('zlib'); + const { promisify } = require('util'); + const zstdCompress = promisify(zlib.zstdCompress); + + const input = 'Reverse compat zstd test. '.repeat(100); + const compressed = await zstdCompress(input); + const result = await text(pull(from(compressed), decompressZstd())); + assert.strictEqual(result, input); +} + +// ============================================================================= +// Run all tests +// ============================================================================= + +(async () => { + // Gzip + await testGzipRoundTrip(); + await testGzipLargeData(); + await testGzipActuallyCompresses(); + + // Deflate + await testDeflateRoundTrip(); + await testDeflateLargeData(); + await testDeflateActuallyCompresses(); + + // Brotli + await testBrotliRoundTrip(); + await testBrotliLargeData(); + await testBrotliActuallyCompresses(); + + // Zstd + await testZstdRoundTrip(); + await testZstdLargeData(); + await testZstdActuallyCompresses(); + + // Binary data + await testBinaryRoundTripGzip(); + await testBinaryRoundTripDeflate(); + await testBinaryRoundTripBrotli(); + await testBinaryRoundTripZstd(); + + // Empty input + await testEmptyInputGzip(); + await testEmptyInputDeflate(); + await testEmptyInputBrotli(); + await testEmptyInputZstd(); + + // Chained + await testChainedGzipDeflate(); + + // Protocol + await testTransformProtocol(); + + // Cross-compat: new streams compress → zlib decompress + await testGzipCompatWithZlib(); + await testDeflateCompatWithZlib(); + await testBrotliCompatWithZlib(); + await testZstdCompatWithZlib(); + + // Reverse compat: zlib compress → new streams decompress + await testZlibGzipToNewStreams(); + await testZlibDeflateToNewStreams(); + await testZlibBrotliToNewStreams(); + await testZlibZstdToNewStreams(); +})().then(common.mustCall());