Release v0.2.2: Real-time streaming, audio-only encoding, VideoFile audio, transferable objects

- Fix processMediaStreamRealtime implementation for real-time streaming support
- Add video: false option for audio-only encoding scenarios
- Add audio extraction from VideoFile using AudioContext API
- Optimize WorkerCommunicator with transferable objects for VideoFrame/AudioData
- Update types and configuration parsers to support new features
- Add comprehensive tests for v0.2.2 functionality
- Update README with v0.2.2 changelog and examples

🤖 Generated with [Claude Code](https://claude.ai/code)

Co-Authored-By: Claude <noreply@anthropic.com>

Author: romot-co

.gitignore

@@ -110,4 +110,5 @@ coverage/
 coverage-vitest/ # If you use a specific output like coverage/vitest
 
 implements.md
-ref
+ref
+.claude/

README.md

@@ -2,8 +2,8 @@
 
 A TypeScript library to encode video (H.264/AVC, VP9, VP8) and audio (AAC, Opus) using the WebCodecs API and mux them into MP4 or WebM containers with a simple, function-first design.
 
-> **🎉 v0.2.1 Release**  
-> This release includes VideoFile support and improved API stability. The function-first API is production-ready with automatic configuration, quality presets, and progressive enhancement.
+> **🎉 v0.2.2 Release**  
+> Major updates: Real-time streaming support, audio-only encoding, VideoFile audio processing, and optimized transferable objects. See [CHANGELOG](#changelog) for details.
 
 ## Features
 
@@ -17,6 +17,9 @@ A TypeScript library to encode video (H.264/AVC, VP9, VP8) and audio (AAC, Opus)
 - **📦 Optimized Bundle Size**: Import only what you need
 - **🛡️ Type Safety**: Full TypeScript support with comprehensive types
 - **🎵 Audio Support**: AAC and Opus encoding with automatic configuration
+- **🎤 Audio-Only Encoding**: Support for `video: false` option (v0.2.2)
+- **📹 VideoFile Audio**: Extract and encode audio from video files (v0.2.2)  
+- **⚡ Performance Optimized**: Transferable objects for faster data transfer (v0.2.2)
 
 ## Installation
 
@@ -370,6 +373,38 @@ if (!supported) {
 4. **Optimize frame rate** for your use case (30fps is usually sufficient)
 5. **Consider container format**: MP4 for compatibility, WebM for smaller files
 
+## Changelog
+
+### v0.2.2 (2025-01-14)
+
+**🚀 Major Features**
+- **Real-time streaming**: Fixed `encodeStream()` MediaStream processing - no longer throws errors
+- **Audio-only encoding**: Added `video: false` option support for pure audio encoding
+- **VideoFile audio extraction**: Automatic audio track processing from video files using AudioContext
+- **Transferable objects optimization**: Improved performance with optimized VideoFrame/AudioData transfer
+
+**🔧 Improvements**
+- Enhanced MediaStream track detection for audio-only streams
+- Better error handling for AudioContext unavailability
+- Optimized worker communication with transferable objects
+- Extended type definitions for `video: false` configurations
+
+**🐛 Bug Fixes**
+- Fixed real-time MediaStream processing in `encodeStream()`
+- Resolved audio processing issues in VideoFile inputs
+- Improved configuration inference for audio-only scenarios
+
+**📝 Documentation**
+- Added comprehensive examples for new features
+- Updated API documentation with v0.2.2 features
+- Added performance optimization guidelines
+
+### v0.2.1 (2025-01-13)
+- Added VideoFile support and removed AudioWorklet feature
+- Updated MediaStreamRecorder to use MediaStreamTrackProcessor
+- Improved build configuration and exports
+- Enhanced test coverage and documentation
+
 ## License
 
 MIT License - see [LICENSE](LICENSE) file for details.

package.json

@@ -1,6 +1,6 @@
 {
   "name": "webcodecs-encoder",
-  "version": "0.2.1",
+  "version": "0.2.2",
   "description": "A TypeScript library for browser environments to encode video (H.264/AVC, VP9, VP8) and audio (AAC, Opus) using the WebCodecs API and mux them into MP4 or WebM containers with real-time streaming support. New function-first API design.",
   "homepage": "https://github.com/romot-co/webcodecs-encoder",
   "repository": {

src/core/encode.ts

@@ -449,6 +449,31 @@ async function processVideoFile(
     const frameRate = config.frameRate || 30;
     const totalFrames = Math.floor(duration * frameRate);
 
+    // 音声処理の準備（設定で音声が有効な場合）
+    let audioContext: AudioContext | null = null;
+    let audioBuffer: AudioBuffer | null = null;
+
+    if (config.audioBitrate > 0 && typeof AudioContext !== "undefined") {
+      try {
+        audioContext = new AudioContext();
+
+        // ファイルから音声データを読み込み
+        const arrayBuffer = await videoFile.file.arrayBuffer();
+        audioBuffer = await audioContext.decodeAudioData(arrayBuffer);
+
+        // 音声データを分割して送信
+        await processAudioFromFile(
+          communicator,
+          audioBuffer,
+          duration,
+          frameRate,
+        );
+      } catch (audioError) {
+        // 音声処理に失敗した場合はログを出して続行
+        console.warn("Failed to process audio from VideoFile:", audioError);
+      }
+    }
+
     // Canvasを作成してフレームを抽出
     const canvas = document.createElement("canvas");
     canvas.width = videoWidth;
@@ -497,6 +522,10 @@ async function processVideoFile(
     // リソースをクリーンアップ
     URL.revokeObjectURL(objectUrl);
     video.remove();
+
+    if (audioContext) {
+      audioContext.close();
+    }
   } catch (error) {
     throw new EncodeError(
       "invalid-input",
@@ -505,3 +534,71 @@ async function processVideoFile(
     );
   }
 }
+
+/**
+ * AudioBufferから音声データを処理してワーカーに送信
+ */
+async function processAudioFromFile(
+  communicator: WorkerCommunicator,
+  audioBuffer: AudioBuffer,
+  duration: number,
+  frameRate: number,
+): Promise<void> {
+  const sampleRate = audioBuffer.sampleRate;
+  const numberOfChannels = audioBuffer.numberOfChannels;
+  const totalSamples = audioBuffer.length;
+
+  // 音声データを適切なチャンクサイズに分割
+  const chunkDurationMs = 1000 / frameRate; // フレームレートに合わせる
+  const samplesPerChunk = Math.floor((sampleRate * chunkDurationMs) / 1000);
+
+  for (let offset = 0; offset < totalSamples; offset += samplesPerChunk) {
+    const remainingSamples = Math.min(samplesPerChunk, totalSamples - offset);
+    const timestamp = (offset / sampleRate) * 1000000; // マイクロ秒
+
+    // チャンネルデータを取得
+    const channelData: Float32Array[] = [];
+    for (let channel = 0; channel < numberOfChannels; channel++) {
+      const sourceData = audioBuffer.getChannelData(channel);
+      const chunkData = new Float32Array(remainingSamples);
+      chunkData.set(sourceData.subarray(offset, offset + remainingSamples));
+      channelData.push(chunkData);
+    }
+
+    try {
+      // AudioDataを作成してワーカーに送信
+      // インターリーブ形式に変換
+      const interleavedData = new Float32Array(
+        remainingSamples * numberOfChannels,
+      );
+      for (let frame = 0; frame < remainingSamples; frame++) {
+        for (let channel = 0; channel < numberOfChannels; channel++) {
+          interleavedData[frame * numberOfChannels + channel] =
+            channelData[channel][frame];
+        }
+      }
+
+      const audioData = new AudioData({
+        format: "f32",
+        sampleRate,
+        numberOfFrames: remainingSamples,
+        numberOfChannels,
+        timestamp,
+        data: interleavedData,
+      });
+
+      communicator.send("addAudioData", {
+        audio: audioData,
+        timestamp,
+        format: "f32",
+        sampleRate,
+        numberOfFrames: remainingSamples,
+        numberOfChannels,
+      });
+
+      audioData.close();
+    } catch (error) {
+      console.warn("Failed to create AudioData chunk:", error);
+    }
+  }
+}

src/stream/encode-stream.ts

@@ -239,16 +239,125 @@ async function processAsyncIterable(
  * MediaStreamをリアルタイム処理
  */
 async function processMediaStreamRealtime(
-  _communicator: WorkerCommunicator,
-  _stream: MediaStream,
+  communicator: WorkerCommunicator,
+  stream: MediaStream,
+  config: any,
+): Promise<void> {
+  const videoTracks = stream.getVideoTracks();
+  const audioTracks = stream.getAudioTracks();
+
+  const readers: ReadableStreamDefaultReader<any>[] = [];
+  const processingPromises: Promise<void>[] = [];
+
+  try {
+    // ビデオトラックの処理
+    if (videoTracks.length > 0) {
+      const videoTrack = videoTracks[0];
+      const processor = new MediaStreamTrackProcessor({ track: videoTrack });
+      const reader =
+        processor.readable.getReader() as ReadableStreamDefaultReader<VideoFrame>;
+      readers.push(reader);
+
+      processingPromises.push(
+        processVideoTrackRealtime(communicator, reader, config),
+      );
+    }
+
+    // オーディオトラックの処理
+    if (audioTracks.length > 0) {
+      const audioTrack = audioTracks[0];
+      const processor = new MediaStreamTrackProcessor({ track: audioTrack });
+      const reader =
+        processor.readable.getReader() as ReadableStreamDefaultReader<AudioData>;
+      readers.push(reader);
+
+      processingPromises.push(processAudioTrackRealtime(communicator, reader));
+    }
+
+    // すべての処理が完了するまで待機
+    await Promise.all(processingPromises);
+  } finally {
+    // リーダーをクリーンアップ
+    for (const reader of readers) {
+      try {
+        reader.cancel();
+      } catch (e) {
+        // エラーは無視（既にキャンセル済みの可能性）
+      }
+    }
+
+    // トラックを停止
+    for (const track of [...videoTracks, ...audioTracks]) {
+      track.stop();
+    }
+  }
+}
+
+/**
+ * VideoTrackをリアルタイム処理
+ */
+async function processVideoTrackRealtime(
+  communicator: WorkerCommunicator,
+  reader: ReadableStreamDefaultReader<VideoFrame>,
   _config: any,
 ): Promise<void> {
-  // MediaStreamRecorderの機能を活用したリアルタイム処理
-  // 実装の詳細は複雑なため、プレースホルダー
-  throw new EncodeError(
-    "invalid-input",
-    "Real-time MediaStream processing requires more complex implementation",
-  );
+  // フレームドロップ機能は将来実装予定
+  // const maxQueueDepth = config.maxQueueDepth || 10;
+
+  try {
+    // eslint-disable-next-line no-constant-condition
+    while (true) {
+      const { value, done } = await reader.read();
+      if (done || !value) break;
+
+      try {
+        await addFrameToWorker(communicator, value, value.timestamp || 0);
+      } finally {
+        value.close();
+      }
+    }
+  } catch (error) {
+    throw new EncodeError(
+      "video-encoding-error",
+      `Real-time video stream processing error: ${error instanceof Error ? error.message : String(error)}`,
+      error,
+    );
+  }
+}
+
+/**
+ * AudioTrackをリアルタイム処理
+ */
+async function processAudioTrackRealtime(
+  communicator: WorkerCommunicator,
+  reader: ReadableStreamDefaultReader<AudioData>,
+): Promise<void> {
+  try {
+    // eslint-disable-next-line no-constant-condition
+    while (true) {
+      const { value, done } = await reader.read();
+      if (done || !value) break;
+
+      try {
+        communicator.send("addAudioData", {
+          audio: value,
+          timestamp: value.timestamp || 0,
+          format: "f32",
+          sampleRate: value.sampleRate,
+          numberOfFrames: value.numberOfFrames,
+          numberOfChannels: value.numberOfChannels,
+        });
+      } finally {
+        value.close();
+      }
+    }
+  } catch (error) {
+    throw new EncodeError(
+      "audio-encoding-error",
+      `Real-time audio stream processing error: ${error instanceof Error ? error.message : String(error)}`,
+      error,
+    );
+  }
 }
 
 /**

src/types.ts

@@ -58,7 +58,7 @@ export interface EncodeOptions {
   quality?: QualityPreset;
 
   // 詳細設定（オプショナル）
-  video?: VideoConfig;
+  video?: VideoConfig | false; // falseでビデオ無効化
   audio?: AudioConfig | false; // falseでオーディオ無効化
   container?: 'mp4' | 'webm';
 

src/utils/can-encode.ts

@@ -135,17 +135,28 @@ async function testVideoCodecSupport(
       codec: codecString,
       width: options?.width || 640,
       height: options?.height || 480,
-      bitrate: options?.video?.bitrate || 1_000_000,
+      bitrate:
+        options?.video === false
+          ? 0
+          : (options?.video as any)?.bitrate || 1_000_000,
       framerate: options?.frameRate || 30,
     };
 
     // オプションの詳細設定を追加
-    if (options?.video?.hardwareAcceleration) {
-      config.hardwareAcceleration = options.video.hardwareAcceleration;
+    if (
+      options &&
+      options.video !== false &&
+      (options.video as any)?.hardwareAcceleration
+    ) {
+      config.hardwareAcceleration = (options.video as any).hardwareAcceleration;
     }
 
-    if (options?.video?.latencyMode) {
-      config.latencyMode = options.video.latencyMode;
+    if (
+      options &&
+      options.video !== false &&
+      (options.video as any)?.latencyMode
+    ) {
+      config.latencyMode = (options.video as any).latencyMode;
     }
 
     const support = await VideoEncoder.isConfigSupported(config);