From 1eab526025af4cac7e2b31e1d4406fea6ee4ac0c Mon Sep 17 00:00:00 2001 From: Marc Rousavy Date: Wed, 27 May 2026 17:53:28 +0200 Subject: [PATCH 1/5] fix: Improve `hasPixelBuffer` correctness --- .../__tests__/visioncamera.frame.harness.ts | 31 ++++----- .../extensions/ImageProxy+getPixelBuffer.kt | 63 ++++++++++--------- .../src/specs/instances/Frame.nitro.ts | 27 +++++--- 3 files changed, 64 insertions(+), 57 deletions(-) diff --git a/apps/simple-camera/__tests__/visioncamera.frame.harness.ts b/apps/simple-camera/__tests__/visioncamera.frame.harness.ts index 32752770d3..e678514d5f 100644 --- a/apps/simple-camera/__tests__/visioncamera.frame.harness.ts +++ b/apps/simple-camera/__tests__/visioncamera.frame.harness.ts @@ -76,7 +76,7 @@ describe('VisionCamera - Frame', () => { expect(framesReceived).toBeGreaterThanOrEqual(3) }) - it('reports and conditionally reads native frame buffers', async (context) => { + it('reports native buffers and conditionally reads pixel buffers', async (context) => { const session = await VisionCamera.createCameraSession(false) const frameOutput = VisionCamera.createFrameOutput({ targetResolution: CommonResolutions.HD_16_9, @@ -131,24 +131,6 @@ describe('VisionCamera - Frame', () => { runtime.setOnFrameCallback(frameOutput, (frame) => { 'worklet' try { - if (!frame.hasPixelBuffer) { - scheduleOnRN(report, { - state: 'skip', - reason: - 'native frame buffers: device does not expose readable pixel buffers', - }) - return - } - - const pixelBufferBytes = frame.getPixelBuffer().byteLength - if (pixelBufferBytes <= 0) { - scheduleOnRN(report, { - state: 'error', - errorMessage: 'Frame pixel buffer was empty.', - }) - return - } - if (!frame.hasNativeBuffer) { scheduleOnRN(report, { state: 'skip', @@ -171,6 +153,17 @@ describe('VisionCamera - Frame', () => { nativeBuffer.release() } + if (frame.hasPixelBuffer) { + const pixelBufferBytes = frame.getPixelBuffer().byteLength + if (pixelBufferBytes <= 0) { + scheduleOnRN(report, { + state: 'error', + errorMessage: 'Frame pixel buffer was empty.', + }) + return + } + } + scheduleOnRN(report, { state: 'success', }) diff --git a/packages/react-native-vision-camera/android/src/main/java/com/margelo/nitro/camera/extensions/ImageProxy+getPixelBuffer.kt b/packages/react-native-vision-camera/android/src/main/java/com/margelo/nitro/camera/extensions/ImageProxy+getPixelBuffer.kt index 83f6acbe91..a8955a3504 100644 --- a/packages/react-native-vision-camera/android/src/main/java/com/margelo/nitro/camera/extensions/ImageProxy+getPixelBuffer.kt +++ b/packages/react-native-vision-camera/android/src/main/java/com/margelo/nitro/camera/extensions/ImageProxy+getPixelBuffer.kt @@ -1,29 +1,39 @@ package com.margelo.nitro.camera.extensions +import android.hardware.HardwareBuffer import android.os.Build -import android.util.Log import androidx.annotation.OptIn +import androidx.annotation.RequiresApi import androidx.camera.core.ExperimentalGetImage import androidx.camera.core.ImageProxy import com.margelo.nitro.camera.utils.DirectByteBufferPool import com.margelo.nitro.core.ArrayBuffer import java.nio.ByteBuffer +private val HardwareBuffer.isCpuReadable: Boolean + @RequiresApi(Build.VERSION_CODES.O) + get() { + val readableUsageFlags = HardwareBuffer.USAGE_CPU_READ_RARELY or HardwareBuffer.USAGE_CPU_READ_OFTEN + return (usage and readableUsageFlags) != 0L + } + +@OptIn(ExperimentalGetImage::class) +private inline fun ImageProxy.withReadableHardwareBuffer(block: (HardwareBuffer) -> T): T? { + if (Build.VERSION.SDK_INT < Build.VERSION_CODES.P) return null + val hardwareBuffer = image?.hardwareBuffer ?: return null + return hardwareBuffer.use { buffer -> + if (buffer.isCpuReadable) { + block(buffer) + } else { + null + } + } +} + val ImageProxy.hasPixelBuffer: Boolean @OptIn(ExperimentalGetImage::class) get() { - if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.P) { - image?.hardwareBuffer?.use { - // We have a buffer - return true - return true - } - } - if (planes.size >= 1) { - // We have CPU accessible planes - return true - } - // We have nothing. - return false + return withReadableHardwareBuffer { true } ?: planes.isNotEmpty() } data class DisposableArrayBuffer( @@ -50,23 +60,20 @@ private fun ByteBuffer.wrapOrCopyIntoArrayBuffer(): DisposableArrayBuffer { @OptIn(ExperimentalGetImage::class) fun ImageProxy.getPixelBuffer(): DisposableArrayBuffer { - if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.P) { - image?.hardwareBuffer?.use { hardwareBuffer -> - try { - // Fast Path: We have a GPU-accessible Buffer - val arrayBuffer = ArrayBuffer.wrap(hardwareBuffer) - return DisposableArrayBuffer(arrayBuffer) { - // no release - } - } catch (e: Throwable) { - Log.e("ImageProxy", "Failed to wrap zero-copy HardwareBuffer! Falling back to ByteBuffer copy...", e) - } + withReadableHardwareBuffer { hardwareBuffer -> + // Fast Path: We have a CPU-readable HardwareBuffer. + val arrayBuffer = ArrayBuffer.wrap(hardwareBuffer) + DisposableArrayBuffer(arrayBuffer) { + // no release } + }?.let { arrayBuffer -> + return arrayBuffer } - when { + + return when { planes.size == 1 -> { // Medium Path: We can wrap a single direct plane as a ByteBuffer, or copy it into one if needed. - return planes.single().buffer.wrapOrCopyIntoArrayBuffer() + planes.single().buffer.wrapOrCopyIntoArrayBuffer() } planes.size > 1 -> { // Slow Path: We have to copy all planes into a new ByteBuffer. @@ -77,10 +84,10 @@ fun ImageProxy.getPixelBuffer(): DisposableArrayBuffer { byteBuffer.put(buffer) } val arrayBuffer = ArrayBuffer.wrap(byteBuffer) - return DisposableArrayBuffer(arrayBuffer) { + DisposableArrayBuffer(arrayBuffer) { DirectByteBufferPool.Shared.release(byteBuffer) } } - else -> throw Error("ImageProxy does not contain any GPU- or CPU-Pixel Data!") + else -> throw Error("ImageProxy does not contain any readable Pixel Data!") } } diff --git a/packages/react-native-vision-camera/src/specs/instances/Frame.nitro.ts b/packages/react-native-vision-camera/src/specs/instances/Frame.nitro.ts index 4bfdf7b253..1c99d4c0db 100644 --- a/packages/react-native-vision-camera/src/specs/instances/Frame.nitro.ts +++ b/packages/react-native-vision-camera/src/specs/instances/Frame.nitro.ts @@ -251,27 +251,34 @@ export interface Frame readonly isPlanar: boolean /** - * Get whether this {@linkcode Frame} has a readable Pixel Buffer + * Get whether this {@linkcode Frame} has readable Pixel Data * attached to it. * * @discussion - * Usually a {@linkcode Frame} has an application-accessible Pixel Buffer - * if its {@linkcode pixelFormat} is application-accessible - aka - * every {@linkcode PixelFormat} except for {@linkcode PixelFormat | 'private'}. - * On iOS, every Frame has an application-accessible Pixel Buffer. + * If this is true, the {@linkcode Frame}'s pixels can be read from JS via + * {@linkcode getPixelBuffer | getPixelBuffer()}. + * + * On Android, {@linkcode PixelFormat | 'private'} Frames are usually native + * buffers only and therefore return `false` here. Use + * {@linkcode hasNativeBuffer | hasNativeBuffer} and + * {@linkcode getNativeBuffer | getNativeBuffer()} for GPU/native interop. + * + * On iOS, every valid Frame has readable Pixel Data. * * @see {@linkcode getPixelBuffer | getPixelBuffer()} */ readonly hasPixelBuffer: boolean /** - * Get whether this {@linkcode Frame} has a Native Buffer + * Get whether this {@linkcode Frame} has a Native Buffer handle * attached to it. * * @discussion - * Usually a {@linkcode Frame} has a Native Buffer if - * its {@linkcode pixelFormat} is application-accessible - aka - * every {@linkcode PixelFormat} except for {@linkcode PixelFormat | 'private'}. - * On iOS, every Frame has a Native Buffer. + * A Native Buffer is the platform-native image handle used for zero-copy + * GPU/native interop. It is independent from readable Pixel Data: + * a {@linkcode PixelFormat | 'private'} Frame may have a Native Buffer even + * when {@linkcode hasPixelBuffer | hasPixelBuffer} is false. + * + * On iOS, every valid Frame has a Native Buffer. * * @see {@linkcode getNativeBuffer | getNativeBuffer()} * @see {@linkcode NativeBuffer} From 5bf609a4c2479094262c7171c2a45ba4b0be5c11 Mon Sep 17 00:00:00 2001 From: Marc Rousavy Date: Wed, 27 May 2026 18:45:21 +0200 Subject: [PATCH 2/5] cleanup --- .../extensions/ImageProxy+getPixelBuffer.kt | 42 +++++++++---------- 1 file changed, 20 insertions(+), 22 deletions(-) diff --git a/packages/react-native-vision-camera/android/src/main/java/com/margelo/nitro/camera/extensions/ImageProxy+getPixelBuffer.kt b/packages/react-native-vision-camera/android/src/main/java/com/margelo/nitro/camera/extensions/ImageProxy+getPixelBuffer.kt index a8955a3504..8352338e78 100644 --- a/packages/react-native-vision-camera/android/src/main/java/com/margelo/nitro/camera/extensions/ImageProxy+getPixelBuffer.kt +++ b/packages/react-native-vision-camera/android/src/main/java/com/margelo/nitro/camera/extensions/ImageProxy+getPixelBuffer.kt @@ -10,30 +10,26 @@ import com.margelo.nitro.camera.utils.DirectByteBufferPool import com.margelo.nitro.core.ArrayBuffer import java.nio.ByteBuffer -private val HardwareBuffer.isCpuReadable: Boolean +val HardwareBuffer.isCpuReadable: Boolean @RequiresApi(Build.VERSION_CODES.O) get() { val readableUsageFlags = HardwareBuffer.USAGE_CPU_READ_RARELY or HardwareBuffer.USAGE_CPU_READ_OFTEN return (usage and readableUsageFlags) != 0L } -@OptIn(ExperimentalGetImage::class) -private inline fun ImageProxy.withReadableHardwareBuffer(block: (HardwareBuffer) -> T): T? { - if (Build.VERSION.SDK_INT < Build.VERSION_CODES.P) return null - val hardwareBuffer = image?.hardwareBuffer ?: return null - return hardwareBuffer.use { buffer -> - if (buffer.isCpuReadable) { - block(buffer) - } else { - null - } - } -} - val ImageProxy.hasPixelBuffer: Boolean @OptIn(ExperimentalGetImage::class) get() { - return withReadableHardwareBuffer { true } ?: planes.isNotEmpty() + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.P) { + image?.hardwareBuffer?.use { hardwareBuffer -> + if (hardwareBuffer.isCpuReadable) { + // We have CPU-readable GPU-backed Pixel Data. + return true + } + } + } + // We have CPU-accessible planes. + return planes.isNotEmpty() } data class DisposableArrayBuffer( @@ -60,14 +56,16 @@ private fun ByteBuffer.wrapOrCopyIntoArrayBuffer(): DisposableArrayBuffer { @OptIn(ExperimentalGetImage::class) fun ImageProxy.getPixelBuffer(): DisposableArrayBuffer { - withReadableHardwareBuffer { hardwareBuffer -> - // Fast Path: We have a CPU-readable HardwareBuffer. - val arrayBuffer = ArrayBuffer.wrap(hardwareBuffer) - DisposableArrayBuffer(arrayBuffer) { - // no release + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.P) { + image?.hardwareBuffer?.use { hardwareBuffer -> + if (hardwareBuffer.isCpuReadable) { + // Fast Path: We have a CPU-readable HardwareBuffer. + val arrayBuffer = ArrayBuffer.wrap(hardwareBuffer) + return DisposableArrayBuffer(arrayBuffer) { + // no release + } + } } - }?.let { arrayBuffer -> - return arrayBuffer } return when { From 3b88d7fc064d8e17334c6eb1358ade319179c2c1 Mon Sep 17 00:00:00 2001 From: Marc Rousavy Date: Thu, 28 May 2026 18:32:43 +0200 Subject: [PATCH 3/5] better kotlin --- .../nitro/camera/extensions/ImageProxy+getPixelBuffer.kt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/react-native-vision-camera/android/src/main/java/com/margelo/nitro/camera/extensions/ImageProxy+getPixelBuffer.kt b/packages/react-native-vision-camera/android/src/main/java/com/margelo/nitro/camera/extensions/ImageProxy+getPixelBuffer.kt index 8352338e78..a7a8713008 100644 --- a/packages/react-native-vision-camera/android/src/main/java/com/margelo/nitro/camera/extensions/ImageProxy+getPixelBuffer.kt +++ b/packages/react-native-vision-camera/android/src/main/java/com/margelo/nitro/camera/extensions/ImageProxy+getPixelBuffer.kt @@ -68,10 +68,10 @@ fun ImageProxy.getPixelBuffer(): DisposableArrayBuffer { } } - return when { + when { planes.size == 1 -> { // Medium Path: We can wrap a single direct plane as a ByteBuffer, or copy it into one if needed. - planes.single().buffer.wrapOrCopyIntoArrayBuffer() + return planes.single().buffer.wrapOrCopyIntoArrayBuffer() } planes.size > 1 -> { // Slow Path: We have to copy all planes into a new ByteBuffer. @@ -82,7 +82,7 @@ fun ImageProxy.getPixelBuffer(): DisposableArrayBuffer { byteBuffer.put(buffer) } val arrayBuffer = ArrayBuffer.wrap(byteBuffer) - DisposableArrayBuffer(arrayBuffer) { + return DisposableArrayBuffer(arrayBuffer) { DirectByteBufferPool.Shared.release(byteBuffer) } } From 548774cf92f69c49e78cf2d1cc7ed542b653c9e8 Mon Sep 17 00:00:00 2001 From: Marc Rousavy Date: Thu, 28 May 2026 18:35:04 +0200 Subject: [PATCH 4/5] adjust jsdoc wording its better now --- .../src/specs/instances/Frame.nitro.ts | 27 +++++++------------ 1 file changed, 10 insertions(+), 17 deletions(-) diff --git a/packages/react-native-vision-camera/src/specs/instances/Frame.nitro.ts b/packages/react-native-vision-camera/src/specs/instances/Frame.nitro.ts index 1c99d4c0db..4bfdf7b253 100644 --- a/packages/react-native-vision-camera/src/specs/instances/Frame.nitro.ts +++ b/packages/react-native-vision-camera/src/specs/instances/Frame.nitro.ts @@ -251,34 +251,27 @@ export interface Frame readonly isPlanar: boolean /** - * Get whether this {@linkcode Frame} has readable Pixel Data + * Get whether this {@linkcode Frame} has a readable Pixel Buffer * attached to it. * * @discussion - * If this is true, the {@linkcode Frame}'s pixels can be read from JS via - * {@linkcode getPixelBuffer | getPixelBuffer()}. - * - * On Android, {@linkcode PixelFormat | 'private'} Frames are usually native - * buffers only and therefore return `false` here. Use - * {@linkcode hasNativeBuffer | hasNativeBuffer} and - * {@linkcode getNativeBuffer | getNativeBuffer()} for GPU/native interop. - * - * On iOS, every valid Frame has readable Pixel Data. + * Usually a {@linkcode Frame} has an application-accessible Pixel Buffer + * if its {@linkcode pixelFormat} is application-accessible - aka + * every {@linkcode PixelFormat} except for {@linkcode PixelFormat | 'private'}. + * On iOS, every Frame has an application-accessible Pixel Buffer. * * @see {@linkcode getPixelBuffer | getPixelBuffer()} */ readonly hasPixelBuffer: boolean /** - * Get whether this {@linkcode Frame} has a Native Buffer handle + * Get whether this {@linkcode Frame} has a Native Buffer * attached to it. * * @discussion - * A Native Buffer is the platform-native image handle used for zero-copy - * GPU/native interop. It is independent from readable Pixel Data: - * a {@linkcode PixelFormat | 'private'} Frame may have a Native Buffer even - * when {@linkcode hasPixelBuffer | hasPixelBuffer} is false. - * - * On iOS, every valid Frame has a Native Buffer. + * Usually a {@linkcode Frame} has a Native Buffer if + * its {@linkcode pixelFormat} is application-accessible - aka + * every {@linkcode PixelFormat} except for {@linkcode PixelFormat | 'private'}. + * On iOS, every Frame has a Native Buffer. * * @see {@linkcode getNativeBuffer | getNativeBuffer()} * @see {@linkcode NativeBuffer} From 0b3dcd49182ee5fd598f31a3ccdfa9b4df9fe99f Mon Sep 17 00:00:00 2001 From: Marc Rousavy Date: Thu, 28 May 2026 18:48:06 +0200 Subject: [PATCH 5/5] more docs around native buffer ig --- .../src/specs/instances/Frame.nitro.ts | 48 ++++++++++++++++--- 1 file changed, 41 insertions(+), 7 deletions(-) diff --git a/packages/react-native-vision-camera/src/specs/instances/Frame.nitro.ts b/packages/react-native-vision-camera/src/specs/instances/Frame.nitro.ts index 4bfdf7b253..91134e5432 100644 --- a/packages/react-native-vision-camera/src/specs/instances/Frame.nitro.ts +++ b/packages/react-native-vision-camera/src/specs/instances/Frame.nitro.ts @@ -251,8 +251,8 @@ export interface Frame readonly isPlanar: boolean /** - * Get whether this {@linkcode Frame} has a readable Pixel Buffer - * attached to it. + * Get whether this {@linkcode Frame} has a CPU-accessible + * Pixel Buffer attached to it. * * @discussion * Usually a {@linkcode Frame} has an application-accessible Pixel Buffer @@ -268,10 +268,12 @@ export interface Frame * attached to it. * * @discussion - * Usually a {@linkcode Frame} has a Native Buffer if - * its {@linkcode pixelFormat} is application-accessible - aka - * every {@linkcode PixelFormat} except for {@linkcode PixelFormat | 'private'}. - * On iOS, every Frame has a Native Buffer. + * A Native Buffer can be sampled on the GPU as an external texture, + * or used in Media Encoder APIs. + * + * @discussion + * Usually a {@linkcode Frame} has a Native Buffer if the platform + * supports it. Only legacy platforms don't support Native Buffers. * * @see {@linkcode getNativeBuffer | getNativeBuffer()} * @see {@linkcode NativeBuffer} @@ -321,12 +323,18 @@ export interface Frame * this {@linkcode Frame}. * * This is a shared contract between libraries to pass - * native buffers around without natively typed bindings. + * Native Buffers around without natively typed bindings. + * + * The Native Buffer can be sampled by the GPU as an + * external texture, or passed to Media Encoder APIs. * * The {@linkcode NativeBuffer} must be released * again by its consumer via {@linkcode NativeBuffer.release | release()}, * otherwise the Camera pipeline might stall. * + * @discussion + * Libraries like Skia or WebGPU implement Native Buffer APIs. + * * @throws If {@linkcode hasNativeBuffer | hasNativeBuffer} is false. * * @example @@ -337,6 +345,32 @@ export interface Frame * nativeBuffer.release() * } * ``` + * @example + * Import a `Frame` into Skia via `NativeBuffer` + * ```ts + * if (frame.hasNativeBuffer) { + * const nativeBuffer = frame.getNativeBuffer() + * const image = Skia.Image.MakeImageFromNativeBuffer(nativeBuffer.pointer) + * // Render `image` via Skia APIs + * image.dispose() + * nativeBuffer.release() + * } + * ``` + * @example + * Import a `Frame` into WebGPU via `NativeBuffer` + * ```ts + * const device = ... // WebGPU device + * if (frame.hasNativeBuffer) { + * const nativeBuffer = frame.getNativeBuffer() + * const image = device.importExternalTexture({ + * source: nativeBuffer, + * label: 'camera-frame' + * }) + * // Render `image` via Skia APIs + * image.dispose() + * nativeBuffer.release() + * } + * ``` */ getNativeBuffer(): NativeBuffer