diff --git a/apps/simple-camera/__tests__/visioncamera.frame.harness.ts b/apps/simple-camera/__tests__/visioncamera.frame.harness.ts index 32752770d3..e678514d5f 100644 --- a/apps/simple-camera/__tests__/visioncamera.frame.harness.ts +++ b/apps/simple-camera/__tests__/visioncamera.frame.harness.ts @@ -76,7 +76,7 @@ describe('VisionCamera - Frame', () => { expect(framesReceived).toBeGreaterThanOrEqual(3) }) - it('reports and conditionally reads native frame buffers', async (context) => { + it('reports native buffers and conditionally reads pixel buffers', async (context) => { const session = await VisionCamera.createCameraSession(false) const frameOutput = VisionCamera.createFrameOutput({ targetResolution: CommonResolutions.HD_16_9, @@ -131,24 +131,6 @@ describe('VisionCamera - Frame', () => { runtime.setOnFrameCallback(frameOutput, (frame) => { 'worklet' try { - if (!frame.hasPixelBuffer) { - scheduleOnRN(report, { - state: 'skip', - reason: - 'native frame buffers: device does not expose readable pixel buffers', - }) - return - } - - const pixelBufferBytes = frame.getPixelBuffer().byteLength - if (pixelBufferBytes <= 0) { - scheduleOnRN(report, { - state: 'error', - errorMessage: 'Frame pixel buffer was empty.', - }) - return - } - if (!frame.hasNativeBuffer) { scheduleOnRN(report, { state: 'skip', @@ -171,6 +153,17 @@ describe('VisionCamera - Frame', () => { nativeBuffer.release() } + if (frame.hasPixelBuffer) { + const pixelBufferBytes = frame.getPixelBuffer().byteLength + if (pixelBufferBytes <= 0) { + scheduleOnRN(report, { + state: 'error', + errorMessage: 'Frame pixel buffer was empty.', + }) + return + } + } + scheduleOnRN(report, { state: 'success', }) diff --git a/packages/react-native-vision-camera/android/src/main/java/com/margelo/nitro/camera/extensions/ImageProxy+getPixelBuffer.kt b/packages/react-native-vision-camera/android/src/main/java/com/margelo/nitro/camera/extensions/ImageProxy+getPixelBuffer.kt index 83f6acbe91..a7a8713008 100644 --- a/packages/react-native-vision-camera/android/src/main/java/com/margelo/nitro/camera/extensions/ImageProxy+getPixelBuffer.kt +++ b/packages/react-native-vision-camera/android/src/main/java/com/margelo/nitro/camera/extensions/ImageProxy+getPixelBuffer.kt @@ -1,29 +1,35 @@ package com.margelo.nitro.camera.extensions +import android.hardware.HardwareBuffer import android.os.Build -import android.util.Log import androidx.annotation.OptIn +import androidx.annotation.RequiresApi import androidx.camera.core.ExperimentalGetImage import androidx.camera.core.ImageProxy import com.margelo.nitro.camera.utils.DirectByteBufferPool import com.margelo.nitro.core.ArrayBuffer import java.nio.ByteBuffer +val HardwareBuffer.isCpuReadable: Boolean + @RequiresApi(Build.VERSION_CODES.O) + get() { + val readableUsageFlags = HardwareBuffer.USAGE_CPU_READ_RARELY or HardwareBuffer.USAGE_CPU_READ_OFTEN + return (usage and readableUsageFlags) != 0L + } + val ImageProxy.hasPixelBuffer: Boolean @OptIn(ExperimentalGetImage::class) get() { if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.P) { - image?.hardwareBuffer?.use { - // We have a buffer - return true - return true + image?.hardwareBuffer?.use { hardwareBuffer -> + if (hardwareBuffer.isCpuReadable) { + // We have CPU-readable GPU-backed Pixel Data. + return true + } } } - if (planes.size >= 1) { - // We have CPU accessible planes - return true - } - // We have nothing. - return false + // We have CPU-accessible planes. + return planes.isNotEmpty() } data class DisposableArrayBuffer( @@ -52,17 +58,16 @@ private fun ByteBuffer.wrapOrCopyIntoArrayBuffer(): DisposableArrayBuffer { fun ImageProxy.getPixelBuffer(): DisposableArrayBuffer { if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.P) { image?.hardwareBuffer?.use { hardwareBuffer -> - try { - // Fast Path: We have a GPU-accessible Buffer + if (hardwareBuffer.isCpuReadable) { + // Fast Path: We have a CPU-readable HardwareBuffer. val arrayBuffer = ArrayBuffer.wrap(hardwareBuffer) return DisposableArrayBuffer(arrayBuffer) { // no release } - } catch (e: Throwable) { - Log.e("ImageProxy", "Failed to wrap zero-copy HardwareBuffer! Falling back to ByteBuffer copy...", e) } } } + when { planes.size == 1 -> { // Medium Path: We can wrap a single direct plane as a ByteBuffer, or copy it into one if needed. @@ -81,6 +86,6 @@ fun ImageProxy.getPixelBuffer(): DisposableArrayBuffer { DirectByteBufferPool.Shared.release(byteBuffer) } } - else -> throw Error("ImageProxy does not contain any GPU- or CPU-Pixel Data!") + else -> throw Error("ImageProxy does not contain any readable Pixel Data!") } } diff --git a/packages/react-native-vision-camera/src/specs/instances/Frame.nitro.ts b/packages/react-native-vision-camera/src/specs/instances/Frame.nitro.ts index 4bfdf7b253..91134e5432 100644 --- a/packages/react-native-vision-camera/src/specs/instances/Frame.nitro.ts +++ b/packages/react-native-vision-camera/src/specs/instances/Frame.nitro.ts @@ -251,8 +251,8 @@ export interface Frame readonly isPlanar: boolean /** - * Get whether this {@linkcode Frame} has a readable Pixel Buffer - * attached to it. + * Get whether this {@linkcode Frame} has a CPU-accessible + * Pixel Buffer attached to it. * * @discussion * Usually a {@linkcode Frame} has an application-accessible Pixel Buffer @@ -268,10 +268,12 @@ export interface Frame * attached to it. * * @discussion - * Usually a {@linkcode Frame} has a Native Buffer if - * its {@linkcode pixelFormat} is application-accessible - aka - * every {@linkcode PixelFormat} except for {@linkcode PixelFormat | 'private'}. - * On iOS, every Frame has a Native Buffer. + * A Native Buffer can be sampled on the GPU as an external texture, + * or used in Media Encoder APIs. + * + * @discussion + * Usually a {@linkcode Frame} has a Native Buffer if the platform + * supports it. Only legacy platforms don't support Native Buffers. * * @see {@linkcode getNativeBuffer | getNativeBuffer()} * @see {@linkcode NativeBuffer} @@ -321,12 +323,18 @@ export interface Frame * this {@linkcode Frame}. * * This is a shared contract between libraries to pass - * native buffers around without natively typed bindings. + * Native Buffers around without natively typed bindings. + * + * The Native Buffer can be sampled by the GPU as an + * external texture, or passed to Media Encoder APIs. * * The {@linkcode NativeBuffer} must be released * again by its consumer via {@linkcode NativeBuffer.release | release()}, * otherwise the Camera pipeline might stall. * + * @discussion + * Libraries like Skia or WebGPU implement Native Buffer APIs. + * * @throws If {@linkcode hasNativeBuffer | hasNativeBuffer} is false. * * @example @@ -337,6 +345,32 @@ export interface Frame * nativeBuffer.release() * } * ``` + * @example + * Import a `Frame` into Skia via `NativeBuffer` + * ```ts + * if (frame.hasNativeBuffer) { + * const nativeBuffer = frame.getNativeBuffer() + * const image = Skia.Image.MakeImageFromNativeBuffer(nativeBuffer.pointer) + * // Render `image` via Skia APIs + * image.dispose() + * nativeBuffer.release() + * } + * ``` + * @example + * Import a `Frame` into WebGPU via `NativeBuffer` + * ```ts + * const device = ... // WebGPU device + * if (frame.hasNativeBuffer) { + * const nativeBuffer = frame.getNativeBuffer() + * const image = device.importExternalTexture({ + * source: nativeBuffer, + * label: 'camera-frame' + * }) + * // Render `image` via Skia APIs + * image.dispose() + * nativeBuffer.release() + * } + * ``` */ getNativeBuffer(): NativeBuffer