fix: Improve hasPixelBuffer correctness

apps/simple-camera/__tests__/visioncamera.frame.harness.ts

@@ -76,7 +76,7 @@ describe('VisionCamera - Frame', () => {
     expect(framesReceived).toBeGreaterThanOrEqual(3)
   })
 
-  it('reports and conditionally reads native frame buffers', async (context) => {
+  it('reports native buffers and conditionally reads pixel buffers', async (context) => {
     const session = await VisionCamera.createCameraSession(false)
     const frameOutput = VisionCamera.createFrameOutput({
       targetResolution: CommonResolutions.HD_16_9,
@@ -131,24 +131,6 @@ describe('VisionCamera - Frame', () => {
     runtime.setOnFrameCallback(frameOutput, (frame) => {
       'worklet'
       try {
-        if (!frame.hasPixelBuffer) {
-          scheduleOnRN(report, {
-            state: 'skip',
-            reason:
-              'native frame buffers: device does not expose readable pixel buffers',
-          })
-          return
-        }
-
-        const pixelBufferBytes = frame.getPixelBuffer().byteLength
-        if (pixelBufferBytes <= 0) {
-          scheduleOnRN(report, {
-            state: 'error',
-            errorMessage: 'Frame pixel buffer was empty.',
-          })
-          return
-        }
-
         if (!frame.hasNativeBuffer) {
           scheduleOnRN(report, {
             state: 'skip',
@@ -171,6 +153,17 @@ describe('VisionCamera - Frame', () => {
           nativeBuffer.release()
         }
 
+        if (frame.hasPixelBuffer) {
+          const pixelBufferBytes = frame.getPixelBuffer().byteLength
+          if (pixelBufferBytes <= 0) {
+            scheduleOnRN(report, {
+              state: 'error',
+              errorMessage: 'Frame pixel buffer was empty.',
+            })
+            return
+          }
+        }
+
         scheduleOnRN(report, {
           state: 'success',
         })

packages/react-native-vision-camera/android/src/main/java/com/margelo/nitro/camera/extensions/ImageProxy+getPixelBuffer.kt

@@ -1,29 +1,35 @@
 package com.margelo.nitro.camera.extensions
 
+import android.hardware.HardwareBuffer
 import android.os.Build
-import android.util.Log
 import androidx.annotation.OptIn
+import androidx.annotation.RequiresApi
 import androidx.camera.core.ExperimentalGetImage
 import androidx.camera.core.ImageProxy
 import com.margelo.nitro.camera.utils.DirectByteBufferPool
 import com.margelo.nitro.core.ArrayBuffer
 import java.nio.ByteBuffer
 
+val HardwareBuffer.isCpuReadable: Boolean
+  @RequiresApi(Build.VERSION_CODES.O)
+  get() {
+    val readableUsageFlags = HardwareBuffer.USAGE_CPU_READ_RARELY or HardwareBuffer.USAGE_CPU_READ_OFTEN
+    return (usage and readableUsageFlags) != 0L
+  }
+
 val ImageProxy.hasPixelBuffer: Boolean
   @OptIn(ExperimentalGetImage::class)
   get() {
     if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.P) {
-      image?.hardwareBuffer?.use {
-        // We have a buffer - return true
-        return true
+      image?.hardwareBuffer?.use { hardwareBuffer ->
+        if (hardwareBuffer.isCpuReadable) {
+          // We have CPU-readable GPU-backed Pixel Data.
+          return true
+        }
       }
     }
-    if (planes.size >= 1) {
-      // We have CPU accessible planes
-      return true
-    }
-    // We have nothing.
-    return false
+    // We have CPU-accessible planes.
+    return planes.isNotEmpty()
   }
 
 data class DisposableArrayBuffer(
@@ -52,17 +58,16 @@ private fun ByteBuffer.wrapOrCopyIntoArrayBuffer(): DisposableArrayBuffer {
 fun ImageProxy.getPixelBuffer(): DisposableArrayBuffer {
   if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.P) {
     image?.hardwareBuffer?.use { hardwareBuffer ->
-      try {
-        // Fast Path: We have a GPU-accessible Buffer
+      if (hardwareBuffer.isCpuReadable) {
+        // Fast Path: We have a CPU-readable HardwareBuffer.
         val arrayBuffer = ArrayBuffer.wrap(hardwareBuffer)
         return DisposableArrayBuffer(arrayBuffer) {
           // no release
         }
-      } catch (e: Throwable) {
-        Log.e("ImageProxy", "Failed to wrap zero-copy HardwareBuffer! Falling back to ByteBuffer copy...", e)
       }
     }
   }
+
   when {
     planes.size == 1 -> {
       // Medium Path: We can wrap a single direct plane as a ByteBuffer, or copy it into one if needed.
@@ -81,6 +86,6 @@ fun ImageProxy.getPixelBuffer(): DisposableArrayBuffer {
         DirectByteBufferPool.Shared.release(byteBuffer)
       }
     }
-    else -> throw Error("ImageProxy does not contain any GPU- or CPU-Pixel Data!")
+    else -> throw Error("ImageProxy does not contain any readable Pixel Data!")
   }
 }

packages/react-native-vision-camera/src/specs/instances/Frame.nitro.ts

@@ -251,8 +251,8 @@ export interface Frame
   readonly isPlanar: boolean
 
   /**
-   * Get whether this {@linkcode Frame} has a readable Pixel Buffer
-   * attached to it.
+   * Get whether this {@linkcode Frame} has a CPU-accessible
+   * Pixel Buffer attached to it.
    *
    * @discussion
    * Usually a {@linkcode Frame} has an application-accessible Pixel Buffer
@@ -268,10 +268,12 @@ export interface Frame
    * attached to it.
    *
    * @discussion
-   * Usually a {@linkcode Frame} has a Native Buffer if
-   * its {@linkcode pixelFormat} is application-accessible - aka
-   * every {@linkcode PixelFormat} except for {@linkcode PixelFormat | 'private'}.
-   * On iOS, every Frame has a Native Buffer.
+   * A Native Buffer can be sampled on the GPU as an external texture,
+   * or used in Media Encoder APIs.
+   *
+   * @discussion
+   * Usually a {@linkcode Frame} has a Native Buffer if the platform
+   * supports it. Only legacy platforms don't support Native Buffers.
    *
    * @see {@linkcode getNativeBuffer | getNativeBuffer()}
    * @see {@linkcode NativeBuffer}
@@ -321,12 +323,18 @@ export interface Frame
    * this {@linkcode Frame}.
    *
    * This is a shared contract between libraries to pass
-   * native buffers around without natively typed bindings.
+   * Native Buffers around without natively typed bindings.
+   *
+   * The Native Buffer can be sampled by the GPU as an
+   * external texture, or passed to Media Encoder APIs.
    *
    * The {@linkcode NativeBuffer} must be released
    * again by its consumer via {@linkcode NativeBuffer.release | release()},
    * otherwise the Camera pipeline might stall.
    *
+   * @discussion
+   * Libraries like Skia or WebGPU implement Native Buffer APIs.
+   *
    * @throws If {@linkcode hasNativeBuffer | hasNativeBuffer} is false.
    *
    * @example
@@ -337,6 +345,32 @@ export interface Frame
    *   nativeBuffer.release()
    * }
    * ```
+   * @example
+   * Import a `Frame` into Skia via `NativeBuffer`
+   * ```ts
+   * if (frame.hasNativeBuffer) {
+   *   const nativeBuffer = frame.getNativeBuffer()
+   *   const image = Skia.Image.MakeImageFromNativeBuffer(nativeBuffer.pointer)
+   *   // Render `image` via Skia APIs
+   *   image.dispose()
+   *   nativeBuffer.release()
+   * }
+   * ```
+   * @example
+   * Import a `Frame` into WebGPU via `NativeBuffer`
+   * ```ts
+   * const device = ... // WebGPU device
+   * if (frame.hasNativeBuffer) {
+   *   const nativeBuffer = frame.getNativeBuffer()
+   *   const image = device.importExternalTexture({
+   *     source: nativeBuffer,
+   *     label: 'camera-frame'
+   *   })
+   *   // Render `image` via Skia APIs
+   *   image.dispose()
+   *   nativeBuffer.release()
+   * }
+   * ```
    */
   getNativeBuffer(): NativeBuffer