diff --git a/packages/example/lib/vision_detector_views/camera_view.dart b/packages/example/lib/vision_detector_views/camera_view.dart index 051c1e1c..dd44226b 100644 --- a/packages/example/lib/vision_detector_views/camera_view.dart +++ b/packages/example/lib/vision_detector_views/camera_view.dart @@ -357,29 +357,126 @@ class _CameraViewState extends State { // get image format final format = InputImageFormatValue.fromRawValue(image.format.raw); - // validate format depending on platform - // only supported formats: - // * nv21 for Android - // * bgra8888 for iOS - if (format == null || - (Platform.isAndroid && format != InputImageFormat.nv21) || + if (format == null) { + print('could not find format from raw value: ${image.format.raw}'); + return null; + } + // Validate format depending on platform + const androidSupportedFormats = [ + InputImageFormat.nv21, + InputImageFormat.yv12, + InputImageFormat.yuv_420_888 + ]; + + if ((Platform.isAndroid && !androidSupportedFormats.contains(format)) || (Platform.isIOS && format != InputImageFormat.bgra8888)) { + print('image format is not supported: $format'); return null; } - // since format is constraint to nv21 or bgra8888, both only have one plane - if (image.planes.length != 1) return null; - final plane = image.planes.first; + InputImageFormat resolvedFormat = format; + final Uint8List bytes; + + if (image.planes.length == 1) { + bytes = image.planes.first.bytes; + } else if (Platform.isAndroid && + (format == InputImageFormat.yuv_420_888 || + format == InputImageFormat.yv12) && + image.planes.length == 3) { + bytes = _convertYUV420ToNV21(image); + resolvedFormat = InputImageFormat.nv21; + } else { + bytes = _concatenatePlanes(image); + } - // compose InputImage using bytes return InputImage.fromBytes( - bytes: plane.bytes, + bytes: bytes, metadata: InputImageMetadata( size: Size(image.width.toDouble(), image.height.toDouble()), - rotation: rotation, // used only in Android - format: format, // used only in iOS - bytesPerRow: plane.bytesPerRow, // used only in iOS + rotation: rotation, + format: resolvedFormat, + bytesPerRow: image.planes.first.bytesPerRow, ), ); } + + // Reusable buffer to avoid per-frame allocations when concatenating planes. + Uint8List? _reusablePlaneBuffer; + + Uint8List _concatenatePlanes(CameraImage image) { + // Calculate the total number of bytes across all planes. + final int totalBytes = image.planes.fold( + 0, + (int sum, Plane plane) => sum + plane.bytes.length, + ); + + // Ensure the reusable buffer is allocated and large enough. + var buffer = _reusablePlaneBuffer; + if (buffer == null || buffer.length < totalBytes) { + buffer = Uint8List(totalBytes); + _reusablePlaneBuffer = buffer; + } + + // Copy each plane's bytes into the reusable buffer. + var offset = 0; + for (final Plane plane in image.planes) { + final bytes = plane.bytes; + buffer.setRange(offset, offset + bytes.length, bytes); + offset += bytes.length; + } + + // Return the reusable buffer directly when sizes match, or a zero-copy view otherwise. + if (totalBytes == buffer.length) { + return buffer; + } + return Uint8List.sublistView(buffer, 0, totalBytes); + } + + Uint8List? _reusableNv21Buffer; + int _lastNv21Size = 0; + Uint8List _convertYUV420ToNV21(CameraImage image) { + final int width = image.width; + final int height = image.height; + final int ySize = width * height; + final int uvSize = ySize ~/ 2; + final int requiredSize = ySize + uvSize; + + if (_reusableNv21Buffer == null || _lastNv21Size != requiredSize) { + _reusableNv21Buffer = Uint8List(requiredSize); + _lastNv21Size = requiredSize; + } + + final Uint8List nv21 = _reusableNv21Buffer!; + + // Copy Y plane (strip row padding) + final Plane yPlane = image.planes[0]; + int destIndex = 0; + for (int row = 0; row < height; row++) { + final int srcRowStart = row * yPlane.bytesPerRow; + nv21.setRange(destIndex, destIndex + width, yPlane.bytes, srcRowStart); + destIndex += width; + } + + // Interleave V and U planes into NV21 (VU order) + final Plane uPlane = image.planes[1]; + final Plane vPlane = image.planes[2]; + final int uvPixelStride = uPlane.bytesPerPixel ?? 1; + final int vPixelStride = vPlane.bytesPerPixel ?? 1; + + int uvIndex = ySize; + for (int row = 0; row < height ~/ 2; row++) { + final int uRowStart = row * uPlane.bytesPerRow; + final int vRowStart = row * vPlane.bytesPerRow; + + for (int col = 0; col < width ~/ 2; col++) { + final int uIndex = uRowStart + col * uvPixelStride; + final int vIndex = vRowStart + col * vPixelStride; + + nv21[uvIndex++] = vPlane.bytes[vIndex]; + nv21[uvIndex++] = uPlane.bytes[uIndex]; + } + } + + return nv21; + } } diff --git a/packages/google_mlkit_commons/android/src/main/kotlin/com/google_mlkit_commons/InputImageConverter.kt b/packages/google_mlkit_commons/android/src/main/kotlin/com/google_mlkit_commons/InputImageConverter.kt index f1d63b33..cf938d09 100644 --- a/packages/google_mlkit_commons/android/src/main/kotlin/com/google_mlkit_commons/InputImageConverter.kt +++ b/packages/google_mlkit_commons/android/src/main/kotlin/com/google_mlkit_commons/InputImageConverter.kt @@ -157,19 +157,33 @@ object InputImageConverter { val height = metaData["height"]?.toString()?.toDouble()?.toInt() ?: throw IllegalArgumentException("Height is null") - if ( - imageFormat == ImageFormat.NV21 || - imageFormat == ImageFormat.YUV_420_888 || - imageFormat == ImageFormat.YV12 - ) { - InputImage.fromByteArray(data, width, height, rotationDegrees, imageFormat) - } else { - result.error( - "InputImageConverterError", - "ImageFormat is not supported. Supported: NV21(17), YUV_420_888(35), YV12. Got: $imageFormat", - null, - ) - null + when (imageFormat) { + ImageFormat.NV21, ImageFormat.YV12 -> { + InputImage.fromByteArray(data, width, height, rotationDegrees, imageFormat) + } + + ImageFormat.YUV_420_888 -> { + // Create an InputImage directly from a YUV_420_888 byte array using the reported image format. + InputImage.fromByteArray( + data, + width, + height, + rotationDegrees, + imageFormat, + ) + } + + else -> { + result.error( + "InputImageConverterError", + "ImageFormat $imageFormat is not supported. Supported formats are: " + + "NV21 (${ImageFormat.NV21}), " + + "YV12 (${ImageFormat.YV12}), " + + "YUV_420_888 (${ImageFormat.YUV_420_888}).", + null, + ) + null + } } } catch (e: Exception) { Log.e("ImageError", "Getting Image failed")