Expose cpu_fallback in VideoDecoder API

docs/source/api_ref_decoders.rst

@@ -33,3 +33,4 @@ For an audio decoder tutorial, see: :ref:`sphx_glr_generated_examples_decoding_a
 
     VideoStreamMetadata
     AudioStreamMetadata
+    CpuFallbackStatus

src/torchcodec/_core/CudaDeviceInterface.cpp

@@ -241,6 +241,8 @@ void CudaDeviceInterface::convertAVFrameToFrameOutput(
     std::optional<torch::Tensor> preAllocatedOutputTensor) {
   validatePreAllocatedTensorShape(preAllocatedOutputTensor, avFrame);
 
+  hasDecodedFrame_ = true;
+
   // All of our CUDA decoding assumes NV12 format. We handle non-NV12 formats by
   // converting them to NV12.
   avFrame = maybeConvertAVFrameToNV12OrRGB24(avFrame);
@@ -358,6 +360,10 @@ std::string CudaDeviceInterface::getDetails() {
   // Note: for this interface specifically the fallback is only known after a
   // frame has been decoded, not before: that's when FFmpeg decides to fallback,
   // so we can't know earlier.
+  if (!hasDecodedFrame_) {
+    return std::string(
+        "FFmpeg CUDA Device Interface. Fallback status unknown (no frames decoded).");
+  }
   return std::string("FFmpeg CUDA Device Interface. Using ") +
       (usingCPUFallback_ ? "CPU fallback." : "NVDEC.");
 }

src/torchcodec/_core/CudaDeviceInterface.h

@@ -63,6 +63,7 @@ class CudaDeviceInterface : public DeviceInterface {
   std::unique_ptr<FilterGraph> nv12Conversion_;
 
   bool usingCPUFallback_ = false;
+  bool hasDecodedFrame_ = false;
 };
 
 } // namespace facebook::torchcodec

src/torchcodec/decoders/__init__.py

@@ -7,6 +7,6 @@
 from .._core import AudioStreamMetadata, VideoStreamMetadata
 from ._audio_decoder import AudioDecoder  # noqa
 from ._decoder_utils import set_cuda_backend  # noqa
-from ._video_decoder import VideoDecoder  # noqa
+from ._video_decoder import CpuFallbackStatus, VideoDecoder  # noqa
 
 SimpleVideoDecoder = VideoDecoder

src/torchcodec/decoders/_video_decoder.py

@@ -9,6 +9,7 @@
 import json
 import numbers
 from collections.abc import Sequence
+from dataclasses import dataclass, field
 from pathlib import Path
 from typing import Literal
 
@@ -25,6 +26,55 @@
 from torchcodec.transforms._decoder_transforms import _make_transform_specs
 
 
+@dataclass
+class CpuFallbackStatus:
+    """Information about CPU fallback status.
+
+    This class tracks whether the decoder fell back to CPU decoding.
+    Users should not instantiate this class directly; instead, access it
+    via the :attr:`VideoDecoder.cpu_fallback` attribute.
+
+    Usage:
+
+    - Use ``str(cpu_fallback_status)`` or ``print(cpu_fallback_status)`` to see the cpu fallback status
+    - Use ``if cpu_fallback_status:`` to check if any fallback occurred
+    """
+
+    status_known: bool = False
+    """Whether the fallback status has been determined.
+    For the Beta CUDA backend (see :func:`~torchcodec.decoders.set_cuda_backend`),
+    this is always ``True`` immediately after decoder creation.
+    For the FFmpeg CUDA backend, this becomes ``True`` after decoding
+    the first frame."""
+    _nvcuvid_unavailable: bool = field(default=False, init=False)
+    _video_not_supported: bool = field(default=False, init=False)
+    _backend: str = field(default="", init=False)
+
+    def __bool__(self):
+        """Returns True if fallback occurred."""
+        return self.status_known and (
+            self._nvcuvid_unavailable or self._video_not_supported
+        )
+
+    def __str__(self):
+        """Returns a human-readable string representation of the cpu fallback status."""
+        if not self.status_known:
+            return f"[{self._backend}] Fallback status: Unknown"
+
+        reasons = []
+        if self._nvcuvid_unavailable:
+            reasons.append("NVcuvid unavailable")
+        if self._video_not_supported:
+            reasons.append("Video not supported")
+
+        if reasons:
+            return (
+                f"[{self._backend}] Fallback status: Falling back due to: "
+                + ", ".join(reasons)
+            )
+        return f"[{self._backend}] Fallback status: No fallback required"
+
+
 class VideoDecoder:
     """A single-stream video decoder.
 
@@ -103,6 +153,10 @@ class VideoDecoder:
         stream_index (int): The stream index that this decoder is retrieving frames from. If a
             stream index was provided at initialization, this is the same value. If it was left
             unspecified, this is the :term:`best stream`.
+        cpu_fallback (CpuFallbackStatus): Information about whether the decoder fell back to CPU
+            decoding. Use ``bool(cpu_fallback)`` to check if fallback occurred, or
+            ``str(cpu_fallback)`` to get a human-readable status message. The status is only
+            determined after at least one frame has been decoded.
     """
 
     def __init__(
@@ -186,9 +240,41 @@ def __init__(
             custom_frame_mappings=custom_frame_mappings_data,
         )
 
+        self._cpu_fallback = CpuFallbackStatus()
+        if device.startswith("cuda"):
+            if device_variant == "beta":
+                self._cpu_fallback._backend = "Beta CUDA"
+            else:
+                self._cpu_fallback._backend = "FFmpeg CUDA"
+        else:
+            self._cpu_fallback._backend = "CPU"
+
     def __len__(self) -> int:
         return self._num_frames
 
+    @property
+    def cpu_fallback(self) -> CpuFallbackStatus:
+        # We only query the CPU fallback info if status is unknown. That happens
+        # either when:
+        # - this @property has never been called before
+        # - no frame has been decoded yet on the FFmpeg interface.
+        # Note that for the beta interface, we're able to know the fallback status
+        # right when the VideoDecoder is instantiated, but the status_known
+        # attribute is initialized to False.
+        if not self._cpu_fallback.status_known:
+            backend_details = core._get_backend_details(self._decoder)
+
+            if "status unknown" not in backend_details:
+                self._cpu_fallback.status_known = True
+
+                if "CPU fallback" in backend_details:
+                    if "NVCUVID not available" in backend_details:
+                        self._cpu_fallback._nvcuvid_unavailable = True
+                    else:
+                        self._cpu_fallback._video_not_supported = True
+
+        return self._cpu_fallback
+
     def _getitem_int(self, key: int) -> Tensor:
         assert isinstance(key, int)
 

test/test_decoders.py

@@ -27,6 +27,7 @@
     assert_frames_equal,
     AV1_VIDEO,
     BT709_FULL_RANGE,
+    cuda_devices,
     cuda_version_used_for_building_torch,
     get_ffmpeg_major_version,
     get_python_version,
@@ -1672,22 +1673,27 @@ def test_beta_cuda_interface_cpu_fallback(self):
         # to the CPU path, too.
 
         ref_dec = VideoDecoder(H265_VIDEO.path, device="cuda")
-        ref_frames = ref_dec.get_frame_at(0)
-        assert (
-            _core._get_backend_details(ref_dec._decoder)
-            == "FFmpeg CUDA Device Interface. Using CPU fallback."
-        )
+
+        # Before accessing any frames, status should be unknown
+        assert not ref_dec.cpu_fallback.status_known
+
+        ref_frame = ref_dec.get_frame_at(0)
+
+        assert "FFmpeg CUDA" in str(ref_dec.cpu_fallback)
+        assert ref_dec.cpu_fallback.status_known
+        assert ref_dec.cpu_fallback
 
         with set_cuda_backend("beta"):
             beta_dec = VideoDecoder(H265_VIDEO.path, device="cuda")
 
-        assert (
-            _core._get_backend_details(beta_dec._decoder)
-            == "Beta CUDA Device Interface. Using CPU fallback."
-        )
+        assert "Beta CUDA" in str(beta_dec.cpu_fallback)
+        # For beta interface, status is known immediately
+        assert beta_dec.cpu_fallback.status_known
+        assert beta_dec.cpu_fallback
+
         beta_frame = beta_dec.get_frame_at(0)
 
-        assert psnr(ref_frames.data, beta_frame.data) > 25
+        assert psnr(ref_frame.data, beta_frame.data) > 25
 
     @needs_cuda
     def test_beta_cuda_interface_error(self):
@@ -1715,7 +1721,7 @@ def test_set_cuda_backend(self):
         # Check that the default is the ffmpeg backend
         assert _get_cuda_backend() == "ffmpeg"
         dec = VideoDecoder(H265_VIDEO.path, device="cuda")
-        assert _core._get_backend_details(dec._decoder).startswith("FFmpeg CUDA")
+        assert "FFmpeg CUDA" in str(dec.cpu_fallback)
 
         # Check the setting "beta" effectively uses the BETA backend.
         # We also show that the affects decoder creation only. When the decoder
@@ -1724,9 +1730,9 @@ def test_set_cuda_backend(self):
         with set_cuda_backend("beta"):
             dec = VideoDecoder(H265_VIDEO.path, device="cuda")
         assert _get_cuda_backend() == "ffmpeg"
-        assert _core._get_backend_details(dec._decoder).startswith("Beta CUDA")
+        assert "Beta CUDA" in str(dec.cpu_fallback)
         with set_cuda_backend("ffmpeg"):
-            assert _core._get_backend_details(dec._decoder).startswith("Beta CUDA")
+            assert "Beta CUDA" in str(dec.cpu_fallback)
 
         # Hacky way to ensure passing "cuda:1" is supported by both backends. We
         # just check that there's an error when passing cuda:N where N is too
@@ -1737,6 +1743,48 @@ def test_set_cuda_backend(self):
                 with set_cuda_backend(backend):
                     VideoDecoder(H265_VIDEO.path, device=f"cuda:{bad_device_number}")
 
+    def test_cpu_fallback_no_fallback_on_cpu_device(self):
+        """Test that CPU device doesn't trigger fallback (it's not a fallback scenario)."""
+        decoder = VideoDecoder(NASA_VIDEO.path, device="cpu")
+
+        assert decoder.cpu_fallback.status_known
+        _ = decoder[0]
+
+        assert not decoder.cpu_fallback
+        assert "No fallback required" in str(decoder.cpu_fallback)
+
+    @needs_cuda
+    @pytest.mark.parametrize("device", cuda_devices())
+    def test_cpu_fallback_h265_video(self, device):
+        """Test that H265 video triggers CPU fallback on CUDA interfaces."""
+        # H265_VIDEO is known to trigger CPU fallback on CUDA
+        # because its dimensions are too small
+        decoder, _ = make_video_decoder(H265_VIDEO.path, device=device)
+
+        if "beta" in device:
+            # For beta interface, status is known immediately
+            assert decoder.cpu_fallback.status_known
+        else:
+            # For FFmpeg interface, status is unknown until first frame is decoded
+            assert not decoder.cpu_fallback.status_known
+
+        decoder.get_frame_at(0)
+
+        assert decoder.cpu_fallback.status_known
+        assert decoder.cpu_fallback
+        assert "Video not supported" in str(decoder.cpu_fallback)
+
+    @needs_cuda
+    @pytest.mark.parametrize("device", cuda_devices())
+    def test_cpu_fallback_no_fallback_on_supported_video(self, device):
+        """Test that supported videos don't trigger fallback on CUDA."""
+        decoder, _ = make_video_decoder(NASA_VIDEO.path, device=device)
+
+        decoder[0]
+
+        assert not decoder.cpu_fallback
+        assert "No fallback required" in str(decoder.cpu_fallback)
+
 
 class TestAudioDecoder:
     @pytest.mark.parametrize("asset", (NASA_AUDIO, NASA_AUDIO_MP3, SINE_MONO_S32))

test/utils.py

@@ -51,6 +51,13 @@ def all_supported_devices():
     )
 
 
+def cuda_devices():
+    return (
+        pytest.param("cuda", marks=pytest.mark.needs_cuda),
+        pytest.param(_CUDA_BETA_DEVICE_STR, marks=pytest.mark.needs_cuda),
+    )
+
+
 def unsplit_device_str(device_str: str) -> str:
     # helper meant to be used as
     # device, device_variant = unsplit_device_str(device)