Fix H.264 HW decode for AVC3 in-band streams (V4L2/m2m)

src/Session.cpp

@@ -780,25 +780,45 @@ void CSession::UpdateStream(CStream& stream)
 
   stream.m_isEncrypted = rep->GetPsshSetPos() != PSSHSET_POS_DEFAULT;
   stream.m_info.SetExtraData(nullptr, 0);
-
   if (!rep->GetCodecPrivateData().empty())
   {
-    std::vector<uint8_t> annexb;
-    const std::vector<uint8_t>* extraData(&annexb);
-
+    const auto& cpd = rep->GetCodecPrivateData();
     const DRM::DecrypterCapabilites& caps{GetDecrypterCaps(rep->m_psshSetPos)};
+    bool isAvcC = !cpd.empty() && cpd[0] == 0x01;
+    bool hasNoSPS = isAvcC && cpd.size() >= 6 && (cpd[5] & 0x1f) == 0;
 
-    if ((caps.flags & DRM::DecrypterCapabilites::SSD_ANNEXB_REQUIRED) &&
-        stream.m_info.GetStreamType() == INPUTSTREAM_TYPE_VIDEO)
+    if ((caps.flags & DRM::DecrypterCapabilites::SSD_ANNEXB_REQUIRED) && !hasNoSPS)
+    {
+      std::vector<uint8_t> annexb = AvcToAnnexb(cpd);
+      if (!annexb.empty())
+        stream.m_info.SetExtraData(annexb.data(), annexb.size());
+      else
+      {
+        LOG::LogF(LOGWARNING, "UpdateStream: AvcToAnnexb failed, using raw CPD");
+        stream.m_info.SetExtraData(cpd.data(), cpd.size());
+      }
+    }
+    else if (isAvcC && hasNoSPS)
+    {
+      // avcC with 0 SPS (AVC3 in-band) - skip extradata as V4L2 h264_xd_copy
+      // rejects avcC with 0 SPS/0 PPS; decoder gets SPS/PPS from in-band NALUs
+      stream.m_info.SetExtraData(nullptr, 0);
+    }
+    else if (isAvcC)
     {
-      LOG::Log(LOGDEBUG, "UpdateStream: Convert avc -> annexb");
-      annexb = AvcToAnnexb(rep->GetCodecPrivateData());
+      std::vector<uint8_t> annexb = AvcToAnnexb(cpd);
+      if (!annexb.empty())
+        stream.m_info.SetExtraData(annexb.data(), annexb.size());
+      else
+      {
+        LOG::LogF(LOGWARNING, "UpdateStream: AvcToAnnexb failed, using raw CPD");
+        stream.m_info.SetExtraData(cpd.data(), cpd.size());
+      }
     }
     else
     {
-      extraData = &rep->GetCodecPrivateData();
+      stream.m_info.SetExtraData(cpd.data(), cpd.size());
     }
-    stream.m_info.SetExtraData(extraData->data(), extraData->size());
   }
 
   stream.m_info.SetCodecFourCC(0);

src/codechandler/AVCCodecHandler.cpp

@@ -238,3 +238,68 @@ bool AVCCodecHandler::GetInformation(kodi::addon::InputstreamInfo& info)
   }
   return isChanged;
 };
+
+bool AVCCodecHandler::Transform(AP4_UI64 pts, AP4_UI32 duration, AP4_DataBuffer& buf, AP4_UI64 timescale)
+{
+  if (!m_needAnnexBTransform || m_naluLengthSize == 0)
+    return false;
+
+  const AP4_Byte* src = buf.GetData();
+  AP4_Size srcSize = buf.GetDataSize();
+  if (srcSize < m_naluLengthSize)
+    return false;
+
+  // Calculate output size: each NALU loses naluLengthSize bytes and gains 4 bytes (start code)
+  // Worst case: many small NALUs -> output grows. Allocate conservatively.
+  AP4_Size outSize = srcSize + (srcSize / m_naluLengthSize) * (4 - m_naluLengthSize) + 64;
+  AP4_DataBuffer outBuf;
+  outBuf.SetDataSize(outSize);
+  AP4_Byte* dst = outBuf.UseData();
+  AP4_Size dstUsed = 0;
+
+  const AP4_Byte startCode[4] = {0x00, 0x00, 0x00, 0x01};
+
+  while (srcSize >= m_naluLengthSize)
+  {
+    AP4_UI32 naluSize = 0;
+    switch (m_naluLengthSize)
+    {
+      case 1: naluSize = src[0]; break;
+      case 2: naluSize = (src[0] << 8) | src[1]; break;
+      case 4: naluSize = (src[0] << 24) | (src[1] << 16) | (src[2] << 8) | src[3]; break;
+      default: return false;
+    }
+
+    if (naluSize == 0 || naluSize > srcSize - m_naluLengthSize)
+      break;
+
+    // Ensure enough space in output buffer
+    AP4_Size needed = dstUsed + 4 + naluSize;
+    if (needed > outBuf.GetDataSize())
+    {
+      outBuf.SetDataSize(needed + 256);
+      dst = outBuf.UseData();
+    }
+
+    // Write Annex B start code
+    memcpy(dst + dstUsed, startCode, 4);
+    dstUsed += 4;
+
+    // Copy NALU data
+    memcpy(dst + dstUsed, src + m_naluLengthSize, naluSize);
+    dstUsed += naluSize;
+
+    // Advance source
+    src += m_naluLengthSize + naluSize;
+    srcSize -= m_naluLengthSize + naluSize;
+  }
+
+  if (dstUsed > 0)
+  {
+    buf.SetDataSize(dstUsed);
+    memcpy(buf.UseData(), outBuf.GetData(), dstUsed);
+    return true;
+  }
+
+  return false;
+}

src/codechandler/AVCCodecHandler.h

@@ -18,9 +18,12 @@ class ATTR_DLL_LOCAL AVCCodecHandler : public CodecHandler
   void UpdatePPSId(const AP4_DataBuffer& buffer) override;
   bool GetInformation(kodi::addon::InputstreamInfo& info) override;
   STREAMCODEC_PROFILE GetProfile() override { return m_codecProfile; };
+  bool Transform(AP4_UI64 pts, AP4_UI32 duration, AP4_DataBuffer& buf, AP4_UI64 timescale) override;
+  void SetAnnexBTransformNeeded(bool needed) { m_needAnnexBTransform = needed; }
 
 private:
   unsigned int m_countPictureSetIds;
   STREAMCODEC_PROFILE m_codecProfile;
   bool m_needSliceInfo;
+  bool m_needAnnexBTransform = false;
 };

src/samplereader/FragmentedSampleReader.cpp

@@ -521,7 +521,32 @@ void CFragmentedSampleReader::UpdateSampleDescription()
   }
 
   if ((m_decrypterCaps.flags & DRM::DecrypterCapabilites::SSD_ANNEXB_REQUIRED) != 0)
+  {
     m_codecHandler->ExtraDataToAnnexB();
+  }
+  else if (desc->GetFormat() == AP4_SAMPLE_FORMAT_AVC1 ||
+           desc->GetFormat() == AP4_SAMPLE_FORMAT_AVC2 ||
+           desc->GetFormat() == AP4_SAMPLE_FORMAT_AVC3 ||
+           desc->GetFormat() == AP4_SAMPLE_FORMAT_AVC4)
+  {
+    bool hasNoSPS = m_codecHandler->m_extraData.GetDataSize() >= 6 &&
+                    m_codecHandler->m_extraData.GetData()[0] == 0x01 &&
+                    (m_codecHandler->m_extraData.GetData()[5] & 0x1f) == 0;
+    if (hasNoSPS)
+    {
+      m_codecHandler->m_extraData.SetDataSize(0);
+      static_cast<AVCCodecHandler*>(m_codecHandler)->SetAnnexBTransformNeeded(true);
+    }
+    else
+    {
+      if (!m_codecHandler->ExtraDataToAnnexB() || m_codecHandler->m_extraData.GetDataSize() == 0)
+      {
+        LOG::LogF(LOGWARNING, "UpdateSampleDescription: ExtraDataToAnnexB failed, clearing extradata");
+        m_codecHandler->m_extraData.SetDataSize(0);
+      }
+      static_cast<AVCCodecHandler*>(m_codecHandler)->SetAnnexBTransformNeeded(true);
+    }
+  }
 }
 
 void CFragmentedSampleReader::ParseTrafTfrf(AP4_UuidAtom* uuidAtom)