From 0d80a12bd72556a8cfdeb59a59349b562294d544 Mon Sep 17 00:00:00 2001 From: Jakob Blomer Date: Fri, 29 May 2026 10:35:05 +0200 Subject: [PATCH 1/6] [ntuple] add RFieldBase::GetOnDiskFieldVersion() --- tree/ntuple/inc/ROOT/RFieldBase.hxx | 5 +++++ tree/ntuple/src/RFieldBase.cxx | 1 + tree/ntuple/test/rfield_class.cxx | 10 +++++++++- 3 files changed, 15 insertions(+), 1 deletion(-) diff --git a/tree/ntuple/inc/ROOT/RFieldBase.hxx b/tree/ntuple/inc/ROOT/RFieldBase.hxx index f82e83ea7061a..6427e22d43362 100644 --- a/tree/ntuple/inc/ROOT/RFieldBase.hxx +++ b/tree/ntuple/inc/ROOT/RFieldBase.hxx @@ -137,6 +137,7 @@ protected: }; public: + static constexpr std::uint32_t kInvalidFieldVersion = -1U; static constexpr std::uint32_t kInvalidTypeVersion = -1U; enum { /// No constructor needs to be called, i.e. any bit pattern in the allocated memory represents a valid type @@ -320,6 +321,8 @@ protected: std::string fTypeAlias; /// List of functions to be called after reading a value std::vector fReadCallbacks; + /// Field version cached from the descriptor after a call to ConnectPageSource() + std::uint32_t fOnDiskFieldVersion = kInvalidFieldVersion; /// C++ type version cached from the descriptor after a call to ConnectPageSource() std::uint32_t fOnDiskTypeVersion = kInvalidTypeVersion; /// TClass checksum cached from the descriptor after a call to ConnectPageSource(). Only set @@ -664,6 +667,8 @@ public: virtual std::uint32_t GetTypeVersion() const { return 0; } /// Return the current TClass reported checksum of this class. Only valid if `kTraitTypeChecksum` is set. virtual std::uint32_t GetTypeChecksum() const { return 0; } + /// Return the field version stored in the field descriptor; only valid after a call to ConnectPageSource() + std::uint32_t GetOnDiskFieldVersion() const { return fOnDiskFieldVersion; } /// Return the C++ type version stored in the field descriptor; only valid after a call to ConnectPageSource() std::uint32_t GetOnDiskTypeVersion() const { return fOnDiskTypeVersion; } /// Return checksum stored in the field descriptor; only valid after a call to ConnectPageSource(), diff --git a/tree/ntuple/src/RFieldBase.cxx b/tree/ntuple/src/RFieldBase.cxx index 38b70e5ad60b5..e00991af4c5b6 100644 --- a/tree/ntuple/src/RFieldBase.cxx +++ b/tree/ntuple/src/RFieldBase.cxx @@ -1023,6 +1023,7 @@ void ROOT::RFieldBase::ConnectPageSource(ROOT::Internal::RPageSource &pageSource R__ASSERT(!fColumnRepresentatives.empty()); if (fOnDiskId != ROOT::kInvalidDescriptorId) { const auto &fieldDesc = desc.GetFieldDescriptor(fOnDiskId); + fOnDiskFieldVersion = fieldDesc.GetFieldVersion(); fOnDiskTypeVersion = fieldDesc.GetTypeVersion(); if (fieldDesc.GetTypeChecksum().has_value()) fOnDiskTypeChecksum = *fieldDesc.GetTypeChecksum(); diff --git a/tree/ntuple/test/rfield_class.cxx b/tree/ntuple/test/rfield_class.cxx index 326a6140f394e..e9dd6297e89a5 100644 --- a/tree/ntuple/test/rfield_class.cxx +++ b/tree/ntuple/test/rfield_class.cxx @@ -36,7 +36,15 @@ TEST(RNTuple, TClass) { EXPECT_THROW(model->MakeField("datime"), ROOT::RException); FileRaii fileGuard("test_ntuple_tclass.root"); - auto ntuple = RNTupleWriter::Recreate(std::move(model), "f", fileGuard.GetPath()); + { + RNTupleWriter::Recreate(std::move(model), "ntpl", fileGuard.GetPath()); + } + auto reader = RNTupleReader::Open("ntpl", fileGuard.GetPath()); + const auto &f = reader->GetModel().GetConstField("klass"); + EXPECT_EQ(0u, f.GetFieldVersion()); + EXPECT_EQ(0u, f.GetOnDiskFieldVersion()); + EXPECT_EQ(TClass::GetClass("CustomStruct")->GetClassVersion(), f.GetOnDiskTypeVersion()); + EXPECT_EQ(TClass::GetClass("CustomStruct")->GetClassVersion(), f.GetTypeVersion()); } TEST(RNTuple, CyclicClass) From 2f86c4fc8268035dff223e0cad24c7c4a09107f0 Mon Sep 17 00:00:00 2001 From: Jakob Blomer Date: Fri, 29 May 2026 13:30:51 +0200 Subject: [PATCH 2/6] [ntuple] add partial support for v1 streamer field The version 0 and version 1 streamer fields are exactly the same for small objects (<=1GiB). For larger objects, the version 1 streamer field stores the byte count stack before the data. This is up to a future patch. For the moment, we assert that the streamed buffer is never >1GiB. --- tree/ntuple/inc/ROOT/RField.hxx | 4 ++ tree/ntuple/src/RFieldMeta.cxx | 18 ++++++- tree/ntuple/test/CustomStruct.hxx | 5 ++ tree/ntuple/test/CustomStructLinkDef.h | 2 + tree/ntuple/test/rfield_streamer.cxx | 71 ++++++++++++++++++++++++++ 5 files changed, 99 insertions(+), 1 deletion(-) diff --git a/tree/ntuple/inc/ROOT/RField.hxx b/tree/ntuple/inc/ROOT/RField.hxx index bc0c4f952877d..3b43c7e8ef49a 100644 --- a/tree/ntuple/inc/ROOT/RField.hxx +++ b/tree/ntuple/inc/ROOT/RField.hxx @@ -237,6 +237,8 @@ public: /// The field for a class using ROOT standard streaming class RStreamerField final : public RFieldBase { private: + static constexpr std::size_t kMaxSmallBuffer = 1024 * 1024 * 1024; ///< maximum buffer size for v0 streamer field + class RStreamerFieldDeleter : public RDeleter { private: TClass *fClass; @@ -283,6 +285,8 @@ public: size_t GetValueSize() const final; size_t GetAlignment() const final; + // As of field version 1, the byte stream contains the byte count stack for large objects (see binary specs) + std::uint32_t GetFieldVersion() const final { return 0; } std::uint32_t GetTypeVersion() const final; std::uint32_t GetTypeChecksum() const final; TClass *GetClass() const { return fClass; } diff --git a/tree/ntuple/src/RFieldMeta.cxx b/tree/ntuple/src/RFieldMeta.cxx index ef334f2c552d1..ce8ee9aaefda3 100644 --- a/tree/ntuple/src/RFieldMeta.cxx +++ b/tree/ntuple/src/RFieldMeta.cxx @@ -1317,6 +1317,13 @@ std::size_t ROOT::RStreamerField::AppendImpl(const void *from) fClass->Streamer(const_cast(from), buffer); auto nbytes = buffer.Length(); + R__ASSERT(nbytes >= 0); + if (static_cast(nbytes) > kMaxSmallBuffer) { + throw RException(R__FAIL("large objects (>1GiB) not supported by the version 0 streamer field")); + } else { + assert(buffer.GetByteCounts().empty()); + } + fAuxiliaryColumn->AppendV(buffer.Buffer(), buffer.Length()); fIndex += nbytes; fPrincipalColumn->Append(&fIndex); @@ -1329,6 +1336,9 @@ void ROOT::RStreamerField::ReadGlobalImpl(ROOT::NTupleSize_t globalIndex, void * ROOT::NTupleSize_t nbytes; fPrincipalColumn->GetCollectionInfo(globalIndex, &collectionStart, &nbytes); + if (nbytes > kMaxSmallBuffer) + throw RException(R__FAIL("large objects (>1GiB) not supported by the version 0 streamer field")); + TBufferFile buffer(TBuffer::kRead, nbytes); fAuxiliaryColumn->ReadV(collectionStart, nbytes, buffer.Buffer()); fClass->Streamer(to, buffer); @@ -1362,7 +1372,13 @@ std::unique_ptr ROOT::RStreamerField::BeforeConnectPageSource( void ROOT::RStreamerField::ReconcileOnDiskField(const RNTupleDescriptor &desc) { - EnsureMatchingOnDiskField(desc, kDiffTypeName | kDiffTypeVersion).ThrowOnError(); + EnsureMatchingOnDiskField(desc, kDiffTypeName | kDiffTypeVersion | kDiffFieldVersion).ThrowOnError(); + const auto &fieldDesc = desc.GetFieldDescriptor(GetOnDiskId()); + if (fieldDesc.GetFieldVersion() > 1) { + throw RException(R__FAIL("RStreamerField " + GetQualifiedFieldName() + " has unsupported field version " + + std::to_string(fieldDesc.GetFieldVersion()) + "\n" + + Internal::GetTypeTraceReport(*this, desc))); + } } void ROOT::RStreamerField::ConstructValue(void *where) const diff --git a/tree/ntuple/test/CustomStruct.hxx b/tree/ntuple/test/CustomStruct.hxx index c3d6e633f356b..da8a79601eacb 100644 --- a/tree/ntuple/test/CustomStruct.hxx +++ b/tree/ntuple/test/CustomStruct.hxx @@ -478,4 +478,9 @@ struct MemberWithCustomStreamer { ClassDefNV(MemberWithCustomStreamer, 2); }; +struct VersionedStreamerField +{ + ClassDefNV(VersionedStreamerField, 2); +}; + #endif diff --git a/tree/ntuple/test/CustomStructLinkDef.h b/tree/ntuple/test/CustomStructLinkDef.h index 51aadb41c34c2..815bf71b5f940 100644 --- a/tree/ntuple/test/CustomStructLinkDef.h +++ b/tree/ntuple/test/CustomStructLinkDef.h @@ -181,4 +181,6 @@ #pragma link C++ class MemberWithCustomStreamer+; +#pragma link C++ class VersionedStreamerField+; + #endif diff --git a/tree/ntuple/test/rfield_streamer.cxx b/tree/ntuple/test/rfield_streamer.cxx index 2f6e82ae85e33..3051198ba41ad 100644 --- a/tree/ntuple/test/rfield_streamer.cxx +++ b/tree/ntuple/test/rfield_streamer.cxx @@ -407,3 +407,74 @@ TEST(RField, StreamerClassMismatch) false /* matchFullMessage */); reader->LoadEntry(0); } + +namespace { + +/// Used to create on-disk streamer fields with different field versions +class RVersionedStreamerField : public RFieldBase { +protected: + std::unique_ptr CloneImpl(std::string_view newName) const final + { + return std::make_unique(newName, fCustomVersion); + } + + const RColumnRepresentations &GetColumnRepresentations() const final + { + static RColumnRepresentations representations( + {{ROOT::ENTupleColumnType::kSplitIndex64, ROOT::ENTupleColumnType::kByte}}, {}); + return representations; + } + + void GenerateColumns() final + { + GenerateColumnsImpl(); + } + void GenerateColumns(const ROOT::RNTupleDescriptor &) final {} + + void ConstructValue(void *) const final {} + + std::size_t AppendImpl(const void *) final { return 0; } + +public: + std::uint32_t fCustomVersion = 0; + + RVersionedStreamerField(std::string_view name, std::uint32_t version) + : RFieldBase(name, "VersionedStreamerField", ROOT::ENTupleStructure::kStreamer, /*isSimple=*/false), + fCustomVersion(version) + {} + + std::uint32_t GetFieldVersion() const final { return fCustomVersion; } + std::uint32_t GetTypeVersion() const final { return 137; } + std::size_t GetValueSize() const final { return 0; } + std::size_t GetAlignment() const final { return 0; } +}; + +} // anonymous namespace + +TEST(RField, StreamerFieldVersion) +{ + for (std::uint32_t version: {0, 1, 2}) { + FileRaii fileGuard("test_ntuple_rfield_streamer_version.root"); + { + auto model = RNTupleModel::Create(); + model->AddField(std::make_unique("f", version)); + auto writer = RNTupleWriter::Recreate(std::move(model), "ntpl", fileGuard.GetPath()); + } + auto reader = RNTupleReader::Open("ntpl", fileGuard.GetPath()); + if (version < 2) { + const auto &f = reader->GetModel().GetConstField("f"); + EXPECT_TRUE(dynamic_cast(&f)); + EXPECT_EQ(0u, f.GetFieldVersion()); + EXPECT_EQ(version, f.GetOnDiskFieldVersion()); + EXPECT_EQ(2u, f.GetTypeVersion()); + EXPECT_EQ(137u, f.GetOnDiskTypeVersion()); + } else { + try { + reader->GetModel().GetConstField("f"); + FAIL() << "creating model from unsupported field version should fail"; + } catch (const ROOT::RException &e) { + EXPECT_THAT(e.what(), ::testing::HasSubstr("RStreamerField f has unsupported field version 2")); + } + } + } +} From 879dd05d75d79383e7315f318bb355503948fdf2 Mon Sep 17 00:00:00 2001 From: Jakob Blomer Date: Fri, 29 May 2026 13:39:19 +0200 Subject: [PATCH 3/6] [NFC][ntuple] bump spec version to 1.0.3.0 Add version 1 streamer field. --- tree/ntuple/doc/BinaryFormatSpecification.md | 14 +++++++++++++- tree/ntuple/inc/ROOT/RNTuple.hxx | 2 +- tree/ntuple/test/ntuple_minifile.cxx | 2 +- 3 files changed, 15 insertions(+), 3 deletions(-) diff --git a/tree/ntuple/doc/BinaryFormatSpecification.md b/tree/ntuple/doc/BinaryFormatSpecification.md index 967f0580be0c1..1751ea036ffd0 100644 --- a/tree/ntuple/doc/BinaryFormatSpecification.md +++ b/tree/ntuple/doc/BinaryFormatSpecification.md @@ -1,4 +1,4 @@ -# RNTuple Binary Format Specification 1.0.2.0 +# RNTuple Binary Format Specification 1.0.3.0 ## Versioning Notes @@ -1130,6 +1130,18 @@ The first (principal) column is of type `(Split)Index[32|64]`. The second column is of type `Byte`. In effect, the column representation is identical to a collection of `std::byte`. +There are two field versions for the streamer field, version 0 and version 1. +Both versions have an identical on-disk representation when the streamed object is smaller than 1GiB. +Only version 1 supports larger streamed objects. +For large objects, the version 1 streamer field prepends the large byte counts ("byte count stack") to the byte stream. +The format for the version 1 byte stream is + + - 64bit unsigned integer: number of elements in the large byte count list + - List if 64bit unsigned integer pairs with the byte count location and byte count value` + - Regular ROOT object stream + +The integer values before the regular ROOT object stream are stored in little endianess. + ### Untyped collections and records Untyped collections and records are fields with a collection or record role and an empty type name. diff --git a/tree/ntuple/inc/ROOT/RNTuple.hxx b/tree/ntuple/inc/ROOT/RNTuple.hxx index 4ded76524b036..f69c689470f84 100644 --- a/tree/ntuple/inc/ROOT/RNTuple.hxx +++ b/tree/ntuple/inc/ROOT/RNTuple.hxx @@ -78,7 +78,7 @@ class RNTuple final { public: static constexpr std::uint16_t kVersionEpoch = 1; static constexpr std::uint16_t kVersionMajor = 0; - static constexpr std::uint16_t kVersionMinor = 2; + static constexpr std::uint16_t kVersionMinor = 3; static constexpr std::uint16_t kVersionPatch = 0; /// Returns the RNTuple version in the following form: diff --git a/tree/ntuple/test/ntuple_minifile.cxx b/tree/ntuple/test/ntuple_minifile.cxx index 6dd2ab7ef4c70..97a4967c71a62 100644 --- a/tree/ntuple/test/ntuple_minifile.cxx +++ b/tree/ntuple/test/ntuple_minifile.cxx @@ -17,7 +17,7 @@ namespace { // as a double check that we increment the version correctly. constexpr auto kVersionEpoch = 1; constexpr auto kVersionMajor = 0; -constexpr auto kVersionMinor = 2; +constexpr auto kVersionMinor = 3; constexpr auto kVersionPatch = 0; bool IsEqual(const ROOT::RNTuple &a, const ROOT::RNTuple &b) From fd5b8ff9dc14335a43d6f03b6d8923eb18c7101d Mon Sep 17 00:00:00 2001 From: Jakob Blomer Date: Fri, 29 May 2026 16:17:51 +0200 Subject: [PATCH 4/6] [io] make byte count typedefs public --- io/io/inc/TBufferFile.h | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/io/io/inc/TBufferFile.h b/io/io/inc/TBufferFile.h index ca6363d188264..651da476c3a0a 100644 --- a/io/io/inc/TBufferFile.h +++ b/io/io/inc/TBufferFile.h @@ -45,6 +45,10 @@ namespace TStreamerInfoActions { } class TBufferFile : public TBufferIO { +public: + using ByteCountLocator_t = std::size_t; // This might become a pair if we implement chunked keys + using ByteCount_t = std::uint64_t; ///< Type used to store byte count values, can be changed to uint32_t if we implement chunked keys + using ByteCountFinder_t = std::unordered_map; protected: typedef std::vector InfoList_t; @@ -52,7 +56,6 @@ class TBufferFile : public TBufferIO { TStreamerInfo *fInfo{nullptr}; ///< Pointer to TStreamerInfo object writing/reading the buffer InfoList_t fInfoStack; ///< Stack of pointers to the TStreamerInfos - using ByteCountLocator_t = std::size_t; // This might become a pair if we implement chunked keys struct ByteCountLocationInfo { ///< Position where the byte count value is stored ByteCountLocator_t locator; @@ -66,8 +69,6 @@ class TBufferFile : public TBufferIO { using ByteCountStack_t = std::vector; ByteCountStack_t fByteCountStack; ///; // fByteCounts will be stored either in the header/summary tkey or at the end // of the last segment/chunk for a large TKey. ByteCountFinder_t fByteCounts; ///< Map to find the byte count value for a given position From 6bfe28cd087e707e255db9328b237567a6b4829f Mon Sep 17 00:00:00 2001 From: Jakob Blomer Date: Fri, 29 May 2026 16:52:44 +0200 Subject: [PATCH 5/6] [ntuple] add support for large objects in streamer field --- tree/ntuple/inc/ROOT/RField.hxx | 2 +- tree/ntuple/src/RFieldMeta.cxx | 53 +++++++++++++++++++++++++--- tree/ntuple/test/rfield_streamer.cxx | 2 +- 3 files changed, 50 insertions(+), 7 deletions(-) diff --git a/tree/ntuple/inc/ROOT/RField.hxx b/tree/ntuple/inc/ROOT/RField.hxx index 3b43c7e8ef49a..e240eed72c6b8 100644 --- a/tree/ntuple/inc/ROOT/RField.hxx +++ b/tree/ntuple/inc/ROOT/RField.hxx @@ -286,7 +286,7 @@ public: size_t GetValueSize() const final; size_t GetAlignment() const final; // As of field version 1, the byte stream contains the byte count stack for large objects (see binary specs) - std::uint32_t GetFieldVersion() const final { return 0; } + std::uint32_t GetFieldVersion() const final { return 1; } std::uint32_t GetTypeVersion() const final; std::uint32_t GetTypeChecksum() const final; TClass *GetClass() const { return fClass; } diff --git a/tree/ntuple/src/RFieldMeta.cxx b/tree/ntuple/src/RFieldMeta.cxx index ce8ee9aaefda3..a36e4273afcf2 100644 --- a/tree/ntuple/src/RFieldMeta.cxx +++ b/tree/ntuple/src/RFieldMeta.cxx @@ -19,6 +19,7 @@ #include #include #include +#include #include #include @@ -1316,10 +1317,21 @@ std::size_t ROOT::RStreamerField::AppendImpl(const void *from) [this](TVirtualStreamerInfo *info) { fStreamerInfos[info->GetNumber()] = info; }); fClass->Streamer(const_cast(from), buffer); - auto nbytes = buffer.Length(); + const auto nbytes = buffer.Length(); + std::size_t szBufCounts = 0; R__ASSERT(nbytes >= 0); if (static_cast(nbytes) > kMaxSmallBuffer) { - throw RException(R__FAIL("large objects (>1GiB) not supported by the version 0 streamer field")); + const std::uint64_t nCounts = buffer.GetByteCounts().size(); + szBufCounts = sizeof(std::uint64_t) * (2 * nCounts + 1); + auto bufCounts = Internal::MakeUninitArray(szBufCounts); + std::size_t pos = Internal::RNTupleSerializer::SerializeUInt64(nCounts, bufCounts.get()); + for (const auto &[bcountLoc, bcountVal] : buffer.GetByteCounts()) { + pos += Internal::RNTupleSerializer::SerializeUInt64(bcountLoc, bufCounts.get() + pos); + pos += Internal::RNTupleSerializer::SerializeUInt64(bcountVal, bufCounts.get() + pos); + } + assert(pos == szBufCounts); + fAuxiliaryColumn->AppendV(bufCounts.get(), szBufCounts); + fIndex += szBufCounts; } else { assert(buffer.GetByteCounts().empty()); } @@ -1327,7 +1339,7 @@ std::size_t ROOT::RStreamerField::AppendImpl(const void *from) fAuxiliaryColumn->AppendV(buffer.Buffer(), buffer.Length()); fIndex += nbytes; fPrincipalColumn->Append(&fIndex); - return nbytes + fPrincipalColumn->GetElement()->GetPackedSize(); + return szBufCounts + nbytes + fPrincipalColumn->GetElement()->GetPackedSize(); } void ROOT::RStreamerField::ReadGlobalImpl(ROOT::NTupleSize_t globalIndex, void *to) @@ -1336,10 +1348,41 @@ void ROOT::RStreamerField::ReadGlobalImpl(ROOT::NTupleSize_t globalIndex, void * ROOT::NTupleSize_t nbytes; fPrincipalColumn->GetCollectionInfo(globalIndex, &collectionStart, &nbytes); - if (nbytes > kMaxSmallBuffer) - throw RException(R__FAIL("large objects (>1GiB) not supported by the version 0 streamer field")); + TBufferFile::ByteCountFinder_t byteCounts; + if (nbytes > kMaxSmallBuffer) { + std::vector bufCounts(sizeof(std::uint64_t)); + fAuxiliaryColumn->ReadV(collectionStart, sizeof(std::uint64_t), bufCounts.data()); + std::uint64_t nCounts; + std::size_t pos = Internal::RNTupleSerializer::DeserializeUInt64(bufCounts.data(), nCounts); + if (nCounts > (std::numeric_limits::max() / sizeof(std::uint64_t)) / 2 - 1) + throw RException(R__FAIL("invalid byte count size in streamer field: " + std::to_string(nCounts))); + const std::size_t szBufCounts = sizeof(std::uint64_t) * (2 * nCounts + 1); + if (szBufCounts > nbytes) + throw RException(R__FAIL("invalid byte count size in streamer field: " + std::to_string(nCounts))); + bufCounts.resize(szBufCounts); + nbytes -= szBufCounts; + fAuxiliaryColumn->ReadV(collectionStart + sizeof(uint64_t), szBufCounts - sizeof(uint64_t), + bufCounts.data() + sizeof(uint64_t)); + collectionStart = collectionStart + szBufCounts; + + byteCounts.reserve(nCounts); + for (std::uint64_t i = 0; i < nCounts; ++i) { + std::uint64_t bcountLoc, bcountVal; + pos += Internal::RNTupleSerializer::DeserializeUInt64(bufCounts.data() + pos, bcountLoc); + pos += Internal::RNTupleSerializer::DeserializeUInt64(bufCounts.data() + pos, bcountVal); + if ((bcountLoc > nbytes) || (bcountVal > nbytes) || (nbytes - bcountVal < bcountLoc)) { + throw RException(R__FAIL("invalid byte count record: " + std::to_string(bcountLoc) + ", " + + std::to_string(bcountVal))); + } + byteCounts.emplace(bcountLoc, bcountVal); + } + assert(pos == szBufCounts); + if (byteCounts.size() != nCounts) + throw RException(R__FAIL("duplicate byte counts")); + } TBufferFile buffer(TBuffer::kRead, nbytes); + buffer.SetByteCounts(std::move(byteCounts)); fAuxiliaryColumn->ReadV(collectionStart, nbytes, buffer.Buffer()); fClass->Streamer(to, buffer); } diff --git a/tree/ntuple/test/rfield_streamer.cxx b/tree/ntuple/test/rfield_streamer.cxx index 3051198ba41ad..411e9b956a95c 100644 --- a/tree/ntuple/test/rfield_streamer.cxx +++ b/tree/ntuple/test/rfield_streamer.cxx @@ -464,7 +464,7 @@ TEST(RField, StreamerFieldVersion) if (version < 2) { const auto &f = reader->GetModel().GetConstField("f"); EXPECT_TRUE(dynamic_cast(&f)); - EXPECT_EQ(0u, f.GetFieldVersion()); + EXPECT_EQ(1u, f.GetFieldVersion()); EXPECT_EQ(version, f.GetOnDiskFieldVersion()); EXPECT_EQ(2u, f.GetTypeVersion()); EXPECT_EQ(137u, f.GetOnDiskTypeVersion()); From 3b67b13df7267fdaa4c60f810ec4dfc947a9fb30 Mon Sep 17 00:00:00 2001 From: Jakob Blomer Date: Tue, 2 Dec 2025 11:45:50 +0100 Subject: [PATCH 6/6] [io] add unit test infrastructure for streaming large objects Uses RStreamerField as a test bed for (de-)serializing large objects with TBufferFile. --- tree/ntuple/test/CMakeLists.txt | 7 +++ tree/ntuple/test/StreamerBeyond.cxx | 1 + tree/ntuple/test/StreamerBeyond.hxx | 14 +++++ tree/ntuple/test/StreamerBeyondLinkDef.h | 5 ++ tree/ntuple/test/rfield_streamer_beyond.cxx | 70 +++++++++++++++++++++ 5 files changed, 97 insertions(+) create mode 100644 tree/ntuple/test/StreamerBeyond.cxx create mode 100644 tree/ntuple/test/StreamerBeyond.hxx create mode 100644 tree/ntuple/test/StreamerBeyondLinkDef.h create mode 100644 tree/ntuple/test/rfield_streamer_beyond.cxx diff --git a/tree/ntuple/test/CMakeLists.txt b/tree/ntuple/test/CMakeLists.txt index 34e8b373149be..c1d398b801af1 100644 --- a/tree/ntuple/test/CMakeLists.txt +++ b/tree/ntuple/test/CMakeLists.txt @@ -114,6 +114,13 @@ ROOT_GENERATE_DICTIONARY(StreamerFieldDict ${CMAKE_CURRENT_SOURCE_DIR}/StreamerF MODULE rfield_streamer LINKDEF StreamerFieldLinkDef.h OPTIONS -inlineInputHeader DEPENDENCIES RIO) +if (CMAKE_SIZEOF_VOID_P EQUAL 8) + ROOT_ADD_GTEST(rfield_streamer_beyond rfield_streamer_beyond.cxx StreamerBeyond.cxx LIBRARIES ROOTNTuple) + ROOT_GENERATE_DICTIONARY(StreamerBeyondDict ${CMAKE_CURRENT_SOURCE_DIR}/StreamerBeyond.hxx + MODULE rfield_streamer_beyond LINKDEF StreamerBeyondLinkDef.h OPTIONS -inlineInputHeader + DEPENDENCIES RIO) +endif() + if(MSVC) set(command ${CMAKE_COMMAND} -E env "ROOTIGNOREPREFIX=1" $) else() diff --git a/tree/ntuple/test/StreamerBeyond.cxx b/tree/ntuple/test/StreamerBeyond.cxx new file mode 100644 index 0000000000000..dbe871afe35c7 --- /dev/null +++ b/tree/ntuple/test/StreamerBeyond.cxx @@ -0,0 +1 @@ +#include "StreamerBeyond.hxx" diff --git a/tree/ntuple/test/StreamerBeyond.hxx b/tree/ntuple/test/StreamerBeyond.hxx new file mode 100644 index 0000000000000..eca5facb34bd6 --- /dev/null +++ b/tree/ntuple/test/StreamerBeyond.hxx @@ -0,0 +1,14 @@ +#ifndef ROOT_RNTuple_Test_StreamerBeyond +#define ROOT_RNTuple_Test_StreamerBeyond + +#include + +#include +#include + +struct StreamerBeyond { + std::vector fOne; + std::vector fTwo; +}; + +#endif diff --git a/tree/ntuple/test/StreamerBeyondLinkDef.h b/tree/ntuple/test/StreamerBeyondLinkDef.h new file mode 100644 index 0000000000000..605c9f1a3bc66 --- /dev/null +++ b/tree/ntuple/test/StreamerBeyondLinkDef.h @@ -0,0 +1,5 @@ +#ifdef __CLING__ + +#pragma link C++ options=rntupleStreamerMode(true) struct StreamerBeyond+; + +#endif diff --git a/tree/ntuple/test/rfield_streamer_beyond.cxx b/tree/ntuple/test/rfield_streamer_beyond.cxx new file mode 100644 index 0000000000000..a655aab12a7aa --- /dev/null +++ b/tree/ntuple/test/rfield_streamer_beyond.cxx @@ -0,0 +1,70 @@ +#include +#include +#include +#include + +#include +#include +#include + +#include "StreamerBeyond.hxx" +#include "gtest/gtest.h" + +namespace { + +class FileRaii { +private: + std::string fPath; + bool fPreserveFile = false; + +public: + explicit FileRaii(const std::string &path) : fPath(path) {} + FileRaii(FileRaii &&) = default; + FileRaii(const FileRaii &) = delete; + FileRaii &operator=(FileRaii &&) = default; + FileRaii &operator=(const FileRaii &) = delete; + ~FileRaii() + { + if (!fPreserveFile) + std::remove(fPath.c_str()); + } + std::string GetPath() const { return fPath; } + + // Useful if you want to keep a test file after the test has finished running + // for debugging purposes. Should only be used locally and never pushed. + void PreserveFile() { fPreserveFile = true; } +}; + +} // anonymous namespace + +TEST(RField, StreamerBeyond) +{ + FileRaii fileGuard("test_ntuple_rfield_streamer_beyond.root"); + + { + auto model = ROOT::RNTupleModel::Create(); + auto f = ROOT::RFieldBase::Create("f", "StreamerBeyond").Unwrap(); + EXPECT_TRUE(dynamic_cast(f.get())); + model->AddField(std::move(f)); + auto writer = ROOT::RNTupleWriter::Recreate(std::move(model), "ntpl", fileGuard.GetPath()); + + auto ptr = writer->GetModel().GetDefaultEntry().GetPtr("f"); + ptr->fOne = std::vector(100000000, -1); + ptr->fTwo = std::vector(100000000, -2); + + writer->Fill(); + } + + auto reader = ROOT::RNTupleReader::Open("ntpl", fileGuard.GetPath()); + ASSERT_EQ(1u, reader->GetNEntries()); + StreamerBeyond sb; + auto view = reader->GetView("f", &sb, "StreamerBeyond"); + + view(0); + + auto ptr = view.GetValue().GetPtr(); + EXPECT_EQ(100000000u, ptr->fOne.size()); + EXPECT_EQ(-1, ptr->fOne.at(1000)); + EXPECT_EQ(100000000u, ptr->fTwo.size()); + EXPECT_EQ(-2, ptr->fTwo.at(2000)); +}