Switch to BufferedWriter for write_rds() and related functions.

This eliminates the need for a separate buffer argument. We also allow users to
control the buffer size and whether or not the write is parallelized.

Author: LTLA

examples/src/write_dgCMatrix.cpp

@@ -47,6 +47,6 @@ int main () {
     auto factors = new rds2cpp::GenericVector;
     obj.attributes.values.emplace_back(factors);
 
-    rds2cpp::write_rds(file_info, "my_matrix.rds");
+    rds2cpp::write_rds(file_info, "my_matrix.rds", {});
     return 0;
 }

include/rds2cpp/SharedParseInfo.hpp

@@ -29,8 +29,7 @@ struct SharedParseInfo {
 
 private:
     std::size_t compute_reference_index(const Header& header) const {
-        // Shouldn't matter that we use a signed integer here,
-        // as the left-shifts should never get to the sign bit.
+        // Shouldn't matter that we use a signed integer here, as the left-shifts should never get to the sign bit.
         std::int32_t index = 0;
         for (int i = 0; i < 3; ++i) {
             index <<= 8;

include/rds2cpp/SharedWriteInfo.hpp

@@ -19,8 +19,8 @@ namespace rds2cpp {
 
 struct SharedWriteInfo;
 
-template<class Writer>
-void write_object(const RObject* object, Writer& writer, std::vector<unsigned char>& buffer, SharedWriteInfo& shared);
+template<class BufferedWriter_>
+void write_object(const RObject* object, BufferedWriter_& bufwriter, SharedWriteInfo& shared);
 
 struct SharedWriteInfo {
     std::size_t reference_count;
@@ -52,34 +52,32 @@ struct SharedWriteInfo {
     {}
 
 private:
-    template<class Writer>
-    static void write_reference(std::size_t ref, Writer& writer, std::vector<unsigned char>& buffer) {
-        buffer.resize(4);
-        buffer[2] = ref & 255;
-        ref >>= 8;
-        buffer[1] = ref & 255;
-        ref >>= 8;
-        buffer[0] = ref & 255;
-        buffer[3] = static_cast<unsigned char>(SEXPType::REF);
-        writer.write(buffer.data(), buffer.size());
-        return;
+    template<class BufferedWriter_>
+    static void write_reference(std::size_t ref, BufferedWriter_& bufwriter) {
+        Header details;
+
+        // Opposite of SharedParseInfo::compute_reference_index.
+        for (int i = 0; i < 3; ++i) {
+            details[2 - i] = ref & 255;
+            ref >>= 8;
+        }
+
+        details[3] = static_cast<unsigned char>(SEXPType::REF);
+        bufwriter.write(details.data(), details.size());
     }
 
 public:
-    template<class Writer>
-    std::size_t write_symbol(const std::string& value, StringEncoding encoding, Writer& writer, std::vector<unsigned char>& buffer) {
+    template<class BufferedWriter_>
+    std::size_t write_symbol(const std::string& value, StringEncoding encoding, BufferedWriter_& bufwriter) {
         auto& host = symbol_mappings[static_cast<int>(encoding)];
         auto it = host.find(value);
         if (it != host.end()) {
-            write_reference(it->second, writer, buffer);
+            write_reference(it->second, bufwriter);
             return it->second;
         }
 
-        buffer.clear();
-        inject_header(SEXPType::SYM, buffer);
-        writer.write(buffer.data(), buffer.size());
-
-        write_single_string(value, encoding, false, writer, buffer);
+        inject_header(SEXPType::SYM, bufwriter);
+        write_single_string(value, encoding, false, bufwriter);
 
         const auto old_reference_count = reference_count;
         host[value] = reference_count;
@@ -88,26 +86,26 @@ struct SharedWriteInfo {
         return old_reference_count;
     }
 
-    template<class Writer>
-    void write_symbol(const RObject* obj, Writer& writer, std::vector<unsigned char>& buffer) {
+    template<class BufferedWriter_>
+    void write_symbol(const RObject* obj, BufferedWriter_& bufwriter) {
         auto ptr = static_cast<const SymbolIndex*>(obj);
-        auto index = ptr->index;
+        const auto index = ptr->index;
         if (index >= known_symbol_mappings.size()) {
             throw std::runtime_error("symbol index out of range for supplied Symbol objects");
         }
 
         auto& candidate = known_symbol_mappings[index];
         if (candidate != 0) {
-            write_reference(candidate, writer, buffer);
+            write_reference(candidate, bufwriter);
         } else {
             const auto& sym = (*known_symbols)[index];
-            candidate = write_symbol(sym.name, sym.encoding, writer, buffer);
+            candidate = write_symbol(sym.name, sym.encoding, bufwriter);
         }
     }
 
 public:
-    template<class Writer>
-    void write_external_pointer(const RObject* obj, Writer& writer, std::vector<unsigned char>& buffer) {
+    template<class BufferedWriter_>
+    void write_external_pointer(const RObject* obj, BufferedWriter_& bufwriter) {
         auto ptr = static_cast<const ExternalPointerIndex*>(obj);
         auto index = ptr->index;
         if (index >= known_external_pointer_mappings.size()) {
@@ -116,36 +114,28 @@ struct SharedWriteInfo {
 
         auto& candidate = known_external_pointer_mappings[index];
         if (candidate != 0) {
-            write_reference(candidate, writer, buffer);
+            write_reference(candidate, bufwriter);
             return;
         }
         candidate = reference_count;
         reference_count = sanisizer::sum<I<decltype(reference_count)> >(reference_count, 1); // safely incrementing this count.
 
         const auto& ext = (*known_external_pointers)[index];
-
-        buffer.clear();
-        inject_header(SEXPType::EXTPTR, ext.attributes, buffer);
-        writer.write(buffer.data(), buffer.size());
-
-        write_object(ext.protection.get(), writer, buffer, *this);
-        write_object(ext.tag.get(), writer, buffer, *this);
-        write_attributes(ext.attributes, writer, buffer, *this);
-
-        return;
+        inject_header(SEXPType::EXTPTR, ext.attributes, bufwriter);
+        write_object(ext.protection.get(), bufwriter, *this);
+        write_object(ext.tag.get(), bufwriter, *this);
+        write_attributes(ext.attributes, bufwriter, *this);
     }
 
 public:
-    template<class Writer>
-    void write_environment(const RObject* obj, Writer& writer, std::vector<unsigned char>& buffer) {
+    template<class BufferedWriter_>
+    void write_environment(const RObject* obj, BufferedWriter_& bufwriter) {
         auto ptr = static_cast<const EnvironmentIndex*>(obj);
         auto index = ptr->index;
         auto env_type = ptr->env_type;
 
         if (env_type == SEXPType::GLOBALENV_ || env_type == SEXPType::BASEENV_ || env_type == SEXPType::EMPTYENV_) {
-            buffer.clear();
-            inject_header(env_type, buffer);
-            writer.write(buffer.data(), buffer.size());
+            inject_header(env_type, bufwriter);
             return;
         }
 
@@ -155,25 +145,20 @@ struct SharedWriteInfo {
 
         auto& candidate = known_environment_mappings[index];
         if (candidate != 0) {
-            write_reference(candidate, writer, buffer);
+            write_reference(candidate, bufwriter);
             return;
         }
-
         candidate = reference_count++;
-        const auto& env = (*known_environments)[index];
 
-        buffer.clear();
-        inject_header(SEXPType::ENV, env.attributes, buffer);
-
-        buffer.insert(buffer.end(), 3, 0);
-        buffer.push_back(env.locked);
-        writer.write(buffer.data(), buffer.size());
+        const auto& env = (*known_environments)[index];
+        inject_header(SEXPType::ENV, env.attributes, bufwriter);
+        inject_integer<std::int32_t, std::int32_t>(env.locked, bufwriter);
 
         {
             EnvironmentIndex parent;
             parent.index = env.parent;
             parent.env_type = env.parent_type;
-            write_environment(&parent, writer, buffer);
+            write_environment(&parent, bufwriter);
         }
 
         const auto& names = env.variable_names;
@@ -184,31 +169,22 @@ struct SharedWriteInfo {
         if (len) {
             // Creating a tagged pairlist per element.
             for (I<decltype(len)> i = 0; i < len; ++i) {
-                buffer.clear();
-                inject_next_pairlist_header(true, buffer);
-                writer.write(buffer.data(), buffer.size());
-
-                write_symbol(names[i], encodings[i], writer, buffer);
-                write_object(values[i].get(), writer, buffer, *this);
+                inject_next_pairlist_header(true, bufwriter);
+                write_symbol(names[i], encodings[i], bufwriter);
+                write_object(values[i].get(), bufwriter, *this);
             }
         }
 
         // Terminating the pairlist.
-        buffer.clear();
-        inject_header(SEXPType::NILVALUE_, buffer);
-        writer.write(buffer.data(), buffer.size());
+        inject_header(SEXPType::NILVALUE_, bufwriter);
 
         // We're not saving a hash table, because I don't want to have to
         // reproduce R's environment hashing logic.
-        buffer.clear();
-        inject_header(SEXPType::NILVALUE_, buffer);
-        writer.write(buffer.data(), buffer.size());
+        inject_header(SEXPType::NILVALUE_, bufwriter);
 
-        if (!write_attributes(env.attributes, writer, buffer, *this)) {
+        if (!write_attributes(env.attributes, bufwriter, *this)) {
             // Finishing with NULL if there aren't any attributes.
-            buffer.clear();
-            inject_header(SEXPType::NILVALUE_, buffer);
-            writer.write(buffer.data(), buffer.size());
+            inject_header(SEXPType::NILVALUE_, bufwriter);
         }
     }
 };

include/rds2cpp/utils_write.hpp

@@ -5,42 +5,33 @@
 #include <vector>
 #include <cstdint>
 #include <cstddef>
+#include <type_traits>
 
 #include "utils_parse.hpp"
 
 namespace rds2cpp {
 
-inline void inject_integer(std::int32_t value, std::vector<unsigned char>& buffer) {
+template<typename Target_, typename Type_, class BufferedWriter_>
+void inject_integer(Type_ value, BufferedWriter_& bufwriter) {
+    static_assert(std::is_same<Target_, Type_>::value); // force users to set the type to avoid implicit typing.
     auto ptr = reinterpret_cast<unsigned char*>(&value);
-    constexpr std::size_t width = 4;
+    constexpr std::size_t width = sizeof(Target_);
     if (little_endian()) {
         std::reverse(ptr, ptr + width);
     }
-    buffer.insert(buffer.end(), ptr, ptr + width);
+    bufwriter.write(ptr, width);
 }
 
-inline void inject_length(std::size_t value, std::vector<unsigned char>& buffer) {
+template<class BufferedWriter_>
+void inject_length(std::size_t value, BufferedWriter_& bufwriter) {
     if (value <= 2147483647) {
-        inject_integer(value, buffer);
-        return;
-    }
-
-    inject_integer(-1, buffer);
-    uint64_t big = value;
-
-    auto ptr = reinterpret_cast<unsigned char*>(&big);
-    constexpr std::size_t width = 8;
-    if (little_endian()) {
-        std::reverse(ptr, ptr + width/2);
-        std::reverse(ptr + width/2, ptr + width);
+        inject_integer<std::int32_t, std::int32_t>(value, bufwriter);
+    } else {
+        // See get_length() for the inverse logic.
+        inject_integer<std::int32_t, std::int32_t>(-1, bufwriter);
+        inject_integer<std::uint32_t>(sanisizer::cast<std::uint32_t>(value >> 32), bufwriter);
+        inject_integer<std::uint32_t>(static_cast<std::uint32_t>(value & 0xFFFFFFFF), bufwriter); // must fit in a uint32 as we're taking the lowest 32 bits.
     }
-
-    buffer.insert(buffer.end(), ptr, ptr + width);
-}
-
-inline void inject_string(const char* ptr, std::size_t n, std::vector<unsigned char>& buffer) {
-    auto p = reinterpret_cast<const unsigned char*>(ptr);
-    buffer.insert(buffer.end(), p, p + n);
 }
 
 template<class Object, typename = int>
@@ -70,40 +61,50 @@ inline unsigned char inject_attribute_header(Attributes& attributes) {
     return bit;
 }
 
-template<class Object>
-void inject_header(Object& vec, std::vector<unsigned char>& buffer) {
-    buffer.insert(buffer.end(), 2, 0);
+template<class Object_, class BufferedWriter_>
+void inject_header(Object_& vec, BufferedWriter_& bufwriter) {
+    Header details;
+    details[0] = 0;
+    details[1] = 0;
 
-    if constexpr(has_attributes_for_writing<Object>::value) {
-        buffer.push_back(inject_attribute_header(vec.attributes));
+    if constexpr(has_attributes_for_writing<Object_>::value) {
+        details[2] = inject_attribute_header(vec.attributes);
     } else {
-        buffer.push_back(0);
+        details[2] = 0;
     }
 
-    // cast from enum should be safe, as SEXPTypes are also unsigned chars.
-    buffer.push_back(static_cast<unsigned char>(vec.type())); 
-    return;
+    details[3] = static_cast<unsigned char>(vec.type()); // Cast from enum should be safe, as SEXPTypes are also unsigned chars.
+    bufwriter.write(details.data(), details.size());
 }
 
-inline void inject_header(SEXPType type, std::vector<unsigned char>& buffer) {
-    buffer.insert(buffer.end(), 3, 0);
-    buffer.push_back(static_cast<unsigned char>(type));
-    return;
+template<class BufferedWriter_>
+void inject_header(SEXPType type, BufferedWriter_& bufwriter) {
+    Header details;
+    details[0] = 0;
+    details[1] = 0;
+    details[2] = 0;
+    details[3] = static_cast<unsigned char>(type);
+    bufwriter.write(details.data(), details.size());
 }
 
-template<class Attributes>
-void inject_header(SEXPType type, Attributes& attributes, std::vector<unsigned char>& buffer) {
-    buffer.insert(buffer.end(), 2, 0);
-    buffer.push_back(inject_attribute_header(attributes));
-    buffer.push_back(static_cast<unsigned char>(type));
-    return;
+template<class Attributes_, class BufferedWriter_>
+void inject_header(SEXPType type, Attributes_& attributes, BufferedWriter_& bufwriter) {
+    Header details;
+    details[0] = 0;
+    details[1] = 0;
+    details[2] = inject_attribute_header(attributes);
+    details[3] = static_cast<unsigned char>(type);
+    bufwriter.write(details.data(), details.size());
 }
 
-inline void inject_next_pairlist_header(bool tagged, std::vector<unsigned char>& buffer) {
-    buffer.insert(buffer.end(), 2, 0);
-    buffer.push_back(tagged ? 0x4 : 0); // has tag.
-    buffer.push_back(static_cast<unsigned char>(SEXPType::LIST));
-    return;
+template<class BufferedWriter_>
+void inject_next_pairlist_header(bool tagged, BufferedWriter_& bufwriter) {
+    Header details;
+    details[0] = 0;
+    details[1] = 0;
+    details[2] = (tagged ? 0x4 : 0); // has tag.
+    details[3] = static_cast<unsigned char>(SEXPType::LIST);
+    bufwriter.write(details.data(), details.size());
 }
 
 }