support write parallel.

Author: ColinLeeo

cpp/CMakeLists.txt

@@ -150,6 +150,13 @@ if (ENABLE_ZLIB)
     add_definitions(-DENABLE_GZIP)
 endif()
 
+option(ENABLE_THREADS "Enable multi-threaded read/write (requires pthreads)" ON)
+message("cmake using: ENABLE_THREADS=${ENABLE_THREADS}")
+
+if (ENABLE_THREADS)
+    add_definitions(-DENABLE_THREADS)
+endif()
+
 option(ENABLE_SIMDE "Enable SIMDe (SIMD Everywhere)" OFF)
 message("cmake using: ENABLE_SIMDE=${ENABLE_SIMDE}")
 

cpp/src/common/config/config.h

@@ -46,6 +46,8 @@ typedef struct ConfigValue {
     TSEncoding double_encoding_type_;
     TSEncoding string_encoding_type_;
     CompressionType default_compression_type_;
+    bool parallel_write_enabled_;
+    int32_t write_thread_count_;
     // When true, aligned writer enforces page size limit strictly by
     // interleaving time/value writes and sealing pages together when any side
     // becomes full.

cpp/src/common/global.cc

@@ -54,12 +54,15 @@ void init_config_value() {
     g_config_value_.int64_encoding_type_ = TS_2DIFF;
     g_config_value_.float_encoding_type_ = GORILLA;
     g_config_value_.double_encoding_type_ = GORILLA;
+    g_config_value_.string_encoding_type_ = PLAIN;
     // Default compression type is LZ4
 #ifdef ENABLE_LZ4
     g_config_value_.default_compression_type_ = LZ4;
 #else
     g_config_value_.default_compression_type_ = UNCOMPRESSED;
 #endif
+    g_config_value_.parallel_write_enabled_ = true;
+    g_config_value_.write_thread_count_ = 6;
     // Enforce aligned page size limits strictly by default.
     g_config_value_.strict_page_size_ = true;
 }

cpp/src/common/global.h

@@ -163,6 +163,20 @@ FORCE_INLINE uint8_t get_global_compression() {
     return static_cast<uint8_t>(g_config_value_.default_compression_type_);
 }
 
+FORCE_INLINE void set_parallel_write_enabled(bool enabled) {
+    g_config_value_.parallel_write_enabled_ = enabled;
+}
+
+FORCE_INLINE bool get_parallel_write_enabled() {
+    return g_config_value_.parallel_write_enabled_;
+}
+
+FORCE_INLINE int set_write_thread_count(int32_t count) {
+    if (count < 1 || count > 64) return E_INVALID_ARG;
+    g_config_value_.write_thread_count_ = count;
+    return E_OK;
+}
+
 extern int init_common();
 extern bool is_timestamp_column_name(const char* time_col_name);
 extern void cols_to_json(ByteStream* byte_stream,

cpp/src/common/tablet.cc

@@ -19,7 +19,9 @@
 
 #include "tablet.h"
 
+#include <algorithm>
 #include <cstdlib>
+#include <numeric>
 
 #include "allocator/alloc_base.h"
 #include "datatype/date_converter.h"
@@ -491,6 +493,135 @@ void Tablet::set_column_categories(
     }
 }
 
+namespace {
+
+template <typename T>
+void permute_array(T* arr, const std::vector<uint32_t>& perm, uint32_t n) {
+    std::vector<T> tmp(n);
+    for (uint32_t i = 0; i < n; i++) tmp[i] = arr[perm[i]];
+    std::copy(tmp.begin(), tmp.end(), arr);
+}
+
+void permute_string_column(Tablet::StringColumn* sc, BitMap& bm,
+                           const std::vector<uint32_t>& perm, uint32_t n,
+                           uint32_t max_rows) {
+    Tablet::StringColumn tmp;
+    tmp.init(max_rows, sc->buf_used > 0 ? sc->buf_used : 64);
+    for (uint32_t i = 0; i < n; i++) {
+        uint32_t r = perm[i];
+        if (bm.test(r)) {
+            // Null row — write a zero-length entry to keep offsets valid.
+            tmp.append(i, "", 0);
+        } else {
+            int32_t off = sc->offsets[r];
+            uint32_t len =
+                static_cast<uint32_t>(sc->offsets[r + 1] - off);
+            tmp.append(i, sc->buffer + off, len);
+        }
+    }
+    // Swap contents.
+    std::swap(sc->offsets, tmp.offsets);
+    std::swap(sc->buffer, tmp.buffer);
+    std::swap(sc->buf_capacity, tmp.buf_capacity);
+    std::swap(sc->buf_used, tmp.buf_used);
+    tmp.destroy();
+}
+
+void permute_bitmap(BitMap& bm, const std::vector<uint32_t>& perm,
+                    uint32_t n) {
+    if (!bm.get_bitmap()) return;
+    uint32_t size_bytes = bm.get_size();
+    // Save original bits.
+    std::vector<char> orig(bm.get_bitmap(), bm.get_bitmap() + size_bytes);
+    // Clear all bits (= all non-null).
+    bm.clear_all();
+    // Re-set null bits through the permutation.
+    for (uint32_t i = 0; i < n; i++) {
+        uint32_t src = perm[i];
+        if (orig[src >> 3] & (1 << (src & 7))) {
+            bm.set(i);
+        }
+    }
+}
+
+}  // anonymous namespace
+
+void Tablet::sort_by_device() {
+    if (id_column_indexes_.empty() || cur_row_size_ <= 1) return;
+
+    const uint32_t n = cur_row_size_;
+
+    // Build permutation sorted by tag column values (stable sort keeps
+    // timestamp order within each device).
+    std::vector<uint32_t> perm(n);
+    std::iota(perm.begin(), perm.end(), 0);
+
+    std::stable_sort(perm.begin(), perm.end(), [this](uint32_t a, uint32_t b) {
+        for (int idx : id_column_indexes_) {
+            bool a_null = bitmaps_[idx].test(a);
+            bool b_null = bitmaps_[idx].test(b);
+            if (a_null != b_null) return a_null > b_null;  // null sorts last
+            if (a_null) continue;  // both null — equal on this column
+            const StringColumn& sc = *value_matrix_[idx].string_col;
+            int32_t a_off = sc.offsets[a];
+            uint32_t a_len = static_cast<uint32_t>(sc.offsets[a + 1] - a_off);
+            int32_t b_off = sc.offsets[b];
+            uint32_t b_len = static_cast<uint32_t>(sc.offsets[b + 1] - b_off);
+            uint32_t min_len = std::min(a_len, b_len);
+            int cmp = (min_len > 0)
+                          ? memcmp(sc.buffer + a_off, sc.buffer + b_off, min_len)
+                          : 0;
+            if (cmp != 0) return cmp < 0;
+            if (a_len != b_len) return a_len < b_len;
+        }
+        return false;
+    });
+
+    // Check if already sorted.
+    bool sorted = true;
+    for (uint32_t i = 0; i < n && sorted; i++) {
+        if (perm[i] != i) sorted = false;
+    }
+    if (sorted) return;
+
+    // Apply permutation to timestamps.
+    permute_array(timestamps_, perm, n);
+
+    // Apply permutation to each column.
+    uint32_t col_count = static_cast<uint32_t>(schema_vec_->size());
+    for (uint32_t c = 0; c < col_count; c++) {
+        TSDataType dt = schema_vec_->at(c).data_type_;
+        switch (dt) {
+            case BOOLEAN:
+                permute_array(value_matrix_[c].bool_data, perm, n);
+                break;
+            case INT32:
+            case DATE:
+                permute_array(value_matrix_[c].int32_data, perm, n);
+                break;
+            case INT64:
+            case TIMESTAMP:
+                permute_array(value_matrix_[c].int64_data, perm, n);
+                break;
+            case FLOAT:
+                permute_array(value_matrix_[c].float_data, perm, n);
+                break;
+            case DOUBLE:
+                permute_array(value_matrix_[c].double_data, perm, n);
+                break;
+            case STRING:
+            case TEXT:
+            case BLOB:
+                permute_string_column(value_matrix_[c].string_col, bitmaps_[c],
+                                      perm, n, max_row_num_);
+                break;
+            default:
+                break;
+        }
+        permute_bitmap(bitmaps_[c], perm, n);
+    }
+}
+
 void Tablet::reset_string_columns() {
     size_t schema_count = schema_vec_->size();
     for (size_t c = 0; c < schema_count; c++) {

cpp/src/common/tablet.h

@@ -46,6 +46,7 @@ class TabletColIterator;
  * with their associated metadata such as column names and types.
  */
 class Tablet {
+   public:
     // Arrow-style string column: offsets + contiguous buffer.
     // string[i] = buffer + offsets[i], len = offsets[i+1] - offsets[i]
     struct StringColumn {
@@ -284,6 +285,10 @@ class Tablet {
 
     void set_column_categories(
         const std::vector<common::ColumnCategory>& column_categories);
+    // Sort rows so that rows belonging to the same device (same TAG column
+    // values) are contiguous.  Stable sort: preserves timestamp order within
+    // each device.  No-op when there are no TAG columns or ≤1 rows.
+    void sort_by_device();
     std::shared_ptr<IDeviceID> get_device_id(int i) const;
     std::vector<uint32_t> find_all_device_boundaries() const;
 

cpp/src/common/thread_pool.h

@@ -0,0 +1,122 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * License); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+#ifndef COMMON_THREAD_POOL_H
+#define COMMON_THREAD_POOL_H
+
+#ifdef ENABLE_THREADS
+
+#include <condition_variable>
+#include <functional>
+#include <future>
+#include <mutex>
+#include <queue>
+#include <thread>
+#include <vector>
+
+namespace common {
+
+// Unified fixed-size thread pool supporting both fire-and-forget tasks
+// (submit void + wait_all) and future-returning tasks (submit<F>).
+// Used by both write path (column-parallel encoding) and read path
+// (column-parallel decoding).
+class ThreadPool {
+   public:
+    explicit ThreadPool(size_t num_threads) : stop_(false), active_(0) {
+        for (size_t i = 0; i < num_threads; i++) {
+            workers_.emplace_back([this] { worker_loop(); });
+        }
+    }
+
+    ~ThreadPool() {
+        {
+            std::lock_guard<std::mutex> lk(mu_);
+            stop_ = true;
+        }
+        cv_work_.notify_all();
+        for (auto& w : workers_) {
+            if (w.joinable()) w.join();
+        }
+    }
+
+    // Submit a fire-and-forget task (no return value).
+    void submit(std::function<void()> task) {
+        {
+            std::lock_guard<std::mutex> lk(mu_);
+            tasks_.push(std::move(task));
+            active_++;
+        }
+        cv_work_.notify_one();
+    }
+
+    // Submit a task that returns a value via std::future.
+    template <typename F>
+    std::future<typename std::result_of<F()>::type> submit(F&& f) {
+        using RetType = typename std::result_of<F()>::type;
+        auto task =
+            std::make_shared<std::packaged_task<RetType()>>(std::forward<F>(f));
+        std::future<RetType> result = task->get_future();
+        {
+            std::lock_guard<std::mutex> lk(mu_);
+            tasks_.emplace([task]() { (*task)(); });
+            active_++;
+        }
+        cv_work_.notify_one();
+        return result;
+    }
+
+    // Block until all submitted tasks have completed.
+    void wait_all() {
+        std::unique_lock<std::mutex> lk(mu_);
+        cv_done_.wait(lk, [this] { return active_ == 0 && tasks_.empty(); });
+    }
+
+   private:
+    void worker_loop() {
+        while (true) {
+            std::function<void()> task;
+            {
+                std::unique_lock<std::mutex> lk(mu_);
+                cv_work_.wait(lk, [this] { return stop_ || !tasks_.empty(); });
+                if (stop_ && tasks_.empty()) return;
+                task = std::move(tasks_.front());
+                tasks_.pop();
+            }
+            task();
+            {
+                std::lock_guard<std::mutex> lk(mu_);
+                active_--;
+            }
+            cv_done_.notify_one();
+        }
+    }
+
+    std::vector<std::thread> workers_;
+    std::queue<std::function<void()>> tasks_;
+    std::mutex mu_;
+    std::condition_variable cv_work_;
+    std::condition_variable cv_done_;
+    bool stop_;
+    int active_;
+};
+
+}  // namespace common
+
+#endif  // ENABLE_THREADS
+
+#endif  // COMMON_THREAD_POOL_H