From 2f4036b38009824e38798d6214ce4083a7fc089f Mon Sep 17 00:00:00 2001 From: Mikhail Fomichev Date: Wed, 6 May 2026 19:18:44 +0300 Subject: [PATCH 01/18] huge project redesign --- .gitignore | 1 - src/AggregationFunctions.h | 360 -------- src/CMakeLists.txt | 93 ++- src/Engine.cpp | 15 - src/Engine.h | 19 - src/ExecutionLogic.h | 225 ----- src/FilterFunctions.h | 86 -- src/Query.h | 54 -- src/ScalarExpressions.h | 42 - src/Types.h | 780 ------------------ src/column/CharColumn.h | 68 ++ src/{ => column}/Codec.h | 0 src/column/Column.h | 14 + src/column/ColumnBuilder.h | 13 + src/column/ColumnFactory.h | 34 + src/column/NumericColumn.h | 198 +++++ src/{ => column}/Schema.h | 6 +- src/column/StringColumn.h | 76 ++ src/column/TemporalColumn.h | 82 ++ .../ExecutionApi.h} | 25 +- src/execution/ExecutionHelper.h | 183 ++++ src/execution/ExecutionLogic.h | 294 +++++++ src/{ => execution}/OperatorsBase.cpp | 60 +- src/{ => execution}/OperatorsBase.h | 34 +- .../expressions}/AggregationExpressions.h | 173 +++- .../expressions/AggregationFunctions.h | 339 ++++++++ .../expressions}/FilterExpressions.h | 13 +- src/execution/expressions/FilterFunctions.h | 58 ++ src/execution/expressions/ScalarExpressions.h | 60 ++ src/execution/expressions/ScalarFunctions.h | 34 + src/{read_write => io}/ColumnarReader.cpp | 80 +- src/io/ColumnarReader.h | 29 + src/io/ColumnarWriter.cpp | 143 ++++ src/io/ColumnarWriter.h | 46 ++ src/io/CsvReader.cpp | 60 ++ src/io/CsvReader.h | 20 + src/io/CsvToColumnar.cpp | 93 +++ src/io/CsvToColumnar.h | 11 + .../CSVTokenizer.cpp => io/CsvTokenizer.cpp} | 23 +- .../CSVTokenizer.h => io/CsvTokenizer.h} | 19 +- src/io/test/TestCsvParser.cpp | 138 ++++ src/io/test/TestReaderWriter.cpp | 128 +++ src/main.cpp | 23 +- src/read_write/CSVReader.cpp | 39 - src/read_write/CSVReader.h | 25 - src/read_write/CSVToColumnar.cpp | 138 ---- src/read_write/CSVToColumnar.h | 31 - src/read_write/ColumnarReader.h | 32 - src/read_write/ColumnarWriter.cpp | 88 -- src/read_write/ColumnarWriter.h | 34 - src/read_write/test/test_csv_parser.cpp | 111 --- src/read_write/test/test_reader_writer.cpp | 496 ----------- src/types/ColumnType.h | 48 ++ src/types/Date.h | 53 ++ src/types/Timestamp.h | 69 ++ src/{macro.h => utils/Assert.h} | 37 +- src/utils/Macro.h | 25 + src/{ => utils}/VectorOfStrings.h | 11 +- .../test/TestVectorOfStrings.cpp} | 52 +- 59 files changed, 2699 insertions(+), 2842 deletions(-) delete mode 100644 src/AggregationFunctions.h delete mode 100644 src/Engine.cpp delete mode 100644 src/Engine.h delete mode 100644 src/ExecutionLogic.h delete mode 100644 src/FilterFunctions.h delete mode 100644 src/Query.h delete mode 100644 src/ScalarExpressions.h delete mode 100644 src/Types.h create mode 100644 src/column/CharColumn.h rename src/{ => column}/Codec.h (100%) create mode 100644 src/column/Column.h create mode 100644 src/column/ColumnBuilder.h create mode 100644 src/column/ColumnFactory.h create mode 100644 src/column/NumericColumn.h rename src/{ => column}/Schema.h (97%) create mode 100644 src/column/StringColumn.h create mode 100644 src/column/TemporalColumn.h rename src/{ExecutionAPI.h => execution/ExecutionApi.h} (86%) create mode 100644 src/execution/ExecutionHelper.h create mode 100644 src/execution/ExecutionLogic.h rename src/{ => execution}/OperatorsBase.cpp (83%) rename src/{ => execution}/OperatorsBase.h (76%) rename src/{ => execution/expressions}/AggregationExpressions.h (50%) create mode 100644 src/execution/expressions/AggregationFunctions.h rename src/{ => execution/expressions}/FilterExpressions.h (94%) create mode 100644 src/execution/expressions/FilterFunctions.h create mode 100644 src/execution/expressions/ScalarExpressions.h create mode 100644 src/execution/expressions/ScalarFunctions.h rename src/{read_write => io}/ColumnarReader.cpp (63%) create mode 100644 src/io/ColumnarReader.h create mode 100644 src/io/ColumnarWriter.cpp create mode 100644 src/io/ColumnarWriter.h create mode 100644 src/io/CsvReader.cpp create mode 100644 src/io/CsvReader.h create mode 100644 src/io/CsvToColumnar.cpp create mode 100644 src/io/CsvToColumnar.h rename src/{read_write/CSVTokenizer.cpp => io/CsvTokenizer.cpp} (90%) rename src/{read_write/CSVTokenizer.h => io/CsvTokenizer.h} (65%) create mode 100644 src/io/test/TestCsvParser.cpp create mode 100644 src/io/test/TestReaderWriter.cpp delete mode 100644 src/read_write/CSVReader.cpp delete mode 100644 src/read_write/CSVReader.h delete mode 100644 src/read_write/CSVToColumnar.cpp delete mode 100644 src/read_write/CSVToColumnar.h delete mode 100644 src/read_write/ColumnarReader.h delete mode 100644 src/read_write/ColumnarWriter.cpp delete mode 100644 src/read_write/ColumnarWriter.h delete mode 100644 src/read_write/test/test_csv_parser.cpp delete mode 100644 src/read_write/test/test_reader_writer.cpp create mode 100644 src/types/ColumnType.h create mode 100644 src/types/Date.h create mode 100644 src/types/Timestamp.h rename src/{macro.h => utils/Assert.h} (52%) create mode 100644 src/utils/Macro.h rename src/{ => utils}/VectorOfStrings.h (96%) rename src/{test_vector_of_strings.cpp => utils/test/TestVectorOfStrings.cpp} (76%) diff --git a/.gitignore b/.gitignore index b9ac6b2..16edb04 100644 --- a/.gitignore +++ b/.gitignore @@ -28,7 +28,6 @@ docker-test-results .vscode/ src/project_bundle.txt - # Prerequisites *.d diff --git a/src/AggregationFunctions.h b/src/AggregationFunctions.h deleted file mode 100644 index cce114f..0000000 --- a/src/AggregationFunctions.h +++ /dev/null @@ -1,360 +0,0 @@ -// -// Created by ragnarokk on 30.03.2026. -// - -#ifndef COLUMNAR_ENGINE_AGGREGATIONFUNCTIONS_H -#define COLUMNAR_ENGINE_AGGREGATIONFUNCTIONS_H - -#include "OperatorsBase.h" - -#include -#include -#include - -class AggregationFunction { -public: - virtual ~AggregationFunction() = default; - - virtual void Update(const RecordBatch& batch, - const std::vector* group_ids = nullptr) = 0; - virtual std::shared_ptr Finalize() = 0; -}; - -struct SumOperation { - template - static inline void Apply(ResultType& result, const ValueType& value) { - result += value; - } - - template - static constexpr ResultType GetInitValue() { - return 0; - } -}; - -struct MaxOperation { - template - static inline void Apply(ResultType& result, const ValueType& value) { - if (value > result) { - result = value; - } - } - - template - static constexpr ResultType GetInitValue() { - return std::numeric_limits::lowest(); - } -}; - -struct MinOperation { - template - static inline void Apply(ResultType& result, const ValueType& value) { - if (value < result) { - result = value; - } - } - - template - static constexpr ResultType GetInitValue() { - return std::numeric_limits::max(); - } -}; - -// OPTIMIZE: maybe redundant -struct CountOperation { - template - static inline void Apply(ResultType& result, const ValueType&) { - ++result; - } -}; - -template -struct AggregationFunctionTraits; - -template <> -struct AggregationFunctionTraits { - using ValueType = int16_t; - using StateType = int32_t; - using ResultColumnType = Int32Column; -}; - -template <> -struct AggregationFunctionTraits { - using ValueType = int32_t; - using StateType = int64_t; - using ResultColumnType = Int64Column; -}; - -template <> -struct AggregationFunctionTraits { - using ValueType = int64_t; - using StateType = __int128_t; - using ResultColumnType = Int128Column; -}; - -template <> -struct AggregationFunctionTraits { - using ValueType = __int128_t; - using StateType = __int128_t; - using ResultColumnType = Int128Column; -}; - -template <> -struct AggregationFunctionTraits { - using ValueType = float; - using StateType = double; - using ResultColumnType = DoubleColumn; -}; - -template <> -struct AggregationFunctionTraits { - using ValueType = double; - using StateType = long double; - using ResultColumnType = LongDoubleColumn; -}; - -template <> -struct AggregationFunctionTraits { - using ValueType = long double; - using StateType = long double; - using ResultColumnType = LongDoubleColumn; -}; - -template <> -struct AggregationFunctionTraits { - using ValueType = char; - using StateType = int16_t; - using ResultColumnType = Int16Column; -}; - -template <> -struct AggregationFunctionTraits { - using ValueType = int32_t; - using StateType = int32_t; - using ResultColumnType = DateColumn; -}; - -template <> -struct AggregationFunctionTraits { - using ValueType = int64_t; - using StateType = int64_t; - using ResultColumnType = TimestampColumn; -}; - -template <> -struct AggregationFunctionTraits { - using ValueType = std::string; -}; - -template -class TypedAggregationFunction : public AggregationFunction { - using ResultColumnType = typename AggregationFunctionTraits::ResultColumnType; - using StateType = typename AggregationFunctionTraits::StateType; - -public: - explicit TypedAggregationFunction(size_t column_index) : column_index_(column_index) { - } - - void Update(const RecordBatch& batch, - const std::vector* group_ids = nullptr) override { - auto* column = static_cast(batch.columns[column_index_].get()); - const auto& data = column->GetData(); - - if (!batch.selection_vector) { - for (size_t i = 0; i < batch.num_rows; ++i) { - uint32_t gid = group_ids ? (*group_ids)[i] : 0; - if (gid >= states_.size()) { - states_.resize(gid + 1, Operation::template GetInitValue()); - } - Operation::Apply(states_[gid], data[i]); - } - } else { - for (size_t i = 0; i < batch.num_rows; ++i) { - uint32_t gid = group_ids ? (*group_ids)[i] : 0; - if (gid >= states_.size()) { - states_.resize(gid + 1, Operation::template GetInitValue()); - } - uint32_t ind = (*batch.selection_vector)[i]; - Operation::Apply(states_[gid], data[ind]); - } - } - } - - std::shared_ptr Finalize() override { - auto result_column = std::make_shared(); - if (states_.empty()) { - result_column->Add(Operation::template GetInitValue()); - } else { - for (const auto& state : states_) { - result_column->Add(state); - } - } - return result_column; - } - -private: - size_t column_index_; - std::vector states_; -}; - -template -using SumAggregationFunction = TypedAggregationFunction; - -template -using MinAggregationFunction = TypedAggregationFunction; - -template -using MaxAggregationFunction = TypedAggregationFunction; - -class CountAggregationFunction : public AggregationFunction { -public: - void Update(const RecordBatch& batch, - const std::vector* group_ids = nullptr) override { - for (size_t i = 0; i < batch.num_rows; ++i) { - uint32_t gid = group_ids ? (*group_ids)[i] : 0; - if (gid >= counts_.size()) { - counts_.resize(gid + 1, 0); - } - counts_[gid]++; - } - } - - std::shared_ptr Finalize() override { - auto result_column = std::make_shared(); - if (counts_.empty()) { - result_column->Add(0); - } else { - for (int64_t c : counts_) { - result_column->Add(static_cast(c)); - } - } - return result_column; - } - -private: - std::vector counts_; -}; - -template -class DistinctCountAggregationFunction : public AggregationFunction { - using ResultColumnType = Int64Column; - using ValueType = typename AggregationFunctionTraits::ValueType; - -public: - explicit DistinctCountAggregationFunction(size_t column_index) : column_index_(column_index) { - } - - void Update(const RecordBatch& batch, - const std::vector* group_ids = nullptr) override { - auto* column = static_cast(batch.columns[column_index_].get()); - const auto& data = column->GetData(); - - if (!batch.selection_vector) { - for (size_t i = 0; i < batch.num_rows; ++i) { - uint32_t gid = group_ids ? (*group_ids)[i] : 0; - if (gid >= distinct_values_.size()) { - distinct_values_.resize(gid + 1); - } - if constexpr (std::is_same_v) { - distinct_values_[gid].emplace(data[i]); - } else { - distinct_values_[gid].insert(data[i]); - } - } - } else { - for (size_t i = 0; i < batch.num_rows; ++i) { - uint32_t gid = group_ids ? (*group_ids)[i] : 0; - if (gid >= distinct_values_.size()) { - distinct_values_.resize(gid + 1); - } - uint32_t ind = (*batch.selection_vector)[i]; - if constexpr (std::is_same_v) { - distinct_values_[gid].emplace(data[ind]); - } else { - distinct_values_[gid].insert(data[ind]); - } - } - } - } - - std::shared_ptr Finalize() override { - auto result_column = std::make_shared(); - if (distinct_values_.empty()) { - result_column->Add(0); - } else { - for (const auto& set : distinct_values_) { - result_column->Add(static_cast(set.size())); - } - } - return result_column; - } - -private: - size_t column_index_; - std::vector> distinct_values_; -}; - -template -class AvgAggregationFunction : public AggregationFunction { -public: - using StateType = typename AggregationFunctionTraits::StateType; - - explicit AvgAggregationFunction(size_t column_index) : column_index_(column_index) { - } - - void Update(const RecordBatch& batch, - const std::vector* group_ids = nullptr) override { - auto* column = static_cast(batch.columns[column_index_].get()); - const auto& data = column->GetData(); - - if (!batch.selection_vector) { - for (size_t i = 0; i < batch.num_rows; ++i) { - uint32_t gid = group_ids ? (*group_ids)[i] : 0; - if (gid >= sums_.size()) { - sums_.resize(gid + 1, 0); - counts_.resize(gid + 1, 0); - } - sums_[gid] += static_cast(data[i]); - counts_[gid]++; - } - } else { - for (size_t i = 0; i < batch.num_rows; ++i) { - uint32_t gid = group_ids ? (*group_ids)[i] : 0; - if (gid >= sums_.size()) { - sums_.resize(gid + 1, 0); - counts_.resize(gid + 1, 0); - } - uint32_t ind = (*batch.selection_vector)[i]; - sums_[gid] += static_cast(data[ind]); - counts_[gid]++; - } - } - } - - std::shared_ptr Finalize() override { - auto result_column = std::make_shared(); - if (sums_.empty()) { - result_column->Add(0.0); - } else { - for (size_t gid = 0; gid < sums_.size(); ++gid) { - if (counts_[gid] == 0) { - result_column->Add(0.0); - } else { - StateType quotient = sums_[gid] / counts_[gid]; - long double rem = - static_cast(sums_[gid] - quotient * counts_[gid]) / - counts_[gid]; - result_column->Add(static_cast(quotient) + rem); - } - } - } - return result_column; - } - -private: - size_t column_index_; - std::vector sums_; - std::vector counts_; -}; - -#endif // COLUMNAR_ENGINE_AGGREGATIONFUNCTIONS_H diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 186097d..75a44f9 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -1,47 +1,62 @@ add_library(project_includes INTERFACE) target_include_directories(project_includes INTERFACE ${CMAKE_CURRENT_SOURCE_DIR} - read_write/test - ${CMAKE_CURRENT_SOURCE_DIR}/read_write + ${CMAKE_CURRENT_SOURCE_DIR}/column + ${CMAKE_CURRENT_SOURCE_DIR}/execution + ${CMAKE_CURRENT_SOURCE_DIR}/execution/expressions + ${CMAKE_CURRENT_SOURCE_DIR}/io + ${CMAKE_CURRENT_SOURCE_DIR}/io/test + ${CMAKE_CURRENT_SOURCE_DIR}/types + ${CMAKE_CURRENT_SOURCE_DIR}/utils + ${CMAKE_CURRENT_SOURCE_DIR}/utils/test ) add_executable(columnar-engine main.cpp - read_write/CSVTokenizer.cpp - read_write/CSVTokenizer.h - read_write/ColumnarReader.cpp - read_write/ColumnarReader.h - Types.h - read_write/CSVReader.cpp - read_write/CSVReader.h - read_write/ColumnarWriter.cpp - read_write/ColumnarWriter.h - read_write/CSVToColumnar.cpp - read_write/CSVToColumnar.h - OperatorsBase.h - Engine.cpp - Engine.h - Codec.h - Query.h - ExecutionLogic.h - AggregationFunctions.h - Schema.h - macro.h - ExecutionAPI.h - AggregationExpressions.h - OperatorsBase.cpp - VectorOfStrings.h - FilterExpressions.h - FilterFunctions.h + io/CsvTokenizer.cpp + io/CsvTokenizer.h + io/ColumnarReader.cpp + io/ColumnarReader.h + io/CsvReader.cpp + io/CsvReader.h + io/ColumnarWriter.cpp + io/ColumnarWriter.h + io/CsvToColumnar.cpp + io/CsvToColumnar.h + execution/OperatorsBase.h + column/Codec.h + execution/ExecutionLogic.h + execution/expressions/AggregationFunctions.h + column/Schema.h + utils/Macro.h + execution/ExecutionApi.h + execution/expressions/AggregationExpressions.h + execution/OperatorsBase.cpp + utils/VectorOfStrings.h + execution/expressions/FilterExpressions.h + execution/expressions/FilterFunctions.h + execution/expressions/ScalarFunctions.h + execution/expressions/ScalarExpressions.h + execution/ExecutionHelper.h + column/ColumnFactory.h + column/CharColumn.h + column/NumericColumn.h + column/StringColumn.h + column/TemporalColumn.h + column/Column.h + column/ColumnBuilder.h + types/ColumnType.h + types/Date.h + types/Timestamp.h ) target_link_libraries(columnar-engine PRIVATE project_includes) target_compile_options(columnar-engine PRIVATE -Werror) add_executable(test_csv_parser - read_write/test/test_csv_parser.cpp - read_write/CSVTokenizer.cpp - read_write/CSVReader.cpp + io/test/TestCsvParser.cpp + io/CsvTokenizer.cpp + io/CsvReader.cpp ) target_link_libraries(test_csv_parser PRIVATE @@ -49,12 +64,12 @@ target_link_libraries(test_csv_parser PRIVATE project_includes) add_executable(test_reader_writer - read_write/test/test_reader_writer.cpp - read_write/CSVTokenizer.cpp - read_write/ColumnarReader.cpp - read_write/CSVReader.cpp - read_write/ColumnarWriter.cpp - read_write/CSVToColumnar.cpp + io/test/TestReaderWriter.cpp + io/CsvTokenizer.cpp + io/ColumnarReader.cpp + io/CsvReader.cpp + io/ColumnarWriter.cpp + io/CsvToColumnar.cpp ) target_link_libraries(test_reader_writer PRIVATE @@ -62,8 +77,8 @@ target_link_libraries(test_reader_writer PRIVATE project_includes) add_executable(test_vector_of_strings - test_vector_of_strings.cpp - VectorOfStrings.h + utils/test/TestVectorOfStrings.cpp + utils/VectorOfStrings.h ) target_link_libraries(test_vector_of_strings PRIVATE diff --git a/src/Engine.cpp b/src/Engine.cpp deleted file mode 100644 index 3fcab76..0000000 --- a/src/Engine.cpp +++ /dev/null @@ -1,15 +0,0 @@ -// -// Created by ragnarokk on 06.01.2026. -// - -#include - -#include "read_write/CSVToColumnar.h" -#include "Engine.h" - -void Engine::InitDataFromCSV(const std::string& csv_file_path, const std::string& columnar_file_path) { - CSVToColumnar converter; - converter.Convert(csv_file_path, columnar_file_path); -} - -// void Engine::ExecuteQuery(const std::string& columnar_file_path) {} diff --git a/src/Engine.h b/src/Engine.h deleted file mode 100644 index c34fe51..0000000 --- a/src/Engine.h +++ /dev/null @@ -1,19 +0,0 @@ -// -// Created by ragnarokk on 06.01.2026. -// - -// TODO: after SQL parser do the interaction here - -#ifndef COLUMNAR_ENGINE_ENGINE_H -#define COLUMNAR_ENGINE_ENGINE_H - -#include - -class Engine { -public: - void InitDataFromCSV(const std::string& csv_file_path, const std::string& columnar_file_path); - - void ExecuteQuery(const std::string& columnar_file_path); -}; - -#endif // COLUMNAR_ENGINE_ENGINE_H diff --git a/src/ExecutionLogic.h b/src/ExecutionLogic.h deleted file mode 100644 index 468adb1..0000000 --- a/src/ExecutionLogic.h +++ /dev/null @@ -1,225 +0,0 @@ -// -// Created by ragnarokk on 30.03.2026. -// - -#ifndef COLUMNAR_ENGINE_LOGIC_H -#define COLUMNAR_ENGINE_LOGIC_H - -#include "AggregationFunctions.h" -#include "AggregationExpressions.h" -#include "FilterFunctions.h" -#include "FilterExpressions.h" -#include "OperatorsBase.h" -#include "Schema.h" -#include "macro.h" - -#include -#include -#include -#include -#include - -struct PlanNode { - virtual ~PlanNode() = default; -}; - -struct ScanNode : public PlanNode { - std::string table_path; - std::vector column_names; -}; - -struct AggregateNode : public PlanNode { - std::shared_ptr child; - std::vector group_by_columns; - std::vector> aggregate_expressions; -}; - -struct FilterNode : public PlanNode { - std::shared_ptr child; - std::shared_ptr filter_expression; -}; - -struct OrderByNode : public PlanNode { - std::shared_ptr child; - std::vector> order_by_columns; - std::optional limit; -}; - -struct LimitNode : public PlanNode { - std::shared_ptr child; - size_t limit; -}; - -struct PhysicalOperatorContext { - std::unique_ptr root_operator; - Schema schema; -}; - -inline PhysicalOperatorContext BuildPhysicalPlan( - const std::shared_ptr& plan_node, - std::optional> required_columns = std::nullopt) { - if (auto scan = std::dynamic_pointer_cast(plan_node)) { - std::vector final_columns; - - Schema full_schema = Schema::FromColumnarFile(scan->table_path); - const std::vector& full_columns = full_schema.GetFields(); - - if (required_columns.has_value()) { - final_columns = required_columns.value(); - if (final_columns.empty() && !full_columns.empty()) { - final_columns.push_back(full_columns[0].name); - } - } else { - if (scan->column_names.empty()) { - for (const Field& field : full_columns) { - final_columns.push_back(field.name); - } - } else { - final_columns = scan->column_names; - } - } - - auto op = std::make_unique(scan->table_path, final_columns); - - Schema output_schema; - for (const std::string& name : final_columns) { - bool found = false; - for (const Field& field : full_columns) { - if (field.name == name) { - output_schema.AddField({field.name, field.type}); - found = true; - break; - } - } - if (!found) [[unlikely]] { - THROW_RUNTIME_ERROR("Column " + name + " not found in table schema"); - } - } - - return {std::move(op), std::move(output_schema)}; - } - - if (auto aggregate = std::dynamic_pointer_cast(plan_node)) { - if (!aggregate->group_by_columns.empty()) { - std::vector needed_columns = aggregate->group_by_columns; - for (const std::shared_ptr& expr : - aggregate->aggregate_expressions) { - expr->CollectRequiredColumns(needed_columns); - } - std::sort(needed_columns.begin(), needed_columns.end()); - needed_columns.erase(std::unique(needed_columns.begin(), needed_columns.end()), - needed_columns.end()); - - PhysicalOperatorContext child_context = - BuildPhysicalPlan(aggregate->child, needed_columns); - - std::vector group_col_indices; - std::vector> empty_key_columns; - Schema output_schema; - - for (const auto& col_name : aggregate->group_by_columns) { - size_t ind = child_context.schema.GetColumnIndexByName(col_name); - group_col_indices.push_back(ind); - - ColumnType type = child_context.schema.GetColumnTypeByName(col_name); - output_schema.AddColumn(col_name, type); - -#define HANDLE_TYPE(ENUM_VAL, STR_VAL, CLASS_TYPE) \ - case ColumnType::ENUM_VAL: empty_key_columns.push_back(std::make_shared()); break; - - switch (type) { - FOR_EACH_COLUMN_TYPE(HANDLE_TYPE); - default: THROW_NOT_IMPLEMENTED; - } -#undef HANDLE_TYPE - } - - std::vector> agg_funcs; - for (const std::shared_ptr& expr : - aggregate->aggregate_expressions) { - agg_funcs.push_back( - expr->CreateAggregationFunction(child_context.schema, output_schema)); - } - - auto op = std::make_unique( - std::move(child_context.root_operator), std::move(group_col_indices), - std::move(empty_key_columns), std::move(agg_funcs)); - return {std::move(op), std::move(output_schema)}; - } - - std::vector needed_columns = aggregate->group_by_columns; - for (const std::shared_ptr& expr : aggregate->aggregate_expressions) { - expr->CollectRequiredColumns(needed_columns); - } - std::sort(needed_columns.begin(), needed_columns.end()); - needed_columns.erase(std::unique(needed_columns.begin(), needed_columns.end()), - needed_columns.end()); - - PhysicalOperatorContext child_context = BuildPhysicalPlan(aggregate->child, needed_columns); - std::vector> agg_functions; - Schema output_schema; - for (const std::shared_ptr& expr : aggregate->aggregate_expressions) { - agg_functions.push_back( - expr->CreateAggregationFunction(child_context.schema, output_schema)); - } - auto op = std::make_unique(std::move(child_context.root_operator), - std::move(agg_functions)); - return {std::move(op), std::move(output_schema)}; - } - - if (auto filter = std::dynamic_pointer_cast(plan_node)) { - std::vector needed_columns; - filter->filter_expression->CollectRequiredColumns(needed_columns); - if (required_columns.has_value()) { - needed_columns.insert(needed_columns.end(), required_columns->begin(), - required_columns->end()); - } - - std::sort(needed_columns.begin(), needed_columns.end()); - needed_columns.erase(std::unique(needed_columns.begin(), needed_columns.end()), - needed_columns.end()); - - PhysicalOperatorContext child_context = BuildPhysicalPlan(filter->child, needed_columns); - std::unique_ptr filter_function = - filter->filter_expression->CreateFilterFunction(child_context.schema); - auto op = std::make_unique(std::move(child_context.root_operator), - std::move(filter_function)); - return {std::move(op), std::move(child_context.schema)}; - } - - if (auto order_by = std::dynamic_pointer_cast(plan_node)) { - std::vector needed_columns; - if (required_columns.has_value()) { - needed_columns = required_columns.value(); - } - for (const auto& [col_name, is_desc] : order_by->order_by_columns) { - needed_columns.push_back(col_name); - } - std::sort(needed_columns.begin(), needed_columns.end()); - needed_columns.erase(std::unique(needed_columns.begin(), needed_columns.end()), - needed_columns.end()); - - PhysicalOperatorContext child_context = BuildPhysicalPlan(order_by->child, needed_columns); - std::vector> sort_columns; - for (const auto& [col_name, is_desc] : order_by->order_by_columns) { - size_t sort_col_idx = child_context.schema.GetColumnIndexByName(col_name); - sort_columns.push_back({sort_col_idx, is_desc}); - } - - auto op = - std::make_unique(std::move(child_context.root_operator), - std::move(sort_columns), std::move(order_by->limit)); - return {std::move(op), std::move(child_context.schema)}; - } - - if (auto limit = std::dynamic_pointer_cast(plan_node)) { - PhysicalOperatorContext child_context = BuildPhysicalPlan(limit->child, required_columns); - auto op = - std::make_unique(std::move(child_context.root_operator), limit->limit); - return {std::move(op), std::move(child_context.schema)}; - } - - THROW_RUNTIME_ERROR("Unknown plan node type"); -} - -#endif // COLUMNAR_ENGINE_LOGIC_H diff --git a/src/FilterFunctions.h b/src/FilterFunctions.h deleted file mode 100644 index e537748..0000000 --- a/src/FilterFunctions.h +++ /dev/null @@ -1,86 +0,0 @@ -// -// Created by ragnarokk on 22.04.2026. -// - -#ifndef COLUMNAR_ENGINE_FILTERFUNCTIONS_H -#define COLUMNAR_ENGINE_FILTERFUNCTIONS_H - -#include "OperatorsBase.h" - -class FilterFunction { -public: - virtual ~FilterFunction() = default; - virtual std::vector Evaluate(const RecordBatch& batch) = 0; -}; - -template -class NotEqFilterFunction : public FilterFunction { -public: - NotEqFilterFunction(size_t column_ind, ValueType target_val) - : column_ind_(column_ind), target_val_(target_val) { - } - - std::vector Evaluate(const RecordBatch& batch) override { - auto* column = static_cast(batch.columns[column_ind_].get()); - const auto& data = column->GetData(); - - std::vector selection_vector; - selection_vector.reserve(batch.num_rows); - - if (batch.selection_vector) { - for (uint32_t ind : *batch.selection_vector) { - if (data[ind] != target_val_) { - selection_vector.push_back(ind); - } - } - } else { - for (size_t i = 0; i < batch.num_rows; ++i) { - if (data[i] != target_val_) { - selection_vector.push_back(i); - } - } - } - return selection_vector; - } - -private: - size_t column_ind_; - ValueType target_val_; -}; - -template -class EqFilterFunction : public FilterFunction { -public: - EqFilterFunction(size_t column_ind, ValueType target_val) - : column_ind_(column_ind), target_val_(target_val) { - } - - std::vector Evaluate(const RecordBatch& batch) override { - auto* column = static_cast(batch.columns[column_ind_].get()); - const auto& data = column->GetData(); - - std::vector selection_vector; - selection_vector.reserve(batch.num_rows); - - if (batch.selection_vector) { - for (uint32_t ind : *batch.selection_vector) { - if (data[ind] == target_val_) { - selection_vector.push_back(ind); - } - } - } else { - for (size_t i = 0; i < batch.num_rows; ++i) { - if (data[i] == target_val_) { - selection_vector.push_back(i); - } - } - } - return selection_vector; - } - -private: - size_t column_ind_; - ValueType target_val_; -}; - -#endif // COLUMNAR_ENGINE_FILTERFUNCTIONS_H diff --git a/src/Query.h b/src/Query.h deleted file mode 100644 index f0ec178..0000000 --- a/src/Query.h +++ /dev/null @@ -1,54 +0,0 @@ -// -// Created by ragnarokk on 30.03.2026. -// - -#ifndef COLUMNAR_ENGINE_QUERY_H -#define COLUMNAR_ENGINE_QUERY_H - -enum class QueryId { - Q0, - Q1, - Q2, - Q3, - Q4, - Q5, - Q6, - Q7, - Q8, - Q9, - Q10, - Q11, - Q12, - Q13, - Q14, - Q15, - Q16, - Q17, - Q18, - Q19, - Q20, - Q21, - Q22, - Q23, - Q24, - Q25, - Q26, - Q27, - Q28, - Q29, - Q30, - Q31, - Q32, - Q33, - Q34, - Q35, - Q36, - Q37, - Q38, - Q39, - Q40, - Q41, - Q42 -}; - -#endif // COLUMNAR_ENGINE_QUERY_H diff --git a/src/ScalarExpressions.h b/src/ScalarExpressions.h deleted file mode 100644 index 17d8a1f..0000000 --- a/src/ScalarExpressions.h +++ /dev/null @@ -1,42 +0,0 @@ -// -// Created by ragnarokk on 5/3/26. -// - -#ifndef COLUMNAR_ENGINE_SCALAREXPRESSIONS_H -#define COLUMNAR_ENGINE_SCALAREXPRESSIONS_H - -class ScalarExpression { -public: - virtual ~ScalarExpression() = default; - virtual std::shared_ptr Evaluate(const RecordBatch& batch) = 0; -}; - -class ExtractMinuteExpression : public ScalarExpression { -public: - explicit ExtractMinuteExpression(size_t input_col_idx) : input_col_idx_(input_col_idx) {} - - std::shared_ptr Evaluate(const RecordBatch& batch) override { - auto* input_col = static_cast(batch.columns[input_col_idx_].get()); - const auto& input_data = input_col->GetData(); - - auto result_col = std::make_shared(); // Минуты отлично влезут в Int32 - - // Оптимизация: учитываем selection_vector, если он есть (после Filter) - if (batch.selection_vector) { - for (size_t i = 0; i < batch.num_rows; ++i) { - size_t actual_idx = (*batch.selection_vector)[i]; - result_col->Add(TimestampColumn::ExtractMinute(input_data[actual_idx])); - } - } else { - for (size_t i = 0; i < input_col->Size(); ++i) { - result_col->Add(TimestampColumn::ExtractMinute(input_data[i])); - } - } - return result_col; - } - -private: - size_t input_col_idx_; -}; - -#endif // COLUMNAR_ENGINE_SCALAREXPRESSIONS_H diff --git a/src/Types.h b/src/Types.h deleted file mode 100644 index 2e81897..0000000 --- a/src/Types.h +++ /dev/null @@ -1,780 +0,0 @@ -// -// Created by ragnarokk on 03.01.2026. -// - -// OPTIMIZE: function AddBatch may be done with better logic of reserve (as in StringColumn) - -#ifndef COLUMNAR_ENGINE_OBJECT_H -#define COLUMNAR_ENGINE_OBJECT_H - -#include "macro.h" -#include "VectorOfStrings.h" - -#include -#include -#include -#include -#include -#include -#include -#include - -enum class ColumnType : uint8_t { - INT16, - INT32, - INT64, - INT128, - FLOAT, - DOUBLE, - LONGDOUBLE, - CHAR, - STRING, - DATE, - TIMESTAMP, -}; - -struct ColumnMetadata { - std::string name; - ColumnType type; - std::vector offsets; - std::vector sizes; -}; - -class Column { -public: - virtual ~Column() = default; - virtual ColumnType GetType() const = 0; - virtual size_t Size() const = 0; - virtual void Add(const std::string& value) = 0; // TODO: DO NOT USE IN REAL CODE - virtual void AddView(std::string_view value) = 0; - virtual void AddBatch(const VectorOfStrings2D& batch, size_t j) = 0; - virtual uint64_t WriteToFile(std::ofstream& file) = 0; - virtual std::string GetDataAsString(size_t index) const = 0; // TODO: DO NOT USE IN REAL CODE - virtual const void* GetRawData() const = 0; - virtual void SerializeValue(size_t index, - std::string& buffer) const = 0; // TODO: DO NOT USE IN REAL CODE - virtual void SerializeBatch(std::vector& keys, - const std::vector* selection = nullptr) const = 0; - virtual void ReadFromRawData(const std::vector& buffer) = 0; - virtual std::shared_ptr CloneEmpty() const = 0; - virtual void CopyFrom(const Column& other, size_t index) = 0; // TODO: DO NOT USE IN REAL CODE - virtual void CopyBatchFrom(const Column& other, - const std::vector* indices = nullptr) = 0; - virtual void Clear() = 0; -}; - -template -struct ColumnTypeTraits; - -template <> -struct ColumnTypeTraits { - static constexpr ColumnType kType = ColumnType::INT16; - static int16_t FromString(const std::string_view& value) { - int16_t parsed = 0; - const auto [ptr, ec] = std::from_chars(value.data(), value.data() + value.size(), parsed); - if (ec != std::errc() || ptr != value.data() + value.size()) { - THROW_RUNTIME_ERROR("Invalid INT16 value: " + std::string(value)); - } - return parsed; - } -}; - -template <> -struct ColumnTypeTraits { - static constexpr ColumnType kType = ColumnType::INT32; - static int32_t FromString(const std::string_view& value) { - int32_t parsed = 0; - const auto [ptr, ec] = std::from_chars(value.data(), value.data() + value.size(), parsed); - if (ec != std::errc() || ptr != value.data() + value.size()) { - THROW_RUNTIME_ERROR("Invalid INT32 value: " + std::string(value)); - } - return parsed; - } -}; - -template <> -struct ColumnTypeTraits { - static constexpr ColumnType kType = ColumnType::INT64; - static int64_t FromString(const std::string_view& value) { - int64_t parsed = 0; - const auto [ptr, ec] = std::from_chars(value.data(), value.data() + value.size(), parsed); - if (ec != std::errc() || ptr != value.data() + value.size()) { - THROW_RUNTIME_ERROR("Invalid INT64 value: " + std::string(value)); - } - return parsed; - } -}; - -template <> -struct ColumnTypeTraits<__int128_t> { - static constexpr ColumnType kType = ColumnType::INT128; - static __int128_t FromString(const std::string_view& value) { - __int128_t result = 0; - bool negative = false; - std::string_view value_view = value; - if (value[0] == '-') { - negative = true; - value_view = value_view.substr(1); - } - for (char c : value_view) { - if (c < '0' || c > '9') [[unlikely]] { - THROW_RUNTIME_ERROR("Invalid integer value: " + std::string(value)); - } - result = result * 10 + (c - '0'); - } - if (negative) { - result = -result; - } - return result; - } -}; - -template <> -struct ColumnTypeTraits { - static constexpr ColumnType kType = ColumnType::FLOAT; - static float FromString(const std::string_view& value) { - return std::stof(std::string(value)); - } -}; - -template <> -struct ColumnTypeTraits { - static constexpr ColumnType kType = ColumnType::DOUBLE; - static double FromString(const std::string_view& value) { - return std::stod(std::string(value)); - } -}; - -template <> -struct ColumnTypeTraits { - static constexpr ColumnType kType = ColumnType::LONGDOUBLE; - static long double FromString(const std::string_view& value) { - return std::stold(std::string(value)); - } -}; - -template - requires std::is_integral_v || std::is_floating_point_v -std::string ToString(T value) { - return std::to_string(value); -} - -template <> -inline std::string ToString<__int128_t>(__int128_t value) { - if (value == 0) { - return "0"; - } - std::string result; - bool negative = value < 0; - if (negative) { - value = -value; - } - while (value > 0) { - result.push_back('0' + (value % 10)); - value /= 10; - } - if (negative) { - result.push_back('-'); - } - std::reverse(result.begin(), result.end()); - return result; -} - -template -class NumericColumn : public Column { -public: - using ValueType = T; - using ContainerType = std::vector; - - ColumnType GetType() const override { - return ColumnTypeTraits::kType; - } - - size_t Size() const override { - return data_.size(); - } - - void Add(T value) { - data_.push_back(value); - } - - void Add(const std::string& value) override { - data_.push_back(ColumnTypeTraits::FromString(value)); - } - - void AddView(std::string_view value) override { - data_.push_back(ColumnTypeTraits::FromString(value)); - } - - void AddBatch(const VectorOfStrings2D& batch, size_t j) override { - data_.reserve(data_.size() + batch.Height()); - for (size_t i = 0; i < batch.Height(); ++i) { - data_.push_back(ColumnTypeTraits::FromString(batch.GetString2D(i, j))); - } - } - - // TODO: add decoding logic - uint64_t WriteToFile(std::ofstream& file) override { - if (!data_.empty()) { - file.write(reinterpret_cast(data_.data()), data_.size() * sizeof(T)); - } - return data_.size() * sizeof(T); - } - - std::string GetDataAsString(size_t index) const override { - if (index >= data_.size()) { - THROW_RUNTIME_ERROR("Index out of range"); - } - return ToString(data_[index]); - } - - const void* GetRawData() const override { - return &data_; - } - - void SerializeValue(size_t index, std::string& buffer) const override { - buffer.append(reinterpret_cast(&data_[index]), sizeof(T)); - } - - void SerializeBatch(std::vector& keys, - const std::vector* selection = nullptr) const override { - if (!selection) { - size_t n = data_.size(); - keys.reserve(n); - for (size_t i = 0; i < n; ++i) { - keys[i].append(reinterpret_cast(&data_[i]), sizeof(T)); - } - } else { - size_t n = selection->size(); - keys.reserve(n); - for (size_t i = 0; i < n; ++i) { - size_t idx = (*selection)[i]; - keys[i].append(reinterpret_cast(&data_[idx]), sizeof(T)); - } - } - } - - void ReadFromRawData(const std::vector& buffer) override { - if (buffer.size() % sizeof(T) != 0) { - THROW_RUNTIME_ERROR("Raw buffer size is not aligned with type size"); - } - size_t n = buffer.size() / sizeof(T); - data_.resize(n); - std::memcpy(data_.data(), buffer.data(), buffer.size()); - } - - std::shared_ptr CloneEmpty() const override { - return std::make_shared>(); - } - - void CopyFrom(const Column& other, size_t index) override { - data_.push_back(static_cast&>(other).GetData()[index]); - } - - void CopyBatchFrom(const Column& other, const std::vector* indices = nullptr) override { - const auto& other_data = static_cast&>(other).GetData(); - if (!indices) { - data_.insert(data_.end(), other_data.begin(), other_data.end()); - } else { - data_.reserve(data_.size() + indices->size()); - for (size_t idx : *indices) { - data_.push_back(other_data[idx]); - } - } - } - - void Clear() override { - data_.clear(); - } - - const std::vector& GetData() const { - return data_; - } - -private: - ContainerType data_; -}; - -using Int16Column = NumericColumn; -using Int32Column = NumericColumn; -using Int64Column = NumericColumn; -using Int128Column = NumericColumn<__int128_t>; -using FloatColumn = NumericColumn; -using DoubleColumn = NumericColumn; -using LongDoubleColumn = NumericColumn; - -class CharColumn : public Column { -public: - using ValueType = char; - using ContainerType = std::vector; - - ColumnType GetType() const override { - return ColumnType::CHAR; - } - size_t Size() const override { - return data_.size(); - } - - void Add(const std::string& value) override { - ASSERT(value.size() == 1); - data_.push_back(value[0]); - } - - void AddView(std::string_view value) override { - ASSERT(value.size() == 1); - data_.push_back(value[0]); - } - - void AddBatch(const VectorOfStrings2D& batch, size_t j) override { - data_.reserve(data_.size() + batch.Height()); - for (size_t i = 0; i < batch.Height(); ++i) { - std::string_view val = batch.GetString2D(i, j); - ASSERT(val.size() == 1); - data_.push_back(val[0]); - } - } - - uint64_t WriteToFile(std::ofstream& file) override { - if (!data_.empty()) { - file.write(data_.data(), data_.size()); - } - return data_.size(); - } - - std::string GetDataAsString(size_t index) const override { - if (index >= data_.size()) { - THROW_RUNTIME_ERROR("Index out of range"); - } - return std::string(1, data_[index]); - } - - void SerializeValue(size_t index, std::string& buffer) const override { - buffer.push_back(data_[index]); - } - - void SerializeBatch(std::vector& keys, - const std::vector* selection = nullptr) const override { - size_t n = !selection ? data_.size() : selection->size(); - for (size_t i = 0; i < n; ++i) { - size_t idx = !selection ? i : (*selection)[i]; - keys[i].push_back(data_[idx]); - } - } - - void ReadFromRawData(const std::vector& buffer) override { - data_.clear(); - data_.insert(data_.end(), buffer.begin(), buffer.end()); - } - - const void* GetRawData() const override { - return data_.data(); - } - - std::shared_ptr CloneEmpty() const override { - return std::make_shared(); - } - - void CopyFrom(const Column& other, size_t index) override { - data_.push_back(static_cast(other).GetData()[index]); - } - - void CopyBatchFrom(const Column& other, const std::vector* indices = nullptr) override { - const auto& other_data = static_cast(other).GetData(); - if (!indices) { - data_.insert(data_.end(), other_data.begin(), other_data.end()); - } else { - data_.reserve(data_.size() + indices->size()); - for (size_t idx : *indices) { - data_.push_back(other_data[idx]); - } - } - } - - void Clear() override { - data_.clear(); - } - - const ContainerType& GetData() const { - return data_; - } - -private: - ContainerType data_; -}; - -// OPTIMIZE: perf decreased because i made VectorOfStrings logic and moved this logic into a -// particular class -class StringColumn : public Column { -public: - using ValueType = std::string; - using ContainerType = VectorOfStrings; - - ColumnType GetType() const override { - return ColumnType::STRING; - } - - size_t Size() const override { - return data_.Size(); - } - - void Add(const std::string& value) override { - data_.PushBack(value); - } - - void AddView(std::string_view value) override { - data_.PushBack(value); - } - - void AddBatch(const VectorOfStrings2D& batch, size_t j) override { - size_t h = batch.Height(); - EnsureCapacity(batch.ColumnSize(j), h); - // data_.ReserveOffsets(data_.Size() + h + 1); - // data_.ReserveData(data_.DataSize() + batch.ColumnSize(j)); - for (size_t i = 0; i < h; ++i) { - data_.PushBack(batch.GetString2D(i, j)); - } - } - - uint64_t WriteToFile(std::ofstream& file) override { - const std::vector& offsets = data_.GetOffsets(); - const std::vector& data = data_.GetData(); - - uint64_t total_size = 0; - std::vector buffer; - size_t h = Size(); - size_t total_len = 0; - for (size_t i = 0; i < h; ++i) { - total_len += sizeof(uint32_t) + (offsets[i + 1] - offsets[i]); - } - buffer.reserve(total_len); - for (size_t i = 0; i < h; ++i) { - size_t start = offsets[i]; - size_t end = offsets[i + 1]; - uint32_t length = end - start; - buffer.insert(buffer.end(), reinterpret_cast(&length), - reinterpret_cast(&length) + sizeof(length)); - if (length > 0) { - buffer.insert(buffer.end(), data.begin() + start, data.begin() + end); - } - total_size += sizeof(length) + length; - } - file.write(buffer.data(), buffer.size()); - return total_size; - } - - std::string GetDataAsString(size_t index) const override { - if (index >= Size()) [[unlikely]] { - THROW_RUNTIME_ERROR("Index out of range"); - } - return std::string{data_[index]}; - } - - const void* GetRawData() const override { - return &data_; - } - - void SerializeValue(size_t index, std::string& buffer) const override { - std::string_view val = data_[index]; - uint32_t len = val.size(); - buffer.append(reinterpret_cast(&len), sizeof(len)); - buffer.append(val.data(), val.size()); - } - - void SerializeBatch(std::vector& keys, - const std::vector* selection = nullptr) const override { - if (!selection) { - size_t n = data_.Size(); - keys.reserve(n); - for (size_t i = 0; i < n; ++i) { - std::string_view val = data_[i]; - uint32_t len = val.size(); - keys[i].append(reinterpret_cast(&len), sizeof(len)); - keys[i].append(val.data(), val.size()); - } - } else { - size_t n = selection->size(); - keys.reserve(n); - for (size_t i = 0; i < n; ++i) { - size_t idx = (*selection)[i]; - std::string_view val = data_[idx]; - uint32_t len = val.size(); - keys[i].append(reinterpret_cast(&len), sizeof(len)); - keys[i].append(val.data(), val.size()); - } - } - } - - void ReadFromRawData(const std::vector& buffer) override { - Clear(); - size_t offset = 0; - while (offset < buffer.size()) { - uint32_t length; - std::memcpy(&length, buffer.data() + offset, sizeof(length)); - offset += sizeof(length); - std::string_view str(buffer.data() + offset, length); - data_.PushBack(str); - offset += length; - } - } - - std::shared_ptr CloneEmpty() const override { - return std::make_shared(); - } - - void CopyFrom(const Column& other, size_t index) override { - data_.PushBack(static_cast(other).data_[index]); - } - - void CopyBatchFrom(const Column& other, const std::vector* indices = nullptr) override { - const auto& other_data = static_cast(other).data_; - if (!indices) { - for (size_t i = 0; i < other_data.Size(); ++i) { - data_.PushBack(other_data[i]); - } - } else { - for (size_t idx : *indices) { - data_.PushBack(other_data[idx]); - } - } - } - - void Clear() override { - data_.Clear(); - } - - const ContainerType& GetData() const { - return data_; - } - -private: - ContainerType data_; - - void EnsureCapacity(size_t data_size, size_t elements) { - size_t required_sz = data_.DataSize() + data_size; - size_t data_cap = data_.DataCapacity(); - if (required_sz > data_cap) { - size_t cap = data_cap == 0 ? 1024 : data_cap * 2; - data_.ReserveData(std::max(required_sz, cap)); - } - - size_t required_offset_size = data_.Size() + elements; - size_t off_cap = data_.OffsetsCapacity(); - if (required_offset_size > off_cap) { - size_t cap = off_cap == 0 ? 1024 : off_cap * 2; - data_.ReserveOffsets(std::max(required_offset_size, cap)); - } - } -}; - -template -class TemporalColumn : public Column { -public: - using ValueType = T; - using ContainerType = std::vector; - - ColumnType GetType() const override { - return CType; - } - - size_t Size() const override { - return data_.size(); - } - - void Add(T value) { - data_.push_back(value); - } - - void Add(const std::string& value) override { - data_.push_back(Derived::Parse(value)); - } - - void AddView(std::string_view value) override { - data_.push_back(Derived::Parse(value)); - } - - void AddBatch(const VectorOfStrings2D& batch, size_t j) override { - size_t h = batch.Height(); - data_.reserve(data_.size() + h); - for (size_t i = 0; i < h; ++i) { - std::string_view cur = batch.GetString2D(i, j); - data_.push_back(Derived::Parse(cur)); - } - } - - uint64_t WriteToFile(std::ofstream& file) override { - if (!data_.empty()) { - file.write(reinterpret_cast(data_.data()), data_.size() * sizeof(T)); - } - return data_.size() * sizeof(T); - } - - std::string GetDataAsString(size_t index) const override { - if (index >= data_.size()) [[unlikely]] { - THROW_RUNTIME_ERROR("Index out of range"); - } - return Derived::Format(data_[index]); - } - - const void* GetRawData() const override { - return &data_; - } - - void SerializeValue(size_t index, std::string& buffer) const override { - buffer.append(reinterpret_cast(&data_[index]), sizeof(T)); - } - - void SerializeBatch(std::vector& keys, - const std::vector* selection = nullptr) const override { - size_t n = !selection ? data_.size() : selection->size(); - for (size_t i = 0; i < n; ++i) { - size_t idx = !selection ? i : (*selection)[i]; - keys[i].append(reinterpret_cast(&data_[idx]), sizeof(T)); - } - } - - void ReadFromRawData(const std::vector& buffer) override { - if (buffer.size() % sizeof(T) != 0) { - THROW_RUNTIME_ERROR("Raw buffer size is not aligned with type size"); - } - size_t n = buffer.size() / sizeof(T); - data_.resize(n); - std::memcpy(data_.data(), buffer.data(), buffer.size()); - } - - std::shared_ptr CloneEmpty() const override { - return std::make_shared>(); - } - - void CopyFrom(const Column& other, size_t index) override { - data_.push_back( - static_cast&>(other).GetData()[index]); - } - - void CopyBatchFrom(const Column& other, const std::vector* indices = nullptr) override { - const auto& other_data = - static_cast&>(other).GetData(); - if (!indices) { - data_.insert(data_.end(), other_data.begin(), other_data.end()); - } else { - data_.reserve(data_.size() + indices->size()); - for (size_t idx : *indices) { - data_.push_back(other_data[idx]); - } - } - } - - void Clear() override { - data_.clear(); - } - - const ContainerType& GetData() const { - return data_; - } - -private: - ContainerType data_; -}; - -class DateColumn : public TemporalColumn { -public: - static int32_t Parse(std::string_view unit) { - if (unit.size() != 10 || unit[4] != '-' || unit[7] != '-') [[unlikely]] { - THROW_RUNTIME_ERROR("Invalid date format: " + std::string(unit)); - } - - int32_t year = - (unit[0] - '0') * 1000 + (unit[1] - '0') * 100 + (unit[2] - '0') * 10 + (unit[3] - '0'); - int32_t month = (unit[5] - '0') * 10 + (unit[6] - '0'); - int32_t day = (unit[8] - '0') * 10 + (unit[9] - '0'); - - year -= (month <= 2); - const int era = (year >= 0 ? year : year - 399) / 400; - const unsigned yoe = static_cast(year - era * 400); - const unsigned doy = (153 * (month + (month > 2 ? -3 : 9)) + 2) / 5 + day - 1; - const unsigned doe = yoe * 365 + yoe / 4 - yoe / 100 + doy; - - return era * 146097 + static_cast(doe) - 719468; - } - - static std::string Format(int32_t days) { - days += 719468; - const int era = (days >= 0 ? days : days - 146096) / 146097; - const unsigned doe = static_cast(days - era * 146097); - const unsigned yoe = (doe - doe / 1460 + doe / 36524 - doe / 146096) / 365; - const int y = static_cast(yoe) + era * 400; - const unsigned doy = doe - (365 * yoe + yoe / 4 - yoe / 100); - const unsigned mp = (5 * doy + 2) / 153; - const unsigned d = doy - (153 * mp + 2) / 5 + 1; - const unsigned m = mp + (mp < 10 ? 3 : -9); - const int year = y + (m <= 2); - - std::string res = "0000-00-00"; - auto write_digit = [&](int val, int pos, int len) { - for (int i = 0; i < len; ++i) { - res[pos + len - 1 - i] = (val % 10) + '0'; - val /= 10; - } - }; - write_digit(year, 0, 4); - write_digit(m, 5, 2); - write_digit(d, 8, 2); - return res; - } -}; - -class TimestampColumn : public TemporalColumn { -public: - static int64_t Parse(std::string_view unit) { - if (unit.size() != 19 || unit[4] != '-' || unit[7] != '-' || unit[10] != ' ' || - unit[13] != ':' || unit[16] != ':') [[unlikely]] { - THROW_RUNTIME_ERROR("Invalid date format: " + std::string(unit)); - } - - auto to_int2 = [](char a, char b) { return (a - '0') * 10 + (b - '0'); }; - - int year = - (unit[0] - '0') * 1000 + (unit[1] - '0') * 100 + (unit[2] - '0') * 10 + (unit[3] - '0'); - int month = to_int2(unit[5], unit[6]); - int day = to_int2(unit[8], unit[9]); - int hour = to_int2(unit[11], unit[12]); - int minute = to_int2(unit[14], unit[15]); - int second = to_int2(unit[17], unit[18]); - - year -= (month <= 2); - const int era = (year >= 0 ? year : year - 399) / 400; - const unsigned yoe = static_cast(year - era * 400); - const unsigned doy = (153 * (month + (month > 2 ? -3 : 9)) + 2) / 5 + day - 1; - const unsigned doe = yoe * 365 + yoe / 4 - yoe / 100 + doy; - - int32_t days = era * 146097 + static_cast(doe) - 719468; - int64_t seconds = hour * 3600 + minute * 60 + second; - return static_cast(days) * 86400 + seconds; - } - - static std::string Format(int64_t total_seconds) { - int64_t days = total_seconds / 86400; - int64_t seconds_in_day = total_seconds % 86400; - if (seconds_in_day < 0) { - seconds_in_day += 86400; - days -= 1; - } - - std::string res = DateColumn::Format(static_cast(days)); - res += " 00:00:00"; - int hour = seconds_in_day / 3600; - int minute = (seconds_in_day % 3600) / 60; - int second = seconds_in_day % 60; - auto write_digit = [&](int val, int pos) { - res[pos] = (val / 10) + '0'; - res[pos + 1] = (val % 10) + '0'; - }; - - write_digit(hour, 11); - write_digit(minute, 14); - write_digit(second, 17); - - return res; - } -}; - -#endif // COLUMNAR_ENGINE_OBJECT_H diff --git a/src/column/CharColumn.h b/src/column/CharColumn.h new file mode 100644 index 0000000..2db0ccf --- /dev/null +++ b/src/column/CharColumn.h @@ -0,0 +1,68 @@ +#pragma once + +#include "column/Column.h" +#include "column/ColumnBuilder.h" + +#include +#include + +class CharColumn : public Column { +public: + using ValueType = char; + using ContainerType = std::vector; + + ColumnType GetType() const override { + return ColumnType::CHAR; + } + + size_t Size() const override { + return data_.size(); + } + + const void* GetRawData() const override { + return data_.data(); + } + + void Clear() override { + data_.clear(); + } + + const ContainerType& GetData() const { + return data_; + } + + void AddValue(char value) { + data_.push_back(value); + } + + void ReadFromBuffer(const std::vector& buffer) { + data_.clear(); + data_.insert(data_.end(), buffer.begin(), buffer.end()); + } + +private: + ContainerType data_; +}; + +class CharColumnBuilder : public ColumnBuilder { +public: + CharColumnBuilder() : column_(std::make_shared()) {} + + void AddBatch(const VectorOfStrings2D& batch, size_t j) override { + size_t h = batch.Height(); + for (size_t i = 0; i < h; ++i) { + std::string_view val = batch.GetString2D(i, j); + ASSERT(val.size() == 1); + column_->AddValue(val[0]); + } + } + + std::shared_ptr Finish() override { + auto result = column_; + column_ = std::make_shared(); + return result; + } + +private: + std::shared_ptr column_; +}; diff --git a/src/Codec.h b/src/column/Codec.h similarity index 100% rename from src/Codec.h rename to src/column/Codec.h diff --git a/src/column/Column.h b/src/column/Column.h new file mode 100644 index 0000000..c911043 --- /dev/null +++ b/src/column/Column.h @@ -0,0 +1,14 @@ +#pragma once + +#include "types/ColumnType.h" + +#include + +class Column { +public: + virtual ~Column() = default; + virtual ColumnType GetType() const = 0; + virtual size_t Size() const = 0; + virtual const void* GetRawData() const = 0; + virtual void Clear() = 0; +}; diff --git a/src/column/ColumnBuilder.h b/src/column/ColumnBuilder.h new file mode 100644 index 0000000..1c9b989 --- /dev/null +++ b/src/column/ColumnBuilder.h @@ -0,0 +1,13 @@ +#pragma once + +#include "column/Column.h" +#include "utils/VectorOfStrings.h" + +#include + +class ColumnBuilder { +public: + virtual ~ColumnBuilder() = default; + virtual void AddBatch(const VectorOfStrings2D& batch, size_t column_index) = 0; + virtual std::shared_ptr Finish() = 0; +}; diff --git a/src/column/ColumnFactory.h b/src/column/ColumnFactory.h new file mode 100644 index 0000000..bdb8926 --- /dev/null +++ b/src/column/ColumnFactory.h @@ -0,0 +1,34 @@ +#pragma once + +#include "column/ColumnBuilder.h" +#include "types/ColumnType.h" +#include "utils/Macro.h" + +#include + +class ColumnFactory { +public: + static std::shared_ptr MakeColumn(ColumnType type) { + switch (type) { +#define HANDLE_TYPE(ENUM_VAL, STR_VAL, CLASS_TYPE) \ + case ColumnType::ENUM_VAL: return std::make_shared(); + + FOR_EACH_COLUMN_TYPE(HANDLE_TYPE) +#undef HANDLE_TYPE + default: + THROW_RUNTIME_ERROR("Unknown column type"); + } + } + + static std::shared_ptr MakeColumnBuilder(ColumnType type) { + switch (type) { +#define HANDLE_TYPE(ENUM_VAL, STR_VAL, CLASS_TYPE) \ + case ColumnType::ENUM_VAL: return std::make_shared(); + + FOR_EACH_COLUMN_TYPE(HANDLE_TYPE) +#undef HANDLE_TYPE + default: + THROW_RUNTIME_ERROR("Unknown column type"); + } + } +}; diff --git a/src/column/NumericColumn.h b/src/column/NumericColumn.h new file mode 100644 index 0000000..59f6ae9 --- /dev/null +++ b/src/column/NumericColumn.h @@ -0,0 +1,198 @@ +#pragma once + +#include "column/Column.h" +#include "column/ColumnBuilder.h" +#include "utils/Assert.h" + +#include +#include +#include +#include +#include + +using Int128 = __int128_t; + +template +struct ColumnTypeTraits; + +template <> +struct ColumnTypeTraits { + static constexpr ColumnType kType = ColumnType::INT16; + static int16_t FromString(std::string_view value) { + int16_t parsed = 0; + const auto [ptr, ec] = std::from_chars(value.data(), value.data() + value.size(), parsed); + if (ec != std::errc() || ptr != value.data() + value.size()) { + THROW_RUNTIME_ERROR("Invalid INT16 value: " + std::string(value)); + } + return parsed; + } +}; + +template <> +struct ColumnTypeTraits { + static constexpr ColumnType kType = ColumnType::INT32; + static int32_t FromString(std::string_view value) { + int32_t parsed = 0; + const auto [ptr, ec] = std::from_chars(value.data(), value.data() + value.size(), parsed); + if (ec != std::errc() || ptr != value.data() + value.size()) { + THROW_RUNTIME_ERROR("Invalid INT32 value: " + std::string(value)); + } + return parsed; + } +}; + +template <> +struct ColumnTypeTraits { + static constexpr ColumnType kType = ColumnType::INT64; + static int64_t FromString(std::string_view value) { + int64_t parsed = 0; + const auto [ptr, ec] = std::from_chars(value.data(), value.data() + value.size(), parsed); + if (ec != std::errc() || ptr != value.data() + value.size()) { + THROW_RUNTIME_ERROR("Invalid INT64 value: " + std::string(value)); + } + return parsed; + } +}; + +template <> +struct ColumnTypeTraits { + static constexpr ColumnType kType = ColumnType::INT128; + static Int128 FromString(std::string_view value) { + Int128 result = 0; + bool negative = false; + std::string_view value_view = value; + if (value[0] == '-') { + negative = true; + value_view = value_view.substr(1); + } + for (char c : value_view) { + if (c < '0' || c > '9') [[unlikely]] { + THROW_RUNTIME_ERROR("Invalid integer value: " + std::string(value)); + } + result = result * 10 + (c - '0'); + } + if (negative) { + result = -result; + } + return result; + } +}; + +template <> +struct ColumnTypeTraits { + static constexpr ColumnType kType = ColumnType::FLOAT; + static float FromString(std::string_view value) { + return std::stof(std::string(value)); + } +}; + +template <> +struct ColumnTypeTraits { + static constexpr ColumnType kType = ColumnType::DOUBLE; + static double FromString(std::string_view value) { + return std::stod(std::string(value)); + } +}; + +template <> +struct ColumnTypeTraits { + static constexpr ColumnType kType = ColumnType::LONGDOUBLE; + static long double FromString(std::string_view value) { + return std::stold(std::string(value)); + } +}; + +template +class NumericColumn : public Column { +public: + using ValueType = T; + using ContainerType = std::vector; + + NumericColumn() = default; + NumericColumn(const ContainerType& data) : data_(data) {} + NumericColumn(ContainerType&& data) : data_(std::move(data)) {} + NumericColumn operator=(const ContainerType& data) { + data_ = data; + return *this; + } + NumericColumn operator=(ContainerType&& data) { + data_ = std::move(data); + return *this; + } + + ~NumericColumn() override = default; + + ColumnType GetType() const override { + return ColumnTypeTraits::kType; + } + + size_t Size() const override { + return data_.size(); + } + + const void* GetRawData() const override { + return &data_; + } + + void Clear() override { + data_.clear(); + } + + const ContainerType& GetData() const { + return data_; + } + + void AddValue(T value) { + data_.push_back(value); + } + + void ReadFromBuffer(const std::vector& buffer) { + if (buffer.size() % sizeof(T) != 0) { + THROW_RUNTIME_ERROR("Raw buffer size is not aligned with type size"); + } + size_t n = buffer.size() / sizeof(T); + data_.resize(n); + std::memcpy(data_.data(), buffer.data(), buffer.size()); + } + +private: + ContainerType data_; +}; + +template +class NumericColumnBuilder : public ColumnBuilder { +public: + NumericColumnBuilder() : column_(std::make_shared>()) {} + + void AddBatch(const VectorOfStrings2D& batch, size_t j) override { + size_t h = batch.Height(); + for (size_t i = 0; i < h; ++i) { + column_->AddValue(ColumnTypeTraits::FromString(batch.GetString2D(i, j))); + } + } + + std::shared_ptr Finish() override { + auto result = column_; + column_ = std::make_shared>(); + return result; + } + +private: + std::shared_ptr> column_; +}; + +using Int16Column = NumericColumn; +using Int32Column = NumericColumn; +using Int64Column = NumericColumn; +using Int128Column = NumericColumn; +using FloatColumn = NumericColumn; +using DoubleColumn = NumericColumn; +using LongDoubleColumn = NumericColumn; + +using Int16ColumnBuilder = NumericColumnBuilder; +using Int32ColumnBuilder = NumericColumnBuilder; +using Int64ColumnBuilder = NumericColumnBuilder; +using Int128ColumnBuilder = NumericColumnBuilder; +using FloatColumnBuilder = NumericColumnBuilder; +using DoubleColumnBuilder = NumericColumnBuilder; +using LongDoubleColumnBuilder = NumericColumnBuilder; diff --git a/src/Schema.h b/src/column/Schema.h similarity index 97% rename from src/Schema.h rename to src/column/Schema.h index 70b67fb..8b4e2ea 100644 --- a/src/Schema.h +++ b/src/column/Schema.h @@ -5,9 +5,9 @@ #ifndef COLUMNAR_ENGINE_SCHEMA_H #define COLUMNAR_ENGINE_SCHEMA_H -#include "ColumnarReader.h" -#include "Types.h" -#include "macro.h" +#include "io/ColumnarReader.h" +#include "types/ColumnType.h" +#include "utils/Macro.h" #include #include diff --git a/src/column/StringColumn.h b/src/column/StringColumn.h new file mode 100644 index 0000000..bf398f1 --- /dev/null +++ b/src/column/StringColumn.h @@ -0,0 +1,76 @@ +#pragma once + +#include "column/Column.h" +#include "column/ColumnBuilder.h" +#include "utils/VectorOfStrings.h" +#include "utils/Macro.h" + +#include +#include + +class StringColumn : public Column { +public: + using ValueType = std::string; + using ContainerType = VectorOfStrings; + + ColumnType GetType() const override { + return ColumnType::STRING; + } + + size_t Size() const override { + return data_.Size(); + } + + const void* GetRawData() const override { + return &data_; + } + + void Clear() override { + data_.Clear(); + } + + const ContainerType& GetData() const { + return data_; + } + + void AddValue(std::string_view value) { + data_.PushBack(value); + } + + void ReadFromBuffer(const std::vector& buffer) { + Clear(); + size_t offset = 0; + while (offset < buffer.size()) { + uint32_t length; + std::memcpy(&length, buffer.data() + offset, sizeof(length)); + offset += sizeof(length); + std::string_view str(buffer.data() + offset, length); + data_.PushBack(str); + offset += length; + } + } + +private: + ContainerType data_; +}; + +class StringColumnBuilder : public ColumnBuilder { +public: + StringColumnBuilder() : column_(std::make_shared()) {} + + void AddBatch(const VectorOfStrings2D& batch, size_t j) override { + size_t h = batch.Height(); + for (size_t i = 0; i < h; ++i) { + column_->AddValue(batch.GetString2D(i, j)); + } + } + + std::shared_ptr Finish() override { + auto result = column_; + column_ = std::make_shared(); + return result; + } + +private: + std::shared_ptr column_; +}; diff --git a/src/column/TemporalColumn.h b/src/column/TemporalColumn.h new file mode 100644 index 0000000..00b6b0b --- /dev/null +++ b/src/column/TemporalColumn.h @@ -0,0 +1,82 @@ +#pragma once + +#include "column/Column.h" +#include "column/ColumnBuilder.h" +#include "types/Date.h" +#include "types/Timestamp.h" + +#include +#include +#include + +template +class TemporalColumn : public Column { +public: + using ValueType = T; + using ContainerType = std::vector; + + ColumnType GetType() const override { + return CType; + } + + size_t Size() const override { + return data_.size(); + } + + const void* GetRawData() const override { + return &data_; + } + + void Clear() override { + data_.clear(); + } + + const ContainerType& GetData() const { + return data_; + } + + void AddValue(T value) { + data_.push_back(value); + } + + void ReadFromBuffer(const std::vector& buffer) { + if (buffer.size() % sizeof(T) != 0) { + THROW_RUNTIME_ERROR("Raw buffer size is not aligned with type size"); + } + size_t n = buffer.size() / sizeof(T); + data_.resize(n); + std::memcpy(data_.data(), buffer.data(), buffer.size()); + } + +private: + ContainerType data_; +}; + +class DateColumn : public TemporalColumn {}; +class TimestampColumn : public TemporalColumn {}; + +template +class TemporalColumnBuilder : public ColumnBuilder { +public: + TemporalColumnBuilder() : column_(std::make_shared()) {} + + void AddBatch(const VectorOfStrings2D& batch, size_t j) override { + size_t h = batch.Height(); + for (size_t i = 0; i < h; ++i) { + std::string_view cur = batch.GetString2D(i, j); + column_->AddValue(ParseStruct::Parse(cur)); + } + } + + std::shared_ptr Finish() override { + auto result = column_; + column_ = std::make_shared(); + return result; + } + +private: + std::shared_ptr column_; +}; + +using DateColumnBuilder = TemporalColumnBuilder; +using TimestampColumnBuilder = TemporalColumnBuilder; diff --git a/src/ExecutionAPI.h b/src/execution/ExecutionApi.h similarity index 86% rename from src/ExecutionAPI.h rename to src/execution/ExecutionApi.h index 6c9102b..b1df5d0 100644 --- a/src/ExecutionAPI.h +++ b/src/execution/ExecutionApi.h @@ -1,13 +1,9 @@ -// -// Created by ragnarokk on 31.03.2026. -// +#pragma once -#ifndef COLUMNAR_ENGINE_EXECUTIONAPI_H -#define COLUMNAR_ENGINE_EXECUTIONAPI_H - -#include "AggregationExpressions.h" -#include "ExecutionLogic.h" -#include "FilterExpressions.h" +#include "execution/expressions/AggregationExpressions.h" +#include "execution/expressions/FilterExpressions.h" +#include "execution/ExecutionLogic.h" +#include "execution/ExecutionHelper.h" #include @@ -42,7 +38,7 @@ class DataResult { for (size_t i = 0; i < rows; ++i) { size_t ind = batch->selection_vector ? (*batch->selection_vector)[i] : i; for (const std::shared_ptr& column : batch->columns) { - std::cout << column->GetDataAsString(ind) << " "; + std::cout << ExecutionHelper::FormatValue(*column, ind) << " "; } std::cout << "\n"; } @@ -99,6 +95,13 @@ class DataFrame { return DataFrame(limit_node); } + DataFrame Project(std::vector> scalar_expressions) { + auto scalar_node = std::make_shared(); + scalar_node->child = logical_plan_; + scalar_node->scalar_expressions = std::move(scalar_expressions); + return DataFrame(scalar_node); + } + DataResult Collect() { PhysicalOperatorContext context = BuildPhysicalPlan(logical_plan_); std::unique_ptr physical_plan_root = std::move(context.root_operator); @@ -114,5 +117,3 @@ class DataFrame { private: std::shared_ptr logical_plan_; }; - -#endif // COLUMNAR_ENGINE_EXECUTIONAPI_H diff --git a/src/execution/ExecutionHelper.h b/src/execution/ExecutionHelper.h new file mode 100644 index 0000000..a90728f --- /dev/null +++ b/src/execution/ExecutionHelper.h @@ -0,0 +1,183 @@ +#pragma once + +#include "execution/OperatorsBase.h" +#include "column/Column.h" +#include "column/NumericColumn.h" +#include "column/StringColumn.h" +#include "column/CharColumn.h" +#include "column/TemporalColumn.h" +#include "utils/Macro.h" + +#include +#include +#include +#include + +class ExecutionHelper { +public: + template + static void IterateColumnData(const RecordBatch& batch, size_t column_index, Func&& func) { + auto* column = static_cast(batch.columns[column_index].get()); + const auto& data = column->GetData(); + + if (!batch.selection_vector) { + for (size_t i = 0; i < batch.num_rows; ++i) { + func(data[i], i, i); + } + } else { + const auto& sel = *batch.selection_vector; + for (size_t i = 0; i < batch.num_rows; ++i) { + func(data[sel[i]], i, sel[i]); + } + } + } + + static std::string FormatValue(const Column& col, size_t index) { + ColumnType type = col.GetType(); + if (type == ColumnType::STRING) { + return std::string(static_cast(col).GetData()[index]); + } else if (type == ColumnType::CHAR) { + return std::string(1, static_cast(col).GetData()[index]); + } else if (type == ColumnType::DATE) { + return Date::Format(static_cast(col).GetData()[index]); + } else if (type == ColumnType::TIMESTAMP) { + return Timestamp::Format(static_cast(col).GetData()[index]); + } else { +#define HANDLE_TYPE(ENUM_VAL, STR_VAL, CLASS_TYPE) \ + case ColumnType::ENUM_VAL: { \ + auto val = static_cast(col).GetData()[index]; \ + return std::to_string(val); \ + } + + switch (type) { + HANDLE_TYPE(INT16, "INT16", Int16Column) + HANDLE_TYPE(INT32, "INT32", Int32Column) + HANDLE_TYPE(INT64, "INT64", Int64Column) + HANDLE_TYPE(FLOAT, "FLOAT", FloatColumn) + HANDLE_TYPE(DOUBLE, "DOUBLE", DoubleColumn) + HANDLE_TYPE(LONGDOUBLE, "LONGDOUBLE", LongDoubleColumn) + case ColumnType::INT128: { + Int128 val = static_cast(col).GetData()[index]; + if (val == 0) { + return "0"; + } + std::string res; + bool neg = val < 0; + if (neg) { + val = -val; + } + while (val > 0) { + res += std::to_string(static_cast(val % 10)); + val /= 10; + } + if (neg) { + res += "-"; + } + std::reverse(res.begin(), res.end()); + return res; + } + default: return ""; + } +#undef HANDLE_TYPE + } + } + static void HashKeys(const Column& col, std::vector& keys, + const std::vector* selection = nullptr) { + ColumnType type = col.GetType(); + + if (type == ColumnType::STRING) { + auto* str_col = static_cast(&col); + const auto& data = str_col->GetData(); + size_t n = selection ? selection->size() : data.Size(); + if (keys.capacity() < n) { + keys.reserve(n); + } + for (size_t i = 0; i < n; ++i) { + size_t idx = selection ? (*selection)[i] : i; + std::string_view val = data[idx]; + size_t len = val.size(); + keys[i].append(reinterpret_cast(&len), sizeof(len)); + keys[i].append(val.data(), val.size()); + } + } else if (type == ColumnType::CHAR) { + auto* char_col = static_cast(&col); + const auto& data = char_col->GetData(); + size_t n = selection ? selection->size() : data.size(); + if (keys.capacity() < n) { + keys.reserve(n); + } + for (size_t i = 0; i < n; ++i) { + size_t idx = selection ? (*selection)[i] : i; + keys[i].push_back(data[idx]); + } + } else { + size_t elem_size = 0; + const void* raw_data = col.GetRawData(); + +#define HANDLE_TYPE(ENUM_VAL, STR_VAL, CLASS_TYPE) \ + case ColumnType::ENUM_VAL: \ + elem_size = sizeof(CLASS_TYPE::ValueType); \ + { \ + auto* vec = static_cast(raw_data); \ + size_t n = selection ? selection->size() : vec->size(); \ + if (keys.capacity() < n) \ + keys.reserve(n); \ + for (size_t i = 0; i < n; ++i) { \ + size_t idx = selection ? (*selection)[i] : i; \ + keys[i].append(reinterpret_cast(&(*vec)[idx]), elem_size); \ + } \ + } \ + break; + + switch (type) { + FOR_NUMERIC_COLUMN_TYPE(HANDLE_TYPE) + default: break; + } +#undef HANDLE_TYPE + } + } + + static void CopySelection(Column& dest, const Column& src, + const std::vector* indices = nullptr) { + ColumnType type = src.GetType(); + if (dest.GetType() != type) { + THROW_RUNTIME_ERROR("Type mismatch in CopySelection"); + } + + if (type == ColumnType::STRING) { + auto& d = static_cast(dest); + const auto& s = static_cast(src).GetData(); + if (!indices) { + for (size_t i = 0; i < s.Size(); ++i) { + d.AddValue(s[i]); + } + } else { + for (size_t idx : *indices) { + d.AddValue(s[idx]); + } + } + } else { +#define HANDLE_TYPE(ENUM_VAL, STR_VAL, CLASS_TYPE) \ + case ColumnType::ENUM_VAL: { \ + auto& d = static_cast(dest); \ + auto* vec = static_cast(src.GetRawData()); \ + if (!indices) { \ + for (size_t i = 0; i < vec->size(); ++i) { \ + d.AddValue((*vec)[i]); \ + } \ + } else { \ + for (size_t idx : *indices) { \ + d.AddValue((*vec)[idx]); \ + } \ + } \ + break; \ + } + + switch (type) { + FOR_NUMERIC_COLUMN_TYPE(HANDLE_TYPE) + default: break; + } +#undef HANDLE_TYPE + } + } +}; diff --git a/src/execution/ExecutionLogic.h b/src/execution/ExecutionLogic.h new file mode 100644 index 0000000..b3abfe6 --- /dev/null +++ b/src/execution/ExecutionLogic.h @@ -0,0 +1,294 @@ +#pragma once + +#include "execution/OperatorsBase.h" +#include "execution/expressions/AggregationFunctions.h" +#include "execution/expressions/AggregationExpressions.h" +#include "execution/expressions/FilterFunctions.h" +#include "execution/expressions/FilterExpressions.h" +#include "execution/expressions/ScalarExpressions.h" + +#include "column/Schema.h" +#include "column/NumericColumn.h" +#include "column/StringColumn.h" +#include "column/CharColumn.h" +#include "column/TemporalColumn.h" + +#include "utils/Macro.h" + +#include +#include +#include +#include +#include + +enum class PlanNodeType { + SCAN, + AGGREGATE, + FILTER, + ORDER_BY, + LIMIT, + SCALAR +}; + +struct PlanNode { + PlanNodeType type; +protected: + explicit PlanNode(PlanNodeType node_type) : type(node_type) {} +}; + +struct ScanNode : public PlanNode { + ScanNode() : PlanNode(PlanNodeType::SCAN) {} + std::string table_path; + std::vector column_names; +}; + +struct AggregateNode : public PlanNode { + AggregateNode() : PlanNode(PlanNodeType::AGGREGATE) {} + std::shared_ptr child; + std::vector group_by_columns; + std::vector> aggregate_expressions; +}; + +struct FilterNode : public PlanNode { + FilterNode() : PlanNode(PlanNodeType::FILTER) {} + std::shared_ptr child; + std::shared_ptr filter_expression; +}; + +struct OrderByNode : public PlanNode { + OrderByNode() : PlanNode(PlanNodeType::ORDER_BY) {} + std::shared_ptr child; + std::vector> order_by_columns; + std::optional limit; +}; + +struct LimitNode : public PlanNode { + LimitNode() : PlanNode(PlanNodeType::LIMIT), limit(0) { + } + std::shared_ptr child; + size_t limit; +}; + +struct ScalarNode : public PlanNode { + ScalarNode() : PlanNode(PlanNodeType::SCALAR) {} + std::shared_ptr child; + std::vector> scalar_expressions; +}; + +struct PhysicalOperatorContext { + std::unique_ptr root_operator; + Schema schema; +}; + +inline PhysicalOperatorContext BuildPhysicalPlan( + const std::shared_ptr& plan_node, + std::optional> required_columns = std::nullopt) { + switch (plan_node->type) { + case PlanNodeType::SCAN: { + auto scan = std::static_pointer_cast(plan_node); + std::vector final_columns; + + Schema full_schema = Schema::FromColumnarFile(scan->table_path); + const std::vector& full_columns = full_schema.GetFields(); + + if (required_columns.has_value()) { + final_columns = required_columns.value(); + if (final_columns.empty() && !full_columns.empty()) { + final_columns.push_back(full_columns[0].name); + } + } else { + if (scan->column_names.empty()) { + for (const Field& field : full_columns) { + final_columns.push_back(field.name); + } + } else { + final_columns = scan->column_names; + } + } + + auto op = std::make_unique(scan->table_path, final_columns); + + Schema output_schema; + for (const std::string& name : final_columns) { + bool found = false; + for (const Field& field : full_columns) { + if (field.name == name) { + output_schema.AddField({field.name, field.type}); + found = true; + break; + } + } + if (!found) [[unlikely]] { + THROW_RUNTIME_ERROR("Column " + name + " not found in table schema"); + } + } + + return {std::move(op), std::move(output_schema)}; + } + + case PlanNodeType::AGGREGATE: { + auto aggregate = std::static_pointer_cast(plan_node); + if (!aggregate->group_by_columns.empty()) { + std::vector needed_columns = aggregate->group_by_columns; + for (const std::shared_ptr& expr : + aggregate->aggregate_expressions) { + expr->CollectRequiredColumns(needed_columns); + } + std::sort(needed_columns.begin(), needed_columns.end()); + needed_columns.erase(std::unique(needed_columns.begin(), needed_columns.end()), + needed_columns.end()); + + PhysicalOperatorContext child_context = + BuildPhysicalPlan(aggregate->child, needed_columns); + + std::vector group_col_indices; + std::vector> empty_key_columns; + Schema output_schema; + + for (const auto& col_name : aggregate->group_by_columns) { + size_t ind = child_context.schema.GetColumnIndexByName(col_name); + group_col_indices.push_back(ind); + + ColumnType type = child_context.schema.GetColumnTypeByName(col_name); + output_schema.AddColumn(col_name, type); + +#define HANDLE_TYPE(ENUM_VAL, STR_VAL, CLASS_TYPE) \ + case ColumnType::ENUM_VAL: empty_key_columns.push_back(std::make_shared()); break; + + switch (type) { + FOR_EACH_COLUMN_TYPE(HANDLE_TYPE); + default: THROW_NOT_IMPLEMENTED; + } +#undef HANDLE_TYPE + } + + std::vector> agg_funcs; + for (const std::shared_ptr& expr : + aggregate->aggregate_expressions) { + agg_funcs.push_back( + expr->CreateGroupedAggregationFunction(child_context.schema, output_schema)); + } + + auto op = std::make_unique( + std::move(child_context.root_operator), std::move(group_col_indices), + std::move(empty_key_columns), std::move(agg_funcs)); + return {std::move(op), std::move(output_schema)}; + } + + std::vector needed_columns = aggregate->group_by_columns; + for (const std::shared_ptr& expr : aggregate->aggregate_expressions) { + expr->CollectRequiredColumns(needed_columns); + } + std::sort(needed_columns.begin(), needed_columns.end()); + needed_columns.erase(std::unique(needed_columns.begin(), needed_columns.end()), + needed_columns.end()); + + PhysicalOperatorContext child_context = BuildPhysicalPlan(aggregate->child, needed_columns); + std::vector> agg_functions; + Schema output_schema; + for (const std::shared_ptr& expr : aggregate->aggregate_expressions) { + agg_functions.push_back( + expr->CreateGlobalAggregationFunction(child_context.schema, output_schema)); + } + auto op = std::make_unique(std::move(child_context.root_operator), + std::move(agg_functions)); + return {std::move(op), std::move(output_schema)}; + } + + case PlanNodeType::FILTER: { + auto filter = std::static_pointer_cast(plan_node); + std::vector needed_columns; + filter->filter_expression->CollectRequiredColumns(needed_columns); + if (required_columns.has_value()) { + needed_columns.insert(needed_columns.end(), required_columns->begin(), + required_columns->end()); + } + + std::sort(needed_columns.begin(), needed_columns.end()); + needed_columns.erase(std::unique(needed_columns.begin(), needed_columns.end()), + needed_columns.end()); + + PhysicalOperatorContext child_context = BuildPhysicalPlan(filter->child, needed_columns); + std::unique_ptr filter_function = + filter->filter_expression->CreateFilterFunction(child_context.schema); + auto op = std::make_unique(std::move(child_context.root_operator), + std::move(filter_function)); + return {std::move(op), std::move(child_context.schema)}; + } + + case PlanNodeType::ORDER_BY: { + auto order_by = std::static_pointer_cast(plan_node); + std::vector needed_columns; + if (required_columns.has_value()) { + needed_columns = required_columns.value(); + } + for (const auto& [col_name, is_desc] : order_by->order_by_columns) { + needed_columns.push_back(col_name); + } + std::sort(needed_columns.begin(), needed_columns.end()); + needed_columns.erase(std::unique(needed_columns.begin(), needed_columns.end()), + needed_columns.end()); + + PhysicalOperatorContext child_context = BuildPhysicalPlan(order_by->child, needed_columns); + std::vector> sort_columns; + for (const auto& [col_name, is_desc] : order_by->order_by_columns) { + size_t sort_col_idx = child_context.schema.GetColumnIndexByName(col_name); + sort_columns.push_back({sort_col_idx, is_desc}); + } + + auto op = + std::make_unique(std::move(child_context.root_operator), + std::move(sort_columns), std::move(order_by->limit)); + return {std::move(op), std::move(child_context.schema)}; + } + + case PlanNodeType::LIMIT: { + auto limit = std::static_pointer_cast(plan_node); + PhysicalOperatorContext child_context = BuildPhysicalPlan(limit->child, required_columns); + auto op = + std::make_unique(std::move(child_context.root_operator), limit->limit); + return {std::move(op), std::move(child_context.schema)}; + } + + case PlanNodeType::SCALAR: { + auto scalar = std::static_pointer_cast(plan_node); + std::vector needed_columns; + if (required_columns.has_value()) { + for (const auto& req_col : required_columns.value()) { + bool is_generated_here = false; + for (const auto& expr : scalar->scalar_expressions) { + if (expr->GetOutputName() == req_col) { + is_generated_here = true; + break; + } + } + if (!is_generated_here) { + needed_columns.push_back(req_col); + } + } + } + for (const auto& expr : scalar->scalar_expressions) { + expr->CollectRequiredColumns(needed_columns); + } + + std::sort(needed_columns.begin(), needed_columns.end()); + needed_columns.erase(std::unique(needed_columns.begin(), needed_columns.end()), + needed_columns.end()); + + PhysicalOperatorContext child_context = BuildPhysicalPlan(scalar->child, needed_columns); + + std::vector> scalar_functions; + + for (const auto& expr : scalar->scalar_expressions) { + scalar_functions.push_back(expr->CreateScalarFunction(child_context.schema)); + } + + auto op = std::make_unique(std::move(child_context.root_operator), + std::move(scalar_functions)); + return {std::move(op), std::move(child_context.schema)}; + } + + default: + THROW_RUNTIME_ERROR("Unknown plan node type"); + } +} diff --git a/src/OperatorsBase.cpp b/src/execution/OperatorsBase.cpp similarity index 83% rename from src/OperatorsBase.cpp rename to src/execution/OperatorsBase.cpp index ef8e42a..f03ea6d 100644 --- a/src/OperatorsBase.cpp +++ b/src/execution/OperatorsBase.cpp @@ -1,10 +1,9 @@ -// -// Created by ragnarokk on 31.03.2026. -// - -#include "AggregationFunctions.h" -#include "FilterFunctions.h" -#include "OperatorsBase.h" +#include "execution/expressions/AggregationFunctions.h" +#include "execution/expressions/FilterFunctions.h" +#include "execution/OperatorsBase.h" +#include "execution/expressions/ScalarFunctions.h" +#include "execution/ExecutionHelper.h" +#include "column/ColumnFactory.h" #include @@ -60,7 +59,7 @@ std::unique_ptr ScanOperator::Run() { AggregationOperator::AggregationOperator( std::unique_ptr child, - std::vector> aggregation_functions) + std::vector> aggregation_functions) : child_(std::move(child)), aggregation_functions_(std::move(aggregation_functions)) { } @@ -69,14 +68,14 @@ std::unique_ptr AggregationOperator::Run() { return nullptr; } while (std::unique_ptr batch = child_->Run()) { - for (std::unique_ptr& aggregation_function : aggregation_functions_) { + for (std::unique_ptr& aggregation_function : aggregation_functions_) { aggregation_function->Update(*batch); } } auto result_batch = std::make_unique(); result_batch->num_rows = 1; - for (std::unique_ptr& aggregation_function : aggregation_functions_) { + for (std::unique_ptr& aggregation_function : aggregation_functions_) { result_batch->columns.push_back(aggregation_function->Finalize()); } @@ -106,7 +105,7 @@ std::unique_ptr FilterOperator::Run() { GroupByOperator::GroupByOperator(std::unique_ptr child, std::vector group_by_col_indices, std::vector> empty_key_columns, - std::vector> agg_funcs) + std::vector> agg_funcs) : child_(std::move(child)), group_by_col_indices_(std::move(group_by_col_indices)), key_columns_(std::move(empty_key_columns)), @@ -123,7 +122,7 @@ std::unique_ptr GroupByOperator::Run() { const std::shared_ptr>& sel_vec = batch->selection_vector; std::vector composite_keys(batch_size); for (size_t col_idx : group_by_col_indices_) { - batch->columns[col_idx]->SerializeBatch(composite_keys, sel_vec.get()); + ExecutionHelper::HashKeys(*batch->columns[col_idx], composite_keys, sel_vec.get()); } std::vector new_group_indices; @@ -148,13 +147,14 @@ std::unique_ptr GroupByOperator::Run() { if (!new_group_indices.empty()) { for (size_t k = 0; k < group_by_col_indices_.size(); ++k) { - key_columns_[k]->CopyBatchFrom(*batch->columns[group_by_col_indices_[k]], - &new_group_indices); + ExecutionHelper::CopySelection(*key_columns_[k], *batch->columns[group_by_col_indices_[k]], &new_group_indices); } } + size_t total_groups = hash_table_.size(); for (auto& func : agg_funcs_) { - func->Update(*batch, &group_ids); + func->Resize(total_groups); + func->Update(*batch, group_ids); } } accumulated_ = true; @@ -210,7 +210,7 @@ std::unique_ptr OrderByOperator::Run() { accumulated_batch_ = std::make_unique(); size_t sz = batch->columns.size(); for (size_t c = 0; c < sz; ++c) { - accumulated_batch_->columns.push_back(batch->columns[c]->CloneEmpty()); + accumulated_batch_->columns.push_back(ColumnFactory::MakeColumn(batch->columns[c]->GetType())); } accumulated_batch_->num_rows = 0; } @@ -218,7 +218,7 @@ std::unique_ptr OrderByOperator::Run() { const std::shared_ptr>& sel = batch->selection_vector; size_t sz = batch->columns.size(); for (size_t c = 0; c < sz; ++c) { - accumulated_batch_->columns[c]->CopyBatchFrom(*batch->columns[c], sel.get()); + ExecutionHelper::CopySelection(*accumulated_batch_->columns[c], *batch->columns[c], sel.get()); } accumulated_batch_->num_rows += batch->num_rows; } @@ -232,8 +232,8 @@ std::unique_ptr OrderByOperator::Run() { std::vector cols_info; cols_info.reserve(sort_columns_.size()); - for (const auto& [col_idx, is_desc] : sort_columns_) { - const Column* raw_column = accumulated_batch_->columns[col_idx].get(); + for (const auto& [col_ind, is_desc] : sort_columns_) { + const Column* raw_column = accumulated_batch_->columns[col_ind].get(); #define HANDLE_TYPE(ENUM_VAL, STR_VAL, CLASS_TYPE) \ case ColumnType::ENUM_VAL: { \ @@ -272,8 +272,8 @@ std::unique_ptr OrderByOperator::Run() { auto sorted_batch = std::make_unique(); sorted_batch->num_rows = accumulated_batch_->num_rows; for (size_t c = 0; c < accumulated_batch_->columns.size(); ++c) { - auto new_col = accumulated_batch_->columns[c]->CloneEmpty(); - new_col->CopyBatchFrom(*accumulated_batch_->columns[c], &indices_); + auto new_col = ColumnFactory::MakeColumn(accumulated_batch_->columns[c]->GetType()); + ExecutionHelper::CopySelection(*new_col, *accumulated_batch_->columns[c], &indices_); sorted_batch->columns.push_back(std::move(new_col)); } accumulated_batch_ = std::move(sorted_batch); @@ -323,3 +323,21 @@ std::unique_ptr LimitOperator::Run() { return batch; } } + +ScalarOperator::ScalarOperator(std::unique_ptr child, + std::vector> scalar_functions) + : child_(std::move(child)), scalar_functions_(std::move(scalar_functions)) { +} + +std::unique_ptr ScalarOperator::Run() { + auto batch = child_->Run(); + if (!batch) { + return nullptr; + } + + for (auto& func : scalar_functions_) { + batch->columns.push_back(func->Evaluate(*batch)); + } + + return batch; +} diff --git a/src/OperatorsBase.h b/src/execution/OperatorsBase.h similarity index 76% rename from src/OperatorsBase.h rename to src/execution/OperatorsBase.h index e09c3d6..d14ddc6 100644 --- a/src/OperatorsBase.h +++ b/src/execution/OperatorsBase.h @@ -1,22 +1,16 @@ -// -// Created by ragnarokk on 06.01.2026. -// +#pragma once -#ifndef COLUMNAR_ENGINE_OPERATORS_BASE_H -#define COLUMNAR_ENGINE_OPERATORS_BASE_H +#include "io/ColumnarReader.h" #include #include #include #include -#include "ColumnarReader.h" -#include "macro.h" -#include "Types.h" - -class AggregationFunction; +class GlobalAggregationFunction; class GroupedAggregationFunction; class FilterFunction; +class ScalarFunction; struct RecordBatch { size_t num_rows; @@ -47,13 +41,13 @@ class ScanOperator : public Operator { class AggregationOperator : public Operator { public: AggregationOperator(std::unique_ptr child, - std::vector> aggregation_functions); + std::vector> aggregation_functions); std::unique_ptr Run() override; private: std::unique_ptr child_; - std::vector> aggregation_functions_; + std::vector> aggregation_functions_; bool finished_ = false; }; @@ -73,7 +67,7 @@ class GroupByOperator : public Operator { public: GroupByOperator(std::unique_ptr child, std::vector group_by_col_indices, std::vector> empty_key_columns, - std::vector> agg_funcs); + std::vector> agg_funcs); std::unique_ptr Run() override; @@ -81,7 +75,7 @@ class GroupByOperator : public Operator { std::unique_ptr child_; std::vector group_by_col_indices_; std::vector> key_columns_; - std::vector> agg_funcs_; + std::vector> agg_funcs_; bool accumulated_ = false; std::unordered_map hash_table_; @@ -117,4 +111,14 @@ class LimitOperator : public Operator { size_t cur_rows_ = 0; }; -#endif // COLUMNAR_ENGINE_OPERATORS_BASE_H +class ScalarOperator : public Operator { +public: + ScalarOperator(std::unique_ptr child, + std::vector> scalar_functions); + + std::unique_ptr Run() override; + +private: + std::unique_ptr child_; + std::vector> scalar_functions_; +}; diff --git a/src/AggregationExpressions.h b/src/execution/expressions/AggregationExpressions.h similarity index 50% rename from src/AggregationExpressions.h rename to src/execution/expressions/AggregationExpressions.h index ba08624..8102b2d 100644 --- a/src/AggregationExpressions.h +++ b/src/execution/expressions/AggregationExpressions.h @@ -1,12 +1,7 @@ -// -// Created by ragnarokk on 31.03.2026. -// +#pragma once -#ifndef COLUMNAR_ENGINE_EXPRESSIONS_H -#define COLUMNAR_ENGINE_EXPRESSIONS_H - -#include "AggregationFunctions.h" -#include "Schema.h" +#include "execution/expressions/AggregationFunctions.h" +#include "column/Schema.h" #include #include @@ -34,7 +29,10 @@ class AggregateExpression { return output_name_.empty() ? column_name_ : output_name_; } - virtual std::unique_ptr CreateAggregationFunction( + virtual std::unique_ptr CreateGlobalAggregationFunction( + const Schema& child_schema, Schema& output_schema) = 0; + + virtual std::unique_ptr CreateGroupedAggregationFunction( const Schema& child_schema, Schema& output_schema) = 0; protected: @@ -48,10 +46,16 @@ class CountExpression : public AggregateExpression { : AggregateExpression(std::move(column_name), std::move(output_name)) { } - std::unique_ptr CreateAggregationFunction( + std::unique_ptr CreateGlobalAggregationFunction( [[maybe_unused]] const Schema& child_schema, Schema& output_schema) override { output_schema.AddColumn(GetOutputName(), ColumnType::INT32); - return std::make_unique(); + return std::make_unique(); + } + + std::unique_ptr CreateGroupedAggregationFunction( + [[maybe_unused]] const Schema& child_schema, Schema& output_schema) override { + output_schema.AddColumn(GetOutputName(), ColumnType::INT32); + return std::make_unique(); } }; @@ -68,15 +72,32 @@ class DistinctCountExpression : public AggregateExpression { return "distinct_count_" + column_name_; } - std::unique_ptr CreateAggregationFunction(const Schema& child_schema, - Schema& output_schema) override { + std::unique_ptr CreateGlobalAggregationFunction(const Schema& child_schema, + Schema& output_schema) override { + size_t column_ind = child_schema.GetColumnIndexByName(GetName()); + ColumnType column_type = child_schema.GetColumnTypeByName(GetName()); + output_schema.AddColumn(GetOutputName(), ColumnType::INT32); + +#define HANDLE_TYPE(ENUM_VAL, STR_VAL, CLASS_TYPE) \ + case ColumnType::ENUM_VAL: \ + return std::make_unique>(column_ind); + + switch (column_type) { + FOR_EACH_COLUMN_TYPE(HANDLE_TYPE); + default: THROW_NOT_IMPLEMENTED; + } +#undef HANDLE_TYPE + } + + std::unique_ptr CreateGroupedAggregationFunction(const Schema& child_schema, + Schema& output_schema) override { size_t column_ind = child_schema.GetColumnIndexByName(GetName()); ColumnType column_type = child_schema.GetColumnTypeByName(GetName()); - output_schema.AddColumn(GetOutputName(), ColumnType::INT64); + output_schema.AddColumn(GetOutputName(), ColumnType::INT32); #define HANDLE_TYPE(ENUM_VAL, STR_VAL, CLASS_TYPE) \ case ColumnType::ENUM_VAL: \ - return std::make_unique>(column_ind); + return std::make_unique>(column_ind); switch (column_type) { FOR_EACH_COLUMN_TYPE(HANDLE_TYPE); @@ -99,22 +120,37 @@ class MinExpression : public AggregateExpression { return "min_" + column_name_; } - std::unique_ptr CreateAggregationFunction(const Schema& child_schema, - Schema& output_schema) override { + std::unique_ptr CreateGlobalAggregationFunction(const Schema& child_schema, + Schema& output_schema) override { size_t column_ind = child_schema.GetColumnIndexByName(GetName()); ColumnType column_type = child_schema.GetColumnTypeByName(GetName()); #define HANDLE_TYPE(ENUM_VAL, STR_VAL, CLASS_TYPE) \ case ColumnType::ENUM_VAL: \ output_schema.AddColumn(GetOutputName(), ColumnType::ENUM_VAL); \ - return std::make_unique>(column_ind); - // END_HANDLE_TYPE + return std::make_unique>(column_ind); switch (column_type) { FOR_NUMERIC_COLUMN_TYPE(HANDLE_TYPE); default: THROW_NOT_IMPLEMENTED; } +#undef HANDLE_TYPE + } + + std::unique_ptr CreateGroupedAggregationFunction(const Schema& child_schema, + Schema& output_schema) override { + size_t column_ind = child_schema.GetColumnIndexByName(GetName()); + ColumnType column_type = child_schema.GetColumnTypeByName(GetName()); +#define HANDLE_TYPE(ENUM_VAL, STR_VAL, CLASS_TYPE) \ + case ColumnType::ENUM_VAL: \ + output_schema.AddColumn(GetOutputName(), ColumnType::ENUM_VAL); \ + return std::make_unique>(column_ind); + + switch (column_type) { + FOR_NUMERIC_COLUMN_TYPE(HANDLE_TYPE); + default: THROW_NOT_IMPLEMENTED; + } #undef HANDLE_TYPE } }; @@ -132,22 +168,37 @@ class MaxExpression : public AggregateExpression { return "max_" + column_name_; } - std::unique_ptr CreateAggregationFunction(const Schema& child_schema, - Schema& output_schema) override { + std::unique_ptr CreateGlobalAggregationFunction(const Schema& child_schema, + Schema& output_schema) override { size_t column_ind = child_schema.GetColumnIndexByName(GetName()); ColumnType column_type = child_schema.GetColumnTypeByName(GetName()); #define HANDLE_TYPE(ENUM_VAL, STR_VAL, CLASS_TYPE) \ case ColumnType::ENUM_VAL: \ output_schema.AddColumn(GetOutputName(), ColumnType::ENUM_VAL); \ - return std::make_unique>(column_ind); \ - // END_HANDLE_TYPE + return std::make_unique>(column_ind); switch (column_type) { FOR_NUMERIC_COLUMN_TYPE(HANDLE_TYPE); default: THROW_NOT_IMPLEMENTED; } +#undef HANDLE_TYPE + } + std::unique_ptr CreateGroupedAggregationFunction(const Schema& child_schema, + Schema& output_schema) override { + size_t column_ind = child_schema.GetColumnIndexByName(GetName()); + ColumnType column_type = child_schema.GetColumnTypeByName(GetName()); + +#define HANDLE_TYPE(ENUM_VAL, STR_VAL, CLASS_TYPE) \ + case ColumnType::ENUM_VAL: \ + output_schema.AddColumn(GetOutputName(), ColumnType::ENUM_VAL); \ + return std::make_unique>(column_ind); + + switch (column_type) { + FOR_NUMERIC_COLUMN_TYPE(HANDLE_TYPE); + default: THROW_NOT_IMPLEMENTED; + } #undef HANDLE_TYPE } }; @@ -165,33 +216,64 @@ class SumExpression : public AggregateExpression { return "sum_" + column_name_; } - std::unique_ptr CreateAggregationFunction(const Schema& child_schema, - Schema& output_schema) override { + std::unique_ptr CreateGlobalAggregationFunction(const Schema& child_schema, + Schema& output_schema) override { const size_t column_ind = child_schema.GetColumnIndexByName(GetName()); const ColumnType column_type = child_schema.GetColumnTypeByName(GetName()); switch (column_type) { case ColumnType::INT16: output_schema.AddColumn(GetOutputName(), ColumnType::INT32); - return std::make_unique>(column_ind); + return std::make_unique>(column_ind); case ColumnType::INT32: output_schema.AddColumn(GetOutputName(), ColumnType::INT64); - return std::make_unique>(column_ind); + return std::make_unique>(column_ind); case ColumnType::INT64: output_schema.AddColumn(GetOutputName(), ColumnType::INT128); - return std::make_unique>(column_ind); + return std::make_unique>(column_ind); case ColumnType::INT128: output_schema.AddColumn(GetOutputName(), ColumnType::INT128); - return std::make_unique>(column_ind); + return std::make_unique>(column_ind); case ColumnType::FLOAT: output_schema.AddColumn(GetOutputName(), ColumnType::DOUBLE); - return std::make_unique>(column_ind); + return std::make_unique>(column_ind); case ColumnType::DOUBLE: output_schema.AddColumn(GetOutputName(), ColumnType::LONGDOUBLE); - return std::make_unique>(column_ind); + return std::make_unique>(column_ind); case ColumnType::LONGDOUBLE: output_schema.AddColumn(GetOutputName(), ColumnType::LONGDOUBLE); - return std::make_unique>(column_ind); + return std::make_unique>(column_ind); + default: THROW_NOT_IMPLEMENTED; + } + } + + std::unique_ptr CreateGroupedAggregationFunction(const Schema& child_schema, + Schema& output_schema) override { + const size_t column_ind = child_schema.GetColumnIndexByName(GetName()); + const ColumnType column_type = child_schema.GetColumnTypeByName(GetName()); + + switch (column_type) { + case ColumnType::INT16: + output_schema.AddColumn(GetOutputName(), ColumnType::INT32); + return std::make_unique>(column_ind); + case ColumnType::INT32: + output_schema.AddColumn(GetOutputName(), ColumnType::INT64); + return std::make_unique>(column_ind); + case ColumnType::INT64: + output_schema.AddColumn(GetOutputName(), ColumnType::INT128); + return std::make_unique>(column_ind); + case ColumnType::INT128: + output_schema.AddColumn(GetOutputName(), ColumnType::INT128); + return std::make_unique>(column_ind); + case ColumnType::FLOAT: + output_schema.AddColumn(GetOutputName(), ColumnType::DOUBLE); + return std::make_unique>(column_ind); + case ColumnType::DOUBLE: + output_schema.AddColumn(GetOutputName(), ColumnType::LONGDOUBLE); + return std::make_unique>(column_ind); + case ColumnType::LONGDOUBLE: + output_schema.AddColumn(GetOutputName(), ColumnType::LONGDOUBLE); + return std::make_unique>(column_ind); default: THROW_NOT_IMPLEMENTED; } } @@ -209,24 +291,39 @@ class AvgExpression : public AggregateExpression { return "avg_" + column_name_; } - std::unique_ptr CreateAggregationFunction(const Schema& child_schema, - Schema& output_schema) override { + std::unique_ptr CreateGlobalAggregationFunction(const Schema& child_schema, + Schema& output_schema) override { const size_t column_ind = child_schema.GetColumnIndexByName(GetName()); const ColumnType column_type = child_schema.GetColumnTypeByName(GetName()); - output_schema.AddColumn(GetOutputName(), ColumnType::LONGDOUBLE); + output_schema.AddColumn(GetOutputName(), ColumnType::DOUBLE); #define HANDLE_TYPE(ENUM_VAL, STR_VAL, CLASS_TYPE) \ case ColumnType::ENUM_VAL: \ - return std::make_unique>(column_ind); - // END_HANDLE_TYPE + return std::make_unique>(column_ind); switch (column_type) { FOR_NUMERIC_COLUMN_TYPE(HANDLE_TYPE); default: THROW_NOT_IMPLEMENTED; } +#undef HANDLE_TYPE } + std::unique_ptr CreateGroupedAggregationFunction(const Schema& child_schema, + Schema& output_schema) override { + const size_t column_ind = child_schema.GetColumnIndexByName(GetName()); + const ColumnType column_type = child_schema.GetColumnTypeByName(GetName()); + output_schema.AddColumn(GetOutputName(), ColumnType::DOUBLE); + +#define HANDLE_TYPE(ENUM_VAL, STR_VAL, CLASS_TYPE) \ + case ColumnType::ENUM_VAL: \ + return std::make_unique>(column_ind); + + switch (column_type) { + FOR_NUMERIC_COLUMN_TYPE(HANDLE_TYPE); + default: THROW_NOT_IMPLEMENTED; + } #undef HANDLE_TYPE + } }; inline std::shared_ptr Count(std::string column_name = "*", @@ -254,5 +351,3 @@ inline std::shared_ptr DistinctCount(std::string column_nam return std::make_shared(std::move(column_name), std::move(output_name)); } - -#endif // COLUMNAR_ENGINE_EXPRESSIONS_H diff --git a/src/execution/expressions/AggregationFunctions.h b/src/execution/expressions/AggregationFunctions.h new file mode 100644 index 0000000..691fbef --- /dev/null +++ b/src/execution/expressions/AggregationFunctions.h @@ -0,0 +1,339 @@ +#pragma once + +#include "execution/ExecutionHelper.h" +#include "execution/OperatorsBase.h" + +#include "column/Column.h" +#include "column/CharColumn.h" +#include "column/NumericColumn.h" +#include "column/StringColumn.h" +#include "column/TemporalColumn.h" + +#include +#include +#include +#include + +class GlobalAggregationFunction { +public: + virtual ~GlobalAggregationFunction() = default; + virtual void Update(const RecordBatch& batch) = 0; + virtual std::shared_ptr Finalize() = 0; +}; + +class GroupedAggregationFunction { +public: + virtual ~GroupedAggregationFunction() = default; + virtual void Resize(size_t num_groups) = 0; + virtual void Update(const RecordBatch& batch, const std::vector& group_ids) = 0; + virtual std::shared_ptr Finalize() = 0; +}; + +template +struct AggregationFunctionTraits; + +#define DEFINE_AGG_TRAIT(COL_TYPE, VAL_TYPE, STATE_TYPE, RES_TYPE) \ + template <> \ + struct AggregationFunctionTraits { \ + using ValueType = VAL_TYPE; \ + using StateType = STATE_TYPE; \ + using ResultColumnType = RES_TYPE; \ + } + +DEFINE_AGG_TRAIT(Int16Column, int16_t, int32_t, Int32Column); +DEFINE_AGG_TRAIT(Int32Column, int32_t, int64_t, Int64Column); +DEFINE_AGG_TRAIT(Int64Column, int64_t, Int128, Int128Column); +DEFINE_AGG_TRAIT(Int128Column, Int128, Int128, Int128Column); +DEFINE_AGG_TRAIT(FloatColumn, float, double, DoubleColumn); +DEFINE_AGG_TRAIT(DoubleColumn, double, long double, LongDoubleColumn); +DEFINE_AGG_TRAIT(LongDoubleColumn, long double, long double, LongDoubleColumn); +DEFINE_AGG_TRAIT(CharColumn, char, int16_t, Int16Column); +DEFINE_AGG_TRAIT(DateColumn, int32_t, int32_t, DateColumn); +DEFINE_AGG_TRAIT(TimestampColumn, int64_t, int64_t, TimestampColumn); + +template <> +struct AggregationFunctionTraits { + using ValueType = std::string; +}; + +#undef DEFINE_AGG_TRAIT + +struct SumOperation { + template + static void Apply(ResultType& result, const ValueType& value) { + result += value; + } + template + static constexpr ResultType GetInitValue() { + return 0; + } +}; + +struct MaxOperation { + template + static void Apply(ResultType& result, const ValueType& value) { + if (value > result) { + result = value; + } + } + template + static constexpr ResultType GetInitValue() { + return std::numeric_limits::lowest(); + } +}; + +struct MinOperation { + template + static inline void Apply(ResultType& result, const ValueType& value) { + if (value < result) { + result = value; + } + } + template + static constexpr ResultType GetInitValue() { + return std::numeric_limits::max(); + } +}; + +template +class TypedGlobalAggregationFunction : public GlobalAggregationFunction { + using ResultColumnType = AggregationFunctionTraits::ResultColumnType; + using StateType = AggregationFunctionTraits::StateType; + +public: + explicit TypedGlobalAggregationFunction(size_t column_index) : column_index_(column_index) { + state_ = Operation::template GetInitValue(); + } + + void Update(const RecordBatch& batch) override { + ExecutionHelper::IterateColumnData(batch, column_index_, [&](const auto& value, size_t, size_t) { + Operation::Apply(state_, value); + }); + has_data_ = true; + } + + std::shared_ptr Finalize() override { + auto result_column = std::make_shared(); + result_column->AddValue(has_data_ ? state_ : Operation::template GetInitValue()); + return result_column; + } + +private: + size_t column_index_; + StateType state_; + bool has_data_ = false; +}; + +template +class TypedGroupedAggregationFunction : public GroupedAggregationFunction { + using ResultColumnType = AggregationFunctionTraits::ResultColumnType; + using StateType = AggregationFunctionTraits::StateType; + +public: + explicit TypedGroupedAggregationFunction(size_t column_index) : column_index_(column_index) { + } + + void Resize(size_t num_groups) override { + if (num_groups > states_.size()) { + states_.resize(num_groups, Operation::template GetInitValue()); + } + } + + void Update(const RecordBatch& batch, const std::vector& group_ids) override { + ExecutionHelper::IterateColumnData(batch, column_index_, [&](const auto& value, size_t row_idx, size_t) { + uint32_t gid = group_ids[row_idx]; + Operation::Apply(states_[gid], value); + }); + } + + std::shared_ptr Finalize() override { + auto result_column = std::make_shared(); + for (const auto& state : states_) { + result_column->AddValue(state); + } + return result_column; + } + +private: + size_t column_index_; + std::vector states_; +}; + +class CountGlobalAggregationFunction : public GlobalAggregationFunction { +public: + void Update(const RecordBatch& batch) override { + count_ += batch.num_rows; + } + std::shared_ptr Finalize() override { + auto result_column = std::make_shared(); + result_column->AddValue(count_); + return result_column; + } + +private: + int64_t count_ = 0; +}; + +class CountGroupedAggregationFunction : public GroupedAggregationFunction { +public: + void Resize(size_t num_groups) override { + if (num_groups > counts_.size()) { + counts_.resize(num_groups, 0); + } + } + + void Update(const RecordBatch& batch, const std::vector& group_ids) override { + for (size_t i = 0; i < batch.num_rows; ++i) { + uint32_t gid = group_ids[i]; + ++counts_[gid]; + } + } + std::shared_ptr Finalize() override { + auto result_column = std::make_shared(); + for (int64_t c : counts_) { + result_column->AddValue(c); + } + return result_column; + } + +private: + std::vector counts_; +}; + +template +class AvgGlobalAggregationFunction : public GlobalAggregationFunction { + using StateType = AggregationFunctionTraits::StateType; + +public: + explicit AvgGlobalAggregationFunction(size_t column_index) : column_index_(column_index) { + } + + void Update(const RecordBatch& batch) override { + ExecutionHelper::IterateColumnData(batch, column_index_, [&](const auto& value, size_t, size_t) { + sum_ += value; + ++count_; + }); + has_data_ = true; + } + + std::shared_ptr Finalize() override { + auto result_column = std::make_shared(); + if (has_data_ && count_ > 0) { + result_column->AddValue(static_cast(sum_) / static_cast(count_)); + } else { + result_column->AddValue(0.0); + } + return result_column; + } + +private: + size_t column_index_; + StateType sum_ = 0; + int64_t count_ = 0; + bool has_data_ = false; +}; + +template +class AvgGroupedAggregationFunction : public GroupedAggregationFunction { + using StateType = AggregationFunctionTraits::StateType; + +public: + explicit AvgGroupedAggregationFunction(size_t column_index) : column_index_(column_index) { + } + + void Resize(size_t num_groups) override { + if (num_groups > states_.size()) { + states_.resize(num_groups); + } + } + + void Update(const RecordBatch& batch, const std::vector& group_ids) override { + ExecutionHelper::IterateColumnData(batch, column_index_, [&](const auto& value, size_t row_idx, size_t) { + uint32_t gid = group_ids[row_idx]; + states_[gid].sum += value; + ++states_[gid].count; + }); + } + + std::shared_ptr Finalize() override { + auto result_column = std::make_shared(); + for (const auto& state : states_) { + if (state.count > 0) { + result_column->AddValue(static_cast(state.sum) / + static_cast(state.count)); + } else { + result_column->AddValue(0.0); + } + } + return result_column; + } + +private: + struct State { + StateType sum = 0; + int64_t count = 0; + }; + size_t column_index_; + std::vector states_; +}; + +template +class DistinctCountGlobalAggregationFunction : public GlobalAggregationFunction { + using ValueType = AggregationFunctionTraits::ValueType; + +public: + explicit DistinctCountGlobalAggregationFunction(size_t column_index) + : column_index_(column_index) { + } + + void Update(const RecordBatch& batch) override { + ExecutionHelper::IterateColumnData(batch, column_index_, [&](const auto& value, size_t, size_t) { + distinct_values_.insert(ValueType(value)); + }); + } + + std::shared_ptr Finalize() override { + auto result_column = std::make_shared(); + result_column->AddValue(static_cast(distinct_values_.size())); + return result_column; + } + +private: + size_t column_index_; + std::unordered_set distinct_values_; +}; + +template +class DistinctCountGroupedAggregationFunction : public GroupedAggregationFunction { + using ValueType = AggregationFunctionTraits::ValueType; + +public: + explicit DistinctCountGroupedAggregationFunction(size_t column_index) + : column_index_(column_index) { + } + + void Resize(size_t num_groups) override { + if (num_groups > distinct_values_.size()) { + distinct_values_.resize(num_groups); + } + } + + void Update(const RecordBatch& batch, const std::vector& group_ids) override { + ExecutionHelper::IterateColumnData(batch, column_index_, [&](const auto& value, size_t row_idx, size_t) { + uint32_t gid = group_ids[row_idx]; + distinct_values_[gid].insert(ValueType(value)); + }); + } + + std::shared_ptr Finalize() override { + auto result_column = std::make_shared(); + for (const auto& set : distinct_values_) { + result_column->AddValue(static_cast(set.size())); + } + return result_column; + } + +private: + size_t column_index_; + std::vector> distinct_values_; +}; diff --git a/src/FilterExpressions.h b/src/execution/expressions/FilterExpressions.h similarity index 94% rename from src/FilterExpressions.h rename to src/execution/expressions/FilterExpressions.h index a6a693d..4f4a71d 100644 --- a/src/FilterExpressions.h +++ b/src/execution/expressions/FilterExpressions.h @@ -1,13 +1,8 @@ -// -// Created by ragnarokk on 21.04.2026. -// - -#ifndef COLUMNAR_ENGINE_FILTEREXPRESSIONS_H -#define COLUMNAR_ENGINE_FILTEREXPRESSIONS_H +#pragma once #include "FilterFunctions.h" -#include "macro.h" -#include "Schema.h" +#include "utils/Macro.h" +#include "column/Schema.h" #include #include @@ -110,5 +105,3 @@ inline std::shared_ptr Eq(const std::string& column_name, T ta return std::make_shared>(column_name, std::move(target_val)); } } - -#endif // COLUMNAR_ENGINE_FILTEREXPRESSIONS_H diff --git a/src/execution/expressions/FilterFunctions.h b/src/execution/expressions/FilterFunctions.h new file mode 100644 index 0000000..ad3c01c --- /dev/null +++ b/src/execution/expressions/FilterFunctions.h @@ -0,0 +1,58 @@ +#pragma once + +#include "execution/OperatorsBase.h" +#include "execution/ExecutionHelper.h" + +class FilterFunction { +public: + virtual ~FilterFunction() = default; + virtual std::vector Evaluate(const RecordBatch& batch) = 0; +}; + +template +class NotEqFilterFunction : public FilterFunction { +public: + NotEqFilterFunction(size_t column_ind, ValueType target_val) + : column_ind_(column_ind), target_val_(target_val) { + } + + std::vector Evaluate(const RecordBatch& batch) override { + std::vector selection_vector; + selection_vector.reserve(batch.num_rows); + + ExecutionHelper::IterateColumnData(batch, column_ind_, [&](const auto& value, size_t, size_t physical_idx) { + if (value != target_val_) { + selection_vector.push_back(physical_idx); + } + }); + return selection_vector; + } + +private: + size_t column_ind_; + ValueType target_val_; +}; + +template +class EqFilterFunction : public FilterFunction { +public: + EqFilterFunction(size_t column_ind, ValueType target_val) + : column_ind_(column_ind), target_val_(target_val) { + } + + std::vector Evaluate(const RecordBatch& batch) override { + std::vector selection_vector; + selection_vector.reserve(batch.num_rows); + + ExecutionHelper::IterateColumnData(batch, column_ind_, [&](const auto& value, size_t, size_t real_ind) { + if (value == target_val_) { + selection_vector.push_back(real_ind); + } + }); + return selection_vector; + } + +private: + size_t column_ind_; + ValueType target_val_; +}; diff --git a/src/execution/expressions/ScalarExpressions.h b/src/execution/expressions/ScalarExpressions.h new file mode 100644 index 0000000..8e344c8 --- /dev/null +++ b/src/execution/expressions/ScalarExpressions.h @@ -0,0 +1,60 @@ +#pragma once + +#include "execution/expressions/ScalarFunctions.h" +#include "column/Schema.h" + +#include +#include +#include + +class ScalarExpression { +public: + virtual ~ScalarExpression() = default; + + explicit ScalarExpression(std::string column_name, std::string output_name = "") + : column_name_(std::move(column_name)), output_name_(std::move(output_name)) { + } + + void CollectRequiredColumns(std::vector& required_columns) { + required_columns.push_back(column_name_); + } + + virtual std::string GetOutputName() const { + return output_name_.empty() ? column_name_ : output_name_; + } + + virtual std::unique_ptr CreateScalarFunction(Schema& child_schema) = 0; + +protected: + std::string column_name_; + std::string output_name_; +}; + +class ExtractMinuteExpression : public ScalarExpression { +public: + explicit ExtractMinuteExpression(std::string column_name, std::string output_name = "") + : ScalarExpression(std::move(column_name), std::move(output_name)) { + } + + std::string GetOutputName() const override { + return output_name_.empty() ? "minutes_" + column_name_ : output_name_; + } + + std::unique_ptr CreateScalarFunction(Schema& child_schema) override { + size_t ind = child_schema.GetColumnIndexByName(column_name_); + ColumnType type = child_schema.GetColumnTypeByName(column_name_); + + if (type != ColumnType::TIMESTAMP) [[unlikely]] { + THROW_RUNTIME_ERROR("ExtractMinute requires TIMESTAMP column"); + } + + child_schema.AddColumn(GetOutputName(), ColumnType::INT32); + return std::make_unique(ind); + } +}; + +inline std::shared_ptr ExtractMinute(std::string column_name, + std::string output_name = "") { + return std::make_shared(std::move(column_name), + std::move(output_name)); +} diff --git a/src/execution/expressions/ScalarFunctions.h b/src/execution/expressions/ScalarFunctions.h new file mode 100644 index 0000000..26d1013 --- /dev/null +++ b/src/execution/expressions/ScalarFunctions.h @@ -0,0 +1,34 @@ +#pragma once + +#include "execution/OperatorsBase.h" +#include "execution/ExecutionHelper.h" +#include "column/TemporalColumn.h" +#include "column/NumericColumn.h" + +#include + +class ScalarFunction { +public: + virtual ~ScalarFunction() = default; + virtual std::shared_ptr Evaluate(const RecordBatch& batch) = 0; +}; + +class ExtractMinuteFunction : public ScalarFunction { +public: + explicit ExtractMinuteFunction(size_t col_ind) : col_ind_(col_ind) {} + + std::shared_ptr Evaluate(const RecordBatch& batch) override { + std::vector result_data; + result_data.reserve(batch.num_rows); + + ExecutionHelper::IterateColumnData(batch, col_ind_, [&](const auto& value, size_t, size_t) { + result_data.push_back(Timestamp::ExtractMinute(value)); + }); + + auto result_col = std::make_shared(std::move(result_data)); + return result_col; + } + +private: + size_t col_ind_; +}; diff --git a/src/read_write/ColumnarReader.cpp b/src/io/ColumnarReader.cpp similarity index 63% rename from src/read_write/ColumnarReader.cpp rename to src/io/ColumnarReader.cpp index 1fccf16..bce5dc4 100644 --- a/src/read_write/ColumnarReader.cpp +++ b/src/io/ColumnarReader.cpp @@ -1,36 +1,41 @@ -// -// Created by ragnarokk on 02.01.2026. -// +#include "column/ColumnFactory.h" +#include "io/ColumnarReader.h" -#include +#include "utils/Assert.h" +#include "utils/Macro.h" -#include "ColumnarReader.h" +#include -ColumnarReader::ColumnarReader(const std::string& file_name) : file_(file_name) { +ColumnarReader::ColumnarReader(const std::string& file_name) : file_(file_name, std::ios::binary) { if (!file_.is_open()) { - throw std::runtime_error("Could not open file for reading"); + THROW_RUNTIME_ERROR("Could not open file for reading"); } + std::streamoff file_size; + ValidateMagicAndGetFileSize(file_size); + ReadMetadata(); +} + +void ColumnarReader::ValidateMagicAndGetFileSize(std::streamoff& file_size) { file_.seekg(0, std::ios::end); - const std::streamoff file_size = file_.tellg(); - if (file_size < 16) { - throw std::runtime_error("File is too small to contain a valid footer"); + file_size = file_.tellg(); + if (file_size < 8) { + THROW_RUNTIME_ERROR("File is too small to contain a valid footer"); } file_.seekg(-4, std::ios::end); std::array magic{}; file_.read(magic.data(), magic.size()); if (!file_ || std::string(magic.data(), magic.size()) != "TUFF") { - throw std::runtime_error("Invalid file footer magic"); + THROW_RUNTIME_ERROR("Invalid file footer magic"); } +} +void ColumnarReader::ReadMetadata() { file_.seekg(-12, std::ios::end); uint64_t metadata_start; file_.read(reinterpret_cast(&metadata_start), sizeof(metadata_start)); if (!file_) { - throw std::runtime_error("Failed to read metadata start offset"); - } - if (metadata_start >= static_cast(file_size)) { - throw std::runtime_error("Corrupted metadata offset"); + THROW_RUNTIME_ERROR("Failed to read metadata start offset"); } file_.seekg(metadata_start, std::ios::beg); @@ -38,7 +43,7 @@ ColumnarReader::ColumnarReader(const std::string& file_name) : file_(file_name) uint32_t num_cols; file_.read(reinterpret_cast(&num_cols), sizeof(num_cols)); if (!file_) { - throw std::runtime_error("Failed to read number of columns"); + THROW_RUNTIME_ERROR("Failed to read number of columns"); } for (uint32_t i = 0; i < num_cols; ++i) { @@ -46,34 +51,34 @@ ColumnarReader::ColumnarReader(const std::string& file_name) : file_(file_name) uint32_t name_length; file_.read(reinterpret_cast(&name_length), sizeof(name_length)); if (!file_) { - throw std::runtime_error("Failed to read column name length"); + THROW_RUNTIME_ERROR("Failed to read column name length"); } column_meta.name.resize(name_length); if (name_length > 0) { file_.read(&column_meta.name[0], name_length); if (!file_) { - throw std::runtime_error("Failed to read column name"); + THROW_RUNTIME_ERROR("Failed to read column name"); } } uint8_t type; file_.read(reinterpret_cast(&type), sizeof(type)); if (!file_) { - throw std::runtime_error("Failed to read column type"); + THROW_RUNTIME_ERROR("Failed to read column type"); } column_meta.type = static_cast(type); uint32_t num_chunks; file_.read(reinterpret_cast(&num_chunks), sizeof(num_chunks)); if (!file_) { - throw std::runtime_error("Failed to read number of chunks"); + THROW_RUNTIME_ERROR("Failed to read number of chunks"); } for (uint32_t j = 0; j < num_chunks; ++j) { uint64_t offset, size; file_.read(reinterpret_cast(&offset), sizeof(offset)); file_.read(reinterpret_cast(&size), sizeof(size)); if (!file_) { - throw std::runtime_error("Failed to read chunk metadata"); + THROW_RUNTIME_ERROR("Failed to read chunk metadata"); } column_meta.offsets.push_back(offset); column_meta.sizes.push_back(size); @@ -83,14 +88,7 @@ ColumnarReader::ColumnarReader(const std::string& file_name) : file_(file_name) } } -ColumnarReader::~ColumnarReader() { - if (file_.is_open()) { - file_.close(); - } -} - -std::vector ColumnarReader::GetRawColumnData(size_t column_index, size_t chunk_index) { - std::vector buffer; +void ColumnarReader::ReadRawColumnData(size_t column_index, size_t chunk_index, std::vector& buffer) const { if (column_index >= metadata_.size()) { THROW_RUNTIME_ERROR("Column index out of range"); } @@ -103,25 +101,27 @@ std::vector ColumnarReader::GetRawColumnData(size_t column_index, size_t c buffer.resize(size); file_.seekg(offset, std::ios::beg); file_.read(buffer.data(), size); - return buffer; } -std::shared_ptr ColumnarReader::GetColumnData(size_t column_index, size_t chunk_index) { - std::vector raw_data = GetRawColumnData(column_index, chunk_index); +std::shared_ptr ColumnarReader::GetColumnData(size_t column_index, size_t chunk_index) const { + std::vector raw_data; + ReadRawColumnData(column_index, chunk_index, raw_data); ColumnType type = metadata_[column_index].type; - std::shared_ptr column; + + std::shared_ptr column = ColumnFactory::MakeColumn(type); #define HANDLE_TYPE(ENUM_VAL, STR_VAL, CLASS_TYPE) \ - if (type == ColumnType::ENUM_VAL) { \ - column = std::make_shared(); \ - } else - - FOR_EACH_COLUMN_TYPE(HANDLE_TYPE) { - THROW_RUNTIME_ERROR("Unknown column type"); + case ColumnType::ENUM_VAL: \ + std::static_pointer_cast(column)->ReadFromBuffer(raw_data); \ + break; + + switch (type) { + FOR_EACH_COLUMN_TYPE(HANDLE_TYPE) + default: + THROW_RUNTIME_ERROR("Unknown column type"); } #undef HANDLE_TYPE - column->ReadFromRawData(raw_data); return column; } diff --git a/src/io/ColumnarReader.h b/src/io/ColumnarReader.h new file mode 100644 index 0000000..9afcb2a --- /dev/null +++ b/src/io/ColumnarReader.h @@ -0,0 +1,29 @@ +#pragma once + +#include "types/ColumnType.h" +#include "column/Column.h" + +#include +#include +#include +#include + +class ColumnarReader { +public: + explicit ColumnarReader(const std::string& file_name); + + void ReadRawColumnData(size_t column_index, size_t chunk_index, std::vector& buffer) const; + + std::shared_ptr GetColumnData(size_t column_index, size_t chunk_index) const; + + const std::vector& GetMetadata() const; + + ColumnType GetColumnTypeByName(const std::string& name) const; + +private: + void ValidateMagicAndGetFileSize(std::streamoff& file_size); + void ReadMetadata(); + + mutable std::ifstream file_; + std::vector metadata_; +}; diff --git a/src/io/ColumnarWriter.cpp b/src/io/ColumnarWriter.cpp new file mode 100644 index 0000000..55fa526 --- /dev/null +++ b/src/io/ColumnarWriter.cpp @@ -0,0 +1,143 @@ +#include "io/ColumnarWriter.h" + +#include "column/NumericColumn.h" +#include "column/StringColumn.h" +#include "column/CharColumn.h" +#include "column/TemporalColumn.h" + +#include "utils/Macro.h" + +ColumnarWriter::ColumnarWriter(const std::string& file_path) { + file_.open(file_path, std::ios::binary); + if (!file_.is_open()) { + THROW_RUNTIME_ERROR("Could not open file for writing"); + } +} + +void ColumnarWriter::WriteHeader(const VectorOfStrings& column_names, + const std::vector& column_types) { + if (header_written_) { + THROW_RUNTIME_ERROR("Header already written"); + } + + file_.write("TUFF", 4); + current_offset_ = 4; + + for (size_t i = 0; i < column_names.Size(); ++i) { + ColumnMetadata column_meta; + column_meta.name = column_names.GetString(i); + column_meta.type = column_types[i]; + metadata_.push_back(column_meta); + } + + header_written_ = true; +} + +void ColumnarWriter::WriteBatch(const std::vector>& columns) { + if (!header_written_) { + THROW_RUNTIME_ERROR("Header must be written before writing batches"); + } + if (columns.size() != metadata_.size()) { + THROW_RUNTIME_ERROR("Number of columns does not match metadata"); + } + for (size_t i = 0; i < columns.size(); ++i) { + uint64_t offset = current_offset_; + uint64_t size = 0; + + ColumnType type = metadata_[i].type; + + if (type == ColumnType::STRING) { + const auto* data = static_cast(columns[i]->GetRawData()); + const std::vector& offsets = data->GetOffsets(); + const std::vector& raw = data->GetData(); + + uint64_t total_size = 0; + std::vector buffer; + size_t h = columns[i]->Size(); + size_t total_len = 0; + for (size_t k = 0; k < h; ++k) { + total_len += sizeof(uint32_t) + (offsets[k + 1] - offsets[k]); + } + buffer.reserve(total_len); + for (size_t k = 0; k < h; ++k) { + size_t start = offsets[k]; + size_t end = offsets[k + 1]; + uint32_t length = end - start; + buffer.insert(buffer.end(), reinterpret_cast(&length), + reinterpret_cast(&length) + sizeof(length)); + if (length > 0) { + buffer.insert(buffer.end(), raw.begin() + start, raw.begin() + end); + } + total_size += sizeof(length) + length; + } + file_.write(buffer.data(), buffer.size()); + size = total_size; + } else if (type == ColumnType::CHAR) { + size = columns[i]->Size(); + if (size > 0) { + file_.write(static_cast(columns[i]->GetRawData()), size); + } + } else { + size_t elem_size = 0; +#define HANDLE_TYPE(ENUM_VAL, STR_VAL, CLASS_TYPE) \ + case ColumnType::ENUM_VAL: { \ + elem_size = sizeof(CLASS_TYPE::ValueType); \ + size = columns[i]->Size() * elem_size; \ + if (size > 0) { \ + auto* vec = static_cast(columns[i]->GetRawData()); \ + file_.write(reinterpret_cast(vec->data()), size); \ + } \ + break; \ + } + + switch (type) { + FOR_NUMERIC_COLUMN_TYPE(HANDLE_TYPE) + default: break; + } +#undef HANDLE_TYPE + } + + metadata_[i].offsets.push_back(offset); + metadata_[i].sizes.push_back(size); + current_offset_ += size; + } + if (!columns.empty()) { + num_rows_ += columns[0]->Size(); + } +} + +void ColumnarWriter::Finalize() && { + if (!header_written_) { + THROW_RUNTIME_ERROR("Header must be written before finalizing"); + } + + WriteMetadata(); + + file_.write("TUFF", 4); +} + +void ColumnarWriter::WriteMetadata() { + uint64_t metadata_start = current_offset_; + + uint32_t num_columns = metadata_.size(); + file_.write(reinterpret_cast(&num_columns), sizeof(num_columns)); + + for (const auto& meta : metadata_) { + uint32_t name_length = meta.name.size(); + file_.write(reinterpret_cast(&name_length), sizeof(name_length)); + file_.write(meta.name.data(), name_length); + + uint8_t type = static_cast(meta.type); + file_.write(reinterpret_cast(&type), sizeof(type)); + + uint32_t num_batches = meta.offsets.size(); + file_.write(reinterpret_cast(&num_batches), sizeof(num_batches)); + for (size_t i = 0; i < num_batches; ++i) { + file_.write(reinterpret_cast(&meta.offsets[i]), sizeof(uint64_t)); + file_.write(reinterpret_cast(&meta.sizes[i]), sizeof(uint64_t)); + } + } + + file_.write(reinterpret_cast(&num_rows_), sizeof(num_rows_)); + file_.write(reinterpret_cast(&metadata_start), sizeof(metadata_start)); +} diff --git a/src/io/ColumnarWriter.h b/src/io/ColumnarWriter.h new file mode 100644 index 0000000..6648fbf --- /dev/null +++ b/src/io/ColumnarWriter.h @@ -0,0 +1,46 @@ +#pragma once + +#include "types/ColumnType.h" +#include "column/Column.h" + +#include "utils/VectorOfStrings.h" + +#include +#include +#include +#include + +class ColumnarWriter { +public: + explicit ColumnarWriter(const std::string& file_path); + + void WriteHeader(const VectorOfStrings& column_names, + const std::vector& column_types); + + void WriteBatch(const std::vector>& columns); + + // File structure: + // - magic "TUFF" + // - columns by batches + // - metadata: + // - number of columns + // - for each column: + // - name length + // - name + // - type + // - number of chunks + // - for each chunk: offset, size + // - number of rows + // - metadata start offset + // - magic "TUFF" + void Finalize() &&; + +private: + void WriteMetadata(); + + std::ofstream file_; + std::vector metadata_; + bool header_written_ = false; + uint64_t current_offset_ = 0; + uint64_t num_rows_ = 0; +}; \ No newline at end of file diff --git a/src/io/CsvReader.cpp b/src/io/CsvReader.cpp new file mode 100644 index 0000000..610536c --- /dev/null +++ b/src/io/CsvReader.cpp @@ -0,0 +1,60 @@ +#include "io/CsvReader.h" +#include "utils/Macro.h" + +#include +#include + +CsvReader::CsvReader(const std::string& file_path, char delim) : tokenizer_(file_path, delim) { +} + +void CsvReader::ReadHeader(VectorOfStrings& header_names, std::vector& header_types) { + VectorOfStrings2D raw_header; + tokenizer_.GetNextRow(raw_header); + tokenizer_.ResetLineFlag(); + expected_columns_ = raw_header.Width(); + + header_names.Clear(); + header_types.clear(); + + for (size_t i = 0; i < expected_columns_; ++i) { + std::string_view token = raw_header.GetString(i); + size_t colon_pos = token.find(':'); + if (colon_pos == std::string_view::npos) { + THROW_RUNTIME_ERROR("Invalid header token: " + std::string(token)); + } + std::string_view type_str = token.substr(0, colon_pos); + std::string_view name_str = token.substr(colon_pos + 1); + +#define HANDLE_TYPE(ENUM_VAL, STR_VAL, CLASS_TYPE) \ + if (type_str == STR_VAL) { \ + header_names.PushBack(name_str); \ + header_types.push_back(ColumnType::ENUM_VAL); \ + } else + + FOR_EACH_COLUMN_TYPE(HANDLE_TYPE) { + THROW_RUNTIME_ERROR("Invalid header token: " + std::string(token)); + } +#undef HANDLE_TYPE + } +} + +bool CsvReader::ReadRow(VectorOfStrings2D& row) { + if (tokenizer_.IsEOF()) { + return false; + } + + tokenizer_.GetNextRow(row); + + if (row.ColumnsInCurrentLine() == 0 && tokenizer_.IsEOF()) { + return false; + } + + if (expected_columns_ != 0 && row.ColumnsInCurrentLine() != expected_columns_) { + THROW_RUNTIME_ERROR( + "CSV row width mismatch: expected " + std::to_string(expected_columns_) + ", got " + + std::to_string(row.ColumnsInCurrentLine()) + " row: " + std::to_string(row.Height())); + } + + tokenizer_.ResetLineFlag(); + return true; +} diff --git a/src/io/CsvReader.h b/src/io/CsvReader.h new file mode 100644 index 0000000..a23f7be --- /dev/null +++ b/src/io/CsvReader.h @@ -0,0 +1,20 @@ +#pragma once + +#include "types/ColumnType.h" +#include "io/CsvTokenizer.h" +#include "utils/VectorOfStrings.h" + +#include +#include + +class CsvReader { +public: + CsvReader(const std::string& file_path, char delim = ','); + + void ReadHeader(VectorOfStrings& header_names, std::vector& header_types); + bool ReadRow(VectorOfStrings2D& row); + +private: + CsvTokenizer tokenizer_; + size_t expected_columns_ = 0; +}; diff --git a/src/io/CsvToColumnar.cpp b/src/io/CsvToColumnar.cpp new file mode 100644 index 0000000..b285599 --- /dev/null +++ b/src/io/CsvToColumnar.cpp @@ -0,0 +1,93 @@ +#include "column/Schema.h" +#include "column/ColumnFactory.h" + +#include "io/CsvToColumnar.h" +#include "io/CsvReader.h" +#include "io/ColumnarWriter.h" + +#include +#include + +void WriteToColumnar(const std::string& columnar_file_path, const VectorOfStrings& column_names, + const std::vector& column_types, CsvReader& reader, + std::vector>& builders) { + ColumnarWriter writer(columnar_file_path); + writer.WriteHeader(column_names, column_types); + + static constexpr size_t kByteSize = 1 << 20; + bool end_flag = true; + VectorOfStrings2D column_batch; + + while (end_flag) { + size_t cur_batch_size = 0; + column_batch.Clear(); + + while (column_batch.ApproxByteSize() < kByteSize) { + bool has_row = reader.ReadRow(column_batch); + if (has_row) { + column_batch.StartNewLine(); + ++cur_batch_size; + } else { + end_flag = false; + break; + } + } + + if (cur_batch_size == 0) { + break; + } + + size_t batch_width = column_batch.Width(); + for (size_t j = 0; j < batch_width; ++j) { + builders[j]->AddBatch(column_batch, j); + } + + std::vector> columns; + for (auto& builder : builders) { + columns.push_back(builder->Finish()); + } + + if (!columns.empty()) { + writer.WriteBatch(columns); + } + } + + std::move(writer).Finalize(); +} + +void CsvToColumnar::Convert(const std::string& csv_file_path, + const std::string& columnar_file_path) { + CsvReader reader(csv_file_path); + VectorOfStrings column_names; + std::vector column_types; + reader.ReadHeader(column_names, column_types); + + std::vector> builders; + for (ColumnType type : column_types) { + builders.push_back(ColumnFactory::MakeColumnBuilder(type)); + } + + WriteToColumnar(columnar_file_path, column_names, column_types, reader, builders); +} + +void CsvToColumnar::ConvertWithSchema(const std::string& csv_file_path, + const std::string& columnar_file_path, + const std::string& schema_file_path) { + CsvReader reader(csv_file_path); + Schema schema = Schema::DeserializeSchema(schema_file_path); + const std::vector fields = schema.GetFields(); + + std::vector> builders; + VectorOfStrings column_names; + std::vector column_types; + + for (const Field& field : fields) { + std::string_view header = field.name; + ColumnType type = field.type; + column_names.PushBack(header); + column_types.push_back(type); + builders.push_back(ColumnFactory::MakeColumnBuilder(type)); + } + + WriteToColumnar(columnar_file_path, column_names, column_types, reader, builders); +} diff --git a/src/io/CsvToColumnar.h b/src/io/CsvToColumnar.h new file mode 100644 index 0000000..96199fc --- /dev/null +++ b/src/io/CsvToColumnar.h @@ -0,0 +1,11 @@ +#pragma once + +#include + +class CsvToColumnar { +public: + static void Convert(const std::string& csv_file_path, const std::string& columnar_file_path); + static void ConvertWithSchema(const std::string& csv_file_path, + const std::string& columnar_file_path, + const std::string& schema_file_path); +}; diff --git a/src/read_write/CSVTokenizer.cpp b/src/io/CsvTokenizer.cpp similarity index 90% rename from src/read_write/CSVTokenizer.cpp rename to src/io/CsvTokenizer.cpp index 227c069..e733438 100644 --- a/src/read_write/CSVTokenizer.cpp +++ b/src/io/CsvTokenizer.cpp @@ -1,12 +1,6 @@ -// -// Created by ragnarokk on 03.01.2026. -// +#include "io/CsvTokenizer.h" -#include "CSVTokenizer.h" - -#include "macro.h" - -CSVTokenizer::CSVTokenizer(const std::string& file_path, char delim) +CsvTokenizer::CsvTokenizer(const std::string& file_path, char delim) : file_(file_path), buffer_(kBufferSize), delim_(delim) { if (!file_.is_open()) { THROW_RUNTIME_ERROR("Could not open CSV file"); @@ -20,7 +14,7 @@ CSVTokenizer::CSVTokenizer(const std::string& file_path, char delim) is_whitespace_[static_cast('\r')] = true; } -bool CSVTokenizer::RefillBuffer(char& ch) { +bool CsvTokenizer::RefillBuffer(char& ch) { if (eof_reached_) { return false; } @@ -35,8 +29,7 @@ bool CSVTokenizer::RefillBuffer(char& ch) { return true; } -void CSVTokenizer::GetNextRow(VectorOfStrings2D& vector_of_strings) { - char ch; +void CsvTokenizer::GetNextRow(VectorOfStrings2D& vector_of_strings) { bool in_quotes = false; bool token_started = false; bool token_materialized = false; @@ -81,7 +74,7 @@ void CSVTokenizer::GetNextRow(VectorOfStrings2D& vector_of_strings) { } } - ch = buffer_[buffer_pos_++]; + char ch = buffer_[buffer_pos_++]; if (!token_started && (ch == ' ' || ch == '\t')) { continue; @@ -147,14 +140,14 @@ void CSVTokenizer::GetNextRow(VectorOfStrings2D& vector_of_strings) { vector_of_strings.EndAddString(); } -bool CSVTokenizer::IsEndOfLine() const { +bool CsvTokenizer::IsEndOfLine() const { return end_of_line_; } -bool CSVTokenizer::IsEOF() const { +bool CsvTokenizer::IsEOF() const { return eof_reached_; } -void CSVTokenizer::ResetLineFlag() { +void CsvTokenizer::ResetLineFlag() { end_of_line_ = false; } diff --git a/src/read_write/CSVTokenizer.h b/src/io/CsvTokenizer.h similarity index 65% rename from src/read_write/CSVTokenizer.h rename to src/io/CsvTokenizer.h index b521661..c86b70b 100644 --- a/src/read_write/CSVTokenizer.h +++ b/src/io/CsvTokenizer.h @@ -1,20 +1,15 @@ -// -// Created by ragnarokk on 03.01.2026. -// +#pragma once -#ifndef COLUMNAR_ENGINE_CSVTOKENIZER_H -#define COLUMNAR_ENGINE_CSVTOKENIZER_H - -#include "VectorOfStrings.h" +#include "utils/VectorOfStrings.h" #include #include -class CSVTokenizer { +class CsvTokenizer { public: - CSVTokenizer(const std::string& file_path, char delim = ','); - CSVTokenizer(const CSVTokenizer& other) = delete; - CSVTokenizer& operator=(const CSVTokenizer& other) = delete; + CsvTokenizer(const std::string& file_path, char delim = ','); + CsvTokenizer(const CsvTokenizer& other) = delete; + CsvTokenizer& operator=(const CsvTokenizer& other) = delete; void GetNextRow(VectorOfStrings2D& vector_of_strings); bool IsEndOfLine() const; @@ -43,5 +38,3 @@ class CSVTokenizer { bool RefillBuffer(char& ch); }; - -#endif // COLUMNAR_ENGINE_CSVTOKENIZER_H diff --git a/src/io/test/TestCsvParser.cpp b/src/io/test/TestCsvParser.cpp new file mode 100644 index 0000000..4615117 --- /dev/null +++ b/src/io/test/TestCsvParser.cpp @@ -0,0 +1,138 @@ +// Generated by AI + +#include "io/CsvReader.h" +#include "io/CsvTokenizer.h" + +#include +#include +#include +#include + +namespace fs = std::filesystem; + +namespace { + +class CreateTempCsvFile { +public: + CreateTempCsvFile(const std::string& content, const std::string& filename) { + path_ = "/tmp/" + filename; + std::ofstream out(path_); + out << content; + out.close(); + } + + ~CreateTempCsvFile() { + fs::remove(path_); + } + + std::string path() const { + return path_.string(); + } + +private: + fs::path path_; +}; + +} // namespace + +TEST_CASE("Simple Csv reading", "[CsvReader]") { + CreateTempCsvFile tempFile("INT32:h1,INT32:h2\n1,2", "test_reader_simple.csv"); + CsvReader reader(tempFile.path()); + + VectorOfStrings header_names; + std::vector header_types; + reader.ReadHeader(header_names, header_types); + REQUIRE(header_names.Size() == 2); + REQUIRE(header_names.GetString(0) == std::string_view("h1")); + REQUIRE(header_names.GetString(1) == std::string_view("h2")); + REQUIRE(header_types[0] == ColumnType::INT32); + + VectorOfStrings2D row; + REQUIRE(reader.ReadRow(row)); + REQUIRE(row.ColumnsInCurrentLine() == 2); + REQUIRE(row.GetString(0) == std::string_view("1")); + REQUIRE(row.GetString(1) == std::string_view("2")); + + row.StartNewLine(); // Emulate how batching works + REQUIRE_FALSE(reader.ReadRow(row)); +} + +TEST_CASE("CsvTokenizer does not materialize empty token on hard EOF", "[CsvTokenizer]") { + CreateTempCsvFile tempFile("", "test_tokenizer_hard_eof.csv"); + CsvTokenizer tokenizer(tempFile.path()); + VectorOfStrings2D row; + + const size_t width_before = row.Width(); + tokenizer.GetNextRow(row); + + REQUIRE(row.Width() == width_before); + REQUIRE(tokenizer.IsEOF()); +} + +TEST_CASE("CsvTokenizer keeps last token without trailing newline", "[CsvTokenizer]") { + CreateTempCsvFile tempFile("a,b", "test_tokenizer_no_trailing_newline.csv"); + CsvTokenizer tokenizer(tempFile.path()); + VectorOfStrings2D row; + + tokenizer.GetNextRow(row); + row.StartNewLine(); + tokenizer.GetNextRow(row); + + REQUIRE(row.Width() == 2); + REQUIRE(row.GetString(0) == std::string_view("a")); + REQUIRE(row.GetString(1) == std::string_view("b")); + REQUIRE(tokenizer.IsEOF()); +} + +TEST_CASE("CsvReader does not return fake empty row at EOF", "[CsvReader]") { + CreateTempCsvFile tempFile("INT32:h1,INT32:h2\n1,2", "test_reader_eof.csv"); + CsvReader reader(tempFile.path()); + + VectorOfStrings header_names; + std::vector header_types; + reader.ReadHeader(header_names, header_types); + REQUIRE(header_names.Size() == 2); + + VectorOfStrings2D row; + REQUIRE(reader.ReadRow(row)); + REQUIRE(row.ColumnsInCurrentLine() == 2); + REQUIRE(row.GetString(0) == std::string_view("1")); + REQUIRE(row.GetString(1) == std::string_view("2")); + row.StartNewLine(); + + REQUIRE_FALSE(reader.ReadRow(row)); + REQUIRE(row.ColumnsInCurrentLine() == 0); +} + +TEST_CASE("CsvReader does not add extra row for trailing newline", "[CsvReader]") { + CreateTempCsvFile tempFile("INT32:h1\nx\n", "test_reader_trailing_newline.csv"); + CsvReader reader(tempFile.path()); + + VectorOfStrings header_names; + std::vector header_types; + reader.ReadHeader(header_names, header_types); + + VectorOfStrings2D row; + REQUIRE(reader.ReadRow(row)); + REQUIRE(row.ColumnsInCurrentLine() == 1); + REQUIRE(row.GetString(0) == std::string_view("x")); + row.StartNewLine(); + REQUIRE_FALSE(reader.ReadRow(row)); +} + +TEST_CASE("CsvReader preserves empty fields", "[CsvReader]") { + CreateTempCsvFile tempFile("STRING:a,STRING:b\n,\n", "test_reader_empty_fields.csv"); + CsvReader reader(tempFile.path()); + + VectorOfStrings header_names; + std::vector header_types; + reader.ReadHeader(header_names, header_types); + + VectorOfStrings2D row; + REQUIRE(reader.ReadRow(row)); + REQUIRE(row.ColumnsInCurrentLine() == 2); + REQUIRE(row.GetString(0) == std::string_view("")); + REQUIRE(row.GetString(1) == std::string_view("")); + row.StartNewLine(); + REQUIRE_FALSE(reader.ReadRow(row)); +} diff --git a/src/io/test/TestReaderWriter.cpp b/src/io/test/TestReaderWriter.cpp new file mode 100644 index 0000000..15fe6c5 --- /dev/null +++ b/src/io/test/TestReaderWriter.cpp @@ -0,0 +1,128 @@ +// Generated by AI + +#include "io/CsvToColumnar.h" +#include "io/ColumnarReader.h" +#include "types/ColumnType.h" + +#include +#include +#include + +namespace fs = std::filesystem; + +std::string CreateTempCSVFile(const std::string& content, + const std::string& filename = "test_rw.csv") { + std::string filepath = "/tmp/" + filename; + std::ofstream file(filepath); + file << content; + file.close(); + return filepath; +} + +void RemoveTempFile(const std::string& filepath) { + if (fs::exists(filepath)) { + fs::remove(filepath); + } +} + +TEST_CASE("CsvToColumnar: Basic write operation", "[CsvToColumnar]") { + std::string csv_content = + "INT32:id,STRING:name\n" + "1,Alice\n" + "2,Bob\n" + "3,Charlie\n"; + std::string csv_path = CreateTempCSVFile(csv_content, "test_CsvToColumnar_basic.csv"); + std::string output_path = "/tmp/test_CsvToColumnar_basic.bin"; + + SECTION("Write CSV to binary format") { + REQUIRE_NOTHROW(CsvToColumnar::Convert(csv_path, output_path)); + REQUIRE(fs::exists(output_path)); + + std::ifstream file(output_path, std::ios::binary); + file.seekg(0, std::ios::end); + size_t file_size = file.tellg(); + REQUIRE(file_size > 0); + + file.seekg(0, std::ios::beg); + char magic[5] = {0}; + file.read(magic, 4); + REQUIRE(std::string(magic) == "TUFF"); + + file.close(); + } + + RemoveTempFile(csv_path); + RemoveTempFile(output_path); +} + +TEST_CASE("CsvToColumnar: Multiple types", "[CsvToColumnar]") { + std::string csv_content = + "INT32:id,STRING:name,FLOAT:score\n" + "1,Alice,95.5\n" + "2,Bob,87.3\n"; + std::string csv_path = CreateTempCSVFile(csv_content, "test_CsvToColumnar_multi.csv"); + std::string output_path = "/tmp/test_CsvToColumnar_multi.bin"; + + CsvToColumnar::Convert(csv_path, output_path); + + REQUIRE(fs::exists(output_path)); + REQUIRE(fs::file_size(output_path) > 0); + + RemoveTempFile(csv_path); + RemoveTempFile(output_path); +} + +TEST_CASE("Reader: Read metadata", "[Reader]") { + std::string csv_content = + "INT32:id,STRING:name\n" + "1,Alice\n" + "2,Bob\n"; + std::string csv_path = CreateTempCSVFile(csv_content, "test_reader_meta.csv"); + std::string output_path = "/tmp/test_reader_meta.bin"; + + CsvToColumnar::Convert(csv_path, output_path); + + SECTION("Read and verify metadata") { + ColumnarReader reader(output_path); + const auto& metadata = reader.GetMetadata(); + + REQUIRE(metadata.size() == 2); + REQUIRE(metadata[0].name == "id"); + REQUIRE(metadata[0].type == ColumnType::INT32); + REQUIRE(metadata[1].name == "name"); + REQUIRE(metadata[1].type == ColumnType::STRING); + } + + RemoveTempFile(csv_path); + RemoveTempFile(output_path); +} + +TEST_CASE("Reader: Read typed column data", "[Reader]") { + std::string csv_content = + "INT32:id,STRING:city\n" + "1,Moscow\n" + "2,London\n"; + std::string csv_path = CreateTempCSVFile(csv_content, "test_reader_typed.csv"); + std::string output_path = "/tmp/test_reader_typed.bin"; + + CsvToColumnar::Convert(csv_path, output_path); + + ColumnarReader reader(output_path); + + SECTION("Read INT32 column") { + auto column = reader.GetColumnData(0, 0); + REQUIRE(column != nullptr); + REQUIRE(column->GetType() == ColumnType::INT32); + REQUIRE(column->Size() == 2); + } + + SECTION("Read STRING column") { + auto column = reader.GetColumnData(1, 0); + REQUIRE(column != nullptr); + REQUIRE(column->GetType() == ColumnType::STRING); + REQUIRE(column->Size() == 2); + } + + RemoveTempFile(csv_path); + RemoveTempFile(output_path); +} diff --git a/src/main.cpp b/src/main.cpp index dd0d7df..71788f5 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -1,5 +1,5 @@ -#include "CSVToColumnar.h" -#include "ExecutionAPI.h" +#include "io/CsvToColumnar.h" +#include "execution/ExecutionApi.h" #include @@ -189,11 +189,15 @@ class Query { df.Display(); } - // TODO: 18 - // SELECT UserID, extract(minute FROM EventTime) AS m, SearchPhrase, COUNT(*) FROM hits GROUP BY // UserID, m, SearchPhrase ORDER BY COUNT(*) DESC LIMIT 10; void Query18() { + auto df = DataFrame::Select(columnar_file_path_, {}) + .Project({ExtractMinute("EventTime", "m")}) + .Aggregate({"UserID", "m", "SearchPhrase"}, {Count()}) + .OrderBy({{"count", true}}, 10) + .Collect(); + df.Display(); } // SELECT UserID FROM hits WHERE UserID = 435090932899640449; @@ -204,7 +208,11 @@ class Query { .Collect(); } - // TODO: 20-23 + // SELECT COUNT(*) FROM hits WHERE URL LIKE '%google%'; + void Query20() { + } + + // TODO: 21-23 // SELECT SearchPhrase FROM hits WHERE SearchPhrase <> '' ORDER BY EventTime LIMIT 10; void Query24() { @@ -254,7 +262,7 @@ class Query { case 15: Query15(); break; case 16: Query16(); break; case 17: Query17(); break; - // TODO: 18 + case 18: Query18(); break; case 19: Query19(); break; // TODO: 20-23 case 24: Query24(); break; @@ -288,8 +296,7 @@ int main(int argc, char** argv) { try { std::cerr << "Converting CSV to columnar format...\n"; const auto start = std::chrono::steady_clock::now(); - CSVToColumnar converter; - converter.ConvertWithSchema(argv[2], argv[3], argv[4]); + CsvToColumnar::ConvertWithSchema(argv[2], argv[3], argv[4]); const auto time = std::chrono::duration(std::chrono::steady_clock::now() - start).count() * 1000; diff --git a/src/read_write/CSVReader.cpp b/src/read_write/CSVReader.cpp deleted file mode 100644 index e4f6eb9..0000000 --- a/src/read_write/CSVReader.cpp +++ /dev/null @@ -1,39 +0,0 @@ -// -// Created by ragnarokk on 05.01.2026. -// - -#include "CSVReader.h" -#include "macro.h" - -#include - -CSVReader::CSVReader(const std::string& file_path, char delim) - : tokenizer_(file_path, delim) { -} - -void CSVReader::ReadHeader(VectorOfStrings2D& header) { - header.Clear(); - tokenizer_.GetNextRow(header); - tokenizer_.ResetLineFlag(); - expected_columns_ = header.Width(); -} - -bool CSVReader::ReadRow(VectorOfStrings2D& row) { - if (tokenizer_.IsEOF()) { - return false; - } - - tokenizer_.GetNextRow(row); - - if (row.ColumnsInCurrentLine() == 0 && tokenizer_.IsEOF()) { - return false; - } - - if (expected_columns_ != 0 && row.ColumnsInCurrentLine() != expected_columns_) { - THROW_RUNTIME_ERROR("CSV row width mismatch: expected " + std::to_string(expected_columns_) + - ", got " + std::to_string(row.ColumnsInCurrentLine()) + " row: " + std::to_string(row.Height())); - } - - tokenizer_.ResetLineFlag(); - return true; -} diff --git a/src/read_write/CSVReader.h b/src/read_write/CSVReader.h deleted file mode 100644 index c4ac68e..0000000 --- a/src/read_write/CSVReader.h +++ /dev/null @@ -1,25 +0,0 @@ -// -// Created by ragnarokk on 05.01.2026. -// - -#ifndef COLUMNAR_ENGINE_RAWCSVREADER_H -#define COLUMNAR_ENGINE_RAWCSVREADER_H - -#include "CSVTokenizer.h" -#include "VectorOfStrings.h" - -#include - -class CSVReader { -public: - CSVReader(const std::string& file_path, char delim = ','); - - void ReadHeader(VectorOfStrings2D& header); - bool ReadRow(VectorOfStrings2D& row); - -private: - CSVTokenizer tokenizer_; - size_t expected_columns_ = 0; -}; - -#endif // COLUMNAR_ENGINE_RAWCSVREADER_H \ No newline at end of file diff --git a/src/read_write/CSVToColumnar.cpp b/src/read_write/CSVToColumnar.cpp deleted file mode 100644 index a61a627..0000000 --- a/src/read_write/CSVToColumnar.cpp +++ /dev/null @@ -1,138 +0,0 @@ -// -// Created by ragnarokk on 05.01.2026. -// - -#include - -#include "CSVToColumnar.h" -#include "CSVReader.h" -#include "ColumnarWriter.h" -#include "Schema.h" -#include "macro.h" -#include "VectorOfStrings.h" - -#include - -void CSVToColumnar::Convert(const std::string& csv_file_path, - const std::string& columnar_file_path) { - CSVReader reader(csv_file_path); - VectorOfStrings2D raw_header; - reader.ReadHeader(raw_header); - - std::vector> columns; - VectorOfStrings2D column_names; - std::vector column_types; - - size_t num_columns = raw_header.Size(); - for (size_t i = 0; i < num_columns; ++i) { - std::string_view token = raw_header.GetString(i); - ParseHeaderToken(token, column_names, column_types); - auto type = column_types.back(); - AddColumn(type, columns); - } - - WriteToColumnar(columnar_file_path, column_names, column_types, reader, columns); -} - -void CSVToColumnar::ConvertWithSchema(const std::string& csv_file_path, - const std::string& columnar_file_path, - const std::string& schema_file_path) { - CSVReader reader(csv_file_path); - Schema schema = Schema::DeserializeSchema(schema_file_path); - const std::vector fields = schema.GetFields(); - - std::vector> columns; - VectorOfStrings2D column_names; - std::vector column_types; - - for (const Field& field : fields) { - std::string_view header = field.name; - ColumnType type = field.type; - column_names.AddString(header); - column_types.push_back(type); - AddColumn(type, columns); - } - - WriteToColumnar(columnar_file_path, column_names, column_types, reader, columns); -} - -void CSVToColumnar::ParseHeaderToken(std::string_view token, VectorOfStrings2D& column_names, - std::vector& column_types) { - size_t colon_pos = token.find(':'); - if (colon_pos == std::string::npos) { - THROW_RUNTIME_ERROR("Invalid header token: " + std::string(token)); - } - std::string_view type_str = token.substr(0, colon_pos); - std::string_view name_str = token.substr(colon_pos + 1); - - // OPTIMIZE: copying in AddString -#define HANDLE_TYPE(ENUM_VAL, STR_VAL, CLASS_TYPE) \ - if (type_str == STR_VAL) { \ - column_names.AddString(name_str); \ - column_types.push_back(ColumnType::ENUM_VAL); \ - } else - FOR_EACH_COLUMN_TYPE(HANDLE_TYPE) { - THROW_RUNTIME_ERROR("Invalid header token: " + std::string(token)); - } -#undef HANDLE_TYPE -} - -void CSVToColumnar::AddColumn(ColumnType type, std::vector>& columns) { - switch (type) { -#define HANDLE_TYPE(ENUM_VAL, STR_VAL, CLASS_TYPE) \ - case ColumnType::ENUM_VAL: columns.push_back(std::make_shared()); break; - - FOR_EACH_COLUMN_TYPE(HANDLE_TYPE) -#undef HANDLE_TYPE - - default: - throw std::runtime_error("Unsupported column type"); - } -} - -void CSVToColumnar::WriteToColumnar(const std::string& columnar_file_path, - const VectorOfStrings2D& column_names, - const std::vector& column_types, CSVReader& reader, - const std::vector>& columns) { - ColumnarWriter writer(columnar_file_path); - writer.WriteHeader(column_names, column_types); - - static constexpr size_t kByteSize = 1 << 20; - bool end_flag = true; - VectorOfStrings2D column_batch; - - while (end_flag) { - size_t cur_batch_size = 0; - column_batch.Clear(); - - while (column_batch.ApproxByteSize() < kByteSize) { - bool has_row = reader.ReadRow(column_batch); - if (has_row) { - column_batch.StartNewLine(); - ++cur_batch_size; - } else { - end_flag = false; - break; - } - } - - if (cur_batch_size == 0) { - break; - } - - size_t batch_width = column_batch.Width(); - for (size_t j = 0; j < batch_width; ++j) { - columns[j]->AddBatch(column_batch, j); - } - - if (!columns.empty()) { - writer.WriteBatch(columns); - } - for (auto& col : columns) { - col->Clear(); - } - } - - - writer.Finalize(); -} diff --git a/src/read_write/CSVToColumnar.h b/src/read_write/CSVToColumnar.h deleted file mode 100644 index 80d6dfb..0000000 --- a/src/read_write/CSVToColumnar.h +++ /dev/null @@ -1,31 +0,0 @@ -// -// Created by ragnarokk on 05.01.2026. -// - -#ifndef COLUMNAR_ENGINE_CSVTOCOLUMNAR_H -#define COLUMNAR_ENGINE_CSVTOCOLUMNAR_H - -#include -#include - -#include "CSVReader.h" -#include "Types.h" - -class CSVToColumnar { -public: - void Convert(const std::string& csv_file_path, const std::string& columnar_file_path); - void ConvertWithSchema(const std::string& csv_file_path, const std::string& columnar_file_path, - const std::string& schema_file_path); - -private: - void ParseHeaderToken(std::string_view token, VectorOfStrings2D& column_names, - std::vector& column_types); - void AddColumn(ColumnType type, std::vector>& columns); - - void WriteToColumnar(const std::string& columnar_file_path, - const VectorOfStrings2D& column_names, - const std::vector& column_types, CSVReader& reader, - const std::vector>& columns); -}; - -#endif // COLUMNAR_ENGINE_CSVTOCOLUMNAR_H \ No newline at end of file diff --git a/src/read_write/ColumnarReader.h b/src/read_write/ColumnarReader.h deleted file mode 100644 index 6fc75a8..0000000 --- a/src/read_write/ColumnarReader.h +++ /dev/null @@ -1,32 +0,0 @@ -// -// Created by ragnarokk on 02.01.2026. -// - -#ifndef COLUMNAR_ENGINE_READER_H -#define COLUMNAR_ENGINE_READER_H - -#include -#include -#include - -#include "Types.h" - -class ColumnarReader { -public: - ColumnarReader(const std::string& file_name); - ~ColumnarReader(); - - std::vector GetRawColumnData(size_t column_index, size_t chunk_index); - - std::shared_ptr GetColumnData(size_t column_index, size_t chunk_index); - - const std::vector& GetMetadata() const; - - ColumnType GetColumnTypeByName(const std::string& name) const; - -private: - std::ifstream file_; - std::vector metadata_; -}; - -#endif // COLUMNAR_ENGINE_READER_H diff --git a/src/read_write/ColumnarWriter.cpp b/src/read_write/ColumnarWriter.cpp deleted file mode 100644 index 5085ec4..0000000 --- a/src/read_write/ColumnarWriter.cpp +++ /dev/null @@ -1,88 +0,0 @@ -// -// Created by ragnarokk on 05.01.2026. -// - -#include "ColumnarWriter.h" - -ColumnarWriter::ColumnarWriter(const std::string& file_path) { - file_.open(file_path, std::ios::binary); - if (!file_.is_open()) { - throw std::runtime_error("Could not open file for writing"); - } -} - -ColumnarWriter::~ColumnarWriter() { - if (file_.is_open()) { - file_.close(); - } -} - -void ColumnarWriter::WriteHeader(const VectorOfStrings2D& column_names, - const std::vector& column_types) { - if (header_written_) { - throw std::runtime_error("Header already written"); - } - - file_.write("TUFF", 4); - current_offset_ = 4; - - for (size_t i = 0; i < column_names.Size(); ++i) { - ColumnMetadata column_meta; - column_meta.name = column_names.GetString(i); - column_meta.type = column_types[i]; - metadata_.push_back(column_meta); - } - - header_written_ = true; -} - -void ColumnarWriter::WriteBatch(const std::vector>& columns) { - if (!header_written_) { - throw std::runtime_error("Header must be written before writing batches"); - } - if (columns.size() != metadata_.size()) { - throw std::runtime_error("Number of columns does not match metadata"); - } - for (size_t i = 0; i < columns.size(); ++i) { - uint64_t offset = current_offset_; - uint64_t size = columns[i]->WriteToFile(file_); - metadata_[i].offsets.push_back(offset); - metadata_[i].sizes.push_back(size); - current_offset_ += size; - } - if (!columns.empty()) { - num_rows_ += columns[0]->Size(); - } -} - -void ColumnarWriter::Finalize() { - if (!header_written_) { - throw std::runtime_error("Header must be written before finalizing"); - } - - uint64_t metadata_start = current_offset_; - - uint32_t num_columns = metadata_.size(); - file_.write(reinterpret_cast(&num_columns), sizeof(num_columns)); - - for (const auto& meta : metadata_) { - uint32_t name_length = meta.name.size(); - file_.write(reinterpret_cast(&name_length), sizeof(name_length)); - file_.write(meta.name.data(), name_length); - - uint8_t type = static_cast(meta.type); - file_.write(reinterpret_cast(&type), sizeof(type)); - - uint32_t num_batches = meta.offsets.size(); - file_.write(reinterpret_cast(&num_batches), sizeof(num_batches)); - for (size_t i = 0; i < num_batches; ++i) { - file_.write(reinterpret_cast(&meta.offsets[i]), sizeof(uint64_t)); - file_.write(reinterpret_cast(&meta.sizes[i]), sizeof(uint64_t)); - } - } - - file_.write(reinterpret_cast(&num_rows_), sizeof(num_rows_)); - - file_.write(reinterpret_cast(&metadata_start), sizeof(metadata_start)); - file_.write("TUFF", 4); -} diff --git a/src/read_write/ColumnarWriter.h b/src/read_write/ColumnarWriter.h deleted file mode 100644 index 7b60d71..0000000 --- a/src/read_write/ColumnarWriter.h +++ /dev/null @@ -1,34 +0,0 @@ -// -// Created by ragnarokk on 05.01.2026. -// - -#ifndef COLUMNAR_ENGINE_COLUMNARWRITER_H -#define COLUMNAR_ENGINE_COLUMNARWRITER_H - -#include -#include - -#include "Types.h" - -class ColumnarWriter { -public: - ColumnarWriter(const std::string& file_path); - ~ColumnarWriter(); - - void WriteHeader(const VectorOfStrings2D& column_names, - const std::vector& column_types); - void WriteBatch(const std::vector>& columns); - - // magic, columns by batches, number of columns, name, type, offsets, sizes, number of rows, - // metadata_start, magic - void Finalize(); - -private: - std::ofstream file_; - std::vector metadata_; - uint64_t current_offset_ = 0; - uint32_t num_rows_ = 0; - bool header_written_ = false; -}; - -#endif // COLUMNAR_ENGINE_COLUMNARWRITER_H \ No newline at end of file diff --git a/src/read_write/test/test_csv_parser.cpp b/src/read_write/test/test_csv_parser.cpp deleted file mode 100644 index ca53b27..0000000 --- a/src/read_write/test/test_csv_parser.cpp +++ /dev/null @@ -1,111 +0,0 @@ -// -// Created by ragnarokk on 04.01.2026. -// - -// Generated by AI - -#include - -#include "CSVReader.h" -#include "CSVTokenizer.h" - -#include -#include -#include - -namespace fs = std::filesystem; - -namespace { - -std::string CreateTempCSVFile(const std::string& content, const std::string& filename) { - const std::string path = "/tmp/" + filename; - std::ofstream out(path); - out << content; - out.close(); - return path; -} - -} // namespace - -TEST_CASE("CSVTokenizer does not materialize empty token on hard EOF", "[CSVTokenizer]") { - const std::string path = CreateTempCSVFile("", "test_tokenizer_hard_eof.csv"); - CSVTokenizer tokenizer(path); - VectorOfStrings2D row; - - const size_t width_before = row.Width(); - tokenizer.GetNextRow(row); - - REQUIRE(row.Width() == width_before); - REQUIRE(tokenizer.IsEOF()); - - fs::remove(path); -} - -TEST_CASE("CSVTokenizer keeps last token without trailing newline", "[CSVTokenizer]") { - const std::string path = CreateTempCSVFile("a,b", "test_tokenizer_no_trailing_newline.csv"); - CSVTokenizer tokenizer(path); - VectorOfStrings2D row; - - tokenizer.GetNextRow(row); - tokenizer.GetNextRow(row); - - REQUIRE(row.Width() == 2); - REQUIRE(row.GetString(0) == std::string_view("a")); - REQUIRE(row.GetString(1) == std::string_view("b")); - REQUIRE(tokenizer.IsEOF()); - - fs::remove(path); -} - -TEST_CASE("CSVReader does not return fake empty row at EOF", "[CSVReader]") { - const std::string path = CreateTempCSVFile("h1,h2\n1,2", "test_reader_eof.csv"); - CSVReader reader(path); - - VectorOfStrings2D header; - reader.ReadHeader(header); - REQUIRE(header.Width() == 2); - - VectorOfStrings2D row; - REQUIRE(reader.ReadRow(row)); - REQUIRE(row.Width() == 2); - REQUIRE(row.GetString(0) == std::string_view("1")); - REQUIRE(row.GetString(1) == std::string_view("2")); - - REQUIRE_FALSE(reader.ReadRow(row)); - REQUIRE(row.Width() == 0); - - fs::remove(path); -} - -TEST_CASE("CSVReader does not add extra row for trailing newline", "[CSVReader]") { - const std::string path = CreateTempCSVFile("h1\nx\n", "test_reader_trailing_newline.csv"); - CSVReader reader(path); - - VectorOfStrings2D header; - reader.ReadHeader(header); - - VectorOfStrings2D row; - REQUIRE(reader.ReadRow(row)); - REQUIRE(row.Width() == 1); - REQUIRE(row.GetString(0) == std::string_view("x")); - REQUIRE_FALSE(reader.ReadRow(row)); - - fs::remove(path); -} - -TEST_CASE("CSVReader preserves empty fields", "[CSVReader]") { - const std::string path = CreateTempCSVFile("a,b\n,\n", "test_reader_empty_fields.csv"); - CSVReader reader(path); - - VectorOfStrings2D header; - reader.ReadHeader(header); - - VectorOfStrings2D row; - REQUIRE(reader.ReadRow(row)); - REQUIRE(row.Width() == 2); - REQUIRE(row.GetString(0) == std::string_view("")); - REQUIRE(row.GetString(1) == std::string_view("")); - REQUIRE_FALSE(reader.ReadRow(row)); - - fs::remove(path); -} diff --git a/src/read_write/test/test_reader_writer.cpp b/src/read_write/test/test_reader_writer.cpp deleted file mode 100644 index 8c38c65..0000000 --- a/src/read_write/test/test_reader_writer.cpp +++ /dev/null @@ -1,496 +0,0 @@ -// -// Created by ragnarokk on 04.01.2026. -// - -// Generated by AI - -#include -#include -#include - -#include "../read_write/CSVToColumnar.h" -#include "../read_write/ColumnarReader.h" -#include "../Types.h" - -namespace fs = std::filesystem; - -std::string CreateTempCSVFile(const std::string& content, - const std::string& filename = "test_rw.csv") { - std::string filepath = "/tmp/" + filename; - std::ofstream file(filepath); - file << content; - file.close(); - return filepath; -} - -void RemoveTempFile(const std::string& filepath) { - if (fs::exists(filepath)) { - fs::remove(filepath); - } -} - -std::vector ExtractInt32Values(const std::shared_ptr& column) { - Int32Column* int_col = dynamic_cast(column.get()); - if (!int_col) { - return {}; - } - - std::vector result; - std::ofstream temp_file("/tmp/temp_int.bin", std::ios::binary); - int_col->WriteToFile(temp_file); - temp_file.close(); - - std::ifstream in_file("/tmp/temp_int.bin", std::ios::binary); - in_file.seekg(0, std::ios::end); - size_t size = in_file.tellg(); - in_file.seekg(0, std::ios::beg); - - result.resize(size / sizeof(int32_t)); - in_file.read(reinterpret_cast(result.data()), size); - in_file.close(); - fs::remove("/tmp/temp_int.bin"); - - return result; -} - -std::vector ExtractStringValues(const std::shared_ptr& column) { - StringColumn* str_col = dynamic_cast(column.get()); - if (!str_col) { - return {}; - } - - std::vector buffer; - std::ofstream temp_file("/tmp/temp_str.bin", std::ios::binary); - str_col->WriteToFile(temp_file); - temp_file.close(); - - std::ifstream in_file("/tmp/temp_str.bin", std::ios::binary); - in_file.seekg(0, std::ios::end); - size_t size = in_file.tellg(); - in_file.seekg(0, std::ios::beg); - - buffer.resize(size); - in_file.read(buffer.data(), size); - in_file.close(); - fs::remove("/tmp/temp_str.bin"); - - std::vector result; - size_t offset = 0; - while (offset < buffer.size()) { - uint32_t length; - std::memcpy(&length, buffer.data() + offset, sizeof(length)); - offset += sizeof(length); - std::string str(buffer.data() + offset, length); - result.push_back(str); - offset += length; - } - - return result; -} - -TEST_CASE("CSVToColumnar: Basic write operation", "[CSVToColumnar]") { - std::string csv_content = - "INT32:id,STRING:name\n" - "1,Alice\n" - "2,Bob\n" - "3,Charlie\n"; - std::string csv_path = CreateTempCSVFile(csv_content, "test_CSVToColumnar_basic.csv"); - std::string output_path = "/tmp/test_CSVToColumnar_basic.bin"; - - SECTION("Write CSV to binary format") { - CSVToColumnar csv_to_columnar; - REQUIRE_NOTHROW(csv_to_columnar.Convert(csv_path, output_path)); - REQUIRE(fs::exists(output_path)); - - std::ifstream file(output_path, std::ios::binary); - file.seekg(0, std::ios::end); - size_t file_size = file.tellg(); - REQUIRE(file_size > 0); - - file.seekg(0, std::ios::beg); - char magic[5] = {0}; - file.read(magic, 4); - REQUIRE(std::string(magic) == "TUFF"); - - file.close(); - } - - RemoveTempFile(csv_path); - RemoveTempFile(output_path); -} - -TEST_CASE("CSVToColumnar: Multiple types", "[CSVToColumnar]") { - std::string csv_content = - "INT32:id,STRING:name,FLOAT:score\n" - "1,Alice,95.5\n" - "2,Bob,87.3\n"; - std::string csv_path = CreateTempCSVFile(csv_content, "test_CSVToColumnar_multi.csv"); - std::string output_path = "/tmp/test_CSVToColumnar_multi.bin"; - - CSVToColumnar csv_to_columnar; - csv_to_columnar.Convert(csv_path, output_path); - - REQUIRE(fs::exists(output_path)); - REQUIRE(fs::file_size(output_path) > 0); - - RemoveTempFile(csv_path); - RemoveTempFile(output_path); -} - -TEST_CASE("Reader: Read metadata", "[Reader]") { - std::string csv_content = - "INT32:id,STRING:name\n" - "1,Alice\n" - "2,Bob\n"; - std::string csv_path = CreateTempCSVFile(csv_content, "test_reader_meta.csv"); - std::string output_path = "/tmp/test_reader_meta.bin"; - - CSVToColumnar csv_to_columnar; - csv_to_columnar.Convert(csv_path, output_path); - - SECTION("Read and verify metadata") { - ColumnarReader reader(output_path); - const auto& metadata = reader.GetMetadata(); - - REQUIRE(metadata.size() == 2); - REQUIRE(metadata[0].name == "id"); - REQUIRE(metadata[0].type == ColumnType::INT32); - REQUIRE(metadata[1].name == "name"); - REQUIRE(metadata[1].type == ColumnType::STRING); - } - - RemoveTempFile(csv_path); - RemoveTempFile(output_path); -} - -TEST_CASE("Reader: Read raw column data", "[Reader]") { - std::string csv_content = - "INT32:value\n" - "100\n" - "200\n" - "300\n"; - std::string csv_path = CreateTempCSVFile(csv_content, "test_reader_raw.csv"); - std::string output_path = "/tmp/test_reader_raw.bin"; - - CSVToColumnar csv_to_columnar; - csv_to_columnar.Convert(csv_path, output_path); - - ColumnarReader reader(output_path); - - SECTION("Get raw data for column") { - std::vector raw_data = reader.GetRawColumnData(0, 0); - REQUIRE(!raw_data.empty()); - REQUIRE(raw_data.size() == 3 * sizeof(int32_t)); - } - - RemoveTempFile(csv_path); - RemoveTempFile(output_path); -} - -TEST_CASE("Reader: Read typed column data", "[Reader]") { - std::string csv_content = - "INT32:id,STRING:city\n" - "1,Moscow\n" - "2,London\n"; - std::string csv_path = CreateTempCSVFile(csv_content, "test_reader_typed.csv"); - std::string output_path = "/tmp/test_reader_typed.bin"; - - CSVToColumnar csv_to_columnar; - csv_to_columnar.Convert(csv_path, output_path); - - ColumnarReader reader(output_path); - - SECTION("Read INT32 column") { - auto column = reader.GetColumnData(0, 0); - REQUIRE(column != nullptr); - REQUIRE(column->GetType() == ColumnType::INT32); - REQUIRE(column->Size() == 2); - } - - SECTION("Read STRING column") { - auto column = reader.GetColumnData(1, 0); - REQUIRE(column != nullptr); - REQUIRE(column->GetType() == ColumnType::STRING); - REQUIRE(column->Size() == 2); - } - - RemoveTempFile(csv_path); - RemoveTempFile(output_path); -} - -TEST_CASE("CSVToColumnar+Reader: Round-trip test with INT data", - "[CSVToColumnar][Reader][Integration]") { - std::string csv_content = - "INT32:numbers\n" - "10\n" - "20\n" - "30\n"; - std::string csv_path = CreateTempCSVFile(csv_content, "test_roundtrip_int.csv"); - std::string output_path = "/tmp/test_roundtrip_int.bin"; - - CSVToColumnar csv_to_columnar; - csv_to_columnar.Convert(csv_path, output_path); - - ColumnarReader reader(output_path); - auto column = reader.GetColumnData(0, 0); - - REQUIRE(column->GetType() == ColumnType::INT32); - REQUIRE(column->Size() == 3); - - RemoveTempFile(csv_path); - RemoveTempFile(output_path); -} - -TEST_CASE("CSVToColumnar+Reader: Round-trip test with STRING data", - "[CSVToColumnar][Reader][Integration]") { - std::string csv_content = - "STRING:cities\n" - "Moscow\n" - "London\n" - "Paris\n"; - std::string csv_path = CreateTempCSVFile(csv_content, "test_roundtrip_str.csv"); - std::string output_path = "/tmp/test_roundtrip_str.bin"; - - CSVToColumnar csv_to_columnar; - csv_to_columnar.Convert(csv_path, output_path); - - ColumnarReader reader(output_path); - auto column = reader.GetColumnData(0, 0); - - REQUIRE(column->GetType() == ColumnType::STRING); - REQUIRE(column->Size() == 3); - - RemoveTempFile(csv_path); - RemoveTempFile(output_path); -} - -TEST_CASE("CSVToColumnar+Reader: Multiple columns round-trip", - "[CSVToColumnar][Reader][Integration]") { - std::string csv_content = - "INT32:id,STRING:name,FLOAT:score\n" - "1,Alice,95.5\n" - "2,Bob,87.3\n" - "3,Charlie,92.1\n"; - std::string csv_path = CreateTempCSVFile(csv_content, "test_roundtrip_multi.csv"); - std::string output_path = "/tmp/test_roundtrip_multi.bin"; - - CSVToColumnar csv_to_columnar; - csv_to_columnar.Convert(csv_path, output_path); - - ColumnarReader reader(output_path); - const auto& metadata = reader.GetMetadata(); - - SECTION("Verify metadata") { - REQUIRE(metadata.size() == 3); - REQUIRE(metadata[0].name == "id"); - REQUIRE(metadata[0].type == ColumnType::INT32); - REQUIRE(metadata[1].name == "name"); - REQUIRE(metadata[1].type == ColumnType::STRING); - REQUIRE(metadata[2].name == "score"); - REQUIRE(metadata[2].type == ColumnType::FLOAT); - } - - SECTION("Verify all columns have data") { - auto col0 = reader.GetColumnData(0, 0); - auto col1 = reader.GetColumnData(1, 0); - auto col2 = reader.GetColumnData(2, 0); - - REQUIRE(col0->Size() == 3); - REQUIRE(col1->Size() == 3); - REQUIRE(col2->Size() == 3); - } - - RemoveTempFile(csv_path); - RemoveTempFile(output_path); -} - -TEST_CASE("Reader: Invalid file", "[Reader][errors]") { - REQUIRE_THROWS_AS(ColumnarReader("/nonexistent/file.bin"), std::runtime_error); -} - -TEST_CASE("Reader: Invalid chunk index", "[Reader][errors]") { - std::string csv_content = "INT32:value\n100\n"; - std::string csv_path = CreateTempCSVFile(csv_content, "test_invalid_chunk.csv"); - std::string output_path = "/tmp/test_invalid_chunk.bin"; - - CSVToColumnar csv_to_columnar; - csv_to_columnar.Convert(csv_path, output_path); - - ColumnarReader reader(output_path); - - REQUIRE_THROWS_AS(reader.GetRawColumnData(0, 999), std::runtime_error); - - RemoveTempFile(csv_path); - RemoveTempFile(output_path); -} - -TEST_CASE("CSVToColumnar: Empty CSV file", "[CSVToColumnar]") { - std::string csv_content = "INT32:id,STRING:name\n"; - std::string csv_path = CreateTempCSVFile(csv_content, "test_empty.csv"); - std::string output_path = "/tmp/test_empty.bin"; - - CSVToColumnar csv_to_columnar; - csv_to_columnar.Convert(csv_path, output_path); - - REQUIRE(fs::exists(output_path)); - - ColumnarReader reader(output_path); - const auto& metadata = reader.GetMetadata(); - REQUIRE(metadata.size() == 2); - - RemoveTempFile(csv_path); - RemoveTempFile(output_path); -} - -TEST_CASE("CSVToColumnar+Reader: Large dataset", "[CSVToColumnar][Reader][Integration]") { - std::string csv_content = "INT32:id,STRING:name\n"; - for (int i = 0; i < 1000; ++i) { - csv_content += std::to_string(i) + ",User" + std::to_string(i) + "\n"; - } - - std::string csv_path = CreateTempCSVFile(csv_content, "test_large.csv"); - std::string output_path = "/tmp/test_large.bin"; - - CSVToColumnar csv_to_columnar; - csv_to_columnar.Convert(csv_path, output_path); - - ColumnarReader reader(output_path); - auto col0 = reader.GetColumnData(0, 0); - auto col1 = reader.GetColumnData(1, 0); - - REQUIRE(col0->Size() == 1000); - REQUIRE(col1->Size() == 1000); - - RemoveTempFile(csv_path); - RemoveTempFile(output_path); -} - -TEST_CASE("CSVToColumnar+Reader: Special characters in strings", - "[CSVToColumnar][Reader][Integration]") { - std::string csv_content = - "STRING:text\n" - "\"Hello, World!\"\n" - "Simple text\n" - "\"Text with, comma\"\n"; - std::string csv_path = CreateTempCSVFile(csv_content, "test_special.csv"); - std::string output_path = "/tmp/test_special.bin"; - - CSVToColumnar csv_to_columnar; - csv_to_columnar.Convert(csv_path, output_path); - - ColumnarReader reader(output_path); - auto column = reader.GetColumnData(0, 0); - - REQUIRE(column->GetType() == ColumnType::STRING); - REQUIRE(column->Size() == 3); - - RemoveTempFile(csv_path); - RemoveTempFile(output_path); -} - -TEST_CASE("CSVToColumnar+Reader: Float precision", "[CSVToColumnar][Reader][Integration]") { - std::string csv_content = - "FLOAT:values\n" - "3.14159\n" - "2.71828\n" - "1.618\n"; - std::string csv_path = CreateTempCSVFile(csv_content, "test_float.csv"); - std::string output_path = "/tmp/test_float.bin"; - - CSVToColumnar csv_to_columnar; - csv_to_columnar.Convert(csv_path, output_path); - - ColumnarReader reader(output_path); - auto column = reader.GetColumnData(0, 0); - - REQUIRE(column->GetType() == ColumnType::FLOAT); - REQUIRE(column->Size() == 3); - - RemoveTempFile(csv_path); - RemoveTempFile(output_path); -} - -TEST_CASE("Reader: Metadata chunks count", "[Reader]") { - std::string csv_content = "INT32:id\n1\n2\n3\n"; - std::string csv_path = CreateTempCSVFile(csv_content, "test_chunks.csv"); - std::string output_path = "/tmp/test_chunks.bin"; - - CSVToColumnar csv_to_columnar; - csv_to_columnar.Convert(csv_path, output_path); - - ColumnarReader reader(output_path); - const auto& metadata = reader.GetMetadata(); - - REQUIRE(metadata[0].offsets.size() == metadata[0].sizes.size()); - REQUIRE(!metadata[0].offsets.empty()); - - RemoveTempFile(csv_path); - RemoveTempFile(output_path); -} - -TEST_CASE("CSVToColumnar: Convert with schema", "[CSVToColumnar][Schema]") { - std::string csv_content = - "Alice,1\n" - "Bob,2\n" - "Charlie,3\n"; - std::string csv_path = CreateTempCSVFile(csv_content, "test_schema_source.csv"); - - std::string schema_content = - "name,STRING\n" - "id,INT32\n"; - std::string schema_path = CreateTempCSVFile(schema_content, "test_schema_desc.schema"); - - std::string output_path = "/tmp/test_schema_output.bin"; - - SECTION("Convert using schema and check columns order") { - CSVToColumnar csv_to_columnar; - REQUIRE_NOTHROW(csv_to_columnar.ConvertWithSchema(csv_path, output_path, schema_path)); - REQUIRE(fs::exists(output_path)); - - ColumnarReader reader(output_path); - const auto& metadata = reader.GetMetadata(); - - REQUIRE(metadata.size() == 2); - // Header ordering depends on CSV: name then id in header, but types must match CSV positions mapped via names. - // Wait, CSVToColumnar maps the CSV headers to schema types via `column_type_map` and pushes exactly in the CSV order: - // column_names.push_back(header); - // column_types.push_back(type); - // Let's verify that columns match CSV order "name, id" but correct types. - REQUIRE(metadata[0].name == "name"); - REQUIRE(metadata[0].type == ColumnType::STRING); - REQUIRE(metadata[1].name == "id"); - REQUIRE(metadata[1].type == ColumnType::INT32); - - auto name_col = reader.GetColumnData(0, 0); - REQUIRE(name_col->GetType() == ColumnType::STRING); - REQUIRE(name_col->Size() == 3); - - auto id_col = reader.GetColumnData(1, 0); - REQUIRE(id_col->GetType() == ColumnType::INT32); - REQUIRE(id_col->Size() == 3); - } - - RemoveTempFile(csv_path); - RemoveTempFile(schema_path); - RemoveTempFile(output_path); -} - -TEST_CASE("CSVToColumnar: Convert with missing schema column", "[CSVToColumnar][Schema][errors]") { - std::string csv_content = - "Alice,1,25\n"; - std::string csv_path = CreateTempCSVFile(csv_content, "test_schema_err.csv"); - - std::string schema_content = - "name,STRING\n" - "id,INT32\n"; - std::string schema_path = CreateTempCSVFile(schema_content, "test_schema_desc_err.schema"); - - std::string output_path = "/tmp/test_schema_output_err.bin"; - - CSVToColumnar csv_to_columnar; - REQUIRE_THROWS_AS(csv_to_columnar.ConvertWithSchema(csv_path, schema_path, output_path), std::runtime_error); - - RemoveTempFile(csv_path); - RemoveTempFile(schema_path); - RemoveTempFile(output_path); -} diff --git a/src/types/ColumnType.h b/src/types/ColumnType.h new file mode 100644 index 0000000..ae6842b --- /dev/null +++ b/src/types/ColumnType.h @@ -0,0 +1,48 @@ +#pragma once + +#include +#include +#include + +enum class ColumnType : uint8_t { + INT16, + INT32, + INT64, + INT128, + FLOAT, + DOUBLE, + LONGDOUBLE, + CHAR, + STRING, + DATE, + TIMESTAMP, +}; + +struct ColumnMetadata { + std::string name; + ColumnType type; + std::vector offsets; + std::vector sizes; +}; + +// TODO: for macro if German is gonna tell i coded that xyevo +// template +// struct ColumnTraits; +// +// template <> +// struct ColumnTraits { +// using ValueType = int16_t; +// static constexpr std::string Name = "INT16"; +// }; +// +// template <> +// struct ColumnTraits { +// using ValueType = int32_t; +// static constexpr std::string Name = "INT32"; +// }; +// +// template <> +// struct ColumnTraits { +// using ValueType = int64_t; +// static constexpr std::string Name = "INT64"; +// }; diff --git a/src/types/Date.h b/src/types/Date.h new file mode 100644 index 0000000..674f90a --- /dev/null +++ b/src/types/Date.h @@ -0,0 +1,53 @@ +#pragma once + +#include "utils/Assert.h" + +#include +#include +#include + +struct Date { + static int32_t Parse(std::string_view unit) { + if (unit.size() != 10 || unit[4] != '-' || unit[7] != '-') [[unlikely]] { + THROW_RUNTIME_ERROR("Invalid date format: " + std::string(unit)); + } + + int32_t year = + (unit[0] - '0') * 1000 + (unit[1] - '0') * 100 + (unit[2] - '0') * 10 + (unit[3] - '0'); + int32_t month = (unit[5] - '0') * 10 + (unit[6] - '0'); + int32_t day = (unit[8] - '0') * 10 + (unit[9] - '0'); + + year -= (month <= 2); + const int era = (year >= 0 ? year : year - 399) / 400; + const unsigned yoe = static_cast(year - era * 400); + const unsigned doy = (153 * (month + (month > 2 ? -3 : 9)) + 2) / 5 + day - 1; + const unsigned doe = yoe * 365 + yoe / 4 - yoe / 100 + doy; + + return era * 146097 + static_cast(doe) - 719468; + } + + static std::string Format(int32_t days) { + days += 719468; + const int era = (days >= 0 ? days : days - 146096) / 146097; + const unsigned doe = static_cast(days - era * 146097); + const unsigned yoe = (doe - doe / 1460 + doe / 36524 - doe / 146096) / 365; + const int y = static_cast(yoe) + era * 400; + const unsigned doy = doe - (365 * yoe + yoe / 4 - yoe / 100); + const unsigned mp = (5 * doy + 2) / 153; + const unsigned d = doy - (153 * mp + 2) / 5 + 1; + const unsigned m = mp + (mp < 10 ? 3 : -9); + const int year = y + (m <= 2); + + std::string res = "0000-00-00"; + auto write_digit = [&](int val, int pos, int len) { + for (int i = 0; i < len; ++i) { + res[pos + len - 1 - i] = (val % 10) + '0'; + val /= 10; + } + }; + write_digit(year, 0, 4); + write_digit(m, 5, 2); + write_digit(d, 8, 2); + return res; + } +}; diff --git a/src/types/Timestamp.h b/src/types/Timestamp.h new file mode 100644 index 0000000..ae4ab6f --- /dev/null +++ b/src/types/Timestamp.h @@ -0,0 +1,69 @@ +#pragma once + +#include "Date.h" + +#include +#include +#include + +struct Timestamp { + static int64_t Parse(std::string_view unit) { + if (unit.size() != 19 || unit[4] != '-' || unit[7] != '-' || unit[10] != ' ' || + unit[13] != ':' || unit[16] != ':') [[unlikely]] { + THROW_RUNTIME_ERROR("Invalid date format: " + std::string(unit)); + } + + auto to_int2 = [](char a, char b) { return (a - '0') * 10 + (b - '0'); }; + + int year = + (unit[0] - '0') * 1000 + (unit[1] - '0') * 100 + (unit[2] - '0') * 10 + (unit[3] - '0'); + int month = to_int2(unit[5], unit[6]); + int day = to_int2(unit[8], unit[9]); + int hour = to_int2(unit[11], unit[12]); + int minute = to_int2(unit[14], unit[15]); + int second = to_int2(unit[17], unit[18]); + + year -= (month <= 2); + const int era = (year >= 0 ? year : year - 399) / 400; + const unsigned yoe = static_cast(year - era * 400); + const unsigned doy = (153 * (month + (month > 2 ? -3 : 9)) + 2) / 5 + day - 1; + const unsigned doe = yoe * 365 + yoe / 4 - yoe / 100 + doy; + + int32_t days = era * 146097 + static_cast(doe) - 719468; + int64_t seconds = hour * 3600 + minute * 60 + second; + return static_cast(days) * 86400 + seconds; + } + + static std::string Format(int64_t total_seconds) { + int64_t days = total_seconds / 86400; + int64_t seconds_in_day = total_seconds % 86400; + if (seconds_in_day < 0) { + seconds_in_day += 86400; + days -= 1; + } + + std::string res = Date::Format(static_cast(days)); + res += " 00:00:00"; + int hour = seconds_in_day / 3600; + int minute = (seconds_in_day % 3600) / 60; + int second = seconds_in_day % 60; + auto write_digit = [&](int val, int pos) { + res[pos] = (val / 10) + '0'; + res[pos + 1] = (val % 10) + '0'; + }; + + write_digit(hour, 11); + write_digit(minute, 14); + write_digit(second, 17); + + return res; + } + + static int32_t ExtractMinute(int64_t total_seconds) { + int64_t seconds_in_day = total_seconds % 86400; + if (seconds_in_day < 0) { + seconds_in_day += 86400; + } + return (seconds_in_day % 3600) / 60; + } +}; diff --git a/src/macro.h b/src/utils/Assert.h similarity index 52% rename from src/macro.h rename to src/utils/Assert.h index 972e325..3dc4e11 100644 --- a/src/macro.h +++ b/src/utils/Assert.h @@ -1,36 +1,7 @@ -// -// Created by ragnarokk on 30.03.2026. -// +#pragma once -#ifndef COLUMNAR_ENGINE_MACRO_H -#define COLUMNAR_ENGINE_MACRO_H - -// USE THIS EVERYWHERE FOR SWITCHING -// Прости, Герман, но я не хочу везде исправлять переборы типов, если новый добавился... -#define FOR_EACH_COLUMN_TYPE(M) \ - M(INT16, "INT16", Int16Column) \ - M(INT32, "INT32", Int32Column) \ - M(INT64, "INT64", Int64Column) \ - M(INT128, "INT128", Int128Column) \ - M(FLOAT, "FLOAT", FloatColumn) \ - M(DOUBLE, "DOUBLE", DoubleColumn) \ - M(LONGDOUBLE, "LONGDOUBLE", LongDoubleColumn) \ - M(CHAR, "CHAR", CharColumn) \ - M(STRING, "STRING", StringColumn) \ - M(DATE, "DATE", DateColumn) \ - M(TIMESTAMP, "TIMESTAMP", TimestampColumn) - -#define FOR_NUMERIC_COLUMN_TYPE(M) \ - M(INT16, "INT16", Int16Column) \ - M(INT32, "INT32", Int32Column) \ - M(INT64, "INT64", Int64Column) \ - M(INT128, "INT128", Int128Column) \ - M(FLOAT, "FLOAT", FloatColumn) \ - M(DOUBLE, "DOUBLE", DoubleColumn) \ - M(LONGDOUBLE, "LONGDOUBLE", LongDoubleColumn) \ - M(CHAR, "CHAR", CharColumn) \ - M(DATE, "DATE", DateColumn) \ - M(TIMESTAMP, "TIMESTAMP", TimestampColumn) +#include +#include #define ASSERT(cond) \ do { \ @@ -55,5 +26,3 @@ #define THROW_RUNTIME_ERROR(msg) \ throw std::runtime_error(std::string(__FILE__) + ":" + std::to_string(__LINE__) + ": " + (msg)) - -#endif // COLUMNAR_ENGINE_MACRO_H diff --git a/src/utils/Macro.h b/src/utils/Macro.h new file mode 100644 index 0000000..40888b6 --- /dev/null +++ b/src/utils/Macro.h @@ -0,0 +1,25 @@ +#pragma once + +#include "column/Column.h" +#include "column/CharColumn.h" +#include "column/NumericColumn.h" +#include "column/StringColumn.h" +#include "column/TemporalColumn.h" + +// USE THIS EVERYWHERE FOR SWITCHING +#define FOR_NUMERIC_COLUMN_TYPE(M) \ + M(INT16, "INT16", Int16Column) \ + M(INT32, "INT32", Int32Column) \ + M(INT64, "INT64", Int64Column) \ + M(INT128, "INT128", Int128Column) \ + M(FLOAT, "FLOAT", FloatColumn) \ + M(DOUBLE, "DOUBLE", DoubleColumn) \ + M(LONGDOUBLE, "LONGDOUBLE", LongDoubleColumn) \ + M(CHAR, "CHAR", CharColumn) \ + M(DATE, "DATE", DateColumn) \ + M(TIMESTAMP, "TIMESTAMP", TimestampColumn) + +// USE THIS EVERYWHERE FOR SWITCHING +#define FOR_EACH_COLUMN_TYPE(M) \ + FOR_NUMERIC_COLUMN_TYPE(M) \ + M(STRING, "STRING", StringColumn) diff --git a/src/VectorOfStrings.h b/src/utils/VectorOfStrings.h similarity index 96% rename from src/VectorOfStrings.h rename to src/utils/VectorOfStrings.h index 8f7d6c6..ffed34f 100644 --- a/src/VectorOfStrings.h +++ b/src/utils/VectorOfStrings.h @@ -1,11 +1,6 @@ -// -// Created by ragnarokk on 01.04.2026. -// +#pragma once -#ifndef COLUMNAR_ENGINE_VECTOROFSTRINGS_H -#define COLUMNAR_ENGINE_VECTOROFSTRINGS_H - -#include "macro.h" +#include "utils/Assert.h" #include #include @@ -233,5 +228,3 @@ class VectorOfStrings2D { size_t columns_in_line_; bool adding_string_; }; - -#endif // COLUMNAR_ENGINE_VECTOROFSTRINGS_H diff --git a/src/test_vector_of_strings.cpp b/src/utils/test/TestVectorOfStrings.cpp similarity index 76% rename from src/test_vector_of_strings.cpp rename to src/utils/test/TestVectorOfStrings.cpp index dedca7c..64f48a4 100644 --- a/src/test_vector_of_strings.cpp +++ b/src/utils/test/TestVectorOfStrings.cpp @@ -1,14 +1,51 @@ -// -// Created by ragnarokk on 02.04.2026. -// -// AI generated +#include "utils/VectorOfStrings.h" #include +#include +#include -#include "VectorOfStrings.h" +TEST_CASE("VectorOfStrings: Basic operations", "[VectorOfStrings]") { + VectorOfStrings vec; + REQUIRE(vec.Size() == 0); + REQUIRE(vec.DataSize() == 0); + + vec.PushBack("hello"); + vec.PushBack("world"); + + REQUIRE(vec.Size() == 2); + REQUIRE(vec.GetString(0) == "hello"); + REQUIRE(vec.GetString(1) == "world"); +} -#include -#include +TEST_CASE("VectorOfStrings2D: Basic operations", "[VectorOfStrings2D]") { + VectorOfStrings2D vec; + vec.StartAddString(); + vec.ContinueAddString("hello"); + vec.EndAddString(); + + vec.StartAddString(); + vec.ContinueAddString("world"); + vec.EndAddString(); + + vec.StartNewLine(); + + vec.StartAddString(); + vec.ContinueAddString("foo"); + vec.EndAddString(); + + vec.StartAddString(); + vec.ContinueAddString("bar"); + vec.EndAddString(); + + vec.StartNewLine(); + + REQUIRE(vec.Height() == 2); + REQUIRE(vec.Width() == 2); + REQUIRE(vec.GetString2D(0, 0) == "hello"); + REQUIRE(vec.GetString2D(0, 1) == "world"); + REQUIRE(vec.GetString2D(1, 0) == "foo"); + REQUIRE(vec.GetString2D(1, 1) == "bar"); +} TEST_CASE("VectorOfStrings2D supports 1D vector mode", "[vector_of_strings]") { VectorOfStrings2D values; @@ -127,4 +164,3 @@ TEST_CASE("VectorOfStrings2D reads non square 2D table correctly", "[vector_of_s REQUIRE(table.GetString2D(1, 1) == std::string_view("r1c1")); REQUIRE(table.GetString2D(1, 2) == std::string_view("r1c2")); } - From 4f734e47a0722d382bf2124111b3e05513c13618 Mon Sep 17 00:00:00 2001 From: Mikhail Fomichev Date: Wed, 6 May 2026 21:44:58 +0300 Subject: [PATCH 02/18] feat: like, not like filters added, queries 20-23 done --- src/CMakeLists.txt | 1 + src/column/NumericColumn.h | 4 +- src/execution/ExecutionHelper.h | 17 -- src/execution/expressions/AggExpHelper.h | 48 ++++ .../expressions/AggregationExpressions.h | 213 ++++++++---------- .../expressions/AggregationFunctions.h | 97 ++++---- src/execution/expressions/FilterExpressions.h | 117 +++++++--- src/execution/expressions/FilterFunctions.h | 78 +++++-- src/execution/expressions/ScalarFunctions.h | 4 +- src/main.cpp | 47 +++- 10 files changed, 410 insertions(+), 216 deletions(-) create mode 100644 src/execution/expressions/AggExpHelper.h diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 75a44f9..18664df 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -48,6 +48,7 @@ add_executable(columnar-engine types/ColumnType.h types/Date.h types/Timestamp.h + execution/expressions/AggExpHelper.h ) target_link_libraries(columnar-engine PRIVATE project_includes) diff --git a/src/column/NumericColumn.h b/src/column/NumericColumn.h index 59f6ae9..f834313 100644 --- a/src/column/NumericColumn.h +++ b/src/column/NumericColumn.h @@ -110,12 +110,12 @@ class NumericColumn : public Column { NumericColumn() = default; NumericColumn(const ContainerType& data) : data_(data) {} - NumericColumn(ContainerType&& data) : data_(std::move(data)) {} + NumericColumn(ContainerType&& data) noexcept : data_(std::move(data)) {} NumericColumn operator=(const ContainerType& data) { data_ = data; return *this; } - NumericColumn operator=(ContainerType&& data) { + NumericColumn operator=(ContainerType&& data) noexcept { data_ = std::move(data); return *this; } diff --git a/src/execution/ExecutionHelper.h b/src/execution/ExecutionHelper.h index a90728f..6a55f04 100644 --- a/src/execution/ExecutionHelper.h +++ b/src/execution/ExecutionHelper.h @@ -15,23 +15,6 @@ class ExecutionHelper { public: - template - static void IterateColumnData(const RecordBatch& batch, size_t column_index, Func&& func) { - auto* column = static_cast(batch.columns[column_index].get()); - const auto& data = column->GetData(); - - if (!batch.selection_vector) { - for (size_t i = 0; i < batch.num_rows; ++i) { - func(data[i], i, i); - } - } else { - const auto& sel = *batch.selection_vector; - for (size_t i = 0; i < batch.num_rows; ++i) { - func(data[sel[i]], i, sel[i]); - } - } - } - static std::string FormatValue(const Column& col, size_t index) { ColumnType type = col.GetType(); if (type == ColumnType::STRING) { diff --git a/src/execution/expressions/AggExpHelper.h b/src/execution/expressions/AggExpHelper.h new file mode 100644 index 0000000..66a85b0 --- /dev/null +++ b/src/execution/expressions/AggExpHelper.h @@ -0,0 +1,48 @@ +#pragma once + +#include "Macro.h" +#include "execution/OperatorsBase.h" + +class AggExpHelper { +public: + template + static void IterateColumnData(const RecordBatch& batch, size_t column_index, Func&& func) { + auto* column = static_cast(batch.columns[column_index].get()); + const auto& data = column->GetData(); + + if (!batch.selection_vector) { + for (size_t i = 0; i < batch.num_rows; ++i) { + func(data[i], i, i); + } + } else { + const auto& sel = *batch.selection_vector; + for (size_t i = 0; i < batch.num_rows; ++i) { + func(data[sel[i]], i, sel[i]); + } + } + } + + template + static auto DispatchNumeric(ColumnType type, Functor&& f) { +#define HANDLE_TYPE(ENUM_VAL, STR_VAL, CLASS_TYPE) \ + case ColumnType::ENUM_VAL: return f.template operator()(ColumnType::ENUM_VAL); + + switch (type) { + FOR_NUMERIC_COLUMN_TYPE(HANDLE_TYPE) + default: THROW_NOT_IMPLEMENTED; + } +#undef HANDLE_TYPE + } + + template + static auto DispatchAll(ColumnType type, Functor&& f) { +#define HANDLE_TYPE(ENUM_VAL, STR_VAL, CLASS_TYPE) \ + case ColumnType::ENUM_VAL: return f.template operator()(ColumnType::ENUM_VAL); + + switch (type) { + FOR_EACH_COLUMN_TYPE(HANDLE_TYPE) + default: THROW_NOT_IMPLEMENTED; + } +#undef HANDLE_TYPE + } +}; \ No newline at end of file diff --git a/src/execution/expressions/AggregationExpressions.h b/src/execution/expressions/AggregationExpressions.h index 8102b2d..83a66d8 100644 --- a/src/execution/expressions/AggregationExpressions.h +++ b/src/execution/expressions/AggregationExpressions.h @@ -1,6 +1,7 @@ #pragma once #include "execution/expressions/AggregationFunctions.h" +#include "execution/expressions/AggExpHelper.h" #include "column/Schema.h" #include @@ -72,38 +73,29 @@ class DistinctCountExpression : public AggregateExpression { return "distinct_count_" + column_name_; } - std::unique_ptr CreateGlobalAggregationFunction(const Schema& child_schema, - Schema& output_schema) override { + std::unique_ptr CreateGlobalAggregationFunction( + const Schema& child_schema, Schema& output_schema) override { size_t column_ind = child_schema.GetColumnIndexByName(GetName()); ColumnType column_type = child_schema.GetColumnTypeByName(GetName()); - output_schema.AddColumn(GetOutputName(), ColumnType::INT32); - -#define HANDLE_TYPE(ENUM_VAL, STR_VAL, CLASS_TYPE) \ - case ColumnType::ENUM_VAL: \ - return std::make_unique>(column_ind); - switch (column_type) { - FOR_EACH_COLUMN_TYPE(HANDLE_TYPE); - default: THROW_NOT_IMPLEMENTED; - } -#undef HANDLE_TYPE + return AggExpHelper::DispatchAll( + column_type, [&](ColumnType) -> std::unique_ptr { + output_schema.AddColumn(GetOutputName(), ColumnType::INT64); + return std::make_unique>(column_ind); + }); } - std::unique_ptr CreateGroupedAggregationFunction(const Schema& child_schema, - Schema& output_schema) override { + std::unique_ptr CreateGroupedAggregationFunction( + const Schema& child_schema, Schema& output_schema) override { size_t column_ind = child_schema.GetColumnIndexByName(GetName()); ColumnType column_type = child_schema.GetColumnTypeByName(GetName()); - output_schema.AddColumn(GetOutputName(), ColumnType::INT32); -#define HANDLE_TYPE(ENUM_VAL, STR_VAL, CLASS_TYPE) \ - case ColumnType::ENUM_VAL: \ - return std::make_unique>(column_ind); - - switch (column_type) { - FOR_EACH_COLUMN_TYPE(HANDLE_TYPE); - default: THROW_NOT_IMPLEMENTED; - } -#undef HANDLE_TYPE + return AggExpHelper::DispatchAll( + column_type, + [&](ColumnType) -> std::unique_ptr { + output_schema.AddColumn(GetOutputName(), ColumnType::INT64); + return std::make_unique>(column_ind); + }); } }; @@ -120,38 +112,32 @@ class MinExpression : public AggregateExpression { return "min_" + column_name_; } - std::unique_ptr CreateGlobalAggregationFunction(const Schema& child_schema, - Schema& output_schema) override { + std::unique_ptr CreateGlobalAggregationFunction( + const Schema& child_schema, Schema& output_schema) override { size_t column_ind = child_schema.GetColumnIndexByName(GetName()); ColumnType column_type = child_schema.GetColumnTypeByName(GetName()); -#define HANDLE_TYPE(ENUM_VAL, STR_VAL, CLASS_TYPE) \ - case ColumnType::ENUM_VAL: \ - output_schema.AddColumn(GetOutputName(), ColumnType::ENUM_VAL); \ - return std::make_unique>(column_ind); - - switch (column_type) { - FOR_NUMERIC_COLUMN_TYPE(HANDLE_TYPE); - default: THROW_NOT_IMPLEMENTED; - } -#undef HANDLE_TYPE + return AggExpHelper::DispatchAll( + column_type, + [&](ColumnType t) -> std::unique_ptr { + output_schema.AddColumn(GetOutputName(), t); + return std::make_unique>( + column_ind); + }); } - std::unique_ptr CreateGroupedAggregationFunction(const Schema& child_schema, - Schema& output_schema) override { + std::unique_ptr CreateGroupedAggregationFunction( + const Schema& child_schema, Schema& output_schema) override { size_t column_ind = child_schema.GetColumnIndexByName(GetName()); ColumnType column_type = child_schema.GetColumnTypeByName(GetName()); -#define HANDLE_TYPE(ENUM_VAL, STR_VAL, CLASS_TYPE) \ - case ColumnType::ENUM_VAL: \ - output_schema.AddColumn(GetOutputName(), ColumnType::ENUM_VAL); \ - return std::make_unique>(column_ind); - - switch (column_type) { - FOR_NUMERIC_COLUMN_TYPE(HANDLE_TYPE); - default: THROW_NOT_IMPLEMENTED; - } -#undef HANDLE_TYPE + return AggExpHelper::DispatchAll( + column_type, + [&](ColumnType t) -> std::unique_ptr { + output_schema.AddColumn(GetOutputName(), t); + return std::make_unique>( + column_ind); + }); } }; @@ -168,38 +154,32 @@ class MaxExpression : public AggregateExpression { return "max_" + column_name_; } - std::unique_ptr CreateGlobalAggregationFunction(const Schema& child_schema, - Schema& output_schema) override { + std::unique_ptr CreateGlobalAggregationFunction( + const Schema& child_schema, Schema& output_schema) override { size_t column_ind = child_schema.GetColumnIndexByName(GetName()); ColumnType column_type = child_schema.GetColumnTypeByName(GetName()); -#define HANDLE_TYPE(ENUM_VAL, STR_VAL, CLASS_TYPE) \ - case ColumnType::ENUM_VAL: \ - output_schema.AddColumn(GetOutputName(), ColumnType::ENUM_VAL); \ - return std::make_unique>(column_ind); - - switch (column_type) { - FOR_NUMERIC_COLUMN_TYPE(HANDLE_TYPE); - default: THROW_NOT_IMPLEMENTED; - } -#undef HANDLE_TYPE + return AggExpHelper::DispatchAll( + column_type, + [&](ColumnType t) -> std::unique_ptr { + output_schema.AddColumn(GetOutputName(), t); + return std::make_unique>( + column_ind); + }); } - std::unique_ptr CreateGroupedAggregationFunction(const Schema& child_schema, - Schema& output_schema) override { + std::unique_ptr CreateGroupedAggregationFunction( + const Schema& child_schema, Schema& output_schema) override { size_t column_ind = child_schema.GetColumnIndexByName(GetName()); ColumnType column_type = child_schema.GetColumnTypeByName(GetName()); -#define HANDLE_TYPE(ENUM_VAL, STR_VAL, CLASS_TYPE) \ - case ColumnType::ENUM_VAL: \ - output_schema.AddColumn(GetOutputName(), ColumnType::ENUM_VAL); \ - return std::make_unique>(column_ind); - - switch (column_type) { - FOR_NUMERIC_COLUMN_TYPE(HANDLE_TYPE); - default: THROW_NOT_IMPLEMENTED; - } -#undef HANDLE_TYPE + return AggExpHelper::DispatchAll( + column_type, + [&](ColumnType t) -> std::unique_ptr { + output_schema.AddColumn(GetOutputName(), t); + return std::make_unique>( + column_ind); + }); } }; @@ -216,64 +196,78 @@ class SumExpression : public AggregateExpression { return "sum_" + column_name_; } - std::unique_ptr CreateGlobalAggregationFunction(const Schema& child_schema, - Schema& output_schema) override { + std::unique_ptr CreateGlobalAggregationFunction( + const Schema& child_schema, Schema& output_schema) override { const size_t column_ind = child_schema.GetColumnIndexByName(GetName()); const ColumnType column_type = child_schema.GetColumnTypeByName(GetName()); switch (column_type) { case ColumnType::INT16: output_schema.AddColumn(GetOutputName(), ColumnType::INT32); - return std::make_unique>(column_ind); + return std::make_unique>( + column_ind); case ColumnType::INT32: output_schema.AddColumn(GetOutputName(), ColumnType::INT64); - return std::make_unique>(column_ind); + return std::make_unique>( + column_ind); case ColumnType::INT64: output_schema.AddColumn(GetOutputName(), ColumnType::INT128); - return std::make_unique>(column_ind); + return std::make_unique>( + column_ind); case ColumnType::INT128: output_schema.AddColumn(GetOutputName(), ColumnType::INT128); - return std::make_unique>(column_ind); + return std::make_unique>( + column_ind); case ColumnType::FLOAT: output_schema.AddColumn(GetOutputName(), ColumnType::DOUBLE); - return std::make_unique>(column_ind); + return std::make_unique>( + column_ind); case ColumnType::DOUBLE: output_schema.AddColumn(GetOutputName(), ColumnType::LONGDOUBLE); - return std::make_unique>(column_ind); + return std::make_unique>( + column_ind); case ColumnType::LONGDOUBLE: output_schema.AddColumn(GetOutputName(), ColumnType::LONGDOUBLE); - return std::make_unique>(column_ind); + return std::make_unique< + TypedGlobalAggregationFunction>(column_ind); default: THROW_NOT_IMPLEMENTED; } } - std::unique_ptr CreateGroupedAggregationFunction(const Schema& child_schema, - Schema& output_schema) override { + std::unique_ptr CreateGroupedAggregationFunction( + const Schema& child_schema, Schema& output_schema) override { const size_t column_ind = child_schema.GetColumnIndexByName(GetName()); const ColumnType column_type = child_schema.GetColumnTypeByName(GetName()); switch (column_type) { case ColumnType::INT16: output_schema.AddColumn(GetOutputName(), ColumnType::INT32); - return std::make_unique>(column_ind); + return std::make_unique>( + column_ind); case ColumnType::INT32: output_schema.AddColumn(GetOutputName(), ColumnType::INT64); - return std::make_unique>(column_ind); + return std::make_unique>( + column_ind); case ColumnType::INT64: output_schema.AddColumn(GetOutputName(), ColumnType::INT128); - return std::make_unique>(column_ind); + return std::make_unique>( + column_ind); case ColumnType::INT128: output_schema.AddColumn(GetOutputName(), ColumnType::INT128); - return std::make_unique>(column_ind); + return std::make_unique< + TypedGroupedAggregationFunction>(column_ind); case ColumnType::FLOAT: output_schema.AddColumn(GetOutputName(), ColumnType::DOUBLE); - return std::make_unique>(column_ind); + return std::make_unique>( + column_ind); case ColumnType::DOUBLE: output_schema.AddColumn(GetOutputName(), ColumnType::LONGDOUBLE); - return std::make_unique>(column_ind); + return std::make_unique< + TypedGroupedAggregationFunction>(column_ind); case ColumnType::LONGDOUBLE: output_schema.AddColumn(GetOutputName(), ColumnType::LONGDOUBLE); - return std::make_unique>(column_ind); + return std::make_unique< + TypedGroupedAggregationFunction>(column_ind); default: THROW_NOT_IMPLEMENTED; } } @@ -291,38 +285,29 @@ class AvgExpression : public AggregateExpression { return "avg_" + column_name_; } - std::unique_ptr CreateGlobalAggregationFunction(const Schema& child_schema, - Schema& output_schema) override { + std::unique_ptr CreateGlobalAggregationFunction( + const Schema& child_schema, Schema& output_schema) override { const size_t column_ind = child_schema.GetColumnIndexByName(GetName()); const ColumnType column_type = child_schema.GetColumnTypeByName(GetName()); - output_schema.AddColumn(GetOutputName(), ColumnType::DOUBLE); - -#define HANDLE_TYPE(ENUM_VAL, STR_VAL, CLASS_TYPE) \ - case ColumnType::ENUM_VAL: \ - return std::make_unique>(column_ind); - switch (column_type) { - FOR_NUMERIC_COLUMN_TYPE(HANDLE_TYPE); - default: THROW_NOT_IMPLEMENTED; - } -#undef HANDLE_TYPE + return AggExpHelper::DispatchNumeric( + column_type, [&](ColumnType) -> std::unique_ptr { + output_schema.AddColumn(GetOutputName(), ColumnType::DOUBLE); + return std::make_unique>(column_ind); + }); } - std::unique_ptr CreateGroupedAggregationFunction(const Schema& child_schema, - Schema& output_schema) override { + std::unique_ptr CreateGroupedAggregationFunction( + const Schema& child_schema, Schema& output_schema) override { const size_t column_ind = child_schema.GetColumnIndexByName(GetName()); const ColumnType column_type = child_schema.GetColumnTypeByName(GetName()); - output_schema.AddColumn(GetOutputName(), ColumnType::DOUBLE); - -#define HANDLE_TYPE(ENUM_VAL, STR_VAL, CLASS_TYPE) \ - case ColumnType::ENUM_VAL: \ - return std::make_unique>(column_ind); - switch (column_type) { - FOR_NUMERIC_COLUMN_TYPE(HANDLE_TYPE); - default: THROW_NOT_IMPLEMENTED; - } -#undef HANDLE_TYPE + return AggExpHelper::DispatchNumeric( + column_type, + [&](ColumnType) -> std::unique_ptr { + output_schema.AddColumn(GetOutputName(), ColumnType::DOUBLE); + return std::make_unique>(column_ind); + }); } }; diff --git a/src/execution/expressions/AggregationFunctions.h b/src/execution/expressions/AggregationFunctions.h index 691fbef..9f46322 100644 --- a/src/execution/expressions/AggregationFunctions.h +++ b/src/execution/expressions/AggregationFunctions.h @@ -1,6 +1,6 @@ #pragma once -#include "execution/ExecutionHelper.h" +#include "execution/expressions/AggExpHelper.h" #include "execution/OperatorsBase.h" #include "column/Column.h" @@ -12,7 +12,6 @@ #include #include #include -#include class GlobalAggregationFunction { public: @@ -47,13 +46,15 @@ DEFINE_AGG_TRAIT(Int128Column, Int128, Int128, Int128Column); DEFINE_AGG_TRAIT(FloatColumn, float, double, DoubleColumn); DEFINE_AGG_TRAIT(DoubleColumn, double, long double, LongDoubleColumn); DEFINE_AGG_TRAIT(LongDoubleColumn, long double, long double, LongDoubleColumn); -DEFINE_AGG_TRAIT(CharColumn, char, int16_t, Int16Column); +DEFINE_AGG_TRAIT(CharColumn, char, char, CharColumn); DEFINE_AGG_TRAIT(DateColumn, int32_t, int32_t, DateColumn); DEFINE_AGG_TRAIT(TimestampColumn, int64_t, int64_t, TimestampColumn); template <> struct AggregationFunctionTraits { using ValueType = std::string; + using StateType = std::string; + using ResultColumnType = StringColumn; }; #undef DEFINE_AGG_TRAIT @@ -76,10 +77,6 @@ struct MaxOperation { result = value; } } - template - static constexpr ResultType GetInitValue() { - return std::numeric_limits::lowest(); - } }; struct MinOperation { @@ -89,10 +86,6 @@ struct MinOperation { result = value; } } - template - static constexpr ResultType GetInitValue() { - return std::numeric_limits::max(); - } }; template @@ -102,19 +95,36 @@ class TypedGlobalAggregationFunction : public GlobalAggregationFunction { public: explicit TypedGlobalAggregationFunction(size_t column_index) : column_index_(column_index) { - state_ = Operation::template GetInitValue(); + if constexpr (requires { Operation::template GetInitValue(); }) { + state_ = Operation::template GetInitValue(); + } else { + state_ = StateType{}; + } } void Update(const RecordBatch& batch) override { - ExecutionHelper::IterateColumnData(batch, column_index_, [&](const auto& value, size_t, size_t) { - Operation::Apply(state_, value); - }); - has_data_ = true; + AggExpHelper::IterateColumnData(batch, column_index_, + [&](const auto& value, size_t, size_t) { + if (!has_data_) { + state_ = value; + has_data_ = true; + } else { + Operation::Apply(state_, value); + } + }); } std::shared_ptr Finalize() override { auto result_column = std::make_shared(); - result_column->AddValue(has_data_ ? state_ : Operation::template GetInitValue()); + if (has_data_) { + result_column->AddValue(state_); + } else { + if constexpr (requires { Operation::template GetInitValue(); }) { + result_column->AddValue(Operation::template GetInitValue()); + } else { + result_column->AddValue(StateType{}); + } + } return result_column; } @@ -135,15 +145,22 @@ class TypedGroupedAggregationFunction : public GroupedAggregationFunction { void Resize(size_t num_groups) override { if (num_groups > states_.size()) { - states_.resize(num_groups, Operation::template GetInitValue()); + states_.resize(num_groups); + has_data_.resize(num_groups, 0); } } void Update(const RecordBatch& batch, const std::vector& group_ids) override { - ExecutionHelper::IterateColumnData(batch, column_index_, [&](const auto& value, size_t row_idx, size_t) { - uint32_t gid = group_ids[row_idx]; - Operation::Apply(states_[gid], value); - }); + AggExpHelper::IterateColumnData(batch, column_index_, + [&](const auto& value, size_t row_idx, size_t) { + uint32_t gid = group_ids[row_idx]; + if (!has_data_[gid]) { + states_[gid] = value; + has_data_[gid] = 1; + } else { + Operation::Apply(states_[gid], value); + } + }); } std::shared_ptr Finalize() override { @@ -157,6 +174,7 @@ class TypedGroupedAggregationFunction : public GroupedAggregationFunction { private: size_t column_index_; std::vector states_; + std::vector has_data_; }; class CountGlobalAggregationFunction : public GlobalAggregationFunction { @@ -209,10 +227,11 @@ class AvgGlobalAggregationFunction : public GlobalAggregationFunction { } void Update(const RecordBatch& batch) override { - ExecutionHelper::IterateColumnData(batch, column_index_, [&](const auto& value, size_t, size_t) { - sum_ += value; - ++count_; - }); + AggExpHelper::IterateColumnData(batch, column_index_, + [&](const auto& value, size_t, size_t) { + sum_ += value; + ++count_; + }); has_data_ = true; } @@ -248,11 +267,12 @@ class AvgGroupedAggregationFunction : public GroupedAggregationFunction { } void Update(const RecordBatch& batch, const std::vector& group_ids) override { - ExecutionHelper::IterateColumnData(batch, column_index_, [&](const auto& value, size_t row_idx, size_t) { - uint32_t gid = group_ids[row_idx]; - states_[gid].sum += value; - ++states_[gid].count; - }); + AggExpHelper::IterateColumnData(batch, column_index_, + [&](const auto& value, size_t row_idx, size_t) { + uint32_t gid = group_ids[row_idx]; + states_[gid].sum += value; + ++states_[gid].count; + }); } std::shared_ptr Finalize() override { @@ -287,9 +307,9 @@ class DistinctCountGlobalAggregationFunction : public GlobalAggregationFunction } void Update(const RecordBatch& batch) override { - ExecutionHelper::IterateColumnData(batch, column_index_, [&](const auto& value, size_t, size_t) { - distinct_values_.insert(ValueType(value)); - }); + AggExpHelper::IterateColumnData( + batch, column_index_, + [&](const auto& value, size_t, size_t) { distinct_values_.insert(ValueType(value)); }); } std::shared_ptr Finalize() override { @@ -319,10 +339,11 @@ class DistinctCountGroupedAggregationFunction : public GroupedAggregationFunctio } void Update(const RecordBatch& batch, const std::vector& group_ids) override { - ExecutionHelper::IterateColumnData(batch, column_index_, [&](const auto& value, size_t row_idx, size_t) { - uint32_t gid = group_ids[row_idx]; - distinct_values_[gid].insert(ValueType(value)); - }); + AggExpHelper::IterateColumnData( + batch, column_index_, [&](const auto& value, size_t row_idx, size_t) { + uint32_t gid = group_ids[row_idx]; + distinct_values_[gid].insert(ValueType(value)); + }); } std::shared_ptr Finalize() override { diff --git a/src/execution/expressions/FilterExpressions.h b/src/execution/expressions/FilterExpressions.h index 4f4a71d..312e59b 100644 --- a/src/execution/expressions/FilterExpressions.h +++ b/src/execution/expressions/FilterExpressions.h @@ -28,27 +28,22 @@ template class NotEqFilterExpression : public FilterExpression { public: NotEqFilterExpression(std::string column_name, ValueType value) - : FilterExpression(column_name), value_(std::move(value)) { + : FilterExpression(std::move(column_name)), value_(std::move(value)) { } std::unique_ptr CreateFilterFunction(const Schema& child_schema) override { size_t ind = child_schema.GetColumnIndexByName(column_name_); ColumnType column_type = child_schema.GetColumnTypeByName(column_name_); -#define HANDLE_TYPE(ENUM_VAL, STR_VAL, CLASS_TYPE) \ - case ColumnType::ENUM_VAL: \ - if constexpr (std::is_same_v) { \ - return std::make_unique>(ind, value_); \ - } else { \ - THROW_NOT_IMPLEMENTED; \ - } - - switch (column_type) { - FOR_EACH_COLUMN_TYPE(HANDLE_TYPE); - default: THROW_NOT_IMPLEMENTED; - } - -#undef HANDLE_TYPE + return AggExpHelper::DispatchAll( + column_type, [&](ColumnType) -> std::unique_ptr { + if constexpr (std::is_same_v) { + return std::make_unique>(ind, + value_); + } else { + THROW_NOT_IMPLEMENTED; + } + }); } private: @@ -59,31 +54,79 @@ template class EqFilterExpression : public FilterExpression { public: EqFilterExpression(std::string column_name, ValueType value) + : FilterExpression(std::move(column_name)), value_(std::move(value)) { + } + + std::unique_ptr CreateFilterFunction(const Schema& child_schema) override { + size_t ind = child_schema.GetColumnIndexByName(column_name_); + ColumnType column_type = child_schema.GetColumnTypeByName(column_name_); + + return AggExpHelper::DispatchAll( + column_type, [&](ColumnType) -> std::unique_ptr { + if constexpr (std::is_same_v) { + return std::make_unique>(ind, value_); + } else { + THROW_NOT_IMPLEMENTED; + } + }); + } + +private: + ValueType value_; +}; + +class LikeExpression : public FilterExpression { +public: + LikeExpression(std::string column_name, std::string value) : FilterExpression(column_name), value_(std::move(value)) { + value_.erase(value_.begin()); + value_.pop_back(); } std::unique_ptr CreateFilterFunction(const Schema& child_schema) override { size_t ind = child_schema.GetColumnIndexByName(column_name_); ColumnType column_type = child_schema.GetColumnTypeByName(column_name_); -#define HANDLE_TYPE(ENUM_VAL, STR_VAL, CLASS_TYPE) \ - case ColumnType::ENUM_VAL: \ - if constexpr (std::is_same_v) { \ - return std::make_unique>(ind, value_); \ - } else { \ - THROW_NOT_IMPLEMENTED; \ - } + return AggExpHelper::DispatchAll( + column_type, [&](ColumnType) -> std::unique_ptr { + if constexpr (std::is_same_v) { + return std::make_unique>(ind, + value_); + } else { + THROW_NOT_IMPLEMENTED; + } + }); + } + +private: + std::string value_; +}; + +class NotLikeExpression : public FilterExpression { +public: + NotLikeExpression(std::string column_name, std::string value) + : FilterExpression(column_name), value_(std::move(value)) { + value_.erase(value_.begin()); + value_.pop_back(); + } - switch (column_type) { - FOR_EACH_COLUMN_TYPE(HANDLE_TYPE); - default: THROW_NOT_IMPLEMENTED; - } + std::unique_ptr CreateFilterFunction(const Schema& child_schema) override { + size_t ind = child_schema.GetColumnIndexByName(column_name_); + ColumnType column_type = child_schema.GetColumnTypeByName(column_name_); -#undef HANDLE_TYPE + return AggExpHelper::DispatchAll( + column_type, [&](ColumnType) -> std::unique_ptr { + if constexpr (std::is_same_v) { + return std::make_unique>(ind, + value_); + } else { + THROW_NOT_IMPLEMENTED; + } + }); } private: - ValueType value_; + std::string value_; }; template @@ -105,3 +148,21 @@ inline std::shared_ptr Eq(const std::string& column_name, T ta return std::make_shared>(column_name, std::move(target_val)); } } + +template +inline std::shared_ptr Like(const std::string& column_name, T target_val) { + if constexpr (std::is_convertible_v) { + return std::make_shared(column_name, std::string(target_val)); + } else { + THROW_RUNTIME_ERROR("Like pattern must be a string"); + } +} + +template +inline std::shared_ptr NotLike(const std::string& column_name, T target_val) { + if constexpr (std::is_convertible_v) { + return std::make_shared(column_name, std::string(target_val)); + } else { + THROW_RUNTIME_ERROR("Like pattern must be a string"); + } +} diff --git a/src/execution/expressions/FilterFunctions.h b/src/execution/expressions/FilterFunctions.h index ad3c01c..eb5e567 100644 --- a/src/execution/expressions/FilterFunctions.h +++ b/src/execution/expressions/FilterFunctions.h @@ -1,7 +1,7 @@ #pragma once #include "execution/OperatorsBase.h" -#include "execution/ExecutionHelper.h" +#include "execution/expressions/AggExpHelper.h" class FilterFunction { public: @@ -13,18 +13,19 @@ template class NotEqFilterFunction : public FilterFunction { public: NotEqFilterFunction(size_t column_ind, ValueType target_val) - : column_ind_(column_ind), target_val_(target_val) { + : column_ind_(column_ind), target_val_(std::move(target_val)) { } std::vector Evaluate(const RecordBatch& batch) override { std::vector selection_vector; selection_vector.reserve(batch.num_rows); - ExecutionHelper::IterateColumnData(batch, column_ind_, [&](const auto& value, size_t, size_t physical_idx) { - if (value != target_val_) { - selection_vector.push_back(physical_idx); - } - }); + AggExpHelper::IterateColumnData( + batch, column_ind_, [&](const auto& value, size_t, size_t real_ind) { + if (value != target_val_) { + selection_vector.push_back(real_ind); + } + }); return selection_vector; } @@ -37,18 +38,19 @@ template class EqFilterFunction : public FilterFunction { public: EqFilterFunction(size_t column_ind, ValueType target_val) - : column_ind_(column_ind), target_val_(target_val) { + : column_ind_(column_ind), target_val_(std::move(target_val)) { } std::vector Evaluate(const RecordBatch& batch) override { std::vector selection_vector; selection_vector.reserve(batch.num_rows); - ExecutionHelper::IterateColumnData(batch, column_ind_, [&](const auto& value, size_t, size_t real_ind) { - if (value == target_val_) { - selection_vector.push_back(real_ind); - } - }); + AggExpHelper::IterateColumnData( + batch, column_ind_, [&](const auto& value, size_t, size_t real_ind) { + if (value == target_val_) { + selection_vector.push_back(real_ind); + } + }); return selection_vector; } @@ -56,3 +58,53 @@ class EqFilterFunction : public FilterFunction { size_t column_ind_; ValueType target_val_; }; + +template +class LikeFilterFunction : public FilterFunction { +public: + LikeFilterFunction(size_t column_ind, ValueType pattern) + : column_ind_(column_ind), pattern_(std::move(pattern)) { + } + + std::vector Evaluate(const RecordBatch& batch) override { + std::vector selection_vector; + selection_vector.reserve(batch.num_rows); + + AggExpHelper::IterateColumnData( + batch, column_ind_, [&](const auto& value, size_t, size_t real_ind) { + if (value.find(pattern_) != std::string::npos) { + selection_vector.push_back(real_ind); + } + }); + return selection_vector; + } + +private: + size_t column_ind_; + ValueType pattern_; +}; + +template +class NotLikeFilterFunction : public FilterFunction { +public: + NotLikeFilterFunction(size_t column_ind, ValueType pattern) + : column_ind_(column_ind), pattern_(std::move(pattern)) { + } + + std::vector Evaluate(const RecordBatch& batch) override { + std::vector selection_vector; + selection_vector.reserve(batch.num_rows); + + AggExpHelper::IterateColumnData( + batch, column_ind_, [&](const auto& value, size_t, size_t real_ind) { + if (value.find(pattern_) == std::string::npos) { + selection_vector.push_back(real_ind); + } + }); + return selection_vector; + } + +private: + size_t column_ind_; + ValueType pattern_; +}; diff --git a/src/execution/expressions/ScalarFunctions.h b/src/execution/expressions/ScalarFunctions.h index 26d1013..4d76846 100644 --- a/src/execution/expressions/ScalarFunctions.h +++ b/src/execution/expressions/ScalarFunctions.h @@ -1,7 +1,7 @@ #pragma once #include "execution/OperatorsBase.h" -#include "execution/ExecutionHelper.h" +#include "execution/expressions/AggExpHelper.h" #include "column/TemporalColumn.h" #include "column/NumericColumn.h" @@ -21,7 +21,7 @@ class ExtractMinuteFunction : public ScalarFunction { std::vector result_data; result_data.reserve(batch.num_rows); - ExecutionHelper::IterateColumnData(batch, col_ind_, [&](const auto& value, size_t, size_t) { + AggExpHelper::IterateColumnData(batch, col_ind_, [&](const auto& value, size_t, size_t) { result_data.push_back(Timestamp::ExtractMinute(value)); }); diff --git a/src/main.cpp b/src/main.cpp index 71788f5..5b1a3d8 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -210,9 +210,49 @@ class Query { // SELECT COUNT(*) FROM hits WHERE URL LIKE '%google%'; void Query20() { + auto df = DataFrame::Select(columnar_file_path_, {"URL"}) + .Filter(Like("URL", "%google%")) + .Aggregate({}, {Count()}) + .Collect(); + df.Display(); + } + + // SELECT SearchPhrase, MIN(URL), COUNT(*) AS c FROM hits WHERE URL LIKE '%google%' AND + // SearchPhrase <> '' GROUP BY SearchPhrase ORDER BY c DESC LIMIT 10; + void Query21() { + auto df = DataFrame::Select(columnar_file_path_, {"SearchPhrase", "URL"}) + .Filter(NotEq("SearchPhrase", "")) + .Filter(Like("URL", "%google%")) + .Aggregate({"SearchPhrase"}, {Min("URL"), Count("*", "c")}) + .OrderBy({{"c", true}}, 10) + .Collect(); + df.Display(); } - // TODO: 21-23 + // SELECT SearchPhrase, MIN(URL), MIN(Title), COUNT(*) AS c, COUNT(DISTINCT UserID) FROM hits + // WHERE Title LIKE '%Google%' AND URL NOT LIKE '%.google.%' AND SearchPhrase <> '' GROUP BY + // SearchPhrase ORDER BY c DESC LIMIT 10; + void Query22() { + auto df = DataFrame::Select(columnar_file_path_, {"SearchPhrase", "URL", "Title", "UserID"}) + .Filter(NotEq("SearchPhrase", "")) + .Filter(Like("Title", "%Google%")) + .Filter(NotLike("URL", "%.google.%")) + .Aggregate({"SearchPhrase"}, {Min("URL"), Min("Title"), Count("*", "c"), + DistinctCount("UserID")}) + .OrderBy({{"c", true}}, 10) + .Collect(); + df.Display(); + } + + // TODO: support SELECT * + // SELECT * FROM hits WHERE URL LIKE '%google%' ORDER BY EventTime LIMIT 10; + void Query23() { + auto df = DataFrame::Select(columnar_file_path_, {"URL", "EventTime"}) + .Filter(Like("URL", "%google%")) + .OrderBy({{"EventTime", false}}, 10) + .Collect(); + df.Display(); + } // SELECT SearchPhrase FROM hits WHERE SearchPhrase <> '' ORDER BY EventTime LIMIT 10; void Query24() { @@ -264,7 +304,10 @@ class Query { case 17: Query17(); break; case 18: Query18(); break; case 19: Query19(); break; - // TODO: 20-23 + case 20: Query20(); break; + case 21: Query21(); break; + case 22: Query22(); break; + case 23: Query23(); break; case 24: Query24(); break; case 25: Query25(); break; case 26: Query26(); break; From 9c49b01dc28821b948b0100a2c30f284fdda8575 Mon Sep 17 00:00:00 2001 From: Mikhail Fomichev Date: Sat, 9 May 2026 22:59:09 +0300 Subject: [PATCH 03/18] feat: query 27: length project function supported, having function supported. Huge remake of aggregation function is done --- src/CMakeLists.txt | 1 + src/column/CharColumn.h | 27 +- src/column/ColumnBuilder.h | 3 + src/column/NumericColumn.h | 34 ++- src/column/StringColumn.h | 26 +- src/column/TemporalColumn.h | 45 ++- src/execution/ExecutionHelper.h | 9 +- src/execution/ExecutionLogic.h | 14 +- src/execution/OperatorsBase.cpp | 56 ++-- src/execution/OperatorsBase.h | 9 +- src/execution/expressions/AggExpHelper.h | 4 +- .../expressions/AggregationExpressions.h | 104 ++++--- .../expressions/AggregationFunctions.h | 266 ++++++++++-------- src/execution/expressions/FilterExpressions.h | 49 +++- src/execution/expressions/FilterFunctions.h | 43 ++- src/execution/expressions/ScalarExpressions.h | 27 ++ src/execution/expressions/ScalarFunctions.h | 31 +- src/main.cpp | 16 ++ src/types/String.h | 10 + 19 files changed, 504 insertions(+), 270 deletions(-) create mode 100644 src/types/String.h diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 18664df..e4baac1 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -49,6 +49,7 @@ add_executable(columnar-engine types/Date.h types/Timestamp.h execution/expressions/AggExpHelper.h + types/String.h ) target_link_libraries(columnar-engine PRIVATE project_includes) diff --git a/src/column/CharColumn.h b/src/column/CharColumn.h index 2db0ccf..e42bcd3 100644 --- a/src/column/CharColumn.h +++ b/src/column/CharColumn.h @@ -11,6 +11,9 @@ class CharColumn : public Column { using ValueType = char; using ContainerType = std::vector; + CharColumn() = default; + CharColumn(ContainerType&& data) noexcept : data_(std::move(data)) {} + ColumnType GetType() const override { return ColumnType::CHAR; } @@ -30,10 +33,6 @@ class CharColumn : public Column { const ContainerType& GetData() const { return data_; } - - void AddValue(char value) { - data_.push_back(value); - } void ReadFromBuffer(const std::vector& buffer) { data_.clear(); @@ -46,23 +45,33 @@ class CharColumn : public Column { class CharColumnBuilder : public ColumnBuilder { public: - CharColumnBuilder() : column_(std::make_shared()) {} + CharColumnBuilder() = default; + + void AddValue(char value) { + data_.push_back(value); + } void AddBatch(const VectorOfStrings2D& batch, size_t j) override { size_t h = batch.Height(); + data_.reserve(data_.size() + h); for (size_t i = 0; i < h; ++i) { std::string_view val = batch.GetString2D(i, j); ASSERT(val.size() == 1); - column_->AddValue(val[0]); + data_.push_back(val[0]); } } std::shared_ptr Finish() override { - auto result = column_; - column_ = std::make_shared(); + auto result = std::make_shared(std::move(data_)); + data_.clear(); return result; } private: - std::shared_ptr column_; + std::vector data_; +}; + +template <> +struct BuilderTypeTrait { + using Type = CharColumnBuilder; }; diff --git a/src/column/ColumnBuilder.h b/src/column/ColumnBuilder.h index 1c9b989..672c39f 100644 --- a/src/column/ColumnBuilder.h +++ b/src/column/ColumnBuilder.h @@ -11,3 +11,6 @@ class ColumnBuilder { virtual void AddBatch(const VectorOfStrings2D& batch, size_t column_index) = 0; virtual std::shared_ptr Finish() = 0; }; + +template +struct BuilderTypeTrait; diff --git a/src/column/NumericColumn.h b/src/column/NumericColumn.h index f834313..311d13d 100644 --- a/src/column/NumericColumn.h +++ b/src/column/NumericColumn.h @@ -109,15 +109,7 @@ class NumericColumn : public Column { using ContainerType = std::vector; NumericColumn() = default; - NumericColumn(const ContainerType& data) : data_(data) {} - NumericColumn(ContainerType&& data) noexcept : data_(std::move(data)) {} - NumericColumn operator=(const ContainerType& data) { - data_ = data; - return *this; - } - NumericColumn operator=(ContainerType&& data) noexcept { - data_ = std::move(data); - return *this; + NumericColumn(ContainerType&& data) noexcept : data_(std::move(data)) { } ~NumericColumn() override = default; @@ -142,10 +134,6 @@ class NumericColumn : public Column { return data_; } - void AddValue(T value) { - data_.push_back(value); - } - void ReadFromBuffer(const std::vector& buffer) { if (buffer.size() % sizeof(T) != 0) { THROW_RUNTIME_ERROR("Raw buffer size is not aligned with type size"); @@ -162,23 +150,28 @@ class NumericColumn : public Column { template class NumericColumnBuilder : public ColumnBuilder { public: - NumericColumnBuilder() : column_(std::make_shared>()) {} + NumericColumnBuilder() = default; + + void AddValue(T value) { + data_.push_back(value); + } void AddBatch(const VectorOfStrings2D& batch, size_t j) override { size_t h = batch.Height(); + data_.reserve(data_.size() + h); for (size_t i = 0; i < h; ++i) { - column_->AddValue(ColumnTypeTraits::FromString(batch.GetString2D(i, j))); + data_.push_back(ColumnTypeTraits::FromString(batch.GetString2D(i, j))); } } std::shared_ptr Finish() override { - auto result = column_; - column_ = std::make_shared>(); + auto result = std::make_shared>(std::move(data_)); + data_.clear(); return result; } private: - std::shared_ptr> column_; + std::vector data_; }; using Int16Column = NumericColumn; @@ -196,3 +189,8 @@ using Int128ColumnBuilder = NumericColumnBuilder; using FloatColumnBuilder = NumericColumnBuilder; using DoubleColumnBuilder = NumericColumnBuilder; using LongDoubleColumnBuilder = NumericColumnBuilder; + +template +struct BuilderTypeTrait> { + using Type = NumericColumnBuilder; +}; diff --git a/src/column/StringColumn.h b/src/column/StringColumn.h index bf398f1..a037fe6 100644 --- a/src/column/StringColumn.h +++ b/src/column/StringColumn.h @@ -13,6 +13,9 @@ class StringColumn : public Column { using ValueType = std::string; using ContainerType = VectorOfStrings; + StringColumn() = default; + StringColumn(ContainerType&& data) noexcept : data_(std::move(data)) {} + ColumnType GetType() const override { return ColumnType::STRING; } @@ -33,10 +36,6 @@ class StringColumn : public Column { return data_; } - void AddValue(std::string_view value) { - data_.PushBack(value); - } - void ReadFromBuffer(const std::vector& buffer) { Clear(); size_t offset = 0; @@ -56,21 +55,30 @@ class StringColumn : public Column { class StringColumnBuilder : public ColumnBuilder { public: - StringColumnBuilder() : column_(std::make_shared()) {} + StringColumnBuilder() = default; + + void AddValue(std::string_view value) { + data_.PushBack(value); + } void AddBatch(const VectorOfStrings2D& batch, size_t j) override { size_t h = batch.Height(); for (size_t i = 0; i < h; ++i) { - column_->AddValue(batch.GetString2D(i, j)); + data_.PushBack(batch.GetString2D(i, j)); } } std::shared_ptr Finish() override { - auto result = column_; - column_ = std::make_shared(); + auto result = std::make_shared(std::move(data_)); + data_.Clear(); return result; } private: - std::shared_ptr column_; + VectorOfStrings data_; +}; + +template <> +struct BuilderTypeTrait { + using Type = StringColumnBuilder; }; diff --git a/src/column/TemporalColumn.h b/src/column/TemporalColumn.h index 00b6b0b..246597b 100644 --- a/src/column/TemporalColumn.h +++ b/src/column/TemporalColumn.h @@ -15,6 +15,9 @@ class TemporalColumn : public Column { using ValueType = T; using ContainerType = std::vector; + TemporalColumn() = default; + TemporalColumn(ContainerType&& data) noexcept : data_(std::move(data)) {} + ColumnType GetType() const override { return CType; } @@ -35,10 +38,6 @@ class TemporalColumn : public Column { return data_; } - void AddValue(T value) { - data_.push_back(value); - } - void ReadFromBuffer(const std::vector& buffer) { if (buffer.size() % sizeof(T) != 0) { THROW_RUNTIME_ERROR("Raw buffer size is not aligned with type size"); @@ -52,31 +51,55 @@ class TemporalColumn : public Column { ContainerType data_; }; -class DateColumn : public TemporalColumn {}; -class TimestampColumn : public TemporalColumn {}; +class DateColumn : public TemporalColumn { +public: + using TemporalColumn::TemporalColumn; +}; + +class TimestampColumn : public TemporalColumn { +public: + using TemporalColumn::TemporalColumn; +}; template class TemporalColumnBuilder : public ColumnBuilder { + using ValueType = typename ColumnTypeT::ValueType; + public: - TemporalColumnBuilder() : column_(std::make_shared()) {} + TemporalColumnBuilder() = default; + + void AddValue(ValueType value) { + data_.push_back(value); + } void AddBatch(const VectorOfStrings2D& batch, size_t j) override { size_t h = batch.Height(); + data_.reserve(data_.size() + h); for (size_t i = 0; i < h; ++i) { std::string_view cur = batch.GetString2D(i, j); - column_->AddValue(ParseStruct::Parse(cur)); + data_.push_back(ParseStruct::Parse(cur)); } } std::shared_ptr Finish() override { - auto result = column_; - column_ = std::make_shared(); + auto result = std::make_shared(std::move(data_)); + data_.clear(); return result; } private: - std::shared_ptr column_; + std::vector data_; }; using DateColumnBuilder = TemporalColumnBuilder; using TimestampColumnBuilder = TemporalColumnBuilder; + +template <> +struct BuilderTypeTrait { + using Type = DateColumnBuilder; +}; + +template <> +struct BuilderTypeTrait { + using Type = TimestampColumnBuilder; +}; diff --git a/src/execution/ExecutionHelper.h b/src/execution/ExecutionHelper.h index 6a55f04..0ab31a5 100644 --- a/src/execution/ExecutionHelper.h +++ b/src/execution/ExecutionHelper.h @@ -120,15 +120,12 @@ class ExecutionHelper { } } - static void CopySelection(Column& dest, const Column& src, + static void CopySelection(ColumnBuilder& dest, const Column& src, const std::vector* indices = nullptr) { ColumnType type = src.GetType(); - if (dest.GetType() != type) { - THROW_RUNTIME_ERROR("Type mismatch in CopySelection"); - } if (type == ColumnType::STRING) { - auto& d = static_cast(dest); + auto& d = static_cast(dest); const auto& s = static_cast(src).GetData(); if (!indices) { for (size_t i = 0; i < s.Size(); ++i) { @@ -142,7 +139,7 @@ class ExecutionHelper { } else { #define HANDLE_TYPE(ENUM_VAL, STR_VAL, CLASS_TYPE) \ case ColumnType::ENUM_VAL: { \ - auto& d = static_cast(dest); \ + auto& d = static_cast(dest); \ auto* vec = static_cast(src.GetRawData()); \ if (!indices) { \ for (size_t i = 0; i < vec->size(); ++i) { \ diff --git a/src/execution/ExecutionLogic.h b/src/execution/ExecutionLogic.h index b3abfe6..176523c 100644 --- a/src/execution/ExecutionLogic.h +++ b/src/execution/ExecutionLogic.h @@ -142,7 +142,7 @@ inline PhysicalOperatorContext BuildPhysicalPlan( BuildPhysicalPlan(aggregate->child, needed_columns); std::vector group_col_indices; - std::vector> empty_key_columns; + std::vector> key_builders; Schema output_schema; for (const auto& col_name : aggregate->group_by_columns) { @@ -151,15 +151,7 @@ inline PhysicalOperatorContext BuildPhysicalPlan( ColumnType type = child_context.schema.GetColumnTypeByName(col_name); output_schema.AddColumn(col_name, type); - -#define HANDLE_TYPE(ENUM_VAL, STR_VAL, CLASS_TYPE) \ - case ColumnType::ENUM_VAL: empty_key_columns.push_back(std::make_shared()); break; - - switch (type) { - FOR_EACH_COLUMN_TYPE(HANDLE_TYPE); - default: THROW_NOT_IMPLEMENTED; - } -#undef HANDLE_TYPE + key_builders.push_back(ColumnFactory::MakeColumnBuilder(type)); } std::vector> agg_funcs; @@ -171,7 +163,7 @@ inline PhysicalOperatorContext BuildPhysicalPlan( auto op = std::make_unique( std::move(child_context.root_operator), std::move(group_col_indices), - std::move(empty_key_columns), std::move(agg_funcs)); + std::move(key_builders), std::move(agg_funcs)); return {std::move(op), std::move(output_schema)}; } diff --git a/src/execution/OperatorsBase.cpp b/src/execution/OperatorsBase.cpp index f03ea6d..cf478ea 100644 --- a/src/execution/OperatorsBase.cpp +++ b/src/execution/OperatorsBase.cpp @@ -104,11 +104,11 @@ std::unique_ptr FilterOperator::Run() { GroupByOperator::GroupByOperator(std::unique_ptr child, std::vector group_by_col_indices, - std::vector> empty_key_columns, + std::vector> key_builders, std::vector> agg_funcs) : child_(std::move(child)), group_by_col_indices_(std::move(group_by_col_indices)), - key_columns_(std::move(empty_key_columns)), + key_builders_(std::move(key_builders)), agg_funcs_(std::move(agg_funcs)) { } @@ -126,7 +126,7 @@ std::unique_ptr GroupByOperator::Run() { } std::vector new_group_indices; - size_t sz = key_columns_.empty() ? 0 : key_columns_[0]->Size(); + size_t sz = num_groups_; bool is_filtered = sel_vec != nullptr; for (size_t i = 0; i < batch_size; ++i) { @@ -147,28 +147,26 @@ std::unique_ptr GroupByOperator::Run() { if (!new_group_indices.empty()) { for (size_t k = 0; k < group_by_col_indices_.size(); ++k) { - ExecutionHelper::CopySelection(*key_columns_[k], *batch->columns[group_by_col_indices_[k]], &new_group_indices); + ExecutionHelper::CopySelection(*key_builders_[k], *batch->columns[group_by_col_indices_[k]], &new_group_indices); } } - size_t total_groups = hash_table_.size(); + num_groups_ = hash_table_.size(); for (auto& func : agg_funcs_) { - func->Resize(total_groups); + func->Resize(num_groups_); func->Update(*batch, group_ids); } } accumulated_ = true; - size_t total_groups = key_columns_.empty() ? 0 : key_columns_[0]->Size(); auto result_batch = std::make_unique(); - result_batch->num_rows = total_groups; + result_batch->num_rows = num_groups_; - for (size_t k = 0; k < key_columns_.size(); ++k) { - result_batch->columns.push_back(std::move(key_columns_[k])); + for (size_t k = 0; k < key_builders_.size(); ++k) { + result_batch->columns.push_back(key_builders_[k]->Finish()); } for (size_t a = 0; a < agg_funcs_.size(); ++a) { - auto full_agg_col = agg_funcs_[a]->Finalize(); - result_batch->columns.push_back(std::move(full_agg_col)); + result_batch->columns.push_back(agg_funcs_[a]->Finalize()); } return result_batch; } @@ -190,9 +188,9 @@ struct SortColumnInfo { bool is_desc; }; -template +template int Compare(const void* raw_data, uint32_t a, uint32_t b) { - using Container = ColumnType::ContainerType; + using Container = ColumnT::ContainerType; const Container* data = static_cast(raw_data); if ((*data)[a] < (*data)[b]) { return -1; @@ -206,24 +204,32 @@ int Compare(const void* raw_data, uint32_t a, uint32_t b) { std::unique_ptr OrderByOperator::Run() { if (!accumulated_) { while (auto batch = child_->Run()) { - if (!accumulated_batch_) { - accumulated_batch_ = std::make_unique(); + if (accum_builders_.empty()) { size_t sz = batch->columns.size(); for (size_t c = 0; c < sz; ++c) { - accumulated_batch_->columns.push_back(ColumnFactory::MakeColumn(batch->columns[c]->GetType())); + accum_builders_.push_back(ColumnFactory::MakeColumnBuilder(batch->columns[c]->GetType())); + accum_types_.push_back(batch->columns[c]->GetType()); } - accumulated_batch_->num_rows = 0; + accum_num_rows_ = 0; } const std::shared_ptr>& sel = batch->selection_vector; - size_t sz = batch->columns.size(); + size_t sz = accum_builders_.size(); for (size_t c = 0; c < sz; ++c) { - ExecutionHelper::CopySelection(*accumulated_batch_->columns[c], *batch->columns[c], sel.get()); + ExecutionHelper::CopySelection(*accum_builders_[c], *batch->columns[c], sel.get()); } - accumulated_batch_->num_rows += batch->num_rows; + accum_num_rows_ += batch->num_rows; } - if (accumulated_batch_) { + if (!accum_builders_.empty()) { + // Finish builders into immutable columns for sorting + accumulated_batch_ = std::make_unique(); + accumulated_batch_->num_rows = accum_num_rows_; + for (auto& builder : accum_builders_) { + accumulated_batch_->columns.push_back(builder->Finish()); + } + accum_builders_.clear(); + size_t sz = accumulated_batch_->num_rows; indices_.reserve(sz); for (uint32_t i = 0; i < sz; ++i) { @@ -272,9 +278,9 @@ std::unique_ptr OrderByOperator::Run() { auto sorted_batch = std::make_unique(); sorted_batch->num_rows = accumulated_batch_->num_rows; for (size_t c = 0; c < accumulated_batch_->columns.size(); ++c) { - auto new_col = ColumnFactory::MakeColumn(accumulated_batch_->columns[c]->GetType()); - ExecutionHelper::CopySelection(*new_col, *accumulated_batch_->columns[c], &indices_); - sorted_batch->columns.push_back(std::move(new_col)); + auto builder = ColumnFactory::MakeColumnBuilder(accumulated_batch_->columns[c]->GetType()); + ExecutionHelper::CopySelection(*builder, *accumulated_batch_->columns[c], &indices_); + sorted_batch->columns.push_back(builder->Finish()); } accumulated_batch_ = std::move(sorted_batch); diff --git a/src/execution/OperatorsBase.h b/src/execution/OperatorsBase.h index d14ddc6..a9f68fe 100644 --- a/src/execution/OperatorsBase.h +++ b/src/execution/OperatorsBase.h @@ -1,6 +1,7 @@ #pragma once #include "io/ColumnarReader.h" +#include "column/ColumnBuilder.h" #include #include @@ -66,7 +67,7 @@ class FilterOperator : public Operator { class GroupByOperator : public Operator { public: GroupByOperator(std::unique_ptr child, std::vector group_by_col_indices, - std::vector> empty_key_columns, + std::vector> key_builders, std::vector> agg_funcs); std::unique_ptr Run() override; @@ -74,9 +75,10 @@ class GroupByOperator : public Operator { private: std::unique_ptr child_; std::vector group_by_col_indices_; - std::vector> key_columns_; + std::vector> key_builders_; std::vector> agg_funcs_; bool accumulated_ = false; + size_t num_groups_ = 0; std::unordered_map hash_table_; }; @@ -93,6 +95,9 @@ class OrderByOperator : public Operator { std::unique_ptr child_; std::vector> sort_columns_; std::unique_ptr accumulated_batch_; + std::vector> accum_builders_; + std::vector accum_types_; + size_t accum_num_rows_ = 0; std::vector indices_; size_t current_idx_ = 0; std::optional limit_; diff --git a/src/execution/expressions/AggExpHelper.h b/src/execution/expressions/AggExpHelper.h index 66a85b0..fa0cb82 100644 --- a/src/execution/expressions/AggExpHelper.h +++ b/src/execution/expressions/AggExpHelper.h @@ -5,9 +5,9 @@ class AggExpHelper { public: - template + template static void IterateColumnData(const RecordBatch& batch, size_t column_index, Func&& func) { - auto* column = static_cast(batch.columns[column_index].get()); + auto* column = static_cast(batch.columns[column_index].get()); const auto& data = column->GetData(); if (!batch.selection_vector) { diff --git a/src/execution/expressions/AggregationExpressions.h b/src/execution/expressions/AggregationExpressions.h index 83a66d8..7074fe7 100644 --- a/src/execution/expressions/AggregationExpressions.h +++ b/src/execution/expressions/AggregationExpressions.h @@ -49,14 +49,14 @@ class CountExpression : public AggregateExpression { std::unique_ptr CreateGlobalAggregationFunction( [[maybe_unused]] const Schema& child_schema, Schema& output_schema) override { - output_schema.AddColumn(GetOutputName(), ColumnType::INT32); - return std::make_unique(); + output_schema.AddColumn(GetOutputName(), ColumnType::INT64); + return std::make_unique>(ColumnType::INT64); } std::unique_ptr CreateGroupedAggregationFunction( [[maybe_unused]] const Schema& child_schema, Schema& output_schema) override { - output_schema.AddColumn(GetOutputName(), ColumnType::INT32); - return std::make_unique(); + output_schema.AddColumn(GetOutputName(), ColumnType::INT64); + return std::make_unique>(ColumnType::INT64); } }; @@ -81,7 +81,8 @@ class DistinctCountExpression : public AggregateExpression { return AggExpHelper::DispatchAll( column_type, [&](ColumnType) -> std::unique_ptr { output_schema.AddColumn(GetOutputName(), ColumnType::INT64); - return std::make_unique>(column_ind); + return std::make_unique>( + column_ind, ColumnType::INT64); }); } @@ -94,7 +95,8 @@ class DistinctCountExpression : public AggregateExpression { column_type, [&](ColumnType) -> std::unique_ptr { output_schema.AddColumn(GetOutputName(), ColumnType::INT64); - return std::make_unique>(column_ind); + return std::make_unique>( + column_ind, ColumnType::INT64); }); } }; @@ -121,8 +123,8 @@ class MinExpression : public AggregateExpression { column_type, [&](ColumnType t) -> std::unique_ptr { output_schema.AddColumn(GetOutputName(), t); - return std::make_unique>( - column_ind); + return std::make_unique>( + column_ind, t); }); } @@ -135,8 +137,8 @@ class MinExpression : public AggregateExpression { column_type, [&](ColumnType t) -> std::unique_ptr { output_schema.AddColumn(GetOutputName(), t); - return std::make_unique>( - column_ind); + return std::make_unique>( + column_ind, t); }); } }; @@ -163,8 +165,8 @@ class MaxExpression : public AggregateExpression { column_type, [&](ColumnType t) -> std::unique_ptr { output_schema.AddColumn(GetOutputName(), t); - return std::make_unique>( - column_ind); + return std::make_unique>( + column_ind, t); }); } @@ -177,8 +179,8 @@ class MaxExpression : public AggregateExpression { column_type, [&](ColumnType t) -> std::unique_ptr { output_schema.AddColumn(GetOutputName(), t); - return std::make_unique>( - column_ind); + return std::make_unique>( + column_ind, t); }); } }; @@ -204,32 +206,40 @@ class SumExpression : public AggregateExpression { switch (column_type) { case ColumnType::INT16: output_schema.AddColumn(GetOutputName(), ColumnType::INT32); - return std::make_unique>( - column_ind); + return std::make_unique< + TypedGlobalAggregationFunction>( + column_ind, ColumnType::INT32); case ColumnType::INT32: output_schema.AddColumn(GetOutputName(), ColumnType::INT64); - return std::make_unique>( - column_ind); + return std::make_unique< + TypedGlobalAggregationFunction>( + column_ind, ColumnType::INT64); case ColumnType::INT64: output_schema.AddColumn(GetOutputName(), ColumnType::INT128); - return std::make_unique>( - column_ind); + return std::make_unique< + TypedGlobalAggregationFunction>( + column_ind, ColumnType::INT128); case ColumnType::INT128: output_schema.AddColumn(GetOutputName(), ColumnType::INT128); - return std::make_unique>( - column_ind); + return std::make_unique< + TypedGlobalAggregationFunction>( + column_ind, ColumnType::INT128); case ColumnType::FLOAT: output_schema.AddColumn(GetOutputName(), ColumnType::DOUBLE); - return std::make_unique>( - column_ind); + return std::make_unique< + TypedGlobalAggregationFunction>( + column_ind, ColumnType::DOUBLE); case ColumnType::DOUBLE: output_schema.AddColumn(GetOutputName(), ColumnType::LONGDOUBLE); - return std::make_unique>( - column_ind); + return std::make_unique< + TypedGlobalAggregationFunction>( + column_ind, ColumnType::LONGDOUBLE); case ColumnType::LONGDOUBLE: output_schema.AddColumn(GetOutputName(), ColumnType::LONGDOUBLE); - return std::make_unique< - TypedGlobalAggregationFunction>(column_ind); + return std::make_unique>( + column_ind, ColumnType::LONGDOUBLE); default: THROW_NOT_IMPLEMENTED; } } @@ -242,32 +252,40 @@ class SumExpression : public AggregateExpression { switch (column_type) { case ColumnType::INT16: output_schema.AddColumn(GetOutputName(), ColumnType::INT32); - return std::make_unique>( - column_ind); + return std::make_unique< + TypedGroupedAggregationFunction>( + column_ind, ColumnType::INT32); case ColumnType::INT32: output_schema.AddColumn(GetOutputName(), ColumnType::INT64); - return std::make_unique>( - column_ind); + return std::make_unique< + TypedGroupedAggregationFunction>( + column_ind, ColumnType::INT64); case ColumnType::INT64: output_schema.AddColumn(GetOutputName(), ColumnType::INT128); - return std::make_unique>( - column_ind); + return std::make_unique< + TypedGroupedAggregationFunction>( + column_ind, ColumnType::INT128); case ColumnType::INT128: output_schema.AddColumn(GetOutputName(), ColumnType::INT128); return std::make_unique< - TypedGroupedAggregationFunction>(column_ind); + TypedGroupedAggregationFunction>( + column_ind, ColumnType::INT128); case ColumnType::FLOAT: output_schema.AddColumn(GetOutputName(), ColumnType::DOUBLE); - return std::make_unique>( - column_ind); + return std::make_unique< + TypedGroupedAggregationFunction>( + column_ind, ColumnType::DOUBLE); case ColumnType::DOUBLE: output_schema.AddColumn(GetOutputName(), ColumnType::LONGDOUBLE); return std::make_unique< - TypedGroupedAggregationFunction>(column_ind); + TypedGroupedAggregationFunction>( + column_ind, ColumnType::LONGDOUBLE); case ColumnType::LONGDOUBLE: output_schema.AddColumn(GetOutputName(), ColumnType::LONGDOUBLE); - return std::make_unique< - TypedGroupedAggregationFunction>(column_ind); + return std::make_unique>( + column_ind, ColumnType::LONGDOUBLE); default: THROW_NOT_IMPLEMENTED; } } @@ -293,7 +311,8 @@ class AvgExpression : public AggregateExpression { return AggExpHelper::DispatchNumeric( column_type, [&](ColumnType) -> std::unique_ptr { output_schema.AddColumn(GetOutputName(), ColumnType::DOUBLE); - return std::make_unique>(column_ind); + return std::make_unique>( + column_ind, ColumnType::DOUBLE); }); } @@ -306,7 +325,8 @@ class AvgExpression : public AggregateExpression { column_type, [&](ColumnType) -> std::unique_ptr { output_schema.AddColumn(GetOutputName(), ColumnType::DOUBLE); - return std::make_unique>(column_ind); + return std::make_unique>( + column_ind, ColumnType::DOUBLE); }); } }; diff --git a/src/execution/expressions/AggregationFunctions.h b/src/execution/expressions/AggregationFunctions.h index 9f46322..3e3e8c9 100644 --- a/src/execution/expressions/AggregationFunctions.h +++ b/src/execution/expressions/AggregationFunctions.h @@ -9,65 +9,51 @@ #include "column/StringColumn.h" #include "column/TemporalColumn.h" +#include "column/ColumnFactory.h" + #include +#include #include #include -class GlobalAggregationFunction { +class AggregationFunction { +public: + virtual ~AggregationFunction() = default; + AggregationFunction(ColumnType column_type) : column_type_(column_type) { + } + + std::shared_ptr Finalize() { + auto builder = ColumnFactory::MakeColumnBuilder(column_type_); + WriteResult(builder); + return builder->Finish(); + } + +protected: + ColumnType column_type_; + + virtual void WriteResult(const std::shared_ptr& builder) = 0; +}; + +class GlobalAggregationFunction : public AggregationFunction { public: - virtual ~GlobalAggregationFunction() = default; + GlobalAggregationFunction(ColumnType column_type) : AggregationFunction(column_type) {} + virtual void Update(const RecordBatch& batch) = 0; - virtual std::shared_ptr Finalize() = 0; }; -class GroupedAggregationFunction { +class GroupedAggregationFunction : public AggregationFunction { public: - virtual ~GroupedAggregationFunction() = default; + GroupedAggregationFunction(ColumnType column_type) : AggregationFunction(column_type) {} + virtual void Resize(size_t num_groups) = 0; virtual void Update(const RecordBatch& batch, const std::vector& group_ids) = 0; - virtual std::shared_ptr Finalize() = 0; -}; - -template -struct AggregationFunctionTraits; - -#define DEFINE_AGG_TRAIT(COL_TYPE, VAL_TYPE, STATE_TYPE, RES_TYPE) \ - template <> \ - struct AggregationFunctionTraits { \ - using ValueType = VAL_TYPE; \ - using StateType = STATE_TYPE; \ - using ResultColumnType = RES_TYPE; \ - } - -DEFINE_AGG_TRAIT(Int16Column, int16_t, int32_t, Int32Column); -DEFINE_AGG_TRAIT(Int32Column, int32_t, int64_t, Int64Column); -DEFINE_AGG_TRAIT(Int64Column, int64_t, Int128, Int128Column); -DEFINE_AGG_TRAIT(Int128Column, Int128, Int128, Int128Column); -DEFINE_AGG_TRAIT(FloatColumn, float, double, DoubleColumn); -DEFINE_AGG_TRAIT(DoubleColumn, double, long double, LongDoubleColumn); -DEFINE_AGG_TRAIT(LongDoubleColumn, long double, long double, LongDoubleColumn); -DEFINE_AGG_TRAIT(CharColumn, char, char, CharColumn); -DEFINE_AGG_TRAIT(DateColumn, int32_t, int32_t, DateColumn); -DEFINE_AGG_TRAIT(TimestampColumn, int64_t, int64_t, TimestampColumn); - -template <> -struct AggregationFunctionTraits { - using ValueType = std::string; - using StateType = std::string; - using ResultColumnType = StringColumn; }; -#undef DEFINE_AGG_TRAIT - struct SumOperation { template static void Apply(ResultType& result, const ValueType& value) { result += value; } - template - static constexpr ResultType GetInitValue() { - return 0; - } }; struct MaxOperation { @@ -88,22 +74,18 @@ struct MinOperation { } }; -template +template class TypedGlobalAggregationFunction : public GlobalAggregationFunction { - using ResultColumnType = AggregationFunctionTraits::ResultColumnType; - using StateType = AggregationFunctionTraits::StateType; + using StateType = OutputColumn::ValueType; + using OutputBuilder = BuilderTypeTrait::Type; public: - explicit TypedGlobalAggregationFunction(size_t column_index) : column_index_(column_index) { - if constexpr (requires { Operation::template GetInitValue(); }) { - state_ = Operation::template GetInitValue(); - } else { - state_ = StateType{}; - } + TypedGlobalAggregationFunction(size_t column_index, ColumnType column_type) + : GlobalAggregationFunction(column_type), column_index_(column_index) { } void Update(const RecordBatch& batch) override { - AggExpHelper::IterateColumnData(batch, column_index_, + AggExpHelper::IterateColumnData(batch, column_index_, [&](const auto& value, size_t, size_t) { if (!has_data_) { state_ = value; @@ -114,33 +96,34 @@ class TypedGlobalAggregationFunction : public GlobalAggregationFunction { }); } - std::shared_ptr Finalize() override { - auto result_column = std::make_shared(); +protected: + void WriteResult(const std::shared_ptr& builder) override { + auto* typed = static_cast(builder.get()); if (has_data_) { - result_column->AddValue(state_); + typed->AddValue(state_); } else { if constexpr (requires { Operation::template GetInitValue(); }) { - result_column->AddValue(Operation::template GetInitValue()); + typed->AddValue(Operation::template GetInitValue()); } else { - result_column->AddValue(StateType{}); + typed->AddValue(StateType{}); } } - return result_column; } private: size_t column_index_; - StateType state_; + StateType state_{}; bool has_data_ = false; }; -template +template class TypedGroupedAggregationFunction : public GroupedAggregationFunction { - using ResultColumnType = AggregationFunctionTraits::ResultColumnType; - using StateType = AggregationFunctionTraits::StateType; + using StateType = OutputColumn::ValueType; + using OutputBuilder = BuilderTypeTrait::Type; public: - explicit TypedGroupedAggregationFunction(size_t column_index) : column_index_(column_index) { + TypedGroupedAggregationFunction(size_t column_index, ColumnType column_type) + : GroupedAggregationFunction(column_type), column_index_(column_index) { } void Resize(size_t num_groups) override { @@ -151,7 +134,7 @@ class TypedGroupedAggregationFunction : public GroupedAggregationFunction { } void Update(const RecordBatch& batch, const std::vector& group_ids) override { - AggExpHelper::IterateColumnData(batch, column_index_, + AggExpHelper::IterateColumnData(batch, column_index_, [&](const auto& value, size_t row_idx, size_t) { uint32_t gid = group_ids[row_idx]; if (!has_data_[gid]) { @@ -163,12 +146,12 @@ class TypedGroupedAggregationFunction : public GroupedAggregationFunction { }); } - std::shared_ptr Finalize() override { - auto result_column = std::make_shared(); +protected: + void WriteResult(const std::shared_ptr& builder) override { + auto* typed = static_cast(builder.get()); for (const auto& state : states_) { - result_column->AddValue(state); + typed->AddValue(state); } - return result_column; } private: @@ -177,23 +160,40 @@ class TypedGroupedAggregationFunction : public GroupedAggregationFunction { std::vector has_data_; }; +template class CountGlobalAggregationFunction : public GlobalAggregationFunction { + using ValueType = OutputColumn::ValueType; + using OutputBuilder = BuilderTypeTrait::Type; + public: + CountGlobalAggregationFunction(ColumnType column_type) + : GlobalAggregationFunction(column_type) { + } + void Update(const RecordBatch& batch) override { count_ += batch.num_rows; } - std::shared_ptr Finalize() override { - auto result_column = std::make_shared(); - result_column->AddValue(count_); - return result_column; + +protected: + void WriteResult(const std::shared_ptr& builder) override { + auto* typed = static_cast(builder.get()); + typed->AddValue(static_cast(count_)); } private: int64_t count_ = 0; }; +template class CountGroupedAggregationFunction : public GroupedAggregationFunction { + using ValueType = OutputColumn::ValueType; + using OutputBuilder = BuilderTypeTrait::Type; + public: + CountGroupedAggregationFunction(ColumnType column_type) + : GroupedAggregationFunction(column_type) { + } + void Resize(size_t num_groups) override { if (num_groups > counts_.size()) { counts_.resize(num_groups, 0); @@ -206,28 +206,32 @@ class CountGroupedAggregationFunction : public GroupedAggregationFunction { ++counts_[gid]; } } - std::shared_ptr Finalize() override { - auto result_column = std::make_shared(); + +protected: + void WriteResult(const std::shared_ptr& builder) override { + auto* typed = static_cast(builder.get()); for (int64_t c : counts_) { - result_column->AddValue(c); + typed->AddValue(static_cast(c)); } - return result_column; } private: std::vector counts_; }; -template +template class AvgGlobalAggregationFunction : public GlobalAggregationFunction { - using StateType = AggregationFunctionTraits::StateType; + using StateType = InputColumn::ValueType; + using OutputType = OutputColumn::ValueType; + using OutputBuilder = BuilderTypeTrait::Type; public: - explicit AvgGlobalAggregationFunction(size_t column_index) : column_index_(column_index) { + AvgGlobalAggregationFunction(size_t column_index, ColumnType column_type) + : GlobalAggregationFunction(column_type), column_index_(column_index) { } void Update(const RecordBatch& batch) override { - AggExpHelper::IterateColumnData(batch, column_index_, + AggExpHelper::IterateColumnData(batch, column_index_, [&](const auto& value, size_t, size_t) { sum_ += value; ++count_; @@ -235,29 +239,43 @@ class AvgGlobalAggregationFunction : public GlobalAggregationFunction { has_data_ = true; } - std::shared_ptr Finalize() override { - auto result_column = std::make_shared(); +protected: + void WriteResult(const std::shared_ptr& builder) override { + auto* typed = static_cast(builder.get()); if (has_data_ && count_ > 0) { - result_column->AddValue(static_cast(sum_) / static_cast(count_)); + typed->AddValue(ComputeAvg()); } else { - result_column->AddValue(0.0); + typed->AddValue(OutputType{}); } - return result_column; } private: + OutputType ComputeAvg() const { + if constexpr (std::is_integral_v) { + OutputType int_part = static_cast(sum_ / count_); + OutputType rem = + static_cast(static_cast(sum_ % count_) / count_); + return int_part + rem; + } else { + return static_cast(sum_) / static_cast(count_); + } + } + size_t column_index_; StateType sum_ = 0; int64_t count_ = 0; bool has_data_ = false; }; -template +template class AvgGroupedAggregationFunction : public GroupedAggregationFunction { - using StateType = AggregationFunctionTraits::StateType; + using StateType = InputColumn::ValueType; + using OutputType = OutputColumn::ValueType; + using OutputBuilder = BuilderTypeTrait::Type; public: - explicit AvgGroupedAggregationFunction(size_t column_index) : column_index_(column_index) { + AvgGroupedAggregationFunction(size_t column_index, ColumnType column_type) + : GroupedAggregationFunction(column_type), column_index_(column_index) { } void Resize(size_t num_groups) override { @@ -267,7 +285,7 @@ class AvgGroupedAggregationFunction : public GroupedAggregationFunction { } void Update(const RecordBatch& batch, const std::vector& group_ids) override { - AggExpHelper::IterateColumnData(batch, column_index_, + AggExpHelper::IterateColumnData(batch, column_index_, [&](const auto& value, size_t row_idx, size_t) { uint32_t gid = group_ids[row_idx]; states_[gid].sum += value; @@ -275,20 +293,30 @@ class AvgGroupedAggregationFunction : public GroupedAggregationFunction { }); } - std::shared_ptr Finalize() override { - auto result_column = std::make_shared(); +protected: + void WriteResult(const std::shared_ptr& builder) override { + auto* typed = static_cast(builder.get()); for (const auto& state : states_) { if (state.count > 0) { - result_column->AddValue(static_cast(state.sum) / - static_cast(state.count)); + typed->AddValue(ComputeAvg(state.sum, state.count)); } else { - result_column->AddValue(0.0); + typed->AddValue(OutputType{}); } } - return result_column; } private: + static OutputType ComputeAvg(StateType sum, int64_t count) { + if constexpr (std::is_integral_v) { + OutputType int_part = static_cast(sum / count); + OutputType rem = + static_cast(static_cast(sum % count) / count); + return int_part + rem; + } else { + return static_cast(sum) / static_cast(count); + } + } + struct State { StateType sum = 0; int64_t count = 0; @@ -297,39 +325,45 @@ class AvgGroupedAggregationFunction : public GroupedAggregationFunction { std::vector states_; }; -template +template class DistinctCountGlobalAggregationFunction : public GlobalAggregationFunction { - using ValueType = AggregationFunctionTraits::ValueType; + using InputValueType = InputColumn::ValueType; + using OutputValueType = OutputColumn::ValueType; + using OutputBuilder = BuilderTypeTrait::Type; public: - explicit DistinctCountGlobalAggregationFunction(size_t column_index) - : column_index_(column_index) { + DistinctCountGlobalAggregationFunction(size_t column_index, ColumnType column_type) + : GlobalAggregationFunction(column_type), column_index_(column_index) { } void Update(const RecordBatch& batch) override { - AggExpHelper::IterateColumnData( + AggExpHelper::IterateColumnData( batch, column_index_, - [&](const auto& value, size_t, size_t) { distinct_values_.insert(ValueType(value)); }); + [&](const auto& value, size_t, size_t) { + distinct_values_.insert(InputValueType(value)); + }); } - std::shared_ptr Finalize() override { - auto result_column = std::make_shared(); - result_column->AddValue(static_cast(distinct_values_.size())); - return result_column; +protected: + void WriteResult(const std::shared_ptr& builder) override { + auto* typed = static_cast(builder.get()); + typed->AddValue(static_cast(distinct_values_.size())); } private: size_t column_index_; - std::unordered_set distinct_values_; + std::unordered_set distinct_values_; }; -template +template class DistinctCountGroupedAggregationFunction : public GroupedAggregationFunction { - using ValueType = AggregationFunctionTraits::ValueType; + using InputValueType = InputColumn::ValueType; + using OutputValueType = OutputColumn::ValueType; + using OutputBuilder = BuilderTypeTrait::Type; public: - explicit DistinctCountGroupedAggregationFunction(size_t column_index) - : column_index_(column_index) { + DistinctCountGroupedAggregationFunction(size_t column_index, ColumnType column_type) + : GroupedAggregationFunction(column_type), column_index_(column_index) { } void Resize(size_t num_groups) override { @@ -339,22 +373,22 @@ class DistinctCountGroupedAggregationFunction : public GroupedAggregationFunctio } void Update(const RecordBatch& batch, const std::vector& group_ids) override { - AggExpHelper::IterateColumnData( + AggExpHelper::IterateColumnData( batch, column_index_, [&](const auto& value, size_t row_idx, size_t) { uint32_t gid = group_ids[row_idx]; - distinct_values_[gid].insert(ValueType(value)); + distinct_values_[gid].insert(InputValueType(value)); }); } - std::shared_ptr Finalize() override { - auto result_column = std::make_shared(); +protected: + void WriteResult(const std::shared_ptr& builder) override { + auto* typed = static_cast(builder.get()); for (const auto& set : distinct_values_) { - result_column->AddValue(static_cast(set.size())); + typed->AddValue(static_cast(set.size())); } - return result_column; } private: size_t column_index_; - std::vector> distinct_values_; + std::vector> distinct_values_; }; diff --git a/src/execution/expressions/FilterExpressions.h b/src/execution/expressions/FilterExpressions.h index 312e59b..35ebc25 100644 --- a/src/execution/expressions/FilterExpressions.h +++ b/src/execution/expressions/FilterExpressions.h @@ -75,6 +75,31 @@ class EqFilterExpression : public FilterExpression { ValueType value_; }; +template +class GreaterFilterExpression : public FilterExpression { +public: + GreaterFilterExpression(std::string column_name, ValueType value) + : FilterExpression(column_name), value_(std::move(value)) { + } + + std::unique_ptr CreateFilterFunction(const Schema& child_schema) override { + size_t ind = child_schema.GetColumnIndexByName(column_name_); + ColumnType column_type = child_schema.GetColumnTypeByName(column_name_); + + return AggExpHelper::DispatchAll( + column_type, [&](ColumnType) -> std::unique_ptr { + if constexpr (std::is_same_v) { + return std::make_unique>(ind, value_); + } else { + THROW_NOT_IMPLEMENTED; + } + }); + } + +private: + ValueType value_; +}; + class LikeExpression : public FilterExpression { public: LikeExpression(std::string column_name, std::string value) @@ -88,10 +113,12 @@ class LikeExpression : public FilterExpression { ColumnType column_type = child_schema.GetColumnTypeByName(column_name_); return AggExpHelper::DispatchAll( - column_type, [&](ColumnType) -> std::unique_ptr { + column_type, + [&]( + ColumnType) -> std::unique_ptr { if constexpr (std::is_same_v) { return std::make_unique>(ind, - value_); + value_); } else { THROW_NOT_IMPLEMENTED; } @@ -115,10 +142,12 @@ class NotLikeExpression : public FilterExpression { ColumnType column_type = child_schema.GetColumnTypeByName(column_name_); return AggExpHelper::DispatchAll( - column_type, [&](ColumnType) -> std::unique_ptr { + column_type, + [&]( + ColumnType) -> std::unique_ptr { if constexpr (std::is_same_v) { - return std::make_unique>(ind, - value_); + return std::make_unique>( + ind, value_); } else { THROW_NOT_IMPLEMENTED; } @@ -149,6 +178,16 @@ inline std::shared_ptr Eq(const std::string& column_name, T ta } } +template +inline std::shared_ptr Greater(const std::string& column_name, T target_val) { + if constexpr (std::is_convertible_v) { + return std::make_shared>(column_name, + std::string(target_val)); + } else { + return std::make_shared>(column_name, std::move(target_val)); + } +} + template inline std::shared_ptr Like(const std::string& column_name, T target_val) { if constexpr (std::is_convertible_v) { diff --git a/src/execution/expressions/FilterFunctions.h b/src/execution/expressions/FilterFunctions.h index eb5e567..284a022 100644 --- a/src/execution/expressions/FilterFunctions.h +++ b/src/execution/expressions/FilterFunctions.h @@ -9,7 +9,7 @@ class FilterFunction { virtual std::vector Evaluate(const RecordBatch& batch) = 0; }; -template +template class NotEqFilterFunction : public FilterFunction { public: NotEqFilterFunction(size_t column_ind, ValueType target_val) @@ -20,7 +20,7 @@ class NotEqFilterFunction : public FilterFunction { std::vector selection_vector; selection_vector.reserve(batch.num_rows); - AggExpHelper::IterateColumnData( + AggExpHelper::IterateColumnData( batch, column_ind_, [&](const auto& value, size_t, size_t real_ind) { if (value != target_val_) { selection_vector.push_back(real_ind); @@ -34,7 +34,7 @@ class NotEqFilterFunction : public FilterFunction { ValueType target_val_; }; -template +template class EqFilterFunction : public FilterFunction { public: EqFilterFunction(size_t column_ind, ValueType target_val) @@ -45,7 +45,7 @@ class EqFilterFunction : public FilterFunction { std::vector selection_vector; selection_vector.reserve(batch.num_rows); - AggExpHelper::IterateColumnData( + AggExpHelper::IterateColumnData( batch, column_ind_, [&](const auto& value, size_t, size_t real_ind) { if (value == target_val_) { selection_vector.push_back(real_ind); @@ -59,7 +59,32 @@ class EqFilterFunction : public FilterFunction { ValueType target_val_; }; -template +template +class GreaterFilterFunction : public FilterFunction { +public: + GreaterFilterFunction(size_t column_ind, ValueType target_val) + : column_ind_(column_ind), target_val_(std::move(target_val)) { + } + + std::vector Evaluate(const RecordBatch& batch) override { + std::vector selection_vector; + selection_vector.reserve(batch.num_rows); + + AggExpHelper::IterateColumnData( + batch, column_ind_, [&](const auto& value, size_t, size_t real_ind) { + if (value > target_val_) { + selection_vector.push_back(real_ind); + } + }); + return selection_vector; + } + +private: + size_t column_ind_; + ValueType target_val_; +}; + +template class LikeFilterFunction : public FilterFunction { public: LikeFilterFunction(size_t column_ind, ValueType pattern) @@ -70,7 +95,7 @@ class LikeFilterFunction : public FilterFunction { std::vector selection_vector; selection_vector.reserve(batch.num_rows); - AggExpHelper::IterateColumnData( + AggExpHelper::IterateColumnData( batch, column_ind_, [&](const auto& value, size_t, size_t real_ind) { if (value.find(pattern_) != std::string::npos) { selection_vector.push_back(real_ind); @@ -84,7 +109,7 @@ class LikeFilterFunction : public FilterFunction { ValueType pattern_; }; -template +template class NotLikeFilterFunction : public FilterFunction { public: NotLikeFilterFunction(size_t column_ind, ValueType pattern) @@ -95,7 +120,7 @@ class NotLikeFilterFunction : public FilterFunction { std::vector selection_vector; selection_vector.reserve(batch.num_rows); - AggExpHelper::IterateColumnData( + AggExpHelper::IterateColumnData( batch, column_ind_, [&](const auto& value, size_t, size_t real_ind) { if (value.find(pattern_) == std::string::npos) { selection_vector.push_back(real_ind); @@ -108,3 +133,5 @@ class NotLikeFilterFunction : public FilterFunction { size_t column_ind_; ValueType pattern_; }; + + diff --git a/src/execution/expressions/ScalarExpressions.h b/src/execution/expressions/ScalarExpressions.h index 8e344c8..af78a82 100644 --- a/src/execution/expressions/ScalarExpressions.h +++ b/src/execution/expressions/ScalarExpressions.h @@ -53,8 +53,35 @@ class ExtractMinuteExpression : public ScalarExpression { } }; +class LengthExpression : public ScalarExpression { +public: + explicit LengthExpression(std::string column_name, std::string output_name = "") + : ScalarExpression(std::move(column_name), std::move(output_name)) { + } + + std::string GetOutputName() const override { + return output_name_.empty() ? "length_" + column_name_ : output_name_; + } + + std::unique_ptr CreateScalarFunction(Schema& child_schema) override { + size_t ind = child_schema.GetColumnIndexByName(column_name_); + ColumnType type = child_schema.GetColumnTypeByName(column_name_); + + if (type != ColumnType::STRING) [[unlikely]] { + THROW_RUNTIME_ERROR("Length requires STRING column"); + } + + child_schema.AddColumn(GetOutputName(), ColumnType::INT64); + return std::make_unique(ind); + } +}; + inline std::shared_ptr ExtractMinute(std::string column_name, std::string output_name = "") { return std::make_shared(std::move(column_name), std::move(output_name)); } + +inline std::shared_ptr Length(std::string column_name, std::string output_name = "") { + return std::make_shared(std::move(column_name), std::move(output_name)); +} diff --git a/src/execution/expressions/ScalarFunctions.h b/src/execution/expressions/ScalarFunctions.h index 4d76846..d0883bd 100644 --- a/src/execution/expressions/ScalarFunctions.h +++ b/src/execution/expressions/ScalarFunctions.h @@ -4,6 +4,7 @@ #include "execution/expressions/AggExpHelper.h" #include "column/TemporalColumn.h" #include "column/NumericColumn.h" +#include "types/String.h" #include @@ -18,15 +19,33 @@ class ExtractMinuteFunction : public ScalarFunction { explicit ExtractMinuteFunction(size_t col_ind) : col_ind_(col_ind) {} std::shared_ptr Evaluate(const RecordBatch& batch) override { - std::vector result_data; - result_data.reserve(batch.num_rows); + size_t physical_size = batch.columns[0]->Size(); + std::vector result_data(physical_size); - AggExpHelper::IterateColumnData(batch, col_ind_, [&](const auto& value, size_t, size_t) { - result_data.push_back(Timestamp::ExtractMinute(value)); + AggExpHelper::IterateColumnData(batch, col_ind_, [&](const auto& value, size_t, size_t real_ind) { + result_data[real_ind] = Timestamp::ExtractMinute(value); }); - auto result_col = std::make_shared(std::move(result_data)); - return result_col; + return std::make_shared(std::move(result_data)); + } + +private: + size_t col_ind_; +}; + +class LengthFunction : public ScalarFunction { +public: + explicit LengthFunction(size_t col_ind) : col_ind_(col_ind) {} + + std::shared_ptr Evaluate(const RecordBatch& batch) override { + size_t physical_size = batch.columns[0]->Size(); + std::vector result_data(physical_size); + + AggExpHelper::IterateColumnData(batch, col_ind_, [&](const auto& value, size_t, size_t real_ind) { + result_data[real_ind] = String::Length(value); + }); + + return std::make_shared(std::move(result_data)); } private: diff --git a/src/main.cpp b/src/main.cpp index 5b1a3d8..bd82016 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -2,6 +2,7 @@ #include "execution/ExecutionApi.h" #include +#include class Query { public: @@ -282,6 +283,20 @@ class Query { df.Display(); } + // SELECT CounterID, AVG(length(URL)) AS l, COUNT(*) AS c FROM hits WHERE URL <> '' GROUP BY + // CounterID HAVING COUNT(*) > 100000 ORDER BY l DESC LIMIT 25; + void Query27() { + auto df = DataFrame::Select(columnar_file_path_, {"CounterID", "URL"}) + .Filter(NotEq("URL", "")) + .Project({Length("URL", "length_URL")}) + .Aggregate({"CounterID"}, {Avg("length_URL", "l"), Count("*", "c")}) + .Filter(Greater("c", static_cast(100000))) + .OrderBy({{"l", true}}, 25) + .Collect(); + + df.Display(); + } + void RunQuery(int query_number) { switch (query_number) { case 0: Query00(); break; @@ -311,6 +326,7 @@ class Query { case 24: Query24(); break; case 25: Query25(); break; case 26: Query26(); break; + case 27: Query27(); break; default: THROW_RUNTIME_ERROR("Unknown query number: " + std::to_string(query_number)); } diff --git a/src/types/String.h b/src/types/String.h new file mode 100644 index 0000000..384d701 --- /dev/null +++ b/src/types/String.h @@ -0,0 +1,10 @@ +#pragma once + +#include + +class String { +public: + static size_t Length(std::string_view string) { + return string.size(); + } +}; From d815b84415d8545ffb6e828f34f21c95b4c5d0ad Mon Sep 17 00:00:00 2001 From: Mikhail Fomichev Date: Sun, 10 May 2026 13:46:36 +0300 Subject: [PATCH 04/18] fix: query 23 fixed, OrderByOperator batch processing fixed (it works with every memory limit now), SELECT(*) supported --- src/column/CharColumn.h | 2 +- src/execution/ExecutionLogic.h | 84 +++++++++------- src/execution/OperatorsBase.cpp | 172 +++++++++++++++++--------------- src/execution/OperatorsBase.h | 2 + src/io/ColumnarWriter.cpp | 5 - src/main.cpp | 3 +- 6 files changed, 144 insertions(+), 124 deletions(-) diff --git a/src/column/CharColumn.h b/src/column/CharColumn.h index e42bcd3..0328e5d 100644 --- a/src/column/CharColumn.h +++ b/src/column/CharColumn.h @@ -23,7 +23,7 @@ class CharColumn : public Column { } const void* GetRawData() const override { - return data_.data(); + return &data_; } void Clear() override { diff --git a/src/execution/ExecutionLogic.h b/src/execution/ExecutionLogic.h index 176523c..2cb1306 100644 --- a/src/execution/ExecutionLogic.h +++ b/src/execution/ExecutionLogic.h @@ -8,10 +8,6 @@ #include "execution/expressions/ScalarExpressions.h" #include "column/Schema.h" -#include "column/NumericColumn.h" -#include "column/StringColumn.h" -#include "column/CharColumn.h" -#include "column/TemporalColumn.h" #include "utils/Macro.h" @@ -32,31 +28,37 @@ enum class PlanNodeType { struct PlanNode { PlanNodeType type; + protected: - explicit PlanNode(PlanNodeType node_type) : type(node_type) {} + explicit PlanNode(PlanNodeType node_type) : type(node_type) { + } }; struct ScanNode : public PlanNode { - ScanNode() : PlanNode(PlanNodeType::SCAN) {} + ScanNode() : PlanNode(PlanNodeType::SCAN) { + } std::string table_path; std::vector column_names; }; struct AggregateNode : public PlanNode { - AggregateNode() : PlanNode(PlanNodeType::AGGREGATE) {} + AggregateNode() : PlanNode(PlanNodeType::AGGREGATE) { + } std::shared_ptr child; std::vector group_by_columns; std::vector> aggregate_expressions; }; struct FilterNode : public PlanNode { - FilterNode() : PlanNode(PlanNodeType::FILTER) {} + FilterNode() : PlanNode(PlanNodeType::FILTER) { + } std::shared_ptr child; std::shared_ptr filter_expression; }; struct OrderByNode : public PlanNode { - OrderByNode() : PlanNode(PlanNodeType::ORDER_BY) {} + OrderByNode() : PlanNode(PlanNodeType::ORDER_BY) { + } std::shared_ptr child; std::vector> order_by_columns; std::optional limit; @@ -70,7 +72,8 @@ struct LimitNode : public PlanNode { }; struct ScalarNode : public PlanNode { - ScalarNode() : PlanNode(PlanNodeType::SCALAR) {} + ScalarNode() : PlanNode(PlanNodeType::SCALAR) { + } std::shared_ptr child; std::vector> scalar_expressions; }; @@ -91,23 +94,22 @@ inline PhysicalOperatorContext BuildPhysicalPlan( Schema full_schema = Schema::FromColumnarFile(scan->table_path); const std::vector& full_columns = full_schema.GetFields(); - if (required_columns.has_value()) { - final_columns = required_columns.value(); - if (final_columns.empty() && !full_columns.empty()) { - final_columns.push_back(full_columns[0].name); + std::vector user_cols; + if (scan->column_names.size() == 1 && scan->column_names[0] == "*") { + for (const Field& field : full_columns) { + user_cols.push_back(field.name); } } else { - if (scan->column_names.empty()) { - for (const Field& field : full_columns) { - final_columns.push_back(field.name); - } - } else { - final_columns = scan->column_names; - } + user_cols = scan->column_names; } - auto op = std::make_unique(scan->table_path, final_columns); + final_columns = user_cols; + if (required_columns.has_value()) { + final_columns.insert(final_columns.end(), required_columns->begin(), + required_columns->end()); + } + auto op = std::make_unique(scan->table_path, final_columns); Schema output_schema; for (const std::string& name : final_columns) { bool found = false; @@ -157,8 +159,8 @@ inline PhysicalOperatorContext BuildPhysicalPlan( std::vector> agg_funcs; for (const std::shared_ptr& expr : aggregate->aggregate_expressions) { - agg_funcs.push_back( - expr->CreateGroupedAggregationFunction(child_context.schema, output_schema)); + agg_funcs.push_back(expr->CreateGroupedAggregationFunction(child_context.schema, + output_schema)); } auto op = std::make_unique( @@ -168,17 +170,20 @@ inline PhysicalOperatorContext BuildPhysicalPlan( } std::vector needed_columns = aggregate->group_by_columns; - for (const std::shared_ptr& expr : aggregate->aggregate_expressions) { + for (const std::shared_ptr& expr : + aggregate->aggregate_expressions) { expr->CollectRequiredColumns(needed_columns); } std::sort(needed_columns.begin(), needed_columns.end()); needed_columns.erase(std::unique(needed_columns.begin(), needed_columns.end()), needed_columns.end()); - PhysicalOperatorContext child_context = BuildPhysicalPlan(aggregate->child, needed_columns); + PhysicalOperatorContext child_context = + BuildPhysicalPlan(aggregate->child, needed_columns); std::vector> agg_functions; Schema output_schema; - for (const std::shared_ptr& expr : aggregate->aggregate_expressions) { + for (const std::shared_ptr& expr : + aggregate->aggregate_expressions) { agg_functions.push_back( expr->CreateGlobalAggregationFunction(child_context.schema, output_schema)); } @@ -200,7 +205,8 @@ inline PhysicalOperatorContext BuildPhysicalPlan( needed_columns.erase(std::unique(needed_columns.begin(), needed_columns.end()), needed_columns.end()); - PhysicalOperatorContext child_context = BuildPhysicalPlan(filter->child, needed_columns); + PhysicalOperatorContext child_context = + BuildPhysicalPlan(filter->child, needed_columns); std::unique_ptr filter_function = filter->filter_expression->CreateFilterFunction(child_context.schema); auto op = std::make_unique(std::move(child_context.root_operator), @@ -221,24 +227,26 @@ inline PhysicalOperatorContext BuildPhysicalPlan( needed_columns.erase(std::unique(needed_columns.begin(), needed_columns.end()), needed_columns.end()); - PhysicalOperatorContext child_context = BuildPhysicalPlan(order_by->child, needed_columns); + PhysicalOperatorContext child_context = + BuildPhysicalPlan(order_by->child, needed_columns); std::vector> sort_columns; for (const auto& [col_name, is_desc] : order_by->order_by_columns) { size_t sort_col_idx = child_context.schema.GetColumnIndexByName(col_name); sort_columns.push_back({sort_col_idx, is_desc}); } - auto op = - std::make_unique(std::move(child_context.root_operator), - std::move(sort_columns), std::move(order_by->limit)); + auto op = std::make_unique(std::move(child_context.root_operator), + std::move(sort_columns), + std::move(order_by->limit)); return {std::move(op), std::move(child_context.schema)}; } case PlanNodeType::LIMIT: { auto limit = std::static_pointer_cast(plan_node); - PhysicalOperatorContext child_context = BuildPhysicalPlan(limit->child, required_columns); - auto op = - std::make_unique(std::move(child_context.root_operator), limit->limit); + PhysicalOperatorContext child_context = + BuildPhysicalPlan(limit->child, required_columns); + auto op = std::make_unique(std::move(child_context.root_operator), + limit->limit); return {std::move(op), std::move(child_context.schema)}; } @@ -267,7 +275,8 @@ inline PhysicalOperatorContext BuildPhysicalPlan( needed_columns.erase(std::unique(needed_columns.begin(), needed_columns.end()), needed_columns.end()); - PhysicalOperatorContext child_context = BuildPhysicalPlan(scalar->child, needed_columns); + PhysicalOperatorContext child_context = + BuildPhysicalPlan(scalar->child, needed_columns); std::vector> scalar_functions; @@ -280,7 +289,6 @@ inline PhysicalOperatorContext BuildPhysicalPlan( return {std::move(op), std::move(child_context.schema)}; } - default: - THROW_RUNTIME_ERROR("Unknown plan node type"); + default: THROW_RUNTIME_ERROR("Unknown plan node type"); } } diff --git a/src/execution/OperatorsBase.cpp b/src/execution/OperatorsBase.cpp index cf478ea..ce5100d 100644 --- a/src/execution/OperatorsBase.cpp +++ b/src/execution/OperatorsBase.cpp @@ -12,7 +12,7 @@ ScanOperator::ScanOperator(const std::string& table_path, : reader_(table_path) { const std::vector& metadata = reader_.GetMetadata(); - if (column_names.empty()) { + if (column_names == std::vector{"*"}) { for (size_t i = 0; i < metadata.size(); ++i) { column_indices_.push_back(i); } @@ -68,14 +68,16 @@ std::unique_ptr AggregationOperator::Run() { return nullptr; } while (std::unique_ptr batch = child_->Run()) { - for (std::unique_ptr& aggregation_function : aggregation_functions_) { + for (std::unique_ptr& aggregation_function : + aggregation_functions_) { aggregation_function->Update(*batch); } } auto result_batch = std::make_unique(); result_batch->num_rows = 1; - for (std::unique_ptr& aggregation_function : aggregation_functions_) { + for (std::unique_ptr& aggregation_function : + aggregation_functions_) { result_batch->columns.push_back(aggregation_function->Finalize()); } @@ -147,7 +149,9 @@ std::unique_ptr GroupByOperator::Run() { if (!new_group_indices.empty()) { for (size_t k = 0; k < group_by_col_indices_.size(); ++k) { - ExecutionHelper::CopySelection(*key_builders_[k], *batch->columns[group_by_col_indices_[k]], &new_group_indices); + ExecutionHelper::CopySelection(*key_builders_[k], + *batch->columns[group_by_col_indices_[k]], + &new_group_indices); } } @@ -201,45 +205,27 @@ int Compare(const void* raw_data, uint32_t a, uint32_t b) { return 0; } -std::unique_ptr OrderByOperator::Run() { - if (!accumulated_) { - while (auto batch = child_->Run()) { - if (accum_builders_.empty()) { - size_t sz = batch->columns.size(); - for (size_t c = 0; c < sz; ++c) { - accum_builders_.push_back(ColumnFactory::MakeColumnBuilder(batch->columns[c]->GetType())); - accum_types_.push_back(batch->columns[c]->GetType()); - } - accum_num_rows_ = 0; - } - - const std::shared_ptr>& sel = batch->selection_vector; - size_t sz = accum_builders_.size(); - for (size_t c = 0; c < sz; ++c) { - ExecutionHelper::CopySelection(*accum_builders_[c], *batch->columns[c], sel.get()); - } - accum_num_rows_ += batch->num_rows; - } +void OrderByOperator::TrimCurBatch(bool is_final) { + if (accum_num_rows_ == 0) { + return; + } - if (!accum_builders_.empty()) { - // Finish builders into immutable columns for sorting - accumulated_batch_ = std::make_unique(); - accumulated_batch_->num_rows = accum_num_rows_; - for (auto& builder : accum_builders_) { - accumulated_batch_->columns.push_back(builder->Finish()); - } - accum_builders_.clear(); + auto temp_batch = std::make_unique(); + temp_batch->num_rows = accum_num_rows_; + for (auto& builder : accum_builders_) { + temp_batch->columns.push_back(builder->Finish()); + } + accum_builders_.clear(); - size_t sz = accumulated_batch_->num_rows; - indices_.reserve(sz); - for (uint32_t i = 0; i < sz; ++i) { - indices_.push_back(i); - } + std::vector indices(accum_num_rows_); + for (size_t i = 0; i < accum_num_rows_; ++i) { + indices[i] = i; + } - std::vector cols_info; - cols_info.reserve(sort_columns_.size()); - for (const auto& [col_ind, is_desc] : sort_columns_) { - const Column* raw_column = accumulated_batch_->columns[col_ind].get(); + std::vector cols_info; + cols_info.reserve(sort_columns_.size()); + for (const auto& [col_ind, is_desc] : sort_columns_) { + const Column* raw_column = temp_batch->columns[col_ind].get(); #define HANDLE_TYPE(ENUM_VAL, STR_VAL, CLASS_TYPE) \ case ColumnType::ENUM_VAL: { \ @@ -247,50 +233,82 @@ std::unique_ptr OrderByOperator::Run() { cols_info.push_back({data_ptr, &Compare, is_desc}); \ break; \ } - auto col_type = raw_column->GetType(); - switch (col_type) { - FOR_EACH_COLUMN_TYPE(HANDLE_TYPE); - default: THROW_NOT_IMPLEMENTED; - } + auto col_type = raw_column->GetType(); + switch (col_type) { + FOR_EACH_COLUMN_TYPE(HANDLE_TYPE); + default: THROW_NOT_IMPLEMENTED; + } #undef HANDLE_TYPE - } + } - auto cmp = [&cols_info](uint32_t a, uint32_t b) { - for (const auto& info : cols_info) { - int res = info.cmp_func(info.raw_data, a, b); - if (res != 0) { - return info.is_desc ? (res > 0) : (res < 0); - } - } - return false; - }; - - if (limit_.has_value() && limit_.value() < sz) { - uint32_t lim = limit_.value(); - std::nth_element(indices_.begin(), indices_.begin() + lim, indices_.end(), cmp); - std::sort(indices_.begin(), indices_.begin() + lim, cmp); - indices_.resize(lim); - accumulated_batch_->num_rows = lim; - } else { - std::sort(indices_.begin(), indices_.end(), cmp); + auto cmp = [&cols_info](uint32_t a, uint32_t b) { + for (const auto& info : cols_info) { + int res = info.cmp_func(info.raw_data, a, b); + if (res != 0) { + return info.is_desc ? (res > 0) : (res < 0); } + } + return false; + }; - auto sorted_batch = std::make_unique(); - sorted_batch->num_rows = accumulated_batch_->num_rows; - for (size_t c = 0; c < accumulated_batch_->columns.size(); ++c) { - auto builder = ColumnFactory::MakeColumnBuilder(accumulated_batch_->columns[c]->GetType()); - ExecutionHelper::CopySelection(*builder, *accumulated_batch_->columns[c], &indices_); - sorted_batch->columns.push_back(builder->Finish()); + if (limit_.has_value() && accum_num_rows_ > limit_.value()) { + uint32_t lim = limit_.value(); + std::nth_element(indices.begin(), indices.begin() + lim, indices.end(), cmp); + indices.resize(lim); + } + if (is_final) { + std::sort(indices.begin(), indices.end(), cmp); + } + + for (auto type : accum_types_) { + accum_builders_.push_back(ColumnFactory::MakeColumnBuilder(type)); + } + for (size_t c = 0; c < accum_builders_.size(); ++c) { + ExecutionHelper::CopySelection(*accum_builders_[c], *temp_batch->columns[c], &indices); + } + accum_num_rows_ = indices.size(); +} + +std::unique_ptr OrderByOperator::Run() { + if (accumulated_) { + return nullptr; + } + + while (auto batch = child_->Run()) { + if (accum_builders_.empty()) { + for (size_t c = 0; c < batch->columns.size(); ++c) { + accum_builders_.push_back( + ColumnFactory::MakeColumnBuilder(batch->columns[c]->GetType())); + accum_types_.push_back(batch->columns[c]->GetType()); } - accumulated_batch_ = std::move(sorted_batch); + accum_num_rows_ = 0; + } - accumulated_ = true; + const std::shared_ptr>& sel = batch->selection_vector; + for (size_t c = 0; c < accum_builders_.size(); ++c) { + ExecutionHelper::CopySelection(*accum_builders_[c], *batch->columns[c], sel.get()); } - if (!accumulated_batch_) { - return nullptr; + accum_num_rows_ += batch->num_rows; + + if (limit_.has_value() && accum_num_rows_ > limit_.value() * 10) { + TrimCurBatch(false); } - return std::move(accumulated_batch_); } + + if (accum_num_rows_ > 0) { + TrimCurBatch(true); + + auto final_batch = std::make_unique(); + final_batch->num_rows = accum_num_rows_; + for (auto& builder : accum_builders_) { + final_batch->columns.push_back(builder->Finish()); + } + + accumulated_ = true; + return final_batch; + } + + accumulated_ = true; return nullptr; } @@ -316,9 +334,7 @@ std::unique_ptr LimitOperator::Run() { batch->num_rows = remaining; if (batch->selection_vector) { auto new_sel_vec = std::make_shared>( - batch->selection_vector->begin(), - batch->selection_vector->begin() + remaining - ); + batch->selection_vector->begin(), batch->selection_vector->begin() + remaining); batch->selection_vector = std::move(new_sel_vec); } else { auto sel_vec = std::make_shared>(remaining); diff --git a/src/execution/OperatorsBase.h b/src/execution/OperatorsBase.h index a9f68fe..0034407 100644 --- a/src/execution/OperatorsBase.h +++ b/src/execution/OperatorsBase.h @@ -92,6 +92,8 @@ class OrderByOperator : public Operator { std::unique_ptr Run() override; private: + void TrimCurBatch(bool is_final); + std::unique_ptr child_; std::vector> sort_columns_; std::unique_ptr accumulated_batch_; diff --git a/src/io/ColumnarWriter.cpp b/src/io/ColumnarWriter.cpp index 55fa526..64296f1 100644 --- a/src/io/ColumnarWriter.cpp +++ b/src/io/ColumnarWriter.cpp @@ -72,11 +72,6 @@ void ColumnarWriter::WriteBatch(const std::vector>& colu } file_.write(buffer.data(), buffer.size()); size = total_size; - } else if (type == ColumnType::CHAR) { - size = columns[i]->Size(); - if (size > 0) { - file_.write(static_cast(columns[i]->GetRawData()), size); - } } else { size_t elem_size = 0; #define HANDLE_TYPE(ENUM_VAL, STR_VAL, CLASS_TYPE) \ diff --git a/src/main.cpp b/src/main.cpp index bd82016..ba3e055 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -245,10 +245,9 @@ class Query { df.Display(); } - // TODO: support SELECT * // SELECT * FROM hits WHERE URL LIKE '%google%' ORDER BY EventTime LIMIT 10; void Query23() { - auto df = DataFrame::Select(columnar_file_path_, {"URL", "EventTime"}) + auto df = DataFrame::Select(columnar_file_path_, {"*"}) .Filter(Like("URL", "%google%")) .OrderBy({{"EventTime", false}}, 10) .Collect(); From 1baaa286c61d4c5f5c85c84f9d0ef749cf7c2200 Mon Sep 17 00:00:00 2001 From: Mikhail Fomichev Date: Sun, 10 May 2026 21:52:41 +0300 Subject: [PATCH 05/18] feat: queries 29-35 --- src/execution/ExecutionApi.h | 31 ++- src/execution/ExecutionLogic.h | 92 ++++++++- src/execution/OperatorsBase.cpp | 21 ++ src/execution/OperatorsBase.h | 45 ++++- .../expressions/AggregationExpressions.h | 181 ++++++++++-------- .../expressions/AggregationFunctions.h | 8 +- src/execution/expressions/ScalarExpressions.h | 78 +++++++- src/execution/expressions/ScalarFunctions.h | 52 +++++ src/main.cpp | 149 ++++++++++++++ 9 files changed, 561 insertions(+), 96 deletions(-) diff --git a/src/execution/ExecutionApi.h b/src/execution/ExecutionApi.h index b1df5d0..5424d9d 100644 --- a/src/execution/ExecutionApi.h +++ b/src/execution/ExecutionApi.h @@ -25,7 +25,7 @@ class DataResult { const std::vector& fields = schema_.GetFields(); if (!fields.empty()) { for (const Field& field : fields) { - std::cout << field.name << " "; + std::cout << field.name << ","; } std::cout << "\n"; } @@ -38,13 +38,26 @@ class DataResult { for (size_t i = 0; i < rows; ++i) { size_t ind = batch->selection_vector ? (*batch->selection_vector)[i] : i; for (const std::shared_ptr& column : batch->columns) { - std::cout << ExecutionHelper::FormatValue(*column, ind) << " "; + std::cout << ExecutionHelper::FormatValue(*column, ind) << ","; } std::cout << "\n"; } } } + std::shared_ptr GetResult(std::string column_name) const { + const std::vector& fields = schema_.GetFields(); + for (size_t i = 0; i < fields.size(); ++i) { + if (fields[i].name == column_name) { + if (!batches_[0]->columns.empty()) { + return batches_[0]->columns[i]; + } + break; + } + } + THROW_RUNTIME_ERROR("No charoncik bebe"); + } + private: std::vector> batches_; Schema schema_; @@ -102,6 +115,20 @@ class DataFrame { return DataFrame(scalar_node); } + DataFrame Drop(std::vector columns_to_drop) { + auto drop_node = std::make_shared(); + drop_node->child = logical_plan_; + drop_node->columns_to_drop = std::move(columns_to_drop); + return DataFrame(drop_node); + } + + DataFrame Reoder(std::vector desired_order) { + auto node = std::make_shared(); + node->child = logical_plan_; + node->desired_order_ = std::move(desired_order); + return DataFrame(node); + } + DataResult Collect() { PhysicalOperatorContext context = BuildPhysicalPlan(logical_plan_); std::unique_ptr physical_plan_root = std::move(context.root_operator); diff --git a/src/execution/ExecutionLogic.h b/src/execution/ExecutionLogic.h index 2cb1306..f0e59e0 100644 --- a/src/execution/ExecutionLogic.h +++ b/src/execution/ExecutionLogic.h @@ -23,7 +23,9 @@ enum class PlanNodeType { FILTER, ORDER_BY, LIMIT, - SCALAR + SCALAR, + DROP, + REORDER }; struct PlanNode { @@ -78,6 +80,20 @@ struct ScalarNode : public PlanNode { std::vector> scalar_expressions; }; +struct DropNode : public PlanNode { + DropNode() : PlanNode(PlanNodeType::DROP) { + } + std::shared_ptr child; + std::vector columns_to_drop; +}; + +struct ReorderNode : public PlanNode { + ReorderNode() : PlanNode(PlanNodeType::REORDER) { + } + std::shared_ptr child; + std::vector desired_order_; +}; + struct PhysicalOperatorContext { std::unique_ptr root_operator; Schema schema; @@ -289,6 +305,78 @@ inline PhysicalOperatorContext BuildPhysicalPlan( return {std::move(op), std::move(child_context.schema)}; } - default: THROW_RUNTIME_ERROR("Unknown plan node type"); + case PlanNodeType::DROP: { + auto drop = std::static_pointer_cast(plan_node); + + PhysicalOperatorContext child_context = + BuildPhysicalPlan(drop->child, required_columns); + + std::vector cols_to_keep_indices; + Schema output_schema; + const auto& child_fields = child_context.schema.GetFields(); + + for (const std::string& drop_col : drop->columns_to_drop) { + bool found = false; + for (const auto& field : child_fields) { + if (field.name == drop_col) { + found = true; + break; + } + } + if (!found) { + THROW_RUNTIME_ERROR("Cannot drop column '" + drop_col + + "': not found in dataframe"); + } + } + + for (size_t i = 0; i < child_fields.size(); ++i) { + const auto& field = child_fields[i]; + if (std::find(drop->columns_to_drop.begin(), drop->columns_to_drop.end(), + field.name) == drop->columns_to_drop.end()) { + cols_to_keep_indices.push_back(i); + output_schema.AddField({field.name, field.type}); + } + } + + auto op = std::make_unique(std::move(child_context.root_operator), + std::move(cols_to_keep_indices)); + return {std::move(op), std::move(output_schema)}; + } + + case PlanNodeType::REORDER: { + auto reorder = std::static_pointer_cast(plan_node); + + PhysicalOperatorContext child_context = BuildPhysicalPlan(reorder->child, required_columns); + + std::vector new_indices; + Schema output_schema; + const auto& child_fields = child_context.schema.GetFields(); + + for (const std::string& reorder_col : reorder->desired_order_) { + bool found = false; + for (size_t i = 0; i < child_fields.size(); ++i) { + const auto& field = child_fields[i]; + if (field.name == reorder_col) { + found = true; + new_indices.push_back(i); + output_schema.AddField({field.name, field.type}); + break; + } + } + + if (!found) { + THROW_RUNTIME_ERROR("Cannot reorder column '" + reorder_col + "': not found in dataframe"); + } + } + + auto op = std::make_unique( + std::move(child_context.root_operator), + std::move(new_indices) + ); + return {std::move(op), std::move(output_schema)}; + } + + default: + THROW_RUNTIME_ERROR("Unknown plan node type"); } } diff --git a/src/execution/OperatorsBase.cpp b/src/execution/OperatorsBase.cpp index ce5100d..5cebb1c 100644 --- a/src/execution/OperatorsBase.cpp +++ b/src/execution/OperatorsBase.cpp @@ -363,3 +363,24 @@ std::unique_ptr ScalarOperator::Run() { return batch; } + +DropOperator::DropOperator(std::unique_ptr child, std::vector column_stay_indices) + : child_(std::move(child)), column_stay_indices_(std::move(column_stay_indices)) { +} + +std::unique_ptr DropOperator::Run() { + auto batch = child_->Run(); + if (!batch) { + return nullptr; + } + + std::vector> new_columns; + new_columns.reserve(column_stay_indices_.size()); + + for (size_t ind : column_stay_indices_) { + new_columns.push_back(std::move(batch->columns[ind])); + } + + batch->columns = std::move(new_columns); + return batch; +} diff --git a/src/execution/OperatorsBase.h b/src/execution/OperatorsBase.h index 0034407..1218a4f 100644 --- a/src/execution/OperatorsBase.h +++ b/src/execution/OperatorsBase.h @@ -41,8 +41,9 @@ class ScanOperator : public Operator { class AggregationOperator : public Operator { public: - AggregationOperator(std::unique_ptr child, - std::vector> aggregation_functions); + AggregationOperator( + std::unique_ptr child, + std::vector> aggregation_functions); std::unique_ptr Run() override; @@ -129,3 +130,43 @@ class ScalarOperator : public Operator { std::unique_ptr child_; std::vector> scalar_functions_; }; + +class DropOperator : public Operator { +public: + DropOperator(std::unique_ptr child, std::vector column_stay_indices); + + std::unique_ptr Run() override; + +private: + std::unique_ptr child_; + std::vector column_stay_indices_; +}; + +class ReorderOperator : public Operator { +public: + ReorderOperator(std::unique_ptr child, std::vector new_indices) + : child_(std::move(child)), new_indices_(std::move(new_indices)) { + } + + std::unique_ptr Run() override { + auto batch = child_->Run(); + if (!batch) { + return nullptr; + } + + auto new_batch = std::make_unique(); + new_batch->num_rows = batch->num_rows; + new_batch->selection_vector = batch->selection_vector; + + new_batch->columns.reserve(new_indices_.size()); + for (size_t ind : new_indices_) { + new_batch->columns.push_back(batch->columns[ind]); + } + + return new_batch; + } + +private: + std::unique_ptr child_; + std::vector new_indices_; +}; diff --git a/src/execution/expressions/AggregationExpressions.h b/src/execution/expressions/AggregationExpressions.h index 7074fe7..f21cfba 100644 --- a/src/execution/expressions/AggregationExpressions.h +++ b/src/execution/expressions/AggregationExpressions.h @@ -8,6 +8,48 @@ #include #include +template +struct SumResultTrait { + using Type = InputColumn; + static constexpr ColumnType kOutTypeEnum = ColumnType::INT64; +}; + +template <> +struct SumResultTrait { + using Type = Int32Column; + static constexpr ColumnType kOutTypeEnum = ColumnType::INT32; +}; +template <> +struct SumResultTrait { + using Type = Int64Column; + static constexpr ColumnType kOutTypeEnum = ColumnType::INT64; +}; +template <> +struct SumResultTrait { + using Type = Int128Column; + static constexpr ColumnType kOutTypeEnum = ColumnType::INT128; +}; +template <> +struct SumResultTrait { + using Type = Int128Column; + static constexpr ColumnType kOutTypeEnum = ColumnType::INT128; +}; +template <> +struct SumResultTrait { + using Type = DoubleColumn; + static constexpr ColumnType kOutTypeEnum = ColumnType::DOUBLE; +}; +template <> +struct SumResultTrait { + using Type = LongDoubleColumn; + static constexpr ColumnType kOutTypeEnum = ColumnType::LONGDOUBLE; +}; +template <> +struct SumResultTrait { + using Type = LongDoubleColumn; + static constexpr ColumnType kOutTypeEnum = ColumnType::LONGDOUBLE; +}; + class AggregateExpression { public: virtual ~AggregateExpression() = default; @@ -203,45 +245,22 @@ class SumExpression : public AggregateExpression { const size_t column_ind = child_schema.GetColumnIndexByName(GetName()); const ColumnType column_type = child_schema.GetColumnTypeByName(GetName()); - switch (column_type) { - case ColumnType::INT16: - output_schema.AddColumn(GetOutputName(), ColumnType::INT32); - return std::make_unique< - TypedGlobalAggregationFunction>( - column_ind, ColumnType::INT32); - case ColumnType::INT32: - output_schema.AddColumn(GetOutputName(), ColumnType::INT64); - return std::make_unique< - TypedGlobalAggregationFunction>( - column_ind, ColumnType::INT64); - case ColumnType::INT64: - output_schema.AddColumn(GetOutputName(), ColumnType::INT128); - return std::make_unique< - TypedGlobalAggregationFunction>( - column_ind, ColumnType::INT128); - case ColumnType::INT128: - output_schema.AddColumn(GetOutputName(), ColumnType::INT128); - return std::make_unique< - TypedGlobalAggregationFunction>( - column_ind, ColumnType::INT128); - case ColumnType::FLOAT: - output_schema.AddColumn(GetOutputName(), ColumnType::DOUBLE); - return std::make_unique< - TypedGlobalAggregationFunction>( - column_ind, ColumnType::DOUBLE); - case ColumnType::DOUBLE: - output_schema.AddColumn(GetOutputName(), ColumnType::LONGDOUBLE); - return std::make_unique< - TypedGlobalAggregationFunction>( - column_ind, ColumnType::LONGDOUBLE); - case ColumnType::LONGDOUBLE: - output_schema.AddColumn(GetOutputName(), ColumnType::LONGDOUBLE); - return std::make_unique>( - column_ind, ColumnType::LONGDOUBLE); - default: THROW_NOT_IMPLEMENTED; + if (column_type == ColumnType::STRING || column_type == ColumnType::CHAR || + column_type == ColumnType::DATE || column_type == ColumnType::TIMESTAMP) { + THROW_RUNTIME_ERROR("SUM operation is not supported for this column type"); } + + return AggExpHelper::DispatchNumeric( + column_type, + [&](ColumnType) -> std::unique_ptr { + using OutputColumn = SumResultTrait::Type; + ColumnType out_type = SumResultTrait::kOutTypeEnum; + + output_schema.AddColumn(GetOutputName(), out_type); + return std::make_unique< + TypedGlobalAggregationFunction>( + column_ind, out_type); + }); } std::unique_ptr CreateGroupedAggregationFunction( @@ -249,45 +268,22 @@ class SumExpression : public AggregateExpression { const size_t column_ind = child_schema.GetColumnIndexByName(GetName()); const ColumnType column_type = child_schema.GetColumnTypeByName(GetName()); - switch (column_type) { - case ColumnType::INT16: - output_schema.AddColumn(GetOutputName(), ColumnType::INT32); - return std::make_unique< - TypedGroupedAggregationFunction>( - column_ind, ColumnType::INT32); - case ColumnType::INT32: - output_schema.AddColumn(GetOutputName(), ColumnType::INT64); - return std::make_unique< - TypedGroupedAggregationFunction>( - column_ind, ColumnType::INT64); - case ColumnType::INT64: - output_schema.AddColumn(GetOutputName(), ColumnType::INT128); - return std::make_unique< - TypedGroupedAggregationFunction>( - column_ind, ColumnType::INT128); - case ColumnType::INT128: - output_schema.AddColumn(GetOutputName(), ColumnType::INT128); - return std::make_unique< - TypedGroupedAggregationFunction>( - column_ind, ColumnType::INT128); - case ColumnType::FLOAT: - output_schema.AddColumn(GetOutputName(), ColumnType::DOUBLE); - return std::make_unique< - TypedGroupedAggregationFunction>( - column_ind, ColumnType::DOUBLE); - case ColumnType::DOUBLE: - output_schema.AddColumn(GetOutputName(), ColumnType::LONGDOUBLE); - return std::make_unique< - TypedGroupedAggregationFunction>( - column_ind, ColumnType::LONGDOUBLE); - case ColumnType::LONGDOUBLE: - output_schema.AddColumn(GetOutputName(), ColumnType::LONGDOUBLE); - return std::make_unique>( - column_ind, ColumnType::LONGDOUBLE); - default: THROW_NOT_IMPLEMENTED; + if (column_type == ColumnType::STRING || column_type == ColumnType::CHAR || + column_type == ColumnType::DATE || column_type == ColumnType::TIMESTAMP) { + THROW_RUNTIME_ERROR("SUM operation is not supported for this column type"); } + + return AggExpHelper::DispatchNumeric( + column_type, + [&](ColumnType) -> std::unique_ptr { + using OutputColumn = SumResultTrait::Type; + ColumnType out_type = SumResultTrait::kOutTypeEnum; + + output_schema.AddColumn(GetOutputName(), out_type); + return std::make_unique< + TypedGroupedAggregationFunction>( + column_ind, out_type); + }); } }; @@ -308,11 +304,20 @@ class AvgExpression : public AggregateExpression { const size_t column_ind = child_schema.GetColumnIndexByName(GetName()); const ColumnType column_type = child_schema.GetColumnTypeByName(GetName()); + if (column_type == ColumnType::STRING || column_type == ColumnType::CHAR || + column_type == ColumnType::DATE || column_type == ColumnType::TIMESTAMP) { + THROW_RUNTIME_ERROR("AVG operation is not supported for non-numeric columns"); + } + return AggExpHelper::DispatchNumeric( - column_type, [&](ColumnType) -> std::unique_ptr { - output_schema.AddColumn(GetOutputName(), ColumnType::DOUBLE); - return std::make_unique>( - column_ind, ColumnType::DOUBLE); + column_type, + [&](ColumnType) -> std::unique_ptr { + using StateColumn = SumResultTrait::Type; + + output_schema.AddColumn(GetOutputName(), ColumnType::LONGDOUBLE); + return std::make_unique< + AvgGlobalAggregationFunction>( + column_ind, ColumnType::LONGDOUBLE); }); } @@ -321,12 +326,20 @@ class AvgExpression : public AggregateExpression { const size_t column_ind = child_schema.GetColumnIndexByName(GetName()); const ColumnType column_type = child_schema.GetColumnTypeByName(GetName()); + if (column_type == ColumnType::STRING || column_type == ColumnType::CHAR || + column_type == ColumnType::DATE || column_type == ColumnType::TIMESTAMP) { + THROW_RUNTIME_ERROR("AVG operation is not supported for non-numeric columns"); + } + return AggExpHelper::DispatchNumeric( column_type, - [&](ColumnType) -> std::unique_ptr { - output_schema.AddColumn(GetOutputName(), ColumnType::DOUBLE); - return std::make_unique>( - column_ind, ColumnType::DOUBLE); + [&](ColumnType) -> std::unique_ptr { + using StateColumn = SumResultTrait::Type; + + output_schema.AddColumn(GetOutputName(), ColumnType::LONGDOUBLE); + return std::make_unique< + AvgGroupedAggregationFunction>( + column_ind, ColumnType::LONGDOUBLE); }); } }; diff --git a/src/execution/expressions/AggregationFunctions.h b/src/execution/expressions/AggregationFunctions.h index 3e3e8c9..db9950d 100644 --- a/src/execution/expressions/AggregationFunctions.h +++ b/src/execution/expressions/AggregationFunctions.h @@ -219,9 +219,9 @@ class CountGroupedAggregationFunction : public GroupedAggregationFunction { std::vector counts_; }; -template +template class AvgGlobalAggregationFunction : public GlobalAggregationFunction { - using StateType = InputColumn::ValueType; + using StateType = StateColumn::ValueType; using OutputType = OutputColumn::ValueType; using OutputBuilder = BuilderTypeTrait::Type; @@ -267,9 +267,9 @@ class AvgGlobalAggregationFunction : public GlobalAggregationFunction { bool has_data_ = false; }; -template +template class AvgGroupedAggregationFunction : public GroupedAggregationFunction { - using StateType = InputColumn::ValueType; + using StateType = StateColumn::ValueType; using OutputType = OutputColumn::ValueType; using OutputBuilder = BuilderTypeTrait::Type; diff --git a/src/execution/expressions/ScalarExpressions.h b/src/execution/expressions/ScalarExpressions.h index af78a82..c305d67 100644 --- a/src/execution/expressions/ScalarExpressions.h +++ b/src/execution/expressions/ScalarExpressions.h @@ -15,7 +15,7 @@ class ScalarExpression { : column_name_(std::move(column_name)), output_name_(std::move(output_name)) { } - void CollectRequiredColumns(std::vector& required_columns) { + virtual void CollectRequiredColumns(std::vector& required_columns) { required_columns.push_back(column_name_); } @@ -76,12 +76,86 @@ class LengthExpression : public ScalarExpression { } }; +template +class ConstantExpression : public ScalarExpression { +public: + ConstantExpression(std::string output_name, T constant) + : ScalarExpression("", std::move(output_name)), constant_(std::move(constant)) { + } + + void CollectRequiredColumns(std::vector&) override { + // yes charonchik bebe + } + + std::unique_ptr CreateScalarFunction(Schema& child_schema) override { + if constexpr (std::is_constructible_v) { + child_schema.AddColumn(GetOutputName(), ColumnType::STRING); + return std::make_unique>(ColumnType::STRING, + std::string(constant_)); + } else { + ColumnType type = ColumnTypeTraits::kType; + child_schema.AddColumn(GetOutputName(), type); + return std::make_unique>>(type, constant_); + } + } + +private: + T constant_; +}; + +template +class AddConstantExpression : public ScalarExpression { +public: + AddConstantExpression(std::string column_name, T constant, std::string output_name = "") + : ScalarExpression(std::move(column_name), std::move(output_name)), + constant_(std::move(constant)) { + } + + std::string GetOutputName() const override { + if (!output_name_.empty()) { + return output_name_; + } + return column_name_ + "_plus_" + std::to_string(constant_); + } + + std::unique_ptr CreateScalarFunction(Schema& child_schema) override { + size_t ind = child_schema.GetColumnIndexByName(column_name_); + ColumnType column_type = child_schema.GetColumnTypeByName(column_name_); + + if (column_type == ColumnType::STRING || column_type == ColumnType::CHAR || + column_type == ColumnType::DATE || column_type == ColumnType::TIMESTAMP) { + THROW_RUNTIME_ERROR("AddConst is not supported for non-numeric columns"); + } + + return AggExpHelper::DispatchNumeric( + column_type, + [&](ColumnType) -> std::unique_ptr { + child_schema.AddColumn(GetOutputName(), column_type); + return std::make_unique>(ind, constant_); + }); + } + +private: + T constant_; +}; + inline std::shared_ptr ExtractMinute(std::string column_name, std::string output_name = "") { return std::make_shared(std::move(column_name), std::move(output_name)); } -inline std::shared_ptr Length(std::string column_name, std::string output_name = "") { +inline std::shared_ptr Length(std::string column_name, + std::string output_name = "") { return std::make_shared(std::move(column_name), std::move(output_name)); } + +template +inline std::shared_ptr Literal(std::string output_name, T value) { + return std::make_shared>(std::move(output_name), std::move(value)); +} + +template +inline std::shared_ptr AddConst(std::string column_name, T value, std::string output_name = "") { + return std::make_shared>(std::move(column_name), std::move(value), std::move(output_name)); +} diff --git a/src/execution/expressions/ScalarFunctions.h b/src/execution/expressions/ScalarFunctions.h index d0883bd..0813358 100644 --- a/src/execution/expressions/ScalarFunctions.h +++ b/src/execution/expressions/ScalarFunctions.h @@ -51,3 +51,55 @@ class LengthFunction : public ScalarFunction { private: size_t col_ind_; }; + +template +class ConstantFunction : public ScalarFunction { + using ValueType = ColumnClass::ValueType; + using BuilderType = BuilderTypeTrait::Type; + +public: + ConstantFunction(ColumnType type, ValueType constant) + : type_(type), constant_(std::move(constant)) {} + + std::shared_ptr Evaluate(const RecordBatch& batch) override { + size_t physical_size = batch.num_rows; + + auto builder = ColumnFactory::MakeColumnBuilder(type_); + auto* typed_builder = static_cast(builder.get()); + + for (size_t i = 0; i < physical_size; ++i) { + typed_builder->AddValue(constant_); + } + + return builder->Finish(); + } + +private: + ColumnType type_; + ValueType constant_; +}; + +template +class AddConstantFunction : public ScalarFunction { + using ValueType = ColumnClass::ValueType; + +public: + AddConstantFunction(size_t col_ind, ValueType constant) + : col_ind_(col_ind), constant_(std::move(constant)) {} + + std::shared_ptr Evaluate(const RecordBatch& batch) override { + size_t physical_size = batch.columns.empty() ? batch.num_rows : batch.columns[0]->Size(); + std::vector result_data(physical_size); + + AggExpHelper::IterateColumnData( + batch, col_ind_, [&](const auto& value, size_t, size_t real_ind) { + result_data[real_ind] = static_cast(value + constant_); + }); + + return std::make_shared(std::move(result_data)); + } + +private: + size_t col_ind_; + ValueType constant_; +}; diff --git a/src/main.cpp b/src/main.cpp index ba3e055..321e526 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -296,6 +296,147 @@ class Query { df.Display(); } + // TODO: + // SELECT REGEXP_REPLACE(Referer, '^https?://(?:www\.)?([^/]+)/.*$', '\1') AS k, + // AVG(length(Referer)) AS l, COUNT(*) AS c, MIN(Referer) FROM hits WHERE Referer <> '' GROUP BY + // k HAVING COUNT(*) > 100000 ORDER BY l DESC LIMIT 25; + void Query28() { + THROW_NOT_IMPLEMENTED; + } + + // WARNING: DataFrame::Display output does not affect that query + // SELECT SUM(ResolutionWidth), SUM(ResolutionWidth + 1), SUM(ResolutionWidth + 2), + // SUM(ResolutionWidth + 3), SUM(ResolutionWidth + 4), SUM(ResolutionWidth + 5), + // SUM(ResolutionWidth + 6), SUM(ResolutionWidth + 7), SUM(ResolutionWidth + 8), + // SUM(ResolutionWidth + 9), SUM(ResolutionWidth + 10), SUM(ResolutionWidth + 11), + // SUM(ResolutionWidth + 12), SUM(ResolutionWidth + 13), SUM(ResolutionWidth + 14), + // SUM(ResolutionWidth + 15), SUM(ResolutionWidth + 16), SUM(ResolutionWidth + 17), + // SUM(ResolutionWidth + 18), SUM(ResolutionWidth + 19), SUM(ResolutionWidth + 20), + // SUM(ResolutionWidth + 21), SUM(ResolutionWidth + 22), SUM(ResolutionWidth + 23), + // SUM(ResolutionWidth + 24), SUM(ResolutionWidth + 25), SUM(ResolutionWidth + 26), + // SUM(ResolutionWidth + 27), SUM(ResolutionWidth + 28), SUM(ResolutionWidth + 29), + // SUM(ResolutionWidth + 30), SUM(ResolutionWidth + 31), SUM(ResolutionWidth + 32), + // SUM(ResolutionWidth + 33), SUM(ResolutionWidth + 34), SUM(ResolutionWidth + 35), + // SUM(ResolutionWidth + 36), SUM(ResolutionWidth + 37), SUM(ResolutionWidth + 38), + // SUM(ResolutionWidth + 39), SUM(ResolutionWidth + 40), SUM(ResolutionWidth + 41), + // SUM(ResolutionWidth + 42), SUM(ResolutionWidth + 43), SUM(ResolutionWidth + 44), + // SUM(ResolutionWidth + 45), SUM(ResolutionWidth + 46), SUM(ResolutionWidth + 47), + // SUM(ResolutionWidth + 48), SUM(ResolutionWidth + 49), SUM(ResolutionWidth + 50), + // SUM(ResolutionWidth + 51), SUM(ResolutionWidth + 52), SUM(ResolutionWidth + 53), + // SUM(ResolutionWidth + 54), SUM(ResolutionWidth + 55), SUM(ResolutionWidth + 56), + // SUM(ResolutionWidth + 57), SUM(ResolutionWidth + 58), SUM(ResolutionWidth + 59), + // SUM(ResolutionWidth + 60), SUM(ResolutionWidth + 61), SUM(ResolutionWidth + 62), + // SUM(ResolutionWidth + 63), SUM(ResolutionWidth + 64), SUM(ResolutionWidth + 65), + // SUM(ResolutionWidth + 66), SUM(ResolutionWidth + 67), SUM(ResolutionWidth + 68), + // SUM(ResolutionWidth + 69), SUM(ResolutionWidth + 70), SUM(ResolutionWidth + 71), + // SUM(ResolutionWidth + 72), SUM(ResolutionWidth + 73), SUM(ResolutionWidth + 74), + // SUM(ResolutionWidth + 75), SUM(ResolutionWidth + 76), SUM(ResolutionWidth + 77), + // SUM(ResolutionWidth + 78), SUM(ResolutionWidth + 79), SUM(ResolutionWidth + 80), + // SUM(ResolutionWidth + 81), SUM(ResolutionWidth + 82), SUM(ResolutionWidth + 83), + // SUM(ResolutionWidth + 84), SUM(ResolutionWidth + 85), SUM(ResolutionWidth + 86), + // SUM(ResolutionWidth + 87), SUM(ResolutionWidth + 88), SUM(ResolutionWidth + 89) FROM hits; + void Query29() { + auto df = DataFrame::Select(columnar_file_path_, {"ResolutionWidth"}) + .Aggregate({}, {Sum("ResolutionWidth", "a"), Count("*", "b")}) + .Collect(); + + std::shared_ptr cnt = df.GetResult("b"); + std::shared_ptr sum = df.GetResult("a"); + + const auto* cnt_vec = static_cast*>(cnt->GetRawData()); + int64_t delta = (*cnt_vec)[0]; + + int64_t cur = 0; + const auto* sum_vec = static_cast*>(sum->GetRawData()); + cur = (*sum_vec)[0]; + + for (size_t i = 0; i < 90; ++i) { + std::cout << cur << ","; + cur += delta; + } + std::cout << "\n"; + } + + // SELECT SearchEngineID, ClientIP, COUNT(*) AS c, SUM(IsRefresh), AVG(ResolutionWidth) FROM + // hits WHERE SearchPhrase <> '' GROUP BY SearchEngineID, ClientIP ORDER BY c DESC LIMIT 10; + void Query30() { + auto df = DataFrame::Select(columnar_file_path_, + {"SearchEngineID", "ClientIP", "IsRefresh", "ResolutionWidth"}) + .Filter(NotEq("SearchPhrase", "")) + .Aggregate({"SearchEngineID", "ClientIP"}, + {Count("*", "c"), Sum("IsRefresh", "sum_refresh"), + Avg("ResolutionWidth", "avg_res_width")}) + .OrderBy({{"c", true}}, 10) + .Collect(); + + df.Display(); + } + + // SELECT WatchID, ClientIP, COUNT(*) AS c, SUM(IsRefresh), AVG(ResolutionWidth) FROM hits WHERE + // SearchPhrase <> '' GROUP BY WatchID, ClientIP ORDER BY c DESC LIMIT 10; + void Query31() { + auto df = DataFrame::Select(columnar_file_path_, + {"WatchID", "ClientIP", "IsRefresh", "ResolutionWidth"}) + .Filter(NotEq("SearchPhrase", "")) + .Aggregate({"WatchID", "ClientIP"}, + {Count("*", "c"), Sum("IsRefresh", "sum_refresh"), + Avg("ResolutionWidth", "avg_res_width")}) + .OrderBy({{"c", true}}, 10) + .Collect(); + + df.Display(); + } + + // SELECT WatchID, ClientIP, COUNT(*) AS c, SUM(IsRefresh), AVG(ResolutionWidth) FROM hits GROUP + // BY WatchID, ClientIP ORDER BY c DESC LIMIT 10; + void Query32() { + auto df = DataFrame::Select(columnar_file_path_, + {"WatchID", "ClientIP", "IsRefresh", "ResolutionWidth"}) + .Aggregate({"WatchID", "ClientIP"}, + {Count("*", "c"), Sum("IsRefresh", "sum_refresh"), + Avg("ResolutionWidth", "avg_res_width")}) + .OrderBy({{"c", true}}, 10) + .Collect(); + + df.Display(); + } + + // SELECT URL, COUNT(*) AS c FROM hits GROUP BY URL ORDER BY c DESC LIMIT 10; + void Query33() { + auto df = DataFrame::Select(columnar_file_path_, {"URL"}) + .Aggregate({"URL"}, {Count("*", "c")}) + .OrderBy({{"c", true}}, 10) + .Collect(); + df.Display(); + } + + // SELECT 1, URL, COUNT(*) AS c FROM hits GROUP BY 1, URL ORDER BY c DESC LIMIT 10; + void Query34() { + auto df = DataFrame::Select(columnar_file_path_, {"URL"}) + .Aggregate({"URL"}, {Count("*", "c")}) + .OrderBy({{"c", true}}, 10) + .Project({Literal("const_1", 1)}) + .Reoder({"const_1", "URL", "c"}) + .Collect(); + df.Display(); + } + + // SELECT ClientIP, ClientIP - 1, ClientIP - 2, ClientIP - 3, COUNT(*) AS c FROM hits GROUP BY + // ClientIP, ClientIP - 1, ClientIP - 2, ClientIP - 3 ORDER BY c DESC LIMIT 10; + void Query35() { + auto df = DataFrame::Select(columnar_file_path_, {"ClientIP"}) + .Aggregate({"ClientIP"}, {Count("*", "c")}) + .OrderBy({{"c", true}}, 10) + .Project({AddConst("ClientIP", -1, "ClientIP_minus_1"), + AddConst("ClientIP", -2, "ClientIP_minus_2"), + AddConst("ClientIP", -3, "ClientIP_minus_3")}) + .Reoder({"ClientIP", "ClientIP_minus_1", "ClientIP_minus_2", + "ClientIP_minus_3", "c"}) + .Collect(); + + df.Display(); + } + void RunQuery(int query_number) { switch (query_number) { case 0: Query00(); break; @@ -326,6 +467,14 @@ class Query { case 25: Query25(); break; case 26: Query26(); break; case 27: Query27(); break; + case 28: Query28(); break; + case 29: Query29(); break; + case 30: Query30(); break; + case 31: Query31(); break; + case 32: Query32(); break; + case 33: Query33(); break; + case 34: Query34(); break; + case 35: Query35(); break; default: THROW_RUNTIME_ERROR("Unknown query number: " + std::to_string(query_number)); } From 267e61b986db3e7b2220b7a7435b168ac0f5a448 Mon Sep 17 00:00:00 2001 From: Mikhail Fomichev Date: Mon, 11 May 2026 23:32:47 +0300 Subject: [PATCH 06/18] feat: queries 28, 36-39, 41 --- src/CMakeLists.txt | 1 + src/column/ColumnView.h | 28 ++++ src/execution/ExecutionApi.h | 3 +- src/execution/ExecutionLogic.h | 3 +- src/execution/OperatorsBase.cpp | 112 +++++++++------ src/execution/OperatorsBase.h | 4 +- src/execution/expressions/AggExpHelper.h | 18 ++- src/execution/expressions/FilterExpressions.h | 108 +++++++++++++- src/execution/expressions/FilterFunctions.h | 105 +++++++++++--- src/execution/expressions/ScalarExpressions.h | 135 +++++++++++++++++- src/execution/expressions/ScalarFunctions.h | 125 ++++++++++++++-- src/main.cpp | 135 +++++++++++++++++- 12 files changed, 685 insertions(+), 92 deletions(-) create mode 100644 src/column/ColumnView.h diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index e4baac1..1c32baf 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -50,6 +50,7 @@ add_executable(columnar-engine types/Timestamp.h execution/expressions/AggExpHelper.h types/String.h + column/ColumnView.h ) target_link_libraries(columnar-engine PRIVATE project_includes) diff --git a/src/column/ColumnView.h b/src/column/ColumnView.h new file mode 100644 index 0000000..20d1ed7 --- /dev/null +++ b/src/column/ColumnView.h @@ -0,0 +1,28 @@ +#pragma once + +#include + +template +struct FlatColumnView { + const ContainerType* container; + + explicit FlatColumnView(const ContainerType* c) : container(c) {} + + inline ReturnType operator[](size_t i) const { + return (*container)[i]; + } +}; + +template +struct ConstColumnView { + ReturnType value; + + explicit ConstColumnView(ReturnType val) : value(std::move(val)) {} + + inline ReturnType operator[](size_t) const { + return value; + } +}; + +template +using ViewVariant = std::variant, ConstColumnView>; \ No newline at end of file diff --git a/src/execution/ExecutionApi.h b/src/execution/ExecutionApi.h index 5424d9d..5863ece 100644 --- a/src/execution/ExecutionApi.h +++ b/src/execution/ExecutionApi.h @@ -93,11 +93,12 @@ class DataFrame { } DataFrame OrderBy(std::vector> order_by_columns, - std::optional limit = std::nullopt) { + std::optional limit = std::nullopt, std::optional offset = std::nullopt) { auto order_by_node = std::make_shared(); order_by_node->child = logical_plan_; order_by_node->order_by_columns = std::move(order_by_columns); order_by_node->limit = limit; + order_by_node->offset = offset; return DataFrame(order_by_node); } diff --git a/src/execution/ExecutionLogic.h b/src/execution/ExecutionLogic.h index f0e59e0..7e0fc20 100644 --- a/src/execution/ExecutionLogic.h +++ b/src/execution/ExecutionLogic.h @@ -64,6 +64,7 @@ struct OrderByNode : public PlanNode { std::shared_ptr child; std::vector> order_by_columns; std::optional limit; + std::optional offset; }; struct LimitNode : public PlanNode { @@ -253,7 +254,7 @@ inline PhysicalOperatorContext BuildPhysicalPlan( auto op = std::make_unique(std::move(child_context.root_operator), std::move(sort_columns), - std::move(order_by->limit)); + std::move(order_by->limit), std::move(order_by->offset)); return {std::move(op), std::move(child_context.schema)}; } diff --git a/src/execution/OperatorsBase.cpp b/src/execution/OperatorsBase.cpp index 5cebb1c..4ad3740 100644 --- a/src/execution/OperatorsBase.cpp +++ b/src/execution/OperatorsBase.cpp @@ -180,8 +180,19 @@ std::unique_ptr GroupByOperator::Run() { OrderByOperator::OrderByOperator(std::unique_ptr child, std::vector> sort_columns, - std::optional limit) - : child_(std::move(child)), sort_columns_(std::move(sort_columns)), limit_(limit) { + std::optional limit, std::optional offset) + : child_(std::move(child)), + sort_columns_(std::move(sort_columns)), + limit_(limit), + offset_(offset) { + total_limit_ = limit_; + if (total_limit_.has_value()) { + if (offset_.has_value()) { + total_limit_.value() += offset_.value(); + } + } else { + total_limit_ = offset_; + } } using CompareFunc = int (*)(const void*, uint32_t, uint32_t); @@ -205,6 +216,49 @@ int Compare(const void* raw_data, uint32_t a, uint32_t b) { return 0; } +std::unique_ptr OrderByOperator::Run() { + if (accumulated_) { + return nullptr; + } + + while (auto batch = child_->Run()) { + if (accum_builders_.empty()) { + for (size_t c = 0; c < batch->columns.size(); ++c) { + accum_builders_.push_back( + ColumnFactory::MakeColumnBuilder(batch->columns[c]->GetType())); + accum_types_.push_back(batch->columns[c]->GetType()); + } + accum_num_rows_ = 0; + } + + const std::shared_ptr>& sel = batch->selection_vector; + for (size_t c = 0; c < accum_builders_.size(); ++c) { + ExecutionHelper::CopySelection(*accum_builders_[c], *batch->columns[c], sel.get()); + } + accum_num_rows_ += batch->num_rows; + + if (total_limit_.has_value() && accum_num_rows_ > total_limit_.value() * 10) { + TrimCurBatch(false); + } + } + + if (accum_num_rows_ > 0) { + TrimCurBatch(true); + + auto final_batch = std::make_unique(); + final_batch->num_rows = accum_num_rows_; + for (auto& builder : accum_builders_) { + final_batch->columns.push_back(builder->Finish()); + } + + accumulated_ = true; + return final_batch; + } + + accumulated_ = true; + return nullptr; +} + void OrderByOperator::TrimCurBatch(bool is_final) { if (accum_num_rows_ == 0) { return; @@ -251,13 +305,20 @@ void OrderByOperator::TrimCurBatch(bool is_final) { return false; }; - if (limit_.has_value() && accum_num_rows_ > limit_.value()) { - uint32_t lim = limit_.value(); + if (total_limit_.has_value() && accum_num_rows_ > total_limit_.value()) { + uint32_t lim = total_limit_.value(); std::nth_element(indices.begin(), indices.begin() + lim, indices.end(), cmp); indices.resize(lim); } if (is_final) { std::sort(indices.begin(), indices.end(), cmp); + if (offset_.has_value()) { + if (offset_.value() < indices.size()) { + indices.erase(indices.begin(), indices.begin() + offset_.value()); + } else { + indices.clear(); + } + } } for (auto type : accum_types_) { @@ -269,49 +330,6 @@ void OrderByOperator::TrimCurBatch(bool is_final) { accum_num_rows_ = indices.size(); } -std::unique_ptr OrderByOperator::Run() { - if (accumulated_) { - return nullptr; - } - - while (auto batch = child_->Run()) { - if (accum_builders_.empty()) { - for (size_t c = 0; c < batch->columns.size(); ++c) { - accum_builders_.push_back( - ColumnFactory::MakeColumnBuilder(batch->columns[c]->GetType())); - accum_types_.push_back(batch->columns[c]->GetType()); - } - accum_num_rows_ = 0; - } - - const std::shared_ptr>& sel = batch->selection_vector; - for (size_t c = 0; c < accum_builders_.size(); ++c) { - ExecutionHelper::CopySelection(*accum_builders_[c], *batch->columns[c], sel.get()); - } - accum_num_rows_ += batch->num_rows; - - if (limit_.has_value() && accum_num_rows_ > limit_.value() * 10) { - TrimCurBatch(false); - } - } - - if (accum_num_rows_ > 0) { - TrimCurBatch(true); - - auto final_batch = std::make_unique(); - final_batch->num_rows = accum_num_rows_; - for (auto& builder : accum_builders_) { - final_batch->columns.push_back(builder->Finish()); - } - - accumulated_ = true; - return final_batch; - } - - accumulated_ = true; - return nullptr; -} - LimitOperator::LimitOperator(std::unique_ptr child, size_t limit) : child_(std::move(child)), limit_(limit) { } diff --git a/src/execution/OperatorsBase.h b/src/execution/OperatorsBase.h index 1218a4f..02ade27 100644 --- a/src/execution/OperatorsBase.h +++ b/src/execution/OperatorsBase.h @@ -88,7 +88,7 @@ class OrderByOperator : public Operator { public: OrderByOperator(std::unique_ptr child, std::vector> sort_columns, - std::optional limit); + std::optional limit, std::optional offset); std::unique_ptr Run() override; @@ -104,6 +104,8 @@ class OrderByOperator : public Operator { std::vector indices_; size_t current_idx_ = 0; std::optional limit_; + std::optional offset_; + std::optional total_limit_; bool accumulated_ = false; }; diff --git a/src/execution/expressions/AggExpHelper.h b/src/execution/expressions/AggExpHelper.h index fa0cb82..8c9fb8a 100644 --- a/src/execution/expressions/AggExpHelper.h +++ b/src/execution/expressions/AggExpHelper.h @@ -22,10 +22,24 @@ class AggExpHelper { } } + template + static void IterateViewColData(const RecordBatch& batch, const ViewT& view, Func&& func) { + if (!batch.selection_vector) { + for (size_t i = 0; i < batch.num_rows; ++i) { + func(view[i], i, i); + } + } else { + const auto& sel = *batch.selection_vector; + for (size_t i = 0; i < batch.num_rows; ++i) { + func(view[sel[i]], i, sel[i]); + } + } + } + template static auto DispatchNumeric(ColumnType type, Functor&& f) { #define HANDLE_TYPE(ENUM_VAL, STR_VAL, CLASS_TYPE) \ - case ColumnType::ENUM_VAL: return f.template operator()(ColumnType::ENUM_VAL); + case ColumnType::ENUM_VAL: return f.template operator()(ColumnType::ENUM_VAL); switch (type) { FOR_NUMERIC_COLUMN_TYPE(HANDLE_TYPE) @@ -37,7 +51,7 @@ class AggExpHelper { template static auto DispatchAll(ColumnType type, Functor&& f) { #define HANDLE_TYPE(ENUM_VAL, STR_VAL, CLASS_TYPE) \ - case ColumnType::ENUM_VAL: return f.template operator()(ColumnType::ENUM_VAL); + case ColumnType::ENUM_VAL: return f.template operator()(ColumnType::ENUM_VAL); switch (type) { FOR_EACH_COLUMN_TYPE(HANDLE_TYPE) diff --git a/src/execution/expressions/FilterExpressions.h b/src/execution/expressions/FilterExpressions.h index 35ebc25..6834fdd 100644 --- a/src/execution/expressions/FilterExpressions.h +++ b/src/execution/expressions/FilterExpressions.h @@ -14,8 +14,10 @@ class FilterExpression { explicit FilterExpression(std::string column_name) : column_name_(std::move(column_name)) { } - void CollectRequiredColumns(std::vector& required_columns) { - required_columns.push_back(column_name_); + virtual void CollectRequiredColumns(std::vector& required_columns) { + if (!column_name_.empty()) { + required_columns.push_back(column_name_); + } } virtual std::unique_ptr CreateFilterFunction(const Schema& child_schema) = 0; @@ -89,7 +91,60 @@ class GreaterFilterExpression : public FilterExpression { return AggExpHelper::DispatchAll( column_type, [&](ColumnType) -> std::unique_ptr { if constexpr (std::is_same_v) { - return std::make_unique>(ind, value_); + return std::make_unique>(ind, + value_); + } else { + THROW_NOT_IMPLEMENTED; + } + }); + } + +private: + ValueType value_; +}; + +template +class GreaterEqFilterExpression : public FilterExpression { +public: + GreaterEqFilterExpression(std::string column_name, ValueType value) + : FilterExpression(column_name), value_(std::move(value)) { + } + + std::unique_ptr CreateFilterFunction(const Schema& child_schema) override { + size_t ind = child_schema.GetColumnIndexByName(column_name_); + ColumnType column_type = child_schema.GetColumnTypeByName(column_name_); + + return AggExpHelper::DispatchAll( + column_type, [&](ColumnType) -> std::unique_ptr { + if constexpr (std::is_same_v) { + return std::make_unique>( + ind, value_); + } else { + THROW_NOT_IMPLEMENTED; + } + }); + } + +private: + ValueType value_; +}; + +template +class LessEqFilterExpression : public FilterExpression { +public: + LessEqFilterExpression(std::string column_name, ValueType value) + : FilterExpression(column_name), value_(std::move(value)) { + } + + std::unique_ptr CreateFilterFunction(const Schema& child_schema) override { + size_t ind = child_schema.GetColumnIndexByName(column_name_); + ColumnType column_type = child_schema.GetColumnTypeByName(column_name_); + + return AggExpHelper::DispatchAll( + column_type, [&](ColumnType) -> std::unique_ptr { + if constexpr (std::is_same_v) { + return std::make_unique>(ind, + value_); } else { THROW_NOT_IMPLEMENTED; } @@ -158,6 +213,27 @@ class NotLikeExpression : public FilterExpression { std::string value_; }; +class AndExpression : public FilterExpression { +public: + AndExpression(std::shared_ptr left, std::shared_ptr right) + : FilterExpression(""), left_(std::move(left)), right_(std::move(right)) { + } + + void CollectRequiredColumns(std::vector& required_columns) override { + left_->CollectRequiredColumns(required_columns); + right_->CollectRequiredColumns(required_columns); + } + + std::unique_ptr CreateFilterFunction(const Schema& child_schema) override { + return std::make_unique(left_->CreateFilterFunction(child_schema), + right_->CreateFilterFunction(child_schema)); + } + +private: + std::shared_ptr left_; + std::shared_ptr right_; +}; + template inline std::shared_ptr NotEq(const std::string& column_name, T target_val) { if constexpr (std::is_convertible_v) { @@ -182,12 +258,32 @@ template inline std::shared_ptr Greater(const std::string& column_name, T target_val) { if constexpr (std::is_convertible_v) { return std::make_shared>(column_name, - std::string(target_val)); + std::string(target_val)); } else { return std::make_shared>(column_name, std::move(target_val)); } } +template +inline std::shared_ptr GreaterEq(const std::string& column_name, T target_val) { + if constexpr (std::is_convertible_v) { + return std::make_shared>(column_name, + std::string(target_val)); + } else { + return std::make_shared>(column_name, std::move(target_val)); + } +} + +template +inline std::shared_ptr LessEq(const std::string& column_name, T target_val) { + if constexpr (std::is_convertible_v) { + return std::make_shared>(column_name, + std::string(target_val)); + } else { + return std::make_shared>(column_name, std::move(target_val)); + } +} + template inline std::shared_ptr Like(const std::string& column_name, T target_val) { if constexpr (std::is_convertible_v) { @@ -205,3 +301,7 @@ inline std::shared_ptr NotLike(const std::string& column_name, THROW_RUNTIME_ERROR("Like pattern must be a string"); } } + +inline std::shared_ptr And(std::shared_ptr left, std::shared_ptr right) { + return std::make_shared(std::move(left), std::move(right)); +} diff --git a/src/execution/expressions/FilterFunctions.h b/src/execution/expressions/FilterFunctions.h index 284a022..78b283d 100644 --- a/src/execution/expressions/FilterFunctions.h +++ b/src/execution/expressions/FilterFunctions.h @@ -20,12 +20,12 @@ class NotEqFilterFunction : public FilterFunction { std::vector selection_vector; selection_vector.reserve(batch.num_rows); - AggExpHelper::IterateColumnData( - batch, column_ind_, [&](const auto& value, size_t, size_t real_ind) { - if (value != target_val_) { - selection_vector.push_back(real_ind); - } - }); + AggExpHelper::IterateColumnData(batch, column_ind_, + [&](const auto& value, size_t, size_t real_ind) { + if (value != target_val_) { + selection_vector.push_back(real_ind); + } + }); return selection_vector; } @@ -45,12 +45,12 @@ class EqFilterFunction : public FilterFunction { std::vector selection_vector; selection_vector.reserve(batch.num_rows); - AggExpHelper::IterateColumnData( - batch, column_ind_, [&](const auto& value, size_t, size_t real_ind) { - if (value == target_val_) { - selection_vector.push_back(real_ind); - } - }); + AggExpHelper::IterateColumnData(batch, column_ind_, + [&](const auto& value, size_t, size_t real_ind) { + if (value == target_val_) { + selection_vector.push_back(real_ind); + } + }); return selection_vector; } @@ -70,12 +70,62 @@ class GreaterFilterFunction : public FilterFunction { std::vector selection_vector; selection_vector.reserve(batch.num_rows); - AggExpHelper::IterateColumnData( - batch, column_ind_, [&](const auto& value, size_t, size_t real_ind) { - if (value > target_val_) { - selection_vector.push_back(real_ind); - } - }); + AggExpHelper::IterateColumnData(batch, column_ind_, + [&](const auto& value, size_t, size_t real_ind) { + if (value > target_val_) { + selection_vector.push_back(real_ind); + } + }); + return selection_vector; + } + +private: + size_t column_ind_; + ValueType target_val_; +}; + +template +class GreaterEqFilterFunction : public FilterFunction { +public: + GreaterEqFilterFunction(size_t column_ind, ValueType target_val) + : column_ind_(column_ind), target_val_(std::move(target_val)) { + } + + std::vector Evaluate(const RecordBatch& batch) override { + std::vector selection_vector; + selection_vector.reserve(batch.num_rows); + + AggExpHelper::IterateColumnData(batch, column_ind_, + [&](const auto& value, size_t, size_t real_ind) { + if (value >= target_val_) { + selection_vector.push_back(real_ind); + } + }); + return selection_vector; + } + +private: + size_t column_ind_; + ValueType target_val_; +}; + +template +class LessEqFilterFunction : public FilterFunction { +public: + LessEqFilterFunction(size_t column_ind, ValueType target_val) + : column_ind_(column_ind), target_val_(std::move(target_val)) { + } + + std::vector Evaluate(const RecordBatch& batch) override { + std::vector selection_vector; + selection_vector.reserve(batch.num_rows); + + AggExpHelper::IterateColumnData(batch, column_ind_, + [&](const auto& value, size_t, size_t real_ind) { + if (value <= target_val_) { + selection_vector.push_back(real_ind); + } + }); return selection_vector; } @@ -134,4 +184,23 @@ class NotLikeFilterFunction : public FilterFunction { ValueType pattern_; }; +class AndFilterFunction : public FilterFunction { +public: + AndFilterFunction(std::unique_ptr left, std::unique_ptr right) + : left_(std::move(left)), right_(std::move(right)) { + } + + std::vector Evaluate(const RecordBatch& batch) override { + std::vector left_res = left_->Evaluate(batch); + std::vector right_res = right_->Evaluate(batch); + std::vector result; + std::set_intersection(left_res.begin(), left_res.end(), right_res.begin(), right_res.end(), + std::back_inserter(result)); + return result; + } + +private: + std::unique_ptr left_; + std::unique_ptr right_; +}; diff --git a/src/execution/expressions/ScalarExpressions.h b/src/execution/expressions/ScalarExpressions.h index c305d67..fca1069 100644 --- a/src/execution/expressions/ScalarExpressions.h +++ b/src/execution/expressions/ScalarExpressions.h @@ -16,7 +16,9 @@ class ScalarExpression { } virtual void CollectRequiredColumns(std::vector& required_columns) { - required_columns.push_back(column_name_); + if (!column_name_.empty()) { + required_columns.push_back(column_name_); + } } virtual std::string GetOutputName() const { @@ -99,6 +101,10 @@ class ConstantExpression : public ScalarExpression { } } + const T& GetValue() const { + return constant_; + } + private: T constant_; }; @@ -125,11 +131,10 @@ class AddConstantExpression : public ScalarExpression { if (column_type == ColumnType::STRING || column_type == ColumnType::CHAR || column_type == ColumnType::DATE || column_type == ColumnType::TIMESTAMP) { THROW_RUNTIME_ERROR("AddConst is not supported for non-numeric columns"); - } + } return AggExpHelper::DispatchNumeric( - column_type, - [&](ColumnType) -> std::unique_ptr { + column_type, [&](ColumnType) -> std::unique_ptr { child_schema.AddColumn(GetOutputName(), column_type); return std::make_unique>(ind, constant_); }); @@ -139,6 +144,101 @@ class AddConstantExpression : public ScalarExpression { T constant_; }; +class ColumnRefExpression : public ScalarExpression { +public: + explicit ColumnRefExpression(std::string column_name) + : ScalarExpression(std::move(column_name)) { + } + + std::unique_ptr CreateScalarFunction(Schema&) override { + THROW_RUNTIME_ERROR( + "ColumnRefExpression is a marker and should not be evaluated directly."); + } +}; + +template +class CaseWhenExpression : public ScalarExpression { +public: + CaseWhenExpression(std::string output_name, std::shared_ptr cond_expr, + std::shared_ptr true_expr, + std::shared_ptr false_expr, ColumnType return_type) + : ScalarExpression("", std::move(output_name)), + cond_expr_(std::move(cond_expr)), + true_expr_(std::move(true_expr)), + false_expr_(std::move(false_expr)), + return_type_(return_type) { + } + + void CollectRequiredColumns(std::vector& required_columns) override { + cond_expr_->CollectRequiredColumns(required_columns); + true_expr_->CollectRequiredColumns(required_columns); + false_expr_->CollectRequiredColumns(required_columns); + } + + std::unique_ptr CreateScalarFunction(Schema& child_schema) override { + auto cond_func = cond_expr_->CreateFilterFunction(child_schema); + + using ResolvedBranch = std::variant; + + auto resolve_branch = [&](const std::shared_ptr& expr) -> ResolvedBranch { + if (auto col_ref = std::dynamic_pointer_cast(expr)) { + return child_schema.GetColumnIndexByName(col_ref->GetOutputName()); + } else if (auto const_expr = + std::dynamic_pointer_cast>(expr)) { + return const_expr->GetValue(); + } + THROW_RUNTIME_ERROR("CASE WHEN branch must be ColRef or Literal"); + }; + + ResolvedBranch true_resolved = resolve_branch(true_expr_); + ResolvedBranch false_resolved = resolve_branch(false_expr_); + + child_schema.AddColumn(GetOutputName(), return_type_); + + return AggExpHelper::DispatchAll( + return_type_, [&](ColumnType) -> std::unique_ptr { + using ContainerType = ColumnClass::ContainerType; + using RetT = std::decay_t()[0])>; + if constexpr (std::is_constructible_v || + std::is_convertible_v) { + return std::make_unique>( + std::move(cond_func), true_resolved, false_resolved, return_type_); + } else { + THROW_RUNTIME_ERROR("Unreachable: type mismatch between column and constant"); + } + }); + } + +private: + std::shared_ptr cond_expr_; + std::shared_ptr true_expr_; + std::shared_ptr false_expr_; + ColumnType return_type_; +}; + +class RegexpReplaceExpression : public ScalarExpression { +public: + RegexpReplaceExpression(std::string column_name, std::string pattern, std::string replacement, std::string output_name = "") + : ScalarExpression(std::move(column_name), std::move(output_name)), + pattern_(std::move(pattern)), replacement_(std::move(replacement)) {} + + std::unique_ptr CreateScalarFunction(Schema& child_schema) override { + size_t ind = child_schema.GetColumnIndexByName(column_name_); + ColumnType type = child_schema.GetColumnTypeByName(column_name_); + + if (type != ColumnType::STRING) [[unlikely]] { + THROW_RUNTIME_ERROR("REGEXP_REPLACE requires STRING column"); + } + + child_schema.AddColumn(GetOutputName(), ColumnType::STRING); + return std::make_unique(ind, pattern_, replacement_); + } + +private: + std::string pattern_; + std::string replacement_; +}; + inline std::shared_ptr ExtractMinute(std::string column_name, std::string output_name = "") { return std::make_shared(std::move(column_name), @@ -156,6 +256,29 @@ inline std::shared_ptr Literal(std::string output_name, T valu } template -inline std::shared_ptr AddConst(std::string column_name, T value, std::string output_name = "") { - return std::make_shared>(std::move(column_name), std::move(value), std::move(output_name)); +inline std::shared_ptr AddConst(std::string column_name, T value, + std::string output_name = "") { + return std::make_shared>(std::move(column_name), std::move(value), + std::move(output_name)); +} + +inline std::shared_ptr ColRef(std::string column_name) { + return std::make_shared(std::move(column_name)); +} + +template +inline std::shared_ptr CaseWhen(std::string output_name, + std::shared_ptr cond, + std::shared_ptr true_expr, + std::shared_ptr false_expr, + ColumnType return_type) { + return std::make_shared>(std::move(output_name), + std::move(cond), std::move(true_expr), + std::move(false_expr), return_type); +} + +inline std::shared_ptr RegexpReplace( + std::string column_name, std::string pattern, std::string replacement, std::string output_name = "") { + return std::make_shared( + std::move(column_name), std::move(pattern), std::move(replacement), std::move(output_name)); } diff --git a/src/execution/expressions/ScalarFunctions.h b/src/execution/expressions/ScalarFunctions.h index 0813358..936b6c3 100644 --- a/src/execution/expressions/ScalarFunctions.h +++ b/src/execution/expressions/ScalarFunctions.h @@ -2,10 +2,12 @@ #include "execution/OperatorsBase.h" #include "execution/expressions/AggExpHelper.h" +#include "column/ColumnView.h" #include "column/TemporalColumn.h" #include "column/NumericColumn.h" #include "types/String.h" +#include #include class ScalarFunction { @@ -16,15 +18,17 @@ class ScalarFunction { class ExtractMinuteFunction : public ScalarFunction { public: - explicit ExtractMinuteFunction(size_t col_ind) : col_ind_(col_ind) {} + explicit ExtractMinuteFunction(size_t col_ind) : col_ind_(col_ind) { + } std::shared_ptr Evaluate(const RecordBatch& batch) override { size_t physical_size = batch.columns[0]->Size(); std::vector result_data(physical_size); - AggExpHelper::IterateColumnData(batch, col_ind_, [&](const auto& value, size_t, size_t real_ind) { - result_data[real_ind] = Timestamp::ExtractMinute(value); - }); + AggExpHelper::IterateColumnData( + batch, col_ind_, [&](const auto& value, size_t, size_t real_ind) { + result_data[real_ind] = Timestamp::ExtractMinute(value); + }); return std::make_shared(std::move(result_data)); } @@ -35,15 +39,17 @@ class ExtractMinuteFunction : public ScalarFunction { class LengthFunction : public ScalarFunction { public: - explicit LengthFunction(size_t col_ind) : col_ind_(col_ind) {} + explicit LengthFunction(size_t col_ind) : col_ind_(col_ind) { + } std::shared_ptr Evaluate(const RecordBatch& batch) override { size_t physical_size = batch.columns[0]->Size(); std::vector result_data(physical_size); - AggExpHelper::IterateColumnData(batch, col_ind_, [&](const auto& value, size_t, size_t real_ind) { - result_data[real_ind] = String::Length(value); - }); + AggExpHelper::IterateColumnData( + batch, col_ind_, [&](const auto& value, size_t, size_t real_ind) { + result_data[real_ind] = String::Length(value); + }); return std::make_shared(std::move(result_data)); } @@ -59,10 +65,11 @@ class ConstantFunction : public ScalarFunction { public: ConstantFunction(ColumnType type, ValueType constant) - : type_(type), constant_(std::move(constant)) {} + : type_(type), constant_(std::move(constant)) { + } std::shared_ptr Evaluate(const RecordBatch& batch) override { - size_t physical_size = batch.num_rows; + size_t physical_size = batch.columns.empty() ? batch.num_rows : batch.columns[0]->Size(); auto builder = ColumnFactory::MakeColumnBuilder(type_); auto* typed_builder = static_cast(builder.get()); @@ -85,7 +92,8 @@ class AddConstantFunction : public ScalarFunction { public: AddConstantFunction(size_t col_ind, ValueType constant) - : col_ind_(col_ind), constant_(std::move(constant)) {} + : col_ind_(col_ind), constant_(std::move(constant)) { + } std::shared_ptr Evaluate(const RecordBatch& batch) override { size_t physical_size = batch.columns.empty() ? batch.num_rows : batch.columns[0]->Size(); @@ -103,3 +111,98 @@ class AddConstantFunction : public ScalarFunction { size_t col_ind_; ValueType constant_; }; + +template +class CaseWhenFunction : public ScalarFunction { + using ContainerType = ColumnClass::ContainerType; + using ReturnType = std::decay_t()[0])>; + using BuilderType = BuilderTypeTrait::Type; + + using ResolvedBranch = + std::variant; // либо индекс колонки, либо константа + using ViewType = ViewVariant; + +public: + CaseWhenFunction(std::unique_ptr cond_func, ResolvedBranch true_branch, + ResolvedBranch false_branch, ColumnType type) + : cond_func_(std::move(cond_func)), + true_branch_(std::move(true_branch)), + false_branch_(std::move(false_branch)), + type_(type) { + } + + std::shared_ptr Evaluate(const RecordBatch& batch) override { + std::vector true_indices = cond_func_->Evaluate(batch); + + auto build_view = [&](const ResolvedBranch& b) -> ViewType { + if (std::holds_alternative(b)) { + size_t col_idx = std::get(b); + auto* col = static_cast(batch.columns[col_idx].get()); + return FlatColumnView(&col->GetData()); + } else { + return ConstColumnView(std::get(b)); + } + }; + + ViewType true_view = build_view(true_branch_); + ViewType false_view = build_view(false_branch_); + + auto builder = ColumnFactory::MakeColumnBuilder(type_); + auto* typed_builder = static_cast(builder.get()); + + std::visit( + [&](const auto& true_v, const auto& false_v) { + size_t physical_size = + batch.columns.empty() ? batch.num_rows : batch.columns[0]->Size(); + size_t cur_ind = 0; + for (size_t real_ind = 0; real_ind < physical_size; ++real_ind) { + if (cur_ind < true_indices.size() && true_indices[cur_ind] == real_ind) { + typed_builder->AddValue(true_v[real_ind]); + ++cur_ind; + } else { + typed_builder->AddValue(false_v[real_ind]); + } + } + }, + true_view, false_view); + + return builder->Finish(); + } + +private: + std::unique_ptr cond_func_; + ResolvedBranch true_branch_; + ResolvedBranch false_branch_; + ColumnType type_; +}; + +class RegexpReplaceFunction : public ScalarFunction { +public: + RegexpReplaceFunction(size_t col_ind, const std::string& pattern, + const std::string& replacement) + : col_ind_(col_ind), regex_(pattern), replacement_(replacement) { + } + + std::shared_ptr Evaluate(const RecordBatch& batch) override { + size_t physical_size = batch.columns.empty() ? batch.num_rows : batch.columns[0]->Size(); + std::vector temp_res(physical_size); + + AggExpHelper::IterateColumnData( + batch, col_ind_, [&](const auto& value, size_t, size_t real_ind) { + std::string str(value); + temp_res[real_ind] = + std::regex_replace(str, regex_, replacement_, std::regex_constants::format_sed); + }); + + StringColumnBuilder builder; + for (size_t i = 0; i < physical_size; ++i) { + builder.AddValue(temp_res[i]); + } + return builder.Finish(); + } + +private: + size_t col_ind_; + std::regex regex_; + std::string replacement_; +}; diff --git a/src/main.cpp b/src/main.cpp index 321e526..7090033 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -301,7 +301,18 @@ class Query { // AVG(length(Referer)) AS l, COUNT(*) AS c, MIN(Referer) FROM hits WHERE Referer <> '' GROUP BY // k HAVING COUNT(*) > 100000 ORDER BY l DESC LIMIT 25; void Query28() { - THROW_NOT_IMPLEMENTED; + auto df = + DataFrame::Select(columnar_file_path_, {"Referer"}) + .Filter(NotEq("Referer", "")) + .Project({RegexpReplace("Referer", "^https?://(?:www\\.)?([^/]+)/.*$", "\\1", "k"), + Length("Referer", "length_ref")}) + .Aggregate({"k"}, + {Avg("length_ref", "l"), Count("*", "c"), Min("Referer", "min_ref")}) + .Filter(Greater("c", 100000)) + .OrderBy({{"l", true}}, 25) + .Collect(); + + df.Display(); } // WARNING: DataFrame::Display output does not affect that query @@ -437,6 +448,121 @@ class Query { df.Display(); } + // SELECT URL, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= + // '2013-07-01' AND EventDate <= '2013-07-31' AND DontCountHits = 0 AND IsRefresh = 0 AND URL <> + // '' GROUP BY URL ORDER BY PageViews DESC LIMIT 10; + void Query36() { + auto df = DataFrame::Select(columnar_file_path_, {"URL"}) + .Filter(NotEq("URL", "")) + .Filter(Eq("CounterID", 62)) + .Filter(GreaterEq("EventDate", Date::Parse("2013-07-01"))) + .Filter(LessEq("EventDate", Date::Parse("2013-07-31"))) + .Filter(Eq("DontCountHits", 0)) + .Filter(Eq("IsRefresh", 0)) + .Aggregate({"URL"}, {Count("*", "PageViews")}) + .OrderBy({{"PageViews", true}}, 10) + .Collect(); + + df.Display(); + } + + // SELECT Title, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= + // '2013-07-01' AND EventDate <= '2013-07-31' AND DontCountHits = 0 AND IsRefresh = 0 AND Title + // <> '' GROUP BY Title ORDER BY PageViews DESC LIMIT 10; + void Query37() { + auto df = DataFrame::Select(columnar_file_path_, {"Title"}) + .Filter(NotEq("Title", "")) + .Filter(Eq("CounterID", 62)) + .Filter(GreaterEq("EventDate", Date::Parse("2013-07-01"))) + .Filter(LessEq("EventDate", Date::Parse("2013-07-31"))) + .Filter(Eq("DontCountHits", 0)) + .Filter(Eq("IsRefresh", 0)) + .Aggregate({"Title"}, {Count("*", "PageViews")}) + .OrderBy({{"PageViews", true}}, 10) + .Collect(); + + df.Display(); + } + + // SELECT URL, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= + // '2013-07-01' AND EventDate <= '2013-07-31' AND IsRefresh = 0 AND IsLink <> 0 AND IsDownload = + // 0 GROUP BY URL ORDER BY PageViews DESC LIMIT 10 OFFSET 1000; + void Query38() { + auto df = DataFrame::Select(columnar_file_path_, {"URL"}) + .Filter(NotEq("IsLink", 0)) + .Filter(Eq("IsDownload", 0)) + .Filter(Eq("CounterID", 62)) + .Filter(GreaterEq("EventDate", Date::Parse("2013-07-01"))) + .Filter(LessEq("EventDate", Date::Parse("2013-07-31"))) + .Filter(Eq("IsRefresh", 0)) + .Aggregate({"URL"}, {Count("*", "PageViews")}) + .OrderBy({{"PageViews", true}}, 10, 1000) + .Collect(); + + df.Display(); + } + + // SELECT TraficSourceID, SearchEngineID, AdvEngineID, CASE WHEN (SearchEngineID = 0 AND + // AdvEngineID = 0) THEN Referer ELSE '' END AS Src, URL AS Dst, COUNT(*) AS PageViews FROM hits + // WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND + // IsRefresh = 0 GROUP BY TraficSourceID, SearchEngineID, AdvEngineID, Src, Dst ORDER BY + // PageViews DESC LIMIT 10 OFFSET 1000; + void Query39() { + auto df = + DataFrame::Select(columnar_file_path_, {}) + .Filter(Eq("CounterID", 62)) + .Filter(GreaterEq("EventDate", Date::Parse("2013-07-01"))) + .Filter(LessEq("EventDate", Date::Parse("2013-07-31"))) + .Filter(Eq("IsRefresh", 0)) + .Project({CaseWhen( + "Src", And(Eq("SearchEngineID", 0), Eq("AdvEngineID", 0)), + ColRef("Referer"), Literal("dummy_name", ""), ColumnType::STRING)}) + .Aggregate({"TraficSourceID", "SearchEngineID", "AdvEngineID", "Src", "URL"}, + {Count("*", "PageViews")}) + .OrderBy({{"PageViews", true}}, 10, 1000) + .Collect(); + + df.Display(); + } + + // TODO: + // SELECT URLHash, EventDate, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate + // >= '2013-07-01' AND EventDate <= '2013-07-31' AND IsRefresh = 0 AND TraficSourceID IN (-1, 6) + // AND RefererHash = 3594120000172545465 GROUP BY URLHash, EventDate ORDER BY PageViews DESC + // LIMIT 10 OFFSET 100; + void Query40() const { + THROW_NOT_IMPLEMENTED; + } + + // SELECT WindowClientWidth, WindowClientHeight, COUNT(*) AS PageViews FROM hits WHERE CounterID + // = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND IsRefresh = 0 AND + // DontCountHits = 0 AND URLHash = 2868770270353813622 GROUP BY WindowClientWidth, + // WindowClientHeight ORDER BY PageViews DESC LIMIT 10 OFFSET 10000; + void Query41() const { + auto df = + DataFrame::Select(columnar_file_path_, {}) + .Filter(Eq("URLHash", 2868770270353813622)) + .Filter(Eq("CounterID", 62)) + .Filter(GreaterEq("EventDate", Date::Parse("2013-07-01"))) + .Filter(LessEq("EventDate", Date::Parse("2013-07-31"))) + .Filter(Eq("IsRefresh", 0)) + .Filter(Eq("DontCountHits", 0)) + .Aggregate({"WindowClientWidth", "WindowClientHeight"}, {Count("*", "PageViews")}) + .OrderBy({{"PageViews", true}}, 10, 10000) + .Collect(); + + df.Display(); + } + + // TODO: + // SELECT DATE_TRUNC('minute', EventTime) AS M, COUNT(*) AS PageViews FROM hits WHERE CounterID + // = 62 AND EventDate >= '2013-07-14' AND EventDate <= '2013-07-15' AND IsRefresh = 0 AND + // DontCountHits = 0 GROUP BY DATE_TRUNC('minute', EventTime) ORDER BY DATE_TRUNC('minute', + // EventTime) LIMIT 10 OFFSET 1000; + void Query42() const { + THROW_NOT_IMPLEMENTED; + } + void RunQuery(int query_number) { switch (query_number) { case 0: Query00(); break; @@ -475,6 +601,13 @@ class Query { case 33: Query33(); break; case 34: Query34(); break; case 35: Query35(); break; + case 36: Query36(); break; + case 37: Query37(); break; + case 38: Query38(); break; + case 39: Query39(); break; + case 40: Query40(); break; + case 41: Query41(); break; + case 42: Query42(); break; default: THROW_RUNTIME_ERROR("Unknown query number: " + std::to_string(query_number)); } From b92a604e32965e09f64e7441f82ddc10efa73eb0 Mon Sep 17 00:00:00 2001 From: Mikhail Fomichev Date: Tue, 12 May 2026 16:18:23 +0300 Subject: [PATCH 07/18] feat: queries 40, 42 [ALL QUERIES DONE, LEGENDARY MOMENT] --- src/CMakeLists.txt | 1 + src/execution/expressions/FilterExpressions.h | 25 ++++++++++ src/execution/expressions/FilterFunctions.h | 23 +++++++++ src/execution/expressions/ScalarExpressions.h | 49 +++++++++++++++++-- src/execution/expressions/ScalarFunctions.h | 23 +++++++++ src/main.cpp | 28 +++++++++-- src/types/Date.h | 33 +++++++++++++ src/types/TimeUnit.h | 42 ++++++++++++++++ src/types/Timestamp.h | 36 +++++++++++++- 9 files changed, 250 insertions(+), 10 deletions(-) create mode 100644 src/types/TimeUnit.h diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 1c32baf..cfd56ab 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -51,6 +51,7 @@ add_executable(columnar-engine execution/expressions/AggExpHelper.h types/String.h column/ColumnView.h + types/TimeUnit.h ) target_link_libraries(columnar-engine PRIVATE project_includes) diff --git a/src/execution/expressions/FilterExpressions.h b/src/execution/expressions/FilterExpressions.h index 6834fdd..9d1454a 100644 --- a/src/execution/expressions/FilterExpressions.h +++ b/src/execution/expressions/FilterExpressions.h @@ -234,6 +234,27 @@ class AndExpression : public FilterExpression { std::shared_ptr right_; }; +class OrExpression : public FilterExpression { +public: + OrExpression(std::shared_ptr left, std::shared_ptr right) + : FilterExpression(""), left_(std::move(left)), right_(std::move(right)) { + } + + void CollectRequiredColumns(std::vector& required_columns) override { + left_->CollectRequiredColumns(required_columns); + right_->CollectRequiredColumns(required_columns); + } + + std::unique_ptr CreateFilterFunction(const Schema& child_schema) override { + return std::make_unique(left_->CreateFilterFunction(child_schema), + right_->CreateFilterFunction(child_schema)); + } + +private: + std::shared_ptr left_; + std::shared_ptr right_; +}; + template inline std::shared_ptr NotEq(const std::string& column_name, T target_val) { if constexpr (std::is_convertible_v) { @@ -305,3 +326,7 @@ inline std::shared_ptr NotLike(const std::string& column_name, inline std::shared_ptr And(std::shared_ptr left, std::shared_ptr right) { return std::make_shared(std::move(left), std::move(right)); } + +inline std::shared_ptr Or(std::shared_ptr left, std::shared_ptr right) { + return std::make_shared(std::move(left), std::move(right)); +} diff --git a/src/execution/expressions/FilterFunctions.h b/src/execution/expressions/FilterFunctions.h index 78b283d..c07fbf5 100644 --- a/src/execution/expressions/FilterFunctions.h +++ b/src/execution/expressions/FilterFunctions.h @@ -195,6 +195,7 @@ class AndFilterFunction : public FilterFunction { std::vector right_res = right_->Evaluate(batch); std::vector result; + result.reserve(left_res.size() + right_res.size()); std::set_intersection(left_res.begin(), left_res.end(), right_res.begin(), right_res.end(), std::back_inserter(result)); return result; @@ -204,3 +205,25 @@ class AndFilterFunction : public FilterFunction { std::unique_ptr left_; std::unique_ptr right_; }; + +class OrFilterFunction : public FilterFunction { +public: + OrFilterFunction(std::unique_ptr left, std::unique_ptr right) + : left_(std::move(left)), right_(std::move(right)) { + } + + std::vector Evaluate(const RecordBatch& batch) override { + std::vector left_res = left_->Evaluate(batch); + std::vector right_res = right_->Evaluate(batch); + + std::vector result; + result.reserve(left_res.size() + right_res.size()); + std::set_union(left_res.begin(), left_res.end(), right_res.begin(), right_res.end(), + std::back_inserter(result)); + return result; + } + +private: + std::unique_ptr left_; + std::unique_ptr right_; +}; diff --git a/src/execution/expressions/ScalarExpressions.h b/src/execution/expressions/ScalarExpressions.h index fca1069..6ac74da 100644 --- a/src/execution/expressions/ScalarExpressions.h +++ b/src/execution/expressions/ScalarExpressions.h @@ -1,7 +1,8 @@ #pragma once -#include "execution/expressions/ScalarFunctions.h" #include "column/Schema.h" +#include "execution/expressions/ScalarFunctions.h" +#include "types/TimeUnit.h" #include #include @@ -32,6 +33,7 @@ class ScalarExpression { std::string output_name_; }; +// // TODO: remove ExtractMinute, make Extract with uniform interface (like TruncateFunction) class ExtractMinuteExpression : public ScalarExpression { public: explicit ExtractMinuteExpression(std::string column_name, std::string output_name = "") @@ -55,6 +57,33 @@ class ExtractMinuteExpression : public ScalarExpression { } }; +class TimeTruncExpression : public ScalarExpression { +public: + TimeTruncExpression(std::string column_name, std::string date_part, std::string output_name = "") + : ScalarExpression(std::move(column_name), std::move(output_name)), + date_part_str_(std::move(date_part)) {} + + std::string GetOutputName() const override { + return output_name_.empty() ? "date_trunc_" + column_name_ : output_name_; + } + + std::unique_ptr CreateScalarFunction(Schema& child_schema) override { + size_t ind = child_schema.GetColumnIndexByName(column_name_); + ColumnType type = child_schema.GetColumnTypeByName(column_name_); + + if (type != ColumnType::TIMESTAMP && type != ColumnType::DATE) [[unlikely]] { + THROW_RUNTIME_ERROR("DATE_TRUNC requires TIMESTAMP or DATE column"); + } + + TimeUnitType part = TimeUnit::ParseTimeUnit(date_part_str_); + child_schema.AddColumn(GetOutputName(), type); + return std::make_unique(ind, part); + } + +private: + std::string date_part_str_; +}; + class LengthExpression : public ScalarExpression { public: explicit LengthExpression(std::string column_name, std::string output_name = "") @@ -218,9 +247,12 @@ class CaseWhenExpression : public ScalarExpression { class RegexpReplaceExpression : public ScalarExpression { public: - RegexpReplaceExpression(std::string column_name, std::string pattern, std::string replacement, std::string output_name = "") + RegexpReplaceExpression(std::string column_name, std::string pattern, std::string replacement, + std::string output_name = "") : ScalarExpression(std::move(column_name), std::move(output_name)), - pattern_(std::move(pattern)), replacement_(std::move(replacement)) {} + pattern_(std::move(pattern)), + replacement_(std::move(replacement)) { + } std::unique_ptr CreateScalarFunction(Schema& child_schema) override { size_t ind = child_schema.GetColumnIndexByName(column_name_); @@ -245,6 +277,12 @@ inline std::shared_ptr ExtractMinute(std::string column_name, std::move(output_name)); } +inline std::shared_ptr TimeTrunc(std::string column_name, std::string date_part, + std::string output_name = "") { + return std::make_shared(std::move(column_name), std::move(date_part), + std::move(output_name)); +} + inline std::shared_ptr Length(std::string column_name, std::string output_name = "") { return std::make_shared(std::move(column_name), std::move(output_name)); @@ -277,8 +315,9 @@ inline std::shared_ptr CaseWhen(std::string output_name, std::move(false_expr), return_type); } -inline std::shared_ptr RegexpReplace( - std::string column_name, std::string pattern, std::string replacement, std::string output_name = "") { +inline std::shared_ptr RegexpReplace(std::string column_name, std::string pattern, + std::string replacement, + std::string output_name = "") { return std::make_shared( std::move(column_name), std::move(pattern), std::move(replacement), std::move(output_name)); } diff --git a/src/execution/expressions/ScalarFunctions.h b/src/execution/expressions/ScalarFunctions.h index 936b6c3..deae3db 100644 --- a/src/execution/expressions/ScalarFunctions.h +++ b/src/execution/expressions/ScalarFunctions.h @@ -16,6 +16,7 @@ class ScalarFunction { virtual std::shared_ptr Evaluate(const RecordBatch& batch) = 0; }; +// TODO: remove ExtractMinute, make Extract with uniform interface (like TruncateFunction) class ExtractMinuteFunction : public ScalarFunction { public: explicit ExtractMinuteFunction(size_t col_ind) : col_ind_(col_ind) { @@ -37,6 +38,28 @@ class ExtractMinuteFunction : public ScalarFunction { size_t col_ind_; }; +class TimeTruncFunction : public ScalarFunction { +public: + TimeTruncFunction(size_t col_ind, TimeUnitType part) : col_ind_(col_ind), part_(part) { + } + + std::shared_ptr Evaluate(const RecordBatch& batch) override { + size_t physical_size = batch.columns[0]->Size(); + std::vector result_data(physical_size); + + AggExpHelper::IterateColumnData( + batch, col_ind_, [&](const auto& value, size_t, size_t real_ind) { + result_data[real_ind] = Timestamp::Truncate(value, part_); + }); + + return std::make_shared(std::move(result_data)); + } + +private: + size_t col_ind_; + TimeUnitType part_; +}; + class LengthFunction : public ScalarFunction { public: explicit LengthFunction(size_t col_ind) : col_ind_(col_ind) { diff --git a/src/main.cpp b/src/main.cpp index 7090033..a84ea39 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -525,13 +525,23 @@ class Query { df.Display(); } - // TODO: // SELECT URLHash, EventDate, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate // >= '2013-07-01' AND EventDate <= '2013-07-31' AND IsRefresh = 0 AND TraficSourceID IN (-1, 6) // AND RefererHash = 3594120000172545465 GROUP BY URLHash, EventDate ORDER BY PageViews DESC // LIMIT 10 OFFSET 100; void Query40() const { - THROW_NOT_IMPLEMENTED; + auto df = + DataFrame::Select(columnar_file_path_, {"Title"}) + .Filter(Eq("RefererHash", 3594120000172545465)) + .Filter(Eq("CounterID", 62)) + .Filter(GreaterEq("EventDate", Date::Parse("2013-07-01"))) + .Filter(LessEq("EventDate", Date::Parse("2013-07-31"))) + .Filter(Eq("IsRefresh", 0)) + .Filter(Or(Eq("TraficSourceID", -1), Eq("TraficSourceID", 6))) + .Aggregate({"URLHash", "EventDate"}, {Count("*", "PageViews")}) + .OrderBy({{"PageViews", true}}, 10, 100) + .Collect(); + df.Display(); } // SELECT WindowClientWidth, WindowClientHeight, COUNT(*) AS PageViews FROM hits WHERE CounterID @@ -554,13 +564,23 @@ class Query { df.Display(); } - // TODO: // SELECT DATE_TRUNC('minute', EventTime) AS M, COUNT(*) AS PageViews FROM hits WHERE CounterID // = 62 AND EventDate >= '2013-07-14' AND EventDate <= '2013-07-15' AND IsRefresh = 0 AND // DontCountHits = 0 GROUP BY DATE_TRUNC('minute', EventTime) ORDER BY DATE_TRUNC('minute', // EventTime) LIMIT 10 OFFSET 1000; void Query42() const { - THROW_NOT_IMPLEMENTED; + auto df = DataFrame::Select(columnar_file_path_, {}) + .Filter(Eq("CounterID", 62)) + .Filter(GreaterEq("EventDate", Date::Parse("2013-07-01"))) + .Filter(LessEq("EventDate", Date::Parse("2013-07-31"))) + .Filter(Eq("IsRefresh", 0)) + .Filter(Eq("DontCountHits", 0)) + .Project({TimeTrunc("EventTime", "minute", "M")}) + .Aggregate({"M"}, {Count("*", "PageViews")}) + .OrderBy({{"M", false}}, 10, 1000) + .Collect(); + + df.Display(); } void RunQuery(int query_number) { diff --git a/src/types/Date.h b/src/types/Date.h index 674f90a..ecf8232 100644 --- a/src/types/Date.h +++ b/src/types/Date.h @@ -1,5 +1,6 @@ #pragma once +#include "TimeUnit.h" #include "utils/Assert.h" #include @@ -50,4 +51,36 @@ struct Date { write_digit(d, 8, 2); return res; } + + static int32_t Truncate(int32_t total_days, TimeUnitType unit) { + if (unit == TimeUnitType::DAY || unit == TimeUnitType::HOUR || unit == TimeUnitType::MINUTE || + unit == TimeUnitType::SECOND) { + return total_days; + } + + int32_t days = total_days + 719468; + const int era = (days >= 0 ? days : days - 146096) / 146097; + const unsigned doe = static_cast(days - era * 146097); + const unsigned yoe = (doe - doe / 1460 + doe / 36524 - doe / 146096) / 365; + const int y = static_cast(yoe) + era * 400; + const unsigned doy = doe - (365 * yoe + yoe / 4 - yoe / 100); + const unsigned mp = (5 * doy + 2) / 153; + const unsigned d = doy - (153 * mp + 2) / 5 + 1; + const unsigned m = mp + (mp < 10 ? 3 : -9); + int year = y + (m <= 2); + + switch (unit) { + case TimeUnitType::MONTH: { + return total_days - static_cast(d) + 1; + } + case TimeUnitType::YEAR: { + const int e = (year >= 0 ? year : year - 399) / 400; + const unsigned ye = static_cast(year - e * 400); + const unsigned de = ye * 365 + ye / 4 - ye / 100 + 306; + return e * 146097 + static_cast(de) - 719468; + } + default: + return total_days; + } + } }; diff --git a/src/types/TimeUnit.h b/src/types/TimeUnit.h new file mode 100644 index 0000000..b377d20 --- /dev/null +++ b/src/types/TimeUnit.h @@ -0,0 +1,42 @@ +#pragma once + +#include "utils/Assert.h" + +#include +#include + +enum class TimeUnitType { + YEAR, + MONTH, + DAY, + HOUR, + MINUTE, + SECOND +}; + + +class TimeUnit { +public: + static TimeUnitType ParseTimeUnit(std::string_view unit_str) { + if (unit_str == "year") { + return TimeUnitType::YEAR; + } + if (unit_str == "month") { + return TimeUnitType::MONTH; + } + if (unit_str == "day") { + return TimeUnitType::DAY; + } + if (unit_str == "hour") { + return TimeUnitType::HOUR; + } + if (unit_str == "minute") { + return TimeUnitType::MINUTE; + } + if (unit_str == "second") { + return TimeUnitType::SECOND; + } + + THROW_RUNTIME_ERROR("Unknown time unit: " + std::string(unit_str)); + } +}; \ No newline at end of file diff --git a/src/types/Timestamp.h b/src/types/Timestamp.h index ae4ab6f..25784e5 100644 --- a/src/types/Timestamp.h +++ b/src/types/Timestamp.h @@ -1,8 +1,8 @@ #pragma once #include "Date.h" +#include "TimeUnit.h" -#include #include #include @@ -59,6 +59,7 @@ struct Timestamp { return res; } + // TODO: make uniform interface called Extract static int32_t ExtractMinute(int64_t total_seconds) { int64_t seconds_in_day = total_seconds % 86400; if (seconds_in_day < 0) { @@ -66,4 +67,37 @@ struct Timestamp { } return (seconds_in_day % 3600) / 60; } + + static int64_t Truncate(int64_t total_seconds, TimeUnitType part) { + int64_t seconds_in_day = total_seconds % 86400; + if (seconds_in_day < 0) { + seconds_in_day += 86400; + } + + switch (part) { + case TimeUnitType::MINUTE: { + int64_t extra_seconds = seconds_in_day % 60; + return total_seconds - extra_seconds; + } + case TimeUnitType::HOUR: { + int64_t extra_seconds = seconds_in_day % 3600; + return total_seconds - extra_seconds; + } + case TimeUnitType::DAY: { + return total_seconds - seconds_in_day; + } + case TimeUnitType::MONTH: { + int32_t days = total_seconds / 86400; + int32_t trunc = Date::Truncate(days, TimeUnitType::MONTH); + return static_cast(trunc) * 86400; + } + case TimeUnitType::YEAR: { + int32_t days = total_seconds / 86400; + int32_t trunc = Date::Truncate(days, TimeUnitType::YEAR); + return static_cast(trunc) * 86400; + } + default: break; + } + return total_seconds; + } }; From 9705d07789ebca3349d7b6d32724d2acf1be3944 Mon Sep 17 00:00:00 2001 From: Mikhail Fomichev Date: Fri, 15 May 2026 20:28:42 +0300 Subject: [PATCH 08/18] script: run all queries in one script --- script/run_all_queries.sh | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100755 script/run_all_queries.sh diff --git a/script/run_all_queries.sh b/script/run_all_queries.sh new file mode 100755 index 0000000..61f77a6 --- /dev/null +++ b/script/run_all_queries.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +cd "$(dirname "$0")" || exit 1 + +echo ">>> Запуск setup.sh..." +./setup.sh + +echo ">>> Запуск build.sh..." +./build.sh + +echo ">>> Запуск convert.sh..." +./convert.sh + +echo ">>> Запуск запросов от 0 до 42..." +for i in {0..42} +do + echo "=========================================" + echo ">>> Выполнение запроса $i" + echo "=========================================" + ./run_query.sh "$i" ../columnar_hits_sample.tuff test.csv test.log +done + +echo ">>> Все запросы выполнены успешно." From a255c75f20ba385e0e1730b058832125b8b0bf20 Mon Sep 17 00:00:00 2001 From: Mikhail Fomichev Date: Fri, 15 May 2026 22:04:32 +0300 Subject: [PATCH 09/18] optimize: problem with multiple parsing of metadata fixed, execution logic is readable now --- script/run_all_queries.sh | 6 +- src/CMakeLists.txt | 1 + src/column/Schema.h | 11 - src/execution/ExecutionApi.h | 49 +-- src/execution/ExecutionLogic.cpp | 259 +++++++++++++ src/execution/ExecutionLogic.h | 389 ++++---------------- src/execution/OperatorsBase.cpp | 49 +-- src/execution/OperatorsBase.h | 9 +- src/execution/expressions/ScalarFunctions.h | 4 +- src/io/ColumnarReader.cpp | 151 ++++---- src/io/ColumnarReader.h | 45 ++- src/main.cpp | 4 +- 12 files changed, 501 insertions(+), 476 deletions(-) create mode 100644 src/execution/ExecutionLogic.cpp diff --git a/script/run_all_queries.sh b/script/run_all_queries.sh index 61f77a6..6e22b9f 100755 --- a/script/run_all_queries.sh +++ b/script/run_all_queries.sh @@ -1,5 +1,7 @@ #!/bin/bash +set -e + cd "$(dirname "$0")" || exit 1 echo ">>> Запуск setup.sh..." @@ -9,7 +11,7 @@ echo ">>> Запуск build.sh..." ./build.sh echo ">>> Запуск convert.sh..." -./convert.sh +./convert.sh ../hits_sample.csv ../columnar_hits_sample.tuff ../hits.schema echo ">>> Запуск запросов от 0 до 42..." for i in {0..42} @@ -20,4 +22,4 @@ do ./run_query.sh "$i" ../columnar_hits_sample.tuff test.csv test.log done -echo ">>> Все запросы выполнены успешно." +echo ">>> Все запросы выполнены успешно." \ No newline at end of file diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index cfd56ab..ee8ccdd 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -26,6 +26,7 @@ add_executable(columnar-engine execution/OperatorsBase.h column/Codec.h execution/ExecutionLogic.h + execution/ExecutionLogic.cpp execution/expressions/AggregationFunctions.h column/Schema.h utils/Macro.h diff --git a/src/column/Schema.h b/src/column/Schema.h index 8b4e2ea..b43843d 100644 --- a/src/column/Schema.h +++ b/src/column/Schema.h @@ -72,17 +72,6 @@ class Schema { return Schema(std::move(fields)); } - static Schema FromColumnarFile(const std::string& columnar_file_path) { - ColumnarReader reader(columnar_file_path); - const std::vector metadata = reader.GetMetadata(); - - std::vector fields; - for (const ColumnMetadata& meta : metadata) { - fields.push_back({meta.name, meta.type}); - } - return Schema(std::move(fields)); - } - private: std::vector fields_; diff --git a/src/execution/ExecutionApi.h b/src/execution/ExecutionApi.h index 5863ece..6c1f288 100644 --- a/src/execution/ExecutionApi.h +++ b/src/execution/ExecutionApi.h @@ -70,68 +70,59 @@ class DataFrame { } static DataFrame Select(std::string columnar_file_path, std::vector column_names) { - auto scan_node = std::make_shared(); - scan_node->table_path = std::move(columnar_file_path); - scan_node->column_names = std::move(column_names); + auto table_meta = std::make_shared(columnar_file_path); + + auto scan_node = std::make_shared(std::move(columnar_file_path), + std::move(column_names), std::move(table_meta)); return DataFrame(scan_node); } DataFrame Aggregate(std::vector group_by_columns, std::vector> aggregate_expressions) { - auto aggregate_node = std::make_shared(); - aggregate_node->child = logical_plan_; - aggregate_node->group_by_columns = std::move(group_by_columns); - aggregate_node->aggregate_expressions = std::move(aggregate_expressions); + auto aggregate_node = std::make_shared( + logical_plan_, std::move(group_by_columns), std::move(aggregate_expressions)); + return DataFrame(aggregate_node); } DataFrame Filter(std::shared_ptr filter_expression) { - auto filter_node = std::make_shared(); - filter_node->child = logical_plan_; - filter_node->filter_expression = std::move(filter_expression); + auto filter_node = + std::make_shared(logical_plan_, std::move(filter_expression)); return DataFrame(filter_node); } DataFrame OrderBy(std::vector> order_by_columns, - std::optional limit = std::nullopt, std::optional offset = std::nullopt) { - auto order_by_node = std::make_shared(); - order_by_node->child = logical_plan_; - order_by_node->order_by_columns = std::move(order_by_columns); - order_by_node->limit = limit; - order_by_node->offset = offset; + std::optional limit = std::nullopt, + std::optional offset = std::nullopt) { + auto order_by_node = std::make_shared( + logical_plan_, std::move(order_by_columns), std::move(limit), std::move(offset)); return DataFrame(order_by_node); } DataFrame Limit(size_t limit) { - auto limit_node = std::make_shared(); - limit_node->child = logical_plan_; - limit_node->limit = limit; + auto limit_node = std::make_shared(logical_plan_, limit); return DataFrame(limit_node); } DataFrame Project(std::vector> scalar_expressions) { - auto scalar_node = std::make_shared(); - scalar_node->child = logical_plan_; - scalar_node->scalar_expressions = std::move(scalar_expressions); + auto scalar_node = + std::make_shared(logical_plan_, std::move(scalar_expressions)); return DataFrame(scalar_node); } DataFrame Drop(std::vector columns_to_drop) { - auto drop_node = std::make_shared(); - drop_node->child = logical_plan_; - drop_node->columns_to_drop = std::move(columns_to_drop); + auto drop_node = std::make_shared(logical_plan_, std::move(columns_to_drop)); return DataFrame(drop_node); } DataFrame Reoder(std::vector desired_order) { - auto node = std::make_shared(); - node->child = logical_plan_; - node->desired_order_ = std::move(desired_order); + auto node = std::make_shared(logical_plan_, std::move(desired_order)); return DataFrame(node); } DataResult Collect() { - PhysicalOperatorContext context = BuildPhysicalPlan(logical_plan_); + PhysicalOperatorContext context = logical_plan_->BuildPhysicalPlan(); + std::unique_ptr physical_plan_root = std::move(context.root_operator); std::vector> result; while (std::unique_ptr batch = physical_plan_root->Run()) { diff --git a/src/execution/ExecutionLogic.cpp b/src/execution/ExecutionLogic.cpp new file mode 100644 index 0000000..44ee699 --- /dev/null +++ b/src/execution/ExecutionLogic.cpp @@ -0,0 +1,259 @@ +#include "execution/ExecutionLogic.h" + +PhysicalOperatorContext ScanNode::BuildPhysicalPlan( + std::optional> required_columns) const { + const auto& full_columns = table_meta->GetColumns(); + + std::vector user_cols; + if (column_names.size() == 1 && column_names[0] == "*") { + for (const auto& field : full_columns) { + user_cols.push_back(field.name); + } + } else { + user_cols = column_names; + } + + std::vector final_columns = user_cols; + if (required_columns.has_value()) { + final_columns.insert(final_columns.end(), required_columns->begin(), + required_columns->end()); + std::sort(final_columns.begin(), final_columns.end()); + final_columns.erase(std::unique(final_columns.begin(), final_columns.end()), + final_columns.end()); + } + + auto op = std::make_unique(table_path, final_columns, table_meta); + + Schema output_schema; + for (const std::string& name : final_columns) { + bool found = false; + for (const auto& field : full_columns) { + if (field.name == name) { + output_schema.AddField({field.name, field.type}); + found = true; + break; + } + } + if (!found) [[unlikely]] { + THROW_RUNTIME_ERROR("Column " + name + " not found in table schema"); + } + } + + return {std::move(op), std::move(output_schema)}; +} + +PhysicalOperatorContext AggregateNode::BuildPhysicalPlan( + std::optional>) const { + if (!group_by_columns.empty()) { + std::vector needed_columns = group_by_columns; + for (const std::shared_ptr& expr : aggregate_expressions) { + expr->CollectRequiredColumns(needed_columns); + } + std::sort(needed_columns.begin(), needed_columns.end()); + needed_columns.erase(std::unique(needed_columns.begin(), needed_columns.end()), + needed_columns.end()); + + PhysicalOperatorContext child_context = child->BuildPhysicalPlan(needed_columns); + + std::vector group_col_indices; + std::vector> key_builders; + Schema output_schema; + + for (const std::string& col_name : group_by_columns) { + size_t ind = child_context.schema.GetColumnIndexByName(col_name); + group_col_indices.push_back(ind); + + ColumnType col_type = child_context.schema.GetColumnTypeByName(col_name); + output_schema.AddColumn(col_name, col_type); + key_builders.push_back(ColumnFactory::MakeColumnBuilder(col_type)); + } + + std::vector> agg_funcs; + for (const std::shared_ptr& expr : aggregate_expressions) { + agg_funcs.push_back( + expr->CreateGroupedAggregationFunction(child_context.schema, output_schema)); + } + + auto op = std::make_unique(std::move(child_context.root_operator), + std::move(group_col_indices), + std::move(key_builders), std::move(agg_funcs)); + return {std::move(op), std::move(output_schema)}; + } + + std::vector needed_columns = group_by_columns; + for (const std::shared_ptr& expr : aggregate_expressions) { + expr->CollectRequiredColumns(needed_columns); + } + std::sort(needed_columns.begin(), needed_columns.end()); + needed_columns.erase(std::unique(needed_columns.begin(), needed_columns.end()), + needed_columns.end()); + + PhysicalOperatorContext child_context = child->BuildPhysicalPlan(needed_columns); + std::vector> agg_functions; + Schema output_schema; + for (const std::shared_ptr& expr : aggregate_expressions) { + agg_functions.push_back( + expr->CreateGlobalAggregationFunction(child_context.schema, output_schema)); + } + auto op = std::make_unique(std::move(child_context.root_operator), + std::move(agg_functions)); + return {std::move(op), std::move(output_schema)}; +} + +PhysicalOperatorContext FilterNode::BuildPhysicalPlan( + std::optional> required_columns) const { + std::vector needed_columns; + filter_expression->CollectRequiredColumns(needed_columns); + if (required_columns.has_value()) { + needed_columns.insert(needed_columns.end(), required_columns->begin(), + required_columns->end()); + } + + std::sort(needed_columns.begin(), needed_columns.end()); + needed_columns.erase(std::unique(needed_columns.begin(), needed_columns.end()), + needed_columns.end()); + + PhysicalOperatorContext child_context = child->BuildPhysicalPlan(needed_columns); + std::unique_ptr filter_function = + filter_expression->CreateFilterFunction(child_context.schema); + auto op = std::make_unique(std::move(child_context.root_operator), + std::move(filter_function)); + return {std::move(op), std::move(child_context.schema)}; +} + +PhysicalOperatorContext OrderByNode::BuildPhysicalPlan( + std::optional> required_columns) const { + std::vector needed_columns; + if (required_columns.has_value()) { + needed_columns = required_columns.value(); + } + for (const auto& [col_name, is_desc] : order_by_columns) { + needed_columns.push_back(col_name); + } + std::sort(needed_columns.begin(), needed_columns.end()); + needed_columns.erase(std::unique(needed_columns.begin(), needed_columns.end()), + needed_columns.end()); + + PhysicalOperatorContext child_context = child->BuildPhysicalPlan(needed_columns); + std::vector> sort_columns; + for (const auto& [col_name, is_desc] : order_by_columns) { + size_t sort_col_idx = child_context.schema.GetColumnIndexByName(col_name); + sort_columns.push_back({sort_col_idx, is_desc}); + } + + auto op = std::make_unique(std::move(child_context.root_operator), + std::move(sort_columns), std::move(limit), + std::move(offset)); + return {std::move(op), std::move(child_context.schema)}; +} + +PhysicalOperatorContext LimitNode::BuildPhysicalPlan( + std::optional> required_columns) const { + PhysicalOperatorContext child_context = child->BuildPhysicalPlan(required_columns); + auto op = std::make_unique(std::move(child_context.root_operator), limit); + return {std::move(op), std::move(child_context.schema)}; +} + +PhysicalOperatorContext ScalarNode::BuildPhysicalPlan( + std::optional> required_columns) const { + std::vector needed_columns; + if (required_columns.has_value()) { + for (const auto& req_col : required_columns.value()) { + bool is_generated_here = false; + for (const auto& expr : scalar_expressions) { + if (expr->GetOutputName() == req_col) { + is_generated_here = true; + break; + } + } + if (!is_generated_here) { + needed_columns.push_back(req_col); + } + } + } + for (const auto& expr : scalar_expressions) { + expr->CollectRequiredColumns(needed_columns); + } + + std::sort(needed_columns.begin(), needed_columns.end()); + needed_columns.erase(std::unique(needed_columns.begin(), needed_columns.end()), + needed_columns.end()); + + PhysicalOperatorContext child_context = child->BuildPhysicalPlan(needed_columns); + + std::vector> scalar_functions; + + for (const auto& expr : scalar_expressions) { + scalar_functions.push_back(expr->CreateScalarFunction(child_context.schema)); + } + + auto op = std::make_unique(std::move(child_context.root_operator), + std::move(scalar_functions)); + return {std::move(op), std::move(child_context.schema)}; +} + +PhysicalOperatorContext DropNode::BuildPhysicalPlan( + std::optional> required_columns) const { + PhysicalOperatorContext child_context = child->BuildPhysicalPlan(required_columns); + + std::vector cols_to_keep_indices; + Schema output_schema; + const auto& child_fields = child_context.schema.GetFields(); + + for (const std::string& drop_col : columns_to_drop) { + bool found = false; + for (const auto& field : child_fields) { + if (field.name == drop_col) { + found = true; + break; + } + } + if (!found) { + THROW_RUNTIME_ERROR("Cannot drop column '" + drop_col + "': not found in dataframe"); + } + } + + for (size_t i = 0; i < child_fields.size(); ++i) { + const auto& field = child_fields[i]; + if (std::find(columns_to_drop.begin(), columns_to_drop.end(), field.name) == + columns_to_drop.end()) { + cols_to_keep_indices.push_back(i); + output_schema.AddField({field.name, field.type}); + } + } + + auto op = std::make_unique(std::move(child_context.root_operator), + std::move(cols_to_keep_indices)); + return {std::move(op), std::move(output_schema)}; +} + +PhysicalOperatorContext ReorderNode::BuildPhysicalPlan( + std::optional> required_columns) const { + PhysicalOperatorContext child_context = child->BuildPhysicalPlan(required_columns); + + std::vector new_indices; + Schema output_schema; + const auto& child_fields = child_context.schema.GetFields(); + + for (const std::string& reorder_col : desired_order) { + bool found = false; + for (size_t i = 0; i < child_fields.size(); ++i) { + const auto& field = child_fields[i]; + if (field.name == reorder_col) { + found = true; + new_indices.push_back(i); + output_schema.AddField({field.name, field.type}); + break; + } + } + + if (!found) { + THROW_RUNTIME_ERROR("Cannot reorder column '" + reorder_col + + "': not found in dataframe"); + } + } + + auto op = std::make_unique(std::move(child_context.root_operator), + std::move(new_indices)); + return {std::move(op), std::move(output_schema)}; +} diff --git a/src/execution/ExecutionLogic.h b/src/execution/ExecutionLogic.h index 7e0fc20..7037d0c 100644 --- a/src/execution/ExecutionLogic.h +++ b/src/execution/ExecutionLogic.h @@ -1,383 +1,134 @@ #pragma once #include "execution/OperatorsBase.h" -#include "execution/expressions/AggregationFunctions.h" #include "execution/expressions/AggregationExpressions.h" -#include "execution/expressions/FilterFunctions.h" #include "execution/expressions/FilterExpressions.h" #include "execution/expressions/ScalarExpressions.h" #include "column/Schema.h" -#include "utils/Macro.h" - #include #include #include #include #include -enum class PlanNodeType { - SCAN, - AGGREGATE, - FILTER, - ORDER_BY, - LIMIT, - SCALAR, - DROP, - REORDER +struct PhysicalOperatorContext { + std::unique_ptr root_operator; + Schema schema; }; struct PlanNode { - PlanNodeType type; + virtual ~PlanNode() = default; -protected: - explicit PlanNode(PlanNodeType node_type) : type(node_type) { - } + virtual PhysicalOperatorContext BuildPhysicalPlan( + std::optional> required_columns = std::nullopt) const = 0; }; struct ScanNode : public PlanNode { - ScanNode() : PlanNode(PlanNodeType::SCAN) { - } std::string table_path; std::vector column_names; + std::shared_ptr table_meta; + + ScanNode(std::string table_p, std::vector column_n, + std::shared_ptr table_m) + : table_path(std::move(table_p)), + column_names(std::move(column_n)), + table_meta(std::move(table_m)) { + } + + PhysicalOperatorContext BuildPhysicalPlan( + std::optional> required_columns) const override; }; struct AggregateNode : public PlanNode { - AggregateNode() : PlanNode(PlanNodeType::AGGREGATE) { - } std::shared_ptr child; std::vector group_by_columns; std::vector> aggregate_expressions; + + AggregateNode(std::shared_ptr c, std::vector gb_cols, + std::vector> agg_exprs) + : child(std::move(c)), + group_by_columns(std::move(gb_cols)), + aggregate_expressions(std::move(agg_exprs)) { + } + + PhysicalOperatorContext BuildPhysicalPlan( + std::optional> required_columns) const override; }; struct FilterNode : public PlanNode { - FilterNode() : PlanNode(PlanNodeType::FILTER) { - } std::shared_ptr child; std::shared_ptr filter_expression; + + FilterNode(std::shared_ptr c, std::shared_ptr filter_expr) + : child(std::move(c)), filter_expression(std::move(filter_expr)) { + } + + PhysicalOperatorContext BuildPhysicalPlan( + std::optional> required_columns) const override; }; struct OrderByNode : public PlanNode { - OrderByNode() : PlanNode(PlanNodeType::ORDER_BY) { - } std::shared_ptr child; std::vector> order_by_columns; std::optional limit; std::optional offset; + + OrderByNode(std::shared_ptr c, + std::vector> order_by_cols, std::optional lim, + std::optional off) + : child(std::move(c)), order_by_columns(std::move(order_by_cols)), limit(lim), offset(off) { + } + + PhysicalOperatorContext BuildPhysicalPlan( + std::optional> required_columns) const override; }; struct LimitNode : public PlanNode { - LimitNode() : PlanNode(PlanNodeType::LIMIT), limit(0) { - } std::shared_ptr child; size_t limit; + + LimitNode(std::shared_ptr c, size_t lim) : child(std::move(c)), limit(lim) { + } + + PhysicalOperatorContext BuildPhysicalPlan( + std::optional> required_columns) const override; }; struct ScalarNode : public PlanNode { - ScalarNode() : PlanNode(PlanNodeType::SCALAR) { - } std::shared_ptr child; std::vector> scalar_expressions; + + ScalarNode(std::shared_ptr c, + std::vector> scalar_exprs) + : child(std::move(c)), scalar_expressions(std::move(scalar_exprs)) { + } + + PhysicalOperatorContext BuildPhysicalPlan( + std::optional> required_columns) const override; }; struct DropNode : public PlanNode { - DropNode() : PlanNode(PlanNodeType::DROP) { - } std::shared_ptr child; std::vector columns_to_drop; -}; -struct ReorderNode : public PlanNode { - ReorderNode() : PlanNode(PlanNodeType::REORDER) { + DropNode(std::shared_ptr c, std::vector cols_to_drop) + : child(std::move(c)), columns_to_drop(std::move(cols_to_drop)) { } - std::shared_ptr child; - std::vector desired_order_; -}; -struct PhysicalOperatorContext { - std::unique_ptr root_operator; - Schema schema; + PhysicalOperatorContext BuildPhysicalPlan( + std::optional> required_columns) const override; }; -inline PhysicalOperatorContext BuildPhysicalPlan( - const std::shared_ptr& plan_node, - std::optional> required_columns = std::nullopt) { - switch (plan_node->type) { - case PlanNodeType::SCAN: { - auto scan = std::static_pointer_cast(plan_node); - std::vector final_columns; - - Schema full_schema = Schema::FromColumnarFile(scan->table_path); - const std::vector& full_columns = full_schema.GetFields(); - - std::vector user_cols; - if (scan->column_names.size() == 1 && scan->column_names[0] == "*") { - for (const Field& field : full_columns) { - user_cols.push_back(field.name); - } - } else { - user_cols = scan->column_names; - } - - final_columns = user_cols; - if (required_columns.has_value()) { - final_columns.insert(final_columns.end(), required_columns->begin(), - required_columns->end()); - } - - auto op = std::make_unique(scan->table_path, final_columns); - Schema output_schema; - for (const std::string& name : final_columns) { - bool found = false; - for (const Field& field : full_columns) { - if (field.name == name) { - output_schema.AddField({field.name, field.type}); - found = true; - break; - } - } - if (!found) [[unlikely]] { - THROW_RUNTIME_ERROR("Column " + name + " not found in table schema"); - } - } - - return {std::move(op), std::move(output_schema)}; - } - - case PlanNodeType::AGGREGATE: { - auto aggregate = std::static_pointer_cast(plan_node); - if (!aggregate->group_by_columns.empty()) { - std::vector needed_columns = aggregate->group_by_columns; - for (const std::shared_ptr& expr : - aggregate->aggregate_expressions) { - expr->CollectRequiredColumns(needed_columns); - } - std::sort(needed_columns.begin(), needed_columns.end()); - needed_columns.erase(std::unique(needed_columns.begin(), needed_columns.end()), - needed_columns.end()); - - PhysicalOperatorContext child_context = - BuildPhysicalPlan(aggregate->child, needed_columns); - - std::vector group_col_indices; - std::vector> key_builders; - Schema output_schema; - - for (const auto& col_name : aggregate->group_by_columns) { - size_t ind = child_context.schema.GetColumnIndexByName(col_name); - group_col_indices.push_back(ind); - - ColumnType type = child_context.schema.GetColumnTypeByName(col_name); - output_schema.AddColumn(col_name, type); - key_builders.push_back(ColumnFactory::MakeColumnBuilder(type)); - } - - std::vector> agg_funcs; - for (const std::shared_ptr& expr : - aggregate->aggregate_expressions) { - agg_funcs.push_back(expr->CreateGroupedAggregationFunction(child_context.schema, - output_schema)); - } - - auto op = std::make_unique( - std::move(child_context.root_operator), std::move(group_col_indices), - std::move(key_builders), std::move(agg_funcs)); - return {std::move(op), std::move(output_schema)}; - } - - std::vector needed_columns = aggregate->group_by_columns; - for (const std::shared_ptr& expr : - aggregate->aggregate_expressions) { - expr->CollectRequiredColumns(needed_columns); - } - std::sort(needed_columns.begin(), needed_columns.end()); - needed_columns.erase(std::unique(needed_columns.begin(), needed_columns.end()), - needed_columns.end()); - - PhysicalOperatorContext child_context = - BuildPhysicalPlan(aggregate->child, needed_columns); - std::vector> agg_functions; - Schema output_schema; - for (const std::shared_ptr& expr : - aggregate->aggregate_expressions) { - agg_functions.push_back( - expr->CreateGlobalAggregationFunction(child_context.schema, output_schema)); - } - auto op = std::make_unique(std::move(child_context.root_operator), - std::move(agg_functions)); - return {std::move(op), std::move(output_schema)}; - } - - case PlanNodeType::FILTER: { - auto filter = std::static_pointer_cast(plan_node); - std::vector needed_columns; - filter->filter_expression->CollectRequiredColumns(needed_columns); - if (required_columns.has_value()) { - needed_columns.insert(needed_columns.end(), required_columns->begin(), - required_columns->end()); - } - - std::sort(needed_columns.begin(), needed_columns.end()); - needed_columns.erase(std::unique(needed_columns.begin(), needed_columns.end()), - needed_columns.end()); - - PhysicalOperatorContext child_context = - BuildPhysicalPlan(filter->child, needed_columns); - std::unique_ptr filter_function = - filter->filter_expression->CreateFilterFunction(child_context.schema); - auto op = std::make_unique(std::move(child_context.root_operator), - std::move(filter_function)); - return {std::move(op), std::move(child_context.schema)}; - } - - case PlanNodeType::ORDER_BY: { - auto order_by = std::static_pointer_cast(plan_node); - std::vector needed_columns; - if (required_columns.has_value()) { - needed_columns = required_columns.value(); - } - for (const auto& [col_name, is_desc] : order_by->order_by_columns) { - needed_columns.push_back(col_name); - } - std::sort(needed_columns.begin(), needed_columns.end()); - needed_columns.erase(std::unique(needed_columns.begin(), needed_columns.end()), - needed_columns.end()); - - PhysicalOperatorContext child_context = - BuildPhysicalPlan(order_by->child, needed_columns); - std::vector> sort_columns; - for (const auto& [col_name, is_desc] : order_by->order_by_columns) { - size_t sort_col_idx = child_context.schema.GetColumnIndexByName(col_name); - sort_columns.push_back({sort_col_idx, is_desc}); - } - - auto op = std::make_unique(std::move(child_context.root_operator), - std::move(sort_columns), - std::move(order_by->limit), std::move(order_by->offset)); - return {std::move(op), std::move(child_context.schema)}; - } - - case PlanNodeType::LIMIT: { - auto limit = std::static_pointer_cast(plan_node); - PhysicalOperatorContext child_context = - BuildPhysicalPlan(limit->child, required_columns); - auto op = std::make_unique(std::move(child_context.root_operator), - limit->limit); - return {std::move(op), std::move(child_context.schema)}; - } - - case PlanNodeType::SCALAR: { - auto scalar = std::static_pointer_cast(plan_node); - std::vector needed_columns; - if (required_columns.has_value()) { - for (const auto& req_col : required_columns.value()) { - bool is_generated_here = false; - for (const auto& expr : scalar->scalar_expressions) { - if (expr->GetOutputName() == req_col) { - is_generated_here = true; - break; - } - } - if (!is_generated_here) { - needed_columns.push_back(req_col); - } - } - } - for (const auto& expr : scalar->scalar_expressions) { - expr->CollectRequiredColumns(needed_columns); - } - - std::sort(needed_columns.begin(), needed_columns.end()); - needed_columns.erase(std::unique(needed_columns.begin(), needed_columns.end()), - needed_columns.end()); - - PhysicalOperatorContext child_context = - BuildPhysicalPlan(scalar->child, needed_columns); - - std::vector> scalar_functions; - - for (const auto& expr : scalar->scalar_expressions) { - scalar_functions.push_back(expr->CreateScalarFunction(child_context.schema)); - } - - auto op = std::make_unique(std::move(child_context.root_operator), - std::move(scalar_functions)); - return {std::move(op), std::move(child_context.schema)}; - } - - case PlanNodeType::DROP: { - auto drop = std::static_pointer_cast(plan_node); - - PhysicalOperatorContext child_context = - BuildPhysicalPlan(drop->child, required_columns); - - std::vector cols_to_keep_indices; - Schema output_schema; - const auto& child_fields = child_context.schema.GetFields(); - - for (const std::string& drop_col : drop->columns_to_drop) { - bool found = false; - for (const auto& field : child_fields) { - if (field.name == drop_col) { - found = true; - break; - } - } - if (!found) { - THROW_RUNTIME_ERROR("Cannot drop column '" + drop_col + - "': not found in dataframe"); - } - } - - for (size_t i = 0; i < child_fields.size(); ++i) { - const auto& field = child_fields[i]; - if (std::find(drop->columns_to_drop.begin(), drop->columns_to_drop.end(), - field.name) == drop->columns_to_drop.end()) { - cols_to_keep_indices.push_back(i); - output_schema.AddField({field.name, field.type}); - } - } - - auto op = std::make_unique(std::move(child_context.root_operator), - std::move(cols_to_keep_indices)); - return {std::move(op), std::move(output_schema)}; - } - - case PlanNodeType::REORDER: { - auto reorder = std::static_pointer_cast(plan_node); - - PhysicalOperatorContext child_context = BuildPhysicalPlan(reorder->child, required_columns); - - std::vector new_indices; - Schema output_schema; - const auto& child_fields = child_context.schema.GetFields(); - - for (const std::string& reorder_col : reorder->desired_order_) { - bool found = false; - for (size_t i = 0; i < child_fields.size(); ++i) { - const auto& field = child_fields[i]; - if (field.name == reorder_col) { - found = true; - new_indices.push_back(i); - output_schema.AddField({field.name, field.type}); - break; - } - } - - if (!found) { - THROW_RUNTIME_ERROR("Cannot reorder column '" + reorder_col + "': not found in dataframe"); - } - } - - auto op = std::make_unique( - std::move(child_context.root_operator), - std::move(new_indices) - ); - return {std::move(op), std::move(output_schema)}; - } +struct ReorderNode : public PlanNode { + std::shared_ptr child; + std::vector desired_order; - default: - THROW_RUNTIME_ERROR("Unknown plan node type"); + ReorderNode(std::shared_ptr c, std::vector desired_ord) + : child(std::move(c)), desired_order(std::move(desired_ord)) { } -} + + PhysicalOperatorContext BuildPhysicalPlan( + std::optional> required_columns) const override; +}; diff --git a/src/execution/OperatorsBase.cpp b/src/execution/OperatorsBase.cpp index 4ad3740..ce653ed 100644 --- a/src/execution/OperatorsBase.cpp +++ b/src/execution/OperatorsBase.cpp @@ -8,36 +8,41 @@ #include ScanOperator::ScanOperator(const std::string& table_path, - const std::vector& column_names) - : reader_(table_path) { - const std::vector& metadata = reader_.GetMetadata(); - - if (column_names == std::vector{"*"}) { - for (size_t i = 0; i < metadata.size(); ++i) { - column_indices_.push_back(i); - } - } else { - for (const auto& name : column_names) { - bool found = false; - for (size_t i = 0; i < metadata.size(); ++i) { - if (metadata[i].name == name) { - column_indices_.push_back(i); - found = true; - break; - } - } - if (!found) { - THROW_RUNTIME_ERROR("Column " + name + " not found in metadata"); + const std::vector& column_names, + std::shared_ptr metadata) + : reader_(table_path, metadata), metadata_(metadata) { + const auto& full_columns = metadata_->GetColumns(); + + for (const std::string& name : column_names) { + bool found = false; + for (size_t i = 0; i < full_columns.size(); ++i) { + if (full_columns[i].name == name) { + column_indices_.push_back(i); + found = true; + break; } } + if (!found) { + THROW_RUNTIME_ERROR("Column " + name + " not found in metadata"); + } } - if (!metadata.empty()) { - total_chunks_ = metadata[0].offsets.size(); + if (!full_columns.empty()) { + total_chunks_ = full_columns[0].offsets.size(); } } std::unique_ptr ScanOperator::Run() { + if (column_indices_.empty()) { + if (finished_empty_scan_) { + return nullptr; + } + auto record_batch = std::make_unique(); + record_batch->num_rows = metadata_->GetNumRows(); + finished_empty_scan_ = true; + return record_batch; + } + if (current_chunk_ >= total_chunks_) { return nullptr; } diff --git a/src/execution/OperatorsBase.h b/src/execution/OperatorsBase.h index 02ade27..a3c47f0 100644 --- a/src/execution/OperatorsBase.h +++ b/src/execution/OperatorsBase.h @@ -28,15 +28,18 @@ class Operator { class ScanOperator : public Operator { public: - ScanOperator(const std::string& table_path, const std::vector& column_names); + ScanOperator(const std::string& table_path, const std::vector& column_names, + std::shared_ptr metadata); std::unique_ptr Run() override; private: ColumnarReader reader_; + std::shared_ptr metadata_; std::vector column_indices_; size_t current_chunk_ = 0; size_t total_chunks_ = 0; + bool finished_empty_scan_ = false; }; class AggregationOperator : public Operator { @@ -87,8 +90,8 @@ class GroupByOperator : public Operator { class OrderByOperator : public Operator { public: OrderByOperator(std::unique_ptr child, - std::vector> sort_columns, - std::optional limit, std::optional offset); + std::vector> sort_columns, std::optional limit, + std::optional offset); std::unique_ptr Run() override; diff --git a/src/execution/expressions/ScalarFunctions.h b/src/execution/expressions/ScalarFunctions.h index deae3db..95d2d18 100644 --- a/src/execution/expressions/ScalarFunctions.h +++ b/src/execution/expressions/ScalarFunctions.h @@ -159,8 +159,8 @@ class CaseWhenFunction : public ScalarFunction { auto build_view = [&](const ResolvedBranch& b) -> ViewType { if (std::holds_alternative(b)) { - size_t col_idx = std::get(b); - auto* col = static_cast(batch.columns[col_idx].get()); + size_t col_ind = std::get(b); + auto* col = static_cast(batch.columns[col_ind].get()); return FlatColumnView(&col->GetData()); } else { return ConstColumnView(std::get(b)); diff --git a/src/io/ColumnarReader.cpp b/src/io/ColumnarReader.cpp index bce5dc4..58b71c9 100644 --- a/src/io/ColumnarReader.cpp +++ b/src/io/ColumnarReader.cpp @@ -6,134 +6,133 @@ #include -ColumnarReader::ColumnarReader(const std::string& file_name) : file_(file_name, std::ios::binary) { - if (!file_.is_open()) { - THROW_RUNTIME_ERROR("Could not open file for reading"); +void MetadataTable::ParseMetadata(const std::string& file_path) { + std::ifstream file(file_path, std::ios::binary); + if (!file.is_open()) { + THROW_RUNTIME_ERROR("Could not open file for metadata read"); } - std::streamoff file_size; - ValidateMagicAndGetFileSize(file_size); - ReadMetadata(); -} -void ColumnarReader::ValidateMagicAndGetFileSize(std::streamoff& file_size) { - file_.seekg(0, std::ios::end); - file_size = file_.tellg(); - if (file_size < 8) { + file.seekg(0, std::ios::end); + std::streamoff file_size = file.tellg(); + + if (file_size < 24) { // 20 байт мета + 4 байта магии THROW_RUNTIME_ERROR("File is too small to contain a valid footer"); } - file_.seekg(-4, std::ios::end); + file.seekg(-4, std::ios::end); std::array magic{}; - file_.read(magic.data(), magic.size()); - if (!file_ || std::string(magic.data(), magic.size()) != "TUFF") { + file.read(magic.data(), magic.size()); + if (!file || std::string(magic.data(), magic.size()) != "TUFF") { THROW_RUNTIME_ERROR("Invalid file footer magic"); } -} -void ColumnarReader::ReadMetadata() { - file_.seekg(-12, std::ios::end); + file.seekg(file_size - 20, std::ios::beg); + file.read(reinterpret_cast(&num_rows_), sizeof(num_rows_)); + uint64_t metadata_start; - file_.read(reinterpret_cast(&metadata_start), sizeof(metadata_start)); - if (!file_) { - THROW_RUNTIME_ERROR("Failed to read metadata start offset"); - } + file.read(reinterpret_cast(&metadata_start), sizeof(metadata_start)); + + size_t metadata_size = (file_size - 20) - metadata_start; + std::vector buffer(metadata_size); + file.seekg(metadata_start, std::ios::beg); + file.read(buffer.data(), metadata_size); + + size_t offset = 0; - file_.seekg(metadata_start, std::ios::beg); + auto read_from_buf = [&](void* dest, size_t size) { + std::memcpy(dest, buffer.data() + offset, size); + offset += size; + }; uint32_t num_cols; - file_.read(reinterpret_cast(&num_cols), sizeof(num_cols)); - if (!file_) { - THROW_RUNTIME_ERROR("Failed to read number of columns"); - } + read_from_buf(&num_cols, sizeof(num_cols)); for (uint32_t i = 0; i < num_cols; ++i) { ColumnMetadata column_meta; uint32_t name_length; - file_.read(reinterpret_cast(&name_length), sizeof(name_length)); - if (!file_) { - THROW_RUNTIME_ERROR("Failed to read column name length"); - } + read_from_buf(&name_length, sizeof(name_length)); + column_meta.name.resize(name_length); if (name_length > 0) { - file_.read(&column_meta.name[0], name_length); - if (!file_) { - THROW_RUNTIME_ERROR("Failed to read column name"); - } + read_from_buf(column_meta.name.data(), name_length); } uint8_t type; - file_.read(reinterpret_cast(&type), sizeof(type)); - if (!file_) { - THROW_RUNTIME_ERROR("Failed to read column type"); - } + read_from_buf(&type, sizeof(type)); column_meta.type = static_cast(type); uint32_t num_chunks; - file_.read(reinterpret_cast(&num_chunks), sizeof(num_chunks)); - if (!file_) { - THROW_RUNTIME_ERROR("Failed to read number of chunks"); - } + read_from_buf(&num_chunks, sizeof(num_chunks)); + + column_meta.offsets.reserve(num_chunks); + column_meta.sizes.reserve(num_chunks); + for (uint32_t j = 0; j < num_chunks; ++j) { - uint64_t offset, size; - file_.read(reinterpret_cast(&offset), sizeof(offset)); - file_.read(reinterpret_cast(&size), sizeof(size)); - if (!file_) { - THROW_RUNTIME_ERROR("Failed to read chunk metadata"); - } - column_meta.offsets.push_back(offset); - column_meta.sizes.push_back(size); + uint64_t chunk_offset, chunk_size; + read_from_buf(&chunk_offset, sizeof(chunk_offset)); + read_from_buf(&chunk_size, sizeof(chunk_size)); + + column_meta.offsets.push_back(chunk_offset); + column_meta.sizes.push_back(chunk_size); } + column_meta_.push_back(std::move(column_meta)); + } +} + +ColumnarReader::ColumnarReader(const std::string& file_name, + std::shared_ptr meta) + : file_(file_name, std::ios::binary), metadata_(std::move(meta)) { + if (!file_.is_open()) { + THROW_RUNTIME_ERROR("Could not open file for reading data"); + } +} - metadata_.push_back(std::move(column_meta)); +ColumnType ColumnarReader::GetColumnTypeByName(const std::string& name) const { + for (const auto& meta : metadata_->GetColumns()) { + if (meta.name == name) { + return meta.type; + } } + THROW_RUNTIME_ERROR("Column " + name + " not found"); } -void ColumnarReader::ReadRawColumnData(size_t column_index, size_t chunk_index, std::vector& buffer) const { - if (column_index >= metadata_.size()) { +void ColumnarReader::ReadRawColumnData(size_t column_index, size_t chunk_index, + std::vector& buffer) const { + const ColumnMetadata& meta = (*metadata_)[column_index]; + + if (column_index >= metadata_->GetNumColumns()) { THROW_RUNTIME_ERROR("Column index out of range"); } - if (chunk_index >= metadata_[column_index].offsets.size()) { + if (chunk_index >= meta.offsets.size()) { THROW_RUNTIME_ERROR("Chunk index out of range"); } - uint64_t offset = metadata_[column_index].offsets[chunk_index]; - uint64_t size = metadata_[column_index].sizes[chunk_index]; + + uint64_t offset = meta.offsets[chunk_index]; + uint64_t size = meta.sizes[chunk_index]; buffer.resize(size); file_.seekg(offset, std::ios::beg); file_.read(buffer.data(), size); } -std::shared_ptr ColumnarReader::GetColumnData(size_t column_index, size_t chunk_index) const { +std::shared_ptr ColumnarReader::GetColumnData(size_t column_index, + size_t chunk_index) const { std::vector raw_data; ReadRawColumnData(column_index, chunk_index, raw_data); - ColumnType type = metadata_[column_index].type; - + + ColumnType type = (*metadata_)[column_index].type; std::shared_ptr column = ColumnFactory::MakeColumn(type); -#define HANDLE_TYPE(ENUM_VAL, STR_VAL, CLASS_TYPE) \ - case ColumnType::ENUM_VAL: \ +#define HANDLE_TYPE(ENUM_VAL, STR_VAL, CLASS_TYPE) \ + case ColumnType::ENUM_VAL: \ std::static_pointer_cast(column)->ReadFromBuffer(raw_data); \ break; switch (type) { FOR_EACH_COLUMN_TYPE(HANDLE_TYPE) - default: - THROW_RUNTIME_ERROR("Unknown column type"); + default: THROW_RUNTIME_ERROR("Unknown column type"); } #undef HANDLE_TYPE return column; } - -const std::vector& ColumnarReader::GetMetadata() const { - return metadata_; -} - -ColumnType ColumnarReader::GetColumnTypeByName(const std::string& name) const { - for (const auto& meta : metadata_) { - if (meta.name == name) { - return meta.type; - } - } - THROW_RUNTIME_ERROR("Column " + name + " not found in metadata"); -} diff --git a/src/io/ColumnarReader.h b/src/io/ColumnarReader.h index 9afcb2a..3d3ef0f 100644 --- a/src/io/ColumnarReader.h +++ b/src/io/ColumnarReader.h @@ -6,24 +6,49 @@ #include #include #include +#include #include -class ColumnarReader { +class MetadataTable { public: - explicit ColumnarReader(const std::string& file_name); - - void ReadRawColumnData(size_t column_index, size_t chunk_index, std::vector& buffer) const; + explicit MetadataTable(const std::string& file_path) { + ParseMetadata(file_path); + } + + const std::vector& GetColumns() const { + return column_meta_; + } + uint64_t GetNumRows() const { + return num_rows_; + } + size_t GetNumColumns() const { + return column_meta_.size(); + } + const ColumnMetadata& operator[](size_t ind) const { + return column_meta_[ind]; + } - std::shared_ptr GetColumnData(size_t column_index, size_t chunk_index) const; +private: + void ParseMetadata(const std::string& file_path); + + std::vector column_meta_; + uint64_t num_rows_ = 0; +}; - const std::vector& GetMetadata() const; +class ColumnarReader { +public: + ColumnarReader(const std::string& file_name, std::shared_ptr meta); + + uint64_t GetNumRows() const { + return metadata_->GetNumRows(); + } ColumnType GetColumnTypeByName(const std::string& name) const; + void ReadRawColumnData(size_t column_index, size_t chunk_index, + std::vector& buffer) const; + std::shared_ptr GetColumnData(size_t column_index, size_t chunk_index) const; private: - void ValidateMagicAndGetFileSize(std::streamoff& file_size); - void ReadMetadata(); - mutable std::ifstream file_; - std::vector metadata_; + std::shared_ptr metadata_; }; diff --git a/src/main.cpp b/src/main.cpp index a84ea39..3a4aa65 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -19,7 +19,7 @@ class Query { // SELECT COUNT(*) FROM hits WHERE AdvEngineID <> 0; void Query01() { auto df = DataFrame::Select(columnar_file_path_, {}) - .Filter(NotEq("AdvEngineID", 0)) + .Filter(NotEq("AdvEngineID", 0)) .Aggregate({}, {Count()}) .Collect(); @@ -77,7 +77,7 @@ class Query { // COUNT(*) DESC; void Query07() { auto df = DataFrame::Select(columnar_file_path_, {"AdvEngineID"}) - .Filter(NotEq("AdvEngineID", 0)) + .Filter(NotEq("AdvEngineID", 0)) .Aggregate({"AdvEngineID"}, {Count()}) .OrderBy({{"count", true}}) .Collect(); From 0d2eedc870a66ad8e8fa80e512d74a8c99fa290e Mon Sep 17 00:00:00 2001 From: Mikhail Fomichev Date: Fri, 15 May 2026 22:16:11 +0300 Subject: [PATCH 10/18] fix: tests fix --- src/io/test/TestReaderWriter.cpp | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/io/test/TestReaderWriter.cpp b/src/io/test/TestReaderWriter.cpp index 15fe6c5..e062e7c 100644 --- a/src/io/test/TestReaderWriter.cpp +++ b/src/io/test/TestReaderWriter.cpp @@ -7,6 +7,7 @@ #include #include #include +#include namespace fs = std::filesystem; @@ -83,8 +84,8 @@ TEST_CASE("Reader: Read metadata", "[Reader]") { CsvToColumnar::Convert(csv_path, output_path); SECTION("Read and verify metadata") { - ColumnarReader reader(output_path); - const auto& metadata = reader.GetMetadata(); + auto meta = std::make_shared(output_path); + const auto& metadata = meta->GetColumns(); REQUIRE(metadata.size() == 2); REQUIRE(metadata[0].name == "id"); @@ -107,7 +108,8 @@ TEST_CASE("Reader: Read typed column data", "[Reader]") { CsvToColumnar::Convert(csv_path, output_path); - ColumnarReader reader(output_path); + auto meta = std::make_shared(output_path); + ColumnarReader reader(output_path, meta); SECTION("Read INT32 column") { auto column = reader.GetColumnData(0, 0); @@ -125,4 +127,4 @@ TEST_CASE("Reader: Read typed column data", "[Reader]") { RemoveTempFile(csv_path); RemoveTempFile(output_path); -} +} \ No newline at end of file From b6c57ee2e98613ca7c7256e1477db5c34be3c343 Mon Sep 17 00:00:00 2001 From: Mikhail Fomichev Date: Sat, 16 May 2026 14:59:38 +0300 Subject: [PATCH 11/18] redesign: now push execution implemented instead of pull. Start of implementing multithreading --- CMakeLists.txt | 23 +- script/run_all_queries.sh | 9 +- script/setup.sh | 14 +- src/CMakeLists.txt | 21 +- src/execution/ExecutionApi.h | 27 +- src/execution/ExecutionLogic.cpp | 202 +++++---- src/execution/ExecutionLogic.h | 41 +- src/execution/OperatorsBase.cpp | 418 +++++++++--------- src/execution/OperatorsBase.h | 196 ++++---- src/execution/Pipeline.h | 76 ++++ src/execution/PipelineExecutor.h | 31 ++ .../expressions/AggregationFunctions.h | 7 +- src/execution/expressions/ScalarFunctions.h | 9 +- src/io/ColumnarReader.cpp | 22 +- src/io/ColumnarReader.h | 5 +- src/main.cpp | 6 +- src/utils/Concurrency.h | 76 ++++ src/utils/ThreadPool.h | 78 ++++ 18 files changed, 809 insertions(+), 452 deletions(-) create mode 100644 src/execution/Pipeline.h create mode 100644 src/execution/PipelineExecutor.h create mode 100644 src/utils/Concurrency.h create mode 100644 src/utils/ThreadPool.h diff --git a/CMakeLists.txt b/CMakeLists.txt index 83ce1f4..88f4ac0 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 3.28) +cmake_minimum_required(VERSION 3.30) project(columnar-engine) @@ -15,7 +15,26 @@ include(cmake/TestSolution.cmake) include(cmake/SeminarUtils.cmake) include(FetchContent) -include(cmake/FindGlog.cmake) + +set(ABSL_PROPAGATE_CXX_STD ON) +set(BUILD_TESTING OFF) +set(ABSL_ENABLE_INSTALL ON) + +FetchContent_Declare( + abseil-cpp + GIT_REPOSITORY https://github.com/abseil/abseil-cpp.git + GIT_TAG 20240116.2 + SYSTEM +) +FetchContent_MakeAvailable(abseil-cpp) + +FetchContent_Declare( + re2 + GIT_REPOSITORY https://github.com/google/re2.git + GIT_TAG 2024-02-01 + SYSTEM +) +FetchContent_MakeAvailable(re2) find_package(Catch REQUIRED) find_package(Benchmark REQUIRED) diff --git a/script/run_all_queries.sh b/script/run_all_queries.sh index 6e22b9f..afa983f 100755 --- a/script/run_all_queries.sh +++ b/script/run_all_queries.sh @@ -13,13 +13,16 @@ echo ">>> Запуск build.sh..." echo ">>> Запуск convert.sh..." ./convert.sh ../hits_sample.csv ../columnar_hits_sample.tuff ../hits.schema -echo ">>> Запуск запросов от 0 до 42..." +RESULTS_DIR="query_results" +mkdir -p "${RESULTS_DIR}" + +echo ">>> Запуск запросов от 0 до 42 (результаты в ${RESULTS_DIR}/)..." for i in {0..42} do echo "=========================================" echo ">>> Выполнение запроса $i" echo "=========================================" - ./run_query.sh "$i" ../columnar_hits_sample.tuff test.csv test.log + ./run_query.sh "$i" ../columnar_hits_sample.tuff "${RESULTS_DIR}/query_${i}.csv" "${RESULTS_DIR}/query_${i}.log" done -echo ">>> Все запросы выполнены успешно." \ No newline at end of file +echo ">>> Все запросы выполнены успешно. Результаты в ${RESULTS_DIR}/" \ No newline at end of file diff --git a/script/setup.sh b/script/setup.sh index 9cef495..be555df 100755 --- a/script/setup.sh +++ b/script/setup.sh @@ -3,7 +3,17 @@ set -euo pipefail export DEBIAN_FRONTEND=noninteractive +apt-get update +apt-get install -y --no-install-recommends ca-certificates gpg wget + +wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null \ + | gpg --dearmor - \ + | tee /usr/share/keyrings/kitware-archive-keyring.gpg >/dev/null + +echo 'deb [signed-by=/usr/share/keyrings/kitware-archive-keyring.gpg] https://apt.kitware.com/ubuntu/ noble main' \ + | tee /etc/apt/sources.list.d/kitware.list >/dev/null + apt-get update apt-get install -y --no-install-recommends \ - build-essential cmake git ca-certificates \ - g++ libgoogle-glog-dev + build-essential git ca-certificates \ + g++ libgoogle-glog-dev cmake \ No newline at end of file diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index ee8ccdd..4d2db56 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -1,4 +1,10 @@ add_library(project_includes INTERFACE) + +target_link_libraries(project_includes INTERFACE + absl::flat_hash_map + re2::re2 +) + target_include_directories(project_includes INTERFACE ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/column @@ -11,7 +17,9 @@ target_include_directories(project_includes INTERFACE ${CMAKE_CURRENT_SOURCE_DIR}/utils/test ) -add_executable(columnar-engine +find_package(Threads REQUIRED) + +set(ENGINE_SOURCES main.cpp io/CsvTokenizer.cpp io/CsvTokenizer.h @@ -53,11 +61,22 @@ add_executable(columnar-engine types/String.h column/ColumnView.h types/TimeUnit.h + utils/Concurrency.h + execution/Pipeline.h + execution/PipelineExecutor.h ) +add_executable(columnar-engine ${ENGINE_SOURCES} + utils/ThreadPool.h) target_link_libraries(columnar-engine PRIVATE project_includes) target_compile_options(columnar-engine PRIVATE -Werror) +add_executable(columnar-engine-mt ${ENGINE_SOURCES}) +target_link_libraries(columnar-engine-mt PRIVATE project_includes Threads::Threads) +# ENABLE_MULTITHREADING, it will enable std::atomic and std::mutex +target_compile_definitions(columnar-engine-mt PRIVATE ENABLE_MULTITHREADING) +target_compile_options(columnar-engine-mt PRIVATE -Werror) + add_executable(test_csv_parser io/test/TestCsvParser.cpp io/CsvTokenizer.cpp diff --git a/src/execution/ExecutionApi.h b/src/execution/ExecutionApi.h index 6c1f288..eb218dd 100644 --- a/src/execution/ExecutionApi.h +++ b/src/execution/ExecutionApi.h @@ -4,6 +4,7 @@ #include "execution/expressions/FilterExpressions.h" #include "execution/ExecutionLogic.h" #include "execution/ExecutionHelper.h" +#include "execution/PipelineExecutor.h" #include @@ -45,7 +46,7 @@ class DataResult { } } - std::shared_ptr GetResult(std::string column_name) const { + std::shared_ptr GetResult(const std::string& column_name) const { const std::vector& fields = schema_.GetFields(); for (size_t i = 0; i < fields.size(); ++i) { if (fields[i].name == column_name) { @@ -115,22 +116,26 @@ class DataFrame { return DataFrame(drop_node); } - DataFrame Reoder(std::vector desired_order) { + DataFrame Reorder(std::vector desired_order) { auto node = std::make_shared(logical_plan_, std::move(desired_order)); return DataFrame(node); } - DataResult Collect() { - PhysicalOperatorContext context = logical_plan_->BuildPhysicalPlan(); + DataResult Collect() const { + PipelineBuildContext ctx; - std::unique_ptr physical_plan_root = std::move(context.root_operator); - std::vector> result; - while (std::unique_ptr batch = physical_plan_root->Run()) { - if (batch != nullptr) { - result.push_back(std::move(batch)); - } + auto outer_pipe = std::make_unique(); + auto result_sink = std::make_shared(); + outer_pipe->sink = result_sink; + ctx.current_pipeline = outer_pipe.get(); + + logical_plan_->BuildPipelines(ctx); + ctx.completed_pipelines.push_back(std::move(outer_pipe)); + for (auto& pipeline : ctx.completed_pipelines) { + pipeline->Execute(); } - return DataResult{std::move(result), std::move(context.schema)}; + + return DataResult{result_sink->TakeBatches(), std::move(ctx.schema)}; } private: diff --git a/src/execution/ExecutionLogic.cpp b/src/execution/ExecutionLogic.cpp index 44ee699..4fc03ce 100644 --- a/src/execution/ExecutionLogic.cpp +++ b/src/execution/ExecutionLogic.cpp @@ -1,7 +1,9 @@ #include "execution/ExecutionLogic.h" -PhysicalOperatorContext ScanNode::BuildPhysicalPlan( - std::optional> required_columns) const { +// ========================= ScanNode ========================= + +void ScanNode::BuildPipelines(PipelineBuildContext& ctx, + std::optional> required_columns) const { const auto& full_columns = table_meta->GetColumns(); std::vector user_cols; @@ -15,14 +17,16 @@ PhysicalOperatorContext ScanNode::BuildPhysicalPlan( std::vector final_columns = user_cols; if (required_columns.has_value()) { - final_columns.insert(final_columns.end(), required_columns->begin(), - required_columns->end()); - std::sort(final_columns.begin(), final_columns.end()); - final_columns.erase(std::unique(final_columns.begin(), final_columns.end()), - final_columns.end()); + for (const std::string& req_col : required_columns.value()) { + if (std::find(final_columns.begin(), final_columns.end(), req_col) == + final_columns.end()) { + final_columns.push_back(req_col); + } + } } - auto op = std::make_unique(table_path, final_columns, table_meta); + auto scan_op = std::make_shared(table_path, final_columns, table_meta); + ctx.current_pipeline->source = scan_op; Schema output_schema; for (const std::string& name : final_columns) { @@ -39,12 +43,19 @@ PhysicalOperatorContext ScanNode::BuildPhysicalPlan( } } - return {std::move(op), std::move(output_schema)}; + ctx.schema = std::move(output_schema); } -PhysicalOperatorContext AggregateNode::BuildPhysicalPlan( - std::optional>) const { +// ========================= AggregateNode ========================= + +void AggregateNode::BuildPipelines(PipelineBuildContext& ctx, + std::optional>) const { + Pipeline* outer_pipeline = ctx.current_pipeline; + Pipeline* inner_pipeline = new Pipeline(); + ctx.current_pipeline = inner_pipeline; + if (!group_by_columns.empty()) { + // === GROUP BY path === std::vector needed_columns = group_by_columns; for (const std::shared_ptr& expr : aggregate_expressions) { expr->CollectRequiredColumns(needed_columns); @@ -53,55 +64,67 @@ PhysicalOperatorContext AggregateNode::BuildPhysicalPlan( needed_columns.erase(std::unique(needed_columns.begin(), needed_columns.end()), needed_columns.end()); - PhysicalOperatorContext child_context = child->BuildPhysicalPlan(needed_columns); + child->BuildPipelines(ctx, needed_columns); std::vector group_col_indices; std::vector> key_builders; Schema output_schema; for (const std::string& col_name : group_by_columns) { - size_t ind = child_context.schema.GetColumnIndexByName(col_name); + size_t ind = ctx.schema.GetColumnIndexByName(col_name); group_col_indices.push_back(ind); - ColumnType col_type = child_context.schema.GetColumnTypeByName(col_name); + ColumnType col_type = ctx.schema.GetColumnTypeByName(col_name); output_schema.AddColumn(col_name, col_type); key_builders.push_back(ColumnFactory::MakeColumnBuilder(col_type)); } std::vector> agg_funcs; for (const std::shared_ptr& expr : aggregate_expressions) { - agg_funcs.push_back( - expr->CreateGroupedAggregationFunction(child_context.schema, output_schema)); + agg_funcs.push_back(expr->CreateGroupedAggregationFunction(ctx.schema, output_schema)); } - auto op = std::make_unique(std::move(child_context.root_operator), - std::move(group_col_indices), - std::move(key_builders), std::move(agg_funcs)); - return {std::move(op), std::move(output_schema)}; - } + auto breaker = std::make_shared( + std::move(group_col_indices), std::move(key_builders), std::move(agg_funcs)); - std::vector needed_columns = group_by_columns; - for (const std::shared_ptr& expr : aggregate_expressions) { - expr->CollectRequiredColumns(needed_columns); - } - std::sort(needed_columns.begin(), needed_columns.end()); - needed_columns.erase(std::unique(needed_columns.begin(), needed_columns.end()), - needed_columns.end()); + inner_pipeline->sink = breaker; + ctx.completed_pipelines.push_back(std::unique_ptr(inner_pipeline)); + outer_pipeline->source = breaker; + ctx.current_pipeline = outer_pipeline; + ctx.schema = std::move(output_schema); + } else { + // === Global aggregation path === + std::vector needed_columns; + for (const std::shared_ptr& expr : aggregate_expressions) { + expr->CollectRequiredColumns(needed_columns); + } + std::sort(needed_columns.begin(), needed_columns.end()); + needed_columns.erase(std::unique(needed_columns.begin(), needed_columns.end()), + needed_columns.end()); - PhysicalOperatorContext child_context = child->BuildPhysicalPlan(needed_columns); - std::vector> agg_functions; - Schema output_schema; - for (const std::shared_ptr& expr : aggregate_expressions) { - agg_functions.push_back( - expr->CreateGlobalAggregationFunction(child_context.schema, output_schema)); + child->BuildPipelines(ctx, needed_columns); + + std::vector> agg_functions; + Schema output_schema; + for (const std::shared_ptr& expr : aggregate_expressions) { + agg_functions.push_back( + expr->CreateGlobalAggregationFunction(ctx.schema, output_schema)); + } + + auto breaker = std::make_shared(std::move(agg_functions)); + + inner_pipeline->sink = breaker; + ctx.completed_pipelines.push_back(std::unique_ptr(inner_pipeline)); + outer_pipeline->source = breaker; + ctx.current_pipeline = outer_pipeline; + ctx.schema = std::move(output_schema); } - auto op = std::make_unique(std::move(child_context.root_operator), - std::move(agg_functions)); - return {std::move(op), std::move(output_schema)}; } -PhysicalOperatorContext FilterNode::BuildPhysicalPlan( - std::optional> required_columns) const { +// ========================= FilterNode ========================= + +void FilterNode::BuildPipelines(PipelineBuildContext& ctx, + std::optional> required_columns) const { std::vector needed_columns; filter_expression->CollectRequiredColumns(needed_columns); if (required_columns.has_value()) { @@ -113,16 +136,22 @@ PhysicalOperatorContext FilterNode::BuildPhysicalPlan( needed_columns.erase(std::unique(needed_columns.begin(), needed_columns.end()), needed_columns.end()); - PhysicalOperatorContext child_context = child->BuildPhysicalPlan(needed_columns); + child->BuildPipelines(ctx, needed_columns); + std::unique_ptr filter_function = - filter_expression->CreateFilterFunction(child_context.schema); - auto op = std::make_unique(std::move(child_context.root_operator), - std::move(filter_function)); - return {std::move(op), std::move(child_context.schema)}; + filter_expression->CreateFilterFunction(ctx.schema); + auto filter_op = std::make_shared(std::move(filter_function)); + ctx.current_pipeline->transforms.push_back(filter_op); } -PhysicalOperatorContext OrderByNode::BuildPhysicalPlan( - std::optional> required_columns) const { +// ========================= OrderByNode ========================= + +void OrderByNode::BuildPipelines(PipelineBuildContext& ctx, + std::optional> required_columns) const { + Pipeline* outer_pipeline = ctx.current_pipeline; + Pipeline* inner_pipeline = new Pipeline(); + ctx.current_pipeline = inner_pipeline; + std::vector needed_columns; if (required_columns.has_value()) { needed_columns = required_columns.value(); @@ -134,28 +163,36 @@ PhysicalOperatorContext OrderByNode::BuildPhysicalPlan( needed_columns.erase(std::unique(needed_columns.begin(), needed_columns.end()), needed_columns.end()); - PhysicalOperatorContext child_context = child->BuildPhysicalPlan(needed_columns); + child->BuildPipelines(ctx, needed_columns); + std::vector> sort_columns; for (const auto& [col_name, is_desc] : order_by_columns) { - size_t sort_col_idx = child_context.schema.GetColumnIndexByName(col_name); + size_t sort_col_idx = ctx.schema.GetColumnIndexByName(col_name); sort_columns.push_back({sort_col_idx, is_desc}); } - auto op = std::make_unique(std::move(child_context.root_operator), - std::move(sort_columns), std::move(limit), - std::move(offset)); - return {std::move(op), std::move(child_context.schema)}; + auto breaker = std::make_shared(std::move(sort_columns), + std::move(limit), std::move(offset)); + + inner_pipeline->sink = breaker; + ctx.completed_pipelines.push_back(std::unique_ptr(inner_pipeline)); + outer_pipeline->source = breaker; + ctx.current_pipeline = outer_pipeline; } -PhysicalOperatorContext LimitNode::BuildPhysicalPlan( - std::optional> required_columns) const { - PhysicalOperatorContext child_context = child->BuildPhysicalPlan(required_columns); - auto op = std::make_unique(std::move(child_context.root_operator), limit); - return {std::move(op), std::move(child_context.schema)}; +// ========================= LimitNode ========================= + +void LimitNode::BuildPipelines(PipelineBuildContext& ctx, + std::optional> required_columns) const { + child->BuildPipelines(ctx, required_columns); + auto limit_op = std::make_shared(limit); + ctx.current_pipeline->transforms.push_back(limit_op); } -PhysicalOperatorContext ScalarNode::BuildPhysicalPlan( - std::optional> required_columns) const { +// ========================= ScalarNode ========================= + +void ScalarNode::BuildPipelines(PipelineBuildContext& ctx, + std::optional> required_columns) const { std::vector needed_columns; if (required_columns.has_value()) { for (const auto& req_col : required_columns.value()) { @@ -179,26 +216,24 @@ PhysicalOperatorContext ScalarNode::BuildPhysicalPlan( needed_columns.erase(std::unique(needed_columns.begin(), needed_columns.end()), needed_columns.end()); - PhysicalOperatorContext child_context = child->BuildPhysicalPlan(needed_columns); + child->BuildPipelines(ctx, needed_columns); std::vector> scalar_functions; - for (const auto& expr : scalar_expressions) { - scalar_functions.push_back(expr->CreateScalarFunction(child_context.schema)); + scalar_functions.push_back(expr->CreateScalarFunction(ctx.schema)); } - auto op = std::make_unique(std::move(child_context.root_operator), - std::move(scalar_functions)); - return {std::move(op), std::move(child_context.schema)}; + auto scalar_op = std::make_shared(std::move(scalar_functions)); + ctx.current_pipeline->transforms.push_back(scalar_op); } -PhysicalOperatorContext DropNode::BuildPhysicalPlan( - std::optional> required_columns) const { - PhysicalOperatorContext child_context = child->BuildPhysicalPlan(required_columns); +// ========================= DropNode ========================= - std::vector cols_to_keep_indices; - Schema output_schema; - const auto& child_fields = child_context.schema.GetFields(); +void DropNode::BuildPipelines(PipelineBuildContext& ctx, + std::optional> required_columns) const { + child->BuildPipelines(ctx, required_columns); + + const auto& child_fields = ctx.schema.GetFields(); for (const std::string& drop_col : columns_to_drop) { bool found = false; @@ -213,6 +248,8 @@ PhysicalOperatorContext DropNode::BuildPhysicalPlan( } } + std::vector cols_to_keep_indices; + Schema output_schema; for (size_t i = 0; i < child_fields.size(); ++i) { const auto& field = child_fields[i]; if (std::find(columns_to_drop.begin(), columns_to_drop.end(), field.name) == @@ -222,18 +259,21 @@ PhysicalOperatorContext DropNode::BuildPhysicalPlan( } } - auto op = std::make_unique(std::move(child_context.root_operator), - std::move(cols_to_keep_indices)); - return {std::move(op), std::move(output_schema)}; + auto drop_op = std::make_shared(std::move(cols_to_keep_indices)); + ctx.current_pipeline->transforms.push_back(drop_op); + ctx.schema = std::move(output_schema); } -PhysicalOperatorContext ReorderNode::BuildPhysicalPlan( - std::optional> required_columns) const { - PhysicalOperatorContext child_context = child->BuildPhysicalPlan(required_columns); +// ========================= ReorderNode ========================= + +void ReorderNode::BuildPipelines(PipelineBuildContext& ctx, + std::optional> required_columns) const { + child->BuildPipelines(ctx, required_columns); + + const auto& child_fields = ctx.schema.GetFields(); std::vector new_indices; Schema output_schema; - const auto& child_fields = child_context.schema.GetFields(); for (const std::string& reorder_col : desired_order) { bool found = false; @@ -253,7 +293,7 @@ PhysicalOperatorContext ReorderNode::BuildPhysicalPlan( } } - auto op = std::make_unique(std::move(child_context.root_operator), - std::move(new_indices)); - return {std::move(op), std::move(output_schema)}; + auto reorder_op = std::make_shared(std::move(new_indices)); + ctx.current_pipeline->transforms.push_back(reorder_op); + ctx.schema = std::move(output_schema); } diff --git a/src/execution/ExecutionLogic.h b/src/execution/ExecutionLogic.h index 7037d0c..b59ea67 100644 --- a/src/execution/ExecutionLogic.h +++ b/src/execution/ExecutionLogic.h @@ -1,6 +1,7 @@ #pragma once #include "execution/OperatorsBase.h" +#include "execution/PipelineExecutor.h" #include "execution/expressions/AggregationExpressions.h" #include "execution/expressions/FilterExpressions.h" #include "execution/expressions/ScalarExpressions.h" @@ -13,15 +14,11 @@ #include #include -struct PhysicalOperatorContext { - std::unique_ptr root_operator; - Schema schema; -}; - struct PlanNode { virtual ~PlanNode() = default; - virtual PhysicalOperatorContext BuildPhysicalPlan( + virtual void BuildPipelines( + PipelineBuildContext& ctx, std::optional> required_columns = std::nullopt) const = 0; }; @@ -37,8 +34,8 @@ struct ScanNode : public PlanNode { table_meta(std::move(table_m)) { } - PhysicalOperatorContext BuildPhysicalPlan( - std::optional> required_columns) const override; + void BuildPipelines(PipelineBuildContext& ctx, + std::optional> required_columns) const override; }; struct AggregateNode : public PlanNode { @@ -53,8 +50,8 @@ struct AggregateNode : public PlanNode { aggregate_expressions(std::move(agg_exprs)) { } - PhysicalOperatorContext BuildPhysicalPlan( - std::optional> required_columns) const override; + void BuildPipelines(PipelineBuildContext& ctx, + std::optional> required_columns) const override; }; struct FilterNode : public PlanNode { @@ -65,8 +62,8 @@ struct FilterNode : public PlanNode { : child(std::move(c)), filter_expression(std::move(filter_expr)) { } - PhysicalOperatorContext BuildPhysicalPlan( - std::optional> required_columns) const override; + void BuildPipelines(PipelineBuildContext& ctx, + std::optional> required_columns) const override; }; struct OrderByNode : public PlanNode { @@ -81,8 +78,8 @@ struct OrderByNode : public PlanNode { : child(std::move(c)), order_by_columns(std::move(order_by_cols)), limit(lim), offset(off) { } - PhysicalOperatorContext BuildPhysicalPlan( - std::optional> required_columns) const override; + void BuildPipelines(PipelineBuildContext& ctx, + std::optional> required_columns) const override; }; struct LimitNode : public PlanNode { @@ -92,8 +89,8 @@ struct LimitNode : public PlanNode { LimitNode(std::shared_ptr c, size_t lim) : child(std::move(c)), limit(lim) { } - PhysicalOperatorContext BuildPhysicalPlan( - std::optional> required_columns) const override; + void BuildPipelines(PipelineBuildContext& ctx, + std::optional> required_columns) const override; }; struct ScalarNode : public PlanNode { @@ -105,8 +102,8 @@ struct ScalarNode : public PlanNode { : child(std::move(c)), scalar_expressions(std::move(scalar_exprs)) { } - PhysicalOperatorContext BuildPhysicalPlan( - std::optional> required_columns) const override; + void BuildPipelines(PipelineBuildContext& ctx, + std::optional> required_columns) const override; }; struct DropNode : public PlanNode { @@ -117,8 +114,8 @@ struct DropNode : public PlanNode { : child(std::move(c)), columns_to_drop(std::move(cols_to_drop)) { } - PhysicalOperatorContext BuildPhysicalPlan( - std::optional> required_columns) const override; + void BuildPipelines(PipelineBuildContext& ctx, + std::optional> required_columns) const override; }; struct ReorderNode : public PlanNode { @@ -129,6 +126,6 @@ struct ReorderNode : public PlanNode { : child(std::move(c)), desired_order(std::move(desired_ord)) { } - PhysicalOperatorContext BuildPhysicalPlan( - std::optional> required_columns) const override; + void BuildPipelines(PipelineBuildContext& ctx, + std::optional> required_columns) const override; }; diff --git a/src/execution/OperatorsBase.cpp b/src/execution/OperatorsBase.cpp index ce653ed..82158d0 100644 --- a/src/execution/OperatorsBase.cpp +++ b/src/execution/OperatorsBase.cpp @@ -7,6 +7,8 @@ #include +// ========================= ScanOperator ========================= + ScanOperator::ScanOperator(const std::string& table_path, const std::vector& column_names, std::shared_ptr metadata) @@ -32,18 +34,19 @@ ScanOperator::ScanOperator(const std::string& table_path, } } -std::unique_ptr ScanOperator::Run() { +std::unique_ptr ScanOperator::GetData() { if (column_indices_.empty()) { - if (finished_empty_scan_) { - return nullptr; + bool expected = false; + if (finished_empty_scan_.compare_exchange_strong(expected, true)) { + auto record_batch = std::make_unique(); + record_batch->num_rows = metadata_->GetNumRows(); + return record_batch; } - auto record_batch = std::make_unique(); - record_batch->num_rows = metadata_->GetNumRows(); - finished_empty_scan_ = true; - return record_batch; + return nullptr; } - if (current_chunk_ >= total_chunks_) { + size_t cur_chunk = current_chunk_.fetch_add(1); + if (cur_chunk >= total_chunks_) { return nullptr; } @@ -51,152 +54,218 @@ std::unique_ptr ScanOperator::Run() { record_batch->num_rows = 0; for (size_t col_idx : column_indices_) { - std::shared_ptr column_data = reader_.GetColumnData(col_idx, current_chunk_); + std::shared_ptr column_data = reader_.GetColumnData(col_idx, cur_chunk); if (record_batch->num_rows == 0) { record_batch->num_rows = column_data->Size(); } record_batch->columns.push_back(std::move(column_data)); } - ++current_chunk_; return record_batch; } -AggregationOperator::AggregationOperator( - std::unique_ptr child, - std::vector> aggregation_functions) - : child_(std::move(child)), aggregation_functions_(std::move(aggregation_functions)) { +// ========================= FilterTransformOperator ========================= + +FilterTransformOperator::FilterTransformOperator(std::shared_ptr filter_function) + : filter_function_(std::move(filter_function)) { } -std::unique_ptr AggregationOperator::Run() { - if (finished_) { +std::unique_ptr FilterTransformOperator::Execute(std::unique_ptr batch) { + std::vector selection_vector = filter_function_->Evaluate(*batch); + if (selection_vector.empty()) { return nullptr; } - while (std::unique_ptr batch = child_->Run()) { - for (std::unique_ptr& aggregation_function : - aggregation_functions_) { - aggregation_function->Update(*batch); - } + batch->num_rows = selection_vector.size(); + batch->selection_vector = + std::make_shared>(std::move(selection_vector)); + return batch; +} + +// ========================= ScalarTransformOperator ========================= + +ScalarTransformOperator::ScalarTransformOperator( + std::vector> scalar_functions) + : scalar_functions_(std::move(scalar_functions)) { +} + +std::unique_ptr ScalarTransformOperator::Execute(std::unique_ptr batch) { + for (auto& func : scalar_functions_) { + batch->columns.push_back(func->Evaluate(*batch)); } - auto result_batch = std::make_unique(); - result_batch->num_rows = 1; + return batch; +} - for (std::unique_ptr& aggregation_function : - aggregation_functions_) { - result_batch->columns.push_back(aggregation_function->Finalize()); +// ========================= DropTransformOperator ========================= + +DropTransformOperator::DropTransformOperator(std::vector column_stay_indices) + : column_stay_indices_(std::move(column_stay_indices)) { +} + +std::unique_ptr DropTransformOperator::Execute(std::unique_ptr batch) { + std::vector> new_columns; + new_columns.reserve(column_stay_indices_.size()); + + for (size_t ind : column_stay_indices_) { + new_columns.push_back(std::move(batch->columns[ind])); } - finished_ = true; - return result_batch; + batch->columns = std::move(new_columns); + return batch; } -FilterOperator::FilterOperator(std::unique_ptr child, - std::shared_ptr filter_function) - : child_(std::move(child)), filter_function_(std::move(filter_function)) { +// ========================= LimitTransformOperator ========================= + +LimitTransformOperator::LimitTransformOperator(size_t limit) : limit_(limit) { } -std::unique_ptr FilterOperator::Run() { - while (std::unique_ptr batch = child_->Run()) { - std::vector selection_vector = filter_function_->Evaluate(*batch); - if (selection_vector.empty()) { - continue; +std::unique_ptr LimitTransformOperator::Execute(std::unique_ptr batch) { + if (cur_rows_ >= limit_) { + done_ = true; + return nullptr; + } + + if (cur_rows_ + batch->num_rows <= limit_) { + cur_rows_ += batch->num_rows; + if (cur_rows_ == limit_) { + done_ = true; } - batch->num_rows = selection_vector.size(); - batch->selection_vector = - std::make_shared>(std::move(selection_vector)); return batch; + } else { + size_t remaining = limit_ - cur_rows_; + cur_rows_ = limit_; + done_ = true; + batch->num_rows = remaining; + if (batch->selection_vector) { + auto new_sel_vec = std::make_shared>( + batch->selection_vector->begin(), batch->selection_vector->begin() + remaining); + batch->selection_vector = std::move(new_sel_vec); + } else { + auto sel_vec = std::make_shared>(remaining); + std::iota(sel_vec->begin(), sel_vec->end(), 0); + batch->selection_vector = std::move(sel_vec); + } + return batch; + } +} + +// ========================= AggregationSinkSourceOperator ========================= + +AggregationSinkSourceOperator::AggregationSinkSourceOperator( + std::vector> aggregation_functions) + : aggregation_functions_(std::move(aggregation_functions)) { +} + +void AggregationSinkSourceOperator::Sink(std::unique_ptr batch) { + for (auto& aggregation_function : aggregation_functions_) { + aggregation_function->Update(*batch); + } +} + +void AggregationSinkSourceOperator::Finalize() { + result_batch_ = std::make_unique(); + result_batch_->num_rows = 1; + + for (auto& aggregation_function : aggregation_functions_) { + result_batch_->columns.push_back(aggregation_function->Finalize()); + } +} + +std::unique_ptr AggregationSinkSourceOperator::GetData() { + if (emitted_) { + return nullptr; } - return nullptr; + emitted_ = true; + return std::move(result_batch_); } -GroupByOperator::GroupByOperator(std::unique_ptr child, - std::vector group_by_col_indices, - std::vector> key_builders, - std::vector> agg_funcs) - : child_(std::move(child)), - group_by_col_indices_(std::move(group_by_col_indices)), +// ========================= GroupBySinkSourceOperator ========================= + +GroupBySinkSourceOperator::GroupBySinkSourceOperator( + std::vector group_by_col_indices, + std::vector> key_builders, + std::vector> agg_funcs) + : group_by_col_indices_(std::move(group_by_col_indices)), key_builders_(std::move(key_builders)), agg_funcs_(std::move(agg_funcs)) { } -std::unique_ptr GroupByOperator::Run() { - if (!accumulated_) { - while (auto batch = child_->Run()) { - std::vector group_ids; - size_t batch_size = batch->num_rows; - group_ids.reserve(batch_size); - - const std::shared_ptr>& sel_vec = batch->selection_vector; - std::vector composite_keys(batch_size); - for (size_t col_idx : group_by_col_indices_) { - ExecutionHelper::HashKeys(*batch->columns[col_idx], composite_keys, sel_vec.get()); - } - - std::vector new_group_indices; - size_t sz = num_groups_; - - bool is_filtered = sel_vec != nullptr; - for (size_t i = 0; i < batch_size; ++i) { - const std::string& key = composite_keys[i]; - auto it = hash_table_.find(key); - uint32_t gid; - - if (it == hash_table_.end()) { - gid = sz + new_group_indices.size(); - hash_table_[key] = gid; - size_t actual_idx = is_filtered ? (*sel_vec)[i] : i; - new_group_indices.push_back(actual_idx); - } else { - gid = it->second; - } - group_ids.push_back(gid); - } +void GroupBySinkSourceOperator::Sink(std::unique_ptr batch) { + std::vector group_ids; + size_t batch_size = batch->num_rows; + group_ids.reserve(batch_size); - if (!new_group_indices.empty()) { - for (size_t k = 0; k < group_by_col_indices_.size(); ++k) { - ExecutionHelper::CopySelection(*key_builders_[k], - *batch->columns[group_by_col_indices_[k]], - &new_group_indices); - } - } + const std::shared_ptr>& sel_vec = batch->selection_vector; + std::vector composite_keys(batch_size); + for (size_t col_idx : group_by_col_indices_) { + ExecutionHelper::HashKeys(*batch->columns[col_idx], composite_keys, sel_vec.get()); + } - num_groups_ = hash_table_.size(); - for (auto& func : agg_funcs_) { - func->Resize(num_groups_); - func->Update(*batch, group_ids); - } - } - accumulated_ = true; + std::vector new_group_indices; + size_t sz = num_groups_; - auto result_batch = std::make_unique(); - result_batch->num_rows = num_groups_; + bool is_filtered = sel_vec != nullptr; + for (size_t i = 0; i < batch_size; ++i) { + const std::string& key = composite_keys[i]; + auto it = hash_table_.find(key); + uint32_t gid; - for (size_t k = 0; k < key_builders_.size(); ++k) { - result_batch->columns.push_back(key_builders_[k]->Finish()); + if (it == hash_table_.end()) { + gid = sz + new_group_indices.size(); + hash_table_[key] = gid; + size_t actual_idx = is_filtered ? (*sel_vec)[i] : i; + new_group_indices.push_back(actual_idx); + } else { + gid = it->second; } - for (size_t a = 0; a < agg_funcs_.size(); ++a) { - result_batch->columns.push_back(agg_funcs_[a]->Finalize()); + group_ids.push_back(gid); + } + + if (!new_group_indices.empty()) { + for (size_t k = 0; k < group_by_col_indices_.size(); ++k) { + ExecutionHelper::CopySelection(*key_builders_[k], + *batch->columns[group_by_col_indices_[k]], + &new_group_indices); } - return result_batch; } - return nullptr; + num_groups_ = hash_table_.size(); + for (auto& func : agg_funcs_) { + func->Resize(num_groups_); + func->Update(*batch, group_ids); + } +} + +void GroupBySinkSourceOperator::Finalize() { + result_batch_ = std::make_unique(); + result_batch_->num_rows = num_groups_; + + for (size_t k = 0; k < key_builders_.size(); ++k) { + result_batch_->columns.push_back(key_builders_[k]->Finish()); + } + for (size_t a = 0; a < agg_funcs_.size(); ++a) { + result_batch_->columns.push_back(agg_funcs_[a]->Finalize()); + } } -OrderByOperator::OrderByOperator(std::unique_ptr child, - std::vector> sort_columns, - std::optional limit, std::optional offset) - : child_(std::move(child)), - sort_columns_(std::move(sort_columns)), - limit_(limit), - offset_(offset) { +std::unique_ptr GroupBySinkSourceOperator::GetData() { + if (emitted_) { + return nullptr; + } + emitted_ = true; + return std::move(result_batch_); +} + +// ========================= OrderBySinkSourceOperator ========================= + +OrderBySinkSourceOperator::OrderBySinkSourceOperator( + std::vector> sort_columns, std::optional limit, + std::optional offset) + : sort_columns_(std::move(sort_columns)), limit_(limit), offset_(offset) { total_limit_ = limit_; if (total_limit_.has_value()) { if (offset_.has_value()) { total_limit_.value() += offset_.value(); } - } else { - total_limit_ = offset_; } } @@ -221,50 +290,48 @@ int Compare(const void* raw_data, uint32_t a, uint32_t b) { return 0; } -std::unique_ptr OrderByOperator::Run() { - if (accumulated_) { - return nullptr; - } - - while (auto batch = child_->Run()) { - if (accum_builders_.empty()) { - for (size_t c = 0; c < batch->columns.size(); ++c) { - accum_builders_.push_back( - ColumnFactory::MakeColumnBuilder(batch->columns[c]->GetType())); - accum_types_.push_back(batch->columns[c]->GetType()); - } - accum_num_rows_ = 0; +void OrderBySinkSourceOperator::Sink(std::unique_ptr batch) { + if (accum_builders_.empty()) { + for (size_t c = 0; c < batch->columns.size(); ++c) { + accum_builders_.push_back( + ColumnFactory::MakeColumnBuilder(batch->columns[c]->GetType())); + accum_types_.push_back(batch->columns[c]->GetType()); } + accum_num_rows_ = 0; + } - const std::shared_ptr>& sel = batch->selection_vector; - for (size_t c = 0; c < accum_builders_.size(); ++c) { - ExecutionHelper::CopySelection(*accum_builders_[c], *batch->columns[c], sel.get()); - } - accum_num_rows_ += batch->num_rows; + const std::shared_ptr>& sel = batch->selection_vector; + for (size_t c = 0; c < accum_builders_.size(); ++c) { + ExecutionHelper::CopySelection(*accum_builders_[c], *batch->columns[c], sel.get()); + } + accum_num_rows_ += batch->num_rows; - if (total_limit_.has_value() && accum_num_rows_ > total_limit_.value() * 10) { - TrimCurBatch(false); - } + if (total_limit_.has_value() && accum_num_rows_ > total_limit_.value() * 10) { + TrimCurBatch(false); } +} +void OrderBySinkSourceOperator::Finalize() { if (accum_num_rows_ > 0) { TrimCurBatch(true); - auto final_batch = std::make_unique(); - final_batch->num_rows = accum_num_rows_; + result_batch_ = std::make_unique(); + result_batch_->num_rows = accum_num_rows_; for (auto& builder : accum_builders_) { - final_batch->columns.push_back(builder->Finish()); + result_batch_->columns.push_back(builder->Finish()); } - - accumulated_ = true; - return final_batch; } +} - accumulated_ = true; - return nullptr; +std::unique_ptr OrderBySinkSourceOperator::GetData() { + if (emitted_) { + return nullptr; + } + emitted_ = true; + return std::move(result_batch_); } -void OrderByOperator::TrimCurBatch(bool is_final) { +void OrderBySinkSourceOperator::TrimCurBatch(bool is_final) { if (accum_num_rows_ == 0) { return; } @@ -334,76 +401,3 @@ void OrderByOperator::TrimCurBatch(bool is_final) { } accum_num_rows_ = indices.size(); } - -LimitOperator::LimitOperator(std::unique_ptr child, size_t limit) - : child_(std::move(child)), limit_(limit) { -} - -std::unique_ptr LimitOperator::Run() { - if (cur_rows_ >= limit_) { - return nullptr; - } - std::unique_ptr batch = child_->Run(); - if (!batch) { - return nullptr; - } - - if (cur_rows_ + batch->num_rows <= limit_) { - cur_rows_ += batch->num_rows; - return batch; - } else { - size_t remaining = limit_ - cur_rows_; - cur_rows_ += remaining; - batch->num_rows = remaining; - if (batch->selection_vector) { - auto new_sel_vec = std::make_shared>( - batch->selection_vector->begin(), batch->selection_vector->begin() + remaining); - batch->selection_vector = std::move(new_sel_vec); - } else { - auto sel_vec = std::make_shared>(remaining); - std::iota(sel_vec->begin(), sel_vec->end(), 0); - batch->selection_vector = std::move(sel_vec); - } - - return batch; - } -} - -ScalarOperator::ScalarOperator(std::unique_ptr child, - std::vector> scalar_functions) - : child_(std::move(child)), scalar_functions_(std::move(scalar_functions)) { -} - -std::unique_ptr ScalarOperator::Run() { - auto batch = child_->Run(); - if (!batch) { - return nullptr; - } - - for (auto& func : scalar_functions_) { - batch->columns.push_back(func->Evaluate(*batch)); - } - - return batch; -} - -DropOperator::DropOperator(std::unique_ptr child, std::vector column_stay_indices) - : child_(std::move(child)), column_stay_indices_(std::move(column_stay_indices)) { -} - -std::unique_ptr DropOperator::Run() { - auto batch = child_->Run(); - if (!batch) { - return nullptr; - } - - std::vector> new_columns; - new_columns.reserve(column_stay_indices_.size()); - - for (size_t ind : column_stay_indices_) { - new_columns.push_back(std::move(batch->columns[ind])); - } - - batch->columns = std::move(new_columns); - return batch; -} diff --git a/src/execution/OperatorsBase.h b/src/execution/OperatorsBase.h index a3c47f0..64d6c6b 100644 --- a/src/execution/OperatorsBase.h +++ b/src/execution/OperatorsBase.h @@ -1,177 +1,173 @@ #pragma once +#include "execution/Pipeline.h" #include "io/ColumnarReader.h" #include "column/ColumnBuilder.h" +#include "utils/Concurrency.h" + +#include "absl/container/flat_hash_map.h" #include #include -#include #include +#include +#include class GlobalAggregationFunction; class GroupedAggregationFunction; class FilterFunction; class ScalarFunction; -struct RecordBatch { - size_t num_rows; - std::shared_ptr> selection_vector; - std::vector> columns; -}; - -class Operator { -public: - virtual ~Operator() = default; - - virtual std::unique_ptr Run() = 0; -}; +// ========================= Source Operators ========================= -class ScanOperator : public Operator { +class ScanOperator : public SourceOperator { public: ScanOperator(const std::string& table_path, const std::vector& column_names, std::shared_ptr metadata); - std::unique_ptr Run() override; + std::unique_ptr GetData() override; private: ColumnarReader reader_; std::shared_ptr metadata_; std::vector column_indices_; - size_t current_chunk_ = 0; + + Atomic current_chunk_{0}; size_t total_chunks_ = 0; - bool finished_empty_scan_ = false; + Atomic finished_empty_scan_{false}; }; -class AggregationOperator : public Operator { +// ========================= Transform Operators ========================= + +class FilterTransformOperator : public TransformOperator { public: - AggregationOperator( - std::unique_ptr child, - std::vector> aggregation_functions); + FilterTransformOperator(std::shared_ptr filter_function); - std::unique_ptr Run() override; + std::unique_ptr Execute(std::unique_ptr batch) override; private: - std::unique_ptr child_; - std::vector> aggregation_functions_; - bool finished_ = false; + std::shared_ptr filter_function_; }; -class FilterOperator : public Operator { +class ScalarTransformOperator : public TransformOperator { public: - FilterOperator(std::unique_ptr child, - std::shared_ptr filter_function); + ScalarTransformOperator(std::vector> scalar_functions); - std::unique_ptr Run() override; + std::unique_ptr Execute(std::unique_ptr batch) override; private: - std::unique_ptr child_; - std::shared_ptr filter_function_; + std::vector> scalar_functions_; }; -class GroupByOperator : public Operator { +class DropTransformOperator : public TransformOperator { public: - GroupByOperator(std::unique_ptr child, std::vector group_by_col_indices, - std::vector> key_builders, - std::vector> agg_funcs); + DropTransformOperator(std::vector column_stay_indices); - std::unique_ptr Run() override; + std::unique_ptr Execute(std::unique_ptr batch) override; private: - std::unique_ptr child_; - std::vector group_by_col_indices_; - std::vector> key_builders_; - std::vector> agg_funcs_; - bool accumulated_ = false; - size_t num_groups_ = 0; - - std::unordered_map hash_table_; + std::vector column_stay_indices_; }; -class OrderByOperator : public Operator { +class ReorderTransformOperator : public TransformOperator { public: - OrderByOperator(std::unique_ptr child, - std::vector> sort_columns, std::optional limit, - std::optional offset); + ReorderTransformOperator(std::vector new_indices) + : new_indices_(std::move(new_indices)) { + } - std::unique_ptr Run() override; + std::unique_ptr Execute(std::unique_ptr batch) override { + auto new_batch = std::make_unique(); + new_batch->num_rows = batch->num_rows; + new_batch->selection_vector = batch->selection_vector; -private: - void TrimCurBatch(bool is_final); + new_batch->columns.reserve(new_indices_.size()); + for (size_t ind : new_indices_) { + new_batch->columns.push_back(batch->columns[ind]); + } - std::unique_ptr child_; - std::vector> sort_columns_; - std::unique_ptr accumulated_batch_; - std::vector> accum_builders_; - std::vector accum_types_; - size_t accum_num_rows_ = 0; - std::vector indices_; - size_t current_idx_ = 0; - std::optional limit_; - std::optional offset_; - std::optional total_limit_; - bool accumulated_ = false; + return new_batch; + } + +private: + std::vector new_indices_; }; -class LimitOperator : public Operator { +class LimitTransformOperator : public TransformOperator { public: - LimitOperator(std::unique_ptr child, size_t limit); + LimitTransformOperator(size_t limit); - std::unique_ptr Run() override; + std::unique_ptr Execute(std::unique_ptr batch) override; + + bool IsPipelineDone() const override { + return done_; + } private: - std::unique_ptr child_; size_t limit_; size_t cur_rows_ = 0; + bool done_ = false; }; -class ScalarOperator : public Operator { +// ========================= Pipeline Breakers (Sink + Source) ========================= + +class AggregationSinkSourceOperator : public SinkOperator, public SourceOperator { public: - ScalarOperator(std::unique_ptr child, - std::vector> scalar_functions); + AggregationSinkSourceOperator( + std::vector> aggregation_functions); + + void Sink(std::unique_ptr batch) override; + void Finalize() override; - std::unique_ptr Run() override; + std::unique_ptr GetData() override; private: - std::unique_ptr child_; - std::vector> scalar_functions_; + std::vector> aggregation_functions_; + std::unique_ptr result_batch_; + bool emitted_ = false; }; -class DropOperator : public Operator { +class GroupBySinkSourceOperator : public SinkOperator, public SourceOperator { public: - DropOperator(std::unique_ptr child, std::vector column_stay_indices); + GroupBySinkSourceOperator(std::vector group_by_col_indices, + std::vector> key_builders, + std::vector> agg_funcs); + + void Sink(std::unique_ptr batch) override; + void Finalize() override; - std::unique_ptr Run() override; + std::unique_ptr GetData() override; private: - std::unique_ptr child_; - std::vector column_stay_indices_; + std::vector group_by_col_indices_; + std::vector> key_builders_; + std::vector> agg_funcs_; + size_t num_groups_ = 0; + + absl::flat_hash_map hash_table_; + std::unique_ptr result_batch_; + bool emitted_ = false; }; -class ReorderOperator : public Operator { +class OrderBySinkSourceOperator : public SinkOperator, public SourceOperator { public: - ReorderOperator(std::unique_ptr child, std::vector new_indices) - : child_(std::move(child)), new_indices_(std::move(new_indices)) { - } + OrderBySinkSourceOperator(std::vector> sort_columns, + std::optional limit, std::optional offset); - std::unique_ptr Run() override { - auto batch = child_->Run(); - if (!batch) { - return nullptr; - } + void Sink(std::unique_ptr batch) override; + void Finalize() override; - auto new_batch = std::make_unique(); - new_batch->num_rows = batch->num_rows; - new_batch->selection_vector = batch->selection_vector; - - new_batch->columns.reserve(new_indices_.size()); - for (size_t ind : new_indices_) { - new_batch->columns.push_back(batch->columns[ind]); - } - - return new_batch; - } + std::unique_ptr GetData() override; private: - std::unique_ptr child_; - std::vector new_indices_; + void TrimCurBatch(bool is_final); + + std::vector> sort_columns_; + std::vector> accum_builders_; + std::vector accum_types_; + size_t accum_num_rows_ = 0; + std::optional limit_; + std::optional offset_; + std::optional total_limit_; + std::unique_ptr result_batch_; + bool emitted_ = false; }; diff --git a/src/execution/Pipeline.h b/src/execution/Pipeline.h new file mode 100644 index 0000000..f900bd6 --- /dev/null +++ b/src/execution/Pipeline.h @@ -0,0 +1,76 @@ +#pragma once + +#include "column/Column.h" + +#include +#include + +struct RecordBatch { + size_t num_rows; + std::shared_ptr> selection_vector; + std::vector> columns; +}; + +// generates RecordBatches (for example scan operator) +class SourceOperator { +public: + virtual ~SourceOperator() = default; + virtual std::unique_ptr GetData() = 0; +}; + +// transforms per-batch data +class TransformOperator { +public: + virtual ~TransformOperator() = default; + virtual std::unique_ptr Execute(std::unique_ptr batch) = 0; + + virtual bool IsPipelineDone() const { + return false; + } +}; + +// accumulates all data, breaks pipeline +class SinkOperator { +public: + virtual ~SinkOperator() = default; + + virtual void Sink(std::unique_ptr batch) = 0; + virtual void Finalize() = 0; +}; + +class Pipeline { +public: + std::shared_ptr source; + std::vector> transforms; + std::shared_ptr sink; + + void Execute() { + while (auto batch = source->GetData()) { + bool skip = false; + bool stop = false; + + for (std::shared_ptr& transform : transforms) { + batch = transform->Execute(std::move(batch)); + if (!batch || batch->num_rows == 0) { + skip = true; + break; + } + if (transform->IsPipelineDone()) { + stop = true; + break; + } + } + + if (skip) { + continue; + } + if (batch && batch->num_rows > 0) { + sink->Sink(std::move(batch)); + } + if (stop) { + break; + } + } + sink->Finalize(); + } +}; diff --git a/src/execution/PipelineExecutor.h b/src/execution/PipelineExecutor.h new file mode 100644 index 0000000..c8eec1d --- /dev/null +++ b/src/execution/PipelineExecutor.h @@ -0,0 +1,31 @@ +#pragma once + +#include "execution/Pipeline.h" +#include "column/Schema.h" + +#include +#include + +class ResultSinkOperator : public SinkOperator { +public: + void Sink(std::unique_ptr batch) override { + batches_.push_back(std::move(batch)); + } + + void Finalize() override { + // charonchik bebe + } + + std::vector> TakeBatches() { + return std::move(batches_); + } + +private: + std::vector> batches_; +}; + +struct PipelineBuildContext { + Pipeline* current_pipeline = nullptr; + std::vector> completed_pipelines; + Schema schema; +}; diff --git a/src/execution/expressions/AggregationFunctions.h b/src/execution/expressions/AggregationFunctions.h index db9950d..eec99f9 100644 --- a/src/execution/expressions/AggregationFunctions.h +++ b/src/execution/expressions/AggregationFunctions.h @@ -11,9 +11,10 @@ #include "column/ColumnFactory.h" +#include "absl/container/flat_hash_set.h" + #include #include -#include #include class AggregationFunction { @@ -352,7 +353,7 @@ class DistinctCountGlobalAggregationFunction : public GlobalAggregationFunction private: size_t column_index_; - std::unordered_set distinct_values_; + absl::flat_hash_set distinct_values_; }; template @@ -390,5 +391,5 @@ class DistinctCountGroupedAggregationFunction : public GroupedAggregationFunctio private: size_t column_index_; - std::vector> distinct_values_; + std::vector> distinct_values_; }; diff --git a/src/execution/expressions/ScalarFunctions.h b/src/execution/expressions/ScalarFunctions.h index 95d2d18..ca0a3a2 100644 --- a/src/execution/expressions/ScalarFunctions.h +++ b/src/execution/expressions/ScalarFunctions.h @@ -7,7 +7,8 @@ #include "column/NumericColumn.h" #include "types/String.h" -#include +#include + #include class ScalarFunction { @@ -213,8 +214,8 @@ class RegexpReplaceFunction : public ScalarFunction { AggExpHelper::IterateColumnData( batch, col_ind_, [&](const auto& value, size_t, size_t real_ind) { std::string str(value); - temp_res[real_ind] = - std::regex_replace(str, regex_, replacement_, std::regex_constants::format_sed); + re2::RE2::GlobalReplace(&str, regex_, replacement_); + temp_res[real_ind] = std::move(str); }); StringColumnBuilder builder; @@ -226,6 +227,6 @@ class RegexpReplaceFunction : public ScalarFunction { private: size_t col_ind_; - std::regex regex_; + re2::RE2 regex_; std::string replacement_; }; diff --git a/src/io/ColumnarReader.cpp b/src/io/ColumnarReader.cpp index 58b71c9..f93ff6b 100644 --- a/src/io/ColumnarReader.cpp +++ b/src/io/ColumnarReader.cpp @@ -5,6 +5,9 @@ #include "utils/Macro.h" #include +#include +#include +#include void MetadataTable::ParseMetadata(const std::string& file_path) { std::ifstream file(file_path, std::ios::binary); @@ -81,9 +84,16 @@ void MetadataTable::ParseMetadata(const std::string& file_path) { ColumnarReader::ColumnarReader(const std::string& file_name, std::shared_ptr meta) - : file_(file_name, std::ios::binary), metadata_(std::move(meta)) { - if (!file_.is_open()) { - THROW_RUNTIME_ERROR("Could not open file for reading data"); + : metadata_(std::move(meta)) { + fd_ = open(file_name.c_str(), O_RDONLY); + if (fd_ < 0) { + THROW_RUNTIME_ERROR("Could not open file for reading data: " + file_name); + } +} + +ColumnarReader::~ColumnarReader() { + if (fd_ >= 0) { + close(fd_); } } @@ -111,8 +121,10 @@ void ColumnarReader::ReadRawColumnData(size_t column_index, size_t chunk_index, uint64_t size = meta.sizes[chunk_index]; buffer.resize(size); - file_.seekg(offset, std::ios::beg); - file_.read(buffer.data(), size); + ssize_t bytes_read = pread(fd_, buffer.data(), size, offset); + if (bytes_read != static_cast(size)) { + THROW_RUNTIME_ERROR("Failed to read data"); + } } std::shared_ptr ColumnarReader::GetColumnData(size_t column_index, diff --git a/src/io/ColumnarReader.h b/src/io/ColumnarReader.h index 3d3ef0f..73d69ad 100644 --- a/src/io/ColumnarReader.h +++ b/src/io/ColumnarReader.h @@ -3,10 +3,8 @@ #include "types/ColumnType.h" #include "column/Column.h" -#include #include #include -#include #include class MetadataTable { @@ -38,6 +36,7 @@ class MetadataTable { class ColumnarReader { public: ColumnarReader(const std::string& file_name, std::shared_ptr meta); + ~ColumnarReader(); uint64_t GetNumRows() const { return metadata_->GetNumRows(); @@ -49,6 +48,6 @@ class ColumnarReader { std::shared_ptr GetColumnData(size_t column_index, size_t chunk_index) const; private: - mutable std::ifstream file_; + int fd_ = -1; std::shared_ptr metadata_; }; diff --git a/src/main.cpp b/src/main.cpp index 3a4aa65..06ca8ed 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -2,7 +2,7 @@ #include "execution/ExecutionApi.h" #include -#include +#include class Query { public: @@ -427,7 +427,7 @@ class Query { .Aggregate({"URL"}, {Count("*", "c")}) .OrderBy({{"c", true}}, 10) .Project({Literal("const_1", 1)}) - .Reoder({"const_1", "URL", "c"}) + .Reorder({"const_1", "URL", "c"}) .Collect(); df.Display(); } @@ -441,7 +441,7 @@ class Query { .Project({AddConst("ClientIP", -1, "ClientIP_minus_1"), AddConst("ClientIP", -2, "ClientIP_minus_2"), AddConst("ClientIP", -3, "ClientIP_minus_3")}) - .Reoder({"ClientIP", "ClientIP_minus_1", "ClientIP_minus_2", + .Reorder({"ClientIP", "ClientIP_minus_1", "ClientIP_minus_2", "ClientIP_minus_3", "c"}) .Collect(); diff --git a/src/utils/Concurrency.h b/src/utils/Concurrency.h new file mode 100644 index 0000000..0343105 --- /dev/null +++ b/src/utils/Concurrency.h @@ -0,0 +1,76 @@ +#pragma once + +#ifdef ENABLE_MULTITHREADING + +#include +#include + +template +using Atomic = std::atomic; + +using Mutex = std::mutex; + +#else + +template +class DummyAtomic { +public: + DummyAtomic(T val = 0) : val_(val) { + } + + T fetch_add(T arg, std::memory_order /*order*/ = std::memory_order_seq_cst) { // NOLINT + T old = val_; + val_ += arg; + return old; + } + + T load(std::memory_order /*order*/ = std::memory_order_seq_cst) const { // NOLINT + return val_; + } + void store(T val, std::memory_order /*order*/ = std::memory_order_seq_cst) { // NOLINT + val_ = val; + } + bool compare_exchange_strong(T& expected, T desired, // NOLINT + std::memory_order /*success*/ = std::memory_order_seq_cst, + std::memory_order /*failure*/ = std::memory_order_seq_cst) { + if (val_ == expected) { + val_ = desired; + return true; + } else { + expected = val_; + return false; + } + } + bool compare_exchange_weak(T& expected, T desired, // NOLINT + std::memory_order /*success*/ = std::memory_order_seq_cst, + std::memory_order /*failure*/ = std::memory_order_seq_cst) { + if (val_ == expected) { + val_ = desired; + return true; + } else { + expected = val_; + return false; + } + } + +private: + T val_; +}; + +class DummyMutex { +public: + void lock() { // NOLINT + } + void unlock() { // NOLINT + } + bool try_lock() { // NOLINT + return true; + } +}; + +template +using Atomic = DummyAtomic; + +using Mutex = DummyMutex; + +#endif diff --git a/src/utils/ThreadPool.h b/src/utils/ThreadPool.h new file mode 100644 index 0000000..b57e070 --- /dev/null +++ b/src/utils/ThreadPool.h @@ -0,0 +1,78 @@ +#pragma once + +#include "utils/Concurrency.h" +#include +#include + +#ifdef ENABLE_MULTITHREADING + +#include +#include + +class ThreadPool { +public: + explicit ThreadPool(size_t num_threads) : stop_(false) { + for (size_t i = 0; i < num_threads; ++i) { + workers_.emplace_back([this] { + for (;;) { + std::function task; + + { + std::unique_lock lock(this->queue_mutex_); + this->condition_.wait( + lock, [this] { return this->stop_ || !this->tasks_.empty(); }); + + if (this->stop_ && this->tasks_.empty()) { + return; + } + + task = std::move(this->tasks_.back()); + this->tasks_.pop_back(); + } + + task(); + } + }); + } + } + + void Enqueue(std::function task) { + { + std::unique_lock lock(queue_mutex_); + tasks_.push_back(std::move(task)); + } + condition_.notify_one(); + } + + ~ThreadPool() { + { + std::unique_lock lock(queue_mutex_); + stop_ = true; + } + condition_.notify_all(); + for (std::thread& worker : workers_) { + worker.join(); + } + } + +private: + std::vector workers_; + std::vector> tasks_; + std::mutex queue_mutex_; + std::condition_variable condition_; + bool stop_; +}; + +#else + +class ThreadPool { +public: + explicit ThreadPool(size_t /*num_threads*/) { + } + + void Enqueue(std::function task) { + task(); + } +}; + +#endif From 2983e75b93f42311585b8691dd5250e1c8ae1433 Mon Sep 17 00:00:00 2001 From: Mikhail Fomichev Date: Sun, 17 May 2026 21:02:09 +0300 Subject: [PATCH 12/18] feat: multithread execution implemented --- src/execution/ExecutionApi.h | 20 +- src/execution/ExecutionLogic.cpp | 4 +- src/execution/OperatorsBase.cpp | 87 ++++--- src/execution/OperatorsBase.h | 49 ++-- src/execution/Pipeline.h | 79 ++++-- src/execution/PipelineExecutor.h | 7 +- .../expressions/AggregationExpressions.h | 52 ++-- .../expressions/AggregationFunctions.h | 234 +++++++++++------- src/main.cpp | 1 - src/utils/Concurrency.h | 40 ++- 10 files changed, 366 insertions(+), 207 deletions(-) diff --git a/src/execution/ExecutionApi.h b/src/execution/ExecutionApi.h index eb218dd..947905e 100644 --- a/src/execution/ExecutionApi.h +++ b/src/execution/ExecutionApi.h @@ -7,6 +7,7 @@ #include "execution/PipelineExecutor.h" #include +#include class DataResult { public: @@ -121,8 +122,21 @@ class DataFrame { return DataFrame(node); } - DataResult Collect() const { + DataResult Collect(size_t threads = 0) const { + size_t num_threads = threads; + if (num_threads == 0) { +#ifdef ENABLE_MULTITHREADING + num_threads = std::thread::hardware_concurrency(); + if (num_threads == 0) { + num_threads = 2; + } +#else + num_threads = 1; +#endif + } + PipelineBuildContext ctx; + ctx.num_threads = num_threads; auto outer_pipe = std::make_unique(); auto result_sink = std::make_shared(); @@ -131,8 +145,10 @@ class DataFrame { logical_plan_->BuildPipelines(ctx); ctx.completed_pipelines.push_back(std::move(outer_pipe)); + + ThreadPool thread_pool(num_threads); for (auto& pipeline : ctx.completed_pipelines) { - pipeline->Execute(); + pipeline->Execute(thread_pool, num_threads); } return DataResult{result_sink->TakeBatches(), std::move(ctx.schema)}; diff --git a/src/execution/ExecutionLogic.cpp b/src/execution/ExecutionLogic.cpp index 4fc03ce..e752ee0 100644 --- a/src/execution/ExecutionLogic.cpp +++ b/src/execution/ExecutionLogic.cpp @@ -81,7 +81,7 @@ void AggregateNode::BuildPipelines(PipelineBuildContext& ctx, std::vector> agg_funcs; for (const std::shared_ptr& expr : aggregate_expressions) { - agg_funcs.push_back(expr->CreateGroupedAggregationFunction(ctx.schema, output_schema)); + agg_funcs.push_back(expr->CreateGroupedAggregationFunction(ctx.schema, output_schema, ctx.num_threads)); } auto breaker = std::make_shared( @@ -108,7 +108,7 @@ void AggregateNode::BuildPipelines(PipelineBuildContext& ctx, Schema output_schema; for (const std::shared_ptr& expr : aggregate_expressions) { agg_functions.push_back( - expr->CreateGlobalAggregationFunction(ctx.schema, output_schema)); + expr->CreateGlobalAggregationFunction(ctx.schema, output_schema, ctx.num_threads)); } auto breaker = std::make_shared(std::move(agg_functions)); diff --git a/src/execution/OperatorsBase.cpp b/src/execution/OperatorsBase.cpp index 82158d0..62d7b00 100644 --- a/src/execution/OperatorsBase.cpp +++ b/src/execution/OperatorsBase.cpp @@ -113,27 +113,47 @@ std::unique_ptr DropTransformOperator::Execute(std::unique_ptr new_indices) + : new_indices_(std::move(new_indices)) { +} + +std::unique_ptr ReorderTransformOperator::Execute(std::unique_ptr batch) { + auto new_batch = std::make_unique(); + new_batch->num_rows = batch->num_rows; + new_batch->selection_vector = batch->selection_vector; + + new_batch->columns.reserve(new_indices_.size()); + for (size_t ind : new_indices_) { + new_batch->columns.push_back(batch->columns[ind]); + } + + return new_batch; +} + // ========================= LimitTransformOperator ========================= LimitTransformOperator::LimitTransformOperator(size_t limit) : limit_(limit) { } std::unique_ptr LimitTransformOperator::Execute(std::unique_ptr batch) { - if (cur_rows_ >= limit_) { - done_ = true; + size_t old_rows = cur_rows_.fetch_add(batch->num_rows); + + if (old_rows >= limit_) { + done_.store(true, std::memory_order_relaxed); return nullptr; } - if (cur_rows_ + batch->num_rows <= limit_) { - cur_rows_ += batch->num_rows; - if (cur_rows_ == limit_) { - done_ = true; + size_t remaining = limit_ - old_rows; + + if (batch->num_rows <= remaining) { + if (old_rows + batch->num_rows >= limit_) { + done_.store(true, std::memory_order_relaxed); } return batch; } else { - size_t remaining = limit_ - cur_rows_; - cur_rows_ = limit_; - done_ = true; + done_.store(true, std::memory_order_relaxed); batch->num_rows = remaining; if (batch->selection_vector) { auto new_sel_vec = std::make_shared>( @@ -155,13 +175,13 @@ AggregationSinkSourceOperator::AggregationSinkSourceOperator( : aggregation_functions_(std::move(aggregation_functions)) { } -void AggregationSinkSourceOperator::Sink(std::unique_ptr batch) { +void AggregationSinkSourceOperator::Sink(std::unique_ptr batch, size_t thread_id) { for (auto& aggregation_function : aggregation_functions_) { - aggregation_function->Update(*batch); + aggregation_function->Update(*batch, thread_id); } } -void AggregationSinkSourceOperator::Finalize() { +void AggregationSinkSourceOperator::Finalize() && { result_batch_ = std::make_unique(); result_batch_->num_rows = 1; @@ -171,11 +191,11 @@ void AggregationSinkSourceOperator::Finalize() { } std::unique_ptr AggregationSinkSourceOperator::GetData() { - if (emitted_) { - return nullptr; + bool expected = false; + if (emitted_.compare_exchange_strong(expected, true)) { + return std::move(result_batch_); } - emitted_ = true; - return std::move(result_batch_); + return nullptr; } // ========================= GroupBySinkSourceOperator ========================= @@ -189,7 +209,7 @@ GroupBySinkSourceOperator::GroupBySinkSourceOperator( agg_funcs_(std::move(agg_funcs)) { } -void GroupBySinkSourceOperator::Sink(std::unique_ptr batch) { +void GroupBySinkSourceOperator::Sink(std::unique_ptr batch, size_t thread_id) { std::vector group_ids; size_t batch_size = batch->num_rows; group_ids.reserve(batch_size); @@ -200,6 +220,8 @@ void GroupBySinkSourceOperator::Sink(std::unique_ptr batch) { ExecutionHelper::HashKeys(*batch->columns[col_idx], composite_keys, sel_vec.get()); } + LockGuard lock(sink_mutex_); + std::vector new_group_indices; size_t sz = num_groups_; @@ -222,20 +244,19 @@ void GroupBySinkSourceOperator::Sink(std::unique_ptr batch) { if (!new_group_indices.empty()) { for (size_t k = 0; k < group_by_col_indices_.size(); ++k) { - ExecutionHelper::CopySelection(*key_builders_[k], - *batch->columns[group_by_col_indices_[k]], - &new_group_indices); + ExecutionHelper::CopySelection( + *key_builders_[k], *batch->columns[group_by_col_indices_[k]], &new_group_indices); } } num_groups_ = hash_table_.size(); for (auto& func : agg_funcs_) { func->Resize(num_groups_); - func->Update(*batch, group_ids); + func->Update(*batch, group_ids, thread_id); } } -void GroupBySinkSourceOperator::Finalize() { +void GroupBySinkSourceOperator::Finalize() && { result_batch_ = std::make_unique(); result_batch_->num_rows = num_groups_; @@ -248,11 +269,11 @@ void GroupBySinkSourceOperator::Finalize() { } std::unique_ptr GroupBySinkSourceOperator::GetData() { - if (emitted_) { - return nullptr; + bool expected = false; + if (emitted_.compare_exchange_strong(expected, true)) { + return std::move(result_batch_); } - emitted_ = true; - return std::move(result_batch_); + return nullptr; } // ========================= OrderBySinkSourceOperator ========================= @@ -290,7 +311,9 @@ int Compare(const void* raw_data, uint32_t a, uint32_t b) { return 0; } -void OrderBySinkSourceOperator::Sink(std::unique_ptr batch) { +void OrderBySinkSourceOperator::Sink(std::unique_ptr batch, size_t /* thread_id */) { + LockGuard lock(sink_mutex_); + if (accum_builders_.empty()) { for (size_t c = 0; c < batch->columns.size(); ++c) { accum_builders_.push_back( @@ -311,7 +334,7 @@ void OrderBySinkSourceOperator::Sink(std::unique_ptr batch) { } } -void OrderBySinkSourceOperator::Finalize() { +void OrderBySinkSourceOperator::Finalize() && { if (accum_num_rows_ > 0) { TrimCurBatch(true); @@ -324,11 +347,11 @@ void OrderBySinkSourceOperator::Finalize() { } std::unique_ptr OrderBySinkSourceOperator::GetData() { - if (emitted_) { - return nullptr; + bool expected = false; + if (emitted_.compare_exchange_strong(expected, true)) { + return std::move(result_batch_); } - emitted_ = true; - return std::move(result_batch_); + return nullptr; } void OrderBySinkSourceOperator::TrimCurBatch(bool is_final) { diff --git a/src/execution/OperatorsBase.h b/src/execution/OperatorsBase.h index 64d6c6b..d2d9b2c 100644 --- a/src/execution/OperatorsBase.h +++ b/src/execution/OperatorsBase.h @@ -9,8 +9,6 @@ #include #include -#include -#include #include class GlobalAggregationFunction; @@ -71,22 +69,9 @@ class DropTransformOperator : public TransformOperator { class ReorderTransformOperator : public TransformOperator { public: - ReorderTransformOperator(std::vector new_indices) - : new_indices_(std::move(new_indices)) { - } - - std::unique_ptr Execute(std::unique_ptr batch) override { - auto new_batch = std::make_unique(); - new_batch->num_rows = batch->num_rows; - new_batch->selection_vector = batch->selection_vector; + ReorderTransformOperator(std::vector new_indices); - new_batch->columns.reserve(new_indices_.size()); - for (size_t ind : new_indices_) { - new_batch->columns.push_back(batch->columns[ind]); - } - - return new_batch; - } + std::unique_ptr Execute(std::unique_ptr batch) override; private: std::vector new_indices_; @@ -99,31 +84,31 @@ class LimitTransformOperator : public TransformOperator { std::unique_ptr Execute(std::unique_ptr batch) override; bool IsPipelineDone() const override { - return done_; + return done_.load(std::memory_order_relaxed); } private: size_t limit_; - size_t cur_rows_ = 0; - bool done_ = false; + Atomic cur_rows_{0}; + Atomic done_{false}; }; -// ========================= Pipeline Breakers (Sink + Source) ========================= +// ========================= Sink + Source Operators ========================= class AggregationSinkSourceOperator : public SinkOperator, public SourceOperator { public: AggregationSinkSourceOperator( std::vector> aggregation_functions); - void Sink(std::unique_ptr batch) override; - void Finalize() override; + void Sink(std::unique_ptr batch, size_t thread_id) override; + void Finalize() && override; std::unique_ptr GetData() override; private: std::vector> aggregation_functions_; std::unique_ptr result_batch_; - bool emitted_ = false; + Atomic emitted_{false}; }; class GroupBySinkSourceOperator : public SinkOperator, public SourceOperator { @@ -132,12 +117,14 @@ class GroupBySinkSourceOperator : public SinkOperator, public SourceOperator { std::vector> key_builders, std::vector> agg_funcs); - void Sink(std::unique_ptr batch) override; - void Finalize() override; + void Sink(std::unique_ptr batch, size_t thread_id) override; + void Finalize() && override; std::unique_ptr GetData() override; private: + Mutex sink_mutex_; + std::vector group_by_col_indices_; std::vector> key_builders_; std::vector> agg_funcs_; @@ -145,7 +132,7 @@ class GroupBySinkSourceOperator : public SinkOperator, public SourceOperator { absl::flat_hash_map hash_table_; std::unique_ptr result_batch_; - bool emitted_ = false; + Atomic emitted_{false}; }; class OrderBySinkSourceOperator : public SinkOperator, public SourceOperator { @@ -153,14 +140,16 @@ class OrderBySinkSourceOperator : public SinkOperator, public SourceOperator { OrderBySinkSourceOperator(std::vector> sort_columns, std::optional limit, std::optional offset); - void Sink(std::unique_ptr batch) override; - void Finalize() override; + void Sink(std::unique_ptr batch, size_t thread_id) override; + void Finalize() && override; std::unique_ptr GetData() override; private: void TrimCurBatch(bool is_final); + Mutex sink_mutex_; + std::vector> sort_columns_; std::vector> accum_builders_; std::vector accum_types_; @@ -169,5 +158,5 @@ class OrderBySinkSourceOperator : public SinkOperator, public SourceOperator { std::optional offset_; std::optional total_limit_; std::unique_ptr result_batch_; - bool emitted_ = false; + Atomic emitted_{false}; }; diff --git a/src/execution/Pipeline.h b/src/execution/Pipeline.h index f900bd6..fc82853 100644 --- a/src/execution/Pipeline.h +++ b/src/execution/Pipeline.h @@ -1,6 +1,7 @@ #pragma once #include "column/Column.h" +#include "utils/ThreadPool.h" #include #include @@ -34,8 +35,8 @@ class SinkOperator { public: virtual ~SinkOperator() = default; - virtual void Sink(std::unique_ptr batch) = 0; - virtual void Finalize() = 0; + virtual void Sink(std::unique_ptr batch, size_t thread_id) = 0; + virtual void Finalize() && = 0; }; class Pipeline { @@ -44,33 +45,57 @@ class Pipeline { std::vector> transforms; std::shared_ptr sink; - void Execute() { - while (auto batch = source->GetData()) { - bool skip = false; - bool stop = false; + void Execute(ThreadPool& thread_pool, size_t num_threads) { + Atomic num_tasks_finished{0}; + Mutex wait_mutex; + ConditionVariable wait_cond; + for (size_t thread_id = 0; thread_id < num_threads; thread_id++) { + thread_pool.Enqueue( + [this, thread_id, &num_tasks_finished, &wait_mutex, &wait_cond, num_threads]() { + while (auto batch = source->GetData()) { + bool skip = false; + bool stop = false; - for (std::shared_ptr& transform : transforms) { - batch = transform->Execute(std::move(batch)); - if (!batch || batch->num_rows == 0) { - skip = true; - break; - } - if (transform->IsPipelineDone()) { - stop = true; - break; - } - } + for (std::shared_ptr& transform : transforms) { + batch = transform->Execute(std::move(batch)); + if (!batch || batch->num_rows == 0) { + skip = true; + break; + } + if (transform->IsPipelineDone()) { + stop = true; + break; + } + } - if (skip) { - continue; - } - if (batch && batch->num_rows > 0) { - sink->Sink(std::move(batch)); - } - if (stop) { - break; - } + if (skip) { + continue; + } + if (batch && batch->num_rows > 0) { + sink->Sink(std::move(batch), thread_id); + } + if (stop) { + break; + } + } + { + UniqueLock lock(wait_mutex); + size_t finished_before = + num_tasks_finished.fetch_add(1, std::memory_order_release); + if (finished_before + 1 == num_threads) { + wait_cond.notify_one(); + } + } + }); } - sink->Finalize(); + + { + UniqueLock lock(wait_mutex); + wait_cond.wait(lock, [&]() { + return num_tasks_finished.load(std::memory_order_acquire) == num_threads; + }); + } + + std::move(*sink).Finalize(); } }; diff --git a/src/execution/PipelineExecutor.h b/src/execution/PipelineExecutor.h index c8eec1d..d96edfc 100644 --- a/src/execution/PipelineExecutor.h +++ b/src/execution/PipelineExecutor.h @@ -8,11 +8,12 @@ class ResultSinkOperator : public SinkOperator { public: - void Sink(std::unique_ptr batch) override { + void Sink(std::unique_ptr batch, size_t /* thread_id */) override { + LockGuard lock(mutex_); batches_.push_back(std::move(batch)); } - void Finalize() override { + void Finalize() && override { // charonchik bebe } @@ -21,6 +22,7 @@ class ResultSinkOperator : public SinkOperator { } private: + Mutex mutex_; std::vector> batches_; }; @@ -28,4 +30,5 @@ struct PipelineBuildContext { Pipeline* current_pipeline = nullptr; std::vector> completed_pipelines; Schema schema; + size_t num_threads = 1; }; diff --git a/src/execution/expressions/AggregationExpressions.h b/src/execution/expressions/AggregationExpressions.h index f21cfba..931a538 100644 --- a/src/execution/expressions/AggregationExpressions.h +++ b/src/execution/expressions/AggregationExpressions.h @@ -73,10 +73,10 @@ class AggregateExpression { } virtual std::unique_ptr CreateGlobalAggregationFunction( - const Schema& child_schema, Schema& output_schema) = 0; + const Schema& child_schema, Schema& output_schema, size_t num_threads) = 0; virtual std::unique_ptr CreateGroupedAggregationFunction( - const Schema& child_schema, Schema& output_schema) = 0; + const Schema& child_schema, Schema& output_schema, size_t num_threads) = 0; protected: std::string column_name_; @@ -90,15 +90,15 @@ class CountExpression : public AggregateExpression { } std::unique_ptr CreateGlobalAggregationFunction( - [[maybe_unused]] const Schema& child_schema, Schema& output_schema) override { + const Schema&, Schema& output_schema, size_t num_threads) override { output_schema.AddColumn(GetOutputName(), ColumnType::INT64); - return std::make_unique>(ColumnType::INT64); + return std::make_unique>(ColumnType::INT64, num_threads); } std::unique_ptr CreateGroupedAggregationFunction( - [[maybe_unused]] const Schema& child_schema, Schema& output_schema) override { + const Schema&, Schema& output_schema, size_t num_threads) override { output_schema.AddColumn(GetOutputName(), ColumnType::INT64); - return std::make_unique>(ColumnType::INT64); + return std::make_unique>(ColumnType::INT64, num_threads); } }; @@ -116,7 +116,7 @@ class DistinctCountExpression : public AggregateExpression { } std::unique_ptr CreateGlobalAggregationFunction( - const Schema& child_schema, Schema& output_schema) override { + const Schema& child_schema, Schema& output_schema, size_t num_threads) override { size_t column_ind = child_schema.GetColumnIndexByName(GetName()); ColumnType column_type = child_schema.GetColumnTypeByName(GetName()); @@ -124,12 +124,12 @@ class DistinctCountExpression : public AggregateExpression { column_type, [&](ColumnType) -> std::unique_ptr { output_schema.AddColumn(GetOutputName(), ColumnType::INT64); return std::make_unique>( - column_ind, ColumnType::INT64); + column_ind, ColumnType::INT64, num_threads); }); } std::unique_ptr CreateGroupedAggregationFunction( - const Schema& child_schema, Schema& output_schema) override { + const Schema& child_schema, Schema& output_schema, size_t num_threads) override { size_t column_ind = child_schema.GetColumnIndexByName(GetName()); ColumnType column_type = child_schema.GetColumnTypeByName(GetName()); @@ -138,7 +138,7 @@ class DistinctCountExpression : public AggregateExpression { [&](ColumnType) -> std::unique_ptr { output_schema.AddColumn(GetOutputName(), ColumnType::INT64); return std::make_unique>( - column_ind, ColumnType::INT64); + column_ind, ColumnType::INT64, num_threads); }); } }; @@ -157,7 +157,7 @@ class MinExpression : public AggregateExpression { } std::unique_ptr CreateGlobalAggregationFunction( - const Schema& child_schema, Schema& output_schema) override { + const Schema& child_schema, Schema& output_schema, size_t num_threads) override { size_t column_ind = child_schema.GetColumnIndexByName(GetName()); ColumnType column_type = child_schema.GetColumnTypeByName(GetName()); @@ -166,12 +166,12 @@ class MinExpression : public AggregateExpression { [&](ColumnType t) -> std::unique_ptr { output_schema.AddColumn(GetOutputName(), t); return std::make_unique>( - column_ind, t); + column_ind, t, num_threads); }); } std::unique_ptr CreateGroupedAggregationFunction( - const Schema& child_schema, Schema& output_schema) override { + const Schema& child_schema, Schema& output_schema, size_t num_threads) override { size_t column_ind = child_schema.GetColumnIndexByName(GetName()); ColumnType column_type = child_schema.GetColumnTypeByName(GetName()); @@ -180,7 +180,7 @@ class MinExpression : public AggregateExpression { [&](ColumnType t) -> std::unique_ptr { output_schema.AddColumn(GetOutputName(), t); return std::make_unique>( - column_ind, t); + column_ind, t, num_threads); }); } }; @@ -199,7 +199,7 @@ class MaxExpression : public AggregateExpression { } std::unique_ptr CreateGlobalAggregationFunction( - const Schema& child_schema, Schema& output_schema) override { + const Schema& child_schema, Schema& output_schema, size_t num_threads) override { size_t column_ind = child_schema.GetColumnIndexByName(GetName()); ColumnType column_type = child_schema.GetColumnTypeByName(GetName()); @@ -208,12 +208,12 @@ class MaxExpression : public AggregateExpression { [&](ColumnType t) -> std::unique_ptr { output_schema.AddColumn(GetOutputName(), t); return std::make_unique>( - column_ind, t); + column_ind, t, num_threads); }); } std::unique_ptr CreateGroupedAggregationFunction( - const Schema& child_schema, Schema& output_schema) override { + const Schema& child_schema, Schema& output_schema, size_t num_threads) override { size_t column_ind = child_schema.GetColumnIndexByName(GetName()); ColumnType column_type = child_schema.GetColumnTypeByName(GetName()); @@ -222,7 +222,7 @@ class MaxExpression : public AggregateExpression { [&](ColumnType t) -> std::unique_ptr { output_schema.AddColumn(GetOutputName(), t); return std::make_unique>( - column_ind, t); + column_ind, t, num_threads); }); } }; @@ -241,7 +241,7 @@ class SumExpression : public AggregateExpression { } std::unique_ptr CreateGlobalAggregationFunction( - const Schema& child_schema, Schema& output_schema) override { + const Schema& child_schema, Schema& output_schema, size_t num_threads) override { const size_t column_ind = child_schema.GetColumnIndexByName(GetName()); const ColumnType column_type = child_schema.GetColumnTypeByName(GetName()); @@ -259,12 +259,12 @@ class SumExpression : public AggregateExpression { output_schema.AddColumn(GetOutputName(), out_type); return std::make_unique< TypedGlobalAggregationFunction>( - column_ind, out_type); + column_ind, out_type, num_threads); }); } std::unique_ptr CreateGroupedAggregationFunction( - const Schema& child_schema, Schema& output_schema) override { + const Schema& child_schema, Schema& output_schema, size_t num_threads) override { const size_t column_ind = child_schema.GetColumnIndexByName(GetName()); const ColumnType column_type = child_schema.GetColumnTypeByName(GetName()); @@ -282,7 +282,7 @@ class SumExpression : public AggregateExpression { output_schema.AddColumn(GetOutputName(), out_type); return std::make_unique< TypedGroupedAggregationFunction>( - column_ind, out_type); + column_ind, out_type, num_threads); }); } }; @@ -300,7 +300,7 @@ class AvgExpression : public AggregateExpression { } std::unique_ptr CreateGlobalAggregationFunction( - const Schema& child_schema, Schema& output_schema) override { + const Schema& child_schema, Schema& output_schema, size_t num_threads) override { const size_t column_ind = child_schema.GetColumnIndexByName(GetName()); const ColumnType column_type = child_schema.GetColumnTypeByName(GetName()); @@ -317,12 +317,12 @@ class AvgExpression : public AggregateExpression { output_schema.AddColumn(GetOutputName(), ColumnType::LONGDOUBLE); return std::make_unique< AvgGlobalAggregationFunction>( - column_ind, ColumnType::LONGDOUBLE); + column_ind, ColumnType::LONGDOUBLE, num_threads); }); } std::unique_ptr CreateGroupedAggregationFunction( - const Schema& child_schema, Schema& output_schema) override { + const Schema& child_schema, Schema& output_schema, size_t num_threads) override { const size_t column_ind = child_schema.GetColumnIndexByName(GetName()); const ColumnType column_type = child_schema.GetColumnTypeByName(GetName()); @@ -339,7 +339,7 @@ class AvgExpression : public AggregateExpression { output_schema.AddColumn(GetOutputName(), ColumnType::LONGDOUBLE); return std::make_unique< AvgGroupedAggregationFunction>( - column_ind, ColumnType::LONGDOUBLE); + column_ind, ColumnType::LONGDOUBLE, num_threads); }); } }; diff --git a/src/execution/expressions/AggregationFunctions.h b/src/execution/expressions/AggregationFunctions.h index eec99f9..ee3970c 100644 --- a/src/execution/expressions/AggregationFunctions.h +++ b/src/execution/expressions/AggregationFunctions.h @@ -8,7 +8,6 @@ #include "column/NumericColumn.h" #include "column/StringColumn.h" #include "column/TemporalColumn.h" - #include "column/ColumnFactory.h" #include "absl/container/flat_hash_set.h" @@ -20,7 +19,8 @@ class AggregationFunction { public: virtual ~AggregationFunction() = default; - AggregationFunction(ColumnType column_type) : column_type_(column_type) { + AggregationFunction(ColumnType column_type, size_t num_threads) + : column_type_(column_type), num_threads_(num_threads) { } std::shared_ptr Finalize() { @@ -31,23 +31,29 @@ class AggregationFunction { protected: ColumnType column_type_; + size_t num_threads_; virtual void WriteResult(const std::shared_ptr& builder) = 0; }; class GlobalAggregationFunction : public AggregationFunction { public: - GlobalAggregationFunction(ColumnType column_type) : AggregationFunction(column_type) {} + GlobalAggregationFunction(ColumnType column_type, size_t num_threads) + : AggregationFunction(column_type, num_threads) { + } - virtual void Update(const RecordBatch& batch) = 0; + virtual void Update(const RecordBatch& batch, size_t thread_id) = 0; }; class GroupedAggregationFunction : public AggregationFunction { public: - GroupedAggregationFunction(ColumnType column_type) : AggregationFunction(column_type) {} + GroupedAggregationFunction(ColumnType column_type, size_t num_threads) + : AggregationFunction(column_type, num_threads) { + } virtual void Resize(size_t num_groups) = 0; - virtual void Update(const RecordBatch& batch, const std::vector& group_ids) = 0; + virtual void Update(const RecordBatch& batch, const std::vector& group_ids, + size_t thread_id) = 0; }; struct SumOperation { @@ -80,28 +86,51 @@ class TypedGlobalAggregationFunction : public GlobalAggregationFunction { using StateType = OutputColumn::ValueType; using OutputBuilder = BuilderTypeTrait::Type; + struct alignas(64) ThreadState { + StateType value{}; + bool has_data = false; + }; + public: - TypedGlobalAggregationFunction(size_t column_index, ColumnType column_type) - : GlobalAggregationFunction(column_type), column_index_(column_index) { + TypedGlobalAggregationFunction(size_t column_index, ColumnType column_type, size_t num_threads) + : GlobalAggregationFunction(column_type, num_threads), + column_index_(column_index), + states_(num_threads) { } - void Update(const RecordBatch& batch) override { - AggExpHelper::IterateColumnData(batch, column_index_, - [&](const auto& value, size_t, size_t) { - if (!has_data_) { - state_ = value; - has_data_ = true; - } else { - Operation::Apply(state_, value); - } - }); + void Update(const RecordBatch& batch, size_t thread_id) override { + ThreadState& local_state = states_[thread_id]; + + AggExpHelper::IterateColumnData( + batch, column_index_, [&](const auto& value, size_t, size_t) { + if (!local_state.has_data) { + local_state.value = value; + local_state.has_data = true; + } else { + Operation::Apply(local_state.value, value); + } + }); } protected: void WriteResult(const std::shared_ptr& builder) override { + StateType result{}; + bool has_data = false; + + for (size_t i = 0; i < states_.size(); i++) { + if (has_data) { + if (states_[i].has_data) { + Operation::Apply(result, states_[i].value); + } + } else if (states_[i].has_data) { + has_data = true; + result = states_[i].value; + } + } + auto* typed = static_cast(builder.get()); - if (has_data_) { - typed->AddValue(state_); + if (has_data) { + typed->AddValue(result); } else { if constexpr (requires { Operation::template GetInitValue(); }) { typed->AddValue(Operation::template GetInitValue()); @@ -112,9 +141,8 @@ class TypedGlobalAggregationFunction : public GlobalAggregationFunction { } private: - size_t column_index_; - StateType state_{}; - bool has_data_ = false; + const size_t column_index_; + std::vector states_; }; template @@ -123,8 +151,8 @@ class TypedGroupedAggregationFunction : public GroupedAggregationFunction { using OutputBuilder = BuilderTypeTrait::Type; public: - TypedGroupedAggregationFunction(size_t column_index, ColumnType column_type) - : GroupedAggregationFunction(column_type), column_index_(column_index) { + TypedGroupedAggregationFunction(size_t column_index, ColumnType column_type, size_t num_threads) + : GroupedAggregationFunction(column_type, num_threads), column_index_(column_index) { } void Resize(size_t num_groups) override { @@ -134,17 +162,18 @@ class TypedGroupedAggregationFunction : public GroupedAggregationFunction { } } - void Update(const RecordBatch& batch, const std::vector& group_ids) override { - AggExpHelper::IterateColumnData(batch, column_index_, - [&](const auto& value, size_t row_idx, size_t) { - uint32_t gid = group_ids[row_idx]; - if (!has_data_[gid]) { - states_[gid] = value; - has_data_[gid] = 1; - } else { - Operation::Apply(states_[gid], value); - } - }); + void Update(const RecordBatch& batch, const std::vector& group_ids, + size_t /* thread_id */) override { + AggExpHelper::IterateColumnData( + batch, column_index_, [&](const auto& value, size_t row_idx, size_t) { + uint32_t gid = group_ids[row_idx]; + if (!has_data_[gid]) { + states_[gid] = value; + has_data_[gid] = 1; + } else { + Operation::Apply(states_[gid], value); + } + }); } protected: @@ -166,23 +195,31 @@ class CountGlobalAggregationFunction : public GlobalAggregationFunction { using ValueType = OutputColumn::ValueType; using OutputBuilder = BuilderTypeTrait::Type; + struct alignas(64) ThreadState { + int64_t count = 0; + }; + public: - CountGlobalAggregationFunction(ColumnType column_type) - : GlobalAggregationFunction(column_type) { + CountGlobalAggregationFunction(ColumnType column_type, size_t num_threads) + : GlobalAggregationFunction(column_type, num_threads), states_(num_threads) { } - void Update(const RecordBatch& batch) override { - count_ += batch.num_rows; + void Update(const RecordBatch& batch, size_t thread_id) override { + states_[thread_id].count += batch.num_rows; } protected: void WriteResult(const std::shared_ptr& builder) override { + int64_t total = 0; + for (const auto& state : states_) { + total += state.count; + } auto* typed = static_cast(builder.get()); - typed->AddValue(static_cast(count_)); + typed->AddValue(static_cast(total)); } private: - int64_t count_ = 0; + std::vector states_; }; template @@ -191,8 +228,8 @@ class CountGroupedAggregationFunction : public GroupedAggregationFunction { using OutputBuilder = BuilderTypeTrait::Type; public: - CountGroupedAggregationFunction(ColumnType column_type) - : GroupedAggregationFunction(column_type) { + CountGroupedAggregationFunction(ColumnType column_type, size_t num_threads) + : GroupedAggregationFunction(column_type, num_threads) { } void Resize(size_t num_groups) override { @@ -201,7 +238,8 @@ class CountGroupedAggregationFunction : public GroupedAggregationFunction { } } - void Update(const RecordBatch& batch, const std::vector& group_ids) override { + void Update(const RecordBatch& batch, const std::vector& group_ids, + size_t /* thread_id */) override { for (size_t i = 0; i < batch.num_rows; ++i) { uint32_t gid = group_ids[i]; ++counts_[gid]; @@ -226,46 +264,58 @@ class AvgGlobalAggregationFunction : public GlobalAggregationFunction { using OutputType = OutputColumn::ValueType; using OutputBuilder = BuilderTypeTrait::Type; + struct alignas(64) ThreadState { + StateType sum = 0; + int64_t count = 0; + }; + public: - AvgGlobalAggregationFunction(size_t column_index, ColumnType column_type) - : GlobalAggregationFunction(column_type), column_index_(column_index) { + AvgGlobalAggregationFunction(size_t column_index, ColumnType column_type, size_t num_threads) + : GlobalAggregationFunction(column_type, num_threads), + column_index_(column_index), + states_(num_threads) { } - void Update(const RecordBatch& batch) override { + void Update(const RecordBatch& batch, size_t thread_id) override { + ThreadState& local = states_[thread_id]; AggExpHelper::IterateColumnData(batch, column_index_, - [&](const auto& value, size_t, size_t) { - sum_ += value; - ++count_; - }); - has_data_ = true; + [&](const auto& value, size_t, size_t) { + local.sum += value; + ++local.count; + }); } protected: void WriteResult(const std::shared_ptr& builder) override { + StateType total_sum = 0; + int64_t total_count = 0; + for (const auto& state : states_) { + total_sum += state.sum; + total_count += state.count; + } + auto* typed = static_cast(builder.get()); - if (has_data_ && count_ > 0) { - typed->AddValue(ComputeAvg()); + if (total_count > 0) { + typed->AddValue(ComputeAvg(total_sum, total_count)); } else { typed->AddValue(OutputType{}); } } private: - OutputType ComputeAvg() const { + static OutputType ComputeAvg(StateType sum, int64_t count) { if constexpr (std::is_integral_v) { - OutputType int_part = static_cast(sum_ / count_); + OutputType int_part = static_cast(sum / count); OutputType rem = - static_cast(static_cast(sum_ % count_) / count_); + static_cast(static_cast(sum % count) / count); return int_part + rem; } else { - return static_cast(sum_) / static_cast(count_); + return static_cast(sum) / static_cast(count); } } size_t column_index_; - StateType sum_ = 0; - int64_t count_ = 0; - bool has_data_ = false; + std::vector states_; }; template @@ -275,8 +325,8 @@ class AvgGroupedAggregationFunction : public GroupedAggregationFunction { using OutputBuilder = BuilderTypeTrait::Type; public: - AvgGroupedAggregationFunction(size_t column_index, ColumnType column_type) - : GroupedAggregationFunction(column_type), column_index_(column_index) { + AvgGroupedAggregationFunction(size_t column_index, ColumnType column_type, size_t num_threads) + : GroupedAggregationFunction(column_type, num_threads), column_index_(column_index) { } void Resize(size_t num_groups) override { @@ -285,13 +335,14 @@ class AvgGroupedAggregationFunction : public GroupedAggregationFunction { } } - void Update(const RecordBatch& batch, const std::vector& group_ids) override { - AggExpHelper::IterateColumnData(batch, column_index_, - [&](const auto& value, size_t row_idx, size_t) { - uint32_t gid = group_ids[row_idx]; - states_[gid].sum += value; - ++states_[gid].count; - }); + void Update(const RecordBatch& batch, const std::vector& group_ids, + size_t /* thread_id */) override { + AggExpHelper::IterateColumnData( + batch, column_index_, [&](const auto& value, size_t row_idx, size_t) { + uint32_t gid = group_ids[row_idx]; + states_[gid].sum += value; + ++states_[gid].count; + }); } protected: @@ -310,8 +361,7 @@ class AvgGroupedAggregationFunction : public GroupedAggregationFunction { static OutputType ComputeAvg(StateType sum, int64_t count) { if constexpr (std::is_integral_v) { OutputType int_part = static_cast(sum / count); - OutputType rem = - static_cast(static_cast(sum % count) / count); + OutputType rem = static_cast(static_cast(sum % count) / count); return int_part + rem; } else { return static_cast(sum) / static_cast(count); @@ -332,28 +382,42 @@ class DistinctCountGlobalAggregationFunction : public GlobalAggregationFunction using OutputValueType = OutputColumn::ValueType; using OutputBuilder = BuilderTypeTrait::Type; + struct alignas(64) Set { + absl::flat_hash_set set; + }; + public: - DistinctCountGlobalAggregationFunction(size_t column_index, ColumnType column_type) - : GlobalAggregationFunction(column_type), column_index_(column_index) { + DistinctCountGlobalAggregationFunction(size_t column_index, ColumnType column_type, + size_t num_threads) + : GlobalAggregationFunction(column_type, num_threads), + column_index_(column_index), + thread_sets_(num_threads) { } - void Update(const RecordBatch& batch) override { + void Update(const RecordBatch& batch, size_t thread_id) override { + auto& local_set = thread_sets_[thread_id].set; AggExpHelper::IterateColumnData( - batch, column_index_, - [&](const auto& value, size_t, size_t) { - distinct_values_.insert(InputValueType(value)); + batch, column_index_, [&](const auto& value, size_t, size_t) { + local_set.insert(InputValueType(value)); }); } protected: void WriteResult(const std::shared_ptr& builder) override { + auto& merged = thread_sets_[0].set; + for (size_t i = 1; i < thread_sets_.size(); ++i) { + for (auto& val : thread_sets_[i].set) { + merged.insert(std::move(val)); + } + } + auto* typed = static_cast(builder.get()); - typed->AddValue(static_cast(distinct_values_.size())); + typed->AddValue(static_cast(merged.size())); } private: size_t column_index_; - absl::flat_hash_set distinct_values_; + std::vector thread_sets_; }; template @@ -363,8 +427,9 @@ class DistinctCountGroupedAggregationFunction : public GroupedAggregationFunctio using OutputBuilder = BuilderTypeTrait::Type; public: - DistinctCountGroupedAggregationFunction(size_t column_index, ColumnType column_type) - : GroupedAggregationFunction(column_type), column_index_(column_index) { + DistinctCountGroupedAggregationFunction(size_t column_index, ColumnType column_type, + size_t num_threads) + : GroupedAggregationFunction(column_type, num_threads), column_index_(column_index) { } void Resize(size_t num_groups) override { @@ -373,7 +438,8 @@ class DistinctCountGroupedAggregationFunction : public GroupedAggregationFunctio } } - void Update(const RecordBatch& batch, const std::vector& group_ids) override { + void Update(const RecordBatch& batch, const std::vector& group_ids, + size_t /* thread_id */) override { AggExpHelper::IterateColumnData( batch, column_index_, [&](const auto& value, size_t row_idx, size_t) { uint32_t gid = group_ids[row_idx]; diff --git a/src/main.cpp b/src/main.cpp index 06ca8ed..c182e4e 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -296,7 +296,6 @@ class Query { df.Display(); } - // TODO: // SELECT REGEXP_REPLACE(Referer, '^https?://(?:www\.)?([^/]+)/.*$', '\1') AS k, // AVG(length(Referer)) AS l, COUNT(*) AS c, MIN(Referer) FROM hits WHERE Referer <> '' GROUP BY // k HAVING COUNT(*) > 100000 ORDER BY l DESC LIMIT 25; diff --git a/src/utils/Concurrency.h b/src/utils/Concurrency.h index 0343105..180e89d 100644 --- a/src/utils/Concurrency.h +++ b/src/utils/Concurrency.h @@ -3,15 +3,23 @@ #ifdef ENABLE_MULTITHREADING #include +#include #include template using Atomic = std::atomic; - using Mutex = std::mutex; +using ConditionVariable = std::condition_variable; + +template +using LockGuard = std::lock_guard; +template +using UniqueLock = std::unique_lock; #else +#include "utils/Assert.h" + template class DummyAtomic { public: @@ -68,9 +76,39 @@ class DummyMutex { } }; +class DummyConditionVariable { +public: + template + void wait(Lock& /*lock*/, Predicate pred) { + ASSERT(pred()); + } + void notify_one() {} + void notify_all() {} +}; + template using Atomic = DummyAtomic; using Mutex = DummyMutex; +using ConditionVariable = DummyConditionVariable; + +template +class LockGuard { +public: + explicit LockGuard(MutexT& /*m*/) { + } + LockGuard(const LockGuard&) = delete; + LockGuard& operator=(const LockGuard&) = delete; +}; + +template +class UniqueLock { +public: + explicit UniqueLock(MutexT& /*m*/) { + } + UniqueLock(const UniqueLock&) = delete; + UniqueLock& operator=(const UniqueLock&) = delete; +}; + #endif From 8ecdd4ed7d570acd01205450c485798c0406dddf Mon Sep 17 00:00:00 2001 From: Mikhail Fomichev Date: Sun, 17 May 2026 21:38:38 +0300 Subject: [PATCH 13/18] script: run all queries for multithreading supported --- .gitignore | 1 + script/build.sh | 2 +- script/run_all_queries.sh | 34 ++++++++++++++++++++-------------- script/run_all_queries_mt.sh | 34 ++++++++++++++++++++++++++++++++++ script/run_query_mt.sh | 34 ++++++++++++++++++++++++++++++++++ 5 files changed, 90 insertions(+), 15 deletions(-) create mode 100755 script/run_all_queries_mt.sh create mode 100755 script/run_query_mt.sh diff --git a/.gitignore b/.gitignore index 16edb04..9e8403f 100644 --- a/.gitignore +++ b/.gitignore @@ -18,6 +18,7 @@ compile_commands.json .clangd test_csv test.csv +test.log test_csv2 test_main.cpp hits_sample.csv diff --git a/script/build.sh b/script/build.sh index 1d3d964..27893d6 100755 --- a/script/build.sh +++ b/script/build.sh @@ -12,4 +12,4 @@ cmake -S "${ROOT_DIR}" -B "${BUILD_DIR}" \ -DCMAKE_BUILD_TYPE="${BUILD_TYPE}" \ -DCMAKE_CXX_STANDARD=23 -cmake --build "${BUILD_DIR}" --target columnar-engine -j "$(nproc)" +cmake --build "${BUILD_DIR}" -j "$(nproc)" diff --git a/script/run_all_queries.sh b/script/run_all_queries.sh index afa983f..4f30ac8 100755 --- a/script/run_all_queries.sh +++ b/script/run_all_queries.sh @@ -1,28 +1,34 @@ #!/bin/bash - set -e cd "$(dirname "$0")" || exit 1 -echo ">>> Запуск setup.sh..." -./setup.sh - -echo ">>> Запуск build.sh..." -./build.sh - -echo ">>> Запуск convert.sh..." -./convert.sh ../hits_sample.csv ../columnar_hits_sample.tuff ../hits.schema +echo ">>> Running queries 0-42 one thread..." +ITERATIONS=${1:-1} RESULTS_DIR="query_results" mkdir -p "${RESULTS_DIR}" -echo ">>> Запуск запросов от 0 до 42 (результаты в ${RESULTS_DIR}/)..." -for i in {0..42} +for (( iter=1; iter<=ITERATIONS; iter++ )) do echo "=========================================" - echo ">>> Выполнение запроса $i" + echo ">>> Iteration $iter / $ITERATIONS" echo "=========================================" - ./run_query.sh "$i" ../columnar_hits_sample.tuff "${RESULTS_DIR}/query_${i}.csv" "${RESULTS_DIR}/query_${i}.log" + for i in {0..42} + do + echo -n ">>> Executing query $i ... " + set +e + ./run_query.sh "$i" ../columnar_hits_sample.tuff "${RESULTS_DIR}/query_${i}_iter_${iter}.csv" "${RESULTS_DIR}/query_${i}_iter_${iter}.log" + exit_code=$? + set -e + + if [ $exit_code -ne 0 ]; then + echo "FAILED with exit code $exit_code" + echo "See log: ${RESULTS_DIR}/query_${i}_iter_${iter}.log" + exit $exit_code + fi + echo "OK" + done done -echo ">>> Все запросы выполнены успешно. Результаты в ${RESULTS_DIR}/" \ No newline at end of file +echo ">>> All $ITERATIONS iterations of all queries completed successfully." diff --git a/script/run_all_queries_mt.sh b/script/run_all_queries_mt.sh new file mode 100755 index 0000000..9d8ba96 --- /dev/null +++ b/script/run_all_queries_mt.sh @@ -0,0 +1,34 @@ +#!/bin/bash +set -e + +cd "$(dirname "$0")" || exit 1 + +echo ">>> Running queries 0-42 multithreaded..." + +ITERATIONS=${1:-1} +RESULTS_DIR="query_results_mt" +mkdir -p "${RESULTS_DIR}" + +for (( iter=1; iter<=ITERATIONS; iter++ )) +do + echo "=========================================" + echo ">>> Iteration $iter / $ITERATIONS" + echo "=========================================" + for i in {0..42} + do + echo -n ">>> Executing query $i ... " + set +e + ./run_query_mt.sh "$i" ../columnar_hits_sample.tuff "${RESULTS_DIR}/query_${i}_iter_${iter}.csv" "${RESULTS_DIR}/query_${i}_iter_${iter}.log" + exit_code=$? + set -e + + if [ $exit_code -ne 0 ]; then + echo "FAILED with exit code $exit_code" + echo "See log: ${RESULTS_DIR}/query_${i}_iter_${iter}.log" + exit $exit_code + fi + echo "OK" + done +done + +echo ">>> All $ITERATIONS iterations of all queries completed successfully." diff --git a/script/run_query_mt.sh b/script/run_query_mt.sh new file mode 100755 index 0000000..499dd7e --- /dev/null +++ b/script/run_query_mt.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" + +if [[ $# -lt 4 ]]; then + echo "Usage: script/run_query_mt.sh " >&2 + exit 2 +fi + +QUERY_NUM="$1" +COLUMNAR="$2" +OUTPUT_CSV="$3" +LOG_FILE="$4" + +BUILD_DIR="${ROOT_DIR}/cmake-build-release" +BIN="${BUILD_DIR}/columnar-engine-mt" + +if [[ ! -x "${BIN}" ]]; then + echo "ERROR: columnar-engine-mt not found at ${BIN}" >&2 + echo "Run script/build.sh first" >&2 + exit 1 +fi + +if [[ ! -f "${COLUMNAR}" ]]; then + echo "ERROR: columnar file not found: ${COLUMNAR}" >&2 + exit 2 +fi + +mkdir -p "$(dirname "${OUTPUT_CSV}")" +mkdir -p "$(dirname "${LOG_FILE}")" + +# Run the query, capture stdout (CSV result) and tee stderr+stdout to log +"${BIN}" query "${COLUMNAR}" "${QUERY_NUM}" > "${OUTPUT_CSV}" 2> >(tee "${LOG_FILE}" >&2) From eef1da397af5d480576d5c4a2ed796d1a219e5f6 Mon Sep 17 00:00:00 2001 From: Mikhail Fomichev Date: Mon, 18 May 2026 22:34:59 +0300 Subject: [PATCH 14/18] feat: every operator is under multithreading now. implementing multithread execution officially finished. --- script/run_all_queries.sh | 50 ++- script/run_all_queries_mt.sh | 52 +++- src/execution/ExecutionApi.h | 17 +- src/execution/ExecutionLogic.cpp | 24 +- src/execution/OperatorsBase.cpp | 284 +++++++++++++----- src/execution/OperatorsBase.h | 32 +- src/execution/Pipeline.h | 11 +- .../expressions/AggregationFunctions.h | 214 ++++++++----- 8 files changed, 500 insertions(+), 184 deletions(-) diff --git a/script/run_all_queries.sh b/script/run_all_queries.sh index 4f30ac8..2b53cf7 100755 --- a/script/run_all_queries.sh +++ b/script/run_all_queries.sh @@ -1,11 +1,31 @@ #!/bin/bash set -e +# usage: +# -i 3 - count of iterations (or --iterations) +# --answers - print answers for each query +# --debug - print time of execution for each query + cd "$(dirname "$0")" || exit 1 -echo ">>> Running queries 0-42 one thread..." +# Значения по умолчанию +ITERATIONS=1 +SHOW_DEBUG=0 +SHOW_ANSWERS=0 + +# Парсинг аргументов командной строки +while [[ "$#" -gt 0 ]]; do + case $1 in + --debug) SHOW_DEBUG=1; shift ;; + --answers) SHOW_ANSWERS=1; shift ;; + -i|--iterations) ITERATIONS="$2"; shift 2 ;; + *) echo "Unknown parameter passed: $1"; exit 1 ;; + esac +done + +echo ">>> Running queries 0-42 (single thread)..." +echo ">>> Iterations: $ITERATIONS | Show Debug: $SHOW_DEBUG | Show Answers: $SHOW_ANSWERS" -ITERATIONS=${1:-1} RESULTS_DIR="query_results" mkdir -p "${RESULTS_DIR}" @@ -17,18 +37,38 @@ do for i in {0..42} do echo -n ">>> Executing query $i ... " + + CSV_FILE="${RESULTS_DIR}/query_${i}_iter_${iter}.csv" + LOG_FILE="${RESULTS_DIR}/query_${i}_iter_${iter}.log" + set +e - ./run_query.sh "$i" ../columnar_hits_sample.tuff "${RESULTS_DIR}/query_${i}_iter_${iter}.csv" "${RESULTS_DIR}/query_${i}_iter_${iter}.log" + ./run_query.sh "$i" ../columnar_hits_sample.tuff "$CSV_FILE" "$LOG_FILE" exit_code=$? set -e if [ $exit_code -ne 0 ]; then echo "FAILED with exit code $exit_code" - echo "See log: ${RESULTS_DIR}/query_${i}_iter_${iter}.log" + echo "See log: $LOG_FILE" + echo "--- CRASH LOG ---" + cat "$LOG_FILE" # При падении всегда выводим лог, чтобы сразу видеть ошибку exit $exit_code fi echo "OK" + + # Если запрошен вывод ответов + if [ "$SHOW_ANSWERS" -eq 1 ]; then + echo "------ ANSWERS (Query $i) ------" + cat "$CSV_FILE" + echo "--------------------------------" + fi + + # Если запрошен вывод дебаг информации (времени выполнения и т.д.) + if [ "$SHOW_DEBUG" -eq 1 ]; then + echo "------- DEBUG (Query $i) -------" + cat "$LOG_FILE" + echo "--------------------------------" + fi done done -echo ">>> All $ITERATIONS iterations of all queries completed successfully." +echo ">>> All $ITERATIONS iterations of all queries completed successfully." \ No newline at end of file diff --git a/script/run_all_queries_mt.sh b/script/run_all_queries_mt.sh index 9d8ba96..c8d2db0 100755 --- a/script/run_all_queries_mt.sh +++ b/script/run_all_queries_mt.sh @@ -1,11 +1,31 @@ #!/bin/bash set -e +# usage: +# -i 3 - count of iterations (or --iterations) +# --answers - print answers for each query +# --debug - print time of execution for each query + cd "$(dirname "$0")" || exit 1 -echo ">>> Running queries 0-42 multithreaded..." +# Значения по умолчанию +ITERATIONS=1 +SHOW_DEBUG=0 +SHOW_ANSWERS=0 + +# Парсинг аргументов командной строки +while [[ "$#" -gt 0 ]]; do + case $1 in + --debug) SHOW_DEBUG=1; shift ;; + --answers) SHOW_ANSWERS=1; shift ;; + -i|--iterations) ITERATIONS="$2"; shift 2 ;; + *) echo "Unknown parameter passed: $1"; exit 1 ;; + esac +done + +echo ">>> Running queries 0-42 (multithreaded)..." +echo ">>> Iterations: $ITERATIONS | Show Debug: $SHOW_DEBUG | Show Answers: $SHOW_ANSWERS" -ITERATIONS=${1:-1} RESULTS_DIR="query_results_mt" mkdir -p "${RESULTS_DIR}" @@ -17,18 +37,38 @@ do for i in {0..42} do echo -n ">>> Executing query $i ... " + + CSV_FILE="${RESULTS_DIR}/query_${i}_iter_${iter}.csv" + LOG_FILE="${RESULTS_DIR}/query_${i}_iter_${iter}.log" + set +e - ./run_query_mt.sh "$i" ../columnar_hits_sample.tuff "${RESULTS_DIR}/query_${i}_iter_${iter}.csv" "${RESULTS_DIR}/query_${i}_iter_${iter}.log" + ./run_query_mt.sh "$i" ../columnar_hits_sample.tuff "$CSV_FILE" "$LOG_FILE" exit_code=$? set -e - + if [ $exit_code -ne 0 ]; then echo "FAILED with exit code $exit_code" - echo "See log: ${RESULTS_DIR}/query_${i}_iter_${iter}.log" + echo "See log: $LOG_FILE" + echo "--- CRASH LOG ---" + cat "$LOG_FILE" # При падении всегда выводим лог, чтобы сразу видеть ошибку exit $exit_code fi echo "OK" + + # Если запрошен вывод ответов + if [ "$SHOW_ANSWERS" -eq 1 ]; then + echo "------ ANSWERS (Query $i) ------" + cat "$CSV_FILE" + echo "--------------------------------" + fi + + # Если запрошен вывод дебаг информации (времени выполнения и т.д.) + if [ "$SHOW_DEBUG" -eq 1 ]; then + echo "------- DEBUG (Query $i) -------" + cat "$LOG_FILE" + echo "--------------------------------" + fi done done -echo ">>> All $ITERATIONS iterations of all queries completed successfully." +echo ">>> All $ITERATIONS iterations of all queries completed successfully." \ No newline at end of file diff --git a/src/execution/ExecutionApi.h b/src/execution/ExecutionApi.h index 947905e..2dd95d4 100644 --- a/src/execution/ExecutionApi.h +++ b/src/execution/ExecutionApi.h @@ -123,20 +123,19 @@ class DataFrame { } DataResult Collect(size_t threads = 0) const { - size_t num_threads = threads; - if (num_threads == 0) { + if (threads == 0) { #ifdef ENABLE_MULTITHREADING - num_threads = std::thread::hardware_concurrency(); - if (num_threads == 0) { - num_threads = 2; + threads = std::thread::hardware_concurrency(); + if (threads == 0) { + threads = 2; } #else - num_threads = 1; + threads = 1; #endif } PipelineBuildContext ctx; - ctx.num_threads = num_threads; + ctx.num_threads = threads; auto outer_pipe = std::make_unique(); auto result_sink = std::make_shared(); @@ -146,9 +145,9 @@ class DataFrame { logical_plan_->BuildPipelines(ctx); ctx.completed_pipelines.push_back(std::move(outer_pipe)); - ThreadPool thread_pool(num_threads); + ThreadPool thread_pool(threads); for (auto& pipeline : ctx.completed_pipelines) { - pipeline->Execute(thread_pool, num_threads); + pipeline->Execute(thread_pool, threads); } return DataResult{result_sink->TakeBatches(), std::move(ctx.schema)}; diff --git a/src/execution/ExecutionLogic.cpp b/src/execution/ExecutionLogic.cpp index e752ee0..615c1d7 100644 --- a/src/execution/ExecutionLogic.cpp +++ b/src/execution/ExecutionLogic.cpp @@ -51,7 +51,7 @@ void ScanNode::BuildPipelines(PipelineBuildContext& ctx, void AggregateNode::BuildPipelines(PipelineBuildContext& ctx, std::optional>) const { Pipeline* outer_pipeline = ctx.current_pipeline; - Pipeline* inner_pipeline = new Pipeline(); + Pipeline* inner_pipeline = new Pipeline(); // TODO: this is bad, should not use raw pointers ctx.current_pipeline = inner_pipeline; if (!group_by_columns.empty()) { @@ -67,7 +67,7 @@ void AggregateNode::BuildPipelines(PipelineBuildContext& ctx, child->BuildPipelines(ctx, needed_columns); std::vector group_col_indices; - std::vector> key_builders; + std::vector key_types; Schema output_schema; for (const std::string& col_name : group_by_columns) { @@ -76,16 +76,18 @@ void AggregateNode::BuildPipelines(PipelineBuildContext& ctx, ColumnType col_type = ctx.schema.GetColumnTypeByName(col_name); output_schema.AddColumn(col_name, col_type); - key_builders.push_back(ColumnFactory::MakeColumnBuilder(col_type)); + key_types.push_back(col_type); } std::vector> agg_funcs; for (const std::shared_ptr& expr : aggregate_expressions) { - agg_funcs.push_back(expr->CreateGroupedAggregationFunction(ctx.schema, output_schema, ctx.num_threads)); + agg_funcs.push_back( + expr->CreateGroupedAggregationFunction(ctx.schema, output_schema, ctx.num_threads)); } auto breaker = std::make_shared( - std::move(group_col_indices), std::move(key_builders), std::move(agg_funcs)); + std::move(group_col_indices), std::move(key_types), std::move(agg_funcs), + ctx.num_threads); inner_pipeline->sink = breaker; ctx.completed_pipelines.push_back(std::unique_ptr(inner_pipeline)); @@ -149,7 +151,7 @@ void FilterNode::BuildPipelines(PipelineBuildContext& ctx, void OrderByNode::BuildPipelines(PipelineBuildContext& ctx, std::optional> required_columns) const { Pipeline* outer_pipeline = ctx.current_pipeline; - Pipeline* inner_pipeline = new Pipeline(); + Pipeline* inner_pipeline = new Pipeline(); // TODO: this is bad, should not use raw pointers ctx.current_pipeline = inner_pipeline; std::vector needed_columns; @@ -171,8 +173,14 @@ void OrderByNode::BuildPipelines(PipelineBuildContext& ctx, sort_columns.push_back({sort_col_idx, is_desc}); } - auto breaker = std::make_shared(std::move(sort_columns), - std::move(limit), std::move(offset)); + std::vector accum_types; + for (const auto& field : ctx.schema.GetFields()) { + accum_types.push_back(field.type); + } + + auto breaker = std::make_shared( + std::move(sort_columns), std::move(accum_types), std::move(limit), std::move(offset), + ctx.num_threads); inner_pipeline->sink = breaker; ctx.completed_pipelines.push_back(std::unique_ptr(inner_pipeline)); diff --git a/src/execution/OperatorsBase.cpp b/src/execution/OperatorsBase.cpp index 62d7b00..d33b8e7 100644 --- a/src/execution/OperatorsBase.cpp +++ b/src/execution/OperatorsBase.cpp @@ -201,39 +201,44 @@ std::unique_ptr AggregationSinkSourceOperator::GetData() { // ========================= GroupBySinkSourceOperator ========================= GroupBySinkSourceOperator::GroupBySinkSourceOperator( - std::vector group_by_col_indices, - std::vector> key_builders, - std::vector> agg_funcs) + std::vector group_by_col_indices, std::vector key_types, + std::vector> agg_funcs, size_t num_threads) : group_by_col_indices_(std::move(group_by_col_indices)), - key_builders_(std::move(key_builders)), - agg_funcs_(std::move(agg_funcs)) { + key_types_(std::move(key_types)), + agg_funcs_(std::move(agg_funcs)), + thread_states_(num_threads) { } void GroupBySinkSourceOperator::Sink(std::unique_ptr batch, size_t thread_id) { + GroupByThreadState& local = thread_states_[thread_id]; + + if (local.key_builders.empty()) { + for (auto type : key_types_) { + local.key_builders.push_back(ColumnFactory::MakeColumnBuilder(type)); + } + } + std::vector group_ids; size_t batch_size = batch->num_rows; group_ids.reserve(batch_size); const std::shared_ptr>& sel_vec = batch->selection_vector; std::vector composite_keys(batch_size); - for (size_t col_idx : group_by_col_indices_) { - ExecutionHelper::HashKeys(*batch->columns[col_idx], composite_keys, sel_vec.get()); + for (size_t col_ind : group_by_col_indices_) { + ExecutionHelper::HashKeys(*batch->columns[col_ind], composite_keys, sel_vec.get()); } - LockGuard lock(sink_mutex_); - std::vector new_group_indices; - size_t sz = num_groups_; - bool is_filtered = sel_vec != nullptr; + for (size_t i = 0; i < batch_size; ++i) { const std::string& key = composite_keys[i]; - auto it = hash_table_.find(key); + auto it = local.hash_table.find(key); uint32_t gid; - if (it == hash_table_.end()) { - gid = sz + new_group_indices.size(); - hash_table_[key] = gid; + if (it == local.hash_table.end()) { + gid = local.num_groups++; + local.hash_table[key] = gid; size_t actual_idx = is_filtered ? (*sel_vec)[i] : i; new_group_indices.push_back(actual_idx); } else { @@ -244,24 +249,72 @@ void GroupBySinkSourceOperator::Sink(std::unique_ptr batch, size_t if (!new_group_indices.empty()) { for (size_t k = 0; k < group_by_col_indices_.size(); ++k) { - ExecutionHelper::CopySelection( - *key_builders_[k], *batch->columns[group_by_col_indices_[k]], &new_group_indices); + ExecutionHelper::CopySelection(*local.key_builders[k], + *batch->columns[group_by_col_indices_[k]], + &new_group_indices); } } - num_groups_ = hash_table_.size(); for (auto& func : agg_funcs_) { - func->Resize(num_groups_); + func->Resize(local.num_groups, thread_id); func->Update(*batch, group_ids, thread_id); } } void GroupBySinkSourceOperator::Finalize() && { + GroupByThreadState& global = thread_states_[0]; + + if (global.key_builders.empty()) { + for (auto type : key_types_) { + global.key_builders.push_back(ColumnFactory::MakeColumnBuilder(type)); + } + } + + for (size_t t = 1; t < thread_states_.size(); ++t) { + GroupByThreadState& local = thread_states_[t]; + if (local.num_groups == 0) { + continue; + } + + std::vector> local_key_cols; + for (std::shared_ptr& key_builder : local.key_builders) { + local_key_cols.push_back(key_builder->Finish()); + } + + for (const auto& [key, local_gid] : local.hash_table) { + auto it = global.hash_table.find(key); + uint32_t target_global_gid; + + if (it == global.hash_table.end()) { + target_global_gid = global.num_groups++; + global.hash_table[key] = target_global_gid; + + std::vector single_ind = {local_gid}; + for (size_t k = 0; k < global.key_builders.size(); ++k) { + ExecutionHelper::CopySelection(*global.key_builders[k], *local_key_cols[k], + &single_ind); + } + + for (auto& func : agg_funcs_) { + func->Resize(global.num_groups, 0); + } + } else { + target_global_gid = it->second; + } + + for (auto& func : agg_funcs_) { + func->CombineState(t, local_gid, target_global_gid); + } + } + + local.hash_table.clear(); + } + result_batch_ = std::make_unique(); - result_batch_->num_rows = num_groups_; + result_batch_->num_rows = global.num_groups; - for (size_t k = 0; k < key_builders_.size(); ++k) { - result_batch_->columns.push_back(key_builders_[k]->Finish()); + for (size_t k = 0; k < global.key_builders.size(); ++k) { + result_batch_->columns.push_back(global.key_builders[k]->Finish()); } for (size_t a = 0; a < agg_funcs_.size(); ++a) { result_batch_->columns.push_back(agg_funcs_[a]->Finalize()); @@ -279,9 +332,13 @@ std::unique_ptr GroupBySinkSourceOperator::GetData() { // ========================= OrderBySinkSourceOperator ========================= OrderBySinkSourceOperator::OrderBySinkSourceOperator( - std::vector> sort_columns, std::optional limit, - std::optional offset) - : sort_columns_(std::move(sort_columns)), limit_(limit), offset_(offset) { + std::vector> sort_columns, std::vector column_types, + std::optional limit, std::optional offset, size_t num_threads) + : sort_columns_(std::move(sort_columns)), + column_types_(column_types), + limit_(limit), + offset_(offset), + thread_states_(num_threads) { total_limit_ = limit_; if (total_limit_.has_value()) { if (offset_.has_value()) { @@ -311,39 +368,125 @@ int Compare(const void* raw_data, uint32_t a, uint32_t b) { return 0; } -void OrderBySinkSourceOperator::Sink(std::unique_ptr batch, size_t /* thread_id */) { - LockGuard lock(sink_mutex_); +void OrderBySinkSourceOperator::Sink(std::unique_ptr batch, size_t thread_id) { + OrderByThreadState& local = thread_states_[thread_id]; - if (accum_builders_.empty()) { - for (size_t c = 0; c < batch->columns.size(); ++c) { - accum_builders_.push_back( - ColumnFactory::MakeColumnBuilder(batch->columns[c]->GetType())); - accum_types_.push_back(batch->columns[c]->GetType()); + if (local.column_builders.empty()) { + for (auto type : column_types_) { + local.column_builders.push_back(ColumnFactory::MakeColumnBuilder(type)); } - accum_num_rows_ = 0; + local.accum_num_rows = 0; } const std::shared_ptr>& sel = batch->selection_vector; - for (size_t c = 0; c < accum_builders_.size(); ++c) { - ExecutionHelper::CopySelection(*accum_builders_[c], *batch->columns[c], sel.get()); + for (size_t c = 0; c < local.column_builders.size(); ++c) { + ExecutionHelper::CopySelection(*local.column_builders[c], *batch->columns[c], sel.get()); } - accum_num_rows_ += batch->num_rows; + local.accum_num_rows += batch->num_rows; - if (total_limit_.has_value() && accum_num_rows_ > total_limit_.value() * 10) { - TrimCurBatch(false); + if (total_limit_.has_value() && local.accum_num_rows > total_limit_.value() * 10) { + TrimLocalBatch(thread_id); } } +// TODO: IT DOES NOT USE MULTITHREADING! if we don't have limit in a query, then OrderBy will work +// with a speed of one thread algorithm... void OrderBySinkSourceOperator::Finalize() && { - if (accum_num_rows_ > 0) { - TrimCurBatch(true); + std::vector> global_builders; + for (ColumnType type : column_types_) { + global_builders.push_back(ColumnFactory::MakeColumnBuilder(type)); + } + + size_t num_rows = 0; + for (size_t t = 0; t < thread_states_.size(); ++t) { + OrderByThreadState& local = thread_states_[t]; + if (local.accum_num_rows == 0) { + continue; + } + std::vector> local_cols; + for (auto& builder : local.column_builders) { + local_cols.push_back(builder->Finish()); + } + + for (size_t c = 0; c < global_builders.size(); ++c) { + ExecutionHelper::CopySelection(*global_builders[c], *local_cols[c], nullptr); + } + num_rows += local.accum_num_rows; + local.column_builders.clear(); + } + if (num_rows == 0) { result_batch_ = std::make_unique(); - result_batch_->num_rows = accum_num_rows_; - for (auto& builder : accum_builders_) { - result_batch_->columns.push_back(builder->Finish()); + result_batch_->num_rows = 0; + return; + } + + auto temp_batch = std::make_unique(); + temp_batch->num_rows = num_rows; + for (auto& builder : global_builders) { + temp_batch->columns.push_back(builder->Finish()); + } + + std::vector indices(num_rows); + std::iota(indices.begin(), indices.end(), 0); + + std::vector cols_info; + cols_info.reserve(sort_columns_.size()); + for (const auto& [col_ind, is_desc] : sort_columns_) { + const Column* raw_column = temp_batch->columns[col_ind].get(); + auto col_type = raw_column->GetType(); +#define HANDLE_TYPE(ENUM_VAL, STR_VAL, CLASS_TYPE) \ + case ColumnType::ENUM_VAL: { \ + const void* data_ptr = raw_column->GetRawData(); \ + cols_info.push_back({data_ptr, &Compare, is_desc}); \ + break; \ + } + switch (col_type) { + FOR_EACH_COLUMN_TYPE(HANDLE_TYPE); + default: THROW_NOT_IMPLEMENTED; + } +#undef HANDLE_TYPE + } + + auto cmp = [&cols_info](uint32_t a, uint32_t b) { + for (const auto& info : cols_info) { + int res = info.cmp_func(info.raw_data, a, b); + if (res != 0) { + return info.is_desc ? (res > 0) : (res < 0); + } + } + return false; + }; + + if (total_limit_.has_value() && num_rows > total_limit_.value()) { + uint32_t lim = total_limit_.value(); + std::partial_sort(indices.begin(), indices.begin() + lim, indices.end(), cmp); + indices.resize(lim); + } else { + std::sort(indices.begin(), indices.end(), cmp); + } + + if (offset_.has_value() && offset_.value() > 0) { + size_t off = offset_.value(); + if (off >= indices.size()) { + indices.clear(); + } else { + indices.erase(indices.begin(), indices.begin() + off); } } + + result_batch_ = std::make_unique(); + result_batch_->num_rows = indices.size(); + + std::vector> final_builders; + for (auto type : column_types_) { + final_builders.push_back(ColumnFactory::MakeColumnBuilder(type)); + } + + for (size_t c = 0; c < final_builders.size(); ++c) { + ExecutionHelper::CopySelection(*final_builders[c], *temp_batch->columns[c], &indices); + result_batch_->columns.push_back(final_builders[c]->Finish()); + } } std::unique_ptr OrderBySinkSourceOperator::GetData() { @@ -354,20 +497,22 @@ std::unique_ptr OrderBySinkSourceOperator::GetData() { return nullptr; } -void OrderBySinkSourceOperator::TrimCurBatch(bool is_final) { - if (accum_num_rows_ == 0) { +void OrderBySinkSourceOperator::TrimLocalBatch(size_t thread_id) { + OrderByThreadState& local = thread_states_[thread_id]; + + if (local.accum_num_rows == 0) { return; } auto temp_batch = std::make_unique(); - temp_batch->num_rows = accum_num_rows_; - for (auto& builder : accum_builders_) { + temp_batch->num_rows = local.accum_num_rows; + for (auto& builder : local.column_builders) { temp_batch->columns.push_back(builder->Finish()); } - accum_builders_.clear(); + local.column_builders.clear(); - std::vector indices(accum_num_rows_); - for (size_t i = 0; i < accum_num_rows_; ++i) { + std::vector indices(local.accum_num_rows); + for (size_t i = 0; i < local.accum_num_rows; ++i) { indices[i] = i; } @@ -400,27 +545,28 @@ void OrderBySinkSourceOperator::TrimCurBatch(bool is_final) { return false; }; - if (total_limit_.has_value() && accum_num_rows_ > total_limit_.value()) { + if (total_limit_.has_value() && local.accum_num_rows > total_limit_.value()) { uint32_t lim = total_limit_.value(); std::nth_element(indices.begin(), indices.begin() + lim, indices.end(), cmp); indices.resize(lim); } - if (is_final) { - std::sort(indices.begin(), indices.end(), cmp); - if (offset_.has_value()) { - if (offset_.value() < indices.size()) { - indices.erase(indices.begin(), indices.begin() + offset_.value()); - } else { - indices.clear(); - } - } - } - - for (auto type : accum_types_) { - accum_builders_.push_back(ColumnFactory::MakeColumnBuilder(type)); - } - for (size_t c = 0; c < accum_builders_.size(); ++c) { - ExecutionHelper::CopySelection(*accum_builders_[c], *temp_batch->columns[c], &indices); - } - accum_num_rows_ = indices.size(); + // if (is_final) { + // std::sort(indices.begin(), indices.end(), cmp); + // if (offset_.has_value()) { + // if (offset_.value() < indices.size()) { + // indices.erase(indices.begin(), indices.begin() + offset_.value()); + // } else { + // indices.clear(); + // } + // } + // } + + for (auto type : column_types_) { + local.column_builders.push_back(ColumnFactory::MakeColumnBuilder(type)); + } + for (size_t c = 0; c < local.column_builders.size(); ++c) { + ExecutionHelper::CopySelection(*local.column_builders[c], *temp_batch->columns[c], + &indices); + } + local.accum_num_rows = indices.size(); } diff --git a/src/execution/OperatorsBase.h b/src/execution/OperatorsBase.h index d2d9b2c..10aafae 100644 --- a/src/execution/OperatorsBase.h +++ b/src/execution/OperatorsBase.h @@ -114,8 +114,9 @@ class AggregationSinkSourceOperator : public SinkOperator, public SourceOperator class GroupBySinkSourceOperator : public SinkOperator, public SourceOperator { public: GroupBySinkSourceOperator(std::vector group_by_col_indices, - std::vector> key_builders, - std::vector> agg_funcs); + std::vector key_types, + std::vector> agg_funcs, + size_t num_threads); void Sink(std::unique_ptr batch, size_t thread_id) override; void Finalize() && override; @@ -123,14 +124,17 @@ class GroupBySinkSourceOperator : public SinkOperator, public SourceOperator { std::unique_ptr GetData() override; private: - Mutex sink_mutex_; + struct alignas(64) GroupByThreadState { + absl::flat_hash_map hash_table; + std::vector> key_builders; + size_t num_groups = 0; + }; std::vector group_by_col_indices_; - std::vector> key_builders_; + std::vector key_types_; std::vector> agg_funcs_; - size_t num_groups_ = 0; - absl::flat_hash_map hash_table_; + std::vector thread_states_; std::unique_ptr result_batch_; Atomic emitted_{false}; }; @@ -138,7 +142,8 @@ class GroupBySinkSourceOperator : public SinkOperator, public SourceOperator { class OrderBySinkSourceOperator : public SinkOperator, public SourceOperator { public: OrderBySinkSourceOperator(std::vector> sort_columns, - std::optional limit, std::optional offset); + std::vector column_types, std::optional limit, + std::optional offset, size_t num_threads); void Sink(std::unique_ptr batch, size_t thread_id) override; void Finalize() && override; @@ -146,17 +151,20 @@ class OrderBySinkSourceOperator : public SinkOperator, public SourceOperator { std::unique_ptr GetData() override; private: - void TrimCurBatch(bool is_final); + void TrimLocalBatch(size_t thread_id); - Mutex sink_mutex_; + struct alignas(64) OrderByThreadState { + std::vector> column_builders; + size_t accum_num_rows = 0; + }; std::vector> sort_columns_; - std::vector> accum_builders_; - std::vector accum_types_; - size_t accum_num_rows_ = 0; + const std::vector column_types_; std::optional limit_; std::optional offset_; std::optional total_limit_; + + std::vector thread_states_; std::unique_ptr result_batch_; Atomic emitted_{false}; }; diff --git a/src/execution/Pipeline.h b/src/execution/Pipeline.h index fc82853..daf2fea 100644 --- a/src/execution/Pipeline.h +++ b/src/execution/Pipeline.h @@ -46,10 +46,10 @@ class Pipeline { std::shared_ptr sink; void Execute(ThreadPool& thread_pool, size_t num_threads) { - Atomic num_tasks_finished{0}; + size_t num_tasks_finished{0}; Mutex wait_mutex; ConditionVariable wait_cond; - for (size_t thread_id = 0; thread_id < num_threads; thread_id++) { + for (size_t thread_id = 0; thread_id < num_threads; ++thread_id) { thread_pool.Enqueue( [this, thread_id, &num_tasks_finished, &wait_mutex, &wait_cond, num_threads]() { while (auto batch = source->GetData()) { @@ -80,9 +80,8 @@ class Pipeline { } { UniqueLock lock(wait_mutex); - size_t finished_before = - num_tasks_finished.fetch_add(1, std::memory_order_release); - if (finished_before + 1 == num_threads) { + ++num_tasks_finished; + if (num_tasks_finished == num_threads) { wait_cond.notify_one(); } } @@ -92,7 +91,7 @@ class Pipeline { { UniqueLock lock(wait_mutex); wait_cond.wait(lock, [&]() { - return num_tasks_finished.load(std::memory_order_acquire) == num_threads; + return num_tasks_finished == num_threads; }); } diff --git a/src/execution/expressions/AggregationFunctions.h b/src/execution/expressions/AggregationFunctions.h index ee3970c..0ead79b 100644 --- a/src/execution/expressions/AggregationFunctions.h +++ b/src/execution/expressions/AggregationFunctions.h @@ -51,9 +51,10 @@ class GroupedAggregationFunction : public AggregationFunction { : AggregationFunction(column_type, num_threads) { } - virtual void Resize(size_t num_groups) = 0; + virtual void Resize(size_t num_groups, size_t thread_id) = 0; virtual void Update(const RecordBatch& batch, const std::vector& group_ids, size_t thread_id) = 0; + virtual void CombineState(size_t src_thread, uint32_t src_gid, uint32_t global_gid) = 0; }; struct SumOperation { @@ -95,11 +96,11 @@ class TypedGlobalAggregationFunction : public GlobalAggregationFunction { TypedGlobalAggregationFunction(size_t column_index, ColumnType column_type, size_t num_threads) : GlobalAggregationFunction(column_type, num_threads), column_index_(column_index), - states_(num_threads) { + thread_states_(num_threads) { } void Update(const RecordBatch& batch, size_t thread_id) override { - ThreadState& local_state = states_[thread_id]; + ThreadState& local_state = thread_states_[thread_id]; AggExpHelper::IterateColumnData( batch, column_index_, [&](const auto& value, size_t, size_t) { @@ -117,14 +118,14 @@ class TypedGlobalAggregationFunction : public GlobalAggregationFunction { StateType result{}; bool has_data = false; - for (size_t i = 0; i < states_.size(); i++) { + for (size_t i = 0; i < thread_states_.size(); i++) { if (has_data) { - if (states_[i].has_data) { - Operation::Apply(result, states_[i].value); + if (thread_states_[i].has_data) { + Operation::Apply(result, thread_states_[i].value); } - } else if (states_[i].has_data) { + } else if (thread_states_[i].has_data) { has_data = true; - result = states_[i].value; + result = thread_states_[i].value; } } @@ -142,7 +143,7 @@ class TypedGlobalAggregationFunction : public GlobalAggregationFunction { private: const size_t column_index_; - std::vector states_; + std::vector thread_states_; }; template @@ -152,42 +153,65 @@ class TypedGroupedAggregationFunction : public GroupedAggregationFunction { public: TypedGroupedAggregationFunction(size_t column_index, ColumnType column_type, size_t num_threads) - : GroupedAggregationFunction(column_type, num_threads), column_index_(column_index) { + : GroupedAggregationFunction(column_type, num_threads), + column_index_(column_index), + thread_states_(num_threads) { } - void Resize(size_t num_groups) override { - if (num_groups > states_.size()) { - states_.resize(num_groups); - has_data_.resize(num_groups, 0); + void Resize(size_t num_groups, size_t thread_id) override { + auto& local = thread_states_[thread_id]; + if (num_groups > local.states.size()) { + local.states.resize(num_groups); + local.has_data.resize(num_groups, 0); } } void Update(const RecordBatch& batch, const std::vector& group_ids, - size_t /* thread_id */) override { + size_t thread_id) override { + auto& local = thread_states_[thread_id]; AggExpHelper::IterateColumnData( batch, column_index_, [&](const auto& value, size_t row_idx, size_t) { uint32_t gid = group_ids[row_idx]; - if (!has_data_[gid]) { - states_[gid] = value; - has_data_[gid] = 1; + if (!local.has_data[gid]) { + local.states[gid] = value; + local.has_data[gid] = 1; } else { - Operation::Apply(states_[gid], value); + Operation::Apply(local.states[gid], value); } }); } + void CombineState(size_t src_thread, uint32_t src_gid, uint32_t global_gid) override { + auto& src = thread_states_[src_thread]; + auto& global = thread_states_[0]; + + if (src.has_data[src_gid]) { + if (!global.has_data[global_gid]) { + global.states[global_gid] = src.states[src_gid]; + global.has_data[global_gid] = 1; + } else { + Operation::Apply(global.states[global_gid], src.states[src_gid]); + } + } + } + protected: void WriteResult(const std::shared_ptr& builder) override { auto* typed = static_cast(builder.get()); - for (const auto& state : states_) { + auto& global = thread_states_[0]; + for (const StateType& state : global.states) { typed->AddValue(state); } } private: + struct alignas(64) ThreadState { + std::vector states; + std::vector has_data; + }; + size_t column_index_; - std::vector states_; - std::vector has_data_; + std::vector thread_states_; }; template @@ -201,17 +225,17 @@ class CountGlobalAggregationFunction : public GlobalAggregationFunction { public: CountGlobalAggregationFunction(ColumnType column_type, size_t num_threads) - : GlobalAggregationFunction(column_type, num_threads), states_(num_threads) { + : GlobalAggregationFunction(column_type, num_threads), thread_states_(num_threads) { } void Update(const RecordBatch& batch, size_t thread_id) override { - states_[thread_id].count += batch.num_rows; + thread_states_[thread_id].count += batch.num_rows; } protected: void WriteResult(const std::shared_ptr& builder) override { int64_t total = 0; - for (const auto& state : states_) { + for (const auto& state : thread_states_) { total += state.count; } auto* typed = static_cast(builder.get()); @@ -219,7 +243,7 @@ class CountGlobalAggregationFunction : public GlobalAggregationFunction { } private: - std::vector states_; + std::vector thread_states_; }; template @@ -229,33 +253,47 @@ class CountGroupedAggregationFunction : public GroupedAggregationFunction { public: CountGroupedAggregationFunction(ColumnType column_type, size_t num_threads) - : GroupedAggregationFunction(column_type, num_threads) { + : GroupedAggregationFunction(column_type, num_threads), thread_states_(num_threads) { } - void Resize(size_t num_groups) override { - if (num_groups > counts_.size()) { - counts_.resize(num_groups, 0); + void Resize(size_t num_groups, size_t thread_id) override { + auto& local = thread_states_[thread_id]; + if (num_groups > local.counts.size()) { + local.counts.resize(num_groups, 0); } } void Update(const RecordBatch& batch, const std::vector& group_ids, - size_t /* thread_id */) override { + size_t thread_id) override { + auto& local = thread_states_[thread_id]; for (size_t i = 0; i < batch.num_rows; ++i) { uint32_t gid = group_ids[i]; - ++counts_[gid]; + ++local.counts[gid]; } } + void CombineState(size_t src_thread, uint32_t src_gid, uint32_t global_gid) override { + ThreadState& src = thread_states_[src_thread]; + ThreadState& global = thread_states_[0]; + global.counts[global_gid] += src.counts[src_gid]; + } + protected: void WriteResult(const std::shared_ptr& builder) override { auto* typed = static_cast(builder.get()); - for (int64_t c : counts_) { + auto& global = thread_states_[0]; + + for (int64_t c : global.counts) { typed->AddValue(static_cast(c)); } } private: - std::vector counts_; + struct alignas(64) ThreadState { + std::vector counts; + }; + + std::vector thread_states_; }; template @@ -273,11 +311,11 @@ class AvgGlobalAggregationFunction : public GlobalAggregationFunction { AvgGlobalAggregationFunction(size_t column_index, ColumnType column_type, size_t num_threads) : GlobalAggregationFunction(column_type, num_threads), column_index_(column_index), - states_(num_threads) { + thread_states_(num_threads) { } void Update(const RecordBatch& batch, size_t thread_id) override { - ThreadState& local = states_[thread_id]; + ThreadState& local = thread_states_[thread_id]; AggExpHelper::IterateColumnData(batch, column_index_, [&](const auto& value, size_t, size_t) { local.sum += value; @@ -289,7 +327,7 @@ class AvgGlobalAggregationFunction : public GlobalAggregationFunction { void WriteResult(const std::shared_ptr& builder) override { StateType total_sum = 0; int64_t total_count = 0; - for (const auto& state : states_) { + for (const auto& state : thread_states_) { total_sum += state.sum; total_count += state.count; } @@ -306,8 +344,7 @@ class AvgGlobalAggregationFunction : public GlobalAggregationFunction { static OutputType ComputeAvg(StateType sum, int64_t count) { if constexpr (std::is_integral_v) { OutputType int_part = static_cast(sum / count); - OutputType rem = - static_cast(static_cast(sum % count) / count); + OutputType rem = static_cast(static_cast(sum % count) / count); return int_part + rem; } else { return static_cast(sum) / static_cast(count); @@ -315,7 +352,7 @@ class AvgGlobalAggregationFunction : public GlobalAggregationFunction { } size_t column_index_; - std::vector states_; + std::vector thread_states_; }; template @@ -326,29 +363,42 @@ class AvgGroupedAggregationFunction : public GroupedAggregationFunction { public: AvgGroupedAggregationFunction(size_t column_index, ColumnType column_type, size_t num_threads) - : GroupedAggregationFunction(column_type, num_threads), column_index_(column_index) { + : GroupedAggregationFunction(column_type, num_threads), + column_index_(column_index), + thread_states_(num_threads) { } - void Resize(size_t num_groups) override { - if (num_groups > states_.size()) { - states_.resize(num_groups); + void Resize(size_t num_groups, size_t thread_id) override { + auto& local = thread_states_[thread_id]; + if (num_groups > local.states.size()) { + local.states.resize(num_groups); } } void Update(const RecordBatch& batch, const std::vector& group_ids, - size_t /* thread_id */) override { + size_t thread_id) override { + auto& local = thread_states_[thread_id]; AggExpHelper::IterateColumnData( batch, column_index_, [&](const auto& value, size_t row_idx, size_t) { uint32_t gid = group_ids[row_idx]; - states_[gid].sum += value; - ++states_[gid].count; + local.states[gid].sum += value; + ++local.states[gid].count; }); } + void CombineState(size_t src_thread, uint32_t src_gid, uint32_t global_gid) override { + ThreadState& src = thread_states_[src_thread]; + ThreadState& global = thread_states_[0]; + global.states[global_gid].sum += src.states[src_gid].sum; + global.states[global_gid].count += src.states[src_gid].count; + } + protected: void WriteResult(const std::shared_ptr& builder) override { auto* typed = static_cast(builder.get()); - for (const auto& state : states_) { + auto& global = thread_states_[0]; + + for (const auto& state : global.states) { if (state.count > 0) { typed->AddValue(ComputeAvg(state.sum, state.count)); } else { @@ -368,12 +418,17 @@ class AvgGroupedAggregationFunction : public GroupedAggregationFunction { } } - struct State { - StateType sum = 0; - int64_t count = 0; + struct alignas(64) ThreadState { + struct State { + StateType sum; + int64_t count; + }; + + std::vector states; }; + size_t column_index_; - std::vector states_; + std::vector thread_states_; }; template @@ -382,10 +437,6 @@ class DistinctCountGlobalAggregationFunction : public GlobalAggregationFunction using OutputValueType = OutputColumn::ValueType; using OutputBuilder = BuilderTypeTrait::Type; - struct alignas(64) Set { - absl::flat_hash_set set; - }; - public: DistinctCountGlobalAggregationFunction(size_t column_index, ColumnType column_type, size_t num_threads) @@ -397,9 +448,8 @@ class DistinctCountGlobalAggregationFunction : public GlobalAggregationFunction void Update(const RecordBatch& batch, size_t thread_id) override { auto& local_set = thread_sets_[thread_id].set; AggExpHelper::IterateColumnData( - batch, column_index_, [&](const auto& value, size_t, size_t) { - local_set.insert(InputValueType(value)); - }); + batch, column_index_, + [&](const auto& value, size_t, size_t) { local_set.insert(InputValueType(value)); }); } protected: @@ -416,6 +466,10 @@ class DistinctCountGlobalAggregationFunction : public GlobalAggregationFunction } private: + struct alignas(64) Set { + absl::flat_hash_set set; + }; + size_t column_index_; std::vector thread_sets_; }; @@ -429,33 +483,55 @@ class DistinctCountGroupedAggregationFunction : public GroupedAggregationFunctio public: DistinctCountGroupedAggregationFunction(size_t column_index, ColumnType column_type, size_t num_threads) - : GroupedAggregationFunction(column_type, num_threads), column_index_(column_index) { + : GroupedAggregationFunction(column_type, num_threads), + column_index_(column_index), + thread_states_(num_threads) { } - void Resize(size_t num_groups) override { - if (num_groups > distinct_values_.size()) { - distinct_values_.resize(num_groups); + void Resize(size_t num_groups, size_t thread_id) override { + auto& local = thread_states_[thread_id]; + if (num_groups > local.sets.size()) { + local.sets.resize(num_groups); } } void Update(const RecordBatch& batch, const std::vector& group_ids, - size_t /* thread_id */) override { + size_t thread_id) override { + auto& local = thread_states_[thread_id]; AggExpHelper::IterateColumnData( batch, column_index_, [&](const auto& value, size_t row_idx, size_t) { uint32_t gid = group_ids[row_idx]; - distinct_values_[gid].insert(InputValueType(value)); + local.sets[gid].set.insert(InputValueType(value)); }); } + void CombineState(size_t src_thread, uint32_t src_gid, uint32_t global_gid) override { + ThreadState& src = thread_states_[src_thread]; + ThreadState& global = thread_states_[0]; + for (const auto& val : src.sets[src_gid].set) { + global.sets[global_gid].set.insert(val); + } + } + protected: void WriteResult(const std::shared_ptr& builder) override { - auto* typed = static_cast(builder.get()); - for (const auto& set : distinct_values_) { - typed->AddValue(static_cast(set.size())); + OutputBuilder* typed = static_cast(builder.get()); + ThreadState& global = thread_states_[0]; + + for (const auto& wrapper : global.sets) { + typed->AddValue(static_cast(wrapper.set.size())); } } private: + struct alignas(64) ThreadState { + struct Set { + absl::flat_hash_set set; + }; + + std::vector sets; + }; + size_t column_index_; - std::vector> distinct_values_; + std::vector thread_states_; }; From df00b8180056a0d401c81b1aca6a6158590957b6 Mon Sep 17 00:00:00 2001 From: Mikhail Fomichev Date: Wed, 27 May 2026 18:05:30 +0300 Subject: [PATCH 15/18] fix: CsvTokenizer fix, script fix --- script/run_all_queries.sh | 39 ++++++++-- script/run_all_queries_mt.sh | 43 ++++++++--- script/run_query.sh | 13 +++- script/run_query_mt.sh | 15 ++-- src/CMakeLists.txt | 1 - src/column/CharColumn.h | 4 +- src/column/NumericColumn.h | 4 +- src/column/TemporalColumn.h | 4 +- src/execution/Pipeline.h | 2 +- src/io/CsvToColumnar.cpp | 32 ++++---- src/io/CsvTokenizer.cpp | 139 ++++++++++++++--------------------- src/io/CsvTokenizer.h | 6 +- 12 files changed, 171 insertions(+), 131 deletions(-) diff --git a/script/run_all_queries.sh b/script/run_all_queries.sh index 2b53cf7..660463a 100755 --- a/script/run_all_queries.sh +++ b/script/run_all_queries.sh @@ -27,7 +27,9 @@ echo ">>> Running queries 0-42 (single thread)..." echo ">>> Iterations: $ITERATIONS | Show Debug: $SHOW_DEBUG | Show Answers: $SHOW_ANSWERS" RESULTS_DIR="query_results" -mkdir -p "${RESULTS_DIR}" +if [ "$SHOW_ANSWERS" -eq 1 ] || [ "$SHOW_DEBUG" -eq 1 ]; then + mkdir -p "${RESULTS_DIR}" +fi for (( iter=1; iter<=ITERATIONS; iter++ )) do @@ -38,35 +40,56 @@ do do echo -n ">>> Executing query $i ... " - CSV_FILE="${RESULTS_DIR}/query_${i}_iter_${iter}.csv" - LOG_FILE="${RESULTS_DIR}/query_${i}_iter_${iter}.log" + # Куда пишем ответы + if [ "$SHOW_ANSWERS" -eq 1 ]; then + CSV_FILE="${RESULTS_DIR}/query_${i}_iter_${iter}.csv" + else + CSV_FILE="/dev/null" + fi + + # Куда пишем логи (чтобы вытащить время) + if [ "$SHOW_DEBUG" -eq 1 ]; then + LOG_FILE="${RESULTS_DIR}/query_${i}_iter_${iter}.log" + else + LOG_FILE=$(mktemp) # Временный файл, который мы потом удалим + fi set +e ./run_query.sh "$i" ../columnar_hits_sample.tuff "$CSV_FILE" "$LOG_FILE" exit_code=$? set -e + # Вытаскиваем время из лога (формат: "Query 0 completed in 2.57227 ms") + EXEC_TIME=$(grep "completed in" "$LOG_FILE" | awk '{print $5, $6}' || true) + if [ $exit_code -ne 0 ]; then echo "FAILED with exit code $exit_code" - echo "See log: $LOG_FILE" echo "--- CRASH LOG ---" - cat "$LOG_FILE" # При падении всегда выводим лог, чтобы сразу видеть ошибку + cat "$LOG_FILE" + if [ "$SHOW_DEBUG" -eq 0 ]; then rm -f "$LOG_FILE"; fi exit $exit_code fi - echo "OK" - # Если запрошен вывод ответов + # Красиво выводим время + if [ -n "$EXEC_TIME" ]; then + echo "${EXEC_TIME} ... OK" + else + echo "OK" + fi + if [ "$SHOW_ANSWERS" -eq 1 ]; then echo "------ ANSWERS (Query $i) ------" cat "$CSV_FILE" echo "--------------------------------" fi - # Если запрошен вывод дебаг информации (времени выполнения и т.д.) if [ "$SHOW_DEBUG" -eq 1 ]; then echo "------- DEBUG (Query $i) -------" cat "$LOG_FILE" echo "--------------------------------" + else + # Удаляем мусор, если дебаг не нужен + rm -f "$LOG_FILE" fi done done diff --git a/script/run_all_queries_mt.sh b/script/run_all_queries_mt.sh index c8d2db0..b3d8651 100755 --- a/script/run_all_queries_mt.sh +++ b/script/run_all_queries_mt.sh @@ -23,11 +23,13 @@ while [[ "$#" -gt 0 ]]; do esac done -echo ">>> Running queries 0-42 (multithreaded)..." +echo ">>> Running queries 0-42 (single thread)..." echo ">>> Iterations: $ITERATIONS | Show Debug: $SHOW_DEBUG | Show Answers: $SHOW_ANSWERS" -RESULTS_DIR="query_results_mt" -mkdir -p "${RESULTS_DIR}" +RESULTS_DIR="query_mt_results" +if [ "$SHOW_ANSWERS" -eq 1 ] || [ "$SHOW_DEBUG" -eq 1 ]; then + mkdir -p "${RESULTS_DIR}" +fi for (( iter=1; iter<=ITERATIONS; iter++ )) do @@ -38,35 +40,56 @@ do do echo -n ">>> Executing query $i ... " - CSV_FILE="${RESULTS_DIR}/query_${i}_iter_${iter}.csv" - LOG_FILE="${RESULTS_DIR}/query_${i}_iter_${iter}.log" + # Куда пишем ответы + if [ "$SHOW_ANSWERS" -eq 1 ]; then + CSV_FILE="${RESULTS_DIR}/query_${i}_iter_${iter}.csv" + else + CSV_FILE="/dev/null" + fi + + # Куда пишем логи (чтобы вытащить время) + if [ "$SHOW_DEBUG" -eq 1 ]; then + LOG_FILE="${RESULTS_DIR}/query_${i}_iter_${iter}.log" + else + LOG_FILE=$(mktemp) # Временный файл, который мы потом удалим + fi set +e ./run_query_mt.sh "$i" ../columnar_hits_sample.tuff "$CSV_FILE" "$LOG_FILE" exit_code=$? set -e + # Вытаскиваем время из лога (формат: "Query 0 completed in 2.57227 ms") + EXEC_TIME=$(grep "completed in" "$LOG_FILE" | awk '{print $5, $6}' || true) + if [ $exit_code -ne 0 ]; then echo "FAILED with exit code $exit_code" - echo "See log: $LOG_FILE" echo "--- CRASH LOG ---" - cat "$LOG_FILE" # При падении всегда выводим лог, чтобы сразу видеть ошибку + cat "$LOG_FILE" + if [ "$SHOW_DEBUG" -eq 0 ]; then rm -f "$LOG_FILE"; fi exit $exit_code fi - echo "OK" - # Если запрошен вывод ответов + # Красиво выводим время + if [ -n "$EXEC_TIME" ]; then + echo "${EXEC_TIME} ... OK" + else + echo "OK" + fi + if [ "$SHOW_ANSWERS" -eq 1 ]; then echo "------ ANSWERS (Query $i) ------" cat "$CSV_FILE" echo "--------------------------------" fi - # Если запрошен вывод дебаг информации (времени выполнения и т.д.) if [ "$SHOW_DEBUG" -eq 1 ]; then echo "------- DEBUG (Query $i) -------" cat "$LOG_FILE" echo "--------------------------------" + else + # Удаляем мусор, если дебаг не нужен + rm -f "$LOG_FILE" fi done done diff --git a/script/run_query.sh b/script/run_query.sh index 9ade173..69e6103 100755 --- a/script/run_query.sh +++ b/script/run_query.sh @@ -27,8 +27,13 @@ if [[ ! -f "${COLUMNAR}" ]]; then exit 2 fi -mkdir -p "$(dirname "${OUTPUT_CSV}")" -mkdir -p "$(dirname "${LOG_FILE}")" +# Создаем директории, только если это реальные файлы, а не /dev/null +if [[ "${OUTPUT_CSV}" != "/dev/null" ]]; then + mkdir -p "$(dirname "${OUTPUT_CSV}")" +fi +if [[ "${LOG_FILE}" != "/dev/null" ]]; then + mkdir -p "$(dirname "${LOG_FILE}")" +fi -# Run the query, capture stdout (CSV result) and tee stderr+stdout to log -"${BIN}" query "${COLUMNAR}" "${QUERY_NUM}" > "${OUTPUT_CSV}" 2> >(tee "${LOG_FILE}" >&2) +# Run the query, capture stdout to CSV and stderr to log +"${BIN}" query "${COLUMNAR}" "${QUERY_NUM}" > "${OUTPUT_CSV}" 2> "${LOG_FILE}" \ No newline at end of file diff --git a/script/run_query_mt.sh b/script/run_query_mt.sh index 499dd7e..440e01d 100755 --- a/script/run_query_mt.sh +++ b/script/run_query_mt.sh @@ -17,7 +17,7 @@ BUILD_DIR="${ROOT_DIR}/cmake-build-release" BIN="${BUILD_DIR}/columnar-engine-mt" if [[ ! -x "${BIN}" ]]; then - echo "ERROR: columnar-engine-mt not found at ${BIN}" >&2 + echo "ERROR: columnar-engine not found at ${BIN}" >&2 echo "Run script/build.sh first" >&2 exit 1 fi @@ -27,8 +27,13 @@ if [[ ! -f "${COLUMNAR}" ]]; then exit 2 fi -mkdir -p "$(dirname "${OUTPUT_CSV}")" -mkdir -p "$(dirname "${LOG_FILE}")" +# Создаем директории, только если это реальные файлы, а не /dev/null +if [[ "${OUTPUT_CSV}" != "/dev/null" ]]; then + mkdir -p "$(dirname "${OUTPUT_CSV}")" +fi +if [[ "${LOG_FILE}" != "/dev/null" ]]; then + mkdir -p "$(dirname "${LOG_FILE}")" +fi -# Run the query, capture stdout (CSV result) and tee stderr+stdout to log -"${BIN}" query "${COLUMNAR}" "${QUERY_NUM}" > "${OUTPUT_CSV}" 2> >(tee "${LOG_FILE}" >&2) +# Run the query, capture stdout to CSV and stderr to log +"${BIN}" query "${COLUMNAR}" "${QUERY_NUM}" > "${OUTPUT_CSV}" 2> "${LOG_FILE}" \ No newline at end of file diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 4d2db56..23508a8 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -73,7 +73,6 @@ target_compile_options(columnar-engine PRIVATE -Werror) add_executable(columnar-engine-mt ${ENGINE_SOURCES}) target_link_libraries(columnar-engine-mt PRIVATE project_includes Threads::Threads) -# ENABLE_MULTITHREADING, it will enable std::atomic and std::mutex target_compile_definitions(columnar-engine-mt PRIVATE ENABLE_MULTITHREADING) target_compile_options(columnar-engine-mt PRIVATE -Werror) diff --git a/src/column/CharColumn.h b/src/column/CharColumn.h index 0328e5d..d28eb1b 100644 --- a/src/column/CharColumn.h +++ b/src/column/CharColumn.h @@ -53,7 +53,9 @@ class CharColumnBuilder : public ColumnBuilder { void AddBatch(const VectorOfStrings2D& batch, size_t j) override { size_t h = batch.Height(); - data_.reserve(data_.size() + h); + if (data_.size() < h) { + data_.reserve(data_.size() + h); + } for (size_t i = 0; i < h; ++i) { std::string_view val = batch.GetString2D(i, j); ASSERT(val.size() == 1); diff --git a/src/column/NumericColumn.h b/src/column/NumericColumn.h index 311d13d..5a00652 100644 --- a/src/column/NumericColumn.h +++ b/src/column/NumericColumn.h @@ -158,7 +158,9 @@ class NumericColumnBuilder : public ColumnBuilder { void AddBatch(const VectorOfStrings2D& batch, size_t j) override { size_t h = batch.Height(); - data_.reserve(data_.size() + h); + if (data_.size() < h) { + data_.reserve(data_.size() + h); + } for (size_t i = 0; i < h; ++i) { data_.push_back(ColumnTypeTraits::FromString(batch.GetString2D(i, j))); } diff --git a/src/column/TemporalColumn.h b/src/column/TemporalColumn.h index 246597b..e94b3ac 100644 --- a/src/column/TemporalColumn.h +++ b/src/column/TemporalColumn.h @@ -74,7 +74,9 @@ class TemporalColumnBuilder : public ColumnBuilder { void AddBatch(const VectorOfStrings2D& batch, size_t j) override { size_t h = batch.Height(); - data_.reserve(data_.size() + h); + if (data_.size() < h) { + data_.reserve(data_.size() + h); + } for (size_t i = 0; i < h; ++i) { std::string_view cur = batch.GetString2D(i, j); data_.push_back(ParseStruct::Parse(cur)); diff --git a/src/execution/Pipeline.h b/src/execution/Pipeline.h index daf2fea..3830721 100644 --- a/src/execution/Pipeline.h +++ b/src/execution/Pipeline.h @@ -46,7 +46,7 @@ class Pipeline { std::shared_ptr sink; void Execute(ThreadPool& thread_pool, size_t num_threads) { - size_t num_tasks_finished{0}; + size_t num_tasks_finished = 0; Mutex wait_mutex; ConditionVariable wait_cond; for (size_t thread_id = 0; thread_id < num_threads; ++thread_id) { diff --git a/src/io/CsvToColumnar.cpp b/src/io/CsvToColumnar.cpp index b285599..7578cb4 100644 --- a/src/io/CsvToColumnar.cpp +++ b/src/io/CsvToColumnar.cpp @@ -14,26 +14,28 @@ void WriteToColumnar(const std::string& columnar_file_path, const VectorOfString ColumnarWriter writer(columnar_file_path); writer.WriteHeader(column_names, column_types); - static constexpr size_t kByteSize = 1 << 20; + static constexpr size_t kRowsPerChunk = 8192; + static constexpr size_t kBytePerCache = 1 << 20; + bool end_flag = true; VectorOfStrings2D column_batch; + size_t accumulated_rows = 0; while (end_flag) { - size_t cur_batch_size = 0; column_batch.Clear(); + size_t cache_rows = 0; - while (column_batch.ApproxByteSize() < kByteSize) { - bool has_row = reader.ReadRow(column_batch); - if (has_row) { + while (column_batch.ApproxByteSize() < kBytePerCache && accumulated_rows + cache_rows < kRowsPerChunk) { + if (reader.ReadRow(column_batch)) { column_batch.StartNewLine(); - ++cur_batch_size; + ++cache_rows; } else { end_flag = false; break; } } - if (cur_batch_size == 0) { + if (cache_rows == 0) { break; } @@ -42,13 +44,17 @@ void WriteToColumnar(const std::string& columnar_file_path, const VectorOfString builders[j]->AddBatch(column_batch, j); } - std::vector> columns; - for (auto& builder : builders) { - columns.push_back(builder->Finish()); - } + accumulated_rows += cache_rows; - if (!columns.empty()) { - writer.WriteBatch(columns); + if (accumulated_rows >= kRowsPerChunk || !end_flag) { + std::vector> columns; + for (auto& builder : builders) { + columns.push_back(builder->Finish()); + } + if (!columns.empty() && columns[0]->Size() > 0) { + writer.WriteBatch(columns); + } + accumulated_rows = 0; } } diff --git a/src/io/CsvTokenizer.cpp b/src/io/CsvTokenizer.cpp index e733438..116b4e7 100644 --- a/src/io/CsvTokenizer.cpp +++ b/src/io/CsvTokenizer.cpp @@ -5,9 +5,6 @@ CsvTokenizer::CsvTokenizer(const std::string& file_path, char delim) if (!file_.is_open()) { THROW_RUNTIME_ERROR("Could not open CSV file"); } - is_special_[static_cast(delim_)] = true; - is_special_[static_cast('\n')] = true; - is_special_[static_cast('"')] = true; is_whitespace_[static_cast(' ')] = true; is_whitespace_[static_cast('\t')] = true; @@ -29,20 +26,19 @@ bool CsvTokenizer::RefillBuffer(char& ch) { return true; } -void CsvTokenizer::GetNextRow(VectorOfStrings2D& vector_of_strings) { - bool in_quotes = false; +void CsvTokenizer::GetNextRow(VectorOfStrings2D& rows) { bool token_started = false; - bool token_materialized = false; + bool in_quotes = false; auto start_token_if_needed = [&]() { - if (!token_materialized) { - vector_of_strings.StartAddString(); - token_materialized = true; + if (!token_started) { + rows.StartAddString(); + token_started = true; } }; while (true) { - if (buffer_pos_ >= buffer_end_) [[unlikely]] { + if (buffer_pos_ >= buffer_end_) { char tmp; if (!RefillBuffer(tmp)) { break; @@ -50,94 +46,71 @@ void CsvTokenizer::GetNextRow(VectorOfStrings2D& vector_of_strings) { --buffer_pos_; } - if (!in_quotes) { - if (!token_started) { - while (buffer_pos_ < buffer_end_ && (buffer_[buffer_pos_] == ' ' || buffer_[buffer_pos_] == '\t')) { - ++buffer_pos_; - } - } - - size_t start = buffer_pos_; - while (buffer_pos_ < buffer_end_) { - if (is_special_[static_cast(buffer_[buffer_pos_])]) { - break; - } - ++buffer_pos_; - } - if (buffer_pos_ > start) { - start_token_if_needed(); - vector_of_strings.ContinueAddString(std::string_view(buffer_.data() + start, buffer_pos_ - start)); - token_started = true; - } - if (buffer_pos_ >= buffer_end_) { - continue; - } - } - char ch = buffer_[buffer_pos_++]; - if (!token_started && (ch == ' ' || ch == '\t')) { + if (!token_started && !in_quotes && is_whitespace_[static_cast(ch)]) { continue; } - if (ch == '"' && !in_quotes && !token_started) { - start_token_if_needed(); - in_quotes = true; - token_started = true; - continue; - } + start_token_if_needed(); + if (in_quotes) { + if (ch == '"') { + bool is_escaped_quote = false; + if (buffer_pos_ < buffer_end_) { + if (buffer_[buffer_pos_] == '"') { + is_escaped_quote = true; + } + } else { + char tmp; + if (RefillBuffer(tmp)) { + --buffer_pos_; + if (buffer_[buffer_pos_] == '"') { + is_escaped_quote = true; + } + } + } - if (ch == '"' && in_quotes) { - in_quotes = false; - while (GetChar(ch) && ch != delim_ && ch != '\n') { - if (!is_whitespace_[static_cast(ch)]) { - THROW_RUNTIME_ERROR( - "Invalid CSV format: unexpected character after closing quote"); + if (is_escaped_quote) { + rows.ContinueAddString('"'); + buffer_pos_++; + } else { + in_quotes = false; } + } else if (ch != '\r') { + rows.ContinueAddString(ch); } - start_token_if_needed(); - vector_of_strings.EndAddString(); - token_started = false; - token_materialized = false; - if (ch == '\n' || eof_reached_) { - end_of_line_ = true; - return; - } - continue; - } + } else { + if (ch == '"') { + in_quotes = true; + } else if (ch == delim_ || ch == '\n') { + while (!rows.EmptyLastString() && + is_whitespace_[static_cast( + rows.BackLastString())]) { + rows.PopLastChar(); + } - if (!in_quotes && (ch == delim_ || ch == '\n')) { - start_token_if_needed(); - while (!vector_of_strings.EmptyLastString() && - is_whitespace_[static_cast(vector_of_strings.BackLastString())]) { - vector_of_strings.PopLastChar(); - } - vector_of_strings.EndAddString(); - token_started = false; - token_materialized = false; - if (ch == '\n') { - end_of_line_ = true; - return; + rows.EndAddString(); + token_started = false; + + if (ch == '\n') { + end_of_line_ = true; + return; + } + } else if (ch != '\r') { + rows.ContinueAddString(ch); } - continue; } - - start_token_if_needed(); - vector_of_strings.ContinueAddString(ch); - token_started = true; } - if (!token_materialized) { - end_of_line_ = true; - return; - } - - while (!vector_of_strings.EmptyLastString() && - is_whitespace_[static_cast(vector_of_strings.BackLastString())]) { - vector_of_strings.PopLastChar(); + if (token_started || !eof_reached_) { + start_token_if_needed(); + while (!rows.EmptyLastString() && + is_whitespace_[static_cast(rows.BackLastString())]) { + rows.PopLastChar(); + } + rows.EndAddString(); } end_of_line_ = true; - vector_of_strings.EndAddString(); } bool CsvTokenizer::IsEndOfLine() const { diff --git a/src/io/CsvTokenizer.h b/src/io/CsvTokenizer.h index c86b70b..a0348d2 100644 --- a/src/io/CsvTokenizer.h +++ b/src/io/CsvTokenizer.h @@ -11,15 +11,14 @@ class CsvTokenizer { CsvTokenizer(const CsvTokenizer& other) = delete; CsvTokenizer& operator=(const CsvTokenizer& other) = delete; - void GetNextRow(VectorOfStrings2D& vector_of_strings); + void GetNextRow(VectorOfStrings2D& rows); bool IsEndOfLine() const; bool IsEOF() const; void ResetLineFlag(); private: - static constexpr size_t kBufferSize = 1 << 20; + static constexpr size_t kBufferSize = 1 << 27; - bool is_special_[256] = {false}; bool is_whitespace_[256] = {false}; std::ifstream file_; std::vector buffer_; @@ -28,6 +27,7 @@ class CsvTokenizer { char delim_; bool end_of_line_ = false; bool eof_reached_ = false; + bool GetChar(char& ch) { if (buffer_pos_ >= buffer_end_) [[unlikely]] { return RefillBuffer(ch); From 0e82a99d32c6c068b62f7f12aa7c2c1bbcb1ac21 Mon Sep 17 00:00:00 2001 From: Mikhail Fomichev Date: Wed, 27 May 2026 18:53:48 +0300 Subject: [PATCH 16/18] feat: engine no longer prints column names --- src/execution/ExecutionApi.h | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/execution/ExecutionApi.h b/src/execution/ExecutionApi.h index 2dd95d4..9a86fb6 100644 --- a/src/execution/ExecutionApi.h +++ b/src/execution/ExecutionApi.h @@ -23,13 +23,15 @@ class DataResult { return schema_; } - void Display() const { + void Display(bool display_names = false) const { const std::vector& fields = schema_.GetFields(); - if (!fields.empty()) { - for (const Field& field : fields) { - std::cout << field.name << ","; + if (display_names) { + if (!fields.empty()) { + for (const Field& field : fields) { + std::cout << field.name << ","; + } + std::cout << "\n"; } - std::cout << "\n"; } for (const std::unique_ptr& batch : batches_) { From b80430f03fb83dec4e95e65b45cd85856713c479 Mon Sep 17 00:00:00 2001 From: Mikhail Fomichev Date: Sat, 30 May 2026 23:15:56 +0300 Subject: [PATCH 17/18] script: run all queries support --cold so queries can be executed with cold caches --- .gitignore | 1 + script/run_all_queries.sh | 52 ++++++++++++++++++++++++++++++++-- script/run_all_queries_mt.sh | 54 ++++++++++++++++++++++++++++++++++-- script/run_query.sh | 9 +++++- script/run_query_mt.sh | 9 +++++- src/main.cpp | 2 +- 6 files changed, 119 insertions(+), 8 deletions(-) diff --git a/.gitignore b/.gitignore index 9e8403f..17b5463 100644 --- a/.gitignore +++ b/.gitignore @@ -22,6 +22,7 @@ test.log test_csv2 test_main.cpp hits_sample.csv +hits_10m.csv columnar_hits_sample.tuff report.txt docker-test diff --git a/script/run_all_queries.sh b/script/run_all_queries.sh index 660463a..99a3bfb 100755 --- a/script/run_all_queries.sh +++ b/script/run_all_queries.sh @@ -5,6 +5,7 @@ set -e # -i 3 - count of iterations (or --iterations) # --answers - print answers for each query # --debug - print time of execution for each query +# --cold - drop OS page cache before each query (requires sudo) cd "$(dirname "$0")" || exit 1 @@ -12,30 +13,39 @@ cd "$(dirname "$0")" || exit 1 ITERATIONS=1 SHOW_DEBUG=0 SHOW_ANSWERS=0 +COLD_CACHE=0 # Парсинг аргументов командной строки while [[ "$#" -gt 0 ]]; do case $1 in --debug) SHOW_DEBUG=1; shift ;; --answers) SHOW_ANSWERS=1; shift ;; + --cold) COLD_CACHE=1; shift ;; -i|--iterations) ITERATIONS="$2"; shift 2 ;; *) echo "Unknown parameter passed: $1"; exit 1 ;; esac done echo ">>> Running queries 0-42 (single thread)..." -echo ">>> Iterations: $ITERATIONS | Show Debug: $SHOW_DEBUG | Show Answers: $SHOW_ANSWERS" +echo ">>> Iterations: $ITERATIONS | Show Debug: $SHOW_DEBUG | Show Answers: $SHOW_ANSWERS | Cold Cache: $COLD_CACHE" RESULTS_DIR="query_results" if [ "$SHOW_ANSWERS" -eq 1 ] || [ "$SHOW_DEBUG" -eq 1 ]; then mkdir -p "${RESULTS_DIR}" fi +# Массив для хранения всех времён (для общего geomean) +ALL_TIMES=() + for (( iter=1; iter<=ITERATIONS; iter++ )) do echo "=========================================" echo ">>> Iteration $iter / $ITERATIONS" echo "=========================================" + + # Массив времён для текущей итерации + ITER_TIMES=() + for i in {0..42} do echo -n ">>> Executing query $i ... " @@ -54,13 +64,20 @@ do LOG_FILE=$(mktemp) # Временный файл, который мы потом удалим fi + # Формируем аргументы для run_query.sh + COLD_ARG="" + if [ "$COLD_CACHE" -eq 1 ]; then + COLD_ARG="--cold" + fi + set +e - ./run_query.sh "$i" ../columnar_hits_sample.tuff "$CSV_FILE" "$LOG_FILE" + ./run_query.sh "$i" ../columnar_hits_sample.tuff "$CSV_FILE" "$LOG_FILE" $COLD_ARG exit_code=$? set -e # Вытаскиваем время из лога (формат: "Query 0 completed in 2.57227 ms") EXEC_TIME=$(grep "completed in" "$LOG_FILE" | awk '{print $5, $6}' || true) + EXEC_TIME_MS=$(grep "completed in" "$LOG_FILE" | awk '{print $5}' || true) if [ $exit_code -ne 0 ]; then echo "FAILED with exit code $exit_code" @@ -77,6 +94,12 @@ do echo "OK" fi + # Сохраняем время для geomean + if [ -n "$EXEC_TIME_MS" ]; then + ITER_TIMES+=("$EXEC_TIME_MS") + ALL_TIMES+=("$EXEC_TIME_MS") + fi + if [ "$SHOW_ANSWERS" -eq 1 ]; then echo "------ ANSWERS (Query $i) ------" cat "$CSV_FILE" @@ -92,6 +115,31 @@ do rm -f "$LOG_FILE" fi done + + # Считаем geomean для текущей итерации + if [ ${#ITER_TIMES[@]} -gt 0 ]; then + GEOMEAN=$(printf '%s\n' "${ITER_TIMES[@]}" | awk '{ + sum += log($1); n++ + } END { + if (n > 0) printf "%.5f", exp(sum / n) + }') + echo "=========================================" + echo ">>> Geomean for iteration $iter: ${GEOMEAN} ms" + echo "=========================================" + fi done +# Считаем общий geomean по всем итерациям +if [ ${#ALL_TIMES[@]} -gt 0 ]; then + TOTAL_GEOMEAN=$(printf '%s\n' "${ALL_TIMES[@]}" | awk '{ + sum += log($1); n++ + } END { + if (n > 0) printf "%.5f", exp(sum / n) + }') + echo "" + echo "=========================================" + echo ">>> Overall geomean ($ITERATIONS iterations): ${TOTAL_GEOMEAN} ms" + echo "=========================================" +fi + echo ">>> All $ITERATIONS iterations of all queries completed successfully." \ No newline at end of file diff --git a/script/run_all_queries_mt.sh b/script/run_all_queries_mt.sh index b3d8651..f76fa9b 100755 --- a/script/run_all_queries_mt.sh +++ b/script/run_all_queries_mt.sh @@ -5,6 +5,7 @@ set -e # -i 3 - count of iterations (or --iterations) # --answers - print answers for each query # --debug - print time of execution for each query +# --cold - drop OS page cache before each query (requires sudo) cd "$(dirname "$0")" || exit 1 @@ -12,30 +13,39 @@ cd "$(dirname "$0")" || exit 1 ITERATIONS=1 SHOW_DEBUG=0 SHOW_ANSWERS=0 +COLD_CACHE=0 # Парсинг аргументов командной строки while [[ "$#" -gt 0 ]]; do case $1 in --debug) SHOW_DEBUG=1; shift ;; --answers) SHOW_ANSWERS=1; shift ;; + --cold) COLD_CACHE=1; shift ;; -i|--iterations) ITERATIONS="$2"; shift 2 ;; *) echo "Unknown parameter passed: $1"; exit 1 ;; esac done -echo ">>> Running queries 0-42 (single thread)..." -echo ">>> Iterations: $ITERATIONS | Show Debug: $SHOW_DEBUG | Show Answers: $SHOW_ANSWERS" +echo ">>> Running queries 0-42 (multi thread)..." +echo ">>> Iterations: $ITERATIONS | Show Debug: $SHOW_DEBUG | Show Answers: $SHOW_ANSWERS | Cold Cache: $COLD_CACHE" RESULTS_DIR="query_mt_results" if [ "$SHOW_ANSWERS" -eq 1 ] || [ "$SHOW_DEBUG" -eq 1 ]; then mkdir -p "${RESULTS_DIR}" fi +# Массив для хранения всех времён (для общего geomean) +ALL_TIMES=() + for (( iter=1; iter<=ITERATIONS; iter++ )) do echo "=========================================" echo ">>> Iteration $iter / $ITERATIONS" echo "=========================================" + + # Массив времён для текущей итерации + ITER_TIMES=() + for i in {0..42} do echo -n ">>> Executing query $i ... " @@ -54,13 +64,20 @@ do LOG_FILE=$(mktemp) # Временный файл, который мы потом удалим fi + # Формируем аргументы для run_query_mt.sh + COLD_ARG="" + if [ "$COLD_CACHE" -eq 1 ]; then + COLD_ARG="--cold" + fi + set +e - ./run_query_mt.sh "$i" ../columnar_hits_sample.tuff "$CSV_FILE" "$LOG_FILE" + ./run_query_mt.sh "$i" ../columnar_hits_sample.tuff "$CSV_FILE" "$LOG_FILE" $COLD_ARG exit_code=$? set -e # Вытаскиваем время из лога (формат: "Query 0 completed in 2.57227 ms") EXEC_TIME=$(grep "completed in" "$LOG_FILE" | awk '{print $5, $6}' || true) + EXEC_TIME_MS=$(grep "completed in" "$LOG_FILE" | awk '{print $5}' || true) if [ $exit_code -ne 0 ]; then echo "FAILED with exit code $exit_code" @@ -77,6 +94,12 @@ do echo "OK" fi + # Сохраняем время для geomean + if [ -n "$EXEC_TIME_MS" ]; then + ITER_TIMES+=("$EXEC_TIME_MS") + ALL_TIMES+=("$EXEC_TIME_MS") + fi + if [ "$SHOW_ANSWERS" -eq 1 ]; then echo "------ ANSWERS (Query $i) ------" cat "$CSV_FILE" @@ -92,6 +115,31 @@ do rm -f "$LOG_FILE" fi done + + # Считаем geomean для текущей итерации + if [ ${#ITER_TIMES[@]} -gt 0 ]; then + GEOMEAN=$(printf '%s\n' "${ITER_TIMES[@]}" | awk '{ + sum += log($1); n++ + } END { + if (n > 0) printf "%.5f", exp(sum / n) + }') + echo "=========================================" + echo ">>> Geomean for iteration $iter: ${GEOMEAN} ms" + echo "=========================================" + fi done +# Считаем общий geomean по всем итерациям +if [ ${#ALL_TIMES[@]} -gt 0 ]; then + TOTAL_GEOMEAN=$(printf '%s\n' "${ALL_TIMES[@]}" | awk '{ + sum += log($1); n++ + } END { + if (n > 0) printf "%.5f", exp(sum / n) + }') + echo "" + echo "=========================================" + echo ">>> Overall geomean ($ITERATIONS iterations): ${TOTAL_GEOMEAN} ms" + echo "=========================================" +fi + echo ">>> All $ITERATIONS iterations of all queries completed successfully." \ No newline at end of file diff --git a/script/run_query.sh b/script/run_query.sh index 69e6103..ea9513b 100755 --- a/script/run_query.sh +++ b/script/run_query.sh @@ -4,7 +4,7 @@ set -euo pipefail ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" if [[ $# -lt 4 ]]; then - echo "Usage: script/run_query.sh " >&2 + echo "Usage: script/run_query.sh [--cold]" >&2 exit 2 fi @@ -12,6 +12,7 @@ QUERY_NUM="$1" COLUMNAR="$2" OUTPUT_CSV="$3" LOG_FILE="$4" +DROP_CACHE="${5:-}" BUILD_DIR="${ROOT_DIR}/cmake-build-release" BIN="${BUILD_DIR}/columnar-engine" @@ -35,5 +36,11 @@ if [[ "${LOG_FILE}" != "/dev/null" ]]; then mkdir -p "$(dirname "${LOG_FILE}")" fi +# Очищаем кэш ОС для холодного запуска +if [[ "${DROP_CACHE}" == "--cold" ]]; then + sync + sudo sh -c 'echo 3 > /proc/sys/vm/drop_caches' +fi + # Run the query, capture stdout to CSV and stderr to log "${BIN}" query "${COLUMNAR}" "${QUERY_NUM}" > "${OUTPUT_CSV}" 2> "${LOG_FILE}" \ No newline at end of file diff --git a/script/run_query_mt.sh b/script/run_query_mt.sh index 440e01d..0ba357c 100755 --- a/script/run_query_mt.sh +++ b/script/run_query_mt.sh @@ -4,7 +4,7 @@ set -euo pipefail ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" if [[ $# -lt 4 ]]; then - echo "Usage: script/run_query_mt.sh " >&2 + echo "Usage: script/run_query_mt.sh [--cold]" >&2 exit 2 fi @@ -12,6 +12,7 @@ QUERY_NUM="$1" COLUMNAR="$2" OUTPUT_CSV="$3" LOG_FILE="$4" +DROP_CACHE="${5:-}" BUILD_DIR="${ROOT_DIR}/cmake-build-release" BIN="${BUILD_DIR}/columnar-engine-mt" @@ -35,5 +36,11 @@ if [[ "${LOG_FILE}" != "/dev/null" ]]; then mkdir -p "$(dirname "${LOG_FILE}")" fi +# Очищаем кэш ОС для холодного запуска +if [[ "${DROP_CACHE}" == "--cold" ]]; then + sync + sudo sh -c 'echo 3 > /proc/sys/vm/drop_caches' +fi + # Run the query, capture stdout to CSV and stderr to log "${BIN}" query "${COLUMNAR}" "${QUERY_NUM}" > "${OUTPUT_CSV}" 2> "${LOG_FILE}" \ No newline at end of file diff --git a/src/main.cpp b/src/main.cpp index c182e4e..6efbd89 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -441,7 +441,7 @@ class Query { AddConst("ClientIP", -2, "ClientIP_minus_2"), AddConst("ClientIP", -3, "ClientIP_minus_3")}) .Reorder({"ClientIP", "ClientIP_minus_1", "ClientIP_minus_2", - "ClientIP_minus_3", "c"}) + "ClientIP_minus_3", "c"}) .Collect(); df.Display(); From 761f0949fec5824ef005bc3791cf1134fa137ba6 Mon Sep 17 00:00:00 2001 From: Mikhail Fomichev Date: Thu, 11 Jun 2026 18:24:57 +0300 Subject: [PATCH 18/18] some unknown change --- src/execution/expressions/ScalarFunctions.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/execution/expressions/ScalarFunctions.h b/src/execution/expressions/ScalarFunctions.h index ca0a3a2..aa6aadf 100644 --- a/src/execution/expressions/ScalarFunctions.h +++ b/src/execution/expressions/ScalarFunctions.h @@ -171,11 +171,11 @@ class CaseWhenFunction : public ScalarFunction { ViewType true_view = build_view(true_branch_); ViewType false_view = build_view(false_branch_); - auto builder = ColumnFactory::MakeColumnBuilder(type_); + std::shared_ptr builder = ColumnFactory::MakeColumnBuilder(type_); auto* typed_builder = static_cast(builder.get()); std::visit( - [&](const auto& true_v, const auto& false_v) { + [&](const ViewType& true_v, const ViewType& false_v) { size_t physical_size = batch.columns.empty() ? batch.num_rows : batch.columns[0]->Size(); size_t cur_ind = 0;