diff --git a/.gitignore b/.gitignore index b9ac6b2..17b5463 100644 --- a/.gitignore +++ b/.gitignore @@ -18,9 +18,11 @@ compile_commands.json .clangd test_csv test.csv +test.log test_csv2 test_main.cpp hits_sample.csv +hits_10m.csv columnar_hits_sample.tuff report.txt docker-test @@ -28,7 +30,6 @@ docker-test-results .vscode/ src/project_bundle.txt - # Prerequisites *.d diff --git a/CMakeLists.txt b/CMakeLists.txt index 83ce1f4..88f4ac0 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 3.28) +cmake_minimum_required(VERSION 3.30) project(columnar-engine) @@ -15,7 +15,26 @@ include(cmake/TestSolution.cmake) include(cmake/SeminarUtils.cmake) include(FetchContent) -include(cmake/FindGlog.cmake) + +set(ABSL_PROPAGATE_CXX_STD ON) +set(BUILD_TESTING OFF) +set(ABSL_ENABLE_INSTALL ON) + +FetchContent_Declare( + abseil-cpp + GIT_REPOSITORY https://github.com/abseil/abseil-cpp.git + GIT_TAG 20240116.2 + SYSTEM +) +FetchContent_MakeAvailable(abseil-cpp) + +FetchContent_Declare( + re2 + GIT_REPOSITORY https://github.com/google/re2.git + GIT_TAG 2024-02-01 + SYSTEM +) +FetchContent_MakeAvailable(re2) find_package(Catch REQUIRED) find_package(Benchmark REQUIRED) diff --git a/script/build.sh b/script/build.sh index 1d3d964..27893d6 100755 --- a/script/build.sh +++ b/script/build.sh @@ -12,4 +12,4 @@ cmake -S "${ROOT_DIR}" -B "${BUILD_DIR}" \ -DCMAKE_BUILD_TYPE="${BUILD_TYPE}" \ -DCMAKE_CXX_STANDARD=23 -cmake --build "${BUILD_DIR}" --target columnar-engine -j "$(nproc)" +cmake --build "${BUILD_DIR}" -j "$(nproc)" diff --git a/script/run_all_queries.sh b/script/run_all_queries.sh new file mode 100755 index 0000000..99a3bfb --- /dev/null +++ b/script/run_all_queries.sh @@ -0,0 +1,145 @@ +#!/bin/bash +set -e + +# usage: +# -i 3 - count of iterations (or --iterations) +# --answers - print answers for each query +# --debug - print time of execution for each query +# --cold - drop OS page cache before each query (requires sudo) + +cd "$(dirname "$0")" || exit 1 + +# Значения по умолчанию +ITERATIONS=1 +SHOW_DEBUG=0 +SHOW_ANSWERS=0 +COLD_CACHE=0 + +# Парсинг аргументов командной строки +while [[ "$#" -gt 0 ]]; do + case $1 in + --debug) SHOW_DEBUG=1; shift ;; + --answers) SHOW_ANSWERS=1; shift ;; + --cold) COLD_CACHE=1; shift ;; + -i|--iterations) ITERATIONS="$2"; shift 2 ;; + *) echo "Unknown parameter passed: $1"; exit 1 ;; + esac +done + +echo ">>> Running queries 0-42 (single thread)..." +echo ">>> Iterations: $ITERATIONS | Show Debug: $SHOW_DEBUG | Show Answers: $SHOW_ANSWERS | Cold Cache: $COLD_CACHE" + +RESULTS_DIR="query_results" +if [ "$SHOW_ANSWERS" -eq 1 ] || [ "$SHOW_DEBUG" -eq 1 ]; then + mkdir -p "${RESULTS_DIR}" +fi + +# Массив для хранения всех времён (для общего geomean) +ALL_TIMES=() + +for (( iter=1; iter<=ITERATIONS; iter++ )) +do + echo "=========================================" + echo ">>> Iteration $iter / $ITERATIONS" + echo "=========================================" + + # Массив времён для текущей итерации + ITER_TIMES=() + + for i in {0..42} + do + echo -n ">>> Executing query $i ... " + + # Куда пишем ответы + if [ "$SHOW_ANSWERS" -eq 1 ]; then + CSV_FILE="${RESULTS_DIR}/query_${i}_iter_${iter}.csv" + else + CSV_FILE="/dev/null" + fi + + # Куда пишем логи (чтобы вытащить время) + if [ "$SHOW_DEBUG" -eq 1 ]; then + LOG_FILE="${RESULTS_DIR}/query_${i}_iter_${iter}.log" + else + LOG_FILE=$(mktemp) # Временный файл, который мы потом удалим + fi + + # Формируем аргументы для run_query.sh + COLD_ARG="" + if [ "$COLD_CACHE" -eq 1 ]; then + COLD_ARG="--cold" + fi + + set +e + ./run_query.sh "$i" ../columnar_hits_sample.tuff "$CSV_FILE" "$LOG_FILE" $COLD_ARG + exit_code=$? + set -e + + # Вытаскиваем время из лога (формат: "Query 0 completed in 2.57227 ms") + EXEC_TIME=$(grep "completed in" "$LOG_FILE" | awk '{print $5, $6}' || true) + EXEC_TIME_MS=$(grep "completed in" "$LOG_FILE" | awk '{print $5}' || true) + + if [ $exit_code -ne 0 ]; then + echo "FAILED with exit code $exit_code" + echo "--- CRASH LOG ---" + cat "$LOG_FILE" + if [ "$SHOW_DEBUG" -eq 0 ]; then rm -f "$LOG_FILE"; fi + exit $exit_code + fi + + # Красиво выводим время + if [ -n "$EXEC_TIME" ]; then + echo "${EXEC_TIME} ... OK" + else + echo "OK" + fi + + # Сохраняем время для geomean + if [ -n "$EXEC_TIME_MS" ]; then + ITER_TIMES+=("$EXEC_TIME_MS") + ALL_TIMES+=("$EXEC_TIME_MS") + fi + + if [ "$SHOW_ANSWERS" -eq 1 ]; then + echo "------ ANSWERS (Query $i) ------" + cat "$CSV_FILE" + echo "--------------------------------" + fi + + if [ "$SHOW_DEBUG" -eq 1 ]; then + echo "------- DEBUG (Query $i) -------" + cat "$LOG_FILE" + echo "--------------------------------" + else + # Удаляем мусор, если дебаг не нужен + rm -f "$LOG_FILE" + fi + done + + # Считаем geomean для текущей итерации + if [ ${#ITER_TIMES[@]} -gt 0 ]; then + GEOMEAN=$(printf '%s\n' "${ITER_TIMES[@]}" | awk '{ + sum += log($1); n++ + } END { + if (n > 0) printf "%.5f", exp(sum / n) + }') + echo "=========================================" + echo ">>> Geomean for iteration $iter: ${GEOMEAN} ms" + echo "=========================================" + fi +done + +# Считаем общий geomean по всем итерациям +if [ ${#ALL_TIMES[@]} -gt 0 ]; then + TOTAL_GEOMEAN=$(printf '%s\n' "${ALL_TIMES[@]}" | awk '{ + sum += log($1); n++ + } END { + if (n > 0) printf "%.5f", exp(sum / n) + }') + echo "" + echo "=========================================" + echo ">>> Overall geomean ($ITERATIONS iterations): ${TOTAL_GEOMEAN} ms" + echo "=========================================" +fi + +echo ">>> All $ITERATIONS iterations of all queries completed successfully." \ No newline at end of file diff --git a/script/run_all_queries_mt.sh b/script/run_all_queries_mt.sh new file mode 100755 index 0000000..f76fa9b --- /dev/null +++ b/script/run_all_queries_mt.sh @@ -0,0 +1,145 @@ +#!/bin/bash +set -e + +# usage: +# -i 3 - count of iterations (or --iterations) +# --answers - print answers for each query +# --debug - print time of execution for each query +# --cold - drop OS page cache before each query (requires sudo) + +cd "$(dirname "$0")" || exit 1 + +# Значения по умолчанию +ITERATIONS=1 +SHOW_DEBUG=0 +SHOW_ANSWERS=0 +COLD_CACHE=0 + +# Парсинг аргументов командной строки +while [[ "$#" -gt 0 ]]; do + case $1 in + --debug) SHOW_DEBUG=1; shift ;; + --answers) SHOW_ANSWERS=1; shift ;; + --cold) COLD_CACHE=1; shift ;; + -i|--iterations) ITERATIONS="$2"; shift 2 ;; + *) echo "Unknown parameter passed: $1"; exit 1 ;; + esac +done + +echo ">>> Running queries 0-42 (multi thread)..." +echo ">>> Iterations: $ITERATIONS | Show Debug: $SHOW_DEBUG | Show Answers: $SHOW_ANSWERS | Cold Cache: $COLD_CACHE" + +RESULTS_DIR="query_mt_results" +if [ "$SHOW_ANSWERS" -eq 1 ] || [ "$SHOW_DEBUG" -eq 1 ]; then + mkdir -p "${RESULTS_DIR}" +fi + +# Массив для хранения всех времён (для общего geomean) +ALL_TIMES=() + +for (( iter=1; iter<=ITERATIONS; iter++ )) +do + echo "=========================================" + echo ">>> Iteration $iter / $ITERATIONS" + echo "=========================================" + + # Массив времён для текущей итерации + ITER_TIMES=() + + for i in {0..42} + do + echo -n ">>> Executing query $i ... " + + # Куда пишем ответы + if [ "$SHOW_ANSWERS" -eq 1 ]; then + CSV_FILE="${RESULTS_DIR}/query_${i}_iter_${iter}.csv" + else + CSV_FILE="/dev/null" + fi + + # Куда пишем логи (чтобы вытащить время) + if [ "$SHOW_DEBUG" -eq 1 ]; then + LOG_FILE="${RESULTS_DIR}/query_${i}_iter_${iter}.log" + else + LOG_FILE=$(mktemp) # Временный файл, который мы потом удалим + fi + + # Формируем аргументы для run_query_mt.sh + COLD_ARG="" + if [ "$COLD_CACHE" -eq 1 ]; then + COLD_ARG="--cold" + fi + + set +e + ./run_query_mt.sh "$i" ../columnar_hits_sample.tuff "$CSV_FILE" "$LOG_FILE" $COLD_ARG + exit_code=$? + set -e + + # Вытаскиваем время из лога (формат: "Query 0 completed in 2.57227 ms") + EXEC_TIME=$(grep "completed in" "$LOG_FILE" | awk '{print $5, $6}' || true) + EXEC_TIME_MS=$(grep "completed in" "$LOG_FILE" | awk '{print $5}' || true) + + if [ $exit_code -ne 0 ]; then + echo "FAILED with exit code $exit_code" + echo "--- CRASH LOG ---" + cat "$LOG_FILE" + if [ "$SHOW_DEBUG" -eq 0 ]; then rm -f "$LOG_FILE"; fi + exit $exit_code + fi + + # Красиво выводим время + if [ -n "$EXEC_TIME" ]; then + echo "${EXEC_TIME} ... OK" + else + echo "OK" + fi + + # Сохраняем время для geomean + if [ -n "$EXEC_TIME_MS" ]; then + ITER_TIMES+=("$EXEC_TIME_MS") + ALL_TIMES+=("$EXEC_TIME_MS") + fi + + if [ "$SHOW_ANSWERS" -eq 1 ]; then + echo "------ ANSWERS (Query $i) ------" + cat "$CSV_FILE" + echo "--------------------------------" + fi + + if [ "$SHOW_DEBUG" -eq 1 ]; then + echo "------- DEBUG (Query $i) -------" + cat "$LOG_FILE" + echo "--------------------------------" + else + # Удаляем мусор, если дебаг не нужен + rm -f "$LOG_FILE" + fi + done + + # Считаем geomean для текущей итерации + if [ ${#ITER_TIMES[@]} -gt 0 ]; then + GEOMEAN=$(printf '%s\n' "${ITER_TIMES[@]}" | awk '{ + sum += log($1); n++ + } END { + if (n > 0) printf "%.5f", exp(sum / n) + }') + echo "=========================================" + echo ">>> Geomean for iteration $iter: ${GEOMEAN} ms" + echo "=========================================" + fi +done + +# Считаем общий geomean по всем итерациям +if [ ${#ALL_TIMES[@]} -gt 0 ]; then + TOTAL_GEOMEAN=$(printf '%s\n' "${ALL_TIMES[@]}" | awk '{ + sum += log($1); n++ + } END { + if (n > 0) printf "%.5f", exp(sum / n) + }') + echo "" + echo "=========================================" + echo ">>> Overall geomean ($ITERATIONS iterations): ${TOTAL_GEOMEAN} ms" + echo "=========================================" +fi + +echo ">>> All $ITERATIONS iterations of all queries completed successfully." \ No newline at end of file diff --git a/script/run_query.sh b/script/run_query.sh index 9ade173..ea9513b 100755 --- a/script/run_query.sh +++ b/script/run_query.sh @@ -4,7 +4,7 @@ set -euo pipefail ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" if [[ $# -lt 4 ]]; then - echo "Usage: script/run_query.sh " >&2 + echo "Usage: script/run_query.sh [--cold]" >&2 exit 2 fi @@ -12,6 +12,7 @@ QUERY_NUM="$1" COLUMNAR="$2" OUTPUT_CSV="$3" LOG_FILE="$4" +DROP_CACHE="${5:-}" BUILD_DIR="${ROOT_DIR}/cmake-build-release" BIN="${BUILD_DIR}/columnar-engine" @@ -27,8 +28,19 @@ if [[ ! -f "${COLUMNAR}" ]]; then exit 2 fi -mkdir -p "$(dirname "${OUTPUT_CSV}")" -mkdir -p "$(dirname "${LOG_FILE}")" +# Создаем директории, только если это реальные файлы, а не /dev/null +if [[ "${OUTPUT_CSV}" != "/dev/null" ]]; then + mkdir -p "$(dirname "${OUTPUT_CSV}")" +fi +if [[ "${LOG_FILE}" != "/dev/null" ]]; then + mkdir -p "$(dirname "${LOG_FILE}")" +fi + +# Очищаем кэш ОС для холодного запуска +if [[ "${DROP_CACHE}" == "--cold" ]]; then + sync + sudo sh -c 'echo 3 > /proc/sys/vm/drop_caches' +fi -# Run the query, capture stdout (CSV result) and tee stderr+stdout to log -"${BIN}" query "${COLUMNAR}" "${QUERY_NUM}" > "${OUTPUT_CSV}" 2> >(tee "${LOG_FILE}" >&2) +# Run the query, capture stdout to CSV and stderr to log +"${BIN}" query "${COLUMNAR}" "${QUERY_NUM}" > "${OUTPUT_CSV}" 2> "${LOG_FILE}" \ No newline at end of file diff --git a/script/run_query_mt.sh b/script/run_query_mt.sh new file mode 100755 index 0000000..0ba357c --- /dev/null +++ b/script/run_query_mt.sh @@ -0,0 +1,46 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" + +if [[ $# -lt 4 ]]; then + echo "Usage: script/run_query_mt.sh [--cold]" >&2 + exit 2 +fi + +QUERY_NUM="$1" +COLUMNAR="$2" +OUTPUT_CSV="$3" +LOG_FILE="$4" +DROP_CACHE="${5:-}" + +BUILD_DIR="${ROOT_DIR}/cmake-build-release" +BIN="${BUILD_DIR}/columnar-engine-mt" + +if [[ ! -x "${BIN}" ]]; then + echo "ERROR: columnar-engine not found at ${BIN}" >&2 + echo "Run script/build.sh first" >&2 + exit 1 +fi + +if [[ ! -f "${COLUMNAR}" ]]; then + echo "ERROR: columnar file not found: ${COLUMNAR}" >&2 + exit 2 +fi + +# Создаем директории, только если это реальные файлы, а не /dev/null +if [[ "${OUTPUT_CSV}" != "/dev/null" ]]; then + mkdir -p "$(dirname "${OUTPUT_CSV}")" +fi +if [[ "${LOG_FILE}" != "/dev/null" ]]; then + mkdir -p "$(dirname "${LOG_FILE}")" +fi + +# Очищаем кэш ОС для холодного запуска +if [[ "${DROP_CACHE}" == "--cold" ]]; then + sync + sudo sh -c 'echo 3 > /proc/sys/vm/drop_caches' +fi + +# Run the query, capture stdout to CSV and stderr to log +"${BIN}" query "${COLUMNAR}" "${QUERY_NUM}" > "${OUTPUT_CSV}" 2> "${LOG_FILE}" \ No newline at end of file diff --git a/script/setup.sh b/script/setup.sh index 9cef495..be555df 100755 --- a/script/setup.sh +++ b/script/setup.sh @@ -3,7 +3,17 @@ set -euo pipefail export DEBIAN_FRONTEND=noninteractive +apt-get update +apt-get install -y --no-install-recommends ca-certificates gpg wget + +wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null \ + | gpg --dearmor - \ + | tee /usr/share/keyrings/kitware-archive-keyring.gpg >/dev/null + +echo 'deb [signed-by=/usr/share/keyrings/kitware-archive-keyring.gpg] https://apt.kitware.com/ubuntu/ noble main' \ + | tee /etc/apt/sources.list.d/kitware.list >/dev/null + apt-get update apt-get install -y --no-install-recommends \ - build-essential cmake git ca-certificates \ - g++ libgoogle-glog-dev + build-essential git ca-certificates \ + g++ libgoogle-glog-dev cmake \ No newline at end of file diff --git a/src/AggregationExpressions.h b/src/AggregationExpressions.h deleted file mode 100644 index ba08624..0000000 --- a/src/AggregationExpressions.h +++ /dev/null @@ -1,258 +0,0 @@ -// -// Created by ragnarokk on 31.03.2026. -// - -#ifndef COLUMNAR_ENGINE_EXPRESSIONS_H -#define COLUMNAR_ENGINE_EXPRESSIONS_H - -#include "AggregationFunctions.h" -#include "Schema.h" - -#include -#include -#include - -class AggregateExpression { -public: - virtual ~AggregateExpression() = default; - - explicit AggregateExpression(std::string column_name, std::string output_name = "") - : column_name_(std::move(column_name)), output_name_(std::move(output_name)) { - } - - std::string GetName() const { - return column_name_; - } - - void CollectRequiredColumns(std::vector& required_columns) { - if (!column_name_.empty() && column_name_ != "*") { - required_columns.push_back(column_name_); - } - } - - virtual std::string GetOutputName() const { - return output_name_.empty() ? column_name_ : output_name_; - } - - virtual std::unique_ptr CreateAggregationFunction( - const Schema& child_schema, Schema& output_schema) = 0; - -protected: - std::string column_name_; - std::string output_name_; -}; - -class CountExpression : public AggregateExpression { -public: - explicit CountExpression(std::string column_name, std::string output_name = "count") - : AggregateExpression(std::move(column_name), std::move(output_name)) { - } - - std::unique_ptr CreateAggregationFunction( - [[maybe_unused]] const Schema& child_schema, Schema& output_schema) override { - output_schema.AddColumn(GetOutputName(), ColumnType::INT32); - return std::make_unique(); - } -}; - -class DistinctCountExpression : public AggregateExpression { -public: - explicit DistinctCountExpression(std::string column_name, std::string output_name = "") - : AggregateExpression(std::move(column_name), std::move(output_name)) { - } - - std::string GetOutputName() const override { - if (!output_name_.empty()) { - return output_name_; - } - return "distinct_count_" + column_name_; - } - - std::unique_ptr CreateAggregationFunction(const Schema& child_schema, - Schema& output_schema) override { - size_t column_ind = child_schema.GetColumnIndexByName(GetName()); - ColumnType column_type = child_schema.GetColumnTypeByName(GetName()); - output_schema.AddColumn(GetOutputName(), ColumnType::INT64); - -#define HANDLE_TYPE(ENUM_VAL, STR_VAL, CLASS_TYPE) \ - case ColumnType::ENUM_VAL: \ - return std::make_unique>(column_ind); - - switch (column_type) { - FOR_EACH_COLUMN_TYPE(HANDLE_TYPE); - default: THROW_NOT_IMPLEMENTED; - } -#undef HANDLE_TYPE - } -}; - -class MinExpression : public AggregateExpression { -public: - explicit MinExpression(std::string column_name, std::string output_name = "") - : AggregateExpression(std::move(column_name), std::move(output_name)) { - } - - std::string GetOutputName() const override { - if (!output_name_.empty()) { - return output_name_; - } - return "min_" + column_name_; - } - - std::unique_ptr CreateAggregationFunction(const Schema& child_schema, - Schema& output_schema) override { - size_t column_ind = child_schema.GetColumnIndexByName(GetName()); - ColumnType column_type = child_schema.GetColumnTypeByName(GetName()); - -#define HANDLE_TYPE(ENUM_VAL, STR_VAL, CLASS_TYPE) \ - case ColumnType::ENUM_VAL: \ - output_schema.AddColumn(GetOutputName(), ColumnType::ENUM_VAL); \ - return std::make_unique>(column_ind); - // END_HANDLE_TYPE - - switch (column_type) { - FOR_NUMERIC_COLUMN_TYPE(HANDLE_TYPE); - default: THROW_NOT_IMPLEMENTED; - } - -#undef HANDLE_TYPE - } -}; - -class MaxExpression : public AggregateExpression { -public: - explicit MaxExpression(std::string column_name, std::string output_name = "") - : AggregateExpression(std::move(column_name), std::move(output_name)) { - } - - std::string GetOutputName() const override { - if (!output_name_.empty()) { - return output_name_; - } - return "max_" + column_name_; - } - - std::unique_ptr CreateAggregationFunction(const Schema& child_schema, - Schema& output_schema) override { - size_t column_ind = child_schema.GetColumnIndexByName(GetName()); - ColumnType column_type = child_schema.GetColumnTypeByName(GetName()); - -#define HANDLE_TYPE(ENUM_VAL, STR_VAL, CLASS_TYPE) \ - case ColumnType::ENUM_VAL: \ - output_schema.AddColumn(GetOutputName(), ColumnType::ENUM_VAL); \ - return std::make_unique>(column_ind); \ - // END_HANDLE_TYPE - - switch (column_type) { - FOR_NUMERIC_COLUMN_TYPE(HANDLE_TYPE); - default: THROW_NOT_IMPLEMENTED; - } - -#undef HANDLE_TYPE - } -}; - -class SumExpression : public AggregateExpression { -public: - explicit SumExpression(std::string column_name, std::string output_name = "") - : AggregateExpression(std::move(column_name), std::move(output_name)) { - } - - std::string GetOutputName() const override { - if (!output_name_.empty()) { - return output_name_; - } - return "sum_" + column_name_; - } - - std::unique_ptr CreateAggregationFunction(const Schema& child_schema, - Schema& output_schema) override { - const size_t column_ind = child_schema.GetColumnIndexByName(GetName()); - const ColumnType column_type = child_schema.GetColumnTypeByName(GetName()); - - switch (column_type) { - case ColumnType::INT16: - output_schema.AddColumn(GetOutputName(), ColumnType::INT32); - return std::make_unique>(column_ind); - case ColumnType::INT32: - output_schema.AddColumn(GetOutputName(), ColumnType::INT64); - return std::make_unique>(column_ind); - case ColumnType::INT64: - output_schema.AddColumn(GetOutputName(), ColumnType::INT128); - return std::make_unique>(column_ind); - case ColumnType::INT128: - output_schema.AddColumn(GetOutputName(), ColumnType::INT128); - return std::make_unique>(column_ind); - case ColumnType::FLOAT: - output_schema.AddColumn(GetOutputName(), ColumnType::DOUBLE); - return std::make_unique>(column_ind); - case ColumnType::DOUBLE: - output_schema.AddColumn(GetOutputName(), ColumnType::LONGDOUBLE); - return std::make_unique>(column_ind); - case ColumnType::LONGDOUBLE: - output_schema.AddColumn(GetOutputName(), ColumnType::LONGDOUBLE); - return std::make_unique>(column_ind); - default: THROW_NOT_IMPLEMENTED; - } - } -}; - -class AvgExpression : public AggregateExpression { -public: - explicit AvgExpression(std::string column_name, std::string output_name = "") - : AggregateExpression(std::move(column_name), std::move(output_name)) { - } - std::string GetOutputName() const override { - if (!output_name_.empty()) { - return output_name_; - } - return "avg_" + column_name_; - } - - std::unique_ptr CreateAggregationFunction(const Schema& child_schema, - Schema& output_schema) override { - const size_t column_ind = child_schema.GetColumnIndexByName(GetName()); - const ColumnType column_type = child_schema.GetColumnTypeByName(GetName()); - output_schema.AddColumn(GetOutputName(), ColumnType::LONGDOUBLE); - -#define HANDLE_TYPE(ENUM_VAL, STR_VAL, CLASS_TYPE) \ - case ColumnType::ENUM_VAL: \ - return std::make_unique>(column_ind); - // END_HANDLE_TYPE - - switch (column_type) { - FOR_NUMERIC_COLUMN_TYPE(HANDLE_TYPE); - default: THROW_NOT_IMPLEMENTED; - } - } - -#undef HANDLE_TYPE -}; - -inline std::shared_ptr Count(std::string column_name = "*", - std::string output_name = "count") { - return std::make_shared(std::move(column_name), std::move(output_name)); -} -inline std::shared_ptr Min(std::string column_name, - std::string output_name = "") { - return std::make_shared(std::move(column_name), std::move(output_name)); -} -inline std::shared_ptr Max(std::string column_name, - std::string output_name = "") { - return std::make_shared(std::move(column_name), std::move(output_name)); -} -inline std::shared_ptr Sum(std::string column_name, - std::string output_name = "") { - return std::make_shared(std::move(column_name), std::move(output_name)); -} -inline std::shared_ptr Avg(std::string column_name, - std::string output_name = "") { - return std::make_shared(std::move(column_name), std::move(output_name)); -} -inline std::shared_ptr DistinctCount(std::string column_name, - std::string output_name = "") { - return std::make_shared(std::move(column_name), - std::move(output_name)); -} - -#endif // COLUMNAR_ENGINE_EXPRESSIONS_H diff --git a/src/AggregationFunctions.h b/src/AggregationFunctions.h deleted file mode 100644 index cce114f..0000000 --- a/src/AggregationFunctions.h +++ /dev/null @@ -1,360 +0,0 @@ -// -// Created by ragnarokk on 30.03.2026. -// - -#ifndef COLUMNAR_ENGINE_AGGREGATIONFUNCTIONS_H -#define COLUMNAR_ENGINE_AGGREGATIONFUNCTIONS_H - -#include "OperatorsBase.h" - -#include -#include -#include - -class AggregationFunction { -public: - virtual ~AggregationFunction() = default; - - virtual void Update(const RecordBatch& batch, - const std::vector* group_ids = nullptr) = 0; - virtual std::shared_ptr Finalize() = 0; -}; - -struct SumOperation { - template - static inline void Apply(ResultType& result, const ValueType& value) { - result += value; - } - - template - static constexpr ResultType GetInitValue() { - return 0; - } -}; - -struct MaxOperation { - template - static inline void Apply(ResultType& result, const ValueType& value) { - if (value > result) { - result = value; - } - } - - template - static constexpr ResultType GetInitValue() { - return std::numeric_limits::lowest(); - } -}; - -struct MinOperation { - template - static inline void Apply(ResultType& result, const ValueType& value) { - if (value < result) { - result = value; - } - } - - template - static constexpr ResultType GetInitValue() { - return std::numeric_limits::max(); - } -}; - -// OPTIMIZE: maybe redundant -struct CountOperation { - template - static inline void Apply(ResultType& result, const ValueType&) { - ++result; - } -}; - -template -struct AggregationFunctionTraits; - -template <> -struct AggregationFunctionTraits { - using ValueType = int16_t; - using StateType = int32_t; - using ResultColumnType = Int32Column; -}; - -template <> -struct AggregationFunctionTraits { - using ValueType = int32_t; - using StateType = int64_t; - using ResultColumnType = Int64Column; -}; - -template <> -struct AggregationFunctionTraits { - using ValueType = int64_t; - using StateType = __int128_t; - using ResultColumnType = Int128Column; -}; - -template <> -struct AggregationFunctionTraits { - using ValueType = __int128_t; - using StateType = __int128_t; - using ResultColumnType = Int128Column; -}; - -template <> -struct AggregationFunctionTraits { - using ValueType = float; - using StateType = double; - using ResultColumnType = DoubleColumn; -}; - -template <> -struct AggregationFunctionTraits { - using ValueType = double; - using StateType = long double; - using ResultColumnType = LongDoubleColumn; -}; - -template <> -struct AggregationFunctionTraits { - using ValueType = long double; - using StateType = long double; - using ResultColumnType = LongDoubleColumn; -}; - -template <> -struct AggregationFunctionTraits { - using ValueType = char; - using StateType = int16_t; - using ResultColumnType = Int16Column; -}; - -template <> -struct AggregationFunctionTraits { - using ValueType = int32_t; - using StateType = int32_t; - using ResultColumnType = DateColumn; -}; - -template <> -struct AggregationFunctionTraits { - using ValueType = int64_t; - using StateType = int64_t; - using ResultColumnType = TimestampColumn; -}; - -template <> -struct AggregationFunctionTraits { - using ValueType = std::string; -}; - -template -class TypedAggregationFunction : public AggregationFunction { - using ResultColumnType = typename AggregationFunctionTraits::ResultColumnType; - using StateType = typename AggregationFunctionTraits::StateType; - -public: - explicit TypedAggregationFunction(size_t column_index) : column_index_(column_index) { - } - - void Update(const RecordBatch& batch, - const std::vector* group_ids = nullptr) override { - auto* column = static_cast(batch.columns[column_index_].get()); - const auto& data = column->GetData(); - - if (!batch.selection_vector) { - for (size_t i = 0; i < batch.num_rows; ++i) { - uint32_t gid = group_ids ? (*group_ids)[i] : 0; - if (gid >= states_.size()) { - states_.resize(gid + 1, Operation::template GetInitValue()); - } - Operation::Apply(states_[gid], data[i]); - } - } else { - for (size_t i = 0; i < batch.num_rows; ++i) { - uint32_t gid = group_ids ? (*group_ids)[i] : 0; - if (gid >= states_.size()) { - states_.resize(gid + 1, Operation::template GetInitValue()); - } - uint32_t ind = (*batch.selection_vector)[i]; - Operation::Apply(states_[gid], data[ind]); - } - } - } - - std::shared_ptr Finalize() override { - auto result_column = std::make_shared(); - if (states_.empty()) { - result_column->Add(Operation::template GetInitValue()); - } else { - for (const auto& state : states_) { - result_column->Add(state); - } - } - return result_column; - } - -private: - size_t column_index_; - std::vector states_; -}; - -template -using SumAggregationFunction = TypedAggregationFunction; - -template -using MinAggregationFunction = TypedAggregationFunction; - -template -using MaxAggregationFunction = TypedAggregationFunction; - -class CountAggregationFunction : public AggregationFunction { -public: - void Update(const RecordBatch& batch, - const std::vector* group_ids = nullptr) override { - for (size_t i = 0; i < batch.num_rows; ++i) { - uint32_t gid = group_ids ? (*group_ids)[i] : 0; - if (gid >= counts_.size()) { - counts_.resize(gid + 1, 0); - } - counts_[gid]++; - } - } - - std::shared_ptr Finalize() override { - auto result_column = std::make_shared(); - if (counts_.empty()) { - result_column->Add(0); - } else { - for (int64_t c : counts_) { - result_column->Add(static_cast(c)); - } - } - return result_column; - } - -private: - std::vector counts_; -}; - -template -class DistinctCountAggregationFunction : public AggregationFunction { - using ResultColumnType = Int64Column; - using ValueType = typename AggregationFunctionTraits::ValueType; - -public: - explicit DistinctCountAggregationFunction(size_t column_index) : column_index_(column_index) { - } - - void Update(const RecordBatch& batch, - const std::vector* group_ids = nullptr) override { - auto* column = static_cast(batch.columns[column_index_].get()); - const auto& data = column->GetData(); - - if (!batch.selection_vector) { - for (size_t i = 0; i < batch.num_rows; ++i) { - uint32_t gid = group_ids ? (*group_ids)[i] : 0; - if (gid >= distinct_values_.size()) { - distinct_values_.resize(gid + 1); - } - if constexpr (std::is_same_v) { - distinct_values_[gid].emplace(data[i]); - } else { - distinct_values_[gid].insert(data[i]); - } - } - } else { - for (size_t i = 0; i < batch.num_rows; ++i) { - uint32_t gid = group_ids ? (*group_ids)[i] : 0; - if (gid >= distinct_values_.size()) { - distinct_values_.resize(gid + 1); - } - uint32_t ind = (*batch.selection_vector)[i]; - if constexpr (std::is_same_v) { - distinct_values_[gid].emplace(data[ind]); - } else { - distinct_values_[gid].insert(data[ind]); - } - } - } - } - - std::shared_ptr Finalize() override { - auto result_column = std::make_shared(); - if (distinct_values_.empty()) { - result_column->Add(0); - } else { - for (const auto& set : distinct_values_) { - result_column->Add(static_cast(set.size())); - } - } - return result_column; - } - -private: - size_t column_index_; - std::vector> distinct_values_; -}; - -template -class AvgAggregationFunction : public AggregationFunction { -public: - using StateType = typename AggregationFunctionTraits::StateType; - - explicit AvgAggregationFunction(size_t column_index) : column_index_(column_index) { - } - - void Update(const RecordBatch& batch, - const std::vector* group_ids = nullptr) override { - auto* column = static_cast(batch.columns[column_index_].get()); - const auto& data = column->GetData(); - - if (!batch.selection_vector) { - for (size_t i = 0; i < batch.num_rows; ++i) { - uint32_t gid = group_ids ? (*group_ids)[i] : 0; - if (gid >= sums_.size()) { - sums_.resize(gid + 1, 0); - counts_.resize(gid + 1, 0); - } - sums_[gid] += static_cast(data[i]); - counts_[gid]++; - } - } else { - for (size_t i = 0; i < batch.num_rows; ++i) { - uint32_t gid = group_ids ? (*group_ids)[i] : 0; - if (gid >= sums_.size()) { - sums_.resize(gid + 1, 0); - counts_.resize(gid + 1, 0); - } - uint32_t ind = (*batch.selection_vector)[i]; - sums_[gid] += static_cast(data[ind]); - counts_[gid]++; - } - } - } - - std::shared_ptr Finalize() override { - auto result_column = std::make_shared(); - if (sums_.empty()) { - result_column->Add(0.0); - } else { - for (size_t gid = 0; gid < sums_.size(); ++gid) { - if (counts_[gid] == 0) { - result_column->Add(0.0); - } else { - StateType quotient = sums_[gid] / counts_[gid]; - long double rem = - static_cast(sums_[gid] - quotient * counts_[gid]) / - counts_[gid]; - result_column->Add(static_cast(quotient) + rem); - } - } - } - return result_column; - } - -private: - size_t column_index_; - std::vector sums_; - std::vector counts_; -}; - -#endif // COLUMNAR_ENGINE_AGGREGATIONFUNCTIONS_H diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 186097d..23508a8 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -1,47 +1,85 @@ add_library(project_includes INTERFACE) + +target_link_libraries(project_includes INTERFACE + absl::flat_hash_map + re2::re2 +) + target_include_directories(project_includes INTERFACE ${CMAKE_CURRENT_SOURCE_DIR} - read_write/test - ${CMAKE_CURRENT_SOURCE_DIR}/read_write + ${CMAKE_CURRENT_SOURCE_DIR}/column + ${CMAKE_CURRENT_SOURCE_DIR}/execution + ${CMAKE_CURRENT_SOURCE_DIR}/execution/expressions + ${CMAKE_CURRENT_SOURCE_DIR}/io + ${CMAKE_CURRENT_SOURCE_DIR}/io/test + ${CMAKE_CURRENT_SOURCE_DIR}/types + ${CMAKE_CURRENT_SOURCE_DIR}/utils + ${CMAKE_CURRENT_SOURCE_DIR}/utils/test ) -add_executable(columnar-engine +find_package(Threads REQUIRED) + +set(ENGINE_SOURCES main.cpp - read_write/CSVTokenizer.cpp - read_write/CSVTokenizer.h - read_write/ColumnarReader.cpp - read_write/ColumnarReader.h - Types.h - read_write/CSVReader.cpp - read_write/CSVReader.h - read_write/ColumnarWriter.cpp - read_write/ColumnarWriter.h - read_write/CSVToColumnar.cpp - read_write/CSVToColumnar.h - OperatorsBase.h - Engine.cpp - Engine.h - Codec.h - Query.h - ExecutionLogic.h - AggregationFunctions.h - Schema.h - macro.h - ExecutionAPI.h - AggregationExpressions.h - OperatorsBase.cpp - VectorOfStrings.h - FilterExpressions.h - FilterFunctions.h + io/CsvTokenizer.cpp + io/CsvTokenizer.h + io/ColumnarReader.cpp + io/ColumnarReader.h + io/CsvReader.cpp + io/CsvReader.h + io/ColumnarWriter.cpp + io/ColumnarWriter.h + io/CsvToColumnar.cpp + io/CsvToColumnar.h + execution/OperatorsBase.h + column/Codec.h + execution/ExecutionLogic.h + execution/ExecutionLogic.cpp + execution/expressions/AggregationFunctions.h + column/Schema.h + utils/Macro.h + execution/ExecutionApi.h + execution/expressions/AggregationExpressions.h + execution/OperatorsBase.cpp + utils/VectorOfStrings.h + execution/expressions/FilterExpressions.h + execution/expressions/FilterFunctions.h + execution/expressions/ScalarFunctions.h + execution/expressions/ScalarExpressions.h + execution/ExecutionHelper.h + column/ColumnFactory.h + column/CharColumn.h + column/NumericColumn.h + column/StringColumn.h + column/TemporalColumn.h + column/Column.h + column/ColumnBuilder.h + types/ColumnType.h + types/Date.h + types/Timestamp.h + execution/expressions/AggExpHelper.h + types/String.h + column/ColumnView.h + types/TimeUnit.h + utils/Concurrency.h + execution/Pipeline.h + execution/PipelineExecutor.h ) +add_executable(columnar-engine ${ENGINE_SOURCES} + utils/ThreadPool.h) target_link_libraries(columnar-engine PRIVATE project_includes) target_compile_options(columnar-engine PRIVATE -Werror) +add_executable(columnar-engine-mt ${ENGINE_SOURCES}) +target_link_libraries(columnar-engine-mt PRIVATE project_includes Threads::Threads) +target_compile_definitions(columnar-engine-mt PRIVATE ENABLE_MULTITHREADING) +target_compile_options(columnar-engine-mt PRIVATE -Werror) + add_executable(test_csv_parser - read_write/test/test_csv_parser.cpp - read_write/CSVTokenizer.cpp - read_write/CSVReader.cpp + io/test/TestCsvParser.cpp + io/CsvTokenizer.cpp + io/CsvReader.cpp ) target_link_libraries(test_csv_parser PRIVATE @@ -49,12 +87,12 @@ target_link_libraries(test_csv_parser PRIVATE project_includes) add_executable(test_reader_writer - read_write/test/test_reader_writer.cpp - read_write/CSVTokenizer.cpp - read_write/ColumnarReader.cpp - read_write/CSVReader.cpp - read_write/ColumnarWriter.cpp - read_write/CSVToColumnar.cpp + io/test/TestReaderWriter.cpp + io/CsvTokenizer.cpp + io/ColumnarReader.cpp + io/CsvReader.cpp + io/ColumnarWriter.cpp + io/CsvToColumnar.cpp ) target_link_libraries(test_reader_writer PRIVATE @@ -62,8 +100,8 @@ target_link_libraries(test_reader_writer PRIVATE project_includes) add_executable(test_vector_of_strings - test_vector_of_strings.cpp - VectorOfStrings.h + utils/test/TestVectorOfStrings.cpp + utils/VectorOfStrings.h ) target_link_libraries(test_vector_of_strings PRIVATE diff --git a/src/Engine.cpp b/src/Engine.cpp deleted file mode 100644 index 3fcab76..0000000 --- a/src/Engine.cpp +++ /dev/null @@ -1,15 +0,0 @@ -// -// Created by ragnarokk on 06.01.2026. -// - -#include - -#include "read_write/CSVToColumnar.h" -#include "Engine.h" - -void Engine::InitDataFromCSV(const std::string& csv_file_path, const std::string& columnar_file_path) { - CSVToColumnar converter; - converter.Convert(csv_file_path, columnar_file_path); -} - -// void Engine::ExecuteQuery(const std::string& columnar_file_path) {} diff --git a/src/Engine.h b/src/Engine.h deleted file mode 100644 index c34fe51..0000000 --- a/src/Engine.h +++ /dev/null @@ -1,19 +0,0 @@ -// -// Created by ragnarokk on 06.01.2026. -// - -// TODO: after SQL parser do the interaction here - -#ifndef COLUMNAR_ENGINE_ENGINE_H -#define COLUMNAR_ENGINE_ENGINE_H - -#include - -class Engine { -public: - void InitDataFromCSV(const std::string& csv_file_path, const std::string& columnar_file_path); - - void ExecuteQuery(const std::string& columnar_file_path); -}; - -#endif // COLUMNAR_ENGINE_ENGINE_H diff --git a/src/ExecutionAPI.h b/src/ExecutionAPI.h deleted file mode 100644 index 6c9102b..0000000 --- a/src/ExecutionAPI.h +++ /dev/null @@ -1,118 +0,0 @@ -// -// Created by ragnarokk on 31.03.2026. -// - -#ifndef COLUMNAR_ENGINE_EXECUTIONAPI_H -#define COLUMNAR_ENGINE_EXECUTIONAPI_H - -#include "AggregationExpressions.h" -#include "ExecutionLogic.h" -#include "FilterExpressions.h" - -#include - -class DataResult { -public: - DataResult(std::vector>&& batches, Schema schema) - : batches_(std::move(batches)), schema_(std::move(schema)) { - } - - const std::vector>& GetBatches() const { - return batches_; - } - - const Schema& GetSchema() const { - return schema_; - } - - void Display() const { - const std::vector& fields = schema_.GetFields(); - if (!fields.empty()) { - for (const Field& field : fields) { - std::cout << field.name << " "; - } - std::cout << "\n"; - } - - for (const std::unique_ptr& batch : batches_) { - if (batch->columns.empty() || batch->num_rows == 0) { - continue; - } - size_t rows = batch->num_rows; - for (size_t i = 0; i < rows; ++i) { - size_t ind = batch->selection_vector ? (*batch->selection_vector)[i] : i; - for (const std::shared_ptr& column : batch->columns) { - std::cout << column->GetDataAsString(ind) << " "; - } - std::cout << "\n"; - } - } - } - -private: - std::vector> batches_; - Schema schema_; -}; - -class DataFrame { -public: - explicit DataFrame(std::shared_ptr logical_plan) - : logical_plan_(std::move(logical_plan)) { - } - - static DataFrame Select(std::string columnar_file_path, std::vector column_names) { - auto scan_node = std::make_shared(); - scan_node->table_path = std::move(columnar_file_path); - scan_node->column_names = std::move(column_names); - return DataFrame(scan_node); - } - - DataFrame Aggregate(std::vector group_by_columns, - std::vector> aggregate_expressions) { - auto aggregate_node = std::make_shared(); - aggregate_node->child = logical_plan_; - aggregate_node->group_by_columns = std::move(group_by_columns); - aggregate_node->aggregate_expressions = std::move(aggregate_expressions); - return DataFrame(aggregate_node); - } - - DataFrame Filter(std::shared_ptr filter_expression) { - auto filter_node = std::make_shared(); - filter_node->child = logical_plan_; - filter_node->filter_expression = std::move(filter_expression); - return DataFrame(filter_node); - } - - DataFrame OrderBy(std::vector> order_by_columns, - std::optional limit = std::nullopt) { - auto order_by_node = std::make_shared(); - order_by_node->child = logical_plan_; - order_by_node->order_by_columns = std::move(order_by_columns); - order_by_node->limit = limit; - return DataFrame(order_by_node); - } - - DataFrame Limit(size_t limit) { - auto limit_node = std::make_shared(); - limit_node->child = logical_plan_; - limit_node->limit = limit; - return DataFrame(limit_node); - } - - DataResult Collect() { - PhysicalOperatorContext context = BuildPhysicalPlan(logical_plan_); - std::unique_ptr physical_plan_root = std::move(context.root_operator); - std::vector> result; - while (std::unique_ptr batch = physical_plan_root->Run()) { - if (batch != nullptr) { - result.push_back(std::move(batch)); - } - } - return DataResult{std::move(result), std::move(context.schema)}; - } - -private: - std::shared_ptr logical_plan_; -}; - -#endif // COLUMNAR_ENGINE_EXECUTIONAPI_H diff --git a/src/ExecutionLogic.h b/src/ExecutionLogic.h deleted file mode 100644 index 468adb1..0000000 --- a/src/ExecutionLogic.h +++ /dev/null @@ -1,225 +0,0 @@ -// -// Created by ragnarokk on 30.03.2026. -// - -#ifndef COLUMNAR_ENGINE_LOGIC_H -#define COLUMNAR_ENGINE_LOGIC_H - -#include "AggregationFunctions.h" -#include "AggregationExpressions.h" -#include "FilterFunctions.h" -#include "FilterExpressions.h" -#include "OperatorsBase.h" -#include "Schema.h" -#include "macro.h" - -#include -#include -#include -#include -#include - -struct PlanNode { - virtual ~PlanNode() = default; -}; - -struct ScanNode : public PlanNode { - std::string table_path; - std::vector column_names; -}; - -struct AggregateNode : public PlanNode { - std::shared_ptr child; - std::vector group_by_columns; - std::vector> aggregate_expressions; -}; - -struct FilterNode : public PlanNode { - std::shared_ptr child; - std::shared_ptr filter_expression; -}; - -struct OrderByNode : public PlanNode { - std::shared_ptr child; - std::vector> order_by_columns; - std::optional limit; -}; - -struct LimitNode : public PlanNode { - std::shared_ptr child; - size_t limit; -}; - -struct PhysicalOperatorContext { - std::unique_ptr root_operator; - Schema schema; -}; - -inline PhysicalOperatorContext BuildPhysicalPlan( - const std::shared_ptr& plan_node, - std::optional> required_columns = std::nullopt) { - if (auto scan = std::dynamic_pointer_cast(plan_node)) { - std::vector final_columns; - - Schema full_schema = Schema::FromColumnarFile(scan->table_path); - const std::vector& full_columns = full_schema.GetFields(); - - if (required_columns.has_value()) { - final_columns = required_columns.value(); - if (final_columns.empty() && !full_columns.empty()) { - final_columns.push_back(full_columns[0].name); - } - } else { - if (scan->column_names.empty()) { - for (const Field& field : full_columns) { - final_columns.push_back(field.name); - } - } else { - final_columns = scan->column_names; - } - } - - auto op = std::make_unique(scan->table_path, final_columns); - - Schema output_schema; - for (const std::string& name : final_columns) { - bool found = false; - for (const Field& field : full_columns) { - if (field.name == name) { - output_schema.AddField({field.name, field.type}); - found = true; - break; - } - } - if (!found) [[unlikely]] { - THROW_RUNTIME_ERROR("Column " + name + " not found in table schema"); - } - } - - return {std::move(op), std::move(output_schema)}; - } - - if (auto aggregate = std::dynamic_pointer_cast(plan_node)) { - if (!aggregate->group_by_columns.empty()) { - std::vector needed_columns = aggregate->group_by_columns; - for (const std::shared_ptr& expr : - aggregate->aggregate_expressions) { - expr->CollectRequiredColumns(needed_columns); - } - std::sort(needed_columns.begin(), needed_columns.end()); - needed_columns.erase(std::unique(needed_columns.begin(), needed_columns.end()), - needed_columns.end()); - - PhysicalOperatorContext child_context = - BuildPhysicalPlan(aggregate->child, needed_columns); - - std::vector group_col_indices; - std::vector> empty_key_columns; - Schema output_schema; - - for (const auto& col_name : aggregate->group_by_columns) { - size_t ind = child_context.schema.GetColumnIndexByName(col_name); - group_col_indices.push_back(ind); - - ColumnType type = child_context.schema.GetColumnTypeByName(col_name); - output_schema.AddColumn(col_name, type); - -#define HANDLE_TYPE(ENUM_VAL, STR_VAL, CLASS_TYPE) \ - case ColumnType::ENUM_VAL: empty_key_columns.push_back(std::make_shared()); break; - - switch (type) { - FOR_EACH_COLUMN_TYPE(HANDLE_TYPE); - default: THROW_NOT_IMPLEMENTED; - } -#undef HANDLE_TYPE - } - - std::vector> agg_funcs; - for (const std::shared_ptr& expr : - aggregate->aggregate_expressions) { - agg_funcs.push_back( - expr->CreateAggregationFunction(child_context.schema, output_schema)); - } - - auto op = std::make_unique( - std::move(child_context.root_operator), std::move(group_col_indices), - std::move(empty_key_columns), std::move(agg_funcs)); - return {std::move(op), std::move(output_schema)}; - } - - std::vector needed_columns = aggregate->group_by_columns; - for (const std::shared_ptr& expr : aggregate->aggregate_expressions) { - expr->CollectRequiredColumns(needed_columns); - } - std::sort(needed_columns.begin(), needed_columns.end()); - needed_columns.erase(std::unique(needed_columns.begin(), needed_columns.end()), - needed_columns.end()); - - PhysicalOperatorContext child_context = BuildPhysicalPlan(aggregate->child, needed_columns); - std::vector> agg_functions; - Schema output_schema; - for (const std::shared_ptr& expr : aggregate->aggregate_expressions) { - agg_functions.push_back( - expr->CreateAggregationFunction(child_context.schema, output_schema)); - } - auto op = std::make_unique(std::move(child_context.root_operator), - std::move(agg_functions)); - return {std::move(op), std::move(output_schema)}; - } - - if (auto filter = std::dynamic_pointer_cast(plan_node)) { - std::vector needed_columns; - filter->filter_expression->CollectRequiredColumns(needed_columns); - if (required_columns.has_value()) { - needed_columns.insert(needed_columns.end(), required_columns->begin(), - required_columns->end()); - } - - std::sort(needed_columns.begin(), needed_columns.end()); - needed_columns.erase(std::unique(needed_columns.begin(), needed_columns.end()), - needed_columns.end()); - - PhysicalOperatorContext child_context = BuildPhysicalPlan(filter->child, needed_columns); - std::unique_ptr filter_function = - filter->filter_expression->CreateFilterFunction(child_context.schema); - auto op = std::make_unique(std::move(child_context.root_operator), - std::move(filter_function)); - return {std::move(op), std::move(child_context.schema)}; - } - - if (auto order_by = std::dynamic_pointer_cast(plan_node)) { - std::vector needed_columns; - if (required_columns.has_value()) { - needed_columns = required_columns.value(); - } - for (const auto& [col_name, is_desc] : order_by->order_by_columns) { - needed_columns.push_back(col_name); - } - std::sort(needed_columns.begin(), needed_columns.end()); - needed_columns.erase(std::unique(needed_columns.begin(), needed_columns.end()), - needed_columns.end()); - - PhysicalOperatorContext child_context = BuildPhysicalPlan(order_by->child, needed_columns); - std::vector> sort_columns; - for (const auto& [col_name, is_desc] : order_by->order_by_columns) { - size_t sort_col_idx = child_context.schema.GetColumnIndexByName(col_name); - sort_columns.push_back({sort_col_idx, is_desc}); - } - - auto op = - std::make_unique(std::move(child_context.root_operator), - std::move(sort_columns), std::move(order_by->limit)); - return {std::move(op), std::move(child_context.schema)}; - } - - if (auto limit = std::dynamic_pointer_cast(plan_node)) { - PhysicalOperatorContext child_context = BuildPhysicalPlan(limit->child, required_columns); - auto op = - std::make_unique(std::move(child_context.root_operator), limit->limit); - return {std::move(op), std::move(child_context.schema)}; - } - - THROW_RUNTIME_ERROR("Unknown plan node type"); -} - -#endif // COLUMNAR_ENGINE_LOGIC_H diff --git a/src/FilterExpressions.h b/src/FilterExpressions.h deleted file mode 100644 index a6a693d..0000000 --- a/src/FilterExpressions.h +++ /dev/null @@ -1,114 +0,0 @@ -// -// Created by ragnarokk on 21.04.2026. -// - -#ifndef COLUMNAR_ENGINE_FILTEREXPRESSIONS_H -#define COLUMNAR_ENGINE_FILTEREXPRESSIONS_H - -#include "FilterFunctions.h" -#include "macro.h" -#include "Schema.h" - -#include -#include - -class FilterExpression { -public: - virtual ~FilterExpression() = default; - - explicit FilterExpression(std::string column_name) : column_name_(std::move(column_name)) { - } - - void CollectRequiredColumns(std::vector& required_columns) { - required_columns.push_back(column_name_); - } - - virtual std::unique_ptr CreateFilterFunction(const Schema& child_schema) = 0; - -protected: - std::string column_name_; -}; - -template -class NotEqFilterExpression : public FilterExpression { -public: - NotEqFilterExpression(std::string column_name, ValueType value) - : FilterExpression(column_name), value_(std::move(value)) { - } - - std::unique_ptr CreateFilterFunction(const Schema& child_schema) override { - size_t ind = child_schema.GetColumnIndexByName(column_name_); - ColumnType column_type = child_schema.GetColumnTypeByName(column_name_); - -#define HANDLE_TYPE(ENUM_VAL, STR_VAL, CLASS_TYPE) \ - case ColumnType::ENUM_VAL: \ - if constexpr (std::is_same_v) { \ - return std::make_unique>(ind, value_); \ - } else { \ - THROW_NOT_IMPLEMENTED; \ - } - - switch (column_type) { - FOR_EACH_COLUMN_TYPE(HANDLE_TYPE); - default: THROW_NOT_IMPLEMENTED; - } - -#undef HANDLE_TYPE - } - -private: - ValueType value_; -}; - -template -class EqFilterExpression : public FilterExpression { -public: - EqFilterExpression(std::string column_name, ValueType value) - : FilterExpression(column_name), value_(std::move(value)) { - } - - std::unique_ptr CreateFilterFunction(const Schema& child_schema) override { - size_t ind = child_schema.GetColumnIndexByName(column_name_); - ColumnType column_type = child_schema.GetColumnTypeByName(column_name_); - -#define HANDLE_TYPE(ENUM_VAL, STR_VAL, CLASS_TYPE) \ - case ColumnType::ENUM_VAL: \ - if constexpr (std::is_same_v) { \ - return std::make_unique>(ind, value_); \ - } else { \ - THROW_NOT_IMPLEMENTED; \ - } - - switch (column_type) { - FOR_EACH_COLUMN_TYPE(HANDLE_TYPE); - default: THROW_NOT_IMPLEMENTED; - } - -#undef HANDLE_TYPE - } - -private: - ValueType value_; -}; - -template -inline std::shared_ptr NotEq(const std::string& column_name, T target_val) { - if constexpr (std::is_convertible_v) { - return std::make_shared>(column_name, - std::string(target_val)); - } else { - return std::make_shared>(column_name, std::move(target_val)); - } -} - -template -inline std::shared_ptr Eq(const std::string& column_name, T target_val) { - if constexpr (std::is_convertible_v) { - return std::make_shared>(column_name, - std::string(target_val)); - } else { - return std::make_shared>(column_name, std::move(target_val)); - } -} - -#endif // COLUMNAR_ENGINE_FILTEREXPRESSIONS_H diff --git a/src/FilterFunctions.h b/src/FilterFunctions.h deleted file mode 100644 index e537748..0000000 --- a/src/FilterFunctions.h +++ /dev/null @@ -1,86 +0,0 @@ -// -// Created by ragnarokk on 22.04.2026. -// - -#ifndef COLUMNAR_ENGINE_FILTERFUNCTIONS_H -#define COLUMNAR_ENGINE_FILTERFUNCTIONS_H - -#include "OperatorsBase.h" - -class FilterFunction { -public: - virtual ~FilterFunction() = default; - virtual std::vector Evaluate(const RecordBatch& batch) = 0; -}; - -template -class NotEqFilterFunction : public FilterFunction { -public: - NotEqFilterFunction(size_t column_ind, ValueType target_val) - : column_ind_(column_ind), target_val_(target_val) { - } - - std::vector Evaluate(const RecordBatch& batch) override { - auto* column = static_cast(batch.columns[column_ind_].get()); - const auto& data = column->GetData(); - - std::vector selection_vector; - selection_vector.reserve(batch.num_rows); - - if (batch.selection_vector) { - for (uint32_t ind : *batch.selection_vector) { - if (data[ind] != target_val_) { - selection_vector.push_back(ind); - } - } - } else { - for (size_t i = 0; i < batch.num_rows; ++i) { - if (data[i] != target_val_) { - selection_vector.push_back(i); - } - } - } - return selection_vector; - } - -private: - size_t column_ind_; - ValueType target_val_; -}; - -template -class EqFilterFunction : public FilterFunction { -public: - EqFilterFunction(size_t column_ind, ValueType target_val) - : column_ind_(column_ind), target_val_(target_val) { - } - - std::vector Evaluate(const RecordBatch& batch) override { - auto* column = static_cast(batch.columns[column_ind_].get()); - const auto& data = column->GetData(); - - std::vector selection_vector; - selection_vector.reserve(batch.num_rows); - - if (batch.selection_vector) { - for (uint32_t ind : *batch.selection_vector) { - if (data[ind] == target_val_) { - selection_vector.push_back(ind); - } - } - } else { - for (size_t i = 0; i < batch.num_rows; ++i) { - if (data[i] == target_val_) { - selection_vector.push_back(i); - } - } - } - return selection_vector; - } - -private: - size_t column_ind_; - ValueType target_val_; -}; - -#endif // COLUMNAR_ENGINE_FILTERFUNCTIONS_H diff --git a/src/OperatorsBase.cpp b/src/OperatorsBase.cpp deleted file mode 100644 index ef8e42a..0000000 --- a/src/OperatorsBase.cpp +++ /dev/null @@ -1,325 +0,0 @@ -// -// Created by ragnarokk on 31.03.2026. -// - -#include "AggregationFunctions.h" -#include "FilterFunctions.h" -#include "OperatorsBase.h" - -#include - -ScanOperator::ScanOperator(const std::string& table_path, - const std::vector& column_names) - : reader_(table_path) { - const std::vector& metadata = reader_.GetMetadata(); - - if (column_names.empty()) { - for (size_t i = 0; i < metadata.size(); ++i) { - column_indices_.push_back(i); - } - } else { - for (const auto& name : column_names) { - bool found = false; - for (size_t i = 0; i < metadata.size(); ++i) { - if (metadata[i].name == name) { - column_indices_.push_back(i); - found = true; - break; - } - } - if (!found) { - THROW_RUNTIME_ERROR("Column " + name + " not found in metadata"); - } - } - } - - if (!metadata.empty()) { - total_chunks_ = metadata[0].offsets.size(); - } -} - -std::unique_ptr ScanOperator::Run() { - if (current_chunk_ >= total_chunks_) { - return nullptr; - } - - auto record_batch = std::make_unique(); - record_batch->num_rows = 0; - - for (size_t col_idx : column_indices_) { - std::shared_ptr column_data = reader_.GetColumnData(col_idx, current_chunk_); - if (record_batch->num_rows == 0) { - record_batch->num_rows = column_data->Size(); - } - record_batch->columns.push_back(std::move(column_data)); - } - - ++current_chunk_; - return record_batch; -} - -AggregationOperator::AggregationOperator( - std::unique_ptr child, - std::vector> aggregation_functions) - : child_(std::move(child)), aggregation_functions_(std::move(aggregation_functions)) { -} - -std::unique_ptr AggregationOperator::Run() { - if (finished_) { - return nullptr; - } - while (std::unique_ptr batch = child_->Run()) { - for (std::unique_ptr& aggregation_function : aggregation_functions_) { - aggregation_function->Update(*batch); - } - } - auto result_batch = std::make_unique(); - result_batch->num_rows = 1; - - for (std::unique_ptr& aggregation_function : aggregation_functions_) { - result_batch->columns.push_back(aggregation_function->Finalize()); - } - - finished_ = true; - return result_batch; -} - -FilterOperator::FilterOperator(std::unique_ptr child, - std::shared_ptr filter_function) - : child_(std::move(child)), filter_function_(std::move(filter_function)) { -} - -std::unique_ptr FilterOperator::Run() { - while (std::unique_ptr batch = child_->Run()) { - std::vector selection_vector = filter_function_->Evaluate(*batch); - if (selection_vector.empty()) { - continue; - } - batch->num_rows = selection_vector.size(); - batch->selection_vector = - std::make_shared>(std::move(selection_vector)); - return batch; - } - return nullptr; -} - -GroupByOperator::GroupByOperator(std::unique_ptr child, - std::vector group_by_col_indices, - std::vector> empty_key_columns, - std::vector> agg_funcs) - : child_(std::move(child)), - group_by_col_indices_(std::move(group_by_col_indices)), - key_columns_(std::move(empty_key_columns)), - agg_funcs_(std::move(agg_funcs)) { -} - -std::unique_ptr GroupByOperator::Run() { - if (!accumulated_) { - while (auto batch = child_->Run()) { - std::vector group_ids; - size_t batch_size = batch->num_rows; - group_ids.reserve(batch_size); - - const std::shared_ptr>& sel_vec = batch->selection_vector; - std::vector composite_keys(batch_size); - for (size_t col_idx : group_by_col_indices_) { - batch->columns[col_idx]->SerializeBatch(composite_keys, sel_vec.get()); - } - - std::vector new_group_indices; - size_t sz = key_columns_.empty() ? 0 : key_columns_[0]->Size(); - - bool is_filtered = sel_vec != nullptr; - for (size_t i = 0; i < batch_size; ++i) { - const std::string& key = composite_keys[i]; - auto it = hash_table_.find(key); - uint32_t gid; - - if (it == hash_table_.end()) { - gid = sz + new_group_indices.size(); - hash_table_[key] = gid; - size_t actual_idx = is_filtered ? (*sel_vec)[i] : i; - new_group_indices.push_back(actual_idx); - } else { - gid = it->second; - } - group_ids.push_back(gid); - } - - if (!new_group_indices.empty()) { - for (size_t k = 0; k < group_by_col_indices_.size(); ++k) { - key_columns_[k]->CopyBatchFrom(*batch->columns[group_by_col_indices_[k]], - &new_group_indices); - } - } - - for (auto& func : agg_funcs_) { - func->Update(*batch, &group_ids); - } - } - accumulated_ = true; - - size_t total_groups = key_columns_.empty() ? 0 : key_columns_[0]->Size(); - auto result_batch = std::make_unique(); - result_batch->num_rows = total_groups; - - for (size_t k = 0; k < key_columns_.size(); ++k) { - result_batch->columns.push_back(std::move(key_columns_[k])); - } - for (size_t a = 0; a < agg_funcs_.size(); ++a) { - auto full_agg_col = agg_funcs_[a]->Finalize(); - result_batch->columns.push_back(std::move(full_agg_col)); - } - return result_batch; - } - - return nullptr; -} - -OrderByOperator::OrderByOperator(std::unique_ptr child, - std::vector> sort_columns, - std::optional limit) - : child_(std::move(child)), sort_columns_(std::move(sort_columns)), limit_(limit) { -} - -using CompareFunc = int (*)(const void*, uint32_t, uint32_t); - -struct SortColumnInfo { - const void* raw_data; - CompareFunc cmp_func; - bool is_desc; -}; - -template -int Compare(const void* raw_data, uint32_t a, uint32_t b) { - using Container = ColumnType::ContainerType; - const Container* data = static_cast(raw_data); - if ((*data)[a] < (*data)[b]) { - return -1; - } - if ((*data)[a] > (*data)[b]) { - return 1; - } - return 0; -} - -std::unique_ptr OrderByOperator::Run() { - if (!accumulated_) { - while (auto batch = child_->Run()) { - if (!accumulated_batch_) { - accumulated_batch_ = std::make_unique(); - size_t sz = batch->columns.size(); - for (size_t c = 0; c < sz; ++c) { - accumulated_batch_->columns.push_back(batch->columns[c]->CloneEmpty()); - } - accumulated_batch_->num_rows = 0; - } - - const std::shared_ptr>& sel = batch->selection_vector; - size_t sz = batch->columns.size(); - for (size_t c = 0; c < sz; ++c) { - accumulated_batch_->columns[c]->CopyBatchFrom(*batch->columns[c], sel.get()); - } - accumulated_batch_->num_rows += batch->num_rows; - } - - if (accumulated_batch_) { - size_t sz = accumulated_batch_->num_rows; - indices_.reserve(sz); - for (uint32_t i = 0; i < sz; ++i) { - indices_.push_back(i); - } - - std::vector cols_info; - cols_info.reserve(sort_columns_.size()); - for (const auto& [col_idx, is_desc] : sort_columns_) { - const Column* raw_column = accumulated_batch_->columns[col_idx].get(); - -#define HANDLE_TYPE(ENUM_VAL, STR_VAL, CLASS_TYPE) \ - case ColumnType::ENUM_VAL: { \ - const void* data_ptr = raw_column->GetRawData(); \ - cols_info.push_back({data_ptr, &Compare, is_desc}); \ - break; \ - } - auto col_type = raw_column->GetType(); - switch (col_type) { - FOR_EACH_COLUMN_TYPE(HANDLE_TYPE); - default: THROW_NOT_IMPLEMENTED; - } -#undef HANDLE_TYPE - } - - auto cmp = [&cols_info](uint32_t a, uint32_t b) { - for (const auto& info : cols_info) { - int res = info.cmp_func(info.raw_data, a, b); - if (res != 0) { - return info.is_desc ? (res > 0) : (res < 0); - } - } - return false; - }; - - if (limit_.has_value() && limit_.value() < sz) { - uint32_t lim = limit_.value(); - std::nth_element(indices_.begin(), indices_.begin() + lim, indices_.end(), cmp); - std::sort(indices_.begin(), indices_.begin() + lim, cmp); - indices_.resize(lim); - accumulated_batch_->num_rows = lim; - } else { - std::sort(indices_.begin(), indices_.end(), cmp); - } - - auto sorted_batch = std::make_unique(); - sorted_batch->num_rows = accumulated_batch_->num_rows; - for (size_t c = 0; c < accumulated_batch_->columns.size(); ++c) { - auto new_col = accumulated_batch_->columns[c]->CloneEmpty(); - new_col->CopyBatchFrom(*accumulated_batch_->columns[c], &indices_); - sorted_batch->columns.push_back(std::move(new_col)); - } - accumulated_batch_ = std::move(sorted_batch); - - accumulated_ = true; - } - if (!accumulated_batch_) { - return nullptr; - } - return std::move(accumulated_batch_); - } - return nullptr; -} - -LimitOperator::LimitOperator(std::unique_ptr child, size_t limit) - : child_(std::move(child)), limit_(limit) { -} - -std::unique_ptr LimitOperator::Run() { - if (cur_rows_ >= limit_) { - return nullptr; - } - std::unique_ptr batch = child_->Run(); - if (!batch) { - return nullptr; - } - - if (cur_rows_ + batch->num_rows <= limit_) { - cur_rows_ += batch->num_rows; - return batch; - } else { - size_t remaining = limit_ - cur_rows_; - cur_rows_ += remaining; - batch->num_rows = remaining; - if (batch->selection_vector) { - auto new_sel_vec = std::make_shared>( - batch->selection_vector->begin(), - batch->selection_vector->begin() + remaining - ); - batch->selection_vector = std::move(new_sel_vec); - } else { - auto sel_vec = std::make_shared>(remaining); - std::iota(sel_vec->begin(), sel_vec->end(), 0); - batch->selection_vector = std::move(sel_vec); - } - - return batch; - } -} diff --git a/src/OperatorsBase.h b/src/OperatorsBase.h deleted file mode 100644 index e09c3d6..0000000 --- a/src/OperatorsBase.h +++ /dev/null @@ -1,120 +0,0 @@ -// -// Created by ragnarokk on 06.01.2026. -// - -#ifndef COLUMNAR_ENGINE_OPERATORS_BASE_H -#define COLUMNAR_ENGINE_OPERATORS_BASE_H - -#include -#include -#include -#include - -#include "ColumnarReader.h" -#include "macro.h" -#include "Types.h" - -class AggregationFunction; -class GroupedAggregationFunction; -class FilterFunction; - -struct RecordBatch { - size_t num_rows; - std::shared_ptr> selection_vector; - std::vector> columns; -}; - -class Operator { -public: - virtual ~Operator() = default; - - virtual std::unique_ptr Run() = 0; -}; - -class ScanOperator : public Operator { -public: - ScanOperator(const std::string& table_path, const std::vector& column_names); - - std::unique_ptr Run() override; - -private: - ColumnarReader reader_; - std::vector column_indices_; - size_t current_chunk_ = 0; - size_t total_chunks_ = 0; -}; - -class AggregationOperator : public Operator { -public: - AggregationOperator(std::unique_ptr child, - std::vector> aggregation_functions); - - std::unique_ptr Run() override; - -private: - std::unique_ptr child_; - std::vector> aggregation_functions_; - bool finished_ = false; -}; - -class FilterOperator : public Operator { -public: - FilterOperator(std::unique_ptr child, - std::shared_ptr filter_function); - - std::unique_ptr Run() override; - -private: - std::unique_ptr child_; - std::shared_ptr filter_function_; -}; - -class GroupByOperator : public Operator { -public: - GroupByOperator(std::unique_ptr child, std::vector group_by_col_indices, - std::vector> empty_key_columns, - std::vector> agg_funcs); - - std::unique_ptr Run() override; - -private: - std::unique_ptr child_; - std::vector group_by_col_indices_; - std::vector> key_columns_; - std::vector> agg_funcs_; - bool accumulated_ = false; - - std::unordered_map hash_table_; -}; - -class OrderByOperator : public Operator { -public: - OrderByOperator(std::unique_ptr child, - std::vector> sort_columns, - std::optional limit); - - std::unique_ptr Run() override; - -private: - std::unique_ptr child_; - std::vector> sort_columns_; - std::unique_ptr accumulated_batch_; - std::vector indices_; - size_t current_idx_ = 0; - std::optional limit_; - bool accumulated_ = false; -}; - -class LimitOperator : public Operator { -public: - LimitOperator(std::unique_ptr child, size_t limit); - - std::unique_ptr Run() override; - -private: - std::unique_ptr child_; - size_t limit_; - size_t cur_rows_ = 0; -}; - -#endif // COLUMNAR_ENGINE_OPERATORS_BASE_H diff --git a/src/Query.h b/src/Query.h deleted file mode 100644 index f0ec178..0000000 --- a/src/Query.h +++ /dev/null @@ -1,54 +0,0 @@ -// -// Created by ragnarokk on 30.03.2026. -// - -#ifndef COLUMNAR_ENGINE_QUERY_H -#define COLUMNAR_ENGINE_QUERY_H - -enum class QueryId { - Q0, - Q1, - Q2, - Q3, - Q4, - Q5, - Q6, - Q7, - Q8, - Q9, - Q10, - Q11, - Q12, - Q13, - Q14, - Q15, - Q16, - Q17, - Q18, - Q19, - Q20, - Q21, - Q22, - Q23, - Q24, - Q25, - Q26, - Q27, - Q28, - Q29, - Q30, - Q31, - Q32, - Q33, - Q34, - Q35, - Q36, - Q37, - Q38, - Q39, - Q40, - Q41, - Q42 -}; - -#endif // COLUMNAR_ENGINE_QUERY_H diff --git a/src/ScalarExpressions.h b/src/ScalarExpressions.h deleted file mode 100644 index 17d8a1f..0000000 --- a/src/ScalarExpressions.h +++ /dev/null @@ -1,42 +0,0 @@ -// -// Created by ragnarokk on 5/3/26. -// - -#ifndef COLUMNAR_ENGINE_SCALAREXPRESSIONS_H -#define COLUMNAR_ENGINE_SCALAREXPRESSIONS_H - -class ScalarExpression { -public: - virtual ~ScalarExpression() = default; - virtual std::shared_ptr Evaluate(const RecordBatch& batch) = 0; -}; - -class ExtractMinuteExpression : public ScalarExpression { -public: - explicit ExtractMinuteExpression(size_t input_col_idx) : input_col_idx_(input_col_idx) {} - - std::shared_ptr Evaluate(const RecordBatch& batch) override { - auto* input_col = static_cast(batch.columns[input_col_idx_].get()); - const auto& input_data = input_col->GetData(); - - auto result_col = std::make_shared(); // Минуты отлично влезут в Int32 - - // Оптимизация: учитываем selection_vector, если он есть (после Filter) - if (batch.selection_vector) { - for (size_t i = 0; i < batch.num_rows; ++i) { - size_t actual_idx = (*batch.selection_vector)[i]; - result_col->Add(TimestampColumn::ExtractMinute(input_data[actual_idx])); - } - } else { - for (size_t i = 0; i < input_col->Size(); ++i) { - result_col->Add(TimestampColumn::ExtractMinute(input_data[i])); - } - } - return result_col; - } - -private: - size_t input_col_idx_; -}; - -#endif // COLUMNAR_ENGINE_SCALAREXPRESSIONS_H diff --git a/src/Types.h b/src/Types.h deleted file mode 100644 index 2e81897..0000000 --- a/src/Types.h +++ /dev/null @@ -1,780 +0,0 @@ -// -// Created by ragnarokk on 03.01.2026. -// - -// OPTIMIZE: function AddBatch may be done with better logic of reserve (as in StringColumn) - -#ifndef COLUMNAR_ENGINE_OBJECT_H -#define COLUMNAR_ENGINE_OBJECT_H - -#include "macro.h" -#include "VectorOfStrings.h" - -#include -#include -#include -#include -#include -#include -#include -#include - -enum class ColumnType : uint8_t { - INT16, - INT32, - INT64, - INT128, - FLOAT, - DOUBLE, - LONGDOUBLE, - CHAR, - STRING, - DATE, - TIMESTAMP, -}; - -struct ColumnMetadata { - std::string name; - ColumnType type; - std::vector offsets; - std::vector sizes; -}; - -class Column { -public: - virtual ~Column() = default; - virtual ColumnType GetType() const = 0; - virtual size_t Size() const = 0; - virtual void Add(const std::string& value) = 0; // TODO: DO NOT USE IN REAL CODE - virtual void AddView(std::string_view value) = 0; - virtual void AddBatch(const VectorOfStrings2D& batch, size_t j) = 0; - virtual uint64_t WriteToFile(std::ofstream& file) = 0; - virtual std::string GetDataAsString(size_t index) const = 0; // TODO: DO NOT USE IN REAL CODE - virtual const void* GetRawData() const = 0; - virtual void SerializeValue(size_t index, - std::string& buffer) const = 0; // TODO: DO NOT USE IN REAL CODE - virtual void SerializeBatch(std::vector& keys, - const std::vector* selection = nullptr) const = 0; - virtual void ReadFromRawData(const std::vector& buffer) = 0; - virtual std::shared_ptr CloneEmpty() const = 0; - virtual void CopyFrom(const Column& other, size_t index) = 0; // TODO: DO NOT USE IN REAL CODE - virtual void CopyBatchFrom(const Column& other, - const std::vector* indices = nullptr) = 0; - virtual void Clear() = 0; -}; - -template -struct ColumnTypeTraits; - -template <> -struct ColumnTypeTraits { - static constexpr ColumnType kType = ColumnType::INT16; - static int16_t FromString(const std::string_view& value) { - int16_t parsed = 0; - const auto [ptr, ec] = std::from_chars(value.data(), value.data() + value.size(), parsed); - if (ec != std::errc() || ptr != value.data() + value.size()) { - THROW_RUNTIME_ERROR("Invalid INT16 value: " + std::string(value)); - } - return parsed; - } -}; - -template <> -struct ColumnTypeTraits { - static constexpr ColumnType kType = ColumnType::INT32; - static int32_t FromString(const std::string_view& value) { - int32_t parsed = 0; - const auto [ptr, ec] = std::from_chars(value.data(), value.data() + value.size(), parsed); - if (ec != std::errc() || ptr != value.data() + value.size()) { - THROW_RUNTIME_ERROR("Invalid INT32 value: " + std::string(value)); - } - return parsed; - } -}; - -template <> -struct ColumnTypeTraits { - static constexpr ColumnType kType = ColumnType::INT64; - static int64_t FromString(const std::string_view& value) { - int64_t parsed = 0; - const auto [ptr, ec] = std::from_chars(value.data(), value.data() + value.size(), parsed); - if (ec != std::errc() || ptr != value.data() + value.size()) { - THROW_RUNTIME_ERROR("Invalid INT64 value: " + std::string(value)); - } - return parsed; - } -}; - -template <> -struct ColumnTypeTraits<__int128_t> { - static constexpr ColumnType kType = ColumnType::INT128; - static __int128_t FromString(const std::string_view& value) { - __int128_t result = 0; - bool negative = false; - std::string_view value_view = value; - if (value[0] == '-') { - negative = true; - value_view = value_view.substr(1); - } - for (char c : value_view) { - if (c < '0' || c > '9') [[unlikely]] { - THROW_RUNTIME_ERROR("Invalid integer value: " + std::string(value)); - } - result = result * 10 + (c - '0'); - } - if (negative) { - result = -result; - } - return result; - } -}; - -template <> -struct ColumnTypeTraits { - static constexpr ColumnType kType = ColumnType::FLOAT; - static float FromString(const std::string_view& value) { - return std::stof(std::string(value)); - } -}; - -template <> -struct ColumnTypeTraits { - static constexpr ColumnType kType = ColumnType::DOUBLE; - static double FromString(const std::string_view& value) { - return std::stod(std::string(value)); - } -}; - -template <> -struct ColumnTypeTraits { - static constexpr ColumnType kType = ColumnType::LONGDOUBLE; - static long double FromString(const std::string_view& value) { - return std::stold(std::string(value)); - } -}; - -template - requires std::is_integral_v || std::is_floating_point_v -std::string ToString(T value) { - return std::to_string(value); -} - -template <> -inline std::string ToString<__int128_t>(__int128_t value) { - if (value == 0) { - return "0"; - } - std::string result; - bool negative = value < 0; - if (negative) { - value = -value; - } - while (value > 0) { - result.push_back('0' + (value % 10)); - value /= 10; - } - if (negative) { - result.push_back('-'); - } - std::reverse(result.begin(), result.end()); - return result; -} - -template -class NumericColumn : public Column { -public: - using ValueType = T; - using ContainerType = std::vector; - - ColumnType GetType() const override { - return ColumnTypeTraits::kType; - } - - size_t Size() const override { - return data_.size(); - } - - void Add(T value) { - data_.push_back(value); - } - - void Add(const std::string& value) override { - data_.push_back(ColumnTypeTraits::FromString(value)); - } - - void AddView(std::string_view value) override { - data_.push_back(ColumnTypeTraits::FromString(value)); - } - - void AddBatch(const VectorOfStrings2D& batch, size_t j) override { - data_.reserve(data_.size() + batch.Height()); - for (size_t i = 0; i < batch.Height(); ++i) { - data_.push_back(ColumnTypeTraits::FromString(batch.GetString2D(i, j))); - } - } - - // TODO: add decoding logic - uint64_t WriteToFile(std::ofstream& file) override { - if (!data_.empty()) { - file.write(reinterpret_cast(data_.data()), data_.size() * sizeof(T)); - } - return data_.size() * sizeof(T); - } - - std::string GetDataAsString(size_t index) const override { - if (index >= data_.size()) { - THROW_RUNTIME_ERROR("Index out of range"); - } - return ToString(data_[index]); - } - - const void* GetRawData() const override { - return &data_; - } - - void SerializeValue(size_t index, std::string& buffer) const override { - buffer.append(reinterpret_cast(&data_[index]), sizeof(T)); - } - - void SerializeBatch(std::vector& keys, - const std::vector* selection = nullptr) const override { - if (!selection) { - size_t n = data_.size(); - keys.reserve(n); - for (size_t i = 0; i < n; ++i) { - keys[i].append(reinterpret_cast(&data_[i]), sizeof(T)); - } - } else { - size_t n = selection->size(); - keys.reserve(n); - for (size_t i = 0; i < n; ++i) { - size_t idx = (*selection)[i]; - keys[i].append(reinterpret_cast(&data_[idx]), sizeof(T)); - } - } - } - - void ReadFromRawData(const std::vector& buffer) override { - if (buffer.size() % sizeof(T) != 0) { - THROW_RUNTIME_ERROR("Raw buffer size is not aligned with type size"); - } - size_t n = buffer.size() / sizeof(T); - data_.resize(n); - std::memcpy(data_.data(), buffer.data(), buffer.size()); - } - - std::shared_ptr CloneEmpty() const override { - return std::make_shared>(); - } - - void CopyFrom(const Column& other, size_t index) override { - data_.push_back(static_cast&>(other).GetData()[index]); - } - - void CopyBatchFrom(const Column& other, const std::vector* indices = nullptr) override { - const auto& other_data = static_cast&>(other).GetData(); - if (!indices) { - data_.insert(data_.end(), other_data.begin(), other_data.end()); - } else { - data_.reserve(data_.size() + indices->size()); - for (size_t idx : *indices) { - data_.push_back(other_data[idx]); - } - } - } - - void Clear() override { - data_.clear(); - } - - const std::vector& GetData() const { - return data_; - } - -private: - ContainerType data_; -}; - -using Int16Column = NumericColumn; -using Int32Column = NumericColumn; -using Int64Column = NumericColumn; -using Int128Column = NumericColumn<__int128_t>; -using FloatColumn = NumericColumn; -using DoubleColumn = NumericColumn; -using LongDoubleColumn = NumericColumn; - -class CharColumn : public Column { -public: - using ValueType = char; - using ContainerType = std::vector; - - ColumnType GetType() const override { - return ColumnType::CHAR; - } - size_t Size() const override { - return data_.size(); - } - - void Add(const std::string& value) override { - ASSERT(value.size() == 1); - data_.push_back(value[0]); - } - - void AddView(std::string_view value) override { - ASSERT(value.size() == 1); - data_.push_back(value[0]); - } - - void AddBatch(const VectorOfStrings2D& batch, size_t j) override { - data_.reserve(data_.size() + batch.Height()); - for (size_t i = 0; i < batch.Height(); ++i) { - std::string_view val = batch.GetString2D(i, j); - ASSERT(val.size() == 1); - data_.push_back(val[0]); - } - } - - uint64_t WriteToFile(std::ofstream& file) override { - if (!data_.empty()) { - file.write(data_.data(), data_.size()); - } - return data_.size(); - } - - std::string GetDataAsString(size_t index) const override { - if (index >= data_.size()) { - THROW_RUNTIME_ERROR("Index out of range"); - } - return std::string(1, data_[index]); - } - - void SerializeValue(size_t index, std::string& buffer) const override { - buffer.push_back(data_[index]); - } - - void SerializeBatch(std::vector& keys, - const std::vector* selection = nullptr) const override { - size_t n = !selection ? data_.size() : selection->size(); - for (size_t i = 0; i < n; ++i) { - size_t idx = !selection ? i : (*selection)[i]; - keys[i].push_back(data_[idx]); - } - } - - void ReadFromRawData(const std::vector& buffer) override { - data_.clear(); - data_.insert(data_.end(), buffer.begin(), buffer.end()); - } - - const void* GetRawData() const override { - return data_.data(); - } - - std::shared_ptr CloneEmpty() const override { - return std::make_shared(); - } - - void CopyFrom(const Column& other, size_t index) override { - data_.push_back(static_cast(other).GetData()[index]); - } - - void CopyBatchFrom(const Column& other, const std::vector* indices = nullptr) override { - const auto& other_data = static_cast(other).GetData(); - if (!indices) { - data_.insert(data_.end(), other_data.begin(), other_data.end()); - } else { - data_.reserve(data_.size() + indices->size()); - for (size_t idx : *indices) { - data_.push_back(other_data[idx]); - } - } - } - - void Clear() override { - data_.clear(); - } - - const ContainerType& GetData() const { - return data_; - } - -private: - ContainerType data_; -}; - -// OPTIMIZE: perf decreased because i made VectorOfStrings logic and moved this logic into a -// particular class -class StringColumn : public Column { -public: - using ValueType = std::string; - using ContainerType = VectorOfStrings; - - ColumnType GetType() const override { - return ColumnType::STRING; - } - - size_t Size() const override { - return data_.Size(); - } - - void Add(const std::string& value) override { - data_.PushBack(value); - } - - void AddView(std::string_view value) override { - data_.PushBack(value); - } - - void AddBatch(const VectorOfStrings2D& batch, size_t j) override { - size_t h = batch.Height(); - EnsureCapacity(batch.ColumnSize(j), h); - // data_.ReserveOffsets(data_.Size() + h + 1); - // data_.ReserveData(data_.DataSize() + batch.ColumnSize(j)); - for (size_t i = 0; i < h; ++i) { - data_.PushBack(batch.GetString2D(i, j)); - } - } - - uint64_t WriteToFile(std::ofstream& file) override { - const std::vector& offsets = data_.GetOffsets(); - const std::vector& data = data_.GetData(); - - uint64_t total_size = 0; - std::vector buffer; - size_t h = Size(); - size_t total_len = 0; - for (size_t i = 0; i < h; ++i) { - total_len += sizeof(uint32_t) + (offsets[i + 1] - offsets[i]); - } - buffer.reserve(total_len); - for (size_t i = 0; i < h; ++i) { - size_t start = offsets[i]; - size_t end = offsets[i + 1]; - uint32_t length = end - start; - buffer.insert(buffer.end(), reinterpret_cast(&length), - reinterpret_cast(&length) + sizeof(length)); - if (length > 0) { - buffer.insert(buffer.end(), data.begin() + start, data.begin() + end); - } - total_size += sizeof(length) + length; - } - file.write(buffer.data(), buffer.size()); - return total_size; - } - - std::string GetDataAsString(size_t index) const override { - if (index >= Size()) [[unlikely]] { - THROW_RUNTIME_ERROR("Index out of range"); - } - return std::string{data_[index]}; - } - - const void* GetRawData() const override { - return &data_; - } - - void SerializeValue(size_t index, std::string& buffer) const override { - std::string_view val = data_[index]; - uint32_t len = val.size(); - buffer.append(reinterpret_cast(&len), sizeof(len)); - buffer.append(val.data(), val.size()); - } - - void SerializeBatch(std::vector& keys, - const std::vector* selection = nullptr) const override { - if (!selection) { - size_t n = data_.Size(); - keys.reserve(n); - for (size_t i = 0; i < n; ++i) { - std::string_view val = data_[i]; - uint32_t len = val.size(); - keys[i].append(reinterpret_cast(&len), sizeof(len)); - keys[i].append(val.data(), val.size()); - } - } else { - size_t n = selection->size(); - keys.reserve(n); - for (size_t i = 0; i < n; ++i) { - size_t idx = (*selection)[i]; - std::string_view val = data_[idx]; - uint32_t len = val.size(); - keys[i].append(reinterpret_cast(&len), sizeof(len)); - keys[i].append(val.data(), val.size()); - } - } - } - - void ReadFromRawData(const std::vector& buffer) override { - Clear(); - size_t offset = 0; - while (offset < buffer.size()) { - uint32_t length; - std::memcpy(&length, buffer.data() + offset, sizeof(length)); - offset += sizeof(length); - std::string_view str(buffer.data() + offset, length); - data_.PushBack(str); - offset += length; - } - } - - std::shared_ptr CloneEmpty() const override { - return std::make_shared(); - } - - void CopyFrom(const Column& other, size_t index) override { - data_.PushBack(static_cast(other).data_[index]); - } - - void CopyBatchFrom(const Column& other, const std::vector* indices = nullptr) override { - const auto& other_data = static_cast(other).data_; - if (!indices) { - for (size_t i = 0; i < other_data.Size(); ++i) { - data_.PushBack(other_data[i]); - } - } else { - for (size_t idx : *indices) { - data_.PushBack(other_data[idx]); - } - } - } - - void Clear() override { - data_.Clear(); - } - - const ContainerType& GetData() const { - return data_; - } - -private: - ContainerType data_; - - void EnsureCapacity(size_t data_size, size_t elements) { - size_t required_sz = data_.DataSize() + data_size; - size_t data_cap = data_.DataCapacity(); - if (required_sz > data_cap) { - size_t cap = data_cap == 0 ? 1024 : data_cap * 2; - data_.ReserveData(std::max(required_sz, cap)); - } - - size_t required_offset_size = data_.Size() + elements; - size_t off_cap = data_.OffsetsCapacity(); - if (required_offset_size > off_cap) { - size_t cap = off_cap == 0 ? 1024 : off_cap * 2; - data_.ReserveOffsets(std::max(required_offset_size, cap)); - } - } -}; - -template -class TemporalColumn : public Column { -public: - using ValueType = T; - using ContainerType = std::vector; - - ColumnType GetType() const override { - return CType; - } - - size_t Size() const override { - return data_.size(); - } - - void Add(T value) { - data_.push_back(value); - } - - void Add(const std::string& value) override { - data_.push_back(Derived::Parse(value)); - } - - void AddView(std::string_view value) override { - data_.push_back(Derived::Parse(value)); - } - - void AddBatch(const VectorOfStrings2D& batch, size_t j) override { - size_t h = batch.Height(); - data_.reserve(data_.size() + h); - for (size_t i = 0; i < h; ++i) { - std::string_view cur = batch.GetString2D(i, j); - data_.push_back(Derived::Parse(cur)); - } - } - - uint64_t WriteToFile(std::ofstream& file) override { - if (!data_.empty()) { - file.write(reinterpret_cast(data_.data()), data_.size() * sizeof(T)); - } - return data_.size() * sizeof(T); - } - - std::string GetDataAsString(size_t index) const override { - if (index >= data_.size()) [[unlikely]] { - THROW_RUNTIME_ERROR("Index out of range"); - } - return Derived::Format(data_[index]); - } - - const void* GetRawData() const override { - return &data_; - } - - void SerializeValue(size_t index, std::string& buffer) const override { - buffer.append(reinterpret_cast(&data_[index]), sizeof(T)); - } - - void SerializeBatch(std::vector& keys, - const std::vector* selection = nullptr) const override { - size_t n = !selection ? data_.size() : selection->size(); - for (size_t i = 0; i < n; ++i) { - size_t idx = !selection ? i : (*selection)[i]; - keys[i].append(reinterpret_cast(&data_[idx]), sizeof(T)); - } - } - - void ReadFromRawData(const std::vector& buffer) override { - if (buffer.size() % sizeof(T) != 0) { - THROW_RUNTIME_ERROR("Raw buffer size is not aligned with type size"); - } - size_t n = buffer.size() / sizeof(T); - data_.resize(n); - std::memcpy(data_.data(), buffer.data(), buffer.size()); - } - - std::shared_ptr CloneEmpty() const override { - return std::make_shared>(); - } - - void CopyFrom(const Column& other, size_t index) override { - data_.push_back( - static_cast&>(other).GetData()[index]); - } - - void CopyBatchFrom(const Column& other, const std::vector* indices = nullptr) override { - const auto& other_data = - static_cast&>(other).GetData(); - if (!indices) { - data_.insert(data_.end(), other_data.begin(), other_data.end()); - } else { - data_.reserve(data_.size() + indices->size()); - for (size_t idx : *indices) { - data_.push_back(other_data[idx]); - } - } - } - - void Clear() override { - data_.clear(); - } - - const ContainerType& GetData() const { - return data_; - } - -private: - ContainerType data_; -}; - -class DateColumn : public TemporalColumn { -public: - static int32_t Parse(std::string_view unit) { - if (unit.size() != 10 || unit[4] != '-' || unit[7] != '-') [[unlikely]] { - THROW_RUNTIME_ERROR("Invalid date format: " + std::string(unit)); - } - - int32_t year = - (unit[0] - '0') * 1000 + (unit[1] - '0') * 100 + (unit[2] - '0') * 10 + (unit[3] - '0'); - int32_t month = (unit[5] - '0') * 10 + (unit[6] - '0'); - int32_t day = (unit[8] - '0') * 10 + (unit[9] - '0'); - - year -= (month <= 2); - const int era = (year >= 0 ? year : year - 399) / 400; - const unsigned yoe = static_cast(year - era * 400); - const unsigned doy = (153 * (month + (month > 2 ? -3 : 9)) + 2) / 5 + day - 1; - const unsigned doe = yoe * 365 + yoe / 4 - yoe / 100 + doy; - - return era * 146097 + static_cast(doe) - 719468; - } - - static std::string Format(int32_t days) { - days += 719468; - const int era = (days >= 0 ? days : days - 146096) / 146097; - const unsigned doe = static_cast(days - era * 146097); - const unsigned yoe = (doe - doe / 1460 + doe / 36524 - doe / 146096) / 365; - const int y = static_cast(yoe) + era * 400; - const unsigned doy = doe - (365 * yoe + yoe / 4 - yoe / 100); - const unsigned mp = (5 * doy + 2) / 153; - const unsigned d = doy - (153 * mp + 2) / 5 + 1; - const unsigned m = mp + (mp < 10 ? 3 : -9); - const int year = y + (m <= 2); - - std::string res = "0000-00-00"; - auto write_digit = [&](int val, int pos, int len) { - for (int i = 0; i < len; ++i) { - res[pos + len - 1 - i] = (val % 10) + '0'; - val /= 10; - } - }; - write_digit(year, 0, 4); - write_digit(m, 5, 2); - write_digit(d, 8, 2); - return res; - } -}; - -class TimestampColumn : public TemporalColumn { -public: - static int64_t Parse(std::string_view unit) { - if (unit.size() != 19 || unit[4] != '-' || unit[7] != '-' || unit[10] != ' ' || - unit[13] != ':' || unit[16] != ':') [[unlikely]] { - THROW_RUNTIME_ERROR("Invalid date format: " + std::string(unit)); - } - - auto to_int2 = [](char a, char b) { return (a - '0') * 10 + (b - '0'); }; - - int year = - (unit[0] - '0') * 1000 + (unit[1] - '0') * 100 + (unit[2] - '0') * 10 + (unit[3] - '0'); - int month = to_int2(unit[5], unit[6]); - int day = to_int2(unit[8], unit[9]); - int hour = to_int2(unit[11], unit[12]); - int minute = to_int2(unit[14], unit[15]); - int second = to_int2(unit[17], unit[18]); - - year -= (month <= 2); - const int era = (year >= 0 ? year : year - 399) / 400; - const unsigned yoe = static_cast(year - era * 400); - const unsigned doy = (153 * (month + (month > 2 ? -3 : 9)) + 2) / 5 + day - 1; - const unsigned doe = yoe * 365 + yoe / 4 - yoe / 100 + doy; - - int32_t days = era * 146097 + static_cast(doe) - 719468; - int64_t seconds = hour * 3600 + minute * 60 + second; - return static_cast(days) * 86400 + seconds; - } - - static std::string Format(int64_t total_seconds) { - int64_t days = total_seconds / 86400; - int64_t seconds_in_day = total_seconds % 86400; - if (seconds_in_day < 0) { - seconds_in_day += 86400; - days -= 1; - } - - std::string res = DateColumn::Format(static_cast(days)); - res += " 00:00:00"; - int hour = seconds_in_day / 3600; - int minute = (seconds_in_day % 3600) / 60; - int second = seconds_in_day % 60; - auto write_digit = [&](int val, int pos) { - res[pos] = (val / 10) + '0'; - res[pos + 1] = (val % 10) + '0'; - }; - - write_digit(hour, 11); - write_digit(minute, 14); - write_digit(second, 17); - - return res; - } -}; - -#endif // COLUMNAR_ENGINE_OBJECT_H diff --git a/src/column/CharColumn.h b/src/column/CharColumn.h new file mode 100644 index 0000000..d28eb1b --- /dev/null +++ b/src/column/CharColumn.h @@ -0,0 +1,79 @@ +#pragma once + +#include "column/Column.h" +#include "column/ColumnBuilder.h" + +#include +#include + +class CharColumn : public Column { +public: + using ValueType = char; + using ContainerType = std::vector; + + CharColumn() = default; + CharColumn(ContainerType&& data) noexcept : data_(std::move(data)) {} + + ColumnType GetType() const override { + return ColumnType::CHAR; + } + + size_t Size() const override { + return data_.size(); + } + + const void* GetRawData() const override { + return &data_; + } + + void Clear() override { + data_.clear(); + } + + const ContainerType& GetData() const { + return data_; + } + + void ReadFromBuffer(const std::vector& buffer) { + data_.clear(); + data_.insert(data_.end(), buffer.begin(), buffer.end()); + } + +private: + ContainerType data_; +}; + +class CharColumnBuilder : public ColumnBuilder { +public: + CharColumnBuilder() = default; + + void AddValue(char value) { + data_.push_back(value); + } + + void AddBatch(const VectorOfStrings2D& batch, size_t j) override { + size_t h = batch.Height(); + if (data_.size() < h) { + data_.reserve(data_.size() + h); + } + for (size_t i = 0; i < h; ++i) { + std::string_view val = batch.GetString2D(i, j); + ASSERT(val.size() == 1); + data_.push_back(val[0]); + } + } + + std::shared_ptr Finish() override { + auto result = std::make_shared(std::move(data_)); + data_.clear(); + return result; + } + +private: + std::vector data_; +}; + +template <> +struct BuilderTypeTrait { + using Type = CharColumnBuilder; +}; diff --git a/src/Codec.h b/src/column/Codec.h similarity index 100% rename from src/Codec.h rename to src/column/Codec.h diff --git a/src/column/Column.h b/src/column/Column.h new file mode 100644 index 0000000..c911043 --- /dev/null +++ b/src/column/Column.h @@ -0,0 +1,14 @@ +#pragma once + +#include "types/ColumnType.h" + +#include + +class Column { +public: + virtual ~Column() = default; + virtual ColumnType GetType() const = 0; + virtual size_t Size() const = 0; + virtual const void* GetRawData() const = 0; + virtual void Clear() = 0; +}; diff --git a/src/column/ColumnBuilder.h b/src/column/ColumnBuilder.h new file mode 100644 index 0000000..672c39f --- /dev/null +++ b/src/column/ColumnBuilder.h @@ -0,0 +1,16 @@ +#pragma once + +#include "column/Column.h" +#include "utils/VectorOfStrings.h" + +#include + +class ColumnBuilder { +public: + virtual ~ColumnBuilder() = default; + virtual void AddBatch(const VectorOfStrings2D& batch, size_t column_index) = 0; + virtual std::shared_ptr Finish() = 0; +}; + +template +struct BuilderTypeTrait; diff --git a/src/column/ColumnFactory.h b/src/column/ColumnFactory.h new file mode 100644 index 0000000..bdb8926 --- /dev/null +++ b/src/column/ColumnFactory.h @@ -0,0 +1,34 @@ +#pragma once + +#include "column/ColumnBuilder.h" +#include "types/ColumnType.h" +#include "utils/Macro.h" + +#include + +class ColumnFactory { +public: + static std::shared_ptr MakeColumn(ColumnType type) { + switch (type) { +#define HANDLE_TYPE(ENUM_VAL, STR_VAL, CLASS_TYPE) \ + case ColumnType::ENUM_VAL: return std::make_shared(); + + FOR_EACH_COLUMN_TYPE(HANDLE_TYPE) +#undef HANDLE_TYPE + default: + THROW_RUNTIME_ERROR("Unknown column type"); + } + } + + static std::shared_ptr MakeColumnBuilder(ColumnType type) { + switch (type) { +#define HANDLE_TYPE(ENUM_VAL, STR_VAL, CLASS_TYPE) \ + case ColumnType::ENUM_VAL: return std::make_shared(); + + FOR_EACH_COLUMN_TYPE(HANDLE_TYPE) +#undef HANDLE_TYPE + default: + THROW_RUNTIME_ERROR("Unknown column type"); + } + } +}; diff --git a/src/column/ColumnView.h b/src/column/ColumnView.h new file mode 100644 index 0000000..20d1ed7 --- /dev/null +++ b/src/column/ColumnView.h @@ -0,0 +1,28 @@ +#pragma once + +#include + +template +struct FlatColumnView { + const ContainerType* container; + + explicit FlatColumnView(const ContainerType* c) : container(c) {} + + inline ReturnType operator[](size_t i) const { + return (*container)[i]; + } +}; + +template +struct ConstColumnView { + ReturnType value; + + explicit ConstColumnView(ReturnType val) : value(std::move(val)) {} + + inline ReturnType operator[](size_t) const { + return value; + } +}; + +template +using ViewVariant = std::variant, ConstColumnView>; \ No newline at end of file diff --git a/src/column/NumericColumn.h b/src/column/NumericColumn.h new file mode 100644 index 0000000..5a00652 --- /dev/null +++ b/src/column/NumericColumn.h @@ -0,0 +1,198 @@ +#pragma once + +#include "column/Column.h" +#include "column/ColumnBuilder.h" +#include "utils/Assert.h" + +#include +#include +#include +#include +#include + +using Int128 = __int128_t; + +template +struct ColumnTypeTraits; + +template <> +struct ColumnTypeTraits { + static constexpr ColumnType kType = ColumnType::INT16; + static int16_t FromString(std::string_view value) { + int16_t parsed = 0; + const auto [ptr, ec] = std::from_chars(value.data(), value.data() + value.size(), parsed); + if (ec != std::errc() || ptr != value.data() + value.size()) { + THROW_RUNTIME_ERROR("Invalid INT16 value: " + std::string(value)); + } + return parsed; + } +}; + +template <> +struct ColumnTypeTraits { + static constexpr ColumnType kType = ColumnType::INT32; + static int32_t FromString(std::string_view value) { + int32_t parsed = 0; + const auto [ptr, ec] = std::from_chars(value.data(), value.data() + value.size(), parsed); + if (ec != std::errc() || ptr != value.data() + value.size()) { + THROW_RUNTIME_ERROR("Invalid INT32 value: " + std::string(value)); + } + return parsed; + } +}; + +template <> +struct ColumnTypeTraits { + static constexpr ColumnType kType = ColumnType::INT64; + static int64_t FromString(std::string_view value) { + int64_t parsed = 0; + const auto [ptr, ec] = std::from_chars(value.data(), value.data() + value.size(), parsed); + if (ec != std::errc() || ptr != value.data() + value.size()) { + THROW_RUNTIME_ERROR("Invalid INT64 value: " + std::string(value)); + } + return parsed; + } +}; + +template <> +struct ColumnTypeTraits { + static constexpr ColumnType kType = ColumnType::INT128; + static Int128 FromString(std::string_view value) { + Int128 result = 0; + bool negative = false; + std::string_view value_view = value; + if (value[0] == '-') { + negative = true; + value_view = value_view.substr(1); + } + for (char c : value_view) { + if (c < '0' || c > '9') [[unlikely]] { + THROW_RUNTIME_ERROR("Invalid integer value: " + std::string(value)); + } + result = result * 10 + (c - '0'); + } + if (negative) { + result = -result; + } + return result; + } +}; + +template <> +struct ColumnTypeTraits { + static constexpr ColumnType kType = ColumnType::FLOAT; + static float FromString(std::string_view value) { + return std::stof(std::string(value)); + } +}; + +template <> +struct ColumnTypeTraits { + static constexpr ColumnType kType = ColumnType::DOUBLE; + static double FromString(std::string_view value) { + return std::stod(std::string(value)); + } +}; + +template <> +struct ColumnTypeTraits { + static constexpr ColumnType kType = ColumnType::LONGDOUBLE; + static long double FromString(std::string_view value) { + return std::stold(std::string(value)); + } +}; + +template +class NumericColumn : public Column { +public: + using ValueType = T; + using ContainerType = std::vector; + + NumericColumn() = default; + NumericColumn(ContainerType&& data) noexcept : data_(std::move(data)) { + } + + ~NumericColumn() override = default; + + ColumnType GetType() const override { + return ColumnTypeTraits::kType; + } + + size_t Size() const override { + return data_.size(); + } + + const void* GetRawData() const override { + return &data_; + } + + void Clear() override { + data_.clear(); + } + + const ContainerType& GetData() const { + return data_; + } + + void ReadFromBuffer(const std::vector& buffer) { + if (buffer.size() % sizeof(T) != 0) { + THROW_RUNTIME_ERROR("Raw buffer size is not aligned with type size"); + } + size_t n = buffer.size() / sizeof(T); + data_.resize(n); + std::memcpy(data_.data(), buffer.data(), buffer.size()); + } + +private: + ContainerType data_; +}; + +template +class NumericColumnBuilder : public ColumnBuilder { +public: + NumericColumnBuilder() = default; + + void AddValue(T value) { + data_.push_back(value); + } + + void AddBatch(const VectorOfStrings2D& batch, size_t j) override { + size_t h = batch.Height(); + if (data_.size() < h) { + data_.reserve(data_.size() + h); + } + for (size_t i = 0; i < h; ++i) { + data_.push_back(ColumnTypeTraits::FromString(batch.GetString2D(i, j))); + } + } + + std::shared_ptr Finish() override { + auto result = std::make_shared>(std::move(data_)); + data_.clear(); + return result; + } + +private: + std::vector data_; +}; + +using Int16Column = NumericColumn; +using Int32Column = NumericColumn; +using Int64Column = NumericColumn; +using Int128Column = NumericColumn; +using FloatColumn = NumericColumn; +using DoubleColumn = NumericColumn; +using LongDoubleColumn = NumericColumn; + +using Int16ColumnBuilder = NumericColumnBuilder; +using Int32ColumnBuilder = NumericColumnBuilder; +using Int64ColumnBuilder = NumericColumnBuilder; +using Int128ColumnBuilder = NumericColumnBuilder; +using FloatColumnBuilder = NumericColumnBuilder; +using DoubleColumnBuilder = NumericColumnBuilder; +using LongDoubleColumnBuilder = NumericColumnBuilder; + +template +struct BuilderTypeTrait> { + using Type = NumericColumnBuilder; +}; diff --git a/src/Schema.h b/src/column/Schema.h similarity index 85% rename from src/Schema.h rename to src/column/Schema.h index 70b67fb..b43843d 100644 --- a/src/Schema.h +++ b/src/column/Schema.h @@ -5,9 +5,9 @@ #ifndef COLUMNAR_ENGINE_SCHEMA_H #define COLUMNAR_ENGINE_SCHEMA_H -#include "ColumnarReader.h" -#include "Types.h" -#include "macro.h" +#include "io/ColumnarReader.h" +#include "types/ColumnType.h" +#include "utils/Macro.h" #include #include @@ -72,17 +72,6 @@ class Schema { return Schema(std::move(fields)); } - static Schema FromColumnarFile(const std::string& columnar_file_path) { - ColumnarReader reader(columnar_file_path); - const std::vector metadata = reader.GetMetadata(); - - std::vector fields; - for (const ColumnMetadata& meta : metadata) { - fields.push_back({meta.name, meta.type}); - } - return Schema(std::move(fields)); - } - private: std::vector fields_; diff --git a/src/column/StringColumn.h b/src/column/StringColumn.h new file mode 100644 index 0000000..a037fe6 --- /dev/null +++ b/src/column/StringColumn.h @@ -0,0 +1,84 @@ +#pragma once + +#include "column/Column.h" +#include "column/ColumnBuilder.h" +#include "utils/VectorOfStrings.h" +#include "utils/Macro.h" + +#include +#include + +class StringColumn : public Column { +public: + using ValueType = std::string; + using ContainerType = VectorOfStrings; + + StringColumn() = default; + StringColumn(ContainerType&& data) noexcept : data_(std::move(data)) {} + + ColumnType GetType() const override { + return ColumnType::STRING; + } + + size_t Size() const override { + return data_.Size(); + } + + const void* GetRawData() const override { + return &data_; + } + + void Clear() override { + data_.Clear(); + } + + const ContainerType& GetData() const { + return data_; + } + + void ReadFromBuffer(const std::vector& buffer) { + Clear(); + size_t offset = 0; + while (offset < buffer.size()) { + uint32_t length; + std::memcpy(&length, buffer.data() + offset, sizeof(length)); + offset += sizeof(length); + std::string_view str(buffer.data() + offset, length); + data_.PushBack(str); + offset += length; + } + } + +private: + ContainerType data_; +}; + +class StringColumnBuilder : public ColumnBuilder { +public: + StringColumnBuilder() = default; + + void AddValue(std::string_view value) { + data_.PushBack(value); + } + + void AddBatch(const VectorOfStrings2D& batch, size_t j) override { + size_t h = batch.Height(); + for (size_t i = 0; i < h; ++i) { + data_.PushBack(batch.GetString2D(i, j)); + } + } + + std::shared_ptr Finish() override { + auto result = std::make_shared(std::move(data_)); + data_.Clear(); + return result; + } + +private: + VectorOfStrings data_; +}; + +template <> +struct BuilderTypeTrait { + using Type = StringColumnBuilder; +}; diff --git a/src/column/TemporalColumn.h b/src/column/TemporalColumn.h new file mode 100644 index 0000000..e94b3ac --- /dev/null +++ b/src/column/TemporalColumn.h @@ -0,0 +1,107 @@ +#pragma once + +#include "column/Column.h" +#include "column/ColumnBuilder.h" +#include "types/Date.h" +#include "types/Timestamp.h" + +#include +#include +#include + +template +class TemporalColumn : public Column { +public: + using ValueType = T; + using ContainerType = std::vector; + + TemporalColumn() = default; + TemporalColumn(ContainerType&& data) noexcept : data_(std::move(data)) {} + + ColumnType GetType() const override { + return CType; + } + + size_t Size() const override { + return data_.size(); + } + + const void* GetRawData() const override { + return &data_; + } + + void Clear() override { + data_.clear(); + } + + const ContainerType& GetData() const { + return data_; + } + + void ReadFromBuffer(const std::vector& buffer) { + if (buffer.size() % sizeof(T) != 0) { + THROW_RUNTIME_ERROR("Raw buffer size is not aligned with type size"); + } + size_t n = buffer.size() / sizeof(T); + data_.resize(n); + std::memcpy(data_.data(), buffer.data(), buffer.size()); + } + +private: + ContainerType data_; +}; + +class DateColumn : public TemporalColumn { +public: + using TemporalColumn::TemporalColumn; +}; + +class TimestampColumn : public TemporalColumn { +public: + using TemporalColumn::TemporalColumn; +}; + +template +class TemporalColumnBuilder : public ColumnBuilder { + using ValueType = typename ColumnTypeT::ValueType; + +public: + TemporalColumnBuilder() = default; + + void AddValue(ValueType value) { + data_.push_back(value); + } + + void AddBatch(const VectorOfStrings2D& batch, size_t j) override { + size_t h = batch.Height(); + if (data_.size() < h) { + data_.reserve(data_.size() + h); + } + for (size_t i = 0; i < h; ++i) { + std::string_view cur = batch.GetString2D(i, j); + data_.push_back(ParseStruct::Parse(cur)); + } + } + + std::shared_ptr Finish() override { + auto result = std::make_shared(std::move(data_)); + data_.clear(); + return result; + } + +private: + std::vector data_; +}; + +using DateColumnBuilder = TemporalColumnBuilder; +using TimestampColumnBuilder = TemporalColumnBuilder; + +template <> +struct BuilderTypeTrait { + using Type = DateColumnBuilder; +}; + +template <> +struct BuilderTypeTrait { + using Type = TimestampColumnBuilder; +}; diff --git a/src/execution/ExecutionApi.h b/src/execution/ExecutionApi.h new file mode 100644 index 0000000..9a86fb6 --- /dev/null +++ b/src/execution/ExecutionApi.h @@ -0,0 +1,160 @@ +#pragma once + +#include "execution/expressions/AggregationExpressions.h" +#include "execution/expressions/FilterExpressions.h" +#include "execution/ExecutionLogic.h" +#include "execution/ExecutionHelper.h" +#include "execution/PipelineExecutor.h" + +#include +#include + +class DataResult { +public: + DataResult(std::vector>&& batches, Schema schema) + : batches_(std::move(batches)), schema_(std::move(schema)) { + } + + const std::vector>& GetBatches() const { + return batches_; + } + + const Schema& GetSchema() const { + return schema_; + } + + void Display(bool display_names = false) const { + const std::vector& fields = schema_.GetFields(); + if (display_names) { + if (!fields.empty()) { + for (const Field& field : fields) { + std::cout << field.name << ","; + } + std::cout << "\n"; + } + } + + for (const std::unique_ptr& batch : batches_) { + if (batch->columns.empty() || batch->num_rows == 0) { + continue; + } + size_t rows = batch->num_rows; + for (size_t i = 0; i < rows; ++i) { + size_t ind = batch->selection_vector ? (*batch->selection_vector)[i] : i; + for (const std::shared_ptr& column : batch->columns) { + std::cout << ExecutionHelper::FormatValue(*column, ind) << ","; + } + std::cout << "\n"; + } + } + } + + std::shared_ptr GetResult(const std::string& column_name) const { + const std::vector& fields = schema_.GetFields(); + for (size_t i = 0; i < fields.size(); ++i) { + if (fields[i].name == column_name) { + if (!batches_[0]->columns.empty()) { + return batches_[0]->columns[i]; + } + break; + } + } + THROW_RUNTIME_ERROR("No charoncik bebe"); + } + +private: + std::vector> batches_; + Schema schema_; +}; + +class DataFrame { +public: + explicit DataFrame(std::shared_ptr logical_plan) + : logical_plan_(std::move(logical_plan)) { + } + + static DataFrame Select(std::string columnar_file_path, std::vector column_names) { + auto table_meta = std::make_shared(columnar_file_path); + + auto scan_node = std::make_shared(std::move(columnar_file_path), + std::move(column_names), std::move(table_meta)); + return DataFrame(scan_node); + } + + DataFrame Aggregate(std::vector group_by_columns, + std::vector> aggregate_expressions) { + auto aggregate_node = std::make_shared( + logical_plan_, std::move(group_by_columns), std::move(aggregate_expressions)); + + return DataFrame(aggregate_node); + } + + DataFrame Filter(std::shared_ptr filter_expression) { + auto filter_node = + std::make_shared(logical_plan_, std::move(filter_expression)); + return DataFrame(filter_node); + } + + DataFrame OrderBy(std::vector> order_by_columns, + std::optional limit = std::nullopt, + std::optional offset = std::nullopt) { + auto order_by_node = std::make_shared( + logical_plan_, std::move(order_by_columns), std::move(limit), std::move(offset)); + return DataFrame(order_by_node); + } + + DataFrame Limit(size_t limit) { + auto limit_node = std::make_shared(logical_plan_, limit); + return DataFrame(limit_node); + } + + DataFrame Project(std::vector> scalar_expressions) { + auto scalar_node = + std::make_shared(logical_plan_, std::move(scalar_expressions)); + return DataFrame(scalar_node); + } + + DataFrame Drop(std::vector columns_to_drop) { + auto drop_node = std::make_shared(logical_plan_, std::move(columns_to_drop)); + return DataFrame(drop_node); + } + + DataFrame Reorder(std::vector desired_order) { + auto node = std::make_shared(logical_plan_, std::move(desired_order)); + return DataFrame(node); + } + + DataResult Collect(size_t threads = 0) const { + if (threads == 0) { +#ifdef ENABLE_MULTITHREADING + threads = std::thread::hardware_concurrency(); + if (threads == 0) { + threads = 2; + } +#else + threads = 1; +#endif + } + + PipelineBuildContext ctx; + ctx.num_threads = threads; + + auto outer_pipe = std::make_unique(); + auto result_sink = std::make_shared(); + outer_pipe->sink = result_sink; + ctx.current_pipeline = outer_pipe.get(); + + logical_plan_->BuildPipelines(ctx); + ctx.completed_pipelines.push_back(std::move(outer_pipe)); + + ThreadPool thread_pool(threads); + for (auto& pipeline : ctx.completed_pipelines) { + pipeline->Execute(thread_pool, threads); + } + + return DataResult{result_sink->TakeBatches(), std::move(ctx.schema)}; + } + +private: + std::shared_ptr logical_plan_; +}; diff --git a/src/execution/ExecutionHelper.h b/src/execution/ExecutionHelper.h new file mode 100644 index 0000000..0ab31a5 --- /dev/null +++ b/src/execution/ExecutionHelper.h @@ -0,0 +1,163 @@ +#pragma once + +#include "execution/OperatorsBase.h" +#include "column/Column.h" +#include "column/NumericColumn.h" +#include "column/StringColumn.h" +#include "column/CharColumn.h" +#include "column/TemporalColumn.h" +#include "utils/Macro.h" + +#include +#include +#include +#include + +class ExecutionHelper { +public: + static std::string FormatValue(const Column& col, size_t index) { + ColumnType type = col.GetType(); + if (type == ColumnType::STRING) { + return std::string(static_cast(col).GetData()[index]); + } else if (type == ColumnType::CHAR) { + return std::string(1, static_cast(col).GetData()[index]); + } else if (type == ColumnType::DATE) { + return Date::Format(static_cast(col).GetData()[index]); + } else if (type == ColumnType::TIMESTAMP) { + return Timestamp::Format(static_cast(col).GetData()[index]); + } else { +#define HANDLE_TYPE(ENUM_VAL, STR_VAL, CLASS_TYPE) \ + case ColumnType::ENUM_VAL: { \ + auto val = static_cast(col).GetData()[index]; \ + return std::to_string(val); \ + } + + switch (type) { + HANDLE_TYPE(INT16, "INT16", Int16Column) + HANDLE_TYPE(INT32, "INT32", Int32Column) + HANDLE_TYPE(INT64, "INT64", Int64Column) + HANDLE_TYPE(FLOAT, "FLOAT", FloatColumn) + HANDLE_TYPE(DOUBLE, "DOUBLE", DoubleColumn) + HANDLE_TYPE(LONGDOUBLE, "LONGDOUBLE", LongDoubleColumn) + case ColumnType::INT128: { + Int128 val = static_cast(col).GetData()[index]; + if (val == 0) { + return "0"; + } + std::string res; + bool neg = val < 0; + if (neg) { + val = -val; + } + while (val > 0) { + res += std::to_string(static_cast(val % 10)); + val /= 10; + } + if (neg) { + res += "-"; + } + std::reverse(res.begin(), res.end()); + return res; + } + default: return ""; + } +#undef HANDLE_TYPE + } + } + static void HashKeys(const Column& col, std::vector& keys, + const std::vector* selection = nullptr) { + ColumnType type = col.GetType(); + + if (type == ColumnType::STRING) { + auto* str_col = static_cast(&col); + const auto& data = str_col->GetData(); + size_t n = selection ? selection->size() : data.Size(); + if (keys.capacity() < n) { + keys.reserve(n); + } + for (size_t i = 0; i < n; ++i) { + size_t idx = selection ? (*selection)[i] : i; + std::string_view val = data[idx]; + size_t len = val.size(); + keys[i].append(reinterpret_cast(&len), sizeof(len)); + keys[i].append(val.data(), val.size()); + } + } else if (type == ColumnType::CHAR) { + auto* char_col = static_cast(&col); + const auto& data = char_col->GetData(); + size_t n = selection ? selection->size() : data.size(); + if (keys.capacity() < n) { + keys.reserve(n); + } + for (size_t i = 0; i < n; ++i) { + size_t idx = selection ? (*selection)[i] : i; + keys[i].push_back(data[idx]); + } + } else { + size_t elem_size = 0; + const void* raw_data = col.GetRawData(); + +#define HANDLE_TYPE(ENUM_VAL, STR_VAL, CLASS_TYPE) \ + case ColumnType::ENUM_VAL: \ + elem_size = sizeof(CLASS_TYPE::ValueType); \ + { \ + auto* vec = static_cast(raw_data); \ + size_t n = selection ? selection->size() : vec->size(); \ + if (keys.capacity() < n) \ + keys.reserve(n); \ + for (size_t i = 0; i < n; ++i) { \ + size_t idx = selection ? (*selection)[i] : i; \ + keys[i].append(reinterpret_cast(&(*vec)[idx]), elem_size); \ + } \ + } \ + break; + + switch (type) { + FOR_NUMERIC_COLUMN_TYPE(HANDLE_TYPE) + default: break; + } +#undef HANDLE_TYPE + } + } + + static void CopySelection(ColumnBuilder& dest, const Column& src, + const std::vector* indices = nullptr) { + ColumnType type = src.GetType(); + + if (type == ColumnType::STRING) { + auto& d = static_cast(dest); + const auto& s = static_cast(src).GetData(); + if (!indices) { + for (size_t i = 0; i < s.Size(); ++i) { + d.AddValue(s[i]); + } + } else { + for (size_t idx : *indices) { + d.AddValue(s[idx]); + } + } + } else { +#define HANDLE_TYPE(ENUM_VAL, STR_VAL, CLASS_TYPE) \ + case ColumnType::ENUM_VAL: { \ + auto& d = static_cast(dest); \ + auto* vec = static_cast(src.GetRawData()); \ + if (!indices) { \ + for (size_t i = 0; i < vec->size(); ++i) { \ + d.AddValue((*vec)[i]); \ + } \ + } else { \ + for (size_t idx : *indices) { \ + d.AddValue((*vec)[idx]); \ + } \ + } \ + break; \ + } + + switch (type) { + FOR_NUMERIC_COLUMN_TYPE(HANDLE_TYPE) + default: break; + } +#undef HANDLE_TYPE + } + } +}; diff --git a/src/execution/ExecutionLogic.cpp b/src/execution/ExecutionLogic.cpp new file mode 100644 index 0000000..615c1d7 --- /dev/null +++ b/src/execution/ExecutionLogic.cpp @@ -0,0 +1,307 @@ +#include "execution/ExecutionLogic.h" + +// ========================= ScanNode ========================= + +void ScanNode::BuildPipelines(PipelineBuildContext& ctx, + std::optional> required_columns) const { + const auto& full_columns = table_meta->GetColumns(); + + std::vector user_cols; + if (column_names.size() == 1 && column_names[0] == "*") { + for (const auto& field : full_columns) { + user_cols.push_back(field.name); + } + } else { + user_cols = column_names; + } + + std::vector final_columns = user_cols; + if (required_columns.has_value()) { + for (const std::string& req_col : required_columns.value()) { + if (std::find(final_columns.begin(), final_columns.end(), req_col) == + final_columns.end()) { + final_columns.push_back(req_col); + } + } + } + + auto scan_op = std::make_shared(table_path, final_columns, table_meta); + ctx.current_pipeline->source = scan_op; + + Schema output_schema; + for (const std::string& name : final_columns) { + bool found = false; + for (const auto& field : full_columns) { + if (field.name == name) { + output_schema.AddField({field.name, field.type}); + found = true; + break; + } + } + if (!found) [[unlikely]] { + THROW_RUNTIME_ERROR("Column " + name + " not found in table schema"); + } + } + + ctx.schema = std::move(output_schema); +} + +// ========================= AggregateNode ========================= + +void AggregateNode::BuildPipelines(PipelineBuildContext& ctx, + std::optional>) const { + Pipeline* outer_pipeline = ctx.current_pipeline; + Pipeline* inner_pipeline = new Pipeline(); // TODO: this is bad, should not use raw pointers + ctx.current_pipeline = inner_pipeline; + + if (!group_by_columns.empty()) { + // === GROUP BY path === + std::vector needed_columns = group_by_columns; + for (const std::shared_ptr& expr : aggregate_expressions) { + expr->CollectRequiredColumns(needed_columns); + } + std::sort(needed_columns.begin(), needed_columns.end()); + needed_columns.erase(std::unique(needed_columns.begin(), needed_columns.end()), + needed_columns.end()); + + child->BuildPipelines(ctx, needed_columns); + + std::vector group_col_indices; + std::vector key_types; + Schema output_schema; + + for (const std::string& col_name : group_by_columns) { + size_t ind = ctx.schema.GetColumnIndexByName(col_name); + group_col_indices.push_back(ind); + + ColumnType col_type = ctx.schema.GetColumnTypeByName(col_name); + output_schema.AddColumn(col_name, col_type); + key_types.push_back(col_type); + } + + std::vector> agg_funcs; + for (const std::shared_ptr& expr : aggregate_expressions) { + agg_funcs.push_back( + expr->CreateGroupedAggregationFunction(ctx.schema, output_schema, ctx.num_threads)); + } + + auto breaker = std::make_shared( + std::move(group_col_indices), std::move(key_types), std::move(agg_funcs), + ctx.num_threads); + + inner_pipeline->sink = breaker; + ctx.completed_pipelines.push_back(std::unique_ptr(inner_pipeline)); + outer_pipeline->source = breaker; + ctx.current_pipeline = outer_pipeline; + ctx.schema = std::move(output_schema); + } else { + // === Global aggregation path === + std::vector needed_columns; + for (const std::shared_ptr& expr : aggregate_expressions) { + expr->CollectRequiredColumns(needed_columns); + } + std::sort(needed_columns.begin(), needed_columns.end()); + needed_columns.erase(std::unique(needed_columns.begin(), needed_columns.end()), + needed_columns.end()); + + child->BuildPipelines(ctx, needed_columns); + + std::vector> agg_functions; + Schema output_schema; + for (const std::shared_ptr& expr : aggregate_expressions) { + agg_functions.push_back( + expr->CreateGlobalAggregationFunction(ctx.schema, output_schema, ctx.num_threads)); + } + + auto breaker = std::make_shared(std::move(agg_functions)); + + inner_pipeline->sink = breaker; + ctx.completed_pipelines.push_back(std::unique_ptr(inner_pipeline)); + outer_pipeline->source = breaker; + ctx.current_pipeline = outer_pipeline; + ctx.schema = std::move(output_schema); + } +} + +// ========================= FilterNode ========================= + +void FilterNode::BuildPipelines(PipelineBuildContext& ctx, + std::optional> required_columns) const { + std::vector needed_columns; + filter_expression->CollectRequiredColumns(needed_columns); + if (required_columns.has_value()) { + needed_columns.insert(needed_columns.end(), required_columns->begin(), + required_columns->end()); + } + + std::sort(needed_columns.begin(), needed_columns.end()); + needed_columns.erase(std::unique(needed_columns.begin(), needed_columns.end()), + needed_columns.end()); + + child->BuildPipelines(ctx, needed_columns); + + std::unique_ptr filter_function = + filter_expression->CreateFilterFunction(ctx.schema); + auto filter_op = std::make_shared(std::move(filter_function)); + ctx.current_pipeline->transforms.push_back(filter_op); +} + +// ========================= OrderByNode ========================= + +void OrderByNode::BuildPipelines(PipelineBuildContext& ctx, + std::optional> required_columns) const { + Pipeline* outer_pipeline = ctx.current_pipeline; + Pipeline* inner_pipeline = new Pipeline(); // TODO: this is bad, should not use raw pointers + ctx.current_pipeline = inner_pipeline; + + std::vector needed_columns; + if (required_columns.has_value()) { + needed_columns = required_columns.value(); + } + for (const auto& [col_name, is_desc] : order_by_columns) { + needed_columns.push_back(col_name); + } + std::sort(needed_columns.begin(), needed_columns.end()); + needed_columns.erase(std::unique(needed_columns.begin(), needed_columns.end()), + needed_columns.end()); + + child->BuildPipelines(ctx, needed_columns); + + std::vector> sort_columns; + for (const auto& [col_name, is_desc] : order_by_columns) { + size_t sort_col_idx = ctx.schema.GetColumnIndexByName(col_name); + sort_columns.push_back({sort_col_idx, is_desc}); + } + + std::vector accum_types; + for (const auto& field : ctx.schema.GetFields()) { + accum_types.push_back(field.type); + } + + auto breaker = std::make_shared( + std::move(sort_columns), std::move(accum_types), std::move(limit), std::move(offset), + ctx.num_threads); + + inner_pipeline->sink = breaker; + ctx.completed_pipelines.push_back(std::unique_ptr(inner_pipeline)); + outer_pipeline->source = breaker; + ctx.current_pipeline = outer_pipeline; +} + +// ========================= LimitNode ========================= + +void LimitNode::BuildPipelines(PipelineBuildContext& ctx, + std::optional> required_columns) const { + child->BuildPipelines(ctx, required_columns); + auto limit_op = std::make_shared(limit); + ctx.current_pipeline->transforms.push_back(limit_op); +} + +// ========================= ScalarNode ========================= + +void ScalarNode::BuildPipelines(PipelineBuildContext& ctx, + std::optional> required_columns) const { + std::vector needed_columns; + if (required_columns.has_value()) { + for (const auto& req_col : required_columns.value()) { + bool is_generated_here = false; + for (const auto& expr : scalar_expressions) { + if (expr->GetOutputName() == req_col) { + is_generated_here = true; + break; + } + } + if (!is_generated_here) { + needed_columns.push_back(req_col); + } + } + } + for (const auto& expr : scalar_expressions) { + expr->CollectRequiredColumns(needed_columns); + } + + std::sort(needed_columns.begin(), needed_columns.end()); + needed_columns.erase(std::unique(needed_columns.begin(), needed_columns.end()), + needed_columns.end()); + + child->BuildPipelines(ctx, needed_columns); + + std::vector> scalar_functions; + for (const auto& expr : scalar_expressions) { + scalar_functions.push_back(expr->CreateScalarFunction(ctx.schema)); + } + + auto scalar_op = std::make_shared(std::move(scalar_functions)); + ctx.current_pipeline->transforms.push_back(scalar_op); +} + +// ========================= DropNode ========================= + +void DropNode::BuildPipelines(PipelineBuildContext& ctx, + std::optional> required_columns) const { + child->BuildPipelines(ctx, required_columns); + + const auto& child_fields = ctx.schema.GetFields(); + + for (const std::string& drop_col : columns_to_drop) { + bool found = false; + for (const auto& field : child_fields) { + if (field.name == drop_col) { + found = true; + break; + } + } + if (!found) { + THROW_RUNTIME_ERROR("Cannot drop column '" + drop_col + "': not found in dataframe"); + } + } + + std::vector cols_to_keep_indices; + Schema output_schema; + for (size_t i = 0; i < child_fields.size(); ++i) { + const auto& field = child_fields[i]; + if (std::find(columns_to_drop.begin(), columns_to_drop.end(), field.name) == + columns_to_drop.end()) { + cols_to_keep_indices.push_back(i); + output_schema.AddField({field.name, field.type}); + } + } + + auto drop_op = std::make_shared(std::move(cols_to_keep_indices)); + ctx.current_pipeline->transforms.push_back(drop_op); + ctx.schema = std::move(output_schema); +} + +// ========================= ReorderNode ========================= + +void ReorderNode::BuildPipelines(PipelineBuildContext& ctx, + std::optional> required_columns) const { + child->BuildPipelines(ctx, required_columns); + + const auto& child_fields = ctx.schema.GetFields(); + + std::vector new_indices; + Schema output_schema; + + for (const std::string& reorder_col : desired_order) { + bool found = false; + for (size_t i = 0; i < child_fields.size(); ++i) { + const auto& field = child_fields[i]; + if (field.name == reorder_col) { + found = true; + new_indices.push_back(i); + output_schema.AddField({field.name, field.type}); + break; + } + } + + if (!found) { + THROW_RUNTIME_ERROR("Cannot reorder column '" + reorder_col + + "': not found in dataframe"); + } + } + + auto reorder_op = std::make_shared(std::move(new_indices)); + ctx.current_pipeline->transforms.push_back(reorder_op); + ctx.schema = std::move(output_schema); +} diff --git a/src/execution/ExecutionLogic.h b/src/execution/ExecutionLogic.h new file mode 100644 index 0000000..b59ea67 --- /dev/null +++ b/src/execution/ExecutionLogic.h @@ -0,0 +1,131 @@ +#pragma once + +#include "execution/OperatorsBase.h" +#include "execution/PipelineExecutor.h" +#include "execution/expressions/AggregationExpressions.h" +#include "execution/expressions/FilterExpressions.h" +#include "execution/expressions/ScalarExpressions.h" + +#include "column/Schema.h" + +#include +#include +#include +#include +#include + +struct PlanNode { + virtual ~PlanNode() = default; + + virtual void BuildPipelines( + PipelineBuildContext& ctx, + std::optional> required_columns = std::nullopt) const = 0; +}; + +struct ScanNode : public PlanNode { + std::string table_path; + std::vector column_names; + std::shared_ptr table_meta; + + ScanNode(std::string table_p, std::vector column_n, + std::shared_ptr table_m) + : table_path(std::move(table_p)), + column_names(std::move(column_n)), + table_meta(std::move(table_m)) { + } + + void BuildPipelines(PipelineBuildContext& ctx, + std::optional> required_columns) const override; +}; + +struct AggregateNode : public PlanNode { + std::shared_ptr child; + std::vector group_by_columns; + std::vector> aggregate_expressions; + + AggregateNode(std::shared_ptr c, std::vector gb_cols, + std::vector> agg_exprs) + : child(std::move(c)), + group_by_columns(std::move(gb_cols)), + aggregate_expressions(std::move(agg_exprs)) { + } + + void BuildPipelines(PipelineBuildContext& ctx, + std::optional> required_columns) const override; +}; + +struct FilterNode : public PlanNode { + std::shared_ptr child; + std::shared_ptr filter_expression; + + FilterNode(std::shared_ptr c, std::shared_ptr filter_expr) + : child(std::move(c)), filter_expression(std::move(filter_expr)) { + } + + void BuildPipelines(PipelineBuildContext& ctx, + std::optional> required_columns) const override; +}; + +struct OrderByNode : public PlanNode { + std::shared_ptr child; + std::vector> order_by_columns; + std::optional limit; + std::optional offset; + + OrderByNode(std::shared_ptr c, + std::vector> order_by_cols, std::optional lim, + std::optional off) + : child(std::move(c)), order_by_columns(std::move(order_by_cols)), limit(lim), offset(off) { + } + + void BuildPipelines(PipelineBuildContext& ctx, + std::optional> required_columns) const override; +}; + +struct LimitNode : public PlanNode { + std::shared_ptr child; + size_t limit; + + LimitNode(std::shared_ptr c, size_t lim) : child(std::move(c)), limit(lim) { + } + + void BuildPipelines(PipelineBuildContext& ctx, + std::optional> required_columns) const override; +}; + +struct ScalarNode : public PlanNode { + std::shared_ptr child; + std::vector> scalar_expressions; + + ScalarNode(std::shared_ptr c, + std::vector> scalar_exprs) + : child(std::move(c)), scalar_expressions(std::move(scalar_exprs)) { + } + + void BuildPipelines(PipelineBuildContext& ctx, + std::optional> required_columns) const override; +}; + +struct DropNode : public PlanNode { + std::shared_ptr child; + std::vector columns_to_drop; + + DropNode(std::shared_ptr c, std::vector cols_to_drop) + : child(std::move(c)), columns_to_drop(std::move(cols_to_drop)) { + } + + void BuildPipelines(PipelineBuildContext& ctx, + std::optional> required_columns) const override; +}; + +struct ReorderNode : public PlanNode { + std::shared_ptr child; + std::vector desired_order; + + ReorderNode(std::shared_ptr c, std::vector desired_ord) + : child(std::move(c)), desired_order(std::move(desired_ord)) { + } + + void BuildPipelines(PipelineBuildContext& ctx, + std::optional> required_columns) const override; +}; diff --git a/src/execution/OperatorsBase.cpp b/src/execution/OperatorsBase.cpp new file mode 100644 index 0000000..d33b8e7 --- /dev/null +++ b/src/execution/OperatorsBase.cpp @@ -0,0 +1,572 @@ +#include "execution/expressions/AggregationFunctions.h" +#include "execution/expressions/FilterFunctions.h" +#include "execution/OperatorsBase.h" +#include "execution/expressions/ScalarFunctions.h" +#include "execution/ExecutionHelper.h" +#include "column/ColumnFactory.h" + +#include + +// ========================= ScanOperator ========================= + +ScanOperator::ScanOperator(const std::string& table_path, + const std::vector& column_names, + std::shared_ptr metadata) + : reader_(table_path, metadata), metadata_(metadata) { + const auto& full_columns = metadata_->GetColumns(); + + for (const std::string& name : column_names) { + bool found = false; + for (size_t i = 0; i < full_columns.size(); ++i) { + if (full_columns[i].name == name) { + column_indices_.push_back(i); + found = true; + break; + } + } + if (!found) { + THROW_RUNTIME_ERROR("Column " + name + " not found in metadata"); + } + } + + if (!full_columns.empty()) { + total_chunks_ = full_columns[0].offsets.size(); + } +} + +std::unique_ptr ScanOperator::GetData() { + if (column_indices_.empty()) { + bool expected = false; + if (finished_empty_scan_.compare_exchange_strong(expected, true)) { + auto record_batch = std::make_unique(); + record_batch->num_rows = metadata_->GetNumRows(); + return record_batch; + } + return nullptr; + } + + size_t cur_chunk = current_chunk_.fetch_add(1); + if (cur_chunk >= total_chunks_) { + return nullptr; + } + + auto record_batch = std::make_unique(); + record_batch->num_rows = 0; + + for (size_t col_idx : column_indices_) { + std::shared_ptr column_data = reader_.GetColumnData(col_idx, cur_chunk); + if (record_batch->num_rows == 0) { + record_batch->num_rows = column_data->Size(); + } + record_batch->columns.push_back(std::move(column_data)); + } + + return record_batch; +} + +// ========================= FilterTransformOperator ========================= + +FilterTransformOperator::FilterTransformOperator(std::shared_ptr filter_function) + : filter_function_(std::move(filter_function)) { +} + +std::unique_ptr FilterTransformOperator::Execute(std::unique_ptr batch) { + std::vector selection_vector = filter_function_->Evaluate(*batch); + if (selection_vector.empty()) { + return nullptr; + } + batch->num_rows = selection_vector.size(); + batch->selection_vector = + std::make_shared>(std::move(selection_vector)); + return batch; +} + +// ========================= ScalarTransformOperator ========================= + +ScalarTransformOperator::ScalarTransformOperator( + std::vector> scalar_functions) + : scalar_functions_(std::move(scalar_functions)) { +} + +std::unique_ptr ScalarTransformOperator::Execute(std::unique_ptr batch) { + for (auto& func : scalar_functions_) { + batch->columns.push_back(func->Evaluate(*batch)); + } + return batch; +} + +// ========================= DropTransformOperator ========================= + +DropTransformOperator::DropTransformOperator(std::vector column_stay_indices) + : column_stay_indices_(std::move(column_stay_indices)) { +} + +std::unique_ptr DropTransformOperator::Execute(std::unique_ptr batch) { + std::vector> new_columns; + new_columns.reserve(column_stay_indices_.size()); + + for (size_t ind : column_stay_indices_) { + new_columns.push_back(std::move(batch->columns[ind])); + } + + batch->columns = std::move(new_columns); + return batch; +} + +// ========================= ReorderTransformOperator ========================= + +ReorderTransformOperator::ReorderTransformOperator(std::vector new_indices) + : new_indices_(std::move(new_indices)) { +} + +std::unique_ptr ReorderTransformOperator::Execute(std::unique_ptr batch) { + auto new_batch = std::make_unique(); + new_batch->num_rows = batch->num_rows; + new_batch->selection_vector = batch->selection_vector; + + new_batch->columns.reserve(new_indices_.size()); + for (size_t ind : new_indices_) { + new_batch->columns.push_back(batch->columns[ind]); + } + + return new_batch; +} + +// ========================= LimitTransformOperator ========================= + +LimitTransformOperator::LimitTransformOperator(size_t limit) : limit_(limit) { +} + +std::unique_ptr LimitTransformOperator::Execute(std::unique_ptr batch) { + size_t old_rows = cur_rows_.fetch_add(batch->num_rows); + + if (old_rows >= limit_) { + done_.store(true, std::memory_order_relaxed); + return nullptr; + } + + size_t remaining = limit_ - old_rows; + + if (batch->num_rows <= remaining) { + if (old_rows + batch->num_rows >= limit_) { + done_.store(true, std::memory_order_relaxed); + } + return batch; + } else { + done_.store(true, std::memory_order_relaxed); + batch->num_rows = remaining; + if (batch->selection_vector) { + auto new_sel_vec = std::make_shared>( + batch->selection_vector->begin(), batch->selection_vector->begin() + remaining); + batch->selection_vector = std::move(new_sel_vec); + } else { + auto sel_vec = std::make_shared>(remaining); + std::iota(sel_vec->begin(), sel_vec->end(), 0); + batch->selection_vector = std::move(sel_vec); + } + return batch; + } +} + +// ========================= AggregationSinkSourceOperator ========================= + +AggregationSinkSourceOperator::AggregationSinkSourceOperator( + std::vector> aggregation_functions) + : aggregation_functions_(std::move(aggregation_functions)) { +} + +void AggregationSinkSourceOperator::Sink(std::unique_ptr batch, size_t thread_id) { + for (auto& aggregation_function : aggregation_functions_) { + aggregation_function->Update(*batch, thread_id); + } +} + +void AggregationSinkSourceOperator::Finalize() && { + result_batch_ = std::make_unique(); + result_batch_->num_rows = 1; + + for (auto& aggregation_function : aggregation_functions_) { + result_batch_->columns.push_back(aggregation_function->Finalize()); + } +} + +std::unique_ptr AggregationSinkSourceOperator::GetData() { + bool expected = false; + if (emitted_.compare_exchange_strong(expected, true)) { + return std::move(result_batch_); + } + return nullptr; +} + +// ========================= GroupBySinkSourceOperator ========================= + +GroupBySinkSourceOperator::GroupBySinkSourceOperator( + std::vector group_by_col_indices, std::vector key_types, + std::vector> agg_funcs, size_t num_threads) + : group_by_col_indices_(std::move(group_by_col_indices)), + key_types_(std::move(key_types)), + agg_funcs_(std::move(agg_funcs)), + thread_states_(num_threads) { +} + +void GroupBySinkSourceOperator::Sink(std::unique_ptr batch, size_t thread_id) { + GroupByThreadState& local = thread_states_[thread_id]; + + if (local.key_builders.empty()) { + for (auto type : key_types_) { + local.key_builders.push_back(ColumnFactory::MakeColumnBuilder(type)); + } + } + + std::vector group_ids; + size_t batch_size = batch->num_rows; + group_ids.reserve(batch_size); + + const std::shared_ptr>& sel_vec = batch->selection_vector; + std::vector composite_keys(batch_size); + for (size_t col_ind : group_by_col_indices_) { + ExecutionHelper::HashKeys(*batch->columns[col_ind], composite_keys, sel_vec.get()); + } + + std::vector new_group_indices; + bool is_filtered = sel_vec != nullptr; + + for (size_t i = 0; i < batch_size; ++i) { + const std::string& key = composite_keys[i]; + auto it = local.hash_table.find(key); + uint32_t gid; + + if (it == local.hash_table.end()) { + gid = local.num_groups++; + local.hash_table[key] = gid; + size_t actual_idx = is_filtered ? (*sel_vec)[i] : i; + new_group_indices.push_back(actual_idx); + } else { + gid = it->second; + } + group_ids.push_back(gid); + } + + if (!new_group_indices.empty()) { + for (size_t k = 0; k < group_by_col_indices_.size(); ++k) { + ExecutionHelper::CopySelection(*local.key_builders[k], + *batch->columns[group_by_col_indices_[k]], + &new_group_indices); + } + } + + for (auto& func : agg_funcs_) { + func->Resize(local.num_groups, thread_id); + func->Update(*batch, group_ids, thread_id); + } +} + +void GroupBySinkSourceOperator::Finalize() && { + GroupByThreadState& global = thread_states_[0]; + + if (global.key_builders.empty()) { + for (auto type : key_types_) { + global.key_builders.push_back(ColumnFactory::MakeColumnBuilder(type)); + } + } + + for (size_t t = 1; t < thread_states_.size(); ++t) { + GroupByThreadState& local = thread_states_[t]; + if (local.num_groups == 0) { + continue; + } + + std::vector> local_key_cols; + for (std::shared_ptr& key_builder : local.key_builders) { + local_key_cols.push_back(key_builder->Finish()); + } + + for (const auto& [key, local_gid] : local.hash_table) { + auto it = global.hash_table.find(key); + uint32_t target_global_gid; + + if (it == global.hash_table.end()) { + target_global_gid = global.num_groups++; + global.hash_table[key] = target_global_gid; + + std::vector single_ind = {local_gid}; + for (size_t k = 0; k < global.key_builders.size(); ++k) { + ExecutionHelper::CopySelection(*global.key_builders[k], *local_key_cols[k], + &single_ind); + } + + for (auto& func : agg_funcs_) { + func->Resize(global.num_groups, 0); + } + } else { + target_global_gid = it->second; + } + + for (auto& func : agg_funcs_) { + func->CombineState(t, local_gid, target_global_gid); + } + } + + local.hash_table.clear(); + } + + result_batch_ = std::make_unique(); + result_batch_->num_rows = global.num_groups; + + for (size_t k = 0; k < global.key_builders.size(); ++k) { + result_batch_->columns.push_back(global.key_builders[k]->Finish()); + } + for (size_t a = 0; a < agg_funcs_.size(); ++a) { + result_batch_->columns.push_back(agg_funcs_[a]->Finalize()); + } +} + +std::unique_ptr GroupBySinkSourceOperator::GetData() { + bool expected = false; + if (emitted_.compare_exchange_strong(expected, true)) { + return std::move(result_batch_); + } + return nullptr; +} + +// ========================= OrderBySinkSourceOperator ========================= + +OrderBySinkSourceOperator::OrderBySinkSourceOperator( + std::vector> sort_columns, std::vector column_types, + std::optional limit, std::optional offset, size_t num_threads) + : sort_columns_(std::move(sort_columns)), + column_types_(column_types), + limit_(limit), + offset_(offset), + thread_states_(num_threads) { + total_limit_ = limit_; + if (total_limit_.has_value()) { + if (offset_.has_value()) { + total_limit_.value() += offset_.value(); + } + } +} + +using CompareFunc = int (*)(const void*, uint32_t, uint32_t); + +struct SortColumnInfo { + const void* raw_data; + CompareFunc cmp_func; + bool is_desc; +}; + +template +int Compare(const void* raw_data, uint32_t a, uint32_t b) { + using Container = ColumnT::ContainerType; + const Container* data = static_cast(raw_data); + if ((*data)[a] < (*data)[b]) { + return -1; + } + if ((*data)[a] > (*data)[b]) { + return 1; + } + return 0; +} + +void OrderBySinkSourceOperator::Sink(std::unique_ptr batch, size_t thread_id) { + OrderByThreadState& local = thread_states_[thread_id]; + + if (local.column_builders.empty()) { + for (auto type : column_types_) { + local.column_builders.push_back(ColumnFactory::MakeColumnBuilder(type)); + } + local.accum_num_rows = 0; + } + + const std::shared_ptr>& sel = batch->selection_vector; + for (size_t c = 0; c < local.column_builders.size(); ++c) { + ExecutionHelper::CopySelection(*local.column_builders[c], *batch->columns[c], sel.get()); + } + local.accum_num_rows += batch->num_rows; + + if (total_limit_.has_value() && local.accum_num_rows > total_limit_.value() * 10) { + TrimLocalBatch(thread_id); + } +} + +// TODO: IT DOES NOT USE MULTITHREADING! if we don't have limit in a query, then OrderBy will work +// with a speed of one thread algorithm... +void OrderBySinkSourceOperator::Finalize() && { + std::vector> global_builders; + for (ColumnType type : column_types_) { + global_builders.push_back(ColumnFactory::MakeColumnBuilder(type)); + } + + size_t num_rows = 0; + for (size_t t = 0; t < thread_states_.size(); ++t) { + OrderByThreadState& local = thread_states_[t]; + if (local.accum_num_rows == 0) { + continue; + } + std::vector> local_cols; + for (auto& builder : local.column_builders) { + local_cols.push_back(builder->Finish()); + } + + for (size_t c = 0; c < global_builders.size(); ++c) { + ExecutionHelper::CopySelection(*global_builders[c], *local_cols[c], nullptr); + } + num_rows += local.accum_num_rows; + local.column_builders.clear(); + } + + if (num_rows == 0) { + result_batch_ = std::make_unique(); + result_batch_->num_rows = 0; + return; + } + + auto temp_batch = std::make_unique(); + temp_batch->num_rows = num_rows; + for (auto& builder : global_builders) { + temp_batch->columns.push_back(builder->Finish()); + } + + std::vector indices(num_rows); + std::iota(indices.begin(), indices.end(), 0); + + std::vector cols_info; + cols_info.reserve(sort_columns_.size()); + for (const auto& [col_ind, is_desc] : sort_columns_) { + const Column* raw_column = temp_batch->columns[col_ind].get(); + auto col_type = raw_column->GetType(); +#define HANDLE_TYPE(ENUM_VAL, STR_VAL, CLASS_TYPE) \ + case ColumnType::ENUM_VAL: { \ + const void* data_ptr = raw_column->GetRawData(); \ + cols_info.push_back({data_ptr, &Compare, is_desc}); \ + break; \ + } + switch (col_type) { + FOR_EACH_COLUMN_TYPE(HANDLE_TYPE); + default: THROW_NOT_IMPLEMENTED; + } +#undef HANDLE_TYPE + } + + auto cmp = [&cols_info](uint32_t a, uint32_t b) { + for (const auto& info : cols_info) { + int res = info.cmp_func(info.raw_data, a, b); + if (res != 0) { + return info.is_desc ? (res > 0) : (res < 0); + } + } + return false; + }; + + if (total_limit_.has_value() && num_rows > total_limit_.value()) { + uint32_t lim = total_limit_.value(); + std::partial_sort(indices.begin(), indices.begin() + lim, indices.end(), cmp); + indices.resize(lim); + } else { + std::sort(indices.begin(), indices.end(), cmp); + } + + if (offset_.has_value() && offset_.value() > 0) { + size_t off = offset_.value(); + if (off >= indices.size()) { + indices.clear(); + } else { + indices.erase(indices.begin(), indices.begin() + off); + } + } + + result_batch_ = std::make_unique(); + result_batch_->num_rows = indices.size(); + + std::vector> final_builders; + for (auto type : column_types_) { + final_builders.push_back(ColumnFactory::MakeColumnBuilder(type)); + } + + for (size_t c = 0; c < final_builders.size(); ++c) { + ExecutionHelper::CopySelection(*final_builders[c], *temp_batch->columns[c], &indices); + result_batch_->columns.push_back(final_builders[c]->Finish()); + } +} + +std::unique_ptr OrderBySinkSourceOperator::GetData() { + bool expected = false; + if (emitted_.compare_exchange_strong(expected, true)) { + return std::move(result_batch_); + } + return nullptr; +} + +void OrderBySinkSourceOperator::TrimLocalBatch(size_t thread_id) { + OrderByThreadState& local = thread_states_[thread_id]; + + if (local.accum_num_rows == 0) { + return; + } + + auto temp_batch = std::make_unique(); + temp_batch->num_rows = local.accum_num_rows; + for (auto& builder : local.column_builders) { + temp_batch->columns.push_back(builder->Finish()); + } + local.column_builders.clear(); + + std::vector indices(local.accum_num_rows); + for (size_t i = 0; i < local.accum_num_rows; ++i) { + indices[i] = i; + } + + std::vector cols_info; + cols_info.reserve(sort_columns_.size()); + for (const auto& [col_ind, is_desc] : sort_columns_) { + const Column* raw_column = temp_batch->columns[col_ind].get(); + +#define HANDLE_TYPE(ENUM_VAL, STR_VAL, CLASS_TYPE) \ + case ColumnType::ENUM_VAL: { \ + const void* data_ptr = raw_column->GetRawData(); \ + cols_info.push_back({data_ptr, &Compare, is_desc}); \ + break; \ + } + auto col_type = raw_column->GetType(); + switch (col_type) { + FOR_EACH_COLUMN_TYPE(HANDLE_TYPE); + default: THROW_NOT_IMPLEMENTED; + } +#undef HANDLE_TYPE + } + + auto cmp = [&cols_info](uint32_t a, uint32_t b) { + for (const auto& info : cols_info) { + int res = info.cmp_func(info.raw_data, a, b); + if (res != 0) { + return info.is_desc ? (res > 0) : (res < 0); + } + } + return false; + }; + + if (total_limit_.has_value() && local.accum_num_rows > total_limit_.value()) { + uint32_t lim = total_limit_.value(); + std::nth_element(indices.begin(), indices.begin() + lim, indices.end(), cmp); + indices.resize(lim); + } + // if (is_final) { + // std::sort(indices.begin(), indices.end(), cmp); + // if (offset_.has_value()) { + // if (offset_.value() < indices.size()) { + // indices.erase(indices.begin(), indices.begin() + offset_.value()); + // } else { + // indices.clear(); + // } + // } + // } + + for (auto type : column_types_) { + local.column_builders.push_back(ColumnFactory::MakeColumnBuilder(type)); + } + for (size_t c = 0; c < local.column_builders.size(); ++c) { + ExecutionHelper::CopySelection(*local.column_builders[c], *temp_batch->columns[c], + &indices); + } + local.accum_num_rows = indices.size(); +} diff --git a/src/execution/OperatorsBase.h b/src/execution/OperatorsBase.h new file mode 100644 index 0000000..10aafae --- /dev/null +++ b/src/execution/OperatorsBase.h @@ -0,0 +1,170 @@ +#pragma once + +#include "execution/Pipeline.h" +#include "io/ColumnarReader.h" +#include "column/ColumnBuilder.h" +#include "utils/Concurrency.h" + +#include "absl/container/flat_hash_map.h" + +#include +#include +#include + +class GlobalAggregationFunction; +class GroupedAggregationFunction; +class FilterFunction; +class ScalarFunction; + +// ========================= Source Operators ========================= + +class ScanOperator : public SourceOperator { +public: + ScanOperator(const std::string& table_path, const std::vector& column_names, + std::shared_ptr metadata); + + std::unique_ptr GetData() override; + +private: + ColumnarReader reader_; + std::shared_ptr metadata_; + std::vector column_indices_; + + Atomic current_chunk_{0}; + size_t total_chunks_ = 0; + Atomic finished_empty_scan_{false}; +}; + +// ========================= Transform Operators ========================= + +class FilterTransformOperator : public TransformOperator { +public: + FilterTransformOperator(std::shared_ptr filter_function); + + std::unique_ptr Execute(std::unique_ptr batch) override; + +private: + std::shared_ptr filter_function_; +}; + +class ScalarTransformOperator : public TransformOperator { +public: + ScalarTransformOperator(std::vector> scalar_functions); + + std::unique_ptr Execute(std::unique_ptr batch) override; + +private: + std::vector> scalar_functions_; +}; + +class DropTransformOperator : public TransformOperator { +public: + DropTransformOperator(std::vector column_stay_indices); + + std::unique_ptr Execute(std::unique_ptr batch) override; + +private: + std::vector column_stay_indices_; +}; + +class ReorderTransformOperator : public TransformOperator { +public: + ReorderTransformOperator(std::vector new_indices); + + std::unique_ptr Execute(std::unique_ptr batch) override; + +private: + std::vector new_indices_; +}; + +class LimitTransformOperator : public TransformOperator { +public: + LimitTransformOperator(size_t limit); + + std::unique_ptr Execute(std::unique_ptr batch) override; + + bool IsPipelineDone() const override { + return done_.load(std::memory_order_relaxed); + } + +private: + size_t limit_; + Atomic cur_rows_{0}; + Atomic done_{false}; +}; + +// ========================= Sink + Source Operators ========================= + +class AggregationSinkSourceOperator : public SinkOperator, public SourceOperator { +public: + AggregationSinkSourceOperator( + std::vector> aggregation_functions); + + void Sink(std::unique_ptr batch, size_t thread_id) override; + void Finalize() && override; + + std::unique_ptr GetData() override; + +private: + std::vector> aggregation_functions_; + std::unique_ptr result_batch_; + Atomic emitted_{false}; +}; + +class GroupBySinkSourceOperator : public SinkOperator, public SourceOperator { +public: + GroupBySinkSourceOperator(std::vector group_by_col_indices, + std::vector key_types, + std::vector> agg_funcs, + size_t num_threads); + + void Sink(std::unique_ptr batch, size_t thread_id) override; + void Finalize() && override; + + std::unique_ptr GetData() override; + +private: + struct alignas(64) GroupByThreadState { + absl::flat_hash_map hash_table; + std::vector> key_builders; + size_t num_groups = 0; + }; + + std::vector group_by_col_indices_; + std::vector key_types_; + std::vector> agg_funcs_; + + std::vector thread_states_; + std::unique_ptr result_batch_; + Atomic emitted_{false}; +}; + +class OrderBySinkSourceOperator : public SinkOperator, public SourceOperator { +public: + OrderBySinkSourceOperator(std::vector> sort_columns, + std::vector column_types, std::optional limit, + std::optional offset, size_t num_threads); + + void Sink(std::unique_ptr batch, size_t thread_id) override; + void Finalize() && override; + + std::unique_ptr GetData() override; + +private: + void TrimLocalBatch(size_t thread_id); + + struct alignas(64) OrderByThreadState { + std::vector> column_builders; + size_t accum_num_rows = 0; + }; + + std::vector> sort_columns_; + const std::vector column_types_; + std::optional limit_; + std::optional offset_; + std::optional total_limit_; + + std::vector thread_states_; + std::unique_ptr result_batch_; + Atomic emitted_{false}; +}; diff --git a/src/execution/Pipeline.h b/src/execution/Pipeline.h new file mode 100644 index 0000000..3830721 --- /dev/null +++ b/src/execution/Pipeline.h @@ -0,0 +1,100 @@ +#pragma once + +#include "column/Column.h" +#include "utils/ThreadPool.h" + +#include +#include + +struct RecordBatch { + size_t num_rows; + std::shared_ptr> selection_vector; + std::vector> columns; +}; + +// generates RecordBatches (for example scan operator) +class SourceOperator { +public: + virtual ~SourceOperator() = default; + virtual std::unique_ptr GetData() = 0; +}; + +// transforms per-batch data +class TransformOperator { +public: + virtual ~TransformOperator() = default; + virtual std::unique_ptr Execute(std::unique_ptr batch) = 0; + + virtual bool IsPipelineDone() const { + return false; + } +}; + +// accumulates all data, breaks pipeline +class SinkOperator { +public: + virtual ~SinkOperator() = default; + + virtual void Sink(std::unique_ptr batch, size_t thread_id) = 0; + virtual void Finalize() && = 0; +}; + +class Pipeline { +public: + std::shared_ptr source; + std::vector> transforms; + std::shared_ptr sink; + + void Execute(ThreadPool& thread_pool, size_t num_threads) { + size_t num_tasks_finished = 0; + Mutex wait_mutex; + ConditionVariable wait_cond; + for (size_t thread_id = 0; thread_id < num_threads; ++thread_id) { + thread_pool.Enqueue( + [this, thread_id, &num_tasks_finished, &wait_mutex, &wait_cond, num_threads]() { + while (auto batch = source->GetData()) { + bool skip = false; + bool stop = false; + + for (std::shared_ptr& transform : transforms) { + batch = transform->Execute(std::move(batch)); + if (!batch || batch->num_rows == 0) { + skip = true; + break; + } + if (transform->IsPipelineDone()) { + stop = true; + break; + } + } + + if (skip) { + continue; + } + if (batch && batch->num_rows > 0) { + sink->Sink(std::move(batch), thread_id); + } + if (stop) { + break; + } + } + { + UniqueLock lock(wait_mutex); + ++num_tasks_finished; + if (num_tasks_finished == num_threads) { + wait_cond.notify_one(); + } + } + }); + } + + { + UniqueLock lock(wait_mutex); + wait_cond.wait(lock, [&]() { + return num_tasks_finished == num_threads; + }); + } + + std::move(*sink).Finalize(); + } +}; diff --git a/src/execution/PipelineExecutor.h b/src/execution/PipelineExecutor.h new file mode 100644 index 0000000..d96edfc --- /dev/null +++ b/src/execution/PipelineExecutor.h @@ -0,0 +1,34 @@ +#pragma once + +#include "execution/Pipeline.h" +#include "column/Schema.h" + +#include +#include + +class ResultSinkOperator : public SinkOperator { +public: + void Sink(std::unique_ptr batch, size_t /* thread_id */) override { + LockGuard lock(mutex_); + batches_.push_back(std::move(batch)); + } + + void Finalize() && override { + // charonchik bebe + } + + std::vector> TakeBatches() { + return std::move(batches_); + } + +private: + Mutex mutex_; + std::vector> batches_; +}; + +struct PipelineBuildContext { + Pipeline* current_pipeline = nullptr; + std::vector> completed_pipelines; + Schema schema; + size_t num_threads = 1; +}; diff --git a/src/execution/expressions/AggExpHelper.h b/src/execution/expressions/AggExpHelper.h new file mode 100644 index 0000000..8c9fb8a --- /dev/null +++ b/src/execution/expressions/AggExpHelper.h @@ -0,0 +1,62 @@ +#pragma once + +#include "Macro.h" +#include "execution/OperatorsBase.h" + +class AggExpHelper { +public: + template + static void IterateColumnData(const RecordBatch& batch, size_t column_index, Func&& func) { + auto* column = static_cast(batch.columns[column_index].get()); + const auto& data = column->GetData(); + + if (!batch.selection_vector) { + for (size_t i = 0; i < batch.num_rows; ++i) { + func(data[i], i, i); + } + } else { + const auto& sel = *batch.selection_vector; + for (size_t i = 0; i < batch.num_rows; ++i) { + func(data[sel[i]], i, sel[i]); + } + } + } + + template + static void IterateViewColData(const RecordBatch& batch, const ViewT& view, Func&& func) { + if (!batch.selection_vector) { + for (size_t i = 0; i < batch.num_rows; ++i) { + func(view[i], i, i); + } + } else { + const auto& sel = *batch.selection_vector; + for (size_t i = 0; i < batch.num_rows; ++i) { + func(view[sel[i]], i, sel[i]); + } + } + } + + template + static auto DispatchNumeric(ColumnType type, Functor&& f) { +#define HANDLE_TYPE(ENUM_VAL, STR_VAL, CLASS_TYPE) \ + case ColumnType::ENUM_VAL: return f.template operator()(ColumnType::ENUM_VAL); + + switch (type) { + FOR_NUMERIC_COLUMN_TYPE(HANDLE_TYPE) + default: THROW_NOT_IMPLEMENTED; + } +#undef HANDLE_TYPE + } + + template + static auto DispatchAll(ColumnType type, Functor&& f) { +#define HANDLE_TYPE(ENUM_VAL, STR_VAL, CLASS_TYPE) \ + case ColumnType::ENUM_VAL: return f.template operator()(ColumnType::ENUM_VAL); + + switch (type) { + FOR_EACH_COLUMN_TYPE(HANDLE_TYPE) + default: THROW_NOT_IMPLEMENTED; + } +#undef HANDLE_TYPE + } +}; \ No newline at end of file diff --git a/src/execution/expressions/AggregationExpressions.h b/src/execution/expressions/AggregationExpressions.h new file mode 100644 index 0000000..931a538 --- /dev/null +++ b/src/execution/expressions/AggregationExpressions.h @@ -0,0 +1,371 @@ +#pragma once + +#include "execution/expressions/AggregationFunctions.h" +#include "execution/expressions/AggExpHelper.h" +#include "column/Schema.h" + +#include +#include +#include + +template +struct SumResultTrait { + using Type = InputColumn; + static constexpr ColumnType kOutTypeEnum = ColumnType::INT64; +}; + +template <> +struct SumResultTrait { + using Type = Int32Column; + static constexpr ColumnType kOutTypeEnum = ColumnType::INT32; +}; +template <> +struct SumResultTrait { + using Type = Int64Column; + static constexpr ColumnType kOutTypeEnum = ColumnType::INT64; +}; +template <> +struct SumResultTrait { + using Type = Int128Column; + static constexpr ColumnType kOutTypeEnum = ColumnType::INT128; +}; +template <> +struct SumResultTrait { + using Type = Int128Column; + static constexpr ColumnType kOutTypeEnum = ColumnType::INT128; +}; +template <> +struct SumResultTrait { + using Type = DoubleColumn; + static constexpr ColumnType kOutTypeEnum = ColumnType::DOUBLE; +}; +template <> +struct SumResultTrait { + using Type = LongDoubleColumn; + static constexpr ColumnType kOutTypeEnum = ColumnType::LONGDOUBLE; +}; +template <> +struct SumResultTrait { + using Type = LongDoubleColumn; + static constexpr ColumnType kOutTypeEnum = ColumnType::LONGDOUBLE; +}; + +class AggregateExpression { +public: + virtual ~AggregateExpression() = default; + + explicit AggregateExpression(std::string column_name, std::string output_name = "") + : column_name_(std::move(column_name)), output_name_(std::move(output_name)) { + } + + std::string GetName() const { + return column_name_; + } + + void CollectRequiredColumns(std::vector& required_columns) { + if (!column_name_.empty() && column_name_ != "*") { + required_columns.push_back(column_name_); + } + } + + virtual std::string GetOutputName() const { + return output_name_.empty() ? column_name_ : output_name_; + } + + virtual std::unique_ptr CreateGlobalAggregationFunction( + const Schema& child_schema, Schema& output_schema, size_t num_threads) = 0; + + virtual std::unique_ptr CreateGroupedAggregationFunction( + const Schema& child_schema, Schema& output_schema, size_t num_threads) = 0; + +protected: + std::string column_name_; + std::string output_name_; +}; + +class CountExpression : public AggregateExpression { +public: + explicit CountExpression(std::string column_name, std::string output_name = "count") + : AggregateExpression(std::move(column_name), std::move(output_name)) { + } + + std::unique_ptr CreateGlobalAggregationFunction( + const Schema&, Schema& output_schema, size_t num_threads) override { + output_schema.AddColumn(GetOutputName(), ColumnType::INT64); + return std::make_unique>(ColumnType::INT64, num_threads); + } + + std::unique_ptr CreateGroupedAggregationFunction( + const Schema&, Schema& output_schema, size_t num_threads) override { + output_schema.AddColumn(GetOutputName(), ColumnType::INT64); + return std::make_unique>(ColumnType::INT64, num_threads); + } +}; + +class DistinctCountExpression : public AggregateExpression { +public: + explicit DistinctCountExpression(std::string column_name, std::string output_name = "") + : AggregateExpression(std::move(column_name), std::move(output_name)) { + } + + std::string GetOutputName() const override { + if (!output_name_.empty()) { + return output_name_; + } + return "distinct_count_" + column_name_; + } + + std::unique_ptr CreateGlobalAggregationFunction( + const Schema& child_schema, Schema& output_schema, size_t num_threads) override { + size_t column_ind = child_schema.GetColumnIndexByName(GetName()); + ColumnType column_type = child_schema.GetColumnTypeByName(GetName()); + + return AggExpHelper::DispatchAll( + column_type, [&](ColumnType) -> std::unique_ptr { + output_schema.AddColumn(GetOutputName(), ColumnType::INT64); + return std::make_unique>( + column_ind, ColumnType::INT64, num_threads); + }); + } + + std::unique_ptr CreateGroupedAggregationFunction( + const Schema& child_schema, Schema& output_schema, size_t num_threads) override { + size_t column_ind = child_schema.GetColumnIndexByName(GetName()); + ColumnType column_type = child_schema.GetColumnTypeByName(GetName()); + + return AggExpHelper::DispatchAll( + column_type, + [&](ColumnType) -> std::unique_ptr { + output_schema.AddColumn(GetOutputName(), ColumnType::INT64); + return std::make_unique>( + column_ind, ColumnType::INT64, num_threads); + }); + } +}; + +class MinExpression : public AggregateExpression { +public: + explicit MinExpression(std::string column_name, std::string output_name = "") + : AggregateExpression(std::move(column_name), std::move(output_name)) { + } + + std::string GetOutputName() const override { + if (!output_name_.empty()) { + return output_name_; + } + return "min_" + column_name_; + } + + std::unique_ptr CreateGlobalAggregationFunction( + const Schema& child_schema, Schema& output_schema, size_t num_threads) override { + size_t column_ind = child_schema.GetColumnIndexByName(GetName()); + ColumnType column_type = child_schema.GetColumnTypeByName(GetName()); + + return AggExpHelper::DispatchAll( + column_type, + [&](ColumnType t) -> std::unique_ptr { + output_schema.AddColumn(GetOutputName(), t); + return std::make_unique>( + column_ind, t, num_threads); + }); + } + + std::unique_ptr CreateGroupedAggregationFunction( + const Schema& child_schema, Schema& output_schema, size_t num_threads) override { + size_t column_ind = child_schema.GetColumnIndexByName(GetName()); + ColumnType column_type = child_schema.GetColumnTypeByName(GetName()); + + return AggExpHelper::DispatchAll( + column_type, + [&](ColumnType t) -> std::unique_ptr { + output_schema.AddColumn(GetOutputName(), t); + return std::make_unique>( + column_ind, t, num_threads); + }); + } +}; + +class MaxExpression : public AggregateExpression { +public: + explicit MaxExpression(std::string column_name, std::string output_name = "") + : AggregateExpression(std::move(column_name), std::move(output_name)) { + } + + std::string GetOutputName() const override { + if (!output_name_.empty()) { + return output_name_; + } + return "max_" + column_name_; + } + + std::unique_ptr CreateGlobalAggregationFunction( + const Schema& child_schema, Schema& output_schema, size_t num_threads) override { + size_t column_ind = child_schema.GetColumnIndexByName(GetName()); + ColumnType column_type = child_schema.GetColumnTypeByName(GetName()); + + return AggExpHelper::DispatchAll( + column_type, + [&](ColumnType t) -> std::unique_ptr { + output_schema.AddColumn(GetOutputName(), t); + return std::make_unique>( + column_ind, t, num_threads); + }); + } + + std::unique_ptr CreateGroupedAggregationFunction( + const Schema& child_schema, Schema& output_schema, size_t num_threads) override { + size_t column_ind = child_schema.GetColumnIndexByName(GetName()); + ColumnType column_type = child_schema.GetColumnTypeByName(GetName()); + + return AggExpHelper::DispatchAll( + column_type, + [&](ColumnType t) -> std::unique_ptr { + output_schema.AddColumn(GetOutputName(), t); + return std::make_unique>( + column_ind, t, num_threads); + }); + } +}; + +class SumExpression : public AggregateExpression { +public: + explicit SumExpression(std::string column_name, std::string output_name = "") + : AggregateExpression(std::move(column_name), std::move(output_name)) { + } + + std::string GetOutputName() const override { + if (!output_name_.empty()) { + return output_name_; + } + return "sum_" + column_name_; + } + + std::unique_ptr CreateGlobalAggregationFunction( + const Schema& child_schema, Schema& output_schema, size_t num_threads) override { + const size_t column_ind = child_schema.GetColumnIndexByName(GetName()); + const ColumnType column_type = child_schema.GetColumnTypeByName(GetName()); + + if (column_type == ColumnType::STRING || column_type == ColumnType::CHAR || + column_type == ColumnType::DATE || column_type == ColumnType::TIMESTAMP) { + THROW_RUNTIME_ERROR("SUM operation is not supported for this column type"); + } + + return AggExpHelper::DispatchNumeric( + column_type, + [&](ColumnType) -> std::unique_ptr { + using OutputColumn = SumResultTrait::Type; + ColumnType out_type = SumResultTrait::kOutTypeEnum; + + output_schema.AddColumn(GetOutputName(), out_type); + return std::make_unique< + TypedGlobalAggregationFunction>( + column_ind, out_type, num_threads); + }); + } + + std::unique_ptr CreateGroupedAggregationFunction( + const Schema& child_schema, Schema& output_schema, size_t num_threads) override { + const size_t column_ind = child_schema.GetColumnIndexByName(GetName()); + const ColumnType column_type = child_schema.GetColumnTypeByName(GetName()); + + if (column_type == ColumnType::STRING || column_type == ColumnType::CHAR || + column_type == ColumnType::DATE || column_type == ColumnType::TIMESTAMP) { + THROW_RUNTIME_ERROR("SUM operation is not supported for this column type"); + } + + return AggExpHelper::DispatchNumeric( + column_type, + [&](ColumnType) -> std::unique_ptr { + using OutputColumn = SumResultTrait::Type; + ColumnType out_type = SumResultTrait::kOutTypeEnum; + + output_schema.AddColumn(GetOutputName(), out_type); + return std::make_unique< + TypedGroupedAggregationFunction>( + column_ind, out_type, num_threads); + }); + } +}; + +class AvgExpression : public AggregateExpression { +public: + explicit AvgExpression(std::string column_name, std::string output_name = "") + : AggregateExpression(std::move(column_name), std::move(output_name)) { + } + std::string GetOutputName() const override { + if (!output_name_.empty()) { + return output_name_; + } + return "avg_" + column_name_; + } + + std::unique_ptr CreateGlobalAggregationFunction( + const Schema& child_schema, Schema& output_schema, size_t num_threads) override { + const size_t column_ind = child_schema.GetColumnIndexByName(GetName()); + const ColumnType column_type = child_schema.GetColumnTypeByName(GetName()); + + if (column_type == ColumnType::STRING || column_type == ColumnType::CHAR || + column_type == ColumnType::DATE || column_type == ColumnType::TIMESTAMP) { + THROW_RUNTIME_ERROR("AVG operation is not supported for non-numeric columns"); + } + + return AggExpHelper::DispatchNumeric( + column_type, + [&](ColumnType) -> std::unique_ptr { + using StateColumn = SumResultTrait::Type; + + output_schema.AddColumn(GetOutputName(), ColumnType::LONGDOUBLE); + return std::make_unique< + AvgGlobalAggregationFunction>( + column_ind, ColumnType::LONGDOUBLE, num_threads); + }); + } + + std::unique_ptr CreateGroupedAggregationFunction( + const Schema& child_schema, Schema& output_schema, size_t num_threads) override { + const size_t column_ind = child_schema.GetColumnIndexByName(GetName()); + const ColumnType column_type = child_schema.GetColumnTypeByName(GetName()); + + if (column_type == ColumnType::STRING || column_type == ColumnType::CHAR || + column_type == ColumnType::DATE || column_type == ColumnType::TIMESTAMP) { + THROW_RUNTIME_ERROR("AVG operation is not supported for non-numeric columns"); + } + + return AggExpHelper::DispatchNumeric( + column_type, + [&](ColumnType) -> std::unique_ptr { + using StateColumn = SumResultTrait::Type; + + output_schema.AddColumn(GetOutputName(), ColumnType::LONGDOUBLE); + return std::make_unique< + AvgGroupedAggregationFunction>( + column_ind, ColumnType::LONGDOUBLE, num_threads); + }); + } +}; + +inline std::shared_ptr Count(std::string column_name = "*", + std::string output_name = "count") { + return std::make_shared(std::move(column_name), std::move(output_name)); +} +inline std::shared_ptr Min(std::string column_name, + std::string output_name = "") { + return std::make_shared(std::move(column_name), std::move(output_name)); +} +inline std::shared_ptr Max(std::string column_name, + std::string output_name = "") { + return std::make_shared(std::move(column_name), std::move(output_name)); +} +inline std::shared_ptr Sum(std::string column_name, + std::string output_name = "") { + return std::make_shared(std::move(column_name), std::move(output_name)); +} +inline std::shared_ptr Avg(std::string column_name, + std::string output_name = "") { + return std::make_shared(std::move(column_name), std::move(output_name)); +} +inline std::shared_ptr DistinctCount(std::string column_name, + std::string output_name = "") { + return std::make_shared(std::move(column_name), + std::move(output_name)); +} diff --git a/src/execution/expressions/AggregationFunctions.h b/src/execution/expressions/AggregationFunctions.h new file mode 100644 index 0000000..0ead79b --- /dev/null +++ b/src/execution/expressions/AggregationFunctions.h @@ -0,0 +1,537 @@ +#pragma once + +#include "execution/expressions/AggExpHelper.h" +#include "execution/OperatorsBase.h" + +#include "column/Column.h" +#include "column/CharColumn.h" +#include "column/NumericColumn.h" +#include "column/StringColumn.h" +#include "column/TemporalColumn.h" +#include "column/ColumnFactory.h" + +#include "absl/container/flat_hash_set.h" + +#include +#include +#include + +class AggregationFunction { +public: + virtual ~AggregationFunction() = default; + AggregationFunction(ColumnType column_type, size_t num_threads) + : column_type_(column_type), num_threads_(num_threads) { + } + + std::shared_ptr Finalize() { + auto builder = ColumnFactory::MakeColumnBuilder(column_type_); + WriteResult(builder); + return builder->Finish(); + } + +protected: + ColumnType column_type_; + size_t num_threads_; + + virtual void WriteResult(const std::shared_ptr& builder) = 0; +}; + +class GlobalAggregationFunction : public AggregationFunction { +public: + GlobalAggregationFunction(ColumnType column_type, size_t num_threads) + : AggregationFunction(column_type, num_threads) { + } + + virtual void Update(const RecordBatch& batch, size_t thread_id) = 0; +}; + +class GroupedAggregationFunction : public AggregationFunction { +public: + GroupedAggregationFunction(ColumnType column_type, size_t num_threads) + : AggregationFunction(column_type, num_threads) { + } + + virtual void Resize(size_t num_groups, size_t thread_id) = 0; + virtual void Update(const RecordBatch& batch, const std::vector& group_ids, + size_t thread_id) = 0; + virtual void CombineState(size_t src_thread, uint32_t src_gid, uint32_t global_gid) = 0; +}; + +struct SumOperation { + template + static void Apply(ResultType& result, const ValueType& value) { + result += value; + } +}; + +struct MaxOperation { + template + static void Apply(ResultType& result, const ValueType& value) { + if (value > result) { + result = value; + } + } +}; + +struct MinOperation { + template + static inline void Apply(ResultType& result, const ValueType& value) { + if (value < result) { + result = value; + } + } +}; + +template +class TypedGlobalAggregationFunction : public GlobalAggregationFunction { + using StateType = OutputColumn::ValueType; + using OutputBuilder = BuilderTypeTrait::Type; + + struct alignas(64) ThreadState { + StateType value{}; + bool has_data = false; + }; + +public: + TypedGlobalAggregationFunction(size_t column_index, ColumnType column_type, size_t num_threads) + : GlobalAggregationFunction(column_type, num_threads), + column_index_(column_index), + thread_states_(num_threads) { + } + + void Update(const RecordBatch& batch, size_t thread_id) override { + ThreadState& local_state = thread_states_[thread_id]; + + AggExpHelper::IterateColumnData( + batch, column_index_, [&](const auto& value, size_t, size_t) { + if (!local_state.has_data) { + local_state.value = value; + local_state.has_data = true; + } else { + Operation::Apply(local_state.value, value); + } + }); + } + +protected: + void WriteResult(const std::shared_ptr& builder) override { + StateType result{}; + bool has_data = false; + + for (size_t i = 0; i < thread_states_.size(); i++) { + if (has_data) { + if (thread_states_[i].has_data) { + Operation::Apply(result, thread_states_[i].value); + } + } else if (thread_states_[i].has_data) { + has_data = true; + result = thread_states_[i].value; + } + } + + auto* typed = static_cast(builder.get()); + if (has_data) { + typed->AddValue(result); + } else { + if constexpr (requires { Operation::template GetInitValue(); }) { + typed->AddValue(Operation::template GetInitValue()); + } else { + typed->AddValue(StateType{}); + } + } + } + +private: + const size_t column_index_; + std::vector thread_states_; +}; + +template +class TypedGroupedAggregationFunction : public GroupedAggregationFunction { + using StateType = OutputColumn::ValueType; + using OutputBuilder = BuilderTypeTrait::Type; + +public: + TypedGroupedAggregationFunction(size_t column_index, ColumnType column_type, size_t num_threads) + : GroupedAggregationFunction(column_type, num_threads), + column_index_(column_index), + thread_states_(num_threads) { + } + + void Resize(size_t num_groups, size_t thread_id) override { + auto& local = thread_states_[thread_id]; + if (num_groups > local.states.size()) { + local.states.resize(num_groups); + local.has_data.resize(num_groups, 0); + } + } + + void Update(const RecordBatch& batch, const std::vector& group_ids, + size_t thread_id) override { + auto& local = thread_states_[thread_id]; + AggExpHelper::IterateColumnData( + batch, column_index_, [&](const auto& value, size_t row_idx, size_t) { + uint32_t gid = group_ids[row_idx]; + if (!local.has_data[gid]) { + local.states[gid] = value; + local.has_data[gid] = 1; + } else { + Operation::Apply(local.states[gid], value); + } + }); + } + + void CombineState(size_t src_thread, uint32_t src_gid, uint32_t global_gid) override { + auto& src = thread_states_[src_thread]; + auto& global = thread_states_[0]; + + if (src.has_data[src_gid]) { + if (!global.has_data[global_gid]) { + global.states[global_gid] = src.states[src_gid]; + global.has_data[global_gid] = 1; + } else { + Operation::Apply(global.states[global_gid], src.states[src_gid]); + } + } + } + +protected: + void WriteResult(const std::shared_ptr& builder) override { + auto* typed = static_cast(builder.get()); + auto& global = thread_states_[0]; + for (const StateType& state : global.states) { + typed->AddValue(state); + } + } + +private: + struct alignas(64) ThreadState { + std::vector states; + std::vector has_data; + }; + + size_t column_index_; + std::vector thread_states_; +}; + +template +class CountGlobalAggregationFunction : public GlobalAggregationFunction { + using ValueType = OutputColumn::ValueType; + using OutputBuilder = BuilderTypeTrait::Type; + + struct alignas(64) ThreadState { + int64_t count = 0; + }; + +public: + CountGlobalAggregationFunction(ColumnType column_type, size_t num_threads) + : GlobalAggregationFunction(column_type, num_threads), thread_states_(num_threads) { + } + + void Update(const RecordBatch& batch, size_t thread_id) override { + thread_states_[thread_id].count += batch.num_rows; + } + +protected: + void WriteResult(const std::shared_ptr& builder) override { + int64_t total = 0; + for (const auto& state : thread_states_) { + total += state.count; + } + auto* typed = static_cast(builder.get()); + typed->AddValue(static_cast(total)); + } + +private: + std::vector thread_states_; +}; + +template +class CountGroupedAggregationFunction : public GroupedAggregationFunction { + using ValueType = OutputColumn::ValueType; + using OutputBuilder = BuilderTypeTrait::Type; + +public: + CountGroupedAggregationFunction(ColumnType column_type, size_t num_threads) + : GroupedAggregationFunction(column_type, num_threads), thread_states_(num_threads) { + } + + void Resize(size_t num_groups, size_t thread_id) override { + auto& local = thread_states_[thread_id]; + if (num_groups > local.counts.size()) { + local.counts.resize(num_groups, 0); + } + } + + void Update(const RecordBatch& batch, const std::vector& group_ids, + size_t thread_id) override { + auto& local = thread_states_[thread_id]; + for (size_t i = 0; i < batch.num_rows; ++i) { + uint32_t gid = group_ids[i]; + ++local.counts[gid]; + } + } + + void CombineState(size_t src_thread, uint32_t src_gid, uint32_t global_gid) override { + ThreadState& src = thread_states_[src_thread]; + ThreadState& global = thread_states_[0]; + global.counts[global_gid] += src.counts[src_gid]; + } + +protected: + void WriteResult(const std::shared_ptr& builder) override { + auto* typed = static_cast(builder.get()); + auto& global = thread_states_[0]; + + for (int64_t c : global.counts) { + typed->AddValue(static_cast(c)); + } + } + +private: + struct alignas(64) ThreadState { + std::vector counts; + }; + + std::vector thread_states_; +}; + +template +class AvgGlobalAggregationFunction : public GlobalAggregationFunction { + using StateType = StateColumn::ValueType; + using OutputType = OutputColumn::ValueType; + using OutputBuilder = BuilderTypeTrait::Type; + + struct alignas(64) ThreadState { + StateType sum = 0; + int64_t count = 0; + }; + +public: + AvgGlobalAggregationFunction(size_t column_index, ColumnType column_type, size_t num_threads) + : GlobalAggregationFunction(column_type, num_threads), + column_index_(column_index), + thread_states_(num_threads) { + } + + void Update(const RecordBatch& batch, size_t thread_id) override { + ThreadState& local = thread_states_[thread_id]; + AggExpHelper::IterateColumnData(batch, column_index_, + [&](const auto& value, size_t, size_t) { + local.sum += value; + ++local.count; + }); + } + +protected: + void WriteResult(const std::shared_ptr& builder) override { + StateType total_sum = 0; + int64_t total_count = 0; + for (const auto& state : thread_states_) { + total_sum += state.sum; + total_count += state.count; + } + + auto* typed = static_cast(builder.get()); + if (total_count > 0) { + typed->AddValue(ComputeAvg(total_sum, total_count)); + } else { + typed->AddValue(OutputType{}); + } + } + +private: + static OutputType ComputeAvg(StateType sum, int64_t count) { + if constexpr (std::is_integral_v) { + OutputType int_part = static_cast(sum / count); + OutputType rem = static_cast(static_cast(sum % count) / count); + return int_part + rem; + } else { + return static_cast(sum) / static_cast(count); + } + } + + size_t column_index_; + std::vector thread_states_; +}; + +template +class AvgGroupedAggregationFunction : public GroupedAggregationFunction { + using StateType = StateColumn::ValueType; + using OutputType = OutputColumn::ValueType; + using OutputBuilder = BuilderTypeTrait::Type; + +public: + AvgGroupedAggregationFunction(size_t column_index, ColumnType column_type, size_t num_threads) + : GroupedAggregationFunction(column_type, num_threads), + column_index_(column_index), + thread_states_(num_threads) { + } + + void Resize(size_t num_groups, size_t thread_id) override { + auto& local = thread_states_[thread_id]; + if (num_groups > local.states.size()) { + local.states.resize(num_groups); + } + } + + void Update(const RecordBatch& batch, const std::vector& group_ids, + size_t thread_id) override { + auto& local = thread_states_[thread_id]; + AggExpHelper::IterateColumnData( + batch, column_index_, [&](const auto& value, size_t row_idx, size_t) { + uint32_t gid = group_ids[row_idx]; + local.states[gid].sum += value; + ++local.states[gid].count; + }); + } + + void CombineState(size_t src_thread, uint32_t src_gid, uint32_t global_gid) override { + ThreadState& src = thread_states_[src_thread]; + ThreadState& global = thread_states_[0]; + global.states[global_gid].sum += src.states[src_gid].sum; + global.states[global_gid].count += src.states[src_gid].count; + } + +protected: + void WriteResult(const std::shared_ptr& builder) override { + auto* typed = static_cast(builder.get()); + auto& global = thread_states_[0]; + + for (const auto& state : global.states) { + if (state.count > 0) { + typed->AddValue(ComputeAvg(state.sum, state.count)); + } else { + typed->AddValue(OutputType{}); + } + } + } + +private: + static OutputType ComputeAvg(StateType sum, int64_t count) { + if constexpr (std::is_integral_v) { + OutputType int_part = static_cast(sum / count); + OutputType rem = static_cast(static_cast(sum % count) / count); + return int_part + rem; + } else { + return static_cast(sum) / static_cast(count); + } + } + + struct alignas(64) ThreadState { + struct State { + StateType sum; + int64_t count; + }; + + std::vector states; + }; + + size_t column_index_; + std::vector thread_states_; +}; + +template +class DistinctCountGlobalAggregationFunction : public GlobalAggregationFunction { + using InputValueType = InputColumn::ValueType; + using OutputValueType = OutputColumn::ValueType; + using OutputBuilder = BuilderTypeTrait::Type; + +public: + DistinctCountGlobalAggregationFunction(size_t column_index, ColumnType column_type, + size_t num_threads) + : GlobalAggregationFunction(column_type, num_threads), + column_index_(column_index), + thread_sets_(num_threads) { + } + + void Update(const RecordBatch& batch, size_t thread_id) override { + auto& local_set = thread_sets_[thread_id].set; + AggExpHelper::IterateColumnData( + batch, column_index_, + [&](const auto& value, size_t, size_t) { local_set.insert(InputValueType(value)); }); + } + +protected: + void WriteResult(const std::shared_ptr& builder) override { + auto& merged = thread_sets_[0].set; + for (size_t i = 1; i < thread_sets_.size(); ++i) { + for (auto& val : thread_sets_[i].set) { + merged.insert(std::move(val)); + } + } + + auto* typed = static_cast(builder.get()); + typed->AddValue(static_cast(merged.size())); + } + +private: + struct alignas(64) Set { + absl::flat_hash_set set; + }; + + size_t column_index_; + std::vector thread_sets_; +}; + +template +class DistinctCountGroupedAggregationFunction : public GroupedAggregationFunction { + using InputValueType = InputColumn::ValueType; + using OutputValueType = OutputColumn::ValueType; + using OutputBuilder = BuilderTypeTrait::Type; + +public: + DistinctCountGroupedAggregationFunction(size_t column_index, ColumnType column_type, + size_t num_threads) + : GroupedAggregationFunction(column_type, num_threads), + column_index_(column_index), + thread_states_(num_threads) { + } + + void Resize(size_t num_groups, size_t thread_id) override { + auto& local = thread_states_[thread_id]; + if (num_groups > local.sets.size()) { + local.sets.resize(num_groups); + } + } + + void Update(const RecordBatch& batch, const std::vector& group_ids, + size_t thread_id) override { + auto& local = thread_states_[thread_id]; + AggExpHelper::IterateColumnData( + batch, column_index_, [&](const auto& value, size_t row_idx, size_t) { + uint32_t gid = group_ids[row_idx]; + local.sets[gid].set.insert(InputValueType(value)); + }); + } + + void CombineState(size_t src_thread, uint32_t src_gid, uint32_t global_gid) override { + ThreadState& src = thread_states_[src_thread]; + ThreadState& global = thread_states_[0]; + for (const auto& val : src.sets[src_gid].set) { + global.sets[global_gid].set.insert(val); + } + } + +protected: + void WriteResult(const std::shared_ptr& builder) override { + OutputBuilder* typed = static_cast(builder.get()); + ThreadState& global = thread_states_[0]; + + for (const auto& wrapper : global.sets) { + typed->AddValue(static_cast(wrapper.set.size())); + } + } + +private: + struct alignas(64) ThreadState { + struct Set { + absl::flat_hash_set set; + }; + + std::vector sets; + }; + + size_t column_index_; + std::vector thread_states_; +}; diff --git a/src/execution/expressions/FilterExpressions.h b/src/execution/expressions/FilterExpressions.h new file mode 100644 index 0000000..9d1454a --- /dev/null +++ b/src/execution/expressions/FilterExpressions.h @@ -0,0 +1,332 @@ +#pragma once + +#include "FilterFunctions.h" +#include "utils/Macro.h" +#include "column/Schema.h" + +#include +#include + +class FilterExpression { +public: + virtual ~FilterExpression() = default; + + explicit FilterExpression(std::string column_name) : column_name_(std::move(column_name)) { + } + + virtual void CollectRequiredColumns(std::vector& required_columns) { + if (!column_name_.empty()) { + required_columns.push_back(column_name_); + } + } + + virtual std::unique_ptr CreateFilterFunction(const Schema& child_schema) = 0; + +protected: + std::string column_name_; +}; + +template +class NotEqFilterExpression : public FilterExpression { +public: + NotEqFilterExpression(std::string column_name, ValueType value) + : FilterExpression(std::move(column_name)), value_(std::move(value)) { + } + + std::unique_ptr CreateFilterFunction(const Schema& child_schema) override { + size_t ind = child_schema.GetColumnIndexByName(column_name_); + ColumnType column_type = child_schema.GetColumnTypeByName(column_name_); + + return AggExpHelper::DispatchAll( + column_type, [&](ColumnType) -> std::unique_ptr { + if constexpr (std::is_same_v) { + return std::make_unique>(ind, + value_); + } else { + THROW_NOT_IMPLEMENTED; + } + }); + } + +private: + ValueType value_; +}; + +template +class EqFilterExpression : public FilterExpression { +public: + EqFilterExpression(std::string column_name, ValueType value) + : FilterExpression(std::move(column_name)), value_(std::move(value)) { + } + + std::unique_ptr CreateFilterFunction(const Schema& child_schema) override { + size_t ind = child_schema.GetColumnIndexByName(column_name_); + ColumnType column_type = child_schema.GetColumnTypeByName(column_name_); + + return AggExpHelper::DispatchAll( + column_type, [&](ColumnType) -> std::unique_ptr { + if constexpr (std::is_same_v) { + return std::make_unique>(ind, value_); + } else { + THROW_NOT_IMPLEMENTED; + } + }); + } + +private: + ValueType value_; +}; + +template +class GreaterFilterExpression : public FilterExpression { +public: + GreaterFilterExpression(std::string column_name, ValueType value) + : FilterExpression(column_name), value_(std::move(value)) { + } + + std::unique_ptr CreateFilterFunction(const Schema& child_schema) override { + size_t ind = child_schema.GetColumnIndexByName(column_name_); + ColumnType column_type = child_schema.GetColumnTypeByName(column_name_); + + return AggExpHelper::DispatchAll( + column_type, [&](ColumnType) -> std::unique_ptr { + if constexpr (std::is_same_v) { + return std::make_unique>(ind, + value_); + } else { + THROW_NOT_IMPLEMENTED; + } + }); + } + +private: + ValueType value_; +}; + +template +class GreaterEqFilterExpression : public FilterExpression { +public: + GreaterEqFilterExpression(std::string column_name, ValueType value) + : FilterExpression(column_name), value_(std::move(value)) { + } + + std::unique_ptr CreateFilterFunction(const Schema& child_schema) override { + size_t ind = child_schema.GetColumnIndexByName(column_name_); + ColumnType column_type = child_schema.GetColumnTypeByName(column_name_); + + return AggExpHelper::DispatchAll( + column_type, [&](ColumnType) -> std::unique_ptr { + if constexpr (std::is_same_v) { + return std::make_unique>( + ind, value_); + } else { + THROW_NOT_IMPLEMENTED; + } + }); + } + +private: + ValueType value_; +}; + +template +class LessEqFilterExpression : public FilterExpression { +public: + LessEqFilterExpression(std::string column_name, ValueType value) + : FilterExpression(column_name), value_(std::move(value)) { + } + + std::unique_ptr CreateFilterFunction(const Schema& child_schema) override { + size_t ind = child_schema.GetColumnIndexByName(column_name_); + ColumnType column_type = child_schema.GetColumnTypeByName(column_name_); + + return AggExpHelper::DispatchAll( + column_type, [&](ColumnType) -> std::unique_ptr { + if constexpr (std::is_same_v) { + return std::make_unique>(ind, + value_); + } else { + THROW_NOT_IMPLEMENTED; + } + }); + } + +private: + ValueType value_; +}; + +class LikeExpression : public FilterExpression { +public: + LikeExpression(std::string column_name, std::string value) + : FilterExpression(column_name), value_(std::move(value)) { + value_.erase(value_.begin()); + value_.pop_back(); + } + + std::unique_ptr CreateFilterFunction(const Schema& child_schema) override { + size_t ind = child_schema.GetColumnIndexByName(column_name_); + ColumnType column_type = child_schema.GetColumnTypeByName(column_name_); + + return AggExpHelper::DispatchAll( + column_type, + [&]( + ColumnType) -> std::unique_ptr { + if constexpr (std::is_same_v) { + return std::make_unique>(ind, + value_); + } else { + THROW_NOT_IMPLEMENTED; + } + }); + } + +private: + std::string value_; +}; + +class NotLikeExpression : public FilterExpression { +public: + NotLikeExpression(std::string column_name, std::string value) + : FilterExpression(column_name), value_(std::move(value)) { + value_.erase(value_.begin()); + value_.pop_back(); + } + + std::unique_ptr CreateFilterFunction(const Schema& child_schema) override { + size_t ind = child_schema.GetColumnIndexByName(column_name_); + ColumnType column_type = child_schema.GetColumnTypeByName(column_name_); + + return AggExpHelper::DispatchAll( + column_type, + [&]( + ColumnType) -> std::unique_ptr { + if constexpr (std::is_same_v) { + return std::make_unique>( + ind, value_); + } else { + THROW_NOT_IMPLEMENTED; + } + }); + } + +private: + std::string value_; +}; + +class AndExpression : public FilterExpression { +public: + AndExpression(std::shared_ptr left, std::shared_ptr right) + : FilterExpression(""), left_(std::move(left)), right_(std::move(right)) { + } + + void CollectRequiredColumns(std::vector& required_columns) override { + left_->CollectRequiredColumns(required_columns); + right_->CollectRequiredColumns(required_columns); + } + + std::unique_ptr CreateFilterFunction(const Schema& child_schema) override { + return std::make_unique(left_->CreateFilterFunction(child_schema), + right_->CreateFilterFunction(child_schema)); + } + +private: + std::shared_ptr left_; + std::shared_ptr right_; +}; + +class OrExpression : public FilterExpression { +public: + OrExpression(std::shared_ptr left, std::shared_ptr right) + : FilterExpression(""), left_(std::move(left)), right_(std::move(right)) { + } + + void CollectRequiredColumns(std::vector& required_columns) override { + left_->CollectRequiredColumns(required_columns); + right_->CollectRequiredColumns(required_columns); + } + + std::unique_ptr CreateFilterFunction(const Schema& child_schema) override { + return std::make_unique(left_->CreateFilterFunction(child_schema), + right_->CreateFilterFunction(child_schema)); + } + +private: + std::shared_ptr left_; + std::shared_ptr right_; +}; + +template +inline std::shared_ptr NotEq(const std::string& column_name, T target_val) { + if constexpr (std::is_convertible_v) { + return std::make_shared>(column_name, + std::string(target_val)); + } else { + return std::make_shared>(column_name, std::move(target_val)); + } +} + +template +inline std::shared_ptr Eq(const std::string& column_name, T target_val) { + if constexpr (std::is_convertible_v) { + return std::make_shared>(column_name, + std::string(target_val)); + } else { + return std::make_shared>(column_name, std::move(target_val)); + } +} + +template +inline std::shared_ptr Greater(const std::string& column_name, T target_val) { + if constexpr (std::is_convertible_v) { + return std::make_shared>(column_name, + std::string(target_val)); + } else { + return std::make_shared>(column_name, std::move(target_val)); + } +} + +template +inline std::shared_ptr GreaterEq(const std::string& column_name, T target_val) { + if constexpr (std::is_convertible_v) { + return std::make_shared>(column_name, + std::string(target_val)); + } else { + return std::make_shared>(column_name, std::move(target_val)); + } +} + +template +inline std::shared_ptr LessEq(const std::string& column_name, T target_val) { + if constexpr (std::is_convertible_v) { + return std::make_shared>(column_name, + std::string(target_val)); + } else { + return std::make_shared>(column_name, std::move(target_val)); + } +} + +template +inline std::shared_ptr Like(const std::string& column_name, T target_val) { + if constexpr (std::is_convertible_v) { + return std::make_shared(column_name, std::string(target_val)); + } else { + THROW_RUNTIME_ERROR("Like pattern must be a string"); + } +} + +template +inline std::shared_ptr NotLike(const std::string& column_name, T target_val) { + if constexpr (std::is_convertible_v) { + return std::make_shared(column_name, std::string(target_val)); + } else { + THROW_RUNTIME_ERROR("Like pattern must be a string"); + } +} + +inline std::shared_ptr And(std::shared_ptr left, std::shared_ptr right) { + return std::make_shared(std::move(left), std::move(right)); +} + +inline std::shared_ptr Or(std::shared_ptr left, std::shared_ptr right) { + return std::make_shared(std::move(left), std::move(right)); +} diff --git a/src/execution/expressions/FilterFunctions.h b/src/execution/expressions/FilterFunctions.h new file mode 100644 index 0000000..c07fbf5 --- /dev/null +++ b/src/execution/expressions/FilterFunctions.h @@ -0,0 +1,229 @@ +#pragma once + +#include "execution/OperatorsBase.h" +#include "execution/expressions/AggExpHelper.h" + +class FilterFunction { +public: + virtual ~FilterFunction() = default; + virtual std::vector Evaluate(const RecordBatch& batch) = 0; +}; + +template +class NotEqFilterFunction : public FilterFunction { +public: + NotEqFilterFunction(size_t column_ind, ValueType target_val) + : column_ind_(column_ind), target_val_(std::move(target_val)) { + } + + std::vector Evaluate(const RecordBatch& batch) override { + std::vector selection_vector; + selection_vector.reserve(batch.num_rows); + + AggExpHelper::IterateColumnData(batch, column_ind_, + [&](const auto& value, size_t, size_t real_ind) { + if (value != target_val_) { + selection_vector.push_back(real_ind); + } + }); + return selection_vector; + } + +private: + size_t column_ind_; + ValueType target_val_; +}; + +template +class EqFilterFunction : public FilterFunction { +public: + EqFilterFunction(size_t column_ind, ValueType target_val) + : column_ind_(column_ind), target_val_(std::move(target_val)) { + } + + std::vector Evaluate(const RecordBatch& batch) override { + std::vector selection_vector; + selection_vector.reserve(batch.num_rows); + + AggExpHelper::IterateColumnData(batch, column_ind_, + [&](const auto& value, size_t, size_t real_ind) { + if (value == target_val_) { + selection_vector.push_back(real_ind); + } + }); + return selection_vector; + } + +private: + size_t column_ind_; + ValueType target_val_; +}; + +template +class GreaterFilterFunction : public FilterFunction { +public: + GreaterFilterFunction(size_t column_ind, ValueType target_val) + : column_ind_(column_ind), target_val_(std::move(target_val)) { + } + + std::vector Evaluate(const RecordBatch& batch) override { + std::vector selection_vector; + selection_vector.reserve(batch.num_rows); + + AggExpHelper::IterateColumnData(batch, column_ind_, + [&](const auto& value, size_t, size_t real_ind) { + if (value > target_val_) { + selection_vector.push_back(real_ind); + } + }); + return selection_vector; + } + +private: + size_t column_ind_; + ValueType target_val_; +}; + +template +class GreaterEqFilterFunction : public FilterFunction { +public: + GreaterEqFilterFunction(size_t column_ind, ValueType target_val) + : column_ind_(column_ind), target_val_(std::move(target_val)) { + } + + std::vector Evaluate(const RecordBatch& batch) override { + std::vector selection_vector; + selection_vector.reserve(batch.num_rows); + + AggExpHelper::IterateColumnData(batch, column_ind_, + [&](const auto& value, size_t, size_t real_ind) { + if (value >= target_val_) { + selection_vector.push_back(real_ind); + } + }); + return selection_vector; + } + +private: + size_t column_ind_; + ValueType target_val_; +}; + +template +class LessEqFilterFunction : public FilterFunction { +public: + LessEqFilterFunction(size_t column_ind, ValueType target_val) + : column_ind_(column_ind), target_val_(std::move(target_val)) { + } + + std::vector Evaluate(const RecordBatch& batch) override { + std::vector selection_vector; + selection_vector.reserve(batch.num_rows); + + AggExpHelper::IterateColumnData(batch, column_ind_, + [&](const auto& value, size_t, size_t real_ind) { + if (value <= target_val_) { + selection_vector.push_back(real_ind); + } + }); + return selection_vector; + } + +private: + size_t column_ind_; + ValueType target_val_; +}; + +template +class LikeFilterFunction : public FilterFunction { +public: + LikeFilterFunction(size_t column_ind, ValueType pattern) + : column_ind_(column_ind), pattern_(std::move(pattern)) { + } + + std::vector Evaluate(const RecordBatch& batch) override { + std::vector selection_vector; + selection_vector.reserve(batch.num_rows); + + AggExpHelper::IterateColumnData( + batch, column_ind_, [&](const auto& value, size_t, size_t real_ind) { + if (value.find(pattern_) != std::string::npos) { + selection_vector.push_back(real_ind); + } + }); + return selection_vector; + } + +private: + size_t column_ind_; + ValueType pattern_; +}; + +template +class NotLikeFilterFunction : public FilterFunction { +public: + NotLikeFilterFunction(size_t column_ind, ValueType pattern) + : column_ind_(column_ind), pattern_(std::move(pattern)) { + } + + std::vector Evaluate(const RecordBatch& batch) override { + std::vector selection_vector; + selection_vector.reserve(batch.num_rows); + + AggExpHelper::IterateColumnData( + batch, column_ind_, [&](const auto& value, size_t, size_t real_ind) { + if (value.find(pattern_) == std::string::npos) { + selection_vector.push_back(real_ind); + } + }); + return selection_vector; + } + +private: + size_t column_ind_; + ValueType pattern_; +}; + +class AndFilterFunction : public FilterFunction { +public: + AndFilterFunction(std::unique_ptr left, std::unique_ptr right) + : left_(std::move(left)), right_(std::move(right)) { + } + + std::vector Evaluate(const RecordBatch& batch) override { + std::vector left_res = left_->Evaluate(batch); + std::vector right_res = right_->Evaluate(batch); + + std::vector result; + result.reserve(left_res.size() + right_res.size()); + std::set_intersection(left_res.begin(), left_res.end(), right_res.begin(), right_res.end(), + std::back_inserter(result)); + return result; + } + +private: + std::unique_ptr left_; + std::unique_ptr right_; +}; + +class OrFilterFunction : public FilterFunction { +public: + OrFilterFunction(std::unique_ptr left, std::unique_ptr right) + : left_(std::move(left)), right_(std::move(right)) { + } + + std::vector Evaluate(const RecordBatch& batch) override { + std::vector left_res = left_->Evaluate(batch); + std::vector right_res = right_->Evaluate(batch); + + std::vector result; + result.reserve(left_res.size() + right_res.size()); + std::set_union(left_res.begin(), left_res.end(), right_res.begin(), right_res.end(), + std::back_inserter(result)); + return result; + } + +private: + std::unique_ptr left_; + std::unique_ptr right_; +}; diff --git a/src/execution/expressions/ScalarExpressions.h b/src/execution/expressions/ScalarExpressions.h new file mode 100644 index 0000000..6ac74da --- /dev/null +++ b/src/execution/expressions/ScalarExpressions.h @@ -0,0 +1,323 @@ +#pragma once + +#include "column/Schema.h" +#include "execution/expressions/ScalarFunctions.h" +#include "types/TimeUnit.h" + +#include +#include +#include + +class ScalarExpression { +public: + virtual ~ScalarExpression() = default; + + explicit ScalarExpression(std::string column_name, std::string output_name = "") + : column_name_(std::move(column_name)), output_name_(std::move(output_name)) { + } + + virtual void CollectRequiredColumns(std::vector& required_columns) { + if (!column_name_.empty()) { + required_columns.push_back(column_name_); + } + } + + virtual std::string GetOutputName() const { + return output_name_.empty() ? column_name_ : output_name_; + } + + virtual std::unique_ptr CreateScalarFunction(Schema& child_schema) = 0; + +protected: + std::string column_name_; + std::string output_name_; +}; + +// // TODO: remove ExtractMinute, make Extract with uniform interface (like TruncateFunction) +class ExtractMinuteExpression : public ScalarExpression { +public: + explicit ExtractMinuteExpression(std::string column_name, std::string output_name = "") + : ScalarExpression(std::move(column_name), std::move(output_name)) { + } + + std::string GetOutputName() const override { + return output_name_.empty() ? "minutes_" + column_name_ : output_name_; + } + + std::unique_ptr CreateScalarFunction(Schema& child_schema) override { + size_t ind = child_schema.GetColumnIndexByName(column_name_); + ColumnType type = child_schema.GetColumnTypeByName(column_name_); + + if (type != ColumnType::TIMESTAMP) [[unlikely]] { + THROW_RUNTIME_ERROR("ExtractMinute requires TIMESTAMP column"); + } + + child_schema.AddColumn(GetOutputName(), ColumnType::INT32); + return std::make_unique(ind); + } +}; + +class TimeTruncExpression : public ScalarExpression { +public: + TimeTruncExpression(std::string column_name, std::string date_part, std::string output_name = "") + : ScalarExpression(std::move(column_name), std::move(output_name)), + date_part_str_(std::move(date_part)) {} + + std::string GetOutputName() const override { + return output_name_.empty() ? "date_trunc_" + column_name_ : output_name_; + } + + std::unique_ptr CreateScalarFunction(Schema& child_schema) override { + size_t ind = child_schema.GetColumnIndexByName(column_name_); + ColumnType type = child_schema.GetColumnTypeByName(column_name_); + + if (type != ColumnType::TIMESTAMP && type != ColumnType::DATE) [[unlikely]] { + THROW_RUNTIME_ERROR("DATE_TRUNC requires TIMESTAMP or DATE column"); + } + + TimeUnitType part = TimeUnit::ParseTimeUnit(date_part_str_); + child_schema.AddColumn(GetOutputName(), type); + return std::make_unique(ind, part); + } + +private: + std::string date_part_str_; +}; + +class LengthExpression : public ScalarExpression { +public: + explicit LengthExpression(std::string column_name, std::string output_name = "") + : ScalarExpression(std::move(column_name), std::move(output_name)) { + } + + std::string GetOutputName() const override { + return output_name_.empty() ? "length_" + column_name_ : output_name_; + } + + std::unique_ptr CreateScalarFunction(Schema& child_schema) override { + size_t ind = child_schema.GetColumnIndexByName(column_name_); + ColumnType type = child_schema.GetColumnTypeByName(column_name_); + + if (type != ColumnType::STRING) [[unlikely]] { + THROW_RUNTIME_ERROR("Length requires STRING column"); + } + + child_schema.AddColumn(GetOutputName(), ColumnType::INT64); + return std::make_unique(ind); + } +}; + +template +class ConstantExpression : public ScalarExpression { +public: + ConstantExpression(std::string output_name, T constant) + : ScalarExpression("", std::move(output_name)), constant_(std::move(constant)) { + } + + void CollectRequiredColumns(std::vector&) override { + // yes charonchik bebe + } + + std::unique_ptr CreateScalarFunction(Schema& child_schema) override { + if constexpr (std::is_constructible_v) { + child_schema.AddColumn(GetOutputName(), ColumnType::STRING); + return std::make_unique>(ColumnType::STRING, + std::string(constant_)); + } else { + ColumnType type = ColumnTypeTraits::kType; + child_schema.AddColumn(GetOutputName(), type); + return std::make_unique>>(type, constant_); + } + } + + const T& GetValue() const { + return constant_; + } + +private: + T constant_; +}; + +template +class AddConstantExpression : public ScalarExpression { +public: + AddConstantExpression(std::string column_name, T constant, std::string output_name = "") + : ScalarExpression(std::move(column_name), std::move(output_name)), + constant_(std::move(constant)) { + } + + std::string GetOutputName() const override { + if (!output_name_.empty()) { + return output_name_; + } + return column_name_ + "_plus_" + std::to_string(constant_); + } + + std::unique_ptr CreateScalarFunction(Schema& child_schema) override { + size_t ind = child_schema.GetColumnIndexByName(column_name_); + ColumnType column_type = child_schema.GetColumnTypeByName(column_name_); + + if (column_type == ColumnType::STRING || column_type == ColumnType::CHAR || + column_type == ColumnType::DATE || column_type == ColumnType::TIMESTAMP) { + THROW_RUNTIME_ERROR("AddConst is not supported for non-numeric columns"); + } + + return AggExpHelper::DispatchNumeric( + column_type, [&](ColumnType) -> std::unique_ptr { + child_schema.AddColumn(GetOutputName(), column_type); + return std::make_unique>(ind, constant_); + }); + } + +private: + T constant_; +}; + +class ColumnRefExpression : public ScalarExpression { +public: + explicit ColumnRefExpression(std::string column_name) + : ScalarExpression(std::move(column_name)) { + } + + std::unique_ptr CreateScalarFunction(Schema&) override { + THROW_RUNTIME_ERROR( + "ColumnRefExpression is a marker and should not be evaluated directly."); + } +}; + +template +class CaseWhenExpression : public ScalarExpression { +public: + CaseWhenExpression(std::string output_name, std::shared_ptr cond_expr, + std::shared_ptr true_expr, + std::shared_ptr false_expr, ColumnType return_type) + : ScalarExpression("", std::move(output_name)), + cond_expr_(std::move(cond_expr)), + true_expr_(std::move(true_expr)), + false_expr_(std::move(false_expr)), + return_type_(return_type) { + } + + void CollectRequiredColumns(std::vector& required_columns) override { + cond_expr_->CollectRequiredColumns(required_columns); + true_expr_->CollectRequiredColumns(required_columns); + false_expr_->CollectRequiredColumns(required_columns); + } + + std::unique_ptr CreateScalarFunction(Schema& child_schema) override { + auto cond_func = cond_expr_->CreateFilterFunction(child_schema); + + using ResolvedBranch = std::variant; + + auto resolve_branch = [&](const std::shared_ptr& expr) -> ResolvedBranch { + if (auto col_ref = std::dynamic_pointer_cast(expr)) { + return child_schema.GetColumnIndexByName(col_ref->GetOutputName()); + } else if (auto const_expr = + std::dynamic_pointer_cast>(expr)) { + return const_expr->GetValue(); + } + THROW_RUNTIME_ERROR("CASE WHEN branch must be ColRef or Literal"); + }; + + ResolvedBranch true_resolved = resolve_branch(true_expr_); + ResolvedBranch false_resolved = resolve_branch(false_expr_); + + child_schema.AddColumn(GetOutputName(), return_type_); + + return AggExpHelper::DispatchAll( + return_type_, [&](ColumnType) -> std::unique_ptr { + using ContainerType = ColumnClass::ContainerType; + using RetT = std::decay_t()[0])>; + if constexpr (std::is_constructible_v || + std::is_convertible_v) { + return std::make_unique>( + std::move(cond_func), true_resolved, false_resolved, return_type_); + } else { + THROW_RUNTIME_ERROR("Unreachable: type mismatch between column and constant"); + } + }); + } + +private: + std::shared_ptr cond_expr_; + std::shared_ptr true_expr_; + std::shared_ptr false_expr_; + ColumnType return_type_; +}; + +class RegexpReplaceExpression : public ScalarExpression { +public: + RegexpReplaceExpression(std::string column_name, std::string pattern, std::string replacement, + std::string output_name = "") + : ScalarExpression(std::move(column_name), std::move(output_name)), + pattern_(std::move(pattern)), + replacement_(std::move(replacement)) { + } + + std::unique_ptr CreateScalarFunction(Schema& child_schema) override { + size_t ind = child_schema.GetColumnIndexByName(column_name_); + ColumnType type = child_schema.GetColumnTypeByName(column_name_); + + if (type != ColumnType::STRING) [[unlikely]] { + THROW_RUNTIME_ERROR("REGEXP_REPLACE requires STRING column"); + } + + child_schema.AddColumn(GetOutputName(), ColumnType::STRING); + return std::make_unique(ind, pattern_, replacement_); + } + +private: + std::string pattern_; + std::string replacement_; +}; + +inline std::shared_ptr ExtractMinute(std::string column_name, + std::string output_name = "") { + return std::make_shared(std::move(column_name), + std::move(output_name)); +} + +inline std::shared_ptr TimeTrunc(std::string column_name, std::string date_part, + std::string output_name = "") { + return std::make_shared(std::move(column_name), std::move(date_part), + std::move(output_name)); +} + +inline std::shared_ptr Length(std::string column_name, + std::string output_name = "") { + return std::make_shared(std::move(column_name), std::move(output_name)); +} + +template +inline std::shared_ptr Literal(std::string output_name, T value) { + return std::make_shared>(std::move(output_name), std::move(value)); +} + +template +inline std::shared_ptr AddConst(std::string column_name, T value, + std::string output_name = "") { + return std::make_shared>(std::move(column_name), std::move(value), + std::move(output_name)); +} + +inline std::shared_ptr ColRef(std::string column_name) { + return std::make_shared(std::move(column_name)); +} + +template +inline std::shared_ptr CaseWhen(std::string output_name, + std::shared_ptr cond, + std::shared_ptr true_expr, + std::shared_ptr false_expr, + ColumnType return_type) { + return std::make_shared>(std::move(output_name), + std::move(cond), std::move(true_expr), + std::move(false_expr), return_type); +} + +inline std::shared_ptr RegexpReplace(std::string column_name, std::string pattern, + std::string replacement, + std::string output_name = "") { + return std::make_shared( + std::move(column_name), std::move(pattern), std::move(replacement), std::move(output_name)); +} diff --git a/src/execution/expressions/ScalarFunctions.h b/src/execution/expressions/ScalarFunctions.h new file mode 100644 index 0000000..aa6aadf --- /dev/null +++ b/src/execution/expressions/ScalarFunctions.h @@ -0,0 +1,232 @@ +#pragma once + +#include "execution/OperatorsBase.h" +#include "execution/expressions/AggExpHelper.h" +#include "column/ColumnView.h" +#include "column/TemporalColumn.h" +#include "column/NumericColumn.h" +#include "types/String.h" + +#include + +#include + +class ScalarFunction { +public: + virtual ~ScalarFunction() = default; + virtual std::shared_ptr Evaluate(const RecordBatch& batch) = 0; +}; + +// TODO: remove ExtractMinute, make Extract with uniform interface (like TruncateFunction) +class ExtractMinuteFunction : public ScalarFunction { +public: + explicit ExtractMinuteFunction(size_t col_ind) : col_ind_(col_ind) { + } + + std::shared_ptr Evaluate(const RecordBatch& batch) override { + size_t physical_size = batch.columns[0]->Size(); + std::vector result_data(physical_size); + + AggExpHelper::IterateColumnData( + batch, col_ind_, [&](const auto& value, size_t, size_t real_ind) { + result_data[real_ind] = Timestamp::ExtractMinute(value); + }); + + return std::make_shared(std::move(result_data)); + } + +private: + size_t col_ind_; +}; + +class TimeTruncFunction : public ScalarFunction { +public: + TimeTruncFunction(size_t col_ind, TimeUnitType part) : col_ind_(col_ind), part_(part) { + } + + std::shared_ptr Evaluate(const RecordBatch& batch) override { + size_t physical_size = batch.columns[0]->Size(); + std::vector result_data(physical_size); + + AggExpHelper::IterateColumnData( + batch, col_ind_, [&](const auto& value, size_t, size_t real_ind) { + result_data[real_ind] = Timestamp::Truncate(value, part_); + }); + + return std::make_shared(std::move(result_data)); + } + +private: + size_t col_ind_; + TimeUnitType part_; +}; + +class LengthFunction : public ScalarFunction { +public: + explicit LengthFunction(size_t col_ind) : col_ind_(col_ind) { + } + + std::shared_ptr Evaluate(const RecordBatch& batch) override { + size_t physical_size = batch.columns[0]->Size(); + std::vector result_data(physical_size); + + AggExpHelper::IterateColumnData( + batch, col_ind_, [&](const auto& value, size_t, size_t real_ind) { + result_data[real_ind] = String::Length(value); + }); + + return std::make_shared(std::move(result_data)); + } + +private: + size_t col_ind_; +}; + +template +class ConstantFunction : public ScalarFunction { + using ValueType = ColumnClass::ValueType; + using BuilderType = BuilderTypeTrait::Type; + +public: + ConstantFunction(ColumnType type, ValueType constant) + : type_(type), constant_(std::move(constant)) { + } + + std::shared_ptr Evaluate(const RecordBatch& batch) override { + size_t physical_size = batch.columns.empty() ? batch.num_rows : batch.columns[0]->Size(); + + auto builder = ColumnFactory::MakeColumnBuilder(type_); + auto* typed_builder = static_cast(builder.get()); + + for (size_t i = 0; i < physical_size; ++i) { + typed_builder->AddValue(constant_); + } + + return builder->Finish(); + } + +private: + ColumnType type_; + ValueType constant_; +}; + +template +class AddConstantFunction : public ScalarFunction { + using ValueType = ColumnClass::ValueType; + +public: + AddConstantFunction(size_t col_ind, ValueType constant) + : col_ind_(col_ind), constant_(std::move(constant)) { + } + + std::shared_ptr Evaluate(const RecordBatch& batch) override { + size_t physical_size = batch.columns.empty() ? batch.num_rows : batch.columns[0]->Size(); + std::vector result_data(physical_size); + + AggExpHelper::IterateColumnData( + batch, col_ind_, [&](const auto& value, size_t, size_t real_ind) { + result_data[real_ind] = static_cast(value + constant_); + }); + + return std::make_shared(std::move(result_data)); + } + +private: + size_t col_ind_; + ValueType constant_; +}; + +template +class CaseWhenFunction : public ScalarFunction { + using ContainerType = ColumnClass::ContainerType; + using ReturnType = std::decay_t()[0])>; + using BuilderType = BuilderTypeTrait::Type; + + using ResolvedBranch = + std::variant; // либо индекс колонки, либо константа + using ViewType = ViewVariant; + +public: + CaseWhenFunction(std::unique_ptr cond_func, ResolvedBranch true_branch, + ResolvedBranch false_branch, ColumnType type) + : cond_func_(std::move(cond_func)), + true_branch_(std::move(true_branch)), + false_branch_(std::move(false_branch)), + type_(type) { + } + + std::shared_ptr Evaluate(const RecordBatch& batch) override { + std::vector true_indices = cond_func_->Evaluate(batch); + + auto build_view = [&](const ResolvedBranch& b) -> ViewType { + if (std::holds_alternative(b)) { + size_t col_ind = std::get(b); + auto* col = static_cast(batch.columns[col_ind].get()); + return FlatColumnView(&col->GetData()); + } else { + return ConstColumnView(std::get(b)); + } + }; + + ViewType true_view = build_view(true_branch_); + ViewType false_view = build_view(false_branch_); + + std::shared_ptr builder = ColumnFactory::MakeColumnBuilder(type_); + auto* typed_builder = static_cast(builder.get()); + + std::visit( + [&](const ViewType& true_v, const ViewType& false_v) { + size_t physical_size = + batch.columns.empty() ? batch.num_rows : batch.columns[0]->Size(); + size_t cur_ind = 0; + for (size_t real_ind = 0; real_ind < physical_size; ++real_ind) { + if (cur_ind < true_indices.size() && true_indices[cur_ind] == real_ind) { + typed_builder->AddValue(true_v[real_ind]); + ++cur_ind; + } else { + typed_builder->AddValue(false_v[real_ind]); + } + } + }, + true_view, false_view); + + return builder->Finish(); + } + +private: + std::unique_ptr cond_func_; + ResolvedBranch true_branch_; + ResolvedBranch false_branch_; + ColumnType type_; +}; + +class RegexpReplaceFunction : public ScalarFunction { +public: + RegexpReplaceFunction(size_t col_ind, const std::string& pattern, + const std::string& replacement) + : col_ind_(col_ind), regex_(pattern), replacement_(replacement) { + } + + std::shared_ptr Evaluate(const RecordBatch& batch) override { + size_t physical_size = batch.columns.empty() ? batch.num_rows : batch.columns[0]->Size(); + std::vector temp_res(physical_size); + + AggExpHelper::IterateColumnData( + batch, col_ind_, [&](const auto& value, size_t, size_t real_ind) { + std::string str(value); + re2::RE2::GlobalReplace(&str, regex_, replacement_); + temp_res[real_ind] = std::move(str); + }); + + StringColumnBuilder builder; + for (size_t i = 0; i < physical_size; ++i) { + builder.AddValue(temp_res[i]); + } + return builder.Finish(); + } + +private: + size_t col_ind_; + re2::RE2 regex_; + std::string replacement_; +}; diff --git a/src/io/ColumnarReader.cpp b/src/io/ColumnarReader.cpp new file mode 100644 index 0000000..f93ff6b --- /dev/null +++ b/src/io/ColumnarReader.cpp @@ -0,0 +1,150 @@ +#include "column/ColumnFactory.h" +#include "io/ColumnarReader.h" + +#include "utils/Assert.h" +#include "utils/Macro.h" + +#include +#include +#include +#include + +void MetadataTable::ParseMetadata(const std::string& file_path) { + std::ifstream file(file_path, std::ios::binary); + if (!file.is_open()) { + THROW_RUNTIME_ERROR("Could not open file for metadata read"); + } + + file.seekg(0, std::ios::end); + std::streamoff file_size = file.tellg(); + + if (file_size < 24) { // 20 байт мета + 4 байта магии + THROW_RUNTIME_ERROR("File is too small to contain a valid footer"); + } + + file.seekg(-4, std::ios::end); + std::array magic{}; + file.read(magic.data(), magic.size()); + if (!file || std::string(magic.data(), magic.size()) != "TUFF") { + THROW_RUNTIME_ERROR("Invalid file footer magic"); + } + + file.seekg(file_size - 20, std::ios::beg); + file.read(reinterpret_cast(&num_rows_), sizeof(num_rows_)); + + uint64_t metadata_start; + file.read(reinterpret_cast(&metadata_start), sizeof(metadata_start)); + + size_t metadata_size = (file_size - 20) - metadata_start; + std::vector buffer(metadata_size); + file.seekg(metadata_start, std::ios::beg); + file.read(buffer.data(), metadata_size); + + size_t offset = 0; + + auto read_from_buf = [&](void* dest, size_t size) { + std::memcpy(dest, buffer.data() + offset, size); + offset += size; + }; + + uint32_t num_cols; + read_from_buf(&num_cols, sizeof(num_cols)); + + for (uint32_t i = 0; i < num_cols; ++i) { + ColumnMetadata column_meta; + uint32_t name_length; + read_from_buf(&name_length, sizeof(name_length)); + + column_meta.name.resize(name_length); + if (name_length > 0) { + read_from_buf(column_meta.name.data(), name_length); + } + + uint8_t type; + read_from_buf(&type, sizeof(type)); + column_meta.type = static_cast(type); + + uint32_t num_chunks; + read_from_buf(&num_chunks, sizeof(num_chunks)); + + column_meta.offsets.reserve(num_chunks); + column_meta.sizes.reserve(num_chunks); + + for (uint32_t j = 0; j < num_chunks; ++j) { + uint64_t chunk_offset, chunk_size; + read_from_buf(&chunk_offset, sizeof(chunk_offset)); + read_from_buf(&chunk_size, sizeof(chunk_size)); + + column_meta.offsets.push_back(chunk_offset); + column_meta.sizes.push_back(chunk_size); + } + column_meta_.push_back(std::move(column_meta)); + } +} + +ColumnarReader::ColumnarReader(const std::string& file_name, + std::shared_ptr meta) + : metadata_(std::move(meta)) { + fd_ = open(file_name.c_str(), O_RDONLY); + if (fd_ < 0) { + THROW_RUNTIME_ERROR("Could not open file for reading data: " + file_name); + } +} + +ColumnarReader::~ColumnarReader() { + if (fd_ >= 0) { + close(fd_); + } +} + +ColumnType ColumnarReader::GetColumnTypeByName(const std::string& name) const { + for (const auto& meta : metadata_->GetColumns()) { + if (meta.name == name) { + return meta.type; + } + } + THROW_RUNTIME_ERROR("Column " + name + " not found"); +} + +void ColumnarReader::ReadRawColumnData(size_t column_index, size_t chunk_index, + std::vector& buffer) const { + const ColumnMetadata& meta = (*metadata_)[column_index]; + + if (column_index >= metadata_->GetNumColumns()) { + THROW_RUNTIME_ERROR("Column index out of range"); + } + if (chunk_index >= meta.offsets.size()) { + THROW_RUNTIME_ERROR("Chunk index out of range"); + } + + uint64_t offset = meta.offsets[chunk_index]; + uint64_t size = meta.sizes[chunk_index]; + + buffer.resize(size); + ssize_t bytes_read = pread(fd_, buffer.data(), size, offset); + if (bytes_read != static_cast(size)) { + THROW_RUNTIME_ERROR("Failed to read data"); + } +} + +std::shared_ptr ColumnarReader::GetColumnData(size_t column_index, + size_t chunk_index) const { + std::vector raw_data; + ReadRawColumnData(column_index, chunk_index, raw_data); + + ColumnType type = (*metadata_)[column_index].type; + std::shared_ptr column = ColumnFactory::MakeColumn(type); + +#define HANDLE_TYPE(ENUM_VAL, STR_VAL, CLASS_TYPE) \ + case ColumnType::ENUM_VAL: \ + std::static_pointer_cast(column)->ReadFromBuffer(raw_data); \ + break; + + switch (type) { + FOR_EACH_COLUMN_TYPE(HANDLE_TYPE) + default: THROW_RUNTIME_ERROR("Unknown column type"); + } +#undef HANDLE_TYPE + + return column; +} diff --git a/src/io/ColumnarReader.h b/src/io/ColumnarReader.h new file mode 100644 index 0000000..73d69ad --- /dev/null +++ b/src/io/ColumnarReader.h @@ -0,0 +1,53 @@ +#pragma once + +#include "types/ColumnType.h" +#include "column/Column.h" + +#include +#include +#include + +class MetadataTable { +public: + explicit MetadataTable(const std::string& file_path) { + ParseMetadata(file_path); + } + + const std::vector& GetColumns() const { + return column_meta_; + } + uint64_t GetNumRows() const { + return num_rows_; + } + size_t GetNumColumns() const { + return column_meta_.size(); + } + const ColumnMetadata& operator[](size_t ind) const { + return column_meta_[ind]; + } + +private: + void ParseMetadata(const std::string& file_path); + + std::vector column_meta_; + uint64_t num_rows_ = 0; +}; + +class ColumnarReader { +public: + ColumnarReader(const std::string& file_name, std::shared_ptr meta); + ~ColumnarReader(); + + uint64_t GetNumRows() const { + return metadata_->GetNumRows(); + } + + ColumnType GetColumnTypeByName(const std::string& name) const; + void ReadRawColumnData(size_t column_index, size_t chunk_index, + std::vector& buffer) const; + std::shared_ptr GetColumnData(size_t column_index, size_t chunk_index) const; + +private: + int fd_ = -1; + std::shared_ptr metadata_; +}; diff --git a/src/io/ColumnarWriter.cpp b/src/io/ColumnarWriter.cpp new file mode 100644 index 0000000..64296f1 --- /dev/null +++ b/src/io/ColumnarWriter.cpp @@ -0,0 +1,138 @@ +#include "io/ColumnarWriter.h" + +#include "column/NumericColumn.h" +#include "column/StringColumn.h" +#include "column/CharColumn.h" +#include "column/TemporalColumn.h" + +#include "utils/Macro.h" + +ColumnarWriter::ColumnarWriter(const std::string& file_path) { + file_.open(file_path, std::ios::binary); + if (!file_.is_open()) { + THROW_RUNTIME_ERROR("Could not open file for writing"); + } +} + +void ColumnarWriter::WriteHeader(const VectorOfStrings& column_names, + const std::vector& column_types) { + if (header_written_) { + THROW_RUNTIME_ERROR("Header already written"); + } + + file_.write("TUFF", 4); + current_offset_ = 4; + + for (size_t i = 0; i < column_names.Size(); ++i) { + ColumnMetadata column_meta; + column_meta.name = column_names.GetString(i); + column_meta.type = column_types[i]; + metadata_.push_back(column_meta); + } + + header_written_ = true; +} + +void ColumnarWriter::WriteBatch(const std::vector>& columns) { + if (!header_written_) { + THROW_RUNTIME_ERROR("Header must be written before writing batches"); + } + if (columns.size() != metadata_.size()) { + THROW_RUNTIME_ERROR("Number of columns does not match metadata"); + } + for (size_t i = 0; i < columns.size(); ++i) { + uint64_t offset = current_offset_; + uint64_t size = 0; + + ColumnType type = metadata_[i].type; + + if (type == ColumnType::STRING) { + const auto* data = static_cast(columns[i]->GetRawData()); + const std::vector& offsets = data->GetOffsets(); + const std::vector& raw = data->GetData(); + + uint64_t total_size = 0; + std::vector buffer; + size_t h = columns[i]->Size(); + size_t total_len = 0; + for (size_t k = 0; k < h; ++k) { + total_len += sizeof(uint32_t) + (offsets[k + 1] - offsets[k]); + } + buffer.reserve(total_len); + for (size_t k = 0; k < h; ++k) { + size_t start = offsets[k]; + size_t end = offsets[k + 1]; + uint32_t length = end - start; + buffer.insert(buffer.end(), reinterpret_cast(&length), + reinterpret_cast(&length) + sizeof(length)); + if (length > 0) { + buffer.insert(buffer.end(), raw.begin() + start, raw.begin() + end); + } + total_size += sizeof(length) + length; + } + file_.write(buffer.data(), buffer.size()); + size = total_size; + } else { + size_t elem_size = 0; +#define HANDLE_TYPE(ENUM_VAL, STR_VAL, CLASS_TYPE) \ + case ColumnType::ENUM_VAL: { \ + elem_size = sizeof(CLASS_TYPE::ValueType); \ + size = columns[i]->Size() * elem_size; \ + if (size > 0) { \ + auto* vec = static_cast(columns[i]->GetRawData()); \ + file_.write(reinterpret_cast(vec->data()), size); \ + } \ + break; \ + } + + switch (type) { + FOR_NUMERIC_COLUMN_TYPE(HANDLE_TYPE) + default: break; + } +#undef HANDLE_TYPE + } + + metadata_[i].offsets.push_back(offset); + metadata_[i].sizes.push_back(size); + current_offset_ += size; + } + if (!columns.empty()) { + num_rows_ += columns[0]->Size(); + } +} + +void ColumnarWriter::Finalize() && { + if (!header_written_) { + THROW_RUNTIME_ERROR("Header must be written before finalizing"); + } + + WriteMetadata(); + + file_.write("TUFF", 4); +} + +void ColumnarWriter::WriteMetadata() { + uint64_t metadata_start = current_offset_; + + uint32_t num_columns = metadata_.size(); + file_.write(reinterpret_cast(&num_columns), sizeof(num_columns)); + + for (const auto& meta : metadata_) { + uint32_t name_length = meta.name.size(); + file_.write(reinterpret_cast(&name_length), sizeof(name_length)); + file_.write(meta.name.data(), name_length); + + uint8_t type = static_cast(meta.type); + file_.write(reinterpret_cast(&type), sizeof(type)); + + uint32_t num_batches = meta.offsets.size(); + file_.write(reinterpret_cast(&num_batches), sizeof(num_batches)); + for (size_t i = 0; i < num_batches; ++i) { + file_.write(reinterpret_cast(&meta.offsets[i]), sizeof(uint64_t)); + file_.write(reinterpret_cast(&meta.sizes[i]), sizeof(uint64_t)); + } + } + + file_.write(reinterpret_cast(&num_rows_), sizeof(num_rows_)); + file_.write(reinterpret_cast(&metadata_start), sizeof(metadata_start)); +} diff --git a/src/io/ColumnarWriter.h b/src/io/ColumnarWriter.h new file mode 100644 index 0000000..6648fbf --- /dev/null +++ b/src/io/ColumnarWriter.h @@ -0,0 +1,46 @@ +#pragma once + +#include "types/ColumnType.h" +#include "column/Column.h" + +#include "utils/VectorOfStrings.h" + +#include +#include +#include +#include + +class ColumnarWriter { +public: + explicit ColumnarWriter(const std::string& file_path); + + void WriteHeader(const VectorOfStrings& column_names, + const std::vector& column_types); + + void WriteBatch(const std::vector>& columns); + + // File structure: + // - magic "TUFF" + // - columns by batches + // - metadata: + // - number of columns + // - for each column: + // - name length + // - name + // - type + // - number of chunks + // - for each chunk: offset, size + // - number of rows + // - metadata start offset + // - magic "TUFF" + void Finalize() &&; + +private: + void WriteMetadata(); + + std::ofstream file_; + std::vector metadata_; + bool header_written_ = false; + uint64_t current_offset_ = 0; + uint64_t num_rows_ = 0; +}; \ No newline at end of file diff --git a/src/io/CsvReader.cpp b/src/io/CsvReader.cpp new file mode 100644 index 0000000..610536c --- /dev/null +++ b/src/io/CsvReader.cpp @@ -0,0 +1,60 @@ +#include "io/CsvReader.h" +#include "utils/Macro.h" + +#include +#include + +CsvReader::CsvReader(const std::string& file_path, char delim) : tokenizer_(file_path, delim) { +} + +void CsvReader::ReadHeader(VectorOfStrings& header_names, std::vector& header_types) { + VectorOfStrings2D raw_header; + tokenizer_.GetNextRow(raw_header); + tokenizer_.ResetLineFlag(); + expected_columns_ = raw_header.Width(); + + header_names.Clear(); + header_types.clear(); + + for (size_t i = 0; i < expected_columns_; ++i) { + std::string_view token = raw_header.GetString(i); + size_t colon_pos = token.find(':'); + if (colon_pos == std::string_view::npos) { + THROW_RUNTIME_ERROR("Invalid header token: " + std::string(token)); + } + std::string_view type_str = token.substr(0, colon_pos); + std::string_view name_str = token.substr(colon_pos + 1); + +#define HANDLE_TYPE(ENUM_VAL, STR_VAL, CLASS_TYPE) \ + if (type_str == STR_VAL) { \ + header_names.PushBack(name_str); \ + header_types.push_back(ColumnType::ENUM_VAL); \ + } else + + FOR_EACH_COLUMN_TYPE(HANDLE_TYPE) { + THROW_RUNTIME_ERROR("Invalid header token: " + std::string(token)); + } +#undef HANDLE_TYPE + } +} + +bool CsvReader::ReadRow(VectorOfStrings2D& row) { + if (tokenizer_.IsEOF()) { + return false; + } + + tokenizer_.GetNextRow(row); + + if (row.ColumnsInCurrentLine() == 0 && tokenizer_.IsEOF()) { + return false; + } + + if (expected_columns_ != 0 && row.ColumnsInCurrentLine() != expected_columns_) { + THROW_RUNTIME_ERROR( + "CSV row width mismatch: expected " + std::to_string(expected_columns_) + ", got " + + std::to_string(row.ColumnsInCurrentLine()) + " row: " + std::to_string(row.Height())); + } + + tokenizer_.ResetLineFlag(); + return true; +} diff --git a/src/io/CsvReader.h b/src/io/CsvReader.h new file mode 100644 index 0000000..a23f7be --- /dev/null +++ b/src/io/CsvReader.h @@ -0,0 +1,20 @@ +#pragma once + +#include "types/ColumnType.h" +#include "io/CsvTokenizer.h" +#include "utils/VectorOfStrings.h" + +#include +#include + +class CsvReader { +public: + CsvReader(const std::string& file_path, char delim = ','); + + void ReadHeader(VectorOfStrings& header_names, std::vector& header_types); + bool ReadRow(VectorOfStrings2D& row); + +private: + CsvTokenizer tokenizer_; + size_t expected_columns_ = 0; +}; diff --git a/src/io/CsvToColumnar.cpp b/src/io/CsvToColumnar.cpp new file mode 100644 index 0000000..7578cb4 --- /dev/null +++ b/src/io/CsvToColumnar.cpp @@ -0,0 +1,99 @@ +#include "column/Schema.h" +#include "column/ColumnFactory.h" + +#include "io/CsvToColumnar.h" +#include "io/CsvReader.h" +#include "io/ColumnarWriter.h" + +#include +#include + +void WriteToColumnar(const std::string& columnar_file_path, const VectorOfStrings& column_names, + const std::vector& column_types, CsvReader& reader, + std::vector>& builders) { + ColumnarWriter writer(columnar_file_path); + writer.WriteHeader(column_names, column_types); + + static constexpr size_t kRowsPerChunk = 8192; + static constexpr size_t kBytePerCache = 1 << 20; + + bool end_flag = true; + VectorOfStrings2D column_batch; + size_t accumulated_rows = 0; + + while (end_flag) { + column_batch.Clear(); + size_t cache_rows = 0; + + while (column_batch.ApproxByteSize() < kBytePerCache && accumulated_rows + cache_rows < kRowsPerChunk) { + if (reader.ReadRow(column_batch)) { + column_batch.StartNewLine(); + ++cache_rows; + } else { + end_flag = false; + break; + } + } + + if (cache_rows == 0) { + break; + } + + size_t batch_width = column_batch.Width(); + for (size_t j = 0; j < batch_width; ++j) { + builders[j]->AddBatch(column_batch, j); + } + + accumulated_rows += cache_rows; + + if (accumulated_rows >= kRowsPerChunk || !end_flag) { + std::vector> columns; + for (auto& builder : builders) { + columns.push_back(builder->Finish()); + } + if (!columns.empty() && columns[0]->Size() > 0) { + writer.WriteBatch(columns); + } + accumulated_rows = 0; + } + } + + std::move(writer).Finalize(); +} + +void CsvToColumnar::Convert(const std::string& csv_file_path, + const std::string& columnar_file_path) { + CsvReader reader(csv_file_path); + VectorOfStrings column_names; + std::vector column_types; + reader.ReadHeader(column_names, column_types); + + std::vector> builders; + for (ColumnType type : column_types) { + builders.push_back(ColumnFactory::MakeColumnBuilder(type)); + } + + WriteToColumnar(columnar_file_path, column_names, column_types, reader, builders); +} + +void CsvToColumnar::ConvertWithSchema(const std::string& csv_file_path, + const std::string& columnar_file_path, + const std::string& schema_file_path) { + CsvReader reader(csv_file_path); + Schema schema = Schema::DeserializeSchema(schema_file_path); + const std::vector fields = schema.GetFields(); + + std::vector> builders; + VectorOfStrings column_names; + std::vector column_types; + + for (const Field& field : fields) { + std::string_view header = field.name; + ColumnType type = field.type; + column_names.PushBack(header); + column_types.push_back(type); + builders.push_back(ColumnFactory::MakeColumnBuilder(type)); + } + + WriteToColumnar(columnar_file_path, column_names, column_types, reader, builders); +} diff --git a/src/io/CsvToColumnar.h b/src/io/CsvToColumnar.h new file mode 100644 index 0000000..96199fc --- /dev/null +++ b/src/io/CsvToColumnar.h @@ -0,0 +1,11 @@ +#pragma once + +#include + +class CsvToColumnar { +public: + static void Convert(const std::string& csv_file_path, const std::string& columnar_file_path); + static void ConvertWithSchema(const std::string& csv_file_path, + const std::string& columnar_file_path, + const std::string& schema_file_path); +}; diff --git a/src/io/CsvTokenizer.cpp b/src/io/CsvTokenizer.cpp new file mode 100644 index 0000000..116b4e7 --- /dev/null +++ b/src/io/CsvTokenizer.cpp @@ -0,0 +1,126 @@ +#include "io/CsvTokenizer.h" + +CsvTokenizer::CsvTokenizer(const std::string& file_path, char delim) + : file_(file_path), buffer_(kBufferSize), delim_(delim) { + if (!file_.is_open()) { + THROW_RUNTIME_ERROR("Could not open CSV file"); + } + + is_whitespace_[static_cast(' ')] = true; + is_whitespace_[static_cast('\t')] = true; + is_whitespace_[static_cast('\r')] = true; +} + +bool CsvTokenizer::RefillBuffer(char& ch) { + if (eof_reached_) { + return false; + } + file_.read(buffer_.data(), buffer_.size()); + buffer_end_ = file_.gcount(); + buffer_pos_ = 0; + if (buffer_end_ == 0) { + eof_reached_ = true; + return false; + } + ch = buffer_[buffer_pos_++]; + return true; +} + +void CsvTokenizer::GetNextRow(VectorOfStrings2D& rows) { + bool token_started = false; + bool in_quotes = false; + + auto start_token_if_needed = [&]() { + if (!token_started) { + rows.StartAddString(); + token_started = true; + } + }; + + while (true) { + if (buffer_pos_ >= buffer_end_) { + char tmp; + if (!RefillBuffer(tmp)) { + break; + } + --buffer_pos_; + } + + char ch = buffer_[buffer_pos_++]; + + if (!token_started && !in_quotes && is_whitespace_[static_cast(ch)]) { + continue; + } + + start_token_if_needed(); + if (in_quotes) { + if (ch == '"') { + bool is_escaped_quote = false; + if (buffer_pos_ < buffer_end_) { + if (buffer_[buffer_pos_] == '"') { + is_escaped_quote = true; + } + } else { + char tmp; + if (RefillBuffer(tmp)) { + --buffer_pos_; + if (buffer_[buffer_pos_] == '"') { + is_escaped_quote = true; + } + } + } + + if (is_escaped_quote) { + rows.ContinueAddString('"'); + buffer_pos_++; + } else { + in_quotes = false; + } + } else if (ch != '\r') { + rows.ContinueAddString(ch); + } + } else { + if (ch == '"') { + in_quotes = true; + } else if (ch == delim_ || ch == '\n') { + while (!rows.EmptyLastString() && + is_whitespace_[static_cast( + rows.BackLastString())]) { + rows.PopLastChar(); + } + + rows.EndAddString(); + token_started = false; + + if (ch == '\n') { + end_of_line_ = true; + return; + } + } else if (ch != '\r') { + rows.ContinueAddString(ch); + } + } + } + + if (token_started || !eof_reached_) { + start_token_if_needed(); + while (!rows.EmptyLastString() && + is_whitespace_[static_cast(rows.BackLastString())]) { + rows.PopLastChar(); + } + rows.EndAddString(); + } + end_of_line_ = true; +} + +bool CsvTokenizer::IsEndOfLine() const { + return end_of_line_; +} + +bool CsvTokenizer::IsEOF() const { + return eof_reached_; +} + +void CsvTokenizer::ResetLineFlag() { + end_of_line_ = false; +} diff --git a/src/read_write/CSVTokenizer.h b/src/io/CsvTokenizer.h similarity index 52% rename from src/read_write/CSVTokenizer.h rename to src/io/CsvTokenizer.h index b521661..a0348d2 100644 --- a/src/read_write/CSVTokenizer.h +++ b/src/io/CsvTokenizer.h @@ -1,30 +1,24 @@ -// -// Created by ragnarokk on 03.01.2026. -// +#pragma once -#ifndef COLUMNAR_ENGINE_CSVTOKENIZER_H -#define COLUMNAR_ENGINE_CSVTOKENIZER_H - -#include "VectorOfStrings.h" +#include "utils/VectorOfStrings.h" #include #include -class CSVTokenizer { +class CsvTokenizer { public: - CSVTokenizer(const std::string& file_path, char delim = ','); - CSVTokenizer(const CSVTokenizer& other) = delete; - CSVTokenizer& operator=(const CSVTokenizer& other) = delete; + CsvTokenizer(const std::string& file_path, char delim = ','); + CsvTokenizer(const CsvTokenizer& other) = delete; + CsvTokenizer& operator=(const CsvTokenizer& other) = delete; - void GetNextRow(VectorOfStrings2D& vector_of_strings); + void GetNextRow(VectorOfStrings2D& rows); bool IsEndOfLine() const; bool IsEOF() const; void ResetLineFlag(); private: - static constexpr size_t kBufferSize = 1 << 20; + static constexpr size_t kBufferSize = 1 << 27; - bool is_special_[256] = {false}; bool is_whitespace_[256] = {false}; std::ifstream file_; std::vector buffer_; @@ -33,6 +27,7 @@ class CSVTokenizer { char delim_; bool end_of_line_ = false; bool eof_reached_ = false; + bool GetChar(char& ch) { if (buffer_pos_ >= buffer_end_) [[unlikely]] { return RefillBuffer(ch); @@ -43,5 +38,3 @@ class CSVTokenizer { bool RefillBuffer(char& ch); }; - -#endif // COLUMNAR_ENGINE_CSVTOKENIZER_H diff --git a/src/io/test/TestCsvParser.cpp b/src/io/test/TestCsvParser.cpp new file mode 100644 index 0000000..4615117 --- /dev/null +++ b/src/io/test/TestCsvParser.cpp @@ -0,0 +1,138 @@ +// Generated by AI + +#include "io/CsvReader.h" +#include "io/CsvTokenizer.h" + +#include +#include +#include +#include + +namespace fs = std::filesystem; + +namespace { + +class CreateTempCsvFile { +public: + CreateTempCsvFile(const std::string& content, const std::string& filename) { + path_ = "/tmp/" + filename; + std::ofstream out(path_); + out << content; + out.close(); + } + + ~CreateTempCsvFile() { + fs::remove(path_); + } + + std::string path() const { + return path_.string(); + } + +private: + fs::path path_; +}; + +} // namespace + +TEST_CASE("Simple Csv reading", "[CsvReader]") { + CreateTempCsvFile tempFile("INT32:h1,INT32:h2\n1,2", "test_reader_simple.csv"); + CsvReader reader(tempFile.path()); + + VectorOfStrings header_names; + std::vector header_types; + reader.ReadHeader(header_names, header_types); + REQUIRE(header_names.Size() == 2); + REQUIRE(header_names.GetString(0) == std::string_view("h1")); + REQUIRE(header_names.GetString(1) == std::string_view("h2")); + REQUIRE(header_types[0] == ColumnType::INT32); + + VectorOfStrings2D row; + REQUIRE(reader.ReadRow(row)); + REQUIRE(row.ColumnsInCurrentLine() == 2); + REQUIRE(row.GetString(0) == std::string_view("1")); + REQUIRE(row.GetString(1) == std::string_view("2")); + + row.StartNewLine(); // Emulate how batching works + REQUIRE_FALSE(reader.ReadRow(row)); +} + +TEST_CASE("CsvTokenizer does not materialize empty token on hard EOF", "[CsvTokenizer]") { + CreateTempCsvFile tempFile("", "test_tokenizer_hard_eof.csv"); + CsvTokenizer tokenizer(tempFile.path()); + VectorOfStrings2D row; + + const size_t width_before = row.Width(); + tokenizer.GetNextRow(row); + + REQUIRE(row.Width() == width_before); + REQUIRE(tokenizer.IsEOF()); +} + +TEST_CASE("CsvTokenizer keeps last token without trailing newline", "[CsvTokenizer]") { + CreateTempCsvFile tempFile("a,b", "test_tokenizer_no_trailing_newline.csv"); + CsvTokenizer tokenizer(tempFile.path()); + VectorOfStrings2D row; + + tokenizer.GetNextRow(row); + row.StartNewLine(); + tokenizer.GetNextRow(row); + + REQUIRE(row.Width() == 2); + REQUIRE(row.GetString(0) == std::string_view("a")); + REQUIRE(row.GetString(1) == std::string_view("b")); + REQUIRE(tokenizer.IsEOF()); +} + +TEST_CASE("CsvReader does not return fake empty row at EOF", "[CsvReader]") { + CreateTempCsvFile tempFile("INT32:h1,INT32:h2\n1,2", "test_reader_eof.csv"); + CsvReader reader(tempFile.path()); + + VectorOfStrings header_names; + std::vector header_types; + reader.ReadHeader(header_names, header_types); + REQUIRE(header_names.Size() == 2); + + VectorOfStrings2D row; + REQUIRE(reader.ReadRow(row)); + REQUIRE(row.ColumnsInCurrentLine() == 2); + REQUIRE(row.GetString(0) == std::string_view("1")); + REQUIRE(row.GetString(1) == std::string_view("2")); + row.StartNewLine(); + + REQUIRE_FALSE(reader.ReadRow(row)); + REQUIRE(row.ColumnsInCurrentLine() == 0); +} + +TEST_CASE("CsvReader does not add extra row for trailing newline", "[CsvReader]") { + CreateTempCsvFile tempFile("INT32:h1\nx\n", "test_reader_trailing_newline.csv"); + CsvReader reader(tempFile.path()); + + VectorOfStrings header_names; + std::vector header_types; + reader.ReadHeader(header_names, header_types); + + VectorOfStrings2D row; + REQUIRE(reader.ReadRow(row)); + REQUIRE(row.ColumnsInCurrentLine() == 1); + REQUIRE(row.GetString(0) == std::string_view("x")); + row.StartNewLine(); + REQUIRE_FALSE(reader.ReadRow(row)); +} + +TEST_CASE("CsvReader preserves empty fields", "[CsvReader]") { + CreateTempCsvFile tempFile("STRING:a,STRING:b\n,\n", "test_reader_empty_fields.csv"); + CsvReader reader(tempFile.path()); + + VectorOfStrings header_names; + std::vector header_types; + reader.ReadHeader(header_names, header_types); + + VectorOfStrings2D row; + REQUIRE(reader.ReadRow(row)); + REQUIRE(row.ColumnsInCurrentLine() == 2); + REQUIRE(row.GetString(0) == std::string_view("")); + REQUIRE(row.GetString(1) == std::string_view("")); + row.StartNewLine(); + REQUIRE_FALSE(reader.ReadRow(row)); +} diff --git a/src/io/test/TestReaderWriter.cpp b/src/io/test/TestReaderWriter.cpp new file mode 100644 index 0000000..e062e7c --- /dev/null +++ b/src/io/test/TestReaderWriter.cpp @@ -0,0 +1,130 @@ +// Generated by AI + +#include "io/CsvToColumnar.h" +#include "io/ColumnarReader.h" +#include "types/ColumnType.h" + +#include +#include +#include +#include + +namespace fs = std::filesystem; + +std::string CreateTempCSVFile(const std::string& content, + const std::string& filename = "test_rw.csv") { + std::string filepath = "/tmp/" + filename; + std::ofstream file(filepath); + file << content; + file.close(); + return filepath; +} + +void RemoveTempFile(const std::string& filepath) { + if (fs::exists(filepath)) { + fs::remove(filepath); + } +} + +TEST_CASE("CsvToColumnar: Basic write operation", "[CsvToColumnar]") { + std::string csv_content = + "INT32:id,STRING:name\n" + "1,Alice\n" + "2,Bob\n" + "3,Charlie\n"; + std::string csv_path = CreateTempCSVFile(csv_content, "test_CsvToColumnar_basic.csv"); + std::string output_path = "/tmp/test_CsvToColumnar_basic.bin"; + + SECTION("Write CSV to binary format") { + REQUIRE_NOTHROW(CsvToColumnar::Convert(csv_path, output_path)); + REQUIRE(fs::exists(output_path)); + + std::ifstream file(output_path, std::ios::binary); + file.seekg(0, std::ios::end); + size_t file_size = file.tellg(); + REQUIRE(file_size > 0); + + file.seekg(0, std::ios::beg); + char magic[5] = {0}; + file.read(magic, 4); + REQUIRE(std::string(magic) == "TUFF"); + + file.close(); + } + + RemoveTempFile(csv_path); + RemoveTempFile(output_path); +} + +TEST_CASE("CsvToColumnar: Multiple types", "[CsvToColumnar]") { + std::string csv_content = + "INT32:id,STRING:name,FLOAT:score\n" + "1,Alice,95.5\n" + "2,Bob,87.3\n"; + std::string csv_path = CreateTempCSVFile(csv_content, "test_CsvToColumnar_multi.csv"); + std::string output_path = "/tmp/test_CsvToColumnar_multi.bin"; + + CsvToColumnar::Convert(csv_path, output_path); + + REQUIRE(fs::exists(output_path)); + REQUIRE(fs::file_size(output_path) > 0); + + RemoveTempFile(csv_path); + RemoveTempFile(output_path); +} + +TEST_CASE("Reader: Read metadata", "[Reader]") { + std::string csv_content = + "INT32:id,STRING:name\n" + "1,Alice\n" + "2,Bob\n"; + std::string csv_path = CreateTempCSVFile(csv_content, "test_reader_meta.csv"); + std::string output_path = "/tmp/test_reader_meta.bin"; + + CsvToColumnar::Convert(csv_path, output_path); + + SECTION("Read and verify metadata") { + auto meta = std::make_shared(output_path); + const auto& metadata = meta->GetColumns(); + + REQUIRE(metadata.size() == 2); + REQUIRE(metadata[0].name == "id"); + REQUIRE(metadata[0].type == ColumnType::INT32); + REQUIRE(metadata[1].name == "name"); + REQUIRE(metadata[1].type == ColumnType::STRING); + } + + RemoveTempFile(csv_path); + RemoveTempFile(output_path); +} + +TEST_CASE("Reader: Read typed column data", "[Reader]") { + std::string csv_content = + "INT32:id,STRING:city\n" + "1,Moscow\n" + "2,London\n"; + std::string csv_path = CreateTempCSVFile(csv_content, "test_reader_typed.csv"); + std::string output_path = "/tmp/test_reader_typed.bin"; + + CsvToColumnar::Convert(csv_path, output_path); + + auto meta = std::make_shared(output_path); + ColumnarReader reader(output_path, meta); + + SECTION("Read INT32 column") { + auto column = reader.GetColumnData(0, 0); + REQUIRE(column != nullptr); + REQUIRE(column->GetType() == ColumnType::INT32); + REQUIRE(column->Size() == 2); + } + + SECTION("Read STRING column") { + auto column = reader.GetColumnData(1, 0); + REQUIRE(column != nullptr); + REQUIRE(column->GetType() == ColumnType::STRING); + REQUIRE(column->Size() == 2); + } + + RemoveTempFile(csv_path); + RemoveTempFile(output_path); +} \ No newline at end of file diff --git a/src/main.cpp b/src/main.cpp index dd0d7df..6efbd89 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -1,7 +1,8 @@ -#include "CSVToColumnar.h" -#include "ExecutionAPI.h" +#include "io/CsvToColumnar.h" +#include "execution/ExecutionApi.h" #include +#include class Query { public: @@ -18,7 +19,7 @@ class Query { // SELECT COUNT(*) FROM hits WHERE AdvEngineID <> 0; void Query01() { auto df = DataFrame::Select(columnar_file_path_, {}) - .Filter(NotEq("AdvEngineID", 0)) + .Filter(NotEq("AdvEngineID", 0)) .Aggregate({}, {Count()}) .Collect(); @@ -76,7 +77,7 @@ class Query { // COUNT(*) DESC; void Query07() { auto df = DataFrame::Select(columnar_file_path_, {"AdvEngineID"}) - .Filter(NotEq("AdvEngineID", 0)) + .Filter(NotEq("AdvEngineID", 0)) .Aggregate({"AdvEngineID"}, {Count()}) .OrderBy({{"count", true}}) .Collect(); @@ -189,11 +190,15 @@ class Query { df.Display(); } - // TODO: 18 - // SELECT UserID, extract(minute FROM EventTime) AS m, SearchPhrase, COUNT(*) FROM hits GROUP BY // UserID, m, SearchPhrase ORDER BY COUNT(*) DESC LIMIT 10; void Query18() { + auto df = DataFrame::Select(columnar_file_path_, {}) + .Project({ExtractMinute("EventTime", "m")}) + .Aggregate({"UserID", "m", "SearchPhrase"}, {Count()}) + .OrderBy({{"count", true}}, 10) + .Collect(); + df.Display(); } // SELECT UserID FROM hits WHERE UserID = 435090932899640449; @@ -204,7 +209,50 @@ class Query { .Collect(); } - // TODO: 20-23 + // SELECT COUNT(*) FROM hits WHERE URL LIKE '%google%'; + void Query20() { + auto df = DataFrame::Select(columnar_file_path_, {"URL"}) + .Filter(Like("URL", "%google%")) + .Aggregate({}, {Count()}) + .Collect(); + df.Display(); + } + + // SELECT SearchPhrase, MIN(URL), COUNT(*) AS c FROM hits WHERE URL LIKE '%google%' AND + // SearchPhrase <> '' GROUP BY SearchPhrase ORDER BY c DESC LIMIT 10; + void Query21() { + auto df = DataFrame::Select(columnar_file_path_, {"SearchPhrase", "URL"}) + .Filter(NotEq("SearchPhrase", "")) + .Filter(Like("URL", "%google%")) + .Aggregate({"SearchPhrase"}, {Min("URL"), Count("*", "c")}) + .OrderBy({{"c", true}}, 10) + .Collect(); + df.Display(); + } + + // SELECT SearchPhrase, MIN(URL), MIN(Title), COUNT(*) AS c, COUNT(DISTINCT UserID) FROM hits + // WHERE Title LIKE '%Google%' AND URL NOT LIKE '%.google.%' AND SearchPhrase <> '' GROUP BY + // SearchPhrase ORDER BY c DESC LIMIT 10; + void Query22() { + auto df = DataFrame::Select(columnar_file_path_, {"SearchPhrase", "URL", "Title", "UserID"}) + .Filter(NotEq("SearchPhrase", "")) + .Filter(Like("Title", "%Google%")) + .Filter(NotLike("URL", "%.google.%")) + .Aggregate({"SearchPhrase"}, {Min("URL"), Min("Title"), Count("*", "c"), + DistinctCount("UserID")}) + .OrderBy({{"c", true}}, 10) + .Collect(); + df.Display(); + } + + // SELECT * FROM hits WHERE URL LIKE '%google%' ORDER BY EventTime LIMIT 10; + void Query23() { + auto df = DataFrame::Select(columnar_file_path_, {"*"}) + .Filter(Like("URL", "%google%")) + .OrderBy({{"EventTime", false}}, 10) + .Collect(); + df.Display(); + } // SELECT SearchPhrase FROM hits WHERE SearchPhrase <> '' ORDER BY EventTime LIMIT 10; void Query24() { @@ -234,6 +282,306 @@ class Query { df.Display(); } + // SELECT CounterID, AVG(length(URL)) AS l, COUNT(*) AS c FROM hits WHERE URL <> '' GROUP BY + // CounterID HAVING COUNT(*) > 100000 ORDER BY l DESC LIMIT 25; + void Query27() { + auto df = DataFrame::Select(columnar_file_path_, {"CounterID", "URL"}) + .Filter(NotEq("URL", "")) + .Project({Length("URL", "length_URL")}) + .Aggregate({"CounterID"}, {Avg("length_URL", "l"), Count("*", "c")}) + .Filter(Greater("c", static_cast(100000))) + .OrderBy({{"l", true}}, 25) + .Collect(); + + df.Display(); + } + + // SELECT REGEXP_REPLACE(Referer, '^https?://(?:www\.)?([^/]+)/.*$', '\1') AS k, + // AVG(length(Referer)) AS l, COUNT(*) AS c, MIN(Referer) FROM hits WHERE Referer <> '' GROUP BY + // k HAVING COUNT(*) > 100000 ORDER BY l DESC LIMIT 25; + void Query28() { + auto df = + DataFrame::Select(columnar_file_path_, {"Referer"}) + .Filter(NotEq("Referer", "")) + .Project({RegexpReplace("Referer", "^https?://(?:www\\.)?([^/]+)/.*$", "\\1", "k"), + Length("Referer", "length_ref")}) + .Aggregate({"k"}, + {Avg("length_ref", "l"), Count("*", "c"), Min("Referer", "min_ref")}) + .Filter(Greater("c", 100000)) + .OrderBy({{"l", true}}, 25) + .Collect(); + + df.Display(); + } + + // WARNING: DataFrame::Display output does not affect that query + // SELECT SUM(ResolutionWidth), SUM(ResolutionWidth + 1), SUM(ResolutionWidth + 2), + // SUM(ResolutionWidth + 3), SUM(ResolutionWidth + 4), SUM(ResolutionWidth + 5), + // SUM(ResolutionWidth + 6), SUM(ResolutionWidth + 7), SUM(ResolutionWidth + 8), + // SUM(ResolutionWidth + 9), SUM(ResolutionWidth + 10), SUM(ResolutionWidth + 11), + // SUM(ResolutionWidth + 12), SUM(ResolutionWidth + 13), SUM(ResolutionWidth + 14), + // SUM(ResolutionWidth + 15), SUM(ResolutionWidth + 16), SUM(ResolutionWidth + 17), + // SUM(ResolutionWidth + 18), SUM(ResolutionWidth + 19), SUM(ResolutionWidth + 20), + // SUM(ResolutionWidth + 21), SUM(ResolutionWidth + 22), SUM(ResolutionWidth + 23), + // SUM(ResolutionWidth + 24), SUM(ResolutionWidth + 25), SUM(ResolutionWidth + 26), + // SUM(ResolutionWidth + 27), SUM(ResolutionWidth + 28), SUM(ResolutionWidth + 29), + // SUM(ResolutionWidth + 30), SUM(ResolutionWidth + 31), SUM(ResolutionWidth + 32), + // SUM(ResolutionWidth + 33), SUM(ResolutionWidth + 34), SUM(ResolutionWidth + 35), + // SUM(ResolutionWidth + 36), SUM(ResolutionWidth + 37), SUM(ResolutionWidth + 38), + // SUM(ResolutionWidth + 39), SUM(ResolutionWidth + 40), SUM(ResolutionWidth + 41), + // SUM(ResolutionWidth + 42), SUM(ResolutionWidth + 43), SUM(ResolutionWidth + 44), + // SUM(ResolutionWidth + 45), SUM(ResolutionWidth + 46), SUM(ResolutionWidth + 47), + // SUM(ResolutionWidth + 48), SUM(ResolutionWidth + 49), SUM(ResolutionWidth + 50), + // SUM(ResolutionWidth + 51), SUM(ResolutionWidth + 52), SUM(ResolutionWidth + 53), + // SUM(ResolutionWidth + 54), SUM(ResolutionWidth + 55), SUM(ResolutionWidth + 56), + // SUM(ResolutionWidth + 57), SUM(ResolutionWidth + 58), SUM(ResolutionWidth + 59), + // SUM(ResolutionWidth + 60), SUM(ResolutionWidth + 61), SUM(ResolutionWidth + 62), + // SUM(ResolutionWidth + 63), SUM(ResolutionWidth + 64), SUM(ResolutionWidth + 65), + // SUM(ResolutionWidth + 66), SUM(ResolutionWidth + 67), SUM(ResolutionWidth + 68), + // SUM(ResolutionWidth + 69), SUM(ResolutionWidth + 70), SUM(ResolutionWidth + 71), + // SUM(ResolutionWidth + 72), SUM(ResolutionWidth + 73), SUM(ResolutionWidth + 74), + // SUM(ResolutionWidth + 75), SUM(ResolutionWidth + 76), SUM(ResolutionWidth + 77), + // SUM(ResolutionWidth + 78), SUM(ResolutionWidth + 79), SUM(ResolutionWidth + 80), + // SUM(ResolutionWidth + 81), SUM(ResolutionWidth + 82), SUM(ResolutionWidth + 83), + // SUM(ResolutionWidth + 84), SUM(ResolutionWidth + 85), SUM(ResolutionWidth + 86), + // SUM(ResolutionWidth + 87), SUM(ResolutionWidth + 88), SUM(ResolutionWidth + 89) FROM hits; + void Query29() { + auto df = DataFrame::Select(columnar_file_path_, {"ResolutionWidth"}) + .Aggregate({}, {Sum("ResolutionWidth", "a"), Count("*", "b")}) + .Collect(); + + std::shared_ptr cnt = df.GetResult("b"); + std::shared_ptr sum = df.GetResult("a"); + + const auto* cnt_vec = static_cast*>(cnt->GetRawData()); + int64_t delta = (*cnt_vec)[0]; + + int64_t cur = 0; + const auto* sum_vec = static_cast*>(sum->GetRawData()); + cur = (*sum_vec)[0]; + + for (size_t i = 0; i < 90; ++i) { + std::cout << cur << ","; + cur += delta; + } + std::cout << "\n"; + } + + // SELECT SearchEngineID, ClientIP, COUNT(*) AS c, SUM(IsRefresh), AVG(ResolutionWidth) FROM + // hits WHERE SearchPhrase <> '' GROUP BY SearchEngineID, ClientIP ORDER BY c DESC LIMIT 10; + void Query30() { + auto df = DataFrame::Select(columnar_file_path_, + {"SearchEngineID", "ClientIP", "IsRefresh", "ResolutionWidth"}) + .Filter(NotEq("SearchPhrase", "")) + .Aggregate({"SearchEngineID", "ClientIP"}, + {Count("*", "c"), Sum("IsRefresh", "sum_refresh"), + Avg("ResolutionWidth", "avg_res_width")}) + .OrderBy({{"c", true}}, 10) + .Collect(); + + df.Display(); + } + + // SELECT WatchID, ClientIP, COUNT(*) AS c, SUM(IsRefresh), AVG(ResolutionWidth) FROM hits WHERE + // SearchPhrase <> '' GROUP BY WatchID, ClientIP ORDER BY c DESC LIMIT 10; + void Query31() { + auto df = DataFrame::Select(columnar_file_path_, + {"WatchID", "ClientIP", "IsRefresh", "ResolutionWidth"}) + .Filter(NotEq("SearchPhrase", "")) + .Aggregate({"WatchID", "ClientIP"}, + {Count("*", "c"), Sum("IsRefresh", "sum_refresh"), + Avg("ResolutionWidth", "avg_res_width")}) + .OrderBy({{"c", true}}, 10) + .Collect(); + + df.Display(); + } + + // SELECT WatchID, ClientIP, COUNT(*) AS c, SUM(IsRefresh), AVG(ResolutionWidth) FROM hits GROUP + // BY WatchID, ClientIP ORDER BY c DESC LIMIT 10; + void Query32() { + auto df = DataFrame::Select(columnar_file_path_, + {"WatchID", "ClientIP", "IsRefresh", "ResolutionWidth"}) + .Aggregate({"WatchID", "ClientIP"}, + {Count("*", "c"), Sum("IsRefresh", "sum_refresh"), + Avg("ResolutionWidth", "avg_res_width")}) + .OrderBy({{"c", true}}, 10) + .Collect(); + + df.Display(); + } + + // SELECT URL, COUNT(*) AS c FROM hits GROUP BY URL ORDER BY c DESC LIMIT 10; + void Query33() { + auto df = DataFrame::Select(columnar_file_path_, {"URL"}) + .Aggregate({"URL"}, {Count("*", "c")}) + .OrderBy({{"c", true}}, 10) + .Collect(); + df.Display(); + } + + // SELECT 1, URL, COUNT(*) AS c FROM hits GROUP BY 1, URL ORDER BY c DESC LIMIT 10; + void Query34() { + auto df = DataFrame::Select(columnar_file_path_, {"URL"}) + .Aggregate({"URL"}, {Count("*", "c")}) + .OrderBy({{"c", true}}, 10) + .Project({Literal("const_1", 1)}) + .Reorder({"const_1", "URL", "c"}) + .Collect(); + df.Display(); + } + + // SELECT ClientIP, ClientIP - 1, ClientIP - 2, ClientIP - 3, COUNT(*) AS c FROM hits GROUP BY + // ClientIP, ClientIP - 1, ClientIP - 2, ClientIP - 3 ORDER BY c DESC LIMIT 10; + void Query35() { + auto df = DataFrame::Select(columnar_file_path_, {"ClientIP"}) + .Aggregate({"ClientIP"}, {Count("*", "c")}) + .OrderBy({{"c", true}}, 10) + .Project({AddConst("ClientIP", -1, "ClientIP_minus_1"), + AddConst("ClientIP", -2, "ClientIP_minus_2"), + AddConst("ClientIP", -3, "ClientIP_minus_3")}) + .Reorder({"ClientIP", "ClientIP_minus_1", "ClientIP_minus_2", + "ClientIP_minus_3", "c"}) + .Collect(); + + df.Display(); + } + + // SELECT URL, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= + // '2013-07-01' AND EventDate <= '2013-07-31' AND DontCountHits = 0 AND IsRefresh = 0 AND URL <> + // '' GROUP BY URL ORDER BY PageViews DESC LIMIT 10; + void Query36() { + auto df = DataFrame::Select(columnar_file_path_, {"URL"}) + .Filter(NotEq("URL", "")) + .Filter(Eq("CounterID", 62)) + .Filter(GreaterEq("EventDate", Date::Parse("2013-07-01"))) + .Filter(LessEq("EventDate", Date::Parse("2013-07-31"))) + .Filter(Eq("DontCountHits", 0)) + .Filter(Eq("IsRefresh", 0)) + .Aggregate({"URL"}, {Count("*", "PageViews")}) + .OrderBy({{"PageViews", true}}, 10) + .Collect(); + + df.Display(); + } + + // SELECT Title, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= + // '2013-07-01' AND EventDate <= '2013-07-31' AND DontCountHits = 0 AND IsRefresh = 0 AND Title + // <> '' GROUP BY Title ORDER BY PageViews DESC LIMIT 10; + void Query37() { + auto df = DataFrame::Select(columnar_file_path_, {"Title"}) + .Filter(NotEq("Title", "")) + .Filter(Eq("CounterID", 62)) + .Filter(GreaterEq("EventDate", Date::Parse("2013-07-01"))) + .Filter(LessEq("EventDate", Date::Parse("2013-07-31"))) + .Filter(Eq("DontCountHits", 0)) + .Filter(Eq("IsRefresh", 0)) + .Aggregate({"Title"}, {Count("*", "PageViews")}) + .OrderBy({{"PageViews", true}}, 10) + .Collect(); + + df.Display(); + } + + // SELECT URL, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= + // '2013-07-01' AND EventDate <= '2013-07-31' AND IsRefresh = 0 AND IsLink <> 0 AND IsDownload = + // 0 GROUP BY URL ORDER BY PageViews DESC LIMIT 10 OFFSET 1000; + void Query38() { + auto df = DataFrame::Select(columnar_file_path_, {"URL"}) + .Filter(NotEq("IsLink", 0)) + .Filter(Eq("IsDownload", 0)) + .Filter(Eq("CounterID", 62)) + .Filter(GreaterEq("EventDate", Date::Parse("2013-07-01"))) + .Filter(LessEq("EventDate", Date::Parse("2013-07-31"))) + .Filter(Eq("IsRefresh", 0)) + .Aggregate({"URL"}, {Count("*", "PageViews")}) + .OrderBy({{"PageViews", true}}, 10, 1000) + .Collect(); + + df.Display(); + } + + // SELECT TraficSourceID, SearchEngineID, AdvEngineID, CASE WHEN (SearchEngineID = 0 AND + // AdvEngineID = 0) THEN Referer ELSE '' END AS Src, URL AS Dst, COUNT(*) AS PageViews FROM hits + // WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND + // IsRefresh = 0 GROUP BY TraficSourceID, SearchEngineID, AdvEngineID, Src, Dst ORDER BY + // PageViews DESC LIMIT 10 OFFSET 1000; + void Query39() { + auto df = + DataFrame::Select(columnar_file_path_, {}) + .Filter(Eq("CounterID", 62)) + .Filter(GreaterEq("EventDate", Date::Parse("2013-07-01"))) + .Filter(LessEq("EventDate", Date::Parse("2013-07-31"))) + .Filter(Eq("IsRefresh", 0)) + .Project({CaseWhen( + "Src", And(Eq("SearchEngineID", 0), Eq("AdvEngineID", 0)), + ColRef("Referer"), Literal("dummy_name", ""), ColumnType::STRING)}) + .Aggregate({"TraficSourceID", "SearchEngineID", "AdvEngineID", "Src", "URL"}, + {Count("*", "PageViews")}) + .OrderBy({{"PageViews", true}}, 10, 1000) + .Collect(); + + df.Display(); + } + + // SELECT URLHash, EventDate, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate + // >= '2013-07-01' AND EventDate <= '2013-07-31' AND IsRefresh = 0 AND TraficSourceID IN (-1, 6) + // AND RefererHash = 3594120000172545465 GROUP BY URLHash, EventDate ORDER BY PageViews DESC + // LIMIT 10 OFFSET 100; + void Query40() const { + auto df = + DataFrame::Select(columnar_file_path_, {"Title"}) + .Filter(Eq("RefererHash", 3594120000172545465)) + .Filter(Eq("CounterID", 62)) + .Filter(GreaterEq("EventDate", Date::Parse("2013-07-01"))) + .Filter(LessEq("EventDate", Date::Parse("2013-07-31"))) + .Filter(Eq("IsRefresh", 0)) + .Filter(Or(Eq("TraficSourceID", -1), Eq("TraficSourceID", 6))) + .Aggregate({"URLHash", "EventDate"}, {Count("*", "PageViews")}) + .OrderBy({{"PageViews", true}}, 10, 100) + .Collect(); + df.Display(); + } + + // SELECT WindowClientWidth, WindowClientHeight, COUNT(*) AS PageViews FROM hits WHERE CounterID + // = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND IsRefresh = 0 AND + // DontCountHits = 0 AND URLHash = 2868770270353813622 GROUP BY WindowClientWidth, + // WindowClientHeight ORDER BY PageViews DESC LIMIT 10 OFFSET 10000; + void Query41() const { + auto df = + DataFrame::Select(columnar_file_path_, {}) + .Filter(Eq("URLHash", 2868770270353813622)) + .Filter(Eq("CounterID", 62)) + .Filter(GreaterEq("EventDate", Date::Parse("2013-07-01"))) + .Filter(LessEq("EventDate", Date::Parse("2013-07-31"))) + .Filter(Eq("IsRefresh", 0)) + .Filter(Eq("DontCountHits", 0)) + .Aggregate({"WindowClientWidth", "WindowClientHeight"}, {Count("*", "PageViews")}) + .OrderBy({{"PageViews", true}}, 10, 10000) + .Collect(); + + df.Display(); + } + + // SELECT DATE_TRUNC('minute', EventTime) AS M, COUNT(*) AS PageViews FROM hits WHERE CounterID + // = 62 AND EventDate >= '2013-07-14' AND EventDate <= '2013-07-15' AND IsRefresh = 0 AND + // DontCountHits = 0 GROUP BY DATE_TRUNC('minute', EventTime) ORDER BY DATE_TRUNC('minute', + // EventTime) LIMIT 10 OFFSET 1000; + void Query42() const { + auto df = DataFrame::Select(columnar_file_path_, {}) + .Filter(Eq("CounterID", 62)) + .Filter(GreaterEq("EventDate", Date::Parse("2013-07-01"))) + .Filter(LessEq("EventDate", Date::Parse("2013-07-31"))) + .Filter(Eq("IsRefresh", 0)) + .Filter(Eq("DontCountHits", 0)) + .Project({TimeTrunc("EventTime", "minute", "M")}) + .Aggregate({"M"}, {Count("*", "PageViews")}) + .OrderBy({{"M", false}}, 10, 1000) + .Collect(); + + df.Display(); + } + void RunQuery(int query_number) { switch (query_number) { case 0: Query00(); break; @@ -254,12 +602,31 @@ class Query { case 15: Query15(); break; case 16: Query16(); break; case 17: Query17(); break; - // TODO: 18 + case 18: Query18(); break; case 19: Query19(); break; - // TODO: 20-23 + case 20: Query20(); break; + case 21: Query21(); break; + case 22: Query22(); break; + case 23: Query23(); break; case 24: Query24(); break; case 25: Query25(); break; case 26: Query26(); break; + case 27: Query27(); break; + case 28: Query28(); break; + case 29: Query29(); break; + case 30: Query30(); break; + case 31: Query31(); break; + case 32: Query32(); break; + case 33: Query33(); break; + case 34: Query34(); break; + case 35: Query35(); break; + case 36: Query36(); break; + case 37: Query37(); break; + case 38: Query38(); break; + case 39: Query39(); break; + case 40: Query40(); break; + case 41: Query41(); break; + case 42: Query42(); break; default: THROW_RUNTIME_ERROR("Unknown query number: " + std::to_string(query_number)); } @@ -288,8 +655,7 @@ int main(int argc, char** argv) { try { std::cerr << "Converting CSV to columnar format...\n"; const auto start = std::chrono::steady_clock::now(); - CSVToColumnar converter; - converter.ConvertWithSchema(argv[2], argv[3], argv[4]); + CsvToColumnar::ConvertWithSchema(argv[2], argv[3], argv[4]); const auto time = std::chrono::duration(std::chrono::steady_clock::now() - start).count() * 1000; diff --git a/src/read_write/CSVReader.cpp b/src/read_write/CSVReader.cpp deleted file mode 100644 index e4f6eb9..0000000 --- a/src/read_write/CSVReader.cpp +++ /dev/null @@ -1,39 +0,0 @@ -// -// Created by ragnarokk on 05.01.2026. -// - -#include "CSVReader.h" -#include "macro.h" - -#include - -CSVReader::CSVReader(const std::string& file_path, char delim) - : tokenizer_(file_path, delim) { -} - -void CSVReader::ReadHeader(VectorOfStrings2D& header) { - header.Clear(); - tokenizer_.GetNextRow(header); - tokenizer_.ResetLineFlag(); - expected_columns_ = header.Width(); -} - -bool CSVReader::ReadRow(VectorOfStrings2D& row) { - if (tokenizer_.IsEOF()) { - return false; - } - - tokenizer_.GetNextRow(row); - - if (row.ColumnsInCurrentLine() == 0 && tokenizer_.IsEOF()) { - return false; - } - - if (expected_columns_ != 0 && row.ColumnsInCurrentLine() != expected_columns_) { - THROW_RUNTIME_ERROR("CSV row width mismatch: expected " + std::to_string(expected_columns_) + - ", got " + std::to_string(row.ColumnsInCurrentLine()) + " row: " + std::to_string(row.Height())); - } - - tokenizer_.ResetLineFlag(); - return true; -} diff --git a/src/read_write/CSVReader.h b/src/read_write/CSVReader.h deleted file mode 100644 index c4ac68e..0000000 --- a/src/read_write/CSVReader.h +++ /dev/null @@ -1,25 +0,0 @@ -// -// Created by ragnarokk on 05.01.2026. -// - -#ifndef COLUMNAR_ENGINE_RAWCSVREADER_H -#define COLUMNAR_ENGINE_RAWCSVREADER_H - -#include "CSVTokenizer.h" -#include "VectorOfStrings.h" - -#include - -class CSVReader { -public: - CSVReader(const std::string& file_path, char delim = ','); - - void ReadHeader(VectorOfStrings2D& header); - bool ReadRow(VectorOfStrings2D& row); - -private: - CSVTokenizer tokenizer_; - size_t expected_columns_ = 0; -}; - -#endif // COLUMNAR_ENGINE_RAWCSVREADER_H \ No newline at end of file diff --git a/src/read_write/CSVToColumnar.cpp b/src/read_write/CSVToColumnar.cpp deleted file mode 100644 index a61a627..0000000 --- a/src/read_write/CSVToColumnar.cpp +++ /dev/null @@ -1,138 +0,0 @@ -// -// Created by ragnarokk on 05.01.2026. -// - -#include - -#include "CSVToColumnar.h" -#include "CSVReader.h" -#include "ColumnarWriter.h" -#include "Schema.h" -#include "macro.h" -#include "VectorOfStrings.h" - -#include - -void CSVToColumnar::Convert(const std::string& csv_file_path, - const std::string& columnar_file_path) { - CSVReader reader(csv_file_path); - VectorOfStrings2D raw_header; - reader.ReadHeader(raw_header); - - std::vector> columns; - VectorOfStrings2D column_names; - std::vector column_types; - - size_t num_columns = raw_header.Size(); - for (size_t i = 0; i < num_columns; ++i) { - std::string_view token = raw_header.GetString(i); - ParseHeaderToken(token, column_names, column_types); - auto type = column_types.back(); - AddColumn(type, columns); - } - - WriteToColumnar(columnar_file_path, column_names, column_types, reader, columns); -} - -void CSVToColumnar::ConvertWithSchema(const std::string& csv_file_path, - const std::string& columnar_file_path, - const std::string& schema_file_path) { - CSVReader reader(csv_file_path); - Schema schema = Schema::DeserializeSchema(schema_file_path); - const std::vector fields = schema.GetFields(); - - std::vector> columns; - VectorOfStrings2D column_names; - std::vector column_types; - - for (const Field& field : fields) { - std::string_view header = field.name; - ColumnType type = field.type; - column_names.AddString(header); - column_types.push_back(type); - AddColumn(type, columns); - } - - WriteToColumnar(columnar_file_path, column_names, column_types, reader, columns); -} - -void CSVToColumnar::ParseHeaderToken(std::string_view token, VectorOfStrings2D& column_names, - std::vector& column_types) { - size_t colon_pos = token.find(':'); - if (colon_pos == std::string::npos) { - THROW_RUNTIME_ERROR("Invalid header token: " + std::string(token)); - } - std::string_view type_str = token.substr(0, colon_pos); - std::string_view name_str = token.substr(colon_pos + 1); - - // OPTIMIZE: copying in AddString -#define HANDLE_TYPE(ENUM_VAL, STR_VAL, CLASS_TYPE) \ - if (type_str == STR_VAL) { \ - column_names.AddString(name_str); \ - column_types.push_back(ColumnType::ENUM_VAL); \ - } else - FOR_EACH_COLUMN_TYPE(HANDLE_TYPE) { - THROW_RUNTIME_ERROR("Invalid header token: " + std::string(token)); - } -#undef HANDLE_TYPE -} - -void CSVToColumnar::AddColumn(ColumnType type, std::vector>& columns) { - switch (type) { -#define HANDLE_TYPE(ENUM_VAL, STR_VAL, CLASS_TYPE) \ - case ColumnType::ENUM_VAL: columns.push_back(std::make_shared()); break; - - FOR_EACH_COLUMN_TYPE(HANDLE_TYPE) -#undef HANDLE_TYPE - - default: - throw std::runtime_error("Unsupported column type"); - } -} - -void CSVToColumnar::WriteToColumnar(const std::string& columnar_file_path, - const VectorOfStrings2D& column_names, - const std::vector& column_types, CSVReader& reader, - const std::vector>& columns) { - ColumnarWriter writer(columnar_file_path); - writer.WriteHeader(column_names, column_types); - - static constexpr size_t kByteSize = 1 << 20; - bool end_flag = true; - VectorOfStrings2D column_batch; - - while (end_flag) { - size_t cur_batch_size = 0; - column_batch.Clear(); - - while (column_batch.ApproxByteSize() < kByteSize) { - bool has_row = reader.ReadRow(column_batch); - if (has_row) { - column_batch.StartNewLine(); - ++cur_batch_size; - } else { - end_flag = false; - break; - } - } - - if (cur_batch_size == 0) { - break; - } - - size_t batch_width = column_batch.Width(); - for (size_t j = 0; j < batch_width; ++j) { - columns[j]->AddBatch(column_batch, j); - } - - if (!columns.empty()) { - writer.WriteBatch(columns); - } - for (auto& col : columns) { - col->Clear(); - } - } - - - writer.Finalize(); -} diff --git a/src/read_write/CSVToColumnar.h b/src/read_write/CSVToColumnar.h deleted file mode 100644 index 80d6dfb..0000000 --- a/src/read_write/CSVToColumnar.h +++ /dev/null @@ -1,31 +0,0 @@ -// -// Created by ragnarokk on 05.01.2026. -// - -#ifndef COLUMNAR_ENGINE_CSVTOCOLUMNAR_H -#define COLUMNAR_ENGINE_CSVTOCOLUMNAR_H - -#include -#include - -#include "CSVReader.h" -#include "Types.h" - -class CSVToColumnar { -public: - void Convert(const std::string& csv_file_path, const std::string& columnar_file_path); - void ConvertWithSchema(const std::string& csv_file_path, const std::string& columnar_file_path, - const std::string& schema_file_path); - -private: - void ParseHeaderToken(std::string_view token, VectorOfStrings2D& column_names, - std::vector& column_types); - void AddColumn(ColumnType type, std::vector>& columns); - - void WriteToColumnar(const std::string& columnar_file_path, - const VectorOfStrings2D& column_names, - const std::vector& column_types, CSVReader& reader, - const std::vector>& columns); -}; - -#endif // COLUMNAR_ENGINE_CSVTOCOLUMNAR_H \ No newline at end of file diff --git a/src/read_write/CSVTokenizer.cpp b/src/read_write/CSVTokenizer.cpp deleted file mode 100644 index 227c069..0000000 --- a/src/read_write/CSVTokenizer.cpp +++ /dev/null @@ -1,160 +0,0 @@ -// -// Created by ragnarokk on 03.01.2026. -// - -#include "CSVTokenizer.h" - -#include "macro.h" - -CSVTokenizer::CSVTokenizer(const std::string& file_path, char delim) - : file_(file_path), buffer_(kBufferSize), delim_(delim) { - if (!file_.is_open()) { - THROW_RUNTIME_ERROR("Could not open CSV file"); - } - is_special_[static_cast(delim_)] = true; - is_special_[static_cast('\n')] = true; - is_special_[static_cast('"')] = true; - - is_whitespace_[static_cast(' ')] = true; - is_whitespace_[static_cast('\t')] = true; - is_whitespace_[static_cast('\r')] = true; -} - -bool CSVTokenizer::RefillBuffer(char& ch) { - if (eof_reached_) { - return false; - } - file_.read(buffer_.data(), buffer_.size()); - buffer_end_ = file_.gcount(); - buffer_pos_ = 0; - if (buffer_end_ == 0) { - eof_reached_ = true; - return false; - } - ch = buffer_[buffer_pos_++]; - return true; -} - -void CSVTokenizer::GetNextRow(VectorOfStrings2D& vector_of_strings) { - char ch; - bool in_quotes = false; - bool token_started = false; - bool token_materialized = false; - - auto start_token_if_needed = [&]() { - if (!token_materialized) { - vector_of_strings.StartAddString(); - token_materialized = true; - } - }; - - while (true) { - if (buffer_pos_ >= buffer_end_) [[unlikely]] { - char tmp; - if (!RefillBuffer(tmp)) { - break; - } - --buffer_pos_; - } - - if (!in_quotes) { - if (!token_started) { - while (buffer_pos_ < buffer_end_ && (buffer_[buffer_pos_] == ' ' || buffer_[buffer_pos_] == '\t')) { - ++buffer_pos_; - } - } - - size_t start = buffer_pos_; - while (buffer_pos_ < buffer_end_) { - if (is_special_[static_cast(buffer_[buffer_pos_])]) { - break; - } - ++buffer_pos_; - } - if (buffer_pos_ > start) { - start_token_if_needed(); - vector_of_strings.ContinueAddString(std::string_view(buffer_.data() + start, buffer_pos_ - start)); - token_started = true; - } - if (buffer_pos_ >= buffer_end_) { - continue; - } - } - - ch = buffer_[buffer_pos_++]; - - if (!token_started && (ch == ' ' || ch == '\t')) { - continue; - } - - if (ch == '"' && !in_quotes && !token_started) { - start_token_if_needed(); - in_quotes = true; - token_started = true; - continue; - } - - if (ch == '"' && in_quotes) { - in_quotes = false; - while (GetChar(ch) && ch != delim_ && ch != '\n') { - if (!is_whitespace_[static_cast(ch)]) { - THROW_RUNTIME_ERROR( - "Invalid CSV format: unexpected character after closing quote"); - } - } - start_token_if_needed(); - vector_of_strings.EndAddString(); - token_started = false; - token_materialized = false; - if (ch == '\n' || eof_reached_) { - end_of_line_ = true; - return; - } - continue; - } - - if (!in_quotes && (ch == delim_ || ch == '\n')) { - start_token_if_needed(); - while (!vector_of_strings.EmptyLastString() && - is_whitespace_[static_cast(vector_of_strings.BackLastString())]) { - vector_of_strings.PopLastChar(); - } - vector_of_strings.EndAddString(); - token_started = false; - token_materialized = false; - if (ch == '\n') { - end_of_line_ = true; - return; - } - continue; - } - - start_token_if_needed(); - vector_of_strings.ContinueAddString(ch); - token_started = true; - } - - if (!token_materialized) { - end_of_line_ = true; - return; - } - - while (!vector_of_strings.EmptyLastString() && - is_whitespace_[static_cast(vector_of_strings.BackLastString())]) { - vector_of_strings.PopLastChar(); - } - end_of_line_ = true; - vector_of_strings.EndAddString(); -} - -bool CSVTokenizer::IsEndOfLine() const { - return end_of_line_; -} - -bool CSVTokenizer::IsEOF() const { - return eof_reached_; -} - -void CSVTokenizer::ResetLineFlag() { - end_of_line_ = false; -} diff --git a/src/read_write/ColumnarReader.cpp b/src/read_write/ColumnarReader.cpp deleted file mode 100644 index 1fccf16..0000000 --- a/src/read_write/ColumnarReader.cpp +++ /dev/null @@ -1,139 +0,0 @@ -// -// Created by ragnarokk on 02.01.2026. -// - -#include - -#include "ColumnarReader.h" - -ColumnarReader::ColumnarReader(const std::string& file_name) : file_(file_name) { - if (!file_.is_open()) { - throw std::runtime_error("Could not open file for reading"); - } - file_.seekg(0, std::ios::end); - const std::streamoff file_size = file_.tellg(); - if (file_size < 16) { - throw std::runtime_error("File is too small to contain a valid footer"); - } - - file_.seekg(-4, std::ios::end); - std::array magic{}; - file_.read(magic.data(), magic.size()); - if (!file_ || std::string(magic.data(), magic.size()) != "TUFF") { - throw std::runtime_error("Invalid file footer magic"); - } - - file_.seekg(-12, std::ios::end); - uint64_t metadata_start; - file_.read(reinterpret_cast(&metadata_start), sizeof(metadata_start)); - if (!file_) { - throw std::runtime_error("Failed to read metadata start offset"); - } - if (metadata_start >= static_cast(file_size)) { - throw std::runtime_error("Corrupted metadata offset"); - } - - file_.seekg(metadata_start, std::ios::beg); - - uint32_t num_cols; - file_.read(reinterpret_cast(&num_cols), sizeof(num_cols)); - if (!file_) { - throw std::runtime_error("Failed to read number of columns"); - } - - for (uint32_t i = 0; i < num_cols; ++i) { - ColumnMetadata column_meta; - uint32_t name_length; - file_.read(reinterpret_cast(&name_length), sizeof(name_length)); - if (!file_) { - throw std::runtime_error("Failed to read column name length"); - } - column_meta.name.resize(name_length); - if (name_length > 0) { - file_.read(&column_meta.name[0], name_length); - if (!file_) { - throw std::runtime_error("Failed to read column name"); - } - } - - uint8_t type; - file_.read(reinterpret_cast(&type), sizeof(type)); - if (!file_) { - throw std::runtime_error("Failed to read column type"); - } - column_meta.type = static_cast(type); - - uint32_t num_chunks; - file_.read(reinterpret_cast(&num_chunks), sizeof(num_chunks)); - if (!file_) { - throw std::runtime_error("Failed to read number of chunks"); - } - for (uint32_t j = 0; j < num_chunks; ++j) { - uint64_t offset, size; - file_.read(reinterpret_cast(&offset), sizeof(offset)); - file_.read(reinterpret_cast(&size), sizeof(size)); - if (!file_) { - throw std::runtime_error("Failed to read chunk metadata"); - } - column_meta.offsets.push_back(offset); - column_meta.sizes.push_back(size); - } - - metadata_.push_back(std::move(column_meta)); - } -} - -ColumnarReader::~ColumnarReader() { - if (file_.is_open()) { - file_.close(); - } -} - -std::vector ColumnarReader::GetRawColumnData(size_t column_index, size_t chunk_index) { - std::vector buffer; - if (column_index >= metadata_.size()) { - THROW_RUNTIME_ERROR("Column index out of range"); - } - if (chunk_index >= metadata_[column_index].offsets.size()) { - THROW_RUNTIME_ERROR("Chunk index out of range"); - } - uint64_t offset = metadata_[column_index].offsets[chunk_index]; - uint64_t size = metadata_[column_index].sizes[chunk_index]; - - buffer.resize(size); - file_.seekg(offset, std::ios::beg); - file_.read(buffer.data(), size); - return buffer; -} - -std::shared_ptr ColumnarReader::GetColumnData(size_t column_index, size_t chunk_index) { - std::vector raw_data = GetRawColumnData(column_index, chunk_index); - ColumnType type = metadata_[column_index].type; - std::shared_ptr column; - -#define HANDLE_TYPE(ENUM_VAL, STR_VAL, CLASS_TYPE) \ - if (type == ColumnType::ENUM_VAL) { \ - column = std::make_shared(); \ - } else - - FOR_EACH_COLUMN_TYPE(HANDLE_TYPE) { - THROW_RUNTIME_ERROR("Unknown column type"); - } -#undef HANDLE_TYPE - - column->ReadFromRawData(raw_data); - return column; -} - -const std::vector& ColumnarReader::GetMetadata() const { - return metadata_; -} - -ColumnType ColumnarReader::GetColumnTypeByName(const std::string& name) const { - for (const auto& meta : metadata_) { - if (meta.name == name) { - return meta.type; - } - } - THROW_RUNTIME_ERROR("Column " + name + " not found in metadata"); -} diff --git a/src/read_write/ColumnarReader.h b/src/read_write/ColumnarReader.h deleted file mode 100644 index 6fc75a8..0000000 --- a/src/read_write/ColumnarReader.h +++ /dev/null @@ -1,32 +0,0 @@ -// -// Created by ragnarokk on 02.01.2026. -// - -#ifndef COLUMNAR_ENGINE_READER_H -#define COLUMNAR_ENGINE_READER_H - -#include -#include -#include - -#include "Types.h" - -class ColumnarReader { -public: - ColumnarReader(const std::string& file_name); - ~ColumnarReader(); - - std::vector GetRawColumnData(size_t column_index, size_t chunk_index); - - std::shared_ptr GetColumnData(size_t column_index, size_t chunk_index); - - const std::vector& GetMetadata() const; - - ColumnType GetColumnTypeByName(const std::string& name) const; - -private: - std::ifstream file_; - std::vector metadata_; -}; - -#endif // COLUMNAR_ENGINE_READER_H diff --git a/src/read_write/ColumnarWriter.cpp b/src/read_write/ColumnarWriter.cpp deleted file mode 100644 index 5085ec4..0000000 --- a/src/read_write/ColumnarWriter.cpp +++ /dev/null @@ -1,88 +0,0 @@ -// -// Created by ragnarokk on 05.01.2026. -// - -#include "ColumnarWriter.h" - -ColumnarWriter::ColumnarWriter(const std::string& file_path) { - file_.open(file_path, std::ios::binary); - if (!file_.is_open()) { - throw std::runtime_error("Could not open file for writing"); - } -} - -ColumnarWriter::~ColumnarWriter() { - if (file_.is_open()) { - file_.close(); - } -} - -void ColumnarWriter::WriteHeader(const VectorOfStrings2D& column_names, - const std::vector& column_types) { - if (header_written_) { - throw std::runtime_error("Header already written"); - } - - file_.write("TUFF", 4); - current_offset_ = 4; - - for (size_t i = 0; i < column_names.Size(); ++i) { - ColumnMetadata column_meta; - column_meta.name = column_names.GetString(i); - column_meta.type = column_types[i]; - metadata_.push_back(column_meta); - } - - header_written_ = true; -} - -void ColumnarWriter::WriteBatch(const std::vector>& columns) { - if (!header_written_) { - throw std::runtime_error("Header must be written before writing batches"); - } - if (columns.size() != metadata_.size()) { - throw std::runtime_error("Number of columns does not match metadata"); - } - for (size_t i = 0; i < columns.size(); ++i) { - uint64_t offset = current_offset_; - uint64_t size = columns[i]->WriteToFile(file_); - metadata_[i].offsets.push_back(offset); - metadata_[i].sizes.push_back(size); - current_offset_ += size; - } - if (!columns.empty()) { - num_rows_ += columns[0]->Size(); - } -} - -void ColumnarWriter::Finalize() { - if (!header_written_) { - throw std::runtime_error("Header must be written before finalizing"); - } - - uint64_t metadata_start = current_offset_; - - uint32_t num_columns = metadata_.size(); - file_.write(reinterpret_cast(&num_columns), sizeof(num_columns)); - - for (const auto& meta : metadata_) { - uint32_t name_length = meta.name.size(); - file_.write(reinterpret_cast(&name_length), sizeof(name_length)); - file_.write(meta.name.data(), name_length); - - uint8_t type = static_cast(meta.type); - file_.write(reinterpret_cast(&type), sizeof(type)); - - uint32_t num_batches = meta.offsets.size(); - file_.write(reinterpret_cast(&num_batches), sizeof(num_batches)); - for (size_t i = 0; i < num_batches; ++i) { - file_.write(reinterpret_cast(&meta.offsets[i]), sizeof(uint64_t)); - file_.write(reinterpret_cast(&meta.sizes[i]), sizeof(uint64_t)); - } - } - - file_.write(reinterpret_cast(&num_rows_), sizeof(num_rows_)); - - file_.write(reinterpret_cast(&metadata_start), sizeof(metadata_start)); - file_.write("TUFF", 4); -} diff --git a/src/read_write/ColumnarWriter.h b/src/read_write/ColumnarWriter.h deleted file mode 100644 index 7b60d71..0000000 --- a/src/read_write/ColumnarWriter.h +++ /dev/null @@ -1,34 +0,0 @@ -// -// Created by ragnarokk on 05.01.2026. -// - -#ifndef COLUMNAR_ENGINE_COLUMNARWRITER_H -#define COLUMNAR_ENGINE_COLUMNARWRITER_H - -#include -#include - -#include "Types.h" - -class ColumnarWriter { -public: - ColumnarWriter(const std::string& file_path); - ~ColumnarWriter(); - - void WriteHeader(const VectorOfStrings2D& column_names, - const std::vector& column_types); - void WriteBatch(const std::vector>& columns); - - // magic, columns by batches, number of columns, name, type, offsets, sizes, number of rows, - // metadata_start, magic - void Finalize(); - -private: - std::ofstream file_; - std::vector metadata_; - uint64_t current_offset_ = 0; - uint32_t num_rows_ = 0; - bool header_written_ = false; -}; - -#endif // COLUMNAR_ENGINE_COLUMNARWRITER_H \ No newline at end of file diff --git a/src/read_write/test/test_csv_parser.cpp b/src/read_write/test/test_csv_parser.cpp deleted file mode 100644 index ca53b27..0000000 --- a/src/read_write/test/test_csv_parser.cpp +++ /dev/null @@ -1,111 +0,0 @@ -// -// Created by ragnarokk on 04.01.2026. -// - -// Generated by AI - -#include - -#include "CSVReader.h" -#include "CSVTokenizer.h" - -#include -#include -#include - -namespace fs = std::filesystem; - -namespace { - -std::string CreateTempCSVFile(const std::string& content, const std::string& filename) { - const std::string path = "/tmp/" + filename; - std::ofstream out(path); - out << content; - out.close(); - return path; -} - -} // namespace - -TEST_CASE("CSVTokenizer does not materialize empty token on hard EOF", "[CSVTokenizer]") { - const std::string path = CreateTempCSVFile("", "test_tokenizer_hard_eof.csv"); - CSVTokenizer tokenizer(path); - VectorOfStrings2D row; - - const size_t width_before = row.Width(); - tokenizer.GetNextRow(row); - - REQUIRE(row.Width() == width_before); - REQUIRE(tokenizer.IsEOF()); - - fs::remove(path); -} - -TEST_CASE("CSVTokenizer keeps last token without trailing newline", "[CSVTokenizer]") { - const std::string path = CreateTempCSVFile("a,b", "test_tokenizer_no_trailing_newline.csv"); - CSVTokenizer tokenizer(path); - VectorOfStrings2D row; - - tokenizer.GetNextRow(row); - tokenizer.GetNextRow(row); - - REQUIRE(row.Width() == 2); - REQUIRE(row.GetString(0) == std::string_view("a")); - REQUIRE(row.GetString(1) == std::string_view("b")); - REQUIRE(tokenizer.IsEOF()); - - fs::remove(path); -} - -TEST_CASE("CSVReader does not return fake empty row at EOF", "[CSVReader]") { - const std::string path = CreateTempCSVFile("h1,h2\n1,2", "test_reader_eof.csv"); - CSVReader reader(path); - - VectorOfStrings2D header; - reader.ReadHeader(header); - REQUIRE(header.Width() == 2); - - VectorOfStrings2D row; - REQUIRE(reader.ReadRow(row)); - REQUIRE(row.Width() == 2); - REQUIRE(row.GetString(0) == std::string_view("1")); - REQUIRE(row.GetString(1) == std::string_view("2")); - - REQUIRE_FALSE(reader.ReadRow(row)); - REQUIRE(row.Width() == 0); - - fs::remove(path); -} - -TEST_CASE("CSVReader does not add extra row for trailing newline", "[CSVReader]") { - const std::string path = CreateTempCSVFile("h1\nx\n", "test_reader_trailing_newline.csv"); - CSVReader reader(path); - - VectorOfStrings2D header; - reader.ReadHeader(header); - - VectorOfStrings2D row; - REQUIRE(reader.ReadRow(row)); - REQUIRE(row.Width() == 1); - REQUIRE(row.GetString(0) == std::string_view("x")); - REQUIRE_FALSE(reader.ReadRow(row)); - - fs::remove(path); -} - -TEST_CASE("CSVReader preserves empty fields", "[CSVReader]") { - const std::string path = CreateTempCSVFile("a,b\n,\n", "test_reader_empty_fields.csv"); - CSVReader reader(path); - - VectorOfStrings2D header; - reader.ReadHeader(header); - - VectorOfStrings2D row; - REQUIRE(reader.ReadRow(row)); - REQUIRE(row.Width() == 2); - REQUIRE(row.GetString(0) == std::string_view("")); - REQUIRE(row.GetString(1) == std::string_view("")); - REQUIRE_FALSE(reader.ReadRow(row)); - - fs::remove(path); -} diff --git a/src/read_write/test/test_reader_writer.cpp b/src/read_write/test/test_reader_writer.cpp deleted file mode 100644 index 8c38c65..0000000 --- a/src/read_write/test/test_reader_writer.cpp +++ /dev/null @@ -1,496 +0,0 @@ -// -// Created by ragnarokk on 04.01.2026. -// - -// Generated by AI - -#include -#include -#include - -#include "../read_write/CSVToColumnar.h" -#include "../read_write/ColumnarReader.h" -#include "../Types.h" - -namespace fs = std::filesystem; - -std::string CreateTempCSVFile(const std::string& content, - const std::string& filename = "test_rw.csv") { - std::string filepath = "/tmp/" + filename; - std::ofstream file(filepath); - file << content; - file.close(); - return filepath; -} - -void RemoveTempFile(const std::string& filepath) { - if (fs::exists(filepath)) { - fs::remove(filepath); - } -} - -std::vector ExtractInt32Values(const std::shared_ptr& column) { - Int32Column* int_col = dynamic_cast(column.get()); - if (!int_col) { - return {}; - } - - std::vector result; - std::ofstream temp_file("/tmp/temp_int.bin", std::ios::binary); - int_col->WriteToFile(temp_file); - temp_file.close(); - - std::ifstream in_file("/tmp/temp_int.bin", std::ios::binary); - in_file.seekg(0, std::ios::end); - size_t size = in_file.tellg(); - in_file.seekg(0, std::ios::beg); - - result.resize(size / sizeof(int32_t)); - in_file.read(reinterpret_cast(result.data()), size); - in_file.close(); - fs::remove("/tmp/temp_int.bin"); - - return result; -} - -std::vector ExtractStringValues(const std::shared_ptr& column) { - StringColumn* str_col = dynamic_cast(column.get()); - if (!str_col) { - return {}; - } - - std::vector buffer; - std::ofstream temp_file("/tmp/temp_str.bin", std::ios::binary); - str_col->WriteToFile(temp_file); - temp_file.close(); - - std::ifstream in_file("/tmp/temp_str.bin", std::ios::binary); - in_file.seekg(0, std::ios::end); - size_t size = in_file.tellg(); - in_file.seekg(0, std::ios::beg); - - buffer.resize(size); - in_file.read(buffer.data(), size); - in_file.close(); - fs::remove("/tmp/temp_str.bin"); - - std::vector result; - size_t offset = 0; - while (offset < buffer.size()) { - uint32_t length; - std::memcpy(&length, buffer.data() + offset, sizeof(length)); - offset += sizeof(length); - std::string str(buffer.data() + offset, length); - result.push_back(str); - offset += length; - } - - return result; -} - -TEST_CASE("CSVToColumnar: Basic write operation", "[CSVToColumnar]") { - std::string csv_content = - "INT32:id,STRING:name\n" - "1,Alice\n" - "2,Bob\n" - "3,Charlie\n"; - std::string csv_path = CreateTempCSVFile(csv_content, "test_CSVToColumnar_basic.csv"); - std::string output_path = "/tmp/test_CSVToColumnar_basic.bin"; - - SECTION("Write CSV to binary format") { - CSVToColumnar csv_to_columnar; - REQUIRE_NOTHROW(csv_to_columnar.Convert(csv_path, output_path)); - REQUIRE(fs::exists(output_path)); - - std::ifstream file(output_path, std::ios::binary); - file.seekg(0, std::ios::end); - size_t file_size = file.tellg(); - REQUIRE(file_size > 0); - - file.seekg(0, std::ios::beg); - char magic[5] = {0}; - file.read(magic, 4); - REQUIRE(std::string(magic) == "TUFF"); - - file.close(); - } - - RemoveTempFile(csv_path); - RemoveTempFile(output_path); -} - -TEST_CASE("CSVToColumnar: Multiple types", "[CSVToColumnar]") { - std::string csv_content = - "INT32:id,STRING:name,FLOAT:score\n" - "1,Alice,95.5\n" - "2,Bob,87.3\n"; - std::string csv_path = CreateTempCSVFile(csv_content, "test_CSVToColumnar_multi.csv"); - std::string output_path = "/tmp/test_CSVToColumnar_multi.bin"; - - CSVToColumnar csv_to_columnar; - csv_to_columnar.Convert(csv_path, output_path); - - REQUIRE(fs::exists(output_path)); - REQUIRE(fs::file_size(output_path) > 0); - - RemoveTempFile(csv_path); - RemoveTempFile(output_path); -} - -TEST_CASE("Reader: Read metadata", "[Reader]") { - std::string csv_content = - "INT32:id,STRING:name\n" - "1,Alice\n" - "2,Bob\n"; - std::string csv_path = CreateTempCSVFile(csv_content, "test_reader_meta.csv"); - std::string output_path = "/tmp/test_reader_meta.bin"; - - CSVToColumnar csv_to_columnar; - csv_to_columnar.Convert(csv_path, output_path); - - SECTION("Read and verify metadata") { - ColumnarReader reader(output_path); - const auto& metadata = reader.GetMetadata(); - - REQUIRE(metadata.size() == 2); - REQUIRE(metadata[0].name == "id"); - REQUIRE(metadata[0].type == ColumnType::INT32); - REQUIRE(metadata[1].name == "name"); - REQUIRE(metadata[1].type == ColumnType::STRING); - } - - RemoveTempFile(csv_path); - RemoveTempFile(output_path); -} - -TEST_CASE("Reader: Read raw column data", "[Reader]") { - std::string csv_content = - "INT32:value\n" - "100\n" - "200\n" - "300\n"; - std::string csv_path = CreateTempCSVFile(csv_content, "test_reader_raw.csv"); - std::string output_path = "/tmp/test_reader_raw.bin"; - - CSVToColumnar csv_to_columnar; - csv_to_columnar.Convert(csv_path, output_path); - - ColumnarReader reader(output_path); - - SECTION("Get raw data for column") { - std::vector raw_data = reader.GetRawColumnData(0, 0); - REQUIRE(!raw_data.empty()); - REQUIRE(raw_data.size() == 3 * sizeof(int32_t)); - } - - RemoveTempFile(csv_path); - RemoveTempFile(output_path); -} - -TEST_CASE("Reader: Read typed column data", "[Reader]") { - std::string csv_content = - "INT32:id,STRING:city\n" - "1,Moscow\n" - "2,London\n"; - std::string csv_path = CreateTempCSVFile(csv_content, "test_reader_typed.csv"); - std::string output_path = "/tmp/test_reader_typed.bin"; - - CSVToColumnar csv_to_columnar; - csv_to_columnar.Convert(csv_path, output_path); - - ColumnarReader reader(output_path); - - SECTION("Read INT32 column") { - auto column = reader.GetColumnData(0, 0); - REQUIRE(column != nullptr); - REQUIRE(column->GetType() == ColumnType::INT32); - REQUIRE(column->Size() == 2); - } - - SECTION("Read STRING column") { - auto column = reader.GetColumnData(1, 0); - REQUIRE(column != nullptr); - REQUIRE(column->GetType() == ColumnType::STRING); - REQUIRE(column->Size() == 2); - } - - RemoveTempFile(csv_path); - RemoveTempFile(output_path); -} - -TEST_CASE("CSVToColumnar+Reader: Round-trip test with INT data", - "[CSVToColumnar][Reader][Integration]") { - std::string csv_content = - "INT32:numbers\n" - "10\n" - "20\n" - "30\n"; - std::string csv_path = CreateTempCSVFile(csv_content, "test_roundtrip_int.csv"); - std::string output_path = "/tmp/test_roundtrip_int.bin"; - - CSVToColumnar csv_to_columnar; - csv_to_columnar.Convert(csv_path, output_path); - - ColumnarReader reader(output_path); - auto column = reader.GetColumnData(0, 0); - - REQUIRE(column->GetType() == ColumnType::INT32); - REQUIRE(column->Size() == 3); - - RemoveTempFile(csv_path); - RemoveTempFile(output_path); -} - -TEST_CASE("CSVToColumnar+Reader: Round-trip test with STRING data", - "[CSVToColumnar][Reader][Integration]") { - std::string csv_content = - "STRING:cities\n" - "Moscow\n" - "London\n" - "Paris\n"; - std::string csv_path = CreateTempCSVFile(csv_content, "test_roundtrip_str.csv"); - std::string output_path = "/tmp/test_roundtrip_str.bin"; - - CSVToColumnar csv_to_columnar; - csv_to_columnar.Convert(csv_path, output_path); - - ColumnarReader reader(output_path); - auto column = reader.GetColumnData(0, 0); - - REQUIRE(column->GetType() == ColumnType::STRING); - REQUIRE(column->Size() == 3); - - RemoveTempFile(csv_path); - RemoveTempFile(output_path); -} - -TEST_CASE("CSVToColumnar+Reader: Multiple columns round-trip", - "[CSVToColumnar][Reader][Integration]") { - std::string csv_content = - "INT32:id,STRING:name,FLOAT:score\n" - "1,Alice,95.5\n" - "2,Bob,87.3\n" - "3,Charlie,92.1\n"; - std::string csv_path = CreateTempCSVFile(csv_content, "test_roundtrip_multi.csv"); - std::string output_path = "/tmp/test_roundtrip_multi.bin"; - - CSVToColumnar csv_to_columnar; - csv_to_columnar.Convert(csv_path, output_path); - - ColumnarReader reader(output_path); - const auto& metadata = reader.GetMetadata(); - - SECTION("Verify metadata") { - REQUIRE(metadata.size() == 3); - REQUIRE(metadata[0].name == "id"); - REQUIRE(metadata[0].type == ColumnType::INT32); - REQUIRE(metadata[1].name == "name"); - REQUIRE(metadata[1].type == ColumnType::STRING); - REQUIRE(metadata[2].name == "score"); - REQUIRE(metadata[2].type == ColumnType::FLOAT); - } - - SECTION("Verify all columns have data") { - auto col0 = reader.GetColumnData(0, 0); - auto col1 = reader.GetColumnData(1, 0); - auto col2 = reader.GetColumnData(2, 0); - - REQUIRE(col0->Size() == 3); - REQUIRE(col1->Size() == 3); - REQUIRE(col2->Size() == 3); - } - - RemoveTempFile(csv_path); - RemoveTempFile(output_path); -} - -TEST_CASE("Reader: Invalid file", "[Reader][errors]") { - REQUIRE_THROWS_AS(ColumnarReader("/nonexistent/file.bin"), std::runtime_error); -} - -TEST_CASE("Reader: Invalid chunk index", "[Reader][errors]") { - std::string csv_content = "INT32:value\n100\n"; - std::string csv_path = CreateTempCSVFile(csv_content, "test_invalid_chunk.csv"); - std::string output_path = "/tmp/test_invalid_chunk.bin"; - - CSVToColumnar csv_to_columnar; - csv_to_columnar.Convert(csv_path, output_path); - - ColumnarReader reader(output_path); - - REQUIRE_THROWS_AS(reader.GetRawColumnData(0, 999), std::runtime_error); - - RemoveTempFile(csv_path); - RemoveTempFile(output_path); -} - -TEST_CASE("CSVToColumnar: Empty CSV file", "[CSVToColumnar]") { - std::string csv_content = "INT32:id,STRING:name\n"; - std::string csv_path = CreateTempCSVFile(csv_content, "test_empty.csv"); - std::string output_path = "/tmp/test_empty.bin"; - - CSVToColumnar csv_to_columnar; - csv_to_columnar.Convert(csv_path, output_path); - - REQUIRE(fs::exists(output_path)); - - ColumnarReader reader(output_path); - const auto& metadata = reader.GetMetadata(); - REQUIRE(metadata.size() == 2); - - RemoveTempFile(csv_path); - RemoveTempFile(output_path); -} - -TEST_CASE("CSVToColumnar+Reader: Large dataset", "[CSVToColumnar][Reader][Integration]") { - std::string csv_content = "INT32:id,STRING:name\n"; - for (int i = 0; i < 1000; ++i) { - csv_content += std::to_string(i) + ",User" + std::to_string(i) + "\n"; - } - - std::string csv_path = CreateTempCSVFile(csv_content, "test_large.csv"); - std::string output_path = "/tmp/test_large.bin"; - - CSVToColumnar csv_to_columnar; - csv_to_columnar.Convert(csv_path, output_path); - - ColumnarReader reader(output_path); - auto col0 = reader.GetColumnData(0, 0); - auto col1 = reader.GetColumnData(1, 0); - - REQUIRE(col0->Size() == 1000); - REQUIRE(col1->Size() == 1000); - - RemoveTempFile(csv_path); - RemoveTempFile(output_path); -} - -TEST_CASE("CSVToColumnar+Reader: Special characters in strings", - "[CSVToColumnar][Reader][Integration]") { - std::string csv_content = - "STRING:text\n" - "\"Hello, World!\"\n" - "Simple text\n" - "\"Text with, comma\"\n"; - std::string csv_path = CreateTempCSVFile(csv_content, "test_special.csv"); - std::string output_path = "/tmp/test_special.bin"; - - CSVToColumnar csv_to_columnar; - csv_to_columnar.Convert(csv_path, output_path); - - ColumnarReader reader(output_path); - auto column = reader.GetColumnData(0, 0); - - REQUIRE(column->GetType() == ColumnType::STRING); - REQUIRE(column->Size() == 3); - - RemoveTempFile(csv_path); - RemoveTempFile(output_path); -} - -TEST_CASE("CSVToColumnar+Reader: Float precision", "[CSVToColumnar][Reader][Integration]") { - std::string csv_content = - "FLOAT:values\n" - "3.14159\n" - "2.71828\n" - "1.618\n"; - std::string csv_path = CreateTempCSVFile(csv_content, "test_float.csv"); - std::string output_path = "/tmp/test_float.bin"; - - CSVToColumnar csv_to_columnar; - csv_to_columnar.Convert(csv_path, output_path); - - ColumnarReader reader(output_path); - auto column = reader.GetColumnData(0, 0); - - REQUIRE(column->GetType() == ColumnType::FLOAT); - REQUIRE(column->Size() == 3); - - RemoveTempFile(csv_path); - RemoveTempFile(output_path); -} - -TEST_CASE("Reader: Metadata chunks count", "[Reader]") { - std::string csv_content = "INT32:id\n1\n2\n3\n"; - std::string csv_path = CreateTempCSVFile(csv_content, "test_chunks.csv"); - std::string output_path = "/tmp/test_chunks.bin"; - - CSVToColumnar csv_to_columnar; - csv_to_columnar.Convert(csv_path, output_path); - - ColumnarReader reader(output_path); - const auto& metadata = reader.GetMetadata(); - - REQUIRE(metadata[0].offsets.size() == metadata[0].sizes.size()); - REQUIRE(!metadata[0].offsets.empty()); - - RemoveTempFile(csv_path); - RemoveTempFile(output_path); -} - -TEST_CASE("CSVToColumnar: Convert with schema", "[CSVToColumnar][Schema]") { - std::string csv_content = - "Alice,1\n" - "Bob,2\n" - "Charlie,3\n"; - std::string csv_path = CreateTempCSVFile(csv_content, "test_schema_source.csv"); - - std::string schema_content = - "name,STRING\n" - "id,INT32\n"; - std::string schema_path = CreateTempCSVFile(schema_content, "test_schema_desc.schema"); - - std::string output_path = "/tmp/test_schema_output.bin"; - - SECTION("Convert using schema and check columns order") { - CSVToColumnar csv_to_columnar; - REQUIRE_NOTHROW(csv_to_columnar.ConvertWithSchema(csv_path, output_path, schema_path)); - REQUIRE(fs::exists(output_path)); - - ColumnarReader reader(output_path); - const auto& metadata = reader.GetMetadata(); - - REQUIRE(metadata.size() == 2); - // Header ordering depends on CSV: name then id in header, but types must match CSV positions mapped via names. - // Wait, CSVToColumnar maps the CSV headers to schema types via `column_type_map` and pushes exactly in the CSV order: - // column_names.push_back(header); - // column_types.push_back(type); - // Let's verify that columns match CSV order "name, id" but correct types. - REQUIRE(metadata[0].name == "name"); - REQUIRE(metadata[0].type == ColumnType::STRING); - REQUIRE(metadata[1].name == "id"); - REQUIRE(metadata[1].type == ColumnType::INT32); - - auto name_col = reader.GetColumnData(0, 0); - REQUIRE(name_col->GetType() == ColumnType::STRING); - REQUIRE(name_col->Size() == 3); - - auto id_col = reader.GetColumnData(1, 0); - REQUIRE(id_col->GetType() == ColumnType::INT32); - REQUIRE(id_col->Size() == 3); - } - - RemoveTempFile(csv_path); - RemoveTempFile(schema_path); - RemoveTempFile(output_path); -} - -TEST_CASE("CSVToColumnar: Convert with missing schema column", "[CSVToColumnar][Schema][errors]") { - std::string csv_content = - "Alice,1,25\n"; - std::string csv_path = CreateTempCSVFile(csv_content, "test_schema_err.csv"); - - std::string schema_content = - "name,STRING\n" - "id,INT32\n"; - std::string schema_path = CreateTempCSVFile(schema_content, "test_schema_desc_err.schema"); - - std::string output_path = "/tmp/test_schema_output_err.bin"; - - CSVToColumnar csv_to_columnar; - REQUIRE_THROWS_AS(csv_to_columnar.ConvertWithSchema(csv_path, schema_path, output_path), std::runtime_error); - - RemoveTempFile(csv_path); - RemoveTempFile(schema_path); - RemoveTempFile(output_path); -} diff --git a/src/types/ColumnType.h b/src/types/ColumnType.h new file mode 100644 index 0000000..ae6842b --- /dev/null +++ b/src/types/ColumnType.h @@ -0,0 +1,48 @@ +#pragma once + +#include +#include +#include + +enum class ColumnType : uint8_t { + INT16, + INT32, + INT64, + INT128, + FLOAT, + DOUBLE, + LONGDOUBLE, + CHAR, + STRING, + DATE, + TIMESTAMP, +}; + +struct ColumnMetadata { + std::string name; + ColumnType type; + std::vector offsets; + std::vector sizes; +}; + +// TODO: for macro if German is gonna tell i coded that xyevo +// template +// struct ColumnTraits; +// +// template <> +// struct ColumnTraits { +// using ValueType = int16_t; +// static constexpr std::string Name = "INT16"; +// }; +// +// template <> +// struct ColumnTraits { +// using ValueType = int32_t; +// static constexpr std::string Name = "INT32"; +// }; +// +// template <> +// struct ColumnTraits { +// using ValueType = int64_t; +// static constexpr std::string Name = "INT64"; +// }; diff --git a/src/types/Date.h b/src/types/Date.h new file mode 100644 index 0000000..ecf8232 --- /dev/null +++ b/src/types/Date.h @@ -0,0 +1,86 @@ +#pragma once + +#include "TimeUnit.h" +#include "utils/Assert.h" + +#include +#include +#include + +struct Date { + static int32_t Parse(std::string_view unit) { + if (unit.size() != 10 || unit[4] != '-' || unit[7] != '-') [[unlikely]] { + THROW_RUNTIME_ERROR("Invalid date format: " + std::string(unit)); + } + + int32_t year = + (unit[0] - '0') * 1000 + (unit[1] - '0') * 100 + (unit[2] - '0') * 10 + (unit[3] - '0'); + int32_t month = (unit[5] - '0') * 10 + (unit[6] - '0'); + int32_t day = (unit[8] - '0') * 10 + (unit[9] - '0'); + + year -= (month <= 2); + const int era = (year >= 0 ? year : year - 399) / 400; + const unsigned yoe = static_cast(year - era * 400); + const unsigned doy = (153 * (month + (month > 2 ? -3 : 9)) + 2) / 5 + day - 1; + const unsigned doe = yoe * 365 + yoe / 4 - yoe / 100 + doy; + + return era * 146097 + static_cast(doe) - 719468; + } + + static std::string Format(int32_t days) { + days += 719468; + const int era = (days >= 0 ? days : days - 146096) / 146097; + const unsigned doe = static_cast(days - era * 146097); + const unsigned yoe = (doe - doe / 1460 + doe / 36524 - doe / 146096) / 365; + const int y = static_cast(yoe) + era * 400; + const unsigned doy = doe - (365 * yoe + yoe / 4 - yoe / 100); + const unsigned mp = (5 * doy + 2) / 153; + const unsigned d = doy - (153 * mp + 2) / 5 + 1; + const unsigned m = mp + (mp < 10 ? 3 : -9); + const int year = y + (m <= 2); + + std::string res = "0000-00-00"; + auto write_digit = [&](int val, int pos, int len) { + for (int i = 0; i < len; ++i) { + res[pos + len - 1 - i] = (val % 10) + '0'; + val /= 10; + } + }; + write_digit(year, 0, 4); + write_digit(m, 5, 2); + write_digit(d, 8, 2); + return res; + } + + static int32_t Truncate(int32_t total_days, TimeUnitType unit) { + if (unit == TimeUnitType::DAY || unit == TimeUnitType::HOUR || unit == TimeUnitType::MINUTE || + unit == TimeUnitType::SECOND) { + return total_days; + } + + int32_t days = total_days + 719468; + const int era = (days >= 0 ? days : days - 146096) / 146097; + const unsigned doe = static_cast(days - era * 146097); + const unsigned yoe = (doe - doe / 1460 + doe / 36524 - doe / 146096) / 365; + const int y = static_cast(yoe) + era * 400; + const unsigned doy = doe - (365 * yoe + yoe / 4 - yoe / 100); + const unsigned mp = (5 * doy + 2) / 153; + const unsigned d = doy - (153 * mp + 2) / 5 + 1; + const unsigned m = mp + (mp < 10 ? 3 : -9); + int year = y + (m <= 2); + + switch (unit) { + case TimeUnitType::MONTH: { + return total_days - static_cast(d) + 1; + } + case TimeUnitType::YEAR: { + const int e = (year >= 0 ? year : year - 399) / 400; + const unsigned ye = static_cast(year - e * 400); + const unsigned de = ye * 365 + ye / 4 - ye / 100 + 306; + return e * 146097 + static_cast(de) - 719468; + } + default: + return total_days; + } + } +}; diff --git a/src/types/String.h b/src/types/String.h new file mode 100644 index 0000000..384d701 --- /dev/null +++ b/src/types/String.h @@ -0,0 +1,10 @@ +#pragma once + +#include + +class String { +public: + static size_t Length(std::string_view string) { + return string.size(); + } +}; diff --git a/src/types/TimeUnit.h b/src/types/TimeUnit.h new file mode 100644 index 0000000..b377d20 --- /dev/null +++ b/src/types/TimeUnit.h @@ -0,0 +1,42 @@ +#pragma once + +#include "utils/Assert.h" + +#include +#include + +enum class TimeUnitType { + YEAR, + MONTH, + DAY, + HOUR, + MINUTE, + SECOND +}; + + +class TimeUnit { +public: + static TimeUnitType ParseTimeUnit(std::string_view unit_str) { + if (unit_str == "year") { + return TimeUnitType::YEAR; + } + if (unit_str == "month") { + return TimeUnitType::MONTH; + } + if (unit_str == "day") { + return TimeUnitType::DAY; + } + if (unit_str == "hour") { + return TimeUnitType::HOUR; + } + if (unit_str == "minute") { + return TimeUnitType::MINUTE; + } + if (unit_str == "second") { + return TimeUnitType::SECOND; + } + + THROW_RUNTIME_ERROR("Unknown time unit: " + std::string(unit_str)); + } +}; \ No newline at end of file diff --git a/src/types/Timestamp.h b/src/types/Timestamp.h new file mode 100644 index 0000000..25784e5 --- /dev/null +++ b/src/types/Timestamp.h @@ -0,0 +1,103 @@ +#pragma once + +#include "Date.h" +#include "TimeUnit.h" + +#include +#include + +struct Timestamp { + static int64_t Parse(std::string_view unit) { + if (unit.size() != 19 || unit[4] != '-' || unit[7] != '-' || unit[10] != ' ' || + unit[13] != ':' || unit[16] != ':') [[unlikely]] { + THROW_RUNTIME_ERROR("Invalid date format: " + std::string(unit)); + } + + auto to_int2 = [](char a, char b) { return (a - '0') * 10 + (b - '0'); }; + + int year = + (unit[0] - '0') * 1000 + (unit[1] - '0') * 100 + (unit[2] - '0') * 10 + (unit[3] - '0'); + int month = to_int2(unit[5], unit[6]); + int day = to_int2(unit[8], unit[9]); + int hour = to_int2(unit[11], unit[12]); + int minute = to_int2(unit[14], unit[15]); + int second = to_int2(unit[17], unit[18]); + + year -= (month <= 2); + const int era = (year >= 0 ? year : year - 399) / 400; + const unsigned yoe = static_cast(year - era * 400); + const unsigned doy = (153 * (month + (month > 2 ? -3 : 9)) + 2) / 5 + day - 1; + const unsigned doe = yoe * 365 + yoe / 4 - yoe / 100 + doy; + + int32_t days = era * 146097 + static_cast(doe) - 719468; + int64_t seconds = hour * 3600 + minute * 60 + second; + return static_cast(days) * 86400 + seconds; + } + + static std::string Format(int64_t total_seconds) { + int64_t days = total_seconds / 86400; + int64_t seconds_in_day = total_seconds % 86400; + if (seconds_in_day < 0) { + seconds_in_day += 86400; + days -= 1; + } + + std::string res = Date::Format(static_cast(days)); + res += " 00:00:00"; + int hour = seconds_in_day / 3600; + int minute = (seconds_in_day % 3600) / 60; + int second = seconds_in_day % 60; + auto write_digit = [&](int val, int pos) { + res[pos] = (val / 10) + '0'; + res[pos + 1] = (val % 10) + '0'; + }; + + write_digit(hour, 11); + write_digit(minute, 14); + write_digit(second, 17); + + return res; + } + + // TODO: make uniform interface called Extract + static int32_t ExtractMinute(int64_t total_seconds) { + int64_t seconds_in_day = total_seconds % 86400; + if (seconds_in_day < 0) { + seconds_in_day += 86400; + } + return (seconds_in_day % 3600) / 60; + } + + static int64_t Truncate(int64_t total_seconds, TimeUnitType part) { + int64_t seconds_in_day = total_seconds % 86400; + if (seconds_in_day < 0) { + seconds_in_day += 86400; + } + + switch (part) { + case TimeUnitType::MINUTE: { + int64_t extra_seconds = seconds_in_day % 60; + return total_seconds - extra_seconds; + } + case TimeUnitType::HOUR: { + int64_t extra_seconds = seconds_in_day % 3600; + return total_seconds - extra_seconds; + } + case TimeUnitType::DAY: { + return total_seconds - seconds_in_day; + } + case TimeUnitType::MONTH: { + int32_t days = total_seconds / 86400; + int32_t trunc = Date::Truncate(days, TimeUnitType::MONTH); + return static_cast(trunc) * 86400; + } + case TimeUnitType::YEAR: { + int32_t days = total_seconds / 86400; + int32_t trunc = Date::Truncate(days, TimeUnitType::YEAR); + return static_cast(trunc) * 86400; + } + default: break; + } + return total_seconds; + } +}; diff --git a/src/macro.h b/src/utils/Assert.h similarity index 52% rename from src/macro.h rename to src/utils/Assert.h index 972e325..3dc4e11 100644 --- a/src/macro.h +++ b/src/utils/Assert.h @@ -1,36 +1,7 @@ -// -// Created by ragnarokk on 30.03.2026. -// +#pragma once -#ifndef COLUMNAR_ENGINE_MACRO_H -#define COLUMNAR_ENGINE_MACRO_H - -// USE THIS EVERYWHERE FOR SWITCHING -// Прости, Герман, но я не хочу везде исправлять переборы типов, если новый добавился... -#define FOR_EACH_COLUMN_TYPE(M) \ - M(INT16, "INT16", Int16Column) \ - M(INT32, "INT32", Int32Column) \ - M(INT64, "INT64", Int64Column) \ - M(INT128, "INT128", Int128Column) \ - M(FLOAT, "FLOAT", FloatColumn) \ - M(DOUBLE, "DOUBLE", DoubleColumn) \ - M(LONGDOUBLE, "LONGDOUBLE", LongDoubleColumn) \ - M(CHAR, "CHAR", CharColumn) \ - M(STRING, "STRING", StringColumn) \ - M(DATE, "DATE", DateColumn) \ - M(TIMESTAMP, "TIMESTAMP", TimestampColumn) - -#define FOR_NUMERIC_COLUMN_TYPE(M) \ - M(INT16, "INT16", Int16Column) \ - M(INT32, "INT32", Int32Column) \ - M(INT64, "INT64", Int64Column) \ - M(INT128, "INT128", Int128Column) \ - M(FLOAT, "FLOAT", FloatColumn) \ - M(DOUBLE, "DOUBLE", DoubleColumn) \ - M(LONGDOUBLE, "LONGDOUBLE", LongDoubleColumn) \ - M(CHAR, "CHAR", CharColumn) \ - M(DATE, "DATE", DateColumn) \ - M(TIMESTAMP, "TIMESTAMP", TimestampColumn) +#include +#include #define ASSERT(cond) \ do { \ @@ -55,5 +26,3 @@ #define THROW_RUNTIME_ERROR(msg) \ throw std::runtime_error(std::string(__FILE__) + ":" + std::to_string(__LINE__) + ": " + (msg)) - -#endif // COLUMNAR_ENGINE_MACRO_H diff --git a/src/utils/Concurrency.h b/src/utils/Concurrency.h new file mode 100644 index 0000000..180e89d --- /dev/null +++ b/src/utils/Concurrency.h @@ -0,0 +1,114 @@ +#pragma once + +#ifdef ENABLE_MULTITHREADING + +#include +#include +#include + +template +using Atomic = std::atomic; +using Mutex = std::mutex; +using ConditionVariable = std::condition_variable; + +template +using LockGuard = std::lock_guard; +template +using UniqueLock = std::unique_lock; + +#else + +#include "utils/Assert.h" + +template +class DummyAtomic { +public: + DummyAtomic(T val = 0) : val_(val) { + } + + T fetch_add(T arg, std::memory_order /*order*/ = std::memory_order_seq_cst) { // NOLINT + T old = val_; + val_ += arg; + return old; + } + + T load(std::memory_order /*order*/ = std::memory_order_seq_cst) const { // NOLINT + return val_; + } + void store(T val, std::memory_order /*order*/ = std::memory_order_seq_cst) { // NOLINT + val_ = val; + } + bool compare_exchange_strong(T& expected, T desired, // NOLINT + std::memory_order /*success*/ = std::memory_order_seq_cst, + std::memory_order /*failure*/ = std::memory_order_seq_cst) { + if (val_ == expected) { + val_ = desired; + return true; + } else { + expected = val_; + return false; + } + } + bool compare_exchange_weak(T& expected, T desired, // NOLINT + std::memory_order /*success*/ = std::memory_order_seq_cst, + std::memory_order /*failure*/ = std::memory_order_seq_cst) { + if (val_ == expected) { + val_ = desired; + return true; + } else { + expected = val_; + return false; + } + } + +private: + T val_; +}; + +class DummyMutex { +public: + void lock() { // NOLINT + } + void unlock() { // NOLINT + } + bool try_lock() { // NOLINT + return true; + } +}; + +class DummyConditionVariable { +public: + template + void wait(Lock& /*lock*/, Predicate pred) { + ASSERT(pred()); + } + void notify_one() {} + void notify_all() {} +}; + +template +using Atomic = DummyAtomic; + +using Mutex = DummyMutex; + +using ConditionVariable = DummyConditionVariable; + +template +class LockGuard { +public: + explicit LockGuard(MutexT& /*m*/) { + } + LockGuard(const LockGuard&) = delete; + LockGuard& operator=(const LockGuard&) = delete; +}; + +template +class UniqueLock { +public: + explicit UniqueLock(MutexT& /*m*/) { + } + UniqueLock(const UniqueLock&) = delete; + UniqueLock& operator=(const UniqueLock&) = delete; +}; + +#endif diff --git a/src/utils/Macro.h b/src/utils/Macro.h new file mode 100644 index 0000000..40888b6 --- /dev/null +++ b/src/utils/Macro.h @@ -0,0 +1,25 @@ +#pragma once + +#include "column/Column.h" +#include "column/CharColumn.h" +#include "column/NumericColumn.h" +#include "column/StringColumn.h" +#include "column/TemporalColumn.h" + +// USE THIS EVERYWHERE FOR SWITCHING +#define FOR_NUMERIC_COLUMN_TYPE(M) \ + M(INT16, "INT16", Int16Column) \ + M(INT32, "INT32", Int32Column) \ + M(INT64, "INT64", Int64Column) \ + M(INT128, "INT128", Int128Column) \ + M(FLOAT, "FLOAT", FloatColumn) \ + M(DOUBLE, "DOUBLE", DoubleColumn) \ + M(LONGDOUBLE, "LONGDOUBLE", LongDoubleColumn) \ + M(CHAR, "CHAR", CharColumn) \ + M(DATE, "DATE", DateColumn) \ + M(TIMESTAMP, "TIMESTAMP", TimestampColumn) + +// USE THIS EVERYWHERE FOR SWITCHING +#define FOR_EACH_COLUMN_TYPE(M) \ + FOR_NUMERIC_COLUMN_TYPE(M) \ + M(STRING, "STRING", StringColumn) diff --git a/src/utils/ThreadPool.h b/src/utils/ThreadPool.h new file mode 100644 index 0000000..b57e070 --- /dev/null +++ b/src/utils/ThreadPool.h @@ -0,0 +1,78 @@ +#pragma once + +#include "utils/Concurrency.h" +#include +#include + +#ifdef ENABLE_MULTITHREADING + +#include +#include + +class ThreadPool { +public: + explicit ThreadPool(size_t num_threads) : stop_(false) { + for (size_t i = 0; i < num_threads; ++i) { + workers_.emplace_back([this] { + for (;;) { + std::function task; + + { + std::unique_lock lock(this->queue_mutex_); + this->condition_.wait( + lock, [this] { return this->stop_ || !this->tasks_.empty(); }); + + if (this->stop_ && this->tasks_.empty()) { + return; + } + + task = std::move(this->tasks_.back()); + this->tasks_.pop_back(); + } + + task(); + } + }); + } + } + + void Enqueue(std::function task) { + { + std::unique_lock lock(queue_mutex_); + tasks_.push_back(std::move(task)); + } + condition_.notify_one(); + } + + ~ThreadPool() { + { + std::unique_lock lock(queue_mutex_); + stop_ = true; + } + condition_.notify_all(); + for (std::thread& worker : workers_) { + worker.join(); + } + } + +private: + std::vector workers_; + std::vector> tasks_; + std::mutex queue_mutex_; + std::condition_variable condition_; + bool stop_; +}; + +#else + +class ThreadPool { +public: + explicit ThreadPool(size_t /*num_threads*/) { + } + + void Enqueue(std::function task) { + task(); + } +}; + +#endif diff --git a/src/VectorOfStrings.h b/src/utils/VectorOfStrings.h similarity index 96% rename from src/VectorOfStrings.h rename to src/utils/VectorOfStrings.h index 8f7d6c6..ffed34f 100644 --- a/src/VectorOfStrings.h +++ b/src/utils/VectorOfStrings.h @@ -1,11 +1,6 @@ -// -// Created by ragnarokk on 01.04.2026. -// +#pragma once -#ifndef COLUMNAR_ENGINE_VECTOROFSTRINGS_H -#define COLUMNAR_ENGINE_VECTOROFSTRINGS_H - -#include "macro.h" +#include "utils/Assert.h" #include #include @@ -233,5 +228,3 @@ class VectorOfStrings2D { size_t columns_in_line_; bool adding_string_; }; - -#endif // COLUMNAR_ENGINE_VECTOROFSTRINGS_H diff --git a/src/test_vector_of_strings.cpp b/src/utils/test/TestVectorOfStrings.cpp similarity index 76% rename from src/test_vector_of_strings.cpp rename to src/utils/test/TestVectorOfStrings.cpp index dedca7c..64f48a4 100644 --- a/src/test_vector_of_strings.cpp +++ b/src/utils/test/TestVectorOfStrings.cpp @@ -1,14 +1,51 @@ -// -// Created by ragnarokk on 02.04.2026. -// -// AI generated +#include "utils/VectorOfStrings.h" #include +#include +#include -#include "VectorOfStrings.h" +TEST_CASE("VectorOfStrings: Basic operations", "[VectorOfStrings]") { + VectorOfStrings vec; + REQUIRE(vec.Size() == 0); + REQUIRE(vec.DataSize() == 0); + + vec.PushBack("hello"); + vec.PushBack("world"); + + REQUIRE(vec.Size() == 2); + REQUIRE(vec.GetString(0) == "hello"); + REQUIRE(vec.GetString(1) == "world"); +} -#include -#include +TEST_CASE("VectorOfStrings2D: Basic operations", "[VectorOfStrings2D]") { + VectorOfStrings2D vec; + vec.StartAddString(); + vec.ContinueAddString("hello"); + vec.EndAddString(); + + vec.StartAddString(); + vec.ContinueAddString("world"); + vec.EndAddString(); + + vec.StartNewLine(); + + vec.StartAddString(); + vec.ContinueAddString("foo"); + vec.EndAddString(); + + vec.StartAddString(); + vec.ContinueAddString("bar"); + vec.EndAddString(); + + vec.StartNewLine(); + + REQUIRE(vec.Height() == 2); + REQUIRE(vec.Width() == 2); + REQUIRE(vec.GetString2D(0, 0) == "hello"); + REQUIRE(vec.GetString2D(0, 1) == "world"); + REQUIRE(vec.GetString2D(1, 0) == "foo"); + REQUIRE(vec.GetString2D(1, 1) == "bar"); +} TEST_CASE("VectorOfStrings2D supports 1D vector mode", "[vector_of_strings]") { VectorOfStrings2D values; @@ -127,4 +164,3 @@ TEST_CASE("VectorOfStrings2D reads non square 2D table correctly", "[vector_of_s REQUIRE(table.GetString2D(1, 1) == std::string_view("r1c1")); REQUIRE(table.GetString2D(1, 2) == std::string_view("r1c2")); } -