To review

.gitignore

@@ -18,17 +18,18 @@ compile_commands.json
 .clangd
 test_csv
 test.csv
+test.log
 test_csv2
 test_main.cpp
 hits_sample.csv
+hits_10m.csv
 columnar_hits_sample.tuff
 report.txt
 docker-test
 docker-test-results
 .vscode/
 src/project_bundle.txt
 
-
 # Prerequisites
 *.d
 

CMakeLists.txt

@@ -1,4 +1,4 @@
-cmake_minimum_required(VERSION 3.28)
+cmake_minimum_required(VERSION 3.30)
 
 project(columnar-engine)
 
@@ -15,7 +15,26 @@ include(cmake/TestSolution.cmake)
 include(cmake/SeminarUtils.cmake)
 
 include(FetchContent)
-include(cmake/FindGlog.cmake)
+
+set(ABSL_PROPAGATE_CXX_STD ON)
+set(BUILD_TESTING OFF)
+set(ABSL_ENABLE_INSTALL ON)
+
+FetchContent_Declare(
+        abseil-cpp
+        GIT_REPOSITORY https://github.com/abseil/abseil-cpp.git
+        GIT_TAG 20240116.2
+        SYSTEM
+)
+FetchContent_MakeAvailable(abseil-cpp)
+
+FetchContent_Declare(
+        re2
+        GIT_REPOSITORY https://github.com/google/re2.git
+        GIT_TAG 2024-02-01
+        SYSTEM
+)
+FetchContent_MakeAvailable(re2)
 
 find_package(Catch REQUIRED)
 find_package(Benchmark REQUIRED)

script/build.sh

@@ -12,4 +12,4 @@ cmake -S "${ROOT_DIR}" -B "${BUILD_DIR}" \
     -DCMAKE_BUILD_TYPE="${BUILD_TYPE}" \
     -DCMAKE_CXX_STANDARD=23
 
-cmake --build "${BUILD_DIR}" --target columnar-engine -j "$(nproc)"
+cmake --build "${BUILD_DIR}" -j "$(nproc)"

script/run_all_queries.sh

@@ -0,0 +1,145 @@
+#!/bin/bash
+set -e
+
+# usage:
+#   -i 3 - count of iterations (or --iterations)
+#   --answers - print answers for each query
+#   --debug - print time of execution for each query
+#   --cold  - drop OS page cache before each query (requires sudo)
+
+cd "$(dirname "$0")" || exit 1
+
+# Значения по умолчанию
+ITERATIONS=1
+SHOW_DEBUG=0
+SHOW_ANSWERS=0
+COLD_CACHE=0
+
+# Парсинг аргументов командной строки
+while [[ "$#" -gt 0 ]]; do
+    case $1 in
+        --debug) SHOW_DEBUG=1; shift ;;
+        --answers) SHOW_ANSWERS=1; shift ;;
+        --cold) COLD_CACHE=1; shift ;;
+        -i|--iterations) ITERATIONS="$2"; shift 2 ;;
+        *) echo "Unknown parameter passed: $1"; exit 1 ;;
+    esac
+done
+
+echo ">>> Running queries 0-42 (single thread)..."
+echo ">>> Iterations: $ITERATIONS | Show Debug: $SHOW_DEBUG | Show Answers: $SHOW_ANSWERS | Cold Cache: $COLD_CACHE"
+
+RESULTS_DIR="query_results"
+if [ "$SHOW_ANSWERS" -eq 1 ] || [ "$SHOW_DEBUG" -eq 1 ]; then
+    mkdir -p "${RESULTS_DIR}"
+fi
+
+# Массив для хранения всех времён (для общего geomean)
+ALL_TIMES=()
+
+for (( iter=1; iter<=ITERATIONS; iter++ ))
+do
+    echo "========================================="
+    echo ">>> Iteration $iter / $ITERATIONS"
+    echo "========================================="
+
+    # Массив времён для текущей итерации
+    ITER_TIMES=()
+
+    for i in {0..42}
+    do
+        echo -n ">>> Executing query $i ... "
+
+        # Куда пишем ответы
+        if [ "$SHOW_ANSWERS" -eq 1 ]; then
+            CSV_FILE="${RESULTS_DIR}/query_${i}_iter_${iter}.csv"
+        else
+            CSV_FILE="/dev/null"
+        fi
+
+        # Куда пишем логи (чтобы вытащить время)
+        if [ "$SHOW_DEBUG" -eq 1 ]; then
+            LOG_FILE="${RESULTS_DIR}/query_${i}_iter_${iter}.log"
+        else
+            LOG_FILE=$(mktemp) # Временный файл, который мы потом удалим
+        fi
+
+        # Формируем аргументы для run_query.sh
+        COLD_ARG=""
+        if [ "$COLD_CACHE" -eq 1 ]; then
+            COLD_ARG="--cold"
+        fi
+
+        set +e
+        ./run_query.sh "$i" ../columnar_hits_sample.tuff "$CSV_FILE" "$LOG_FILE" $COLD_ARG
+        exit_code=$?
+        set -e
+
+        # Вытаскиваем время из лога (формат: "Query 0 completed in 2.57227 ms")
+        EXEC_TIME=$(grep "completed in" "$LOG_FILE" | awk '{print $5, $6}' || true)
+        EXEC_TIME_MS=$(grep "completed in" "$LOG_FILE" | awk '{print $5}' || true)
+
+        if [ $exit_code -ne 0 ]; then
+            echo "FAILED with exit code $exit_code"
+            echo "--- CRASH LOG ---"
+            cat "$LOG_FILE"
+            if [ "$SHOW_DEBUG" -eq 0 ]; then rm -f "$LOG_FILE"; fi
+            exit $exit_code
+        fi
+
+        # Красиво выводим время
+        if [ -n "$EXEC_TIME" ]; then
+            echo "${EXEC_TIME} ... OK"
+        else
+            echo "OK"
+        fi
+
+        # Сохраняем время для geomean
+        if [ -n "$EXEC_TIME_MS" ]; then
+            ITER_TIMES+=("$EXEC_TIME_MS")
+            ALL_TIMES+=("$EXEC_TIME_MS")
+        fi
+
+        if [ "$SHOW_ANSWERS" -eq 1 ]; then
+            echo "------ ANSWERS (Query $i) ------"
+            cat "$CSV_FILE"
+            echo "--------------------------------"
+        fi
+
+        if [ "$SHOW_DEBUG" -eq 1 ]; then
+            echo "------- DEBUG (Query $i) -------"
+            cat "$LOG_FILE"
+            echo "--------------------------------"
+        else
+            # Удаляем мусор, если дебаг не нужен
+            rm -f "$LOG_FILE"
+        fi
+    done
+
+    # Считаем geomean для текущей итерации
+    if [ ${#ITER_TIMES[@]} -gt 0 ]; then
+        GEOMEAN=$(printf '%s\n' "${ITER_TIMES[@]}" | awk '{
+            sum += log($1); n++
+        } END {
+            if (n > 0) printf "%.5f", exp(sum / n)
+        }')
+        echo "========================================="
+        echo ">>> Geomean for iteration $iter: ${GEOMEAN} ms"
+        echo "========================================="
+    fi
+done
+
+# Считаем общий geomean по всем итерациям
+if [ ${#ALL_TIMES[@]} -gt 0 ]; then
+    TOTAL_GEOMEAN=$(printf '%s\n' "${ALL_TIMES[@]}" | awk '{
+        sum += log($1); n++
+    } END {
+        if (n > 0) printf "%.5f", exp(sum / n)
+    }')
+    echo ""
+    echo "========================================="
+    echo ">>> Overall geomean ($ITERATIONS iterations): ${TOTAL_GEOMEAN} ms"
+    echo "========================================="
+fi
+
+echo ">>> All $ITERATIONS iterations of all queries completed successfully."

script/run_all_queries_mt.sh

@@ -0,0 +1,145 @@
+#!/bin/bash
+set -e
+
+# usage:
+#   -i 3 - count of iterations (or --iterations)
+#   --answers - print answers for each query
+#   --debug - print time of execution for each query
+#   --cold  - drop OS page cache before each query (requires sudo)
+
+cd "$(dirname "$0")" || exit 1
+
+# Значения по умолчанию
+ITERATIONS=1
+SHOW_DEBUG=0
+SHOW_ANSWERS=0
+COLD_CACHE=0
+
+# Парсинг аргументов командной строки
+while [[ "$#" -gt 0 ]]; do
+    case $1 in
+        --debug) SHOW_DEBUG=1; shift ;;
+        --answers) SHOW_ANSWERS=1; shift ;;
+        --cold) COLD_CACHE=1; shift ;;
+        -i|--iterations) ITERATIONS="$2"; shift 2 ;;
+        *) echo "Unknown parameter passed: $1"; exit 1 ;;
+    esac
+done
+
+echo ">>> Running queries 0-42 (multi thread)..."
+echo ">>> Iterations: $ITERATIONS | Show Debug: $SHOW_DEBUG | Show Answers: $SHOW_ANSWERS | Cold Cache: $COLD_CACHE"
+
+RESULTS_DIR="query_mt_results"
+if [ "$SHOW_ANSWERS" -eq 1 ] || [ "$SHOW_DEBUG" -eq 1 ]; then
+    mkdir -p "${RESULTS_DIR}"
+fi
+
+# Массив для хранения всех времён (для общего geomean)
+ALL_TIMES=()
+
+for (( iter=1; iter<=ITERATIONS; iter++ ))
+do
+    echo "========================================="
+    echo ">>> Iteration $iter / $ITERATIONS"
+    echo "========================================="
+
+    # Массив времён для текущей итерации
+    ITER_TIMES=()
+
+    for i in {0..42}
+    do
+        echo -n ">>> Executing query $i ... "
+
+        # Куда пишем ответы
+        if [ "$SHOW_ANSWERS" -eq 1 ]; then
+            CSV_FILE="${RESULTS_DIR}/query_${i}_iter_${iter}.csv"
+        else
+            CSV_FILE="/dev/null"
+        fi
+
+        # Куда пишем логи (чтобы вытащить время)
+        if [ "$SHOW_DEBUG" -eq 1 ]; then
+            LOG_FILE="${RESULTS_DIR}/query_${i}_iter_${iter}.log"
+        else
+            LOG_FILE=$(mktemp) # Временный файл, который мы потом удалим
+        fi
+
+        # Формируем аргументы для run_query_mt.sh
+        COLD_ARG=""
+        if [ "$COLD_CACHE" -eq 1 ]; then
+            COLD_ARG="--cold"
+        fi
+
+        set +e
+        ./run_query_mt.sh "$i" ../columnar_hits_sample.tuff "$CSV_FILE" "$LOG_FILE" $COLD_ARG
+        exit_code=$?
+        set -e
+
+        # Вытаскиваем время из лога (формат: "Query 0 completed in 2.57227 ms")
+        EXEC_TIME=$(grep "completed in" "$LOG_FILE" | awk '{print $5, $6}' || true)
+        EXEC_TIME_MS=$(grep "completed in" "$LOG_FILE" | awk '{print $5}' || true)
+
+        if [ $exit_code -ne 0 ]; then
+            echo "FAILED with exit code $exit_code"
+            echo "--- CRASH LOG ---"
+            cat "$LOG_FILE"
+            if [ "$SHOW_DEBUG" -eq 0 ]; then rm -f "$LOG_FILE"; fi
+            exit $exit_code
+        fi
+
+        # Красиво выводим время
+        if [ -n "$EXEC_TIME" ]; then
+            echo "${EXEC_TIME} ... OK"
+        else
+            echo "OK"
+        fi
+
+        # Сохраняем время для geomean
+        if [ -n "$EXEC_TIME_MS" ]; then
+            ITER_TIMES+=("$EXEC_TIME_MS")
+            ALL_TIMES+=("$EXEC_TIME_MS")
+        fi
+
+        if [ "$SHOW_ANSWERS" -eq 1 ]; then
+            echo "------ ANSWERS (Query $i) ------"
+            cat "$CSV_FILE"
+            echo "--------------------------------"
+        fi
+
+        if [ "$SHOW_DEBUG" -eq 1 ]; then
+            echo "------- DEBUG (Query $i) -------"
+            cat "$LOG_FILE"
+            echo "--------------------------------"
+        else
+            # Удаляем мусор, если дебаг не нужен
+            rm -f "$LOG_FILE"
+        fi
+    done
+
+    # Считаем geomean для текущей итерации
+    if [ ${#ITER_TIMES[@]} -gt 0 ]; then
+        GEOMEAN=$(printf '%s\n' "${ITER_TIMES[@]}" | awk '{
+            sum += log($1); n++
+        } END {
+            if (n > 0) printf "%.5f", exp(sum / n)
+        }')
+        echo "========================================="
+        echo ">>> Geomean for iteration $iter: ${GEOMEAN} ms"
+        echo "========================================="
+    fi
+done
+
+# Считаем общий geomean по всем итерациям
+if [ ${#ALL_TIMES[@]} -gt 0 ]; then
+    TOTAL_GEOMEAN=$(printf '%s\n' "${ALL_TIMES[@]}" | awk '{
+        sum += log($1); n++
+    } END {
+        if (n > 0) printf "%.5f", exp(sum / n)
+    }')
+    echo ""
+    echo "========================================="
+    echo ">>> Overall geomean ($ITERATIONS iterations): ${TOTAL_GEOMEAN} ms"
+    echo "========================================="
+fi
+
+echo ">>> All $ITERATIONS iterations of all queries completed successfully."

script/run_query.sh

@@ -4,14 +4,15 @@ set -euo pipefail
 ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
 
 if [[ $# -lt 4 ]]; then
-    echo "Usage: script/run_query.sh <query_num> <columnar> <output_csv> <log_file>" >&2
+    echo "Usage: script/run_query.sh <query_num> <columnar> <output_csv> <log_file> [--cold]" >&2
     exit 2
 fi
 
 QUERY_NUM="$1"
 COLUMNAR="$2"
 OUTPUT_CSV="$3"
 LOG_FILE="$4"
+DROP_CACHE="${5:-}"
 
 BUILD_DIR="${ROOT_DIR}/cmake-build-release"
 BIN="${BUILD_DIR}/columnar-engine"
@@ -27,8 +28,19 @@ if [[ ! -f "${COLUMNAR}" ]]; then
     exit 2
 fi
 
-mkdir -p "$(dirname "${OUTPUT_CSV}")"
-mkdir -p "$(dirname "${LOG_FILE}")"
+# Создаем директории, только если это реальные файлы, а не /dev/null
+if [[ "${OUTPUT_CSV}" != "/dev/null" ]]; then
+    mkdir -p "$(dirname "${OUTPUT_CSV}")"
+fi
+if [[ "${LOG_FILE}" != "/dev/null" ]]; then
+    mkdir -p "$(dirname "${LOG_FILE}")"
+fi
+
+# Очищаем кэш ОС для холодного запуска
+if [[ "${DROP_CACHE}" == "--cold" ]]; then
+    sync
+    sudo sh -c 'echo 3 > /proc/sys/vm/drop_caches'
+fi
 
-# Run the query, capture stdout (CSV result) and tee stderr+stdout to log
-"${BIN}" query "${COLUMNAR}" "${QUERY_NUM}" > "${OUTPUT_CSV}" 2> >(tee "${LOG_FILE}" >&2)
+# Run the query, capture stdout to CSV and stderr to log
+"${BIN}" query "${COLUMNAR}" "${QUERY_NUM}" > "${OUTPUT_CSV}" 2> "${LOG_FILE}"