Merge pull request #21 from ClickHouse/single-store

Add SingleStore measurements

Author: rschu1ze

singlestore/benchmark.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+
+# Check if the required arguments are provided
+if [[ $# -lt 4 ]]; then
+    echo "Usage: $0 <ROOT_PASSWORD> <DB_NAME> <RESULT_FILE_RUNTIMES> <RESULT_FILE_MEMORY_USAGE>"
+    exit 1
+fi
+
+# Arguments
+ROOT_PASSWORD="$1"
+DB_NAME="$2"
+RESULT_FILE_RUNTIMES="$3"
+RESULT_FILE_MEMORY_USAGE="$4"
+
+# Construct the query log file name using $DB_NAME
+QUERY_LOG_FILE="_query_log_${DB_NAME}.txt"
+
+# Print the database name
+echo "Running queries on database: $DB_NAME"
+
+# Run queries and log the output
+./run_queries.sh "$ROOT_PASSWORD" "$DB_NAME" 2>&1 | tee "$QUERY_LOG_FILE"

singlestore/count.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+
+# Check if the required arguments are provided
+if [[ $# -lt 3 ]]; then
+    echo "Usage: $0 <ROOT_PASSWORD> <DB_NAME> <TABLE_NAME>"
+    exit 1
+fi
+
+# Arguments
+ROOT_PASSWORD="$1"
+DB_NAME="$2"
+TABLE_NAME="$3"
+
+export MYSQL_PWD=${ROOT_PASSWORD}
+
+mysql -h 127.0.0.1 -P 3306 -u root -e "SELECT count(*) FROM $DB_NAME.$TABLE_NAME"

singlestore/create_and_load.sh

@@ -0,0 +1,33 @@
+#!/bin/bash
+
+# Check if the required arguments are provided
+if [[ $# -lt 8 ]]; then
+    echo "Usage: $0 <ROOT_PASSWORD> <DB_NAME> <TABLE_NAME> <DDL_FILE> <DATA_DIRECTORY> <NUM_FILES> <SUCCESS_LOG> <ERROR_LOG>"
+    exit 1
+fi
+
+# Arguments
+ROOT_PASSWORD="$1"
+DB_NAME="$2"
+TABLE_NAME="$3"
+DDL_FILE="$4"
+DATA_DIRECTORY="$5"
+NUM_FILES="$6"
+SUCCESS_LOG="$7"
+ERROR_LOG="$8"
+
+# Validate arguments
+[[ ! -f "$DDL_FILE" ]] && { echo "Error: DDL file '$DDL_FILE' does not exist."; exit 1; }
+[[ ! -d "$DATA_DIRECTORY" ]] && { echo "Error: Data directory '$DATA_DIRECTORY' does not exist."; exit 1; }
+[[ ! "$NUM_FILES" =~ ^[0-9]+$ ]] && { echo "Error: NUM_FILES must be a positive integer."; exit 1; }
+
+export MYSQL_PWD=${ROOT_PASSWORD}
+
+echo "Creating database $DB_NAME"
+mysql -h 127.0.0.1 -P 3306 -u root -e "CREATE DATABASE IF NOT EXISTS $DB_NAME"
+
+echo "Executing DDL for database $DB_NAME"
+mysql -h 127.0.0.1 -P 3306 -u root $DB_NAME < "$DDL_FILE"
+
+echo "Loading data for database $DB_NAME"
+./load_data.sh "$ROOT_PASSWORD" "$DATA_DIRECTORY" "$DB_NAME" "$TABLE_NAME" "$NUM_FILES" "$SUCCESS_LOG" "$ERROR_LOG"

singlestore/data_size.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+
+# Check if the required arguments are provided
+if [[ $# -lt 3 ]]; then
+    echo "Usage: $0 <ROOT_PASSWORD> <DB_NAME> <TABLE_NAME>"
+    exit 1
+fi
+
+# Arguments
+ROOT_PASSWORD="$1"
+DB_NAME="$2"
+TABLE_NAME="$3"
+
+export MYSQL_PWD=${ROOT_PASSWORD}
+
+mysql -h 127.0.0.1 -P 3306 -u root -e "SELECT sum(compressed_size) FROM information_schema.columnar_segments WHERE database_name = '$DB_NAME' AND table_name = '$TABLE_NAME'"

singlestore/ddl.sql

@@ -0,0 +1,8 @@
+CREATE TABLE bluesky
+(
+    data JSON
+);
+-- Notes:
+--   - Not using data structures to speed up scans. In SingleStore, no sort keys or indexes can be created on JSON sub-columns.
+--   - The only physical optimization we use is 'use_seekable_json' but that is implicitly on: https://docs.singlestore.com/db/v8.9/create-a-database/columnstore/columnstore-seekability-using-json/
+--   - We _could_ run OPTIMIZE to force a merge but since we are also not doing this for other benchmarked databases, we omit that.

singlestore/drop_table.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+
+# Check if the required arguments are provided
+if [[ $# -lt 3 ]]; then
+    echo "Usage: $0 <ROOT_PASSWORD> <DB_NAME> <TABLE_NAME>"
+    exit 1
+fi
+
+ROOT_PASSWORD="$1"
+DB_NAME="$2"
+TABLE_NAME="$3"
+
+echo "Dropping table: $DB_NAME.$TABLE_NAME"
+
+export MYSQL_PWD=${ROOT_PASSWORD}
+mysql -h 127.0.0.1 -P 3306 -u root -e "DROP DATABASE IF EXISTS $DB_NAME"

singlestore/index_size.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+
+# Check if the required arguments are provided
+if [[ $# -lt 3 ]]; then
+    echo "Usage: $0 <ROOT_PASSWORD> <DB_NAME> <TABLE_NAME>"
+    exit 1
+fi
+
+# Arguments
+ROOT_PASSWORD="$1"
+DB_NAME="$2"
+TABLE_NAME="$3"
+
+export MYSQL_PWD=${ROOT_PASSWORD}
+
+mysql -h 127.0.0.1 -P 3306 -u root -e "SELECT sum(memory_use) FROM information_schema.index_statistics WHERE database_name = '$DB_NAME' AND table_name = '$TABLE_NAME'"

singlestore/install.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+
+# Check if the required arguments are provided
+if [[ $# -lt 2 ]]; then
+    echo "Usage: $0 <LICENSE_KEY> <ROOT_PASSWORD>"
+    exit 1
+fi
+
+# Arguments
+LICENSE_KEY="$1"
+ROOT_PASSWORD="$2"
+
+docker run -i --init \
+    --name singlestore-ciab \
+    -e LICENSE_KEY="${LICENSE_KEY}" \
+    -e ROOT_PASSWORD="${ROOT_PASSWORD}" \
+    -p 3306:3306 -p 8080:8080 \
+    singlestore/cluster-in-a-box
+
+docker start singlestore-ciab
+
+while true
+do
+    mysql -h 127.0.0.1 -P 3306 -u root --password="${ROOT_PASSWORD}" -e 'SELECT 1' && break
+    sleep 1
+done

singlestore/load_data.sh

@@ -0,0 +1,41 @@
+#!/bin/bash
+
+# Check if the required arguments are provided
+if [[ $# -lt 7 ]]; then
+    echo "Usage: $0 <ROOT_PASSWORD> <DATA_DIRECTORY> <DB_NAME> <TABLE_NAME> <MAX_FILES> <SUCCESS_LOG> <ERROR_LOG>"
+    exit 1
+fi
+
+
+# Arguments
+ROOT_PASSWORD="$1"
+DATA_DIRECTORY="$2"
+DB_NAME="$3"
+TABLE_NAME="$4"
+MAX_FILES="$5"
+SUCCESS_LOG="$6"
+ERROR_LOG="$7"
+
+# Validate arguments
+[[ ! -d "$DATA_DIRECTORY" ]] && { echo "Error: Data directory '$DATA_DIRECTORY' does not exist."; exit 1; }
+[[ ! "$MAX_FILES" =~ ^[0-9]+$ ]] && { echo "Error: MAX_FILES must be a positive integer."; exit 1; }
+
+export MYSQL_PWD=${ROOT_PASSWORD}
+
+# Load data
+counter=0
+for file in $(ls "$DATA_DIRECTORY"/*.json.gz | head -n "$MAX_FILES"); do
+    echo "Processing file: $file"
+
+    # Note: If one or more JSON documents in the currently processed file cannot be parsed (because of extremely deep nesting, line breaks
+    #       in unexpected places, etc.), then SingleStore will skip the _entire_ file. This unfortunately reduces the "data quality" metric
+    #       (= the number of successfully inserted JSON documents) quite a bit. SingleStore's LOAD statement comes with a SKIP PARSER ERRORS
+    #       clause that would theoretically allow to skip individual documents, but it is not supported for JSON
+    #       (https://www.singlestore.com/forum/t/pipeline-skip-parser-errors-with-json/2794).
+    mysql --local-infile=1 -h 127.0.0.1 -P 3306 -u root -D $DB_NAME -e "LOAD DATA LOCAL INFILE \"$file\" INTO TABLE bluesky(data <- %) FORMAT JSON"
+
+    counter=$((counter + 1))
+    if [[ $counter -ge $MAX_FILES ]]; then
+        break
+    fi
+done

singlestore/main.sh

@@ -0,0 +1,76 @@
+#!/bin/bash
+
+# --------------------------------------------------------------------------------------------------------------------------------------------------------------
+# NOTE: Provide license key and PASSWORD here
+LICENSE_KEY=""
+ROOT_PASSWORD=""
+# --------------------------------------------------------------------------------------------------------------------------------------------------------------
+
+# Default data directory
+DEFAULT_DATA_DIRECTORY=~/data/bluesky
+
+# Allow the user to optionally provide the data directory as an argument
+DATA_DIRECTORY="${1:-$DEFAULT_DATA_DIRECTORY}"
+
+# Define success and error log files
+SUCCESS_LOG="${2:-success.log}"
+ERROR_LOG="${3:-error.log}"
+
+# Define prefix for output files
+OUTPUT_PREFIX="${4:-_m6i.8xlarge}"
+
+# Check if the directory exists
+if [[ ! -d "$DATA_DIRECTORY" ]]; then
+    echo "Error: Data directory '$DATA_DIRECTORY' does not exist."
+    exit 1
+fi
+
+echo "Select the dataset size to benchmark:"
+echo "1) 1m (default)"
+echo "2) 10m"
+echo "3) 100m"
+echo "4) 1000m"
+echo "5) all"
+read -p "Enter the number corresponding to your choice: " choice
+
+# ./install.sh "${LICENSE_KEY}" "${ROOT_PASSWORD}"
+
+benchmark() {
+    local size=$1
+    # Check DATA_DIRECTORY contains the required number of files to run the benchmark
+    file_count=$(find "$DATA_DIRECTORY" -type f | wc -l)
+    if (( file_count < size )); then
+        echo "Error: Not enough files in '$DATA_DIRECTORY'. Required: $size, Found: $file_count."
+        exit 1
+    fi
+    ./create_and_load.sh "${ROOT_PASSWORD}" "bluesky_${size}m" bluesky "ddl.sql" "$DATA_DIRECTORY" "$size" "$SUCCESS_LOG" "$ERROR_LOG"
+    ./total_size.sh "${ROOT_PASSWORD}" "bluesky_${size}m" bluesky | tee "${OUTPUT_PREFIX}_bluesky_${size}m.total_size"
+    ./data_size.sh "${ROOT_PASSWORD}" "bluesky_${size}m" bluesky | tee "${OUTPUT_PREFIX}_bluesky_${size}m.data_size"
+    ./index_size.sh "${ROOT_PASSWORD}" "bluesky_${size}m" bluesky | tee "${OUTPUT_PREFIX}_bluesky_${size}m.index_size"
+    ./count.sh "${ROOT_PASSWORD}" "bluesky_${size}m" bluesky | tee "${OUTPUT_PREFIX}_bluesky_${size}m.count"
+    ./query_results.sh "${ROOT_PASSWORD}" "bluesky_${size}m" | tee "${OUTPUT_PREFIX}_bluesky_${size}m.query_results"
+    # ./physical_query_plans.sh "${ROOT_PASSWORD}" "bluesky_${size}m" | tee "${OUTPUT_PREFIX}_bluesky_${size}m.physical_query_plans"
+    ./benchmark.sh "${ROOT_PASSWORD}" "bluesky_${size}m" "${OUTPUT_PREFIX}_bluesky_${size}m.results_runtime" "${OUTPUT_PREFIX}_bluesky_${size}m.results_memory_usage"
+    ./drop_table.sh "${ROOT_PASSWORD}" "bluesky_${size}m" bluesky
+}
+
+case $choice in
+    2)
+        benchmark 10
+        ;;
+    3)
+        benchmark 100
+        ;;
+    4)
+        benchmark 1000
+        ;;
+    5)
+        benchmark 1
+        benchmark 10
+        benchmark 100
+        benchmark 1000
+        ;;
+    *)
+        benchmark 1
+        ;;
+esac