Merge pull request #29 from ClickHouse/dont-trash-my-system

Install ClickHouse in local directory instead of system

Author: rschu1ze

clickhouse/count.sh

@@ -10,4 +10,4 @@ fi
 DB_NAME="$1"
 TABLE_NAME="$2"
 
-clickhouse-client --database="$DB_NAME" --query "SELECT count() FROM '$TABLE_NAME';"
+./clickhouse client --database="$DB_NAME" --query "SELECT count() FROM '$TABLE_NAME';"

clickhouse/create_and_load.sh

@@ -22,10 +22,10 @@ ERROR_LOG="$7"
 
 
 echo "Creating database $DB_NAME"
-clickhouse-client --query "CREATE DATABASE IF NOT EXISTS $DB_NAME"
+./clickhouse client --query "CREATE DATABASE IF NOT EXISTS $DB_NAME"
 
 echo "Executing DDL for database $DB_NAME"
-clickhouse-client --database="$DB_NAME" --enable_json_type=1 --multiquery < "$DDL_FILE"
+./clickhouse client --database="$DB_NAME" --enable_json_type=1 --multiquery < "$DDL_FILE"
 
 echo "Loading data for database $DB_NAME"
 ./load_data.sh "$DATA_DIRECTORY" "$DB_NAME" "$TABLE_NAME" "$NUM_FILES" "$SUCCESS_LOG" "$ERROR_LOG"

clickhouse/data_size.sh

@@ -10,4 +10,4 @@ fi
 DB_NAME="$1"
 TABLE_NAME="$2"
 
-clickhouse-client --query "SELECT sum(data_compressed_bytes) FROM system.parts WHERE database = '$DB_NAME' AND table = '$TABLE_NAME' AND active"
+./clickhouse client --query "SELECT sum(data_compressed_bytes) FROM system.parts WHERE database = '$DB_NAME' AND table = '$TABLE_NAME' AND active"

clickhouse/drop_table.sh

@@ -1,14 +1,9 @@
 #!/bin/bash
 
-# Check if the required arguments are provided
-if [[ $# -lt 2 ]]; then
-    echo "Usage: $0 <DB_NAME> <TABLE_NAME>"
-    exit 1
-fi
+echo "Stopping ClickHouse"
+pidof clickhouse && kill -9 `pidof clickhouse`
 
-DB_NAME="$1"
-TABLE_NAME="$2"
-
-echo "Dropping table: $DB_NAME.$TABLE_NAME"
-
-clickhouse-client --database="$DB_NAME" --query "DROP TABLE IF EXISTS $TABLE_NAME"
+# 'DROP TABLE' has a build-in safety mechanism that prevents users from dropping large tables. We hit that with large
+# numbers of ingested data. Instead, make our lifes easy and remove the persistence manually.
+echo "Dropping all data"
+rm -rf data/ metadata/ store/

clickhouse/index_size.sh

@@ -10,4 +10,4 @@ fi
 DB_NAME="$1"
 TABLE_NAME="$2"
 
-clickhouse-client --query "SELECT sum(primary_key_size) + sum(marks_bytes) FROM system.parts WHERE database = '$DB_NAME' AND table = '$TABLE_NAME' AND active"
+./clickhouse client --query "SELECT sum(primary_key_size) + sum(marks_bytes) FROM system.parts WHERE database = '$DB_NAME' AND table = '$TABLE_NAME' AND active"

clickhouse/index_usage.sh

@@ -18,8 +18,8 @@ cat queries.sql | while read -r query; do
     echo "Index usage for query Q$QUERY_NUM:"
     echo
 
-    clickhouse-client --database="$DB_NAME" --query="EXPLAIN indexes=1 $query"
+    ./clickhouse client --database="$DB_NAME" --query="EXPLAIN indexes=1 $query"
 
     # Increment the query number
     QUERY_NUM=$((QUERY_NUM + 1))
-done;
+done;

clickhouse/install.sh

@@ -1,12 +1,3 @@
 #!/bin/bash
 
 curl https://clickhouse.com/ | sh
-sudo ./clickhouse install --noninteractive
-sudo clickhouse start
-
-while true
-do
-    clickhouse-client --query "SELECT 1" && break
-    sleep 1
-done
-

clickhouse/load_data.sh

@@ -39,7 +39,7 @@ for file in $(ls "$DATA_DIRECTORY"/*.json.gz | head -n "$MAX_FILES"); do
     fi
 
     # Attempt the first import
-    clickhouse-client --query="INSERT INTO $DB_NAME.$TABLE_NAME SETTINGS min_insert_block_size_rows = 1_000_000, min_insert_block_size_bytes = 0 FORMAT JSONAsObject" < "$uncompressed_file"
+    ./clickhouse client --query="INSERT INTO $DB_NAME.$TABLE_NAME SETTINGS min_insert_block_size_rows = 1_000_000, min_insert_block_size_bytes = 0 FORMAT JSONAsObject" < "$uncompressed_file"
     first_attempt=$?
 
     # Check if the first import was successful
@@ -51,7 +51,7 @@ for file in $(ls "$DATA_DIRECTORY"/*.json.gz | head -n "$MAX_FILES"); do
 
         echo "Processing $file... again..."
         # Attempt the second import with a different command
-        clickhouse-client --query="INSERT INTO $DB_NAME.$TABLE_NAME SETTINGS min_insert_block_size_rows = 1_000_000, min_insert_block_size_bytes = 0, input_format_allow_errors_num = 1_000_000_000, input_format_allow_errors_ratio=1 FORMAT JSONAsObject" < "$uncompressed_file"
+        ./clickhouse client --query="INSERT INTO $DB_NAME.$TABLE_NAME SETTINGS min_insert_block_size_rows = 1_000_000, min_insert_block_size_bytes = 0, input_format_allow_errors_num = 1_000_000_000, input_format_allow_errors_ratio=1 FORMAT JSONAsObject" < "$uncompressed_file"
         second_attempt=$?
 
         # Check if the second import was successful

clickhouse/main.sh

@@ -38,6 +38,7 @@ benchmark() {
         echo "Error: Not enough files in '$DATA_DIRECTORY'. Required: $size, Found: $file_count."
         exit 1
     fi
+    ./start.sh
     ./create_and_load.sh "bluesky_${size}m_${suffix}" bluesky "ddl_${suffix}.sql" "$DATA_DIRECTORY" "$size" "$SUCCESS_LOG" "$ERROR_LOG"
     ./total_size.sh "bluesky_${size}m_${suffix}" bluesky | tee "${OUTPUT_PREFIX}_bluesky_${size}m_${suffix}.total_size"
     ./data_size.sh "bluesky_${size}m_${suffix}" bluesky | tee "${OUTPUT_PREFIX}_bluesky_${size}m_${suffix}.data_size"
@@ -47,7 +48,7 @@ benchmark() {
     ./index_usage.sh "bluesky_${size}m_${suffix}" | tee "${OUTPUT_PREFIX}_bluesky_${size}m_${suffix}.index_usage"
     ./physical_query_plans.sh "bluesky_${size}m_${suffix}" | tee "${OUTPUT_PREFIX}_bluesky_${size}m_${suffix}.physical_query_plans"
     ./benchmark.sh "bluesky_${size}m_${suffix}" "${OUTPUT_PREFIX}_bluesky_${size}m_${suffix}.results_runtime" "${OUTPUT_PREFIX}_bluesky_${size}m_${suffix}.results_memory_usage"
-    ./drop_table.sh "bluesky_${size}m_${suffix}" bluesky
+    ./drop_table.sh # also stops ClickHouse
 }
 
 case $choice in

clickhouse/physical_query_plans.sh

@@ -18,8 +18,8 @@ cat queries.sql | while read -r query; do
     echo "Physical query plan for query Q$QUERY_NUM:"
     echo
 
-    clickhouse-client --database="$DB_NAME" --query="EXPLAIN PIPELINE $query"
+    ./clickhouse client --database="$DB_NAME" --query="EXPLAIN PIPELINE $query"
 
     # Increment the query number
     QUERY_NUM=$((QUERY_NUM + 1))
-done;
+done;