Merge pull request #18 from ClickHouse/retain-structure

rschu1ze · web-flow · commit cf269cfc47d3 · 2025-03-11T16:08:10.000+01:00
Record which systems retain the document structure and which don't
diff --git a/README.md b/README.md
@@ -24,7 +24,9 @@ You can easily reproduce every test (although for some systems it may take from
 
 Best efforts should be taken to understand the details of every tested system for a fair comparison. It is allowed to apply various [indexing methods](https://clickhouse.com/blog/json-bench-clickhouse-vs-mongodb-elasticsearch-duckdb-postgresql#some-json-paths-can-be-used-for-indexes-and-data-sorting) whenever appropriate.
 
-It is [not allowed](https://clickhouse.com/blog/json-bench-clickhouse-vs-mongodb-elasticsearch-duckdb-postgresql#no-query-results-cache) to use query results caching or flatten JSON at the insertion time.
+It is [not allowed](https://clickhouse.com/blog/json-bench-clickhouse-vs-mongodb-elasticsearch-duckdb-postgresql#no-query-results-cache) to use query results caching or flatten JSON into multiple non-JSON colums at insertion type.
+
+Some databases do have a JSON data type but they flatten nested JSON documents at insertion time to a single level (typically using `.` as separator between levels). We consider this a grey zone. On the one hand, this removes the possibility to restore the original documents, on the other hand, flattening may in many practical situations be acceptable. The dashboard allows to filter out databases which do not retain the document structure (i.e. which flatten).
 
 ## Goals
 
@@ -85,7 +87,7 @@ For example, for clickhouse:
 
 ```
 cd clickhouse
-./main.sh 
+./main.sh
 
 Select the dataset size to benchmark:
 1) 1m (default)
diff --git a/clickhouse/results/m6i.8xlarge_bluesky_1000m_lz4.json b/clickhouse/results/m6i.8xlarge_bluesky_1000m_lz4.json
@@ -6,6 +6,7 @@
   "machine": "m6i.8xlarge, 10000gib gp3",
   "cluster_size": 1,
   "comment": "",
+  "retains_structure": "yes",
   "tags": [
   ],
   "dataset_size": 1000000000,
diff --git a/clickhouse/results/m6i.8xlarge_bluesky_1000m_zstd.json b/clickhouse/results/m6i.8xlarge_bluesky_1000m_zstd.json
@@ -6,6 +6,7 @@
   "machine": "m6i.8xlarge, 10000gib gp3",
   "cluster_size": 1,
   "comment": "",
+  "retains_structure": "yes",
   "tags": [
   ],
   "dataset_size": 1000000000,
diff --git a/clickhouse/results/m6i.8xlarge_bluesky_100m_lz4.json b/clickhouse/results/m6i.8xlarge_bluesky_100m_lz4.json
@@ -6,6 +6,7 @@
   "machine": "m6i.8xlarge, 10000gib gp3",
   "cluster_size": 1,
   "comment": "",
+  "retains_structure": "yes",
   "tags": [
   ],
   "dataset_size": 100000000,
diff --git a/clickhouse/results/m6i.8xlarge_bluesky_100m_zstd.json b/clickhouse/results/m6i.8xlarge_bluesky_100m_zstd.json
@@ -6,6 +6,7 @@
   "machine": "m6i.8xlarge, 10000gib gp3",
   "cluster_size": 1,
   "comment": "",
+  "retains_structure": "yes",
   "tags": [
   ],
   "dataset_size": 100000000,
diff --git a/clickhouse/results/m6i.8xlarge_bluesky_10m_lz4.json b/clickhouse/results/m6i.8xlarge_bluesky_10m_lz4.json
@@ -6,6 +6,7 @@
   "machine": "m6i.8xlarge, 10000gib gp3",
   "cluster_size": 1,
   "comment": "",
+  "retains_structure": "yes",
   "tags": [
   ],
   "dataset_size": 10000000,
diff --git a/clickhouse/results/m6i.8xlarge_bluesky_10m_zstd.json b/clickhouse/results/m6i.8xlarge_bluesky_10m_zstd.json
@@ -6,6 +6,7 @@
   "machine": "m6i.8xlarge, 10000gib gp3",
   "cluster_size": 1,
   "comment": "",
+  "retains_structure": "yes",
   "tags": [
   ],
   "dataset_size": 10000000,
diff --git a/clickhouse/results/m6i.8xlarge_bluesky_1m_lz4.json b/clickhouse/results/m6i.8xlarge_bluesky_1m_lz4.json
@@ -6,6 +6,7 @@
   "machine": "m6i.8xlarge, 10000gib gp3",
   "cluster_size": 1,
   "comment": "",
+  "retains_structure": "yes",
   "tags": [
   ],
   "dataset_size": 1000000,
diff --git a/clickhouse/results/m6i.8xlarge_bluesky_1m_zstd.json b/clickhouse/results/m6i.8xlarge_bluesky_1m_zstd.json
@@ -6,6 +6,7 @@
   "machine": "m6i.8xlarge, 10000gib gp3",
   "cluster_size": 1,
   "comment": "",
+  "retains_structure": "yes",
   "tags": [
   ],
   "dataset_size": 1000000,
diff --git a/duckdb/results/m6i.8xlarge_bluesky_1000m.json b/duckdb/results/m6i.8xlarge_bluesky_1000m.json
@@ -6,6 +6,7 @@
   "machine": "m6i.8xlarge, 16000gib gp3",
   "cluster_size": 1,
   "comment": "",
+  "retains_structure": "yes",
   "tags": [
   ],
   "dataset_size": 1000000000,
@@ -28,4 +29,4 @@
     "1 hr 2 min 26.50 sec, 1 hr 2 min 15.85 sec, 1 hr 2 min 19.26 sec",
     "1 hr 2 min 28.53 sec, 1 hr 2 min 16.93 sec, 1 hr 2 min 20.10 sec"
   ]
-}
+}
diff --git a/duckdb/results/m6i.8xlarge_bluesky_100m.json b/duckdb/results/m6i.8xlarge_bluesky_100m.json
@@ -6,6 +6,7 @@
   "machine": "m6i.8xlarge, 16000gib gp3",
   "cluster_size": 1,
   "comment": "",
+  "retains_structure": "yes",
   "tags": [
   ],
   "dataset_size": 100000000,
@@ -28,4 +29,4 @@
     "6 min 13.36 sec, 7.55 sec, 7.53 sec",
     "6 min 13.56 sec, 8.07 sec, 8.06 sec"
   ]
-}
+}
diff --git a/duckdb/results/m6i.8xlarge_bluesky_10m.json b/duckdb/results/m6i.8xlarge_bluesky_10m.json
@@ -6,6 +6,7 @@
   "machine": "m6i.8xlarge, 16000gib gp3",
   "cluster_size": 1,
   "comment": "",
+  "retains_structure": "yes",
   "tags": [
   ],
   "dataset_size": 10000000,
@@ -28,4 +29,4 @@
     "37.36 sec, 1.06 sec, 1.06 sec",
     "37.44 sec, 1.18 sec, 1.21 sec"
   ]
-}
+}
diff --git a/duckdb/results/m6i.8xlarge_bluesky_1m.json b/duckdb/results/m6i.8xlarge_bluesky_1m.json
@@ -6,6 +6,7 @@
   "machine": "m6i.8xlarge, 16000gib gp3",
   "cluster_size": 1,
   "comment": "",
+  "retains_structure": "yes",
   "tags": [
   ],
   "dataset_size": 1000000,
@@ -28,4 +29,4 @@
     "2.80 sec, 250.00 msec, 247.00 msec",
     "2.82 sec, 259.00 msec, 256.00 msec"
   ]
-}
+}
diff --git a/elasticsearch/results/m6i.8xlarge_bluesky_no_source_1000m_best_compression.json b/elasticsearch/results/m6i.8xlarge_bluesky_no_source_1000m_best_compression.json
@@ -6,6 +6,7 @@
   "machine": "m6i.8xlarge, 16000gib gp3",
   "cluster_size": 1,
   "comment": "",
+  "retains_structure": "no",
   "tags": [
   ],
   "dataset_size": 1000000000,
diff --git a/elasticsearch/results/m6i.8xlarge_bluesky_no_source_1000m_default_compression.json b/elasticsearch/results/m6i.8xlarge_bluesky_no_source_1000m_default_compression.json
@@ -6,6 +6,7 @@
   "machine": "m6i.8xlarge, 16000gib gp3",
   "cluster_size": 1,
   "comment": "",
+  "retains_structure": "no",
   "tags": [
   ],
   "dataset_size": 1000000000,
diff --git a/elasticsearch/results/m6i.8xlarge_bluesky_no_source_100m_best_compression.json b/elasticsearch/results/m6i.8xlarge_bluesky_no_source_100m_best_compression.json
@@ -6,6 +6,7 @@
   "machine": "m6i.8xlarge, 16000gib gp3",
   "cluster_size": 1,
   "comment": "",
+  "retains_structure": "no",
   "tags": [
   ],
   "dataset_size": 100000000,
diff --git a/elasticsearch/results/m6i.8xlarge_bluesky_no_source_100m_default_compression.json b/elasticsearch/results/m6i.8xlarge_bluesky_no_source_100m_default_compression.json
@@ -6,6 +6,7 @@
   "machine": "m6i.8xlarge, 16000gib gp3",
   "cluster_size": 1,
   "comment": "",
+  "retains_structure": "no",
   "tags": [
   ],
   "dataset_size": 100000000,
diff --git a/elasticsearch/results/m6i.8xlarge_bluesky_no_source_10m_best_compression.json b/elasticsearch/results/m6i.8xlarge_bluesky_no_source_10m_best_compression.json
@@ -6,6 +6,7 @@
   "machine": "m6i.8xlarge, 16000gib gp3",
   "cluster_size": 1,
   "comment": "",
+  "retains_structure": "no",
   "tags": [
   ],
   "dataset_size": 10000000,
diff --git a/elasticsearch/results/m6i.8xlarge_bluesky_no_source_10m_default_compression.json b/elasticsearch/results/m6i.8xlarge_bluesky_no_source_10m_default_compression.json
@@ -6,6 +6,7 @@
   "machine": "m6i.8xlarge, 16000gib gp3",
   "cluster_size": 1,
   "comment": "",
+  "retains_structure": "no",
   "tags": [
   ],
   "dataset_size": 10000000,
diff --git a/elasticsearch/results/m6i.8xlarge_bluesky_no_source_1m_best_compression.json b/elasticsearch/results/m6i.8xlarge_bluesky_no_source_1m_best_compression.json
@@ -6,6 +6,7 @@
   "machine": "m6i.8xlarge, 16000gib gp3",
   "cluster_size": 1,
   "comment": "",
+  "retains_structure": "no",
   "tags": [
   ],
   "dataset_size": 1000000,
diff --git a/elasticsearch/results/m6i.8xlarge_bluesky_no_source_1m_default_compression.json b/elasticsearch/results/m6i.8xlarge_bluesky_no_source_1m_default_compression.json
@@ -6,6 +6,7 @@
   "machine": "m6i.8xlarge, 16000gib gp3",
   "cluster_size": 1,
   "comment": "",
+  "retains_structure": "no",
   "tags": [
   ],
   "dataset_size": 1000000,
diff --git a/elasticsearch/results/m6i.8xlarge_bluesky_source_1000m_best_compression.json b/elasticsearch/results/m6i.8xlarge_bluesky_source_1000m_best_compression.json
@@ -6,6 +6,7 @@
   "machine": "m6i.8xlarge, 16000gib gp3",
   "cluster_size": 1,
   "comment": "",
+  "retains_structure": "yes",
   "tags": [
   ],
   "dataset_size": 1000000000,
diff --git a/elasticsearch/results/m6i.8xlarge_bluesky_source_1000m_default_compression.json b/elasticsearch/results/m6i.8xlarge_bluesky_source_1000m_default_compression.json
@@ -6,6 +6,7 @@
   "machine": "m6i.8xlarge, 16000gib gp3",
   "cluster_size": 1,
   "comment": "",
+  "retains_structure": "yes",
   "tags": [
   ],
   "dataset_size": 1000000000,
diff --git a/elasticsearch/results/m6i.8xlarge_bluesky_source_100m_best_compression.json b/elasticsearch/results/m6i.8xlarge_bluesky_source_100m_best_compression.json
@@ -6,6 +6,7 @@
   "machine": "m6i.8xlarge, 16000gib gp3",
   "cluster_size": 1,
   "comment": "",
+  "retains_structure": "yes",
   "tags": [
   ],
   "dataset_size": 100000000,
diff --git a/elasticsearch/results/m6i.8xlarge_bluesky_source_100m_default_compression.json b/elasticsearch/results/m6i.8xlarge_bluesky_source_100m_default_compression.json
@@ -6,6 +6,7 @@
   "machine": "m6i.8xlarge, 16000gib gp3",
   "cluster_size": 1,
   "comment": "",
+  "retains_structure": "yes",
   "tags": [
   ],
   "dataset_size": 100000000,
diff --git a/elasticsearch/results/m6i.8xlarge_bluesky_source_10m_best_compression.json b/elasticsearch/results/m6i.8xlarge_bluesky_source_10m_best_compression.json
@@ -6,6 +6,7 @@
   "machine": "m6i.8xlarge, 16000gib gp3",
   "cluster_size": 1,
   "comment": "",
+  "retains_structure": "yes",
   "tags": [
   ],
   "dataset_size": 10000000,
diff --git a/elasticsearch/results/m6i.8xlarge_bluesky_source_10m_default_compression.json b/elasticsearch/results/m6i.8xlarge_bluesky_source_10m_default_compression.json
@@ -6,6 +6,7 @@
   "machine": "m6i.8xlarge, 16000gib gp3",
   "cluster_size": 1,
   "comment": "",
+  "retains_structure": "yes",
   "tags": [
   ],
   "dataset_size": 10000000,
diff --git a/elasticsearch/results/m6i.8xlarge_bluesky_source_1m_best_compression.json b/elasticsearch/results/m6i.8xlarge_bluesky_source_1m_best_compression.json
@@ -6,6 +6,7 @@
   "machine": "m6i.8xlarge, 16000gib gp3",
   "cluster_size": 1,
   "comment": "",
+  "retains_structure": "yes",
   "tags": [
   ],
   "dataset_size": 1000000,
diff --git a/elasticsearch/results/m6i.8xlarge_bluesky_source_1m_default_compression.json b/elasticsearch/results/m6i.8xlarge_bluesky_source_1m_default_compression.json
@@ -6,6 +6,7 @@
   "machine": "m6i.8xlarge, 16000gib gp3",
   "cluster_size": 1,
   "comment": "",
+  "retains_structure": "yes",
   "tags": [
   ],
   "dataset_size": 1000000,
@@ -28,4 +29,4 @@
     "48.00 msec, 50.00 msec, 49.00 msec",
     "57.00 msec, 59.00 msec, 58.00 msec"
   ]
-}
+}
diff --git a/mongodb/results/m6i.8xlarge_bluesky_1000m_snappy.json b/mongodb/results/m6i.8xlarge_bluesky_1000m_snappy.json
@@ -6,6 +6,7 @@
   "machine": "m6i.8xlarge, 10000gib gp3",
   "cluster_size": 1,
   "comment": "",
+  "retains_structure": "yes",
   "tags": [
   ],
   "dataset_size": 1000000000,
diff --git a/mongodb/results/m6i.8xlarge_bluesky_1000m_zstd.json b/mongodb/results/m6i.8xlarge_bluesky_1000m_zstd.json
@@ -6,6 +6,7 @@
   "machine": "m6i.8xlarge, 10000gib gp3",
   "cluster_size": 1,
   "comment": "",
+  "retains_structure": "yes",
   "tags": [
   ],
   "dataset_size": 1000000000,
diff --git a/mongodb/results/m6i.8xlarge_bluesky_100m_snappy.json b/mongodb/results/m6i.8xlarge_bluesky_100m_snappy.json
@@ -6,6 +6,7 @@
   "machine": "m6i.8xlarge, 10000gib gp3",
   "cluster_size": 1,
   "comment": "",
+  "retains_structure": "yes",
   "tags": [
   ],
   "dataset_size": 100000000,
diff --git a/mongodb/results/m6i.8xlarge_bluesky_100m_zstd.json b/mongodb/results/m6i.8xlarge_bluesky_100m_zstd.json
@@ -6,6 +6,7 @@
   "machine": "m6i.8xlarge, 10000gib gp3",
   "cluster_size": 1,
   "comment": "",
+  "retains_structure": "yes",
   "tags": [
   ],
   "dataset_size": 100000000,
diff --git a/mongodb/results/m6i.8xlarge_bluesky_10m_snappy.json b/mongodb/results/m6i.8xlarge_bluesky_10m_snappy.json
@@ -6,6 +6,7 @@
   "machine": "m6i.8xlarge, 10000gib gp3",
   "cluster_size": 1,
   "comment": "",
+  "retains_structure": "yes",
   "tags": [
   ],
   "dataset_size": 10000000,
diff --git a/mongodb/results/m6i.8xlarge_bluesky_10m_zstd.json b/mongodb/results/m6i.8xlarge_bluesky_10m_zstd.json
@@ -6,6 +6,7 @@
   "machine": "m6i.8xlarge, 10000gib gp3",
   "cluster_size": 1,
   "comment": "",
+  "retains_structure": "yes",
   "tags": [
   ],
   "dataset_size": 10000000,
diff --git a/mongodb/results/m6i.8xlarge_bluesky_1m_snappy.json b/mongodb/results/m6i.8xlarge_bluesky_1m_snappy.json
@@ -6,6 +6,7 @@
   "machine": "m6i.8xlarge, 10000gib gp3",
   "cluster_size": 1,
   "comment": "",
+  "retains_structure": "yes",
   "tags": [
   ],
   "dataset_size": 1000000,
diff --git a/mongodb/results/m6i.8xlarge_bluesky_1m_zstd.json b/mongodb/results/m6i.8xlarge_bluesky_1m_zstd.json
@@ -6,6 +6,7 @@
   "machine": "m6i.8xlarge, 10000gib gp3",
   "cluster_size": 1,
   "comment": "",
+  "retains_structure": "yes",
   "tags": [
   ],
   "dataset_size": 1000000,
diff --git a/postgresql/results/m6i.8xlarge_bluesky_1000m_lz4.json b/postgresql/results/m6i.8xlarge_bluesky_1000m_lz4.json
@@ -6,6 +6,7 @@
   "machine": "m6i.8xlarge, 10000gib gp3",
   "cluster_size": 1,
   "comment": "",
+  "retains_structure": "yes",
   "tags": [
   ],
   "dataset_size": 1000000000,
diff --git a/postgresql/results/m6i.8xlarge_bluesky_1000m_pglz.json b/postgresql/results/m6i.8xlarge_bluesky_1000m_pglz.json
@@ -6,6 +6,7 @@
   "machine": "m6i.8xlarge, 10000gib gp3",
   "cluster_size": 1,
   "comment": "",
+  "retains_structure": "yes",
   "tags": [
   ],
   "dataset_size": 1000000000,
diff --git a/postgresql/results/m6i.8xlarge_bluesky_100m_lz4.json b/postgresql/results/m6i.8xlarge_bluesky_100m_lz4.json
@@ -6,6 +6,7 @@
   "machine": "m6i.8xlarge, 10000gib gp3",
   "cluster_size": 1,
   "comment": "",
+  "retains_structure": "yes",
   "tags": [
   ],
   "dataset_size": 100000000,
diff --git a/postgresql/results/m6i.8xlarge_bluesky_100m_pglz.json b/postgresql/results/m6i.8xlarge_bluesky_100m_pglz.json
@@ -6,6 +6,7 @@
   "machine": "m6i.8xlarge, 10000gib gp3",
   "cluster_size": 1,
   "comment": "",
+  "retains_structure": "yes",
   "tags": [
   ],
   "dataset_size": 100000000,
diff --git a/postgresql/results/m6i.8xlarge_bluesky_10m_lz4.json b/postgresql/results/m6i.8xlarge_bluesky_10m_lz4.json
@@ -6,6 +6,7 @@
   "machine": "m6i.8xlarge, 10000gib gp3",
   "cluster_size": 1,
   "comment": "",
+  "retains_structure": "yes",
   "tags": [
   ],
   "dataset_size": 10000000,
diff --git a/postgresql/results/m6i.8xlarge_bluesky_10m_pglz.json b/postgresql/results/m6i.8xlarge_bluesky_10m_pglz.json
@@ -6,6 +6,7 @@
   "machine": "m6i.8xlarge, 10000gib gp3",
   "cluster_size": 1,
   "comment": "",
+  "retains_structure": "yes",
   "tags": [
   ],
   "dataset_size": 10000000,
diff --git a/postgresql/results/m6i.8xlarge_bluesky_1m_lz4.json b/postgresql/results/m6i.8xlarge_bluesky_1m_lz4.json
@@ -6,6 +6,7 @@
   "machine": "m6i.8xlarge, 10000gib gp3",
   "cluster_size": 1,
   "comment": "",
+  "retains_structure": "yes",
   "tags": [
   ],
   "dataset_size": 1000000,
diff --git a/postgresql/results/m6i.8xlarge_bluesky_1m_pglz.json b/postgresql/results/m6i.8xlarge_bluesky_1m_pglz.json
diff --git a/victorialogs/results/m6i.8xlarge_bluesky_1000m.json b/victorialogs/results/m6i.8xlarge_bluesky_1000m.json
diff --git a/victorialogs/results/m6i.8xlarge_bluesky_100m.json b/victorialogs/results/m6i.8xlarge_bluesky_100m.json
diff --git a/victorialogs/results/m6i.8xlarge_bluesky_10m.json b/victorialogs/results/m6i.8xlarge_bluesky_10m.json
diff --git a/victorialogs/results/m6i.8xlarge_bluesky_1m.json b/victorialogs/results/m6i.8xlarge_bluesky_1m.json
