diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5e6451d22b1..372e2639bb0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -136,6 +136,22 @@ jobs: uv run --all-packages make html working-directory: docs/ + # Pins the measurement_id hash that scripts/post-ingest.py uses as the primary key of every + # benchmarks-database fact row. The golden vectors are a frozen artifact (see the note inside + # the JSON); this job fails if a change to scripts/_measurement_id.py drifts the hash, which + # would break ON CONFLICT idempotency against the rows already in production. + measurement-id-golden: + name: "measurement_id golden vectors" + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 + - uses: spiraldb/actions/.github/actions/setup-uv@a746510eafaa926484c354541cfc49b2ec06cc63 # 0.18.6 + - name: Pytest - measurement_id golden vectors + run: | + uv run --no-project --with pytest --with xxhash \ + pytest scripts/tests/test_measurement_id.py + python-cuda-test: name: "Python CUDA (test)" if: github.repository == 'vortex-data/vortex' diff --git a/_typos.toml b/_typos.toml index e2a5ff330b8..d5d2a8e4db3 100644 --- a/_typos.toml +++ b/_typos.toml @@ -8,7 +8,10 @@ extend-ignore-re = [ ] [files] -extend-exclude = ["/vortex-bench/**", "/docs/references.bib", "benchmarks/**", "vortex-sqllogictest/slt/**", "encodings/fsst/src/dfa/tests.rs", "encodings/fsst/src/dfa/flat_contains.rs", "benchmarks-website/server/static/**", "benchmarks-website/server/tests/snapshots/**"] +# scripts/measurement_id_golden.json is a frozen hash-pin artifact whose vectors include +# deliberate Unicode edge cases (JSON-escaped, so "café" reads as the token "caf"); its bytes +# must never change, so it is excluded rather than "fixed". +extend-exclude = ["/vortex-bench/**", "/docs/references.bib", "benchmarks/**", "vortex-sqllogictest/slt/**", "encodings/fsst/src/dfa/tests.rs", "encodings/fsst/src/dfa/flat_contains.rs", "benchmarks-website/server/static/**", "benchmarks-website/server/tests/snapshots/**", "scripts/measurement_id_golden.json"] [type.py] extend-ignore-identifiers-re = [ diff --git a/scripts/_measurement_id.py b/scripts/_measurement_id.py index e8834f4c097..6e670c38dd0 100644 --- a/scripts/_measurement_id.py +++ b/scripts/_measurement_id.py @@ -1,20 +1,20 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright the Vortex contributors -"""Python port of the server-internal `measurement_id` xxhash64 functions. - -This is a byte-for-byte port of the `vortex-data/benchmarks-website` repo's -`server/src/db.rs` (`measurement_id_*`, `hasher_for`, -`write_str`, `write_opt_str`, `write_i32`, `write_f64`, `finish`). The v4 ingest -writer computes the same `measurement_id` the v3 Rust server computes, so -re-ingesting an existing `(commit, dim-tuple)` upserts the existing row via -`ON CONFLICT (measurement_id) DO UPDATE` instead of inserting a duplicate. - -The equivalence is NOT assumed -- it is pinned by golden vectors generated from -the Rust source of truth (the `measurement_id_golden_vectors` test in the -`vortex-data/benchmarks-website` repo's `server/src/db.rs`). The Python port is -checked against those vectors; any drift in either implementation fails that -check. +"""The `measurement_id` xxhash64 functions -- the primary-key hash of the benchmarks database. + +This began as a byte-for-byte port of the Rust implementation in the `vortex-data/ +benchmarks-website` repo's `server/src/db.rs` (`measurement_id_*`, `hasher_for`, `write_str`, +`write_opt_str`, `write_i32`, `write_f64`, `finish`); with that server crate retired, this module +is now the only implementation. Re-ingesting an existing `(commit, dim-tuple)` upserts the +existing row via `ON CONFLICT (measurement_id) DO UPDATE` instead of inserting a duplicate, so +the hash is frozen forever: every row already in the production Postgres carries an id computed +this way. + +The freeze is NOT assumed -- it is pinned by `scripts/measurement_id_golden.json`, whose vectors +were generated by the original Rust source of truth and must never be regenerated. +`scripts/tests/test_measurement_id.py` asserts this module reproduces every vector; any drift +fails that check. ## The hash, precisely diff --git a/scripts/measurement_id_golden.json b/scripts/measurement_id_golden.json new file mode 100644 index 00000000000..5df8fe9775d --- /dev/null +++ b/scripts/measurement_id_golden.json @@ -0,0 +1,825 @@ +{ + "note": "Frozen golden vectors for the measurement_id_* hash, generated by the original Rust implementation (server/src/db.rs in the retired vortex-data/benchmarks-website server crate) that wrote the production rows. NEVER regenerate this file: the ids it pins are the primary keys already stored in the benchmarks RDS Postgres, and any drift would break ON CONFLICT (measurement_id) idempotency. scripts/tests/test_measurement_id.py asserts the Python port in scripts/_measurement_id.py reproduces every measurement_id here.", + "seed": 0, + "vectors": [ + { + "fields": { + "commit_sha": "0000000000000000000000000000000000000000", + "dataset": "dataset-0", + "dataset_variant": null, + "engine": "vortex", + "format": "vortex-file-compressed", + "query_idx": -20, + "scale_factor": null, + "storage": "nvme" + }, + "measurement_id": 8941066430020068021, + "table": "query_measurements" + }, + { + "fields": { + "commit_sha": "0000000000000000000000009e3779b97f4a7c15", + "dataset": "dataset-1", + "dataset_variant": "variant-1", + "engine": "vortex", + "format": "vortex-file-compressed", + "query_idx": -19, + "scale_factor": "sf-1", + "storage": "s3" + }, + "measurement_id": 3149995011396621799, + "table": "query_measurements" + }, + { + "fields": { + "commit_sha": "0000000000000000000000003c6ef372fe94f82a", + "dataset": "dataset-2", + "dataset_variant": "variant-2", + "engine": "vortex", + "format": "vortex-file-compressed", + "query_idx": -18, + "scale_factor": null, + "storage": "nvme" + }, + "measurement_id": -6215740072949592228, + "table": "query_measurements" + }, + { + "fields": { + "commit_sha": "000000000000000000000000daa66d2c7ddf743f", + "dataset": "dataset-3", + "dataset_variant": null, + "engine": "vortex", + "format": "vortex-file-compressed", + "query_idx": -17, + "scale_factor": "sf-3", + "storage": "s3" + }, + "measurement_id": 8041805059478031959, + "table": "query_measurements" + }, + { + "fields": { + "commit_sha": "00000000000000000000000078dde6e5fd29f054", + "dataset": "dataset-4", + "dataset_variant": "variant-4", + "engine": "vortex", + "format": "vortex-file-compressed", + "query_idx": -16, + "scale_factor": null, + "storage": "nvme" + }, + "measurement_id": 4772600626526399345, + "table": "query_measurements" + }, + { + "fields": { + "commit_sha": "0000000000000000000000001715609f7c746c69", + "dataset": "dataset-5", + "dataset_variant": "variant-5", + "engine": "vortex", + "format": "vortex-file-compressed", + "query_idx": -15, + "scale_factor": "sf-5", + "storage": "s3" + }, + "measurement_id": -9016235375010445132, + "table": "query_measurements" + }, + { + "fields": { + "commit_sha": "000000000000000000000000b54cda58fbbee87e", + "dataset": "dataset-6", + "dataset_variant": null, + "engine": "vortex", + "format": "vortex-file-compressed", + "query_idx": -14, + "scale_factor": null, + "storage": "nvme" + }, + "measurement_id": -6411293753624450917, + "table": "query_measurements" + }, + { + "fields": { + "commit_sha": "000000000000000000000000538454127b096493", + "dataset": "dataset-7", + "dataset_variant": "variant-7", + "engine": "vortex", + "format": "vortex-file-compressed", + "query_idx": -13, + "scale_factor": "sf-7", + "storage": "s3" + }, + "measurement_id": 8460757218662476151, + "table": "query_measurements" + }, + { + "fields": { + "commit_sha": "000000000000000000000000f1bbcdcbfa53e0a8", + "dataset": "dataset-8", + "dataset_variant": "variant-8", + "engine": "vortex", + "format": "vortex-file-compressed", + "query_idx": -12, + "scale_factor": null, + "storage": "nvme" + }, + "measurement_id": 7670946045315319280, + "table": "query_measurements" + }, + { + "fields": { + "commit_sha": "0000000000000000000000008ff34785799e5cbd", + "dataset": "dataset-9", + "dataset_variant": null, + "engine": "vortex", + "format": "vortex-file-compressed", + "query_idx": -11, + "scale_factor": "sf-9", + "storage": "s3" + }, + "measurement_id": 1978441881557979666, + "table": "query_measurements" + }, + { + "fields": { + "commit_sha": "0000000000000000000000002e2ac13ef8e8d8d2", + "dataset": "dataset-10", + "dataset_variant": "variant-10", + "engine": "vortex", + "format": "vortex-file-compressed", + "query_idx": -10, + "scale_factor": null, + "storage": "nvme" + }, + "measurement_id": 8135912901467482164, + "table": "query_measurements" + }, + { + "fields": { + "commit_sha": "000000000000000000000000cc623af8783354e7", + "dataset": "dataset-11", + "dataset_variant": "variant-11", + "engine": "vortex", + "format": "vortex-file-compressed", + "query_idx": -9, + "scale_factor": "sf-11", + "storage": "s3" + }, + "measurement_id": 805733765350646006, + "table": "query_measurements" + }, + { + "fields": { + "commit_sha": "0000000000000000000000006a99b4b1f77dd0fc", + "dataset": "dataset-12", + "dataset_variant": null, + "engine": "vortex", + "format": "vortex-file-compressed", + "query_idx": -8, + "scale_factor": null, + "storage": "nvme" + }, + "measurement_id": -4091026328830870798, + "table": "query_measurements" + }, + { + "fields": { + "commit_sha": "00000000000000000000000008d12e6b76c84d11", + "dataset": "dataset-13", + "dataset_variant": "variant-13", + "engine": "vortex", + "format": "vortex-file-compressed", + "query_idx": -7, + "scale_factor": "sf-13", + "storage": "s3" + }, + "measurement_id": -6061385115965648155, + "table": "query_measurements" + }, + { + "fields": { + "commit_sha": "000000000000000000000000a708a824f612c926", + "dataset": "dataset-14", + "dataset_variant": "variant-14", + "engine": "vortex", + "format": "vortex-file-compressed", + "query_idx": -6, + "scale_factor": null, + "storage": "nvme" + }, + "measurement_id": -6684883584448916863, + "table": "query_measurements" + }, + { + "fields": { + "commit_sha": "000000000000000000000000454021de755d453b", + "dataset": "dataset-15", + "dataset_variant": null, + "engine": "vortex", + "format": "vortex-file-compressed", + "query_idx": -5, + "scale_factor": "sf-15", + "storage": "s3" + }, + "measurement_id": 8814896453978445645, + "table": "query_measurements" + }, + { + "fields": { + "commit_sha": "000000000000000000000000e3779b97f4a7c150", + "dataset": "dataset-16", + "dataset_variant": "variant-16", + "engine": "vortex", + "format": "vortex-file-compressed", + "query_idx": -4, + "scale_factor": null, + "storage": "nvme" + }, + "measurement_id": 6686567714742361044, + "table": "query_measurements" + }, + { + "fields": { + "commit_sha": "00000000000000000000000081af155173f23d65", + "dataset": "dataset-17", + "dataset_variant": "variant-17", + "engine": "vortex", + "format": "vortex-file-compressed", + "query_idx": -3, + "scale_factor": "sf-17", + "storage": "s3" + }, + "measurement_id": 3557917168552872470, + "table": "query_measurements" + }, + { + "fields": { + "commit_sha": "0000000000000000000000001fe68f0af33cb97a", + "dataset": "dataset-18", + "dataset_variant": null, + "engine": "vortex", + "format": "vortex-file-compressed", + "query_idx": -2, + "scale_factor": null, + "storage": "nvme" + }, + "measurement_id": -6165762661706299592, + "table": "query_measurements" + }, + { + "fields": { + "commit_sha": "000000000000000000000000be1e08c47287358f", + "dataset": "dataset-19", + "dataset_variant": "variant-19", + "engine": "vortex", + "format": "vortex-file-compressed", + "query_idx": -1, + "scale_factor": "sf-19", + "storage": "s3" + }, + "measurement_id": -535031789163192453, + "table": "query_measurements" + }, + { + "fields": { + "commit_sha": "0000000000000000000000005c55827df1d1b1a4", + "dataset": "dataset-20", + "dataset_variant": "variant-20", + "engine": "vortex", + "format": "vortex-file-compressed", + "query_idx": 0, + "scale_factor": null, + "storage": "nvme" + }, + "measurement_id": 4634990895302954655, + "table": "query_measurements" + }, + { + "fields": { + "commit_sha": "000000000000000000000000fa8cfc37711c2db9", + "dataset": "dataset-21", + "dataset_variant": null, + "engine": "vortex", + "format": "vortex-file-compressed", + "query_idx": 1, + "scale_factor": "sf-21", + "storage": "s3" + }, + "measurement_id": 5936866269340399925, + "table": "query_measurements" + }, + { + "fields": { + "commit_sha": "00000000000000000000000098c475f0f066a9ce", + "dataset": "dataset-22", + "dataset_variant": "variant-22", + "engine": "vortex", + "format": "vortex-file-compressed", + "query_idx": 2, + "scale_factor": null, + "storage": "nvme" + }, + "measurement_id": 4471015533681449689, + "table": "query_measurements" + }, + { + "fields": { + "commit_sha": "00000000000000000000000036fbefaa6fb125e3", + "dataset": "dataset-23", + "dataset_variant": "variant-23", + "engine": "vortex", + "format": "vortex-file-compressed", + "query_idx": 3, + "scale_factor": "sf-23", + "storage": "s3" + }, + "measurement_id": 3270457410965634007, + "table": "query_measurements" + }, + { + "fields": { + "commit_sha": "000000000000000000000000d5336963eefba1f8", + "dataset": "dataset-24", + "dataset_variant": null, + "engine": "vortex", + "format": "vortex-file-compressed", + "query_idx": 4, + "scale_factor": null, + "storage": "nvme" + }, + "measurement_id": -6630843892626754319, + "table": "query_measurements" + }, + { + "fields": { + "commit_sha": "000000000000000000000000736ae31d6e461e0d", + "dataset": "dataset-25", + "dataset_variant": "variant-25", + "engine": "vortex", + "format": "vortex-file-compressed", + "query_idx": 5, + "scale_factor": "sf-25", + "storage": "s3" + }, + "measurement_id": 5238548701197127815, + "table": "query_measurements" + }, + { + "fields": { + "commit_sha": "00000000000000000000000011a25cd6ed909a22", + "dataset": "dataset-26", + "dataset_variant": "variant-26", + "engine": "vortex", + "format": "vortex-file-compressed", + "query_idx": 6, + "scale_factor": null, + "storage": "nvme" + }, + "measurement_id": -869863853831808211, + "table": "query_measurements" + }, + { + "fields": { + "commit_sha": "000000000000000000000000afd9d6906cdb1637", + "dataset": "dataset-27", + "dataset_variant": null, + "engine": "vortex", + "format": "vortex-file-compressed", + "query_idx": 7, + "scale_factor": "sf-27", + "storage": "s3" + }, + "measurement_id": -6175971040302506030, + "table": "query_measurements" + }, + { + "fields": { + "commit_sha": "0000000000000000000000004e115049ec25924c", + "dataset": "dataset-28", + "dataset_variant": "variant-28", + "engine": "vortex", + "format": "vortex-file-compressed", + "query_idx": 8, + "scale_factor": null, + "storage": "nvme" + }, + "measurement_id": 4758474844754348116, + "table": "query_measurements" + }, + { + "fields": { + "commit_sha": "000000000000000000000000ec48ca036b700e61", + "dataset": "dataset-29", + "dataset_variant": "variant-29", + "engine": "vortex", + "format": "vortex-file-compressed", + "query_idx": 9, + "scale_factor": "sf-29", + "storage": "s3" + }, + "measurement_id": 4027371045171197062, + "table": "query_measurements" + }, + { + "fields": { + "commit_sha": "0000000000000000000000008a8043bceaba8a76", + "dataset": "dataset-30", + "dataset_variant": null, + "engine": "vortex", + "format": "vortex-file-compressed", + "query_idx": 10, + "scale_factor": null, + "storage": "nvme" + }, + "measurement_id": 6213625536713947544, + "table": "query_measurements" + }, + { + "fields": { + "commit_sha": "00000000000000000000000028b7bd766a05068b", + "dataset": "dataset-31", + "dataset_variant": "variant-31", + "engine": "vortex", + "format": "vortex-file-compressed", + "query_idx": 11, + "scale_factor": "sf-31", + "storage": "s3" + }, + "measurement_id": 3264730627889711995, + "table": "query_measurements" + }, + { + "fields": { + "commit_sha": "000000000000000000000000c6ef372fe94f82a0", + "dataset": "dataset-32", + "dataset_variant": "variant-32", + "engine": "vortex", + "format": "vortex-file-compressed", + "query_idx": 12, + "scale_factor": null, + "storage": "nvme" + }, + "measurement_id": 6426500585598625566, + "table": "query_measurements" + }, + { + "fields": { + "commit_sha": "0000000000000000000000006526b0e96899feb5", + "dataset": "dataset-33", + "dataset_variant": null, + "engine": "vortex", + "format": "vortex-file-compressed", + "query_idx": 13, + "scale_factor": "sf-33", + "storage": "s3" + }, + "measurement_id": -470855441762601549, + "table": "query_measurements" + }, + { + "fields": { + "commit_sha": "000000000000000000000000035e2aa2e7e47aca", + "dataset": "dataset-34", + "dataset_variant": "variant-34", + "engine": "vortex", + "format": "vortex-file-compressed", + "query_idx": 14, + "scale_factor": null, + "storage": "nvme" + }, + "measurement_id": -8702944831802119265, + "table": "query_measurements" + }, + { + "fields": { + "commit_sha": "000000000000000000000000a195a45c672ef6df", + "dataset": "dataset-35", + "dataset_variant": "variant-35", + "engine": "vortex", + "format": "vortex-file-compressed", + "query_idx": 15, + "scale_factor": "sf-35", + "storage": "s3" + }, + "measurement_id": 6237767006190293538, + "table": "query_measurements" + }, + { + "fields": { + "commit_sha": "0000000000000000000000003fcd1e15e67972f4", + "dataset": "dataset-36", + "dataset_variant": null, + "engine": "vortex", + "format": "vortex-file-compressed", + "query_idx": 16, + "scale_factor": null, + "storage": "nvme" + }, + "measurement_id": 5322839569260259127, + "table": "query_measurements" + }, + { + "fields": { + "commit_sha": "000000000000000000000000de0497cf65c3ef09", + "dataset": "dataset-37", + "dataset_variant": "variant-37", + "engine": "vortex", + "format": "vortex-file-compressed", + "query_idx": 17, + "scale_factor": "sf-37", + "storage": "s3" + }, + "measurement_id": 1995931045924102080, + "table": "query_measurements" + }, + { + "fields": { + "commit_sha": "0000000000000000000000007c3c1188e50e6b1e", + "dataset": "dataset-38", + "dataset_variant": "variant-38", + "engine": "vortex", + "format": "vortex-file-compressed", + "query_idx": 18, + "scale_factor": null, + "storage": "nvme" + }, + "measurement_id": -7932518466629682266, + "table": "query_measurements" + }, + { + "fields": { + "commit_sha": "0000000000000000000000001a738b426458e733", + "dataset": "dataset-39", + "dataset_variant": null, + "engine": "vortex", + "format": "vortex-file-compressed", + "query_idx": 19, + "scale_factor": "sf-39", + "storage": "s3" + }, + "measurement_id": 7227626056758470989, + "table": "query_measurements" + }, + { + "fields": { + "commit_sha": "0123456789abcdef0123456789abcdef01234567", + "dataset": "tpch", + "dataset_variant": null, + "engine": "vortex", + "format": "parquet", + "query_idx": 7, + "scale_factor": "1", + "storage": "nvme" + }, + "measurement_id": 6778969912115220139, + "table": "query_measurements" + }, + { + "fields": { + "commit_sha": "0123456789abcdef0123456789abcdef01234567", + "dataset": "\ub370\uc774\ud130\uc14b-\u65e5\u672c\u8a9e", + "dataset_variant": "caf\u00e9-\u03a9", + "engine": "\u30f4", + "format": "format-\u2713", + "query_idx": 0, + "scale_factor": "\u03c3", + "storage": "s3" + }, + "measurement_id": 8038081617766937373, + "table": "query_measurements" + }, + { + "fields": { + "commit_sha": "0123456789abcdef0123456789abcdef01234567", + "dataset": "boundary", + "dataset_variant": null, + "engine": "e", + "format": "f", + "query_idx": -2147483648, + "scale_factor": null, + "storage": "nvme" + }, + "measurement_id": -1997262439178828726, + "table": "query_measurements" + }, + { + "fields": { + "commit_sha": "0123456789abcdef0123456789abcdef01234567", + "dataset": "boundary", + "dataset_variant": null, + "engine": "e", + "format": "f", + "query_idx": 2147483647, + "scale_factor": null, + "storage": "nvme" + }, + "measurement_id": -1111737216055603595, + "table": "query_measurements" + }, + { + "fields": { + "commit_sha": "0123456789abcdef0123456789abcdef01234567", + "dataset": "", + "dataset_variant": "", + "engine": "", + "format": "", + "query_idx": 0, + "scale_factor": "", + "storage": "" + }, + "measurement_id": -4121125782605521286, + "table": "query_measurements" + }, + { + "fields": { + "commit_sha": "0123456789abcdef0123456789abcdef01234567", + "dataset": "chimp", + "dataset_variant": null, + "format": "vortex", + "op": "encode" + }, + "measurement_id": -1351591591404870516, + "table": "compression_times" + }, + { + "fields": { + "commit_sha": "0123456789abcdef0123456789abcdef01234567", + "dataset": "chimp", + "dataset_variant": null, + "format": "vortex", + "op": "decode" + }, + "measurement_id": 8221793816172450272, + "table": "compression_times" + }, + { + "fields": { + "commit_sha": "0123456789abcdef0123456789abcdef01234567", + "dataset": "taxi", + "dataset_variant": "v2", + "format": "parquet", + "op": "encode" + }, + "measurement_id": -192124532873811017, + "table": "compression_times" + }, + { + "fields": { + "commit_sha": "0123456789abcdef0123456789abcdef01234567", + "dataset": "\uc555\ucd95-\u30c6\u30b9\u30c8", + "dataset_variant": "\u5909\u7a2e", + "format": "vortex-\u2713", + "op": "decode" + }, + "measurement_id": -2942165613664654219, + "table": "compression_times" + }, + { + "fields": { + "commit_sha": "0123456789abcdef0123456789abcdef01234567", + "dataset": "chimp", + "dataset_variant": null, + "format": "vortex" + }, + "measurement_id": 4543757287978490921, + "table": "compression_sizes" + }, + { + "fields": { + "commit_sha": "0123456789abcdef0123456789abcdef01234567", + "dataset": "taxi", + "dataset_variant": "v2", + "format": "parquet" + }, + "measurement_id": -7313339766868556874, + "table": "compression_sizes" + }, + { + "fields": { + "commit_sha": "0123456789abcdef0123456789abcdef01234567", + "dataset": "\ud06c\uae30-\u30b5\u30a4\u30ba", + "dataset_variant": "\u5909\u7a2e-\u03a9", + "format": "vortex-\u2713" + }, + "measurement_id": -8576081510595521142, + "table": "compression_sizes" + }, + { + "fields": { + "commit_sha": "0123456789abcdef0123456789abcdef01234567", + "dataset": "chimp", + "format": "vortex" + }, + "measurement_id": 4734678989716641189, + "table": "random_access_times" + }, + { + "fields": { + "commit_sha": "0123456789abcdef0123456789abcdef01234567", + "dataset": "taxi", + "format": "parquet" + }, + "measurement_id": 164255076009922704, + "table": "random_access_times" + }, + { + "fields": { + "commit_sha": "0123456789abcdef0123456789abcdef01234567", + "dataset": "\u7121\u4f5c\u70ba", + "format": "\u2713" + }, + "measurement_id": 832111591737631247, + "table": "random_access_times" + }, + { + "fields": { + "commit_sha": "0123456789abcdef0123456789abcdef01234567", + "dataset": "cohere-large-10m", + "flavor": "vortex", + "layout": "trained", + "threshold": 0.0 + }, + "measurement_id": 2427216480405807826, + "table": "vector_search_runs" + }, + { + "fields": { + "commit_sha": "0123456789abcdef0123456789abcdef01234567", + "dataset": "cohere-large-10m", + "flavor": "vortex", + "layout": "trained", + "threshold": 0.5 + }, + "measurement_id": -1711405494360824574, + "table": "vector_search_runs" + }, + { + "fields": { + "commit_sha": "0123456789abcdef0123456789abcdef01234567", + "dataset": "cohere-large-10m", + "flavor": "vortex", + "layout": "trained", + "threshold": 0.83 + }, + "measurement_id": 2194658981253663824, + "table": "vector_search_runs" + }, + { + "fields": { + "commit_sha": "0123456789abcdef0123456789abcdef01234567", + "dataset": "cohere-large-10m", + "flavor": "vortex", + "layout": "trained", + "threshold": 1.0 + }, + "measurement_id": -5633397304139473003, + "table": "vector_search_runs" + }, + { + "fields": { + "commit_sha": "0123456789abcdef0123456789abcdef01234567", + "dataset": "cohere-large-10m", + "flavor": "vortex", + "layout": "trained", + "threshold": -1.5 + }, + "measurement_id": -5505640980838817192, + "table": "vector_search_runs" + }, + { + "fields": { + "commit_sha": "0123456789abcdef0123456789abcdef01234567", + "dataset": "cohere-large-10m", + "flavor": "vortex", + "layout": "trained", + "threshold": 1e-09 + }, + "measurement_id": -812426564273547024, + "table": "vector_search_runs" + }, + { + "fields": { + "commit_sha": "0123456789abcdef0123456789abcdef01234567", + "dataset": "cohere-large-10m", + "flavor": "vortex", + "layout": "trained", + "threshold": 1000000000000.0 + }, + "measurement_id": 8044969450004818277, + "table": "vector_search_runs" + }, + { + "fields": { + "commit_sha": "0123456789abcdef0123456789abcdef01234567", + "dataset": "cohere-large-10m", + "flavor": "vortex", + "layout": "trained", + "threshold": 3.141592653589793 + }, + "measurement_id": 3687626387279485390, + "table": "vector_search_runs" + } + ] +} diff --git a/scripts/tests/test_measurement_id.py b/scripts/tests/test_measurement_id.py new file mode 100644 index 00000000000..c8baf0cfb8d --- /dev/null +++ b/scripts/tests/test_measurement_id.py @@ -0,0 +1,62 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright the Vortex contributors + +"""Golden-vector pin for the `measurement_id` hash in `scripts/_measurement_id.py`. + +The `measurement_id` computed by `scripts/post-ingest.py --postgres` is the primary key of every +fact row in the benchmarks-website RDS Postgres. Re-ingesting an existing `(commit, dim-tuple)` +upserts via `ON CONFLICT (measurement_id)`; if the hash ever drifted, re-ingests would insert +duplicate rows next to millions of existing ones instead. The hash is therefore frozen forever. + +`scripts/measurement_id_golden.json` pins it: the vectors were generated by the original Rust +implementation (`server/src/db.rs` in the now-deleted `vortex-data/benchmarks-website` `server/` +crate, the source of truth at the time the production rows were written). With the Rust side +retired, the golden file is a frozen artifact -- it must NEVER be regenerated, because the ids it +pins are the ids already stored in production. This test asserts the Python port reproduces every +vector byte-for-byte. + +Run with: + + uv run --no-project --with pytest --with xxhash pytest scripts/tests/test_measurement_id.py +""" + +import importlib.util +import json +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[2] +MODULE_PATH = REPO_ROOT / "scripts" / "_measurement_id.py" +GOLDEN_PATH = REPO_ROOT / "scripts" / "measurement_id_golden.json" + + +def load_measurement_id_module(): + spec = importlib.util.spec_from_file_location("_measurement_id", MODULE_PATH) + assert spec is not None + module = importlib.util.module_from_spec(spec) + assert spec.loader is not None + spec.loader.exec_module(module) + return module + + +def test_every_golden_vector_reproduces(): + module = load_measurement_id_module() + golden = json.loads(GOLDEN_PATH.read_text()) + assert golden["seed"] == 0 + + for vector in golden["vectors"]: + port_fn = module.MEASUREMENT_ID_BY_TABLE[vector["table"]] + computed = port_fn(**vector["fields"]) + assert computed == vector["measurement_id"], ( + f"measurement_id drift for table {vector['table']!r} fields {vector['fields']!r}: " + f"computed {computed}, golden {vector['measurement_id']}" + ) + + +def test_every_fact_table_is_covered(): + """Every port function must have at least one golden vector, so a new fact table cannot be + wired into `MEASUREMENT_ID_BY_TABLE` without also pinning its hash.""" + module = load_measurement_id_module() + golden = json.loads(GOLDEN_PATH.read_text()) + + covered_tables = {vector["table"] for vector in golden["vectors"]} + assert covered_tables == set(module.MEASUREMENT_ID_BY_TABLE)