diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml new file mode 100644 index 00000000..b16871ab --- /dev/null +++ b/.github/workflows/benchmark.yml @@ -0,0 +1,77 @@ +name: "Scheduled Benchmarks (Tier 2)" + +on: + schedule: + - cron: '0 3 * * 1' # Every Monday 03:00 UTC + workflow_dispatch: + +# Prevent a scheduled run from racing a manual dispatch for the same ports. +concurrency: + group: scheduled-benchmarks + cancel-in-progress: false + +env: + # Matches test-eql.yml — forces JS-based composite actions onto Node 24. + FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true" + +jobs: + benchmark: + name: "100K dataset benchmark (Postgres 17)" + runs-on: ubuntu-latest + timeout-minutes: 60 + + steps: + - uses: actions/checkout@v4 + + - name: Install postgresql-client + # generate.sh uses psql directly against Postgres (port 7433) and Proxy + # (port 6433). jdx/mise-action only installs Rust + Python. + run: | + sudo apt-get update + sudo apt-get install -y postgresql-client + + - uses: jdx/mise-action@v3 + with: + version: 2026.4.0 + install: true + cache: true + + - name: Write Proxy credentials to .env + env: + CS_CLIENT_ACCESS_KEY: ${{ secrets.CS_CLIENT_ACCESS_KEY }} + CS_DEFAULT_KEYSET_ID: ${{ secrets.CS_DEFAULT_KEYSET_ID }} + CS_CLIENT_KEY: ${{ secrets.CS_CLIENT_KEY }} + CS_CLIENT_ID: ${{ secrets.CS_CLIENT_ID }} + CS_WORKSPACE_CRN: ${{ secrets.CS_WORKSPACE_CRN }} + run: | + { + printf 'CS_CLIENT_ACCESS_KEY=%s\n' "$CS_CLIENT_ACCESS_KEY" + printf 'CS_DEFAULT_KEYSET_ID=%s\n' "$CS_DEFAULT_KEYSET_ID" + printf 'CS_CLIENT_KEY=%s\n' "$CS_CLIENT_KEY" + printf 'CS_CLIENT_ID=%s\n' "$CS_CLIENT_ID" + printf 'CS_WORKSPACE_CRN=%s\n' "$CS_WORKSPACE_CRN" + } > tests/benchmarks/.env + + - name: Bring up Postgres + Proxy + run: mise run bench:up + + - name: Generate 100K dataset + run: mise run bench:generate + + - name: Run Tier 2 benchmark suite + run: | + BENCH_REPORT_DATE="$(date -u +%Y-%m-%d)-${{ github.run_id }}" + export BENCH_REPORT_DATE + mise run bench:full + + - name: Tear down containers + if: always() + run: mise run bench:down + + - name: Upload benchmark report + if: always() + uses: actions/upload-artifact@v4 + with: + name: benchmark-report-${{ github.run_id }} + path: tests/benchmarks/reports/ + retention-days: 90 diff --git a/mise.toml b/mise.toml index ff70e4ce..878cb8cc 100644 --- a/mise.toml +++ b/mise.toml @@ -14,7 +14,7 @@ "python" = "3.13" [task_config] -includes = ["tasks", "tasks/postgres.toml"] +includes = ["tasks", "tasks/postgres.toml", "tasks/bench.toml"] [env] POSTGRES_DB = "cipherstash" diff --git a/tasks/bench.toml b/tasks/bench.toml new file mode 100644 index 00000000..f6cd8d13 --- /dev/null +++ b/tasks/bench.toml @@ -0,0 +1,35 @@ +["bench:up"] +description = "Start Postgres + Proxy for benchmark data generation" +dir = "{{config_root}}" +run = """ +if [ ! -f tests/benchmarks/.env ]; then + echo "ERROR: tests/benchmarks/.env missing. Copy .env.example and fill in credentials." >&2 + exit 1 +fi +docker compose --env-file tests/benchmarks/.env -f tests/benchmarks/docker-compose.yml up -d --wait +""" + +["bench:down"] +description = "Stop benchmark Postgres + Proxy" +dir = "{{config_root}}" +run = """ +docker compose -f tests/benchmarks/docker-compose.yml down -v +""" + +["bench:generate"] +description = "Generate 100K encrypted bench dataset (requires bench:up first)" +# `build` produces release/cipherstash-encrypt.sql, which generate.sh +# installs into the bench Postgres container before applying schema.sql. +depends = ["build"] +dir = "{{config_root}}" +run = """ +tests/benchmarks/generate.sh 100k +""" + +["bench:full"] +description = "Run full Tier 2 benchmark suite against bench-postgres" +dir = "{{config_root}}/tests/sqlx" +env = { DATABASE_URL = "postgresql://cipherstash:password@localhost:7433/cipherstash" } +run = """ +cargo test --test bench_perf_tests run_all_benchmarks -- --ignored --nocapture +""" diff --git a/tests/benchmarks/.env.example b/tests/benchmarks/.env.example new file mode 100644 index 00000000..fe41909a --- /dev/null +++ b/tests/benchmarks/.env.example @@ -0,0 +1,7 @@ +# CipherStash Proxy credentials +# Get these from https://dashboard.cipherstash.com +CS_CLIENT_ACCESS_KEY= +CS_DEFAULT_KEYSET_ID= +CS_CLIENT_KEY= +CS_CLIENT_ID= +CS_WORKSPACE_CRN= diff --git a/tests/benchmarks/.gitignore b/tests/benchmarks/.gitignore new file mode 100644 index 00000000..9e7d7623 --- /dev/null +++ b/tests/benchmarks/.gitignore @@ -0,0 +1,6 @@ +# Generated reports (too large for git, regenerated on demand) +reports/* +!reports/.gitkeep + +# Local Proxy credentials +.env diff --git a/tests/benchmarks/README.md b/tests/benchmarks/README.md new file mode 100644 index 00000000..d4aa57db --- /dev/null +++ b/tests/benchmarks/README.md @@ -0,0 +1,48 @@ +# EQL Scheduled Benchmarks (Tier 2) + +Heavy-weight performance benchmarks that run weekly in CI against 100K-row +encrypted datasets. Complements the Tier 1 tests in `tests/sqlx/tests/bench_*`. + +## What this is + +- Brings up Postgres + CipherStash Proxy via docker-compose +- Inserts 100K plaintext rows through the Proxy (which encrypts them) +- Runs each P0/P1/P2 query pattern 10 times +- Reads `pg_stat_statements` for statistical aggregates +- Outputs JSON + Markdown reports + +## Local usage + +```bash +# Populate credentials +cp tests/benchmarks/.env.example tests/benchmarks/.env +# Edit .env with your CipherStash credentials + +# Start Postgres + Proxy +mise run bench:up + +# Build EQL and generate 100K dataset (bench:generate depends on build) +mise run bench:generate + +# Run the full Tier 2 suite +mise run bench:full + +# Results land in tests/benchmarks/reports/ +``` + +## CI usage + +Runs automatically every Monday at 03:00 UTC via +`.github/workflows/benchmark.yml`. Also manually invocable from the +GitHub Actions UI (Run workflow button). + +## Why a separate workflow + +- 100K generation takes ~100 seconds via the Proxy +- The slowest pattern (`bench_ore_order_by_limit`) takes several seconds per run on 100K rows +- Regular PR CI must stay under 10 minutes; this suite would blow that budget + +## Output + +`tests/benchmarks/reports/benchmark-YYYY-MM-DD.{json,md}` — uploaded as +GitHub Actions artifact named `benchmark-report-`. diff --git a/tests/benchmarks/docker-compose.yml b/tests/benchmarks/docker-compose.yml new file mode 100644 index 00000000..bd35ba65 --- /dev/null +++ b/tests/benchmarks/docker-compose.yml @@ -0,0 +1,59 @@ +services: + postgres: + image: postgres:17 + container_name: bench-postgres + command: > + postgres + -c track_functions=all + -c shared_preload_libraries=pg_stat_statements + -c pg_stat_statements.track=all + -c pg_stat_statements.max=10000 + ports: + - "127.0.0.1:7433:5432" + environment: + POSTGRES_DB: cipherstash + POSTGRES_USER: cipherstash + POSTGRES_PASSWORD: password + healthcheck: + test: ["CMD-SHELL", "pg_isready -U cipherstash"] + interval: 1s + timeout: 5s + retries: 10 + networks: + - bench + + proxy: + image: cipherstash/proxy:latest + container_name: bench-proxy + ports: + - "127.0.0.1:6433:6432" + environment: + CS_DATABASE__NAME: cipherstash + CS_DATABASE__USERNAME: cipherstash + CS_DATABASE__PASSWORD: password + CS_DATABASE__HOST: postgres + CS_DATABASE__PORT: 5432 + # EQL install is performed explicitly by generate.sh before schema.sql runs. + # Leaving Proxy's own install off avoids racing against generate.sh. + CS_DATABASE__INSTALL_EQL: "false" + CS_CLIENT_ACCESS_KEY: ${CS_CLIENT_ACCESS_KEY} + CS_DEFAULT_KEYSET_ID: ${CS_DEFAULT_KEYSET_ID} + CS_CLIENT_KEY: ${CS_CLIENT_KEY} + CS_CLIENT_ID: ${CS_CLIENT_ID} + CS_WORKSPACE_CRN: ${CS_WORKSPACE_CRN} + healthcheck: + # Probe the Proxy's pg-protocol listener (no auth handshake required). + # busybox `nc` is present in the cipherstash/proxy image. + test: ["CMD-SHELL", "nc -z localhost 6432"] + interval: 1s + timeout: 5s + retries: 30 + depends_on: + postgres: + condition: service_healthy + networks: + - bench + +networks: + bench: + driver: bridge diff --git a/tests/benchmarks/generate.sh b/tests/benchmarks/generate.sh new file mode 100755 index 00000000..bf6aee96 --- /dev/null +++ b/tests/benchmarks/generate.sh @@ -0,0 +1,58 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Generates a 100K-row encrypted bench dataset via CipherStash Proxy. +# No dump is written in v1 — the Tier 2 workflow regenerates fresh each run. +# +# Prerequisites: +# - mise run build (produces release/cipherstash-encrypt.sql) +# - docker compose -f tests/benchmarks/docker-compose.yml up -d --wait +# - tests/benchmarks/.env populated with CipherStash credentials + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +EQL_SQL="$REPO_ROOT/release/cipherstash-encrypt.sql" +SCALE="${1:-100k}" + +case "$SCALE" in + 100k) ROWS=100000 ;; + *) echo "Unsupported scale: $SCALE (only 100k in v1)" >&2; exit 1 ;; +esac + +if [ ! -f "$EQL_SQL" ]; then + echo "ERROR: $EQL_SQL not found. Run 'mise run build' first." >&2 + exit 1 +fi + +PG_URL="postgresql://cipherstash:password@localhost:7433/cipherstash" +PROXY_URL="postgresql://cipherstash:password@localhost:6433/cipherstash" + +echo "==> Installing EQL into bench-postgres" +psql "$PG_URL" -v ON_ERROR_STOP=1 -f "$EQL_SQL" >/dev/null + +echo "==> Applying bench schema and Proxy search configuration" +psql "$PG_URL" -v ON_ERROR_STOP=1 -f "$SCRIPT_DIR/schema.sql" + +echo "==> Inserting $ROWS plaintext rows through Proxy (this encrypts them)" +# generate_series emits plaintext rows; Proxy intercepts and encrypts each +# column per the search config applied in schema.sql. +psql "$PROXY_URL" -v ON_ERROR_STOP=1 -c " +INSERT INTO bench (encrypted_text, encrypted_int, encrypted_bigint) +SELECT + ('text_' || (((gs - 1) % 1000) + 1))::text, + (((gs - 1) % 1000) + 1)::int, + (((gs - 1) % 1000) + 1)::bigint * 1000000000 +FROM generate_series(1, $ROWS) AS gs; +" + +echo "==> Creating indexes and running ANALYZE" +psql "$PG_URL" -v ON_ERROR_STOP=1 -c " +CREATE INDEX IF NOT EXISTS bench_text_hmac_idx ON bench USING hash (eql_v2.hmac_256(encrypted_text)); +CREATE INDEX IF NOT EXISTS bench_text_ore_idx ON bench USING btree (encrypted_text eql_v2.encrypted_operator_class); +CREATE INDEX IF NOT EXISTS bench_int_ore_idx ON bench USING btree (encrypted_int eql_v2.encrypted_operator_class); +CREATE INDEX IF NOT EXISTS bench_bigint_ore_idx ON bench USING btree (encrypted_bigint eql_v2.encrypted_operator_class); +CREATE INDEX IF NOT EXISTS bench_text_bloom_idx ON bench USING gin (eql_v2.bloom_filter(encrypted_text)); +ANALYZE bench; +" + +echo "==> Done. Rows: $ROWS" diff --git a/tests/benchmarks/reports/.gitkeep b/tests/benchmarks/reports/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/tests/benchmarks/schema.sql b/tests/benchmarks/schema.sql new file mode 100644 index 00000000..e8693ef1 --- /dev/null +++ b/tests/benchmarks/schema.sql @@ -0,0 +1,35 @@ +-- Bench schema for Tier 2 benchmarks. +-- Applied against the bench-postgres container AFTER EQL has been explicitly +-- installed by generate.sh (see Task 4 — generate.sh installs +-- release/cipherstash-encrypt.sql directly, not relying on Proxy's async install). + +DROP TABLE IF EXISTS bench; + +CREATE TABLE bench ( + id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + encrypted_text eql_v2_encrypted, + encrypted_int eql_v2_encrypted, + encrypted_bigint eql_v2_encrypted +); + +-- Proxy search configuration: tells Proxy which index terms to generate +-- for each column when plaintext is inserted. +-- +-- Signature: eql_v2.add_search_config(table, column, index, cast_as) +-- (see src/config/functions.sql). add_search_config calls activate_config +-- internally when migrating=false, so no explicit activate_config call. + +-- text column: equality (hmac), pattern match (bloom), ordering (ore) +SELECT eql_v2.add_search_config('bench', 'encrypted_text', 'unique', 'text'); +SELECT eql_v2.add_search_config('bench', 'encrypted_text', 'match', 'text'); +SELECT eql_v2.add_search_config('bench', 'encrypted_text', 'ore', 'text'); + +-- integer column: equality + ORE range/ordering +SELECT eql_v2.add_search_config('bench', 'encrypted_int', 'unique', 'int'); +SELECT eql_v2.add_search_config('bench', 'encrypted_int', 'ore', 'int'); + +-- bigint column: equality + ORE range/ordering +SELECT eql_v2.add_search_config('bench', 'encrypted_bigint', 'unique', 'big_int'); +SELECT eql_v2.add_search_config('bench', 'encrypted_bigint', 'ore', 'big_int'); + +-- Indexes (created after data load in generate.sh, after ANALYZE) diff --git a/tests/sqlx/Cargo.lock b/tests/sqlx/Cargo.lock index a1060773..66f047d2 100644 --- a/tests/sqlx/Cargo.lock +++ b/tests/sqlx/Cargo.lock @@ -152,6 +152,15 @@ dependencies = [ "zeroize", ] +[[package]] +name = "deranged" +version = "0.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" +dependencies = [ + "powerfmt", +] + [[package]] name = "digest" version = "0.10.7" @@ -198,6 +207,7 @@ dependencies = [ "serde", "serde_json", "sqlx", + "time", "tokio", ] @@ -642,6 +652,12 @@ dependencies = [ "zeroize", ] +[[package]] +name = "num-conv" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6673768db2d862beb9b39a78fdcb1a69439615d5794a1be50caa9bc92c81967" + [[package]] name = "num-integer" version = "0.1.46" @@ -770,6 +786,12 @@ dependencies = [ "zerovec", ] +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + [[package]] name = "ppv-lite86" version = "0.2.21" @@ -1261,6 +1283,37 @@ dependencies = [ "syn", ] +[[package]] +name = "time" +version = "0.3.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "743bd48c283afc0388f9b8827b976905fb217ad9e647fae3a379a9283c4def2c" +dependencies = [ + "deranged", + "itoa", + "num-conv", + "powerfmt", + "serde_core", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7694e1cfe791f8d31026952abf09c69ca6f6fa4e1a1229e18988f06a04a12dca" + +[[package]] +name = "time-macros" +version = "0.2.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e70e4c5a0e0a8a4823ad65dfe1a6930e4f4d756dcd9dd7939022b5e8c501215" +dependencies = [ + "num-conv", + "time-core", +] + [[package]] name = "tinystr" version = "0.8.1" diff --git a/tests/sqlx/Cargo.toml b/tests/sqlx/Cargo.toml index 025d697d..acff2489 100644 --- a/tests/sqlx/Cargo.toml +++ b/tests/sqlx/Cargo.toml @@ -9,6 +9,7 @@ tokio = { version = "1", features = ["full"] } serde = { version = "1", features = ["derive"] } serde_json = "1" anyhow = "1" +time = { version = "0.3", features = ["formatting", "std"] } [dev-dependencies] # None needed - tests live in this crate diff --git a/tests/sqlx/fixtures/FIXTURE_SCHEMA.md b/tests/sqlx/fixtures/FIXTURE_SCHEMA.md index 7988fb23..93c34e05 100644 --- a/tests/sqlx/fixtures/FIXTURE_SCHEMA.md +++ b/tests/sqlx/fixtures/FIXTURE_SCHEMA.md @@ -9,7 +9,8 @@ EQL Extension (via migrations) ├── encrypted_json.sql ├── array_data.sql ├── order_by_null_data.sql (depends on ore migration) - └── ore_data.sql + ├── ore table (migration 002 — not a fixture) + └── bench_data.sql + bench_setup.sql (depend on migration 007) ``` All fixtures depend on the EQL extension being installed via SQLx migrations. @@ -119,7 +120,7 @@ CREATE TABLE ore ( **Helper Functions:** - `get_ore_encrypted(pool, id)` - Selects encrypted value from ore table -- `create_encrypted_json(id)` - Looks up ore table at `id * 10` (valid ids: 1-9 → ore lookups: 10-90) +- `create_encrypted_json(id)` - Looks up ore table at `id * 10` (valid ids: 1-99 → ore lookups: 10-990) **Key Property:** - Sequential numeric values enable deterministic comparison tests @@ -132,6 +133,55 @@ CREATE TABLE ore ( --- +## bench_data.sql + +**Purpose:** Seeds 10K rows into the `bench` table for performance benchmarking. Opt-in fixture — only loaded when a test explicitly includes `scripts("bench_data")`, so other tests don't pay the cost. + +**Dependencies:** +- Requires `bench` table from migration `007_install_bench_data.sql` +- Uses `create_encrypted_json()` from migration `004_install_test_helpers.sql` + +**Schema:** Uses `bench` table (DDL in migration 007): +```sql +CREATE TABLE bench ( + id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + encrypted_text eql_v2_encrypted, + encrypted_int eql_v2_encrypted, + encrypted_bigint eql_v2_encrypted +); +``` + +**Data:** +- 10,000 rows drawn from 99 distinct encrypted values. Caller ids `1..99` are Zipf-skewed and resolve via `create_encrypted_json(id)` to ORE ids `{10, 20, …, 990}` (see the ORE helper description above) +- Zipf-like skew via `setseed(0.42)` + `random()^2` — deterministic and byte-identical across runs +- Top id gets ~5% of rows; tail ids ~0.5% each (top:bottom ratio ~10x) +- Each column draws independently, so column values are decorrelated within a row +- Each row has HMAC, bloom filter, and ORE index terms + +**Used By:** +- bench_data_tests.rs (all tests) + +--- + +## bench_setup.sql + +**Purpose:** Creates the 5 benchmark indexes and refreshes planner statistics. Always loaded after `bench_data.sql` in tests that verify index usage. + +**Dependencies:** +- Requires `bench` table with data from `bench_data.sql` + +**Indexes created:** +- `bench_text_hmac_idx` — hash on `eql_v2.hmac_256(encrypted_text)` for equality +- `bench_text_ore_idx` — btree on `encrypted_text` via operator class for text ordering +- `bench_int_ore_idx` — btree on `encrypted_int` via operator class for range/ORDER BY +- `bench_bigint_ore_idx` — btree on `encrypted_bigint` via operator class +- `bench_text_bloom_idx` — GIN on `eql_v2.bloom_filter(encrypted_text)` for containment + +**Used By:** +- bench_data_tests.rs (index-usage tests: `scripts("bench_data", "bench_setup")`) + +--- + ## Validation Tests Each fixture should have a validation test to ensure correct structure: @@ -148,15 +198,15 @@ async fn fixture_encrypted_json_has_three_records(pool: PgPool) { } ``` -### ore_data Validation +### ore Migration Validation ```rust -#[sqlx::test(fixtures(path = "../fixtures", scripts("ore_data")))] +#[sqlx::test] async fn fixture_ore_data_has_99_records(pool: PgPool) { let count: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM ore") .fetch_one(&pool) .await .unwrap(); - assert_eq!(count, 99, "ore_data fixture should create 99 records"); + assert_eq!(count, 99, "ore migration should provide 99 records"); } ``` @@ -166,7 +216,7 @@ async fn fixture_ore_data_has_99_records(pool: PgPool) { - Use snake_case for fixture file names - Name should describe the data, not the test using it -- Examples: `encrypted_json.sql`, `ore_data.sql`, `array_data.sql` +- Examples: `encrypted_json.sql`, `array_data.sql`, `bench_data.sql` ## Adding New Fixtures diff --git a/tests/sqlx/fixtures/bench_data.sql b/tests/sqlx/fixtures/bench_data.sql new file mode 100644 index 00000000..247d4ed5 --- /dev/null +++ b/tests/sqlx/fixtures/bench_data.sql @@ -0,0 +1,23 @@ +-- Fixture: bench_data.sql +-- +-- Seeds 10K rows into the bench table for performance testing. +-- Each column draws independently from 99 distinct encrypted values (ore ids 1-99) +-- using a Zipf-like skew so the planner sees realistic histograms. +-- +-- Index terms per row: hm (hmac), b3 (blake3), bf (bloom filter), ob (ORE blocks), sv (STE vec) +-- Data generated via create_encrypted_json() from 004_install_test_helpers.sql. +-- +-- Distribution: +-- Deterministic via setseed(0.42) — byte-identical across runs. +-- random()^2 produces a power-law skew: P(id=k) is proportional to 1/sqrt(k). +-- Top id gets ~5% of rows (~500); tail ids get ~0.5% each (~50). Ratio ~10x. +-- Three independent draws per row decorrelate the columns. + +SELECT setseed(0.42); + +INSERT INTO bench (encrypted_text, encrypted_int, encrypted_bigint) +SELECT + create_encrypted_json(1 + floor(99 * power(random(), 2))::int), + create_encrypted_json(1 + floor(99 * power(random(), 2))::int), + create_encrypted_json(1 + floor(99 * power(random(), 2))::int) +FROM generate_series(1, 10000); diff --git a/tests/sqlx/fixtures/bench_setup.sql b/tests/sqlx/fixtures/bench_setup.sql new file mode 100644 index 00000000..0f997940 --- /dev/null +++ b/tests/sqlx/fixtures/bench_setup.sql @@ -0,0 +1,31 @@ +-- Fixture: bench_setup.sql +-- +-- Creates benchmark indexes and refreshes planner statistics. +-- Table DDL from migration 007_install_bench_data.sql; 10K rows from bench_data.sql fixture. +-- +-- Indexes: +-- bench_text_hmac_idx - hash on eql_v2.hmac_256(encrypted_text) for equality +-- bench_text_ore_idx - btree on encrypted_text via operator class for text ordering +-- bench_int_ore_idx - btree on encrypted_int via operator class for range/ORDER BY +-- bench_bigint_ore_idx - btree on encrypted_bigint via operator class +-- bench_text_bloom_idx - GIN on eql_v2.bloom_filter(encrypted_text) for containment +-- +-- Pattern follows containment_with_index_tests.rs: indexes in fixture (not migration) +-- so tests can verify before/after index creation. + +CREATE INDEX IF NOT EXISTS bench_text_hmac_idx + ON bench USING hash (eql_v2.hmac_256(encrypted_text)); + +CREATE INDEX IF NOT EXISTS bench_text_ore_idx + ON bench USING btree (encrypted_text eql_v2.encrypted_operator_class); + +CREATE INDEX IF NOT EXISTS bench_int_ore_idx + ON bench USING btree (encrypted_int eql_v2.encrypted_operator_class); + +CREATE INDEX IF NOT EXISTS bench_bigint_ore_idx + ON bench USING btree (encrypted_bigint eql_v2.encrypted_operator_class); + +CREATE INDEX IF NOT EXISTS bench_text_bloom_idx + ON bench USING gin (eql_v2.bloom_filter(encrypted_text)); + +ANALYZE bench; diff --git a/tests/sqlx/migrations/007_install_bench_data.sql b/tests/sqlx/migrations/007_install_bench_data.sql new file mode 100644 index 00000000..49ff6975 --- /dev/null +++ b/tests/sqlx/migrations/007_install_bench_data.sql @@ -0,0 +1,17 @@ +-- Migration: 007_install_bench_data.sql +-- +-- Creates benchmark table for performance testing. +-- DDL only — data is loaded by the bench_data.sql fixture so that +-- only bench tests pay the 10K-row seeding cost, not the entire suite. +-- +-- Columns: +-- encrypted_text - text equality (hmac), pattern match (bloom), ordering (ore) +-- encrypted_int - integer ORE range/equality/ordering +-- encrypted_bigint - bigint ORE at scale + +CREATE TABLE bench ( + id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + encrypted_text eql_v2_encrypted, + encrypted_int eql_v2_encrypted, + encrypted_bigint eql_v2_encrypted +); diff --git a/tests/sqlx/migrations/README.md b/tests/sqlx/migrations/README.md index a03dcaa0..abfc7471 100644 --- a/tests/sqlx/migrations/README.md +++ b/tests/sqlx/migrations/README.md @@ -10,16 +10,19 @@ These migrations install EQL and test helpers into the test database using a **h - In `.gitignore` - never commit this file - Ensures tests always use current EQL version -**Migrations 002-004 are static fixtures**: -- 002: Test helpers (`test_helpers.sql`) -- 003: ORE test data (`ore.sql`) -- 004: STE Vec test data (`ste_vec.sql`) +**Migrations 002-007 are static fixtures**: +- 002: ORE test data (`ore.sql`) +- 003: STE Vec test data (`ste_vec.sql`) +- 004: Test helpers (`test_helpers.sql`) +- 005: STE Vec vast data +- 006: ORE text data +- 007: Benchmark table DDL (`bench` table with 3 encrypted columns — DDL only, no rows) ## How SQLx Uses These Migrations When using `#[sqlx::test]`: - Each test gets a fresh database -- All migrations (001-004) run automatically before each test +- All migrations (001-007) run automatically before each test - Migration 001 contains the latest built EQL - No need to manually reset database between tests @@ -36,7 +39,7 @@ cp release/cipherstash-encrypt.sql tests/sqlx/migrations/001_install_eql.sql ## Adding New Test Fixtures To add new test data or helpers: -1. Create a new migration: `tests/sqlx/migrations/005_my_fixture.sql` +1. Create a new migration using the next unused number (e.g. `tests/sqlx/migrations/008_my_fixture.sql`) 2. Add your SQL fixtures 3. Commit it (static migrations are version-controlled) 4. SQLx will apply it automatically in test runs diff --git a/tests/sqlx/src/helpers.rs b/tests/sqlx/src/helpers.rs index a2f454e9..e3864433 100644 --- a/tests/sqlx/src/helpers.rs +++ b/tests/sqlx/src/helpers.rs @@ -40,6 +40,32 @@ pub async fn get_ore_text_encrypted(pool: &PgPool, id: i32) -> Result { result.with_context(|| format!("ore_text returned NULL for id={}", id)) } +/// Fetch encrypted_int value from the bench table by id +/// +/// The bench table is created by the bench_data fixture (10K rows, ids 1-10000). +pub async fn get_bench_encrypted_int(pool: &PgPool, id: i32) -> Result { + let result: Option = + sqlx::query_scalar("SELECT (encrypted_int).data::text FROM bench WHERE id = $1") + .bind(id) + .fetch_one(pool) + .await + .with_context(|| format!("fetching bench encrypted_int for id={id}"))?; + result.with_context(|| format!("bench.encrypted_int is NULL for id={id}")) +} + +/// Fetch encrypted_text value from the bench table by id +/// +/// The bench table is created by the bench_data fixture (10K rows, ids 1-10000). +pub async fn get_bench_encrypted_text(pool: &PgPool, id: i32) -> Result { + let result: Option = + sqlx::query_scalar("SELECT (encrypted_text).data::text FROM bench WHERE id = $1") + .bind(id) + .fetch_one(pool) + .await + .with_context(|| format!("fetching bench encrypted_text for id={id}"))?; + result.with_context(|| format!("bench.encrypted_text is NULL for id={id}")) +} + /// Assert sorted rows match expected sequential id range pub fn assert_sequential_ids(rows: &[sqlx::postgres::PgRow], start: i64, end: i64) { let ids: Vec = rows.iter().map(|r| r.try_get(0).unwrap()).collect(); @@ -501,6 +527,35 @@ pub async fn explain_json(pool: &PgPool, query: &str) -> Result Result { + let sql = format!("EXPLAIN (FORMAT JSON) {}", query); + let mut q = sqlx::query_scalar::<_, serde_json::Value>(&sql); + for p in params { + q = q.bind(*p); + } + let plan = q + .fetch_one(pool) + .await + .with_context(|| format!("running EXPLAIN (FORMAT JSON) on query: {}", query))?; + Ok(plan[0]["Plan"]["Node Type"] + .as_str() + .with_context(|| format!("extracting Plan.Node Type from EXPLAIN on query: {}", query))? + .to_string()) +} + /// Run EXPLAIN ANALYZE multiple times and return averaged statistics /// /// Executes `EXPLAIN (ANALYZE, FORMAT JSON) {query}` the specified number of times @@ -531,6 +586,37 @@ pub async fn explain_json(pool: &PgPool, query: &str) -> Result Result { + explain_analyze_avg_bound(pool, query, &[], runs).await +} + +/// Run EXPLAIN ANALYZE multiple times with bound parameters and return averaged statistics +/// +/// Like `explain_analyze_avg`, but supports `$1`, `$2`, ... placeholders in the +/// query. String parameters are bound via sqlx, avoiding SQL injection and +/// quoting issues when the value may contain `'` characters (for example, +/// encrypted JSON payloads surfaced via `::jsonb::eql_v2_encrypted`). +/// +/// # Arguments +/// * `pool` - Database connection pool +/// * `query` - SQL query with `$N` placeholders (no EXPLAIN prefix) +/// * `params` - String parameters bound in order ($1 → params[0], etc.) +/// * `runs` - Number of times to execute (must be >= 1) +/// +/// # Example +/// ```ignore +/// let stats = explain_analyze_avg_bound( +/// &pool, +/// "SELECT * FROM bench WHERE eql_v2.hmac_256(col) = eql_v2.hmac_256($1::jsonb::eql_v2_encrypted)", +/// &[&encrypted], +/// 5, +/// ).await?; +/// ``` +pub async fn explain_analyze_avg_bound( + pool: &PgPool, + query: &str, + params: &[&str], + runs: usize, +) -> Result { anyhow::ensure!(runs >= 1, "runs must be >= 1, got {}", runs); let sql = format!("EXPLAIN (ANALYZE, FORMAT JSON) {}", query); @@ -540,39 +626,27 @@ pub async fn explain_analyze_avg(pool: &PgPool, query: &str, runs: usize) -> Res let mut node_type = String::new(); for i in 0..runs { - let plan: serde_json::Value = sqlx::query_scalar(&sql) - .fetch_one(pool) - .await - .with_context(|| { - format!( - "running EXPLAIN ANALYZE (run {}/{}) on query: {}", - i + 1, - runs, - query - ) - })?; + let mut q = sqlx::query_scalar::<_, serde_json::Value>(&sql); + for p in params { + q = q.bind(*p); + } + let plan = q.fetch_one(pool).await.with_context(|| { + format!( + "running EXPLAIN ANALYZE (run {}/{}) on query: {}", + i + 1, + runs, + query + ) + })?; // EXPLAIN (ANALYZE, FORMAT JSON) returns: // [{"Plan": {...}, "Planning Time": N, "Execution Time": N}] let entry = &plan[0]; - - let exec_time = entry["Execution Time"] - .as_f64() - .with_context(|| format!("extracting Execution Time on run {}/{}", i + 1, runs))?; - - let plan_time = entry["Planning Time"] - .as_f64() - .with_context(|| format!("extracting Planning Time on run {}/{}", i + 1, runs))?; - + let (exec_time, plan_time, nt) = parse_explain_entry(entry, i + 1, runs)?; total_execution_ms += exec_time; total_planning_ms += plan_time; - - // Capture node type from first run only if i == 0 { - node_type = entry["Plan"]["Node Type"] - .as_str() - .with_context(|| "extracting Node Type from first run")? - .to_string(); + node_type = nt; } } @@ -584,6 +658,27 @@ pub async fn explain_analyze_avg(pool: &PgPool, query: &str, runs: usize) -> Res }) } +fn parse_explain_entry( + entry: &serde_json::Value, + run_num: usize, + total_runs: usize, +) -> Result<(f64, f64, String)> { + let exec_time = entry["Execution Time"].as_f64().with_context(|| { + format!( + "extracting Execution Time on run {}/{}", + run_num, total_runs + ) + })?; + let plan_time = entry["Planning Time"] + .as_f64() + .with_context(|| format!("extracting Planning Time on run {}/{}", run_num, total_runs))?; + let node_type = entry["Plan"]["Node Type"] + .as_str() + .with_context(|| format!("extracting Node Type on run {}/{}", run_num, total_runs))? + .to_string(); + Ok((exec_time, plan_time, node_type)) +} + /// Assert that a JSON EXPLAIN plan does not use any sequential scan /// /// Recursively walks the JSON plan tree checking all "Node Type" fields. @@ -701,7 +796,7 @@ pub async fn ensure_pg_stat_statements(pool: &PgPool) -> Result<()> { /// let stats = read_pg_stat_statements(&pool, "%FROM bench%").await?; /// ``` pub async fn reset_pg_stat_statements(pool: &PgPool) -> Result<()> { - sqlx::query("SELECT pg_stat_statements_reset(NULL::oid, NULL::oid, (SELECT oid FROM pg_database WHERE datname = current_database()))") + sqlx::query("SELECT pg_stat_statements_reset(NULL::oid, (SELECT oid FROM pg_database WHERE datname = current_database()), 0::bigint)") .execute(pool) .await .with_context(|| "resetting pg_stat_statements counters for current database")?; diff --git a/tests/sqlx/src/lib.rs b/tests/sqlx/src/lib.rs index f72cc45f..973a94a9 100644 --- a/tests/sqlx/src/lib.rs +++ b/tests/sqlx/src/lib.rs @@ -7,19 +7,22 @@ use sqlx::PgPool; pub mod assertions; pub mod helpers; pub mod index_types; +pub mod reports; pub mod selectors; pub use assertions::QueryAssertion; pub use helpers::{ analyze_table, assert_no_seq_scan, assert_sequential_ids, assert_uses_index, assert_uses_seq_scan, create_jsonb_gin_index, ensure_pg_stat_statements, explain_analyze_avg, - explain_json, explain_query, get_encrypted_term, get_ore_encrypted, get_ore_encrypted_as_jsonb, - get_ore_text_encrypted, get_ore_text_encrypted_as_jsonb, get_ste_vec_encrypted, - get_ste_vec_encrypted_pair, get_ste_vec_selector_term, get_ste_vec_sv_element, - get_ste_vec_term_by_id, read_pg_stat_statements, reset_pg_stat_statements, ExplainStats, - PgStatEntry, + explain_analyze_avg_bound, explain_json, explain_query, fetch_plan_node_type, + get_bench_encrypted_int, get_bench_encrypted_text, get_encrypted_term, get_ore_encrypted, + get_ore_encrypted_as_jsonb, get_ore_text_encrypted, get_ore_text_encrypted_as_jsonb, + get_ste_vec_encrypted, get_ste_vec_encrypted_pair, get_ste_vec_selector_term, + get_ste_vec_sv_element, get_ste_vec_term_by_id, read_pg_stat_statements, + reset_pg_stat_statements, ExplainStats, PgStatEntry, }; pub use index_types as IndexTypes; +pub use reports::{append_result, write_reports, PerfResult}; pub use selectors::Selectors; /// Reset pg_stat_user_functions tracking before tests diff --git a/tests/sqlx/src/reports.rs b/tests/sqlx/src/reports.rs new file mode 100644 index 00000000..1dbd9b56 --- /dev/null +++ b/tests/sqlx/src/reports.rs @@ -0,0 +1,117 @@ +//! Benchmark report writer for Tier 2 scheduled benchmarks. +//! +//! Each `#[ignore]` benchmark in `bench_perf_tests.rs` pushes a `PerfResult` +//! into `append_result`. The `run_all_benchmarks` orchestrator invokes each +//! benchmark in sequence and then calls `write_reports` to flush all +//! accumulated results to JSON + Markdown. +//! +//! Output shape matches the design doc (.work/eql-index-performance/ +//! 2026-03-30-benchmarking-design.md §Report Format) with one caveat: the +//! design doc lists `p95_ms` / `p99_ms` fields; Postgres `pg_stat_statements` +//! does not expose percentiles — only mean / stddev / total. v1 omits them +//! and documents the gap. Adding percentiles would require a different timing +//! strategy (e.g. client-side histograms) deferred to a follow-up. + +use anyhow::{Context, Result}; +use serde::Serialize; +use std::fs; +use std::path::PathBuf; +use std::sync::Mutex; +use time::format_description::well_known::Rfc3339; +use time::OffsetDateTime; + +/// One benchmark case result. +#[derive(Debug, Clone, Serialize)] +pub struct PerfResult { + /// Test name (e.g. "hmac_256_equality") + pub name: String, + /// Priority tier (P0, P1, P2) + pub priority: String, + /// Number of executions + pub runs: i64, + /// Plan node type (e.g. "Index Scan", "Seq Scan") + pub plan_type: String, + /// Mean execution time in milliseconds + pub mean_ms: f64, + /// Population standard deviation in milliseconds + pub stddev_ms: f64, + /// Total execution time across all runs in milliseconds + pub total_ms: f64, +} + +/// Top-level report structure — matches the design doc's JSON shape. +#[derive(Debug, Clone, Serialize)] +pub struct BenchmarkReport { + /// RFC3339 UTC timestamp at report-write time + pub timestamp: String, + /// Postgres major version (e.g. "17") + pub postgres_version: String, + /// Dataset size this report was produced against + pub dataset_rows: i64, + /// One entry per benchmark case + pub results: Vec, +} + +static RESULTS: Mutex> = Mutex::new(Vec::new()); + +/// Push a result onto the shared in-memory accumulator. +pub fn append_result(r: PerfResult) { + RESULTS.lock().expect("results mutex poisoned").push(r); +} + +/// Write JSON + Markdown reports for all accumulated results. +/// +/// Output paths: +/// `/benchmark-.json` +/// `/benchmark-.md` +/// +/// `date` is used only as a filename suffix (any caller-supplied string, +/// typically `YYYY-MM-DD` with an optional run-id suffix for uniqueness). +/// The report's `timestamp` field is captured at write time as RFC3339 UTC +/// and is independent of `date`. +pub fn write_reports( + output_dir: &str, + date: &str, + postgres_version: &str, + dataset_rows: i64, +) -> Result<(PathBuf, PathBuf)> { + let results = RESULTS.lock().expect("results mutex poisoned").clone(); + let timestamp = OffsetDateTime::now_utc() + .format(&Rfc3339) + .context("formatting RFC3339 write-time timestamp")?; + let report = BenchmarkReport { + timestamp, + postgres_version: postgres_version.to_string(), + dataset_rows, + results, + }; + + fs::create_dir_all(output_dir).with_context(|| format!("creating output dir {output_dir}"))?; + + let json_path = PathBuf::from(output_dir).join(format!("benchmark-{date}.json")); + let md_path = PathBuf::from(output_dir).join(format!("benchmark-{date}.md")); + + let json = serde_json::to_string_pretty(&report).context("serializing report to JSON")?; + fs::write(&json_path, json).with_context(|| format!("writing {}", json_path.display()))?; + + fs::write(&md_path, render_markdown(&report)) + .with_context(|| format!("writing {}", md_path.display()))?; + + Ok((json_path, md_path)) +} + +fn render_markdown(report: &BenchmarkReport) -> String { + let mut out = String::new(); + out.push_str(&format!("# Benchmark Report — {}\n\n", report.timestamp)); + out.push_str(&format!("- Postgres: {}\n", report.postgres_version)); + out.push_str(&format!("- Dataset rows: {}\n\n", report.dataset_rows)); + out.push_str("| Query Pattern | Priority | Plan | Runs | Mean (ms) | Stddev (ms) |\n"); + out.push_str("|---|---|---|---|---|---|\n"); + for r in &report.results { + out.push_str(&format!( + "| {} | {} | {} | {} | {:.3} | {:.3} |\n", + r.name, r.priority, r.plan_type, r.runs, r.mean_ms, r.stddev_ms + )); + } + out +} diff --git a/tests/sqlx/tests/bench_data_tests.rs b/tests/sqlx/tests/bench_data_tests.rs new file mode 100644 index 00000000..a6912b46 --- /dev/null +++ b/tests/sqlx/tests/bench_data_tests.rs @@ -0,0 +1,186 @@ +//! Benchmark data verification tests +//! +//! Validates bench_data fixture (10K rows) and bench_setup fixture (indexes): +//! - 10K rows seeded correctly across 3 encrypted columns +//! - Index terms (hmac, bloom, ORE) are extractable +//! - Indexes are used by the query planner (EXPLAIN assertions) +//! - Sequential scan baseline without indexes + +use anyhow::Result; +use eql_tests::{analyze_table, assert_uses_index, assert_uses_seq_scan, explain_query}; +use sqlx::PgPool; + +const BENCH_ROW_COUNT: i64 = 10000; + +// ========== Data Integrity Tests ========== + +/// Verify fixture seeded exactly 10K rows +#[sqlx::test(fixtures(path = "../fixtures", scripts("bench_data")))] +async fn bench_table_has_expected_row_count(pool: PgPool) -> Result<()> { + let count: (i64,) = sqlx::query_as("SELECT COUNT(*) FROM bench") + .fetch_one(&pool) + .await?; + assert_eq!( + count.0, BENCH_ROW_COUNT, + "bench table should have 10000 rows" + ); + Ok(()) +} + +/// Verify all three columns have non-null encrypted data +#[sqlx::test(fixtures(path = "../fixtures", scripts("bench_data")))] +async fn bench_columns_are_populated(pool: PgPool) -> Result<()> { + let count: (i64,) = sqlx::query_as( + "SELECT COUNT(*) FROM bench + WHERE encrypted_text IS NOT NULL + AND encrypted_int IS NOT NULL + AND encrypted_bigint IS NOT NULL", + ) + .fetch_one(&pool) + .await?; + assert_eq!( + count.0, BENCH_ROW_COUNT, + "all rows should have non-null encrypted columns" + ); + Ok(()) +} + +/// Verify hmac_256 index terms are extractable from encrypted_text +#[sqlx::test(fixtures(path = "../fixtures", scripts("bench_data")))] +async fn bench_encrypted_text_has_hmac_terms(pool: PgPool) -> Result<()> { + let count: (i64,) = sqlx::query_as( + "SELECT COUNT(*) FROM bench WHERE eql_v2.hmac_256(encrypted_text) IS NOT NULL", + ) + .fetch_one(&pool) + .await?; + assert_eq!( + count.0, BENCH_ROW_COUNT, + "all rows should have hmac_256 index terms" + ); + Ok(()) +} + +/// Verify bloom_filter index terms are extractable from encrypted_text +#[sqlx::test(fixtures(path = "../fixtures", scripts("bench_data")))] +async fn bench_encrypted_text_has_bloom_filter_terms(pool: PgPool) -> Result<()> { + let count: (i64,) = sqlx::query_as( + "SELECT COUNT(*) FROM bench WHERE eql_v2.bloom_filter(encrypted_text) IS NOT NULL", + ) + .fetch_one(&pool) + .await?; + assert_eq!( + count.0, BENCH_ROW_COUNT, + "all rows should have bloom_filter index terms" + ); + Ok(()) +} + +/// Verify ORE terms are extractable from encrypted_int (3 of 5 indexes are ORE btree) +#[sqlx::test(fixtures(path = "../fixtures", scripts("bench_data")))] +async fn bench_encrypted_int_has_ore_terms(pool: PgPool) -> Result<()> { + let count: (i64,) = sqlx::query_as( + "SELECT COUNT(*) FROM bench WHERE eql_v2.ore_block_u64_8_256(encrypted_int) IS NOT NULL", + ) + .fetch_one(&pool) + .await?; + assert_eq!( + count.0, BENCH_ROW_COUNT, + "all rows should have ORE block index terms" + ); + Ok(()) +} + +/// Verify ORE terms are extractable from encrypted_bigint +/// +/// Both int and bigint columns use the same eql_v2_encrypted type and ob index structure. +/// These tests verify that data seeding populated both columns, not that encoding differs. +#[sqlx::test(fixtures(path = "../fixtures", scripts("bench_data")))] +async fn bench_encrypted_bigint_has_ore_terms(pool: PgPool) -> Result<()> { + let count: (i64,) = sqlx::query_as( + "SELECT COUNT(*) FROM bench WHERE eql_v2.ore_block_u64_8_256(encrypted_bigint) IS NOT NULL", + ) + .fetch_one(&pool) + .await?; + assert_eq!( + count.0, BENCH_ROW_COUNT, + "all rows should have ORE block index terms" + ); + Ok(()) +} + +// ========== Index Usage Tests (with fixture) ========== + +/// Verify hash index is used for hmac_256 equality lookup +#[sqlx::test(fixtures(path = "../fixtures", scripts("bench_data", "bench_setup")))] +async fn bench_hmac_equality_uses_hash_index(pool: PgPool) -> Result<()> { + let encrypted: String = + sqlx::query_scalar("SELECT (encrypted_text).data::text FROM bench WHERE id = 1") + .fetch_one(&pool) + .await?; + + let sql = format!( + "SELECT * FROM bench WHERE eql_v2.hmac_256(encrypted_text) = eql_v2.hmac_256('{}'::jsonb::eql_v2_encrypted)", + encrypted + ); + assert_uses_index(&pool, &sql, "bench_text_hmac_idx").await?; + Ok(()) +} + +/// Verify btree index is used for ORDER BY with LIMIT on encrypted_int +#[sqlx::test(fixtures(path = "../fixtures", scripts("bench_data", "bench_setup")))] +async fn bench_ore_order_uses_btree_index(pool: PgPool) -> Result<()> { + let sql = "SELECT * FROM bench ORDER BY encrypted_int LIMIT 10"; + assert_uses_index(&pool, sql, "bench_int_ore_idx").await?; + Ok(()) +} + +/// Verify GIN index is used for bloom_filter containment +#[sqlx::test(fixtures(path = "../fixtures", scripts("bench_data", "bench_setup")))] +async fn bench_bloom_containment_uses_gin_index(pool: PgPool) -> Result<()> { + let encrypted: String = + sqlx::query_scalar("SELECT (encrypted_text).data::text FROM bench WHERE id = 1") + .fetch_one(&pool) + .await?; + + let sql = format!( + "SELECT * FROM bench WHERE eql_v2.bloom_filter(encrypted_text) @> eql_v2.bloom_filter('{}'::jsonb::eql_v2_encrypted)", + encrypted + ); + assert_uses_index(&pool, &sql, "bench_text_bloom_idx").await?; + Ok(()) +} + +/// Verify btree index is used for ORDER BY with LIMIT on encrypted_text +#[sqlx::test(fixtures(path = "../fixtures", scripts("bench_data", "bench_setup")))] +async fn bench_ore_text_order_uses_btree_index(pool: PgPool) -> Result<()> { + let sql = "SELECT * FROM bench ORDER BY encrypted_text LIMIT 10"; + assert_uses_index(&pool, sql, "bench_text_ore_idx").await?; + Ok(()) +} + +/// Verify btree index is used for ORDER BY with LIMIT on encrypted_bigint +#[sqlx::test(fixtures(path = "../fixtures", scripts("bench_data", "bench_setup")))] +async fn bench_ore_bigint_order_uses_btree_index(pool: PgPool) -> Result<()> { + let sql = "SELECT * FROM bench ORDER BY encrypted_bigint LIMIT 10"; + assert_uses_index(&pool, sql, "bench_bigint_ore_idx").await?; + Ok(()) +} + +/// Verify sequential scan without indexes (before/after pattern sanity check) +#[sqlx::test(fixtures(path = "../fixtures", scripts("bench_data")))] +async fn bench_hmac_without_index_uses_seq_scan(pool: PgPool) -> Result<()> { + analyze_table(&pool, "bench").await?; + + let encrypted: String = + sqlx::query_scalar("SELECT (encrypted_text).data::text FROM bench WHERE id = 1") + .fetch_one(&pool) + .await?; + + let sql = format!( + "SELECT * FROM bench WHERE eql_v2.hmac_256(encrypted_text) = eql_v2.hmac_256('{}'::jsonb::eql_v2_encrypted)", + encrypted + ); + let explain = explain_query(&pool, &sql).await?; + assert_uses_seq_scan(&explain); + Ok(()) +} diff --git a/tests/sqlx/tests/bench_perf_tests.rs b/tests/sqlx/tests/bench_perf_tests.rs new file mode 100644 index 00000000..52f7a0f7 --- /dev/null +++ b/tests/sqlx/tests/bench_perf_tests.rs @@ -0,0 +1,330 @@ +//! Tier 2 scheduled benchmarks. +//! +//! All tests are marked #[ignore] so regular CI doesn't run them. The scheduled +//! workflow in .github/workflows/benchmark.yml invokes the orchestrator: +//! `cargo test --test bench_perf_tests run_all_benchmarks -- --ignored`. +//! +//! Unlike Tier 1 tests, these use #[tokio::test] with a manual pool connected +//! via DATABASE_URL against a pre-loaded 100K-row dataset (set by `mise run bench:full`). +//! +//! Each benchmark: +//! 1. Resets pg_stat_statements +//! 2. Captures the actual query plan via EXPLAIN (FORMAT JSON) +//! 3. Runs its query pattern `RUNS` times (currently 10) +//! 4. Reads pg_stat_statements for the match +//! 5. Appends a PerfResult to the shared accumulator +//! +//! The `run_all_benchmarks` orchestrator invokes each benchmark helper in +//! sequence and then calls `flush_reports` to write JSON + Markdown. Individual +//! `#[tokio::test] #[ignore]` wrappers are retained so developers can run a +//! single benchmark in isolation, but they do NOT write reports (the +//! orchestrator owns report emission). + +use anyhow::Result; +use eql_tests::{ + append_result, ensure_pg_stat_statements, fetch_plan_node_type, read_pg_stat_statements, + reset_pg_stat_statements, write_reports, PerfResult, +}; +use sqlx::postgres::PgPoolOptions; +use sqlx::PgPool; + +const RUNS: i64 = 10; +const DATASET_ROWS: i64 = 100_000; + +async fn connect() -> Result { + let url = std::env::var("DATABASE_URL") + .expect("DATABASE_URL must be set (run `mise run bench:full`)"); + let pool = PgPoolOptions::new() + .max_connections(4) + .connect(&url) + .await?; + ensure_pg_stat_statements(&pool).await?; + Ok(pool) +} + +// ============================================================================ +// Benchmark bodies — each is an async fn that takes a &PgPool. Thin test +// wrappers below allow running one benchmark in isolation; the orchestrator +// invokes the bodies directly. +// ============================================================================ + +/// P0 baseline: hmac_256 equality should stay ~0.5ms at 100K rows. +async fn bench_hmac_256_equality(pool: &PgPool) -> Result<()> { + let encrypted: String = + sqlx::query_scalar("SELECT (encrypted_text).data::text FROM bench WHERE id = 1") + .fetch_one(pool) + .await?; + + let query = "SELECT * FROM bench WHERE eql_v2.hmac_256(encrypted_text) = eql_v2.hmac_256($1::jsonb::eql_v2_encrypted)"; + let plan_type = fetch_plan_node_type(pool, query, &[&encrypted]).await?; + + reset_pg_stat_statements(pool).await?; + + for _ in 0..RUNS { + sqlx::query(query).bind(&encrypted).fetch_all(pool).await?; + } + + let stats = read_pg_stat_statements( + pool, + "%FROM bench WHERE eql_v2.hmac_256(encrypted_text) = eql_v2.hmac_256($%", + ) + .await?; + + append_result(PerfResult { + name: "hmac_256_equality".into(), + priority: "P0".into(), + runs: stats.calls, + plan_type, + mean_ms: stats.mean_exec_time, + stddev_ms: stats.stddev_exec_time, + total_ms: stats.total_exec_time, + }); + + assert_eq!(stats.calls, RUNS, "expected {RUNS} recorded calls"); + Ok(()) +} + +/// P2: bloom_filter containment — expected ~3.35ms at 100K rows. +async fn bench_bloom_filter_containment(pool: &PgPool) -> Result<()> { + let encrypted: String = + sqlx::query_scalar("SELECT (encrypted_text).data::text FROM bench WHERE id = 1") + .fetch_one(pool) + .await?; + + let query = "SELECT * FROM bench WHERE eql_v2.bloom_filter(encrypted_text) @> eql_v2.bloom_filter($1::jsonb::eql_v2_encrypted)"; + let plan_type = fetch_plan_node_type(pool, query, &[&encrypted]).await?; + + reset_pg_stat_statements(pool).await?; + for _ in 0..RUNS { + sqlx::query(query).bind(&encrypted).fetch_all(pool).await?; + } + let stats = read_pg_stat_statements( + pool, + "%eql_v2.bloom_filter(encrypted_text) @> eql_v2.bloom_filter($%", + ) + .await?; + + append_result(PerfResult { + name: "bloom_filter_containment".into(), + priority: "P2".into(), + runs: stats.calls, + plan_type, + mean_ms: stats.mean_exec_time, + stddev_ms: stats.stddev_exec_time, + total_ms: stats.total_exec_time, + }); + assert_eq!(stats.calls, RUNS, "expected {RUNS} recorded calls"); + Ok(()) +} + +/// P0: eql_cast equality — currently seq scans (CIP-2831). Report records the +/// actual plan + timing so the number is visible week-over-week until the fix ships. +async fn bench_eql_cast_equality(pool: &PgPool) -> Result<()> { + let encrypted: String = + sqlx::query_scalar("SELECT (encrypted_text).data::text FROM bench WHERE id = 1") + .fetch_one(pool) + .await?; + + let query = "SELECT * FROM bench WHERE encrypted_text = $1::jsonb::eql_v2_encrypted"; + let plan_type = fetch_plan_node_type(pool, query, &[&encrypted]).await?; + + reset_pg_stat_statements(pool).await?; + for _ in 0..RUNS { + sqlx::query(query).bind(&encrypted).fetch_all(pool).await?; + } + let stats = read_pg_stat_statements( + pool, + "%FROM bench WHERE encrypted_text = $%::jsonb::eql_v2_encrypted%", + ) + .await?; + + append_result(PerfResult { + name: "eql_cast_equality".into(), + priority: "P0".into(), + runs: stats.calls, + plan_type, + mean_ms: stats.mean_exec_time, + stddev_ms: stats.stddev_exec_time, + total_ms: stats.total_exec_time, + }); + assert_eq!(stats.calls, RUNS, "expected {RUNS} recorded calls"); + Ok(()) +} + +/// P0: ORE equality via operator class — currently seq scans (CIP-2831). +async fn bench_ore_equality_opclass(pool: &PgPool) -> Result<()> { + let encrypted: String = + sqlx::query_scalar("SELECT (encrypted_int).data::text FROM bench WHERE id = 1") + .fetch_one(pool) + .await?; + + let query = "SELECT * FROM bench WHERE encrypted_int = $1::jsonb::eql_v2_encrypted"; + let plan_type = fetch_plan_node_type(pool, query, &[&encrypted]).await?; + + reset_pg_stat_statements(pool).await?; + for _ in 0..RUNS { + sqlx::query(query).bind(&encrypted).fetch_all(pool).await?; + } + let stats = read_pg_stat_statements( + pool, + "%FROM bench WHERE encrypted_int = $%::jsonb::eql_v2_encrypted%", + ) + .await?; + + append_result(PerfResult { + name: "ore_equality_opclass".into(), + priority: "P0".into(), + runs: stats.calls, + plan_type, + mean_ms: stats.mean_exec_time, + stddev_ms: stats.stddev_exec_time, + total_ms: stats.total_exec_time, + }); + assert_eq!(stats.calls, RUNS, "expected {RUNS} recorded calls"); + Ok(()) +} + +/// P1: ORE range < with LIMIT — expected ~1.93ms at 100K rows. +async fn bench_ore_range_lt_limit(pool: &PgPool) -> Result<()> { + let encrypted: String = + sqlx::query_scalar("SELECT (encrypted_int).data::text FROM bench WHERE id = 50000") + .fetch_one(pool) + .await?; + + let query = "SELECT * FROM bench WHERE encrypted_int < $1::jsonb::eql_v2_encrypted ORDER BY encrypted_int LIMIT 10"; + let plan_type = fetch_plan_node_type(pool, query, &[&encrypted]).await?; + + reset_pg_stat_statements(pool).await?; + for _ in 0..RUNS { + sqlx::query(query).bind(&encrypted).fetch_all(pool).await?; + } + let stats = read_pg_stat_statements( + pool, + "%FROM bench WHERE encrypted_int < $%ORDER BY encrypted_int LIMIT %", + ) + .await?; + + append_result(PerfResult { + name: "ore_range_lt_limit".into(), + priority: "P1".into(), + runs: stats.calls, + plan_type, + mean_ms: stats.mean_exec_time, + stddev_ms: stats.stddev_exec_time, + total_ms: stats.total_exec_time, + }); + assert_eq!(stats.calls, RUNS, "expected {RUNS} recorded calls"); + Ok(()) +} + +/// P1: ORE ORDER BY encrypted_int LIMIT 10 — design doc observes ~543ms at 10K, +/// so expect several seconds at 100K. Report captures actual number. +async fn bench_ore_order_by_limit(pool: &PgPool) -> Result<()> { + let query = "SELECT * FROM bench ORDER BY encrypted_int LIMIT 10"; + let plan_type = fetch_plan_node_type(pool, query, &[]).await?; + + reset_pg_stat_statements(pool).await?; + for _ in 0..RUNS { + sqlx::query(query).fetch_all(pool).await?; + } + let stats = read_pg_stat_statements(pool, "%FROM bench ORDER BY encrypted_int LIMIT %").await?; + + append_result(PerfResult { + name: "ore_order_by_limit".into(), + priority: "P1".into(), + runs: stats.calls, + plan_type, + mean_ms: stats.mean_exec_time, + stddev_ms: stats.stddev_exec_time, + total_ms: stats.total_exec_time, + }); + assert_eq!(stats.calls, RUNS, "expected {RUNS} recorded calls"); + Ok(()) +} + +async fn flush_reports(pool: &PgPool) -> Result<()> { + let pg_version: String = sqlx::query_scalar("SHOW server_version_num") + .fetch_one(pool) + .await?; + // server_version_num is "170004" etc — take the major version digits + let pg_major = pg_version + .get(..pg_version.len().saturating_sub(4)) + .unwrap_or(&pg_version) + .to_string(); + + let date = std::env::var("BENCH_REPORT_DATE").unwrap_or_else(|_| today_utc()); + let output_dir = std::env::var("BENCH_REPORT_DIR") + .unwrap_or_else(|_| "../../tests/benchmarks/reports".into()); + let (json, md) = write_reports(&output_dir, &date, &pg_major, DATASET_ROWS)?; + eprintln!("wrote {} and {}", json.display(), md.display()); + Ok(()) +} + +fn today_utc() -> String { + // Avoid adding the `chrono` dep; shell out to `date -u` for UTC. + let out = std::process::Command::new("date") + .args(["-u", "+%Y-%m-%d"]) + .output() + .expect("invoking date"); + String::from_utf8(out.stdout).unwrap().trim().to_string() +} + +// ============================================================================ +// Orchestrator — scheduled CI entry point. Runs every benchmark in sequence +// and emits the report. +// ============================================================================ + +#[tokio::test] +#[ignore = "Tier 2: run all benchmarks + write reports (invoked by `mise run bench:full`)"] +async fn run_all_benchmarks() -> Result<()> { + let pool = connect().await?; + bench_hmac_256_equality(&pool).await?; + bench_bloom_filter_containment(&pool).await?; + bench_eql_cast_equality(&pool).await?; + bench_ore_equality_opclass(&pool).await?; + bench_ore_range_lt_limit(&pool).await?; + bench_ore_order_by_limit(&pool).await?; + flush_reports(&pool).await +} + +// ============================================================================ +// Individual test wrappers — allow running one benchmark in isolation via +// `cargo test --test bench_perf_tests -- --ignored`. These do NOT +// flush reports; only `run_all_benchmarks` does that. +// ============================================================================ + +#[tokio::test] +#[ignore = "Tier 2: run via `mise run bench:full` (requires pre-loaded bench data)"] +async fn hmac_256_equality() -> Result<()> { + bench_hmac_256_equality(&connect().await?).await +} + +#[tokio::test] +#[ignore = "Tier 2: run via `mise run bench:full` (requires pre-loaded bench data)"] +async fn bloom_filter_containment() -> Result<()> { + bench_bloom_filter_containment(&connect().await?).await +} + +#[tokio::test] +#[ignore = "Tier 2: run via `mise run bench:full` (requires pre-loaded bench data)"] +async fn eql_cast_equality() -> Result<()> { + bench_eql_cast_equality(&connect().await?).await +} + +#[tokio::test] +#[ignore = "Tier 2: run via `mise run bench:full` (requires pre-loaded bench data)"] +async fn ore_equality_opclass() -> Result<()> { + bench_ore_equality_opclass(&connect().await?).await +} + +#[tokio::test] +#[ignore = "Tier 2: run via `mise run bench:full` (requires pre-loaded bench data)"] +async fn ore_range_lt_limit() -> Result<()> { + bench_ore_range_lt_limit(&connect().await?).await +} + +#[tokio::test] +#[ignore = "Tier 2: run via `mise run bench:full` (requires pre-loaded bench data)"] +async fn ore_order_by_limit() -> Result<()> { + bench_ore_order_by_limit(&connect().await?).await +} diff --git a/tests/sqlx/tests/bench_plan_tests.rs b/tests/sqlx/tests/bench_plan_tests.rs new file mode 100644 index 00000000..7d2dc163 --- /dev/null +++ b/tests/sqlx/tests/bench_plan_tests.rs @@ -0,0 +1,110 @@ +//! Tier 1 benchmark plan assertions +//! +//! EXPLAIN-based tests asserting each P0/P1 query pattern uses the expected +//! index access method. Tests for known-broken patterns are marked #[ignore]. +//! +//! ANALYZE is run by the bench_setup fixture — planner statistics are populated at fixture load. + +use anyhow::Result; +use eql_tests::{assert_uses_index, get_bench_encrypted_int, get_bench_encrypted_text}; +use sqlx::PgPool; + +const BENCH_INT_ORE_IDX: &str = "bench_int_ore_idx"; +const BENCH_TEXT_HMAC_IDX: &str = "bench_text_hmac_idx"; + +/// ORE range query (less-than) uses btree index +#[sqlx::test(fixtures(path = "../fixtures", scripts("bench_data", "bench_setup")))] +async fn ore_int_range_lt_uses_btree_index(pool: PgPool) -> Result<()> { + let encrypted = get_bench_encrypted_int(&pool, 50).await?; + + let sql = format!( + "SELECT * FROM bench WHERE encrypted_int < '{}'::jsonb::eql_v2_encrypted \ + ORDER BY encrypted_int LIMIT 10", + encrypted + ); + assert_uses_index(&pool, &sql, BENCH_INT_ORE_IDX).await?; + Ok(()) +} + +/// ORE range query (greater-than) uses btree index +#[sqlx::test(fixtures(path = "../fixtures", scripts("bench_data", "bench_setup")))] +async fn ore_int_range_gt_uses_btree_index(pool: PgPool) -> Result<()> { + let encrypted = get_bench_encrypted_int(&pool, 50).await?; + + let sql = format!( + "SELECT * FROM bench WHERE encrypted_int > '{}'::jsonb::eql_v2_encrypted \ + ORDER BY encrypted_int LIMIT 10", + encrypted + ); + assert_uses_index(&pool, &sql, BENCH_INT_ORE_IDX).await?; + Ok(()) +} + +/// ORE combined range (>= low AND <= high) uses btree index +/// +/// Uses explicit >= / <= rather than BETWEEN — BETWEEN's operator resolution +/// against eql_v2_encrypted is untested and may not resolve to the btree family. +#[sqlx::test(fixtures(path = "../fixtures", scripts("bench_data", "bench_setup")))] +async fn ore_int_range_combined_uses_btree_index(pool: PgPool) -> Result<()> { + let low = get_bench_encrypted_int(&pool, 10).await?; + let high = get_bench_encrypted_int(&pool, 90).await?; + + let sql = format!( + "SELECT * FROM bench \ + WHERE encrypted_int >= '{}'::jsonb::eql_v2_encrypted \ + AND encrypted_int <= '{}'::jsonb::eql_v2_encrypted \ + ORDER BY encrypted_int LIMIT 10", + low, high + ); + assert_uses_index(&pool, &sql, BENCH_INT_ORE_IDX).await?; + Ok(()) +} + +/// eql_cast equality should use hash index — currently seq scans (CIP-2831) +/// +/// "eql_cast" refers to the implicit JSONB-to-eql_v2_encrypted assignment cast +/// defined in `src/encrypted/casts.sql` (`CREATE CAST (jsonb AS eql_v2_encrypted) +/// WITH FUNCTION eql_v2.to_encrypted(jsonb)`). The SQL under test uses +/// `'...'::jsonb::eql_v2_encrypted`, which invokes that cast. PostgreSQL does not +/// recognise this cast path as equivalent to the indexed `hmac_256` term, so the +/// planner falls back to a sequential scan instead of using `bench_text_hmac_idx`. +/// +/// Remove #[ignore] when eql_cast index usage is fixed. At 1M rows this query +/// takes 7.83s vs 0.4ms for hmac_256 — a 19,500x regression. +/// Passing with the 10K-row fixture confirms index usage — timing data above was measured at 1M rows. +#[sqlx::test(fixtures(path = "../fixtures", scripts("bench_data", "bench_setup")))] +#[ignore = "CIP-2831: eql_cast equality performs full seq scan, no index used"] +async fn eql_cast_equality_uses_hash_index(pool: PgPool) -> Result<()> { + let encrypted = get_bench_encrypted_text(&pool, 1).await?; + + let sql = format!( + "SELECT * FROM bench WHERE encrypted_text = '{}'::jsonb::eql_v2_encrypted", + encrypted + ); + assert_uses_index(&pool, &sql, BENCH_TEXT_HMAC_IDX).await?; + Ok(()) +} + +/// ORE equality via operator class should use btree — currently seq scans (CIP-2831) +/// +/// Like `eql_cast_equality_uses_hash_index`, the SQL uses `'...'::jsonb::eql_v2_encrypted` +/// (the implicit JSONB assignment cast from `src/encrypted/casts.sql`). For integer +/// columns with ORE index terms the planner should satisfy equality via the btree +/// operator class, but the cast path prevents index recognition and causes a seq scan. +/// +/// CIP-2831 covers both this and `eql_cast_equality_uses_hash_index` as a single root cause fix. +/// Remove #[ignore] when ORE equality index usage is fixed. At 1M rows this +/// query takes 18.47s vs 0.4ms for hmac_256. +/// Passing with the 10K-row fixture confirms index usage — timing data above was measured at 1M rows. +#[sqlx::test(fixtures(path = "../fixtures", scripts("bench_data", "bench_setup")))] +#[ignore = "CIP-2831: ORE equality via operator class performs full seq scan"] +async fn ore_equality_uses_btree_index(pool: PgPool) -> Result<()> { + let encrypted = get_bench_encrypted_int(&pool, 1).await?; + + let sql = format!( + "SELECT * FROM bench WHERE encrypted_int = '{}'::jsonb::eql_v2_encrypted", + encrypted + ); + assert_uses_index(&pool, &sql, BENCH_INT_ORE_IDX).await?; + Ok(()) +} diff --git a/tests/sqlx/tests/bench_regression_tests.rs b/tests/sqlx/tests/bench_regression_tests.rs new file mode 100644 index 00000000..b9c3eb44 --- /dev/null +++ b/tests/sqlx/tests/bench_regression_tests.rs @@ -0,0 +1,104 @@ +//! Tier 1 benchmark magnitude regression tests +//! +//! Asserts execution time stays under generous thresholds to catch catastrophic regressions +//! while tolerating CI runner variance. Most thresholds are ~100x the expected baseline; +//! ore_order_by uses 4x (543ms observed baseline leaves little headroom for a 100x multiple +//! without creating a test that never fails). +//! Uses EXPLAIN ANALYZE averaged over 5 runs for server-side timing. +//! +//! Patterns known to be broken (P0 seq scans) are NOT included here — encoding +//! bad performance as "acceptable" defeats the purpose. See bench_plan_tests.rs +//! for their #[ignore] plan assertions. + +use anyhow::Result; +use eql_tests::{ + explain_analyze_avg, explain_analyze_avg_bound, get_bench_encrypted_int, + get_bench_encrypted_text, ExplainStats, +}; +use sqlx::PgPool; + +/// hmac_256 equality must stay under 50ms on 10K rows (expected ~0.5ms) +#[sqlx::test(fixtures(path = "../fixtures", scripts("bench_data", "bench_setup")))] +async fn hmac_equality_under_threshold(pool: PgPool) -> Result<()> { + // id=1 maps to 1 of 100 distinct values → ~100 matching rows at 10K + let encrypted = get_bench_encrypted_text(&pool, 1).await?; + + let stats: ExplainStats = explain_analyze_avg_bound( + &pool, + "SELECT * FROM bench WHERE eql_v2.hmac_256(encrypted_text) = eql_v2.hmac_256($1::jsonb::eql_v2_encrypted)", + &[&encrypted], + 5, + ) + .await?; + assert!( + stats.execution_time_ms < 50.0, + "hmac_256 equality took {:.1}ms, threshold 50ms (expected ~0.5ms at 10K rows, node_type={})", + stats.execution_time_ms, stats.node_type + ); + Ok(()) +} + +/// bloom_filter containment must stay under 100ms on 10K rows (expected ~1ms) +#[sqlx::test(fixtures(path = "../fixtures", scripts("bench_data", "bench_setup")))] +async fn bloom_filter_containment_under_threshold(pool: PgPool) -> Result<()> { + // id=1 maps to 1 of 100 distinct values → ~100 matching rows at 10K + let encrypted = get_bench_encrypted_text(&pool, 1).await?; + + let stats: ExplainStats = explain_analyze_avg_bound( + &pool, + "SELECT * FROM bench WHERE eql_v2.bloom_filter(encrypted_text) @> eql_v2.bloom_filter($1::jsonb::eql_v2_encrypted)", + &[&encrypted], + 5, + ) + .await?; + assert!( + stats.execution_time_ms < 100.0, + "bloom_filter containment took {:.1}ms, threshold 100ms (expected ~1ms at 10K rows, node_type={})", + stats.execution_time_ms, stats.node_type + ); + Ok(()) +} + +/// ORE range query (< LIMIT 10) must stay under 200ms on 10K rows (expected ~2ms) +#[sqlx::test(fixtures(path = "../fixtures", scripts("bench_data", "bench_setup")))] +async fn ore_range_lt_under_threshold(pool: PgPool) -> Result<()> { + // id=50 is the bench row midpoint; encrypted_int uses a +33 offset so this maps + // to ore id 83, but the 10K distribution still yields ~4,900 rows below the predicate + let encrypted = get_bench_encrypted_int(&pool, 50).await?; + + let stats: ExplainStats = explain_analyze_avg_bound( + &pool, + "SELECT * FROM bench WHERE encrypted_int < $1::jsonb::eql_v2_encrypted \ + ORDER BY encrypted_int LIMIT 10", + &[&encrypted], + 5, + ) + .await?; + assert!( + stats.execution_time_ms < 200.0, + "ORE range < LIMIT 10 took {:.1}ms, threshold 200ms (expected ~2ms at 10K rows, node_type={})", + stats.execution_time_ms, stats.node_type + ); + Ok(()) +} + +/// ORE ORDER BY LIMIT 10 must stay under 2000ms on 10K rows +/// +/// The design doc's observed baseline for this pattern is ~543ms at 10K rows +/// ("Full-set comparison before sort"). Threshold is set at 2000ms — 4x the +/// observed baseline — to absorb CI variance while catching catastrophic regressions. +#[sqlx::test(fixtures(path = "../fixtures", scripts("bench_data", "bench_setup")))] +async fn ore_order_by_under_threshold(pool: PgPool) -> Result<()> { + let stats: ExplainStats = explain_analyze_avg( + &pool, + "SELECT * FROM bench ORDER BY encrypted_int LIMIT 10", + 5, + ) + .await?; + assert!( + stats.execution_time_ms < 2000.0, + "ORE ORDER BY LIMIT 10 took {:.1}ms, threshold 2000ms (observed ~543ms baseline at 10K rows, node_type={})", + stats.execution_time_ms, stats.node_type + ); + Ok(()) +} diff --git a/tests/sqlx/tests/order_by_no_opclass_tests.rs b/tests/sqlx/tests/order_by_no_opclass_tests.rs index 136a4ce4..201e59d3 100644 --- a/tests/sqlx/tests/order_by_no_opclass_tests.rs +++ b/tests/sqlx/tests/order_by_no_opclass_tests.rs @@ -169,6 +169,7 @@ async fn direct_order_by_desc_wrong_order_without_opclass(pool: PgPool) -> Resul // ============================================================================ #[sqlx::test(fixtures(path = "../fixtures", scripts("drop_operator_classes")))] +#[ignore = "slow: O(n²) correlated subquery over 1000 rows; run with --ignored"] async fn correlated_subquery_ranking_asc_without_opclass(pool: PgPool) -> Result<()> { // eql_v2.compare() is a standalone function (not an operator), so it survives // the operator class drops. A correlated subquery counts how many rows have a @@ -198,6 +199,7 @@ async fn correlated_subquery_ranking_asc_without_opclass(pool: PgPool) -> Result } #[sqlx::test(fixtures(path = "../fixtures", scripts("drop_operator_classes")))] +#[ignore = "slow: O(n²) correlated subquery over 1000 rows; run with --ignored"] async fn correlated_subquery_ranking_desc_without_opclass(pool: PgPool) -> Result<()> { // Same correlated subquery with DESC — should return highest-ranked rows first. @@ -220,6 +222,7 @@ async fn correlated_subquery_ranking_desc_without_opclass(pool: PgPool) -> Resul } #[sqlx::test(fixtures(path = "../fixtures", scripts("drop_operator_classes")))] +#[ignore = "slow: O(n²) correlated subquery over 1000 rows; run with --ignored"] async fn correlated_subquery_ranking_with_limit_without_opclass(pool: PgPool) -> Result<()> { // LIMIT 1 with ASC subquery ranking should return the smallest value (id=1) @@ -238,6 +241,7 @@ async fn correlated_subquery_ranking_with_limit_without_opclass(pool: PgPool) -> } #[sqlx::test(fixtures(path = "../fixtures", scripts("drop_operator_classes")))] +#[ignore = "slow: O(n²) correlated subquery over 1000 rows; run with --ignored"] async fn correlated_subquery_ranking_with_where_without_opclass(pool: PgPool) -> Result<()> { // WHERE clause filters rows, then correlated subquery orders the result correctly. // Note: the subquery counts over the full table to produce a global rank. diff --git a/tests/sqlx/tests/order_by_sort_tests.rs b/tests/sqlx/tests/order_by_sort_tests.rs index 33b69cbf..853e9e4a 100644 --- a/tests/sqlx/tests/order_by_sort_tests.rs +++ b/tests/sqlx/tests/order_by_sort_tests.rs @@ -522,6 +522,7 @@ async fn sort_compare_table_ref_matches_order_by_compare(pool: PgPool) -> Result // ============================================================================ #[sqlx::test(fixtures(path = "../fixtures", scripts("drop_operator_classes")))] +#[ignore = "slow: O(n²) correlated subquery over 1000 rows (~7m); run with --ignored"] async fn filtered_inner_query_correct_order(pool: PgPool) -> Result<()> { // Optimized: inner query also filters, producing correct relative ordering // within the filtered set @@ -602,6 +603,7 @@ async fn filtered_inner_query_with_range(pool: PgPool) -> Result<()> { // ============================================================================ #[sqlx::test(fixtures(path = "../fixtures", scripts("drop_operator_classes")))] +#[ignore = "perf: relative timing assertion at 1000 rows; run with --ignored"] async fn sort_compare_faster_than_correlated_subquery(pool: PgPool) -> Result<()> { // Warm up: run each query once to populate caches let sort_sql = "SELECT * FROM eql_v2.sort_compare( @@ -644,6 +646,7 @@ async fn sort_compare_faster_than_correlated_subquery(pool: PgPool) -> Result<() } #[sqlx::test(fixtures(path = "../fixtures", scripts("drop_operator_classes")))] +#[ignore = "perf: relative timing assertion at 1000 rows; run with --ignored"] async fn filtered_inner_query_faster_than_unfiltered(pool: PgPool) -> Result<()> { let ore_term = get_ore_encrypted(&pool, 42).await?; @@ -703,6 +706,7 @@ async fn filtered_inner_query_faster_than_unfiltered(pool: PgPool) -> Result<()> // ============================================================================ #[sqlx::test(fixtures(path = "../fixtures", scripts("drop_operator_classes")))] +#[ignore = "perf: O(n log n) vs O(n²) demonstration at 1000 rows; run with --ignored"] async fn sort_compare_performance_at_scale(pool: PgPool) -> Result<()> { // 1000 rows is sufficient scale to demonstrate O(n log n) vs O(n²) let sort_sql = "SELECT * FROM eql_v2.sort_compare( @@ -738,6 +742,7 @@ async fn sort_compare_performance_at_scale(pool: PgPool) -> Result<()> { } #[sqlx::test(fixtures(path = "../fixtures", scripts("drop_operator_classes")))] +#[ignore = "perf: timing assertion at 1000 rows; run with --ignored"] async fn filtered_inner_query_performance_at_scale(pool: PgPool) -> Result<()> { let ore_term = get_ore_encrypted(&pool, 42).await?;