From 4cd348067a738f292da948b1536e6a08fef17d94 Mon Sep 17 00:00:00 2001 From: jamals86 Date: Mon, 4 May 2026 15:42:41 +0300 Subject: [PATCH] Enable EXECUTE AS for STREAM tables and CLI tweaks Allow EXECUTE AS USER to operate on STREAM tables (backend error messages and core checks updated) and add integration tests to verify stream-table impersonation. Improve CLI UX: normalize/validate server URLs with sensible http/https defaults, preserve SDK keepalive behavior, add meta-command aliases and SQL keyword completions, and surface the query "As: " footer in table output. Add configurable chat benchmark message-rate pacing and CLI/script flags. Run SQL Studio regression tests in CI and wire up related UI test/files. Also bump several dependencies and add tokio-netem for network impairment testing. Minor test and fixture updates included. --- .github/workflows/release.yml | 11 +- Cargo.lock | 121 ++-- Cargo.toml | 32 +- .../src/http/sql/execution_paths.rs | 4 +- .../kalamdb-core/src/cluster_handler.rs | 3 +- backend/tests/common/testserver/fixtures.rs | 2 + backend/tests/docs/AUTH_SCENARIOS.md | 4 +- .../misc/auth/test_as_user_impersonation.rs | 76 +- benchv2/Cargo.lock | 1 + benchv2/run-chat-realtime.sh | 13 + benchv2/src/benchmarks/chat_realtime_bench.rs | 96 ++- cli/README.md | 15 +- cli/src/completer.rs | 59 +- cli/src/connect.rs | 61 +- cli/src/formatter.rs | 78 +- cli/src/parser.rs | 98 ++- cli/src/session.rs | 330 +++++---- cli/src/session/cluster/actions.rs | 8 +- cli/src/session/commands.rs | 415 ++++++----- cli/src/session/info.rs | 147 ++-- cli/tests/cli/test_cli_doc_matrix.rs | 29 +- .../adr-019-subject-scoped-user-identity.md | 13 +- docs/getting-started/cli.md | 11 +- docs/getting-started/quick-start.md | 8 +- docs/reference/sql.md | 8 +- examples/chat-with-ai/package-lock.json | 7 +- examples/chat-with-ai/package.json | 6 +- .../chat-with-ai/scripts/generate-schema.sh | 5 +- link/kalam-client/Cargo.toml | 3 +- link/kalam-client/tests/common/tcp_proxy.rs | 71 +- link/kalam-client/tests/proxied.rs | 2 + .../tests/proxied/topic_consumption_netem.rs | 471 +++++++++++++ .../tests/proxied/transport_impairments.rs | 359 ++++++++++ link/kalam-link-dart/src/tests.rs | 2 + link/link-common/Cargo.toml | 3 +- link/link-common/src/client/builder.rs | 12 +- link/link-common/src/client/tests.rs | 12 + .../src/query/models/query_result.rs | 4 + link/sdks/typescript/client/README.md | 4 +- link/sdks/typescript/client/package-lock.json | 4 +- link/sdks/typescript/client/src/client.ts | 3 +- link/sdks/typescript/orm/package-lock.json | 8 +- nextest.toml | 24 + ui/package.json | 2 + .../input-form/StudioEditorPanel.tsx | 122 ++-- .../input-form/sqlCompletionCatalog.test.ts | 162 +++++ .../input-form/sqlCompletionCatalog.ts | 666 ++++++++++++++++++ ui/src/pages/SqlStudio.test.tsx | 40 ++ ui/src/services/sqlStudioService.ts | 2 +- ui/tsconfig.json | 1 + ui/vite.config.ts | 9 +- 51 files changed, 3036 insertions(+), 611 deletions(-) create mode 100644 link/kalam-client/tests/proxied/topic_consumption_netem.rs create mode 100644 ui/src/components/sql-studio-v2/input-form/sqlCompletionCatalog.test.ts create mode 100644 ui/src/components/sql-studio-v2/input-form/sqlCompletionCatalog.ts diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 181d93e73..22ee788c6 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -230,6 +230,15 @@ jobs: cd ui npm install + - name: Run SQL Studio regression tests + shell: bash + env: + CI: "true" + run: | + set -euo pipefail + cd ui + npm run test:sql-studio + - name: Run Admin UI tests shell: bash env: @@ -237,7 +246,7 @@ jobs: run: | set -euo pipefail cd ui - npm test -- --run + npm run test:ci - name: Build Admin UI shell: bash diff --git a/Cargo.lock b/Cargo.lock index eeb9de7af..05d491686 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -498,9 +498,9 @@ checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" [[package]] name = "arrow" -version = "58.1.0" +version = "58.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d441fdda254b65f3e9025910eb2c2066b6295d9c8ed409522b8d2ace1ff8574c" +checksum = "607e64bb911ee4f90483e044fe78f175989148c2892e659a2cd25429e782ec54" dependencies = [ "arrow-arith", "arrow-array", @@ -519,9 +519,9 @@ dependencies = [ [[package]] name = "arrow-arith" -version = "58.1.0" +version = "58.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ced5406f8b720cc0bc3aa9cf5758f93e8593cda5490677aa194e4b4b383f9a59" +checksum = "e754319ed8a85d817fe7adf183227e0b5308b82790a737b426c1124626b48118" dependencies = [ "arrow-array", "arrow-buffer", @@ -533,9 +533,9 @@ dependencies = [ [[package]] name = "arrow-array" -version = "58.1.0" +version = "58.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "772bd34cacdda8baec9418d80d23d0fb4d50ef0735685bd45158b83dfeb6e62d" +checksum = "841321891f247aa86c6112c80d83d89cb36e0addd020fa2425085b8eb6c3f579" dependencies = [ "ahash 0.8.12", "arrow-buffer", @@ -544,7 +544,7 @@ dependencies = [ "chrono", "chrono-tz", "half 2.7.1", - "hashbrown 0.16.1", + "hashbrown 0.17.0", "num-complex", "num-integer", "num-traits", @@ -552,9 +552,9 @@ dependencies = [ [[package]] name = "arrow-buffer" -version = "58.1.0" +version = "58.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "898f4cf1e9598fdb77f356fdf2134feedfd0ee8d5a4e0a5f573e7d0aec16baa4" +checksum = "f955dfb73fae000425f49c8226d2044dab60fb7ad4af1e24f961756354d996c9" dependencies = [ "bytes", "half 2.7.1", @@ -564,9 +564,9 @@ dependencies = [ [[package]] name = "arrow-cast" -version = "58.1.0" +version = "58.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0127816c96533d20fc938729f48c52d3e48f99717e7a0b5ade77d742510736d" +checksum = "ca5e686972523798f76bef355145bc1ae25a84c731e650268d31ab763c701663" dependencies = [ "arrow-array", "arrow-buffer", @@ -586,9 +586,9 @@ dependencies = [ [[package]] name = "arrow-csv" -version = "58.1.0" +version = "58.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca025bd0f38eeecb57c2153c0123b960494138e6a957bbda10da2b25415209fe" +checksum = "86c276756867fc8186ec380c72c290e6e3b23a1d4fb05df6b1d62d2e62666d48" dependencies = [ "arrow-array", "arrow-cast", @@ -601,9 +601,9 @@ dependencies = [ [[package]] name = "arrow-data" -version = "58.1.0" +version = "58.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42d10beeab2b1c3bb0b53a00f7c944a178b622173a5c7bcabc3cb45d90238df4" +checksum = "db3b5846209775b6dc8056d77ff9a032b27043383dd5488abd0b663e265b9373" dependencies = [ "arrow-buffer", "arrow-schema", @@ -614,9 +614,9 @@ dependencies = [ [[package]] name = "arrow-ipc" -version = "58.1.0" +version = "58.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "609a441080e338147a84e8e6904b6da482cefb957c5cdc0f3398872f69a315d0" +checksum = "fd8907ddd8f9fbabf91ec2c85c1d81fe2874e336d2443eb36373595e28b98dd5" dependencies = [ "arrow-array", "arrow-buffer", @@ -629,15 +629,16 @@ dependencies = [ [[package]] name = "arrow-json" -version = "58.1.0" +version = "58.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ead0914e4861a531be48fe05858265cf854a4880b9ed12618b1d08cba9bebc8" +checksum = "f4518c59acc501f10d7dcae397fe12b8db3d81bc7de94456f8a58f9165d6f502" dependencies = [ "arrow-array", "arrow-buffer", "arrow-cast", - "arrow-data", + "arrow-ord", "arrow-schema", + "arrow-select", "chrono", "half 2.7.1", "indexmap", @@ -653,9 +654,9 @@ dependencies = [ [[package]] name = "arrow-ord" -version = "58.1.0" +version = "58.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "763a7ba279b20b52dad300e68cfc37c17efa65e68623169076855b3a9e941ca5" +checksum = "efa70d9d6b1356f1fb9f1f651b84a725b7e0abb93f188cf7d31f14abfa2f2e6f" dependencies = [ "arrow-array", "arrow-buffer", @@ -666,9 +667,9 @@ dependencies = [ [[package]] name = "arrow-row" -version = "58.1.0" +version = "58.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e14fe367802f16d7668163ff647830258e6e0aeea9a4d79aaedf273af3bdcd3e" +checksum = "faec88a945338192beffbbd4be0def70135422930caa244ac3cec0cd213b26b4" dependencies = [ "arrow-array", "arrow-buffer", @@ -679,9 +680,9 @@ dependencies = [ [[package]] name = "arrow-schema" -version = "58.1.0" +version = "58.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c30a1365d7a7dc50cc847e54154e6af49e4c4b0fddc9f607b687f29212082743" +checksum = "18aa020f6bc8e5201dcd2d4b7f98c68f8a410ef37128263243e6ff2a47a67d4f" dependencies = [ "serde_core", "serde_json", @@ -689,9 +690,9 @@ dependencies = [ [[package]] name = "arrow-select" -version = "58.1.0" +version = "58.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78694888660a9e8ac949853db393af2a8b8fc82c19ce333132dfa2e72cc1a7fe" +checksum = "a657ab5132e9c8ca3b24eb15a823d0ced38017fe3930ff50167466b02e2d592c" dependencies = [ "ahash 0.8.12", "arrow-array", @@ -703,9 +704,9 @@ dependencies = [ [[package]] name = "arrow-string" -version = "58.1.0" +version = "58.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61e04a01f8bb73ce54437514c5fd3ee2aa3e8abe4c777ee5cc55853b1652f79e" +checksum = "f6de2efbbd1a9f9780ceb8d1ff5d20421b35863b361e3386b4f571f1fc69fcb8" dependencies = [ "arrow-array", "arrow-buffer", @@ -3239,6 +3240,12 @@ dependencies = [ "foldhash 0.2.0", ] +[[package]] +name = "hashbrown" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f467dd6dccf739c208452f8014c75c18bb8301b050ad1cfb27153803edb0f51" + [[package]] name = "heck" version = "0.5.0" @@ -3838,7 +3845,7 @@ dependencies = [ "predicates", "rand 0.10.1", "regex", - "reqwest 0.13.2", + "reqwest 0.13.3", "rpassword", "rustyline", "serde", @@ -3862,10 +3869,11 @@ dependencies = [ "kalamdb-server", "link-common", "ntest", - "reqwest 0.13.2", + "reqwest 0.13.3", "serde_json", "tempfile", "tokio", + "tokio-netem", ] [[package]] @@ -4049,7 +4057,7 @@ dependencies = [ "log", "moka", "once_cell", - "reqwest 0.13.2", + "reqwest 0.13.3", "serde", "serde_json", "sha2 0.11.0", @@ -4556,7 +4564,7 @@ dependencies = [ "opentelemetry_sdk", "parking_lot", "rand 0.10.1", - "reqwest 0.13.2", + "reqwest 0.13.3", "serde", "serde_json", "serial_test", @@ -4937,8 +4945,9 @@ dependencies = [ "log", "miniz_oxide 0.9.1", "quinn-proto", - "reqwest 0.13.2", + "reqwest 0.13.3", "rmp-serde", + "rustls", "rustls-webpki", "serde", "serde_json", @@ -5624,9 +5633,9 @@ dependencies = [ [[package]] name = "parquet" -version = "58.1.0" +version = "58.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d3f9f2205199603564127932b89695f52b62322f541d0fc7179d57c2e1c9877" +checksum = "43d7efd3052f7d6ef601085559a246bc991e9a8cc77e02753737df6322ce35f1" dependencies = [ "ahash 0.8.12", "arrow-array", @@ -5642,7 +5651,7 @@ dependencies = [ "flate2", "futures", "half 2.7.1", - "hashbrown 0.16.1", + "hashbrown 0.17.0", "lz4_flex", "num-bigint", "num-integer", @@ -6551,9 +6560,9 @@ dependencies = [ [[package]] name = "reqwest" -version = "0.13.2" +version = "0.13.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab3f43e3283ab1488b624b44b0e988d0acea0b3214e694730a055cb6b2efa801" +checksum = "62e0021ea2c22aed41653bc7e1419abb2c97e038ff2c33d0e1309e49a97deec0" dependencies = [ "base64", "bytes", @@ -6672,9 +6681,9 @@ dependencies = [ [[package]] name = "rpassword" -version = "7.5.1" +version = "7.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2501c67132bd19c3005b0111fba298907ef002c8c1cf68e25634707e38bf66fe" +checksum = "5ac5b223d9738ef56e0b98305410be40fa0941bf6036c56f1506751e43552d64" dependencies = [ "libc", "rtoolbox", @@ -6788,9 +6797,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.37" +version = "0.23.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "758025cb5fccfd3bc2fd74708fd4682be41d99e5dff73c377c0646c6012c73a4" +checksum = "ef86cd5876211988985292b91c96a8f2d298df24e75989a43a3c73f2d4d8168b" dependencies = [ "aws-lc-rs", "log", @@ -7624,6 +7633,26 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "tokio-netem" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ee34c2e83c178d3ffeb53bf51fa480fcdc1dac07a693e786e8677fc6773d8b9" +dependencies = [ + "anyhow", + "bytes", + "clap", + "futures", + "humantime", + "pin-project", + "rand 0.9.2", + "smallvec", + "socket2 0.5.10", + "tokio", + "tracing", + "tracing-subscriber", +] + [[package]] name = "tokio-postgres" version = "0.7.17" @@ -8140,9 +8169,9 @@ dependencies = [ [[package]] name = "usearch" -version = "2.25.1" +version = "2.25.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1025b65e3669950a226b1b5e4e8de77f51ad7af7cf8a693ada62a9ec16c742e0" +checksum = "15d31c8be436d55b9ec4300fe21978435bf59c1026e821257e9e74c651f7f05c" dependencies = [ "cxx", "cxx-build", diff --git a/Cargo.toml b/Cargo.toml index 986760610..0f0671834 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -73,11 +73,12 @@ log = "0.4.29" # Async runtime tokio = { version = "1.52.1", features = ["rt-multi-thread", "macros", "sync", "time", "fs", "io-util", "io-std", "net", "signal", "parking_lot"] } +tokio-netem = "0.1.1" tokio-stream = { version = "0.1", features = ["net"] } # HTTP client (with HTTP/2 support) - using rustls-tls for cross-compilation compatibility # Note: native-tls requires OpenSSL which is difficult to cross-compile for aarch64 -reqwest = { version = "0.13.2", default-features = false, features = ["json", "rustls", "http2", "multipart", "stream", "form"] } +reqwest = { version = "0.13.3", default-features = false, features = ["json", "rustls", "http2", "multipart", "stream", "form"] } hyper = { version = "1.8.1", default-features = false, features = ["client", "http1"] } hyper-util = { version = "0.1.20", default-features = false, features = ["client", "client-legacy", "http1", "tokio"] } url = "2.5.8" @@ -87,6 +88,7 @@ tokio-tungstenite = { version = "0.29.0", features = ["rustls-tls-webpki-roots"] # Security floor pins for vulnerable transitive TLS/QUIC crates. aws-lc-rs = { version = "1.16.3", default-features = false } +rustls = { version = "0.23.40", default-features = false } quinn-proto = { version = "0.11.14", default-features = false } rustls-webpki = { version = "0.103.13", default-features = false } @@ -101,16 +103,16 @@ chrono = { version = "0.4.44", features = ["serde"] } # DataFusion 52.3.0 resolves to Arrow 57.3.0; mixing in Arrow 58 pulls a second # arrow_array crate into the graph and causes FixedSizeListArray type mismatches. # Upgrade DataFusion first before bumping Arrow/Parquet beyond this family. -arrow = { version = "58.1.0", default-features = false } -arrow-ipc = { version = "58.1.0", default-features = false } -arrow-schema = { version = "58.1.0" } +arrow = { version = "58.2.0", default-features = false } +arrow-ipc = { version = "58.2.0", default-features = false } +arrow-schema = { version = "58.2.0" } datafusion = { version = "53.1.0", default-features = false, features = ["sql", "parquet", "recursive_protection", "nested_expressions"] } datafusion-datasource = { version = "53.1.0", default-features = false } datafusion-common = { version = "53.1.0", default-features = false } datafusion-expr = { version = "53.1.0" } datafusion-functions-json = { version = "0.53.1" } sqlparser = { version = "0.61.0" } -parquet = { version = "58.1.0", default-features = false, features = ["snap", "zstd", "arrow", "async"] } +parquet = { version = "58.2.0", default-features = false, features = ["snap", "zstd", "arrow", "async"] } # Web framework actix-web = { version = "4.13.0", features = ["http2"] } @@ -133,7 +135,7 @@ ulid = "1.1" jsonwebtoken = { version = "10.3.0", default-features = false, features = ["aws_lc_rs"] } # Configuration -toml = "1.1.0" +toml = "1.1.2" dotenv = "0.15" # CLI tools @@ -161,14 +163,14 @@ rmp-serde = "1.3" # gRPC for Raft network layer tonic = { version = "0.14.5" } tonic-prost = "0.14.5" -tonic-build = "0.14.4" +tonic-build = "0.14.5" prost = "0.14.3" prost-types = "0.14.3" # Additional dependencies bcrypt = "0.19.0" rand = "0.10.1" -rcgen = "0.14.1" +rcgen = "0.14.7" x509-parser = "0.18.1" kalamdb-commons = { path = "backend/crates/kalamdb-commons", default-features = false } kalamdb-plan-cache = { path = "backend/crates/kalamdb-plan-cache" } @@ -179,7 +181,7 @@ cookie = "0.18" colored = "3.1.1" fern = "0.7" indicatif = "0.18.4" -rpassword = "7.5.1" +rpassword = "7.5.2" dirs = "6.0" term_size = "0.3" crossterm = "0.29.0" @@ -189,23 +191,23 @@ futures-util = { version = "0.3.32", default-features = false } base64 = "0.22" flatbuffers = "25.12.19" flexbuffers = "25.9.23" -async-trait = "0.1.74" +async-trait = "0.1.89" once_cell = "1.21.4" regex = "1.12.3" dashmap = "6.1" bytes = "1.11.1" -http-body = "1.0.0" -http-body-util = "0.1.2" +http-body = "1.0.1" +http-body-util = "0.1.3" cc = "1.2.61" proc-macro2 = "1.0.106" -quote = "1.0.44" +quote = "1.0.45" syn = { version = "2.0.117", features = ["full", "extra-traits"] } # Object storage abstraction (S3/GCS/Azure/local) object_store = { version = "0.13.2" } # Vector ANN engines -usearch = "2.25.1" +usearch = "2.25.2" parking_lot = "0.12" tokio-util = "0.7.18" @@ -246,7 +248,7 @@ tracing-log = "0.2" tracing-opentelemetry = "0.32.1" opentelemetry = "0.31.0" opentelemetry_sdk = { version = "0.31.0", features = ["trace", "rt-tokio"] } -opentelemetry-otlp = { version = "0.31.0", default-features = false, features = ["trace", "grpc-tonic", "http-proto"] } +opentelemetry-otlp = { version = "0.31.1", default-features = false, features = ["trace", "grpc-tonic", "http-proto"] } # Release profile: balanced for production use (good performance + reasonable size) [profile.release] diff --git a/backend/crates/kalamdb-api/src/http/sql/execution_paths.rs b/backend/crates/kalamdb-api/src/http/sql/execution_paths.rs index 8e2fcbbdb..746b1ebe0 100644 --- a/backend/crates/kalamdb-api/src/http/sql/execution_paths.rs +++ b/backend/crates/kalamdb-api/src/http/sql/execution_paths.rs @@ -298,7 +298,7 @@ pub(super) async fn execute_file_upload_path( ErrorCode::SqlExecutionError, &format!( "EXECUTE AS USER is not allowed on SHARED tables (table '{}'). AS USER \ - impersonation is only supported for USER tables.", + impersonation is only supported for USER and STREAM tables.", table_id ), took_ms(start_time), @@ -562,7 +562,7 @@ pub(super) async fn execute_batch_path( ErrorCode::SqlExecutionError, &format!( "EXECUTE AS USER is not allowed on SHARED tables (table '{}'). AS USER \ - impersonation is only supported for USER tables.", + impersonation is only supported for USER and STREAM tables.", table_id ), took_ms(start_time), diff --git a/backend/crates/kalamdb-core/src/cluster_handler.rs b/backend/crates/kalamdb-core/src/cluster_handler.rs index 45fac493e..3c85d1f83 100644 --- a/backend/crates/kalamdb-core/src/cluster_handler.rs +++ b/backend/crates/kalamdb-core/src/cluster_handler.rs @@ -431,7 +431,8 @@ impl ClusterMessageHandler for CoreClusterHandler { "SQL_EXECUTION_ERROR", &format!( "Statement {} failed: EXECUTE AS USER is not allowed on SHARED tables \ - (table '{}'). AS USER impersonation is only supported for USER tables.", + (table '{}'). AS USER impersonation is only supported for USER and \ + STREAM tables.", idx + 1, table_name ), diff --git a/backend/tests/common/testserver/fixtures.rs b/backend/tests/common/testserver/fixtures.rs index f46fe05e5..8b73ea85b 100644 --- a/backend/tests/common/testserver/fixtures.rs +++ b/backend/tests/common/testserver/fixtures.rs @@ -190,6 +190,7 @@ pub async fn create_messages_table( named_rows: None, row_count: 0, message: Some("Table already existed".to_string()), + as_user: None, }], took: Some(0.0), error: None, @@ -311,6 +312,7 @@ pub async fn create_stream_table( named_rows: None, row_count: 0, message: Some("Table already existed".to_string()), + as_user: None, }], took: Some(0.0), error: None, diff --git a/backend/tests/docs/AUTH_SCENARIOS.md b/backend/tests/docs/AUTH_SCENARIOS.md index cf6ddabfa..ef62d13c9 100644 --- a/backend/tests/docs/AUTH_SCENARIOS.md +++ b/backend/tests/docs/AUTH_SCENARIOS.md @@ -35,10 +35,10 @@ This document outlines the test scenarios for authentication, authorization, and - **Agent Friendly**: Check `backend/tests/testserver/tables/test_user_tables_http.rs`. ## 4. EXECUTE AS USER Role Matrix -**Goal**: Direct USER-table access stays subject-scoped, while explicit `EXECUTE AS USER` follows the role hierarchy. +**Goal**: Direct USER-table and STREAM-table access stays subject-scoped, while explicit `EXECUTE AS USER` follows the role hierarchy. - **Step 1**: Authenticate as `user`, `service`, `dba`, and `system` role users. - **Step 2**: Attempt allowed and denied cross-user `EXECUTE AS USER` SELECT/INSERT/UPDATE/DELETE. -- **Step 3**: Verify allowed role edges operate under the target user ID and denied edges leave target rows unchanged. +- **Step 3**: Verify allowed role edges operate under the target user ID for USER and STREAM tables, and denied edges leave target rows unchanged. - **Step 4**: Verify self-targeted `EXECUTE AS USER` is a no-op and remains scoped to the actor. - **Agent Friendly**: Check `backend/tests/misc/auth/test_as_user_impersonation.rs`. diff --git a/backend/tests/misc/auth/test_as_user_impersonation.rs b/backend/tests/misc/auth/test_as_user_impersonation.rs index d6bc35861..90b929de1 100644 --- a/backend/tests/misc/auth/test_as_user_impersonation.rs +++ b/backend/tests/misc/auth/test_as_user_impersonation.rs @@ -1,8 +1,9 @@ //! Integration tests for EXECUTE AS USER subject isolation and role hierarchy. //! -//! Ordinary USER-table reads remain scoped to the authenticated subject. Explicit -//! EXECUTE AS USER can switch the subject only when the actor role is allowed to -//! target the cached privileged-role class for the target user ID. +//! Ordinary USER-table and STREAM-table reads remain scoped to the authenticated +//! subject. Explicit EXECUTE AS USER can switch the subject only when the actor +//! role is allowed to target the cached privileged-role class for the target +//! user ID. use kalam_client::models::{QueryResponse, ResponseStatus}; use kalamdb_commons::models::{AuthType, Role, UserId}; @@ -111,6 +112,21 @@ async fn create_user_table(server: &TestServer, namespace: &str, table: &str) { assert_success(&table_resp, "create user table"); } +async fn create_stream_table(server: &TestServer, namespace: &str, table: &str) { + let ns_resp = server + .execute_sql_as_user(&format!("CREATE NAMESPACE {}", namespace), "root") + .await; + assert_success(&ns_resp, "create namespace"); + + let create_table = format!( + "CREATE TABLE {}.{} (id VARCHAR PRIMARY KEY, value VARCHAR) WITH (TYPE = 'STREAM', \ + TTL_SECONDS = 3600)", + namespace, table + ); + let table_resp = server.execute_sql_as_user(&create_table, "root").await; + assert_success(&table_resp, "create stream table"); +} + #[actix_web::test] #[ntest::timeout(45000)] async fn test_execute_as_user_role_matrix_allows_privileged_roles_and_denies_user() { @@ -347,6 +363,60 @@ async fn test_allowed_execute_as_can_select_update_and_delete_target_rows() { ); } +#[actix_web::test] +#[ntest::timeout(45000)] +async fn test_execute_as_stream_table_uses_target_user_scope() { + let server = TestServer::new_shared().await; + let ns = unique_name("as_user_stream_scope"); + create_stream_table(&server, &ns, "events").await; + + let actor = insert_user(&server, &unique_name("stream_actor"), Role::Service).await; + let target = insert_user(&server, &unique_name("stream_target"), Role::User).await; + + let insert_sql = format!( + "EXECUTE AS USER '{}' (INSERT INTO {}.events (id, value) VALUES ('e1', 'delegated'))", + target.as_str(), + ns + ); + let insert_resp = server.execute_sql_as_user(&insert_sql, actor.as_str()).await; + assert_success(&insert_resp, "stream insert through execute as user"); + + let target_direct = server + .execute_sql_as_user( + &format!("SELECT value FROM {}.events WHERE id = 'e1'", ns), + target.as_str(), + ) + .await; + assert_success(&target_direct, "target direct stream select"); + assert_eq!(row_count(&target_direct), 1, "target should see delegated stream row"); + + let actor_direct = server + .execute_sql_as_user( + &format!("SELECT value FROM {}.events WHERE id = 'e1'", ns), + actor.as_str(), + ) + .await; + assert_success(&actor_direct, "actor direct stream select"); + assert_eq!(row_count(&actor_direct), 0, "actor must not see target stream row directly"); + + let actor_as_target = server + .execute_sql_as_user( + &format!( + "EXECUTE AS USER '{}' (SELECT value FROM {}.events WHERE id = 'e1')", + target.as_str(), + ns + ), + actor.as_str(), + ) + .await; + assert_success(&actor_as_target, "actor select through execute as on stream table"); + assert_eq!( + row_count(&actor_as_target), + 1, + "execute as user should read the target stream partition" + ); +} + #[actix_web::test] #[ntest::timeout(45000)] async fn test_disallowed_execute_as_denial_is_audited() { diff --git a/benchv2/Cargo.lock b/benchv2/Cargo.lock index 4d0d068a8..9595facf3 100644 --- a/benchv2/Cargo.lock +++ b/benchv2/Cargo.lock @@ -1009,6 +1009,7 @@ dependencies = [ "quinn-proto", "reqwest", "rmp-serde", + "rustls", "rustls-webpki", "serde", "serde_json", diff --git a/benchv2/run-chat-realtime.sh b/benchv2/run-chat-realtime.sh index 5f1f86cec..2388f2210 100755 --- a/benchv2/run-chat-realtime.sh +++ b/benchv2/run-chat-realtime.sh @@ -4,6 +4,8 @@ # Usage: # ./run-chat-realtime.sh # ./run-chat-realtime.sh --url http://127.0.0.1:8080 --minutes 10 --users 1000 --realtime-convs 150 +# ./run-chat-realtime.sh --messages-per-minute 20 +# ./run-chat-realtime.sh --messages-per-minute 0 set -euo pipefail @@ -14,6 +16,7 @@ URL="${KALAMDB_URL:-http://127.0.0.1:8080}" MINUTES="${KALAMDB_BENCH_CHAT_MINUTES:-5}" USER_COUNT="${KALAMDB_BENCH_CHAT_USERS:-1000}" REALTIME_CONVS="${KALAMDB_BENCH_CHAT_REALTIME_CONVS:-100}" +MESSAGES_PER_MINUTE="${KALAMDB_BENCH_CHAT_MESSAGES_PER_MINUTE:-20}" BENCH_USER="${KALAMDB_USER:-}" BENCH_PASSWORD="${KALAMDB_PASSWORD:-}" EXTRA_ARGS=() @@ -36,6 +39,10 @@ while [[ $# -gt 0 ]]; do REALTIME_CONVS="$2" shift 2 ;; + --messages-per-minute) + MESSAGES_PER_MINUTE="$2" + shift 2 + ;; --user) BENCH_USER="$2" shift 2 @@ -54,12 +61,18 @@ done export KALAMDB_BENCH_CHAT_MINUTES="$MINUTES" export KALAMDB_BENCH_CHAT_USERS="$USER_COUNT" export KALAMDB_BENCH_CHAT_REALTIME_CONVS="$REALTIME_CONVS" +export KALAMDB_BENCH_CHAT_MESSAGES_PER_MINUTE="$MESSAGES_PER_MINUTE" echo "▸ Running chat_realtime benchmark" echo "▸ URL: $URL" echo "▸ Minutes: $MINUTES" echo "▸ Seeded users: $USER_COUNT" echo "▸ Active conversations: $REALTIME_CONVS" +if [[ "$MESSAGES_PER_MINUTE" == "0" ]]; then + echo "▸ Conversation message rate: idle subscriptions only" +else + echo "▸ Conversation message rate: $MESSAGES_PER_MINUTE messages/min" +fi CMD=(./run-benchmarks.sh --urls "$URL" --bench chat_realtime --iterations 1 --warmup 0) diff --git a/benchv2/src/benchmarks/chat_realtime_bench.rs b/benchv2/src/benchmarks/chat_realtime_bench.rs index 46bea4e6e..fff27830f 100644 --- a/benchv2/src/benchmarks/chat_realtime_bench.rs +++ b/benchv2/src/benchmarks/chat_realtime_bench.rs @@ -26,7 +26,9 @@ pub struct ChatRealtimeBench; const DEFAULT_CHAT_MINUTES: u64 = 5; const DEFAULT_CHAT_USERS: u32 = 1_000; const DEFAULT_CHAT_REALTIME_CONVS: u32 = 100; +const DEFAULT_CHAT_MESSAGES_PER_MINUTE: u32 = 20; const CHAT_USER_PASSWORD: &str = "BenchChatP@ss123"; +const CHAT_MESSAGES_PER_CYCLE: u32 = 2; const CHAT_TYPING_INTERVAL: Duration = Duration::from_secs(2); const CHAT_TYPING_BURSTS: u64 = 3; const CHAT_WORKER_START_STAGGER_MS: u64 = 5; @@ -56,6 +58,7 @@ struct ChatWorkloadSettings { minutes: u64, user_count: u32, realtime_conversations: u32, + messages_per_minute: u32, } impl ChatWorkloadSettings { @@ -66,6 +69,10 @@ impl ChatWorkloadSettings { "KALAMDB_BENCH_CHAT_REALTIME_CONVS", DEFAULT_CHAT_REALTIME_CONVS, )?; + let messages_per_minute = parse_u32_env( + "KALAMDB_BENCH_CHAT_MESSAGES_PER_MINUTE", + DEFAULT_CHAT_MESSAGES_PER_MINUTE, + )?; if minutes == 0 { return Err("KALAMDB_BENCH_CHAT_MINUTES must be greater than zero".to_string()); @@ -83,6 +90,7 @@ impl ChatWorkloadSettings { minutes, user_count, realtime_conversations, + messages_per_minute, }) } @@ -91,12 +99,31 @@ impl ChatWorkloadSettings { minutes: DEFAULT_CHAT_MINUTES, user_count: DEFAULT_CHAT_USERS, realtime_conversations: DEFAULT_CHAT_REALTIME_CONVS, + messages_per_minute: DEFAULT_CHAT_MESSAGES_PER_MINUTE, }) } fn duration(&self) -> Duration { Duration::from_secs(self.minutes.saturating_mul(60)) } + + fn target_cycle_interval(&self) -> Option { + if self.messages_per_minute == 0 { + return None; + } + + Some(Duration::from_secs_f64( + (60.0 * f64::from(CHAT_MESSAGES_PER_CYCLE)) / f64::from(self.messages_per_minute), + )) + } + + fn message_rate_label(&self) -> String { + if self.messages_per_minute == 0 { + "idle subscriptions only".to_string() + } else { + format!("{} messages/min per conversation", self.messages_per_minute) + } + } } #[derive(Default)] @@ -154,18 +181,22 @@ impl Benchmark for ChatRealtimeBench { fn report_description(&self, _config: &Config) -> String { let settings = ChatWorkloadSettings::for_report(); format!( - "Realtime chat scenario for {}m with {} regular users and {} active conversations", - settings.minutes, settings.user_count, settings.realtime_conversations + "Realtime chat scenario for {}m with {} regular users, {} active conversations, and {}", + settings.minutes, + settings.user_count, + settings.realtime_conversations, + settings.message_rate_label() ) } fn report_full_description(&self, _config: &Config) -> String { let settings = ChatWorkloadSettings::for_report(); format!( - "Creates {} regular KalamDB users, then runs {} concurrent chat-session workers for {} minute(s). Each worker logs in as real users, opens a USER-table conversation row for user A, relies on a Rust topic consumer to mirror that conversation into user B's USER table, sends a user A message that the consumer forwards to user B, emits three typing events over six seconds from user B via a STREAM table, sends a user B reply that the consumer forwards back to user A, and keeps user A on live SQL subscriptions for message and typing delivery.", + "Creates {} regular KalamDB users, then runs {} concurrent chat-session workers for {} minute(s). Each worker logs in as real users, opens a USER-table conversation row for user A, keeps user A on live SQL subscriptions for message and typing delivery, and paces each conversation at {}. When message sending is enabled, a Rust topic consumer mirrors the user A conversation row into user B's USER table, forwards a user A message to user B, emits three typing events over six seconds from user B via a STREAM table, and forwards a user B reply back to user A.", settings.user_count, settings.realtime_conversations, settings.minutes, + settings.message_rate_label(), ) } @@ -178,6 +209,10 @@ impl Benchmark for ChatRealtimeBench { "Active Conversations", settings.realtime_conversations.to_string(), ), + BenchmarkDetail::new( + "Conversation Message Rate", + settings.message_rate_label(), + ), BenchmarkDetail::new( "Responder Typing Burst", format!( @@ -354,16 +389,6 @@ impl Benchmark for ChatRealtimeBench { 60_000_000_000 + u64::from(iteration) * 10_000_000, )); - println!( - " Chat workload settings: duration={}m, regular_users={}, target_active_chat_users={}, active_conversations={}, typing_burst={}x{}s", - settings.minutes, - settings.user_count, - target_active_user_count, - settings.realtime_conversations, - CHAT_TYPING_BURSTS, - CHAT_TYPING_INTERVAL.as_secs(), - ); - let prewarmed_active_users = prewarm_user_clients( user_pool.clone(), users.clone(), @@ -384,9 +409,32 @@ impl Benchmark for ChatRealtimeBench { let scenario_started = Instant::now(); let run_deadline = scenario_started + settings.duration(); + let target_cycle_interval = settings.target_cycle_interval(); let delivery_timeout = chat_delivery_wait_timeout(settings.realtime_conversations); let mut handles = Vec::with_capacity(settings.realtime_conversations as usize); + println!( + " Chat workload settings: duration={}m, regular_users={}, target_active_chat_users={}, active_conversations={}, message_rate={}, typing_burst={}x{}s", + settings.minutes, + settings.user_count, + target_active_user_count, + settings.realtime_conversations, + settings.message_rate_label(), + CHAT_TYPING_BURSTS, + CHAT_TYPING_INTERVAL.as_secs(), + ); + + println!( + " Chat workload settings: duration={}m, regular_users={}, target_active_chat_users={}, active_conversations={}, message_rate={}, typing_burst={}x{}s", + settings.minutes, + settings.user_count, + target_active_user_count, + settings.realtime_conversations, + settings.message_rate_label(), + CHAT_TYPING_BURSTS, + CHAT_TYPING_INTERVAL.as_secs(), + ); + for worker_id in 0..settings.realtime_conversations { let namespace = config.namespace.clone(); let worker_stats = stats.clone(); @@ -416,6 +464,7 @@ impl Benchmark for ChatRealtimeBench { worker_conversations, worker_messages, worker_typing, + target_cycle_interval, delivery_timeout, worker_stop.clone(), ) @@ -505,6 +554,7 @@ async fn run_chat_worker( conversation_ids: Arc, message_ids: Arc, typing_ids: Arc, + target_cycle_interval: Option, delivery_timeout: Duration, global_stop: Arc, ) -> Result<(), String> { @@ -556,8 +606,17 @@ async fn run_chat_worker( .create_conversation_timings .record(create_conversation_started.elapsed()); + if target_cycle_interval.is_none() { + while Instant::now() < run_deadline && !global_stop.load(Ordering::Relaxed) { + sleep(Duration::from_secs(1)).await; + } + + return Ok(()); + } + let mut cycle_ordinal = 0_u64; while Instant::now() < run_deadline && !global_stop.load(Ordering::Relaxed) { + let cycle_started = Instant::now(); stats.sessions_started.fetch_add(1, Ordering::Relaxed); cycle_ordinal += 1; @@ -646,6 +705,17 @@ async fn run_chat_worker( } stats.sessions_completed.fetch_add(1, Ordering::Relaxed); + + if let Some(target_cycle_interval) = target_cycle_interval { + let cycle_elapsed = cycle_started.elapsed(); + if cycle_elapsed < target_cycle_interval { + let remaining_run = run_deadline.saturating_duration_since(Instant::now()); + let sleep_for = (target_cycle_interval - cycle_elapsed).min(remaining_run); + if !sleep_for.is_zero() { + sleep(sleep_for).await; + } + } + } } Ok(()) diff --git a/cli/README.md b/cli/README.md index 4caacc002..1b97eed45 100644 --- a/cli/README.md +++ b/cli/README.md @@ -109,11 +109,9 @@ SELECT * FROM app.users; -- Subscribe to changes SUBSCRIBE TO app.messages WHERE user_id = 'user123'; --- Pause subscription -\pause - --- Resume subscription -\continue +-- Or use the interactive CLI aliases +\subscribe SELECT * FROM app.messages WHERE user_id = 'user123'; +\live SELECT * FROM app.messages WHERE user_id = 'user123'; -- Stop subscription (Ctrl+C) ``` @@ -207,14 +205,13 @@ Special commands starting with backslash (`\`): | `\sessions` | Show active pg-extension gRPC sessions | | `\format ` | Set output format | | `\dt` / `\tables` | List tables | -| `\d ` / `\describe
` | Describe a table | +| `\d
` / `\describe
` | Describe a table (`
` or ``) | +| `\as ` | Wrap one statement as `EXECUTE AS USER` | | `\stats` / `\metrics` | Show system stats | | `\health` | Check server health | | `\flush` | Execute STORAGE FLUSH ALL | -| `\pause` | Pause ingestion | -| `\continue` | Resume ingestion | | `\refresh-tables` / `\refresh` | Refresh autocomplete cache | -| `\subscribe ` / `\watch ` | Start live query | +| `\subscribe ` / `\watch ` / `\live ` | Start live query | | `\unsubscribe` / `\unwatch` | Cancel live query | | `\show-credentials` / `\credentials` | Show stored credentials | | `\update-credentials

` | Update stored credentials | diff --git a/cli/src/completer.rs b/cli/src/completer.rs index fc42b7668..ef3171f69 100644 --- a/cli/src/completer.rs +++ b/cli/src/completer.rs @@ -12,6 +12,8 @@ use std::collections::HashMap; use colored::*; use rustyline::completion::{Completer, Pair}; +use crate::parser::META_COMMAND_COMPLETIONS; + pub(crate) const SQL_KEYWORDS: &[&str] = &[ // DML "SELECT", @@ -31,6 +33,8 @@ pub(crate) const SQL_KEYWORDS: &[&str] = &[ "IS", "NULL", "AS", + "EXECUTE", + "USER", "ORDER", "BY", "GROUP", @@ -170,34 +174,10 @@ impl AutoCompleter { pub fn new() -> Self { let keywords = SQL_KEYWORDS.iter().map(|s| s.to_string()).collect::>(); - let meta_commands = vec![ - "\\quit", - "\\q", - "\\help", - "\\?", - "\\history", - "\\h", - "\\sessions", - "\\stats", - "\\metrics", - "\\flush", - "\\health", - "\\pause", - "\\continue", - "\\dt", - "\\tables", - "\\d", - "\\describe", - "\\format", - "\\refresh-tables", - "\\show-credentials", - "\\credentials", - "\\update-credentials", - "\\delete-credentials", - ] - .iter() - .map(|s| s.to_string()) - .collect(); + let meta_commands = META_COMMAND_COMPLETIONS + .iter() + .map(|command| (*command).to_string()) + .collect(); Self { keywords, @@ -498,6 +478,29 @@ mod tests { let completions = completer.get_styled_completions("\\q", line, line.len()); assert!(completions.iter().any(|c| c.replacement == "\\quit")); assert!(completions.iter().any(|c| c.replacement == "\\q")); + + let as_line = "\\a"; + let as_completions = completer.get_styled_completions("\\a", as_line, as_line.len()); + assert!(as_completions.iter().any(|c| c.replacement == "\\as")); + + let live_line = "\\li"; + let live_completions = + completer.get_styled_completions("\\li", live_line, live_line.len()); + assert!(live_completions.iter().any(|c| c.replacement == "\\live")); + } + + #[test] + fn test_execute_as_keyword_completion() { + let completer = AutoCompleter::new(); + + let execute_line = "EXE"; + let execute_completions = + completer.get_styled_completions("EXE", execute_line, execute_line.len()); + assert!(execute_completions.iter().any(|c| c.replacement == "EXECUTE")); + + let user_line = "US"; + let user_completions = completer.get_styled_completions("US", user_line, user_line.len()); + assert!(user_completions.iter().any(|c| c.replacement == "USER")); } #[test] diff --git a/cli/src/connect.rs b/cli/src/connect.rs index 762474adb..de29598ea 100644 --- a/cli/src/connect.rs +++ b/cli/src/connect.rs @@ -36,7 +36,6 @@ fn build_timeouts(cli: &Cli) -> KalamLinkTimeouts { .subscribe_timeout_secs(5) // Keep default for subscribe ack .initial_data_timeout_secs(cli.initial_data_timeout) .idle_timeout_secs(cli.subscription_timeout) // subscription_timeout is the idle timeout - .keepalive_interval_secs(30) // Keep default .build() } @@ -60,8 +59,43 @@ fn is_localhost_url(url: &str) -> bool { normalized_host.parse::().is_ok_and(|ip| ip.is_loopback()) } +fn should_default_to_http(raw: &str) -> bool { + let candidate = format!("http://{}", raw.trim()); + let parsed = match Url::parse(&candidate) { + Ok(url) => url, + Err(_) => return false, + }; + + let host = match parsed.host_str() { + Some(host) => host, + None => return false, + }; + + let normalized_host = host.trim_start_matches('[').trim_end_matches(']'); + normalized_host.eq_ignore_ascii_case("localhost") + || normalized_host.parse::().is_ok_and(|ip| ip.is_loopback()) +} + fn normalize_and_validate_server_url(server_url: &str) -> Result { - let mut parsed = Url::parse(server_url.trim()).map_err(|e| { + let trimmed = server_url.trim(); + if trimmed.is_empty() { + return Err(CLIError::ConfigurationError( + "Server URL must not be empty".to_string(), + )); + } + + let normalized_input = if trimmed.contains("://") { + trimmed.to_string() + } else { + let scheme = if should_default_to_http(trimmed) { + "http" + } else { + "https" + }; + format!("{scheme}://{trimmed}") + }; + + let mut parsed = Url::parse(&normalized_input).map_err(|e| { CLIError::ConfigurationError(format!("Invalid server URL '{}': {}", server_url, e)) })?; @@ -1044,7 +1078,11 @@ pub async fn create_session( #[cfg(test)] mod tests { - use super::{is_localhost_url, normalize_and_validate_server_url}; + use clap::Parser; + + use super::{build_timeouts, is_localhost_url, normalize_and_validate_server_url}; + use crate::args::Cli; + use kalam_client::KalamLinkTimeouts; #[test] fn test_is_localhost_url_accepts_loopback_hosts() { @@ -1072,6 +1110,14 @@ mod tests { normalize_and_validate_server_url("https://db.example.com/base/").unwrap(), "https://db.example.com/base" ); + assert_eq!( + normalize_and_validate_server_url("kalam.masky.app").unwrap(), + "https://kalam.masky.app" + ); + assert_eq!( + normalize_and_validate_server_url("localhost:8080").unwrap(), + "http://localhost:8080" + ); } #[test] @@ -1081,4 +1127,13 @@ mod tests { assert!(normalize_and_validate_server_url("http://localhost:8080?token=secret").is_err()); assert!(normalize_and_validate_server_url("http://localhost:8080/#fragment").is_err()); } + + #[test] + fn test_build_timeouts_preserves_sdk_keepalive_default() { + let cli = Cli::parse_from(["kalam"]); + + let timeouts = build_timeouts(&cli); + + assert_eq!(timeouts.keepalive_interval, KalamLinkTimeouts::default().keepalive_interval); + } } diff --git a/cli/src/formatter.rs b/cli/src/formatter.rs index 197662959..552b94469 100644 --- a/cli/src/formatter.rs +++ b/cli/src/formatter.rs @@ -92,18 +92,40 @@ impl OutputFormatter { return Ok(self.format_error_detail(error)); } + let as_user = Self::query_as_user(response); + match self.format { - OutputFormat::Table => self.format_table(response), + OutputFormat::Table => self.format_table(response, as_user), OutputFormat::Json => self.format_json(response), OutputFormat::Csv => self.format_csv(response), } } + fn query_as_user(response: &QueryResponse) -> Option<&str> { + response + .results + .first() + .and_then(|result| result.as_user.as_deref()) + .filter(|value| !value.trim().is_empty()) + } + + fn format_table_footer(exec_time_ms: f64, as_user: Option<&str>) -> String { + let mut footer = String::new(); + if let Some(as_user) = as_user { + footer.push_str(&format!("As: {as_user}\n")); + } + footer.push_str(&format!("Took: {:.3} ms", exec_time_ms)); + footer + } + /// Format as table - fn format_table(&self, response: &QueryResponse) -> Result { + fn format_table(&self, response: &QueryResponse, as_user: Option<&str>) -> Result { if response.results.is_empty() { let exec_time_ms = response.took.unwrap_or(0.0); - return Ok(format!("Query OK, 0 rows affected\n\nTook: {:.3} ms", exec_time_ms)); + return Ok(format!( + "Query OK, 0 rows affected\n\n{}", + Self::format_table_footer(exec_time_ms, as_user) + )); } let result = &response.results[0]; @@ -114,8 +136,10 @@ impl OutputFormatter { // Format DDL message similar to MySQL/PostgreSQL let row_count = result.row_count; return Ok(format!( - "{}\nQuery OK, {} rows affected\n\nTook: {:.3} ms", - message, row_count, exec_time_ms + "{}\nQuery OK, {} rows affected\n\n{}", + message, + row_count, + Self::format_table_footer(exec_time_ms, as_user) )); } @@ -255,16 +279,18 @@ impl OutputFormatter { output.push_str(&format!("({} {})\n", row_count, row_label)); // Add blank line for psql-style formatting output.push('\n'); - // Display timing in milliseconds like psql - let exec_time_ms = response.took.unwrap_or(0.0); - output.push_str(&format!("Took: {:.3} ms", exec_time_ms)); + output.push_str(&Self::format_table_footer(exec_time_ms, as_user)); Ok(output) } else { // Non-query statement (INSERT, UPDATE, DELETE) let row_count = result.row_count; let exec_time_ms = response.took.unwrap_or(0.0); - Ok(format!("Query OK, {} rows affected\n\nTook: {:.3} ms", row_count, exec_time_ms)) + Ok(format!( + "Query OK, {} rows affected\n\n{}", + row_count, + Self::format_table_footer(exec_time_ms, as_user) + )) } } @@ -519,7 +545,8 @@ impl OutputFormatter { #[cfg(test)] mod tests { - use kalam_client::TimestampFormat; + use kalam_client::{query::models::ResponseStatus, QueryResult, SchemaField, TimestampFormat}; + use serde_json::json; use super::*; @@ -604,4 +631,35 @@ mod tests { assert_eq!(formatter.raw_timestamp_millis(1735689600000000), 1735689600000); assert_eq!(formatter.raw_timestamp_millis(1735689600000000000), 1735689600000); } + + #[test] + fn test_format_table_footer_includes_effective_user() { + let formatter = OutputFormatter::new( + OutputFormat::Table, + false, + TimestampFormatter::new(TimestampFormat::Iso8601), + ); + + let response = QueryResponse { + status: ResponseStatus::Success, + results: vec![QueryResult { + schema: vec![SchemaField { + name: "name".to_string(), + data_type: KalamDataType::Text, + index: 0, + flags: None, + }], + rows: Some(vec![vec![json!("Alice").into()]]), + named_rows: None, + row_count: 1, + message: None, + as_user: Some("alice".to_string()), + }], + took: Some(2.146), + error: None, + }; + + let output = formatter.format_response(&response).unwrap(); + assert!(output.contains("(1 row)\n\nAs: alice\nTook: 2.146 ms")); + } } diff --git a/cli/src/parser.rs b/cli/src/parser.rs index 93af863c3..59ab8425f 100644 --- a/cli/src/parser.rs +++ b/cli/src/parser.rs @@ -12,6 +12,12 @@ pub enum Command { /// SQL statement Sql(String), + /// Execute a single SQL statement as another user + ExecuteAs { + user: String, + sql: String, + }, + /// Meta-commands (backslash commands) Quit, Help, @@ -35,8 +41,6 @@ pub enum Command { }, ClusterRebalance, Health, - Pause, - Continue, ListTables, Describe(String), SetFormat(String), @@ -66,6 +70,41 @@ pub enum Command { Unknown(String), } +pub(crate) const META_COMMAND_COMPLETIONS: &[&str] = &[ + "\\quit", + "\\q", + "\\help", + "\\?", + "\\info", + "\\session", + "\\history", + "\\h", + "\\sessions", + "\\stats", + "\\metrics", + "\\flush", + "\\health", + "\\dt", + "\\tables", + "\\d", + "\\describe", + "\\as", + "\\format", + "\\refresh-tables", + "\\refresh", + "\\subscribe", + "\\watch", + "\\live", + "\\unsubscribe", + "\\unwatch", + "\\show-credentials", + "\\credentials", + "\\update-credentials", + "\\delete-credentials", + "\\cluster", + "\\consume", +]; + /// Command parser pub struct CommandParser; @@ -94,6 +133,10 @@ impl CommandParser { /// Parse meta-commands (backslash commands) fn parse_meta_command(&self, line: &str) -> Result { + if line.starts_with("\\as") { + return Self::parse_execute_as_command(line); + } + let parts: Vec<&str> = line.split_whitespace().collect(); if parts.is_empty() { return Err(CLIError::ParseError("Invalid command".into())); @@ -214,8 +257,6 @@ impl CommandParser { } }, "\\health" => Ok(Command::Health), - "\\pause" => Ok(Command::Pause), - "\\continue" => Ok(Command::Continue), "\\dt" | "\\tables" => Ok(Command::ListTables), "\\d" | "\\describe" => { if args.is_empty() { @@ -231,7 +272,7 @@ impl CommandParser { Ok(Command::SetFormat(args[0].to_string())) } }, - "\\subscribe" | "\\watch" => { + "\\subscribe" | "\\watch" | "\\live" => { if args.is_empty() { Err(CLIError::ParseError("\\subscribe requires a SQL query".into())) } else { @@ -341,6 +382,33 @@ impl CommandParser { _ => Ok(Command::Unknown(command.to_string())), } } + + fn parse_execute_as_command(line: &str) -> Result { + let remainder = line["\\as".len()..].trim_start(); + if remainder.is_empty() { + return Err(CLIError::ParseError( + "\\as requires a target user and SQL query".into(), + )); + } + + let user_end = remainder.find(char::is_whitespace).ok_or_else(|| { + CLIError::ParseError("\\as requires a target user and SQL query".into()) + })?; + + let user = remainder[..user_end].trim(); + let sql = remainder[user_end..].trim_start(); + + if user.is_empty() || sql.is_empty() { + return Err(CLIError::ParseError( + "\\as requires a target user and SQL query".into(), + )); + } + + Ok(Command::ExecuteAs { + user: user.to_string(), + sql: sql.to_string(), + }) + } } impl Default for CommandParser { @@ -381,6 +449,26 @@ mod tests { assert_eq!(cmd, Command::Describe("users".to_string())); } + #[test] + fn test_parse_live_alias() { + let parser = CommandParser::new(); + let cmd = parser.parse("\\live SELECT * FROM chat.messages").unwrap(); + assert_eq!(cmd, Command::Subscribe("SELECT * FROM chat.messages".to_string())); + } + + #[test] + fn test_parse_execute_as_shortcut() { + let parser = CommandParser::new(); + let cmd = parser.parse("\\as alice SELECT * FROM chat.messages WHERE body = 'hi there'"); + assert_eq!( + cmd.unwrap(), + Command::ExecuteAs { + user: "alice".to_string(), + sql: "SELECT * FROM chat.messages WHERE body = 'hi there'".to_string(), + } + ); + } + #[test] fn test_parse_unknown() { let parser = CommandParser::new(); diff --git a/cli/src/session.rs b/cli/src/session.rs index 3982e76cc..13236301b 100644 --- a/cli/src/session.rs +++ b/cli/src/session.rs @@ -133,10 +133,7 @@ struct ClusterNodeDisplay { api_addr: String, is_self: bool, is_leader: bool, - hostname: Option, - memory_usage_mb: Option, - cpu_usage_percent: Option, - uptime_human: Option, + version: Option, } /// Cluster information for CLI display @@ -1059,7 +1056,7 @@ impl CLISession { // NOTE: Do NOT update server_host here — keep it as the address the user connected to, // not the server's internal listening address (e.g. 0.0.0.0:8080). if let Some(cluster_info) = self.fetch_cluster_info().await { - self.cluster_name = Some(cluster_info.cluster_name); + self.adopt_cluster_metadata(&cluster_info); } // Connection and auth successful - print welcome banner @@ -1332,6 +1329,24 @@ impl CLISession { println!(); } + fn adopt_cluster_metadata(&mut self, info: &ClusterInfoDisplay) { + self.cluster_name = Some(info.cluster_name.clone()); + + if self.server_version.is_some() { + return; + } + + let version = info + .current_node + .as_ref() + .and_then(|node| node.version.clone()) + .or_else(|| info.nodes.iter().find(|node| node.is_self).and_then(|node| node.version.clone())); + + if let Some(version) = version.and_then(Self::normalize_server_field) { + self.server_version = Some(version); + } + } + /// Fetch namespaces, table names, and column names from server and update completer async fn refresh_tables(&mut self, completer: &mut AutoCompleter) -> Result<()> { // Query namespaces first from system.namespaces @@ -2378,53 +2393,14 @@ impl CLISession { ); } - fn render_cluster_health_fallback(&self, info: &ClusterInfoDisplay, note: Option<&str>) { - let active_nodes = info - .nodes - .iter() - .filter(|node| node.status.eq_ignore_ascii_case("active")) - .count(); - let offline_nodes = info - .nodes - .iter() - .filter(|node| node.status.eq_ignore_ascii_case("offline")) - .count(); - - println!("{} Cluster health: {}", "✓".green(), "reachable".green()); - println!( - " Cluster: {} | Nodes: {} total, {} active, {} offline", - info.cluster_name.as_str().cyan(), - info.nodes.len(), - active_nodes, - offline_nodes - ); - if let Some(message) = note { - println!(" {}", message.yellow()); - } - println!(); - println!("{}", "Nodes:".yellow().bold()); - - for node in &info.nodes { - let self_marker = if node.is_self { " (connected)" } else { "" }; - let leader_marker = if node.is_leader { " [LEADER]" } else { "" }; - let hostname = node.hostname.as_deref().unwrap_or(node.api_addr.as_str()); - println!( - " Node {}: {} | {} | {}{}{}", - node.node_id, - node.role, - node.status, - node.api_addr, - leader_marker.yellow(), - self_marker.cyan() - ); - println!( - " host={} | {} | {} | {}", - hostname, - Self::format_cluster_memory(node.memory_usage_mb), - Self::format_cluster_cpu(node.cpu_usage_percent), - Self::format_cluster_uptime(node.uptime_human.as_deref()) - ); - } + fn public_probe_client(&self) -> Result { + Ok(KalamLinkClient::builder() + .base_url(&self.server_url) + .timeout(self.timeouts.receive_timeout) + .max_retries(self.config.resolved_server().max_retries) + .timeouts(self.timeouts.clone()) + .connection_options(self.config.to_connection_options()) + .build()?) } /// Fetch cluster information from system.cluster @@ -2434,8 +2410,7 @@ impl CLISession { .client .execute_query( "SELECT cluster_id, node_id, role, status, api_addr, is_self, is_leader, \ - hostname, memory_usage_mb, cpu_usage_percent, uptime_human FROM system.cluster \ - ORDER BY is_leader DESC, node_id ASC", + version FROM system.cluster ORDER BY is_leader DESC, node_id ASC", None, None, None, @@ -2454,8 +2429,8 @@ impl CLISession { if let Some(rows) = &query_result.rows { for row in rows { // row fields: cluster_id, node_id, role, status, api_addr, is_self, - // is_leader, hostname, memory_usage_mb, cpu_usage_percent, uptime_human - if row.len() >= 11 { + // is_leader, version + if row.len() >= 8 { // Extract cluster_id from first row only if cluster_name.is_empty() { cluster_name = @@ -2467,10 +2442,7 @@ impl CLISession { let api_addr = row[4].as_str().unwrap_or("").to_string(); let is_self = row[5].as_bool().unwrap_or(false); let is_leader = row[6].as_bool().unwrap_or(false); - let hostname = row[7].as_str().map(ToString::to_string); - let memory_usage_mb = row[8].as_u64(); - let cpu_usage_percent = row[9].as_f64().map(|cpu| cpu as f32); - let uptime_human = row[10].as_str().map(ToString::to_string); + let version = row[7].as_str().map(ToString::to_string); // Check if this looks like cluster mode (role is leader/follower) if matches!( @@ -2487,10 +2459,7 @@ impl CLISession { api_addr, is_self, is_leader, - hostname, - memory_usage_mb, - cpu_usage_percent, - uptime_human, + version, }; if is_self { @@ -2520,7 +2489,8 @@ impl CLISession { /// Check server health and refresh cached server metadata pub async fn health_check(&mut self) -> Result<()> { - let basic_health = self.client.health_check().await; + let probe_client = self.public_probe_client()?; + let basic_health = probe_client.health_check().await; match &basic_health { Ok(health) => { @@ -2533,7 +2503,7 @@ impl CLISession { Err(KalamLinkError::ServerError { status_code: 403, .. }) => { - // Localhost-only endpoint; fall back to authenticated SQL-based cluster info. + self.connected = true; }, Err(_) => { self.connected = false; @@ -2543,58 +2513,43 @@ impl CLISession { }, } - let cluster_info = self.fetch_cluster_info().await; - if let Some(ref info) = cluster_info { - self.connected = true; - self.cluster_name = Some(info.cluster_name.clone()); - - if info.is_cluster_mode { - match self.client.cluster_health_check().await { - Ok(cluster_health) => { - self.server_version = - Self::normalize_server_field(cluster_health.version.clone()); - self.server_build_date = - Self::normalize_server_field(cluster_health.build_date.clone()); - self.render_cluster_health_response(&cluster_health); - return Ok(()); - }, - Err(KalamLinkError::ServerError { - status_code: 403, .. - }) => { - self.render_cluster_health_fallback( - info, - Some("Cluster health endpoint is localhost-only; using system.cluster"), - ); - return Ok(()); - }, - Err(_) => { - self.render_cluster_health_fallback( - info, - Some("Cluster health endpoint unavailable; using system.cluster"), - ); - return Ok(()); - }, - } - } - - match basic_health { + match probe_client.cluster_health_check().await { + Ok(cluster_health) => { + self.connected = true; + self.server_version = Self::normalize_server_field(cluster_health.version.clone()); + self.server_build_date = + Self::normalize_server_field(cluster_health.build_date.clone()); + self.render_cluster_health_response(&cluster_health); + return Ok(()); + }, + Err(KalamLinkError::ServerError { + status_code: 403, .. + }) => match basic_health { Ok(_) => { println!("✓ Server is healthy"); + println!(" {}", "Cluster health endpoint is restricted to localhost".yellow()); return Ok(()); }, Err(KalamLinkError::ServerError { status_code: 403, .. }) => { - println!("{} Server is reachable", "✓".green()); - println!(" {}", "Health endpoint is restricted to localhost".yellow()); + println!("{}", "Health endpoints are localhost-only for this connection".yellow()); + println!(" {}", "No authenticated SQL fallback was used.".dimmed()); + println!( + " {}", + "Run SELECT * FROM system.cluster manually if you want authenticated cluster state." + .dimmed() + ); return Ok(()); }, - Err(_) => { - println!("{} Server is reachable", "✓".green()); - println!(" {}", "Using authenticated SQL fallback".yellow()); + Err(e) => return Err(e.into()), + }, + Err(_) => { + if basic_health.is_ok() { + println!("✓ Server is healthy"); return Ok(()); - }, - } + } + }, } match basic_health { @@ -3122,10 +3077,27 @@ mod tests { use super::*; use crate::credentials::FileCredentialStore; - #[derive(Debug, Default)] + #[derive(Debug)] struct TestServerState { sql_authorization_headers: Vec, refresh_authorization_headers: Vec, + health_authorization_headers: Vec, + cluster_health_authorization_headers: Vec, + health_status: u16, + cluster_health_status: u16, + } + + impl Default for TestServerState { + fn default() -> Self { + Self { + sql_authorization_headers: Vec::new(), + refresh_authorization_headers: Vec::new(), + health_authorization_headers: Vec::new(), + cluster_health_authorization_headers: Vec::new(), + health_status: 200, + cluster_health_status: 404, + } + } } struct TestServer { @@ -3176,16 +3148,86 @@ mod tests { let authorization = request.headers.get("authorization").cloned(); let (status_line, body) = match request.path.as_str() { - "/v1/api/healthcheck" => ( - "HTTP/1.1 200 OK", - json!({ - "status": "healthy", - "version": "test", - "api_version": "v1", - "build_date": null - }) - .to_string(), - ), + "/v1/api/healthcheck" => { + let status = { + let mut guard = state.lock().await; + guard + .health_authorization_headers + .push(authorization.clone().unwrap_or_default()); + guard.health_status + }; + + match status { + 200 => ( + "HTTP/1.1 200 OK", + json!({ + "status": "healthy", + "version": "test", + "api_version": "v1", + "build_date": null + }) + .to_string(), + ), + 403 => ( + "HTTP/1.1 403 Forbidden", + json!({ "message": "localhost only" }).to_string(), + ), + code => ( + "HTTP/1.1 500 Internal Server Error", + json!({ "message": format!("unexpected status {code}") }).to_string(), + ), + } + }, + "/v1/api/cluster/health" => { + let status = { + let mut guard = state.lock().await; + guard + .cluster_health_authorization_headers + .push(authorization.clone().unwrap_or_default()); + guard.cluster_health_status + }; + + match status { + 200 => ( + "HTTP/1.1 200 OK", + json!({ + "status": "healthy", + "version": "test", + "build_date": "2026-05-04T00:00:00Z", + "is_cluster_mode": true, + "cluster_id": "test-cluster", + "node_id": 0, + "is_leader": true, + "total_groups": 1, + "groups_leading": 1, + "current_term": 1, + "last_applied": 1, + "millis_since_quorum_ack": 0, + "nodes": [{ + "node_id": 0, + "role": "leader", + "status": "active", + "api_addr": "http://127.0.0.1:8080", + "is_self": true, + "is_leader": true, + "replication_lag": null, + "catchup_progress_pct": null, + "hostname": "localhost", + "memory_usage_mb": 64, + "cpu_usage_percent": 0.25, + "uptime_seconds": 42, + "uptime_human": "42s" + }] + }) + .to_string(), + ), + 403 => ( + "HTTP/1.1 403 Forbidden", + json!({ "message": "localhost only" }).to_string(), + ), + _ => ("HTTP/1.1 404 Not Found", "Not found".to_string()), + } + }, "/v1/api/sql" => { if let Some(header) = authorization.clone() { state.lock().await.sql_authorization_headers.push(header.clone()); @@ -3356,6 +3398,56 @@ mod tests { assert!(options.is_some()); } + #[test] + fn test_parse_describe_target_supports_namespace_and_semicolon() { + let (namespace, table_name) = CLISession::parse_describe_target("chat.messages;").unwrap(); + assert_eq!(namespace.as_deref(), Some("chat")); + assert_eq!(table_name, "messages"); + + let (namespace, table_name) = + CLISession::parse_describe_target("\"chat\".\"message logs\"").unwrap(); + assert_eq!(namespace.as_deref(), Some("chat")); + assert_eq!(table_name, "message logs"); + } + + #[tokio::test] + #[timeout(5000)] + async fn test_health_check_does_not_fall_back_to_authenticated_sql() { + let server = TestServer::spawn().await; + { + let mut state = server.state.lock().await; + state.health_status = 403; + state.cluster_health_status = 403; + } + + let mut session = CLISession::with_auth_and_instance( + server.base_url.clone(), + AuthProvider::jwt_token("fresh-token".to_string()), + OutputFormat::Table, + false, + None, + None, + Some("admin".to_string()), + None, + true, + None, + None, + None, + CLIConfiguration::default(), + crate::config::default_config_path(), + false, + ) + .await + .expect("create session"); + + session.health_check().await.expect("health check should succeed"); + + let state = server.state.lock().await; + assert!(state.sql_authorization_headers.is_empty()); + assert_eq!(state.health_authorization_headers, vec![String::new(), String::new()]); + assert_eq!(state.cluster_health_authorization_headers, vec![String::new()]); + } + #[tokio::test] #[timeout(5000)] async fn test_build_auth_refresher_refreshes_and_persists_tokens() { diff --git a/cli/src/session/cluster/actions.rs b/cli/src/session/cluster/actions.rs index 988ce5f60..51a67125c 100644 --- a/cli/src/session/cluster/actions.rs +++ b/cli/src/session/cluster/actions.rs @@ -1,7 +1,3 @@ -use std::collections::HashMap; - -use kalam_client::KalamCellValue; - use super::{cell_bool, cell_text, cell_u64, CLISession}; use crate::{CLIError, Result}; @@ -258,6 +254,10 @@ fn bytes_to_mb(bytes: u64) -> f64 { #[cfg(test)] mod tests { + use std::collections::HashMap; + + use kalam_client::KalamCellValue; + use super::*; #[test] diff --git a/cli/src/session/commands.rs b/cli/src/session/commands.rs index f0e18e5f9..ffe567715 100644 --- a/cli/src/session/commands.rs +++ b/cli/src/session/commands.rs @@ -72,20 +72,6 @@ impl CLISession { Ok(_) => {}, Err(e) => eprintln!("Health check failed: {}", e), }, - Command::Pause => { - println!("Pausing ingestion..."); - match self.execute("PAUSE").await { - Ok(_) => println!("Ingestion paused"), - Err(e) => eprintln!("Pause failed: {}", e), - } - }, - Command::Continue => { - println!("Resuming ingestion..."); - match self.execute("CONTINUE").await { - Ok(_) => println!("Ingestion resumed"), - Err(e) => eprintln!("Resume failed: {}", e), - } - }, Command::ListTables => { self.execute( "SELECT namespace_id AS namespace, table_name, table_type FROM system.tables \ @@ -94,11 +80,11 @@ impl CLISession { .await?; }, Command::Describe(table) => { - let query = format!( - "SELECT * FROM information_schema.columns WHERE table_name = '{}' ORDER BY \ - ordinal_position", - table - ); + let query = Self::build_describe_query(&table)?; + self.execute(&query).await?; + }, + Command::ExecuteAs { user, sql } => { + let query = Self::build_execute_as_query(&user, &sql)?; self.execute(&query).await?; }, Command::SetFormat(format) => match format.to_lowercase().as_str() { @@ -135,7 +121,6 @@ impl CLISession { println!("No active subscription to cancel"); }, Command::RefreshTables => { - // This is handled in run_interactive, shouldn't reach here println!("Table names refreshed"); }, Command::ShowCredentials => { @@ -164,8 +149,6 @@ impl CLISession { .await?; }, Command::History => { - // Handled in run_interactive() where we have access to history - // This should not be reached eprintln!("History command should be handled in interactive mode"); }, Command::Consume { @@ -185,166 +168,250 @@ impl CLISession { Ok(()) } - /// Show help message (styled, sectioned) + fn build_describe_query(target: &str) -> Result { + let (namespace, table_name) = Self::parse_describe_target(target)?; + let columns = "table_schema AS namespace, table_name, column_name, data_type, \ + is_nullable, column_default, ordinal_position AS position"; + let escaped_table = Self::escape_sql_literal(&table_name); + + if let Some(namespace) = namespace { + Ok(format!( + "SELECT {columns} FROM information_schema.columns WHERE table_schema = '{namespace}' \ + AND table_name = '{table_name}' ORDER BY ordinal_position", + namespace = Self::escape_sql_literal(&namespace), + table_name = escaped_table, + )) + } else { + Ok(format!( + "SELECT {columns} FROM information_schema.columns WHERE table_name = '{table_name}' \ + ORDER BY table_schema, ordinal_position", + table_name = escaped_table, + )) + } + } + + fn build_execute_as_query(user: &str, sql: &str) -> Result { + let normalized_user = Self::normalize_execute_as_user(user)?; + let inner_sql = sql.trim().trim_end_matches(';').trim_end(); + + if inner_sql.is_empty() { + return Err(CLIError::ParseError( + "\\as requires a non-empty SQL statement".to_string(), + )); + } + + Ok(format!( + "EXECUTE AS USER '{}' ({})", + Self::escape_sql_literal(&normalized_user), + inner_sql, + )) + } + + fn normalize_execute_as_user(user: &str) -> Result { + let trimmed = user.trim(); + if trimmed.is_empty() { + return Err(CLIError::ParseError( + "\\as requires a target user".to_string(), + )); + } + + let normalized = if trimmed.len() >= 2 + && ((trimmed.starts_with('\'') && trimmed.ends_with('\'')) + || (trimmed.starts_with('"') && trimmed.ends_with('"'))) + { + &trimmed[1..trimmed.len() - 1] + } else { + trimmed + }; + + if normalized.is_empty() { + return Err(CLIError::ParseError( + "\\as requires a target user".to_string(), + )); + } + + Ok(normalized.to_string()) + } + + pub(super) fn parse_describe_target(target: &str) -> Result<(Option, String)> { + let trimmed = target.trim().trim_end_matches(';').trim(); + if trimmed.is_empty() { + return Err(CLIError::ParseError( + "\\describe requires a table name".to_string(), + )); + } + + let parts = Self::split_identifier_parts(trimmed)?; + match parts.as_slice() { + [table_name] => Ok((None, table_name.clone())), + [namespace, table_name] => Ok((Some(namespace.clone()), table_name.clone())), + _ => Err(CLIError::ParseError( + "\\describe expects

or ".to_string(), + )), + } + } + + fn split_identifier_parts(target: &str) -> Result> { + let mut parts = Vec::new(); + let mut current = String::new(); + let mut chars = target.chars().peekable(); + let mut in_quotes = false; + + while let Some(ch) = chars.next() { + match ch { + '"' => { + if in_quotes && chars.peek() == Some(&'"') { + current.push('"'); + chars.next(); + } else { + in_quotes = !in_quotes; + } + }, + '.' if !in_quotes => { + let part = current.trim(); + if part.is_empty() { + return Err(CLIError::ParseError( + "\\describe received an invalid identifier".to_string(), + )); + } + parts.push(part.to_string()); + current.clear(); + }, + _ => current.push(ch), + } + } + + if in_quotes { + return Err(CLIError::ParseError( + "\\describe received an unterminated quoted identifier".to_string(), + )); + } + + let tail = current.trim(); + if tail.is_empty() { + return Err(CLIError::ParseError( + "\\describe received an invalid identifier".to_string(), + )); + } + parts.push(tail.to_string()); + + Ok(parts) + } + + fn escape_sql_literal(value: &str) -> String { + value.replace('\'', "''") + } + + fn print_help_section(title: &str) { + println!("{}", title.yellow().bold()); + } + + fn print_help_row(command: &str, description: &str) { + println!(" {} {}", format!("{command:<38}").cyan(), description); + } + + fn print_help_example(example: &str) { + println!(" {}", example.green()); + } + fn show_help(&self) { println!(); - println!( - "{}", - "╔═══════════════════════════════════════════════════════════╗" - .bright_blue() - .bold() - ); - println!( - "{}{}{}", - "║ ".bright_blue().bold(), - "Commands & Shortcuts".white().bold(), - " ║" - .to_string() - .bright_blue() - .bold() - ); - println!( - "{}", - "╠═══════════════════════════════════════════════════════════╣" - .bright_blue() - .bold() - ); + println!("{}", "Kalam CLI Help".bright_blue().bold()); + println!(); - // Basics - println!("{}", "║ Basics".bright_blue().bold()); - println!("║ • Write SQL; end with ';' to run"); - println!( - "║ • Autocomplete: keywords, namespaces, tables, columns {}", - "(Tab)".dimmed() - ); - println!("║ • Inline hints and SQL highlighting enabled"); + Self::print_help_section("Basics"); + println!(" Write SQL and end with ';' to run it"); + println!(" Press Tab for SQL, table, namespace, and command completion"); + println!(" Press Up on an empty prompt to open command history"); + println!(); - // Meta-commands (two columns) - println!( - "{}", - "╠───────────────────────────────────────────────────────────╣" - .bright_blue() - .bold() - ); - println!("{}", "║ Meta-Commands".bright_blue().bold()); - let left = [ + Self::print_help_section("Meta Commands"); + for (command, description) in [ ("\\help, \\?", "Show this help"), ("\\quit, \\q", "Exit CLI"), - ("\\info, \\session", "Session info"), - ("\\sessions", "PG gRPC sessions"), - ("\\stats, \\metrics", "System stats"), - ("\\health", "Health check"), - ("\\format ", "table|json|csv"), - ("\\history, \\h", "Browse history"), - ("\\refresh-tables, \\refresh", "Refresh autocomplete"), - ]; - let right = [ + ("\\info, \\session", "Show session details"), + ("\\history, \\h", "Browse command history"), + ("\\health", "Run public health probes"), ("\\dt, \\tables", "List tables"), - ("\\d, \\describe
", "Describe table"), + ("\\d, \\describe
", "Describe a table"), + ("\\as ", "Wrap a statement as EXECUTE AS USER"), + ("\\format ", "Change output format"), + ("\\refresh-tables, \\refresh", "Refresh autocomplete caches"), + ("\\stats, \\metrics", "Show system stats"), + ("\\sessions", "Show active sessions"), ("\\flush", "Run STORAGE FLUSH ALL"), - ("\\pause", "Pause ingestion"), - ("\\continue", "Resume ingestion"), - ("\\subscribe, \\watch ", "Start live query"), - ("\\unsubscribe, \\unwatch", "Stop live query"), - ("\\consume ", "Consume topic messages"), ("\\cluster ", "Cluster operations"), - ]; - for i in 0..left.len().max(right.len()) { - let l = left - .get(i) - .map(|(a, b)| format!("{:<28} {:<18}", a.cyan(), b)) - .unwrap_or_default(); - let r = right - .get(i) - .map(|(a, b)| format!("{:<28} {:<18}", a.cyan(), b)) - .unwrap_or_default(); - println!("║ {:<47}{:<47} ║", l, r); + ("\\consume ", "Consume topic messages"), + ] { + Self::print_help_row(command, description); } + println!(); - // Cluster Commands - println!( - "{}", - "╠───────────────────────────────────────────────────────────╣" - .bright_blue() - .bold() - ); - println!("{}", "║ Cluster Commands".bright_blue().bold()); - println!("║ {:<48} Trigger snapshot", "\\cluster snapshot".cyan()); - println!("║ {:<48} Purge logs up to index", "\\cluster purge --upto ".cyan()); - println!("║ {:<48} Trigger cluster election", "\\cluster trigger-election".cyan()); - println!( - "║ {:<48} Transfer cluster leadership", - "\\cluster transfer-leader ".cyan() - ); - println!("║ {:<48} Rebalance data leaders", "\\cluster rebalance".cyan()); - println!("║ {:<48} Leader stepdown", "\\cluster stepdown".cyan()); - println!("║ {:<48} Clear old snapshots", "\\cluster clear".cyan()); - println!("║ {:<48} List cluster nodes", "\\cluster list".cyan()); - println!("║ {:<48} List all raft groups", "\\cluster list groups".cyan()); - println!( - "║ {:<48} Join node at runtime", - "\\cluster join ".cyan() - ); - println!( - "║ {:<48} Live per-node stats", - "\\subscribe SELECT * FROM system.cluster".cyan() - ); + Self::print_help_section("Live Queries"); + for (command, description) in [ + ("\\subscribe ", "Alias of \\subscribe"), + ("\\live
`, `\describe
` | Describe table | +| `\as ` | Wrap one statement as `EXECUTE AS USER` | | `\stats`, `\metrics` | Show `system.stats` | | `\health` | Server healthcheck | -| `\pause` / `\continue` | Pause/resume ingestion | +| `\flush` | Run `STORAGE FLUSH ALL` | | `\format table|json|csv` | Change output format | -| `\subscribe `, `\watch ` | Start WebSocket live subscription | +| `\subscribe `, `\watch `, `\live ` | Start live subscription | | `\unsubscribe`, `\unwatch` | No-op (prints “No active subscription to cancel”) | | `\cluster ...` | Cluster commands (see below) | | `\refresh-tables`, `\refresh` | Refresh autocomplete metadata | +| `\sessions` | Show active sessions | +| `\consume ...` | Consume topic messages | | `\show-credentials`, `\credentials` | Show stored credentials | | `\update-credentials

` | Update stored credentials | | `\delete-credentials` | Delete stored credentials | @@ -367,10 +371,11 @@ kalam --no-color -c "SELECT * FROM users" > output.txt ### 5. Timing Information -Execution time is displayed for all queries: +Execution metadata is displayed for all queries: ``` (10 rows) +As: root Took: 245.123 ms ``` diff --git a/docs/getting-started/quick-start.md b/docs/getting-started/quick-start.md index 0553ea03c..bbd52c0bf 100644 --- a/docs/getting-started/quick-start.md +++ b/docs/getting-started/quick-start.md @@ -79,10 +79,10 @@ JSON ### Optional: `EXECUTE AS USER` -Use wrapper syntax only. Ordinary USER-table reads stay scoped to the -authenticated user; explicit `EXECUTE AS USER` follows the role hierarchy: -system can target any role, DBA can target DBA/service/user, service can target -service/user, and regular users can only target themselves. +Use wrapper syntax only. Ordinary USER-table and STREAM-table access stays +scoped to the authenticated user; explicit `EXECUTE AS USER` follows the role +hierarchy: system can target any role, DBA can target DBA/service/user, +service can target service/user, and regular users can only target themselves. ```bash curl -u root: -X POST http://127.0.0.1:8080/v1/api/sql \ diff --git a/docs/reference/sql.md b/docs/reference/sql.md index 93ae57066..245b66afc 100644 --- a/docs/reference/sql.md +++ b/docs/reference/sql.md @@ -195,9 +195,9 @@ FROM [.] ## Execute As User -`EXECUTE AS USER` syntax is wrapper-only. It switches USER-table execution to a -target user ID only when the authenticated actor role is allowed to target that -ID's cached role class. +`EXECUTE AS USER` syntax is wrapper-only. It switches USER-table or +STREAM-table execution to a target user ID only when the authenticated actor +role is allowed to target that ID's cached role class. ```sql EXECUTE AS USER '' ( @@ -221,7 +221,7 @@ Rules: 4. DBA users may target dba, service, and user accounts. 5. Service users may target service and user accounts. 6. Regular users may only use self-targeted `EXECUTE AS USER` as a no-op identity boundary. -7. The wrapper is valid only for USER tables; shared tables use their table policy directly. +7. The wrapper is valid for USER and STREAM tables; shared tables use their table policy directly. 8. Target role checks are hot-path cached: service, DBA, and system user IDs are tracked in memory from `system.users`; soft-deleted privileged IDs stay classified by their persisted role, and target IDs not present in that privileged cache are treated as regular users. 9. Legacy inline `... AS USER 'name'` syntax is not supported. diff --git a/examples/chat-with-ai/package-lock.json b/examples/chat-with-ai/package-lock.json index 390a48396..1c36c9805 100644 --- a/examples/chat-with-ai/package-lock.json +++ b/examples/chat-with-ai/package-lock.json @@ -34,7 +34,7 @@ }, "../../link/sdks/typescript/client": { "name": "@kalamdb/client", - "version": "0.4.2-rc.3", + "version": "0.4.3-rc.4", "license": "Apache-2.0", "dependencies": { "ws": "^8.20.0" @@ -49,7 +49,7 @@ }, "../../link/sdks/typescript/consumer": { "name": "@kalamdb/consumer", - "version": "0.4.2-rc.3", + "version": "0.4.3-rc.4", "license": "Apache-2.0", "devDependencies": { "@types/node": "^25.5.0", @@ -64,13 +64,14 @@ }, "../../link/sdks/typescript/orm": { "name": "@kalamdb/orm", - "version": "0.4.2-rc.3", + "version": "0.4.3-rc.5", "license": "Apache-2.0", "bin": { "kalamdb-orm": "dist/cli.js" }, "devDependencies": { "@kalamdb/client": "file:../client", + "@kalamdb/consumer": "file:../consumer", "@types/node": "^25.5.2", "drizzle-orm": "^0.41.0", "typescript": "^5.8.0" diff --git a/examples/chat-with-ai/package.json b/examples/chat-with-ai/package.json index 4c0edb00d..857c2575f 100644 --- a/examples/chat-with-ai/package.json +++ b/examples/chat-with-ai/package.json @@ -14,15 +14,15 @@ "preview": "vite preview", "setup": "bash setup.sh", "preagent": "bash scripts/ensure-sdk.sh", - "agent": "tsx src/agent.ts", + "agent": "NODE_OPTIONS=--preserve-symlinks tsx src/agent.ts", "pretest": "bash scripts/ensure-sdk.sh", - "test": "tsc -p tsconfig.test.json && tsx --test tests/agent.test.ts && playwright test" + "test": "tsc -p tsconfig.test.json && NODE_OPTIONS=--preserve-symlinks tsx --test tests/agent.test.ts && playwright test" }, "dependencies": { "@kalamdb/client": "file:../../link/sdks/typescript/client", "@kalamdb/consumer": "file:../../link/sdks/typescript/consumer", "@kalamdb/orm": "file:../../link/sdks/typescript/orm", - "dotenv": "^17.4.1", + "dotenv": "^17.4.2", "drizzle-orm": "^0.41.0", "react": "^19.2.4", "react-dom": "^19.2.4" diff --git a/examples/chat-with-ai/scripts/generate-schema.sh b/examples/chat-with-ai/scripts/generate-schema.sh index 295026337..8714e7de0 100644 --- a/examples/chat-with-ai/scripts/generate-schema.sh +++ b/examples/chat-with-ai/scripts/generate-schema.sh @@ -4,7 +4,8 @@ set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" PROJECT_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" -ORM_CLI="$PROJECT_DIR/../../link/sdks/typescript/orm/dist/cli.js" +ORM_CLI="$PROJECT_DIR/node_modules/@kalamdb/orm/dist/cli.js" +NODE_FLAGS=(--preserve-symlinks --preserve-symlinks-main) if [ -f "$PROJECT_DIR/.env.local" ]; then set -a @@ -13,7 +14,7 @@ if [ -f "$PROJECT_DIR/.env.local" ]; then set +a fi -node "$ORM_CLI" \ +node "${NODE_FLAGS[@]}" "$ORM_CLI" \ --url "${KALAMDB_URL:-http://127.0.0.1:8080}" \ --user "${KALAMDB_USER:-admin}" \ --password "${KALAMDB_PASSWORD:-kalamdb123}" \ diff --git a/link/kalam-client/Cargo.toml b/link/kalam-client/Cargo.toml index 59b194511..22b13e837 100644 --- a/link/kalam-client/Cargo.toml +++ b/link/kalam-client/Cargo.toml @@ -62,6 +62,7 @@ required-features = ["e2e-tests"] [dev-dependencies] tokio = { workspace = true, features = ["test-util", "macros"] } +tokio-netem = { workspace = true } kalamdb-configs = { workspace = true } kalamdb-server = { workspace = true } tempfile = { workspace = true } @@ -73,4 +74,4 @@ serde_json = { workspace = true } crate-type = ["cdylib", "rlib"] [package.metadata.wasm-pack.profile.release] -wasm-opt = false \ No newline at end of file +wasm-opt = false diff --git a/link/kalam-client/tests/common/tcp_proxy.rs b/link/kalam-client/tests/common/tcp_proxy.rs index 115aa988f..2507a7d33 100644 --- a/link/kalam-client/tests/common/tcp_proxy.rs +++ b/link/kalam-client/tests/common/tcp_proxy.rs @@ -15,6 +15,10 @@ use tokio::{ task::JoinHandle, time::sleep, }; +use tokio_netem::{ + delayer::DynamicDuration, io::NetEmWriteExt, probability::DynamicProbability, + slicer::DynamicSize, terminator::Terminator, throttler::DynamicRate, +}; /// A TCP proxy that sits between a client and a real server, allowing tests to /// simulate network failures by pausing new connections and/or forcibly dropping @@ -43,6 +47,10 @@ struct ProxyImpairments { latency_ms: AtomicU64, stall_every_n_chunks: AtomicU64, stall_duration_ms: AtomicU64, + netem_write_delay: Arc, + netem_write_rate: Arc, + netem_write_slice_size: Arc, + netem_termination_probability: Arc, } impl ProxyImpairments { @@ -52,6 +60,11 @@ impl ProxyImpairments { latency_ms: AtomicU64::new(0), stall_every_n_chunks: AtomicU64::new(0), stall_duration_ms: AtomicU64::new(0), + netem_write_delay: DynamicDuration::new(Duration::ZERO), + netem_write_rate: DynamicRate::new(0), + netem_write_slice_size: DynamicSize::new(0), + netem_termination_probability: DynamicProbability::new(0.0) + .expect("zero termination probability should be valid"), } } @@ -96,7 +109,6 @@ impl TcpDisconnectProxy { let accept_task = tokio::spawn(async move { while let Ok((mut inbound, _peer)) = listener.accept().await { if paused_clone.load(Ordering::SeqCst) { - let _ = inbound.set_linger(Some(Duration::ZERO)); let _ = inbound.shutdown().await; drop(inbound); continue; @@ -191,6 +203,54 @@ impl TcpDisconnectProxy { self.impairments.stall_duration_ms.store(0, Ordering::SeqCst); } + /// Add tokio-netem write-side latency in both directions. + pub fn set_netem_write_delay(&self, delay: Duration) { + self.impairments.netem_write_delay.set(delay); + } + + /// Clear tokio-netem write-side latency. + pub fn clear_netem_write_delay(&self) { + self.impairments.netem_write_delay.set(Duration::ZERO); + } + + /// Throttle forwarded writes in both directions to `bytes_per_second`. + /// + /// A value of `0` disables throttling. + pub fn set_netem_write_rate(&self, bytes_per_second: usize) { + self.impairments.netem_write_rate.set(bytes_per_second); + } + + /// Clear any tokio-netem write throttle. + pub fn clear_netem_write_rate(&self) { + self.impairments.netem_write_rate.set(0); + } + + /// Slice forwarded writes in both directions. A size of `0` disables slicing. + pub fn set_netem_write_slice_size(&self, size: usize) { + self.impairments.netem_write_slice_size.set(size); + } + + /// Clear tokio-netem write slicing. + pub fn clear_netem_write_slice_size(&self) { + self.impairments.netem_write_slice_size.set(0); + } + + /// Probabilistically terminate each proxied transport poll in both directions. + pub fn set_netem_termination_probability(&self, probability: f64) { + self.impairments + .netem_termination_probability + .set(probability) + .expect("termination probability should be between 0.0 and 1.0"); + } + + /// Clear tokio-netem transport termination. + pub fn clear_netem_termination_probability(&self) { + self.impairments + .netem_termination_probability + .set(0.0) + .expect("zero termination probability should be valid"); + } + /// Abort all in-flight proxy tasks, forcibly closing both sides of every /// active connection. pub async fn drop_active_connections(&self) { @@ -271,11 +331,18 @@ async fn bind_loopback_listener() -> std::io::Result { async fn relay_with_impairments( mut reader: tokio::net::tcp::ReadHalf<'_>, - mut writer: tokio::net::tcp::WriteHalf<'_>, + writer: tokio::net::tcp::WriteHalf<'_>, impairments: Arc, ) -> std::io::Result<()> { let mut buffer = [0_u8; 16 * 1024]; let mut chunk_index = 0_u64; + let mut writer = Terminator::new( + writer + .delay_writes_dyn(impairments.netem_write_delay.clone()) + .throttle_writes_dyn(impairments.netem_write_rate.clone()) + .slice_writes_dyn(impairments.netem_write_slice_size.clone()), + impairments.netem_termination_probability.clone(), + ); loop { let read = reader.read(&mut buffer).await?; diff --git a/link/kalam-client/tests/proxied.rs b/link/kalam-client/tests/proxied.rs index 415c18047..6d81cf3aa 100644 --- a/link/kalam-client/tests/proxied.rs +++ b/link/kalam-client/tests/proxied.rs @@ -43,6 +43,8 @@ mod socket_drop_resume; mod staggered_outages; #[path = "proxied/subscribe_during_reconnect.rs"] mod subscribe_during_reconnect; +#[path = "proxied/topic_consumption_netem.rs"] +mod topic_consumption_netem; #[path = "proxied/transport_impairments.rs"] mod transport_impairments; #[path = "proxied/unsubscribe_during_outage.rs"] diff --git a/link/kalam-client/tests/proxied/topic_consumption_netem.rs b/link/kalam-client/tests/proxied/topic_consumption_netem.rs new file mode 100644 index 000000000..be985df55 --- /dev/null +++ b/link/kalam-client/tests/proxied/topic_consumption_netem.rs @@ -0,0 +1,471 @@ +use std::{collections::HashSet, time::Duration}; + +use kalam_client::consumer::{AutoOffsetReset, ConsumerRecord, TopicConsumer, TopicOp}; +use serde_json::Value; +use tokio::time::{sleep, timeout, Instant}; + +use super::helpers::*; +use crate::common::tcp_proxy::TcpDisconnectProxy; + +async fn setup_topic_with_sources( + writer: &kalam_client::KalamLinkClient, + namespace: &str, + table: &str, + topic: &str, + operations: &[&str], +) { + writer + .execute_query(&format!("CREATE NAMESPACE IF NOT EXISTS {}", namespace), None, None, None) + .await + .expect("create topic test namespace"); + writer + .execute_query( + &format!( + "CREATE TABLE IF NOT EXISTS {}.{} (id INT PRIMARY KEY, value TEXT, counter INT)", + namespace, table + ), + None, + None, + None, + ) + .await + .expect("create topic source table"); + writer + .execute_query(&format!("CREATE TOPIC {}", topic), None, None, None) + .await + .expect("create topic"); + + for operation in operations { + writer + .execute_query( + &format!( + "ALTER TOPIC {} ADD SOURCE {}.{} ON {}", + topic, namespace, table, operation + ), + None, + None, + None, + ) + .await + .expect("add topic source route"); + } + + wait_for_topic_routes(writer, topic, operations.len()).await; +} + +async fn wait_for_topic_routes( + writer: &kalam_client::KalamLinkClient, + topic: &str, + min_routes: usize, +) { + let deadline = Instant::now() + Duration::from_secs(15); + let sql = format!("SELECT routes FROM system.topics WHERE topic_id = '{}'", topic); + + while Instant::now() < deadline { + if let Ok(result) = writer.execute_query(&sql, None, None, None).await { + if let Some(routes) = result.get_string("routes") { + if serde_json::from_str::(&routes) + .ok() + .and_then(|value| value.as_array().map(|routes| routes.len())) + .unwrap_or(0) + >= min_routes + { + sleep(Duration::from_millis(100)).await; + return; + } + } + } + sleep(Duration::from_millis(100)).await; + } + + panic!("topic {} did not expose at least {} routes", topic, min_routes); +} + +fn build_consumer( + client: &kalam_client::KalamLinkClient, + topic: &str, + group_id: &str, + enable_auto_commit: bool, +) -> TopicConsumer { + client + .consumer() + .topic(topic) + .group_id(group_id) + .auto_offset_reset(AutoOffsetReset::Earliest) + .enable_auto_commit(enable_auto_commit) + .max_poll_records(25) + .poll_timeout(Duration::from_secs(1)) + .request_timeout(Duration::from_secs(10)) + .retry_backoff(Duration::from_millis(100)) + .build() + .expect("build topic consumer") +} + +async fn poll_records_until( + consumer: &mut TopicConsumer, + min_records: usize, + timeout_dur: Duration, +) -> Vec { + let deadline = Instant::now() + timeout_dur; + let mut records = Vec::new(); + let mut seen = HashSet::<(u32, u64)>::new(); + let mut last_error = None; + + while Instant::now() < deadline && records.len() < min_records { + match consumer.poll().await { + Ok(batch) if batch.is_empty() => sleep(Duration::from_millis(50)).await, + Ok(batch) => { + for record in batch { + if seen.insert((record.partition_id, record.offset)) { + records.push(record); + } + } + }, + Err(err) if is_retryable_consumer_error(&err.to_string()) => { + last_error = Some(err.to_string()); + sleep(Duration::from_millis(50)).await; + }, + Err(err) => panic!("topic consumer poll failed: {}", err), + } + } + + assert!( + records.len() >= min_records, + "expected at least {} topic records, got {}; last retryable error: {:?}", + min_records, + records.len(), + last_error + ); + records +} + +fn is_retryable_consumer_error(message: &str) -> bool { + let message = message.to_ascii_lowercase(); + message.contains("network") + || message.contains("timeout") + || message.contains("connect") + || message.contains("error decoding response body") + || message.contains("request") +} + +fn payload_json(record: &ConsumerRecord) -> Value { + serde_json::from_slice(&record.payload).expect("topic payload should be JSON") +} + +fn payload_i64(payload: &Value, key: &str) -> i64 { + let value = payload.get(key).unwrap_or_else(|| panic!("payload should contain {}", key)); + if let Some(number) = value.as_i64() { + return number; + } + if let Some(object) = value.as_object() { + for nested in object.values() { + if let Some(number) = nested.as_i64() { + return number; + } + if let Some(text) = nested.as_str().and_then(|text| text.parse::().ok()) { + return text; + } + } + } + value + .as_str() + .and_then(|text| text.parse::().ok()) + .unwrap_or_else(|| panic!("payload field {} should be integer-compatible: {}", key, value)) +} + +fn payload_string(payload: &Value, key: &str) -> String { + let value = payload.get(key).unwrap_or_else(|| panic!("payload should contain {}", key)); + if let Some(text) = value.as_str() { + return text.to_string(); + } + if let Some(object) = value.as_object() { + for nested in object.values() { + if let Some(text) = nested.as_str() { + return text.to_string(); + } + } + } + panic!("payload field {} should be string-compatible: {}", key, value); +} + +fn records_with_op<'a>(records: &'a [ConsumerRecord], op: TopicOp) -> Vec<&'a ConsumerRecord> { + records.iter().filter(|record| record.op == op).collect() +} + +#[tokio::test] +#[ntest::timeout(10000)] +async fn test_tokio_netem_topic_consume_fragmented_insert_update_delete_and_commit() { + let result = timeout(Duration::from_secs(60), async { + let writer = match create_test_client() { + Ok(client) => client, + Err(err) => { + eprintln!("Skipping test (writer client unavailable): {}", err); + return; + }, + }; + + let proxy = TcpDisconnectProxy::start(upstream_server_url()).await; + let client = match create_test_client_for_base_url(proxy.base_url()) { + Ok(client) => client, + Err(err) => { + eprintln!("Skipping test (proxy client unavailable): {}", err); + proxy.shutdown().await; + return; + }, + }; + + let suffix = unique_suffix(); + let namespace = format!("topic_netem_{}", suffix); + let table = "events"; + let topic = format!("{}.{}", namespace, table); + let group = format!("topic-netem-fragmented-{}", suffix); + + setup_topic_with_sources( + &writer, + &namespace, + table, + &topic, + &["INSERT", "UPDATE", "DELETE"], + ) + .await; + + writer + .execute_query( + &format!( + "INSERT INTO {}.{} (id, value, counter) VALUES (1, 'created', 0)", + namespace, table + ), + None, + None, + None, + ) + .await + .expect("insert topic source row"); + writer + .execute_query( + &format!( + "UPDATE {}.{} SET value = 'updated', counter = 1 WHERE id = 1", + namespace, table + ), + None, + None, + None, + ) + .await + .expect("update topic source row"); + writer + .execute_query( + &format!("DELETE FROM {}.{} WHERE id = 1", namespace, table), + None, + None, + None, + ) + .await + .expect("delete topic source row"); + + proxy.set_netem_write_slice_size(64); + + let mut consumer = build_consumer(&client, &topic, &group, false); + let records = poll_records_until(&mut consumer, 3, Duration::from_secs(20)).await; + let inserts = records_with_op(&records, TopicOp::Insert); + let updates = records_with_op(&records, TopicOp::Update); + let deletes = records_with_op(&records, TopicOp::Delete); + + assert_eq!(inserts.len(), 1); + assert_eq!(updates.len(), 1); + assert_eq!(deletes.len(), 1); + + let insert_payload = payload_json(inserts[0]); + assert_eq!(payload_i64(&insert_payload, "id"), 1); + assert_eq!(payload_string(&insert_payload, "value"), "created"); + + let update_payload = payload_json(updates[0]); + assert_eq!(payload_i64(&update_payload, "id"), 1); + assert_eq!(payload_string(&update_payload, "value"), "updated"); + assert_eq!(payload_i64(&update_payload, "counter"), 1); + + for record in &records { + consumer.mark_processed(record); + } + let commit = consumer.commit_sync().await.expect("commit fragmented topic records"); + assert_eq!(commit.acknowledged_offset, 2); + + let mut resumed = build_consumer(&client, &topic, &group, false); + let replay = resumed + .poll_with_timeout(Duration::from_millis(250)) + .await + .expect("poll after committed offset"); + assert!(replay.is_empty(), "committed topic offsets should not replay records"); + + proxy.clear_netem_write_slice_size(); + consumer.close().await.ok(); + resumed.close().await.ok(); + proxy.shutdown().await; + }) + .await; + + assert!(result.is_ok(), "fragmented topic consume test timed out"); +} + +#[tokio::test] +#[ntest::timeout(17000)] +async fn test_tokio_netem_topic_bandwidth_collapse_poll_recovers_without_losing_offsets() { + let result = timeout(Duration::from_secs(75), async { + let writer = match create_test_client() { + Ok(client) => client, + Err(err) => { + eprintln!("Skipping test (writer client unavailable): {}", err); + return; + }, + }; + + let proxy = TcpDisconnectProxy::start(upstream_server_url()).await; + let client = match create_test_client_for_base_url(proxy.base_url()) { + Ok(client) => client, + Err(err) => { + eprintln!("Skipping test (proxy client unavailable): {}", err); + proxy.shutdown().await; + return; + }, + }; + + let suffix = unique_suffix(); + let namespace = format!("topic_netem_throttle_{}", suffix); + let table = "events"; + let topic = format!("{}.{}", namespace, table); + let group = format!("topic-netem-throttle-{}", suffix); + let impaired_group = format!("topic-netem-throttle-impaired-{}", suffix); + + setup_topic_with_sources(&writer, &namespace, table, &topic, &["INSERT"]).await; + + for id in 0..5 { + writer + .execute_query( + &format!( + "INSERT INTO {}.{} (id, value, counter) VALUES ({}, 'value_{}', {})", + namespace, table, id, id, id + ), + None, + None, + None, + ) + .await + .expect("insert topic row during throttle test"); + } + + let mut impaired_consumer = build_consumer(&client, &topic, &impaired_group, false); + proxy.set_netem_write_rate(8); + + let throttled_poll = timeout(Duration::from_secs(4), impaired_consumer.poll()).await; + assert!( + throttled_poll.is_err() + || throttled_poll + .as_ref() + .ok() + .and_then(|result| result.as_ref().ok()) + .map_or(true, Vec::is_empty), + "severe bandwidth collapse should not successfully drain topic records immediately" + ); + + proxy.clear_netem_write_rate(); + let mut consumer = build_consumer(&client, &topic, &group, false); + let records = poll_records_until(&mut consumer, 5, Duration::from_secs(20)).await; + let mut ids = records + .iter() + .map(|record| payload_i64(&payload_json(record), "id")) + .collect::>(); + ids.sort_unstable(); + assert_eq!(ids, vec![0, 1, 2, 3, 4]); + + for record in &records { + consumer.mark_processed(record); + } + let commit = consumer.commit_sync().await.expect("commit after bandwidth recovery"); + assert_eq!(commit.acknowledged_offset, 4); + + consumer.close().await.ok(); + impaired_consumer.close().await.ok(); + proxy.shutdown().await; + }) + .await; + + assert!(result.is_ok(), "topic bandwidth collapse test timed out"); +} + +#[tokio::test] +#[ntest::timeout(10000)] +async fn test_tokio_netem_topic_commit_failure_can_be_retried_without_replay() { + let result = timeout(Duration::from_secs(75), async { + let writer = match create_test_client() { + Ok(client) => client, + Err(err) => { + eprintln!("Skipping test (writer client unavailable): {}", err); + return; + }, + }; + + let proxy = TcpDisconnectProxy::start(upstream_server_url()).await; + let client = match create_test_client_for_base_url(proxy.base_url()) { + Ok(client) => client, + Err(err) => { + eprintln!("Skipping test (proxy client unavailable): {}", err); + proxy.shutdown().await; + return; + }, + }; + + let suffix = unique_suffix(); + let namespace = format!("topic_netem_commit_{}", suffix); + let table = "events"; + let topic = format!("{}.{}", namespace, table); + let group = format!("topic-netem-commit-{}", suffix); + + setup_topic_with_sources(&writer, &namespace, table, &topic, &["INSERT"]).await; + + for id in 10..13 { + writer + .execute_query( + &format!( + "INSERT INTO {}.{} (id, value, counter) VALUES ({}, 'value_{}', {})", + namespace, table, id, id, id + ), + None, + None, + None, + ) + .await + .expect("insert topic row for commit retry test"); + } + + let mut consumer = build_consumer(&client, &topic, &group, false); + let records = poll_records_until(&mut consumer, 3, Duration::from_secs(20)).await; + for record in &records { + consumer.mark_processed(record); + } + + proxy.set_netem_termination_probability(1.0); + let failed_commit = consumer.commit_sync().await; + assert!( + failed_commit.is_err(), + "netem transport termination should make the topic commit fail" + ); + + proxy.clear_netem_termination_probability(); + let commit = consumer.commit_sync().await.expect("retry commit after netem recovery"); + assert_eq!(commit.acknowledged_offset, 2); + + let mut resumed = build_consumer(&client, &topic, &group, false); + let replay = resumed + .poll_with_timeout(Duration::from_millis(250)) + .await + .expect("poll after retried commit"); + assert!(replay.is_empty(), "retried commit should prevent topic record replay"); + + consumer.close().await.ok(); + resumed.close().await.ok(); + proxy.shutdown().await; + }) + .await; + + assert!(result.is_ok(), "topic commit retry test timed out"); +} diff --git a/link/kalam-client/tests/proxied/transport_impairments.rs b/link/kalam-client/tests/proxied/transport_impairments.rs index eac3406fd..089f19da2 100644 --- a/link/kalam-client/tests/proxied/transport_impairments.rs +++ b/link/kalam-client/tests/proxied/transport_impairments.rs @@ -450,3 +450,362 @@ async fn test_proxy_packet_loss_style_stalls_resume_without_replay() { assert!(result.is_ok(), "packet-loss-style stall test timed out"); } + +/// Very small write slices fragment WebSocket frames at the transport boundary. +/// The client should parse the stream normally and avoid spurious reconnects. +#[tokio::test] +#[ntest::timeout(8000)] +async fn test_tokio_netem_fragmented_writes_preserve_live_stream() { + let result = timeout(Duration::from_secs(45), async { + let writer = match create_test_client() { + Ok(c) => c, + Err(e) => { + eprintln!("Skipping test (writer client unavailable): {}", e); + return; + }, + }; + + let proxy = TcpDisconnectProxy::start(upstream_server_url()).await; + let (client, _connect_count, disconnect_count) = + match create_test_client_with_events_for_base_url(proxy.base_url()) { + Ok(v) => v, + Err(e) => { + eprintln!("Skipping test (proxy client unavailable): {}", e); + proxy.shutdown().await; + return; + }, + }; + + let suffix = unique_suffix(); + let table = format!("default.netem_sliced_{}", suffix); + ensure_table(&writer, &table).await; + + client.connect().await.expect("initial connect through proxy"); + let mut sub = client + .subscribe_with_config(SubscriptionConfig::new( + format!("netem-sliced-{}", suffix), + format!("SELECT id, value FROM {}", table), + )) + .await + .expect("subscribe through proxy"); + + let _ = timeout(TEST_TIMEOUT, sub.next()).await; + let disconnects_before = disconnect_count.load(Ordering::SeqCst); + proxy.set_netem_write_slice_size(3); + + writer + .execute_query( + &format!( + "INSERT INTO {} (id, value) VALUES ('fragmented', '{}')", + table, + "x".repeat(512) + ), + None, + None, + None, + ) + .await + .expect("insert row through fragmented transport"); + + let mut ids = Vec::::new(); + let mut seq = None; + for _ in 0..30 { + if ids.iter().any(|id| id == "fragmented") { + break; + } + match timeout(Duration::from_millis(1000), sub.next()).await { + Ok(Some(Ok(ev))) => { + collect_ids_and_track_seq(&ev, &mut ids, &mut seq, None, "netem slicing"); + }, + Ok(Some(Err(e))) => panic!("netem slicing subscription error: {}", e), + Ok(None) => panic!("netem slicing subscription ended unexpectedly"), + Err(_) => {}, + } + } + + assert!( + ids.iter().any(|id| id == "fragmented"), + "live row should arrive when netem fragments writes" + ); + assert_eq!( + disconnect_count.load(Ordering::SeqCst), + disconnects_before, + "write fragmentation should not cause a reconnect" + ); + + proxy.clear_netem_write_slice_size(); + sub.close().await.ok(); + client.disconnect().await; + proxy.shutdown().await; + }) + .await; + + assert!(result.is_ok(), "netem fragmented write test timed out"); +} + +/// A severe bandwidth collapse should look like a dead link to keepalive. Once +/// the throttle is removed, reconnect resume must deliver only rows after the +/// checkpoint. +#[tokio::test] +#[ntest::timeout(16000)] +async fn test_tokio_netem_bandwidth_collapse_forces_resume_without_replay() { + let result = timeout(Duration::from_secs(75), async { + let writer = match create_test_client() { + Ok(c) => c, + Err(e) => { + eprintln!("Skipping test (writer client unavailable): {}", e); + return; + }, + }; + + let proxy = TcpDisconnectProxy::start(upstream_server_url()).await; + let (client, connect_count, disconnect_count) = + match create_test_client_with_events_for_base_url(proxy.base_url()) { + Ok(v) => v, + Err(e) => { + eprintln!("Skipping test (proxy client unavailable): {}", e); + proxy.shutdown().await; + return; + }, + }; + + let suffix = unique_suffix(); + let table = format!("default.netem_throttle_{}", suffix); + ensure_table(&writer, &table).await; + + client.connect().await.expect("initial connect through proxy"); + let mut sub = client + .subscribe_with_config(SubscriptionConfig::new( + format!("netem-throttle-{}", suffix), + format!("SELECT id, value FROM {}", table), + )) + .await + .expect("subscribe through proxy"); + + let _ = timeout(TEST_TIMEOUT, sub.next()).await; + writer + .execute_query( + &format!("INSERT INTO {} (id, value) VALUES ('baseline-throttle', 'ready')", table), + None, + None, + None, + ) + .await + .expect("insert baseline row"); + + let mut baseline_ids = Vec::::new(); + let mut checkpoint = None; + for _ in 0..12 { + if baseline_ids.iter().any(|id| id == "baseline-throttle") { + break; + } + match timeout(Duration::from_millis(1200), sub.next()).await { + Ok(Some(Ok(ev))) => { + collect_ids_and_track_seq( + &ev, + &mut baseline_ids, + &mut checkpoint, + None, + "netem throttle baseline", + ); + }, + _ => {}, + } + } + assert!(baseline_ids.iter().any(|id| id == "baseline-throttle")); + let resume_from = query_max_seq(&writer, &table).await; + + let disconnects_before = disconnect_count.load(Ordering::SeqCst); + let expected_connects = connect_count.load(Ordering::SeqCst) + 1; + proxy.set_netem_write_rate(8); + + writer + .execute_query( + &format!( + "INSERT INTO {} (id, value) VALUES ('gap-throttle', 'during-collapse')", + table + ), + None, + None, + None, + ) + .await + .expect("insert gap row during bandwidth collapse"); + + for _ in 0..80 { + if disconnect_count.load(Ordering::SeqCst) > disconnects_before { + break; + } + sleep(Duration::from_millis(100)).await; + } + assert!( + disconnect_count.load(Ordering::SeqCst) > disconnects_before, + "severe netem throttle should force a reconnect" + ); + + proxy.clear_netem_write_rate(); + wait_for_reconnect(&client, &connect_count, expected_connects, "netem throttle").await; + + writer + .execute_query( + &format!( + "INSERT INTO {} (id, value) VALUES ('live-throttle', 'after-reconnect')", + table + ), + None, + None, + None, + ) + .await + .expect("insert live row after throttle clears"); + + wait_for_row_after_checkpoint( + &mut sub, + resume_from, + &["gap-throttle", "live-throttle"], + &["baseline-throttle"], + "netem throttle recovery", + ) + .await; + + sub.close().await.ok(); + client.disconnect().await; + proxy.shutdown().await; + }) + .await; + + assert!(result.is_ok(), "netem bandwidth collapse test timed out"); +} + +/// tokio-netem can fail the transport from inside the I/O adapter instead of +/// aborting the proxy task. The client should treat it as a normal disconnect. +#[tokio::test] +#[ntest::timeout(10000)] +async fn test_tokio_netem_forced_transport_termination_recovers() { + let result = timeout(Duration::from_secs(75), async { + let writer = match create_test_client() { + Ok(c) => c, + Err(e) => { + eprintln!("Skipping test (writer client unavailable): {}", e); + return; + }, + }; + + let proxy = TcpDisconnectProxy::start(upstream_server_url()).await; + let (client, connect_count, disconnect_count) = + match create_test_client_with_events_for_base_url(proxy.base_url()) { + Ok(v) => v, + Err(e) => { + eprintln!("Skipping test (proxy client unavailable): {}", e); + proxy.shutdown().await; + return; + }, + }; + + let suffix = unique_suffix(); + let table = format!("default.netem_terminator_{}", suffix); + ensure_table(&writer, &table).await; + + client.connect().await.expect("initial connect through proxy"); + let mut sub = client + .subscribe_with_config(SubscriptionConfig::new( + format!("netem-terminator-{}", suffix), + format!("SELECT id, value FROM {}", table), + )) + .await + .expect("subscribe through proxy"); + + let _ = timeout(TEST_TIMEOUT, sub.next()).await; + writer + .execute_query( + &format!("INSERT INTO {} (id, value) VALUES ('baseline-term', 'ready')", table), + None, + None, + None, + ) + .await + .expect("insert baseline row"); + + let mut baseline_ids = Vec::::new(); + let mut checkpoint = None; + for _ in 0..12 { + if baseline_ids.iter().any(|id| id == "baseline-term") { + break; + } + match timeout(Duration::from_millis(1200), sub.next()).await { + Ok(Some(Ok(ev))) => { + collect_ids_and_track_seq( + &ev, + &mut baseline_ids, + &mut checkpoint, + None, + "netem termination baseline", + ); + }, + _ => {}, + } + } + assert!(baseline_ids.iter().any(|id| id == "baseline-term")); + let resume_from = query_max_seq(&writer, &table).await; + + let disconnects_before = disconnect_count.load(Ordering::SeqCst); + let expected_connects = connect_count.load(Ordering::SeqCst) + 1; + proxy.set_netem_termination_probability(1.0); + + writer + .execute_query( + &format!( + "INSERT INTO {} (id, value) VALUES ('gap-term', 'during-termination')", + table + ), + None, + None, + None, + ) + .await + .expect("insert gap row during transport termination"); + + for _ in 0..80 { + if disconnect_count.load(Ordering::SeqCst) > disconnects_before { + break; + } + sleep(Duration::from_millis(100)).await; + } + assert!( + disconnect_count.load(Ordering::SeqCst) > disconnects_before, + "netem terminator should force a disconnect" + ); + + proxy.clear_netem_termination_probability(); + wait_for_reconnect(&client, &connect_count, expected_connects, "netem terminator").await; + + writer + .execute_query( + &format!( + "INSERT INTO {} (id, value) VALUES ('live-term', 'after-reconnect')", + table + ), + None, + None, + None, + ) + .await + .expect("insert live row after terminator clears"); + + wait_for_row_after_checkpoint( + &mut sub, + resume_from, + &["gap-term", "live-term"], + &["baseline-term"], + "netem terminator recovery", + ) + .await; + + sub.close().await.ok(); + client.disconnect().await; + proxy.shutdown().await; + }) + .await; + + assert!(result.is_ok(), "netem forced termination test timed out"); +} diff --git a/link/kalam-link-dart/src/tests.rs b/link/kalam-link-dart/src/tests.rs index 6292de08e..8945a5f47 100644 --- a/link/kalam-link-dart/src/tests.rs +++ b/link/kalam-link-dart/src/tests.rs @@ -141,6 +141,7 @@ mod tests { named_rows: None, row_count: 1, message: None, + as_user: None, }; let resp = QueryResponse { status: ResponseStatus::Success, @@ -187,6 +188,7 @@ mod tests { named_rows: None, row_count: 0, message: Some("Table created".into()), + as_user: None, }; let dart: DartQueryResult = qr.into(); assert!(dart.columns.is_empty()); diff --git a/link/link-common/Cargo.toml b/link/link-common/Cargo.toml index 64a0bf9b9..30b10a9f3 100644 --- a/link/link-common/Cargo.toml +++ b/link/link-common/Cargo.toml @@ -49,13 +49,14 @@ web-sys = { workspace = true, features = [ [target.'cfg(not(target_arch = "wasm32"))'.dependencies] aws-lc-rs = { workspace = true, optional = true } +rustls = { workspace = true, features = ["aws_lc_rs"], optional = true } quinn-proto = { workspace = true, optional = true } rustls-webpki = { workspace = true, optional = true } [features] default = [] client-core = ["dep:futures-util", "dep:log", "dep:miniz_oxide", "dep:rmp-serde"] -tokio-runtime = ["client-core", "tokio", "reqwest", "tokio-tungstenite", "bytes", "dep:aws-lc-rs", "dep:quinn-proto", "dep:rustls-webpki"] +tokio-runtime = ["client-core", "tokio", "reqwest", "tokio-tungstenite", "bytes", "dep:aws-lc-rs", "dep:rustls", "dep:quinn-proto", "dep:rustls-webpki"] auth-flows = [] setup = [] healthcheck = [] diff --git a/link/link-common/src/client/builder.rs b/link/link-common/src/client/builder.rs index 6274540c8..098dfa1d2 100644 --- a/link/link-common/src/client/builder.rs +++ b/link/link-common/src/client/builder.rs @@ -1,5 +1,5 @@ use std::{ - sync::{Arc, RwLock}, + sync::{Arc, Once, RwLock}, time::Duration, }; @@ -17,6 +17,14 @@ use crate::{ timeouts::KalamLinkTimeouts, }; +fn ensure_rustls_crypto_provider() { + static INSTALL_RUSTLS_PROVIDER: Once = Once::new(); + + INSTALL_RUSTLS_PROVIDER.call_once(|| { + let _ = rustls::crypto::aws_lc_rs::default_provider().install_default(); + }); +} + impl KalamLinkClientBuilder { pub(crate) fn new() -> Self { Self { @@ -107,6 +115,8 @@ impl KalamLinkClientBuilder { /// Build the client pub fn build(self) -> Result { + ensure_rustls_crypto_provider(); + let base_url = self .base_url .ok_or_else(|| KalamLinkError::ConfigurationError("base_url is required".into()))?; diff --git a/link/link-common/src/client/tests.rs b/link/link-common/src/client/tests.rs index 8d54258ed..79c3dc8bc 100644 --- a/link/link-common/src/client/tests.rs +++ b/link/link-common/src/client/tests.rs @@ -21,6 +21,18 @@ fn test_builder_pattern() { assert!(result.is_ok()); } +#[test] +fn test_builder_installs_rustls_provider_idempotently() { + for _ in 0..2 { + let result = KalamLinkClient::builder() + .base_url("https://kalam.masky.app") + .jwt_token("test_token") + .build(); + + assert!(result.is_ok()); + } +} + #[test] fn test_builder_missing_url() { let result = KalamLinkClient::builder().build(); diff --git a/link/link-common/src/query/models/query_result.rs b/link/link-common/src/query/models/query_result.rs index ea40f2ca6..98b32681c 100644 --- a/link/link-common/src/query/models/query_result.rs +++ b/link/link-common/src/query/models/query_result.rs @@ -33,6 +33,10 @@ pub struct QueryResult { /// Optional message for non-query statements. #[serde(skip_serializing_if = "Option::is_none")] pub message: Option, + + /// Effective user identifier this statement executed as. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub as_user: Option, } impl QueryResult { diff --git a/link/sdks/typescript/client/README.md b/link/sdks/typescript/client/README.md index 1baf6a4d5..fb51e8e3f 100644 --- a/link/sdks/typescript/client/README.md +++ b/link/sdks/typescript/client/README.md @@ -187,7 +187,7 @@ This pattern is useful for chat threads, activity feeds, audit logs, and any UI ## Preserve Tenant Boundaries in Worker Writes -Background services should keep the same user isolation guarantees as the browser. Direct USER-table reads stay scoped to the authenticated account; `executeAsUser()` switches subject only when KalamDB authorizes the actor role for the target user's role. +Background services should keep the same user isolation guarantees as the browser. Direct USER-table and STREAM-table access stays scoped to the authenticated account; `executeAsUser()` switches subject only when KalamDB authorizes the actor role for the target user's role. ```ts await client.executeAsUser( @@ -197,7 +197,7 @@ await client.executeAsUser( ); ``` -That keeps the write inside Alice's USER-table partition through an explicit, audited delegation boundary instead of leaking service-side writes into the wrong tenant scope. +That keeps the write inside Alice's USER-table or STREAM-table partition through an explicit, audited delegation boundary instead of leaking service-side writes into the wrong tenant scope. ## Lower-Level Realtime API diff --git a/link/sdks/typescript/client/package-lock.json b/link/sdks/typescript/client/package-lock.json index 40cc5ed95..251b32391 100644 --- a/link/sdks/typescript/client/package-lock.json +++ b/link/sdks/typescript/client/package-lock.json @@ -1,12 +1,12 @@ { "name": "@kalamdb/client", - "version": "0.4.2-rc.3", + "version": "0.4.3-rc.4", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@kalamdb/client", - "version": "0.4.2-rc.3", + "version": "0.4.3-rc.4", "license": "Apache-2.0", "dependencies": { "ws": "^8.20.0" diff --git a/link/sdks/typescript/client/src/client.ts b/link/sdks/typescript/client/src/client.ts index 8ef84ee8b..bf077550c 100644 --- a/link/sdks/typescript/client/src/client.ts +++ b/link/sdks/typescript/client/src/client.ts @@ -701,7 +701,8 @@ export class KalamDBClient { * Execute a single SQL statement with an AS USER wrapper. * * KalamDB authorizes cross-user targets with its EXECUTE AS USER role matrix. - * Regular users can only target themselves. + * Regular users can only target themselves. The wrapper is valid for USER + * and STREAM tables; SHARED tables use their table policy directly. * * Wraps the SQL using: * `EXECUTE AS USER 'user' ( )` diff --git a/link/sdks/typescript/orm/package-lock.json b/link/sdks/typescript/orm/package-lock.json index 7b3b44676..a39752ff1 100644 --- a/link/sdks/typescript/orm/package-lock.json +++ b/link/sdks/typescript/orm/package-lock.json @@ -1,12 +1,12 @@ { "name": "@kalamdb/orm", - "version": "0.4.2-rc.3", + "version": "0.4.3-rc.5", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@kalamdb/orm", - "version": "0.4.2-rc.3", + "version": "0.4.3-rc.5", "license": "Apache-2.0", "bin": { "kalamdb-orm": "dist/cli.js" @@ -28,7 +28,7 @@ }, "../client": { "name": "@kalamdb/client", - "version": "0.4.2-rc.3", + "version": "0.4.3-rc.4", "dev": true, "license": "Apache-2.0", "dependencies": { @@ -44,7 +44,7 @@ }, "../consumer": { "name": "@kalamdb/consumer", - "version": "0.4.2-rc.3", + "version": "0.4.3-rc.4", "dev": true, "license": "Apache-2.0", "devDependencies": { diff --git a/nextest.toml b/nextest.toml index b24a1b2ff..400bb8007 100644 --- a/nextest.toml +++ b/nextest.toml @@ -107,6 +107,30 @@ test-group = "stateful-heavy" filter = 'test(test_shared_connection_recovers_subscriptions_in_different_stages)' test-group = "stateful-heavy" +[[profile.default.overrides]] +filter = 'test(test_tokio_netem_fragmented_writes_preserve_live_stream)' +test-group = "stateful-heavy" + +[[profile.default.overrides]] +filter = 'test(test_tokio_netem_bandwidth_collapse_forces_resume_without_replay)' +test-group = "stateful-heavy" + +[[profile.default.overrides]] +filter = 'test(test_tokio_netem_forced_transport_termination_recovers)' +test-group = "stateful-heavy" + +[[profile.default.overrides]] +filter = 'test(test_tokio_netem_topic_consume_fragmented_insert_update_delete_and_commit)' +test-group = "stateful-heavy" + +[[profile.default.overrides]] +filter = 'test(test_tokio_netem_topic_bandwidth_collapse_poll_recovers_without_losing_offsets)' +test-group = "stateful-heavy" + +[[profile.default.overrides]] +filter = 'test(test_tokio_netem_topic_commit_failure_can_be_retried_without_replay)' +test-group = "stateful-heavy" + [[profile.default.overrides]] filter = 'test(test_cli_syntax_error_handling)' test-group = "stateful-heavy" diff --git a/ui/package.json b/ui/package.json index 7ae962444..8f97e34d0 100644 --- a/ui/package.json +++ b/ui/package.json @@ -19,6 +19,8 @@ "lint": "eslint . --ext ts,tsx --report-unused-disable-directives --max-warnings 0", "preview": "vite preview", "test": "vitest", + "test:ci": "vitest run", + "test:sql-studio": "vitest run src/services/sqlStudioService.test.ts src/pages/SqlStudio.test.tsx src/components/sql-studio-v2/input-form/sqlCompletionCatalog.test.ts", "test:flows": "vitest run src/components/auth/LoginForm.test.tsx src/components/auth/ProtectedRoute.test.tsx src/components/auth/SetupGuard.test.tsx src/pages/SetupWizard.test.tsx src/pages/SqlStudio.test.tsx" }, "dependencies": { diff --git a/ui/src/components/sql-studio-v2/input-form/StudioEditorPanel.tsx b/ui/src/components/sql-studio-v2/input-form/StudioEditorPanel.tsx index 204a69807..342e8250a 100644 --- a/ui/src/components/sql-studio-v2/input-form/StudioEditorPanel.tsx +++ b/ui/src/components/sql-studio-v2/input-form/StudioEditorPanel.tsx @@ -13,9 +13,17 @@ import { DropdownMenuTrigger, } from "@/components/ui/dropdown-menu"; import type { LiveSubscriptionOptions, StudioNamespace } from "../shared/types"; +import { + buildSqlCompletionData, + resolveSqlContextualCompletions, + type SqlCompletionEntry, + type SqlCompletionData, +} from "./sqlCompletionCatalog"; type ExecuteMode = "all" | "selected"; +const EMPTY_COMPLETION_DATA = buildSqlCompletionData([]); + interface StudioEditorPanelProps { schema: StudioNamespace[]; tabTitle: string; @@ -62,40 +70,10 @@ export function StudioEditorPanel({ const onRunRef = useRef(onRun); const sqlRef = useRef(sql); const completionProviderRef = useRef(null); - const completionDataRef = useRef<{ - namespaces: string[]; - tablesByNamespace: Record; - columnsByTable: Record; - keywords: string[]; - }>({ - namespaces: [], - tablesByNamespace: {}, - columnsByTable: {}, - keywords: [], - }); + const completionDataRef = useRef(EMPTY_COMPLETION_DATA); const completionData = useMemo(() => { - const namespaces: string[] = []; - const tablesByNamespace: Record = {}; - const columnsByTable: Record = {}; - - schema.forEach((namespace) => { - const namespaceKey = namespace.name.toLowerCase(); - namespaces.push(namespace.name); - tablesByNamespace[namespaceKey] = namespace.tables.map((table) => table.name); - namespace.tables.forEach((table) => { - columnsByTable[`${namespaceKey}.${table.name.toLowerCase()}`] = table.columns.map((column) => column.name); - }); - }); - - const keywords = [ - "SELECT", "FROM", "WHERE", "JOIN", "LEFT", "RIGHT", "INNER", "OUTER", - "GROUP BY", "ORDER BY", "LIMIT", "INSERT", "UPDATE", "DELETE", - "CREATE", "ALTER", "DROP", "TABLE", "NAMESPACE", "VALUES", "SET", - "AND", "OR", "NOT", "IN", "AS", "ON", - ]; - - return { namespaces, tablesByNamespace, columnsByTable, keywords }; + return buildSqlCompletionData(schema); }, [schema]); useEffect(() => { @@ -192,57 +170,67 @@ export function StudioEditorPanel({ aliasMatch = aliasRegex.exec(textUntilPosition); } - const pushSuggestion = (label: string, kind: languages.CompletionItemKind, detail: string, insertText = label) => { + const pushSuggestion = ( + label: string, + kind: languages.CompletionItemKind, + detail: string, + insertText = label, + sortText?: string, + insertTextRules?: languages.CompletionItemInsertTextRule, + matchText = prefix, + ) => { const key = `${kind}-${label}-${insertText}`; if (seen.has(key)) { return; } - if (prefix && !label.toLowerCase().includes(prefix)) { + if (matchText && !label.toLowerCase().includes(matchText)) { return; } seen.add(key); - suggestions.push({ label, kind, detail, insertText, range }); + suggestions.push({ label, kind, detail, insertText, insertTextRules, range, sortText }); }; - const tableColumnMatch = /([a-zA-Z_][\w]*)\.([a-zA-Z_][\w]*)\.([a-zA-Z_][\w]*)?$/.exec(textUntilPosition); - if (tableColumnMatch) { - const namespaceKey = tableColumnMatch[1].toLowerCase(); - const tableKey = tableColumnMatch[2].toLowerCase(); - const columns = data.columnsByTable[`${namespaceKey}.${tableKey}`] ?? []; - columns.forEach((column) => - pushSuggestion(column, monaco.languages.CompletionItemKind.Field, `${namespaceKey}.${tableKey} column`), + const pushEntry = (entry: SqlCompletionEntry) => { + const snippetRule = entry.isSnippet + ? monaco.languages.CompletionItemInsertTextRule.InsertAsSnippet + : undefined; + const kindByCategory: Record = { + function: monaco.languages.CompletionItemKind.Function, + keyword: monaco.languages.CompletionItemKind.Keyword, + operator: monaco.languages.CompletionItemKind.Operator, + snippet: monaco.languages.CompletionItemKind.Snippet, + type: monaco.languages.CompletionItemKind.TypeParameter, + }; + + pushSuggestion( + entry.label, + kindByCategory[entry.category], + entry.detail, + entry.insertText ?? entry.label, + entry.sortText, + snippetRule, ); - return { suggestions }; - } + }; - const aliasColumnMatch = /([a-zA-Z_][\w]*)\.([a-zA-Z_][\w]*)?$/.exec(textUntilPosition); - if (aliasColumnMatch) { - const aliasKey = aliasColumnMatch[1].toLowerCase(); - const resolvedTable = aliasToTable[aliasKey]; - if (resolvedTable) { - const columns = data.columnsByTable[resolvedTable] ?? []; - columns.forEach((column) => - pushSuggestion(column, monaco.languages.CompletionItemKind.Field, `${aliasKey} alias column`), - ); - return { suggestions }; - } - } + const contextualCompletion = resolveSqlContextualCompletions(data, textUntilPosition, aliasToTable); + if (contextualCompletion) { + const kind = contextualCompletion.kind === "column" + ? monaco.languages.CompletionItemKind.Field + : monaco.languages.CompletionItemKind.Class; - const namespaceTableMatch = /([a-zA-Z_][\w]*)\.([a-zA-Z_][\w]*)?$/.exec(textUntilPosition); - if (namespaceTableMatch) { - const namespaceKey = namespaceTableMatch[1].toLowerCase(); - const namespaceTables = data.tablesByNamespace[namespaceKey]; - if (namespaceTables && namespaceTables.length > 0) { - namespaceTables.forEach((table) => - pushSuggestion(table, monaco.languages.CompletionItemKind.Class, `${namespaceKey} table`), - ); - return { suggestions }; - } + contextualCompletion.labels.forEach((label) => + pushSuggestion(label, kind, contextualCompletion.detail, label, undefined, undefined, contextualCompletion.partial), + ); + return { suggestions }; } data.keywords.forEach((keyword) => - pushSuggestion(keyword, monaco.languages.CompletionItemKind.Keyword, "SQL keyword"), + pushSuggestion(keyword, monaco.languages.CompletionItemKind.Keyword, "SQL keyword", keyword, `2_${keyword}`), ); + data.snippets.forEach(pushEntry); + data.functions.forEach(pushEntry); + data.types.forEach(pushEntry); + data.operators.forEach(pushEntry); data.namespaces.forEach((namespaceName) => pushSuggestion(namespaceName, monaco.languages.CompletionItemKind.Module, "Namespace"), ); diff --git a/ui/src/components/sql-studio-v2/input-form/sqlCompletionCatalog.test.ts b/ui/src/components/sql-studio-v2/input-form/sqlCompletionCatalog.test.ts new file mode 100644 index 000000000..4759be1ae --- /dev/null +++ b/ui/src/components/sql-studio-v2/input-form/sqlCompletionCatalog.test.ts @@ -0,0 +1,162 @@ +import { describe, expect, it } from "vitest"; + +import { buildSqlCompletionData, resolveSqlContextualCompletions } from "./sqlCompletionCatalog"; +import type { StudioNamespace } from "../shared/types"; + +describe("SQL Studio completion catalog", () => { + it("always includes system.live with known columns", () => { + const data = buildSqlCompletionData([]); + + expect(data.namespaces).toContain("system"); + expect(data.tablesByNamespace.system).toContain("live"); + expect(data.columnsByTable["system.live"]).toEqual( + expect.arrayContaining(["live_id", "subscription_id", "namespace_id", "table_name", "user_id", "status"]), + ); + }); + + it("merges schema tables with static KalamDB and DataFusion completions", () => { + const schema: StudioNamespace[] = [ + { + database: "kalam", + name: "app", + tables: [ + { + database: "kalam", + namespace: "app", + name: "messages", + tableType: "user", + columns: [ + { name: "id", dataType: "TEXT", isNullable: false, isPrimaryKey: true, ordinal: 0 }, + { name: "body", dataType: "TEXT", isNullable: true, isPrimaryKey: false, ordinal: 1 }, + ], + }, + ], + }, + ]; + + const data = buildSqlCompletionData(schema); + + expect(data.tablesByNamespace.app).toContain("messages"); + expect(data.columnsByTable["app.messages"]).toEqual(["id", "body"]); + expect(data.functions.map((entry) => entry.label)).toEqual( + expect.arrayContaining(["CURRENT_USER()", "CURRENT_USER_ID()", "CURRENT_ROLE()", "NOW()", "COUNT()", "JSON_GET()"]), + ); + expect(data.snippets.map((entry) => entry.label)).toEqual( + expect.arrayContaining(["EXECUTE AS USER", "EXPLAIN", "DESCRIBE TABLE", "SUBSCRIBE TO", "KILL LIVE QUERY"]), + ); + }); + + it("resolves namespace table completions from the loaded explorer schema", () => { + const schema: StudioNamespace[] = [ + { + database: "kalam", + name: "system", + tables: [ + { + database: "kalam", + namespace: "system", + name: "jobs", + tableType: "system", + columns: [ + { name: "job_id", dataType: "TEXT", isNullable: false, isPrimaryKey: true, ordinal: 0 }, + ], + }, + { + database: "kalam", + namespace: "system", + name: "users", + tableType: "system", + columns: [ + { name: "user_id", dataType: "TEXT", isNullable: false, isPrimaryKey: true, ordinal: 0 }, + ], + }, + ], + }, + { + database: "kalam", + name: "agent_events", + tables: [ + { + database: "kalam", + namespace: "agent_events", + name: "runs", + tableType: "shared", + columns: [ + { name: "id", dataType: "TEXT", isNullable: false, isPrimaryKey: true, ordinal: 0 }, + { name: "status", dataType: "TEXT", isNullable: false, isPrimaryKey: false, ordinal: 1 }, + ], + }, + { + database: "kalam", + namespace: "agent_events", + name: "messages", + tableType: "shared", + columns: [ + { name: "id", dataType: "TEXT", isNullable: false, isPrimaryKey: true, ordinal: 0 }, + { name: "body", dataType: "TEXT", isNullable: false, isPrimaryKey: false, ordinal: 1 }, + ], + }, + ], + }, + ]; + + const data = buildSqlCompletionData(schema); + + expect(resolveSqlContextualCompletions(data, "system.", {})).toEqual({ + kind: "table", + labels: expect.arrayContaining(["jobs", "users", "live"]), + detail: "system table", + partial: "", + }); + + expect(resolveSqlContextualCompletions(data, "SELECT *\nFROM agent_events.", {})).toEqual({ + kind: "table", + labels: ["runs", "messages"], + detail: "agent_events table", + partial: "", + }); + }); + + it("resolves namespace-qualified and alias-qualified column completions", () => { + const schema: StudioNamespace[] = [ + { + database: "kalam", + name: "agent_events", + tables: [ + { + database: "kalam", + namespace: "agent_events", + name: "runs", + tableType: "shared", + columns: [ + { name: "id", dataType: "TEXT", isNullable: false, isPrimaryKey: true, ordinal: 0 }, + { name: "status", dataType: "TEXT", isNullable: false, isPrimaryKey: false, ordinal: 1 }, + ], + }, + ], + }, + ]; + + const data = buildSqlCompletionData(schema); + + expect(resolveSqlContextualCompletions(data, "SELECT agent_events.runs.", {})).toEqual({ + kind: "column", + labels: ["id", "status"], + detail: "agent_events.runs column", + partial: "", + }); + + expect( + resolveSqlContextualCompletions( + data, + "SELECT * FROM agent_events.runs AS r WHERE r.", + { r: "agent_events.runs" }, + ), + ).toEqual({ + kind: "column", + labels: ["id", "status"], + detail: "r alias column", + partial: "", + }); + }); +}); \ No newline at end of file diff --git a/ui/src/components/sql-studio-v2/input-form/sqlCompletionCatalog.ts b/ui/src/components/sql-studio-v2/input-form/sqlCompletionCatalog.ts new file mode 100644 index 000000000..bed521edf --- /dev/null +++ b/ui/src/components/sql-studio-v2/input-form/sqlCompletionCatalog.ts @@ -0,0 +1,666 @@ +import type { StudioNamespace } from "../shared/types"; + +type CompletionCategory = "function" | "keyword" | "operator" | "snippet" | "type"; + +export interface SqlCompletionEntry { + label: string; + detail: string; + category: CompletionCategory; + insertText?: string; + isSnippet?: boolean; + sortText?: string; +} + +export interface SqlCompletionData { + namespaces: string[]; + tablesByNamespace: Record; + columnsByTable: Record; + keywords: readonly string[]; + functions: readonly SqlCompletionEntry[]; + snippets: readonly SqlCompletionEntry[]; + types: readonly SqlCompletionEntry[]; + operators: readonly SqlCompletionEntry[]; +} + +export interface SqlContextualCompletionMatch { + kind: "table" | "column"; + labels: string[]; + detail: string; + partial: string; +} + +const STATIC_SYSTEM_TABLES = [ + { + namespace: "system", + name: "live", + columns: [ + "live_id", + "connection_id", + "subscription_id", + "namespace_id", + "table_name", + "user_id", + "query", + "options", + "status", + "created_at", + "last_update", + "changes", + "node_id", + "last_ping_at", + ], + }, +] as const; + +const SQL_KEYWORDS = [ + "ACK", + "ADD", + "ALL", + "ALTER", + "ANALYZE", + "AND", + "AS", + "ASC", + "BACKUP", + "BEGIN", + "BETWEEN", + "BY", + "CASE", + "CHECK", + "CLEAR", + "CLUSTER", + "COMMIT", + "COMPACT", + "CONSUME", + "CREATE", + "CROSS", + "DATABASE", + "DEFAULT", + "DELETE", + "DESC", + "DESCRIBE", + "DISTINCT", + "DROP", + "ELSE", + "END", + "EXECUTE", + "EXISTS", + "EXPLAIN", + "EXPORT", + "FALSE", + "FLUSH", + "FOLLOWING", + "FOR", + "FROM", + "FULL", + "GROUP BY", + "HAVING", + "IF", + "IN", + "INNER JOIN", + "INSERT INTO", + "IS", + "JOIN", + "KEY", + "KILL", + "LAST", + "LEFT JOIN", + "LIKE", + "LIMIT", + "LIVE", + "NAMESPACE", + "NOT", + "NULL", + "OFFSET", + "ON", + "OPTIONS", + "OR", + "ORDER BY", + "OUTER", + "OVER", + "PARTITION BY", + "PRECEDING", + "PRIMARY KEY", + "QUERY", + "RANGE", + "RESTORE", + "RIGHT JOIN", + "ROLLBACK", + "ROW", + "ROWS", + "SELECT", + "SET", + "SHARED", + "SHOW", + "START TRANSACTION", + "STORAGE", + "STREAM", + "SUBSCRIBE TO", + "TABLE", + "THEN", + "TO", + "TOPIC", + "TRUE", + "UNBOUNDED", + "UNION", + "UPDATE", + "USE", + "USER", + "VALUES", + "VIEW", + "WHEN", + "WHERE", + "WITH", +] as const; + +const SQL_TYPE_NAMES = [ + "ARRAY", + "BIGINT", + "BOOLEAN", + "DATE", + "DOUBLE", + "FLOAT", + "INT", + "INTEGER", + "JSON", + "JSONB", + "LIST", + "MAP", + "REAL", + "SMALLINT", + "STRUCT", + "TEXT", + "TIME", + "TIMESTAMP", + "TIMESTAMPTZ", + "UUID", + "VARCHAR", +] as const; + +function functionEntry(name: string, detail: string, insertText?: string): SqlCompletionEntry { + return { + label: `${name}()`, + detail, + category: "function", + insertText: insertText ?? `${name}()`, + isSnippet: Boolean(insertText), + sortText: `1_${name}`, + }; +} + +function syntaxEntry(label: string, detail: string, insertText?: string): SqlCompletionEntry { + return { + label, + detail, + category: "snippet", + insertText: insertText ?? label, + isSnippet: Boolean(insertText), + sortText: `0_${label}`, + }; +} + +function typeEntry(label: string): SqlCompletionEntry { + return { + label, + detail: "SQL type", + category: "type", + sortText: `3_${label}`, + }; +} + +function operatorEntry(label: string, detail: string): SqlCompletionEntry { + return { + label, + detail, + category: "operator", + sortText: `4_${label}`, + }; +} + +const KALAMDB_FUNCTION_COMPLETIONS = [ + functionEntry("CURRENT_USER", "KalamDB context function"), + functionEntry("CURRENT_USER_ID", "KalamDB context function"), + functionEntry("CURRENT_ROLE", "KalamDB context function"), + functionEntry("SNOWFLAKE_ID", "KalamDB ID generation function"), + functionEntry("UUID_V7", "KalamDB ID generation function"), + functionEntry("ULID", "KalamDB ID generation function"), + functionEntry("COSINE_DISTANCE", "KalamDB vector similarity function", "COSINE_DISTANCE(${1:vector}, ${2:query_vector})"), + functionEntry("VECTOR_SEARCH", "KalamDB vector table function", "VECTOR_SEARCH(${1:table}, ${2:query_vector})"), +] as const; + +const DATAFUSION_SCALAR_FUNCTION_COMPLETIONS = [ + functionEntry("ABS", "DataFusion math function", "ABS(${1:number})"), + functionEntry("ACOS", "DataFusion math function", "ACOS(${1:number})"), + functionEntry("ACOSH", "DataFusion math function", "ACOSH(${1:number})"), + functionEntry("ARROW_CAST", "DataFusion core function", "ARROW_CAST(${1:value}, ${2:type})"), + functionEntry("ARROW_METADATA", "DataFusion core function", "ARROW_METADATA(${1:value})"), + functionEntry("ARROW_TYPEOF", "DataFusion core function", "ARROW_TYPEOF(${1:value})"), + functionEntry("ASCII", "DataFusion string function", "ASCII(${1:string})"), + functionEntry("ASIN", "DataFusion math function", "ASIN(${1:number})"), + functionEntry("ASINH", "DataFusion math function", "ASINH(${1:number})"), + functionEntry("ATAN", "DataFusion math function", "ATAN(${1:number})"), + functionEntry("ATAN2", "DataFusion math function", "ATAN2(${1:y}, ${2:x})"), + functionEntry("ATANH", "DataFusion math function", "ATANH(${1:number})"), + functionEntry("BIT_LENGTH", "DataFusion string function", "BIT_LENGTH(${1:string})"), + functionEntry("BTRIM", "DataFusion string function", "BTRIM(${1:string})"), + functionEntry("CBRT", "DataFusion math function", "CBRT(${1:number})"), + functionEntry("CEIL", "DataFusion math function", "CEIL(${1:number})"), + functionEntry("CHR", "DataFusion string function", "CHR(${1:code_point})"), + functionEntry("COALESCE", "DataFusion core function", "COALESCE(${1:value}, ${2:fallback})"), + functionEntry("CONCAT", "DataFusion string function", "CONCAT(${1:value})"), + functionEntry("CONCAT_WS", "DataFusion string function", "CONCAT_WS(${1:separator}, ${2:value})"), + functionEntry("CONTAINS", "DataFusion string function", "CONTAINS(${1:string}, ${2:search_string})"), + functionEntry("COS", "DataFusion math function", "COS(${1:number})"), + functionEntry("COSH", "DataFusion math function", "COSH(${1:number})"), + functionEntry("COT", "DataFusion math function", "COT(${1:number})"), + functionEntry("CURRENT_DATE", "DataFusion datetime function"), + functionEntry("CURRENT_TIME", "DataFusion datetime function"), + functionEntry("CURRENT_TIMESTAMP", "DataFusion datetime function"), + functionEntry("DATE_BIN", "DataFusion datetime function", "DATE_BIN(${1:stride}, ${2:source}, ${3:origin})"), + functionEntry("DATE_PART", "DataFusion datetime function", "DATE_PART(${1:part}, ${2:date})"), + functionEntry("DATE_TRUNC", "DataFusion datetime function", "DATE_TRUNC(${1:part}, ${2:date})"), + functionEntry("DEGREES", "DataFusion math function", "DEGREES(${1:number})"), + functionEntry("DIGEST", "DataFusion crypto function", "DIGEST(${1:value}, ${2:algorithm})"), + functionEntry("ENDS_WITH", "DataFusion string function", "ENDS_WITH(${1:string}, ${2:suffix})"), + functionEntry("EXP", "DataFusion math function", "EXP(${1:number})"), + functionEntry("FACTORIAL", "DataFusion math function", "FACTORIAL(${1:number})"), + functionEntry("FLOOR", "DataFusion math function", "FLOOR(${1:number})"), + functionEntry("FROM_UNIXTIME", "DataFusion datetime function", "FROM_UNIXTIME(${1:seconds})"), + functionEntry("GCD", "DataFusion math function", "GCD(${1:x}, ${2:y})"), + functionEntry("GET_FIELD", "DataFusion core function", "GET_FIELD(${1:value}, ${2:field})"), + functionEntry("GREATEST", "DataFusion core function", "GREATEST(${1:value})"), + functionEntry("ISNAN", "DataFusion math function", "ISNAN(${1:number})"), + functionEntry("ISZERO", "DataFusion math function", "ISZERO(${1:number})"), + functionEntry("LCM", "DataFusion math function", "LCM(${1:x}, ${2:y})"), + functionEntry("LEAST", "DataFusion core function", "LEAST(${1:value})"), + functionEntry("LEFT", "DataFusion unicode function", "LEFT(${1:string}, ${2:n})"), + functionEntry("LENGTH", "DataFusion string/unicode function", "LENGTH(${1:string})"), + functionEntry("LEVENSHTEIN", "DataFusion string function", "LEVENSHTEIN(${1:left}, ${2:right})"), + functionEntry("LN", "DataFusion math function", "LN(${1:number})"), + functionEntry("LOG", "DataFusion math function", "LOG(${1:base}, ${2:number})"), + functionEntry("LOG2", "DataFusion math function", "LOG2(${1:number})"), + functionEntry("LOG10", "DataFusion math function", "LOG10(${1:number})"), + functionEntry("LOWER", "DataFusion string function", "LOWER(${1:string})"), + functionEntry("LPAD", "DataFusion unicode function", "LPAD(${1:string}, ${2:length}, ${3:fill})"), + functionEntry("LTRIM", "DataFusion string function", "LTRIM(${1:string})"), + functionEntry("MAKE_DATE", "DataFusion datetime function", "MAKE_DATE(${1:year}, ${2:month}, ${3:day})"), + functionEntry("MAKE_TIME", "DataFusion datetime function", "MAKE_TIME(${1:hour}, ${2:minute}, ${3:second})"), + functionEntry("MD5", "DataFusion crypto function", "MD5(${1:value})"), + functionEntry("NANVL", "DataFusion math function", "NANVL(${1:x}, ${2:y})"), + functionEntry("NAMED_STRUCT", "DataFusion core function", "NAMED_STRUCT(${1:name}, ${2:value})"), + functionEntry("NOW", "DataFusion datetime function"), + functionEntry("NULLIF", "DataFusion core function", "NULLIF(${1:left}, ${2:right})"), + functionEntry("NVL", "DataFusion core function", "NVL(${1:value}, ${2:fallback})"), + functionEntry("NVL2", "DataFusion core function", "NVL2(${1:value}, ${2:not_null}, ${3:null_value})"), + functionEntry("OCTET_LENGTH", "DataFusion string function", "OCTET_LENGTH(${1:string})"), + functionEntry("OVERLAY", "DataFusion core/string function", "OVERLAY(${1:string} PLACING ${2:replacement} FROM ${3:start})"), + functionEntry("PI", "DataFusion math function"), + functionEntry("POWER", "DataFusion math function", "POWER(${1:base}, ${2:exponent})"), + functionEntry("RADIANS", "DataFusion math function", "RADIANS(${1:number})"), + functionEntry("RANDOM", "DataFusion math function"), + functionEntry("REGEXP_COUNT", "DataFusion regex function", "REGEXP_COUNT(${1:string}, ${2:pattern})"), + functionEntry("REGEXP_INSTR", "DataFusion regex function", "REGEXP_INSTR(${1:string}, ${2:pattern})"), + functionEntry("REGEXP_LIKE", "DataFusion regex function", "REGEXP_LIKE(${1:string}, ${2:pattern})"), + functionEntry("REGEXP_MATCH", "DataFusion regex function", "REGEXP_MATCH(${1:string}, ${2:pattern})"), + functionEntry("REGEXP_REPLACE", "DataFusion regex function", "REGEXP_REPLACE(${1:string}, ${2:pattern}, ${3:replacement})"), + functionEntry("REPEAT", "DataFusion string function", "REPEAT(${1:string}, ${2:n})"), + functionEntry("REPLACE", "DataFusion string function", "REPLACE(${1:string}, ${2:from}, ${3:to})"), + functionEntry("REVERSE", "DataFusion unicode function", "REVERSE(${1:string})"), + functionEntry("RIGHT", "DataFusion unicode function", "RIGHT(${1:string}, ${2:n})"), + functionEntry("ROUND", "DataFusion math function", "ROUND(${1:number})"), + functionEntry("RPAD", "DataFusion unicode function", "RPAD(${1:string}, ${2:length}, ${3:fill})"), + functionEntry("RTRIM", "DataFusion string function", "RTRIM(${1:string})"), + functionEntry("SHA224", "DataFusion crypto function", "SHA224(${1:value})"), + functionEntry("SHA256", "DataFusion crypto function", "SHA256(${1:value})"), + functionEntry("SHA384", "DataFusion crypto function", "SHA384(${1:value})"), + functionEntry("SHA512", "DataFusion crypto function", "SHA512(${1:value})"), + functionEntry("SIGNUM", "DataFusion math function", "SIGNUM(${1:number})"), + functionEntry("SIN", "DataFusion math function", "SIN(${1:number})"), + functionEntry("SINH", "DataFusion math function", "SINH(${1:number})"), + functionEntry("SPLIT_PART", "DataFusion string function", "SPLIT_PART(${1:string}, ${2:delimiter}, ${3:index})"), + functionEntry("SQRT", "DataFusion math function", "SQRT(${1:number})"), + functionEntry("STARTS_WITH", "DataFusion string function", "STARTS_WITH(${1:string}, ${2:prefix})"), + functionEntry("STRPOS", "DataFusion unicode function", "STRPOS(${1:string}, ${2:substring})"), + functionEntry("STRUCT", "DataFusion core function", "STRUCT(${1:value})"), + functionEntry("SUBSTR", "DataFusion unicode function", "SUBSTR(${1:string}, ${2:start})"), + functionEntry("TAN", "DataFusion math function", "TAN(${1:number})"), + functionEntry("TANH", "DataFusion math function", "TANH(${1:number})"), + functionEntry("TO_CHAR", "DataFusion datetime function", "TO_CHAR(${1:value}, ${2:format})"), + functionEntry("TO_DATE", "DataFusion datetime function", "TO_DATE(${1:value})"), + functionEntry("TO_HEX", "DataFusion string function", "TO_HEX(${1:value})"), + functionEntry("TO_LOCAL_TIME", "DataFusion datetime function", "TO_LOCAL_TIME(${1:value})"), + functionEntry("TO_TIME", "DataFusion datetime function", "TO_TIME(${1:value})"), + functionEntry("TO_TIMESTAMP", "DataFusion datetime function", "TO_TIMESTAMP(${1:value})"), + functionEntry("TO_TIMESTAMP_SECONDS", "DataFusion datetime function", "TO_TIMESTAMP_SECONDS(${1:value})"), + functionEntry("TO_TIMESTAMP_MILLIS", "DataFusion datetime function", "TO_TIMESTAMP_MILLIS(${1:value})"), + functionEntry("TO_TIMESTAMP_MICROS", "DataFusion datetime function", "TO_TIMESTAMP_MICROS(${1:value})"), + functionEntry("TO_TIMESTAMP_NANOS", "DataFusion datetime function", "TO_TIMESTAMP_NANOS(${1:value})"), + functionEntry("TO_UNIXTIME", "DataFusion datetime function", "TO_UNIXTIME(${1:value})"), + functionEntry("TRANSLATE", "DataFusion unicode function", "TRANSLATE(${1:string}, ${2:from}, ${3:to})"), + functionEntry("TRIM", "DataFusion string function", "TRIM(${1:string})"), + functionEntry("TRUNC", "DataFusion math function", "TRUNC(${1:number})"), + functionEntry("UNION_EXTRACT", "DataFusion core function", "UNION_EXTRACT(${1:value}, ${2:field})"), + functionEntry("UNION_TAG", "DataFusion core function", "UNION_TAG(${1:value})"), + functionEntry("UPPER", "DataFusion string function", "UPPER(${1:string})"), + functionEntry("UUID", "DataFusion string function"), + functionEntry("VERSION", "DataFusion core function"), +] as const; + +const DATAFUSION_AGGREGATE_FUNCTION_COMPLETIONS = [ + functionEntry("APPROX_DISTINCT", "DataFusion aggregate function", "APPROX_DISTINCT(${1:expression})"), + functionEntry("APPROX_MEDIAN", "DataFusion aggregate function", "APPROX_MEDIAN(${1:expression})"), + functionEntry("APPROX_PERCENTILE_CONT", "DataFusion aggregate function", "APPROX_PERCENTILE_CONT(${1:expression}, ${2:percentile})"), + functionEntry("APPROX_PERCENTILE_CONT_WITH_WEIGHT", "DataFusion aggregate function", "APPROX_PERCENTILE_CONT_WITH_WEIGHT(${1:expression}, ${2:weight}, ${3:percentile})"), + functionEntry("ARRAY_AGG", "DataFusion aggregate function", "ARRAY_AGG(${1:expression})"), + functionEntry("AVG", "DataFusion aggregate function", "AVG(${1:expression})"), + functionEntry("BIT_AND", "DataFusion aggregate function", "BIT_AND(${1:expression})"), + functionEntry("BIT_OR", "DataFusion aggregate function", "BIT_OR(${1:expression})"), + functionEntry("BIT_XOR", "DataFusion aggregate function", "BIT_XOR(${1:expression})"), + functionEntry("BOOL_AND", "DataFusion aggregate function", "BOOL_AND(${1:expression})"), + functionEntry("BOOL_OR", "DataFusion aggregate function", "BOOL_OR(${1:expression})"), + functionEntry("CORR", "DataFusion aggregate function", "CORR(${1:y}, ${2:x})"), + functionEntry("COUNT", "DataFusion aggregate function", "COUNT(${1:*})"), + functionEntry("COVAR_POP", "DataFusion aggregate function", "COVAR_POP(${1:y}, ${2:x})"), + functionEntry("COVAR_SAMP", "DataFusion aggregate function", "COVAR_SAMP(${1:y}, ${2:x})"), + functionEntry("GROUPING", "DataFusion aggregate function", "GROUPING(${1:expression})"), + functionEntry("MAX", "DataFusion aggregate function", "MAX(${1:expression})"), + functionEntry("MEDIAN", "DataFusion aggregate function", "MEDIAN(${1:expression})"), + functionEntry("MIN", "DataFusion aggregate function", "MIN(${1:expression})"), + functionEntry("NTH_VALUE", "DataFusion aggregate/window function", "NTH_VALUE(${1:expression}, ${2:n})"), + functionEntry("PERCENTILE_CONT", "DataFusion aggregate function", "PERCENTILE_CONT(${1:expression}, ${2:percentile})"), + functionEntry("REGR_AVGX", "DataFusion regression aggregate", "REGR_AVGX(${1:y}, ${2:x})"), + functionEntry("REGR_AVGY", "DataFusion regression aggregate", "REGR_AVGY(${1:y}, ${2:x})"), + functionEntry("REGR_COUNT", "DataFusion regression aggregate", "REGR_COUNT(${1:y}, ${2:x})"), + functionEntry("REGR_INTERCEPT", "DataFusion regression aggregate", "REGR_INTERCEPT(${1:y}, ${2:x})"), + functionEntry("REGR_R2", "DataFusion regression aggregate", "REGR_R2(${1:y}, ${2:x})"), + functionEntry("REGR_SLOPE", "DataFusion regression aggregate", "REGR_SLOPE(${1:y}, ${2:x})"), + functionEntry("REGR_SXX", "DataFusion regression aggregate", "REGR_SXX(${1:y}, ${2:x})"), + functionEntry("REGR_SXY", "DataFusion regression aggregate", "REGR_SXY(${1:y}, ${2:x})"), + functionEntry("REGR_SYY", "DataFusion regression aggregate", "REGR_SYY(${1:y}, ${2:x})"), + functionEntry("STDDEV", "DataFusion aggregate function", "STDDEV(${1:expression})"), + functionEntry("STDDEV_POP", "DataFusion aggregate function", "STDDEV_POP(${1:expression})"), + functionEntry("STRING_AGG", "DataFusion aggregate function", "STRING_AGG(${1:expression}, ${2:delimiter})"), + functionEntry("SUM", "DataFusion aggregate function", "SUM(${1:expression})"), + functionEntry("VAR_POP", "DataFusion aggregate function", "VAR_POP(${1:expression})"), + functionEntry("VAR_SAMPLE", "DataFusion aggregate function", "VAR_SAMPLE(${1:expression})"), +] as const; + +const DATAFUSION_WINDOW_FUNCTION_COMPLETIONS = [ + functionEntry("CUME_DIST", "DataFusion window function"), + functionEntry("DENSE_RANK", "DataFusion window function"), + functionEntry("FIRST_VALUE", "DataFusion window function", "FIRST_VALUE(${1:expression})"), + functionEntry("LAG", "DataFusion window function", "LAG(${1:expression})"), + functionEntry("LAST_VALUE", "DataFusion window function", "LAST_VALUE(${1:expression})"), + functionEntry("LEAD", "DataFusion window function", "LEAD(${1:expression})"), + functionEntry("NTILE", "DataFusion window function", "NTILE(${1:buckets})"), + functionEntry("PERCENT_RANK", "DataFusion window function"), + functionEntry("RANK", "DataFusion window function"), + functionEntry("ROW_NUMBER", "DataFusion window function"), +] as const; + +const DATAFUSION_NESTED_FUNCTION_COMPLETIONS = [ + functionEntry("ARRAY_ANY_VALUE", "DataFusion nested function", "ARRAY_ANY_VALUE(${1:array})"), + functionEntry("ARRAY_APPEND", "DataFusion nested function", "ARRAY_APPEND(${1:array}, ${2:value})"), + functionEntry("ARRAY_CONCAT", "DataFusion nested function", "ARRAY_CONCAT(${1:array})"), + functionEntry("ARRAY_DIMS", "DataFusion nested function", "ARRAY_DIMS(${1:array})"), + functionEntry("ARRAY_DISTINCT", "DataFusion nested function", "ARRAY_DISTINCT(${1:array})"), + functionEntry("ARRAY_DISTANCE", "DataFusion nested function", "ARRAY_DISTANCE(${1:left}, ${2:right})"), + functionEntry("ARRAY_ELEMENT", "DataFusion nested function", "ARRAY_ELEMENT(${1:array}, ${2:index})"), + functionEntry("ARRAY_EMPTY", "DataFusion nested function", "ARRAY_EMPTY(${1:array})"), + functionEntry("ARRAY_EXCEPT", "DataFusion nested function", "ARRAY_EXCEPT(${1:left}, ${2:right})"), + functionEntry("ARRAY_HAS", "DataFusion nested function", "ARRAY_HAS(${1:array}, ${2:value})"), + functionEntry("ARRAY_HAS_ALL", "DataFusion nested function", "ARRAY_HAS_ALL(${1:array}, ${2:values})"), + functionEntry("ARRAY_HAS_ANY", "DataFusion nested function", "ARRAY_HAS_ANY(${1:array}, ${2:values})"), + functionEntry("ARRAY_INTERSECT", "DataFusion nested function", "ARRAY_INTERSECT(${1:left}, ${2:right})"), + functionEntry("ARRAY_LENGTH", "DataFusion nested function", "ARRAY_LENGTH(${1:array})"), + functionEntry("ARRAY_MAX", "DataFusion nested function", "ARRAY_MAX(${1:array})"), + functionEntry("ARRAY_MIN", "DataFusion nested function", "ARRAY_MIN(${1:array})"), + functionEntry("ARRAY_NDIMS", "DataFusion nested function", "ARRAY_NDIMS(${1:array})"), + functionEntry("ARRAY_POP_BACK", "DataFusion nested function", "ARRAY_POP_BACK(${1:array})"), + functionEntry("ARRAY_POP_FRONT", "DataFusion nested function", "ARRAY_POP_FRONT(${1:array})"), + functionEntry("ARRAY_POSITION", "DataFusion nested function", "ARRAY_POSITION(${1:array}, ${2:value})"), + functionEntry("ARRAY_POSITIONS", "DataFusion nested function", "ARRAY_POSITIONS(${1:array}, ${2:value})"), + functionEntry("ARRAY_PREPEND", "DataFusion nested function", "ARRAY_PREPEND(${1:value}, ${2:array})"), + functionEntry("ARRAY_REMOVE", "DataFusion nested function", "ARRAY_REMOVE(${1:array}, ${2:value})"), + functionEntry("ARRAY_REMOVE_ALL", "DataFusion nested function", "ARRAY_REMOVE_ALL(${1:array}, ${2:value})"), + functionEntry("ARRAY_REMOVE_N", "DataFusion nested function", "ARRAY_REMOVE_N(${1:array}, ${2:value}, ${3:n})"), + functionEntry("ARRAY_REPEAT", "DataFusion nested function", "ARRAY_REPEAT(${1:value}, ${2:n})"), + functionEntry("ARRAY_REPLACE", "DataFusion nested function", "ARRAY_REPLACE(${1:array}, ${2:from}, ${3:to})"), + functionEntry("ARRAY_REPLACE_ALL", "DataFusion nested function", "ARRAY_REPLACE_ALL(${1:array}, ${2:from}, ${3:to})"), + functionEntry("ARRAY_REPLACE_N", "DataFusion nested function", "ARRAY_REPLACE_N(${1:array}, ${2:from}, ${3:to}, ${4:n})"), + functionEntry("ARRAY_RESIZE", "DataFusion nested function", "ARRAY_RESIZE(${1:array}, ${2:size})"), + functionEntry("ARRAY_REVERSE", "DataFusion nested function", "ARRAY_REVERSE(${1:array})"), + functionEntry("ARRAY_SLICE", "DataFusion nested function", "ARRAY_SLICE(${1:array}, ${2:start}, ${3:end})"), + functionEntry("ARRAY_SORT", "DataFusion nested function", "ARRAY_SORT(${1:array})"), + functionEntry("ARRAY_TO_STRING", "DataFusion nested function", "ARRAY_TO_STRING(${1:array}, ${2:delimiter})"), + functionEntry("ARRAY_UNION", "DataFusion nested function", "ARRAY_UNION(${1:left}, ${2:right})"), + functionEntry("ARRAYS_ZIP", "DataFusion nested function", "ARRAYS_ZIP(${1:array})"), + functionEntry("CARDINALITY", "DataFusion nested function", "CARDINALITY(${1:value})"), + functionEntry("FLATTEN", "DataFusion nested function", "FLATTEN(${1:array})"), + functionEntry("GEN_SERIES", "DataFusion nested function", "GEN_SERIES(${1:start}, ${2:stop})"), + functionEntry("MAKE_ARRAY", "DataFusion nested function", "MAKE_ARRAY(${1:value})"), + functionEntry("MAP", "DataFusion nested function", "MAP(${1:key}, ${2:value})"), + functionEntry("MAP_ENTRIES", "DataFusion nested function", "MAP_ENTRIES(${1:map})"), + functionEntry("MAP_EXTRACT", "DataFusion nested function", "MAP_EXTRACT(${1:map}, ${2:key})"), + functionEntry("MAP_KEYS", "DataFusion nested function", "MAP_KEYS(${1:map})"), + functionEntry("MAP_VALUES", "DataFusion nested function", "MAP_VALUES(${1:map})"), + functionEntry("RANGE", "DataFusion nested function", "RANGE(${1:start}, ${2:stop})"), + functionEntry("STRING_TO_ARRAY", "DataFusion nested function", "STRING_TO_ARRAY(${1:string}, ${2:delimiter})"), +] as const; + +const DATAFUSION_JSON_FUNCTION_COMPLETIONS = [ + functionEntry("JSON_AS_TEXT", "DataFusion JSON function", "JSON_AS_TEXT(${1:json}, ${2:path})"), + functionEntry("JSON_CONTAINS", "DataFusion JSON function", "JSON_CONTAINS(${1:json}, ${2:key})"), + functionEntry("JSON_FROM_SCALAR", "DataFusion JSON function", "JSON_FROM_SCALAR(${1:value})"), + functionEntry("JSON_GET", "DataFusion JSON function", "JSON_GET(${1:json}, ${2:path})"), + functionEntry("JSON_GET_ARRAY", "DataFusion JSON function", "JSON_GET_ARRAY(${1:json}, ${2:path})"), + functionEntry("JSON_GET_BOOL", "DataFusion JSON function", "JSON_GET_BOOL(${1:json}, ${2:path})"), + functionEntry("JSON_GET_FLOAT", "DataFusion JSON function", "JSON_GET_FLOAT(${1:json}, ${2:path})"), + functionEntry("JSON_GET_INT", "DataFusion JSON function", "JSON_GET_INT(${1:json}, ${2:path})"), + functionEntry("JSON_GET_JSON", "DataFusion JSON function", "JSON_GET_JSON(${1:json}, ${2:path})"), + functionEntry("JSON_GET_STR", "DataFusion JSON function", "JSON_GET_STR(${1:json}, ${2:path})"), + functionEntry("JSON_LENGTH", "DataFusion JSON function", "JSON_LENGTH(${1:json})"), + functionEntry("JSON_OBJECT_KEYS", "DataFusion JSON function", "JSON_OBJECT_KEYS(${1:json})"), +] as const; + +const SQL_FUNCTION_COMPLETIONS = [ + ...KALAMDB_FUNCTION_COMPLETIONS, + ...DATAFUSION_SCALAR_FUNCTION_COMPLETIONS, + ...DATAFUSION_AGGREGATE_FUNCTION_COMPLETIONS, + ...DATAFUSION_WINDOW_FUNCTION_COMPLETIONS, + ...DATAFUSION_NESTED_FUNCTION_COMPLETIONS, + ...DATAFUSION_JSON_FUNCTION_COMPLETIONS, +] as const; + +const SQL_SYNTAX_COMPLETIONS = [ + syntaxEntry("SELECT", "SQL query", "SELECT ${1:*}\nFROM ${2:table}\nWHERE ${3:condition};"), + syntaxEntry("WITH", "SQL common table expression", "WITH ${1:cte} AS (\n SELECT ${2:*}\n FROM ${3:table}\n)\nSELECT * FROM ${1:cte};"), + syntaxEntry("INSERT INTO", "SQL insert", "INSERT INTO ${1:table} (${2:columns})\nVALUES (${3:values});"), + syntaxEntry("UPDATE", "SQL update", "UPDATE ${1:table}\nSET ${2:column} = ${3:value}\nWHERE ${4:condition};"), + syntaxEntry("DELETE FROM", "SQL delete", "DELETE FROM ${1:table}\nWHERE ${2:condition};"), + syntaxEntry("CREATE TABLE", "KalamDB table DDL", "CREATE TABLE ${1:namespace}.${2:table} (\n ${3:id} TEXT PRIMARY KEY,\n ${4:created_at} TIMESTAMP DEFAULT NOW()\n);"), + syntaxEntry("CREATE USER TABLE", "KalamDB user table DDL", "CREATE USER TABLE ${1:namespace}.${2:table} (\n ${3:id} TEXT PRIMARY KEY\n);"), + syntaxEntry("CREATE SHARED TABLE", "KalamDB shared table DDL", "CREATE SHARED TABLE ${1:namespace}.${2:table} (\n ${3:id} TEXT PRIMARY KEY\n);"), + syntaxEntry("CREATE STREAM TABLE", "KalamDB stream table DDL", "CREATE STREAM TABLE ${1:namespace}.${2:table} (\n ${3:id} TEXT PRIMARY KEY\n);"), + syntaxEntry("ALTER TABLE", "KalamDB table DDL", "ALTER TABLE ${1:namespace}.${2:table}\nADD COLUMN ${3:column} ${4:TEXT};"), + syntaxEntry("DROP TABLE", "KalamDB table DDL", "DROP TABLE ${1:namespace}.${2:table};"), + syntaxEntry("CREATE VIEW", "KalamDB view DDL", "CREATE VIEW ${1:namespace}.${2:view_name} AS\nSELECT ${3:*}\nFROM ${4:table};"), + syntaxEntry("CREATE NAMESPACE", "KalamDB namespace DDL", "CREATE NAMESPACE ${1:namespace};"), + syntaxEntry("ALTER NAMESPACE", "KalamDB namespace DDL", "ALTER NAMESPACE ${1:namespace} SET OPTIONS (${2:key} = '${3:value}');"), + syntaxEntry("DROP NAMESPACE", "KalamDB namespace DDL", "DROP NAMESPACE ${1:namespace};"), + syntaxEntry("SHOW NAMESPACES", "KalamDB metadata command", "SHOW NAMESPACES;"), + syntaxEntry("USE NAMESPACE", "KalamDB namespace command", "USE NAMESPACE ${1:namespace};"), + syntaxEntry("SET NAMESPACE", "KalamDB namespace command", "SET NAMESPACE ${1:namespace};"), + syntaxEntry("SHOW TABLES", "KalamDB metadata command", "SHOW TABLES;"), + syntaxEntry("DESCRIBE TABLE", "KalamDB metadata command", "DESCRIBE TABLE ${1:namespace}.${2:table};"), + syntaxEntry("DESC TABLE", "KalamDB metadata command", "DESC TABLE ${1:namespace}.${2:table};"), + syntaxEntry("SHOW STATS FOR TABLE", "KalamDB metadata command", "SHOW STATS FOR TABLE ${1:namespace}.${2:table};"), + syntaxEntry("EXPLAIN", "DataFusion meta command", "EXPLAIN ${1:SELECT * FROM table};"), + syntaxEntry("EXPLAIN ANALYZE", "DataFusion meta command", "EXPLAIN ANALYZE ${1:SELECT * FROM table};"), + syntaxEntry("DESCRIBE", "DataFusion meta command", "DESCRIBE ${1:table};"), + syntaxEntry("DESC", "DataFusion meta command", "DESC ${1:table};"), + syntaxEntry("SHOW COLUMNS", "DataFusion meta command", "SHOW COLUMNS FROM ${1:table};"), + syntaxEntry("SHOW ALL", "DataFusion meta command", "SHOW ALL;"), + syntaxEntry("SET", "DataFusion session setting", "SET ${1:key} = ${2:value};"), + syntaxEntry("EXECUTE AS USER", "KalamDB impersonation wrapper", "EXECUTE AS USER '${1:username}' (\n ${2:SELECT * FROM table}\n);"), + syntaxEntry("BEGIN", "KalamDB transaction control", "BEGIN;"), + syntaxEntry("START TRANSACTION", "KalamDB transaction control", "START TRANSACTION;"), + syntaxEntry("COMMIT", "KalamDB transaction control", "COMMIT;"), + syntaxEntry("ROLLBACK", "KalamDB transaction control", "ROLLBACK;"), + syntaxEntry("SUBSCRIBE TO", "KalamDB live query subscription", "SUBSCRIBE TO ${1:namespace}.${2:table}\nWHERE ${3:user_id = CURRENT_USER()}\nOPTIONS (last_rows=${4:100});"), + syntaxEntry("SUBSCRIBE TO SELECT", "KalamDB live query subscription", "SUBSCRIBE TO SELECT ${1:*}\nFROM ${2:namespace}.${3:table}\nWHERE ${4:user_id = CURRENT_USER()}\nOPTIONS (last_rows=${5:100}, batch_size=${6:50});"), + syntaxEntry("KILL LIVE QUERY", "KalamDB live query command", "KILL LIVE QUERY '${1:live_id}';"), + syntaxEntry("KILL JOB", "KalamDB job command", "KILL JOB '${1:job_id}';"), + syntaxEntry("STORAGE FLUSH ALL", "KalamDB storage maintenance", "STORAGE FLUSH ALL;"), + syntaxEntry("STORAGE FLUSH TABLE", "KalamDB storage maintenance", "STORAGE FLUSH TABLE ${1:namespace}.${2:table};"), + syntaxEntry("STORAGE COMPACT ALL", "KalamDB storage maintenance", "STORAGE COMPACT ALL;"), + syntaxEntry("STORAGE COMPACT TABLE", "KalamDB storage maintenance", "STORAGE COMPACT TABLE ${1:namespace}.${2:table};"), + syntaxEntry("STORAGE CHECK", "KalamDB storage maintenance", "STORAGE CHECK ${1:storage_id};"), + syntaxEntry("SHOW MANIFEST", "KalamDB storage metadata", "SHOW MANIFEST;"), + syntaxEntry("CREATE STORAGE", "KalamDB storage DDL", "CREATE STORAGE ${1:storage_id}\nTYPE filesystem\nNAME '${2:Local Storage}'\nPATH '${3:/var/lib/kalamdb/parquet}'\nSHARED_TABLES_TEMPLATE '${4:{namespace}/{tableName}/}'\nUSER_TABLES_TEMPLATE '${5:{namespace}/{tableName}/{userId}/}';"), + syntaxEntry("ALTER STORAGE", "KalamDB storage DDL", "ALTER STORAGE ${1:storage_id}\nSET NAME '${2:Storage Name}';"), + syntaxEntry("DROP STORAGE", "KalamDB storage DDL", "DROP STORAGE ${1:storage_id};"), + syntaxEntry("SHOW STORAGES", "KalamDB storage metadata", "SHOW STORAGES;"), + syntaxEntry("CREATE TOPIC", "KalamDB topic command", "CREATE TOPIC ${1:topic_name} PARTITIONS ${2:1};"), + syntaxEntry("DROP TOPIC", "KalamDB topic command", "DROP TOPIC ${1:topic_name};"), + syntaxEntry("CLEAR TOPIC", "KalamDB topic command", "CLEAR TOPIC ${1:topic_name};"), + syntaxEntry("ALTER TOPIC ADD SOURCE", "KalamDB topic command", "ALTER TOPIC ${1:topic_name}\nADD SOURCE ${2:namespace}.${3:table}\nON ${4:insert}\nWHERE ${5:condition};"), + syntaxEntry("CONSUME FROM", "KalamDB topic command", "CONSUME FROM ${1:topic_name}\nGROUP '${2:group_id}'\nFROM ${3:latest}\nLIMIT ${4:100};"), + syntaxEntry("ACK", "KalamDB topic command", "ACK ${1:topic_name}\nGROUP '${2:group_id}'\nPARTITION ${3:0}\nUPTO OFFSET ${4:offset};"), + syntaxEntry("BACKUP DATABASE", "KalamDB backup command", "BACKUP DATABASE TO '${1:/backups/kalamdb.tar.gz}';"), + syntaxEntry("RESTORE DATABASE", "KalamDB restore command", "RESTORE DATABASE FROM '${1:/backups/kalamdb.tar.gz}';"), + syntaxEntry("EXPORT USER DATA", "KalamDB export command", "EXPORT USER DATA;"), + syntaxEntry("SHOW EXPORT", "KalamDB export command", "SHOW EXPORT;"), + syntaxEntry("CREATE USER", "KalamDB user management", "CREATE USER ${1:username} WITH PASSWORD '${2:password}' ROLE ${3:user};"), + syntaxEntry("ALTER USER", "KalamDB user management", "ALTER USER ${1:username} SET ${2:ROLE} ${3:user};"), + syntaxEntry("DROP USER", "KalamDB user management", "DROP USER ${1:username};"), + syntaxEntry("CLUSTER SNAPSHOT", "KalamDB cluster command", "CLUSTER SNAPSHOT;"), + syntaxEntry("CLUSTER PURGE", "KalamDB cluster command", "CLUSTER PURGE --upto ${1:index};"), + syntaxEntry("CLUSTER TRIGGER ELECTION", "KalamDB cluster command", "CLUSTER TRIGGER ELECTION;"), + syntaxEntry("CLUSTER TRANSFER LEADER", "KalamDB cluster command", "CLUSTER TRANSFER LEADER ${1:node_id};"), + syntaxEntry("CLUSTER JOIN", "KalamDB cluster command", "CLUSTER JOIN ${1:node_id} RPC ${2:rpc_addr} API ${3:api_addr};"), + syntaxEntry("CLUSTER REBALANCE", "KalamDB cluster command", "CLUSTER REBALANCE;"), + syntaxEntry("CLUSTER STEPDOWN", "KalamDB cluster command", "CLUSTER STEPDOWN;"), + syntaxEntry("CLUSTER CLEAR", "KalamDB cluster command", "CLUSTER CLEAR;"), +] as const; + +const SQL_OPERATOR_COMPLETIONS = [ + operatorEntry("->", "JSON object operator"), + operatorEntry("->>", "JSON text operator"), + operatorEntry("?", "JSON contains operator"), + operatorEntry("=", "SQL comparison operator"), + operatorEntry("!=", "SQL comparison operator"), + operatorEntry("<>", "SQL comparison operator"), + operatorEntry("<=", "SQL comparison operator"), + operatorEntry(">=", "SQL comparison operator"), +] as const; + +const SQL_TYPE_COMPLETIONS = SQL_TYPE_NAMES.map(typeEntry); + +const EMPTY_COMPLETION_DATA: SqlCompletionData = { + namespaces: [], + tablesByNamespace: {}, + columnsByTable: {}, + keywords: SQL_KEYWORDS, + functions: SQL_FUNCTION_COMPLETIONS, + snippets: SQL_SYNTAX_COMPLETIONS, + types: SQL_TYPE_COMPLETIONS, + operators: SQL_OPERATOR_COMPLETIONS, +}; + +function pushUnique(values: string[], value: string) { + if (!values.some((item) => item.toLowerCase() === value.toLowerCase())) { + values.push(value); + } +} + +function addTable( + data: Pick, + namespaceName: string, + tableName: string, + columns: readonly string[], +) { + const namespaceKey = namespaceName.toLowerCase(); + const tableKey = tableName.toLowerCase(); + const qualifiedTable = `${namespaceKey}.${tableKey}`; + + pushUnique(data.namespaces, namespaceName); + if (!data.tablesByNamespace[namespaceKey]) { + data.tablesByNamespace[namespaceKey] = []; + } + pushUnique(data.tablesByNamespace[namespaceKey], tableName); + + if (!data.columnsByTable[qualifiedTable]) { + data.columnsByTable[qualifiedTable] = []; + } + columns.forEach((column) => pushUnique(data.columnsByTable[qualifiedTable], column)); +} + +export function buildSqlCompletionData(schema: StudioNamespace[]): SqlCompletionData { + const data: SqlCompletionData = { + ...EMPTY_COMPLETION_DATA, + namespaces: [], + tablesByNamespace: {}, + columnsByTable: {}, + }; + + schema.forEach((namespace) => { + namespace.tables.forEach((table) => { + addTable(data, namespace.name, table.name, table.columns.map((column) => column.name)); + }); + }); + + STATIC_SYSTEM_TABLES.forEach((table) => { + addTable(data, table.namespace, table.name, table.columns); + }); + + return data; +} + +export function resolveSqlContextualCompletions( + data: Pick, + textUntilPosition: string, + aliasToTable: Record, +): SqlContextualCompletionMatch | null { + const tableColumnMatch = /([a-zA-Z_][\w]*)\.([a-zA-Z_][\w]*)\.([a-zA-Z_][\w]*)?$/.exec(textUntilPosition); + if (tableColumnMatch) { + const namespaceKey = tableColumnMatch[1].toLowerCase(); + const tableKey = tableColumnMatch[2].toLowerCase(); + const columns = data.columnsByTable[`${namespaceKey}.${tableKey}`] ?? []; + return { + kind: "column", + labels: columns, + detail: `${namespaceKey}.${tableKey} column`, + partial: tableColumnMatch[3]?.toLowerCase() ?? "", + }; + } + + const identifierMatch = /([a-zA-Z_][\w]*)\.([a-zA-Z_][\w]*)?$/.exec(textUntilPosition); + if (!identifierMatch) { + return null; + } + + const qualifierKey = identifierMatch[1].toLowerCase(); + const partial = identifierMatch[2]?.toLowerCase() ?? ""; + const resolvedTable = aliasToTable[qualifierKey]; + if (resolvedTable) { + return { + kind: "column", + labels: data.columnsByTable[resolvedTable] ?? [], + detail: `${qualifierKey} alias column`, + partial, + }; + } + + const namespaceTables = data.tablesByNamespace[qualifierKey] ?? []; + if (namespaceTables.length > 0) { + return { + kind: "table", + labels: namespaceTables, + detail: `${qualifierKey} table`, + partial, + }; + } + + return null; +} \ No newline at end of file diff --git a/ui/src/pages/SqlStudio.test.tsx b/ui/src/pages/SqlStudio.test.tsx index 3f2631618..32850963e 100644 --- a/ui/src/pages/SqlStudio.test.tsx +++ b/ui/src/pages/SqlStudio.test.tsx @@ -353,6 +353,16 @@ describe("SqlStudio page", () => { vi.unstubAllGlobals(); }); + it("renders the main SQL Studio controls on load", async () => { + await renderSqlStudio(); + + expect(getSqlEditor()).toBeTruthy(); + expect(screen.getByRole("button", { name: /^execute$/i })).toBeTruthy(); + expect(screen.getByRole("button", { name: /refresh explorer/i })).toBeTruthy(); + expect(screen.getByTitle("New query tab")).toBeTruthy(); + expect(screen.getByTitle("Expand details panel")).toBeTruthy(); + }); + it("runs a query from the SQL Studio page and renders the results grid", async () => { mockExecuteSqlStudioQuery.mockResolvedValue({ status: "success", @@ -380,6 +390,36 @@ describe("SqlStudio page", () => { expect(await screen.findByText("Ada")).toBeTruthy(); }); + it("runs the active query through the editor command binding", async () => { + mockExecuteSqlStudioQuery.mockResolvedValue({ + status: "success", + rows: [{ id: 42, name: "Grace" }], + schema: [ + { name: "id", dataType: "INT", index: 0, isPrimaryKey: true }, + { name: "name", dataType: "TEXT", index: 1, isPrimaryKey: false }, + ], + tookMs: 8, + rowCount: 1, + logs: [], + }); + + await renderSqlStudio(); + + fireEvent.change(getSqlEditor(), { + target: { value: "SELECT id, name FROM default.events WHERE id = 42" }, + }); + + await act(async () => { + latestEditorCommand?.(); + }); + + await waitFor(() => { + expect(mockExecuteSqlStudioQuery).toHaveBeenCalledWith("SELECT id, name FROM default.events WHERE id = 42"); + }); + + expect(await screen.findByText("Grace")).toBeTruthy(); + }); + it("starts with an empty query editor and opens new tabs empty", async () => { const { store } = await renderSqlStudio(); diff --git a/ui/src/services/sqlStudioService.ts b/ui/src/services/sqlStudioService.ts index 7db62f88f..aefe7d898 100644 --- a/ui/src/services/sqlStudioService.ts +++ b/ui/src/services/sqlStudioService.ts @@ -347,7 +347,7 @@ function normalizeTimestampValue(value: unknown): string | number | null { function applyTableMetadata( table: StudioTable, - row: Pick, + row: Partial>, ): void { table.storageId = normalizeTextValue(row.storage_id); table.version = normalizeNumericValue(row.schema_version); diff --git a/ui/tsconfig.json b/ui/tsconfig.json index a2b26562b..261c1e867 100644 --- a/ui/tsconfig.json +++ b/ui/tsconfig.json @@ -19,6 +19,7 @@ "paths": { "@/*": ["./src/*"], "@kalamdb/client": ["../link/sdks/typescript/client/dist/src/index.d.ts"], + "@kalamdb/orm": ["../link/sdks/typescript/orm/dist/index.d.ts"], "drizzle-orm": ["./node_modules/drizzle-orm"], "drizzle-orm/*": ["./node_modules/drizzle-orm/*"] } diff --git a/ui/vite.config.ts b/ui/vite.config.ts index 30e3c8656..787792ce1 100644 --- a/ui/vite.config.ts +++ b/ui/vite.config.ts @@ -48,13 +48,17 @@ export default defineConfig(({ mode }) => { base: "/ui/", resolve: { preserveSymlinks: true, - dedupe: ["drizzle-orm", "@kalamdb/client"], + dedupe: ["drizzle-orm", "@kalamdb/client", "@kalamdb/orm"], alias: { "@": path.resolve(__dirname, "./src"), "@kalamdb/client": path.resolve( __dirname, "../link/sdks/typescript/client/dist/src/index.js", ), + "@kalamdb/orm": path.resolve( + __dirname, + "../link/sdks/typescript/orm/dist/index.js", + ), }, }, server: { @@ -74,6 +78,7 @@ export default defineConfig(({ mode }) => { allow: [ path.resolve(__dirname, "."), path.resolve(__dirname, "../link/sdks/typescript/client"), + path.resolve(__dirname, "../link/sdks/typescript/orm"), ], }, // Disable caching for WASM and SDK files @@ -102,7 +107,7 @@ export default defineConfig(({ mode }) => { force: true, // Exclude the SDK from pre-bundling so WASM files load correctly // When pre-bundled, import.meta.url points to .vite/deps which breaks WASM loading - exclude: ["@kalamdb/client"], + exclude: ["@kalamdb/client", "@kalamdb/orm"], }, // Ensure WASM files are handled correctly assetsInclude: ["**/*.wasm"],