diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 181d93e7..22ee788c 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -230,6 +230,15 @@ jobs: cd ui npm install + - name: Run SQL Studio regression tests + shell: bash + env: + CI: "true" + run: | + set -euo pipefail + cd ui + npm run test:sql-studio + - name: Run Admin UI tests shell: bash env: @@ -237,7 +246,7 @@ jobs: run: | set -euo pipefail cd ui - npm test -- --run + npm run test:ci - name: Build Admin UI shell: bash diff --git a/Cargo.lock b/Cargo.lock index eeb9de7a..05d49168 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -498,9 +498,9 @@ checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" [[package]] name = "arrow" -version = "58.1.0" +version = "58.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d441fdda254b65f3e9025910eb2c2066b6295d9c8ed409522b8d2ace1ff8574c" +checksum = "607e64bb911ee4f90483e044fe78f175989148c2892e659a2cd25429e782ec54" dependencies = [ "arrow-arith", "arrow-array", @@ -519,9 +519,9 @@ dependencies = [ [[package]] name = "arrow-arith" -version = "58.1.0" +version = "58.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ced5406f8b720cc0bc3aa9cf5758f93e8593cda5490677aa194e4b4b383f9a59" +checksum = "e754319ed8a85d817fe7adf183227e0b5308b82790a737b426c1124626b48118" dependencies = [ "arrow-array", "arrow-buffer", @@ -533,9 +533,9 @@ dependencies = [ [[package]] name = "arrow-array" -version = "58.1.0" +version = "58.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "772bd34cacdda8baec9418d80d23d0fb4d50ef0735685bd45158b83dfeb6e62d" +checksum = "841321891f247aa86c6112c80d83d89cb36e0addd020fa2425085b8eb6c3f579" dependencies = [ "ahash 0.8.12", "arrow-buffer", @@ -544,7 +544,7 @@ dependencies = [ "chrono", "chrono-tz", "half 2.7.1", - "hashbrown 0.16.1", + "hashbrown 0.17.0", "num-complex", "num-integer", "num-traits", @@ -552,9 +552,9 @@ dependencies = [ [[package]] name = "arrow-buffer" -version = "58.1.0" +version = "58.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "898f4cf1e9598fdb77f356fdf2134feedfd0ee8d5a4e0a5f573e7d0aec16baa4" +checksum = "f955dfb73fae000425f49c8226d2044dab60fb7ad4af1e24f961756354d996c9" dependencies = [ "bytes", "half 2.7.1", @@ -564,9 +564,9 @@ dependencies = [ [[package]] name = "arrow-cast" -version = "58.1.0" +version = "58.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0127816c96533d20fc938729f48c52d3e48f99717e7a0b5ade77d742510736d" +checksum = "ca5e686972523798f76bef355145bc1ae25a84c731e650268d31ab763c701663" dependencies = [ "arrow-array", "arrow-buffer", @@ -586,9 +586,9 @@ dependencies = [ [[package]] name = "arrow-csv" -version = "58.1.0" +version = "58.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca025bd0f38eeecb57c2153c0123b960494138e6a957bbda10da2b25415209fe" +checksum = "86c276756867fc8186ec380c72c290e6e3b23a1d4fb05df6b1d62d2e62666d48" dependencies = [ "arrow-array", "arrow-cast", @@ -601,9 +601,9 @@ dependencies = [ [[package]] name = "arrow-data" -version = "58.1.0" +version = "58.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42d10beeab2b1c3bb0b53a00f7c944a178b622173a5c7bcabc3cb45d90238df4" +checksum = "db3b5846209775b6dc8056d77ff9a032b27043383dd5488abd0b663e265b9373" dependencies = [ "arrow-buffer", "arrow-schema", @@ -614,9 +614,9 @@ dependencies = [ [[package]] name = "arrow-ipc" -version = "58.1.0" +version = "58.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "609a441080e338147a84e8e6904b6da482cefb957c5cdc0f3398872f69a315d0" +checksum = "fd8907ddd8f9fbabf91ec2c85c1d81fe2874e336d2443eb36373595e28b98dd5" dependencies = [ "arrow-array", "arrow-buffer", @@ -629,15 +629,16 @@ dependencies = [ [[package]] name = "arrow-json" -version = "58.1.0" +version = "58.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ead0914e4861a531be48fe05858265cf854a4880b9ed12618b1d08cba9bebc8" +checksum = "f4518c59acc501f10d7dcae397fe12b8db3d81bc7de94456f8a58f9165d6f502" dependencies = [ "arrow-array", "arrow-buffer", "arrow-cast", - "arrow-data", + "arrow-ord", "arrow-schema", + "arrow-select", "chrono", "half 2.7.1", "indexmap", @@ -653,9 +654,9 @@ dependencies = [ [[package]] name = "arrow-ord" -version = "58.1.0" +version = "58.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "763a7ba279b20b52dad300e68cfc37c17efa65e68623169076855b3a9e941ca5" +checksum = "efa70d9d6b1356f1fb9f1f651b84a725b7e0abb93f188cf7d31f14abfa2f2e6f" dependencies = [ "arrow-array", "arrow-buffer", @@ -666,9 +667,9 @@ dependencies = [ [[package]] name = "arrow-row" -version = "58.1.0" +version = "58.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e14fe367802f16d7668163ff647830258e6e0aeea9a4d79aaedf273af3bdcd3e" +checksum = "faec88a945338192beffbbd4be0def70135422930caa244ac3cec0cd213b26b4" dependencies = [ "arrow-array", "arrow-buffer", @@ -679,9 +680,9 @@ dependencies = [ [[package]] name = "arrow-schema" -version = "58.1.0" +version = "58.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c30a1365d7a7dc50cc847e54154e6af49e4c4b0fddc9f607b687f29212082743" +checksum = "18aa020f6bc8e5201dcd2d4b7f98c68f8a410ef37128263243e6ff2a47a67d4f" dependencies = [ "serde_core", "serde_json", @@ -689,9 +690,9 @@ dependencies = [ [[package]] name = "arrow-select" -version = "58.1.0" +version = "58.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78694888660a9e8ac949853db393af2a8b8fc82c19ce333132dfa2e72cc1a7fe" +checksum = "a657ab5132e9c8ca3b24eb15a823d0ced38017fe3930ff50167466b02e2d592c" dependencies = [ "ahash 0.8.12", "arrow-array", @@ -703,9 +704,9 @@ dependencies = [ [[package]] name = "arrow-string" -version = "58.1.0" +version = "58.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61e04a01f8bb73ce54437514c5fd3ee2aa3e8abe4c777ee5cc55853b1652f79e" +checksum = "f6de2efbbd1a9f9780ceb8d1ff5d20421b35863b361e3386b4f571f1fc69fcb8" dependencies = [ "arrow-array", "arrow-buffer", @@ -3239,6 +3240,12 @@ dependencies = [ "foldhash 0.2.0", ] +[[package]] +name = "hashbrown" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f467dd6dccf739c208452f8014c75c18bb8301b050ad1cfb27153803edb0f51" + [[package]] name = "heck" version = "0.5.0" @@ -3838,7 +3845,7 @@ dependencies = [ "predicates", "rand 0.10.1", "regex", - "reqwest 0.13.2", + "reqwest 0.13.3", "rpassword", "rustyline", "serde", @@ -3862,10 +3869,11 @@ dependencies = [ "kalamdb-server", "link-common", "ntest", - "reqwest 0.13.2", + "reqwest 0.13.3", "serde_json", "tempfile", "tokio", + "tokio-netem", ] [[package]] @@ -4049,7 +4057,7 @@ dependencies = [ "log", "moka", "once_cell", - "reqwest 0.13.2", + "reqwest 0.13.3", "serde", "serde_json", "sha2 0.11.0", @@ -4556,7 +4564,7 @@ dependencies = [ "opentelemetry_sdk", "parking_lot", "rand 0.10.1", - "reqwest 0.13.2", + "reqwest 0.13.3", "serde", "serde_json", "serial_test", @@ -4937,8 +4945,9 @@ dependencies = [ "log", "miniz_oxide 0.9.1", "quinn-proto", - "reqwest 0.13.2", + "reqwest 0.13.3", "rmp-serde", + "rustls", "rustls-webpki", "serde", "serde_json", @@ -5624,9 +5633,9 @@ dependencies = [ [[package]] name = "parquet" -version = "58.1.0" +version = "58.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d3f9f2205199603564127932b89695f52b62322f541d0fc7179d57c2e1c9877" +checksum = "43d7efd3052f7d6ef601085559a246bc991e9a8cc77e02753737df6322ce35f1" dependencies = [ "ahash 0.8.12", "arrow-array", @@ -5642,7 +5651,7 @@ dependencies = [ "flate2", "futures", "half 2.7.1", - "hashbrown 0.16.1", + "hashbrown 0.17.0", "lz4_flex", "num-bigint", "num-integer", @@ -6551,9 +6560,9 @@ dependencies = [ [[package]] name = "reqwest" -version = "0.13.2" +version = "0.13.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab3f43e3283ab1488b624b44b0e988d0acea0b3214e694730a055cb6b2efa801" +checksum = "62e0021ea2c22aed41653bc7e1419abb2c97e038ff2c33d0e1309e49a97deec0" dependencies = [ "base64", "bytes", @@ -6672,9 +6681,9 @@ dependencies = [ [[package]] name = "rpassword" -version = "7.5.1" +version = "7.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2501c67132bd19c3005b0111fba298907ef002c8c1cf68e25634707e38bf66fe" +checksum = "5ac5b223d9738ef56e0b98305410be40fa0941bf6036c56f1506751e43552d64" dependencies = [ "libc", "rtoolbox", @@ -6788,9 +6797,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.37" +version = "0.23.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "758025cb5fccfd3bc2fd74708fd4682be41d99e5dff73c377c0646c6012c73a4" +checksum = "ef86cd5876211988985292b91c96a8f2d298df24e75989a43a3c73f2d4d8168b" dependencies = [ "aws-lc-rs", "log", @@ -7624,6 +7633,26 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "tokio-netem" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ee34c2e83c178d3ffeb53bf51fa480fcdc1dac07a693e786e8677fc6773d8b9" +dependencies = [ + "anyhow", + "bytes", + "clap", + "futures", + "humantime", + "pin-project", + "rand 0.9.2", + "smallvec", + "socket2 0.5.10", + "tokio", + "tracing", + "tracing-subscriber", +] + [[package]] name = "tokio-postgres" version = "0.7.17" @@ -8140,9 +8169,9 @@ dependencies = [ [[package]] name = "usearch" -version = "2.25.1" +version = "2.25.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1025b65e3669950a226b1b5e4e8de77f51ad7af7cf8a693ada62a9ec16c742e0" +checksum = "15d31c8be436d55b9ec4300fe21978435bf59c1026e821257e9e74c651f7f05c" dependencies = [ "cxx", "cxx-build", diff --git a/Cargo.toml b/Cargo.toml index 98676061..0f067183 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -73,11 +73,12 @@ log = "0.4.29" # Async runtime tokio = { version = "1.52.1", features = ["rt-multi-thread", "macros", "sync", "time", "fs", "io-util", "io-std", "net", "signal", "parking_lot"] } +tokio-netem = "0.1.1" tokio-stream = { version = "0.1", features = ["net"] } # HTTP client (with HTTP/2 support) - using rustls-tls for cross-compilation compatibility # Note: native-tls requires OpenSSL which is difficult to cross-compile for aarch64 -reqwest = { version = "0.13.2", default-features = false, features = ["json", "rustls", "http2", "multipart", "stream", "form"] } +reqwest = { version = "0.13.3", default-features = false, features = ["json", "rustls", "http2", "multipart", "stream", "form"] } hyper = { version = "1.8.1", default-features = false, features = ["client", "http1"] } hyper-util = { version = "0.1.20", default-features = false, features = ["client", "client-legacy", "http1", "tokio"] } url = "2.5.8" @@ -87,6 +88,7 @@ tokio-tungstenite = { version = "0.29.0", features = ["rustls-tls-webpki-roots"] # Security floor pins for vulnerable transitive TLS/QUIC crates. aws-lc-rs = { version = "1.16.3", default-features = false } +rustls = { version = "0.23.40", default-features = false } quinn-proto = { version = "0.11.14", default-features = false } rustls-webpki = { version = "0.103.13", default-features = false } @@ -101,16 +103,16 @@ chrono = { version = "0.4.44", features = ["serde"] } # DataFusion 52.3.0 resolves to Arrow 57.3.0; mixing in Arrow 58 pulls a second # arrow_array crate into the graph and causes FixedSizeListArray type mismatches. # Upgrade DataFusion first before bumping Arrow/Parquet beyond this family. -arrow = { version = "58.1.0", default-features = false } -arrow-ipc = { version = "58.1.0", default-features = false } -arrow-schema = { version = "58.1.0" } +arrow = { version = "58.2.0", default-features = false } +arrow-ipc = { version = "58.2.0", default-features = false } +arrow-schema = { version = "58.2.0" } datafusion = { version = "53.1.0", default-features = false, features = ["sql", "parquet", "recursive_protection", "nested_expressions"] } datafusion-datasource = { version = "53.1.0", default-features = false } datafusion-common = { version = "53.1.0", default-features = false } datafusion-expr = { version = "53.1.0" } datafusion-functions-json = { version = "0.53.1" } sqlparser = { version = "0.61.0" } -parquet = { version = "58.1.0", default-features = false, features = ["snap", "zstd", "arrow", "async"] } +parquet = { version = "58.2.0", default-features = false, features = ["snap", "zstd", "arrow", "async"] } # Web framework actix-web = { version = "4.13.0", features = ["http2"] } @@ -133,7 +135,7 @@ ulid = "1.1" jsonwebtoken = { version = "10.3.0", default-features = false, features = ["aws_lc_rs"] } # Configuration -toml = "1.1.0" +toml = "1.1.2" dotenv = "0.15" # CLI tools @@ -161,14 +163,14 @@ rmp-serde = "1.3" # gRPC for Raft network layer tonic = { version = "0.14.5" } tonic-prost = "0.14.5" -tonic-build = "0.14.4" +tonic-build = "0.14.5" prost = "0.14.3" prost-types = "0.14.3" # Additional dependencies bcrypt = "0.19.0" rand = "0.10.1" -rcgen = "0.14.1" +rcgen = "0.14.7" x509-parser = "0.18.1" kalamdb-commons = { path = "backend/crates/kalamdb-commons", default-features = false } kalamdb-plan-cache = { path = "backend/crates/kalamdb-plan-cache" } @@ -179,7 +181,7 @@ cookie = "0.18" colored = "3.1.1" fern = "0.7" indicatif = "0.18.4" -rpassword = "7.5.1" +rpassword = "7.5.2" dirs = "6.0" term_size = "0.3" crossterm = "0.29.0" @@ -189,23 +191,23 @@ futures-util = { version = "0.3.32", default-features = false } base64 = "0.22" flatbuffers = "25.12.19" flexbuffers = "25.9.23" -async-trait = "0.1.74" +async-trait = "0.1.89" once_cell = "1.21.4" regex = "1.12.3" dashmap = "6.1" bytes = "1.11.1" -http-body = "1.0.0" -http-body-util = "0.1.2" +http-body = "1.0.1" +http-body-util = "0.1.3" cc = "1.2.61" proc-macro2 = "1.0.106" -quote = "1.0.44" +quote = "1.0.45" syn = { version = "2.0.117", features = ["full", "extra-traits"] } # Object storage abstraction (S3/GCS/Azure/local) object_store = { version = "0.13.2" } # Vector ANN engines -usearch = "2.25.1" +usearch = "2.25.2" parking_lot = "0.12" tokio-util = "0.7.18" @@ -246,7 +248,7 @@ tracing-log = "0.2" tracing-opentelemetry = "0.32.1" opentelemetry = "0.31.0" opentelemetry_sdk = { version = "0.31.0", features = ["trace", "rt-tokio"] } -opentelemetry-otlp = { version = "0.31.0", default-features = false, features = ["trace", "grpc-tonic", "http-proto"] } +opentelemetry-otlp = { version = "0.31.1", default-features = false, features = ["trace", "grpc-tonic", "http-proto"] } # Release profile: balanced for production use (good performance + reasonable size) [profile.release] diff --git a/backend/crates/kalamdb-api/src/http/sql/execution_paths.rs b/backend/crates/kalamdb-api/src/http/sql/execution_paths.rs index 8e2fcbbd..746b1ebe 100644 --- a/backend/crates/kalamdb-api/src/http/sql/execution_paths.rs +++ b/backend/crates/kalamdb-api/src/http/sql/execution_paths.rs @@ -298,7 +298,7 @@ pub(super) async fn execute_file_upload_path( ErrorCode::SqlExecutionError, &format!( "EXECUTE AS USER is not allowed on SHARED tables (table '{}'). AS USER \ - impersonation is only supported for USER tables.", + impersonation is only supported for USER and STREAM tables.", table_id ), took_ms(start_time), @@ -562,7 +562,7 @@ pub(super) async fn execute_batch_path( ErrorCode::SqlExecutionError, &format!( "EXECUTE AS USER is not allowed on SHARED tables (table '{}'). AS USER \ - impersonation is only supported for USER tables.", + impersonation is only supported for USER and STREAM tables.", table_id ), took_ms(start_time), diff --git a/backend/crates/kalamdb-core/src/cluster_handler.rs b/backend/crates/kalamdb-core/src/cluster_handler.rs index 45fac493..3c85d1f8 100644 --- a/backend/crates/kalamdb-core/src/cluster_handler.rs +++ b/backend/crates/kalamdb-core/src/cluster_handler.rs @@ -431,7 +431,8 @@ impl ClusterMessageHandler for CoreClusterHandler { "SQL_EXECUTION_ERROR", &format!( "Statement {} failed: EXECUTE AS USER is not allowed on SHARED tables \ - (table '{}'). AS USER impersonation is only supported for USER tables.", + (table '{}'). AS USER impersonation is only supported for USER and \ + STREAM tables.", idx + 1, table_name ), diff --git a/backend/tests/common/testserver/fixtures.rs b/backend/tests/common/testserver/fixtures.rs index f46fe05e..8b73ea85 100644 --- a/backend/tests/common/testserver/fixtures.rs +++ b/backend/tests/common/testserver/fixtures.rs @@ -190,6 +190,7 @@ pub async fn create_messages_table( named_rows: None, row_count: 0, message: Some("Table already existed".to_string()), + as_user: None, }], took: Some(0.0), error: None, @@ -311,6 +312,7 @@ pub async fn create_stream_table( named_rows: None, row_count: 0, message: Some("Table already existed".to_string()), + as_user: None, }], took: Some(0.0), error: None, diff --git a/backend/tests/docs/AUTH_SCENARIOS.md b/backend/tests/docs/AUTH_SCENARIOS.md index cf6ddabf..ef62d13c 100644 --- a/backend/tests/docs/AUTH_SCENARIOS.md +++ b/backend/tests/docs/AUTH_SCENARIOS.md @@ -35,10 +35,10 @@ This document outlines the test scenarios for authentication, authorization, and - **Agent Friendly**: Check `backend/tests/testserver/tables/test_user_tables_http.rs`. ## 4. EXECUTE AS USER Role Matrix -**Goal**: Direct USER-table access stays subject-scoped, while explicit `EXECUTE AS USER` follows the role hierarchy. +**Goal**: Direct USER-table and STREAM-table access stays subject-scoped, while explicit `EXECUTE AS USER` follows the role hierarchy. - **Step 1**: Authenticate as `user`, `service`, `dba`, and `system` role users. - **Step 2**: Attempt allowed and denied cross-user `EXECUTE AS USER` SELECT/INSERT/UPDATE/DELETE. -- **Step 3**: Verify allowed role edges operate under the target user ID and denied edges leave target rows unchanged. +- **Step 3**: Verify allowed role edges operate under the target user ID for USER and STREAM tables, and denied edges leave target rows unchanged. - **Step 4**: Verify self-targeted `EXECUTE AS USER` is a no-op and remains scoped to the actor. - **Agent Friendly**: Check `backend/tests/misc/auth/test_as_user_impersonation.rs`. diff --git a/backend/tests/misc/auth/test_as_user_impersonation.rs b/backend/tests/misc/auth/test_as_user_impersonation.rs index d6bc3586..90b929de 100644 --- a/backend/tests/misc/auth/test_as_user_impersonation.rs +++ b/backend/tests/misc/auth/test_as_user_impersonation.rs @@ -1,8 +1,9 @@ //! Integration tests for EXECUTE AS USER subject isolation and role hierarchy. //! -//! Ordinary USER-table reads remain scoped to the authenticated subject. Explicit -//! EXECUTE AS USER can switch the subject only when the actor role is allowed to -//! target the cached privileged-role class for the target user ID. +//! Ordinary USER-table and STREAM-table reads remain scoped to the authenticated +//! subject. Explicit EXECUTE AS USER can switch the subject only when the actor +//! role is allowed to target the cached privileged-role class for the target +//! user ID. use kalam_client::models::{QueryResponse, ResponseStatus}; use kalamdb_commons::models::{AuthType, Role, UserId}; @@ -111,6 +112,21 @@ async fn create_user_table(server: &TestServer, namespace: &str, table: &str) { assert_success(&table_resp, "create user table"); } +async fn create_stream_table(server: &TestServer, namespace: &str, table: &str) { + let ns_resp = server + .execute_sql_as_user(&format!("CREATE NAMESPACE {}", namespace), "root") + .await; + assert_success(&ns_resp, "create namespace"); + + let create_table = format!( + "CREATE TABLE {}.{} (id VARCHAR PRIMARY KEY, value VARCHAR) WITH (TYPE = 'STREAM', \ + TTL_SECONDS = 3600)", + namespace, table + ); + let table_resp = server.execute_sql_as_user(&create_table, "root").await; + assert_success(&table_resp, "create stream table"); +} + #[actix_web::test] #[ntest::timeout(45000)] async fn test_execute_as_user_role_matrix_allows_privileged_roles_and_denies_user() { @@ -347,6 +363,60 @@ async fn test_allowed_execute_as_can_select_update_and_delete_target_rows() { ); } +#[actix_web::test] +#[ntest::timeout(45000)] +async fn test_execute_as_stream_table_uses_target_user_scope() { + let server = TestServer::new_shared().await; + let ns = unique_name("as_user_stream_scope"); + create_stream_table(&server, &ns, "events").await; + + let actor = insert_user(&server, &unique_name("stream_actor"), Role::Service).await; + let target = insert_user(&server, &unique_name("stream_target"), Role::User).await; + + let insert_sql = format!( + "EXECUTE AS USER '{}' (INSERT INTO {}.events (id, value) VALUES ('e1', 'delegated'))", + target.as_str(), + ns + ); + let insert_resp = server.execute_sql_as_user(&insert_sql, actor.as_str()).await; + assert_success(&insert_resp, "stream insert through execute as user"); + + let target_direct = server + .execute_sql_as_user( + &format!("SELECT value FROM {}.events WHERE id = 'e1'", ns), + target.as_str(), + ) + .await; + assert_success(&target_direct, "target direct stream select"); + assert_eq!(row_count(&target_direct), 1, "target should see delegated stream row"); + + let actor_direct = server + .execute_sql_as_user( + &format!("SELECT value FROM {}.events WHERE id = 'e1'", ns), + actor.as_str(), + ) + .await; + assert_success(&actor_direct, "actor direct stream select"); + assert_eq!(row_count(&actor_direct), 0, "actor must not see target stream row directly"); + + let actor_as_target = server + .execute_sql_as_user( + &format!( + "EXECUTE AS USER '{}' (SELECT value FROM {}.events WHERE id = 'e1')", + target.as_str(), + ns + ), + actor.as_str(), + ) + .await; + assert_success(&actor_as_target, "actor select through execute as on stream table"); + assert_eq!( + row_count(&actor_as_target), + 1, + "execute as user should read the target stream partition" + ); +} + #[actix_web::test] #[ntest::timeout(45000)] async fn test_disallowed_execute_as_denial_is_audited() { diff --git a/benchv2/Cargo.lock b/benchv2/Cargo.lock index 4d0d068a..9595facf 100644 --- a/benchv2/Cargo.lock +++ b/benchv2/Cargo.lock @@ -1009,6 +1009,7 @@ dependencies = [ "quinn-proto", "reqwest", "rmp-serde", + "rustls", "rustls-webpki", "serde", "serde_json", diff --git a/benchv2/run-chat-realtime.sh b/benchv2/run-chat-realtime.sh index 5f1f86ce..2388f221 100755 --- a/benchv2/run-chat-realtime.sh +++ b/benchv2/run-chat-realtime.sh @@ -4,6 +4,8 @@ # Usage: # ./run-chat-realtime.sh # ./run-chat-realtime.sh --url http://127.0.0.1:8080 --minutes 10 --users 1000 --realtime-convs 150 +# ./run-chat-realtime.sh --messages-per-minute 20 +# ./run-chat-realtime.sh --messages-per-minute 0 set -euo pipefail @@ -14,6 +16,7 @@ URL="${KALAMDB_URL:-http://127.0.0.1:8080}" MINUTES="${KALAMDB_BENCH_CHAT_MINUTES:-5}" USER_COUNT="${KALAMDB_BENCH_CHAT_USERS:-1000}" REALTIME_CONVS="${KALAMDB_BENCH_CHAT_REALTIME_CONVS:-100}" +MESSAGES_PER_MINUTE="${KALAMDB_BENCH_CHAT_MESSAGES_PER_MINUTE:-20}" BENCH_USER="${KALAMDB_USER:-}" BENCH_PASSWORD="${KALAMDB_PASSWORD:-}" EXTRA_ARGS=() @@ -36,6 +39,10 @@ while [[ $# -gt 0 ]]; do REALTIME_CONVS="$2" shift 2 ;; + --messages-per-minute) + MESSAGES_PER_MINUTE="$2" + shift 2 + ;; --user) BENCH_USER="$2" shift 2 @@ -54,12 +61,18 @@ done export KALAMDB_BENCH_CHAT_MINUTES="$MINUTES" export KALAMDB_BENCH_CHAT_USERS="$USER_COUNT" export KALAMDB_BENCH_CHAT_REALTIME_CONVS="$REALTIME_CONVS" +export KALAMDB_BENCH_CHAT_MESSAGES_PER_MINUTE="$MESSAGES_PER_MINUTE" echo "▸ Running chat_realtime benchmark" echo "▸ URL: $URL" echo "▸ Minutes: $MINUTES" echo "▸ Seeded users: $USER_COUNT" echo "▸ Active conversations: $REALTIME_CONVS" +if [[ "$MESSAGES_PER_MINUTE" == "0" ]]; then + echo "▸ Conversation message rate: idle subscriptions only" +else + echo "▸ Conversation message rate: $MESSAGES_PER_MINUTE messages/min" +fi CMD=(./run-benchmarks.sh --urls "$URL" --bench chat_realtime --iterations 1 --warmup 0) diff --git a/benchv2/src/benchmarks/chat_realtime_bench.rs b/benchv2/src/benchmarks/chat_realtime_bench.rs index 46bea4e6..fff27830 100644 --- a/benchv2/src/benchmarks/chat_realtime_bench.rs +++ b/benchv2/src/benchmarks/chat_realtime_bench.rs @@ -26,7 +26,9 @@ pub struct ChatRealtimeBench; const DEFAULT_CHAT_MINUTES: u64 = 5; const DEFAULT_CHAT_USERS: u32 = 1_000; const DEFAULT_CHAT_REALTIME_CONVS: u32 = 100; +const DEFAULT_CHAT_MESSAGES_PER_MINUTE: u32 = 20; const CHAT_USER_PASSWORD: &str = "BenchChatP@ss123"; +const CHAT_MESSAGES_PER_CYCLE: u32 = 2; const CHAT_TYPING_INTERVAL: Duration = Duration::from_secs(2); const CHAT_TYPING_BURSTS: u64 = 3; const CHAT_WORKER_START_STAGGER_MS: u64 = 5; @@ -56,6 +58,7 @@ struct ChatWorkloadSettings { minutes: u64, user_count: u32, realtime_conversations: u32, + messages_per_minute: u32, } impl ChatWorkloadSettings { @@ -66,6 +69,10 @@ impl ChatWorkloadSettings { "KALAMDB_BENCH_CHAT_REALTIME_CONVS", DEFAULT_CHAT_REALTIME_CONVS, )?; + let messages_per_minute = parse_u32_env( + "KALAMDB_BENCH_CHAT_MESSAGES_PER_MINUTE", + DEFAULT_CHAT_MESSAGES_PER_MINUTE, + )?; if minutes == 0 { return Err("KALAMDB_BENCH_CHAT_MINUTES must be greater than zero".to_string()); @@ -83,6 +90,7 @@ impl ChatWorkloadSettings { minutes, user_count, realtime_conversations, + messages_per_minute, }) } @@ -91,12 +99,31 @@ impl ChatWorkloadSettings { minutes: DEFAULT_CHAT_MINUTES, user_count: DEFAULT_CHAT_USERS, realtime_conversations: DEFAULT_CHAT_REALTIME_CONVS, + messages_per_minute: DEFAULT_CHAT_MESSAGES_PER_MINUTE, }) } fn duration(&self) -> Duration { Duration::from_secs(self.minutes.saturating_mul(60)) } + + fn target_cycle_interval(&self) -> Option { + if self.messages_per_minute == 0 { + return None; + } + + Some(Duration::from_secs_f64( + (60.0 * f64::from(CHAT_MESSAGES_PER_CYCLE)) / f64::from(self.messages_per_minute), + )) + } + + fn message_rate_label(&self) -> String { + if self.messages_per_minute == 0 { + "idle subscriptions only".to_string() + } else { + format!("{} messages/min per conversation", self.messages_per_minute) + } + } } #[derive(Default)] @@ -154,18 +181,22 @@ impl Benchmark for ChatRealtimeBench { fn report_description(&self, _config: &Config) -> String { let settings = ChatWorkloadSettings::for_report(); format!( - "Realtime chat scenario for {}m with {} regular users and {} active conversations", - settings.minutes, settings.user_count, settings.realtime_conversations + "Realtime chat scenario for {}m with {} regular users, {} active conversations, and {}", + settings.minutes, + settings.user_count, + settings.realtime_conversations, + settings.message_rate_label() ) } fn report_full_description(&self, _config: &Config) -> String { let settings = ChatWorkloadSettings::for_report(); format!( - "Creates {} regular KalamDB users, then runs {} concurrent chat-session workers for {} minute(s). Each worker logs in as real users, opens a USER-table conversation row for user A, relies on a Rust topic consumer to mirror that conversation into user B's USER table, sends a user A message that the consumer forwards to user B, emits three typing events over six seconds from user B via a STREAM table, sends a user B reply that the consumer forwards back to user A, and keeps user A on live SQL subscriptions for message and typing delivery.", + "Creates {} regular KalamDB users, then runs {} concurrent chat-session workers for {} minute(s). Each worker logs in as real users, opens a USER-table conversation row for user A, keeps user A on live SQL subscriptions for message and typing delivery, and paces each conversation at {}. When message sending is enabled, a Rust topic consumer mirrors the user A conversation row into user B's USER table, forwards a user A message to user B, emits three typing events over six seconds from user B via a STREAM table, and forwards a user B reply back to user A.", settings.user_count, settings.realtime_conversations, settings.minutes, + settings.message_rate_label(), ) } @@ -178,6 +209,10 @@ impl Benchmark for ChatRealtimeBench { "Active Conversations", settings.realtime_conversations.to_string(), ), + BenchmarkDetail::new( + "Conversation Message Rate", + settings.message_rate_label(), + ), BenchmarkDetail::new( "Responder Typing Burst", format!( @@ -354,16 +389,6 @@ impl Benchmark for ChatRealtimeBench { 60_000_000_000 + u64::from(iteration) * 10_000_000, )); - println!( - " Chat workload settings: duration={}m, regular_users={}, target_active_chat_users={}, active_conversations={}, typing_burst={}x{}s", - settings.minutes, - settings.user_count, - target_active_user_count, - settings.realtime_conversations, - CHAT_TYPING_BURSTS, - CHAT_TYPING_INTERVAL.as_secs(), - ); - let prewarmed_active_users = prewarm_user_clients( user_pool.clone(), users.clone(), @@ -384,9 +409,32 @@ impl Benchmark for ChatRealtimeBench { let scenario_started = Instant::now(); let run_deadline = scenario_started + settings.duration(); + let target_cycle_interval = settings.target_cycle_interval(); let delivery_timeout = chat_delivery_wait_timeout(settings.realtime_conversations); let mut handles = Vec::with_capacity(settings.realtime_conversations as usize); + println!( + " Chat workload settings: duration={}m, regular_users={}, target_active_chat_users={}, active_conversations={}, message_rate={}, typing_burst={}x{}s", + settings.minutes, + settings.user_count, + target_active_user_count, + settings.realtime_conversations, + settings.message_rate_label(), + CHAT_TYPING_BURSTS, + CHAT_TYPING_INTERVAL.as_secs(), + ); + + println!( + " Chat workload settings: duration={}m, regular_users={}, target_active_chat_users={}, active_conversations={}, message_rate={}, typing_burst={}x{}s", + settings.minutes, + settings.user_count, + target_active_user_count, + settings.realtime_conversations, + settings.message_rate_label(), + CHAT_TYPING_BURSTS, + CHAT_TYPING_INTERVAL.as_secs(), + ); + for worker_id in 0..settings.realtime_conversations { let namespace = config.namespace.clone(); let worker_stats = stats.clone(); @@ -416,6 +464,7 @@ impl Benchmark for ChatRealtimeBench { worker_conversations, worker_messages, worker_typing, + target_cycle_interval, delivery_timeout, worker_stop.clone(), ) @@ -505,6 +554,7 @@ async fn run_chat_worker( conversation_ids: Arc, message_ids: Arc, typing_ids: Arc, + target_cycle_interval: Option, delivery_timeout: Duration, global_stop: Arc, ) -> Result<(), String> { @@ -556,8 +606,17 @@ async fn run_chat_worker( .create_conversation_timings .record(create_conversation_started.elapsed()); + if target_cycle_interval.is_none() { + while Instant::now() < run_deadline && !global_stop.load(Ordering::Relaxed) { + sleep(Duration::from_secs(1)).await; + } + + return Ok(()); + } + let mut cycle_ordinal = 0_u64; while Instant::now() < run_deadline && !global_stop.load(Ordering::Relaxed) { + let cycle_started = Instant::now(); stats.sessions_started.fetch_add(1, Ordering::Relaxed); cycle_ordinal += 1; @@ -646,6 +705,17 @@ async fn run_chat_worker( } stats.sessions_completed.fetch_add(1, Ordering::Relaxed); + + if let Some(target_cycle_interval) = target_cycle_interval { + let cycle_elapsed = cycle_started.elapsed(); + if cycle_elapsed < target_cycle_interval { + let remaining_run = run_deadline.saturating_duration_since(Instant::now()); + let sleep_for = (target_cycle_interval - cycle_elapsed).min(remaining_run); + if !sleep_for.is_zero() { + sleep(sleep_for).await; + } + } + } } Ok(()) diff --git a/cli/README.md b/cli/README.md index 4caacc00..1b97eed4 100644 --- a/cli/README.md +++ b/cli/README.md @@ -109,11 +109,9 @@ SELECT * FROM app.users; -- Subscribe to changes SUBSCRIBE TO app.messages WHERE user_id = 'user123'; --- Pause subscription -\pause - --- Resume subscription -\continue +-- Or use the interactive CLI aliases +\subscribe SELECT * FROM app.messages WHERE user_id = 'user123'; +\live SELECT * FROM app.messages WHERE user_id = 'user123'; -- Stop subscription (Ctrl+C) ``` @@ -207,14 +205,13 @@ Special commands starting with backslash (`\`): | `\sessions` | Show active pg-extension gRPC sessions | | `\format ` | Set output format | | `\dt` / `\tables` | List tables | -| `\d ` / `\describe
` | Describe a table | +| `\d
` / `\describe
` | Describe a table (`
` or ``) | +| `\as ` | Wrap one statement as `EXECUTE AS USER` | | `\stats` / `\metrics` | Show system stats | | `\health` | Check server health | | `\flush` | Execute STORAGE FLUSH ALL | -| `\pause` | Pause ingestion | -| `\continue` | Resume ingestion | | `\refresh-tables` / `\refresh` | Refresh autocomplete cache | -| `\subscribe ` / `\watch ` | Start live query | +| `\subscribe ` / `\watch ` / `\live ` | Start live query | | `\unsubscribe` / `\unwatch` | Cancel live query | | `\show-credentials` / `\credentials` | Show stored credentials | | `\update-credentials

` | Update stored credentials | diff --git a/cli/src/completer.rs b/cli/src/completer.rs index fc42b766..ef3171f6 100644 --- a/cli/src/completer.rs +++ b/cli/src/completer.rs @@ -12,6 +12,8 @@ use std::collections::HashMap; use colored::*; use rustyline::completion::{Completer, Pair}; +use crate::parser::META_COMMAND_COMPLETIONS; + pub(crate) const SQL_KEYWORDS: &[&str] = &[ // DML "SELECT", @@ -31,6 +33,8 @@ pub(crate) const SQL_KEYWORDS: &[&str] = &[ "IS", "NULL", "AS", + "EXECUTE", + "USER", "ORDER", "BY", "GROUP", @@ -170,34 +174,10 @@ impl AutoCompleter { pub fn new() -> Self { let keywords = SQL_KEYWORDS.iter().map(|s| s.to_string()).collect::>(); - let meta_commands = vec![ - "\\quit", - "\\q", - "\\help", - "\\?", - "\\history", - "\\h", - "\\sessions", - "\\stats", - "\\metrics", - "\\flush", - "\\health", - "\\pause", - "\\continue", - "\\dt", - "\\tables", - "\\d", - "\\describe", - "\\format", - "\\refresh-tables", - "\\show-credentials", - "\\credentials", - "\\update-credentials", - "\\delete-credentials", - ] - .iter() - .map(|s| s.to_string()) - .collect(); + let meta_commands = META_COMMAND_COMPLETIONS + .iter() + .map(|command| (*command).to_string()) + .collect(); Self { keywords, @@ -498,6 +478,29 @@ mod tests { let completions = completer.get_styled_completions("\\q", line, line.len()); assert!(completions.iter().any(|c| c.replacement == "\\quit")); assert!(completions.iter().any(|c| c.replacement == "\\q")); + + let as_line = "\\a"; + let as_completions = completer.get_styled_completions("\\a", as_line, as_line.len()); + assert!(as_completions.iter().any(|c| c.replacement == "\\as")); + + let live_line = "\\li"; + let live_completions = + completer.get_styled_completions("\\li", live_line, live_line.len()); + assert!(live_completions.iter().any(|c| c.replacement == "\\live")); + } + + #[test] + fn test_execute_as_keyword_completion() { + let completer = AutoCompleter::new(); + + let execute_line = "EXE"; + let execute_completions = + completer.get_styled_completions("EXE", execute_line, execute_line.len()); + assert!(execute_completions.iter().any(|c| c.replacement == "EXECUTE")); + + let user_line = "US"; + let user_completions = completer.get_styled_completions("US", user_line, user_line.len()); + assert!(user_completions.iter().any(|c| c.replacement == "USER")); } #[test] diff --git a/cli/src/connect.rs b/cli/src/connect.rs index 762474ad..de29598e 100644 --- a/cli/src/connect.rs +++ b/cli/src/connect.rs @@ -36,7 +36,6 @@ fn build_timeouts(cli: &Cli) -> KalamLinkTimeouts { .subscribe_timeout_secs(5) // Keep default for subscribe ack .initial_data_timeout_secs(cli.initial_data_timeout) .idle_timeout_secs(cli.subscription_timeout) // subscription_timeout is the idle timeout - .keepalive_interval_secs(30) // Keep default .build() } @@ -60,8 +59,43 @@ fn is_localhost_url(url: &str) -> bool { normalized_host.parse::().is_ok_and(|ip| ip.is_loopback()) } +fn should_default_to_http(raw: &str) -> bool { + let candidate = format!("http://{}", raw.trim()); + let parsed = match Url::parse(&candidate) { + Ok(url) => url, + Err(_) => return false, + }; + + let host = match parsed.host_str() { + Some(host) => host, + None => return false, + }; + + let normalized_host = host.trim_start_matches('[').trim_end_matches(']'); + normalized_host.eq_ignore_ascii_case("localhost") + || normalized_host.parse::().is_ok_and(|ip| ip.is_loopback()) +} + fn normalize_and_validate_server_url(server_url: &str) -> Result { - let mut parsed = Url::parse(server_url.trim()).map_err(|e| { + let trimmed = server_url.trim(); + if trimmed.is_empty() { + return Err(CLIError::ConfigurationError( + "Server URL must not be empty".to_string(), + )); + } + + let normalized_input = if trimmed.contains("://") { + trimmed.to_string() + } else { + let scheme = if should_default_to_http(trimmed) { + "http" + } else { + "https" + }; + format!("{scheme}://{trimmed}") + }; + + let mut parsed = Url::parse(&normalized_input).map_err(|e| { CLIError::ConfigurationError(format!("Invalid server URL '{}': {}", server_url, e)) })?; @@ -1044,7 +1078,11 @@ pub async fn create_session( #[cfg(test)] mod tests { - use super::{is_localhost_url, normalize_and_validate_server_url}; + use clap::Parser; + + use super::{build_timeouts, is_localhost_url, normalize_and_validate_server_url}; + use crate::args::Cli; + use kalam_client::KalamLinkTimeouts; #[test] fn test_is_localhost_url_accepts_loopback_hosts() { @@ -1072,6 +1110,14 @@ mod tests { normalize_and_validate_server_url("https://db.example.com/base/").unwrap(), "https://db.example.com/base" ); + assert_eq!( + normalize_and_validate_server_url("kalam.masky.app").unwrap(), + "https://kalam.masky.app" + ); + assert_eq!( + normalize_and_validate_server_url("localhost:8080").unwrap(), + "http://localhost:8080" + ); } #[test] @@ -1081,4 +1127,13 @@ mod tests { assert!(normalize_and_validate_server_url("http://localhost:8080?token=secret").is_err()); assert!(normalize_and_validate_server_url("http://localhost:8080/#fragment").is_err()); } + + #[test] + fn test_build_timeouts_preserves_sdk_keepalive_default() { + let cli = Cli::parse_from(["kalam"]); + + let timeouts = build_timeouts(&cli); + + assert_eq!(timeouts.keepalive_interval, KalamLinkTimeouts::default().keepalive_interval); + } } diff --git a/cli/src/formatter.rs b/cli/src/formatter.rs index 19766295..552b9446 100644 --- a/cli/src/formatter.rs +++ b/cli/src/formatter.rs @@ -92,18 +92,40 @@ impl OutputFormatter { return Ok(self.format_error_detail(error)); } + let as_user = Self::query_as_user(response); + match self.format { - OutputFormat::Table => self.format_table(response), + OutputFormat::Table => self.format_table(response, as_user), OutputFormat::Json => self.format_json(response), OutputFormat::Csv => self.format_csv(response), } } + fn query_as_user(response: &QueryResponse) -> Option<&str> { + response + .results + .first() + .and_then(|result| result.as_user.as_deref()) + .filter(|value| !value.trim().is_empty()) + } + + fn format_table_footer(exec_time_ms: f64, as_user: Option<&str>) -> String { + let mut footer = String::new(); + if let Some(as_user) = as_user { + footer.push_str(&format!("As: {as_user}\n")); + } + footer.push_str(&format!("Took: {:.3} ms", exec_time_ms)); + footer + } + /// Format as table - fn format_table(&self, response: &QueryResponse) -> Result { + fn format_table(&self, response: &QueryResponse, as_user: Option<&str>) -> Result { if response.results.is_empty() { let exec_time_ms = response.took.unwrap_or(0.0); - return Ok(format!("Query OK, 0 rows affected\n\nTook: {:.3} ms", exec_time_ms)); + return Ok(format!( + "Query OK, 0 rows affected\n\n{}", + Self::format_table_footer(exec_time_ms, as_user) + )); } let result = &response.results[0]; @@ -114,8 +136,10 @@ impl OutputFormatter { // Format DDL message similar to MySQL/PostgreSQL let row_count = result.row_count; return Ok(format!( - "{}\nQuery OK, {} rows affected\n\nTook: {:.3} ms", - message, row_count, exec_time_ms + "{}\nQuery OK, {} rows affected\n\n{}", + message, + row_count, + Self::format_table_footer(exec_time_ms, as_user) )); } @@ -255,16 +279,18 @@ impl OutputFormatter { output.push_str(&format!("({} {})\n", row_count, row_label)); // Add blank line for psql-style formatting output.push('\n'); - // Display timing in milliseconds like psql - let exec_time_ms = response.took.unwrap_or(0.0); - output.push_str(&format!("Took: {:.3} ms", exec_time_ms)); + output.push_str(&Self::format_table_footer(exec_time_ms, as_user)); Ok(output) } else { // Non-query statement (INSERT, UPDATE, DELETE) let row_count = result.row_count; let exec_time_ms = response.took.unwrap_or(0.0); - Ok(format!("Query OK, {} rows affected\n\nTook: {:.3} ms", row_count, exec_time_ms)) + Ok(format!( + "Query OK, {} rows affected\n\n{}", + row_count, + Self::format_table_footer(exec_time_ms, as_user) + )) } } @@ -519,7 +545,8 @@ impl OutputFormatter { #[cfg(test)] mod tests { - use kalam_client::TimestampFormat; + use kalam_client::{query::models::ResponseStatus, QueryResult, SchemaField, TimestampFormat}; + use serde_json::json; use super::*; @@ -604,4 +631,35 @@ mod tests { assert_eq!(formatter.raw_timestamp_millis(1735689600000000), 1735689600000); assert_eq!(formatter.raw_timestamp_millis(1735689600000000000), 1735689600000); } + + #[test] + fn test_format_table_footer_includes_effective_user() { + let formatter = OutputFormatter::new( + OutputFormat::Table, + false, + TimestampFormatter::new(TimestampFormat::Iso8601), + ); + + let response = QueryResponse { + status: ResponseStatus::Success, + results: vec![QueryResult { + schema: vec![SchemaField { + name: "name".to_string(), + data_type: KalamDataType::Text, + index: 0, + flags: None, + }], + rows: Some(vec![vec![json!("Alice").into()]]), + named_rows: None, + row_count: 1, + message: None, + as_user: Some("alice".to_string()), + }], + took: Some(2.146), + error: None, + }; + + let output = formatter.format_response(&response).unwrap(); + assert!(output.contains("(1 row)\n\nAs: alice\nTook: 2.146 ms")); + } } diff --git a/cli/src/parser.rs b/cli/src/parser.rs index 93af863c..59ab8425 100644 --- a/cli/src/parser.rs +++ b/cli/src/parser.rs @@ -12,6 +12,12 @@ pub enum Command { /// SQL statement Sql(String), + /// Execute a single SQL statement as another user + ExecuteAs { + user: String, + sql: String, + }, + /// Meta-commands (backslash commands) Quit, Help, @@ -35,8 +41,6 @@ pub enum Command { }, ClusterRebalance, Health, - Pause, - Continue, ListTables, Describe(String), SetFormat(String), @@ -66,6 +70,41 @@ pub enum Command { Unknown(String), } +pub(crate) const META_COMMAND_COMPLETIONS: &[&str] = &[ + "\\quit", + "\\q", + "\\help", + "\\?", + "\\info", + "\\session", + "\\history", + "\\h", + "\\sessions", + "\\stats", + "\\metrics", + "\\flush", + "\\health", + "\\dt", + "\\tables", + "\\d", + "\\describe", + "\\as", + "\\format", + "\\refresh-tables", + "\\refresh", + "\\subscribe", + "\\watch", + "\\live", + "\\unsubscribe", + "\\unwatch", + "\\show-credentials", + "\\credentials", + "\\update-credentials", + "\\delete-credentials", + "\\cluster", + "\\consume", +]; + /// Command parser pub struct CommandParser; @@ -94,6 +133,10 @@ impl CommandParser { /// Parse meta-commands (backslash commands) fn parse_meta_command(&self, line: &str) -> Result { + if line.starts_with("\\as") { + return Self::parse_execute_as_command(line); + } + let parts: Vec<&str> = line.split_whitespace().collect(); if parts.is_empty() { return Err(CLIError::ParseError("Invalid command".into())); @@ -214,8 +257,6 @@ impl CommandParser { } }, "\\health" => Ok(Command::Health), - "\\pause" => Ok(Command::Pause), - "\\continue" => Ok(Command::Continue), "\\dt" | "\\tables" => Ok(Command::ListTables), "\\d" | "\\describe" => { if args.is_empty() { @@ -231,7 +272,7 @@ impl CommandParser { Ok(Command::SetFormat(args[0].to_string())) } }, - "\\subscribe" | "\\watch" => { + "\\subscribe" | "\\watch" | "\\live" => { if args.is_empty() { Err(CLIError::ParseError("\\subscribe requires a SQL query".into())) } else { @@ -341,6 +382,33 @@ impl CommandParser { _ => Ok(Command::Unknown(command.to_string())), } } + + fn parse_execute_as_command(line: &str) -> Result { + let remainder = line["\\as".len()..].trim_start(); + if remainder.is_empty() { + return Err(CLIError::ParseError( + "\\as requires a target user and SQL query".into(), + )); + } + + let user_end = remainder.find(char::is_whitespace).ok_or_else(|| { + CLIError::ParseError("\\as requires a target user and SQL query".into()) + })?; + + let user = remainder[..user_end].trim(); + let sql = remainder[user_end..].trim_start(); + + if user.is_empty() || sql.is_empty() { + return Err(CLIError::ParseError( + "\\as requires a target user and SQL query".into(), + )); + } + + Ok(Command::ExecuteAs { + user: user.to_string(), + sql: sql.to_string(), + }) + } } impl Default for CommandParser { @@ -381,6 +449,26 @@ mod tests { assert_eq!(cmd, Command::Describe("users".to_string())); } + #[test] + fn test_parse_live_alias() { + let parser = CommandParser::new(); + let cmd = parser.parse("\\live SELECT * FROM chat.messages").unwrap(); + assert_eq!(cmd, Command::Subscribe("SELECT * FROM chat.messages".to_string())); + } + + #[test] + fn test_parse_execute_as_shortcut() { + let parser = CommandParser::new(); + let cmd = parser.parse("\\as alice SELECT * FROM chat.messages WHERE body = 'hi there'"); + assert_eq!( + cmd.unwrap(), + Command::ExecuteAs { + user: "alice".to_string(), + sql: "SELECT * FROM chat.messages WHERE body = 'hi there'".to_string(), + } + ); + } + #[test] fn test_parse_unknown() { let parser = CommandParser::new(); diff --git a/cli/src/session.rs b/cli/src/session.rs index 3982e76c..13236301 100644 --- a/cli/src/session.rs +++ b/cli/src/session.rs @@ -133,10 +133,7 @@ struct ClusterNodeDisplay { api_addr: String, is_self: bool, is_leader: bool, - hostname: Option, - memory_usage_mb: Option, - cpu_usage_percent: Option, - uptime_human: Option, + version: Option, } /// Cluster information for CLI display @@ -1059,7 +1056,7 @@ impl CLISession { // NOTE: Do NOT update server_host here — keep it as the address the user connected to, // not the server's internal listening address (e.g. 0.0.0.0:8080). if let Some(cluster_info) = self.fetch_cluster_info().await { - self.cluster_name = Some(cluster_info.cluster_name); + self.adopt_cluster_metadata(&cluster_info); } // Connection and auth successful - print welcome banner @@ -1332,6 +1329,24 @@ impl CLISession { println!(); } + fn adopt_cluster_metadata(&mut self, info: &ClusterInfoDisplay) { + self.cluster_name = Some(info.cluster_name.clone()); + + if self.server_version.is_some() { + return; + } + + let version = info + .current_node + .as_ref() + .and_then(|node| node.version.clone()) + .or_else(|| info.nodes.iter().find(|node| node.is_self).and_then(|node| node.version.clone())); + + if let Some(version) = version.and_then(Self::normalize_server_field) { + self.server_version = Some(version); + } + } + /// Fetch namespaces, table names, and column names from server and update completer async fn refresh_tables(&mut self, completer: &mut AutoCompleter) -> Result<()> { // Query namespaces first from system.namespaces @@ -2378,53 +2393,14 @@ impl CLISession { ); } - fn render_cluster_health_fallback(&self, info: &ClusterInfoDisplay, note: Option<&str>) { - let active_nodes = info - .nodes - .iter() - .filter(|node| node.status.eq_ignore_ascii_case("active")) - .count(); - let offline_nodes = info - .nodes - .iter() - .filter(|node| node.status.eq_ignore_ascii_case("offline")) - .count(); - - println!("{} Cluster health: {}", "✓".green(), "reachable".green()); - println!( - " Cluster: {} | Nodes: {} total, {} active, {} offline", - info.cluster_name.as_str().cyan(), - info.nodes.len(), - active_nodes, - offline_nodes - ); - if let Some(message) = note { - println!(" {}", message.yellow()); - } - println!(); - println!("{}", "Nodes:".yellow().bold()); - - for node in &info.nodes { - let self_marker = if node.is_self { " (connected)" } else { "" }; - let leader_marker = if node.is_leader { " [LEADER]" } else { "" }; - let hostname = node.hostname.as_deref().unwrap_or(node.api_addr.as_str()); - println!( - " Node {}: {} | {} | {}{}{}", - node.node_id, - node.role, - node.status, - node.api_addr, - leader_marker.yellow(), - self_marker.cyan() - ); - println!( - " host={} | {} | {} | {}", - hostname, - Self::format_cluster_memory(node.memory_usage_mb), - Self::format_cluster_cpu(node.cpu_usage_percent), - Self::format_cluster_uptime(node.uptime_human.as_deref()) - ); - } + fn public_probe_client(&self) -> Result { + Ok(KalamLinkClient::builder() + .base_url(&self.server_url) + .timeout(self.timeouts.receive_timeout) + .max_retries(self.config.resolved_server().max_retries) + .timeouts(self.timeouts.clone()) + .connection_options(self.config.to_connection_options()) + .build()?) } /// Fetch cluster information from system.cluster @@ -2434,8 +2410,7 @@ impl CLISession { .client .execute_query( "SELECT cluster_id, node_id, role, status, api_addr, is_self, is_leader, \ - hostname, memory_usage_mb, cpu_usage_percent, uptime_human FROM system.cluster \ - ORDER BY is_leader DESC, node_id ASC", + version FROM system.cluster ORDER BY is_leader DESC, node_id ASC", None, None, None, @@ -2454,8 +2429,8 @@ impl CLISession { if let Some(rows) = &query_result.rows { for row in rows { // row fields: cluster_id, node_id, role, status, api_addr, is_self, - // is_leader, hostname, memory_usage_mb, cpu_usage_percent, uptime_human - if row.len() >= 11 { + // is_leader, version + if row.len() >= 8 { // Extract cluster_id from first row only if cluster_name.is_empty() { cluster_name = @@ -2467,10 +2442,7 @@ impl CLISession { let api_addr = row[4].as_str().unwrap_or("").to_string(); let is_self = row[5].as_bool().unwrap_or(false); let is_leader = row[6].as_bool().unwrap_or(false); - let hostname = row[7].as_str().map(ToString::to_string); - let memory_usage_mb = row[8].as_u64(); - let cpu_usage_percent = row[9].as_f64().map(|cpu| cpu as f32); - let uptime_human = row[10].as_str().map(ToString::to_string); + let version = row[7].as_str().map(ToString::to_string); // Check if this looks like cluster mode (role is leader/follower) if matches!( @@ -2487,10 +2459,7 @@ impl CLISession { api_addr, is_self, is_leader, - hostname, - memory_usage_mb, - cpu_usage_percent, - uptime_human, + version, }; if is_self { @@ -2520,7 +2489,8 @@ impl CLISession { /// Check server health and refresh cached server metadata pub async fn health_check(&mut self) -> Result<()> { - let basic_health = self.client.health_check().await; + let probe_client = self.public_probe_client()?; + let basic_health = probe_client.health_check().await; match &basic_health { Ok(health) => { @@ -2533,7 +2503,7 @@ impl CLISession { Err(KalamLinkError::ServerError { status_code: 403, .. }) => { - // Localhost-only endpoint; fall back to authenticated SQL-based cluster info. + self.connected = true; }, Err(_) => { self.connected = false; @@ -2543,58 +2513,43 @@ impl CLISession { }, } - let cluster_info = self.fetch_cluster_info().await; - if let Some(ref info) = cluster_info { - self.connected = true; - self.cluster_name = Some(info.cluster_name.clone()); - - if info.is_cluster_mode { - match self.client.cluster_health_check().await { - Ok(cluster_health) => { - self.server_version = - Self::normalize_server_field(cluster_health.version.clone()); - self.server_build_date = - Self::normalize_server_field(cluster_health.build_date.clone()); - self.render_cluster_health_response(&cluster_health); - return Ok(()); - }, - Err(KalamLinkError::ServerError { - status_code: 403, .. - }) => { - self.render_cluster_health_fallback( - info, - Some("Cluster health endpoint is localhost-only; using system.cluster"), - ); - return Ok(()); - }, - Err(_) => { - self.render_cluster_health_fallback( - info, - Some("Cluster health endpoint unavailable; using system.cluster"), - ); - return Ok(()); - }, - } - } - - match basic_health { + match probe_client.cluster_health_check().await { + Ok(cluster_health) => { + self.connected = true; + self.server_version = Self::normalize_server_field(cluster_health.version.clone()); + self.server_build_date = + Self::normalize_server_field(cluster_health.build_date.clone()); + self.render_cluster_health_response(&cluster_health); + return Ok(()); + }, + Err(KalamLinkError::ServerError { + status_code: 403, .. + }) => match basic_health { Ok(_) => { println!("✓ Server is healthy"); + println!(" {}", "Cluster health endpoint is restricted to localhost".yellow()); return Ok(()); }, Err(KalamLinkError::ServerError { status_code: 403, .. }) => { - println!("{} Server is reachable", "✓".green()); - println!(" {}", "Health endpoint is restricted to localhost".yellow()); + println!("{}", "Health endpoints are localhost-only for this connection".yellow()); + println!(" {}", "No authenticated SQL fallback was used.".dimmed()); + println!( + " {}", + "Run SELECT * FROM system.cluster manually if you want authenticated cluster state." + .dimmed() + ); return Ok(()); }, - Err(_) => { - println!("{} Server is reachable", "✓".green()); - println!(" {}", "Using authenticated SQL fallback".yellow()); + Err(e) => return Err(e.into()), + }, + Err(_) => { + if basic_health.is_ok() { + println!("✓ Server is healthy"); return Ok(()); - }, - } + } + }, } match basic_health { @@ -3122,10 +3077,27 @@ mod tests { use super::*; use crate::credentials::FileCredentialStore; - #[derive(Debug, Default)] + #[derive(Debug)] struct TestServerState { sql_authorization_headers: Vec, refresh_authorization_headers: Vec, + health_authorization_headers: Vec, + cluster_health_authorization_headers: Vec, + health_status: u16, + cluster_health_status: u16, + } + + impl Default for TestServerState { + fn default() -> Self { + Self { + sql_authorization_headers: Vec::new(), + refresh_authorization_headers: Vec::new(), + health_authorization_headers: Vec::new(), + cluster_health_authorization_headers: Vec::new(), + health_status: 200, + cluster_health_status: 404, + } + } } struct TestServer { @@ -3176,16 +3148,86 @@ mod tests { let authorization = request.headers.get("authorization").cloned(); let (status_line, body) = match request.path.as_str() { - "/v1/api/healthcheck" => ( - "HTTP/1.1 200 OK", - json!({ - "status": "healthy", - "version": "test", - "api_version": "v1", - "build_date": null - }) - .to_string(), - ), + "/v1/api/healthcheck" => { + let status = { + let mut guard = state.lock().await; + guard + .health_authorization_headers + .push(authorization.clone().unwrap_or_default()); + guard.health_status + }; + + match status { + 200 => ( + "HTTP/1.1 200 OK", + json!({ + "status": "healthy", + "version": "test", + "api_version": "v1", + "build_date": null + }) + .to_string(), + ), + 403 => ( + "HTTP/1.1 403 Forbidden", + json!({ "message": "localhost only" }).to_string(), + ), + code => ( + "HTTP/1.1 500 Internal Server Error", + json!({ "message": format!("unexpected status {code}") }).to_string(), + ), + } + }, + "/v1/api/cluster/health" => { + let status = { + let mut guard = state.lock().await; + guard + .cluster_health_authorization_headers + .push(authorization.clone().unwrap_or_default()); + guard.cluster_health_status + }; + + match status { + 200 => ( + "HTTP/1.1 200 OK", + json!({ + "status": "healthy", + "version": "test", + "build_date": "2026-05-04T00:00:00Z", + "is_cluster_mode": true, + "cluster_id": "test-cluster", + "node_id": 0, + "is_leader": true, + "total_groups": 1, + "groups_leading": 1, + "current_term": 1, + "last_applied": 1, + "millis_since_quorum_ack": 0, + "nodes": [{ + "node_id": 0, + "role": "leader", + "status": "active", + "api_addr": "http://127.0.0.1:8080", + "is_self": true, + "is_leader": true, + "replication_lag": null, + "catchup_progress_pct": null, + "hostname": "localhost", + "memory_usage_mb": 64, + "cpu_usage_percent": 0.25, + "uptime_seconds": 42, + "uptime_human": "42s" + }] + }) + .to_string(), + ), + 403 => ( + "HTTP/1.1 403 Forbidden", + json!({ "message": "localhost only" }).to_string(), + ), + _ => ("HTTP/1.1 404 Not Found", "Not found".to_string()), + } + }, "/v1/api/sql" => { if let Some(header) = authorization.clone() { state.lock().await.sql_authorization_headers.push(header.clone()); @@ -3356,6 +3398,56 @@ mod tests { assert!(options.is_some()); } + #[test] + fn test_parse_describe_target_supports_namespace_and_semicolon() { + let (namespace, table_name) = CLISession::parse_describe_target("chat.messages;").unwrap(); + assert_eq!(namespace.as_deref(), Some("chat")); + assert_eq!(table_name, "messages"); + + let (namespace, table_name) = + CLISession::parse_describe_target("\"chat\".\"message logs\"").unwrap(); + assert_eq!(namespace.as_deref(), Some("chat")); + assert_eq!(table_name, "message logs"); + } + + #[tokio::test] + #[timeout(5000)] + async fn test_health_check_does_not_fall_back_to_authenticated_sql() { + let server = TestServer::spawn().await; + { + let mut state = server.state.lock().await; + state.health_status = 403; + state.cluster_health_status = 403; + } + + let mut session = CLISession::with_auth_and_instance( + server.base_url.clone(), + AuthProvider::jwt_token("fresh-token".to_string()), + OutputFormat::Table, + false, + None, + None, + Some("admin".to_string()), + None, + true, + None, + None, + None, + CLIConfiguration::default(), + crate::config::default_config_path(), + false, + ) + .await + .expect("create session"); + + session.health_check().await.expect("health check should succeed"); + + let state = server.state.lock().await; + assert!(state.sql_authorization_headers.is_empty()); + assert_eq!(state.health_authorization_headers, vec![String::new(), String::new()]); + assert_eq!(state.cluster_health_authorization_headers, vec![String::new()]); + } + #[tokio::test] #[timeout(5000)] async fn test_build_auth_refresher_refreshes_and_persists_tokens() { diff --git a/cli/src/session/cluster/actions.rs b/cli/src/session/cluster/actions.rs index 988ce5f6..51a67125 100644 --- a/cli/src/session/cluster/actions.rs +++ b/cli/src/session/cluster/actions.rs @@ -1,7 +1,3 @@ -use std::collections::HashMap; - -use kalam_client::KalamCellValue; - use super::{cell_bool, cell_text, cell_u64, CLISession}; use crate::{CLIError, Result}; @@ -258,6 +254,10 @@ fn bytes_to_mb(bytes: u64) -> f64 { #[cfg(test)] mod tests { + use std::collections::HashMap; + + use kalam_client::KalamCellValue; + use super::*; #[test] diff --git a/cli/src/session/commands.rs b/cli/src/session/commands.rs index f0e18e5f..ffe56771 100644 --- a/cli/src/session/commands.rs +++ b/cli/src/session/commands.rs @@ -72,20 +72,6 @@ impl CLISession { Ok(_) => {}, Err(e) => eprintln!("Health check failed: {}", e), }, - Command::Pause => { - println!("Pausing ingestion..."); - match self.execute("PAUSE").await { - Ok(_) => println!("Ingestion paused"), - Err(e) => eprintln!("Pause failed: {}", e), - } - }, - Command::Continue => { - println!("Resuming ingestion..."); - match self.execute("CONTINUE").await { - Ok(_) => println!("Ingestion resumed"), - Err(e) => eprintln!("Resume failed: {}", e), - } - }, Command::ListTables => { self.execute( "SELECT namespace_id AS namespace, table_name, table_type FROM system.tables \ @@ -94,11 +80,11 @@ impl CLISession { .await?; }, Command::Describe(table) => { - let query = format!( - "SELECT * FROM information_schema.columns WHERE table_name = '{}' ORDER BY \ - ordinal_position", - table - ); + let query = Self::build_describe_query(&table)?; + self.execute(&query).await?; + }, + Command::ExecuteAs { user, sql } => { + let query = Self::build_execute_as_query(&user, &sql)?; self.execute(&query).await?; }, Command::SetFormat(format) => match format.to_lowercase().as_str() { @@ -135,7 +121,6 @@ impl CLISession { println!("No active subscription to cancel"); }, Command::RefreshTables => { - // This is handled in run_interactive, shouldn't reach here println!("Table names refreshed"); }, Command::ShowCredentials => { @@ -164,8 +149,6 @@ impl CLISession { .await?; }, Command::History => { - // Handled in run_interactive() where we have access to history - // This should not be reached eprintln!("History command should be handled in interactive mode"); }, Command::Consume { @@ -185,166 +168,250 @@ impl CLISession { Ok(()) } - /// Show help message (styled, sectioned) + fn build_describe_query(target: &str) -> Result { + let (namespace, table_name) = Self::parse_describe_target(target)?; + let columns = "table_schema AS namespace, table_name, column_name, data_type, \ + is_nullable, column_default, ordinal_position AS position"; + let escaped_table = Self::escape_sql_literal(&table_name); + + if let Some(namespace) = namespace { + Ok(format!( + "SELECT {columns} FROM information_schema.columns WHERE table_schema = '{namespace}' \ + AND table_name = '{table_name}' ORDER BY ordinal_position", + namespace = Self::escape_sql_literal(&namespace), + table_name = escaped_table, + )) + } else { + Ok(format!( + "SELECT {columns} FROM information_schema.columns WHERE table_name = '{table_name}' \ + ORDER BY table_schema, ordinal_position", + table_name = escaped_table, + )) + } + } + + fn build_execute_as_query(user: &str, sql: &str) -> Result { + let normalized_user = Self::normalize_execute_as_user(user)?; + let inner_sql = sql.trim().trim_end_matches(';').trim_end(); + + if inner_sql.is_empty() { + return Err(CLIError::ParseError( + "\\as requires a non-empty SQL statement".to_string(), + )); + } + + Ok(format!( + "EXECUTE AS USER '{}' ({})", + Self::escape_sql_literal(&normalized_user), + inner_sql, + )) + } + + fn normalize_execute_as_user(user: &str) -> Result { + let trimmed = user.trim(); + if trimmed.is_empty() { + return Err(CLIError::ParseError( + "\\as requires a target user".to_string(), + )); + } + + let normalized = if trimmed.len() >= 2 + && ((trimmed.starts_with('\'') && trimmed.ends_with('\'')) + || (trimmed.starts_with('"') && trimmed.ends_with('"'))) + { + &trimmed[1..trimmed.len() - 1] + } else { + trimmed + }; + + if normalized.is_empty() { + return Err(CLIError::ParseError( + "\\as requires a target user".to_string(), + )); + } + + Ok(normalized.to_string()) + } + + pub(super) fn parse_describe_target(target: &str) -> Result<(Option, String)> { + let trimmed = target.trim().trim_end_matches(';').trim(); + if trimmed.is_empty() { + return Err(CLIError::ParseError( + "\\describe requires a table name".to_string(), + )); + } + + let parts = Self::split_identifier_parts(trimmed)?; + match parts.as_slice() { + [table_name] => Ok((None, table_name.clone())), + [namespace, table_name] => Ok((Some(namespace.clone()), table_name.clone())), + _ => Err(CLIError::ParseError( + "\\describe expects

or ".to_string(), + )), + } + } + + fn split_identifier_parts(target: &str) -> Result> { + let mut parts = Vec::new(); + let mut current = String::new(); + let mut chars = target.chars().peekable(); + let mut in_quotes = false; + + while let Some(ch) = chars.next() { + match ch { + '"' => { + if in_quotes && chars.peek() == Some(&'"') { + current.push('"'); + chars.next(); + } else { + in_quotes = !in_quotes; + } + }, + '.' if !in_quotes => { + let part = current.trim(); + if part.is_empty() { + return Err(CLIError::ParseError( + "\\describe received an invalid identifier".to_string(), + )); + } + parts.push(part.to_string()); + current.clear(); + }, + _ => current.push(ch), + } + } + + if in_quotes { + return Err(CLIError::ParseError( + "\\describe received an unterminated quoted identifier".to_string(), + )); + } + + let tail = current.trim(); + if tail.is_empty() { + return Err(CLIError::ParseError( + "\\describe received an invalid identifier".to_string(), + )); + } + parts.push(tail.to_string()); + + Ok(parts) + } + + fn escape_sql_literal(value: &str) -> String { + value.replace('\'', "''") + } + + fn print_help_section(title: &str) { + println!("{}", title.yellow().bold()); + } + + fn print_help_row(command: &str, description: &str) { + println!(" {} {}", format!("{command:<38}").cyan(), description); + } + + fn print_help_example(example: &str) { + println!(" {}", example.green()); + } + fn show_help(&self) { println!(); - println!( - "{}", - "╔═══════════════════════════════════════════════════════════╗" - .bright_blue() - .bold() - ); - println!( - "{}{}{}", - "║ ".bright_blue().bold(), - "Commands & Shortcuts".white().bold(), - " ║" - .to_string() - .bright_blue() - .bold() - ); - println!( - "{}", - "╠═══════════════════════════════════════════════════════════╣" - .bright_blue() - .bold() - ); + println!("{}", "Kalam CLI Help".bright_blue().bold()); + println!(); - // Basics - println!("{}", "║ Basics".bright_blue().bold()); - println!("║ • Write SQL; end with ';' to run"); - println!( - "║ • Autocomplete: keywords, namespaces, tables, columns {}", - "(Tab)".dimmed() - ); - println!("║ • Inline hints and SQL highlighting enabled"); + Self::print_help_section("Basics"); + println!(" Write SQL and end with ';' to run it"); + println!(" Press Tab for SQL, table, namespace, and command completion"); + println!(" Press Up on an empty prompt to open command history"); + println!(); - // Meta-commands (two columns) - println!( - "{}", - "╠───────────────────────────────────────────────────────────╣" - .bright_blue() - .bold() - ); - println!("{}", "║ Meta-Commands".bright_blue().bold()); - let left = [ + Self::print_help_section("Meta Commands"); + for (command, description) in [ ("\\help, \\?", "Show this help"), ("\\quit, \\q", "Exit CLI"), - ("\\info, \\session", "Session info"), - ("\\sessions", "PG gRPC sessions"), - ("\\stats, \\metrics", "System stats"), - ("\\health", "Health check"), - ("\\format ", "table|json|csv"), - ("\\history, \\h", "Browse history"), - ("\\refresh-tables, \\refresh", "Refresh autocomplete"), - ]; - let right = [ + ("\\info, \\session", "Show session details"), + ("\\history, \\h", "Browse command history"), + ("\\health", "Run public health probes"), ("\\dt, \\tables", "List tables"), - ("\\d, \\describe
", "Describe table"), + ("\\d, \\describe
", "Describe a table"), + ("\\as ", "Wrap a statement as EXECUTE AS USER"), + ("\\format ", "Change output format"), + ("\\refresh-tables, \\refresh", "Refresh autocomplete caches"), + ("\\stats, \\metrics", "Show system stats"), + ("\\sessions", "Show active sessions"), ("\\flush", "Run STORAGE FLUSH ALL"), - ("\\pause", "Pause ingestion"), - ("\\continue", "Resume ingestion"), - ("\\subscribe, \\watch ", "Start live query"), - ("\\unsubscribe, \\unwatch", "Stop live query"), - ("\\consume ", "Consume topic messages"), ("\\cluster ", "Cluster operations"), - ]; - for i in 0..left.len().max(right.len()) { - let l = left - .get(i) - .map(|(a, b)| format!("{:<28} {:<18}", a.cyan(), b)) - .unwrap_or_default(); - let r = right - .get(i) - .map(|(a, b)| format!("{:<28} {:<18}", a.cyan(), b)) - .unwrap_or_default(); - println!("║ {:<47}{:<47} ║", l, r); + ("\\consume ", "Consume topic messages"), + ] { + Self::print_help_row(command, description); } + println!(); - // Cluster Commands - println!( - "{}", - "╠───────────────────────────────────────────────────────────╣" - .bright_blue() - .bold() - ); - println!("{}", "║ Cluster Commands".bright_blue().bold()); - println!("║ {:<48} Trigger snapshot", "\\cluster snapshot".cyan()); - println!("║ {:<48} Purge logs up to index", "\\cluster purge --upto ".cyan()); - println!("║ {:<48} Trigger cluster election", "\\cluster trigger-election".cyan()); - println!( - "║ {:<48} Transfer cluster leadership", - "\\cluster transfer-leader ".cyan() - ); - println!("║ {:<48} Rebalance data leaders", "\\cluster rebalance".cyan()); - println!("║ {:<48} Leader stepdown", "\\cluster stepdown".cyan()); - println!("║ {:<48} Clear old snapshots", "\\cluster clear".cyan()); - println!("║ {:<48} List cluster nodes", "\\cluster list".cyan()); - println!("║ {:<48} List all raft groups", "\\cluster list groups".cyan()); - println!( - "║ {:<48} Join node at runtime", - "\\cluster join ".cyan() - ); - println!( - "║ {:<48} Live per-node stats", - "\\subscribe SELECT * FROM system.cluster".cyan() - ); + Self::print_help_section("Live Queries"); + for (command, description) in [ + ("\\subscribe ", "Alias of \\subscribe"), + ("\\live
`, `\describe
` | Describe table | +| `\as ` | Wrap one statement as `EXECUTE AS USER` | | `\stats`, `\metrics` | Show `system.stats` | | `\health` | Server healthcheck | -| `\pause` / `\continue` | Pause/resume ingestion | +| `\flush` | Run `STORAGE FLUSH ALL` | | `\format table|json|csv` | Change output format | -| `\subscribe `, `\watch ` | Start WebSocket live subscription | +| `\subscribe `, `\watch `, `\live ` | Start live subscription | | `\unsubscribe`, `\unwatch` | No-op (prints “No active subscription to cancel”) | | `\cluster ...` | Cluster commands (see below) | | `\refresh-tables`, `\refresh` | Refresh autocomplete metadata | +| `\sessions` | Show active sessions | +| `\consume ...` | Consume topic messages | | `\show-credentials`, `\credentials` | Show stored credentials | | `\update-credentials

` | Update stored credentials | | `\delete-credentials` | Delete stored credentials | @@ -367,10 +371,11 @@ kalam --no-color -c "SELECT * FROM users" > output.txt ### 5. Timing Information -Execution time is displayed for all queries: +Execution metadata is displayed for all queries: ``` (10 rows) +As: root Took: 245.123 ms ``` diff --git a/docs/getting-started/quick-start.md b/docs/getting-started/quick-start.md index 0553ea03..bbd52c0b 100644 --- a/docs/getting-started/quick-start.md +++ b/docs/getting-started/quick-start.md @@ -79,10 +79,10 @@ JSON ### Optional: `EXECUTE AS USER` -Use wrapper syntax only. Ordinary USER-table reads stay scoped to the -authenticated user; explicit `EXECUTE AS USER` follows the role hierarchy: -system can target any role, DBA can target DBA/service/user, service can target -service/user, and regular users can only target themselves. +Use wrapper syntax only. Ordinary USER-table and STREAM-table access stays +scoped to the authenticated user; explicit `EXECUTE AS USER` follows the role +hierarchy: system can target any role, DBA can target DBA/service/user, +service can target service/user, and regular users can only target themselves. ```bash curl -u root: -X POST http://127.0.0.1:8080/v1/api/sql \ diff --git a/docs/reference/sql.md b/docs/reference/sql.md index 93ae5706..245b66af 100644 --- a/docs/reference/sql.md +++ b/docs/reference/sql.md @@ -195,9 +195,9 @@ FROM [.] ## Execute As User -`EXECUTE AS USER` syntax is wrapper-only. It switches USER-table execution to a -target user ID only when the authenticated actor role is allowed to target that -ID's cached role class. +`EXECUTE AS USER` syntax is wrapper-only. It switches USER-table or +STREAM-table execution to a target user ID only when the authenticated actor +role is allowed to target that ID's cached role class. ```sql EXECUTE AS USER '' ( @@ -221,7 +221,7 @@ Rules: 4. DBA users may target dba, service, and user accounts. 5. Service users may target service and user accounts. 6. Regular users may only use self-targeted `EXECUTE AS USER` as a no-op identity boundary. -7. The wrapper is valid only for USER tables; shared tables use their table policy directly. +7. The wrapper is valid for USER and STREAM tables; shared tables use their table policy directly. 8. Target role checks are hot-path cached: service, DBA, and system user IDs are tracked in memory from `system.users`; soft-deleted privileged IDs stay classified by their persisted role, and target IDs not present in that privileged cache are treated as regular users. 9. Legacy inline `... AS USER 'name'` syntax is not supported. diff --git a/examples/chat-with-ai/package-lock.json b/examples/chat-with-ai/package-lock.json index 390a4839..1c36c980 100644 --- a/examples/chat-with-ai/package-lock.json +++ b/examples/chat-with-ai/package-lock.json @@ -34,7 +34,7 @@ }, "../../link/sdks/typescript/client": { "name": "@kalamdb/client", - "version": "0.4.2-rc.3", + "version": "0.4.3-rc.4", "license": "Apache-2.0", "dependencies": { "ws": "^8.20.0" @@ -49,7 +49,7 @@ }, "../../link/sdks/typescript/consumer": { "name": "@kalamdb/consumer", - "version": "0.4.2-rc.3", + "version": "0.4.3-rc.4", "license": "Apache-2.0", "devDependencies": { "@types/node": "^25.5.0", @@ -64,13 +64,14 @@ }, "../../link/sdks/typescript/orm": { "name": "@kalamdb/orm", - "version": "0.4.2-rc.3", + "version": "0.4.3-rc.5", "license": "Apache-2.0", "bin": { "kalamdb-orm": "dist/cli.js" }, "devDependencies": { "@kalamdb/client": "file:../client", + "@kalamdb/consumer": "file:../consumer", "@types/node": "^25.5.2", "drizzle-orm": "^0.41.0", "typescript": "^5.8.0" diff --git a/examples/chat-with-ai/package.json b/examples/chat-with-ai/package.json index 4c0edb00..857c2575 100644 --- a/examples/chat-with-ai/package.json +++ b/examples/chat-with-ai/package.json @@ -14,15 +14,15 @@ "preview": "vite preview", "setup": "bash setup.sh", "preagent": "bash scripts/ensure-sdk.sh", - "agent": "tsx src/agent.ts", + "agent": "NODE_OPTIONS=--preserve-symlinks tsx src/agent.ts", "pretest": "bash scripts/ensure-sdk.sh", - "test": "tsc -p tsconfig.test.json && tsx --test tests/agent.test.ts && playwright test" + "test": "tsc -p tsconfig.test.json && NODE_OPTIONS=--preserve-symlinks tsx --test tests/agent.test.ts && playwright test" }, "dependencies": { "@kalamdb/client": "file:../../link/sdks/typescript/client", "@kalamdb/consumer": "file:../../link/sdks/typescript/consumer", "@kalamdb/orm": "file:../../link/sdks/typescript/orm", - "dotenv": "^17.4.1", + "dotenv": "^17.4.2", "drizzle-orm": "^0.41.0", "react": "^19.2.4", "react-dom": "^19.2.4" diff --git a/examples/chat-with-ai/scripts/generate-schema.sh b/examples/chat-with-ai/scripts/generate-schema.sh index 29502633..8714e7de 100644 --- a/examples/chat-with-ai/scripts/generate-schema.sh +++ b/examples/chat-with-ai/scripts/generate-schema.sh @@ -4,7 +4,8 @@ set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" PROJECT_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" -ORM_CLI="$PROJECT_DIR/../../link/sdks/typescript/orm/dist/cli.js" +ORM_CLI="$PROJECT_DIR/node_modules/@kalamdb/orm/dist/cli.js" +NODE_FLAGS=(--preserve-symlinks --preserve-symlinks-main) if [ -f "$PROJECT_DIR/.env.local" ]; then set -a @@ -13,7 +14,7 @@ if [ -f "$PROJECT_DIR/.env.local" ]; then set +a fi -node "$ORM_CLI" \ +node "${NODE_FLAGS[@]}" "$ORM_CLI" \ --url "${KALAMDB_URL:-http://127.0.0.1:8080}" \ --user "${KALAMDB_USER:-admin}" \ --password "${KALAMDB_PASSWORD:-kalamdb123}" \ diff --git a/link/kalam-client/Cargo.toml b/link/kalam-client/Cargo.toml index 59b19451..22b13e83 100644 --- a/link/kalam-client/Cargo.toml +++ b/link/kalam-client/Cargo.toml @@ -62,6 +62,7 @@ required-features = ["e2e-tests"] [dev-dependencies] tokio = { workspace = true, features = ["test-util", "macros"] } +tokio-netem = { workspace = true } kalamdb-configs = { workspace = true } kalamdb-server = { workspace = true } tempfile = { workspace = true } @@ -73,4 +74,4 @@ serde_json = { workspace = true } crate-type = ["cdylib", "rlib"] [package.metadata.wasm-pack.profile.release] -wasm-opt = false \ No newline at end of file +wasm-opt = false diff --git a/link/kalam-client/tests/common/tcp_proxy.rs b/link/kalam-client/tests/common/tcp_proxy.rs index 115aa988..2507a7d3 100644 --- a/link/kalam-client/tests/common/tcp_proxy.rs +++ b/link/kalam-client/tests/common/tcp_proxy.rs @@ -15,6 +15,10 @@ use tokio::{ task::JoinHandle, time::sleep, }; +use tokio_netem::{ + delayer::DynamicDuration, io::NetEmWriteExt, probability::DynamicProbability, + slicer::DynamicSize, terminator::Terminator, throttler::DynamicRate, +}; /// A TCP proxy that sits between a client and a real server, allowing tests to /// simulate network failures by pausing new connections and/or forcibly dropping @@ -43,6 +47,10 @@ struct ProxyImpairments { latency_ms: AtomicU64, stall_every_n_chunks: AtomicU64, stall_duration_ms: AtomicU64, + netem_write_delay: Arc, + netem_write_rate: Arc, + netem_write_slice_size: Arc, + netem_termination_probability: Arc, } impl ProxyImpairments { @@ -52,6 +60,11 @@ impl ProxyImpairments { latency_ms: AtomicU64::new(0), stall_every_n_chunks: AtomicU64::new(0), stall_duration_ms: AtomicU64::new(0), + netem_write_delay: DynamicDuration::new(Duration::ZERO), + netem_write_rate: DynamicRate::new(0), + netem_write_slice_size: DynamicSize::new(0), + netem_termination_probability: DynamicProbability::new(0.0) + .expect("zero termination probability should be valid"), } } @@ -96,7 +109,6 @@ impl TcpDisconnectProxy { let accept_task = tokio::spawn(async move { while let Ok((mut inbound, _peer)) = listener.accept().await { if paused_clone.load(Ordering::SeqCst) { - let _ = inbound.set_linger(Some(Duration::ZERO)); let _ = inbound.shutdown().await; drop(inbound); continue; @@ -191,6 +203,54 @@ impl TcpDisconnectProxy { self.impairments.stall_duration_ms.store(0, Ordering::SeqCst); } + /// Add tokio-netem write-side latency in both directions. + pub fn set_netem_write_delay(&self, delay: Duration) { + self.impairments.netem_write_delay.set(delay); + } + + /// Clear tokio-netem write-side latency. + pub fn clear_netem_write_delay(&self) { + self.impairments.netem_write_delay.set(Duration::ZERO); + } + + /// Throttle forwarded writes in both directions to `bytes_per_second`. + /// + /// A value of `0` disables throttling. + pub fn set_netem_write_rate(&self, bytes_per_second: usize) { + self.impairments.netem_write_rate.set(bytes_per_second); + } + + /// Clear any tokio-netem write throttle. + pub fn clear_netem_write_rate(&self) { + self.impairments.netem_write_rate.set(0); + } + + /// Slice forwarded writes in both directions. A size of `0` disables slicing. + pub fn set_netem_write_slice_size(&self, size: usize) { + self.impairments.netem_write_slice_size.set(size); + } + + /// Clear tokio-netem write slicing. + pub fn clear_netem_write_slice_size(&self) { + self.impairments.netem_write_slice_size.set(0); + } + + /// Probabilistically terminate each proxied transport poll in both directions. + pub fn set_netem_termination_probability(&self, probability: f64) { + self.impairments + .netem_termination_probability + .set(probability) + .expect("termination probability should be between 0.0 and 1.0"); + } + + /// Clear tokio-netem transport termination. + pub fn clear_netem_termination_probability(&self) { + self.impairments + .netem_termination_probability + .set(0.0) + .expect("zero termination probability should be valid"); + } + /// Abort all in-flight proxy tasks, forcibly closing both sides of every /// active connection. pub async fn drop_active_connections(&self) { @@ -271,11 +331,18 @@ async fn bind_loopback_listener() -> std::io::Result { async fn relay_with_impairments( mut reader: tokio::net::tcp::ReadHalf<'_>, - mut writer: tokio::net::tcp::WriteHalf<'_>, + writer: tokio::net::tcp::WriteHalf<'_>, impairments: Arc, ) -> std::io::Result<()> { let mut buffer = [0_u8; 16 * 1024]; let mut chunk_index = 0_u64; + let mut writer = Terminator::new( + writer + .delay_writes_dyn(impairments.netem_write_delay.clone()) + .throttle_writes_dyn(impairments.netem_write_rate.clone()) + .slice_writes_dyn(impairments.netem_write_slice_size.clone()), + impairments.netem_termination_probability.clone(), + ); loop { let read = reader.read(&mut buffer).await?; diff --git a/link/kalam-client/tests/proxied.rs b/link/kalam-client/tests/proxied.rs index 415c1804..6d81cf3a 100644 --- a/link/kalam-client/tests/proxied.rs +++ b/link/kalam-client/tests/proxied.rs @@ -43,6 +43,8 @@ mod socket_drop_resume; mod staggered_outages; #[path = "proxied/subscribe_during_reconnect.rs"] mod subscribe_during_reconnect; +#[path = "proxied/topic_consumption_netem.rs"] +mod topic_consumption_netem; #[path = "proxied/transport_impairments.rs"] mod transport_impairments; #[path = "proxied/unsubscribe_during_outage.rs"] diff --git a/link/kalam-client/tests/proxied/topic_consumption_netem.rs b/link/kalam-client/tests/proxied/topic_consumption_netem.rs new file mode 100644 index 00000000..be985df5 --- /dev/null +++ b/link/kalam-client/tests/proxied/topic_consumption_netem.rs @@ -0,0 +1,471 @@ +use std::{collections::HashSet, time::Duration}; + +use kalam_client::consumer::{AutoOffsetReset, ConsumerRecord, TopicConsumer, TopicOp}; +use serde_json::Value; +use tokio::time::{sleep, timeout, Instant}; + +use super::helpers::*; +use crate::common::tcp_proxy::TcpDisconnectProxy; + +async fn setup_topic_with_sources( + writer: &kalam_client::KalamLinkClient, + namespace: &str, + table: &str, + topic: &str, + operations: &[&str], +) { + writer + .execute_query(&format!("CREATE NAMESPACE IF NOT EXISTS {}", namespace), None, None, None) + .await + .expect("create topic test namespace"); + writer + .execute_query( + &format!( + "CREATE TABLE IF NOT EXISTS {}.{} (id INT PRIMARY KEY, value TEXT, counter INT)", + namespace, table + ), + None, + None, + None, + ) + .await + .expect("create topic source table"); + writer + .execute_query(&format!("CREATE TOPIC {}", topic), None, None, None) + .await + .expect("create topic"); + + for operation in operations { + writer + .execute_query( + &format!( + "ALTER TOPIC {} ADD SOURCE {}.{} ON {}", + topic, namespace, table, operation + ), + None, + None, + None, + ) + .await + .expect("add topic source route"); + } + + wait_for_topic_routes(writer, topic, operations.len()).await; +} + +async fn wait_for_topic_routes( + writer: &kalam_client::KalamLinkClient, + topic: &str, + min_routes: usize, +) { + let deadline = Instant::now() + Duration::from_secs(15); + let sql = format!("SELECT routes FROM system.topics WHERE topic_id = '{}'", topic); + + while Instant::now() < deadline { + if let Ok(result) = writer.execute_query(&sql, None, None, None).await { + if let Some(routes) = result.get_string("routes") { + if serde_json::from_str::(&routes) + .ok() + .and_then(|value| value.as_array().map(|routes| routes.len())) + .unwrap_or(0) + >= min_routes + { + sleep(Duration::from_millis(100)).await; + return; + } + } + } + sleep(Duration::from_millis(100)).await; + } + + panic!("topic {} did not expose at least {} routes", topic, min_routes); +} + +fn build_consumer( + client: &kalam_client::KalamLinkClient, + topic: &str, + group_id: &str, + enable_auto_commit: bool, +) -> TopicConsumer { + client + .consumer() + .topic(topic) + .group_id(group_id) + .auto_offset_reset(AutoOffsetReset::Earliest) + .enable_auto_commit(enable_auto_commit) + .max_poll_records(25) + .poll_timeout(Duration::from_secs(1)) + .request_timeout(Duration::from_secs(10)) + .retry_backoff(Duration::from_millis(100)) + .build() + .expect("build topic consumer") +} + +async fn poll_records_until( + consumer: &mut TopicConsumer, + min_records: usize, + timeout_dur: Duration, +) -> Vec { + let deadline = Instant::now() + timeout_dur; + let mut records = Vec::new(); + let mut seen = HashSet::<(u32, u64)>::new(); + let mut last_error = None; + + while Instant::now() < deadline && records.len() < min_records { + match consumer.poll().await { + Ok(batch) if batch.is_empty() => sleep(Duration::from_millis(50)).await, + Ok(batch) => { + for record in batch { + if seen.insert((record.partition_id, record.offset)) { + records.push(record); + } + } + }, + Err(err) if is_retryable_consumer_error(&err.to_string()) => { + last_error = Some(err.to_string()); + sleep(Duration::from_millis(50)).await; + }, + Err(err) => panic!("topic consumer poll failed: {}", err), + } + } + + assert!( + records.len() >= min_records, + "expected at least {} topic records, got {}; last retryable error: {:?}", + min_records, + records.len(), + last_error + ); + records +} + +fn is_retryable_consumer_error(message: &str) -> bool { + let message = message.to_ascii_lowercase(); + message.contains("network") + || message.contains("timeout") + || message.contains("connect") + || message.contains("error decoding response body") + || message.contains("request") +} + +fn payload_json(record: &ConsumerRecord) -> Value { + serde_json::from_slice(&record.payload).expect("topic payload should be JSON") +} + +fn payload_i64(payload: &Value, key: &str) -> i64 { + let value = payload.get(key).unwrap_or_else(|| panic!("payload should contain {}", key)); + if let Some(number) = value.as_i64() { + return number; + } + if let Some(object) = value.as_object() { + for nested in object.values() { + if let Some(number) = nested.as_i64() { + return number; + } + if let Some(text) = nested.as_str().and_then(|text| text.parse::().ok()) { + return text; + } + } + } + value + .as_str() + .and_then(|text| text.parse::().ok()) + .unwrap_or_else(|| panic!("payload field {} should be integer-compatible: {}", key, value)) +} + +fn payload_string(payload: &Value, key: &str) -> String { + let value = payload.get(key).unwrap_or_else(|| panic!("payload should contain {}", key)); + if let Some(text) = value.as_str() { + return text.to_string(); + } + if let Some(object) = value.as_object() { + for nested in object.values() { + if let Some(text) = nested.as_str() { + return text.to_string(); + } + } + } + panic!("payload field {} should be string-compatible: {}", key, value); +} + +fn records_with_op<'a>(records: &'a [ConsumerRecord], op: TopicOp) -> Vec<&'a ConsumerRecord> { + records.iter().filter(|record| record.op == op).collect() +} + +#[tokio::test] +#[ntest::timeout(10000)] +async fn test_tokio_netem_topic_consume_fragmented_insert_update_delete_and_commit() { + let result = timeout(Duration::from_secs(60), async { + let writer = match create_test_client() { + Ok(client) => client, + Err(err) => { + eprintln!("Skipping test (writer client unavailable): {}", err); + return; + }, + }; + + let proxy = TcpDisconnectProxy::start(upstream_server_url()).await; + let client = match create_test_client_for_base_url(proxy.base_url()) { + Ok(client) => client, + Err(err) => { + eprintln!("Skipping test (proxy client unavailable): {}", err); + proxy.shutdown().await; + return; + }, + }; + + let suffix = unique_suffix(); + let namespace = format!("topic_netem_{}", suffix); + let table = "events"; + let topic = format!("{}.{}", namespace, table); + let group = format!("topic-netem-fragmented-{}", suffix); + + setup_topic_with_sources( + &writer, + &namespace, + table, + &topic, + &["INSERT", "UPDATE", "DELETE"], + ) + .await; + + writer + .execute_query( + &format!( + "INSERT INTO {}.{} (id, value, counter) VALUES (1, 'created', 0)", + namespace, table + ), + None, + None, + None, + ) + .await + .expect("insert topic source row"); + writer + .execute_query( + &format!( + "UPDATE {}.{} SET value = 'updated', counter = 1 WHERE id = 1", + namespace, table + ), + None, + None, + None, + ) + .await + .expect("update topic source row"); + writer + .execute_query( + &format!("DELETE FROM {}.{} WHERE id = 1", namespace, table), + None, + None, + None, + ) + .await + .expect("delete topic source row"); + + proxy.set_netem_write_slice_size(64); + + let mut consumer = build_consumer(&client, &topic, &group, false); + let records = poll_records_until(&mut consumer, 3, Duration::from_secs(20)).await; + let inserts = records_with_op(&records, TopicOp::Insert); + let updates = records_with_op(&records, TopicOp::Update); + let deletes = records_with_op(&records, TopicOp::Delete); + + assert_eq!(inserts.len(), 1); + assert_eq!(updates.len(), 1); + assert_eq!(deletes.len(), 1); + + let insert_payload = payload_json(inserts[0]); + assert_eq!(payload_i64(&insert_payload, "id"), 1); + assert_eq!(payload_string(&insert_payload, "value"), "created"); + + let update_payload = payload_json(updates[0]); + assert_eq!(payload_i64(&update_payload, "id"), 1); + assert_eq!(payload_string(&update_payload, "value"), "updated"); + assert_eq!(payload_i64(&update_payload, "counter"), 1); + + for record in &records { + consumer.mark_processed(record); + } + let commit = consumer.commit_sync().await.expect("commit fragmented topic records"); + assert_eq!(commit.acknowledged_offset, 2); + + let mut resumed = build_consumer(&client, &topic, &group, false); + let replay = resumed + .poll_with_timeout(Duration::from_millis(250)) + .await + .expect("poll after committed offset"); + assert!(replay.is_empty(), "committed topic offsets should not replay records"); + + proxy.clear_netem_write_slice_size(); + consumer.close().await.ok(); + resumed.close().await.ok(); + proxy.shutdown().await; + }) + .await; + + assert!(result.is_ok(), "fragmented topic consume test timed out"); +} + +#[tokio::test] +#[ntest::timeout(17000)] +async fn test_tokio_netem_topic_bandwidth_collapse_poll_recovers_without_losing_offsets() { + let result = timeout(Duration::from_secs(75), async { + let writer = match create_test_client() { + Ok(client) => client, + Err(err) => { + eprintln!("Skipping test (writer client unavailable): {}", err); + return; + }, + }; + + let proxy = TcpDisconnectProxy::start(upstream_server_url()).await; + let client = match create_test_client_for_base_url(proxy.base_url()) { + Ok(client) => client, + Err(err) => { + eprintln!("Skipping test (proxy client unavailable): {}", err); + proxy.shutdown().await; + return; + }, + }; + + let suffix = unique_suffix(); + let namespace = format!("topic_netem_throttle_{}", suffix); + let table = "events"; + let topic = format!("{}.{}", namespace, table); + let group = format!("topic-netem-throttle-{}", suffix); + let impaired_group = format!("topic-netem-throttle-impaired-{}", suffix); + + setup_topic_with_sources(&writer, &namespace, table, &topic, &["INSERT"]).await; + + for id in 0..5 { + writer + .execute_query( + &format!( + "INSERT INTO {}.{} (id, value, counter) VALUES ({}, 'value_{}', {})", + namespace, table, id, id, id + ), + None, + None, + None, + ) + .await + .expect("insert topic row during throttle test"); + } + + let mut impaired_consumer = build_consumer(&client, &topic, &impaired_group, false); + proxy.set_netem_write_rate(8); + + let throttled_poll = timeout(Duration::from_secs(4), impaired_consumer.poll()).await; + assert!( + throttled_poll.is_err() + || throttled_poll + .as_ref() + .ok() + .and_then(|result| result.as_ref().ok()) + .map_or(true, Vec::is_empty), + "severe bandwidth collapse should not successfully drain topic records immediately" + ); + + proxy.clear_netem_write_rate(); + let mut consumer = build_consumer(&client, &topic, &group, false); + let records = poll_records_until(&mut consumer, 5, Duration::from_secs(20)).await; + let mut ids = records + .iter() + .map(|record| payload_i64(&payload_json(record), "id")) + .collect::>(); + ids.sort_unstable(); + assert_eq!(ids, vec![0, 1, 2, 3, 4]); + + for record in &records { + consumer.mark_processed(record); + } + let commit = consumer.commit_sync().await.expect("commit after bandwidth recovery"); + assert_eq!(commit.acknowledged_offset, 4); + + consumer.close().await.ok(); + impaired_consumer.close().await.ok(); + proxy.shutdown().await; + }) + .await; + + assert!(result.is_ok(), "topic bandwidth collapse test timed out"); +} + +#[tokio::test] +#[ntest::timeout(10000)] +async fn test_tokio_netem_topic_commit_failure_can_be_retried_without_replay() { + let result = timeout(Duration::from_secs(75), async { + let writer = match create_test_client() { + Ok(client) => client, + Err(err) => { + eprintln!("Skipping test (writer client unavailable): {}", err); + return; + }, + }; + + let proxy = TcpDisconnectProxy::start(upstream_server_url()).await; + let client = match create_test_client_for_base_url(proxy.base_url()) { + Ok(client) => client, + Err(err) => { + eprintln!("Skipping test (proxy client unavailable): {}", err); + proxy.shutdown().await; + return; + }, + }; + + let suffix = unique_suffix(); + let namespace = format!("topic_netem_commit_{}", suffix); + let table = "events"; + let topic = format!("{}.{}", namespace, table); + let group = format!("topic-netem-commit-{}", suffix); + + setup_topic_with_sources(&writer, &namespace, table, &topic, &["INSERT"]).await; + + for id in 10..13 { + writer + .execute_query( + &format!( + "INSERT INTO {}.{} (id, value, counter) VALUES ({}, 'value_{}', {})", + namespace, table, id, id, id + ), + None, + None, + None, + ) + .await + .expect("insert topic row for commit retry test"); + } + + let mut consumer = build_consumer(&client, &topic, &group, false); + let records = poll_records_until(&mut consumer, 3, Duration::from_secs(20)).await; + for record in &records { + consumer.mark_processed(record); + } + + proxy.set_netem_termination_probability(1.0); + let failed_commit = consumer.commit_sync().await; + assert!( + failed_commit.is_err(), + "netem transport termination should make the topic commit fail" + ); + + proxy.clear_netem_termination_probability(); + let commit = consumer.commit_sync().await.expect("retry commit after netem recovery"); + assert_eq!(commit.acknowledged_offset, 2); + + let mut resumed = build_consumer(&client, &topic, &group, false); + let replay = resumed + .poll_with_timeout(Duration::from_millis(250)) + .await + .expect("poll after retried commit"); + assert!(replay.is_empty(), "retried commit should prevent topic record replay"); + + consumer.close().await.ok(); + resumed.close().await.ok(); + proxy.shutdown().await; + }) + .await; + + assert!(result.is_ok(), "topic commit retry test timed out"); +} diff --git a/link/kalam-client/tests/proxied/transport_impairments.rs b/link/kalam-client/tests/proxied/transport_impairments.rs index eac3406f..089f19da 100644 --- a/link/kalam-client/tests/proxied/transport_impairments.rs +++ b/link/kalam-client/tests/proxied/transport_impairments.rs @@ -450,3 +450,362 @@ async fn test_proxy_packet_loss_style_stalls_resume_without_replay() { assert!(result.is_ok(), "packet-loss-style stall test timed out"); } + +/// Very small write slices fragment WebSocket frames at the transport boundary. +/// The client should parse the stream normally and avoid spurious reconnects. +#[tokio::test] +#[ntest::timeout(8000)] +async fn test_tokio_netem_fragmented_writes_preserve_live_stream() { + let result = timeout(Duration::from_secs(45), async { + let writer = match create_test_client() { + Ok(c) => c, + Err(e) => { + eprintln!("Skipping test (writer client unavailable): {}", e); + return; + }, + }; + + let proxy = TcpDisconnectProxy::start(upstream_server_url()).await; + let (client, _connect_count, disconnect_count) = + match create_test_client_with_events_for_base_url(proxy.base_url()) { + Ok(v) => v, + Err(e) => { + eprintln!("Skipping test (proxy client unavailable): {}", e); + proxy.shutdown().await; + return; + }, + }; + + let suffix = unique_suffix(); + let table = format!("default.netem_sliced_{}", suffix); + ensure_table(&writer, &table).await; + + client.connect().await.expect("initial connect through proxy"); + let mut sub = client + .subscribe_with_config(SubscriptionConfig::new( + format!("netem-sliced-{}", suffix), + format!("SELECT id, value FROM {}", table), + )) + .await + .expect("subscribe through proxy"); + + let _ = timeout(TEST_TIMEOUT, sub.next()).await; + let disconnects_before = disconnect_count.load(Ordering::SeqCst); + proxy.set_netem_write_slice_size(3); + + writer + .execute_query( + &format!( + "INSERT INTO {} (id, value) VALUES ('fragmented', '{}')", + table, + "x".repeat(512) + ), + None, + None, + None, + ) + .await + .expect("insert row through fragmented transport"); + + let mut ids = Vec::::new(); + let mut seq = None; + for _ in 0..30 { + if ids.iter().any(|id| id == "fragmented") { + break; + } + match timeout(Duration::from_millis(1000), sub.next()).await { + Ok(Some(Ok(ev))) => { + collect_ids_and_track_seq(&ev, &mut ids, &mut seq, None, "netem slicing"); + }, + Ok(Some(Err(e))) => panic!("netem slicing subscription error: {}", e), + Ok(None) => panic!("netem slicing subscription ended unexpectedly"), + Err(_) => {}, + } + } + + assert!( + ids.iter().any(|id| id == "fragmented"), + "live row should arrive when netem fragments writes" + ); + assert_eq!( + disconnect_count.load(Ordering::SeqCst), + disconnects_before, + "write fragmentation should not cause a reconnect" + ); + + proxy.clear_netem_write_slice_size(); + sub.close().await.ok(); + client.disconnect().await; + proxy.shutdown().await; + }) + .await; + + assert!(result.is_ok(), "netem fragmented write test timed out"); +} + +/// A severe bandwidth collapse should look like a dead link to keepalive. Once +/// the throttle is removed, reconnect resume must deliver only rows after the +/// checkpoint. +#[tokio::test] +#[ntest::timeout(16000)] +async fn test_tokio_netem_bandwidth_collapse_forces_resume_without_replay() { + let result = timeout(Duration::from_secs(75), async { + let writer = match create_test_client() { + Ok(c) => c, + Err(e) => { + eprintln!("Skipping test (writer client unavailable): {}", e); + return; + }, + }; + + let proxy = TcpDisconnectProxy::start(upstream_server_url()).await; + let (client, connect_count, disconnect_count) = + match create_test_client_with_events_for_base_url(proxy.base_url()) { + Ok(v) => v, + Err(e) => { + eprintln!("Skipping test (proxy client unavailable): {}", e); + proxy.shutdown().await; + return; + }, + }; + + let suffix = unique_suffix(); + let table = format!("default.netem_throttle_{}", suffix); + ensure_table(&writer, &table).await; + + client.connect().await.expect("initial connect through proxy"); + let mut sub = client + .subscribe_with_config(SubscriptionConfig::new( + format!("netem-throttle-{}", suffix), + format!("SELECT id, value FROM {}", table), + )) + .await + .expect("subscribe through proxy"); + + let _ = timeout(TEST_TIMEOUT, sub.next()).await; + writer + .execute_query( + &format!("INSERT INTO {} (id, value) VALUES ('baseline-throttle', 'ready')", table), + None, + None, + None, + ) + .await + .expect("insert baseline row"); + + let mut baseline_ids = Vec::::new(); + let mut checkpoint = None; + for _ in 0..12 { + if baseline_ids.iter().any(|id| id == "baseline-throttle") { + break; + } + match timeout(Duration::from_millis(1200), sub.next()).await { + Ok(Some(Ok(ev))) => { + collect_ids_and_track_seq( + &ev, + &mut baseline_ids, + &mut checkpoint, + None, + "netem throttle baseline", + ); + }, + _ => {}, + } + } + assert!(baseline_ids.iter().any(|id| id == "baseline-throttle")); + let resume_from = query_max_seq(&writer, &table).await; + + let disconnects_before = disconnect_count.load(Ordering::SeqCst); + let expected_connects = connect_count.load(Ordering::SeqCst) + 1; + proxy.set_netem_write_rate(8); + + writer + .execute_query( + &format!( + "INSERT INTO {} (id, value) VALUES ('gap-throttle', 'during-collapse')", + table + ), + None, + None, + None, + ) + .await + .expect("insert gap row during bandwidth collapse"); + + for _ in 0..80 { + if disconnect_count.load(Ordering::SeqCst) > disconnects_before { + break; + } + sleep(Duration::from_millis(100)).await; + } + assert!( + disconnect_count.load(Ordering::SeqCst) > disconnects_before, + "severe netem throttle should force a reconnect" + ); + + proxy.clear_netem_write_rate(); + wait_for_reconnect(&client, &connect_count, expected_connects, "netem throttle").await; + + writer + .execute_query( + &format!( + "INSERT INTO {} (id, value) VALUES ('live-throttle', 'after-reconnect')", + table + ), + None, + None, + None, + ) + .await + .expect("insert live row after throttle clears"); + + wait_for_row_after_checkpoint( + &mut sub, + resume_from, + &["gap-throttle", "live-throttle"], + &["baseline-throttle"], + "netem throttle recovery", + ) + .await; + + sub.close().await.ok(); + client.disconnect().await; + proxy.shutdown().await; + }) + .await; + + assert!(result.is_ok(), "netem bandwidth collapse test timed out"); +} + +/// tokio-netem can fail the transport from inside the I/O adapter instead of +/// aborting the proxy task. The client should treat it as a normal disconnect. +#[tokio::test] +#[ntest::timeout(10000)] +async fn test_tokio_netem_forced_transport_termination_recovers() { + let result = timeout(Duration::from_secs(75), async { + let writer = match create_test_client() { + Ok(c) => c, + Err(e) => { + eprintln!("Skipping test (writer client unavailable): {}", e); + return; + }, + }; + + let proxy = TcpDisconnectProxy::start(upstream_server_url()).await; + let (client, connect_count, disconnect_count) = + match create_test_client_with_events_for_base_url(proxy.base_url()) { + Ok(v) => v, + Err(e) => { + eprintln!("Skipping test (proxy client unavailable): {}", e); + proxy.shutdown().await; + return; + }, + }; + + let suffix = unique_suffix(); + let table = format!("default.netem_terminator_{}", suffix); + ensure_table(&writer, &table).await; + + client.connect().await.expect("initial connect through proxy"); + let mut sub = client + .subscribe_with_config(SubscriptionConfig::new( + format!("netem-terminator-{}", suffix), + format!("SELECT id, value FROM {}", table), + )) + .await + .expect("subscribe through proxy"); + + let _ = timeout(TEST_TIMEOUT, sub.next()).await; + writer + .execute_query( + &format!("INSERT INTO {} (id, value) VALUES ('baseline-term', 'ready')", table), + None, + None, + None, + ) + .await + .expect("insert baseline row"); + + let mut baseline_ids = Vec::::new(); + let mut checkpoint = None; + for _ in 0..12 { + if baseline_ids.iter().any(|id| id == "baseline-term") { + break; + } + match timeout(Duration::from_millis(1200), sub.next()).await { + Ok(Some(Ok(ev))) => { + collect_ids_and_track_seq( + &ev, + &mut baseline_ids, + &mut checkpoint, + None, + "netem termination baseline", + ); + }, + _ => {}, + } + } + assert!(baseline_ids.iter().any(|id| id == "baseline-term")); + let resume_from = query_max_seq(&writer, &table).await; + + let disconnects_before = disconnect_count.load(Ordering::SeqCst); + let expected_connects = connect_count.load(Ordering::SeqCst) + 1; + proxy.set_netem_termination_probability(1.0); + + writer + .execute_query( + &format!( + "INSERT INTO {} (id, value) VALUES ('gap-term', 'during-termination')", + table + ), + None, + None, + None, + ) + .await + .expect("insert gap row during transport termination"); + + for _ in 0..80 { + if disconnect_count.load(Ordering::SeqCst) > disconnects_before { + break; + } + sleep(Duration::from_millis(100)).await; + } + assert!( + disconnect_count.load(Ordering::SeqCst) > disconnects_before, + "netem terminator should force a disconnect" + ); + + proxy.clear_netem_termination_probability(); + wait_for_reconnect(&client, &connect_count, expected_connects, "netem terminator").await; + + writer + .execute_query( + &format!( + "INSERT INTO {} (id, value) VALUES ('live-term', 'after-reconnect')", + table + ), + None, + None, + None, + ) + .await + .expect("insert live row after terminator clears"); + + wait_for_row_after_checkpoint( + &mut sub, + resume_from, + &["gap-term", "live-term"], + &["baseline-term"], + "netem terminator recovery", + ) + .await; + + sub.close().await.ok(); + client.disconnect().await; + proxy.shutdown().await; + }) + .await; + + assert!(result.is_ok(), "netem forced termination test timed out"); +} diff --git a/link/kalam-link-dart/src/tests.rs b/link/kalam-link-dart/src/tests.rs index 6292de08..8945a5f4 100644 --- a/link/kalam-link-dart/src/tests.rs +++ b/link/kalam-link-dart/src/tests.rs @@ -141,6 +141,7 @@ mod tests { named_rows: None, row_count: 1, message: None, + as_user: None, }; let resp = QueryResponse { status: ResponseStatus::Success, @@ -187,6 +188,7 @@ mod tests { named_rows: None, row_count: 0, message: Some("Table created".into()), + as_user: None, }; let dart: DartQueryResult = qr.into(); assert!(dart.columns.is_empty()); diff --git a/link/link-common/Cargo.toml b/link/link-common/Cargo.toml index 64a0bf9b..30b10a9f 100644 --- a/link/link-common/Cargo.toml +++ b/link/link-common/Cargo.toml @@ -49,13 +49,14 @@ web-sys = { workspace = true, features = [ [target.'cfg(not(target_arch = "wasm32"))'.dependencies] aws-lc-rs = { workspace = true, optional = true } +rustls = { workspace = true, features = ["aws_lc_rs"], optional = true } quinn-proto = { workspace = true, optional = true } rustls-webpki = { workspace = true, optional = true } [features] default = [] client-core = ["dep:futures-util", "dep:log", "dep:miniz_oxide", "dep:rmp-serde"] -tokio-runtime = ["client-core", "tokio", "reqwest", "tokio-tungstenite", "bytes", "dep:aws-lc-rs", "dep:quinn-proto", "dep:rustls-webpki"] +tokio-runtime = ["client-core", "tokio", "reqwest", "tokio-tungstenite", "bytes", "dep:aws-lc-rs", "dep:rustls", "dep:quinn-proto", "dep:rustls-webpki"] auth-flows = [] setup = [] healthcheck = [] diff --git a/link/link-common/src/client/builder.rs b/link/link-common/src/client/builder.rs index 6274540c..098dfa1d 100644 --- a/link/link-common/src/client/builder.rs +++ b/link/link-common/src/client/builder.rs @@ -1,5 +1,5 @@ use std::{ - sync::{Arc, RwLock}, + sync::{Arc, Once, RwLock}, time::Duration, }; @@ -17,6 +17,14 @@ use crate::{ timeouts::KalamLinkTimeouts, }; +fn ensure_rustls_crypto_provider() { + static INSTALL_RUSTLS_PROVIDER: Once = Once::new(); + + INSTALL_RUSTLS_PROVIDER.call_once(|| { + let _ = rustls::crypto::aws_lc_rs::default_provider().install_default(); + }); +} + impl KalamLinkClientBuilder { pub(crate) fn new() -> Self { Self { @@ -107,6 +115,8 @@ impl KalamLinkClientBuilder { /// Build the client pub fn build(self) -> Result { + ensure_rustls_crypto_provider(); + let base_url = self .base_url .ok_or_else(|| KalamLinkError::ConfigurationError("base_url is required".into()))?; diff --git a/link/link-common/src/client/tests.rs b/link/link-common/src/client/tests.rs index 8d54258e..79c3dc8b 100644 --- a/link/link-common/src/client/tests.rs +++ b/link/link-common/src/client/tests.rs @@ -21,6 +21,18 @@ fn test_builder_pattern() { assert!(result.is_ok()); } +#[test] +fn test_builder_installs_rustls_provider_idempotently() { + for _ in 0..2 { + let result = KalamLinkClient::builder() + .base_url("https://kalam.masky.app") + .jwt_token("test_token") + .build(); + + assert!(result.is_ok()); + } +} + #[test] fn test_builder_missing_url() { let result = KalamLinkClient::builder().build(); diff --git a/link/link-common/src/query/models/query_result.rs b/link/link-common/src/query/models/query_result.rs index ea40f2ca..98b32681 100644 --- a/link/link-common/src/query/models/query_result.rs +++ b/link/link-common/src/query/models/query_result.rs @@ -33,6 +33,10 @@ pub struct QueryResult { /// Optional message for non-query statements. #[serde(skip_serializing_if = "Option::is_none")] pub message: Option, + + /// Effective user identifier this statement executed as. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub as_user: Option, } impl QueryResult { diff --git a/link/sdks/typescript/client/README.md b/link/sdks/typescript/client/README.md index 1baf6a4d..fb51e8e3 100644 --- a/link/sdks/typescript/client/README.md +++ b/link/sdks/typescript/client/README.md @@ -187,7 +187,7 @@ This pattern is useful for chat threads, activity feeds, audit logs, and any UI ## Preserve Tenant Boundaries in Worker Writes -Background services should keep the same user isolation guarantees as the browser. Direct USER-table reads stay scoped to the authenticated account; `executeAsUser()` switches subject only when KalamDB authorizes the actor role for the target user's role. +Background services should keep the same user isolation guarantees as the browser. Direct USER-table and STREAM-table access stays scoped to the authenticated account; `executeAsUser()` switches subject only when KalamDB authorizes the actor role for the target user's role. ```ts await client.executeAsUser( @@ -197,7 +197,7 @@ await client.executeAsUser( ); ``` -That keeps the write inside Alice's USER-table partition through an explicit, audited delegation boundary instead of leaking service-side writes into the wrong tenant scope. +That keeps the write inside Alice's USER-table or STREAM-table partition through an explicit, audited delegation boundary instead of leaking service-side writes into the wrong tenant scope. ## Lower-Level Realtime API diff --git a/link/sdks/typescript/client/package-lock.json b/link/sdks/typescript/client/package-lock.json index 40cc5ed9..251b3239 100644 --- a/link/sdks/typescript/client/package-lock.json +++ b/link/sdks/typescript/client/package-lock.json @@ -1,12 +1,12 @@ { "name": "@kalamdb/client", - "version": "0.4.2-rc.3", + "version": "0.4.3-rc.4", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@kalamdb/client", - "version": "0.4.2-rc.3", + "version": "0.4.3-rc.4", "license": "Apache-2.0", "dependencies": { "ws": "^8.20.0" diff --git a/link/sdks/typescript/client/src/client.ts b/link/sdks/typescript/client/src/client.ts index 8ef84ee8..bf077550 100644 --- a/link/sdks/typescript/client/src/client.ts +++ b/link/sdks/typescript/client/src/client.ts @@ -701,7 +701,8 @@ export class KalamDBClient { * Execute a single SQL statement with an AS USER wrapper. * * KalamDB authorizes cross-user targets with its EXECUTE AS USER role matrix. - * Regular users can only target themselves. + * Regular users can only target themselves. The wrapper is valid for USER + * and STREAM tables; SHARED tables use their table policy directly. * * Wraps the SQL using: * `EXECUTE AS USER 'user' ( )` diff --git a/link/sdks/typescript/orm/package-lock.json b/link/sdks/typescript/orm/package-lock.json index 7b3b4467..a39752ff 100644 --- a/link/sdks/typescript/orm/package-lock.json +++ b/link/sdks/typescript/orm/package-lock.json @@ -1,12 +1,12 @@ { "name": "@kalamdb/orm", - "version": "0.4.2-rc.3", + "version": "0.4.3-rc.5", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@kalamdb/orm", - "version": "0.4.2-rc.3", + "version": "0.4.3-rc.5", "license": "Apache-2.0", "bin": { "kalamdb-orm": "dist/cli.js" @@ -28,7 +28,7 @@ }, "../client": { "name": "@kalamdb/client", - "version": "0.4.2-rc.3", + "version": "0.4.3-rc.4", "dev": true, "license": "Apache-2.0", "dependencies": { @@ -44,7 +44,7 @@ }, "../consumer": { "name": "@kalamdb/consumer", - "version": "0.4.2-rc.3", + "version": "0.4.3-rc.4", "dev": true, "license": "Apache-2.0", "devDependencies": { diff --git a/nextest.toml b/nextest.toml index b24a1b2f..400bb800 100644 --- a/nextest.toml +++ b/nextest.toml @@ -107,6 +107,30 @@ test-group = "stateful-heavy" filter = 'test(test_shared_connection_recovers_subscriptions_in_different_stages)' test-group = "stateful-heavy" +[[profile.default.overrides]] +filter = 'test(test_tokio_netem_fragmented_writes_preserve_live_stream)' +test-group = "stateful-heavy" + +[[profile.default.overrides]] +filter = 'test(test_tokio_netem_bandwidth_collapse_forces_resume_without_replay)' +test-group = "stateful-heavy" + +[[profile.default.overrides]] +filter = 'test(test_tokio_netem_forced_transport_termination_recovers)' +test-group = "stateful-heavy" + +[[profile.default.overrides]] +filter = 'test(test_tokio_netem_topic_consume_fragmented_insert_update_delete_and_commit)' +test-group = "stateful-heavy" + +[[profile.default.overrides]] +filter = 'test(test_tokio_netem_topic_bandwidth_collapse_poll_recovers_without_losing_offsets)' +test-group = "stateful-heavy" + +[[profile.default.overrides]] +filter = 'test(test_tokio_netem_topic_commit_failure_can_be_retried_without_replay)' +test-group = "stateful-heavy" + [[profile.default.overrides]] filter = 'test(test_cli_syntax_error_handling)' test-group = "stateful-heavy" diff --git a/ui/package.json b/ui/package.json index 7ae96244..8f97e34d 100644 --- a/ui/package.json +++ b/ui/package.json @@ -19,6 +19,8 @@ "lint": "eslint . --ext ts,tsx --report-unused-disable-directives --max-warnings 0", "preview": "vite preview", "test": "vitest", + "test:ci": "vitest run", + "test:sql-studio": "vitest run src/services/sqlStudioService.test.ts src/pages/SqlStudio.test.tsx src/components/sql-studio-v2/input-form/sqlCompletionCatalog.test.ts", "test:flows": "vitest run src/components/auth/LoginForm.test.tsx src/components/auth/ProtectedRoute.test.tsx src/components/auth/SetupGuard.test.tsx src/pages/SetupWizard.test.tsx src/pages/SqlStudio.test.tsx" }, "dependencies": { diff --git a/ui/src/components/sql-studio-v2/input-form/StudioEditorPanel.tsx b/ui/src/components/sql-studio-v2/input-form/StudioEditorPanel.tsx index 204a6980..342e8250 100644 --- a/ui/src/components/sql-studio-v2/input-form/StudioEditorPanel.tsx +++ b/ui/src/components/sql-studio-v2/input-form/StudioEditorPanel.tsx @@ -13,9 +13,17 @@ import { DropdownMenuTrigger, } from "@/components/ui/dropdown-menu"; import type { LiveSubscriptionOptions, StudioNamespace } from "../shared/types"; +import { + buildSqlCompletionData, + resolveSqlContextualCompletions, + type SqlCompletionEntry, + type SqlCompletionData, +} from "./sqlCompletionCatalog"; type ExecuteMode = "all" | "selected"; +const EMPTY_COMPLETION_DATA = buildSqlCompletionData([]); + interface StudioEditorPanelProps { schema: StudioNamespace[]; tabTitle: string; @@ -62,40 +70,10 @@ export function StudioEditorPanel({ const onRunRef = useRef(onRun); const sqlRef = useRef(sql); const completionProviderRef = useRef(null); - const completionDataRef = useRef<{ - namespaces: string[]; - tablesByNamespace: Record; - columnsByTable: Record; - keywords: string[]; - }>({ - namespaces: [], - tablesByNamespace: {}, - columnsByTable: {}, - keywords: [], - }); + const completionDataRef = useRef(EMPTY_COMPLETION_DATA); const completionData = useMemo(() => { - const namespaces: string[] = []; - const tablesByNamespace: Record = {}; - const columnsByTable: Record = {}; - - schema.forEach((namespace) => { - const namespaceKey = namespace.name.toLowerCase(); - namespaces.push(namespace.name); - tablesByNamespace[namespaceKey] = namespace.tables.map((table) => table.name); - namespace.tables.forEach((table) => { - columnsByTable[`${namespaceKey}.${table.name.toLowerCase()}`] = table.columns.map((column) => column.name); - }); - }); - - const keywords = [ - "SELECT", "FROM", "WHERE", "JOIN", "LEFT", "RIGHT", "INNER", "OUTER", - "GROUP BY", "ORDER BY", "LIMIT", "INSERT", "UPDATE", "DELETE", - "CREATE", "ALTER", "DROP", "TABLE", "NAMESPACE", "VALUES", "SET", - "AND", "OR", "NOT", "IN", "AS", "ON", - ]; - - return { namespaces, tablesByNamespace, columnsByTable, keywords }; + return buildSqlCompletionData(schema); }, [schema]); useEffect(() => { @@ -192,57 +170,67 @@ export function StudioEditorPanel({ aliasMatch = aliasRegex.exec(textUntilPosition); } - const pushSuggestion = (label: string, kind: languages.CompletionItemKind, detail: string, insertText = label) => { + const pushSuggestion = ( + label: string, + kind: languages.CompletionItemKind, + detail: string, + insertText = label, + sortText?: string, + insertTextRules?: languages.CompletionItemInsertTextRule, + matchText = prefix, + ) => { const key = `${kind}-${label}-${insertText}`; if (seen.has(key)) { return; } - if (prefix && !label.toLowerCase().includes(prefix)) { + if (matchText && !label.toLowerCase().includes(matchText)) { return; } seen.add(key); - suggestions.push({ label, kind, detail, insertText, range }); + suggestions.push({ label, kind, detail, insertText, insertTextRules, range, sortText }); }; - const tableColumnMatch = /([a-zA-Z_][\w]*)\.([a-zA-Z_][\w]*)\.([a-zA-Z_][\w]*)?$/.exec(textUntilPosition); - if (tableColumnMatch) { - const namespaceKey = tableColumnMatch[1].toLowerCase(); - const tableKey = tableColumnMatch[2].toLowerCase(); - const columns = data.columnsByTable[`${namespaceKey}.${tableKey}`] ?? []; - columns.forEach((column) => - pushSuggestion(column, monaco.languages.CompletionItemKind.Field, `${namespaceKey}.${tableKey} column`), + const pushEntry = (entry: SqlCompletionEntry) => { + const snippetRule = entry.isSnippet + ? monaco.languages.CompletionItemInsertTextRule.InsertAsSnippet + : undefined; + const kindByCategory: Record = { + function: monaco.languages.CompletionItemKind.Function, + keyword: monaco.languages.CompletionItemKind.Keyword, + operator: monaco.languages.CompletionItemKind.Operator, + snippet: monaco.languages.CompletionItemKind.Snippet, + type: monaco.languages.CompletionItemKind.TypeParameter, + }; + + pushSuggestion( + entry.label, + kindByCategory[entry.category], + entry.detail, + entry.insertText ?? entry.label, + entry.sortText, + snippetRule, ); - return { suggestions }; - } + }; - const aliasColumnMatch = /([a-zA-Z_][\w]*)\.([a-zA-Z_][\w]*)?$/.exec(textUntilPosition); - if (aliasColumnMatch) { - const aliasKey = aliasColumnMatch[1].toLowerCase(); - const resolvedTable = aliasToTable[aliasKey]; - if (resolvedTable) { - const columns = data.columnsByTable[resolvedTable] ?? []; - columns.forEach((column) => - pushSuggestion(column, monaco.languages.CompletionItemKind.Field, `${aliasKey} alias column`), - ); - return { suggestions }; - } - } + const contextualCompletion = resolveSqlContextualCompletions(data, textUntilPosition, aliasToTable); + if (contextualCompletion) { + const kind = contextualCompletion.kind === "column" + ? monaco.languages.CompletionItemKind.Field + : monaco.languages.CompletionItemKind.Class; - const namespaceTableMatch = /([a-zA-Z_][\w]*)\.([a-zA-Z_][\w]*)?$/.exec(textUntilPosition); - if (namespaceTableMatch) { - const namespaceKey = namespaceTableMatch[1].toLowerCase(); - const namespaceTables = data.tablesByNamespace[namespaceKey]; - if (namespaceTables && namespaceTables.length > 0) { - namespaceTables.forEach((table) => - pushSuggestion(table, monaco.languages.CompletionItemKind.Class, `${namespaceKey} table`), - ); - return { suggestions }; - } + contextualCompletion.labels.forEach((label) => + pushSuggestion(label, kind, contextualCompletion.detail, label, undefined, undefined, contextualCompletion.partial), + ); + return { suggestions }; } data.keywords.forEach((keyword) => - pushSuggestion(keyword, monaco.languages.CompletionItemKind.Keyword, "SQL keyword"), + pushSuggestion(keyword, monaco.languages.CompletionItemKind.Keyword, "SQL keyword", keyword, `2_${keyword}`), ); + data.snippets.forEach(pushEntry); + data.functions.forEach(pushEntry); + data.types.forEach(pushEntry); + data.operators.forEach(pushEntry); data.namespaces.forEach((namespaceName) => pushSuggestion(namespaceName, monaco.languages.CompletionItemKind.Module, "Namespace"), ); diff --git a/ui/src/components/sql-studio-v2/input-form/sqlCompletionCatalog.test.ts b/ui/src/components/sql-studio-v2/input-form/sqlCompletionCatalog.test.ts new file mode 100644 index 00000000..4759be1a --- /dev/null +++ b/ui/src/components/sql-studio-v2/input-form/sqlCompletionCatalog.test.ts @@ -0,0 +1,162 @@ +import { describe, expect, it } from "vitest"; + +import { buildSqlCompletionData, resolveSqlContextualCompletions } from "./sqlCompletionCatalog"; +import type { StudioNamespace } from "../shared/types"; + +describe("SQL Studio completion catalog", () => { + it("always includes system.live with known columns", () => { + const data = buildSqlCompletionData([]); + + expect(data.namespaces).toContain("system"); + expect(data.tablesByNamespace.system).toContain("live"); + expect(data.columnsByTable["system.live"]).toEqual( + expect.arrayContaining(["live_id", "subscription_id", "namespace_id", "table_name", "user_id", "status"]), + ); + }); + + it("merges schema tables with static KalamDB and DataFusion completions", () => { + const schema: StudioNamespace[] = [ + { + database: "kalam", + name: "app", + tables: [ + { + database: "kalam", + namespace: "app", + name: "messages", + tableType: "user", + columns: [ + { name: "id", dataType: "TEXT", isNullable: false, isPrimaryKey: true, ordinal: 0 }, + { name: "body", dataType: "TEXT", isNullable: true, isPrimaryKey: false, ordinal: 1 }, + ], + }, + ], + }, + ]; + + const data = buildSqlCompletionData(schema); + + expect(data.tablesByNamespace.app).toContain("messages"); + expect(data.columnsByTable["app.messages"]).toEqual(["id", "body"]); + expect(data.functions.map((entry) => entry.label)).toEqual( + expect.arrayContaining(["CURRENT_USER()", "CURRENT_USER_ID()", "CURRENT_ROLE()", "NOW()", "COUNT()", "JSON_GET()"]), + ); + expect(data.snippets.map((entry) => entry.label)).toEqual( + expect.arrayContaining(["EXECUTE AS USER", "EXPLAIN", "DESCRIBE TABLE", "SUBSCRIBE TO", "KILL LIVE QUERY"]), + ); + }); + + it("resolves namespace table completions from the loaded explorer schema", () => { + const schema: StudioNamespace[] = [ + { + database: "kalam", + name: "system", + tables: [ + { + database: "kalam", + namespace: "system", + name: "jobs", + tableType: "system", + columns: [ + { name: "job_id", dataType: "TEXT", isNullable: false, isPrimaryKey: true, ordinal: 0 }, + ], + }, + { + database: "kalam", + namespace: "system", + name: "users", + tableType: "system", + columns: [ + { name: "user_id", dataType: "TEXT", isNullable: false, isPrimaryKey: true, ordinal: 0 }, + ], + }, + ], + }, + { + database: "kalam", + name: "agent_events", + tables: [ + { + database: "kalam", + namespace: "agent_events", + name: "runs", + tableType: "shared", + columns: [ + { name: "id", dataType: "TEXT", isNullable: false, isPrimaryKey: true, ordinal: 0 }, + { name: "status", dataType: "TEXT", isNullable: false, isPrimaryKey: false, ordinal: 1 }, + ], + }, + { + database: "kalam", + namespace: "agent_events", + name: "messages", + tableType: "shared", + columns: [ + { name: "id", dataType: "TEXT", isNullable: false, isPrimaryKey: true, ordinal: 0 }, + { name: "body", dataType: "TEXT", isNullable: false, isPrimaryKey: false, ordinal: 1 }, + ], + }, + ], + }, + ]; + + const data = buildSqlCompletionData(schema); + + expect(resolveSqlContextualCompletions(data, "system.", {})).toEqual({ + kind: "table", + labels: expect.arrayContaining(["jobs", "users", "live"]), + detail: "system table", + partial: "", + }); + + expect(resolveSqlContextualCompletions(data, "SELECT *\nFROM agent_events.", {})).toEqual({ + kind: "table", + labels: ["runs", "messages"], + detail: "agent_events table", + partial: "", + }); + }); + + it("resolves namespace-qualified and alias-qualified column completions", () => { + const schema: StudioNamespace[] = [ + { + database: "kalam", + name: "agent_events", + tables: [ + { + database: "kalam", + namespace: "agent_events", + name: "runs", + tableType: "shared", + columns: [ + { name: "id", dataType: "TEXT", isNullable: false, isPrimaryKey: true, ordinal: 0 }, + { name: "status", dataType: "TEXT", isNullable: false, isPrimaryKey: false, ordinal: 1 }, + ], + }, + ], + }, + ]; + + const data = buildSqlCompletionData(schema); + + expect(resolveSqlContextualCompletions(data, "SELECT agent_events.runs.", {})).toEqual({ + kind: "column", + labels: ["id", "status"], + detail: "agent_events.runs column", + partial: "", + }); + + expect( + resolveSqlContextualCompletions( + data, + "SELECT * FROM agent_events.runs AS r WHERE r.", + { r: "agent_events.runs" }, + ), + ).toEqual({ + kind: "column", + labels: ["id", "status"], + detail: "r alias column", + partial: "", + }); + }); +}); \ No newline at end of file diff --git a/ui/src/components/sql-studio-v2/input-form/sqlCompletionCatalog.ts b/ui/src/components/sql-studio-v2/input-form/sqlCompletionCatalog.ts new file mode 100644 index 00000000..bed521ed --- /dev/null +++ b/ui/src/components/sql-studio-v2/input-form/sqlCompletionCatalog.ts @@ -0,0 +1,666 @@ +import type { StudioNamespace } from "../shared/types"; + +type CompletionCategory = "function" | "keyword" | "operator" | "snippet" | "type"; + +export interface SqlCompletionEntry { + label: string; + detail: string; + category: CompletionCategory; + insertText?: string; + isSnippet?: boolean; + sortText?: string; +} + +export interface SqlCompletionData { + namespaces: string[]; + tablesByNamespace: Record; + columnsByTable: Record; + keywords: readonly string[]; + functions: readonly SqlCompletionEntry[]; + snippets: readonly SqlCompletionEntry[]; + types: readonly SqlCompletionEntry[]; + operators: readonly SqlCompletionEntry[]; +} + +export interface SqlContextualCompletionMatch { + kind: "table" | "column"; + labels: string[]; + detail: string; + partial: string; +} + +const STATIC_SYSTEM_TABLES = [ + { + namespace: "system", + name: "live", + columns: [ + "live_id", + "connection_id", + "subscription_id", + "namespace_id", + "table_name", + "user_id", + "query", + "options", + "status", + "created_at", + "last_update", + "changes", + "node_id", + "last_ping_at", + ], + }, +] as const; + +const SQL_KEYWORDS = [ + "ACK", + "ADD", + "ALL", + "ALTER", + "ANALYZE", + "AND", + "AS", + "ASC", + "BACKUP", + "BEGIN", + "BETWEEN", + "BY", + "CASE", + "CHECK", + "CLEAR", + "CLUSTER", + "COMMIT", + "COMPACT", + "CONSUME", + "CREATE", + "CROSS", + "DATABASE", + "DEFAULT", + "DELETE", + "DESC", + "DESCRIBE", + "DISTINCT", + "DROP", + "ELSE", + "END", + "EXECUTE", + "EXISTS", + "EXPLAIN", + "EXPORT", + "FALSE", + "FLUSH", + "FOLLOWING", + "FOR", + "FROM", + "FULL", + "GROUP BY", + "HAVING", + "IF", + "IN", + "INNER JOIN", + "INSERT INTO", + "IS", + "JOIN", + "KEY", + "KILL", + "LAST", + "LEFT JOIN", + "LIKE", + "LIMIT", + "LIVE", + "NAMESPACE", + "NOT", + "NULL", + "OFFSET", + "ON", + "OPTIONS", + "OR", + "ORDER BY", + "OUTER", + "OVER", + "PARTITION BY", + "PRECEDING", + "PRIMARY KEY", + "QUERY", + "RANGE", + "RESTORE", + "RIGHT JOIN", + "ROLLBACK", + "ROW", + "ROWS", + "SELECT", + "SET", + "SHARED", + "SHOW", + "START TRANSACTION", + "STORAGE", + "STREAM", + "SUBSCRIBE TO", + "TABLE", + "THEN", + "TO", + "TOPIC", + "TRUE", + "UNBOUNDED", + "UNION", + "UPDATE", + "USE", + "USER", + "VALUES", + "VIEW", + "WHEN", + "WHERE", + "WITH", +] as const; + +const SQL_TYPE_NAMES = [ + "ARRAY", + "BIGINT", + "BOOLEAN", + "DATE", + "DOUBLE", + "FLOAT", + "INT", + "INTEGER", + "JSON", + "JSONB", + "LIST", + "MAP", + "REAL", + "SMALLINT", + "STRUCT", + "TEXT", + "TIME", + "TIMESTAMP", + "TIMESTAMPTZ", + "UUID", + "VARCHAR", +] as const; + +function functionEntry(name: string, detail: string, insertText?: string): SqlCompletionEntry { + return { + label: `${name}()`, + detail, + category: "function", + insertText: insertText ?? `${name}()`, + isSnippet: Boolean(insertText), + sortText: `1_${name}`, + }; +} + +function syntaxEntry(label: string, detail: string, insertText?: string): SqlCompletionEntry { + return { + label, + detail, + category: "snippet", + insertText: insertText ?? label, + isSnippet: Boolean(insertText), + sortText: `0_${label}`, + }; +} + +function typeEntry(label: string): SqlCompletionEntry { + return { + label, + detail: "SQL type", + category: "type", + sortText: `3_${label}`, + }; +} + +function operatorEntry(label: string, detail: string): SqlCompletionEntry { + return { + label, + detail, + category: "operator", + sortText: `4_${label}`, + }; +} + +const KALAMDB_FUNCTION_COMPLETIONS = [ + functionEntry("CURRENT_USER", "KalamDB context function"), + functionEntry("CURRENT_USER_ID", "KalamDB context function"), + functionEntry("CURRENT_ROLE", "KalamDB context function"), + functionEntry("SNOWFLAKE_ID", "KalamDB ID generation function"), + functionEntry("UUID_V7", "KalamDB ID generation function"), + functionEntry("ULID", "KalamDB ID generation function"), + functionEntry("COSINE_DISTANCE", "KalamDB vector similarity function", "COSINE_DISTANCE(${1:vector}, ${2:query_vector})"), + functionEntry("VECTOR_SEARCH", "KalamDB vector table function", "VECTOR_SEARCH(${1:table}, ${2:query_vector})"), +] as const; + +const DATAFUSION_SCALAR_FUNCTION_COMPLETIONS = [ + functionEntry("ABS", "DataFusion math function", "ABS(${1:number})"), + functionEntry("ACOS", "DataFusion math function", "ACOS(${1:number})"), + functionEntry("ACOSH", "DataFusion math function", "ACOSH(${1:number})"), + functionEntry("ARROW_CAST", "DataFusion core function", "ARROW_CAST(${1:value}, ${2:type})"), + functionEntry("ARROW_METADATA", "DataFusion core function", "ARROW_METADATA(${1:value})"), + functionEntry("ARROW_TYPEOF", "DataFusion core function", "ARROW_TYPEOF(${1:value})"), + functionEntry("ASCII", "DataFusion string function", "ASCII(${1:string})"), + functionEntry("ASIN", "DataFusion math function", "ASIN(${1:number})"), + functionEntry("ASINH", "DataFusion math function", "ASINH(${1:number})"), + functionEntry("ATAN", "DataFusion math function", "ATAN(${1:number})"), + functionEntry("ATAN2", "DataFusion math function", "ATAN2(${1:y}, ${2:x})"), + functionEntry("ATANH", "DataFusion math function", "ATANH(${1:number})"), + functionEntry("BIT_LENGTH", "DataFusion string function", "BIT_LENGTH(${1:string})"), + functionEntry("BTRIM", "DataFusion string function", "BTRIM(${1:string})"), + functionEntry("CBRT", "DataFusion math function", "CBRT(${1:number})"), + functionEntry("CEIL", "DataFusion math function", "CEIL(${1:number})"), + functionEntry("CHR", "DataFusion string function", "CHR(${1:code_point})"), + functionEntry("COALESCE", "DataFusion core function", "COALESCE(${1:value}, ${2:fallback})"), + functionEntry("CONCAT", "DataFusion string function", "CONCAT(${1:value})"), + functionEntry("CONCAT_WS", "DataFusion string function", "CONCAT_WS(${1:separator}, ${2:value})"), + functionEntry("CONTAINS", "DataFusion string function", "CONTAINS(${1:string}, ${2:search_string})"), + functionEntry("COS", "DataFusion math function", "COS(${1:number})"), + functionEntry("COSH", "DataFusion math function", "COSH(${1:number})"), + functionEntry("COT", "DataFusion math function", "COT(${1:number})"), + functionEntry("CURRENT_DATE", "DataFusion datetime function"), + functionEntry("CURRENT_TIME", "DataFusion datetime function"), + functionEntry("CURRENT_TIMESTAMP", "DataFusion datetime function"), + functionEntry("DATE_BIN", "DataFusion datetime function", "DATE_BIN(${1:stride}, ${2:source}, ${3:origin})"), + functionEntry("DATE_PART", "DataFusion datetime function", "DATE_PART(${1:part}, ${2:date})"), + functionEntry("DATE_TRUNC", "DataFusion datetime function", "DATE_TRUNC(${1:part}, ${2:date})"), + functionEntry("DEGREES", "DataFusion math function", "DEGREES(${1:number})"), + functionEntry("DIGEST", "DataFusion crypto function", "DIGEST(${1:value}, ${2:algorithm})"), + functionEntry("ENDS_WITH", "DataFusion string function", "ENDS_WITH(${1:string}, ${2:suffix})"), + functionEntry("EXP", "DataFusion math function", "EXP(${1:number})"), + functionEntry("FACTORIAL", "DataFusion math function", "FACTORIAL(${1:number})"), + functionEntry("FLOOR", "DataFusion math function", "FLOOR(${1:number})"), + functionEntry("FROM_UNIXTIME", "DataFusion datetime function", "FROM_UNIXTIME(${1:seconds})"), + functionEntry("GCD", "DataFusion math function", "GCD(${1:x}, ${2:y})"), + functionEntry("GET_FIELD", "DataFusion core function", "GET_FIELD(${1:value}, ${2:field})"), + functionEntry("GREATEST", "DataFusion core function", "GREATEST(${1:value})"), + functionEntry("ISNAN", "DataFusion math function", "ISNAN(${1:number})"), + functionEntry("ISZERO", "DataFusion math function", "ISZERO(${1:number})"), + functionEntry("LCM", "DataFusion math function", "LCM(${1:x}, ${2:y})"), + functionEntry("LEAST", "DataFusion core function", "LEAST(${1:value})"), + functionEntry("LEFT", "DataFusion unicode function", "LEFT(${1:string}, ${2:n})"), + functionEntry("LENGTH", "DataFusion string/unicode function", "LENGTH(${1:string})"), + functionEntry("LEVENSHTEIN", "DataFusion string function", "LEVENSHTEIN(${1:left}, ${2:right})"), + functionEntry("LN", "DataFusion math function", "LN(${1:number})"), + functionEntry("LOG", "DataFusion math function", "LOG(${1:base}, ${2:number})"), + functionEntry("LOG2", "DataFusion math function", "LOG2(${1:number})"), + functionEntry("LOG10", "DataFusion math function", "LOG10(${1:number})"), + functionEntry("LOWER", "DataFusion string function", "LOWER(${1:string})"), + functionEntry("LPAD", "DataFusion unicode function", "LPAD(${1:string}, ${2:length}, ${3:fill})"), + functionEntry("LTRIM", "DataFusion string function", "LTRIM(${1:string})"), + functionEntry("MAKE_DATE", "DataFusion datetime function", "MAKE_DATE(${1:year}, ${2:month}, ${3:day})"), + functionEntry("MAKE_TIME", "DataFusion datetime function", "MAKE_TIME(${1:hour}, ${2:minute}, ${3:second})"), + functionEntry("MD5", "DataFusion crypto function", "MD5(${1:value})"), + functionEntry("NANVL", "DataFusion math function", "NANVL(${1:x}, ${2:y})"), + functionEntry("NAMED_STRUCT", "DataFusion core function", "NAMED_STRUCT(${1:name}, ${2:value})"), + functionEntry("NOW", "DataFusion datetime function"), + functionEntry("NULLIF", "DataFusion core function", "NULLIF(${1:left}, ${2:right})"), + functionEntry("NVL", "DataFusion core function", "NVL(${1:value}, ${2:fallback})"), + functionEntry("NVL2", "DataFusion core function", "NVL2(${1:value}, ${2:not_null}, ${3:null_value})"), + functionEntry("OCTET_LENGTH", "DataFusion string function", "OCTET_LENGTH(${1:string})"), + functionEntry("OVERLAY", "DataFusion core/string function", "OVERLAY(${1:string} PLACING ${2:replacement} FROM ${3:start})"), + functionEntry("PI", "DataFusion math function"), + functionEntry("POWER", "DataFusion math function", "POWER(${1:base}, ${2:exponent})"), + functionEntry("RADIANS", "DataFusion math function", "RADIANS(${1:number})"), + functionEntry("RANDOM", "DataFusion math function"), + functionEntry("REGEXP_COUNT", "DataFusion regex function", "REGEXP_COUNT(${1:string}, ${2:pattern})"), + functionEntry("REGEXP_INSTR", "DataFusion regex function", "REGEXP_INSTR(${1:string}, ${2:pattern})"), + functionEntry("REGEXP_LIKE", "DataFusion regex function", "REGEXP_LIKE(${1:string}, ${2:pattern})"), + functionEntry("REGEXP_MATCH", "DataFusion regex function", "REGEXP_MATCH(${1:string}, ${2:pattern})"), + functionEntry("REGEXP_REPLACE", "DataFusion regex function", "REGEXP_REPLACE(${1:string}, ${2:pattern}, ${3:replacement})"), + functionEntry("REPEAT", "DataFusion string function", "REPEAT(${1:string}, ${2:n})"), + functionEntry("REPLACE", "DataFusion string function", "REPLACE(${1:string}, ${2:from}, ${3:to})"), + functionEntry("REVERSE", "DataFusion unicode function", "REVERSE(${1:string})"), + functionEntry("RIGHT", "DataFusion unicode function", "RIGHT(${1:string}, ${2:n})"), + functionEntry("ROUND", "DataFusion math function", "ROUND(${1:number})"), + functionEntry("RPAD", "DataFusion unicode function", "RPAD(${1:string}, ${2:length}, ${3:fill})"), + functionEntry("RTRIM", "DataFusion string function", "RTRIM(${1:string})"), + functionEntry("SHA224", "DataFusion crypto function", "SHA224(${1:value})"), + functionEntry("SHA256", "DataFusion crypto function", "SHA256(${1:value})"), + functionEntry("SHA384", "DataFusion crypto function", "SHA384(${1:value})"), + functionEntry("SHA512", "DataFusion crypto function", "SHA512(${1:value})"), + functionEntry("SIGNUM", "DataFusion math function", "SIGNUM(${1:number})"), + functionEntry("SIN", "DataFusion math function", "SIN(${1:number})"), + functionEntry("SINH", "DataFusion math function", "SINH(${1:number})"), + functionEntry("SPLIT_PART", "DataFusion string function", "SPLIT_PART(${1:string}, ${2:delimiter}, ${3:index})"), + functionEntry("SQRT", "DataFusion math function", "SQRT(${1:number})"), + functionEntry("STARTS_WITH", "DataFusion string function", "STARTS_WITH(${1:string}, ${2:prefix})"), + functionEntry("STRPOS", "DataFusion unicode function", "STRPOS(${1:string}, ${2:substring})"), + functionEntry("STRUCT", "DataFusion core function", "STRUCT(${1:value})"), + functionEntry("SUBSTR", "DataFusion unicode function", "SUBSTR(${1:string}, ${2:start})"), + functionEntry("TAN", "DataFusion math function", "TAN(${1:number})"), + functionEntry("TANH", "DataFusion math function", "TANH(${1:number})"), + functionEntry("TO_CHAR", "DataFusion datetime function", "TO_CHAR(${1:value}, ${2:format})"), + functionEntry("TO_DATE", "DataFusion datetime function", "TO_DATE(${1:value})"), + functionEntry("TO_HEX", "DataFusion string function", "TO_HEX(${1:value})"), + functionEntry("TO_LOCAL_TIME", "DataFusion datetime function", "TO_LOCAL_TIME(${1:value})"), + functionEntry("TO_TIME", "DataFusion datetime function", "TO_TIME(${1:value})"), + functionEntry("TO_TIMESTAMP", "DataFusion datetime function", "TO_TIMESTAMP(${1:value})"), + functionEntry("TO_TIMESTAMP_SECONDS", "DataFusion datetime function", "TO_TIMESTAMP_SECONDS(${1:value})"), + functionEntry("TO_TIMESTAMP_MILLIS", "DataFusion datetime function", "TO_TIMESTAMP_MILLIS(${1:value})"), + functionEntry("TO_TIMESTAMP_MICROS", "DataFusion datetime function", "TO_TIMESTAMP_MICROS(${1:value})"), + functionEntry("TO_TIMESTAMP_NANOS", "DataFusion datetime function", "TO_TIMESTAMP_NANOS(${1:value})"), + functionEntry("TO_UNIXTIME", "DataFusion datetime function", "TO_UNIXTIME(${1:value})"), + functionEntry("TRANSLATE", "DataFusion unicode function", "TRANSLATE(${1:string}, ${2:from}, ${3:to})"), + functionEntry("TRIM", "DataFusion string function", "TRIM(${1:string})"), + functionEntry("TRUNC", "DataFusion math function", "TRUNC(${1:number})"), + functionEntry("UNION_EXTRACT", "DataFusion core function", "UNION_EXTRACT(${1:value}, ${2:field})"), + functionEntry("UNION_TAG", "DataFusion core function", "UNION_TAG(${1:value})"), + functionEntry("UPPER", "DataFusion string function", "UPPER(${1:string})"), + functionEntry("UUID", "DataFusion string function"), + functionEntry("VERSION", "DataFusion core function"), +] as const; + +const DATAFUSION_AGGREGATE_FUNCTION_COMPLETIONS = [ + functionEntry("APPROX_DISTINCT", "DataFusion aggregate function", "APPROX_DISTINCT(${1:expression})"), + functionEntry("APPROX_MEDIAN", "DataFusion aggregate function", "APPROX_MEDIAN(${1:expression})"), + functionEntry("APPROX_PERCENTILE_CONT", "DataFusion aggregate function", "APPROX_PERCENTILE_CONT(${1:expression}, ${2:percentile})"), + functionEntry("APPROX_PERCENTILE_CONT_WITH_WEIGHT", "DataFusion aggregate function", "APPROX_PERCENTILE_CONT_WITH_WEIGHT(${1:expression}, ${2:weight}, ${3:percentile})"), + functionEntry("ARRAY_AGG", "DataFusion aggregate function", "ARRAY_AGG(${1:expression})"), + functionEntry("AVG", "DataFusion aggregate function", "AVG(${1:expression})"), + functionEntry("BIT_AND", "DataFusion aggregate function", "BIT_AND(${1:expression})"), + functionEntry("BIT_OR", "DataFusion aggregate function", "BIT_OR(${1:expression})"), + functionEntry("BIT_XOR", "DataFusion aggregate function", "BIT_XOR(${1:expression})"), + functionEntry("BOOL_AND", "DataFusion aggregate function", "BOOL_AND(${1:expression})"), + functionEntry("BOOL_OR", "DataFusion aggregate function", "BOOL_OR(${1:expression})"), + functionEntry("CORR", "DataFusion aggregate function", "CORR(${1:y}, ${2:x})"), + functionEntry("COUNT", "DataFusion aggregate function", "COUNT(${1:*})"), + functionEntry("COVAR_POP", "DataFusion aggregate function", "COVAR_POP(${1:y}, ${2:x})"), + functionEntry("COVAR_SAMP", "DataFusion aggregate function", "COVAR_SAMP(${1:y}, ${2:x})"), + functionEntry("GROUPING", "DataFusion aggregate function", "GROUPING(${1:expression})"), + functionEntry("MAX", "DataFusion aggregate function", "MAX(${1:expression})"), + functionEntry("MEDIAN", "DataFusion aggregate function", "MEDIAN(${1:expression})"), + functionEntry("MIN", "DataFusion aggregate function", "MIN(${1:expression})"), + functionEntry("NTH_VALUE", "DataFusion aggregate/window function", "NTH_VALUE(${1:expression}, ${2:n})"), + functionEntry("PERCENTILE_CONT", "DataFusion aggregate function", "PERCENTILE_CONT(${1:expression}, ${2:percentile})"), + functionEntry("REGR_AVGX", "DataFusion regression aggregate", "REGR_AVGX(${1:y}, ${2:x})"), + functionEntry("REGR_AVGY", "DataFusion regression aggregate", "REGR_AVGY(${1:y}, ${2:x})"), + functionEntry("REGR_COUNT", "DataFusion regression aggregate", "REGR_COUNT(${1:y}, ${2:x})"), + functionEntry("REGR_INTERCEPT", "DataFusion regression aggregate", "REGR_INTERCEPT(${1:y}, ${2:x})"), + functionEntry("REGR_R2", "DataFusion regression aggregate", "REGR_R2(${1:y}, ${2:x})"), + functionEntry("REGR_SLOPE", "DataFusion regression aggregate", "REGR_SLOPE(${1:y}, ${2:x})"), + functionEntry("REGR_SXX", "DataFusion regression aggregate", "REGR_SXX(${1:y}, ${2:x})"), + functionEntry("REGR_SXY", "DataFusion regression aggregate", "REGR_SXY(${1:y}, ${2:x})"), + functionEntry("REGR_SYY", "DataFusion regression aggregate", "REGR_SYY(${1:y}, ${2:x})"), + functionEntry("STDDEV", "DataFusion aggregate function", "STDDEV(${1:expression})"), + functionEntry("STDDEV_POP", "DataFusion aggregate function", "STDDEV_POP(${1:expression})"), + functionEntry("STRING_AGG", "DataFusion aggregate function", "STRING_AGG(${1:expression}, ${2:delimiter})"), + functionEntry("SUM", "DataFusion aggregate function", "SUM(${1:expression})"), + functionEntry("VAR_POP", "DataFusion aggregate function", "VAR_POP(${1:expression})"), + functionEntry("VAR_SAMPLE", "DataFusion aggregate function", "VAR_SAMPLE(${1:expression})"), +] as const; + +const DATAFUSION_WINDOW_FUNCTION_COMPLETIONS = [ + functionEntry("CUME_DIST", "DataFusion window function"), + functionEntry("DENSE_RANK", "DataFusion window function"), + functionEntry("FIRST_VALUE", "DataFusion window function", "FIRST_VALUE(${1:expression})"), + functionEntry("LAG", "DataFusion window function", "LAG(${1:expression})"), + functionEntry("LAST_VALUE", "DataFusion window function", "LAST_VALUE(${1:expression})"), + functionEntry("LEAD", "DataFusion window function", "LEAD(${1:expression})"), + functionEntry("NTILE", "DataFusion window function", "NTILE(${1:buckets})"), + functionEntry("PERCENT_RANK", "DataFusion window function"), + functionEntry("RANK", "DataFusion window function"), + functionEntry("ROW_NUMBER", "DataFusion window function"), +] as const; + +const DATAFUSION_NESTED_FUNCTION_COMPLETIONS = [ + functionEntry("ARRAY_ANY_VALUE", "DataFusion nested function", "ARRAY_ANY_VALUE(${1:array})"), + functionEntry("ARRAY_APPEND", "DataFusion nested function", "ARRAY_APPEND(${1:array}, ${2:value})"), + functionEntry("ARRAY_CONCAT", "DataFusion nested function", "ARRAY_CONCAT(${1:array})"), + functionEntry("ARRAY_DIMS", "DataFusion nested function", "ARRAY_DIMS(${1:array})"), + functionEntry("ARRAY_DISTINCT", "DataFusion nested function", "ARRAY_DISTINCT(${1:array})"), + functionEntry("ARRAY_DISTANCE", "DataFusion nested function", "ARRAY_DISTANCE(${1:left}, ${2:right})"), + functionEntry("ARRAY_ELEMENT", "DataFusion nested function", "ARRAY_ELEMENT(${1:array}, ${2:index})"), + functionEntry("ARRAY_EMPTY", "DataFusion nested function", "ARRAY_EMPTY(${1:array})"), + functionEntry("ARRAY_EXCEPT", "DataFusion nested function", "ARRAY_EXCEPT(${1:left}, ${2:right})"), + functionEntry("ARRAY_HAS", "DataFusion nested function", "ARRAY_HAS(${1:array}, ${2:value})"), + functionEntry("ARRAY_HAS_ALL", "DataFusion nested function", "ARRAY_HAS_ALL(${1:array}, ${2:values})"), + functionEntry("ARRAY_HAS_ANY", "DataFusion nested function", "ARRAY_HAS_ANY(${1:array}, ${2:values})"), + functionEntry("ARRAY_INTERSECT", "DataFusion nested function", "ARRAY_INTERSECT(${1:left}, ${2:right})"), + functionEntry("ARRAY_LENGTH", "DataFusion nested function", "ARRAY_LENGTH(${1:array})"), + functionEntry("ARRAY_MAX", "DataFusion nested function", "ARRAY_MAX(${1:array})"), + functionEntry("ARRAY_MIN", "DataFusion nested function", "ARRAY_MIN(${1:array})"), + functionEntry("ARRAY_NDIMS", "DataFusion nested function", "ARRAY_NDIMS(${1:array})"), + functionEntry("ARRAY_POP_BACK", "DataFusion nested function", "ARRAY_POP_BACK(${1:array})"), + functionEntry("ARRAY_POP_FRONT", "DataFusion nested function", "ARRAY_POP_FRONT(${1:array})"), + functionEntry("ARRAY_POSITION", "DataFusion nested function", "ARRAY_POSITION(${1:array}, ${2:value})"), + functionEntry("ARRAY_POSITIONS", "DataFusion nested function", "ARRAY_POSITIONS(${1:array}, ${2:value})"), + functionEntry("ARRAY_PREPEND", "DataFusion nested function", "ARRAY_PREPEND(${1:value}, ${2:array})"), + functionEntry("ARRAY_REMOVE", "DataFusion nested function", "ARRAY_REMOVE(${1:array}, ${2:value})"), + functionEntry("ARRAY_REMOVE_ALL", "DataFusion nested function", "ARRAY_REMOVE_ALL(${1:array}, ${2:value})"), + functionEntry("ARRAY_REMOVE_N", "DataFusion nested function", "ARRAY_REMOVE_N(${1:array}, ${2:value}, ${3:n})"), + functionEntry("ARRAY_REPEAT", "DataFusion nested function", "ARRAY_REPEAT(${1:value}, ${2:n})"), + functionEntry("ARRAY_REPLACE", "DataFusion nested function", "ARRAY_REPLACE(${1:array}, ${2:from}, ${3:to})"), + functionEntry("ARRAY_REPLACE_ALL", "DataFusion nested function", "ARRAY_REPLACE_ALL(${1:array}, ${2:from}, ${3:to})"), + functionEntry("ARRAY_REPLACE_N", "DataFusion nested function", "ARRAY_REPLACE_N(${1:array}, ${2:from}, ${3:to}, ${4:n})"), + functionEntry("ARRAY_RESIZE", "DataFusion nested function", "ARRAY_RESIZE(${1:array}, ${2:size})"), + functionEntry("ARRAY_REVERSE", "DataFusion nested function", "ARRAY_REVERSE(${1:array})"), + functionEntry("ARRAY_SLICE", "DataFusion nested function", "ARRAY_SLICE(${1:array}, ${2:start}, ${3:end})"), + functionEntry("ARRAY_SORT", "DataFusion nested function", "ARRAY_SORT(${1:array})"), + functionEntry("ARRAY_TO_STRING", "DataFusion nested function", "ARRAY_TO_STRING(${1:array}, ${2:delimiter})"), + functionEntry("ARRAY_UNION", "DataFusion nested function", "ARRAY_UNION(${1:left}, ${2:right})"), + functionEntry("ARRAYS_ZIP", "DataFusion nested function", "ARRAYS_ZIP(${1:array})"), + functionEntry("CARDINALITY", "DataFusion nested function", "CARDINALITY(${1:value})"), + functionEntry("FLATTEN", "DataFusion nested function", "FLATTEN(${1:array})"), + functionEntry("GEN_SERIES", "DataFusion nested function", "GEN_SERIES(${1:start}, ${2:stop})"), + functionEntry("MAKE_ARRAY", "DataFusion nested function", "MAKE_ARRAY(${1:value})"), + functionEntry("MAP", "DataFusion nested function", "MAP(${1:key}, ${2:value})"), + functionEntry("MAP_ENTRIES", "DataFusion nested function", "MAP_ENTRIES(${1:map})"), + functionEntry("MAP_EXTRACT", "DataFusion nested function", "MAP_EXTRACT(${1:map}, ${2:key})"), + functionEntry("MAP_KEYS", "DataFusion nested function", "MAP_KEYS(${1:map})"), + functionEntry("MAP_VALUES", "DataFusion nested function", "MAP_VALUES(${1:map})"), + functionEntry("RANGE", "DataFusion nested function", "RANGE(${1:start}, ${2:stop})"), + functionEntry("STRING_TO_ARRAY", "DataFusion nested function", "STRING_TO_ARRAY(${1:string}, ${2:delimiter})"), +] as const; + +const DATAFUSION_JSON_FUNCTION_COMPLETIONS = [ + functionEntry("JSON_AS_TEXT", "DataFusion JSON function", "JSON_AS_TEXT(${1:json}, ${2:path})"), + functionEntry("JSON_CONTAINS", "DataFusion JSON function", "JSON_CONTAINS(${1:json}, ${2:key})"), + functionEntry("JSON_FROM_SCALAR", "DataFusion JSON function", "JSON_FROM_SCALAR(${1:value})"), + functionEntry("JSON_GET", "DataFusion JSON function", "JSON_GET(${1:json}, ${2:path})"), + functionEntry("JSON_GET_ARRAY", "DataFusion JSON function", "JSON_GET_ARRAY(${1:json}, ${2:path})"), + functionEntry("JSON_GET_BOOL", "DataFusion JSON function", "JSON_GET_BOOL(${1:json}, ${2:path})"), + functionEntry("JSON_GET_FLOAT", "DataFusion JSON function", "JSON_GET_FLOAT(${1:json}, ${2:path})"), + functionEntry("JSON_GET_INT", "DataFusion JSON function", "JSON_GET_INT(${1:json}, ${2:path})"), + functionEntry("JSON_GET_JSON", "DataFusion JSON function", "JSON_GET_JSON(${1:json}, ${2:path})"), + functionEntry("JSON_GET_STR", "DataFusion JSON function", "JSON_GET_STR(${1:json}, ${2:path})"), + functionEntry("JSON_LENGTH", "DataFusion JSON function", "JSON_LENGTH(${1:json})"), + functionEntry("JSON_OBJECT_KEYS", "DataFusion JSON function", "JSON_OBJECT_KEYS(${1:json})"), +] as const; + +const SQL_FUNCTION_COMPLETIONS = [ + ...KALAMDB_FUNCTION_COMPLETIONS, + ...DATAFUSION_SCALAR_FUNCTION_COMPLETIONS, + ...DATAFUSION_AGGREGATE_FUNCTION_COMPLETIONS, + ...DATAFUSION_WINDOW_FUNCTION_COMPLETIONS, + ...DATAFUSION_NESTED_FUNCTION_COMPLETIONS, + ...DATAFUSION_JSON_FUNCTION_COMPLETIONS, +] as const; + +const SQL_SYNTAX_COMPLETIONS = [ + syntaxEntry("SELECT", "SQL query", "SELECT ${1:*}\nFROM ${2:table}\nWHERE ${3:condition};"), + syntaxEntry("WITH", "SQL common table expression", "WITH ${1:cte} AS (\n SELECT ${2:*}\n FROM ${3:table}\n)\nSELECT * FROM ${1:cte};"), + syntaxEntry("INSERT INTO", "SQL insert", "INSERT INTO ${1:table} (${2:columns})\nVALUES (${3:values});"), + syntaxEntry("UPDATE", "SQL update", "UPDATE ${1:table}\nSET ${2:column} = ${3:value}\nWHERE ${4:condition};"), + syntaxEntry("DELETE FROM", "SQL delete", "DELETE FROM ${1:table}\nWHERE ${2:condition};"), + syntaxEntry("CREATE TABLE", "KalamDB table DDL", "CREATE TABLE ${1:namespace}.${2:table} (\n ${3:id} TEXT PRIMARY KEY,\n ${4:created_at} TIMESTAMP DEFAULT NOW()\n);"), + syntaxEntry("CREATE USER TABLE", "KalamDB user table DDL", "CREATE USER TABLE ${1:namespace}.${2:table} (\n ${3:id} TEXT PRIMARY KEY\n);"), + syntaxEntry("CREATE SHARED TABLE", "KalamDB shared table DDL", "CREATE SHARED TABLE ${1:namespace}.${2:table} (\n ${3:id} TEXT PRIMARY KEY\n);"), + syntaxEntry("CREATE STREAM TABLE", "KalamDB stream table DDL", "CREATE STREAM TABLE ${1:namespace}.${2:table} (\n ${3:id} TEXT PRIMARY KEY\n);"), + syntaxEntry("ALTER TABLE", "KalamDB table DDL", "ALTER TABLE ${1:namespace}.${2:table}\nADD COLUMN ${3:column} ${4:TEXT};"), + syntaxEntry("DROP TABLE", "KalamDB table DDL", "DROP TABLE ${1:namespace}.${2:table};"), + syntaxEntry("CREATE VIEW", "KalamDB view DDL", "CREATE VIEW ${1:namespace}.${2:view_name} AS\nSELECT ${3:*}\nFROM ${4:table};"), + syntaxEntry("CREATE NAMESPACE", "KalamDB namespace DDL", "CREATE NAMESPACE ${1:namespace};"), + syntaxEntry("ALTER NAMESPACE", "KalamDB namespace DDL", "ALTER NAMESPACE ${1:namespace} SET OPTIONS (${2:key} = '${3:value}');"), + syntaxEntry("DROP NAMESPACE", "KalamDB namespace DDL", "DROP NAMESPACE ${1:namespace};"), + syntaxEntry("SHOW NAMESPACES", "KalamDB metadata command", "SHOW NAMESPACES;"), + syntaxEntry("USE NAMESPACE", "KalamDB namespace command", "USE NAMESPACE ${1:namespace};"), + syntaxEntry("SET NAMESPACE", "KalamDB namespace command", "SET NAMESPACE ${1:namespace};"), + syntaxEntry("SHOW TABLES", "KalamDB metadata command", "SHOW TABLES;"), + syntaxEntry("DESCRIBE TABLE", "KalamDB metadata command", "DESCRIBE TABLE ${1:namespace}.${2:table};"), + syntaxEntry("DESC TABLE", "KalamDB metadata command", "DESC TABLE ${1:namespace}.${2:table};"), + syntaxEntry("SHOW STATS FOR TABLE", "KalamDB metadata command", "SHOW STATS FOR TABLE ${1:namespace}.${2:table};"), + syntaxEntry("EXPLAIN", "DataFusion meta command", "EXPLAIN ${1:SELECT * FROM table};"), + syntaxEntry("EXPLAIN ANALYZE", "DataFusion meta command", "EXPLAIN ANALYZE ${1:SELECT * FROM table};"), + syntaxEntry("DESCRIBE", "DataFusion meta command", "DESCRIBE ${1:table};"), + syntaxEntry("DESC", "DataFusion meta command", "DESC ${1:table};"), + syntaxEntry("SHOW COLUMNS", "DataFusion meta command", "SHOW COLUMNS FROM ${1:table};"), + syntaxEntry("SHOW ALL", "DataFusion meta command", "SHOW ALL;"), + syntaxEntry("SET", "DataFusion session setting", "SET ${1:key} = ${2:value};"), + syntaxEntry("EXECUTE AS USER", "KalamDB impersonation wrapper", "EXECUTE AS USER '${1:username}' (\n ${2:SELECT * FROM table}\n);"), + syntaxEntry("BEGIN", "KalamDB transaction control", "BEGIN;"), + syntaxEntry("START TRANSACTION", "KalamDB transaction control", "START TRANSACTION;"), + syntaxEntry("COMMIT", "KalamDB transaction control", "COMMIT;"), + syntaxEntry("ROLLBACK", "KalamDB transaction control", "ROLLBACK;"), + syntaxEntry("SUBSCRIBE TO", "KalamDB live query subscription", "SUBSCRIBE TO ${1:namespace}.${2:table}\nWHERE ${3:user_id = CURRENT_USER()}\nOPTIONS (last_rows=${4:100});"), + syntaxEntry("SUBSCRIBE TO SELECT", "KalamDB live query subscription", "SUBSCRIBE TO SELECT ${1:*}\nFROM ${2:namespace}.${3:table}\nWHERE ${4:user_id = CURRENT_USER()}\nOPTIONS (last_rows=${5:100}, batch_size=${6:50});"), + syntaxEntry("KILL LIVE QUERY", "KalamDB live query command", "KILL LIVE QUERY '${1:live_id}';"), + syntaxEntry("KILL JOB", "KalamDB job command", "KILL JOB '${1:job_id}';"), + syntaxEntry("STORAGE FLUSH ALL", "KalamDB storage maintenance", "STORAGE FLUSH ALL;"), + syntaxEntry("STORAGE FLUSH TABLE", "KalamDB storage maintenance", "STORAGE FLUSH TABLE ${1:namespace}.${2:table};"), + syntaxEntry("STORAGE COMPACT ALL", "KalamDB storage maintenance", "STORAGE COMPACT ALL;"), + syntaxEntry("STORAGE COMPACT TABLE", "KalamDB storage maintenance", "STORAGE COMPACT TABLE ${1:namespace}.${2:table};"), + syntaxEntry("STORAGE CHECK", "KalamDB storage maintenance", "STORAGE CHECK ${1:storage_id};"), + syntaxEntry("SHOW MANIFEST", "KalamDB storage metadata", "SHOW MANIFEST;"), + syntaxEntry("CREATE STORAGE", "KalamDB storage DDL", "CREATE STORAGE ${1:storage_id}\nTYPE filesystem\nNAME '${2:Local Storage}'\nPATH '${3:/var/lib/kalamdb/parquet}'\nSHARED_TABLES_TEMPLATE '${4:{namespace}/{tableName}/}'\nUSER_TABLES_TEMPLATE '${5:{namespace}/{tableName}/{userId}/}';"), + syntaxEntry("ALTER STORAGE", "KalamDB storage DDL", "ALTER STORAGE ${1:storage_id}\nSET NAME '${2:Storage Name}';"), + syntaxEntry("DROP STORAGE", "KalamDB storage DDL", "DROP STORAGE ${1:storage_id};"), + syntaxEntry("SHOW STORAGES", "KalamDB storage metadata", "SHOW STORAGES;"), + syntaxEntry("CREATE TOPIC", "KalamDB topic command", "CREATE TOPIC ${1:topic_name} PARTITIONS ${2:1};"), + syntaxEntry("DROP TOPIC", "KalamDB topic command", "DROP TOPIC ${1:topic_name};"), + syntaxEntry("CLEAR TOPIC", "KalamDB topic command", "CLEAR TOPIC ${1:topic_name};"), + syntaxEntry("ALTER TOPIC ADD SOURCE", "KalamDB topic command", "ALTER TOPIC ${1:topic_name}\nADD SOURCE ${2:namespace}.${3:table}\nON ${4:insert}\nWHERE ${5:condition};"), + syntaxEntry("CONSUME FROM", "KalamDB topic command", "CONSUME FROM ${1:topic_name}\nGROUP '${2:group_id}'\nFROM ${3:latest}\nLIMIT ${4:100};"), + syntaxEntry("ACK", "KalamDB topic command", "ACK ${1:topic_name}\nGROUP '${2:group_id}'\nPARTITION ${3:0}\nUPTO OFFSET ${4:offset};"), + syntaxEntry("BACKUP DATABASE", "KalamDB backup command", "BACKUP DATABASE TO '${1:/backups/kalamdb.tar.gz}';"), + syntaxEntry("RESTORE DATABASE", "KalamDB restore command", "RESTORE DATABASE FROM '${1:/backups/kalamdb.tar.gz}';"), + syntaxEntry("EXPORT USER DATA", "KalamDB export command", "EXPORT USER DATA;"), + syntaxEntry("SHOW EXPORT", "KalamDB export command", "SHOW EXPORT;"), + syntaxEntry("CREATE USER", "KalamDB user management", "CREATE USER ${1:username} WITH PASSWORD '${2:password}' ROLE ${3:user};"), + syntaxEntry("ALTER USER", "KalamDB user management", "ALTER USER ${1:username} SET ${2:ROLE} ${3:user};"), + syntaxEntry("DROP USER", "KalamDB user management", "DROP USER ${1:username};"), + syntaxEntry("CLUSTER SNAPSHOT", "KalamDB cluster command", "CLUSTER SNAPSHOT;"), + syntaxEntry("CLUSTER PURGE", "KalamDB cluster command", "CLUSTER PURGE --upto ${1:index};"), + syntaxEntry("CLUSTER TRIGGER ELECTION", "KalamDB cluster command", "CLUSTER TRIGGER ELECTION;"), + syntaxEntry("CLUSTER TRANSFER LEADER", "KalamDB cluster command", "CLUSTER TRANSFER LEADER ${1:node_id};"), + syntaxEntry("CLUSTER JOIN", "KalamDB cluster command", "CLUSTER JOIN ${1:node_id} RPC ${2:rpc_addr} API ${3:api_addr};"), + syntaxEntry("CLUSTER REBALANCE", "KalamDB cluster command", "CLUSTER REBALANCE;"), + syntaxEntry("CLUSTER STEPDOWN", "KalamDB cluster command", "CLUSTER STEPDOWN;"), + syntaxEntry("CLUSTER CLEAR", "KalamDB cluster command", "CLUSTER CLEAR;"), +] as const; + +const SQL_OPERATOR_COMPLETIONS = [ + operatorEntry("->", "JSON object operator"), + operatorEntry("->>", "JSON text operator"), + operatorEntry("?", "JSON contains operator"), + operatorEntry("=", "SQL comparison operator"), + operatorEntry("!=", "SQL comparison operator"), + operatorEntry("<>", "SQL comparison operator"), + operatorEntry("<=", "SQL comparison operator"), + operatorEntry(">=", "SQL comparison operator"), +] as const; + +const SQL_TYPE_COMPLETIONS = SQL_TYPE_NAMES.map(typeEntry); + +const EMPTY_COMPLETION_DATA: SqlCompletionData = { + namespaces: [], + tablesByNamespace: {}, + columnsByTable: {}, + keywords: SQL_KEYWORDS, + functions: SQL_FUNCTION_COMPLETIONS, + snippets: SQL_SYNTAX_COMPLETIONS, + types: SQL_TYPE_COMPLETIONS, + operators: SQL_OPERATOR_COMPLETIONS, +}; + +function pushUnique(values: string[], value: string) { + if (!values.some((item) => item.toLowerCase() === value.toLowerCase())) { + values.push(value); + } +} + +function addTable( + data: Pick, + namespaceName: string, + tableName: string, + columns: readonly string[], +) { + const namespaceKey = namespaceName.toLowerCase(); + const tableKey = tableName.toLowerCase(); + const qualifiedTable = `${namespaceKey}.${tableKey}`; + + pushUnique(data.namespaces, namespaceName); + if (!data.tablesByNamespace[namespaceKey]) { + data.tablesByNamespace[namespaceKey] = []; + } + pushUnique(data.tablesByNamespace[namespaceKey], tableName); + + if (!data.columnsByTable[qualifiedTable]) { + data.columnsByTable[qualifiedTable] = []; + } + columns.forEach((column) => pushUnique(data.columnsByTable[qualifiedTable], column)); +} + +export function buildSqlCompletionData(schema: StudioNamespace[]): SqlCompletionData { + const data: SqlCompletionData = { + ...EMPTY_COMPLETION_DATA, + namespaces: [], + tablesByNamespace: {}, + columnsByTable: {}, + }; + + schema.forEach((namespace) => { + namespace.tables.forEach((table) => { + addTable(data, namespace.name, table.name, table.columns.map((column) => column.name)); + }); + }); + + STATIC_SYSTEM_TABLES.forEach((table) => { + addTable(data, table.namespace, table.name, table.columns); + }); + + return data; +} + +export function resolveSqlContextualCompletions( + data: Pick, + textUntilPosition: string, + aliasToTable: Record, +): SqlContextualCompletionMatch | null { + const tableColumnMatch = /([a-zA-Z_][\w]*)\.([a-zA-Z_][\w]*)\.([a-zA-Z_][\w]*)?$/.exec(textUntilPosition); + if (tableColumnMatch) { + const namespaceKey = tableColumnMatch[1].toLowerCase(); + const tableKey = tableColumnMatch[2].toLowerCase(); + const columns = data.columnsByTable[`${namespaceKey}.${tableKey}`] ?? []; + return { + kind: "column", + labels: columns, + detail: `${namespaceKey}.${tableKey} column`, + partial: tableColumnMatch[3]?.toLowerCase() ?? "", + }; + } + + const identifierMatch = /([a-zA-Z_][\w]*)\.([a-zA-Z_][\w]*)?$/.exec(textUntilPosition); + if (!identifierMatch) { + return null; + } + + const qualifierKey = identifierMatch[1].toLowerCase(); + const partial = identifierMatch[2]?.toLowerCase() ?? ""; + const resolvedTable = aliasToTable[qualifierKey]; + if (resolvedTable) { + return { + kind: "column", + labels: data.columnsByTable[resolvedTable] ?? [], + detail: `${qualifierKey} alias column`, + partial, + }; + } + + const namespaceTables = data.tablesByNamespace[qualifierKey] ?? []; + if (namespaceTables.length > 0) { + return { + kind: "table", + labels: namespaceTables, + detail: `${qualifierKey} table`, + partial, + }; + } + + return null; +} \ No newline at end of file diff --git a/ui/src/pages/SqlStudio.test.tsx b/ui/src/pages/SqlStudio.test.tsx index 3f263161..32850963 100644 --- a/ui/src/pages/SqlStudio.test.tsx +++ b/ui/src/pages/SqlStudio.test.tsx @@ -353,6 +353,16 @@ describe("SqlStudio page", () => { vi.unstubAllGlobals(); }); + it("renders the main SQL Studio controls on load", async () => { + await renderSqlStudio(); + + expect(getSqlEditor()).toBeTruthy(); + expect(screen.getByRole("button", { name: /^execute$/i })).toBeTruthy(); + expect(screen.getByRole("button", { name: /refresh explorer/i })).toBeTruthy(); + expect(screen.getByTitle("New query tab")).toBeTruthy(); + expect(screen.getByTitle("Expand details panel")).toBeTruthy(); + }); + it("runs a query from the SQL Studio page and renders the results grid", async () => { mockExecuteSqlStudioQuery.mockResolvedValue({ status: "success", @@ -380,6 +390,36 @@ describe("SqlStudio page", () => { expect(await screen.findByText("Ada")).toBeTruthy(); }); + it("runs the active query through the editor command binding", async () => { + mockExecuteSqlStudioQuery.mockResolvedValue({ + status: "success", + rows: [{ id: 42, name: "Grace" }], + schema: [ + { name: "id", dataType: "INT", index: 0, isPrimaryKey: true }, + { name: "name", dataType: "TEXT", index: 1, isPrimaryKey: false }, + ], + tookMs: 8, + rowCount: 1, + logs: [], + }); + + await renderSqlStudio(); + + fireEvent.change(getSqlEditor(), { + target: { value: "SELECT id, name FROM default.events WHERE id = 42" }, + }); + + await act(async () => { + latestEditorCommand?.(); + }); + + await waitFor(() => { + expect(mockExecuteSqlStudioQuery).toHaveBeenCalledWith("SELECT id, name FROM default.events WHERE id = 42"); + }); + + expect(await screen.findByText("Grace")).toBeTruthy(); + }); + it("starts with an empty query editor and opens new tabs empty", async () => { const { store } = await renderSqlStudio(); diff --git a/ui/src/services/sqlStudioService.ts b/ui/src/services/sqlStudioService.ts index 7db62f88..aefe7d89 100644 --- a/ui/src/services/sqlStudioService.ts +++ b/ui/src/services/sqlStudioService.ts @@ -347,7 +347,7 @@ function normalizeTimestampValue(value: unknown): string | number | null { function applyTableMetadata( table: StudioTable, - row: Pick, + row: Partial>, ): void { table.storageId = normalizeTextValue(row.storage_id); table.version = normalizeNumericValue(row.schema_version); diff --git a/ui/tsconfig.json b/ui/tsconfig.json index a2b26562..261c1e86 100644 --- a/ui/tsconfig.json +++ b/ui/tsconfig.json @@ -19,6 +19,7 @@ "paths": { "@/*": ["./src/*"], "@kalamdb/client": ["../link/sdks/typescript/client/dist/src/index.d.ts"], + "@kalamdb/orm": ["../link/sdks/typescript/orm/dist/index.d.ts"], "drizzle-orm": ["./node_modules/drizzle-orm"], "drizzle-orm/*": ["./node_modules/drizzle-orm/*"] } diff --git a/ui/vite.config.ts b/ui/vite.config.ts index 30e3c865..787792ce 100644 --- a/ui/vite.config.ts +++ b/ui/vite.config.ts @@ -48,13 +48,17 @@ export default defineConfig(({ mode }) => { base: "/ui/", resolve: { preserveSymlinks: true, - dedupe: ["drizzle-orm", "@kalamdb/client"], + dedupe: ["drizzle-orm", "@kalamdb/client", "@kalamdb/orm"], alias: { "@": path.resolve(__dirname, "./src"), "@kalamdb/client": path.resolve( __dirname, "../link/sdks/typescript/client/dist/src/index.js", ), + "@kalamdb/orm": path.resolve( + __dirname, + "../link/sdks/typescript/orm/dist/index.js", + ), }, }, server: { @@ -74,6 +78,7 @@ export default defineConfig(({ mode }) => { allow: [ path.resolve(__dirname, "."), path.resolve(__dirname, "../link/sdks/typescript/client"), + path.resolve(__dirname, "../link/sdks/typescript/orm"), ], }, // Disable caching for WASM and SDK files @@ -102,7 +107,7 @@ export default defineConfig(({ mode }) => { force: true, // Exclude the SDK from pre-bundling so WASM files load correctly // When pre-bundled, import.meta.url points to .vite/deps which breaks WASM loading - exclude: ["@kalamdb/client"], + exclude: ["@kalamdb/client", "@kalamdb/orm"], }, // Ensure WASM files are handled correctly assetsInclude: ["**/*.wasm"],