diff --git a/.gitignore b/.gitignore index 7ccf2c44..4e8bc9ae 100644 --- a/.gitignore +++ b/.gitignore @@ -127,3 +127,4 @@ link/sdks/typescript/client/.npmrc /link/kalam-client/sdks/typescript/client/.wasm-target-size-current /link/kalam-client/sdks/typescript/client/.wasm-target-size-current2 /benchv2/logs +/backend/kalamdb-nightly diff --git a/Cargo.lock b/Cargo.lock index 5fc6191b..d174b294 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2849,6 +2849,17 @@ version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" +[[package]] +name = "filetime" +version = "0.2.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f98844151eee8917efc50bd9e8318cb963ae8b297431495d3f758616ea5c57db" +dependencies = [ + "cfg-if", + "libc", + "libredox", +] + [[package]] name = "find-msvc-tools" version = "0.1.9" @@ -4362,6 +4373,7 @@ dependencies = [ "async-trait", "chrono", "datafusion", + "flate2", "kalamdb-commons", "kalamdb-core", "kalamdb-observability", @@ -4374,6 +4386,7 @@ dependencies = [ "parking_lot", "serde", "serde_json", + "tar", "tempfile", "tokio", "tracing", @@ -4904,7 +4917,10 @@ version = "0.1.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1744e39d1d6a9948f4f388969627434e31128196de472883b39f148769bfe30a" dependencies = [ + "bitflags 2.11.0", "libc", + "plain", + "redox_syscall 0.7.5", ] [[package]] @@ -5627,7 +5643,7 @@ checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" dependencies = [ "cfg-if", "libc", - "redox_syscall", + "redox_syscall 0.5.18", "smallvec", "windows-link", ] @@ -5908,6 +5924,12 @@ version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" +[[package]] +name = "plain" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4596b6d070b27117e987119b4dac604f3c58cfb0b191112e24771b2faeac1a6" + [[package]] name = "plotters" version = "0.3.7" @@ -6442,6 +6464,15 @@ dependencies = [ "bitflags 2.11.0", ] +[[package]] +name = "redox_syscall" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4666a1a60d8412eab19d94f6d13dcc9cea0a5ef4fdf6a5db306537413c661b1b" +dependencies = [ + "bitflags 2.11.0", +] + [[package]] name = "redox_users" version = "0.5.2" @@ -7418,6 +7449,17 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" +[[package]] +name = "tar" +version = "0.4.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22692a6476a21fa75fdfc11d452fda482af402c008cdbaf3476414e122040973" +dependencies = [ + "filetime", + "libc", + "xattr", +] + [[package]] name = "target-lexicon" version = "0.13.5" @@ -9004,6 +9046,16 @@ dependencies = [ "time", ] +[[package]] +name = "xattr" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32e45ad4206f6d2479085147f02bc2ef834ac85886624a23575ae137c8aa8156" +dependencies = [ + "libc", + "rustix", +] + [[package]] name = "yasna" version = "0.5.2" diff --git a/backend/crates/kalamdb-api/src/http/sql/models/sql_request.rs b/backend/crates/kalamdb-api/src/http/sql/models/sql_request.rs index a23fb402..08a307f5 100644 --- a/backend/crates/kalamdb-api/src/http/sql/models/sql_request.rs +++ b/backend/crates/kalamdb-api/src/http/sql/models/sql_request.rs @@ -42,8 +42,10 @@ pub struct QueryRequest { pub params: Option>, /// Optional namespace ID for unqualified table names. - /// When set, queries like `SELECT * FROM users` resolve to `namespace_id.users`. - /// Set via `USE namespace` command in CLI clients. + /// When set, this request resolves queries like `SELECT * FROM users` to + /// `namespace_id.users`. + /// Interactive clients can populate this after a successful `USE namespace` + /// command; the value is request-scoped on the backend. #[serde(default, skip_serializing_if = "Option::is_none")] pub namespace_id: Option, } diff --git a/backend/crates/kalamdb-api/src/http/sql/statements.rs b/backend/crates/kalamdb-api/src/http/sql/statements.rs index a78453c4..4ab9945d 100644 --- a/backend/crates/kalamdb-api/src/http/sql/statements.rs +++ b/backend/crates/kalamdb-api/src/http/sql/statements.rs @@ -272,6 +272,26 @@ mod tests { assert_eq!(parsed.sql, "INSERT INTO default.t VALUES (1)"); } + #[test] + fn parse_execute_as_short_alias_quoted_username() { + let parsed = parse_execute_statement( + "EXECUTE AS 'alice' (SELECT * FROM default.todos WHERE id = 1);", + ) + .expect("short alias with quoted username should parse"); + + assert_eq!(parsed.execute_as_username, Some("alice".to_string())); + assert_eq!(parsed.sql, "SELECT * FROM default.todos WHERE id = 1"); + } + + #[test] + fn parse_execute_as_short_alias_bare_username() { + let parsed = parse_execute_statement("execute as bob (INSERT INTO default.t VALUES (1))") + .expect("short alias with bare username should parse"); + + assert_eq!(parsed.execute_as_username, Some("bob".to_string())); + assert_eq!(parsed.sql, "INSERT INTO default.t VALUES (1)"); + } + #[test] fn parse_execute_as_user_bare_no_space_before_paren() { let parsed = parse_execute_statement("EXECUTE AS USER alice(SELECT 1)") diff --git a/backend/crates/kalamdb-core/src/sql/context/execution_context.rs b/backend/crates/kalamdb-core/src/sql/context/execution_context.rs index 3da2391f..ed238635 100644 --- a/backend/crates/kalamdb-core/src/sql/context/execution_context.rs +++ b/backend/crates/kalamdb-core/src/sql/context/execution_context.rs @@ -263,8 +263,10 @@ impl ExecutionContext { /// Get the current default namespace (schema) from DataFusion session config /// - /// This reads `datafusion.catalog.default_schema` from the session configuration. - /// The default schema is set to "default" initially and can be changed using: + /// This reads `datafusion.catalog.default_schema` from the request-scoped + /// session configuration. + /// The default schema is set to "default" initially and can be changed within + /// the current request or multi-statement batch using: /// - `USE namespace` /// - `USE NAMESPACE namespace` /// - `SET NAMESPACE namespace` diff --git a/backend/crates/kalamdb-dialect/src/batch_execution.rs b/backend/crates/kalamdb-dialect/src/batch_execution.rs index cafb2ab2..968994d1 100644 --- a/backend/crates/kalamdb-dialect/src/batch_execution.rs +++ b/backend/crates/kalamdb-dialect/src/batch_execution.rs @@ -518,6 +518,15 @@ mod tests { assert_eq!(statements[1], "SELECT 2"); } + #[test] + fn parse_batch_statements_falls_back_for_short_execute_as() { + let sql = "EXECUTE AS alice (SELECT 1); SELECT 2;"; + let statements = parse_batch_statements(sql).unwrap(); + assert_eq!(statements.len(), 2); + assert_eq!(statements[0], "EXECUTE AS alice (SELECT 1)"); + assert_eq!(statements[1], "SELECT 2"); + } + #[test] fn parse_batch_statements_preserves_file_placeholder_statement() { let sql = "INSERT INTO uploads SELECT * FROM FILE('orders.csv');"; @@ -553,12 +562,12 @@ mod tests { #[test] fn parse_execution_batch_reports_statement_index() { - let err = parse_execution_batch("SELECT 1; EXECUTE AS USER '' (SELECT 2)").unwrap_err(); + let err = parse_execution_batch("SELECT 1; EXECUTE AS '' (SELECT 2)").unwrap_err(); assert_eq!( err, ExecutionBatchParseError::Statement { statement_index: 2, - message: "EXECUTE AS USER username cannot be empty".to_string(), + message: "EXECUTE AS username cannot be empty".to_string(), } ); } @@ -566,7 +575,7 @@ mod tests { #[test] fn prepare_execution_batch_preserves_execute_as_username() { let prepared = - prepare_execution_batch("SELECT 1; EXECUTE AS USER alice (SELECT 2)", |statement| { + prepare_execution_batch("SELECT 1; EXECUTE AS alice (SELECT 2)", |statement| { Ok::<_, String>(statement.sql.clone()) }) .unwrap(); diff --git a/backend/crates/kalamdb-dialect/src/ddl/backup_namespace.rs b/backend/crates/kalamdb-dialect/src/ddl/backup_namespace.rs index 4963deaf..fcd7aa42 100644 --- a/backend/crates/kalamdb-dialect/src/ddl/backup_namespace.rs +++ b/backend/crates/kalamdb-dialect/src/ddl/backup_namespace.rs @@ -3,21 +3,26 @@ //! Parses SQL statements like: //! - BACKUP DATABASE TO '/backups/kalamdb_backup.tar.gz' //! -//! Backs up the entire database (data directory + server.toml). +//! Backs up the entire database to either a server-side directory or a +//! `.tar.gz` / `.tgz` archive. use crate::ddl::DdlResult; /// BACKUP DATABASE statement /// -/// Backs up the entire database to a compressed archive. +/// Backs up the entire database. /// The backup includes: /// - RocksDB data directory /// - Parquet storage files /// - Raft snapshots /// - server.toml configuration +/// +/// If the target path ends with `.tar.gz` or `.tgz`, the backup is written as a +/// single archive file. Otherwise KalamDB writes the backup layout into the +/// target directory. #[derive(Debug, Clone, PartialEq)] pub struct BackupDatabaseStatement { - /// Backup destination path (should end in .tar.gz) + /// Backup destination path on the server filesystem. pub backup_path: String, } @@ -125,6 +130,14 @@ mod tests { assert_eq!(stmt.backup_path, "/backups/kalamdb.tar.gz"); } + #[test] + fn test_parse_backup_database_directory_path() { + let stmt = + BackupDatabaseStatement::parse("BACKUP DATABASE TO '/backups/kalamdb-nightly'") + .unwrap(); + assert_eq!(stmt.backup_path, "/backups/kalamdb-nightly"); + } + #[test] fn test_parse_backup_database_missing_to() { let result = BackupDatabaseStatement::parse("BACKUP DATABASE"); diff --git a/backend/crates/kalamdb-dialect/src/ddl/restore_namespace.rs b/backend/crates/kalamdb-dialect/src/ddl/restore_namespace.rs index 5531c62b..b95be6c8 100644 --- a/backend/crates/kalamdb-dialect/src/ddl/restore_namespace.rs +++ b/backend/crates/kalamdb-dialect/src/ddl/restore_namespace.rs @@ -3,13 +3,14 @@ //! Parses SQL statements like: //! - RESTORE DATABASE FROM '/backups/kalamdb_backup.tar.gz' //! -//! Restores the entire database from a compressed archive. +//! Restores the entire database from either a backup directory or a `.tar.gz` +//! / `.tgz` archive. use crate::ddl::DdlResult; /// RESTORE DATABASE statement /// -/// Restores the entire database from a compressed archive backup. +/// Restores the entire database from a backup path on the server filesystem. /// The restore replaces: /// - RocksDB data directory /// - Parquet storage files @@ -17,7 +18,7 @@ use crate::ddl::DdlResult; /// - server.toml configuration #[derive(Debug, Clone, PartialEq)] pub struct RestoreDatabaseStatement { - /// Backup source path (should be a .tar.gz file) + /// Backup source path on the server filesystem. pub backup_path: String, } @@ -107,6 +108,14 @@ mod tests { assert_eq!(stmt.backup_path, "/backups/kalamdb.tar.gz"); } + #[test] + fn test_parse_restore_database_directory_path() { + let stmt = + RestoreDatabaseStatement::parse("RESTORE DATABASE FROM '/backups/kalamdb-nightly'") + .unwrap(); + assert_eq!(stmt.backup_path, "/backups/kalamdb-nightly"); + } + #[test] fn test_parse_restore_database_missing_from() { let result = RestoreDatabaseStatement::parse("RESTORE DATABASE"); diff --git a/backend/crates/kalamdb-dialect/src/ddl/user_commands.rs b/backend/crates/kalamdb-dialect/src/ddl/user_commands.rs index 1013ae3d..ab035f7c 100644 --- a/backend/crates/kalamdb-dialect/src/ddl/user_commands.rs +++ b/backend/crates/kalamdb-dialect/src/ddl/user_commands.rs @@ -7,7 +7,8 @@ //! //! Uses sqlparser-rs tokenizer for consistent identifier and string handling. -use kalamdb_commons::{AuthType, Role}; +use kalamdb_commons::{AuthType, Role, StorageId}; +use kalamdb_system::providers::storages::models::StorageMode; use serde::{Deserialize, Serialize}; use sqlparser::{ dialect::GenericDialect, @@ -54,6 +55,13 @@ fn parse_role(role_str: &str) -> Result { } } +fn parse_storage_mode(storage_mode_str: &str) -> Result { + StorageMode::from_str_opt(storage_mode_str).ok_or_else(|| UserCommandError { + message: format!("Invalid storage mode '{}'", storage_mode_str), + hint: Some("Valid storage modes: table, region".to_string()), + }) +} + /// Extract identifier or string value from a token fn extract_identifier(token: &Token) -> Option { match token { @@ -94,6 +102,7 @@ fn filter_tokens(tokens: Vec) -> Vec { /// CREATE USER username WITH PASSWORD 'password' ROLE role_name [EMAIL 'email']; /// CREATE USER username WITH OAUTH ROLE role_name [EMAIL 'email']; /// CREATE USER username WITH INTERNAL ROLE role_name; +/// CREATE USER username WITH PASSWORD 'password' ROLE role_name [EMAIL 'email'] [STORAGE_MODE table|region] [STORAGE_ID 'storage']; /// ``` #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] pub struct CreateUserStatement { @@ -102,6 +111,8 @@ pub struct CreateUserStatement { pub role: Role, pub email: Option, pub password: Option, + pub storage_mode: StorageMode, + pub storage_id: Option, } impl CreateUserStatement { @@ -197,18 +208,54 @@ impl CreateUserStatement { })?; let role = parse_role(&role_str)?; - // Optional EMAIL - let email = if is_keyword(iter.peek().unwrap_or(&&Token::EOF), "EMAIL") { - iter.next(); // consume EMAIL - Some(extract_identifier(iter.next().unwrap_or(&Token::EOF)).ok_or_else(|| { - UserCommandError { - message: "Expected email address after EMAIL".to_string(), - hint: Some("Email must be quoted: EMAIL 'user@example.com'".to_string()), + let mut email = None; + let mut storage_mode = StorageMode::Table; + let mut storage_id = None; + + while let Some(token) = iter.peek() { + if is_keyword(token, "EMAIL") { + if email.is_some() { + return Err(UserCommandError { + message: "EMAIL specified more than once".to_string(), + hint: Some("Use a single EMAIL clause".to_string()), + }); } - })?) - } else { - None - }; + iter.next(); + email = Some(extract_identifier(iter.next().unwrap_or(&Token::EOF)).ok_or_else( + || UserCommandError { + message: "Expected email address after EMAIL".to_string(), + hint: Some("Email must be quoted: EMAIL 'user@example.com'".to_string()), + }, + )?); + continue; + } + + if is_keyword(token, "STORAGE_MODE") { + iter.next(); + let value = extract_identifier(iter.next().unwrap_or(&Token::EOF)).ok_or_else(|| { + UserCommandError { + message: "Expected storage mode after STORAGE_MODE".to_string(), + hint: Some("Valid storage modes: table, region".to_string()), + } + })?; + storage_mode = parse_storage_mode(&value)?; + continue; + } + + if is_keyword(token, "STORAGE_ID") { + iter.next(); + let value = extract_identifier(iter.next().unwrap_or(&Token::EOF)).ok_or_else(|| { + UserCommandError { + message: "Expected storage ID after STORAGE_ID".to_string(), + hint: Some("Storage ID can be quoted: STORAGE_ID 'local'".to_string()), + } + })?; + storage_id = Some(StorageId::from(value)); + continue; + } + + break; + } Ok(CreateUserStatement { username, @@ -216,6 +263,8 @@ impl CreateUserStatement { role, email, password, + storage_mode, + storage_id, }) } } @@ -231,6 +280,9 @@ impl CreateUserStatement { /// ALTER USER username SET PASSWORD 'new_password'; /// ALTER USER username SET ROLE new_role; /// ALTER USER username SET EMAIL 'new_email@example.com'; +/// ALTER USER username SET STORAGE_MODE table|region; +/// ALTER USER username SET STORAGE_ID 'storage_id'; +/// ALTER USER username SET STORAGE_ID NULL; /// ``` #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] pub struct AlterUserStatement { @@ -244,6 +296,8 @@ pub enum UserModification { SetPassword(String), SetRole(Role), SetEmail(String), + SetStorageMode(StorageMode), + SetStorageId(Option), } impl UserModification { @@ -254,6 +308,13 @@ impl UserModification { UserModification::SetPassword(_) => "SetPassword([REDACTED])".to_string(), UserModification::SetRole(role) => format!("SetRole({:?})", role), UserModification::SetEmail(email) => format!("SetEmail({})", email), + UserModification::SetStorageMode(storage_mode) => { + format!("SetStorageMode({})", storage_mode) + }, + UserModification::SetStorageId(Some(storage_id)) => { + format!("SetStorageId({})", storage_id) + }, + UserModification::SetStorageId(None) => "SetStorageId(NULL)".to_string(), } } } @@ -271,13 +332,19 @@ impl AlterUserStatement { if !is_keyword(iter.next().unwrap_or(&Token::EOF), "ALTER") { return Err(UserCommandError { message: "Expected ALTER".to_string(), - hint: Some("Syntax: ALTER USER username SET PASSWORD|ROLE|EMAIL ...".to_string()), + hint: Some( + "Syntax: ALTER USER username SET PASSWORD|ROLE|EMAIL|STORAGE_MODE|STORAGE_ID ..." + .to_string(), + ), }); } if !is_keyword(iter.next().unwrap_or(&Token::EOF), "USER") { return Err(UserCommandError { message: "Expected USER after ALTER".to_string(), - hint: Some("Syntax: ALTER USER username SET PASSWORD|ROLE|EMAIL ...".to_string()), + hint: Some( + "Syntax: ALTER USER username SET PASSWORD|ROLE|EMAIL|STORAGE_MODE|STORAGE_ID ..." + .to_string(), + ), }); } @@ -293,11 +360,14 @@ impl AlterUserStatement { if !is_keyword(iter.next().unwrap_or(&Token::EOF), "SET") { return Err(UserCommandError { message: "Expected SET after username".to_string(), - hint: Some("Syntax: ALTER USER username SET PASSWORD|ROLE|EMAIL ...".to_string()), + hint: Some( + "Syntax: ALTER USER username SET PASSWORD|ROLE|EMAIL|STORAGE_MODE|STORAGE_ID ..." + .to_string(), + ), }); } - // Modification type: PASSWORD, ROLE, or EMAIL + // Modification type: PASSWORD, ROLE, EMAIL, STORAGE_MODE, or STORAGE_ID let mod_token = iter.next().unwrap_or(&Token::EOF); let modification = if is_keyword(mod_token, "PASSWORD") { let pwd = extract_identifier(iter.next().unwrap_or(&Token::EOF)).ok_or_else(|| { @@ -330,11 +400,34 @@ impl AlterUserStatement { } })?; UserModification::SetEmail(email) + } else if is_keyword(mod_token, "STORAGE_MODE") { + let storage_mode = + extract_identifier(iter.next().unwrap_or(&Token::EOF)).ok_or_else(|| { + UserCommandError { + message: "Expected storage mode after SET STORAGE_MODE".to_string(), + hint: Some("Valid storage modes: table, region".to_string()), + } + })?; + UserModification::SetStorageMode(parse_storage_mode(&storage_mode)?) + } else if is_keyword(mod_token, "STORAGE_ID") { + let token = iter.next().unwrap_or(&Token::EOF); + if is_keyword(token, "NULL") { + UserModification::SetStorageId(None) + } else { + let storage_id = extract_identifier(token).ok_or_else(|| UserCommandError { + message: "Expected storage ID or NULL after SET STORAGE_ID".to_string(), + hint: Some( + "Use SET STORAGE_ID 'local' or SET STORAGE_ID NULL".to_string(), + ), + })?; + UserModification::SetStorageId(Some(StorageId::from(storage_id))) + } } else { return Err(UserCommandError { - message: "Expected PASSWORD, ROLE, or EMAIL after SET".to_string(), + message: "Expected PASSWORD, ROLE, EMAIL, STORAGE_MODE, or STORAGE_ID after SET" + .to_string(), hint: Some( - "Valid modifications: SET PASSWORD 'pass', SET ROLE admin, SET EMAIL 'x@y.com'" + "Valid modifications: SET PASSWORD 'pass', SET ROLE admin, SET EMAIL 'x@y.com', SET STORAGE_MODE region, SET STORAGE_ID 'local'" .to_string(), ), }); @@ -435,6 +528,16 @@ mod tests { assert_eq!(stmt.password, Some("secure123".to_string())); assert_eq!(stmt.role, Role::Service); assert_eq!(stmt.email, Some("alice@example.com".to_string())); + assert_eq!(stmt.storage_mode, StorageMode::Table); + assert_eq!(stmt.storage_id, None); + } + + #[test] + fn test_create_user_with_storage_options() { + let sql = "CREATE USER 'alice' WITH PASSWORD 'secure123' ROLE user STORAGE_MODE region STORAGE_ID 's3_eu'"; + let stmt = CreateUserStatement::parse(sql).unwrap(); + assert_eq!(stmt.storage_mode, StorageMode::Region); + assert_eq!(stmt.storage_id, Some(StorageId::from("s3_eu"))); } #[test] @@ -542,12 +645,50 @@ mod tests { } } + #[test] + fn test_alter_user_set_storage_mode() { + let sql = "ALTER USER bob SET STORAGE_MODE region"; + let stmt = AlterUserStatement::parse(sql).unwrap(); + assert_eq!(stmt.username, "bob"); + if let UserModification::SetStorageMode(storage_mode) = stmt.modification { + assert_eq!(storage_mode, StorageMode::Region); + } else { + panic!("Expected SetStorageMode"); + } + } + + #[test] + fn test_alter_user_set_storage_id() { + let sql = "ALTER USER bob SET STORAGE_ID 's3_eu'"; + let stmt = AlterUserStatement::parse(sql).unwrap(); + assert_eq!(stmt.username, "bob"); + if let UserModification::SetStorageId(storage_id) = stmt.modification { + assert_eq!(storage_id, Some(StorageId::from("s3_eu"))); + } else { + panic!("Expected SetStorageId"); + } + } + + #[test] + fn test_alter_user_set_storage_id_null() { + let sql = "ALTER USER bob SET STORAGE_ID NULL"; + let stmt = AlterUserStatement::parse(sql).unwrap(); + assert_eq!(stmt.username, "bob"); + if let UserModification::SetStorageId(storage_id) = stmt.modification { + assert_eq!(storage_id, None); + } else { + panic!("Expected SetStorageId"); + } + } + #[test] fn test_alter_user_invalid_modification() { let sql = "ALTER USER alice SET UNKNOWN 'value'"; let result = AlterUserStatement::parse(sql); assert!(result.is_err()); - assert!(result.unwrap_err().contains("PASSWORD, ROLE, or EMAIL")); + assert!(result + .unwrap_err() + .contains("PASSWORD, ROLE, EMAIL, STORAGE_MODE, or STORAGE_ID")); } // DROP USER tests @@ -622,4 +763,20 @@ mod tests { assert_eq!(display, "SetEmail(alice@example.com)"); } + + #[test] + fn test_user_modification_display_for_audit_storage_mode() { + let modification = UserModification::SetStorageMode(StorageMode::Region); + let display = modification.display_for_audit(); + + assert_eq!(display, "SetStorageMode(region)"); + } + + #[test] + fn test_user_modification_display_for_audit_storage_id() { + let modification = UserModification::SetStorageId(Some(StorageId::from("s3_eu"))); + let display = modification.display_for_audit(); + + assert_eq!(display, "SetStorageId(s3_eu)"); + } } diff --git a/backend/crates/kalamdb-dialect/src/execute_as.rs b/backend/crates/kalamdb-dialect/src/execute_as.rs index abf411f2..ef092185 100644 --- a/backend/crates/kalamdb-dialect/src/execute_as.rs +++ b/backend/crates/kalamdb-dialect/src/execute_as.rs @@ -1,57 +1,60 @@ -//! EXECUTE AS USER parsing utilities +//! EXECUTE AS parsing utilities //! -//! Provides a single shared implementation for parsing `EXECUTE AS USER` SQL -//! wrappers. Both the API execution handler and the cluster forwarding logic -//! delegate to these functions so there is **one** canonical parser. +//! Provides a single shared implementation for parsing `EXECUTE AS` SQL +//! wrappers. Both the API execution handler and the cluster forwarding logic +//! delegate to these functions so there is one canonical parser. //! //! Grammar: //! ```text -//! EXECUTE AS USER ( ) +//! EXECUTE AS ( ) +//! EXECUTE AS USER ( ) // legacy compatibility //! ``` -//! `` may be single-quoted (`'alice'`) or bare (`alice`). +//! `` may be single-quoted (`'alice'`) or bare (`alice`). //! The inner SQL must be exactly one statement. use crate::split_statements; -/// Prefix used for case-insensitive matching. -const EXECUTE_AS_PREFIX: &str = "EXECUTE AS USER"; -/// Byte length of the prefix — used for fast slice comparisons. -const EXECUTE_AS_PREFIX_LEN: usize = EXECUTE_AS_PREFIX.len(); // 15 +/// Prefixes used for case-insensitive matching. +const EXECUTE_AS_PREFIXES: [&str; 2] = ["EXECUTE AS USER", "EXECUTE AS"]; -/// Result of parsing an `EXECUTE AS USER` envelope. +/// Result of parsing an `EXECUTE AS` envelope. #[derive(Debug, Clone, PartialEq, Eq)] pub struct ExecuteAsEnvelope { - /// The user identifier that follows `EXECUTE AS USER`. + /// The user identifier that follows `EXECUTE AS` or legacy `EXECUTE AS USER`. pub username: String, /// The inner SQL statement (parentheses stripped). pub inner_sql: String, } -/// Check whether `sql` starts with `EXECUTE AS USER` (case-insensitive). +#[inline] +fn match_execute_as_prefix(sql: &str) -> Option { + let trimmed = sql.trim(); + EXECUTE_AS_PREFIXES.iter().find_map(|prefix| { + let prefix_len = prefix.len(); + (trimmed.len() >= prefix_len + && trimmed.as_bytes()[..prefix_len].eq_ignore_ascii_case(prefix.as_bytes())) + .then_some(prefix_len) + }) +} + +/// Check whether `sql` starts with canonical `EXECUTE AS` or legacy `EXECUTE AS USER` +/// (case-insensitive). /// /// This is a very cheap check that avoids any allocation. #[inline] pub fn is_execute_as(sql: &str) -> bool { - let trimmed = sql.trim(); - trimmed.len() >= EXECUTE_AS_PREFIX_LEN - && trimmed.as_bytes()[..EXECUTE_AS_PREFIX_LEN] - .eq_ignore_ascii_case(EXECUTE_AS_PREFIX.as_bytes()) + match_execute_as_prefix(sql).is_some() } -/// Extract just the inner SQL from an `EXECUTE AS USER '...' (...)` wrapper. +/// Extract just the inner SQL from an `EXECUTE AS '...' (...)` wrapper. /// -/// Returns `Some(inner_sql)` if the SQL matches the EXECUTE AS USER pattern, +/// Returns `Some(inner_sql)` if the SQL matches an EXECUTE AS pattern, /// `None` otherwise. This is intentionally lightweight — it only needs to /// strip the wrapper so callers (e.g. statement classifier) can inspect the /// actual SQL statement without allocating a full parsed result. pub fn extract_inner_sql(sql: &str) -> Option { let trimmed = sql.trim(); - if trimmed.len() < EXECUTE_AS_PREFIX_LEN - || !trimmed.as_bytes()[..EXECUTE_AS_PREFIX_LEN] - .eq_ignore_ascii_case(EXECUTE_AS_PREFIX.as_bytes()) - { - return None; - } + match_execute_as_prefix(trimmed)?; let open_paren = trimmed.find('(')?; let close_paren = find_matching_close_paren(trimmed, open_paren)?; @@ -92,7 +95,7 @@ fn parse_single_quoted(input: &str) -> Result<(String, &str), String> { return Ok((value, &input[idx + 1..])); } - Err("EXECUTE AS USER username quote was not closed".to_string()) + Err("EXECUTE AS username quote was not closed".to_string()) } fn find_matching_close_paren(input: &str, open_idx: usize) -> Option { @@ -148,14 +151,16 @@ fn find_matching_close_paren(input: &str, open_idx: usize) -> Option { None } -/// Fully parse an `EXECUTE AS USER` envelope, extracting the username and the +/// Fully parse an `EXECUTE AS` envelope, extracting the username and the /// parenthesised inner SQL. /// -/// The username may be quoted (`'alice'`) or bare (`alice`). +/// Supports both canonical `EXECUTE AS (...)` and +/// legacy `EXECUTE AS USER (...)`. +/// The user id may be quoted (`'alice'`) or bare (`alice`). /// Bare usernames extend up to the first whitespace or `(` character. /// /// Returns: -/// - `Ok(Some(envelope))` when the statement is an EXECUTE AS USER wrapper +/// - `Ok(Some(envelope))` when the statement is an EXECUTE AS wrapper /// - `Ok(None)` when the statement is a normal SQL statement (no wrapper) /// - `Err(msg)` on syntax errors inside the wrapper pub fn parse_execute_as(statement: &str) -> Result, String> { @@ -164,58 +169,54 @@ pub fn parse_execute_as(statement: &str) -> Result, St return Err("Empty SQL statement".to_string()); } - // Fast prefix check without allocating an uppercased copy. - if trimmed.len() < EXECUTE_AS_PREFIX_LEN - || !trimmed.as_bytes()[..EXECUTE_AS_PREFIX_LEN] - .eq_ignore_ascii_case(EXECUTE_AS_PREFIX.as_bytes()) - { + let Some(prefix_len) = match_execute_as_prefix(trimmed) else { return Ok(None); - } + }; - let after_prefix = trimmed[EXECUTE_AS_PREFIX_LEN..].trim_start(); + let after_prefix = trimmed[prefix_len..].trim_start(); // --- Username extraction (quoted or bare) --- let (username, rest): (String, &str) = if after_prefix.starts_with('\'') { let (parsed_username, rest) = parse_single_quoted(after_prefix)?; let uname = parsed_username.trim(); if uname.is_empty() { - return Err("EXECUTE AS USER username cannot be empty".to_string()); + return Err("EXECUTE AS username cannot be empty".to_string()); } (uname.to_string(), rest.trim_start()) } else { - // Bare: EXECUTE AS USER alice (...) + // Bare: EXECUTE AS alice (...) or EXECUTE AS USER alice (...) // Username extends until whitespace or '('. let end = after_prefix .find(|c: char| c.is_whitespace() || c == '(') .unwrap_or(after_prefix.len()); let uname = after_prefix[..end].trim(); if uname.is_empty() { - return Err("EXECUTE AS USER username cannot be empty".to_string()); + return Err("EXECUTE AS username cannot be empty".to_string()); } (uname.to_string(), after_prefix[end..].trim_start()) }; // --- Parenthesised SQL body --- if !rest.starts_with('(') { - return Err("EXECUTE AS USER must wrap SQL in parentheses".to_string()); + return Err("EXECUTE AS must wrap SQL in parentheses".to_string()); } let close_idx = find_matching_close_paren(rest, 0) - .ok_or_else(|| "EXECUTE AS USER missing closing ')'".to_string())?; + .ok_or_else(|| "EXECUTE AS missing closing ')'".to_string())?; let inner_sql = rest[1..close_idx].trim(); if inner_sql.is_empty() { - return Err("EXECUTE AS USER requires a non-empty inner SQL statement".to_string()); + return Err("EXECUTE AS requires a non-empty inner SQL statement".to_string()); } let trailing = rest[close_idx + 1..].trim(); if !trailing.is_empty() { - return Err("EXECUTE AS USER must contain exactly one wrapped SQL statement".to_string()); + return Err("EXECUTE AS must contain exactly one wrapped SQL statement".to_string()); } let inner_statements = split_statements(inner_sql) - .map_err(|e| format!("Failed to parse inner SQL for EXECUTE AS USER: {}", e))?; + .map_err(|e| format!("Failed to parse inner SQL for EXECUTE AS: {}", e))?; if inner_statements.len() != 1 { - return Err("EXECUTE AS USER can only wrap a single SQL statement".to_string()); + return Err("EXECUTE AS can only wrap a single SQL statement".to_string()); } Ok(Some(ExecuteAsEnvelope { @@ -235,7 +236,9 @@ mod tests { #[test] fn is_execute_as_positive() { assert!(is_execute_as("EXECUTE AS USER 'alice' (SELECT 1)")); + assert!(is_execute_as("EXECUTE AS 'alice' (SELECT 1)")); assert!(is_execute_as("execute as user bob (SELECT 1)")); + assert!(is_execute_as("execute as bob (SELECT 1)")); assert!(is_execute_as(" EXECUTE AS USER 'x' (SELECT 1) ")); } @@ -256,6 +259,12 @@ mod tests { assert_eq!(inner.as_deref(), Some("SELECT * FROM t")); } + #[test] + fn extract_inner_sql_short_alias() { + let inner = extract_inner_sql("EXECUTE AS 'alice' (SELECT * FROM t)"); + assert_eq!(inner.as_deref(), Some("SELECT * FROM t")); + } + #[test] fn extract_inner_sql_bare_username() { let inner = extract_inner_sql("EXECUTE AS USER bob (INSERT INTO t VALUES (1, 'x'))"); @@ -320,6 +329,27 @@ mod tests { assert_eq!(result.inner_sql, "INSERT INTO default.t VALUES (1)"); } + #[test] + fn parse_short_alias_quoted_username() { + let result = + parse_execute_as("EXECUTE AS 'alice' (SELECT * FROM default.todos WHERE id = 1);") + .expect("should parse") + .expect("should be an envelope"); + + assert_eq!(result.username, "alice"); + assert_eq!(result.inner_sql, "SELECT * FROM default.todos WHERE id = 1"); + } + + #[test] + fn parse_short_alias_bare_username() { + let result = parse_execute_as("execute as bob (INSERT INTO default.t VALUES (1))") + .expect("should parse") + .expect("should be an envelope"); + + assert_eq!(result.username, "bob"); + assert_eq!(result.inner_sql, "INSERT INTO default.t VALUES (1)"); + } + #[test] fn parse_escaped_quote_in_username() { let result = diff --git a/backend/crates/kalamdb-handlers/crates/admin/src/backup/restore_database.rs b/backend/crates/kalamdb-handlers/crates/admin/src/backup/restore_database.rs index 7ec9dd58..b75389c4 100644 --- a/backend/crates/kalamdb-handlers/crates/admin/src/backup/restore_database.rs +++ b/backend/crates/kalamdb-handlers/crates/admin/src/backup/restore_database.rs @@ -24,6 +24,15 @@ impl RestoreDatabaseHandler { pub fn new(app_context: Arc) -> Self { Self { app_context } } + + fn is_archive_path(path: &std::path::Path) -> bool { + let value = path + .to_string_lossy() + .trim() + .trim_end_matches(['/', '\\']) + .to_ascii_lowercase(); + value.ends_with(".tar.gz") || value.ends_with(".tgz") + } } impl TypedStatementHandler for RestoreDatabaseHandler { @@ -33,17 +42,19 @@ impl TypedStatementHandler for RestoreDatabaseHandler _params: Vec, _context: &ExecutionContext, ) -> Result { - // Verify backup directory exists before creating job + // Verify backup source exists before creating job let backup_path = std::path::Path::new(&statement.backup_path); if !backup_path.exists() { return Err(KalamDbError::NotFound(format!( - "Backup directory '{}' not found", + "Backup path '{}' not found", statement.backup_path ))); } - if !backup_path.is_dir() { + if !backup_path.is_dir() + && !(backup_path.is_file() && Self::is_archive_path(backup_path)) + { return Err(KalamDbError::InvalidOperation(format!( - "'{}' is not a directory. RESTORE DATABASE expects a backup directory, not a file.", + "'{}' must be a backup directory or a .tar.gz/.tgz backup archive.", statement.backup_path ))); } diff --git a/backend/crates/kalamdb-handlers/crates/admin/src/export/show_export.rs b/backend/crates/kalamdb-handlers/crates/admin/src/export/show_export.rs index d0b51baf..5a47c281 100644 --- a/backend/crates/kalamdb-handlers/crates/admin/src/export/show_export.rs +++ b/backend/crates/kalamdb-handlers/crates/admin/src/export/show_export.rs @@ -47,12 +47,9 @@ impl ShowExportHandler { created_at_millis.saturating_mul(1_000) } - /// Build a download URL for a completed export - fn build_download_url(&self, user_id: &str, export_id: &str) -> String { - let config = self.app_context.config(); - let host = &config.server.host; - let port = config.server.port; - format!("http://{}:{}/v1/exports/{}/{}", host, port, user_id, export_id) + /// Build a download URI for a completed export. + fn build_download_url(user_id: &str, export_id: &str) -> String { + format!("/v1/exports/{}/{}", user_id, export_id) } /// Extract export_id from job parameters JSON @@ -124,7 +121,7 @@ impl TypedStatementHandler for ShowExportHandler { // Build download URL only for completed jobs let url = if job.status == kalamdb_system::JobStatus::Completed { Self::extract_export_id(job) - .map(|eid| self.build_download_url(&user_id, &eid)) + .map(|eid| Self::build_download_url(&user_id, &eid)) .unwrap_or_default() } else { String::new() @@ -181,4 +178,11 @@ mod tests { fn show_export_created_at_converts_millis_to_micros() { assert_eq!(ShowExportHandler::created_at_micros(1_741_900_245_123), 1_741_900_245_123_000); } + + #[test] + fn show_export_download_url_is_relative_uri() { + let url = ShowExportHandler::build_download_url("alice", "export-123"); + + assert_eq!(url, "/v1/exports/alice/export-123"); + } } diff --git a/backend/crates/kalamdb-handlers/crates/ddl/src/namespace/use_namespace.rs b/backend/crates/kalamdb-handlers/crates/ddl/src/namespace/use_namespace.rs index ec4ea98b..b4453268 100644 --- a/backend/crates/kalamdb-handlers/crates/ddl/src/namespace/use_namespace.rs +++ b/backend/crates/kalamdb-handlers/crates/ddl/src/namespace/use_namespace.rs @@ -1,10 +1,13 @@ //! USE NAMESPACE handler //! -//! This handler sets the default schema for unqualified table names using -//! DataFusion's native configuration system. +//! This handler sets the default schema for unqualified table names for the +//! current request or multi-statement batch using DataFusion's native +//! configuration system. //! //! After executing `USE namespace1`, queries like `SELECT * FROM users` //! will resolve to `kalam.namespace1.users`. +//! Interactive clients such as the CLI persist the chosen namespace on the +//! client and send it back on later requests via `namespace_id`. use std::sync::Arc; @@ -21,7 +24,8 @@ use kalamdb_sql::ddl::UseNamespaceStatement; /// Handler for USE NAMESPACE / USE / SET NAMESPACE statements /// /// Uses DataFusion's native `datafusion.catalog.default_schema` configuration -/// to change the default schema for the current session. +/// to change the default schema for the current request or multi-statement +/// batch. This is not a long-lived backend connection session. pub struct UseNamespaceHandler { app_context: Arc, } diff --git a/backend/crates/kalamdb-handlers/crates/user/src/lib.rs b/backend/crates/kalamdb-handlers/crates/user/src/lib.rs index c4679806..06dc9ec0 100644 --- a/backend/crates/kalamdb-handlers/crates/user/src/lib.rs +++ b/backend/crates/kalamdb-handlers/crates/user/src/lib.rs @@ -24,6 +24,8 @@ pub fn register_user_handlers( role: kalamdb_commons::Role::User, email: None, password: None, + storage_mode: kalamdb_system::providers::storages::models::StorageMode::Table, + storage_id: None, }), user::CreateUserHandler::new(app_context.clone(), enforce_password_complexity), SqlStatementKind::CreateUser, diff --git a/backend/crates/kalamdb-handlers/crates/user/src/user/alter.rs b/backend/crates/kalamdb-handlers/crates/user/src/user/alter.rs index 23e65cfc..e35d2ed1 100644 --- a/backend/crates/kalamdb-handlers/crates/user/src/user/alter.rs +++ b/backend/crates/kalamdb-handlers/crates/user/src/user/alter.rs @@ -98,6 +98,32 @@ impl TypedStatementHandler for AlterUserHandler { } updated.email = Some(new_email.clone()); }, + UserModification::SetStorageMode(new_storage_mode) => { + updated.storage_mode = new_storage_mode; + }, + UserModification::SetStorageId(ref new_storage_id) => { + if let Some(storage_id) = new_storage_id { + let app_ctx = self.app_context.clone(); + let storage_lookup_id = storage_id.clone(); + let storage = tokio::task::spawn_blocking(move || { + app_ctx + .system_tables() + .storages() + .get_storage_by_id(&storage_lookup_id) + }) + .await + .map_err(|e| KalamDbError::ExecutionError(format!("Task join error: {}", e)))??; + + if storage.is_none() { + return Err(KalamDbError::InvalidOperation(format!( + "Storage '{}' does not exist", + storage_id.as_str() + ))); + } + } + + updated.storage_id = new_storage_id.clone(); + }, } updated.updated_at = chrono::Utc::now().timestamp_millis(); diff --git a/backend/crates/kalamdb-handlers/crates/user/src/user/create.rs b/backend/crates/kalamdb-handlers/crates/user/src/user/create.rs index 0208fdb0..e959b049 100644 --- a/backend/crates/kalamdb-handlers/crates/user/src/user/create.rs +++ b/backend/crates/kalamdb-handlers/crates/user/src/user/create.rs @@ -56,6 +56,27 @@ impl TypedStatementHandler for CreateUserHandler { ))); } + let storage_id = if let Some(storage_id) = statement.storage_id.clone() { + let app_ctx = self.app_context.clone(); + let storage_lookup_id = storage_id.clone(); + let storage = tokio::task::spawn_blocking(move || { + app_ctx.system_tables().storages().get_storage_by_id(&storage_lookup_id) + }) + .await + .map_err(|e| KalamDbError::ExecutionError(format!("Task join error: {}", e)))??; + + if storage.is_none() { + return Err(KalamDbError::InvalidOperation(format!( + "Storage '{}' does not exist", + storage_id.as_str() + ))); + } + + Some(storage_id) + } else { + None + }; + // Hash password if auth_type = Password, or extract auth_data for OAuth let (password_hash, auth_data) = match statement.auth_type { AuthType::Password => { @@ -123,8 +144,8 @@ impl TypedStatementHandler for CreateUserHandler { email: statement.email.clone(), auth_type: statement.auth_type, auth_data, - storage_mode: kalamdb_system::providers::storages::models::StorageMode::Table, - storage_id: None, + storage_mode: statement.storage_mode, + storage_id, failed_login_attempts: 0, locked_until: None, last_login_at: None, @@ -148,7 +169,16 @@ impl TypedStatementHandler for CreateUserHandler { "CREATE", "USER", &statement.username, - Some(format!("Role: {:?}", statement.role)), + Some(format!( + "Role: {:?}, storage_mode: {}, storage_id: {}", + statement.role, + statement.storage_mode, + statement + .storage_id + .as_ref() + .map(|storage_id| storage_id.as_str()) + .unwrap_or("NULL") + )), None, ); audit::persist_audit_entry(&self.app_context, &audit_entry).await?; diff --git a/backend/crates/kalamdb-jobs/Cargo.toml b/backend/crates/kalamdb-jobs/Cargo.toml index 4e9f88c0..43d6aea2 100644 --- a/backend/crates/kalamdb-jobs/Cargo.toml +++ b/backend/crates/kalamdb-jobs/Cargo.toml @@ -41,6 +41,8 @@ uuid = { workspace = true } # ZIP archive creation zip = { workspace = true } +flate2 = { workspace = true } +tar = { workspace = true } # Concurrent parking_lot = { workspace = true } diff --git a/backend/crates/kalamdb-jobs/src/executors/backup.rs b/backend/crates/kalamdb-jobs/src/executors/backup.rs index c9dacaaa..ccdcbb0f 100644 --- a/backend/crates/kalamdb-jobs/src/executors/backup.rs +++ b/backend/crates/kalamdb-jobs/src/executors/backup.rs @@ -4,7 +4,11 @@ //! for the key-value store, plus a recursive directory copy for Parquet storage, //! snapshots, streams, and the server.toml configuration file. //! -//! ## Backup Directory Layout +//! When the target path ends with `.tar.gz` or `.tgz`, the executor writes a +//! single archive file containing the following layout. Otherwise the same +//! layout is written directly into the target directory. +//! +//! ## Backup Layout //! ```text //! / //! rocksdb/ ← RocksDB BackupEngine output (hot, consistent, deduplicated) @@ -21,20 +25,29 @@ //! } //! ``` -use std::{fs, path::Path}; +use std::{ + fs::{self, File}, + path::{Path, PathBuf}, +}; use async_trait::async_trait; +use flate2::{write::GzEncoder, Compression}; use kalamdb_core::error::KalamDbError; use kalamdb_system::JobType; use serde::{Deserialize, Serialize}; +use tar::Builder; +use uuid::Uuid; use crate::executors::{JobContext, JobDecision, JobExecutor, JobParams}; /// Typed parameters for full database backup operations #[derive(Debug, Clone, Serialize, Deserialize)] pub struct BackupParams { - /// Destination directory for the backup (created if it does not exist). - /// RocksDB native BackupEngine output goes into `/rocksdb/`. + /// Destination backup path on the server filesystem. + /// + /// If the path ends with `.tar.gz` or `.tgz`, KalamDB writes a single + /// archive file. Otherwise a backup directory is created and the layout is + /// written under `/rocksdb`, `/storage`, and so on. pub backup_path: String, } @@ -58,6 +71,145 @@ impl BackupExecutor { Self } + fn is_archive_path(path: &Path) -> bool { + let value = path + .to_string_lossy() + .trim() + .trim_end_matches(['/', '\\']) + .to_ascii_lowercase(); + value.ends_with(".tar.gz") || value.ends_with(".tgz") + } + + fn create_archive_staging_dir(archive_path: &Path) -> Result { + let parent = archive_path.parent().unwrap_or_else(|| Path::new(".")); + fs::create_dir_all(parent).map_err(|e| { + KalamDbError::InvalidOperation(format!( + "Failed to create archive parent directory '{}': {}", + parent.display(), + e + )) + })?; + + let file_name = archive_path + .file_name() + .and_then(|value| value.to_str()) + .unwrap_or("kalamdb-backup"); + let staging_dir = parent.join(format!(".{}.staging-{}", file_name, Uuid::new_v4())); + + fs::create_dir_all(&staging_dir).map_err(|e| { + KalamDbError::InvalidOperation(format!( + "Failed to create backup staging directory '{}': {}", + staging_dir.display(), + e + )) + })?; + + Ok(staging_dir) + } + + fn create_tar_gz_archive(src_dir: &Path, archive_path: &Path) -> Result<(), KalamDbError> { + if let Some(parent) = archive_path.parent() { + fs::create_dir_all(parent).map_err(|e| { + KalamDbError::InvalidOperation(format!( + "Failed to create archive output directory '{}': {}", + parent.display(), + e + )) + })?; + } + + if archive_path.exists() { + if archive_path.is_dir() { + return Err(KalamDbError::InvalidOperation(format!( + "Archive output path '{}' is a directory", + archive_path.display() + ))); + } + + fs::remove_file(archive_path).map_err(|e| { + KalamDbError::InvalidOperation(format!( + "Failed to replace existing archive '{}': {}", + archive_path.display(), + e + )) + })?; + } + + let archive_file = File::create(archive_path).map_err(|e| { + KalamDbError::InvalidOperation(format!( + "Failed to create archive '{}': {}", + archive_path.display(), + e + )) + })?; + let encoder = GzEncoder::new(archive_file, Compression::default()); + let mut builder = Builder::new(encoder); + + for entry in fs::read_dir(src_dir).map_err(|e| { + KalamDbError::InvalidOperation(format!( + "Failed to read archive staging directory '{}': {}", + src_dir.display(), + e + )) + })? { + let entry = entry.map_err(|e| { + KalamDbError::InvalidOperation(format!("Directory entry error: {}", e)) + })?; + let src_path = entry.path(); + let name = entry.file_name(); + let archive_name = Path::new(name.as_os_str()); + + if src_path.is_dir() { + builder.append_dir_all(archive_name, &src_path).map_err(|e| { + KalamDbError::InvalidOperation(format!( + "Failed to archive directory '{}': {}", + src_path.display(), + e + )) + })?; + } else { + let mut file = File::open(&src_path).map_err(|e| { + KalamDbError::InvalidOperation(format!( + "Failed to open archive input '{}': {}", + src_path.display(), + e + )) + })?; + builder.append_file(archive_name, &mut file).map_err(|e| { + KalamDbError::InvalidOperation(format!( + "Failed to archive file '{}': {}", + src_path.display(), + e + )) + })?; + } + } + + builder.finish().map_err(|e| { + KalamDbError::InvalidOperation(format!( + "Failed to finish archive '{}': {}", + archive_path.display(), + e + )) + })?; + let encoder = builder.into_inner().map_err(|e| { + KalamDbError::InvalidOperation(format!( + "Failed to finalize archive '{}': {}", + archive_path.display(), + e + )) + })?; + encoder.finish().map_err(|e| { + KalamDbError::InvalidOperation(format!( + "Failed to flush archive '{}': {}", + archive_path.display(), + e + )) + })?; + + Ok(()) + } + /// Recursively copy `src` into `dst`, creating `dst` if necessary. fn copy_dir(src: &Path, dst: &Path) -> Result { if !src.exists() { @@ -115,46 +267,73 @@ impl JobExecutor for BackupExecutor { async fn execute(&self, ctx: &JobContext) -> Result { let params = ctx.params(); - let backup_dir = std::path::PathBuf::from(¶ms.backup_path); + let backup_target = std::path::PathBuf::from(¶ms.backup_path); let config = ctx.app_ctx.config(); let storage = &config.storage; - ctx.log_info(&format!("Starting full database backup to '{}'", backup_dir.display())); + ctx.log_info(&format!("Starting full database backup to '{}'", backup_target.display())); let storage_backend = ctx.app_ctx.storage_backend(); - let rocksdb_backup_dir = backup_dir.join("rocksdb"); let src_storage = storage.storage_dir(); let src_snapshots = storage.resolved_snapshots_dir(); let src_streams = storage.streams_dir(); let result = tokio::task::spawn_blocking(move || -> Result { + let archive_output = Self::is_archive_path(&backup_target); + let backup_root = if archive_output { + Self::create_archive_staging_dir(&backup_target)? + } else { + backup_target.clone() + }; + + fs::create_dir_all(&backup_root).map_err(|e| { + KalamDbError::InvalidOperation(format!( + "Failed to create backup root '{}': {}", + backup_root.display(), + e + )) + })?; + // 1. RocksDB native backup (hot, consistent, with flush) - storage_backend.backup_to(&rocksdb_backup_dir).map_err(|e| { + storage_backend.backup_to(&backup_root.join("rocksdb")).map_err(|e| { KalamDbError::InvalidOperation(format!("RocksDB backup failed: {}", e)) })?; let mut total_bytes: u64 = 0; // 2. Parquet storage files - total_bytes += Self::copy_dir(&src_storage, &backup_dir.join("storage"))?; + total_bytes += Self::copy_dir(&src_storage, &backup_root.join("storage"))?; // 3. Snapshot files - total_bytes += Self::copy_dir(&src_snapshots, &backup_dir.join("snapshots"))?; + total_bytes += Self::copy_dir(&src_snapshots, &backup_root.join("snapshots"))?; // 4. Stream commit log files - total_bytes += Self::copy_dir(&src_streams, &backup_dir.join("streams"))?; + total_bytes += Self::copy_dir(&src_streams, &backup_root.join("streams"))?; // 5. server.toml (look next to the binary / CWD) let server_toml = Path::new("server.toml"); if server_toml.exists() { - fs::copy(server_toml, backup_dir.join("server.toml")).map_err(|e| { + fs::copy(server_toml, backup_root.join("server.toml")).map_err(|e| { KalamDbError::InvalidOperation(format!("Failed to copy server.toml: {}", e)) })?; total_bytes += server_toml.metadata().map(|m| m.len()).unwrap_or(0); } - Ok(total_bytes) + if archive_output { + Self::create_tar_gz_archive(&backup_root, &backup_target)?; + let archive_size = backup_target.metadata().map(|m| m.len()).map_err(|e| { + KalamDbError::InvalidOperation(format!( + "Failed to inspect archive '{}': {}", + backup_target.display(), + e + )) + })?; + let _ = fs::remove_dir_all(&backup_root); + Ok(archive_size) + } else { + Ok(total_bytes) + } }) .await .map_err(|e| KalamDbError::InvalidOperation(format!("Backup task panicked: {}", e)))?; @@ -199,7 +378,13 @@ impl Default for BackupExecutor { #[cfg(test)] mod tests { + use flate2::read::GzDecoder; + use kalamdb_core::test_helpers::test_app_context_simple; + use tar::Archive; + use tempfile::tempdir; + use super::*; + use crate::executors::{JobContext, JobDecision}; #[test] fn test_executor_properties() { @@ -207,4 +392,65 @@ mod tests { assert_eq!(executor.job_type(), JobType::Backup); assert_eq!(executor.name(), "BackupExecutor"); } + + #[test] + fn test_create_tar_gz_archive_preserves_backup_layout() { + let temp_dir = tempdir().expect("temp dir"); + let source_root = temp_dir.path().join("backup-root"); + fs::create_dir_all(source_root.join("rocksdb")).expect("rocksdb dir"); + fs::create_dir_all(source_root.join("storage/app/messages")).expect("storage dir"); + fs::write(source_root.join("rocksdb/CURRENT"), "manifest").expect("rocksdb file"); + fs::write( + source_root.join("storage/app/messages/part-1.parquet"), + "parquet", + ) + .expect("storage file"); + fs::write(source_root.join("server.toml"), "port = 8080\n").expect("config file"); + + let archive_path = temp_dir.path().join("backup.tar.gz"); + BackupExecutor::create_tar_gz_archive(&source_root, &archive_path) + .expect("archive creation"); + + let extract_root = temp_dir.path().join("extract"); + fs::create_dir_all(&extract_root).expect("extract root"); + let archive_file = File::open(&archive_path).expect("open archive"); + let decoder = GzDecoder::new(archive_file); + let mut archive = Archive::new(decoder); + archive.unpack(&extract_root).expect("unpack archive"); + + assert_eq!( + fs::read_to_string(extract_root.join("rocksdb/CURRENT")).expect("read rocksdb file"), + "manifest" + ); + assert_eq!( + fs::read_to_string(extract_root.join("storage/app/messages/part-1.parquet")) + .expect("read storage file"), + "parquet" + ); + assert_eq!( + fs::read_to_string(extract_root.join("server.toml")).expect("read config file"), + "port = 8080\n" + ); + } + + #[test] + fn test_is_archive_path_accepts_trailing_separator() { + assert!(BackupExecutor::is_archive_path(Path::new("/tmp/kalamdb.tar.gz/"))); + assert!(BackupExecutor::is_archive_path(Path::new("/tmp/kalamdb.tgz/"))); + } + + #[tokio::test] + async fn test_execute_writes_archive_for_archive_path() { + let app_ctx = test_app_context_simple(); + let temp_dir = tempdir().expect("temp dir"); + let archive_path = temp_dir.path().join("kalamdb-backup.tar.gz"); + let params = BackupParams { + backup_path: archive_path.to_string_lossy().to_string(), + }; + let ctx = JobContext::new(app_ctx, "BK-test-archive".to_string(), params); + + let decision = BackupExecutor::new().execute(&ctx).await.expect("backup execute"); + assert!(matches!(decision, JobDecision::Completed { .. })); + assert!(archive_path.is_file(), "expected archive file at {}", archive_path.display()); + } } diff --git a/backend/crates/kalamdb-jobs/src/executors/registry.rs b/backend/crates/kalamdb-jobs/src/executors/registry.rs index f4065a05..1724983f 100644 --- a/backend/crates/kalamdb-jobs/src/executors/registry.rs +++ b/backend/crates/kalamdb-jobs/src/executors/registry.rs @@ -471,8 +471,10 @@ mod tests { use kalamdb_core::test_helpers::test_app_context_simple; use kalamdb_system::JobStatus; use serde::{Deserialize, Serialize}; + use tempfile::tempdir; use super::*; + use crate::executors::backup::BackupExecutor; use crate::executors::JobParams; #[derive(Clone, Serialize, Deserialize)] @@ -597,6 +599,23 @@ mod tests { } } + #[tokio::test] + async fn test_registry_execute_backup_archive_path() { + let app_ctx = test_app_context_simple(); + let temp_dir = tempdir().expect("temp dir"); + let archive_path = temp_dir.path().join("registry-backup.tar.gz"); + + let registry = JobRegistry::new(); + registry.register(Arc::new(BackupExecutor::new())); + + let params = format!(r#"{{"backup_path":"{}"}}"#, archive_path.display()); + let job = make_test_job(JobType::Backup, ¶ms); + let result = registry.execute(app_ctx, &job).await.expect("registry execute backup"); + + assert!(matches!(result, JobDecision::Completed { .. })); + assert!(archive_path.is_file(), "expected archive file at {}", archive_path.display()); + } + #[tokio::test] async fn test_registry_execute_not_found() { let app_ctx = test_app_context_simple(); diff --git a/backend/crates/kalamdb-jobs/src/executors/restore.rs b/backend/crates/kalamdb-jobs/src/executors/restore.rs index 0877f05e..2096cb3d 100644 --- a/backend/crates/kalamdb-jobs/src/executors/restore.rs +++ b/backend/crates/kalamdb-jobs/src/executors/restore.rs @@ -1,10 +1,11 @@ //! Restore Job Executor //! -//! Restores the entire KalamDB database from a backup directory created by the -//! backup job. Uses RocksDB's native `BackupEngine` for the key-value store and -//! a recursive directory copy for Parquet and stream files. +//! Restores the entire KalamDB database from a backup directory or `.tar.gz` +//! / `.tgz` archive created by the backup job. Uses RocksDB's native +//! `BackupEngine` for the key-value store and a recursive directory copy for +//! Parquet and stream files. //! -//! ## Backup Directory Layout Expected +//! ## Backup Layout Expected //! ```text //! / //! rocksdb/ ← RocksDB BackupEngine output @@ -24,19 +25,28 @@ //! ## IMPORTANT //! Restore requires a server restart after completion to reload the restored data. -use std::{fs, path::Path}; +use std::{ + fs::{self, File}, + path::{Component, Path, PathBuf}, +}; use async_trait::async_trait; +use flate2::read::GzDecoder; use kalamdb_core::error::KalamDbError; use kalamdb_system::JobType; use serde::{Deserialize, Serialize}; +use tar::Archive; +use uuid::Uuid; use crate::executors::{JobContext, JobDecision, JobExecutor, JobParams}; /// Typed parameters for full database restore operations #[derive(Debug, Clone, Serialize, Deserialize)] pub struct RestoreParams { - /// Source backup directory (created by BACKUP DATABASE TO '') + /// Source backup path on the server filesystem. + /// + /// The path may be a backup directory or a `.tar.gz` / `.tgz` archive + /// created by `BACKUP DATABASE TO ''`. pub backup_path: String, } @@ -60,6 +70,114 @@ impl RestoreExecutor { Self } + fn is_archive_path(path: &Path) -> bool { + let value = path + .to_string_lossy() + .trim() + .trim_end_matches(['/', '\\']) + .to_ascii_lowercase(); + value.ends_with(".tar.gz") || value.ends_with(".tgz") + } + + fn create_archive_staging_dir(archive_path: &Path) -> Result { + let parent = archive_path.parent().unwrap_or_else(|| Path::new(".")); + fs::create_dir_all(parent).map_err(|e| { + KalamDbError::InvalidOperation(format!( + "Failed to create restore staging parent '{}': {}", + parent.display(), + e + )) + })?; + + let file_name = archive_path + .file_name() + .and_then(|value| value.to_str()) + .unwrap_or("kalamdb-restore"); + let staging_dir = parent.join(format!(".{}.restore-{}", file_name, Uuid::new_v4())); + fs::create_dir_all(&staging_dir).map_err(|e| { + KalamDbError::InvalidOperation(format!( + "Failed to create restore staging directory '{}': {}", + staging_dir.display(), + e + )) + })?; + + Ok(staging_dir) + } + + fn extract_tar_gz_archive(archive_path: &Path, output_dir: &Path) -> Result<(), KalamDbError> { + fs::create_dir_all(output_dir).map_err(|e| { + KalamDbError::InvalidOperation(format!( + "Failed to create restore extraction directory '{}': {}", + output_dir.display(), + e + )) + })?; + + let archive_file = File::open(archive_path).map_err(|e| { + KalamDbError::InvalidOperation(format!( + "Failed to open backup archive '{}': {}", + archive_path.display(), + e + )) + })?; + let decoder = GzDecoder::new(archive_file); + let mut archive = Archive::new(decoder); + + for entry in archive.entries().map_err(|e| { + KalamDbError::InvalidOperation(format!( + "Failed to enumerate backup archive '{}': {}", + archive_path.display(), + e + )) + })? { + let mut entry = entry.map_err(|e| { + KalamDbError::InvalidOperation(format!("Archive entry error: {}", e)) + })?; + + let entry_type = entry.header().entry_type(); + if entry_type.is_symlink() || entry_type.is_hard_link() { + return Err(KalamDbError::InvalidOperation( + "Backup archive cannot contain symlinks".to_string(), + )); + } + + let relative_path = entry.path().map_err(|e| { + KalamDbError::InvalidOperation(format!("Invalid archive path: {}", e)) + })?; + let relative_path = relative_path.into_owned(); + + if relative_path.components().any(|component| { + matches!(component, Component::ParentDir | Component::RootDir | Component::Prefix(_)) + }) { + return Err(KalamDbError::InvalidOperation( + "Backup archive contains an unsafe path".to_string(), + )); + } + + let output_path = output_dir.join(&relative_path); + if let Some(parent) = output_path.parent() { + fs::create_dir_all(parent).map_err(|e| { + KalamDbError::InvalidOperation(format!( + "Failed to create restore path '{}': {}", + parent.display(), + e + )) + })?; + } + + entry.unpack(&output_path).map_err(|e| { + KalamDbError::InvalidOperation(format!( + "Failed to extract '{}' from backup archive: {}", + relative_path.display(), + e + )) + })?; + } + + Ok(()) + } + /// Recursively copy `src` into `dst`, creating `dst` if needed. /// Overwrites existing files. Skips if `src` doesn't exist. fn copy_dir(src: &Path, dst: &Path) -> Result<(), KalamDbError> { @@ -116,21 +234,31 @@ impl JobExecutor for RestoreExecutor { async fn execute(&self, ctx: &JobContext) -> Result { let params = ctx.params(); - let backup_dir = std::path::PathBuf::from(¶ms.backup_path); + let backup_source = std::path::PathBuf::from(¶ms.backup_path); let config = ctx.app_ctx.config(); let storage = &config.storage; - ctx.log_info(&format!("Starting full database restore from '{}'", backup_dir.display())); + ctx.log_info(&format!("Starting full database restore from '{}'", backup_source.display())); let storage_backend = ctx.app_ctx.storage_backend(); - let rocksdb_backup_dir = backup_dir.join("rocksdb"); let dst_storage = storage.storage_dir(); let dst_snapshots = storage.resolved_snapshots_dir(); let dst_streams = storage.streams_dir(); let result = tokio::task::spawn_blocking(move || -> Result<(), KalamDbError> { + let archive_input = Self::is_archive_path(&backup_source); + let restore_root = if archive_input { + let staging_dir = Self::create_archive_staging_dir(&backup_source)?; + Self::extract_tar_gz_archive(&backup_source, &staging_dir)?; + staging_dir + } else { + backup_source.clone() + }; + + let rocksdb_backup_dir = restore_root.join("rocksdb"); + // 1. RocksDB native restore (overwrites current data) if rocksdb_backup_dir.exists() { storage_backend.restore_from(&rocksdb_backup_dir).map_err(|e| { @@ -139,22 +267,26 @@ impl JobExecutor for RestoreExecutor { } // 2. Parquet storage files - Self::copy_dir(&backup_dir.join("storage"), &dst_storage)?; + Self::copy_dir(&restore_root.join("storage"), &dst_storage)?; // 3. Snapshot files - Self::copy_dir(&backup_dir.join("snapshots"), &dst_snapshots)?; + Self::copy_dir(&restore_root.join("snapshots"), &dst_snapshots)?; // 4. Stream commit log files - Self::copy_dir(&backup_dir.join("streams"), &dst_streams)?; + Self::copy_dir(&restore_root.join("streams"), &dst_streams)?; // 5. server.toml (optional) - let backup_toml = backup_dir.join("server.toml"); + let backup_toml = restore_root.join("server.toml"); if backup_toml.exists() { fs::copy(&backup_toml, Path::new("server.toml")).map_err(|e| { KalamDbError::InvalidOperation(format!("Failed to restore server.toml: {}", e)) })?; } + if archive_input { + let _ = fs::remove_dir_all(&restore_root); + } + Ok(()) }) .await @@ -199,6 +331,10 @@ impl Default for RestoreExecutor { #[cfg(test)] mod tests { + use flate2::{write::GzEncoder, Compression}; + use tar::Builder; + use tempfile::tempdir; + use super::*; #[test] @@ -207,4 +343,52 @@ mod tests { assert_eq!(executor.job_type(), JobType::Restore); assert_eq!(executor.name(), "RestoreExecutor"); } + + #[test] + fn test_extract_tar_gz_archive_restores_backup_layout() { + let temp_dir = tempdir().expect("temp dir"); + let source_root = temp_dir.path().join("source-root"); + fs::create_dir_all(source_root.join("rocksdb")).expect("rocksdb dir"); + fs::create_dir_all(source_root.join("storage/app/messages")).expect("storage dir"); + fs::write(source_root.join("rocksdb/CURRENT"), "manifest").expect("rocksdb file"); + fs::write( + source_root.join("storage/app/messages/part-1.parquet"), + "parquet", + ) + .expect("storage file"); + + let archive_path = temp_dir.path().join("restore.tar.gz"); + let archive_file = File::create(&archive_path).expect("archive file"); + let encoder = GzEncoder::new(archive_file, Compression::default()); + let mut builder = Builder::new(encoder); + builder + .append_dir_all("rocksdb", source_root.join("rocksdb")) + .expect("archive rocksdb"); + builder + .append_dir_all("storage", source_root.join("storage")) + .expect("archive storage"); + builder.finish().expect("finish archive"); + let encoder = builder.into_inner().expect("archive encoder"); + encoder.finish().expect("flush archive"); + + let restore_root = temp_dir.path().join("restore-root"); + RestoreExecutor::extract_tar_gz_archive(&archive_path, &restore_root) + .expect("extract archive"); + + assert_eq!( + fs::read_to_string(restore_root.join("rocksdb/CURRENT")).expect("read rocksdb file"), + "manifest" + ); + assert_eq!( + fs::read_to_string(restore_root.join("storage/app/messages/part-1.parquet")) + .expect("read storage file"), + "parquet" + ); + } + + #[test] + fn test_is_archive_path_accepts_trailing_separator() { + assert!(RestoreExecutor::is_archive_path(Path::new("/tmp/kalamdb.tar.gz/"))); + assert!(RestoreExecutor::is_archive_path(Path::new("/tmp/kalamdb.tgz/"))); + } } diff --git a/backend/crates/kalamdb-live/src/notification.rs b/backend/crates/kalamdb-live/src/notification.rs index d854de76..909ddaae 100644 --- a/backend/crates/kalamdb-live/src/notification.rs +++ b/backend/crates/kalamdb-live/src/notification.rs @@ -111,6 +111,16 @@ fn extract_commit_seq(change_notification: &ChangeNotification) -> Option { }) } +fn projection_includes_column(projections: &Option>>, column: &str) -> bool { + match projections { + None => true, + Some(proj) => { + column == SystemColumnNames::SEQ + || proj.iter().any(|candidate| candidate.eq_ignore_ascii_case(column)) + }, + } +} + /// Convert a Row to a projected RowData map (`HashMap`). /// Includes `_seq` always. When `projections` is `None`, includes all columns. fn project_row(row: &Row, projections: &Option>>) -> Result { @@ -119,11 +129,7 @@ fn project_row(row: &Row, projections: &Option>>) -> Result true, - Some(proj) => col == SystemColumnNames::SEQ || proj.iter().any(|p| p == col), - }; - if include { + if projection_includes_column(projections, col) { let cell = scalar_value_to_json(sv) .map_err(|e| LiveError::SerializationError(e.to_string()))?; map.insert(col.clone(), cell); @@ -163,14 +169,13 @@ fn project_update_delta( } } - let owned_keys; - let col_names: Box> = match projections { - None => { - owned_keys = new_row.values.keys().collect::>(); - Box::new(owned_keys.iter().map(|s| s.as_str())) - }, - Some(cols) => Box::new(cols.iter().map(String::as_str)), - }; + let owned_keys = new_row + .values + .keys() + .filter(|column| projection_includes_column(projections, column)) + .cloned() + .collect::>(); + let col_names = owned_keys.iter().map(String::as_str); for col in col_names { if col.starts_with('_') || pk_columns.iter().any(|pk| pk == col) { @@ -968,6 +973,50 @@ mod tests { assert!(rows[0].get(SystemColumnNames::SEQ).is_some()); } + #[tokio::test] + async fn test_notify_async_projection_matches_case_insensitively() { + let registry = ConnectionsManager::new( + NodeId::new(1), + Duration::from_secs(30), + Duration::from_secs(10), + Duration::from_secs(5), + ); + let service = NotificationService::new(Arc::clone(®istry)); + + let user_id = UserId::new("user-proj-upper"); + let table_id = make_table_id("default", "events"); + let conn_id = ConnectionId::new("c-user-upper"); + let live_id = + LiveQueryId::new(user_id.clone(), conn_id.clone(), "sub_upper".to_string()); + + let (tx, mut rx) = mpsc::channel(8); + let flow = Arc::new(SubscriptionFlowControl::new()); + flow.mark_initial_complete(); + + registry.index_subscription( + &user_id, + &conn_id, + live_id, + table_id.clone(), + make_shared_handle("sub_upper", tx, flow, None, Some(vec!["ID"])), + ); + + let change = ChangeNotification::insert(table_id.clone(), make_row(42, "hello", 42)); + service.notify_async(Some(user_id), table_id, change); + + let delivered = tokio::time::timeout(Duration::from_secs(1), rx.recv()) + .await + .expect("projected subscriber gets message within timeout") + .expect("projected subscriber gets message"); + let json: serde_json::Value = serde_json::from_slice(&delivered.to_json()).unwrap(); + let rows = json["rows"].as_array().expect("rows array"); + + assert_eq!(rows.len(), 1); + assert!(rows[0].get("id").is_some()); + assert!(rows[0].get("body").is_none()); + assert!(rows[0].get(SystemColumnNames::SEQ).is_some()); + } + #[tokio::test] async fn test_notify_async_shared_buffers_when_initial_load_incomplete() { let registry = ConnectionsManager::new( diff --git a/backend/tests/integration_tests/topic_pubsub.rs b/backend/tests/integration_tests/topic_pubsub.rs index afe02372..fa643aeb 100644 --- a/backend/tests/integration_tests/topic_pubsub.rs +++ b/backend/tests/integration_tests/topic_pubsub.rs @@ -279,6 +279,28 @@ fn row_offsets(response: &QueryResponse) -> Vec { .collect() } +fn last_row_offset(response: &QueryResponse) -> u64 { + row_offsets(response) + .into_iter() + .max() + .expect("Expected consume response to contain at least one offset row") as u64 +} + +async fn ack_topic_offset(server: &TestServer, topic: &str, group: &str, upto_offset: u64) { + let response = server + .execute_sql(&format!("ACK {} GROUP '{}' UPTO OFFSET {}", topic, group, upto_offset)) + .await; + assert_eq!( + response.status, + ResponseStatus::Success, + "ACK should succeed for topic='{}' group='{}' upto_offset={}: {:?}", + topic, + group, + upto_offset, + response.error + ); +} + async fn assert_topic_offset_state( server: &TestServer, topic: &str, @@ -866,9 +888,13 @@ async fn test_sql_group_consume_resumes_from_committed_offsets_after_cache_clear 1, "Initial consume should return the first message" ); + assert_topic_offset_state(&server, &topic, &group, None).await; + + ack_topic_offset(&server, &topic, &group, last_row_offset(&first_consume)).await; + let first_updated_at = assert_topic_offset_state(&server, &topic, &group, Some(0)) .await - .expect("Expected first grouped consume to auto-ack offset 0"); + .expect("Expected explicit ACK to persist offset 0"); server.app_context.topic_publisher().clear_cache(); @@ -881,11 +907,19 @@ async fn test_sql_group_consume_resumes_from_committed_offsets_after_cache_clear 1, "After clearing in-memory claims, SQL consume should resume from the committed offset" ); + let second_last_offset = last_row_offset(&second_consume); + assert_eq!(second_last_offset, 1); + let before_second_ack = assert_topic_offset_state(&server, &topic, &group, Some(0)) + .await + .expect("Expected committed offset to remain unchanged before explicit ACK"); + + ack_topic_offset(&server, &topic, &group, second_last_offset).await; + let second_updated_at = assert_topic_offset_state(&server, &topic, &group, Some(1)) .await - .expect("Expected second grouped consume to advance auto-ack to offset 1"); + .expect("Expected explicit ACK to advance committed offset to 1"); assert!( - second_updated_at >= first_updated_at, + second_updated_at >= before_second_ack && second_updated_at >= first_updated_at, "Offset update timestamp should move forward or stay equal across rapid commits" ); @@ -968,6 +1002,9 @@ async fn test_sql_group_offsets_are_isolated_per_group() { "New group '{}' should start from the beginning when consuming FROM EARLIEST", group ); + + assert_topic_offset_state(&server, &topic, group, None).await; + ack_topic_offset(&server, &topic, group, last_row_offset(&response)).await; assert_topic_offset_state(&server, &topic, group, Some(*expected_last_acked)).await; } @@ -1008,6 +1045,9 @@ async fn test_sql_group_offsets_are_isolated_per_group() { "Group '{}' should resume from its own committed cursor after cache clear", group ); + + assert_topic_offset_state(&server, &topic, group, Some(*expected_last_acked)).await; + ack_topic_offset(&server, &topic, group, last_row_offset(&resumed)).await; assert_topic_offset_state(&server, &topic, group, Some(*expected_last_acked + 1)).await; } @@ -1083,6 +1123,9 @@ async fn test_sql_group_from_latest_tails_new_messages_and_then_persists_offset( ) .await; assert_eq!(row_offsets(&tail_consume), vec![5, 6]); + + assert_topic_offset_state(&server, &topic, &group, None).await; + ack_topic_offset(&server, &topic, &group, last_row_offset(&tail_consume)).await; assert_topic_offset_state(&server, &topic, &group, Some(6)).await; server.app_context.topic_publisher().clear_cache(); @@ -1132,6 +1175,9 @@ async fn test_sql_group_from_offset_starts_at_requested_offset_and_persists_resu ) .await; assert_eq!(row_offsets(&offset_consume), vec![3, 4]); + + assert_topic_offset_state(&server, &topic, &group, None).await; + ack_topic_offset(&server, &topic, &group, last_row_offset(&offset_consume)).await; assert_topic_offset_state(&server, &topic, &group, Some(4)).await; server.app_context.topic_publisher().clear_cache(); @@ -1143,6 +1189,9 @@ async fn test_sql_group_from_offset_starts_at_requested_offset_and_persists_resu ) .await; assert_eq!(row_offsets(&resumed), vec![5]); + + assert_topic_offset_state(&server, &topic, &group, Some(4)).await; + ack_topic_offset(&server, &topic, &group, last_row_offset(&resumed)).await; assert_topic_offset_state(&server, &topic, &group, Some(5)).await; } diff --git a/backend/tests/testserver/sql/test_user_sql_commands_http.rs b/backend/tests/testserver/sql/test_user_sql_commands_http.rs index 10b54085..df2d5e24 100644 --- a/backend/tests/testserver/sql/test_user_sql_commands_http.rs +++ b/backend/tests/testserver/sql/test_user_sql_commands_http.rs @@ -19,7 +19,7 @@ async fn test_user_sql_commands_over_http() { let admin_auth = server.bearer_auth_header("root")?; // CREATE USER with password - let sql = "CREATE USER 'alice' WITH PASSWORD 'SecurePass123!' ROLE developer EMAIL 'alice@example.com'"; + let sql = "CREATE USER 'alice' WITH PASSWORD 'SecurePass123!' ROLE developer EMAIL 'alice@example.com' STORAGE_MODE region STORAGE_ID 'local'"; let result = server.execute_sql_with_auth(sql, &admin_auth).await?; assert_eq!( result.status, @@ -42,6 +42,8 @@ async fn test_user_sql_commands_over_http() { row.get("email").unwrap().as_str().unwrap(), "alice@example.com" ); + assert_eq!(row.get("storage_mode").unwrap().as_str().unwrap(), "region"); + assert_eq!(row.get("storage_id").unwrap().as_str().unwrap(), "local"); // CREATE USER with OAuth let sql = r#"CREATE USER 'bob' WITH OAUTH '{"provider": "google", "subject": "12345"}' ROLE viewer EMAIL 'bob@example.com'"#; @@ -114,6 +116,33 @@ async fn test_user_sql_commands_over_http() { let role = rows[0].get("role").unwrap().as_str().unwrap(); assert_eq!(role, "dba"); + // ALTER USER SET STORAGE_MODE / STORAGE_ID + let create_sql = "CREATE USER 'storage_test' WITH PASSWORD 'Password123!S' ROLE user STORAGE_MODE table"; + server.execute_sql_with_auth(create_sql, &admin_auth).await?; + + let alter_sql = "ALTER USER 'storage_test' SET STORAGE_MODE region"; + let result = server.execute_sql_with_auth(alter_sql, &admin_auth).await?; + assert_eq!(result.status, ResponseStatus::Success); + + let alter_sql = "ALTER USER 'storage_test' SET STORAGE_ID 'local'"; + let result = server.execute_sql_with_auth(alter_sql, &admin_auth).await?; + assert_eq!(result.status, ResponseStatus::Success); + + let query = "SELECT storage_mode, storage_id FROM system.users WHERE user_id = 'storage_test'"; + let result = server.execute_sql_with_auth(query, &admin_auth).await?; + let rows = result.rows_as_maps(); + let row = &rows[0]; + assert_eq!(row.get("storage_mode").unwrap().as_str().unwrap(), "region"); + assert_eq!(row.get("storage_id").unwrap().as_str().unwrap(), "local"); + + let alter_sql = "ALTER USER 'storage_test' SET STORAGE_ID NULL"; + let result = server.execute_sql_with_auth(alter_sql, &admin_auth).await?; + assert_eq!(result.status, ResponseStatus::Success); + + let result = server.execute_sql_with_auth(query, &admin_auth).await?; + let rows = result.rows_as_maps(); + assert!(rows[0].get("storage_id").unwrap().is_null()); + // DROP USER soft delete let create_sql = "CREATE USER 'frank' WITH PASSWORD 'Password123!F' ROLE user"; server.execute_sql_with_auth(create_sql, &admin_auth).await?; @@ -159,6 +188,13 @@ async fn test_user_sql_commands_over_http() { let result = server.execute_sql_with_auth(alter_sql, &admin_auth).await?; assert_eq!(result.status, ResponseStatus::Error); + // Invalid storage reference + let invalid_storage_sql = "CREATE USER 'bad_storage' WITH PASSWORD 'Password123!H' ROLE user STORAGE_ID 'missing_storage'"; + let result = server + .execute_sql_with_auth(invalid_storage_sql, &admin_auth) + .await?; + assert_eq!(result.status, ResponseStatus::Error); + // DROP USER IF EXISTS let drop_sql = "DROP USER IF EXISTS 'user_that_never_existed'"; let result = server.execute_sql_with_auth(drop_sql, &admin_auth).await?; diff --git a/benchv2/results/bench-2026-05-05-145041-0-4-3-rc-4.html b/benchv2/results/bench-2026-05-05-145041-0-4-3-rc-4.html new file mode 100644 index 00000000..a7fc39c4 --- /dev/null +++ b/benchv2/results/bench-2026-05-05-145041-0-4-3-rc-4.html @@ -0,0 +1,671 @@ + + + + + +KalamDB Benchmark Report + + + + +
+ +
+

KalamDB Benchmark Report

+
Performance analysis — v0.4.3-rc.4
+
+ http://127.0.0.1:8080 + 2026-05-05T14:50:41.736143+00:00 + 100 iters + 5 warmup + 10 concurrency + 100.0K max subs +
+
+ +
+
+
38
+
Total Benchmarks
+
+
+
38
+
Passed
+
+
+
0
+
Failed
+
+
Verdict Mix
33Excellent
4Acceptable
1Slow
successful benchmarks only
+
Compared To Previous
1Faster
29Same
8Slower
against 2026-04-29T19:14:33.905313+00:00
+
+
237.663s
+
Measured Benchmark Time
+
+
+
271.290s
+
Wall Clock Duration
+
+
+ +
+
System Information
+
+
Hostname
Jamals-MacBook-Pro.local
+
Machine Model
MacBookPro18,3
+
CPU Model
Apple M1 Pro
+
CPU Cores
10 logical / 10 physical
+
Total Memory
16.00 GiB
+
Available Memory
10.74 GiB
+
Used Memory
5.26 GiB
+
Memory Usage
32.9%
+
OS
Darwin macOS 26.5
+
Kernel
25.5.0
+
Architecture
aarch64
+
+
+ +
+
+

Latency by Operation (µs)

+ +
+
+

Throughput (ops/sec)

+ +
+
+

Avg Latency by Category (µs)

+ +
+
+ +
Detailed Results
+
+ + + + + + + + + + + + + + + + + + + + + + + + +
StatusBenchmarkCategoryDescriptionItersMeanP50P95P99MinMaxOps/secTotalVerdictvs Prev
PASScreate_tableDDL
CREATE TABLE with 3 columns
100402µs384µs508µs589µs351µs603µs2.5K40.2ms🟢 Excellent~ 406µs prior
PASSdrop_tableDDL
DROP TABLE on a previously created table
1004.73ms4.74ms5.81ms5.89ms3.34ms6.70ms212472.7ms🟢 Excellent~ 4619µs prior
PASSsingle_insertInsert
INSERT a single row into a table
100270µs269µs319µs416µs211µs451µs3.7K27.1ms🟢 Excellent~ 263µs prior
PASSbulk_insertInsert
One INSERT statement with 50 rows (statement-scoped transaction)
100986µs966µs1.16ms1.20ms827µs1.33ms1.0K98.6ms🟢 Excellent~ 992µs prior
PASStransaction_multi_insertInsert
Explicit BEGIN/COMMIT with 50 single-row INSERT statements
1003.16ms3.13ms3.60ms3.98ms2.82ms4.00ms316316.1ms🟡 Acceptable~ 3126µs prior
PASSselect_allSelect
SELECT * from a 200-row table
100796µs774µs889µs1.03ms753µs1.04ms1.3K79.6ms🟢 Excellent~ 782µs prior
PASSselect_by_filterSelect
SELECT with WHERE clause on a 200-row table
100405µs400µs497µs558µs321µs563µs2.5K40.5ms🟢 Excellent~ 397µs prior
PASSselect_countSelect
SELECT COUNT(*) on a 200-row table
100628µs616µs728µs750µs580µs813µs1.6K62.8ms🟢 Excellent↓3% slower
PASSselect_order_by_limitSelect
SELECT with ORDER BY + LIMIT 10 on a 200-row table
100809µs794µs947µs1.12ms761µs1.15ms1.2K80.9ms🟢 Excellent~ 831µs prior
PASSsingle_updateUpdate
UPDATE a single row by filter condition
100907µs889µs1.09ms1.25ms818µs1.32ms1.1K90.7ms🟢 Excellent↓4% slower
PASSsingle_deleteDelete
DELETE a single row by filter condition
1001.32ms1.30ms1.53ms1.63ms1.21ms1.68ms756132.3ms🟢 Excellent↓5% slower
PASSconcurrent_insertConcurrent
N concurrent INSERT operations in parallel (N = concurrency setting)
1001.37ms1.28ms1.68ms1.91ms1.13ms4.86ms729137.1ms🟢 Excellent~ 1344µs prior
PASSconcurrent_selectConcurrent
N concurrent SELECT operations in parallel (N = concurrency setting)
1001.20ms1.21ms1.39ms1.43ms885µs1.47ms833120.0ms🟢 Excellent↓4% slower
PASSpoint_lookupSelect
SELECT by primary key from a 10K-row table (single row lookup)
Baselinephase-0 performanceQuery Classprimary-key lookupDataset10000 seeded rowsQuery ShapeSELECT * FROM <ns>.point_lookup WHERE id = ?
100392µs394µs452µs514µs322µs536µs2.5K39.2ms🟢 Excellent~ 386µs prior
PASSaggregate_querySelect
GROUP BY + SUM/AVG/COUNT on a 10K-row table (analytical query performance)
10028.3ms28.2ms29.8ms31.1ms27.0ms31.7ms352.835s🔴 Slow↓3% slower
PASSmulti_table_joinSelect
SELECT with subquery across two tables (200 customers, 1000 orders)
1003.14ms3.10ms3.45ms3.72ms3.01ms3.83ms318314.3ms🟢 Excellent~ 3076µs prior
PASSlarge_payload_insertInsert
INSERT rows with ~4KB TEXT payloads (serialization + storage throughput)
1001.66ms1.65ms1.88ms2.05ms1.46ms2.14ms604165.7ms🟢 Excellent~ 1656µs prior
PASSwide_column_insertInsert
INSERT into a 20-column table (wide schema overhead)
100919µs890µs1.08ms1.24ms818µs1.26ms1.1K91.9ms🟢 Excellent~ 903µs prior
PASSbulk_deleteDelete
DELETE 100 rows at once with a range filter (bulk deletion)
10069.1ms69.3ms75.5ms76.8ms61.2ms80.5ms146.905s🟡 Acceptable~ 67553µs prior
PASSsequential_crudDML
INSERT → UPDATE → SELECT → DELETE full DML lifecycle per iteration
1002.28ms2.21ms2.72ms3.24ms1.57ms10.3ms439228.0ms🟢 Excellent↓6% slower
PASSalter_tableDDL
ALTER TABLE ADD COLUMN + DROP COLUMN (schema evolution latency)
1001.23ms1.14ms1.59ms2.32ms1.07ms2.45ms812123.2ms🟢 Excellent↓3% slower
PASSconcurrent_updateConcurrent
N concurrent UPDATE operations on the same table (write contention test)
1003.24ms3.21ms4.29ms5.14ms2.12ms5.47ms309324.0ms🟢 Excellent↑4% faster
PASSconcurrent_mixed_dmlConcurrent
Concurrent INSERT + UPDATE + DELETE on the same table (multi-op contention)
1003.27ms3.26ms4.07ms4.39ms2.31ms4.65ms306326.8ms🟢 Excellent~ 3244µs prior
PASSnamespace_isolationConcurrent
Concurrent queries across 5 different namespaces (isolation test)
1001.82ms1.81ms2.12ms2.23ms1.51ms2.41ms551181.7ms🟢 Excellent~ 1822µs prior
PASSsubscribe_initial_loadSubscribe
Subscribe to a 1000-row user table and receive the full initial data batch
1006.41ms6.33ms6.73ms7.12ms6.25ms7.52ms156640.6ms🟢 Excellent~ 6326µs prior
PASSsubscribe_change_latencySubscribe
Latency from INSERT to subscriber receiving the change notification
10051.9ms50.6ms60.2ms75.0ms46.2ms76.1ms195.188s🟡 Acceptable~ 51379µs prior
PASSreconnect_subscribeSubscribe
Disconnect and re-subscribe to a user table (reconnection overhead)
10010.1ms10.3ms11.1ms11.2ms8.82ms11.4ms991.005s🟢 Excellent~ 10342µs prior
PASSflushed_parquet_queryStorage
SELECT from a shared table with 20 flushed Parquet files (200K rows)
100153.2ms184.6ms246.6ms255.1ms24.2ms257.8ms715.325s🟡 Acceptable~ 152145µs prior
PASSconcurrent_subscribersLoad
N WebSocket live-query subscribers receiving changes from concurrent writes
1001.555s1.555s1.577s1.580s1.524s1.622s12.59m🟢 Excellent~ 1556231µs prior
PASSconcurrent_publishersLoad
N concurrent INSERTs into a topic-sourced table (measures publish overhead)
1001.46ms1.41ms1.81ms1.90ms1.26ms2.07ms683146.5ms🟢 Excellent~ 1491µs prior
PASSconcurrent_consumersLoad
N concurrent topic CONSUME calls pulling messages in parallel
100614µs610µs688µs714µs530µs734µs1.6K61.4ms🟢 Excellent~ 618µs prior
PASSsql_1k_concurrentLoad
1000 concurrent SQL SELECT queries at once (RPS degradation test)
Baselinephase-0 performanceQuery Classconcurrent read burstDataset500 seeded rowsBurst1000 concurrent SQL queriesQuery Mixpk lookup, count, selective order-by limit, narrow projection
100152.3ms151.5ms158.0ms161.5ms145.5ms173.2ms715.229s🟢 Excellent~ 152325µs prior
PASScreate_userLoad
CREATE USER (auth subsystem stress test)
1001.24ms1.21ms1.39ms1.51ms1.16ms1.55ms808123.7ms🟢 Excellent~ 1220µs prior
PASSdrop_userLoad
DROP USER (auth subsystem teardown stress test)
100278µs252µs368µs431µs230µs544µs3.6K27.8ms🟢 Excellent~ 273µs prior
PASSconnection_stormLoad
N simultaneous login + SQL + cycles (connection setup overhead)
100275.4ms274.3ms285.0ms300.6ms266.0ms317.7ms427.541s🟢 Excellent~ 275462µs prior
PASSmixed_read_writeLoad
50/50 concurrent reads + writes on same table (contention test)
Baselinephase-0 performanceQuery Classmixed concurrent read/writeDataset200 seeded rowsMix5 reads / 5 writesRead Shapesrange filter, count-star, order-by desc limit
1002.22ms2.20ms2.83ms3.75ms1.45ms5.77ms451221.7ms🟢 Excellent↓5% slower
PASSwide_fanout_queryLoad
N concurrent large-result-set SELECTs (serialization + memory pressure)
1005.95ms5.92ms6.48ms7.13ms5.33ms7.89ms168595.2ms🟢 Excellent~ 5991µs prior
PASSsubscriber_scaleScale
Progressive live-query subscriber scale and insert fanout verification up to 100.0K
Max100.0KTiers10 checkpoints to 100.0KBatch/Wave1.0K / 500Pause/Timeout0ms / 30.0sShared WS1.0K conns @ 100 subs/ws across 1 targetDelivery Checksall tiers to 10.0K + 25.0K/50.0K/100.0K
12.840s2.840s2.840s2.840s2.840s2.840s02.840s🟢 Excellent~ 2855095µs prior
Whole Bench Totals237.663sWall clock 271.290s
+
+ + + +
+ KalamDB v0.4.3-rc.4 — Generated 2026-05-05T14:50:41.736143+00:00 +
+ +
+ + + + diff --git a/benchv2/results/bench-2026-05-05-145041-0-4-3-rc-4.json b/benchv2/results/bench-2026-05-05-145041-0-4-3-rc-4.json new file mode 100644 index 00000000..bfe75e7a --- /dev/null +++ b/benchv2/results/bench-2026-05-05-145041-0-4-3-rc-4.json @@ -0,0 +1,842 @@ +{ + "version": "0.4.3-rc.4", + "server_url": "http://127.0.0.1:8080", + "timestamp": "2026-05-05T14:50:41.735687+00:00", + "config": { + "iterations": 100, + "warmup": 5, + "concurrency": 10, + "namespace": "bench_144609", + "max_subscribers": 100000 + }, + "system": { + "hostname": "Jamals-MacBook-Pro.local", + "machine_model": "MacBookPro18,3", + "os_name": "Darwin", + "os_version": "macOS 26.5", + "kernel_version": "25.5.0", + "architecture": "aarch64", + "cpu_model": "Apple M1 Pro", + "cpu_logical_cores": 10, + "cpu_physical_cores": 10, + "total_memory_bytes": 17179869184, + "available_memory_bytes": 11530731520, + "used_memory_bytes": 5649137664, + "used_memory_percent": 32.88230895996094 + }, + "results": [ + { + "name": "create_table", + "category": "DDL", + "description": "CREATE TABLE with 3 columns", + "full_description": "CREATE TABLE with 3 columns", + "details": [], + "iterations": 100, + "total_us": 40224, + "mean_us": 402.24, + "median_us": 384.0, + "p95_us": 508.05, + "p99_us": 589.14, + "min_us": 351.0, + "max_us": 603.0, + "stddev_us": 50.16057448159094, + "ops_per_sec": 2486.0779634049322, + "success": true, + "error": null + }, + { + "name": "drop_table", + "category": "DDL", + "description": "DROP TABLE on a previously created table", + "full_description": "DROP TABLE on a previously created table", + "details": [], + "iterations": 100, + "total_us": 472671, + "mean_us": 4726.71, + "median_us": 4742.5, + "p95_us": 5809.55, + "p99_us": 5889.140000000004, + "min_us": 3344.0, + "max_us": 6695.0, + "stddev_us": 720.1989525208442, + "ops_per_sec": 211.56364574936902, + "success": true, + "error": null + }, + { + "name": "single_insert", + "category": "Insert", + "description": "INSERT a single row into a table", + "full_description": "INSERT a single row into a table", + "details": [], + "iterations": 100, + "total_us": 27050, + "mean_us": 270.5, + "median_us": 269.0, + "p95_us": 319.2, + "p99_us": 416.3500000000002, + "min_us": 211.0, + "max_us": 451.0, + "stddev_us": 35.43510215374671, + "ops_per_sec": 3696.857670979667, + "success": true, + "error": null + }, + { + "name": "bulk_insert", + "category": "Insert", + "description": "One INSERT statement with 50 rows (statement-scoped transaction)", + "full_description": "One INSERT statement with 50 rows (statement-scoped transaction)", + "details": [], + "iterations": 100, + "total_us": 98563, + "mean_us": 985.63, + "median_us": 966.5, + "p95_us": 1159.65, + "p99_us": 1195.3200000000006, + "min_us": 827.0, + "max_us": 1326.0, + "stddev_us": 95.90469332634459, + "ops_per_sec": 1014.579507523107, + "success": true, + "error": null + }, + { + "name": "transaction_multi_insert", + "category": "Insert", + "description": "Explicit BEGIN/COMMIT with 50 single-row INSERT statements", + "full_description": "Explicit BEGIN/COMMIT with 50 single-row INSERT statements", + "details": [], + "iterations": 100, + "total_us": 316058, + "mean_us": 3160.58, + "median_us": 3131.0, + "p95_us": 3602.2, + "p99_us": 3978.19, + "min_us": 2822.0, + "max_us": 3997.0, + "stddev_us": 232.96035905039486, + "ops_per_sec": 316.39762322105435, + "success": true, + "error": null + }, + { + "name": "select_all", + "category": "Select", + "description": "SELECT * from a 200-row table", + "full_description": "SELECT * from a 200-row table", + "details": [], + "iterations": 100, + "total_us": 79602, + "mean_us": 796.02, + "median_us": 774.0, + "p95_us": 889.3000000000001, + "p99_us": 1034.06, + "min_us": 753.0, + "max_us": 1040.0, + "stddev_us": 55.84610527823292, + "ops_per_sec": 1256.2498429687696, + "success": true, + "error": null + }, + { + "name": "select_by_filter", + "category": "Select", + "description": "SELECT with WHERE clause on a 200-row table", + "full_description": "SELECT with WHERE clause on a 200-row table", + "details": [], + "iterations": 100, + "total_us": 40536, + "mean_us": 405.36, + "median_us": 399.5, + "p95_us": 497.15, + "p99_us": 558.05, + "min_us": 321.0, + "max_us": 563.0, + "stddev_us": 45.44827673945249, + "ops_per_sec": 2466.942964278666, + "success": true, + "error": null + }, + { + "name": "select_count", + "category": "Select", + "description": "SELECT COUNT(*) on a 200-row table", + "full_description": "SELECT COUNT(*) on a 200-row table", + "details": [], + "iterations": 100, + "total_us": 62847, + "mean_us": 628.47, + "median_us": 615.5, + "p95_us": 728.05, + "p99_us": 749.6400000000003, + "min_us": 580.0, + "max_us": 813.0, + "stddev_us": 42.818021080999635, + "ops_per_sec": 1591.1658472162553, + "success": true, + "error": null + }, + { + "name": "select_order_by_limit", + "category": "Select", + "description": "SELECT with ORDER BY + LIMIT 10 on a 200-row table", + "full_description": "SELECT with ORDER BY + LIMIT 10 on a 200-row table", + "details": [], + "iterations": 100, + "total_us": 80940, + "mean_us": 809.4, + "median_us": 794.5, + "p95_us": 946.8999999999999, + "p99_us": 1118.3500000000001, + "min_us": 761.0, + "max_us": 1153.0, + "stddev_us": 65.57623368448682, + "ops_per_sec": 1235.4830738818878, + "success": true, + "error": null + }, + { + "name": "single_update", + "category": "Update", + "description": "UPDATE a single row by filter condition", + "full_description": "UPDATE a single row by filter condition", + "details": [], + "iterations": 100, + "total_us": 90708, + "mean_us": 907.08, + "median_us": 889.0, + "p95_us": 1093.7999999999997, + "p99_us": 1248.7200000000003, + "min_us": 818.0, + "max_us": 1320.0, + "stddev_us": 92.11100900434204, + "ops_per_sec": 1102.4385941703047, + "success": true, + "error": null + }, + { + "name": "single_delete", + "category": "Delete", + "description": "DELETE a single row by filter condition", + "full_description": "DELETE a single row by filter condition", + "details": [], + "iterations": 100, + "total_us": 132325, + "mean_us": 1323.25, + "median_us": 1300.0, + "p95_us": 1529.8, + "p99_us": 1631.5000000000002, + "min_us": 1208.0, + "max_us": 1681.0, + "stddev_us": 99.69381154618405, + "ops_per_sec": 755.7150954090308, + "success": true, + "error": null + }, + { + "name": "concurrent_insert", + "category": "Concurrent", + "description": "N concurrent INSERT operations in parallel (N = concurrency setting)", + "full_description": "N concurrent INSERT operations in parallel (N = concurrency setting)", + "details": [], + "iterations": 100, + "total_us": 137091, + "mean_us": 1370.91, + "median_us": 1285.0, + "p95_us": 1684.3, + "p99_us": 1911.7800000000152, + "min_us": 1134.0, + "max_us": 4860.0, + "stddev_us": 383.49914977583444, + "ops_per_sec": 729.4424871071041, + "success": true, + "error": null + }, + { + "name": "concurrent_select", + "category": "Concurrent", + "description": "N concurrent SELECT operations in parallel (N = concurrency setting)", + "full_description": "N concurrent SELECT operations in parallel (N = concurrency setting)", + "details": [], + "iterations": 100, + "total_us": 120044, + "mean_us": 1200.44, + "median_us": 1210.0, + "p95_us": 1389.2, + "p99_us": 1433.41, + "min_us": 885.0, + "max_us": 1474.0, + "stddev_us": 112.78757364034732, + "ops_per_sec": 833.0278897737496, + "success": true, + "error": null + }, + { + "name": "point_lookup", + "category": "Select", + "description": "SELECT by primary key from a 10K-row table (single row lookup)", + "full_description": "SELECT by primary key from a 10K-row table (single row lookup)", + "details": [ + { + "label": "Baseline", + "value": "phase-0 performance" + }, + { + "label": "Query Class", + "value": "primary-key lookup" + }, + { + "label": "Dataset", + "value": "10000 seeded rows" + }, + { + "label": "Query Shape", + "value": "SELECT * FROM .point_lookup WHERE id = ?" + } + ], + "iterations": 100, + "total_us": 39239, + "mean_us": 392.39, + "median_us": 393.5, + "p95_us": 452.15, + "p99_us": 514.2200000000001, + "min_us": 322.0, + "max_us": 536.0, + "stddev_us": 39.03246382593895, + "ops_per_sec": 2548.4849257116643, + "success": true, + "error": null + }, + { + "name": "aggregate_query", + "category": "Select", + "description": "GROUP BY + SUM/AVG/COUNT on a 10K-row table (analytical query performance)", + "full_description": "GROUP BY + SUM/AVG/COUNT on a 10K-row table (analytical query performance)", + "details": [], + "iterations": 100, + "total_us": 2834569, + "mean_us": 28345.69, + "median_us": 28173.0, + "p95_us": 29822.149999999998, + "p99_us": 31096.97, + "min_us": 26976.0, + "max_us": 31688.0, + "stddev_us": 861.7272611599698, + "ops_per_sec": 35.27873196948107, + "success": true, + "error": null + }, + { + "name": "multi_table_join", + "category": "Select", + "description": "SELECT with subquery across two tables (200 customers, 1000 orders)", + "full_description": "SELECT with subquery across two tables (200 customers, 1000 orders)", + "details": [], + "iterations": 100, + "total_us": 314311, + "mean_us": 3143.11, + "median_us": 3097.5, + "p95_us": 3453.5999999999995, + "p99_us": 3720.1100000000006, + "min_us": 3006.0, + "max_us": 3830.0, + "stddev_us": 153.6981756297686, + "ops_per_sec": 318.15622106766864, + "success": true, + "error": null + }, + { + "name": "large_payload_insert", + "category": "Insert", + "description": "INSERT rows with ~4KB TEXT payloads (serialization + storage throughput)", + "full_description": "INSERT rows with ~4KB TEXT payloads (serialization + storage throughput)", + "details": [], + "iterations": 100, + "total_us": 165667, + "mean_us": 1656.67, + "median_us": 1645.5, + "p95_us": 1884.1000000000001, + "p99_us": 2050.9200000000005, + "min_us": 1461.0, + "max_us": 2142.0, + "stddev_us": 124.09470601361713, + "ops_per_sec": 603.6205158540928, + "success": true, + "error": null + }, + { + "name": "wide_column_insert", + "category": "Insert", + "description": "INSERT into a 20-column table (wide schema overhead)", + "full_description": "INSERT into a 20-column table (wide schema overhead)", + "details": [], + "iterations": 100, + "total_us": 91947, + "mean_us": 919.47, + "median_us": 890.5, + "p95_us": 1082.1499999999999, + "p99_us": 1238.2200000000003, + "min_us": 818.0, + "max_us": 1260.0, + "stddev_us": 86.98060331783951, + "ops_per_sec": 1087.5830641565249, + "success": true, + "error": null + }, + { + "name": "bulk_delete", + "category": "Delete", + "description": "DELETE 100 rows at once with a range filter (bulk deletion)", + "full_description": "DELETE 100 rows at once with a range filter (bulk deletion)", + "details": [], + "iterations": 100, + "total_us": 6905026, + "mean_us": 69050.26, + "median_us": 69288.5, + "p95_us": 75547.3, + "p99_us": 76753.42000000001, + "min_us": 61249.0, + "max_us": 80458.0, + "stddev_us": 4743.810426867899, + "ops_per_sec": 14.482204701329147, + "success": true, + "error": null + }, + { + "name": "sequential_crud", + "category": "DML", + "description": "INSERT → UPDATE → SELECT → DELETE full DML lifecycle per iteration", + "full_description": "INSERT → UPDATE → SELECT → DELETE full DML lifecycle per iteration", + "details": [], + "iterations": 100, + "total_us": 227969, + "mean_us": 2279.69, + "median_us": 2207.5, + "p95_us": 2719.35, + "p99_us": 3235.1200000000363, + "min_us": 1573.0, + "max_us": 10276.0, + "stddev_us": 863.477520368549, + "ops_per_sec": 438.65613307072454, + "success": true, + "error": null + }, + { + "name": "alter_table", + "category": "DDL", + "description": "ALTER TABLE ADD COLUMN + DROP COLUMN (schema evolution latency)", + "full_description": "ALTER TABLE ADD COLUMN + DROP COLUMN (schema evolution latency)", + "details": [], + "iterations": 100, + "total_us": 123153, + "mean_us": 1231.53, + "median_us": 1138.0, + "p95_us": 1586.5499999999997, + "p99_us": 2324.290000000001, + "min_us": 1068.0, + "max_us": 2452.0, + "stddev_us": 239.5923075526087, + "ops_per_sec": 811.9980836845225, + "success": true, + "error": null + }, + { + "name": "concurrent_update", + "category": "Concurrent", + "description": "N concurrent UPDATE operations on the same table (write contention test)", + "full_description": "N concurrent UPDATE operations on the same table (write contention test)", + "details": [], + "iterations": 100, + "total_us": 324019, + "mean_us": 3240.19, + "median_us": 3211.5, + "p95_us": 4293.599999999999, + "p99_us": 5139.360000000002, + "min_us": 2120.0, + "max_us": 5472.0, + "stddev_us": 707.1247613977451, + "ops_per_sec": 308.6238769948676, + "success": true, + "error": null + }, + { + "name": "concurrent_mixed_dml", + "category": "Concurrent", + "description": "Concurrent INSERT + UPDATE + DELETE on the same table (multi-op contention)", + "full_description": "Concurrent INSERT + UPDATE + DELETE on the same table (multi-op contention)", + "details": [], + "iterations": 100, + "total_us": 326783, + "mean_us": 3267.83, + "median_us": 3258.0, + "p95_us": 4069.45, + "p99_us": 4385.660000000001, + "min_us": 2309.0, + "max_us": 4649.0, + "stddev_us": 527.4424101839044, + "ops_per_sec": 306.0134707129808, + "success": true, + "error": null + }, + { + "name": "namespace_isolation", + "category": "Concurrent", + "description": "Concurrent queries across 5 different namespaces (isolation test)", + "full_description": "Concurrent queries across 5 different namespaces (isolation test)", + "details": [], + "iterations": 100, + "total_us": 181652, + "mean_us": 1816.52, + "median_us": 1807.5, + "p95_us": 2123.6, + "p99_us": 2228.840000000001, + "min_us": 1512.0, + "max_us": 2411.0, + "stddev_us": 163.07465477597376, + "ops_per_sec": 550.5031598881377, + "success": true, + "error": null + }, + { + "name": "subscribe_initial_load", + "category": "Subscribe", + "description": "Subscribe to a 1000-row user table and receive the full initial data batch", + "full_description": "Subscribe to a 1000-row user table and receive the full initial data batch", + "details": [], + "iterations": 100, + "total_us": 640588, + "mean_us": 6405.88, + "median_us": 6327.5, + "p95_us": 6727.349999999999, + "p99_us": 7116.040000000002, + "min_us": 6252.0, + "max_us": 7516.0, + "stddev_us": 188.35643623382524, + "ops_per_sec": 156.10657708230562, + "success": true, + "error": null + }, + { + "name": "subscribe_change_latency", + "category": "Subscribe", + "description": "Latency from INSERT to subscriber receiving the change notification", + "full_description": "Latency from INSERT to subscriber receiving the change notification", + "details": [], + "iterations": 100, + "total_us": 5188492, + "mean_us": 51884.92, + "median_us": 50602.0, + "p95_us": 60177.2, + "p99_us": 75003.40000000001, + "min_us": 46229.0, + "max_us": 76132.0, + "stddev_us": 5134.65751846403, + "ops_per_sec": 19.273422797992172, + "success": true, + "error": null + }, + { + "name": "reconnect_subscribe", + "category": "Subscribe", + "description": "Disconnect and re-subscribe to a user table (reconnection overhead)", + "full_description": "Disconnect and re-subscribe to a user table (reconnection overhead)", + "details": [], + "iterations": 100, + "total_us": 1005135, + "mean_us": 10051.35, + "median_us": 10250.5, + "p95_us": 11058.25, + "p99_us": 11180.75, + "min_us": 8819.0, + "max_us": 11354.0, + "stddev_us": 673.4560100516939, + "ops_per_sec": 99.4891233515896, + "success": true, + "error": null + }, + { + "name": "flushed_parquet_query", + "category": "Storage", + "description": "SELECT from a shared table with 20 flushed Parquet files (200K rows)", + "full_description": "SELECT from a shared table with 20 flushed Parquet files (200K rows)", + "details": [], + "iterations": 100, + "total_us": 15324861, + "mean_us": 153248.61, + "median_us": 184603.5, + "p95_us": 246566.30000000002, + "p99_us": 255056.86000000002, + "min_us": 24227.0, + "max_us": 257815.0, + "stddev_us": 89556.73054504095, + "ops_per_sec": 6.525344667073979, + "success": true, + "error": null + }, + { + "name": "concurrent_subscribers", + "category": "Load", + "description": "N WebSocket live-query subscribers receiving changes from concurrent writes", + "full_description": "N WebSocket live-query subscribers receiving changes from concurrent writes", + "details": [], + "iterations": 100, + "total_us": 155484760, + "mean_us": 1554847.6, + "median_us": 1555271.5, + "p95_us": 1576873.55, + "p99_us": 1579660.1800000004, + "min_us": 1523989.0, + "max_us": 1621753.0, + "stddev_us": 18147.870440780473, + "ops_per_sec": 0.6431498495415242, + "success": true, + "error": null + }, + { + "name": "concurrent_publishers", + "category": "Load", + "description": "N concurrent INSERTs into a topic-sourced table (measures publish overhead)", + "full_description": "N concurrent INSERTs into a topic-sourced table (measures publish overhead)", + "details": [], + "iterations": 100, + "total_us": 146474, + "mean_us": 1464.74, + "median_us": 1411.5, + "p95_us": 1807.1, + "p99_us": 1896.750000000001, + "min_us": 1260.0, + "max_us": 2070.0, + "stddev_us": 161.13841343028136, + "ops_per_sec": 682.7150210958941, + "success": true, + "error": null + }, + { + "name": "concurrent_consumers", + "category": "Load", + "description": "N concurrent topic CONSUME calls pulling messages in parallel", + "full_description": "N concurrent topic CONSUME calls pulling messages in parallel", + "details": [], + "iterations": 100, + "total_us": 61448, + "mean_us": 614.48, + "median_us": 609.5, + "p95_us": 688.05, + "p99_us": 714.2000000000002, + "min_us": 530.0, + "max_us": 734.0, + "stddev_us": 44.06617613076813, + "ops_per_sec": 1627.3922666319488, + "success": true, + "error": null + }, + { + "name": "sql_1k_concurrent", + "category": "Load", + "description": "1000 concurrent SQL SELECT queries at once (RPS degradation test)", + "full_description": "1000 concurrent SQL SELECT queries at once (RPS degradation test)", + "details": [ + { + "label": "Baseline", + "value": "phase-0 performance" + }, + { + "label": "Query Class", + "value": "concurrent read burst" + }, + { + "label": "Dataset", + "value": "500 seeded rows" + }, + { + "label": "Burst", + "value": "1000 concurrent SQL queries" + }, + { + "label": "Query Mix", + "value": "pk lookup, count, selective order-by limit, narrow projection" + } + ], + "iterations": 100, + "total_us": 15229428, + "mean_us": 152294.28, + "median_us": 151536.5, + "p95_us": 158048.25, + "p99_us": 161511.46000000005, + "min_us": 145510.0, + "max_us": 173239.0, + "stddev_us": 3578.683977626017, + "ops_per_sec": 6.566234792271909, + "success": true, + "error": null + }, + { + "name": "create_user", + "category": "Load", + "description": "CREATE USER (auth subsystem stress test)", + "full_description": "CREATE USER (auth subsystem stress test)", + "details": [], + "iterations": 100, + "total_us": 123715, + "mean_us": 1237.15, + "median_us": 1212.5, + "p95_us": 1392.15, + "p99_us": 1514.3200000000002, + "min_us": 1161.0, + "max_us": 1546.0, + "stddev_us": 76.0145354255329, + "ops_per_sec": 808.3094208462999, + "success": true, + "error": null + }, + { + "name": "drop_user", + "category": "Load", + "description": "DROP USER (auth subsystem teardown stress test)", + "full_description": "DROP USER (auth subsystem teardown stress test)", + "details": [], + "iterations": 100, + "total_us": 27816, + "mean_us": 278.16, + "median_us": 252.0, + "p95_us": 368.4, + "p99_us": 431.14000000000055, + "min_us": 230.0, + "max_us": 544.0, + "stddev_us": 53.786281154250105, + "ops_per_sec": 3595.0532067874606, + "success": true, + "error": null + }, + { + "name": "connection_storm", + "category": "Load", + "description": "N simultaneous login + SQL + cycles (connection setup overhead)", + "full_description": "N simultaneous login + SQL + cycles (connection setup overhead)", + "details": [], + "iterations": 100, + "total_us": 27540574, + "mean_us": 275405.74, + "median_us": 274268.5, + "p95_us": 284980.55000000005, + "p99_us": 300631.19000000006, + "min_us": 265960.0, + "max_us": 317678.0, + "stddev_us": 7786.12463558143, + "ops_per_sec": 3.6310063835270827, + "success": true, + "error": null + }, + { + "name": "mixed_read_write", + "category": "Load", + "description": "50/50 concurrent reads + writes on same table (contention test)", + "full_description": "50/50 concurrent reads + writes on same table (contention test)", + "details": [ + { + "label": "Baseline", + "value": "phase-0 performance" + }, + { + "label": "Query Class", + "value": "mixed concurrent read/write" + }, + { + "label": "Dataset", + "value": "200 seeded rows" + }, + { + "label": "Mix", + "value": "5 reads / 5 writes" + }, + { + "label": "Read Shapes", + "value": "range filter, count-star, order-by desc limit" + } + ], + "iterations": 100, + "total_us": 221651, + "mean_us": 2216.51, + "median_us": 2201.5, + "p95_us": 2831.1, + "p99_us": 3745.4200000000105, + "min_us": 1452.0, + "max_us": 5767.0, + "stddev_us": 517.9256702452485, + "ops_per_sec": 451.1597060243356, + "success": true, + "error": null + }, + { + "name": "wide_fanout_query", + "category": "Load", + "description": "N concurrent large-result-set SELECTs (serialization + memory pressure)", + "full_description": "N concurrent large-result-set SELECTs (serialization + memory pressure)", + "details": [], + "iterations": 100, + "total_us": 595211, + "mean_us": 5952.11, + "median_us": 5921.5, + "p95_us": 6479.15, + "p99_us": 7131.680000000004, + "min_us": 5334.0, + "max_us": 7892.0, + "stddev_us": 365.1158912460591, + "ops_per_sec": 168.00764770812367, + "success": true, + "error": null + }, + { + "name": "subscriber_scale", + "category": "Scale", + "description": "Progressive live-query subscriber scale and insert fanout verification up to 100.0K", + "full_description": "Progressively ramps cumulative live-query subscribers across tiers 10 -> 100 -> 500 -> 1.0K -> 2.0K -> 5.0K -> 10.0K -> 25.0K -> 50.0K -> 100.0K to verify connection establishment, subscription completion, and INSERT fanout delivery. Delivery probes run at 10 -> 100 -> 500 -> 1.0K -> 2.0K -> 5.0K -> 10.0K -> 25.0K -> 50.0K -> 100.0K. This run is configured with connect_batch=1.0K, wave_size=500, wave_pause=0ms, connect_timeout=30.0s, 1 shared WebSocket target, 100 subscriptions per shared WebSocket connection, and a pooled shared-connection budget of 1.0K.", + "details": [ + { + "label": "Max", + "value": "100.0K" + }, + { + "label": "Tiers", + "value": "10 checkpoints to 100.0K" + }, + { + "label": "Batch/Wave", + "value": "1.0K / 500" + }, + { + "label": "Pause/Timeout", + "value": "0ms / 30.0s" + }, + { + "label": "Shared WS", + "value": "1.0K conns @ 100 subs/ws across 1 target" + }, + { + "label": "Delivery Checks", + "value": "all tiers to 10.0K + 25.0K/50.0K/100.0K" + } + ], + "iterations": 1, + "total_us": 2839613, + "mean_us": 2839613.0, + "median_us": 2839613.0, + "p95_us": 2839613.0, + "p99_us": 2839613.0, + "min_us": 2839613.0, + "max_us": 2839613.0, + "stddev_us": 0.0, + "ops_per_sec": 0.3521606641468397, + "success": true, + "error": null + } + ], + "summary": { + "total_benchmarks": 38, + "passed": 38, + "failed": 0, + "total_duration_ms": 271289.95200000005, + "measured_duration_ms": 237662.76 + } +} \ No newline at end of file diff --git a/cli/README.md b/cli/README.md index 1b97eed4..3d6d4c56 100644 --- a/cli/README.md +++ b/cli/README.md @@ -139,7 +139,6 @@ EXECUTION: -f, --file Execute SQL from file and exit -c, --command Execute SQL command and exit --subscribe Subscribe to a table or live query - --unsubscribe Unsubscribe from a subscription (non-interactive) --list-subscriptions List active subscriptions (non-interactive) OUTPUT: @@ -206,13 +205,12 @@ Special commands starting with backslash (`\`): | `\format ` | Set output format | | `\dt` / `\tables` | List tables | | `\d ` / `\describe
` | Describe a table (`
` or ``) | -| `\as ` | Wrap one statement as `EXECUTE AS USER` | +| `\as ` | Wrap one statement as `EXECUTE AS ''` | | `\stats` / `\metrics` | Show system stats | | `\health` | Check server health | | `\flush` | Execute STORAGE FLUSH ALL | | `\refresh-tables` / `\refresh` | Refresh autocomplete cache | -| `\subscribe ` / `\watch ` / `\live ` | Start live query | -| `\unsubscribe` / `\unwatch` | Cancel live query | +| `\live ` / `\subscribe ` | Start live query (`\subscribe` is an alias) | | `\show-credentials` / `\credentials` | Show stored credentials | | `\update-credentials

` | Update stored credentials | | `\delete-credentials` | Delete stored credentials | @@ -226,7 +224,9 @@ Tips: - Use **Tab** for auto-completion of SQL keywords, namespaces, tables, and columns. - Use **↑/↓** to navigate command history. - Use `\format json` or `\format csv` to switch output formats. -- Use `\subscribe ` for live updates. +- Use `\live ` for live updates. +- Run `BACKUP DATABASE TO '/tmp/kalamdb-backup.tar.gz';` as SQL for server-side backups. +- Run `EXPORT USER DATA;` followed by `SHOW EXPORT;` to fetch your export download URL. ### Output Formats diff --git a/cli/src/args.rs b/cli/src/args.rs index be08cd4f..db9a9721 100644 --- a/cli/src/args.rs +++ b/cli/src/args.rs @@ -218,7 +218,6 @@ pub struct Cli { "list_instances", "subscribe", "list_subscriptions", - "unsubscribe", "consume" ] )] @@ -249,10 +248,6 @@ pub struct Cli { )] pub watch_interval: Duration, - /// Unsubscribe from a subscription - #[arg(long = "unsubscribe")] - pub unsubscribe: Option, - /// List active subscriptions #[arg(long = "list-subscriptions")] pub list_subscriptions: bool, @@ -285,8 +280,10 @@ pub struct Cli { #[cfg(test)] mod tests { - use super::parse_watch_interval; - use std::time::Duration; + use clap::Parser; + + use super::{parse_watch_interval, Cli}; + use std::{path::Path, time::Duration}; #[test] fn parse_watch_interval_defaults_to_seconds() { @@ -309,4 +306,39 @@ mod tests { fn parse_watch_interval_handles_default_five_seconds_literal() { assert_eq!(parse_watch_interval("5s").unwrap(), Duration::from_secs(5)); } + + #[test] + fn short_connection_and_execution_flags_parse() { + let cli = Cli::try_parse_from([ + "kalam", + "-u", + "http://127.0.0.1:8080", + "-c", + "SELECT 1", + "-v", + ]) + .expect("short flags should parse"); + + assert_eq!(cli.url.as_deref(), Some("http://127.0.0.1:8080")); + assert_eq!(cli.command.as_deref(), Some("SELECT 1")); + assert!(cli.verbose); + } + + #[test] + fn short_host_port_and_file_flags_parse() { + let cli = Cli::try_parse_from([ + "kalam", + "-H", + "127.0.0.1", + "-p", + "8080", + "-f", + "./queries.sql", + ]) + .expect("short flags should parse"); + + assert_eq!(cli.host.as_deref(), Some("127.0.0.1")); + assert_eq!(cli.port, 8080); + assert_eq!(cli.file.as_deref(), Some(Path::new("./queries.sql"))); + } } diff --git a/cli/src/commands/subscriptions.rs b/cli/src/commands/subscriptions.rs index b19ed906..e8730a75 100644 --- a/cli/src/commands/subscriptions.rs +++ b/cli/src/commands/subscriptions.rs @@ -12,16 +12,11 @@ fn print_list_subscriptions() { println!(" • No persistent subscription registry is currently implemented"); } -fn print_unsubscribe_message() { - println!("To unsubscribe from an active subscription, use Ctrl+C in the terminal"); - println!("where the subscription is running, or kill the process."); -} - pub async fn handle_subscriptions( cli: &Cli, credential_store: &mut FileCredentialStore, ) -> Result { - if !(cli.list_subscriptions || cli.subscribe.is_some() || cli.unsubscribe.is_some()) { + if !(cli.list_subscriptions || cli.subscribe.is_some()) { return Ok(false); } @@ -30,11 +25,6 @@ pub async fn handle_subscriptions( return Ok(true); } - if cli.unsubscribe.is_some() { - print_unsubscribe_message(); - return Ok(true); - } - // Only subscriptions require a server session. // Load configuration let config = CLIConfiguration::load(&cli.config)?; diff --git a/cli/src/parser.rs b/cli/src/parser.rs index 59ab8425..4ab505cc 100644 --- a/cli/src/parser.rs +++ b/cli/src/parser.rs @@ -6,6 +6,12 @@ use crate::error::{CLIError, Result}; +#[derive(Debug, Clone, PartialEq)] +pub enum FlushTarget { + All, + Table(String), +} + /// Parsed command #[derive(Debug, Clone, PartialEq)] pub enum Command { @@ -21,7 +27,7 @@ pub enum Command { /// Meta-commands (backslash commands) Quit, Help, - Flush, + Flush(FlushTarget), ClusterSnapshot, ClusterPurge { upto: u64, @@ -45,7 +51,6 @@ pub enum Command { Describe(String), SetFormat(String), Subscribe(String), - Unsubscribe, RefreshTables, ShowCredentials, UpdateCredentials { @@ -93,10 +98,7 @@ pub(crate) const META_COMMAND_COMPLETIONS: &[&str] = &[ "\\refresh-tables", "\\refresh", "\\subscribe", - "\\watch", "\\live", - "\\unsubscribe", - "\\unwatch", "\\show-credentials", "\\credentials", "\\update-credentials", @@ -150,7 +152,7 @@ impl CommandParser { "\\help" | "\\?" => Ok(Command::Help), "\\sessions" => Ok(Command::Sessions), "\\stats" | "\\metrics" => Ok(Command::Stats), - "\\flush" => Ok(Command::Flush), + "\\flush" => Self::parse_flush_command(args), "\\cluster" => { if args.is_empty() { Err(CLIError::ParseError( @@ -279,7 +281,6 @@ impl CommandParser { Ok(Command::Subscribe(args.join(" "))) } }, - "\\unsubscribe" | "\\unwatch" => Ok(Command::Unsubscribe), "\\refresh-tables" | "\\refresh" => Ok(Command::RefreshTables), "\\show-credentials" | "\\credentials" => Ok(Command::ShowCredentials), "\\update-credentials" => { @@ -409,6 +410,27 @@ impl CommandParser { sql: sql.to_string(), }) } + + fn parse_flush_command(args: &[&str]) -> Result { + match args { + [] => Ok(Command::Flush(FlushTarget::All)), + [subcommand] if subcommand.eq_ignore_ascii_case("all") => { + Ok(Command::Flush(FlushTarget::All)) + }, + [subcommand, table @ ..] if subcommand.eq_ignore_ascii_case("table") => { + if table.is_empty() { + Err(CLIError::ParseError( + "\\flush table requires a table name".into(), + )) + } else { + Ok(Command::Flush(FlushTarget::Table(table.join(" ")))) + } + }, + _ => Err(CLIError::ParseError( + "\\flush supports: \\flush, \\flush all, or \\flush table

".into(), + )), + } + } } impl Default for CommandParser { @@ -442,6 +464,24 @@ mod tests { assert_eq!(parser.parse("\\?").unwrap(), Command::Help); } + #[test] + fn test_parse_flush_shortcuts() { + let parser = CommandParser::new(); + + assert_eq!(parser.parse("\\flush").unwrap(), Command::Flush(FlushTarget::All)); + assert_eq!(parser.parse("\\flush all").unwrap(), Command::Flush(FlushTarget::All)); + assert_eq!( + parser.parse("\\flush table messages").unwrap(), + Command::Flush(FlushTarget::Table("messages".to_string())) + ); + } + + #[test] + fn test_parse_flush_table_requires_target() { + let parser = CommandParser::new(); + assert!(parser.parse("\\flush table").is_err()); + } + #[test] fn test_parse_describe() { let parser = CommandParser::new(); @@ -523,6 +563,115 @@ mod tests { assert_eq!(parser.parse("\\cluster rebalance").unwrap(), Command::ClusterRebalance); } + #[test] + fn test_parse_meta_command_aliases() { + let parser = CommandParser::new(); + + assert_eq!(parser.parse("\\health").unwrap(), Command::Health); + assert_eq!(parser.parse("\\dt").unwrap(), Command::ListTables); + assert_eq!(parser.parse("\\tables").unwrap(), Command::ListTables); + assert_eq!(parser.parse("\\info").unwrap(), Command::Info); + assert_eq!(parser.parse("\\session").unwrap(), Command::Info); + assert_eq!(parser.parse("\\show-credentials").unwrap(), Command::ShowCredentials); + assert_eq!(parser.parse("\\credentials").unwrap(), Command::ShowCredentials); + assert_eq!(parser.parse("\\delete-credentials").unwrap(), Command::DeleteCredentials); + assert_eq!(parser.parse("\\refresh-tables").unwrap(), Command::RefreshTables); + assert_eq!(parser.parse("\\refresh").unwrap(), Command::RefreshTables); + } + + #[test] + fn test_parse_format_subscribe_and_credential_update_commands() { + let parser = CommandParser::new(); + + assert_eq!( + parser.parse("\\format json").unwrap(), + Command::SetFormat("json".to_string()) + ); + assert_eq!( + parser.parse("\\subscribe SELECT * FROM app.messages").unwrap(), + Command::Subscribe("SELECT * FROM app.messages".to_string()) + ); + assert_eq!( + parser.parse("\\watch SELECT * FROM app.messages").unwrap(), + Command::Subscribe("SELECT * FROM app.messages".to_string()) + ); + assert_eq!( + parser.parse("\\update-credentials admin kalamdb123").unwrap(), + Command::UpdateCredentials { + username: "admin".to_string(), + password: "kalamdb123".to_string(), + } + ); + } + + #[test] + fn test_parse_consume_command_with_all_options() { + let parser = CommandParser::new(); + + assert_eq!( + parser + .parse("\\consume app.events --group workers --from earliest --limit 5 --timeout 12") + .unwrap(), + Command::Consume { + topic: "app.events".to_string(), + group: Some("workers".to_string()), + from: Some("earliest".to_string()), + limit: Some(5), + timeout: Some(12), + } + ); + } + + #[test] + fn test_parse_consume_command_rejects_invalid_arguments() { + let parser = CommandParser::new(); + + assert!(parser.parse("\\consume").is_err()); + assert!(parser.parse("\\consume app.events --limit nope").is_err()); + assert!(parser.parse("\\consume app.events --timeout nope").is_err()); + assert!(parser.parse("\\consume app.events --bogus 1").is_err()); + } + + #[test] + fn test_parse_cluster_command_aliases() { + let parser = CommandParser::new(); + + assert_eq!(parser.parse("\\cluster snapshot").unwrap(), Command::ClusterSnapshot); + assert_eq!( + parser.parse("\\cluster purge --upto 42").unwrap(), + Command::ClusterPurge { upto: 42 } + ); + assert_eq!( + parser.parse("\\cluster purge 42").unwrap(), + Command::ClusterPurge { upto: 42 } + ); + assert_eq!( + parser.parse("\\cluster trigger-election").unwrap(), + Command::ClusterTriggerElection + ); + assert_eq!( + parser.parse("\\cluster trigger election").unwrap(), + Command::ClusterTriggerElection + ); + assert_eq!( + parser.parse("\\cluster transfer-leader 7").unwrap(), + Command::ClusterTransferLeader { node_id: 7 } + ); + assert_eq!( + parser.parse("\\cluster transfer leader 7").unwrap(), + Command::ClusterTransferLeader { node_id: 7 } + ); + assert_eq!(parser.parse("\\cluster stepdown").unwrap(), Command::ClusterStepdown); + assert_eq!(parser.parse("\\cluster step-down").unwrap(), Command::ClusterStepdown); + assert_eq!(parser.parse("\\cluster clear").unwrap(), Command::ClusterClear); + assert_eq!(parser.parse("\\cluster list").unwrap(), Command::ClusterList); + assert_eq!(parser.parse("\\cluster ls").unwrap(), Command::ClusterList); + assert_eq!( + parser.parse("\\cluster list groups").unwrap(), + Command::ClusterListGroups + ); + } + #[test] fn test_parse_cluster_leave_is_rejected() { let parser = CommandParser::new(); diff --git a/cli/src/session.rs b/cli/src/session.rs index 17852f63..d2edf6df 100644 --- a/cli/src/session.rs +++ b/cli/src/session.rs @@ -180,6 +180,9 @@ pub struct CLISession { /// Session is connected connected: bool, + /// Current namespace for unqualified SQL statements + current_namespace: Option, + /// Active subscription paused state subscription_paused: bool, @@ -370,6 +373,7 @@ impl CLISession { format, color, connected, + current_namespace: None, subscription_paused: false, loading_threshold_ms: loading_threshold_ms.unwrap_or(200), animations, @@ -510,9 +514,13 @@ impl CLISession { None }; + let request_namespace = self.request_namespace(); + // Execute the query let result = if upload_parts.is_empty() { - self.client.execute_query(&sql_to_send, None, None, None).await + self.client + .execute_query(&sql_to_send, None, None, request_namespace) + .await } else { let mut parts_for_send = Vec::with_capacity(upload_parts.len()); for part in upload_parts.iter_mut() { @@ -530,7 +538,7 @@ impl CLISession { &sql_to_send, Some(parts_for_send), None, - None, + request_namespace, upload_progress, ) .await @@ -550,11 +558,16 @@ impl CLISession { pub async fn execute(&mut self, sql: &str) -> Result<()> { let start = Instant::now(); let elapsed = start.elapsed(); + let namespace_switch = Self::parse_namespace_switch(sql); let result = self.execute_query_response(sql).await; match result { Ok(response) => { + if let Some(namespace) = namespace_switch { + self.current_namespace = Some(namespace); + } + if let Some((config, server_message)) = Self::extract_subscription_config(&response)? { @@ -655,6 +668,99 @@ impl CLISession { Ok((modified_sql, uploads)) } + fn parse_namespace_switch(sql: &str) -> Option { + let trimmed = sql.trim().trim_end_matches(';').trim(); + if trimmed.is_empty() { + return None; + } + + if let Some(remainder) = Self::strip_ascii_prefix(trimmed, "USE NAMESPACE") { + return Self::parse_namespace_identifier(remainder); + } + + if let Some(remainder) = Self::strip_ascii_prefix(trimmed, "SET NAMESPACE") { + return Self::parse_namespace_identifier(remainder); + } + + let remainder = Self::strip_ascii_prefix(trimmed, "USE")?; + let remainder = remainder.trim(); + if remainder.eq_ignore_ascii_case("NAMESPACE") { + return None; + } + + Self::parse_namespace_identifier(remainder) + } + + fn strip_ascii_prefix<'a>(value: &'a str, prefix: &str) -> Option<&'a str> { + let prefix_len = prefix.len(); + if value.len() < prefix_len || !value[..prefix_len].eq_ignore_ascii_case(prefix) { + return None; + } + + Some(&value[prefix_len..]) + } + + fn parse_namespace_identifier(input: &str) -> Option { + let trimmed = input.trim(); + if trimmed.is_empty() { + return None; + } + + let first = trimmed.chars().next()?; + if matches!(first, '\'' | '"' | '`') { + return Self::parse_quoted_namespace_identifier(trimmed, first); + } + + let end = trimmed + .char_indices() + .find_map(|(index, ch)| (ch.is_whitespace() || ch == ';').then_some(index)) + .unwrap_or(trimmed.len()); + + let namespace = &trimmed[..end]; + if namespace.is_empty() || namespace.contains('.') { + return None; + } + + if !trimmed[end..].trim().is_empty() { + return None; + } + + Some(namespace.to_string()) + } + + fn parse_quoted_namespace_identifier(input: &str, quote: char) -> Option { + let mut namespace = String::new(); + let mut chars = input.char_indices().peekable(); + let (_, opening_quote) = chars.next()?; + if opening_quote != quote { + return None; + } + + while let Some((index, ch)) = chars.next() { + if ch == quote { + if chars.peek().map(|(_, next)| *next == quote).unwrap_or(false) { + namespace.push(quote); + chars.next(); + continue; + } + + if namespace.is_empty() || namespace.contains('.') { + return None; + } + + if !input[index + ch.len_utf8()..].trim().is_empty() { + return None; + } + + return Some(namespace); + } + + namespace.push(ch); + } + + None + } + fn is_file_call_at(sql: &str, idx: usize) -> bool { let bytes = sql.as_bytes(); let needle = b"file"; @@ -900,6 +1006,18 @@ impl CLISession { } } + fn request_namespace(&self) -> Option<&str> { + self.current_namespace.as_deref() + } + + fn effective_namespace(&self) -> &str { + self.request_namespace().unwrap_or("default") + } + + fn current_namespace_label(&self) -> String { + format!("ns:{}", self.effective_namespace()) + } + fn primary_prompt(&self) -> String { // On Windows, rustyline has critical issues with ANSI color codes in prompts // The terminal cannot properly calculate display width, causing cursor misalignment @@ -954,6 +1072,12 @@ impl CLISession { format!("{}@{}", self.username, self.server_host) }; + let namespace = if use_colors_in_prompt { + self.current_namespace_label().magenta().to_string() + } else { + self.current_namespace_label() + }; + let arrow = if use_colors_in_prompt { if use_unicode { "❯".bright_blue().bold().to_string() @@ -964,7 +1088,7 @@ impl CLISession { ">".to_string() }; - let parts = [status, brand_with_profile, identity]; + let parts = [status, brand_with_profile, identity, namespace]; let body = parts.join(" "); format!("{} {} ", body, arrow) } @@ -2767,24 +2891,10 @@ impl CLISession { .unwrap() .as_nanos() ); - let config = SubscriptionConfig::new(sub_id, query); + let config = self.build_subscription_config(query, sub_id)?; self.run_subscription_with_timeout(config, timeout).await } - /// Unsubscribe from active subscription via command line - /// - /// Since subscriptions run in a blocking loop, this method sends a signal - /// to cancel the active subscription. In practice, this would need to be - /// called from a different thread/context than the running subscription. - pub async fn unsubscribe(&mut self, _subscription_id: &str) -> Result<()> { - // For command-line usage, we can't easily interrupt a running subscription - // from the same process. This would require a more complex signaling mechanism. - // For now, inform the user how to cancel subscriptions. - println!("To unsubscribe from an active subscription, use Ctrl+C in the terminal"); - println!("where the subscription is running, or kill the process."); - Ok(()) - } - /// List active subscriptions via command line /// /// Since subscriptions are managed per CLI session and run in blocking mode, @@ -3080,6 +3190,7 @@ mod tests { #[derive(Debug)] struct TestServerState { sql_authorization_headers: Vec, + sql_request_bodies: Vec, refresh_authorization_headers: Vec, health_authorization_headers: Vec, cluster_health_authorization_headers: Vec, @@ -3091,6 +3202,7 @@ mod tests { fn default() -> Self { Self { sql_authorization_headers: Vec::new(), + sql_request_bodies: Vec::new(), refresh_authorization_headers: Vec::new(), health_authorization_headers: Vec::new(), cluster_health_authorization_headers: Vec::new(), @@ -3230,7 +3342,9 @@ mod tests { }, "/v1/api/sql" => { if let Some(header) = authorization.clone() { - state.lock().await.sql_authorization_headers.push(header.clone()); + let mut guard = state.lock().await; + guard.sql_authorization_headers.push(header.clone()); + guard.sql_request_bodies.push(request.body.clone()); match header.as_str() { "Bearer expired-token" => ( "HTTP/1.1 401 Unauthorized", @@ -3311,6 +3425,7 @@ mod tests { struct TestHttpRequest { path: String, headers: HashMap, + body: String, } async fn read_http_request(stream: &mut TcpStream) -> std::io::Result { @@ -3360,7 +3475,13 @@ mod tests { } } - Ok(TestHttpRequest { path, headers }) + let body = String::from_utf8_lossy(&buffer[header_end..]).to_string(); + + Ok(TestHttpRequest { + path, + headers, + body, + }) } fn create_temp_store() -> (FileCredentialStore, TempDir) { @@ -3410,6 +3531,71 @@ mod tests { assert_eq!(table_name, "message logs"); } + #[test] + fn test_parse_namespace_switch_statements() { + assert_eq!( + CLISession::parse_namespace_switch("USE NAMESPACE chat;"), + Some("chat".to_string()) + ); + assert_eq!( + CLISession::parse_namespace_switch("SET NAMESPACE \"team chat\""), + Some("team chat".to_string()) + ); + assert_eq!( + CLISession::parse_namespace_switch("USE billing"), + Some("billing".to_string()) + ); + assert_eq!( + CLISession::parse_namespace_switch("USE NAMESPACE chat; SELECT 1"), + None + ); + } + + #[tokio::test] + #[timeout(5000)] + async fn test_execute_input_applies_current_namespace_to_followup_queries() { + let server = TestServer::spawn().await; + + let mut session = CLISession::with_auth_and_instance( + server.base_url.clone(), + AuthProvider::jwt_token("fresh-token".to_string()), + OutputFormat::Json, + false, + None, + None, + Some("admin".to_string()), + None, + false, + None, + None, + None, + CLIConfiguration::default(), + crate::config::default_config_path(), + false, + ) + .await + .expect("session should initialize"); + + session + .execute_input("USE NAMESPACE chat;") + .await + .expect("use namespace should succeed"); + session + .execute_input("SELECT * FROM messages;") + .await + .expect("follow-up query should succeed"); + + assert_eq!(session.current_namespace.as_deref(), Some("chat")); + + let state = server.state.lock().await; + assert_eq!(state.sql_request_bodies.len(), 2); + assert!( + state.sql_request_bodies[1].contains("\"namespace_id\":\"chat\""), + "expected namespace_id form field in request body: {}", + state.sql_request_bodies[1] + ); + } + #[tokio::test] #[timeout(5000)] async fn test_health_check_does_not_fall_back_to_authenticated_sql() { diff --git a/cli/src/session/commands.rs b/cli/src/session/commands.rs index ffe56771..4f3b3aa6 100644 --- a/cli/src/session/commands.rs +++ b/cli/src/session/commands.rs @@ -6,7 +6,7 @@ use kalam_client::SubscriptionConfig; use super::{CLISession, OutputFormat}; use crate::{ error::{CLIError, Result}, - parser::Command, + parser::{Command, FlushTarget}, }; impl CLISession { @@ -25,9 +25,22 @@ impl CLISession { Command::Help => { self.show_help(); }, - Command::Flush => { - println!("Storage flushing all tables in current namespace..."); - match self.execute("STORAGE FLUSH ALL").await { + Command::Flush(target) => { + let flush_sql = self.build_flush_query(&target)?; + + match &target { + FlushTarget::All => println!( + "Storage flushing all tables in namespace '{}'...", + self.effective_namespace() + ), + FlushTarget::Table(table) => println!( + "Storage flushing table '{}' in namespace context '{}'...", + table, + self.effective_namespace() + ), + } + + match self.execute(&flush_sql).await { Ok(_) => println!("Storage flush completed successfully"), Err(e) => eprintln!("Storage flush failed: {}", e), } @@ -105,7 +118,6 @@ impl CLISession { }, }, Command::Subscribe(query) => { - let (clean_sql, options) = Self::extract_subscribe_options(&query); let sub_id = format!( "sub_{}", std::time::SystemTime::now() @@ -113,13 +125,9 @@ impl CLISession { .unwrap() .as_nanos() ); - let mut config = SubscriptionConfig::new(sub_id, clean_sql); - config.options = options; + let config = self.build_subscription_config(&query, sub_id)?; self.run_subscription(config).await?; }, - Command::Unsubscribe => { - println!("No active subscription to cancel"); - }, Command::RefreshTables => { println!("Table names refreshed"); }, @@ -201,12 +209,155 @@ impl CLISession { } Ok(format!( - "EXECUTE AS USER '{}' ({})", + "EXECUTE AS '{}' ({})", Self::escape_sql_literal(&normalized_user), inner_sql, )) } + fn build_flush_query(&self, target: &FlushTarget) -> Result { + match target { + FlushTarget::All => Ok("STORAGE FLUSH ALL".to_string()), + FlushTarget::Table(table) => { + Self::build_flush_table_query(table, Some(self.effective_namespace())) + }, + } + } + + pub(super) fn build_subscription_config( + &self, + query: &str, + subscription_id: String, + ) -> Result { + let (clean_sql, options) = Self::extract_subscribe_options(query); + let qualified_sql = Self::qualify_subscription_sql(&clean_sql, self.effective_namespace())?; + let mut config = SubscriptionConfig::new(subscription_id, qualified_sql); + config.options = options; + Ok(config) + } + + fn qualify_subscription_sql(sql: &str, default_namespace: &str) -> Result { + let trimmed = sql.trim().trim_end_matches(';').trim(); + if trimmed.is_empty() { + return Err(CLIError::ParseError( + "\\live requires a SELECT query".to_string(), + )); + } + + let Some(from_idx) = Self::find_subscription_from_clause(trimmed) else { + return Ok(trimmed.to_string()); + }; + + let relation_start = trimmed[from_idx..] + .char_indices() + .find_map(|(offset, ch)| (!ch.is_whitespace()).then_some(from_idx + offset)); + let Some(relation_start) = relation_start else { + return Ok(trimmed.to_string()); + }; + + let relation_end = Self::find_subscription_relation_end(trimmed, relation_start); + let relation = trimmed[relation_start..relation_end].trim(); + + let parts = Self::split_identifier_parts(relation).map_err(|_| { + CLIError::ParseError( + "\\live expects SELECT ... FROM
or ".to_string(), + ) + })?; + + if parts.len() != 1 { + return Ok(trimmed.to_string()); + } + + let qualified_relation = format!("{}.{}", Self::quote_identifier(default_namespace), relation); + + Ok(format!( + "{}{}{}", + &trimmed[..relation_start], + qualified_relation, + &trimmed[relation_end..] + )) + } + + fn find_subscription_from_clause(sql: &str) -> Option { + let mut in_single_quotes = false; + let mut in_double_quotes = false; + + for (index, ch) in sql.char_indices() { + match ch { + '\'' if !in_double_quotes => in_single_quotes = !in_single_quotes, + '"' if !in_single_quotes => in_double_quotes = !in_double_quotes, + _ => {}, + } + + if in_single_quotes || in_double_quotes { + continue; + } + + let end = index + 4; + if end > sql.len() || !sql[index..end].eq_ignore_ascii_case("from") { + continue; + } + + let prev = sql[..index].chars().last(); + let next = sql[end..].chars().next(); + let prev_is_ident = prev.is_some_and(Self::is_subscription_identifier_char); + let next_is_ident = next.is_some_and(Self::is_subscription_identifier_char); + + if !prev_is_ident && !next_is_ident { + return Some(end); + } + } + + None + } + + fn find_subscription_relation_end(sql: &str, start: usize) -> usize { + let mut in_double_quotes = false; + + for (offset, ch) in sql[start..].char_indices() { + match ch { + '"' => in_double_quotes = !in_double_quotes, + _ if !in_double_quotes && ch.is_whitespace() => return start + offset, + _ => {}, + } + } + + sql.len() + } + + fn is_subscription_identifier_char(ch: char) -> bool { + ch.is_ascii_alphanumeric() || ch == '_' + } + + fn build_flush_table_query(target: &str, default_namespace: Option<&str>) -> Result { + let trimmed = target.trim().trim_end_matches(';').trim(); + if trimmed.is_empty() { + return Err(CLIError::ParseError( + "\\flush table requires a table name".to_string(), + )); + } + + let parts = Self::split_identifier_parts(trimmed)?; + match parts.as_slice() { + [table_name] => { + let namespace = default_namespace.unwrap_or("default"); + Ok(format!( + "STORAGE FLUSH TABLE {}.{}", + Self::quote_identifier(namespace), + Self::quote_identifier(table_name), + )) + }, + [namespace, table_name] => Ok(format!( + "STORAGE FLUSH TABLE {}.{}", + Self::quote_identifier(namespace), + Self::quote_identifier(table_name), + )), + _ => Err(CLIError::ParseError( + "\\flush table expects
or ".to_string(), + )), + } + } + fn normalize_execute_as_user(user: &str) -> Result { let trimmed = user.trim(); if trimmed.is_empty() { @@ -302,6 +453,10 @@ impl CLISession { value.replace('\'', "''") } + fn quote_identifier(value: &str) -> String { + format!("\"{}\"", value.replace('"', "\"\"")) + } + fn print_help_section(title: &str) { println!("{}", title.yellow().bold()); } @@ -334,12 +489,18 @@ impl CLISession { ("\\health", "Run public health probes"), ("\\dt, \\tables", "List tables"), ("\\d, \\describe
", "Describe a table"), - ("\\as ", "Wrap a statement as EXECUTE AS USER"), + ( + "\\as ", + "Wrap a statement as EXECUTE AS ''", + ), ("\\format ", "Change output format"), ("\\refresh-tables, \\refresh", "Refresh autocomplete caches"), ("\\stats, \\metrics", "Show system stats"), ("\\sessions", "Show active sessions"), - ("\\flush", "Run STORAGE FLUSH ALL"), + ( + "\\flush [all|table
]", + "Run STORAGE FLUSH using the current namespace", + ), ("\\cluster ", "Cluster operations"), ("\\consume ", "Consume topic messages"), ] { @@ -349,10 +510,8 @@ impl CLISession { Self::print_help_section("Live Queries"); for (command, description) in [ - ("\\subscribe ", "Alias of \\subscribe"), - ("\\live ", "Start a live query"), + ("\\subscribe
`, `\describe
` | Describe table | -| `\as ` | Wrap one statement as `EXECUTE AS USER` | +| `\as ` | Wrap one statement as `EXECUTE AS ''` | | `\stats`, `\metrics` | Show `system.stats` | | `\health` | Server healthcheck | | `\flush` | Run `STORAGE FLUSH ALL` | | `\format table|json|csv` | Change output format | -| `\subscribe `, `\watch `, `\live ` | Start live subscription | -| `\unsubscribe`, `\unwatch` | No-op (prints “No active subscription to cancel”) | +| `\live `, `\subscribe ` | Start live subscription (`\subscribe` is an alias) | | `\cluster ...` | Cluster commands (see below) | | `\refresh-tables`, `\refresh` | Refresh autocomplete metadata | | `\sessions` | Show active sessions | @@ -154,6 +152,14 @@ In interactive mode, meta-commands start with `\`: | `\update-credentials

` | Update stored credentials | | `\delete-credentials` | Delete stored credentials | +Backup/export SQL examples you can run directly from the CLI: + +```sql +BACKUP DATABASE TO '/tmp/kalamdb-backup.tar.gz'; +EXPORT USER DATA; +SHOW EXPORT; +``` + ### Cluster meta-commands - `\cluster snapshot` diff --git a/docs/getting-started/quick-start.md b/docs/getting-started/quick-start.md index bbd52c0b..84db437d 100644 --- a/docs/getting-started/quick-start.md +++ b/docs/getting-started/quick-start.md @@ -77,10 +77,10 @@ JSON ## 7. Next steps -### Optional: `EXECUTE AS USER` +### Optional: `EXECUTE AS` Use wrapper syntax only. Ordinary USER-table and STREAM-table access stays -scoped to the authenticated user; explicit `EXECUTE AS USER` follows the role +scoped to the authenticated user; explicit `EXECUTE AS` follows the role hierarchy: system can target any role, DBA can target DBA/service/user, service can target service/user, and regular users can only target themselves. @@ -88,7 +88,7 @@ service can target service/user, and regular users can only target themselves. curl -u root: -X POST http://127.0.0.1:8080/v1/api/sql \ -H 'Content-Type: application/json' \ -d @- <<'JSON' -{"sql":"EXECUTE AS USER 'root' (SELECT * FROM app.messages LIMIT 1);"} +{"sql":"EXECUTE AS 'user_123' (SELECT * FROM app.messages LIMIT 1);"} JSON ``` diff --git a/docs/reference/sql.md b/docs/reference/sql.md index 97835dcc..66feffdc 100644 --- a/docs/reference/sql.md +++ b/docs/reference/sql.md @@ -39,6 +39,10 @@ ALTER NAMESPACE ### USE / SET NAMESPACE +Changes the default namespace for the current request or multi-statement batch. +In the interactive CLI, a successful `USE` also updates the CLI's local +namespace so later requests automatically send `namespace_id`. + ```sql USE ; USE NAMESPACE ; @@ -193,14 +197,14 @@ FROM [.] [LIMIT ]; ``` -## Execute As User +## Execute As -`EXECUTE AS USER` syntax is wrapper-only. It switches USER-table or +`EXECUTE AS` syntax is wrapper-only. It switches USER-table or STREAM-table execution to a target user ID only when the authenticated actor role is allowed to target that ID's cached role class. ```sql -EXECUTE AS USER '' ( +EXECUTE AS '' ( ); ``` @@ -208,7 +212,7 @@ EXECUTE AS USER '' ( Examples: ```sql -EXECUTE AS USER 'alice' ( +EXECUTE AS 'user_123' ( SELECT * FROM app.messages WHERE conversation_id = 42 ); ``` @@ -216,11 +220,11 @@ EXECUTE AS USER 'alice' ( Rules: 1. The wrapper must contain exactly one SQL statement. -2. Username must be single-quoted. +2. The target user ID must be single-quoted. 3. System users may target system, dba, service, and user accounts. 4. DBA users may target dba, service, and user accounts. 5. Service users may target service and user accounts. -6. Regular users may only use self-targeted `EXECUTE AS USER` as a no-op identity boundary. +6. Regular users may only use self-targeted `EXECUTE AS ''` as a no-op identity boundary. 7. The wrapper is valid for USER and STREAM tables; shared tables use their table policy directly. 8. Target role checks are hot-path cached: service, DBA, and system user IDs are tracked in memory from `system.users`; soft-deleted privileged IDs stay classified by their persisted role, and target IDs not present in that privileged cache are treated as regular users. 9. Legacy inline `... AS USER 'name'` syntax is not supported. @@ -233,7 +237,9 @@ Rules: CREATE USER '' WITH ' | OAUTH | INTERNAL> ROLE - [EMAIL '']; + [EMAIL ''] + [STORAGE_MODE ] + [STORAGE_ID '']; ``` ### ALTER USER @@ -242,6 +248,9 @@ CREATE USER '' ALTER USER '' SET PASSWORD ''; ALTER USER '' SET ROLE ; ALTER USER '' SET EMAIL ''; +ALTER USER '' SET STORAGE_MODE ; +ALTER USER '' SET STORAGE_ID ''; +ALTER USER '' SET STORAGE_ID NULL; ``` ### DROP USER @@ -444,27 +453,44 @@ CLUSTER CLEAR; ## Backup / Restore Commands -### BACKUP DATABASE +### EXPORT USER DATA ```sql -BACKUP DATABASE TO ''; -BACKUP DATABASE IF EXISTS TO ''; +EXPORT USER DATA; ``` -### RESTORE DATABASE +### SHOW EXPORT + +```sql +SHOW EXPORT; +``` + +`SHOW EXPORT` returns a `download_url` URI path such as +`/v1/exports//`. Prefix it with your KalamDB server base URL +when downloading the finished ZIP over HTTP. + +### BACKUP DATABASE ```sql -RESTORE DATABASE FROM ''; -RESTORE DATABASE IF NOT EXISTS FROM ''; +BACKUP DATABASE TO ''; ``` -### SHOW BACKUP +`` is a path on the server filesystem. If it ends with `.tar.gz` +or `.tgz`, KalamDB writes a single archive file there. Otherwise it writes the +backup directory layout directly under that path. `BACKUP DATABASE` requires a +DBA or System role. + +### RESTORE DATABASE ```sql -SHOW BACKUP FOR DATABASE ; -SHOW BACKUPS FOR DATABASE ; +RESTORE DATABASE FROM ''; ``` +`` is a path on the server filesystem and may point to either a +backup directory or a `.tar.gz` / `.tgz` archive created by `BACKUP DATABASE`. +The restore job stages the files, and a server restart is required to activate +the restored data. `RESTORE DATABASE` requires a DBA or System role. + ## Built-in Functions (Common) ```sql diff --git a/docs/reference/storage-id-usage.md b/docs/reference/storage-id-usage.md index c0ce6c5e..085d4dc1 100644 --- a/docs/reference/storage-id-usage.md +++ b/docs/reference/storage-id-usage.md @@ -117,19 +117,26 @@ Important rules: - `STORAGE_ID` is still the fallback storage for the table. - User preference fields exist in `system.users` (`storage_mode`, `storage_id`). -## 5) Changing storage per user (current status) +## 5) Changing storage per user -Today, SQL DML against `system.*` tables is blocked, and `ALTER USER` currently supports only: -- `SET PASSWORD` -- `SET ROLE` -- `SET EMAIL` +Use `CREATE USER` or `ALTER USER` to set per-user storage preferences without writing directly to `system.users`: -That means there is currently no public SQL command to change a user's `storage_mode` / `storage_id` directly. +```sql +CREATE USER 'alice' + WITH PASSWORD 'SecurePass123!' + ROLE user + STORAGE_MODE region + STORAGE_ID 's3_eu'; + +ALTER USER 'alice' SET STORAGE_MODE table; +ALTER USER 'alice' SET STORAGE_ID 'local'; +ALTER USER 'alice' SET STORAGE_ID NULL; +``` -Current developer workflow: -- Decide table-level routing with `STORAGE_ID` (+ optional `USE_USER_STORAGE`). -- For user-level storage overrides, use internal/admin backend flows that update user records, not direct SQL updates. -- Track progress and limitations in [docs/development/user-table-storage.md](../development/user-table-storage.md). +Notes: +- `STORAGE_MODE` accepts `table` or `region`. +- `STORAGE_ID` must reference an existing row in `system.storages`. +- `SET STORAGE_ID NULL` clears the stored preference while preserving the current `storage_mode`. ## 6) Common failures diff --git a/link/kalam-client/LICENSE b/link/kalam-client/LICENSE new file mode 100644 index 00000000..af6a5982 --- /dev/null +++ b/link/kalam-client/LICENSE @@ -0,0 +1,3 @@ +Apache-2.0 + +See ../../LICENSE.txt for the full repository license text. \ No newline at end of file diff --git a/link/kalam-client/tests/proxied/topic_consumption_netem.rs b/link/kalam-client/tests/proxied/topic_consumption_netem.rs index be985df5..0b0c35a3 100644 --- a/link/kalam-client/tests/proxied/topic_consumption_netem.rs +++ b/link/kalam-client/tests/proxied/topic_consumption_netem.rs @@ -193,7 +193,7 @@ fn records_with_op<'a>(records: &'a [ConsumerRecord], op: TopicOp) -> Vec<&'a Co } #[tokio::test] -#[ntest::timeout(10000)] +#[ntest::timeout(15000)] async fn test_tokio_netem_topic_consume_fragmented_insert_update_delete_and_commit() { let result = timeout(Duration::from_secs(60), async { let writer = match create_test_client() { @@ -308,7 +308,7 @@ async fn test_tokio_netem_topic_consume_fragmented_insert_update_delete_and_comm } #[tokio::test] -#[ntest::timeout(17000)] +#[ntest::timeout(30000)] async fn test_tokio_netem_topic_bandwidth_collapse_poll_recovers_without_losing_offsets() { let result = timeout(Duration::from_secs(75), async { let writer = match create_test_client() { @@ -393,7 +393,7 @@ async fn test_tokio_netem_topic_bandwidth_collapse_poll_recovers_without_losing_ } #[tokio::test] -#[ntest::timeout(10000)] +#[ntest::timeout(15000)] async fn test_tokio_netem_topic_commit_failure_can_be_retried_without_replay() { let result = timeout(Duration::from_secs(75), async { let writer = match create_test_client() { diff --git a/link/kalam-client/tests/proxied/transport_impairments.rs b/link/kalam-client/tests/proxied/transport_impairments.rs index 089f19da..4f18a20e 100644 --- a/link/kalam-client/tests/proxied/transport_impairments.rs +++ b/link/kalam-client/tests/proxied/transport_impairments.rs @@ -454,7 +454,7 @@ async fn test_proxy_packet_loss_style_stalls_resume_without_replay() { /// Very small write slices fragment WebSocket frames at the transport boundary. /// The client should parse the stream normally and avoid spurious reconnects. #[tokio::test] -#[ntest::timeout(8000)] +#[ntest::timeout(15000)] async fn test_tokio_netem_fragmented_writes_preserve_live_stream() { let result = timeout(Duration::from_secs(45), async { let writer = match create_test_client() { @@ -547,7 +547,7 @@ async fn test_tokio_netem_fragmented_writes_preserve_live_stream() { /// the throttle is removed, reconnect resume must deliver only rows after the /// checkpoint. #[tokio::test] -#[ntest::timeout(16000)] +#[ntest::timeout(30000)] async fn test_tokio_netem_bandwidth_collapse_forces_resume_without_replay() { let result = timeout(Duration::from_secs(75), async { let writer = match create_test_client() { @@ -680,7 +680,7 @@ async fn test_tokio_netem_bandwidth_collapse_forces_resume_without_replay() { /// tokio-netem can fail the transport from inside the I/O adapter instead of /// aborting the proxy task. The client should treat it as a normal disconnect. #[tokio::test] -#[ntest::timeout(10000)] +#[ntest::timeout(15000)] async fn test_tokio_netem_forced_transport_termination_recovers() { let result = timeout(Duration::from_secs(75), async { let writer = match create_test_client() { diff --git a/link/link-common/src/subscription/live_rows_config.rs b/link/link-common/src/subscription/live_rows_config.rs index cc2b517a..d3d92d1c 100644 --- a/link/link-common/src/subscription/live_rows_config.rs +++ b/link/link-common/src/subscription/live_rows_config.rs @@ -25,11 +25,13 @@ impl LiveRowsConfig { continue; } - if normalized.iter().any(|existing| existing == trimmed) { + let normalized_column = trimmed.to_ascii_lowercase(); + + if normalized.iter().any(|existing| existing == &normalized_column) { continue; } - normalized.push(trimmed.to_owned()); + normalized.push(normalized_column); } } diff --git a/link/link-common/src/subscription/live_rows_materializer.rs b/link/link-common/src/subscription/live_rows_materializer.rs index c89a2002..af00bbd6 100644 --- a/link/link-common/src/subscription/live_rows_materializer.rs +++ b/link/link-common/src/subscription/live_rows_materializer.rs @@ -137,9 +137,17 @@ impl LiveRowsMaterializer { } } +fn row_value_by_column<'a>(row: &'a RowData, column: &str) -> Option<&'a KalamCellValue> { + row.get(column).or_else(|| { + row.iter() + .find(|(key, _)| key.eq_ignore_ascii_case(column)) + .map(|(_, value)| value) + }) +} + fn rows_match_on_key_columns(left: &RowData, right: &RowData, key_columns: &[String]) -> bool { for column in key_columns { - match (left.get(column), right.get(column)) { + match (row_value_by_column(left, column), row_value_by_column(right, column)) { (Some(KalamCellValue(left_value)), Some(KalamCellValue(right_value))) if left_value == right_value => {}, _ => return false, @@ -327,6 +335,40 @@ mod tests { } } + #[test] + fn matches_key_columns_case_insensitively() { + fn row_with_key(column_name: &str, id: &str, value: &str) -> RowData { + let mut row = RowData::new(); + row.insert(column_name.to_string(), KalamCellValue::text(id)); + row.insert("value".to_string(), KalamCellValue::text(value)); + row + } + + let mut materializer = LiveRowsMaterializer::new(LiveRowsConfig::default()); + + let _ = materializer.apply(ChangeEvent::InitialDataBatch { + subscription_id: "sub-4".to_string(), + rows: vec![row_with_key("ID", "1", "one")], + batch_control: batch_control(BatchStatus::Ready), + }); + + let updated = materializer + .apply(ChangeEvent::Update { + subscription_id: "sub-4".to_string(), + rows: vec![row_with_key("id", "1", "updated")], + old_rows: vec![row_with_key("ID", "1", "one")], + }) + .expect("update should emit"); + + match updated { + LiveRowsEvent::Rows { rows, .. } => { + assert_eq!(rows.len(), 1); + assert_eq!(rows[0].get("value").and_then(KalamCellValue::as_text), Some("updated")); + }, + other => panic!("unexpected event: {:?}", other), + } + } + #[test] fn emits_empty_snapshot_for_empty_ready_ack() { let mut materializer = LiveRowsMaterializer::new(LiveRowsConfig::default()); diff --git a/link/sdks/typescript/consumer/package-lock.json b/link/sdks/typescript/consumer/package-lock.json index fd7ddc03..962d842d 100644 --- a/link/sdks/typescript/consumer/package-lock.json +++ b/link/sdks/typescript/consumer/package-lock.json @@ -1,12 +1,12 @@ { "name": "@kalamdb/consumer", - "version": "0.4.2-rc.3", + "version": "0.4.3-rc.4", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@kalamdb/consumer", - "version": "0.4.2-rc.3", + "version": "0.4.3-rc.4", "license": "Apache-2.0", "devDependencies": { "@types/node": "^25.5.0", diff --git a/link/sdks/typescript/orm/package-lock.json b/link/sdks/typescript/orm/package-lock.json index a39752ff..a33255af 100644 --- a/link/sdks/typescript/orm/package-lock.json +++ b/link/sdks/typescript/orm/package-lock.json @@ -15,7 +15,7 @@ "@kalamdb/client": "file:../client", "@kalamdb/consumer": "file:../consumer", "@types/node": "^25.5.2", - "drizzle-orm": "^0.41.0", + "drizzle-orm": "^0.45.2", "typescript": "^5.8.0" }, "engines": { @@ -23,7 +23,7 @@ }, "peerDependencies": { "@kalamdb/client": ">=0.4.0", - "drizzle-orm": ">=0.36.0" + "drizzle-orm": ">=0.45.2" } }, "../client": { @@ -77,9 +77,9 @@ } }, "node_modules/drizzle-orm": { - "version": "0.41.0", - "resolved": "https://registry.npmjs.org/drizzle-orm/-/drizzle-orm-0.41.0.tgz", - "integrity": "sha512-7A4ZxhHk9gdlXmTdPj/lREtP+3u8KvZ4yEN6MYVxBzZGex5Wtdc+CWSbu7btgF6TB0N+MNPrvW7RKBbxJchs/Q==", + "version": "0.45.2", + "resolved": "https://registry.npmjs.org/drizzle-orm/-/drizzle-orm-0.45.2.tgz", + "integrity": "sha512-kY0BSaTNYWnoDMVoyY8uxmyHjpJW1geOmBMdSSicKo9CIIWkSxMIj2rkeSR51b8KAPB7m+qysjuHme5nKP+E5Q==", "dev": true, "license": "Apache-2.0", "peerDependencies": { @@ -91,12 +91,13 @@ "@neondatabase/serverless": ">=0.10.0", "@op-engineering/op-sqlite": ">=2", "@opentelemetry/api": "^1.4.1", - "@planetscale/database": ">=1", + "@planetscale/database": ">=1.13", "@prisma/client": "*", "@tidbcloud/serverless": "*", "@types/better-sqlite3": "*", "@types/pg": "*", "@types/sql.js": "*", + "@upstash/redis": ">=1.34.7", "@vercel/postgres": ">=0.8.0", "@xata.io/client": "*", "better-sqlite3": ">=7", @@ -154,6 +155,9 @@ "@types/sql.js": { "optional": true }, + "@upstash/redis": { + "optional": true + }, "@vercel/postgres": { "optional": true }, diff --git a/link/sdks/typescript/orm/package.json b/link/sdks/typescript/orm/package.json index 1313d622..721e7362 100644 --- a/link/sdks/typescript/orm/package.json +++ b/link/sdks/typescript/orm/package.json @@ -42,13 +42,13 @@ }, "peerDependencies": { "@kalamdb/client": ">=0.4.0", - "drizzle-orm": ">=0.36.0" + "drizzle-orm": ">=0.45.2" }, "devDependencies": { "@kalamdb/client": "file:../client", "@kalamdb/consumer": "file:../consumer", "@types/node": "^25.5.2", - "drizzle-orm": "^0.41.0", + "drizzle-orm": "^0.45.2", "typescript": "^5.8.0" }, "keywords": [ diff --git a/ui/index.html b/ui/index.html index 74036318..4feace35 100644 --- a/ui/index.html +++ b/ui/index.html @@ -8,7 +8,6 @@

- diff --git a/ui/package-lock.json b/ui/package-lock.json index 6d42ddd9..f7ebc7e6 100644 --- a/ui/package-lock.json +++ b/ui/package-lock.json @@ -87,7 +87,7 @@ "@kalamdb/client": "file:../client", "@kalamdb/consumer": "file:../consumer", "@types/node": "^25.5.2", - "drizzle-orm": "^0.41.0", + "drizzle-orm": "^0.45.2", "typescript": "^5.8.0" }, "engines": { @@ -95,7 +95,7 @@ }, "peerDependencies": { "@kalamdb/client": ">=0.4.0", - "drizzle-orm": ">=0.36.0" + "drizzle-orm": ">=0.45.2" } }, "node_modules/@adobe/css-tools": { diff --git a/ui/src/components/sql-preview/SqlPreviewDialog.tsx b/ui/src/components/sql-preview/SqlPreviewDialog.tsx index d45478e1..ba030e1e 100644 --- a/ui/src/components/sql-preview/SqlPreviewDialog.tsx +++ b/ui/src/components/sql-preview/SqlPreviewDialog.tsx @@ -396,7 +396,7 @@ export function SqlPreviewDialog({ open, options, onClose }: SqlPreviewDialogPro return ( { if (!isOpen) handleCancel(); }}> - + {options?.title ?? 'SQL Preview'} diff --git a/ui/src/components/sql-studio-v2/preview/logs/StudioExecutionLog.test.tsx b/ui/src/components/sql-studio-v2/preview/logs/StudioExecutionLog.test.tsx index 4cb422c6..cb897adb 100644 --- a/ui/src/components/sql-studio-v2/preview/logs/StudioExecutionLog.test.tsx +++ b/ui/src/components/sql-studio-v2/preview/logs/StudioExecutionLog.test.tsx @@ -79,5 +79,6 @@ describe("StudioExecutionLog", () => { fireEvent.click(timelineButtons[0]); expect(within(details as HTMLElement).getByText(/select id, name from default.events/i)).toBeTruthy(); + expect(screen.queryByRole("dialog")).toBeNull(); }); }); \ No newline at end of file diff --git a/ui/src/components/sql-studio-v2/preview/logs/StudioExecutionLog.tsx b/ui/src/components/sql-studio-v2/preview/logs/StudioExecutionLog.tsx index 4aef80f2..eaf49aa2 100644 --- a/ui/src/components/sql-studio-v2/preview/logs/StudioExecutionLog.tsx +++ b/ui/src/components/sql-studio-v2/preview/logs/StudioExecutionLog.tsx @@ -3,13 +3,6 @@ import type { LucideIcon } from "lucide-react"; import { AlertCircle, ArrowDown, ArrowUp, CheckCircle2, Info } from "lucide-react"; import { Badge } from "@/components/ui/badge"; import { CodeBlock } from "@/components/ui/code-block"; -import { - Dialog, - DialogContent, - DialogDescription, - DialogHeader, - DialogTitle, -} from "@/components/ui/dialog"; import { ScrollArea } from "@/components/ui/scroll-area"; import { cn } from "@/lib/utils"; import type { QueryLogEntry } from "../../shared/types"; @@ -225,11 +218,9 @@ function describeFlow(kind: LogEntryKind): string { export function StudioExecutionLog({ logs, status }: StudioExecutionLogProps) { const [selectedLogId, setSelectedLogId] = useState(null); const previousLastLogIdRef = useRef(null); - const [detailModalId, setDetailModalId] = useState(null); const handleSelectLog = (id: string) => { setSelectedLogId(id); - setDetailModalId(id); }; const decoratedLogs = useMemo( @@ -392,35 +383,6 @@ export function StudioExecutionLog({ logs, status }: StudioExecutionLogProps) { - - {(() => { - const modalEntry = detailModalId - ? decoratedLogs.find((item) => item.entry.id === detailModalId) - : null; - return ( - !o && setDetailModalId(null)}> - - - - {modalEntry?.label} • {modalEntry?.title} - - - Full payload for the selected event. - - -
- {modalEntry && ( - - )} -
-
-
- ); - })()} ); } diff --git a/ui/src/components/storages/StorageForm.tsx b/ui/src/components/storages/StorageForm.tsx index b3cf803f..09580386 100644 --- a/ui/src/components/storages/StorageForm.tsx +++ b/ui/src/components/storages/StorageForm.tsx @@ -2,6 +2,7 @@ import { useEffect, useMemo, useState } from "react"; import { AlertCircle, Cloud, Database, HardDrive, Loader2 } from "lucide-react"; import { useCreateStorageMutation, useUpdateStorageMutation } from "@/store/apiSlice"; import type { Storage } from "@/services/storageService"; +import { getErrorMessage } from "@/lib/errors"; import { Alert, AlertDescription, AlertTitle } from "@/components/ui/alert"; import { Button } from "@/components/ui/button"; import { Input } from "@/components/ui/input"; @@ -202,7 +203,7 @@ export function StorageForm({ open, onOpenChange, storage, onSuccess }: StorageF onSuccess(); onOpenChange(false); } catch (submitError) { - setError(submitError instanceof Error ? submitError.message : "Failed to save storage"); + setError(getErrorMessage(submitError, "Failed to save storage")); } finally { setIsSubmitting(false); } diff --git a/ui/src/components/ui/dialog.tsx b/ui/src/components/ui/dialog.tsx index 63de5143..2e0a1070 100644 --- a/ui/src/components/ui/dialog.tsx +++ b/ui/src/components/ui/dialog.tsx @@ -61,7 +61,7 @@ function DialogContent({ & { size?: "sm" | "default" }) { + const isSmall = size === "sm" + return ( ) diff --git a/ui/src/components/users/DeleteUserDialog.tsx b/ui/src/components/users/DeleteUserDialog.tsx index 2fb47654..e0a0fd9e 100644 --- a/ui/src/components/users/DeleteUserDialog.tsx +++ b/ui/src/components/users/DeleteUserDialog.tsx @@ -8,6 +8,7 @@ import { DialogTitle, } from '@/components/ui/dialog'; import { Button } from '@/components/ui/button'; +import { getErrorMessage } from '@/lib/errors'; import type { User } from '@/services/userService'; import { Loader2, AlertTriangle } from 'lucide-react'; @@ -34,7 +35,7 @@ export function DeleteUserDialog({ await onConfirm(); onOpenChange(false); } catch (err) { - setError(err instanceof Error ? err.message : 'Failed to delete user'); + setError(getErrorMessage(err, 'Failed to delete user')); } finally { setIsDeleting(false); } diff --git a/ui/src/components/users/UserForm.tsx b/ui/src/components/users/UserForm.tsx index f1a90898..3d75c2a7 100644 --- a/ui/src/components/users/UserForm.tsx +++ b/ui/src/components/users/UserForm.tsx @@ -3,6 +3,7 @@ import { AlertCircle, Loader2 } from "lucide-react"; import { useCreateUserMutation, useGetStoragesQuery, useUpdateUserMutation } from "@/store/apiSlice"; import type { User } from "@/services/userService"; import { formatTimestamp } from "@/lib/formatters"; +import { getErrorMessage } from "@/lib/errors"; import { Alert, AlertDescription, AlertTitle } from "@/components/ui/alert"; import { Button } from "@/components/ui/button"; import { Input } from "@/components/ui/input"; @@ -189,7 +190,6 @@ export function UserForm({ open, onOpenChange, user, onSuccess }: UserFormProps) if (normalizedStorageId !== (user.storage_id ?? null)) { updateInput.storage_id = normalizedStorageId; } - await updateUserMutation({ username: user.user_id, input: updateInput }).unwrap(); } else { await createUserMutation({ @@ -207,7 +207,7 @@ export function UserForm({ open, onOpenChange, user, onSuccess }: UserFormProps) onSuccess(); onOpenChange(false); } catch (submitError) { - setError(submitError instanceof Error ? submitError.message : "Failed to save user"); + setError(getErrorMessage(submitError, "Failed to save user")); } finally { setIsSubmitting(false); } diff --git a/ui/src/lib/errors.test.ts b/ui/src/lib/errors.test.ts new file mode 100644 index 00000000..458e2637 --- /dev/null +++ b/ui/src/lib/errors.test.ts @@ -0,0 +1,31 @@ +import { describe, expect, it } from "vitest"; +import { getErrorMessage } from "@/lib/errors"; + +describe("getErrorMessage", () => { + it("returns the RTK custom error message used by unwrap rejections", () => { + expect( + getErrorMessage( + { status: "CUSTOM_ERROR", error: "Statement 1 failed: Invalid operation" }, + "fallback", + ), + ).toBe("Statement 1 failed: Invalid operation"); + }); + + it("includes nested backend details when present", () => { + expect( + getErrorMessage( + { + data: { + error: { + message: "Statement 1 failed: Invalid operation", + details: "UPDATE system.users SET storage_mode = 'table'", + }, + }, + }, + "fallback", + ), + ).toBe( + "Statement 1 failed: Invalid operation\nUPDATE system.users SET storage_mode = 'table'", + ); + }); +}); \ No newline at end of file diff --git a/ui/src/lib/errors.ts b/ui/src/lib/errors.ts new file mode 100644 index 00000000..b9c1b4ce --- /dev/null +++ b/ui/src/lib/errors.ts @@ -0,0 +1,65 @@ +interface CustomQueryError { + status?: string; + error?: unknown; + data?: unknown; + message?: unknown; +} + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null; +} + +function appendDetails(message: string, details: unknown): string { + if (typeof details !== "string") { + return message; + } + + const trimmedDetails = details.trim(); + if (!trimmedDetails || message.includes(trimmedDetails)) { + return message; + } + + return `${message}\n${trimmedDetails}`; +} + +export function getErrorMessage(error: unknown, fallback: string): string { + if (typeof error === "string") { + return error; + } + + if (error instanceof Error) { + return error.message || fallback; + } + + if (isRecord(error)) { + const customError = error as CustomQueryError; + + if (typeof customError.error === "string") { + return customError.error; + } + + if (isRecord(customError.error)) { + const nestedError = customError.error as Record; + if (typeof nestedError.message === "string") { + return appendDetails(nestedError.message, nestedError.details); + } + } + + if (isRecord(customError.data)) { + const data = customError.data as Record; + if (typeof data.message === "string") { + return appendDetails(data.message, data.details); + } + if (isRecord(data.error) && typeof data.error.message === "string") { + const nestedError = data.error as Record; + return appendDetails(nestedError.message as string, nestedError.details); + } + } + + if (typeof customError.message === "string") { + return customError.message; + } + } + + return fallback; +} \ No newline at end of file diff --git a/ui/src/services/sql/queries/userQueries.ts b/ui/src/services/sql/queries/userQueries.ts index 69bdfec0..1dfd7341 100644 --- a/ui/src/services/sql/queries/userQueries.ts +++ b/ui/src/services/sql/queries/userQueries.ts @@ -46,6 +46,12 @@ export function buildCreateUserSql(input: CreateUserInput): string { if (input.email?.trim()) { sql += ` EMAIL '${escapeSqlLiteral(input.email.trim())}'`; } + if (input.storage_mode) { + sql += ` STORAGE_MODE '${escapeSqlLiteral(input.storage_mode)}'`; + } + if (input.storage_id?.trim()) { + sql += ` STORAGE_ID '${escapeSqlLiteral(input.storage_id.trim())}'`; + } return sql; } @@ -61,35 +67,22 @@ export function buildUpdateUserEmailSql(username: string, email: string): string return `ALTER USER '${escapeSqlLiteral(username)}' SET EMAIL '${escapeSqlLiteral(email)}'`; } -export function buildUpdateUserStorageSql( +export function buildUpdateUserStorageModeSql( username: string, - storageMode: "table" | "region" | null | undefined, - storageId: string | null | undefined, -): string | null { - const setClauses: string[] = []; - - if (storageMode !== undefined) { - setClauses.push( - storageMode === null - ? "storage_mode = NULL" - : `storage_mode = '${escapeSqlLiteral(storageMode)}'`, - ); - } - - if (storageId !== undefined) { - const normalizedStorageId = storageId?.trim() ?? ""; - setClauses.push( - normalizedStorageId.length === 0 - ? "storage_id = NULL" - : `storage_id = '${escapeSqlLiteral(normalizedStorageId)}'`, - ); - } + storageMode: "table" | "region", +): string { + return `ALTER USER '${escapeSqlLiteral(username)}' SET STORAGE_MODE '${escapeSqlLiteral(storageMode)}'`; +} - if (setClauses.length === 0) { - return null; +export function buildUpdateUserStorageIdSql( + username: string, + storageId: string | null, +): string { + if (storageId === null) { + return `ALTER USER '${escapeSqlLiteral(username)}' SET STORAGE_ID NULL`; } - return `UPDATE system.users SET ${setClauses.join(", ")} WHERE user_id = '${escapeSqlLiteral(username)}'`; + return `ALTER USER '${escapeSqlLiteral(username)}' SET STORAGE_ID '${escapeSqlLiteral(storageId.trim())}'`; } export function buildDeleteUserSql(username: string): string { diff --git a/ui/src/services/userService.test.ts b/ui/src/services/userService.test.ts index 787993fc..11a95023 100644 --- a/ui/src/services/userService.test.ts +++ b/ui/src/services/userService.test.ts @@ -1,5 +1,16 @@ -import { describe, expect, it } from "vitest"; +import { afterEach, describe, expect, it, vi } from "vitest"; import type { User } from "@/services/userService"; +import { createUser, updateUser } from "@/services/userService"; + +const executeSqlMock = vi.fn(); + +vi.mock("@/lib/kalam-client", () => ({ + executeSql: (sql: string) => executeSqlMock(sql), +})); + +afterEach(() => { + executeSqlMock.mockReset(); +}); describe("User type", () => { it("uses the schema-backed user_id field directly", () => { @@ -22,4 +33,47 @@ describe("User type", () => { expect(user.user_id).toBe("root"); }); + + it("creates users without issuing a direct system.users update", async () => { + executeSqlMock.mockResolvedValue([]); + + await createUser({ + username: "test2", + password: "Password123!", + role: "user", + storage_mode: "table", + storage_id: "local", + }); + + expect(executeSqlMock).toHaveBeenCalledTimes(1); + expect(executeSqlMock).toHaveBeenCalledWith( + "CREATE USER 'test2' WITH PASSWORD 'Password123!' ROLE 'user' STORAGE_MODE 'table' STORAGE_ID 'local'", + ); + }); + + it("updates supported fields with alter-user storage statements", async () => { + executeSqlMock.mockResolvedValue([]); + + await updateUser("test2", { + role: "dba", + storage_mode: "region", + storage_id: "archive", + }); + + expect(executeSqlMock).toHaveBeenCalledTimes(3); + expect(executeSqlMock).toHaveBeenNthCalledWith(1, "ALTER USER 'test2' SET ROLE 'dba'"); + expect(executeSqlMock).toHaveBeenNthCalledWith(2, "ALTER USER 'test2' SET STORAGE_MODE 'region'"); + expect(executeSqlMock).toHaveBeenNthCalledWith(3, "ALTER USER 'test2' SET STORAGE_ID 'archive'"); + }); + + it("can clear a user storage id via alter user", async () => { + executeSqlMock.mockResolvedValue([]); + + await updateUser("test2", { + storage_id: null, + }); + + expect(executeSqlMock).toHaveBeenCalledTimes(1); + expect(executeSqlMock).toHaveBeenCalledWith("ALTER USER 'test2' SET STORAGE_ID NULL"); + }); }); diff --git a/ui/src/services/userService.ts b/ui/src/services/userService.ts index cd6d25cd..9a53982f 100644 --- a/ui/src/services/userService.ts +++ b/ui/src/services/userService.ts @@ -9,7 +9,8 @@ import { buildUpdateUserEmailSql, buildUpdateUserPasswordSql, buildUpdateUserRoleSql, - buildUpdateUserStorageSql, + buildUpdateUserStorageIdSql, + buildUpdateUserStorageModeSql, type CreateUserInput, type UpdateUserInput, } from "@/services/sql/queries/userQueries"; @@ -44,10 +45,6 @@ export async function fetchUsers(): Promise { export async function createUser(input: CreateUserInput): Promise { await executeSql(buildCreateUserSql(input)); - const storageSql = buildUpdateUserStorageSql(input.username, input.storage_mode, input.storage_id); - if (storageSql) { - await executeSql(storageSql); - } } export async function updateUser(username: string, input: UpdateUserInput): Promise { @@ -60,9 +57,11 @@ export async function updateUser(username: string, input: UpdateUserInput): Prom if (input.email !== undefined) { await executeSql(buildUpdateUserEmailSql(username, input.email)); } - const storageSql = buildUpdateUserStorageSql(username, input.storage_mode, input.storage_id); - if (storageSql) { - await executeSql(storageSql); + if (input.storage_mode !== undefined && input.storage_mode !== null) { + await executeSql(buildUpdateUserStorageModeSql(username, input.storage_mode)); + } + if (input.storage_id !== undefined) { + await executeSql(buildUpdateUserStorageIdSql(username, input.storage_id)); } } diff --git a/ui/vite.config.ts b/ui/vite.config.ts index 787792ce..ed4bf095 100644 --- a/ui/vite.config.ts +++ b/ui/vite.config.ts @@ -2,6 +2,7 @@ import { defineConfig, loadEnv } from "vite"; import react from "@vitejs/plugin-react"; import path from "path"; import fs from "fs"; +import type { PluginOption } from "vite"; function normalizeOrigin(url: string): string { return url.replace(/\/+$/, ""); @@ -36,6 +37,21 @@ const cleanupPlugin = () => ({ } }); +const runtimeConfigScriptPlugin = (): PluginOption => ({ + name: "runtime-config-script", + transformIndexHtml() { + return [ + { + tag: "script", + attrs: { + src: "%BASE_URL%runtime-config.js", + }, + injectTo: "body", + }, + ]; + }, +}); + // https://vite.dev/config/ export default defineConfig(({ mode }) => { const env = loadEnv(mode, __dirname, ""); @@ -43,7 +59,7 @@ export default defineConfig(({ mode }) => { const backendWebSocketOrigin = toWebSocketOrigin(backendOrigin); return { - plugins: [react(), cleanupPlugin()], + plugins: [react(), cleanupPlugin(), runtimeConfigScriptPlugin()], // Base path for production build (embedded in server at /ui/) base: "/ui/", resolve: { @@ -90,6 +106,7 @@ export default defineConfig(({ mode }) => { outDir: "dist", sourcemap: false, target: "esnext", + chunkSizeWarningLimit: 700, rollupOptions: { output: { // Ensure WASM files have consistent naming