From 9a653ee8888b026b24aeb2535f49f64872db45d4 Mon Sep 17 00:00:00 2001 From: jamals86 Date: Wed, 13 May 2026 16:38:38 +0300 Subject: [PATCH 1/3] Fixes to the cli --- cli/src/commands/credentials.rs | 17 ++++-------- cli/src/connect.rs | 46 +++++++++++---------------------- cli/src/main.rs | 4 ++- cli/src/terminal_input.rs | 43 ++++++++++++++++++++++++++++++ 4 files changed, 66 insertions(+), 44 deletions(-) create mode 100644 cli/src/terminal_input.rs diff --git a/cli/src/commands/credentials.rs b/cli/src/commands/credentials.rs index b00767e9..64ce1664 100644 --- a/cli/src/commands/credentials.rs +++ b/cli/src/commands/credentials.rs @@ -1,7 +1,4 @@ -use std::{ - io::{self, Write}, - time::Duration, -}; +use std::time::Duration; use kalam_cli::{CLIError, FileCredentialStore, Result}; use kalam_client::{ @@ -10,6 +7,7 @@ use kalam_client::{ }; use crate::args::Cli; +use crate::terminal_input::{prompt_line, prompt_password}; pub fn handle_credentials(cli: &Cli, credential_store: &mut FileCredentialStore) -> Result { if cli.list_instances { @@ -104,19 +102,14 @@ pub async fn login_and_store_credentials( let user = if let Some(user) = &cli.user { user.clone() } else { - print!("User: "); - io::stdout().flush().unwrap(); - let mut input = String::new(); - io::stdin() - .read_line(&mut input) - .map_err(|e| CLIError::FileError(format!("Failed to read user: {}", e)))?; - input.trim().to_string() + prompt_line("User: ") + .map_err(|e| CLIError::FileError(format!("Failed to read user: {}", e)))? }; let password = if let Some(pass) = &cli.password { pass.clone() } else { - rpassword::prompt_password("Password: ") + prompt_password("Password: ") .map_err(|e| CLIError::FileError(format!("Failed to read password: {}", e)))? }; diff --git a/cli/src/connect.rs b/cli/src/connect.rs index 3949f246..cd37ad78 100644 --- a/cli/src/connect.rs +++ b/cli/src/connect.rs @@ -1,5 +1,5 @@ use std::{ - io::{self, IsTerminal, Write}, + io::IsTerminal, net::IpAddr, time::Duration, }; @@ -16,6 +16,7 @@ use kalam_client::{ use url::Url; use crate::args::Cli; +use crate::terminal_input::{prompt_line, prompt_password}; /// Build timeouts configuration from CLI arguments fn build_timeouts(cli: &Cli) -> KalamLinkTimeouts { @@ -257,11 +258,8 @@ pub async fn create_session( println!(); // Get DBA user - print!("Enter the user for your DBA account: "); - io::stdout().flush().map_err(|e| e.to_string())?; - let mut username = String::new(); - io::stdin().read_line(&mut username).map_err(|e| e.to_string())?; - let username = username.trim().to_string(); + let username = prompt_line("Enter the user for your DBA account: ") + .map_err(|e| e.to_string())?; if username.is_empty() { return Err("User cannot be empty".to_string()); } @@ -270,15 +268,14 @@ pub async fn create_session( } // Get DBA password - let password = rpassword::prompt_password("Enter password for your DBA account: ") + let password = prompt_password("Enter password for your DBA account: ") .map_err(|e| e.to_string())?; if password.is_empty() { return Err("Password cannot be empty".to_string()); } // Confirm password - let password_confirm = - rpassword::prompt_password("Confirm password: ").map_err(|e| e.to_string())?; + let password_confirm = prompt_password("Confirm password: ").map_err(|e| e.to_string())?; if password != password_confirm { return Err("Passwords do not match".to_string()); } @@ -286,25 +283,21 @@ pub async fn create_session( // Get root password println!(); println!("Now set the root password (for system administration):"); - let root_password = - rpassword::prompt_password("Enter root password: ").map_err(|e| e.to_string())?; + let root_password = prompt_password("Enter root password: ").map_err(|e| e.to_string())?; if root_password.is_empty() { return Err("Root password cannot be empty".to_string()); } // Confirm root password let root_password_confirm = - rpassword::prompt_password("Confirm root password: ").map_err(|e| e.to_string())?; + prompt_password("Confirm root password: ").map_err(|e| e.to_string())?; if root_password != root_password_confirm { return Err("Root passwords do not match".to_string()); } // Optional email - print!("Enter email (optional, press Enter to skip): "); - io::stdout().flush().map_err(|e| e.to_string())?; - let mut email = String::new(); - io::stdin().read_line(&mut email).map_err(|e| e.to_string())?; - let email = email.trim().to_string(); + let email = prompt_line("Enter email (optional, press Enter to skip): ") + .map_err(|e| e.to_string())?; let email = if email.is_empty() { None } else { Some(email) }; println!(); @@ -448,22 +441,15 @@ pub async fn create_session( println!(); // Prompt for user - print!("User: "); - io::stdout() - .flush() - .map_err(|e| CLIError::FileError(format!("Failed to flush stdout: {}", e)))?; - let mut username = String::new(); - io::stdin() - .read_line(&mut username) + let username = prompt_line("User: ") .map_err(|e| CLIError::FileError(format!("Failed to read user: {}", e)))?; - let username = username.trim().to_string(); if username.is_empty() { return Err(CLIError::ConfigurationError("User cannot be empty".to_string())); } // Prompt for password - let password = rpassword::prompt_password("Password: ") + let password = prompt_password("Password: ") .map_err(|e| CLIError::FileError(format!("Failed to read password: {}", e)))?; // Try to login with provided credentials @@ -472,10 +458,8 @@ pub async fn create_session( let authenticated_user = login_response.user.id.to_string(); // Ask if user wants to save credentials - print!("\nSave credentials for future use? (y/N): "); - io::stdout().flush().ok(); - let mut save_choice = String::new(); - io::stdin().read_line(&mut save_choice).ok(); + let save_choice = prompt_line("\nSave credentials for future use? (y/N): ") + .unwrap_or_default(); if save_choice.trim().eq_ignore_ascii_case("y") || save_choice.trim().eq_ignore_ascii_case("yes") @@ -575,7 +559,7 @@ pub async fn create_session( } else if std::io::stdin().is_terminal() { println!(); println!("User: {}", username); - rpassword::prompt_password("Password: ") + prompt_password("Password: ") .map_err(|e| CLIError::FileError(format!("Failed to read password: {}", e)))? } else { // Non-interactive mode without password - use empty password diff --git a/cli/src/main.rs b/cli/src/main.rs index 4fca21ec..4f748c6b 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -23,6 +23,7 @@ use kalam_cli::{CLIConfiguration, CLIError, FileCredentialStore, Result}; mod args; mod commands; mod connect; +mod terminal_input; use args::Cli; use commands::{ @@ -31,6 +32,7 @@ use commands::{ watch_schema::handle_watch_schema, }; use connect::create_session; +use terminal_input::prompt_password; #[tokio::main] async fn main() { @@ -51,7 +53,7 @@ async fn run() -> Result<()> { let is_interactive_mode = cli.command.is_none() && cli.file.is_none(); if cli.password.as_deref() == Some("") && is_interactive_mode && std::io::stdin().is_terminal() { - let password = rpassword::prompt_password("Password: ") + let password = prompt_password("Password: ") .map_err(|e| CLIError::FileError(format!("Failed to read password: {}", e)))?; cli.password = Some(password); } diff --git a/cli/src/terminal_input.rs b/cli/src/terminal_input.rs new file mode 100644 index 00000000..ea2431cb --- /dev/null +++ b/cli/src/terminal_input.rs @@ -0,0 +1,43 @@ +use std::io::{self, Write}; + +use crossterm::terminal; + +struct CookedTerminalGuard { + restore_raw_mode: bool, +} + +impl CookedTerminalGuard { + fn acquire() -> io::Result { + let restore_raw_mode = terminal::is_raw_mode_enabled()?; + if restore_raw_mode { + terminal::disable_raw_mode()?; + } + + Ok(Self { restore_raw_mode }) + } +} + +impl Drop for CookedTerminalGuard { + fn drop(&mut self) { + if self.restore_raw_mode { + let _ = terminal::enable_raw_mode(); + } + } +} + +pub fn prompt_line(prompt: &str) -> io::Result { + let _guard = CookedTerminalGuard::acquire()?; + + print!("{prompt}"); + io::stdout().flush()?; + + let mut input = String::new(); + io::stdin().read_line(&mut input)?; + + Ok(input.trim().to_string()) +} + +pub fn prompt_password(prompt: &str) -> io::Result { + let _guard = CookedTerminalGuard::acquire()?; + rpassword::prompt_password(prompt) +} \ No newline at end of file From a5648e89bf44a279b997bb125b3317b87a3c38c5 Mon Sep 17 00:00:00 2001 From: jamals86 Date: Wed, 13 May 2026 22:01:00 +0300 Subject: [PATCH 2/3] Route batch SQL to group leaders; add CLI E2E workflow Add routing and forwarding improvements for batch SQL execution and leader-forwarding, plus CI and test tweaks. Key changes: - Add should_route_batch_statements_individually and forward_batch_statement_to_group to forward per-statement queries to the appropriate group leader with retry/backoff for transient metadata visibility errors; integrate into execute_batch_path and forward logic. - Refactor forward.rs to return raw ForwardSqlResponse, convert forwarded responses to HTTP, and expose forward_sql_to_group_leader_raw helper. - Introduce helpers for accumulating batch results and detecting meta-mutating statements; adjust batching semantics to sync transaction state when needed. - Separate non-Raft cluster RPC transport in RaftManager (cluster_network_factory) and use it for peer channels and registrations; use SERVER_VERSION in node metadata and executor responses. - Add a GitHub Actions workflow (cli-cluster-e2e.yml) to run CLI cluster end-to-end tests using a released server binary, and extend nextest.toml overrides to mark many tests as stateful-heavy. - Miscellaneous formatting, minor test and code cleanups across multiple modules. These changes improve correctness and stability when executing multi-statement requests that target different raft groups and add CI coverage for the CLI cluster e2e scenario. --- .config/nextest.toml | 90 +++++++ .github/workflows/cli-cluster-e2e.yml | 191 ++++++++++++++ .../src/http/sql/execution_paths.rs | 239 +++++++++++++++-- .../kalamdb-api/src/http/sql/forward.rs | 124 ++++++++- .../src/http/sql/models/sql_response.rs | 15 +- .../kalamdb-api/src/ws/events/subscription.rs | 11 +- .../kalamdb-core/src/cluster_handler.rs | 7 +- .../src/parser/query_parser.rs | 5 +- .../crates/stream/src/topics/cleanup.rs | 2 +- .../crates/stream/src/topics/clear.rs | 8 +- .../crates/stream/src/topics/drop.rs | 8 +- .../crates/stream/src/topics/mod.rs | 2 +- .../kalamdb-jobs/src/topic_retention.rs | 40 +-- .../crates/kalamdb-publisher/src/service.rs | 23 +- .../crates/kalamdb-raft/src/executor/raft.rs | 14 +- .../kalamdb-raft/src/manager/raft_manager.rs | 26 +- .../crates/kalamdb-raft/src/storage/types.rs | 3 +- .../tests/integration_tests/topic_pubsub.rs | 9 +- backend/tests/misc/auth/test_soft_delete.rs | 20 +- .../tests/misc/production/test_mvcc_phase2.rs | 80 +++--- backend/tests/misc/schema/test_alter_table.rs | 81 +++--- .../schema/test_alter_table_after_flush.rs | 4 +- .../misc/sql/test_pk_index_efficiency.rs | 50 ++-- .../tests/misc/sql/test_row_count_behavior.rs | 50 ++-- .../misc/sql/test_sql_error_redaction.rs | 5 +- .../test_update_delete_version_resolution.rs | 20 +- .../storage/test_cold_storage_manifest.rs | 10 +- cargo_check_output.txt | 2 + cli/run-tests.sh | 88 ++++++- cli/src/connect.rs | 18 +- cli/src/session.rs | 13 +- cli/src/terminal_input.rs | 2 +- cli/tests/cluster.rs | 246 +++++++++++++++--- .../cluster/cluster_test_multi_node_smoke.rs | 66 +++++ cli/tests/cluster/cluster_test_node_rejoin.rs | 8 +- cli/tests/common/mod.rs | 199 +++++++++----- .../ddl/smoke_test_datatype_preservation.rs | 6 +- .../smoke/security/smoke_test_rpc_auth.rs | 47 ++-- .../usecases/smoke_test_file_datatype.rs | 30 ++- docs/architecture/raft-replication.md | 52 ++-- link/link-common/src/query/models/mod.rs | 2 +- pg/crates/kalam-pg-common/src/config.rs | 2 +- pg/crates/kalam-pg-fdw/tests/options.rs | 22 +- scripts/cluster.sh | 94 ++++++- specs/026-postgres-extension/README | 2 +- .../legacy-dual-mode-reference.md | 2 +- 46 files changed, 1508 insertions(+), 530 deletions(-) create mode 100644 .github/workflows/cli-cluster-e2e.yml create mode 100644 cargo_check_output.txt diff --git a/.config/nextest.toml b/.config/nextest.toml index 5b63f7ef..69897c56 100644 --- a/.config/nextest.toml +++ b/.config/nextest.toml @@ -119,6 +119,96 @@ filter = 'test(test_connection_timeout_option)' # handles here; keep it isolated from subprocess-heavy tests. test-group = "stateful-heavy" +[[profile.default.overrides]] +filter = 'test(test_rapid_connect_disconnect)' +test-group = "stateful-heavy" + +[[profile.default.overrides]] +filter = 'test(test_concurrent_websocket_subscriptions)' +test-group = "stateful-heavy" + +[[profile.default.overrides]] +filter = 'test(smoke_test_leader_read_shared_table)' +test-group = "stateful-heavy" + +[[profile.default.overrides]] +filter = 'test(cluster_test_table_identity_updates)' +test-group = "stateful-heavy" + +[[profile.default.overrides]] +filter = 'test(smoke_all_datatypes_user_shared_stream)' +test-group = "stateful-heavy" + +[[profile.default.overrides]] +filter = 'test(cluster_test_table_identity_mixed_operations)' +test-group = "stateful-heavy" + +[[profile.default.overrides]] +filter = 'test(cluster_test_ws_follower_receives_leader_changes)' +test-group = "stateful-heavy" + +[[profile.default.overrides]] +filter = 'test(cluster_test_table_identity_user_tables)' +test-group = "stateful-heavy" + +[[profile.default.overrides]] +filter = 'test(smoke_security_regular_user_cannot_impersonate_privileged_users_in_batch)' +test-group = "stateful-heavy" + +[[profile.default.overrides]] +filter = 'test(smoke_as_user_chat_delete_flow)' +test-group = "stateful-heavy" + +[[profile.default.overrides]] +filter = 'test(smoke_insert_returning_seq_multi_row)' +test-group = "stateful-heavy" + +[[profile.default.overrides]] +filter = 'test(smoke_rpc_login_nonexistent_user_matches_wrong_password_response)' +test-group = "stateful-heavy" + +[[profile.default.overrides]] +filter = 'test(smoke_security_private_shared_table_blocked_in_batch)' +test-group = "stateful-heavy" + +[[profile.default.overrides]] +filter = 'test(smoke_security_system_tables_blocked_in_batch)' +test-group = "stateful-heavy" + +[[profile.default.overrides]] +filter = 'test(smoke_storage_custom_templates)' +test-group = "stateful-heavy" + +[[profile.default.overrides]] +filter = 'test(smoke_storage_check_dba_access)' +test-group = "stateful-heavy" + +[[profile.default.overrides]] +filter = 'test(smoke_storage_check_authorization)' +test-group = "stateful-heavy" + +[[profile.default.overrides]] +filter = 'test(test_topic_consume_update_events)' +test-group = "stateful-heavy" + +[[profile.default.overrides]] +filter = 'test(test_topic_consume_offset_persistence)' +test-group = "stateful-heavy" + +[[profile.default.overrides]] +filter = 'test(smoke_export_download_forbidden_for_other_user)' +test-group = "stateful-heavy" + +[[profile.default.overrides]] +filter = 'test(cluster_test_leader_only_flush_jobs)' +# Uses system.jobs queries — must not race with other tests' flush jobs or background scheduler. +test-group = "stateful-heavy" + +[[profile.default.overrides]] +filter = 'test(cluster_test_jobs_table_consistency)' +# Queries global system.jobs count — must not race with concurrent tests creating jobs. +test-group = "stateful-heavy" + [profile.ci] # Store test results in JUnit format junit.path = "junit.xml" diff --git a/.github/workflows/cli-cluster-e2e.yml b/.github/workflows/cli-cluster-e2e.yml new file mode 100644 index 00000000..7f81c77c --- /dev/null +++ b/.github/workflows/cli-cluster-e2e.yml @@ -0,0 +1,191 @@ +name: CLI Cluster E2E + +on: + push: + branches: [ main ] + paths: + - 'cli/**' + - 'scripts/cluster.sh' + - 'scripts/download-release-server.sh' + - 'scripts/resolve-latest-release-asset.py' + - 'versions.json' + - 'nextest.toml' + - '.github/workflows/cli-cluster-e2e.yml' + pull_request: + paths: + - 'cli/**' + - 'scripts/cluster.sh' + - 'scripts/download-release-server.sh' + - 'scripts/resolve-latest-release-asset.py' + - 'versions.json' + - 'nextest.toml' + - '.github/workflows/cli-cluster-e2e.yml' + workflow_dispatch: + +permissions: + contents: read + +env: + CARGO_TERM_COLOR: always + RUST_VERSION: "1.92.0" + RUSTC_WRAPPER: "" + CARGO_BUILD_RUSTC_WRAPPER: "" + +jobs: + resolve_versions: + name: Resolve Cluster E2E Server Version + runs-on: ubuntu-latest + outputs: + root_version: ${{ steps.versions.outputs.root_version }} + latest_server_release_tag: ${{ steps.latest_server.outputs.release_tag }} + selected_server_version: ${{ steps.latest_server.outputs.selected_version }} + selected_server_selection_reason: ${{ steps.latest_server.outputs.selection_reason }} + latest_server_asset_name: ${{ steps.latest_server.outputs.asset_name }} + latest_server_asset_api_url: ${{ steps.latest_server.outputs.asset_api_url }} + steps: + - name: Checkout + uses: actions/checkout@v6 + + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Verify versions.json + shell: bash + run: | + set -euo pipefail + python3 scripts/versions.py verify + + - name: Resolve versions from versions.json + id: versions + shell: bash + run: | + set -euo pipefail + python3 scripts/versions.py github-outputs --github-output "$GITHUB_OUTPUT" --repository "$GITHUB_REPOSITORY" + + - name: Resolve latest released server asset + id: latest_server + shell: bash + env: + GITHUB_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + python3 scripts/resolve-latest-release-asset.py \ + --repository "$GITHUB_REPOSITORY" \ + --github-output "$GITHUB_OUTPUT" \ + --token "$GITHUB_TOKEN" \ + --preferred-version "${{ steps.versions.outputs.root_version }}" \ + --asset-pattern '^kalamdb-server-.*-linux-x86_64\.tar\.gz$' + + download_release_server_binary: + name: Download Released Cluster Server Binary + runs-on: ubuntu-latest + needs: resolve_versions + steps: + - name: Checkout + uses: actions/checkout@v6 + + - name: Download latest release server asset + shell: bash + env: + GITHUB_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + mkdir -p target/release + bash scripts/download-release-server.sh \ + "${{ needs.resolve_versions.outputs.latest_server_asset_name }}" \ + "${{ needs.resolve_versions.outputs.latest_server_asset_api_url }}" \ + "$PWD/target/release/kalamdb-server" + echo "Using cluster e2e server release ${{ needs.resolve_versions.outputs.latest_server_release_tag }} (${{ needs.resolve_versions.outputs.selected_server_version }}, ${{ needs.resolve_versions.outputs.selected_server_selection_reason }}) for requested version ${{ needs.resolve_versions.outputs.root_version }}" + + - name: Upload released cluster server binary + uses: actions/upload-artifact@v6 + with: + name: cluster-e2e-server-linux-x86_64 + path: target/release/kalamdb-server + if-no-files-found: error + + test: + name: CLI Cluster E2E + runs-on: ubuntu-latest + needs: + - resolve_versions + - download_release_server_binary + steps: + - name: Checkout + uses: actions/checkout@v6 + + - name: Download released server binary + uses: actions/download-artifact@v6 + with: + name: cluster-e2e-server-linux-x86_64 + path: target/release + + - name: Install system dependencies + shell: bash + run: | + sudo apt-get update + sudo apt-get install -y --no-install-recommends \ + clang \ + libclang-dev \ + pkg-config \ + libssl-dev + + - name: Setup Rust + uses: dtolnay/rust-toolchain@stable + with: + toolchain: ${{ env.RUST_VERSION }} + + - name: Cache Rust + uses: Swatinem/rust-cache@v2 + with: + shared-key: cli-cluster-e2e + cache-on-failure: true + + - name: Install cargo-nextest + shell: bash + run: | + set -euo pipefail + curl -LsSf https://get.nexte.st/latest/linux | tar zxf - -C ${CARGO_HOME:-~/.cargo}/bin + cargo nextest --version + + - name: Prepare released server binary + shell: bash + run: | + set -euo pipefail + chmod +x target/release/kalamdb-server + ./target/release/kalamdb-server --version + echo "Cluster e2e server source: latest-release" + + - name: Run cluster workspace-e2e workflow + shell: bash + env: + KALAMDB_ROOT_PASSWORD: kalamdb123 + KALAMDB_ADMIN_PASSWORD: kalamdb123 + run: | + set -euo pipefail + ./scripts/cluster.sh workspace-e2e + + - name: Print cluster logs + if: failure() + shell: bash + run: | + set -euo pipefail + for node in 1 2 3; do + echo "=== cluster node ${node} log ===" + cat ".cluster-local/node${node}/logs/stdout.log" || true + echo + done + + - name: Upload cluster logs + if: always() + uses: actions/upload-artifact@v6 + with: + name: cli-cluster-e2e-logs + path: | + .cluster-local/node1/logs/stdout.log + .cluster-local/node2/logs/stdout.log + .cluster-local/node3/logs/stdout.log + cli/.env + if-no-files-found: warn diff --git a/backend/crates/kalamdb-api/src/http/sql/execution_paths.rs b/backend/crates/kalamdb-api/src/http/sql/execution_paths.rs index 5833beb2..10551ba4 100644 --- a/backend/crates/kalamdb-api/src/http/sql/execution_paths.rs +++ b/backend/crates/kalamdb-api/src/http/sql/execution_paths.rs @@ -1,4 +1,8 @@ -use std::{collections::HashMap, sync::Arc, time::Instant}; +use std::{ + collections::HashMap, + sync::Arc, + time::{Duration, Instant}, +}; use actix_web::{http::StatusCode, HttpRequest, HttpResponse}; use bytes::Bytes; @@ -16,12 +20,16 @@ use kalamdb_core::{ SqlImpersonationService, }, }; +use kalamdb_raft::GroupId; use kalamdb_sql::classifier::SqlStatementKind; use kalamdb_system::FileSubfolderState; use super::{ file_utils::{stage_and_finalize_files, substitute_file_placeholders}, - forward::handle_not_leader_error, + forward::{ + forward_sql_to_group_leader_raw, forwarded_sql_response_to_http, handle_not_leader_error, + prepared_statement_target_group, should_route_batch_statements_individually, + }, helpers::{ cleanup_files, execute_single_statement, execute_single_statement_raw, execution_result_to_query_result, stream_sql_rows_response, @@ -174,6 +182,138 @@ fn build_kalamdb_error_response(err: &KalamDbError, took: f64, is_admin: bool) - build_sql_error_response(status, code, message.as_ref(), None, took, is_admin, preserve_message) } +fn push_or_accumulate_batch_result( + result: QueryResult, + is_batch: bool, + total_inserted: &mut usize, + total_updated: &mut usize, + total_deleted: &mut usize, + results: &mut Vec, +) { + if is_batch { + if let Some(message) = result.message.as_deref() { + if message.contains("Inserted") { + *total_inserted += result.row_count; + return; + } + if message.contains("Updated") { + *total_updated += result.row_count; + return; + } + if message.contains("Deleted") { + *total_deleted += result.row_count; + return; + } + } + } + + results.push(result); +} + +fn statement_mutates_meta( + statement: &PreparedApiExecutionStatement, + app_context: &AppContext, + routing_user_id: &kalamdb_commons::models::UserId, +) -> bool { + let target_group = prepared_statement_target_group(statement, app_context, routing_user_id); + if target_group != Some(GroupId::Meta) { + return false; + } + + statement + .prepared_statement + .classified_statement + .as_ref() + .is_some_and(|classified| classified.is_write_operation()) +} + +fn is_transient_forwarded_metadata_error(response: &SqlResponse) -> bool { + let Some(error) = response.error.as_ref() else { + return false; + }; + + if !matches!(error.code, ErrorCode::SqlExecutionError | ErrorCode::TableNotFound) { + return false; + } + + let mut message = error.message.to_ascii_lowercase(); + if let Some(details) = error.details.as_deref() { + message.push(' '); + message.push_str(&details.to_ascii_lowercase()); + } + + message.contains("table") && message.contains("not found") + || message.contains("relation") && message.contains("does not exist") + || message.contains("unknown table") + || message.contains("namespace") && message.contains("not found") + || message.contains("schema") && message.contains("not found") +} + +#[allow(clippy::too_many_arguments)] +async fn forward_batch_statement_to_group( + target_group: GroupId, + statement: &PreparedApiExecutionStatement, + http_req: &HttpRequest, + req_for_forward: &QueryRequest, + app_context: &AppContext, + request_id: Option<&str>, + start_time: Instant, + retry_metadata_lag: bool, +) -> Result { + const MAX_ATTEMPTS: u32 = 5; + const INITIAL_BACKOFF_MS: u64 = 5; + + let request = QueryRequest { + sql: statement.prepared_statement.sql.clone(), + params: None, + namespace_id: req_for_forward.namespace_id.clone(), + }; + + for attempt in 0..MAX_ATTEMPTS { + let response = forward_sql_to_group_leader_raw( + target_group, + http_req, + &request, + app_context, + request_id, + start_time, + ) + .await?; + + let status = + StatusCode::from_u16(response.status_code as u16).unwrap_or(StatusCode::BAD_GATEWAY); + let parsed = serde_json::from_slice::(&response.body); + + if status.is_success() { + return parsed.map_err(|err| { + HttpResponse::BadGateway().json(SqlResponse::error( + ErrorCode::ForwardFailed, + &format!("Failed to decode forwarded SQL response: {}", err), + took_ms(start_time), + )) + }); + } + + if retry_metadata_lag && attempt + 1 < MAX_ATTEMPTS { + if let Ok(parsed_response) = parsed.as_ref() { + if is_transient_forwarded_metadata_error(parsed_response) { + let backoff_ms = INITIAL_BACKOFF_MS * (1 << attempt); + tokio::time::sleep(Duration::from_millis(backoff_ms)).await; + continue; + } + } + } + + return Err(forwarded_sql_response_to_http(response, start_time)); + } + + Err(HttpResponse::GatewayTimeout().json(SqlResponse::error( + ErrorCode::ForwardFailed, + "Timed out waiting for forwarded SQL metadata visibility", + took_ms(start_time), + ))) +} + fn build_statement_error_response( err: &(dyn std::error::Error + 'static), statement_index: usize, @@ -454,10 +594,17 @@ pub(super) async fn execute_batch_path( ) -> HttpResponse { let is_batch = prepared_statements.len() > 1; let stmt_count = prepared_statements.len(); + let route_statements_individually = should_route_batch_statements_individually( + prepared_statements, + &req_for_forward.params, + app_context.as_ref(), + exec_ctx.user_id(), + ); let mut results = Vec::with_capacity(stmt_count); let mut total_inserted = 0usize; let mut total_updated = 0usize; let mut total_deleted = 0usize; + let mut meta_changed_in_batch = false; let mut params_remaining = Some(params); let mut request_transaction_state = match RequestTransactionState::from_execution_context(exec_ctx) { @@ -576,6 +723,58 @@ pub(super) async fn execute_batch_path( } } + let routing_user_id = execute_as_user.as_ref().unwrap_or_else(|| exec_ctx.user_id()); + + if route_statements_individually { + if let Some(target_group) = + prepared_statement_target_group(stmt, app_context.as_ref(), routing_user_id) + { + if !app_context.executor().is_leader(target_group).await { + match forward_batch_statement_to_group( + target_group, + stmt, + http_req, + req_for_forward, + app_context.as_ref(), + exec_ctx.request_id(), + start_time, + meta_changed_in_batch, + ) + .await + { + Ok(forwarded_response) => { + for result in forwarded_response.results { + push_or_accumulate_batch_result( + result, + is_batch, + &mut total_inserted, + &mut total_updated, + &mut total_deleted, + &mut results, + ); + } + + if statement_mutates_meta(stmt, app_context.as_ref(), routing_user_id) { + meta_changed_in_batch = true; + } + + if let Some(state) = request_transaction_state.as_mut() { + state.sync_from_coordinator(app_context); + } + idx += 1; + continue; + }, + Err(response) => { + if let Some(state) = request_transaction_state.as_mut() { + let _ = state.rollback_if_active(app_context); + } + return response; + }, + } + } + } + } + let stmt_start = Instant::now(); let effective_username = resolve_result_username(authorized_username, stmt.execute_as_username.as_deref()); @@ -678,34 +877,18 @@ pub(super) async fn execute_batch_path( }, }; - if is_batch { - if let Some(ref msg) = result.message { - if msg.contains("Inserted") { - total_inserted += result.row_count; - if let Some(state) = request_transaction_state.as_mut() { - state.sync_from_coordinator(app_context); - } - idx += 1; - continue; - } else if msg.contains("Updated") { - total_updated += result.row_count; - if let Some(state) = request_transaction_state.as_mut() { - state.sync_from_coordinator(app_context); - } - idx += 1; - continue; - } else if msg.contains("Deleted") { - total_deleted += result.row_count; - if let Some(state) = request_transaction_state.as_mut() { - state.sync_from_coordinator(app_context); - } - idx += 1; - continue; - } - } + if statement_mutates_meta(stmt, app_context.as_ref(), routing_user_id) { + meta_changed_in_batch = true; } - results.push(result); + push_or_accumulate_batch_result( + result, + is_batch, + &mut total_inserted, + &mut total_updated, + &mut total_deleted, + &mut results, + ); }, Err(err) => { if let Some(state) = request_transaction_state.as_mut() { diff --git a/backend/crates/kalamdb-api/src/http/sql/forward.rs b/backend/crates/kalamdb-api/src/http/sql/forward.rs index e41d8b14..301be4cc 100644 --- a/backend/crates/kalamdb-api/src/http/sql/forward.rs +++ b/backend/crates/kalamdb-api/src/http/sql/forward.rs @@ -8,7 +8,9 @@ use kalamdb_commons::{ schemas::TableType, }; use kalamdb_core::{app_context::AppContext, error::KalamDbError}; -use kalamdb_raft::{ClusterClient, ForwardSqlRequest, GroupId, RaftExecutor, ShardRouter}; +use kalamdb_raft::{ + ClusterClient, ForwardSqlRequest, ForwardSqlResponse, GroupId, RaftExecutor, ShardRouter, +}; use kalamdb_sql::classifier::SqlStatementKind; use serde_json::Value as JsonValue; use uuid::Uuid; @@ -60,6 +62,21 @@ enum ForwardTarget { Node(NodeId), } +fn is_transaction_control(statement: &PreparedApiExecutionStatement) -> bool { + statement + .prepared_statement + .classified_statement + .as_ref() + .is_some_and(|classified| { + matches!( + classified.kind(), + SqlStatementKind::BeginTransaction + | SqlStatementKind::CommitTransaction + | SqlStatementKind::RollbackTransaction + ) + }) +} + fn data_group_for_table_type( app_context: &AppContext, table_type: TableType, @@ -127,23 +144,58 @@ pub(crate) fn prepared_statement_target_group( } } -async fn forward_sql_grpc( +pub(crate) fn should_route_batch_statements_individually( + prepared_statements: &[PreparedApiExecutionStatement], + params_json: &Option>, + app_context: &AppContext, + user_id: &UserId, +) -> bool { + if prepared_statements.len() <= 1 + || params_json.as_ref().is_some_and(|params| !params.is_empty()) + || prepared_statements.iter().any(is_transaction_control) + { + return false; + } + + if prepared_statements + .iter() + .any(|statement| statement.execute_as_username.is_some()) + { + return true; + } + + let mut targets = prepared_statements + .iter() + .map(|statement| prepared_statement_target_group(statement, app_context, user_id)); + + let Some(Some(first_target)) = targets.next() else { + return false; + }; + + targets.all(|target| target.is_some()) + && prepared_statements.iter().any(|statement| { + prepared_statement_target_group(statement, app_context, user_id) + .is_some_and(|target| target != first_target) + }) +} + +async fn forward_sql_grpc_response( target: ForwardTarget, http_req: &HttpRequest, req: &QueryRequest, app_context: &AppContext, request_id: Option<&str>, start_time: Instant, -) -> Option { +) -> Result { let client = match cluster_client_for(app_context) { Ok(c) => c, - Err(resp) => return Some(resp), + Err(resp) => return Err(resp), }; let params = match parse_forward_params(&req.params) { Ok(v) => v, Err(e) => { - return Some(HttpResponse::BadRequest().json(SqlResponse::error( + return Err(HttpResponse::BadRequest().json(SqlResponse::error( ErrorCode::InvalidParameter, &e, start_time.elapsed().as_secs_f64() * 1000.0, @@ -172,7 +224,7 @@ async fn forward_sql_grpc( Ok(resp) => resp, Err(err) => { log::warn!("Failed to forward SQL over gRPC: {}", err); - return Some(HttpResponse::ServiceUnavailable().json(SqlResponse::error( + return Err(HttpResponse::ServiceUnavailable().json(SqlResponse::error( ErrorCode::ForwardFailed, "Failed to forward request to cluster leader", start_time.elapsed().as_secs_f64() * 1000.0, @@ -180,17 +232,62 @@ async fn forward_sql_grpc( }, }; + Ok(response) +} + +pub(crate) fn forwarded_sql_response_to_http( + response: ForwardSqlResponse, + start_time: Instant, +) -> HttpResponse { if !response.error.is_empty() && response.body.is_empty() { - return Some(HttpResponse::BadGateway().json(SqlResponse::error( + return HttpResponse::BadGateway().json(SqlResponse::error( ErrorCode::ForwardFailed, &response.error, start_time.elapsed().as_secs_f64() * 1000.0, - ))); + )); } let status = actix_web::http::StatusCode::from_u16(response.status_code as u16) .unwrap_or(actix_web::http::StatusCode::BAD_GATEWAY); - Some(HttpResponse::build(status).content_type("application/json").body(response.body)) + HttpResponse::build(status).content_type("application/json").body(response.body) +} + +pub(crate) async fn forward_sql_to_group_leader_raw( + group_id: GroupId, + http_req: &HttpRequest, + req: &QueryRequest, + app_context: &AppContext, + request_id: Option<&str>, + start_time: Instant, +) -> Result { + forward_sql_grpc_response( + ForwardTarget::GroupLeader(group_id), + http_req, + req, + app_context, + request_id, + start_time, + ) + .await +} + +async fn forward_sql_grpc( + target: ForwardTarget, + http_req: &HttpRequest, + req: &QueryRequest, + app_context: &AppContext, + request_id: Option<&str>, + start_time: Instant, +) -> Option { + let response = + match forward_sql_grpc_response(target, http_req, req, app_context, request_id, start_time) + .await + { + Ok(response) => response, + Err(response) => return Some(response), + }; + + Some(forwarded_sql_response_to_http(response, start_time)) } /// Forwards leader-routed operations to the appropriate leader node in cluster mode. @@ -207,6 +304,15 @@ pub async fn forward_sql_if_follower( let start_time = Instant::now(); let executor = app_context.executor(); + if should_route_batch_statements_individually( + prepared_statements, + params_json, + app_context.as_ref(), + user_id, + ) { + return None; + } + let write_targets: Vec = prepared_statements .iter() .filter_map(|statement| { diff --git a/backend/crates/kalamdb-api/src/http/sql/models/sql_response.rs b/backend/crates/kalamdb-api/src/http/sql/models/sql_response.rs index 20fe2bb2..a8641087 100644 --- a/backend/crates/kalamdb-api/src/http/sql/models/sql_response.rs +++ b/backend/crates/kalamdb-api/src/http/sql/models/sql_response.rs @@ -456,11 +456,13 @@ mod tests { let rows = result.rows.as_ref().expect("subscription result should include a row"); assert_eq!(rows[0][0].as_str(), Some("subscription_required")); assert_eq!(rows[0][1].as_str(), Some("ws://localhost:2900/v1/ws")); - assert_eq!(rows[0][3].as_str(), Some("Subscription created. Connect to ws_url to receive updates.")); + assert_eq!( + rows[0][3].as_str(), + Some("Subscription created. Connect to ws_url to receive updates.") + ); - let subscription = rows[0][2] - .as_object() - .expect("subscription column should be a JSON object"); + let subscription = + rows[0][2].as_object().expect("subscription column should be a JSON object"); assert_eq!(subscription.get("id").and_then(|value| value.as_str()), Some("sub-1")); assert_eq!( subscription.get("sql").and_then(|value| value.as_str()), @@ -529,10 +531,7 @@ mod tests { let error = response.error.expect("error response should include an error payload"); assert_eq!(error.code, ErrorCode::SqlExecutionError); - assert_eq!( - error.message, - "SQL statement failed. Review the statement and try again.", - ); + assert_eq!(error.message, "SQL statement failed. Review the statement and try again.",); assert!(error.details.is_none()); } diff --git a/backend/crates/kalamdb-api/src/ws/events/subscription.rs b/backend/crates/kalamdb-api/src/ws/events/subscription.rs index b29a0a38..704572e4 100644 --- a/backend/crates/kalamdb-api/src/ws/events/subscription.rs +++ b/backend/crates/kalamdb-api/src/ws/events/subscription.rs @@ -216,14 +216,9 @@ pub async fn handle_subscribe( ); }, } - let _ = send_error( - session, - &subscription_id, - code, - message.as_ref(), - compression_enabled, - ) - .await; + let _ = + send_error(session, &subscription_id, code, message.as_ref(), compression_enabled) + .await; Ok(()) }, } diff --git a/backend/crates/kalamdb-core/src/cluster_handler.rs b/backend/crates/kalamdb-core/src/cluster_handler.rs index c962152e..539ca324 100644 --- a/backend/crates/kalamdb-core/src/cluster_handler.rs +++ b/backend/crates/kalamdb-core/src/cluster_handler.rs @@ -350,12 +350,7 @@ impl ClusterMessageHandler for CoreClusterHandler { }, Err(e) => { let message = e.to_string(); - return Ok(Self::error_payload( - 400, - "BATCH_PARSE_ERROR", - &message, - started_at, - )); + return Ok(Self::error_payload(400, "BATCH_PARSE_ERROR", &message, started_at)); }, }; diff --git a/backend/crates/kalamdb-dialect/src/parser/query_parser.rs b/backend/crates/kalamdb-dialect/src/parser/query_parser.rs index ef760932..b31b7b86 100644 --- a/backend/crates/kalamdb-dialect/src/parser/query_parser.rs +++ b/backend/crates/kalamdb-dialect/src/parser/query_parser.rs @@ -259,9 +259,8 @@ impl QueryParser { // Wrap in a dummy SELECT to parse the expression let dummy_query = format!("SELECT * FROM dummy WHERE {}", expr_str); - let statements = parse_sql_statements(&dummy_query, &dialect).map_err(|e| { - QueryParseError::ParseError(e.to_string()) - })?; + let statements = parse_sql_statements(&dummy_query, &dialect) + .map_err(|e| QueryParseError::ParseError(e.to_string()))?; if statements.is_empty() { return Err(QueryParseError::InvalidSql("Failed to parse expression".to_string())); diff --git a/backend/crates/kalamdb-handlers/crates/stream/src/topics/cleanup.rs b/backend/crates/kalamdb-handlers/crates/stream/src/topics/cleanup.rs index 932d374a..b6a298e9 100644 --- a/backend/crates/kalamdb-handlers/crates/stream/src/topics/cleanup.rs +++ b/backend/crates/kalamdb-handlers/crates/stream/src/topics/cleanup.rs @@ -11,4 +11,4 @@ pub(super) fn clear_topic_data( .topic_publisher() .clear_topic_data(topic_id) .map_err(|e| KalamDbError::ExecutionError(e.to_string())) -} \ No newline at end of file +} diff --git a/backend/crates/kalamdb-handlers/crates/stream/src/topics/clear.rs b/backend/crates/kalamdb-handlers/crates/stream/src/topics/clear.rs index 777eaa26..eb17bf0f 100644 --- a/backend/crates/kalamdb-handlers/crates/stream/src/topics/clear.rs +++ b/backend/crates/kalamdb-handlers/crates/stream/src/topics/clear.rs @@ -42,8 +42,8 @@ impl TypedStatementHandler for ClearTopicHandler { let topic_name = topic.expect("checked is_some").name; - let (offsets_deleted, messages_deleted) = - clear_topic_data(&self.app_context, topic_id).map_err(|e| { + let (offsets_deleted, messages_deleted) = clear_topic_data(&self.app_context, topic_id) + .map_err(|e| { KalamDbError::ExecutionError(format!( "Failed to clear topic '{}' ({}): {}", topic_name, @@ -63,9 +63,7 @@ impl TypedStatementHandler for ClearTopicHandler { Ok(ExecutionResult::Success { message: format!( "Cleared topic '{}' - {} consumer group offsets deleted, {} messages deleted", - topic_name, - offsets_deleted, - messages_deleted + topic_name, offsets_deleted, messages_deleted ), }) } diff --git a/backend/crates/kalamdb-handlers/crates/stream/src/topics/drop.rs b/backend/crates/kalamdb-handlers/crates/stream/src/topics/drop.rs index 8289d79a..4364a0fd 100644 --- a/backend/crates/kalamdb-handlers/crates/stream/src/topics/drop.rs +++ b/backend/crates/kalamdb-handlers/crates/stream/src/topics/drop.rs @@ -43,8 +43,8 @@ impl TypedStatementHandler for DropTopicHandler { let topic_name = topic.expect("checked is_some").name; - let (offsets_deleted, messages_deleted) = - clear_topic_data(&self.app_context, &topic_id).map_err(|e| { + let (offsets_deleted, messages_deleted) = clear_topic_data(&self.app_context, &topic_id) + .map_err(|e| { KalamDbError::ExecutionError(format!( "Failed to clean up dropped topic '{}' ({}): {}", topic_name, @@ -66,9 +66,7 @@ impl TypedStatementHandler for DropTopicHandler { Ok(ExecutionResult::Success { message: format!( "Dropped topic '{}' - {} consumer group offsets deleted, {} messages deleted", - topic_name, - offsets_deleted, - messages_deleted + topic_name, offsets_deleted, messages_deleted ), }) } diff --git a/backend/crates/kalamdb-handlers/crates/stream/src/topics/mod.rs b/backend/crates/kalamdb-handlers/crates/stream/src/topics/mod.rs index 0103859c..76147fd0 100644 --- a/backend/crates/kalamdb-handlers/crates/stream/src/topics/mod.rs +++ b/backend/crates/kalamdb-handlers/crates/stream/src/topics/mod.rs @@ -1,7 +1,7 @@ mod ack; mod add_source; -mod clear; mod cleanup; +mod clear; mod consume; mod create; mod drop; diff --git a/backend/crates/kalamdb-jobs/src/topic_retention.rs b/backend/crates/kalamdb-jobs/src/topic_retention.rs index 0874698c..c26e89a7 100644 --- a/backend/crates/kalamdb-jobs/src/topic_retention.rs +++ b/backend/crates/kalamdb-jobs/src/topic_retention.rs @@ -100,14 +100,12 @@ mod tests { }; use datafusion::scalar::ScalarValue; - use kalamdb_commons::{ - models::{rows::Row, NamespaceId, PayloadMode, TableId, TableName, TopicId, TopicOp}, + use kalamdb_commons::models::{ + rows::Row, NamespaceId, PayloadMode, TableId, TableName, TopicId, TopicOp, }; use kalamdb_core::test_helpers::test_app_context; use kalamdb_system::{ - providers::{ - topics::{models::Topic, TopicRoute}, - }, + providers::topics::{models::Topic, TopicRoute}, JobType, }; @@ -117,10 +115,7 @@ mod tests { fn create_test_row(id: i32, payload: &str) -> Row { let mut values = BTreeMap::new(); values.insert("id".to_string(), ScalarValue::Int32(Some(id))); - values.insert( - "payload".to_string(), - ScalarValue::Utf8(Some(payload.to_string())), - ); + values.insert("payload".to_string(), ScalarValue::Utf8(Some(payload.to_string()))); Row { values } } @@ -131,10 +126,8 @@ mod tests { retention_seconds: Option, retention_max_bytes: Option, ) -> TableId { - let table_id = TableId::new( - NamespaceId::new("topic_retention_jobs"), - TableName::new(table_name), - ); + let table_id = + TableId::new(NamespaceId::new("topic_retention_jobs"), TableName::new(table_name)); let mut topic = Topic::new(topic_id.clone(), topic_id.as_str().to_string()); topic.partitions = 1; topic.retention_seconds = retention_seconds; @@ -168,8 +161,7 @@ mod tests { let unique = chrono::Utc::now().timestamp_nanos_opt().unwrap_or(0); let retained_topic_id = TopicId::new(&format!("topic.scheduler.retained.{unique}")); - let within_limit_topic_id = - TopicId::new(&format!("topic.scheduler.within.limit.{unique}")); + let within_limit_topic_id = TopicId::new(&format!("topic.scheduler.within.limit.{unique}")); let disabled_topic_id = TopicId::new(&format!("topic.scheduler.disabled.{unique}")); let retained_table_id = create_topic_with_route( @@ -243,18 +235,12 @@ mod tests { "topic retention job should use the TR:: idempotency format" ); assert!( - !app_ctx - .system_tables() - .jobs() - .list_jobs() - .unwrap() - .into_iter() - .any(|job| { - job.job_type == JobType::TopicRetention - && job.idempotency_key.as_deref().is_some_and(|key| { - key.starts_with(&format!("TR:{}:", within_limit_topic_id.as_str())) - }) - }), + !app_ctx.system_tables().jobs().list_jobs().unwrap().into_iter().any(|job| { + job.job_type == JobType::TopicRetention + && job.idempotency_key.as_deref().is_some_and(|key| { + key.starts_with(&format!("TR:{}:", within_limit_topic_id.as_str())) + }) + }), "scheduler should skip retained topics that are already within limits" ); } diff --git a/backend/crates/kalamdb-publisher/src/service.rs b/backend/crates/kalamdb-publisher/src/service.rs index 2e8156e1..86657382 100644 --- a/backend/crates/kalamdb-publisher/src/service.rs +++ b/backend/crates/kalamdb-publisher/src/service.rs @@ -340,9 +340,10 @@ impl TopicPublisherService { /// /// Returns `(offsets_deleted, messages_deleted)`. pub fn clear_topic_data(&self, topic_id: &TopicId) -> Result<(usize, usize)> { - let offsets_deleted = self.offset_store.delete_topic_offsets(topic_id).map_err(|e| { - CommonError::Internal(format!("Failed to delete topic offsets: {}", e)) - })?; + let offsets_deleted = self + .offset_store + .delete_topic_offsets(topic_id) + .map_err(|e| CommonError::Internal(format!("Failed to delete topic offsets: {}", e)))?; let messages_deleted = self.message_store.delete_topic_messages(topic_id).map_err(|e| { CommonError::Internal(format!("Failed to delete topic messages: {}", e)) })?; @@ -1783,8 +1784,12 @@ mod tests { let topic_id = TopicId::new("group_claim_topic"); let group_id = ConsumerGroupId::new("test_group"); - let topic = - create_test_topic_with_partitions(topic_id.clone(), table_id.clone(), TopicOp::Insert, 1); + let topic = create_test_topic_with_partitions( + topic_id.clone(), + table_id.clone(), + TopicOp::Insert, + 1, + ); service.add_topic(topic); for idx in 0..10 { @@ -1995,8 +2000,12 @@ mod tests { let topic_id = TopicId::new("partial_ack_topic"); let group_id = ConsumerGroupId::new("partial_ack_group"); - let topic = - create_test_topic_with_partitions(topic_id.clone(), table_id.clone(), TopicOp::Insert, 1); + let topic = create_test_topic_with_partitions( + topic_id.clone(), + table_id.clone(), + TopicOp::Insert, + 1, + ); service.add_topic(topic); for idx in 0..20 { diff --git a/backend/crates/kalamdb-raft/src/executor/raft.rs b/backend/crates/kalamdb-raft/src/executor/raft.rs index 8c017c60..5a870d84 100644 --- a/backend/crates/kalamdb-raft/src/executor/raft.rs +++ b/backend/crates/kalamdb-raft/src/executor/raft.rs @@ -12,7 +12,7 @@ use std::{ use async_trait::async_trait; use dashmap::DashMap; use kalamdb_commons::models::{NodeId, UserId}; -use kalamdb_observability::collect_runtime_metrics; +use kalamdb_observability::{collect_runtime_metrics, SERVER_VERSION}; use kalamdb_pg::KalamPgService; use kalamdb_sharding::ShardRouter; use openraft::ServerState; @@ -426,10 +426,14 @@ impl CommandExecutor for RaftExecutor { .as_ref() .and_then(|p| p.hostname.clone()) .or_else(|| node.hostname.clone()), - version: peer_cache_entry - .as_ref() - .and_then(|p| p.version.clone()) - .or_else(|| node.version.clone()), + version: if is_self { + Some(SERVER_VERSION.to_string()) + } else { + peer_cache_entry + .as_ref() + .and_then(|p| p.version.clone()) + .or_else(|| node.version.clone()) + }, memory_mb: if is_self { Some(local_runtime.system_total_memory_mb) } else { diff --git a/backend/crates/kalamdb-raft/src/manager/raft_manager.rs b/backend/crates/kalamdb-raft/src/manager/raft_manager.rs index 0851827f..39a2e51f 100644 --- a/backend/crates/kalamdb-raft/src/manager/raft_manager.rs +++ b/backend/crates/kalamdb-raft/src/manager/raft_manager.rs @@ -99,6 +99,10 @@ pub struct RaftManager { /// Shared data shards (configurable, default 1) shared_data_shards: Vec>>, + /// Dedicated transport for non-Raft cluster RPCs such as SQL forwarding + /// and diagnostics. Raft groups keep their own shared consensus pool. + cluster_network_factory: RaftNetworkFactory, + /// Whether the manager has been started started: RwLock, @@ -175,6 +179,7 @@ impl RaftManager { let user_shards_count = config.user_shards; let shared_shards_count = config.shared_shards; let channel_pool = RaftNetworkFactory::new_channel_pool(); + let cluster_network_factory = RaftNetworkFactory::new(GroupId::Meta); // Create unified meta group let meta = Arc::new(RaftGroup::new_with_channel_pool( @@ -210,6 +215,7 @@ impl RaftManager { meta, user_data_shards, shared_data_shards, + cluster_network_factory, started: RwLock::new(false), config, runtime_peers: RwLock::new(HashMap::new()), @@ -233,6 +239,7 @@ impl RaftManager { let user_shards_count = config.user_shards; let shared_shards_count = config.shared_shards; let channel_pool = RaftNetworkFactory::new_channel_pool(); + let cluster_network_factory = RaftNetworkFactory::new(GroupId::Meta); // Ensure the raft_data partition exists let partition = Partition::new(RAFT_PARTITION_NAME); @@ -298,6 +305,7 @@ impl RaftManager { node_id: config.node_id, user_data_shards, shared_data_shards, + cluster_network_factory, started: RwLock::new(false), config, runtime_peers: RwLock::new(HashMap::new()), @@ -386,6 +394,8 @@ impl RaftManager { // Register this node for leader forwarding (covers self-forward when leader detection // lags). + self.cluster_network_factory + .configure_rpc_tls(&self.config.rpc_tls, &self.config.peers)?; self.register_peer( self.node_id, self.config.rpc_addr.clone(), @@ -1317,6 +1327,9 @@ impl RaftManager { self.runtime_peers.write().insert(node_id, node.clone()); + // Register with non-Raft cluster RPC transport (SQL forwarding, ping, diagnostics). + self.cluster_network_factory.register_node(node_id, node.clone()); + // Register with unified meta group self.meta.register_peer(node_id, node.clone()); @@ -1331,25 +1344,26 @@ impl RaftManager { /// Get a gRPC channel to a specific peer node. /// - /// Uses the Meta group's network factory channel pool (all groups share - /// the same `rpc_addr` per node, so one channel per node is sufficient). + /// Uses a dedicated non-Raft cluster RPC channel pool so SQL forwarding and + /// diagnostics cannot block Raft consensus traffic on the shared consensus + /// HTTP/2 transport. /// /// Returns `None` if the node is not registered. pub fn get_peer_channel(&self, node_id: NodeId) -> Option { - if let Some(channel) = self.meta.network_factory().get_or_create_channel(node_id) { + if let Some(channel) = self.cluster_network_factory.get_or_create_channel(node_id) { return Some(channel); } let node = self.node_for_node(node_id)?; - self.meta.register_peer(node_id, node); - self.meta.network_factory().get_or_create_channel(node_id) + self.cluster_network_factory.register_node(node_id, node); + self.cluster_network_factory.get_or_create_channel(node_id) } /// Get all registered peer nodes (id + node info) from the Meta group. /// /// Used by [`crate::network::cluster_client::ClusterClient`] for broadcasting. pub fn get_all_peers(&self) -> Vec<(NodeId, KalamNode)> { - self.meta.network_factory().get_all_peers() + self.cluster_network_factory.get_all_peers() } /// Get all group IDs diff --git a/backend/crates/kalamdb-raft/src/storage/types.rs b/backend/crates/kalamdb-raft/src/storage/types.rs index 5318049a..3219ef8f 100644 --- a/backend/crates/kalamdb-raft/src/storage/types.rs +++ b/backend/crates/kalamdb-raft/src/storage/types.rs @@ -6,6 +6,7 @@ use std::io::Cursor; +use kalamdb_observability::SERVER_VERSION; use openraft::{Entry, RaftTypeConfig}; use serde::{Deserialize, Serialize}; @@ -108,7 +109,7 @@ impl KalamNode { rpc_addr: rpc_addr.into(), api_addr: api_addr.into(), hostname: Self::detect_hostname(), - version: Some(env!("CARGO_PKG_VERSION").to_string()), + version: Some(SERVER_VERSION.to_string()), memory_mb: Self::detect_memory_mb(), os: Some(std::env::consts::OS.to_string()), arch: Some(std::env::consts::ARCH.to_string()), diff --git a/backend/tests/integration_tests/topic_pubsub.rs b/backend/tests/integration_tests/topic_pubsub.rs index 6ff8be1b..1b24a0f1 100644 --- a/backend/tests/integration_tests/topic_pubsub.rs +++ b/backend/tests/integration_tests/topic_pubsub.rs @@ -827,7 +827,8 @@ async fn test_consume_user_role_forbidden() { .as_ref() .map(|e| { e.message.contains("service, dba, or system") - || e.message.contains("Regular users may only execute SELECT and DML statements") + || e.message + .contains("Regular users may only execute SELECT and DML statements") }) .unwrap_or(false), "Error message should mention required roles: {:?}", @@ -2107,8 +2108,7 @@ async fn test_clear_topic() { "CLEAR TOPIC should succeed: {:?}", clear_result.error ); - assert_topic_offset_state(&server, "test_clear_ns.messages_topic", "clear_offsets", None) - .await; + assert_topic_offset_state(&server, "test_clear_ns.messages_topic", "clear_offsets", None).await; assert!( server .app_context @@ -2244,8 +2244,7 @@ async fn test_drop_topic_cleans_up_data_immediately() { chrono::Utc::now().timestamp_millis(), )) .expect("Failed to seed topic offset before DROP TOPIC"); - assert_topic_offset_state(&server, "test_drop_ns.events_topic", "drop_offsets", Some(1)) - .await; + assert_topic_offset_state(&server, "test_drop_ns.events_topic", "drop_offsets", Some(1)).await; // Drop the topic let drop_result = server.execute_sql("DROP TOPIC test_drop_ns.events_topic").await; diff --git a/backend/tests/misc/auth/test_soft_delete.rs b/backend/tests/misc/auth/test_soft_delete.rs index 2d053608..273a0b98 100644 --- a/backend/tests/misc/auth/test_soft_delete.rs +++ b/backend/tests/misc/auth/test_soft_delete.rs @@ -254,14 +254,12 @@ async fn test_delete_with_where_clause() { // Setup fixtures::create_namespace(&server, &namespace).await; server - .execute_sql( - &format!( - "CREATE TABLE {}.tasks (\n id TEXT PRIMARY KEY,\n \ + .execute_sql(&format!( + "CREATE TABLE {}.tasks (\n id TEXT PRIMARY KEY,\n \ title TEXT,\n priority INT\n ) WITH (\n \ TYPE = 'USER',\n STORAGE_ID = 'local'\n )", - namespace - ), - ) + namespace + )) .await; // Insert tasks with different priorities @@ -326,14 +324,12 @@ async fn test_count_excludes_deleted_rows() { // Setup fixtures::create_namespace(&server, &namespace).await; server - .execute_sql( - &format!( - "CREATE TABLE {}.tasks (\n id TEXT PRIMARY KEY,\n \ + .execute_sql(&format!( + "CREATE TABLE {}.tasks (\n id TEXT PRIMARY KEY,\n \ title TEXT\n ) WITH (\n TYPE = 'USER',\n \ STORAGE_ID = 'local'\n )", - namespace - ), - ) + namespace + )) .await; // Insert 5 tasks diff --git a/backend/tests/misc/production/test_mvcc_phase2.rs b/backend/tests/misc/production/test_mvcc_phase2.rs index 7a3661c5..c6240d43 100644 --- a/backend/tests/misc/production/test_mvcc_phase2.rs +++ b/backend/tests/misc/production/test_mvcc_phase2.rs @@ -33,9 +33,8 @@ async fn test_create_table_without_pk_rejected() { // Try to create table without PRIMARY KEY specification let response = server - .execute_sql( - &format!( - r#"CREATE TABLE {}.invalid_table ( + .execute_sql(&format!( + r#"CREATE TABLE {}.invalid_table ( id TEXT, name TEXT, value INT @@ -43,9 +42,8 @@ async fn test_create_table_without_pk_rejected() { TYPE = 'USER', STORAGE_ID = 'local' )"#, - ns - ), - ) + ns + )) .await; // Should fail with error about missing primary key @@ -81,9 +79,8 @@ async fn test_create_table_auto_adds_system_columns() { // Create table with user-defined PK let response = server - .execute_sql( - &format!( - r#"CREATE TABLE {}.products ( + .execute_sql(&format!( + r#"CREATE TABLE {}.products ( id TEXT PRIMARY KEY, name TEXT, price INT @@ -91,9 +88,8 @@ async fn test_create_table_auto_adds_system_columns() { TYPE = 'USER', STORAGE_ID = 'local' )"#, - ns - ), - ) + ns + )) .await; assert_eq!( @@ -172,18 +168,16 @@ async fn test_insert_storage_key_format() { // Create user table let resp = server - .execute_sql( - &format!( - r#"CREATE TABLE {}.user_data ( + .execute_sql(&format!( + r#"CREATE TABLE {}.user_data ( id TEXT PRIMARY KEY, content TEXT ) WITH ( TYPE = 'USER', STORAGE_ID = 'local' )"#, - ns - ), - ) + ns + )) .await; assert_eq!( resp.status, @@ -289,9 +283,8 @@ async fn test_user_table_row_structure() { ); let resp = server - .execute_sql( - &format!( - r#"CREATE TABLE {}.user_records ( + .execute_sql(&format!( + r#"CREATE TABLE {}.user_records ( record_id TEXT PRIMARY KEY, title TEXT, priority INT @@ -299,9 +292,8 @@ async fn test_user_table_row_structure() { TYPE = 'USER', STORAGE_ID = 'local' )"#, - ns - ), - ) + ns + )) .await; assert_eq!(resp.status, ResponseStatus::Success, "Table creation failed: {:?}", resp.error); @@ -457,18 +449,16 @@ async fn test_insert_duplicate_pk_rejected() { // Setup fixtures::create_namespace(&server, &ns).await; server - .execute_sql( - &format!( - r#"CREATE TABLE {}.unique_items ( + .execute_sql(&format!( + r#"CREATE TABLE {}.unique_items ( item_id TEXT PRIMARY KEY, name TEXT ) WITH ( TYPE = 'USER', STORAGE_ID = 'local' )"#, - ns - ), - ) + ns + )) .await; // Insert first record WITH explicit PK @@ -568,18 +558,16 @@ async fn test_incremental_sync_seq_threshold() { ); let resp = server - .execute_sql( - &format!( - r#"CREATE TABLE {}.sync_records ( + .execute_sql(&format!( + r#"CREATE TABLE {}.sync_records ( id TEXT PRIMARY KEY, version INT ) WITH ( TYPE = 'USER', STORAGE_ID = 'local' )"#, - ns - ), - ) + ns + )) .await; assert_eq!(resp.status, ResponseStatus::Success, "Table creation failed: {:?}", resp.error); @@ -658,18 +646,16 @@ async fn test_rocksdb_prefix_scan_user_isolation() { // Setup fixtures::create_namespace(&server, &ns).await; server - .execute_sql( - &format!( - r#"CREATE TABLE {}.user_notes ( + .execute_sql(&format!( + r#"CREATE TABLE {}.user_notes ( note_id TEXT PRIMARY KEY, content TEXT ) WITH ( TYPE = 'USER', STORAGE_ID = 'local' )"#, - ns - ), - ) + ns + )) .await; // Insert data for user1 @@ -748,18 +734,16 @@ async fn test_rocksdb_range_scan_efficiency() { // Setup fixtures::create_namespace(&server, &ns).await; server - .execute_sql( - &format!( - r#"CREATE TABLE {}.versioned_data ( + .execute_sql(&format!( + r#"CREATE TABLE {}.versioned_data ( id TEXT PRIMARY KEY, value INT ) WITH ( TYPE = 'USER', STORAGE_ID = 'local' )"#, - ns - ), - ) + ns + )) .await; // Insert initial version diff --git a/backend/tests/misc/schema/test_alter_table.rs b/backend/tests/misc/schema/test_alter_table.rs index aae62bec..2b40f52b 100644 --- a/backend/tests/misc/schema/test_alter_table.rs +++ b/backend/tests/misc/schema/test_alter_table.rs @@ -23,9 +23,8 @@ async fn test_alter_table_add_column() { // Setup fixtures::create_namespace(&server, &ns).await; let create_response = server - .execute_sql( - &format!( - r#"CREATE TABLE {}.products ( + .execute_sql(&format!( + r#"CREATE TABLE {}.products ( id TEXT PRIMARY KEY, name TEXT, price INT @@ -33,9 +32,8 @@ async fn test_alter_table_add_column() { TYPE = 'USER', STORAGE_ID = 'local' )"#, - ns - ), - ) + ns + )) .await; assert_eq!(create_response.status, ResponseStatus::Success); @@ -53,9 +51,7 @@ async fn test_alter_table_add_column() { // ALTER TABLE: ADD COLUMN let alter_response = server - .execute_sql( - &format!(r#"ALTER TABLE {}.products ADD COLUMN stock INT"#, ns), - ) + .execute_sql(&format!(r#"ALTER TABLE {}.products ADD COLUMN stock INT"#, ns)) .await; assert_eq!( @@ -112,9 +108,8 @@ async fn test_alter_table_drop_column() { assert!(server.namespace_exists(&ns).await, "Namespace should exist after creation"); let create_resp = server - .execute_sql( - &format!( - r#"CREATE TABLE {}.inventory ( + .execute_sql(&format!( + r#"CREATE TABLE {}.inventory ( id TEXT PRIMARY KEY, item TEXT, quantity INT, @@ -123,9 +118,8 @@ async fn test_alter_table_drop_column() { TYPE = 'USER', STORAGE_ID = 'local' )"#, - ns - ), - ) + ns + )) .await; assert_eq!( create_resp.status, @@ -148,9 +142,7 @@ async fn test_alter_table_drop_column() { // ALTER TABLE: DROP COLUMN let alter_response = server - .execute_sql( - &format!(r#"ALTER TABLE {}.inventory DROP COLUMN warehouse"#, ns), - ) + .execute_sql(&format!(r#"ALTER TABLE {}.inventory DROP COLUMN warehouse"#, ns)) .await; assert_eq!( @@ -191,9 +183,8 @@ async fn test_alter_table_rename_column() { // Setup fixtures::create_namespace(&server, &ns).await; server - .execute_sql( - &format!( - r#"CREATE TABLE {}.customers ( + .execute_sql(&format!( + r#"CREATE TABLE {}.customers ( id TEXT PRIMARY KEY, customer_name TEXT, email TEXT @@ -201,9 +192,8 @@ async fn test_alter_table_rename_column() { TYPE = 'USER', STORAGE_ID = 'local' )"#, - ns - ), - ) + ns + )) .await; // Insert data @@ -220,9 +210,10 @@ async fn test_alter_table_rename_column() { // ALTER TABLE: RENAME COLUMN let alter_response = server - .execute_sql( - &format!(r#"ALTER TABLE {}.customers RENAME COLUMN customer_name TO name"#, ns), - ) + .execute_sql(&format!( + r#"ALTER TABLE {}.customers RENAME COLUMN customer_name TO name"#, + ns + )) .await; assert_eq!( @@ -236,9 +227,7 @@ async fn test_alter_table_rename_column() { // Full schema evolution support (column aliasing during scan) is not yet implemented. // For now, verify that the schema metadata was updated by checking DESCRIBE TABLE. - let describe_response = server - .execute_sql(&format!("DESCRIBE TABLE {}.customers", ns)) - .await; + let describe_response = server.execute_sql(&format!("DESCRIBE TABLE {}.customers", ns)).await; assert_eq!( describe_response.status, @@ -279,9 +268,8 @@ async fn test_alter_table_modify_column() { // Setup fixtures::create_namespace(&server, &ns).await; server - .execute_sql( - &format!( - r#"CREATE TABLE {}.metrics ( + .execute_sql(&format!( + r#"CREATE TABLE {}.metrics ( id TEXT PRIMARY KEY, value INT, description TEXT @@ -289,16 +277,13 @@ async fn test_alter_table_modify_column() { TYPE = 'USER', STORAGE_ID = 'local' )"#, - ns - ), - ) + ns + )) .await; // ALTER TABLE: MODIFY COLUMN (change type) let alter_response = server - .execute_sql( - &format!(r#"ALTER TABLE {}.metrics MODIFY COLUMN value BIGINT"#, ns), - ) + .execute_sql(&format!(r#"ALTER TABLE {}.metrics MODIFY COLUMN value BIGINT"#, ns)) .await; assert_eq!( @@ -343,31 +328,25 @@ async fn test_alter_table_schema_versioning() { // Setup fixtures::create_namespace(&server, &ns).await; server - .execute_sql( - &format!( - r#"CREATE TABLE {}.versioned ( + .execute_sql(&format!( + r#"CREATE TABLE {}.versioned ( id TEXT PRIMARY KEY, col1 TEXT ) WITH ( TYPE = 'USER', STORAGE_ID = 'local' )"#, - ns - ), - ) + ns + )) .await; // Perform multiple ALTER operations server - .execute_sql( - &format!(r#"ALTER TABLE {}.versioned ADD COLUMN col2 INT"#, ns), - ) + .execute_sql(&format!(r#"ALTER TABLE {}.versioned ADD COLUMN col2 INT"#, ns)) .await; server - .execute_sql( - &format!(r#"ALTER TABLE {}.versioned ADD COLUMN col3 TEXT"#, ns), - ) + .execute_sql(&format!(r#"ALTER TABLE {}.versioned ADD COLUMN col3 TEXT"#, ns)) .await; // Query should work with all columns (schema evolution tracked internally) diff --git a/backend/tests/misc/schema/test_alter_table_after_flush.rs b/backend/tests/misc/schema/test_alter_table_after_flush.rs index e82a173e..7a6b5582 100644 --- a/backend/tests/misc/schema/test_alter_table_after_flush.rs +++ b/backend/tests/misc/schema/test_alter_table_after_flush.rs @@ -248,9 +248,7 @@ async fn test_multiple_alter_operations_with_flushes() { // Second ALTER server - .execute_sql( - &format!("ALTER TABLE {}.events ADD COLUMN timestamp BIGINT", ns), - ) + .execute_sql(&format!("ALTER TABLE {}.events ADD COLUMN timestamp BIGINT", ns)) .await; // Third batch with both new columns diff --git a/backend/tests/misc/sql/test_pk_index_efficiency.rs b/backend/tests/misc/sql/test_pk_index_efficiency.rs index bde29ff4..b6859c30 100644 --- a/backend/tests/misc/sql/test_pk_index_efficiency.rs +++ b/backend/tests/misc/sql/test_pk_index_efficiency.rs @@ -45,9 +45,8 @@ async fn test_user_table_pk_index_update() { // Setup namespace and user table fixtures::create_namespace(&server, &ns).await; let create_response = server - .execute_sql( - &format!( - r#"CREATE TABLE {}.user_items ( + .execute_sql(&format!( + r#"CREATE TABLE {}.user_items ( id INT PRIMARY KEY, name TEXT, value INT @@ -55,9 +54,8 @@ async fn test_user_table_pk_index_update() { TYPE = 'USER', STORAGE_ID = 'local' )"#, - ns - ), - ) + ns + )) .await; assert_eq!( create_response.status, @@ -355,18 +353,16 @@ async fn test_user_table_pk_index_select() { // Setup fixtures::create_namespace(&server, &ns).await; let create_response = server - .execute_sql( - &format!( - r#"CREATE TABLE {}.records ( + .execute_sql(&format!( + r#"CREATE TABLE {}.records ( id INT PRIMARY KEY, data TEXT ) WITH ( TYPE = 'USER', STORAGE_ID = 'local' )"#, - ns - ), - ) + ns + )) .await; assert_eq!( create_response.status, @@ -472,18 +468,16 @@ async fn test_user_table_pk_index_delete() { // Setup fixtures::create_namespace(&server, &ns).await; let create_response = server - .execute_sql( - &format!( - r#"CREATE TABLE {}.items ( + .execute_sql(&format!( + r#"CREATE TABLE {}.items ( id INT PRIMARY KEY, description TEXT ) WITH ( TYPE = 'USER', STORAGE_ID = 'local' )"#, - ns - ), - ) + ns + )) .await; assert_eq!( create_response.status, @@ -605,9 +599,8 @@ async fn test_user_table_pk_index_update_after_flush() { // Setup namespace and user table fixtures::create_namespace(&server, &ns).await; let create_response = server - .execute_sql( - &format!( - r#"CREATE TABLE {}.user_items ( + .execute_sql(&format!( + r#"CREATE TABLE {}.user_items ( id INT PRIMARY KEY, name TEXT, value INT @@ -615,9 +608,8 @@ async fn test_user_table_pk_index_update_after_flush() { TYPE = 'USER', STORAGE_ID = 'local' )"#, - ns - ), - ) + ns + )) .await; assert_eq!( create_response.status, @@ -884,18 +876,16 @@ async fn test_user_table_pk_index_isolation() { // Setup fixtures::create_namespace(&server, &ns).await; let create_response = server - .execute_sql( - &format!( - r#"CREATE TABLE {}.user_data ( + .execute_sql(&format!( + r#"CREATE TABLE {}.user_data ( id INT PRIMARY KEY, secret TEXT ) WITH ( TYPE = 'USER', STORAGE_ID = 'local' )"#, - ns - ), - ) + ns + )) .await; assert_eq!( create_response.status, diff --git a/backend/tests/misc/sql/test_row_count_behavior.rs b/backend/tests/misc/sql/test_row_count_behavior.rs index 7b4353ff..9ba7b494 100644 --- a/backend/tests/misc/sql/test_row_count_behavior.rs +++ b/backend/tests/misc/sql/test_row_count_behavior.rs @@ -48,18 +48,16 @@ async fn test_update_returns_correct_row_count() { // Setup fixtures::create_namespace(&server, &ns).await; server - .execute_sql( - &format!( - "CREATE TABLE {}.users (id TEXT PRIMARY KEY, + .execute_sql(&format!( + "CREATE TABLE {}.users (id TEXT PRIMARY KEY, name TEXT, email TEXT ) WITH ( TYPE = 'USER', STORAGE_ID = 'local' )", - ns - ), - ) + ns + )) .await; // Insert test data @@ -115,17 +113,15 @@ async fn test_update_same_values_returns_zero() { // Setup fixtures::create_namespace(&server, &ns).await; server - .execute_sql( - &format!( - "CREATE TABLE {}.users (id TEXT PRIMARY KEY, + .execute_sql(&format!( + "CREATE TABLE {}.users (id TEXT PRIMARY KEY, name TEXT ) WITH ( TYPE = 'USER', STORAGE_ID = 'local' )", - ns - ), - ) + ns + )) .await; // Insert test data @@ -157,17 +153,15 @@ async fn test_delete_returns_correct_row_count() { // Setup fixtures::create_namespace(&server, &ns).await; server - .execute_sql( - &format!( - "CREATE TABLE {}.tasks (id TEXT PRIMARY KEY, + .execute_sql(&format!( + "CREATE TABLE {}.tasks (id TEXT PRIMARY KEY, title TEXT ) WITH ( TYPE = 'USER', STORAGE_ID = 'local' )", - ns - ), - ) + ns + )) .await; // Insert test data @@ -215,17 +209,15 @@ async fn test_delete_already_deleted_returns_zero() { // Setup fixtures::create_namespace(&server, &ns).await; server - .execute_sql( - &format!( - "CREATE TABLE {}.tasks (id TEXT PRIMARY KEY, + .execute_sql(&format!( + "CREATE TABLE {}.tasks (id TEXT PRIMARY KEY, title TEXT ) WITH ( TYPE = 'USER', STORAGE_ID = 'local' )", - ns - ), - ) + ns + )) .await; // Insert test data @@ -305,17 +297,15 @@ async fn test_delete_multiple_rows_count() { // Setup fixtures::create_namespace(&server, &ns).await; server - .execute_sql( - &format!( - "CREATE TABLE {}.tasks (id TEXT PRIMARY KEY, + .execute_sql(&format!( + "CREATE TABLE {}.tasks (id TEXT PRIMARY KEY, priority INT ) WITH ( TYPE = 'USER', STORAGE_ID = 'local' )", - ns - ), - ) + ns + )) .await; // Insert 5 tasks with priority 1 diff --git a/backend/tests/misc/sql/test_sql_error_redaction.rs b/backend/tests/misc/sql/test_sql_error_redaction.rs index 1410b28b..60b6d811 100644 --- a/backend/tests/misc/sql/test_sql_error_redaction.rs +++ b/backend/tests/misc/sql/test_sql_error_redaction.rs @@ -20,10 +20,7 @@ async fn test_non_admin_sql_errors_redact_table_details() { let user_response = server.execute_sql_as_user(&sql, &username).await; assert_eq!(user_response.status, ResponseStatus::Error); let user_error = user_response.error.expect("user response should include an error payload"); - assert_eq!( - user_error.message, - "SQL statement failed. Review the statement and try again.", - ); + assert_eq!(user_error.message, "SQL statement failed. Review the statement and try again.",); assert!(user_error.details.is_none()); assert!(!user_error.message.contains(&namespace)); assert!(!user_error.message.contains(&table_name)); diff --git a/backend/tests/misc/sql/test_update_delete_version_resolution.rs b/backend/tests/misc/sql/test_update_delete_version_resolution.rs index 0cc2c25a..b61eaf97 100644 --- a/backend/tests/misc/sql/test_update_delete_version_resolution.rs +++ b/backend/tests/misc/sql/test_update_delete_version_resolution.rs @@ -338,9 +338,8 @@ async fn test_delete_excludes_record() { // Setup fixtures::create_namespace(&server, &namespace).await; server - .execute_sql( - &format!( - r#"CREATE TABLE {}.users ( + .execute_sql(&format!( + r#"CREATE TABLE {}.users ( id TEXT PRIMARY KEY, name TEXT, active BOOLEAN @@ -348,9 +347,8 @@ async fn test_delete_excludes_record() { TYPE = 'USER', STORAGE_ID = 'local' )"#, - namespace - ), - ) + namespace + )) .await; // Insert records @@ -601,18 +599,16 @@ async fn test_query_performance_with_multiple_versions() { // Setup fixtures::create_namespace(&server, &namespace).await; server - .execute_sql( - &format!( - r#"CREATE TABLE {}.{} ( + .execute_sql(&format!( + r#"CREATE TABLE {}.{} ( id TEXT PRIMARY KEY, version INT ) WITH ( TYPE = 'USER', STORAGE_ID = 'local' )"#, - namespace, table - ), - ) + namespace, table + )) .await; // Insert initial version diff --git a/backend/tests/misc/storage/test_cold_storage_manifest.rs b/backend/tests/misc/storage/test_cold_storage_manifest.rs index 8f2e099a..33eb8575 100644 --- a/backend/tests/misc/storage/test_cold_storage_manifest.rs +++ b/backend/tests/misc/storage/test_cold_storage_manifest.rs @@ -39,9 +39,8 @@ async fn test_user_table_cold_storage_uses_manifest() { // Setup namespace and user table fixtures::create_namespace(&server, &ns).await; let create_response = server - .execute_sql( - &format!( - r#"CREATE TABLE {}.items ( + .execute_sql(&format!( + r#"CREATE TABLE {}.items ( id INT PRIMARY KEY, name TEXT, value INT @@ -49,9 +48,8 @@ async fn test_user_table_cold_storage_uses_manifest() { TYPE = 'USER', STORAGE_ID = 'local' )"#, - ns - ), - ) + ns + )) .await; assert_eq!( create_response.status, diff --git a/cargo_check_output.txt b/cargo_check_output.txt new file mode 100644 index 00000000..a54b73cd --- /dev/null +++ b/cargo_check_output.txt @@ -0,0 +1,2 @@ + Checking kalamdb-raft v0.5.0-beta.1 (/Users/jamal/git/KalamDB/backend/crates/kalamdb-raft) + Finished `dev` profile [unoptimized] target(s) in 5.61s diff --git a/cli/run-tests.sh b/cli/run-tests.sh index 279de416..91be42ff 100755 --- a/cli/run-tests.sh +++ b/cli/run-tests.sh @@ -35,6 +35,7 @@ if [ "${KALAMDB_ROOT_PASSWORD+x}" = "x" ]; then ROOT_PASSWORD_SET=true fi TEST_JOBS="${KALAMDB_TEST_JOBS:-}" +TEST_JOBS_AUTO=false TEST_FILTER="" TEST_LIST_FILE="" TEST_TARGET="" @@ -110,6 +111,7 @@ if [ "$SHOW_HELP" = true ]; then echo " --cluster-urls Comma-separated cluster node URLs" echo " --server-type Server mode: fresh | running | cluster" echo " -j, --jobs Override nextest process concurrency" + echo " Cluster mode defaults to KALAMDB_CLUSTER_TEST_JOBS or 4" echo " -P, --package Limit the run to one package (repeatable)" echo " -p, --password Root/admin password" echo " -t, --test Test filter (e.g., 'smoke', 'smoke_test_core')" @@ -159,21 +161,49 @@ detect_cluster_urls_from_health() { return 1 fi - curl -fsS --max-time 2 "${base_url%/}/v1/api/cluster/health" 2>/dev/null | python3 -c ' + curl -fsS --max-time 2 "${base_url%/}/v1/api/cluster/health" 2>/dev/null | python3 - "$base_url" <<'PY' import json import sys +from urllib.parse import urlparse + + +def normalize_api_addr(api_addr: str, base_url: str) -> str: + raw = api_addr.strip() + if not raw: + return "" + + base = urlparse(base_url.strip()) + base_scheme = base.scheme or "http" + base_host = base.hostname or "127.0.0.1" + + parsed = urlparse(raw if "://" in raw else f"{base_scheme}://{raw}") + host = parsed.hostname or "" + if host in {"0.0.0.0", "::", "[::]"}: + host = base_host + + if not host: + return "" + + port = parsed.port + if port is None: + port = 443 if parsed.scheme == "https" else 80 + + scheme = parsed.scheme or base_scheme + return f"{scheme}://{host}:{port}" try: payload = json.load(sys.stdin) except Exception: raise SystemExit(1) +base_url = sys.argv[1].strip() if len(sys.argv) > 1 else "" + if not payload.get("is_cluster_mode"): raise SystemExit(1) urls = [] for node in payload.get("nodes") or []: - api_addr = str(node.get("api_addr") or "").strip() + api_addr = normalize_api_addr(str(node.get("api_addr") or ""), base_url) if api_addr and api_addr not in urls: urls.append(api_addr) @@ -181,7 +211,7 @@ if len(urls) <= 1: raise SystemExit(1) print(",".join(urls)) -' +PY } autodetect_cluster_mode() { @@ -204,6 +234,11 @@ autodetect_cluster_mode() { autodetect_cluster_mode +if [ "$SERVER_TYPE" = "cluster" ] && [ -z "$TEST_JOBS" ]; then + TEST_JOBS="${KALAMDB_CLUSTER_TEST_JOBS:-4}" + TEST_JOBS_AUTO=true +fi + parse_host_port_from_url() { local url="$1" @@ -289,12 +324,38 @@ validate_cluster_health() { import json import sys from urllib.error import URLError +from urllib.parse import urlparse from urllib.request import urlopen target_url = sys.argv[1].rstrip("/") cluster_urls_arg = sys.argv[2].strip() if len(sys.argv) > 2 else "" +def normalize_cluster_url(raw_url, fallback_url): + raw = raw_url.strip() + if not raw: + return "" + + fallback = urlparse(fallback_url) + fallback_scheme = fallback.scheme or "http" + fallback_host = fallback.hostname or "127.0.0.1" + + parsed = urlparse(raw if "://" in raw else f"{fallback_scheme}://{raw}") + host = parsed.hostname or "" + if host in {"0.0.0.0", "::", "[::]"}: + host = fallback_host + + if not host: + return "" + + port = parsed.port + if port is None: + port = 443 if parsed.scheme == "https" else 80 + + scheme = parsed.scheme or fallback_scheme + return f"{scheme}://{host}:{port}" + + def fetch_health(base_url): with urlopen(f"{base_url.rstrip('/')}/v1/api/cluster/health", timeout=3) as response: return json.load(response) @@ -311,10 +372,11 @@ if not target_payload.get("is_cluster_mode"): raise SystemExit(0) cluster_urls = [url.strip() for url in cluster_urls_arg.split(",") if url.strip()] +cluster_urls = [normalized for url in cluster_urls if (normalized := normalize_cluster_url(url, target_url))] if not cluster_urls: seen = set() for node in target_payload.get("nodes") or []: - api_addr = str(node.get("api_addr") or "").strip() + api_addr = normalize_cluster_url(str(node.get("api_addr") or ""), target_url) if api_addr and api_addr not in seen: seen.add(api_addr) cluster_urls.append(api_addr) @@ -469,7 +531,11 @@ if [ -n "$TEST_LIST_FILE" ]; then echo "Test List: $TEST_LIST_FILE" fi if [ -n "$TEST_JOBS" ]; then - echo "Jobs: $TEST_JOBS" + if [ "$TEST_JOBS_AUTO" = true ]; then + echo "Jobs: $TEST_JOBS (cluster default)" + else + echo "Jobs: $TEST_JOBS" + fi fi echo "Mode: $FEATURE_MODE" echo "Supplementary: $SUPPLEMENTARY_MODE" @@ -600,9 +666,17 @@ build_test_cmd() { local test_filter="$1" TEST_CMD=( cargo nextest run - --all-targets ) + local filter_targets_smoke=false + if [ -n "$test_filter" ] && [[ "$test_filter" == smoke* ]]; then + filter_targets_smoke=true + fi + + if [ -z "$TEST_TARGET" ] && [ "$filter_targets_smoke" = false ]; then + TEST_CMD+=(--all-targets) + fi + if [ ${#PACKAGE_FILTERS[@]} -gt 0 ]; then local package for package in "${PACKAGE_FILTERS[@]}"; do @@ -632,7 +706,7 @@ build_test_cmd() { fi if [ -n "$test_filter" ]; then - if [ -z "$TEST_TARGET" ] && [[ "$test_filter" == smoke* ]]; then + if [ -z "$TEST_TARGET" ] && [ "$filter_targets_smoke" = true ]; then TEST_CMD+=(--test smoke) if [[ "$test_filter" != "smoke" ]]; then TEST_CMD+=("$test_filter") diff --git a/cli/src/connect.rs b/cli/src/connect.rs index cd37ad78..b7589180 100644 --- a/cli/src/connect.rs +++ b/cli/src/connect.rs @@ -1,8 +1,4 @@ -use std::{ - io::IsTerminal, - net::IpAddr, - time::Duration, -}; +use std::{io::IsTerminal, net::IpAddr, time::Duration}; use colored::Colorize; use kalam_cli::{ @@ -258,8 +254,8 @@ pub async fn create_session( println!(); // Get DBA user - let username = prompt_line("Enter the user for your DBA account: ") - .map_err(|e| e.to_string())?; + let username = + prompt_line("Enter the user for your DBA account: ").map_err(|e| e.to_string())?; if username.is_empty() { return Err("User cannot be empty".to_string()); } @@ -268,8 +264,8 @@ pub async fn create_session( } // Get DBA password - let password = prompt_password("Enter password for your DBA account: ") - .map_err(|e| e.to_string())?; + let password = + prompt_password("Enter password for your DBA account: ").map_err(|e| e.to_string())?; if password.is_empty() { return Err("Password cannot be empty".to_string()); } @@ -458,8 +454,8 @@ pub async fn create_session( let authenticated_user = login_response.user.id.to_string(); // Ask if user wants to save credentials - let save_choice = prompt_line("\nSave credentials for future use? (y/N): ") - .unwrap_or_default(); + let save_choice = + prompt_line("\nSave credentials for future use? (y/N): ").unwrap_or_default(); if save_choice.trim().eq_ignore_ascii_case("y") || save_choice.trim().eq_ignore_ascii_case("yes") diff --git a/cli/src/session.rs b/cli/src/session.rs index a00bd5dd..6a609a65 100644 --- a/cli/src/session.rs +++ b/cli/src/session.rs @@ -958,14 +958,11 @@ impl CLISession { return Ok(None); }; - let status = serde_json::from_value::( - status_value.clone().into_inner(), - ) - .map_err(|_| { - CLIError::ParseError( - "Subscription status has an invalid format".into(), - ) - })?; + let status = + serde_json::from_value::(status_value.clone().into_inner()) + .map_err(|_| { + CLIError::ParseError("Subscription status has an invalid format".into()) + })?; if status != SqlSubscriptionStatus::SubscriptionRequired { return Ok(None); diff --git a/cli/src/terminal_input.rs b/cli/src/terminal_input.rs index ea2431cb..f2b81c3e 100644 --- a/cli/src/terminal_input.rs +++ b/cli/src/terminal_input.rs @@ -40,4 +40,4 @@ pub fn prompt_line(prompt: &str) -> io::Result { pub fn prompt_password(prompt: &str) -> io::Result { let _guard = CookedTerminalGuard::acquire()?; rpassword::prompt_password(prompt) -} \ No newline at end of file +} diff --git a/cli/tests/cluster.rs b/cli/tests/cluster.rs index 027f631d..d63d85ca 100644 --- a/cli/tests/cluster.rs +++ b/cli/tests/cluster.rs @@ -19,7 +19,10 @@ mod common; mod cluster_common { use std::{ collections::HashMap, - sync::{Mutex, OnceLock}, + sync::{ + atomic::{AtomicU64, Ordering}, + Mutex, OnceLock, + }, time::Duration, }; @@ -28,6 +31,61 @@ mod cluster_common { use crate::common::*; + struct ClusterHelperStats { + calls: AtomicU64, + attempts: AtomicU64, + retryable_errors: AtomicU64, + total_micros: AtomicU64, + } + + impl ClusterHelperStats { + const fn new() -> Self { + Self { + calls: AtomicU64::new(0), + attempts: AtomicU64::new(0), + retryable_errors: AtomicU64::new(0), + total_micros: AtomicU64::new(0), + } + } + } + + static CLUSTER_HELPER_STATS: ClusterHelperStats = ClusterHelperStats::new(); + + fn cluster_helper_timing_enabled() -> bool { + std::env::var("KALAMDB_TRACE_CLUSTER_HELPERS") + .map(|value| matches!(value.trim(), "1" | "true" | "TRUE" | "yes" | "YES")) + .unwrap_or(false) + } + + fn record_cluster_helper_timing( + op: &str, + sql: &str, + elapsed: Duration, + attempts: u64, + retryable_errors: u64, + ) { + CLUSTER_HELPER_STATS.calls.fetch_add(1, Ordering::Relaxed); + CLUSTER_HELPER_STATS.attempts.fetch_add(attempts, Ordering::Relaxed); + CLUSTER_HELPER_STATS + .retryable_errors + .fetch_add(retryable_errors, Ordering::Relaxed); + CLUSTER_HELPER_STATS + .total_micros + .fetch_add(elapsed.as_micros() as u64, Ordering::Relaxed); + + if cluster_helper_timing_enabled() || retryable_errors > 0 { + let first_line = sql.lines().next().unwrap_or(sql).trim(); + eprintln!( + "[cluster-helper] op={} attempts={} retryable={} took_ms={:.3} sql={}", + op, + attempts, + retryable_errors, + elapsed.as_secs_f64() * 1000.0, + first_line + ); + } + } + /// Get cluster node URLs from environment or use defaults pub fn cluster_urls() -> Vec { get_available_server_urls() @@ -50,41 +108,25 @@ mod cluster_common { }) } - /// Per-URL client cache for cluster helpers. + /// Client cache for cluster helpers. /// /// Reusing a `KalamLinkClient` per URL avoids spawning a fresh reqwest connection pool /// on every `execute_on_node` call. Each `KalamLinkClient` owns an `Arc`; /// cloning is cheap and all clones share the same pool. - fn cached_cluster_client(base_url: &str) -> KalamLinkClient { + fn cached_cluster_client(base_url: &str, username: &str, password: &str) -> KalamLinkClient { static CLIENT_CACHE: OnceLock>> = OnceLock::new(); let cache = CLIENT_CACHE.get_or_init(|| Mutex::new(HashMap::new())); + let cache_key = format!("{}\n{}\n{}", base_url, username, password); if let Ok(mut guard) = cache.lock() { - if let Some(client) = guard.get(base_url) { + if let Some(client) = guard.get(&cache_key) { return client.clone(); } - let client = build_cluster_client(base_url); - guard.insert(base_url.to_string(), client.clone()); + let client = build_cluster_client_with_auth(base_url, username, password); + guard.insert(cache_key, client.clone()); return client; } // Fallback if lock poisoned - build_cluster_client(base_url) - } - - fn build_cluster_client(base_url: &str) -> KalamLinkClient { - client_for_user_on_url_with_timeouts( - base_url, - default_username(), - default_password(), - KalamLinkTimeouts::builder() - .connection_timeout_secs(5) - .receive_timeout_secs(30) - .send_timeout_secs(10) - .subscribe_timeout_secs(10) - .auth_timeout_secs(10) - .initial_data_timeout(Duration::from_secs(30)) - .build(), - ) - .expect("Failed to build cluster client") + build_cluster_client_with_auth(base_url, username, password) } fn build_cluster_client_with_auth( @@ -110,7 +152,7 @@ mod cluster_common { /// Create a client connected to a specific cluster node pub fn create_cluster_client(base_url: &str) -> KalamLinkClient { - cached_cluster_client(base_url) + cached_cluster_client(base_url, &default_username(), &default_password()) } /// Create a client connected to a specific cluster node with custom credentials @@ -119,8 +161,7 @@ mod cluster_common { username: &str, password: &str, ) -> KalamLinkClient { - // Custom-auth clients are not pooled (credentials may differ per call). - build_cluster_client_with_auth(base_url, username, password) + cached_cluster_client(base_url, username, password) } /// Execute a query on a specific cluster node and return the count @@ -345,12 +386,16 @@ mod cluster_common { sql: &str, enforce_leader: bool, ) -> Result { + let started_at = std::time::Instant::now(); let sql = sql.to_string(); let mut last_err: Option = None; + let mut attempts = 0u64; + let mut retryable_errors = 0u64; for _ in 0..5 { let urls = ordered_urls_for_query(base_url, &sql, enforce_leader); for url in urls.iter().cloned() { + attempts += 1; let client = create_cluster_client(&url); let sql_value = sql.clone(); match cluster_runtime().block_on(async move { @@ -360,14 +405,23 @@ mod cluster_common { if !response.success() { let err_msg = response_error_message(&response); if is_retryable_cluster_error_for_sql(&sql, &err_msg) { + retryable_errors += 1; last_err = Some(err_msg); continue; } + record_cluster_helper_timing( + "execute_on_node", + &sql, + started_at.elapsed(), + attempts, + retryable_errors, + ); return Err(err_msg); } if is_truncated_read_response(&response, &sql) { if let Some(leader) = leader_url() { if url != leader { + retryable_errors += 1; last_err = Some(format!( "Truncated read response from follower {}", url @@ -377,21 +431,43 @@ mod cluster_common { } } wait_for_cluster_after_sql(&sql); + record_cluster_helper_timing( + "execute_on_node", + &sql, + started_at.elapsed(), + attempts, + retryable_errors, + ); return Ok(serde_json::to_string_pretty(&response) .unwrap_or_else(|_| format!("{:?}", response))); }, Err(e) => { let msg = e.to_string(); if is_retryable_cluster_error_for_sql(&sql, &msg) { + retryable_errors += 1; last_err = Some(msg); continue; } + record_cluster_helper_timing( + "execute_on_node", + &sql, + started_at.elapsed(), + attempts, + retryable_errors, + ); return Err(msg); }, } } } + record_cluster_helper_timing( + "execute_on_node", + &sql, + started_at.elapsed(), + attempts, + retryable_errors, + ); Err(last_err.unwrap_or_else(|| "All cluster nodes failed".to_string())) } @@ -415,12 +491,16 @@ mod cluster_common { sql: &str, enforce_leader: bool, ) -> Result { + let started_at = std::time::Instant::now(); let sql = sql.to_string(); let mut last_err: Option = None; + let mut attempts = 0u64; + let mut retryable_errors = 0u64; for _ in 0..5 { let urls = ordered_urls_for_query(base_url, &sql, enforce_leader); for url in urls.iter().cloned() { + attempts += 1; let client = create_cluster_client(&url); let sql_value = sql.clone(); match cluster_runtime().block_on(async move { @@ -430,14 +510,23 @@ mod cluster_common { if !response.success() { let err_msg = response_error_message(&response); if is_retryable_cluster_error_for_sql(&sql, &err_msg) { + retryable_errors += 1; last_err = Some(err_msg); continue; } + record_cluster_helper_timing( + "execute_on_node_response", + &sql, + started_at.elapsed(), + attempts, + retryable_errors, + ); return Err(err_msg); } if is_truncated_read_response(&response, &sql) { if let Some(leader) = leader_url() { if url != leader { + retryable_errors += 1; last_err = Some(format!( "Truncated read response from follower {}", url @@ -447,20 +536,42 @@ mod cluster_common { } } wait_for_cluster_after_sql(&sql); + record_cluster_helper_timing( + "execute_on_node_response", + &sql, + started_at.elapsed(), + attempts, + retryable_errors, + ); return Ok(response); }, Err(e) => { let msg = e.to_string(); if is_retryable_cluster_error_for_sql(&sql, &msg) { + retryable_errors += 1; last_err = Some(msg); continue; } + record_cluster_helper_timing( + "execute_on_node_response", + &sql, + started_at.elapsed(), + attempts, + retryable_errors, + ); return Err(msg); }, } } } + record_cluster_helper_timing( + "execute_on_node_response", + &sql, + started_at.elapsed(), + attempts, + retryable_errors, + ); Err(last_err.unwrap_or_else(|| "All cluster nodes failed".to_string())) } @@ -492,12 +603,16 @@ mod cluster_common { sql: &str, enforce_leader: bool, ) -> Result { + let started_at = std::time::Instant::now(); let sql = sql.to_string(); let mut last_err: Option = None; + let mut attempts = 0u64; + let mut retryable_errors = 0u64; for _ in 0..5 { let urls = ordered_urls_for_query(base_url, &sql, enforce_leader); for url in urls.iter().cloned() { + attempts += 1; let client = create_cluster_client_with_auth(&url, username, password); let sql_value = sql.clone(); match cluster_runtime().block_on(async move { @@ -507,27 +622,57 @@ mod cluster_common { if !response.success() { let err_msg = response_error_message(&response); if is_retryable_cluster_error_for_sql(&sql, &err_msg) { + retryable_errors += 1; last_err = Some(err_msg); continue; } + record_cluster_helper_timing( + "execute_on_node_as_user", + &sql, + started_at.elapsed(), + attempts, + retryable_errors, + ); return Err(err_msg); } wait_for_cluster_after_sql(&sql); + record_cluster_helper_timing( + "execute_on_node_as_user", + &sql, + started_at.elapsed(), + attempts, + retryable_errors, + ); return Ok(serde_json::to_string_pretty(&response) .unwrap_or_else(|_| format!("{:?}", response))); }, Err(e) => { let msg = e.to_string(); if is_retryable_cluster_error_for_sql(&sql, &msg) { + retryable_errors += 1; last_err = Some(msg); continue; } + record_cluster_helper_timing( + "execute_on_node_as_user", + &sql, + started_at.elapsed(), + attempts, + retryable_errors, + ); return Err(msg); }, } } } + record_cluster_helper_timing( + "execute_on_node_as_user", + &sql, + started_at.elapsed(), + attempts, + retryable_errors, + ); Err(last_err.unwrap_or_else(|| "All cluster nodes failed".to_string())) } @@ -560,12 +705,16 @@ mod cluster_common { sql: &str, enforce_leader: bool, ) -> Result { + let started_at = std::time::Instant::now(); let sql = sql.to_string(); let mut last_err: Option = None; + let mut attempts = 0u64; + let mut retryable_errors = 0u64; for _ in 0..5 { let urls = ordered_urls_for_query(base_url, &sql, enforce_leader); for url in urls.iter().cloned() { + attempts += 1; let client = create_cluster_client_with_auth(&url, username, password); let sql_value = sql.clone(); match cluster_runtime().block_on(async move { @@ -575,26 +724,56 @@ mod cluster_common { if !response.success() { let err_msg = response_error_message(&response); if is_retryable_cluster_error_for_sql(&sql, &err_msg) { + retryable_errors += 1; last_err = Some(err_msg); continue; } + record_cluster_helper_timing( + "execute_on_node_as_user_response", + &sql, + started_at.elapsed(), + attempts, + retryable_errors, + ); return Err(err_msg); } wait_for_cluster_after_sql(&sql); + record_cluster_helper_timing( + "execute_on_node_as_user_response", + &sql, + started_at.elapsed(), + attempts, + retryable_errors, + ); return Ok(response); }, Err(e) => { let msg = e.to_string(); if is_retryable_cluster_error_for_sql(&sql, &msg) { + retryable_errors += 1; last_err = Some(msg); continue; } + record_cluster_helper_timing( + "execute_on_node_as_user_response", + &sql, + started_at.elapsed(), + attempts, + retryable_errors, + ); return Err(msg); }, } } } + record_cluster_helper_timing( + "execute_on_node_as_user_response", + &sql, + started_at.elapsed(), + attempts, + retryable_errors, + ); Err(last_err.unwrap_or_else(|| "All cluster nodes failed".to_string())) } @@ -647,11 +826,7 @@ mod cluster_common { /// Check if a cluster node is healthy pub fn is_node_healthy(base_url: &str) -> bool { - let client = create_cluster_client(base_url); - cluster_runtime() - .block_on(async move { client.execute_query("SELECT 1", None, None, None).await }) - .map(|response| response.success()) - .unwrap_or(false) + crate::common::is_cluster_url_reachable(base_url) } /// Require cluster to be running (skip test if not available) @@ -682,9 +857,10 @@ mod cluster_common { return false; } - // Check if at least one node is reachable - let any_healthy = urls.iter().any(|url| is_node_healthy(url)); - if !any_healthy { + static CLUSTER_REACHABILITY_CHECK: OnceLock = OnceLock::new(); + let reachable = + *CLUSTER_REACHABILITY_CHECK.get_or_init(|| urls.iter().any(|url| is_node_healthy(url))); + if !reachable { if cluster_requested { panic!( "Cluster tests were requested, but no configured cluster node is reachable: \ diff --git a/cli/tests/cluster/cluster_test_multi_node_smoke.rs b/cli/tests/cluster/cluster_test_multi_node_smoke.rs index 08e91828..121d402d 100644 --- a/cli/tests/cluster/cluster_test_multi_node_smoke.rs +++ b/cli/tests/cluster/cluster_test_multi_node_smoke.rs @@ -457,3 +457,69 @@ fn cluster_test_smoke_write_routing() { println!("\n ✅ Write routing test completed\n"); } + +/// Test: mixed Meta + data batches route each statement to the owning Raft group leader. +#[test] +fn cluster_test_mixed_statement_batch_routes_per_group() { + if !require_cluster_running() { + return; + } + + println!("\n=== TEST: Mixed Statement Batch Routing ===\n"); + + let urls = cluster_urls(); + let namespace = generate_unique_namespace("mixed_batch_route"); + let user = format!("mixed_user_{}", namespace); + let password = "test_password_123"; + + let _ = execute_on_node(&urls[0], &format!("DROP NAMESPACE IF EXISTS {} CASCADE", namespace)); + let _ = execute_on_node(&urls[0], &format!("DROP USER {}", user)); + + execute_on_node(&urls[0], &format!("CREATE NAMESPACE {}", namespace)) + .expect("Failed to create namespace"); + execute_on_node( + &urls[0], + &format!( + "CREATE SHARED TABLE {}.mixed_shared (id BIGINT PRIMARY KEY, value STRING)", + namespace + ), + ) + .expect("Failed to create shared table"); + + if !wait_for_table_on_all_nodes(&namespace, "mixed_shared", 10000) { + panic!("Table mixed_shared did not replicate to all nodes"); + } + + let follower_url = leader_url() + .and_then(|leader| urls.iter().find(|url| **url != leader).cloned()) + .or_else(|| urls.get(1).cloned()) + .unwrap_or_else(|| urls[0].clone()); + + let batch = format!( + "CREATE USER {} WITH PASSWORD '{}' ROLE 'user'; \ + INSERT INTO {}.mixed_shared (id, value) VALUES (1, 'initial'); \ + UPDATE {}.mixed_shared SET value = 'updated' WHERE id = 1", + user, password, namespace, namespace + ); + + execute_on_node_raw(&follower_url, &batch) + .unwrap_or_else(|err| panic!("Mixed batch failed from {}: {}", follower_url, err)); + + let user_query = format!("SELECT user_id FROM system.users WHERE user_id = '{}'", user); + let user_visible = urls.iter().all( + |url| matches!(execute_on_node(url, &user_query), Ok(result) if result.contains(&user)), + ); + assert!(user_visible, "Created user was not visible from every cluster node"); + + let rows = fetch_normalized_rows( + &urls[0], + &format!("SELECT id, value FROM {}.mixed_shared ORDER BY id", namespace), + ) + .expect("Failed to read mixed batch data"); + assert_eq!(rows, vec!["1|updated".to_string()]); + + let _ = execute_on_node(&urls[0], &format!("DROP USER {}", user)); + let _ = execute_on_node(&urls[0], &format!("DROP NAMESPACE {} CASCADE", namespace)); + + println!("\n ✅ Mixed statement batch routed per group\n"); +} diff --git a/cli/tests/cluster/cluster_test_node_rejoin.rs b/cli/tests/cluster/cluster_test_node_rejoin.rs index 58abac38..ae3b28cd 100644 --- a/cli/tests/cluster/cluster_test_node_rejoin.rs +++ b/cli/tests/cluster/cluster_test_node_rejoin.rs @@ -505,12 +505,8 @@ fn cluster_test_node_rejoin_user_management() { println!(" ✓ Node3 has user: {}", test_user); // Verify user can authenticate on node3 - let auth_result = execute_on_node_as_user( - stopped_url, - &test_user, - "kalamdb123", - "SELECT 1 AS authenticated", - ); + let auth_result = + execute_on_node_as_user(stopped_url, &test_user, "kalamdb123", "SELECT 1 AS authenticated"); assert!(auth_result.is_ok(), "User should be able to authenticate on node3"); println!(" ✓ User can authenticate on node3"); diff --git a/cli/tests/common/mod.rs b/cli/tests/common/mod.rs index 9dd84a3e..431e86b3 100644 --- a/cli/tests/common/mod.rs +++ b/cli/tests/common/mod.rs @@ -93,14 +93,17 @@ static TEST_CLI_CREDENTIALS_PATH: OnceLock = OnceLock::new(); const LEADER_CACHE_TTL: Duration = Duration::from_secs(5); pub fn shared_http_client() -> Client { - // Tests use separate tokio runtimes, so reusing one global reqwest client can leave pooled - // dispatch tasks bound to a runtime that has already shut down. - Client::builder() - .pool_max_idle_per_host(512) - .pool_idle_timeout(Duration::from_secs(90)) - .tcp_nodelay(true) - .build() - .expect("failed to build test HTTP client") + static CLIENT: OnceLock = OnceLock::new(); + CLIENT + .get_or_init(|| { + Client::builder() + .pool_max_idle_per_host(512) + .pool_idle_timeout(Duration::from_secs(90)) + .tcp_nodelay(true) + .build() + .expect("failed to build test HTTP client") + }) + .clone() } #[derive(Clone, Debug)] @@ -121,6 +124,36 @@ impl TestAuthManager { } } + async fn send_with_retry( + &self, + request: reqwest::RequestBuilder, + ) -> Result { + let max_attempts = if matches!(server_type_from_env(), Some(ServerType::Cluster)) { + 40 + } else { + 5 + }; + for attempt in 0..max_attempts { + let Some(attempt_request) = request.try_clone() else { + return request.send().await; + }; + + match attempt_request.send().await { + Ok(response) => return Ok(response), + Err(error) if attempt + 1 < max_attempts => { + tokio::time::sleep(Duration::from_millis(250)).await; + if error.is_timeout() || error.is_connect() || error.is_request() { + continue; + } + continue; + }, + Err(error) => return Err(error), + } + } + + unreachable!("send_with_retry must return within max_attempts") + } + fn with_shared_token_cache( &self, op: impl FnOnce(&mut HashMap) -> R, @@ -221,15 +254,13 @@ impl TestAuthManager { } let root_password = root_password_from_env(); - let setup_response = client - .post(format!("{}/v1/api/auth/setup", base_url)) - .json(&json!({ + let setup_response = self + .send_with_retry(client.post(format!("{}/v1/api/auth/setup", base_url)).json(&json!({ "user": "admin", "password": "kalamdb123", "root_password": root_password, "email": null - })) - .send() + }))) .await?; if !setup_response.status().is_success() { @@ -341,13 +372,14 @@ impl TestAuthManager { let root_token = self.token_for_url_cached(base_url, "root", root_password).await?; let client = shared_http_client(); - let exists_response = client - .post(format!("{}/v1/api/sql", base_url)) - .bearer_auth(&root_token) - .json(&json!({ - "sql": "SELECT user_id FROM system.users WHERE user_id = 'admin' LIMIT 1" - })) - .send() + let exists_response = self + .send_with_retry( + client.post(format!("{}/v1/api/sql", base_url)).bearer_auth(&root_token).json( + &json!({ + "sql": "SELECT user_id FROM system.users WHERE user_id = 'admin' LIMIT 1" + }), + ), + ) .await?; let admin_exists = if exists_response.status().is_success() { @@ -366,16 +398,17 @@ impl TestAuthManager { }; if admin_exists { - let password_response = client - .post(format!("{}/v1/api/sql", base_url)) - .bearer_auth(&root_token) - .json(&json!({ - "sql": format!( - "ALTER USER admin SET PASSWORD '{}'", - admin_password() - ) - })) - .send() + let password_response = self + .send_with_retry( + client.post(format!("{}/v1/api/sql", base_url)).bearer_auth(&root_token).json( + &json!({ + "sql": format!( + "ALTER USER admin SET PASSWORD '{}'", + admin_password() + ) + }), + ), + ) .await?; if !password_response.status().is_success() { @@ -383,13 +416,14 @@ impl TestAuthManager { return Err(format!("Failed to reset admin password: {}", body).into()); } - let role_response = client - .post(format!("{}/v1/api/sql", base_url)) - .bearer_auth(&root_token) - .json(&json!({ - "sql": "ALTER USER admin SET ROLE 'dba'" - })) - .send() + let role_response = self + .send_with_retry( + client.post(format!("{}/v1/api/sql", base_url)).bearer_auth(&root_token).json( + &json!({ + "sql": "ALTER USER admin SET ROLE 'dba'" + }), + ), + ) .await?; if !role_response.status().is_success() { @@ -400,16 +434,17 @@ impl TestAuthManager { return Ok(()); } - let response = client - .post(format!("{}/v1/api/sql", base_url)) - .bearer_auth(root_token) - .json(&json!({ - "sql": format!( - "CREATE USER admin WITH PASSWORD '{}' ROLE 'dba'", - admin_password() - ) - })) - .send() + let response = self + .send_with_retry( + client.post(format!("{}/v1/api/sql", base_url)).bearer_auth(root_token).json( + &json!({ + "sql": format!( + "CREATE USER admin WITH PASSWORD '{}' ROLE 'dba'", + admin_password() + ) + }), + ), + ) .await?; if !response.status().is_success() { @@ -2480,11 +2515,12 @@ pub fn is_server_running() -> bool { let ctx = test_context(); if ctx.is_cluster { - let urls = if ctx.cluster_urls.is_empty() { - ctx.cluster_urls_raw.clone() - } else { - ctx.cluster_urls.clone() - }; + let mut urls = ctx.cluster_urls.clone(); + for url in &ctx.cluster_urls_raw { + if !urls.contains(url) { + urls.push(url.clone()); + } + } let reachable = wait_for_reachable_cluster_urls(&urls, Duration::from_secs(5)); if !reachable.is_empty() { return true; @@ -2679,15 +2715,14 @@ fn server_requires_auth_for_url(url: &str) -> Option { pub fn get_available_server_urls() -> Vec { let ctx = test_context(); if ctx.is_cluster { - let seed_urls = if ctx.cluster_urls.is_empty() { - ctx.cluster_urls_raw.clone() - } else { - ctx.cluster_urls.clone() - }; - let mut urls = wait_for_sql_ready_cluster_urls(&seed_urls, Duration::from_secs(2)); - if urls.is_empty() { - urls = seed_urls; + let mut seed_urls = ctx.cluster_urls.clone(); + for url in &ctx.cluster_urls_raw { + if !seed_urls.contains(url) { + seed_urls.push(url.clone()); + } } + + let mut urls = seed_urls; if let Some(leader) = leader_url() { urls.retain(|url| url != &leader); urls.insert(0, leader); @@ -2703,6 +2738,10 @@ pub fn cluster_urls_config_order() -> Vec { test_context().cluster_urls_raw.clone() } +pub fn is_cluster_url_reachable(url: &str) -> bool { + url_reachable(url) +} + fn should_wait_for_cluster_after_sql(sql: &str) -> bool { let upper = sql.trim_start().to_ascii_uppercase(); upper.starts_with("CREATE NAMESPACE") @@ -3786,6 +3825,37 @@ fn get_shared_root_client_for_url(base_url: &str) -> KalamLinkClient { build_root_client(base_url) } +fn shared_user_client_cache() -> &'static Mutex> { + static CLIENT_CACHE: OnceLock>> = OnceLock::new(); + CLIENT_CACHE.get_or_init(|| Mutex::new(HashMap::new())) +} + +fn shared_test_client_for_url_with_timeouts( + base_url: &str, + username: &str, + password: &str, + timeouts: &KalamLinkTimeouts, +) -> Result> { + if username == default_username() && password == default_password() { + return Ok(get_shared_root_client_for_url(base_url)); + } + + let cache_key = format!("{}\n{}\n{}", base_url, username, password); + + if let Ok(mut guard) = shared_user_client_cache().lock() { + if let Some(client) = guard.get(&cache_key) { + return Ok(client.clone()); + } + + let client = + build_client_for_url_with_timeouts(base_url, username, password, timeouts.clone())?; + guard.insert(cache_key, client.clone()); + return Ok(client); + } + + build_client_for_url_with_timeouts(base_url, username, password, timeouts.clone()) +} + fn get_shared_root_client() -> KalamLinkClient { let base_url = get_available_server_urls() .first() @@ -3860,12 +3930,9 @@ fn execute_sql_via_client_internal( .initial_data_timeout(Duration::from_secs(30)) .build(); - let client = if username == default_username() && password == default_password() - { - get_shared_root_client_for_url(url) - } else { - build_client_for_url_with_timeouts(url, username, password, timeouts)? - }; + let client = shared_test_client_for_url_with_timeouts( + url, username, password, &timeouts, + )?; let response = client.execute_query(sql, None, params.clone(), None).await?; Ok(response) } diff --git a/cli/tests/smoke/ddl/smoke_test_datatype_preservation.rs b/cli/tests/smoke/ddl/smoke_test_datatype_preservation.rs index 82ee70ae..0f2cdff4 100644 --- a/cli/tests/smoke/ddl/smoke_test_datatype_preservation.rs +++ b/cli/tests/smoke/ddl/smoke_test_datatype_preservation.rs @@ -14,7 +14,7 @@ use serde_json::Value; use crate::common::{ force_auto_test_server_url_async, generate_unique_namespace, get_access_token_for_url, - test_context, + shared_http_client, test_context, }; /// Test that all KalamDataTypes are preserved correctly in query results @@ -22,7 +22,7 @@ use crate::common::{ #[ntest::timeout(60000)] async fn test_all_kalam_datatypes_are_preserved() { let ctx = test_context(); - let client = Client::new(); + let client = shared_http_client(); let base_url = force_auto_test_server_url_async().await; let ns = generate_unique_namespace("dtypes"); let table = "all_types"; @@ -161,7 +161,7 @@ async fn test_all_kalam_datatypes_are_preserved() { #[ntest::timeout(60000)] async fn test_system_tables_shows_correct_datatypes() { let ctx = test_context(); - let client = Client::new(); + let client = shared_http_client(); let base_url = force_auto_test_server_url_async().await; let ns = generate_unique_namespace("systypes"); let table = "type_check"; diff --git a/cli/tests/smoke/security/smoke_test_rpc_auth.rs b/cli/tests/smoke/security/smoke_test_rpc_auth.rs index 774bfd1d..0988a727 100644 --- a/cli/tests/smoke/security/smoke_test_rpc_auth.rs +++ b/cli/tests/smoke/security/smoke_test_rpc_auth.rs @@ -10,23 +10,13 @@ //! Run with: //! cargo nextest run --test smoke smoke_security_rpc_auth -use std::time::Duration; - use base64::{engine::general_purpose, Engine as _}; -use reqwest::Client; use serde_json::json; use crate::common::*; // ─── Helpers ───────────────────────────────────────────────────────────────── -fn http_client() -> Client { - Client::builder() - .timeout(Duration::from_secs(10)) - .build() - .expect("Failed to build reqwest client") -} - fn sql_url() -> String { format!("{}/v1/api/sql", server_url()) } @@ -64,7 +54,7 @@ fn smoke_rpc_sql_no_auth_returns_401() { } let status = block(async { - http_client() + shared_http_client() .post(sql_url()) .json(&json!({ "sql": "SELECT 1" })) .send() @@ -91,7 +81,7 @@ fn smoke_rpc_sql_invalid_bearer_returns_401() { } let status = block(async { - http_client() + shared_http_client() .post(sql_url()) .header("Authorization", "Bearer not.a.valid.jwt") .json(&json!({ "sql": "SELECT 1" })) @@ -124,7 +114,7 @@ fn smoke_rpc_sql_forged_jwt_alg_none_returns_401() { "eyJhbGciOiJub25lIn0.eyJzdWIiOiJhdHRhY2tlciIsInJvbGUiOiJzeXN0ZW0iLCJleHAiOjk5OTk5OTk5OTl9."; let status = block(async { - http_client() + shared_http_client() .post(sql_url()) .header("Authorization", format!("Bearer {}", forged)) .json(&json!({ "sql": "SELECT * FROM system.users" })) @@ -157,7 +147,7 @@ fn smoke_rpc_sql_basic_auth_returns_401() { let basic = "Basic YWRtaW46a2FsYW1kYjEyMw=="; let status = block(async { - http_client() + shared_http_client() .post(sql_url()) .header("Authorization", basic) .json(&json!({ "sql": "SELECT 1" })) @@ -192,7 +182,7 @@ fn smoke_rpc_non_login_auth_endpoints_reject_basic_auth() { )) .expect("valid basic auth header"); - let me_status = http_client() + let me_status = shared_http_client() .get(me_url()) .header(reqwest::header::AUTHORIZATION, auth_header.clone()) .send() @@ -200,7 +190,7 @@ fn smoke_rpc_non_login_auth_endpoints_reject_basic_auth() { .expect("/auth/me request failed") .status(); - let refresh_status = http_client() + let refresh_status = shared_http_client() .post(refresh_url()) .header(reqwest::header::AUTHORIZATION, auth_header) .send() @@ -232,8 +222,14 @@ fn smoke_rpc_me_no_auth_returns_401() { return; } - let status = - block(async { http_client().get(me_url()).send().await.expect("Request failed").status() }); + let status = block(async { + shared_http_client() + .get(me_url()) + .send() + .await + .expect("Request failed") + .status() + }); assert_eq!( status.as_u16(), @@ -254,7 +250,12 @@ fn smoke_rpc_health_is_public() { } let status = block(async { - http_client().get(health_url()).send().await.expect("Request failed").status() + shared_http_client() + .get(health_url()) + .send() + .await + .expect("Request failed") + .status() }); assert!(status.is_success(), "/health must be publicly accessible (2xx), got {}", status); @@ -275,7 +276,7 @@ fn smoke_rpc_login_wrong_password_returns_401_generic_message() { } let (status, body) = block(async { - let response = http_client() + let response = shared_http_client() .post(login_url()) .json(&json!({ "user": "admin", @@ -325,7 +326,7 @@ fn smoke_rpc_login_nonexistent_user_matches_wrong_password_response() { } let (status_real_user, msg_real_user) = block(async { - let resp = http_client() + let resp = shared_http_client() .post(login_url()) .json(&json!({ "user": "admin", "password": "wrong-pass-abc123" })) .send() @@ -343,7 +344,7 @@ fn smoke_rpc_login_nonexistent_user_matches_wrong_password_response() { }); let (status_fake_user, msg_fake_user) = block(async { - let resp = http_client() + let resp = shared_http_client() .post(login_url()) .json(&json!({ "user": "this_user_definitely_does_not_exist_xyz", @@ -478,7 +479,7 @@ fn smoke_rpc_sql_empty_body_returns_error() { }); let status = block(async { - http_client() + shared_http_client() .post(sql_url()) .header("Authorization", format!("Bearer {}", token)) .header("Content-Type", "application/json") diff --git a/cli/tests/smoke/usecases/smoke_test_file_datatype.rs b/cli/tests/smoke/usecases/smoke_test_file_datatype.rs index d4547482..87557827 100644 --- a/cli/tests/smoke/usecases/smoke_test_file_datatype.rs +++ b/cli/tests/smoke/usecases/smoke_test_file_datatype.rs @@ -237,12 +237,23 @@ async fn execute_sql_via_http_as_for_url( }; for url in urls { - let response = client + let response = match client .post(format!("{}/v1/api/sql", url)) .bearer_auth(token) .json(&serde_json::json!({ "sql": sql })) .send() - .await?; + .await + { + Ok(response) => response, + Err(error) => { + let message = error.to_string(); + if is_cluster_mode() && is_retryable_cluster_error_for_sql(sql, &message) { + last_error = Some(message); + continue; + } + return Err(error.into()); + }, + }; let status = response.status(); let body_text = response.text().await?; @@ -320,7 +331,7 @@ async fn execute_multipart_sql_with_cluster_retry( } for base_url in urls { - let response = client + let response = match client .execute(build_multipart_sql_request( client, &base_url, @@ -328,7 +339,18 @@ async fn execute_multipart_sql_with_cluster_retry( content_type, body.to_vec(), )?) - .await?; + .await + { + Ok(response) => response, + Err(error) => { + let message = error.to_string(); + if is_cluster_mode() && is_retryable_cluster_error_for_sql("", &message) { + last_error = Some(message); + continue; + } + return Err(error.into()); + }, + }; let status = response.status(); let response_content_type = response diff --git a/docs/architecture/raft-replication.md b/docs/architecture/raft-replication.md index e0cea18c..715dbc99 100644 --- a/docs/architecture/raft-replication.md +++ b/docs/architecture/raft-replication.md @@ -2,23 +2,23 @@ ## Overview -KalamDB uses a multi-Raft topology (OpenRaft 0.9) to replicate metadata, jobs, user data, and shared data across nodes. The Raft layer lives in [backend/crates/kalamdb-raft](backend/crates/kalamdb-raft/src/lib.rs) and is accessed through the `CommandExecutor` abstraction so handlers do not branch on cluster vs standalone mode. +KalamDB uses a multi-Raft topology (OpenRaft 0.9) to replicate metadata, jobs, user data, and shared data across nodes. The Raft layer lives in [backend/crates/kalamdb-raft](../../backend/crates/kalamdb-raft/src/lib.rs) and is accessed through the `CommandExecutor` abstraction so handlers do not branch on cluster vs standalone mode. -- **Multi-group layout**: 1 unified metadata group, 32 user-data shards, 1 shared-data shard by default. Group identity and sharding helpers live in [backend/crates/kalamdb-sharding/src](backend/crates/kalamdb-sharding/src/lib.rs), and group ID decoding accepts the configured user/shared shard ranges. +- **Multi-group layout**: 1 unified metadata group, 32 user-data shards, 1 shared-data shard by default. Group identity and sharding helpers live in [backend/crates/kalamdb-sharding/src](../../backend/crates/kalamdb-sharding/src/lib.rs), and group ID decoding accepts the configured user/shared shard ranges. - **Command path**: Handler → `CommandExecutor` (`DirectExecutor` in standalone or `RaftExecutor` in cluster) → `RaftManager` → `RaftGroup` → OpenRaft log → State machine → Applier → storage/provider. -- **Replication modes**: `Quorum` (fast, default) or `All` (wait for every member to apply) configured via `ReplicationMode` in [backend/crates/kalamdb-raft/src/manager/config.rs](backend/crates/kalamdb-raft/src/manager/config.rs). -- **Transport**: gRPC service in [backend/crates/kalamdb-raft/src/network/service.rs](backend/crates/kalamdb-raft/src/network/service.rs) handles Raft RPCs and follower→leader proposal forwarding. -- **Storage**: Combined in-memory log + state machine storage in [backend/crates/kalamdb-raft/src/storage/raft_store.rs](backend/crates/kalamdb-raft/src/storage/raft_store.rs); snapshots are used for compaction. Real table writes are persisted by appliers after Raft apply. +- **Replication modes**: `Quorum` (fast, default) or `All` (wait for every member to apply) configured via `ReplicationMode` in [backend/crates/kalamdb-raft/src/manager/config.rs](../../backend/crates/kalamdb-raft/src/manager/config.rs). +- **Transport**: gRPC service in [backend/crates/kalamdb-raft/src/network/service.rs](../../backend/crates/kalamdb-raft/src/network/service.rs) handles Raft RPCs and follower→leader proposal forwarding. +- **Storage**: Combined in-memory log + state machine storage in [backend/crates/kalamdb-raft/src/storage/raft_store.rs](../../backend/crates/kalamdb-raft/src/storage/raft_store.rs); snapshots are used for compaction. Real table writes are persisted by appliers after Raft apply. ## Topology & Sharding - Group IDs encode role and shard: `Meta`, `DataUserShard(n)`, `DataSharedShard(n)`. Numeric IDs are stable for OpenRaft membership and RPC routing. -- User data routing: `hash(user_id) % user_shards` (default 32). Shared tables currently always use shard 0. Helpers live in `ShardRouter` in [backend/crates/kalamdb-sharding/src/lib.rs](backend/crates/kalamdb-sharding/src/lib.rs). +- User data routing: `hash(user_id) % user_shards` (default 32). Shared tables currently always use shard 0. Helpers live in `ShardRouter` in [backend/crates/kalamdb-sharding/src/lib.rs](../../backend/crates/kalamdb-sharding/src/lib.rs). - Table-level helpers: `ShardRouter::route_table` / `table_shard_id` hash `TableId` when table-scoped routing is needed. ## Command Flow (Cluster Mode) -1. Handlers call the `CommandExecutor` interface. `RaftExecutor` (cluster) serializes Raft commands/responses with FlexBuffers and picks the target group (user shard, shared shard, or metadata group) in [backend/crates/kalamdb-raft/src/executor/raft.rs](backend/crates/kalamdb-raft/src/executor/raft.rs). +1. Handlers call the `CommandExecutor` interface. `RaftExecutor` (cluster) serializes Raft commands/responses with FlexBuffers and picks the target group (user shard, shared shard, or metadata group) in [backend/crates/kalamdb-raft/src/executor/raft.rs](../../backend/crates/kalamdb-raft/src/executor/raft.rs). 2. `RaftManager` proposes to the correct `RaftGroup`: - `propose_*` APIs forward to the leader if the local node is a follower via `propose_with_forward`. - In `ReplicationMode::All`, leaders wait for every member to apply the committed log before returning. @@ -29,6 +29,7 @@ KalamDB uses a multi-Raft topology (OpenRaft 0.9) to replicate metadata, jobs, u ## SQL DML & Commit Ordering - SQL autocommit DML for user/shared tables is staged through the transaction mutation path and committed through the appropriate data Raft group. Direct local SQL provider writes are no longer the normal HTTP SQL write path in cluster mode. +- Multi-statement SQL batches keep the existing prepared-statement execution path. When a batch has no bind parameters or explicit transaction control and every statement resolves to a known Raft group, the HTTP SQL layer routes each statement to its owning group leader. This lets mixed metadata/data batches use the Meta group for DDL/user commands and the target data group for DML without adding a SQL rewrite pass to the hot path. - State machines derive `_commit_seq` deterministically from `(group_id, log_index)` via `commit_seq_from_log_position`. Appliers receive that value and stamp inserted, updated, deleted, and transaction-batch rows with the replicated commit order. - `CommitSequenceTracker::observe_committed` advances the local high-watermark after persisted apply, so every node observes the same commit sequence for the same Raft entry. - `_seq` remains the only client-visible live-query resume cursor. `_commit_seq` is internal state-machine metadata used for deterministic visibility and follower-side snapshot gating. @@ -43,8 +44,9 @@ KalamDB uses a multi-Raft topology (OpenRaft 0.9) to replicate metadata, jobs, u ## Network Layer -- Service surface: `RaftRpc` (vote, append_entries, install_snapshot) and `ClientProposal` (follower→leader proposal forwarding) defined in [backend/crates/kalamdb-raft/src/network/service.rs](backend/crates/kalamdb-raft/src/network/service.rs). -- Client side: each `RaftNetworkFactory` produces `RaftNetwork` clients per group, but all groups in a `RaftManager` share one node-level tonic channel pool keyed by peer `NodeId` ([backend/crates/kalamdb-raft/src/network/network.rs](backend/crates/kalamdb-raft/src/network/network.rs)). This keeps the multi-Raft topology from opening one persistent HTTP/2 transport per `(group, peer)` pair. +- Service surface: `RaftRpc` (vote, append_entries, install_snapshot) and `ClientProposal` (follower→leader proposal forwarding) defined in [backend/crates/kalamdb-raft/src/network/service.rs](../../backend/crates/kalamdb-raft/src/network/service.rs). +- Client side: each `RaftNetworkFactory` produces `RaftNetwork` clients per group, but all Raft consensus groups in a `RaftManager` share one node-level tonic channel pool keyed by peer `NodeId` ([backend/crates/kalamdb-raft/src/network/network.rs](../../backend/crates/kalamdb-raft/src/network/network.rs)). This keeps the multi-Raft topology from opening one persistent HTTP/2 transport per `(group, peer)` pair. +- Non-Raft cluster RPCs (`ForwardSql`, `Ping`, and `GetNodeInfo`) use a separate node-level tonic channel pool, also keyed by peer `NodeId`. This keeps long-running follower→leader SQL forwarding from sharing an HTTP/2 transport with Raft heartbeats, append entries, votes, and snapshots. - RPC call construction is centralized in the network layer: OpenRaft RPCs use a typed `RaftRpcKind` plus shared encode/send/decode helpers, follower proposal forwarding uses `RaftNetworkFactory::send_client_proposal`, and non-Raft cluster messages use `ClusterClient` shared request/metadata/error handling. This keeps channel reuse, auth metadata, and serde boundaries consistent across inter-node calls. - Serialization boundaries are intentionally layered: tonic/prost frames the gRPC messages, MessagePack encodes OpenRaft request/response payloads, FlexBuffers encodes committed Raft commands and apply responses, and follower SQL forwarding returns already-serialized HTTP JSON bytes from the leader so followers do not deserialize and reserialize result bodies. - Server startup: `start_rpc_server` binds to the advertised RPC port and serves the Raft gRPC server; invoked by `RaftExecutor::start` before starting groups. @@ -52,7 +54,7 @@ KalamDB uses a multi-Raft topology (OpenRaft 0.9) to replicate metadata, jobs, u ## Raft Manager Lifecycle -Implemented in [backend/crates/kalamdb-raft/src/manager/raft_manager.rs](backend/crates/kalamdb-raft/src/manager/raft_manager.rs). +Implemented in [backend/crates/kalamdb-raft/src/manager/raft_manager.rs](../../backend/crates/kalamdb-raft/src/manager/raft_manager.rs). - **Construction**: Creates 1 meta group + N user shards + M shared shards (defaults 32/1). Each group is a `RaftGroup` wrapping its own storage and network factory; the manager injects a shared channel pool into every factory so peer transports are reused across groups. - **Start**: Registers configured peers with every group, starts the RPC server, then starts all Raft groups with OpenRaft configs (heartbeat/election timeouts from `RaftManagerConfig`). @@ -63,7 +65,7 @@ Implemented in [backend/crates/kalamdb-raft/src/manager/raft_manager.rs](backend ## Raft Groups -Defined in [backend/crates/kalamdb-raft/src/manager/raft_group.rs](backend/crates/kalamdb-raft/src/manager/raft_group.rs). +Defined in [backend/crates/kalamdb-raft/src/manager/raft_group.rs](../../backend/crates/kalamdb-raft/src/manager/raft_group.rs). - Wraps an OpenRaft `Raft` instance plus combined storage and network factory for a single group. - Key operations: `start`, `initialize`, `add_learner`, `wait_for_learner_catchup`, `promote_learner`, `change_membership`, `propose`, `propose_with_all_replicas`, and `propose_with_forward`. @@ -72,26 +74,26 @@ Defined in [backend/crates/kalamdb-raft/src/manager/raft_group.rs](backend/crate ## Storage & Snapshots -Implemented by `KalamRaftStorage` in [backend/crates/kalamdb-raft/src/storage/raft_store.rs](backend/crates/kalamdb-raft/src/storage/raft_store.rs). +Implemented by `KalamRaftStorage` in [backend/crates/kalamdb-raft/src/storage/raft_store.rs](../../backend/crates/kalamdb-raft/src/storage/raft_store.rs). - Combined `RaftStorage` (log + state machine) with in-memory BTreeMap log, vote/commit tracking, and snapshot retention. Snapshot builder captures state machine bytes plus membership metadata; snapshots are served to lagging replicas and purge earlier logs. -- OpenRaft RPC/snapshot serialization continues to use bincode helpers in [backend/crates/kalamdb-raft/src/state_machine/serde_helpers.rs](backend/crates/kalamdb-raft/src/state_machine/serde_helpers.rs). +- OpenRaft RPC/snapshot serialization continues to use bincode helpers in [backend/crates/kalamdb-raft/src/state_machine/serde_helpers.rs](../../backend/crates/kalamdb-raft/src/state_machine/serde_helpers.rs). - State machine apply runs on every node; apply errors are logged and surfaced via response payloads when possible. - **Current limitation**: log/state storage is in-memory; durability relies on higher layers persisting real data via appliers. Crash restart currently depends on application-level recovery. ## State Machines & Appliers -All state machines live under [backend/crates/kalamdb-raft/src/state_machine](backend/crates/kalamdb-raft/src/state_machine). +All state machines live under [backend/crates/kalamdb-raft/src/state_machine](../../backend/crates/kalamdb-raft/src/state_machine). - **MetaStateMachine**: namespaces, tables, storages, users, and jobs; caches snapshot data and drives metadata appliers. -- **UserDataStateMachine**: per-user tables; routes persistence through `UserDataApplier` ([user_data.rs](backend/crates/kalamdb-raft/src/state_machine/user_data.rs)). -- **SharedDataStateMachine**: shared tables; uses `SharedDataApplier` ([shared_data.rs](backend/crates/kalamdb-raft/src/state_machine/shared_data.rs)). +- **UserDataStateMachine**: per-user tables; routes persistence through `UserDataApplier` ([user_data.rs](../../backend/crates/kalamdb-raft/src/state_machine/user_data.rs)). +- **SharedDataStateMachine**: shared tables; uses `SharedDataApplier` ([shared_data.rs](../../backend/crates/kalamdb-raft/src/state_machine/shared_data.rs)). - Every state machine tracks `last_applied_index` for idempotency and produces snapshots capturing its cached state; row data persistence is delegated to appliers so followers apply real writes too. -- Applier traits and no-op defaults live in [backend/crates/kalamdb-raft/src/applier](backend/crates/kalamdb-raft/src/applier/mod.rs). +- Applier traits and no-op defaults live in [backend/crates/kalamdb-raft/src/applier](../../backend/crates/kalamdb-raft/src/applier/mod.rs). ## Command Types -Defined in [backend/crates/kalamdb-raft/src/commands](backend/crates/kalamdb-raft/src/commands/mod.rs) and split per group: +Defined in [backend/crates/kalamdb-raft/src/commands](../../backend/crates/kalamdb-raft/src/commands/mod.rs) and split per group: - Meta: namespaces/tables/storage metadata, user accounts, login events, locks, and jobs. - UserData: per-user DML plus live-query registrations. - SharedData: shared-table DML. @@ -101,7 +103,7 @@ Responses are serialized FlexBuffers payloads returned after apply. - Peer registry: `RaftManager::register_peer` plumbs peer addresses into every `RaftGroup` so OpenRaft can dial peers via the network factory. - Leader discovery: `current_leader` is per-group; followers respond to forwarded proposals with leader hints. HTTP SQL write forwarding uses the target data group directly when possible. `RaftExecutor::get_cluster_info` aggregates OpenRaft metrics for UI/diagnostics. -- Shard mapping: user shard via user_id hash; shared shard fixed to 0. Default counts exposed as `DEFAULT_USER_DATA_SHARDS` and `DEFAULT_SHARED_DATA_SHARDS` in [backend/crates/kalamdb-raft/src/manager/config.rs](backend/crates/kalamdb-raft/src/manager/config.rs). +- Shard mapping: user shard via user_id hash; shared shard fixed to 0. Default counts exposed as `DEFAULT_USER_DATA_SHARDS` and `DEFAULT_SHARED_DATA_SHARDS` in [backend/crates/kalamdb-raft/src/manager/config.rs](../../backend/crates/kalamdb-raft/src/manager/config.rs). ## Operational Notes & Limitations @@ -137,9 +139,9 @@ Responses are serialized FlexBuffers payloads returned after apply. ## At-a-Glance Module Map -- Manager & groups: [backend/crates/kalamdb-raft/src/manager](backend/crates/kalamdb-raft/src/manager) -- Network transport: [backend/crates/kalamdb-raft/src/network](backend/crates/kalamdb-raft/src/network) -- Executor abstraction: [backend/crates/kalamdb-raft/src/executor](backend/crates/kalamdb-raft/src/executor) -- State machines & appliers: [backend/crates/kalamdb-raft/src/state_machine](backend/crates/kalamdb-raft/src/state_machine) and [backend/crates/kalamdb-raft/src/applier](backend/crates/kalamdb-raft/src/applier) -- Storage adapter: [backend/crates/kalamdb-raft/src/storage](backend/crates/kalamdb-raft/src/storage) -- Command types: [backend/crates/kalamdb-raft/src/commands](backend/crates/kalamdb-raft/src/commands) +- Manager & groups: [backend/crates/kalamdb-raft/src/manager](../../backend/crates/kalamdb-raft/src/manager) +- Network transport: [backend/crates/kalamdb-raft/src/network](../../backend/crates/kalamdb-raft/src/network) +- Executor abstraction: [backend/crates/kalamdb-raft/src/executor](../../backend/crates/kalamdb-raft/src/executor) +- State machines & appliers: [backend/crates/kalamdb-raft/src/state_machine](../../backend/crates/kalamdb-raft/src/state_machine) and [backend/crates/kalamdb-raft/src/applier](../../backend/crates/kalamdb-raft/src/applier) +- Storage adapter: [backend/crates/kalamdb-raft/src/storage](../../backend/crates/kalamdb-raft/src/storage) +- Command types: [backend/crates/kalamdb-raft/src/commands](../../backend/crates/kalamdb-raft/src/commands) diff --git a/link/link-common/src/query/models/mod.rs b/link/link-common/src/query/models/mod.rs index 3f29c1c1..44ed0fa0 100644 --- a/link/link-common/src/query/models/mod.rs +++ b/link/link-common/src/query/models/mod.rs @@ -8,7 +8,7 @@ pub mod upload_progress; pub use error_detail::ErrorDetail; pub use kalamdb_commons::{ - ResponseStatus, SqlSubscriptionDescriptor, SqlSubscriptionRow, SqlSubscriptionStatus, + ResponseStatus, SqlSubscriptionDescriptor, SqlSubscriptionRow, SqlSubscriptionStatus, }; pub use query_request::QueryRequest; pub use query_response::QueryResponse; diff --git a/pg/crates/kalam-pg-common/src/config.rs b/pg/crates/kalam-pg-common/src/config.rs index cb3cfd08..bfdf6424 100644 --- a/pg/crates/kalam-pg-common/src/config.rs +++ b/pg/crates/kalam-pg-common/src/config.rs @@ -154,7 +154,7 @@ impl Default for RemoteServerConfig { fn default() -> Self { Self { host: "127.0.0.1".to_string(), - port: 50051, + port: 2910, timeout_ms: 0, auth_mode: RemoteAuthMode::None, auth_header: None, diff --git a/pg/crates/kalam-pg-fdw/tests/options.rs b/pg/crates/kalam-pg-fdw/tests/options.rs index 9fb71dfd..086ddea8 100644 --- a/pg/crates/kalam-pg-fdw/tests/options.rs +++ b/pg/crates/kalam-pg-fdw/tests/options.rs @@ -7,19 +7,19 @@ use kalam_pg_fdw::ServerOptions; fn parses_remote_server_options() { let options = BTreeMap::from([ ("host".to_string(), "127.0.0.1".to_string()), - ("port".to_string(), "50051".to_string()), + ("port".to_string(), "2910".to_string()), ]); let parsed = ServerOptions::parse(&options).expect("parse remote server options"); assert_eq!(parsed.remote.as_ref().expect("remote config").host, "127.0.0.1"); - assert_eq!(parsed.remote.as_ref().expect("remote config").port, 50051); + assert_eq!(parsed.remote.as_ref().expect("remote config").port, 2910); assert_eq!(parsed.remote.as_ref().expect("remote config").auth_mode, RemoteAuthMode::None); } #[test] fn rejects_missing_host() { - let options = BTreeMap::from([("port".to_string(), "50051".to_string())]); + let options = BTreeMap::from([("port".to_string(), "2910".to_string())]); let err = ServerOptions::parse(&options).expect_err("missing host should fail"); assert!(err.to_string().contains("host")); @@ -67,7 +67,7 @@ fn parses_tls_server_options() { fn rejects_client_cert_without_key() { let options = BTreeMap::from([ ("host".to_string(), "127.0.0.1".to_string()), - ("port".to_string(), "50051".to_string()), + ("port".to_string(), "2910".to_string()), ( "client_cert".to_string(), "-----BEGIN CERTIFICATE-----\ncert\n-----END CERTIFICATE-----".to_string(), @@ -83,7 +83,7 @@ fn rejects_client_cert_without_key() { fn rejects_client_key_without_cert() { let options = BTreeMap::from([ ("host".to_string(), "127.0.0.1".to_string()), - ("port".to_string(), "50051".to_string()), + ("port".to_string(), "2910".to_string()), ( "client_key".to_string(), "-----BEGIN PRIVATE KEY-----\nkey\n-----END PRIVATE KEY-----".to_string(), @@ -99,20 +99,20 @@ fn rejects_client_key_without_cert() { fn non_tls_uses_http_scheme() { let options = BTreeMap::from([ ("host".to_string(), "127.0.0.1".to_string()), - ("port".to_string(), "50051".to_string()), + ("port".to_string(), "2910".to_string()), ]); let parsed = ServerOptions::parse(&options).expect("parse non-TLS options"); let remote = parsed.remote.as_ref().expect("remote config"); assert!(!remote.tls_enabled()); - assert_eq!(remote.endpoint_uri(), "http://127.0.0.1:50051"); + assert_eq!(remote.endpoint_uri(), "http://127.0.0.1:2910"); } #[test] fn legacy_auth_header_defaults_to_static_header_mode() { let options = BTreeMap::from([ ("host".to_string(), "127.0.0.1".to_string()), - ("port".to_string(), "50051".to_string()), + ("port".to_string(), "2910".to_string()), ("auth_header".to_string(), "Bearer legacy-secret".to_string()), ]); @@ -126,7 +126,7 @@ fn legacy_auth_header_defaults_to_static_header_mode() { fn parses_account_login_server_options() { let options = BTreeMap::from([ ("host".to_string(), "127.0.0.1".to_string()), - ("port".to_string(), "50051".to_string()), + ("port".to_string(), "2910".to_string()), ("auth_mode".to_string(), "account_login".to_string()), ("login_user".to_string(), "pg_dba".to_string()), ("login_password".to_string(), "super-secret".to_string()), @@ -143,7 +143,7 @@ fn parses_account_login_server_options() { fn account_login_requires_login_user() { let options = BTreeMap::from([ ("host".to_string(), "127.0.0.1".to_string()), - ("port".to_string(), "50051".to_string()), + ("port".to_string(), "2910".to_string()), ("auth_mode".to_string(), "account_login".to_string()), ("login_password".to_string(), "super-secret".to_string()), ]); @@ -157,7 +157,7 @@ fn account_login_requires_login_user() { fn static_header_rejects_account_login_fields() { let options = BTreeMap::from([ ("host".to_string(), "127.0.0.1".to_string()), - ("port".to_string(), "50051".to_string()), + ("port".to_string(), "2910".to_string()), ("auth_mode".to_string(), "static_header".to_string()), ("auth_header".to_string(), "Bearer legacy-secret".to_string()), ("login_user".to_string(), "pg_dba".to_string()), diff --git a/scripts/cluster.sh b/scripts/cluster.sh index 2b1aa862..baf515f6 100755 --- a/scripts/cluster.sh +++ b/scripts/cluster.sh @@ -83,6 +83,7 @@ NODE3_ID=3 ROOT_PASSWORD="${KALAMDB_ROOT_PASSWORD:-kalamdb123}" ADMIN_PASSWORD="${KALAMDB_ADMIN_PASSWORD:-$ROOT_PASSWORD}" CLUSTER_URLS="http://127.0.0.1:$NODE1_HTTP,http://127.0.0.1:$NODE2_HTTP,http://127.0.0.1:$NODE3_HTTP" +CLI_ENV_FILE="$PROJECT_ROOT/cli/.env" print_header() { echo "" @@ -354,7 +355,7 @@ api_addr = \"http://127.0.0.1:$NODE3_HTTP\" [server] host = "127.0.0.1" port = $http_port -workers = 2 +workers = 0 api_version = "v1" [storage] @@ -371,7 +372,7 @@ default_query_limit = 50 [datafusion] memory_limit = 33554432 query_parallelism = 2 -max_partitions = 2 +max_partitions = 4 batch_size = 1024 [logging] @@ -383,8 +384,8 @@ format = "json" request_timeout = 30 keepalive_timeout = 75 max_connections = 25000 -backlog = 2048 -worker_max_blocking_threads = 64 +backlog = 4096 +worker_max_blocking_threads = 32 client_request_timeout = 5 client_disconnect_timeout = 2 max_header_size = 16384 @@ -393,6 +394,14 @@ max_header_size = 16384 max_queries_per_sec = 10000 max_messages_per_sec = 1000 max_subscriptions_per_user = 1000 +max_auth_requests_per_ip_per_sec = 200000 +max_connections_per_ip = 200000 +max_requests_per_ip_per_sec = 200000 +request_body_limit_bytes = 104857600 +ban_duration_seconds = 300 +enable_connection_protection = true +cache_max_entries = 1000 +cache_ttl_seconds = 600 [topics] visibility_timeout_secs = 10 @@ -534,6 +543,31 @@ check_cluster_ready() { curl -sf "http://127.0.0.1:$http_port/v1/api/cluster/health" 2>/dev/null | grep -q '"status":"healthy"' } +check_node_sql_auth_ready() { + local base_url=$1 + get_access_token "$base_url" >/dev/null 2>&1 +} + +wait_for_cluster_sql_auth_ready() { + local ready=0 + + for _i in {1..60}; do + local count=0 + check_node_sql_auth_ready "http://127.0.0.1:$NODE1_HTTP" && ((count++)) || true + check_node_sql_auth_ready "http://127.0.0.1:$NODE2_HTTP" && ((count++)) || true + check_node_sql_auth_ready "http://127.0.0.1:$NODE3_HTTP" && ((count++)) || true + + if [ "$count" -eq 3 ]; then + ready=1 + break + fi + + sleep 0.5 + done + + [ "$ready" -eq 1 ] +} + start_cluster() { print_header echo -e "${GREEN}Starting 3-node local cluster...${NC}" @@ -596,6 +630,13 @@ start_cluster() { exit 1 fi + echo -e "${YELLOW}Waiting for SQL/auth readiness on all nodes...${NC}" + if ! wait_for_cluster_sql_auth_ready; then + echo -e "${RED}Cluster HTTP endpoints are up, but SQL/auth readiness did not converge on all nodes.${NC}" + echo "Check logs with: ./scripts/cluster.sh logs 1" + exit 1 + fi + echo -e "${GREEN}╔═══════════════════════════════════════════════════════════════════╗${NC}" echo -e "${GREEN}║ Cluster Ready! ║${NC}" echo -e "${GREEN}╚═══════════════════════════════════════════════════════════════════╝${NC}" @@ -779,6 +820,20 @@ ensure_admin_user() { fi } +write_cli_cluster_env() { + cat > "$CLI_ENV_FILE" << EOF +# KalamDB Test Configuration +# Auto-generated by scripts/cluster.sh for local 3-node cluster testing. + +KALAMDB_SERVER_URL=http://127.0.0.1:$NODE1_HTTP +KALAMDB_CLUSTER_URLS=$CLUSTER_URLS +KALAMDB_ROOT_PASSWORD=$ROOT_PASSWORD +KALAMDB_SERVER_TYPE=cluster +EOF + + echo -e "${GREEN}✓ Updated $CLI_ENV_FILE for local cluster testing${NC}" +} + run_cluster_tests() { print_header ensure_cluster_healthy @@ -968,6 +1023,29 @@ run_full_test_suite() { echo -e "${GREEN}╚═══════════════════════════════════════════════════════════════════╝${NC}" } +run_workspace_nextest_e2e() { + print_header + echo -e "${YELLOW}Preparing fresh local cluster for CLI nextest e2e run...${NC}" + echo "" + + stop_cluster >/dev/null 2>&1 || true + + if [ -d "$CLUSTER_DATA_DIR" ]; then + echo -e "${YELLOW}Removing existing local cluster state...${NC}" + rm -rf "$CLUSTER_DATA_DIR" + fi + + start_cluster + write_cli_cluster_env + + echo "" + echo -e "${YELLOW}Running CLI-local nextest with e2e feature (includes cluster tests)...${NC}" + echo "" + + cd "$PROJECT_ROOT/cli" + cargo nextest run --features e2e-tests +} + detect_leader_url() { local query="SELECT api_addr, is_leader FROM system.cluster" local response @@ -1040,6 +1118,8 @@ show_help() { echo " smoke-all Run smoke tests against all nodes (local only)" echo " verify Run consistency verification tests (local only)" echo " full Run complete test suite (local only)" + echo " workspace-e2e Clean-start local cluster, rewrite cli/.env, and run CLI nextest" + echo " (includes dedicated cluster tests already)" echo " shell N Open shell in node N (docker only)" echo "" echo "Examples:" @@ -1049,6 +1129,7 @@ show_help() { echo " $0 logs 1 # View node 1 logs" echo " $0 --docker shell 2 # Open shell in Docker node 2" echo " $0 test # Run tests against running cluster" + echo " $0 workspace-e2e # Fresh cluster + cli/.env rewrite + CLI nextest" echo "" } @@ -1089,7 +1170,7 @@ if [ "$MODE" = "docker" ]; then help|--help|-h|"") show_help ;; - smoke|smoke-all|verify|full) + smoke|smoke-all|verify|full|workspace-e2e) echo -e "${RED}Error: '$1' command is only available in local mode${NC}" echo "Use: $0 $1 (without --docker flag)" exit 1 @@ -1146,6 +1227,9 @@ else full) run_full_test_suite ;; + workspace-e2e) + run_workspace_nextest_e2e + ;; shell) echo -e "${RED}Error: 'shell' command is only available in Docker mode${NC}" echo "Use: $0 --docker shell " diff --git a/specs/026-postgres-extension/README b/specs/026-postgres-extension/README index 47997eda..57614277 100644 --- a/specs/026-postgres-extension/README +++ b/specs/026-postgres-extension/README @@ -172,7 +172,7 @@ CREATE EXTENSION pg_kalam; CREATE SERVER kalam_server FOREIGN DATA WRAPPER pg_kalam OPTIONS ( host '127.0.0.1', - port '50051' + port '2910' ); ``` diff --git a/specs/026-postgres-extension/legacy-dual-mode-reference.md b/specs/026-postgres-extension/legacy-dual-mode-reference.md index a2c406d3..0f2b2290 100644 --- a/specs/026-postgres-extension/legacy-dual-mode-reference.md +++ b/specs/026-postgres-extension/legacy-dual-mode-reference.md @@ -204,7 +204,7 @@ CREATE EXTENSION pg_kalam; -- Example: user table "messages" in namespace "app" CREATE SERVER kalam_server FOREIGN DATA WRAPPER pg_kalam - OPTIONS (host 'localhost', port '50051'); + OPTIONS (host 'localhost', port '2910'); CREATE FOREIGN TABLE kalam_messages ( id BIGINT, From 747527677fb2ff63206047614501a692e8a23c24 Mon Sep 17 00:00:00 2001 From: jamals86 Date: Wed, 13 May 2026 22:16:55 +0300 Subject: [PATCH 3/3] Removed any nextest limitations not needed anymore server is stable now --- .config/nextest.toml | 320 +++++++++++-------------------- nextest.toml | 444 +++++++++++++++++++++---------------------- 2 files changed, 337 insertions(+), 427 deletions(-) diff --git a/.config/nextest.toml b/.config/nextest.toml index 69897c56..86ff2317 100644 --- a/.config/nextest.toml +++ b/.config/nextest.toml @@ -3,211 +3,121 @@ retries = 3 test-threads = 15 -[test-groups] -# Keep these known noisy shared-cluster/perf tests from overlapping with each -# other, while still allowing unrelated tests to use the remaining default -# concurrency budget. -stateful-heavy = { max-threads = 1 } -proxied-reconnect = { max-threads = 2 } - -[[profile.default.overrides]] -filter = 'test(test_scenario_08_subscription_reconnect)' -slow-timeout = { period = "60s", terminate-after = 2 } - -[[profile.default.overrides]] -filter = 'test(test_setup_complete_environment)' -test-group = "stateful-heavy" - -[[profile.default.overrides]] -filter = 'test(idle_autocommit_transaction_checks_add_no_extra_allocations)' -# This perf/allocation regression is stable in isolation but noisy under full workspace contention. -test-group = "stateful-heavy" - -[[profile.default.overrides]] -filter = 'test(autocommit_read_write_latency_regression_stays_within_five_percent)' -# This perf regression compares two nearly identical code paths and needs full -# scheduler isolation from unrelated tests to avoid runner-noise false positives. -threads-required = 15 - -[[profile.default.overrides]] -filter = 'test(e2e_perf_sequential_insert_100)' -# This PG perf check is stable in isolation but noisy under full-suite contention. -test-group = "stateful-heavy" - -[[profile.default.overrides]] -filter = 'test(e2e_perf_cross_verify_latency)' -# This latency check is sensitive to concurrent heavy tests and should run in isolation. -test-group = "stateful-heavy" - -[[profile.default.overrides]] -filter = 'test(e2e_perf_local_memory_stays_bounded_under_batch_insert_and_scan)' -# This memory-bound perf check is meaningful only without competing suite load. -test-group = "stateful-heavy" - -[[profile.default.overrides]] -filter = 'test(test_topic_http_consume_direct_multi_user_publishers_no_missing_changes)' -# These topic smoke tests are stable in isolation but can contend under full workspace load. -test-group = "stateful-heavy" - -[[profile.default.overrides]] -filter = 'test(test_topic_http_consume_preserves_impersonated_user_and_payloads)' -test-group = "stateful-heavy" - -[[profile.default.overrides]] -filter = 'test(test_topic_consume_option_matrix_start_batch_auto_ack_modes)' -test-group = "stateful-heavy" - -[[profile.default.overrides]] -filter = 'test(test_topic_high_load_two_consumers_same_group_single_delivery)' -test-group = "stateful-heavy" - -[[profile.default.overrides]] -filter = 'test(test_topic_four_consumers_same_group_no_duplicates)' -test-group = "stateful-heavy" - -[[profile.default.overrides]] -filter = 'test(test_topic_ack_failure_recovery_no_message_loss_with_latency)' -test-group = "stateful-heavy" - -[[profile.default.overrides]] -filter = 'test(test_topic_fan_out_different_groups_receive_all)' -test-group = "stateful-heavy" - -[[profile.default.overrides]] -filter = 'test(test_event_counter_integrity_through_multiple_outages)' -test-group = "stateful-heavy" - -[[profile.default.overrides]] -filter = 'test(test_gradual_latency_ramp_forces_reconnect_then_recovers)' -test-group = "stateful-heavy" - -[[profile.default.overrides]] -filter = 'test(test_heavy_write_burst_during_outage_all_delivered)' -test-group = "stateful-heavy" - -[[profile.default.overrides]] -filter = 'test(test_large_initial_snapshot_survives_repeated_outages)' -test-group = "proxied-reconnect" - -[[profile.default.overrides]] -filter = 'test(test_latency_spike_during_initial_snapshot_recovers)' -test-group = "stateful-heavy" - -[[profile.default.overrides]] -filter = 'test(test_proxy_server_down_during_live_updates_resumes)' -test-group = "stateful-heavy" - -[[profile.default.overrides]] -filter = 'test(test_loading_snapshot_with_live_writes_resumes_without_duplicate_rows)' -test-group = "proxied-reconnect" - -[[profile.default.overrides]] -filter = 'test(test_proxy_three_subscriptions_resume_after_server_bounce)' -test-group = "stateful-heavy" - -[[profile.default.overrides]] -filter = 'test(test_shared_connection_recovers_subscriptions_in_different_stages)' -test-group = "stateful-heavy" - -[[profile.default.overrides]] -filter = 'test(test_cli_syntax_error_handling)' -test-group = "stateful-heavy" - -[[profile.default.overrides]] -filter = 'test(test_connection_timeout_option)' -# Pure options test, but full-suite CLI subprocess load can make nextest assign leaked output -# handles here; keep it isolated from subprocess-heavy tests. -test-group = "stateful-heavy" - -[[profile.default.overrides]] -filter = 'test(test_rapid_connect_disconnect)' -test-group = "stateful-heavy" - -[[profile.default.overrides]] -filter = 'test(test_concurrent_websocket_subscriptions)' -test-group = "stateful-heavy" - -[[profile.default.overrides]] -filter = 'test(smoke_test_leader_read_shared_table)' -test-group = "stateful-heavy" - -[[profile.default.overrides]] -filter = 'test(cluster_test_table_identity_updates)' -test-group = "stateful-heavy" - -[[profile.default.overrides]] -filter = 'test(smoke_all_datatypes_user_shared_stream)' -test-group = "stateful-heavy" - -[[profile.default.overrides]] -filter = 'test(cluster_test_table_identity_mixed_operations)' -test-group = "stateful-heavy" - -[[profile.default.overrides]] -filter = 'test(cluster_test_ws_follower_receives_leader_changes)' -test-group = "stateful-heavy" - -[[profile.default.overrides]] -filter = 'test(cluster_test_table_identity_user_tables)' -test-group = "stateful-heavy" - -[[profile.default.overrides]] -filter = 'test(smoke_security_regular_user_cannot_impersonate_privileged_users_in_batch)' -test-group = "stateful-heavy" - -[[profile.default.overrides]] -filter = 'test(smoke_as_user_chat_delete_flow)' -test-group = "stateful-heavy" - -[[profile.default.overrides]] -filter = 'test(smoke_insert_returning_seq_multi_row)' -test-group = "stateful-heavy" - -[[profile.default.overrides]] -filter = 'test(smoke_rpc_login_nonexistent_user_matches_wrong_password_response)' -test-group = "stateful-heavy" - -[[profile.default.overrides]] -filter = 'test(smoke_security_private_shared_table_blocked_in_batch)' -test-group = "stateful-heavy" - -[[profile.default.overrides]] -filter = 'test(smoke_security_system_tables_blocked_in_batch)' -test-group = "stateful-heavy" - -[[profile.default.overrides]] -filter = 'test(smoke_storage_custom_templates)' -test-group = "stateful-heavy" - -[[profile.default.overrides]] -filter = 'test(smoke_storage_check_dba_access)' -test-group = "stateful-heavy" - -[[profile.default.overrides]] -filter = 'test(smoke_storage_check_authorization)' -test-group = "stateful-heavy" - -[[profile.default.overrides]] -filter = 'test(test_topic_consume_update_events)' -test-group = "stateful-heavy" - -[[profile.default.overrides]] -filter = 'test(test_topic_consume_offset_persistence)' -test-group = "stateful-heavy" - -[[profile.default.overrides]] -filter = 'test(smoke_export_download_forbidden_for_other_user)' -test-group = "stateful-heavy" - -[[profile.default.overrides]] -filter = 'test(cluster_test_leader_only_flush_jobs)' -# Uses system.jobs queries — must not race with other tests' flush jobs or background scheduler. -test-group = "stateful-heavy" - -[[profile.default.overrides]] -filter = 'test(cluster_test_jobs_table_consistency)' -# Queries global system.jobs count — must not race with concurrent tests creating jobs. -test-group = "stateful-heavy" +# [test-groups] +# # Keep these known noisy shared-cluster/perf tests from overlapping with each +# # other, while still allowing unrelated tests to use the remaining default +# # concurrency budget. +# stateful-heavy = { max-threads = 1 } +# proxied-reconnect = { max-threads = 2 } + +# [[profile.default.overrides]] +# filter = 'test(test_scenario_08_subscription_reconnect)' +# slow-timeout = { period = "60s", terminate-after = 2 } + +# [[profile.default.overrides]] +# filter = 'test(test_setup_complete_environment)' +# test-group = "stateful-heavy" + +# [[profile.default.overrides]] +# filter = 'test(idle_autocommit_transaction_checks_add_no_extra_allocations)' +# # This perf/allocation regression is stable in isolation but noisy under full workspace contention. +# test-group = "stateful-heavy" + +# [[profile.default.overrides]] +# filter = 'test(autocommit_read_write_latency_regression_stays_within_five_percent)' +# # This perf regression compares two nearly identical code paths and needs full +# # scheduler isolation from unrelated tests to avoid runner-noise false positives. +# threads-required = 15 + +# [[profile.default.overrides]] +# filter = 'test(e2e_perf_sequential_insert_100)' +# # This PG perf check is stable in isolation but noisy under full-suite contention. +# test-group = "stateful-heavy" + +# [[profile.default.overrides]] +# filter = 'test(e2e_perf_cross_verify_latency)' +# # This latency check is sensitive to concurrent heavy tests and should run in isolation. +# test-group = "stateful-heavy" + +# [[profile.default.overrides]] +# filter = 'test(e2e_perf_local_memory_stays_bounded_under_batch_insert_and_scan)' +# # This memory-bound perf check is meaningful only without competing suite load. +# test-group = "stateful-heavy" + +# [[profile.default.overrides]] +# filter = 'test(test_topic_http_consume_direct_multi_user_publishers_no_missing_changes)' +# # These topic smoke tests are stable in isolation but can contend under full workspace load. +# test-group = "stateful-heavy" + +# [[profile.default.overrides]] +# filter = 'test(test_topic_http_consume_preserves_impersonated_user_and_payloads)' +# test-group = "stateful-heavy" + +# [[profile.default.overrides]] +# filter = 'test(test_topic_consume_option_matrix_start_batch_auto_ack_modes)' +# test-group = "stateful-heavy" + +# [[profile.default.overrides]] +# filter = 'test(test_topic_high_load_two_consumers_same_group_single_delivery)' +# test-group = "stateful-heavy" + +# [[profile.default.overrides]] +# filter = 'test(test_topic_four_consumers_same_group_no_duplicates)' +# test-group = "stateful-heavy" + +# [[profile.default.overrides]] +# filter = 'test(test_topic_ack_failure_recovery_no_message_loss_with_latency)' +# test-group = "stateful-heavy" + +# [[profile.default.overrides]] +# filter = 'test(test_topic_fan_out_different_groups_receive_all)' +# test-group = "stateful-heavy" + +# [[profile.default.overrides]] +# filter = 'test(test_event_counter_integrity_through_multiple_outages)' +# test-group = "stateful-heavy" + +# [[profile.default.overrides]] +# filter = 'test(test_gradual_latency_ramp_forces_reconnect_then_recovers)' +# test-group = "stateful-heavy" + +# [[profile.default.overrides]] +# filter = 'test(test_heavy_write_burst_during_outage_all_delivered)' +# test-group = "stateful-heavy" + +# [[profile.default.overrides]] +# filter = 'test(test_large_initial_snapshot_survives_repeated_outages)' +# test-group = "proxied-reconnect" + +# [[profile.default.overrides]] +# filter = 'test(test_latency_spike_during_initial_snapshot_recovers)' +# test-group = "stateful-heavy" + +# [[profile.default.overrides]] +# filter = 'test(test_proxy_server_down_during_live_updates_resumes)' +# test-group = "stateful-heavy" + +# [[profile.default.overrides]] +# filter = 'test(test_loading_snapshot_with_live_writes_resumes_without_duplicate_rows)' +# test-group = "proxied-reconnect" + +# [[profile.default.overrides]] +# filter = 'test(test_proxy_three_subscriptions_resume_after_server_bounce)' +# test-group = "stateful-heavy" + +# [[profile.default.overrides]] +# filter = 'test(test_shared_connection_recovers_subscriptions_in_different_stages)' +# test-group = "stateful-heavy" + +# [[profile.default.overrides]] +# filter = 'test(test_cli_syntax_error_handling)' +# test-group = "stateful-heavy" + +# [[profile.default.overrides]] +# filter = 'test(test_connection_timeout_option)' +# # Pure options test, but full-suite CLI subprocess load can make nextest assign leaked output +# # handles here; keep it isolated from subprocess-heavy tests. +# test-group = "stateful-heavy" [profile.ci] # Store test results in JUnit format diff --git a/nextest.toml b/nextest.toml index 39d153b3..7a46d8dc 100644 --- a/nextest.toml +++ b/nextest.toml @@ -10,228 +10,228 @@ test-threads = 15 stateful-heavy = { max-threads = 1 } proxied-reconnect = { max-threads = 2 } -[[profile.default.overrides]] -filter = 'test(test_scenario_08_subscription_reconnect)' -slow-timeout = { period = "60s", terminate-after = 2 } - -[[profile.default.overrides]] -filter = 'test(test_setup_complete_environment)' -test-group = "stateful-heavy" - -[[profile.default.overrides]] -filter = 'test(idle_autocommit_transaction_checks_add_no_extra_allocations)' -# This perf/allocation regression is stable in isolation but noisy under full workspace contention. -test-group = "stateful-heavy" - -[[profile.default.overrides]] -filter = 'test(autocommit_read_write_latency_regression_stays_within_five_percent)' -# This perf regression compares two nearly identical code paths and needs full -# scheduler isolation from unrelated tests to avoid runner-noise false positives. -threads-required = 15 - -[[profile.default.overrides]] -filter = 'test(e2e_perf_sequential_insert_100)' -# This PG perf check is stable in isolation but noisy under full-suite contention. -test-group = "stateful-heavy" - -[[profile.default.overrides]] -filter = 'test(e2e_perf_cross_verify_latency)' -# This latency check is sensitive to concurrent heavy tests and should run in isolation. -test-group = "stateful-heavy" - -[[profile.default.overrides]] -filter = 'test(e2e_perf_local_memory_stays_bounded_under_batch_insert_and_scan)' -# This memory-bound perf check is meaningful only without competing suite load. -test-group = "stateful-heavy" - -[[profile.default.overrides]] -filter = 'test(test_topic_http_consume_direct_multi_user_publishers_no_missing_changes)' -# These topic smoke tests are stable in isolation but can contend under full workspace load. -test-group = "stateful-heavy" - -[[profile.default.overrides]] -filter = 'test(test_topic_http_consume_preserves_impersonated_user_and_payloads)' -test-group = "stateful-heavy" - -[[profile.default.overrides]] -filter = 'test(test_topic_consume_option_matrix_start_batch_auto_ack_modes)' -test-group = "stateful-heavy" - -[[profile.default.overrides]] -filter = 'test(test_topic_high_load_two_consumers_same_group_single_delivery)' -test-group = "stateful-heavy" - -[[profile.default.overrides]] -filter = 'test(test_topic_four_consumers_same_group_no_duplicates)' -test-group = "stateful-heavy" - -[[profile.default.overrides]] -filter = 'test(test_topic_ack_failure_recovery_no_message_loss_with_latency)' -test-group = "stateful-heavy" - -[[profile.default.overrides]] -filter = 'test(test_topic_fan_out_different_groups_receive_all)' -test-group = "stateful-heavy" - -[[profile.default.overrides]] -filter = 'test(test_event_counter_integrity_through_multiple_outages)' -test-group = "stateful-heavy" - -[[profile.default.overrides]] -filter = 'test(test_gradual_latency_ramp_forces_reconnect_then_recovers)' -test-group = "stateful-heavy" - -[[profile.default.overrides]] -filter = 'test(test_heavy_write_burst_during_outage_all_delivered)' -test-group = "stateful-heavy" - -[[profile.default.overrides]] -filter = 'test(test_large_initial_snapshot_survives_repeated_outages)' -test-group = "proxied-reconnect" - -[[profile.default.overrides]] -filter = 'test(test_latency_spike_during_initial_snapshot_recovers)' -test-group = "stateful-heavy" - -[[profile.default.overrides]] -filter = 'test(test_proxy_server_down_during_live_updates_resumes)' -test-group = "stateful-heavy" - -[[profile.default.overrides]] -filter = 'test(test_loading_snapshot_with_live_writes_resumes_without_duplicate_rows)' -test-group = "proxied-reconnect" - -[[profile.default.overrides]] -filter = 'test(test_proxy_three_subscriptions_resume_after_server_bounce)' -test-group = "stateful-heavy" - -[[profile.default.overrides]] -filter = 'test(test_shared_connection_recovers_subscriptions_in_different_stages)' -test-group = "stateful-heavy" - -[[profile.default.overrides]] -filter = 'test(test_tokio_netem_fragmented_writes_preserve_live_stream)' -test-group = "stateful-heavy" - -[[profile.default.overrides]] -filter = 'test(test_tokio_netem_bandwidth_collapse_forces_resume_without_replay)' -test-group = "stateful-heavy" - -[[profile.default.overrides]] -filter = 'test(test_tokio_netem_forced_transport_termination_recovers)' -test-group = "stateful-heavy" - -[[profile.default.overrides]] -filter = 'test(test_tokio_netem_topic_consume_fragmented_insert_update_delete_and_commit)' -test-group = "stateful-heavy" - -[[profile.default.overrides]] -filter = 'test(test_tokio_netem_topic_bandwidth_collapse_poll_recovers_without_losing_offsets)' -test-group = "stateful-heavy" - -[[profile.default.overrides]] -filter = 'test(test_tokio_netem_topic_commit_failure_can_be_retried_without_replay)' -test-group = "stateful-heavy" - -[[profile.default.overrides]] -filter = 'test(test_cli_syntax_error_handling)' -test-group = "stateful-heavy" - -[[profile.default.overrides]] -filter = 'test(test_connection_timeout_option)' -# Pure options test, but full-suite CLI subprocess load can make nextest assign leaked output -# handles here; keep it isolated from subprocess-heavy tests. -test-group = "stateful-heavy" - -[[profile.default.overrides]] -filter = 'test(test_rapid_connect_disconnect)' -test-group = "stateful-heavy" - -[[profile.default.overrides]] -filter = 'test(test_concurrent_websocket_subscriptions)' -test-group = "stateful-heavy" - -[[profile.default.overrides]] -filter = 'test(smoke_test_leader_read_shared_table)' -test-group = "stateful-heavy" - -[[profile.default.overrides]] -filter = 'test(cluster_test_table_identity_updates)' -test-group = "stateful-heavy" - -[[profile.default.overrides]] -filter = 'test(smoke_all_datatypes_user_shared_stream)' -test-group = "stateful-heavy" - -[[profile.default.overrides]] -filter = 'test(cluster_test_table_identity_mixed_operations)' -test-group = "stateful-heavy" - -[[profile.default.overrides]] -filter = 'test(cluster_test_ws_follower_receives_leader_changes)' -test-group = "stateful-heavy" - -[[profile.default.overrides]] -filter = 'test(cluster_test_table_identity_user_tables)' -test-group = "stateful-heavy" - -[[profile.default.overrides]] -filter = 'test(smoke_security_regular_user_cannot_impersonate_privileged_users_in_batch)' -test-group = "stateful-heavy" - -[[profile.default.overrides]] -filter = 'test(smoke_as_user_chat_delete_flow)' -test-group = "stateful-heavy" - -[[profile.default.overrides]] -filter = 'test(smoke_insert_returning_seq_multi_row)' -test-group = "stateful-heavy" - -[[profile.default.overrides]] -filter = 'test(smoke_rpc_login_nonexistent_user_matches_wrong_password_response)' -test-group = "stateful-heavy" - -[[profile.default.overrides]] -filter = 'test(smoke_security_private_shared_table_blocked_in_batch)' -test-group = "stateful-heavy" - -[[profile.default.overrides]] -filter = 'test(smoke_security_system_tables_blocked_in_batch)' -test-group = "stateful-heavy" - -[[profile.default.overrides]] -filter = 'test(smoke_storage_custom_templates)' -test-group = "stateful-heavy" - -[[profile.default.overrides]] -filter = 'test(smoke_storage_check_dba_access)' -test-group = "stateful-heavy" - -[[profile.default.overrides]] -filter = 'test(smoke_storage_check_authorization)' -test-group = "stateful-heavy" - -[[profile.default.overrides]] -filter = 'test(test_topic_consume_update_events)' -test-group = "stateful-heavy" - -[[profile.default.overrides]] -filter = 'test(test_topic_consume_offset_persistence)' -test-group = "stateful-heavy" - -[[profile.default.overrides]] -filter = 'test(smoke_export_download_forbidden_for_other_user)' -test-group = "stateful-heavy" - -[[profile.default.overrides]] -filter = 'test(cluster_test_leader_only_flush_jobs)' -# Uses system.jobs queries — must not race with other tests' flush jobs or background scheduler. -test-group = "stateful-heavy" - -[[profile.default.overrides]] -filter = 'test(cluster_test_jobs_table_consistency)' -# Queries global system.jobs count — must not race with concurrent tests creating jobs. -test-group = "stateful-heavy" +# [[profile.default.overrides]] +# filter = 'test(test_scenario_08_subscription_reconnect)' +# slow-timeout = { period = "60s", terminate-after = 2 } + +# [[profile.default.overrides]] +# filter = 'test(test_setup_complete_environment)' +# test-group = "stateful-heavy" + +# [[profile.default.overrides]] +# filter = 'test(idle_autocommit_transaction_checks_add_no_extra_allocations)' +# # This perf/allocation regression is stable in isolation but noisy under full workspace contention. +# test-group = "stateful-heavy" + +# [[profile.default.overrides]] +# filter = 'test(autocommit_read_write_latency_regression_stays_within_five_percent)' +# # This perf regression compares two nearly identical code paths and needs full +# # scheduler isolation from unrelated tests to avoid runner-noise false positives. +# threads-required = 15 + +# [[profile.default.overrides]] +# filter = 'test(e2e_perf_sequential_insert_100)' +# # This PG perf check is stable in isolation but noisy under full-suite contention. +# test-group = "stateful-heavy" + +# [[profile.default.overrides]] +# filter = 'test(e2e_perf_cross_verify_latency)' +# # This latency check is sensitive to concurrent heavy tests and should run in isolation. +# test-group = "stateful-heavy" + +# [[profile.default.overrides]] +# filter = 'test(e2e_perf_local_memory_stays_bounded_under_batch_insert_and_scan)' +# # This memory-bound perf check is meaningful only without competing suite load. +# test-group = "stateful-heavy" + +# [[profile.default.overrides]] +# filter = 'test(test_topic_http_consume_direct_multi_user_publishers_no_missing_changes)' +# # These topic smoke tests are stable in isolation but can contend under full workspace load. +# test-group = "stateful-heavy" + +# [[profile.default.overrides]] +# filter = 'test(test_topic_http_consume_preserves_impersonated_user_and_payloads)' +# test-group = "stateful-heavy" + +# [[profile.default.overrides]] +# filter = 'test(test_topic_consume_option_matrix_start_batch_auto_ack_modes)' +# test-group = "stateful-heavy" + +# [[profile.default.overrides]] +# filter = 'test(test_topic_high_load_two_consumers_same_group_single_delivery)' +# test-group = "stateful-heavy" + +# [[profile.default.overrides]] +# filter = 'test(test_topic_four_consumers_same_group_no_duplicates)' +# test-group = "stateful-heavy" + +# [[profile.default.overrides]] +# filter = 'test(test_topic_ack_failure_recovery_no_message_loss_with_latency)' +# test-group = "stateful-heavy" + +# [[profile.default.overrides]] +# filter = 'test(test_topic_fan_out_different_groups_receive_all)' +# test-group = "stateful-heavy" + +# [[profile.default.overrides]] +# filter = 'test(test_event_counter_integrity_through_multiple_outages)' +# test-group = "stateful-heavy" + +# [[profile.default.overrides]] +# filter = 'test(test_gradual_latency_ramp_forces_reconnect_then_recovers)' +# test-group = "stateful-heavy" + +# [[profile.default.overrides]] +# filter = 'test(test_heavy_write_burst_during_outage_all_delivered)' +# test-group = "stateful-heavy" + +# [[profile.default.overrides]] +# filter = 'test(test_large_initial_snapshot_survives_repeated_outages)' +# test-group = "proxied-reconnect" + +# [[profile.default.overrides]] +# filter = 'test(test_latency_spike_during_initial_snapshot_recovers)' +# test-group = "stateful-heavy" + +# [[profile.default.overrides]] +# filter = 'test(test_proxy_server_down_during_live_updates_resumes)' +# test-group = "stateful-heavy" + +# [[profile.default.overrides]] +# filter = 'test(test_loading_snapshot_with_live_writes_resumes_without_duplicate_rows)' +# test-group = "proxied-reconnect" + +# [[profile.default.overrides]] +# filter = 'test(test_proxy_three_subscriptions_resume_after_server_bounce)' +# test-group = "stateful-heavy" + +# [[profile.default.overrides]] +# filter = 'test(test_shared_connection_recovers_subscriptions_in_different_stages)' +# test-group = "stateful-heavy" + +# [[profile.default.overrides]] +# filter = 'test(test_tokio_netem_fragmented_writes_preserve_live_stream)' +# test-group = "stateful-heavy" + +# [[profile.default.overrides]] +# filter = 'test(test_tokio_netem_bandwidth_collapse_forces_resume_without_replay)' +# test-group = "stateful-heavy" + +# [[profile.default.overrides]] +# filter = 'test(test_tokio_netem_forced_transport_termination_recovers)' +# test-group = "stateful-heavy" + +# [[profile.default.overrides]] +# filter = 'test(test_tokio_netem_topic_consume_fragmented_insert_update_delete_and_commit)' +# test-group = "stateful-heavy" + +# [[profile.default.overrides]] +# filter = 'test(test_tokio_netem_topic_bandwidth_collapse_poll_recovers_without_losing_offsets)' +# test-group = "stateful-heavy" + +# [[profile.default.overrides]] +# filter = 'test(test_tokio_netem_topic_commit_failure_can_be_retried_without_replay)' +# test-group = "stateful-heavy" + +# [[profile.default.overrides]] +# filter = 'test(test_cli_syntax_error_handling)' +# test-group = "stateful-heavy" + +# [[profile.default.overrides]] +# filter = 'test(test_connection_timeout_option)' +# # Pure options test, but full-suite CLI subprocess load can make nextest assign leaked output +# # handles here; keep it isolated from subprocess-heavy tests. +# test-group = "stateful-heavy" + +# [[profile.default.overrides]] +# filter = 'test(test_rapid_connect_disconnect)' +# test-group = "stateful-heavy" + +# [[profile.default.overrides]] +# filter = 'test(test_concurrent_websocket_subscriptions)' +# test-group = "stateful-heavy" + +# [[profile.default.overrides]] +# filter = 'test(smoke_test_leader_read_shared_table)' +# test-group = "stateful-heavy" + +# [[profile.default.overrides]] +# filter = 'test(cluster_test_table_identity_updates)' +# test-group = "stateful-heavy" + +# [[profile.default.overrides]] +# filter = 'test(smoke_all_datatypes_user_shared_stream)' +# test-group = "stateful-heavy" + +# [[profile.default.overrides]] +# filter = 'test(cluster_test_table_identity_mixed_operations)' +# test-group = "stateful-heavy" + +# [[profile.default.overrides]] +# filter = 'test(cluster_test_ws_follower_receives_leader_changes)' +# test-group = "stateful-heavy" + +# [[profile.default.overrides]] +# filter = 'test(cluster_test_table_identity_user_tables)' +# test-group = "stateful-heavy" + +# [[profile.default.overrides]] +# filter = 'test(smoke_security_regular_user_cannot_impersonate_privileged_users_in_batch)' +# test-group = "stateful-heavy" + +# [[profile.default.overrides]] +# filter = 'test(smoke_as_user_chat_delete_flow)' +# test-group = "stateful-heavy" + +# [[profile.default.overrides]] +# filter = 'test(smoke_insert_returning_seq_multi_row)' +# test-group = "stateful-heavy" + +# [[profile.default.overrides]] +# filter = 'test(smoke_rpc_login_nonexistent_user_matches_wrong_password_response)' +# test-group = "stateful-heavy" + +# [[profile.default.overrides]] +# filter = 'test(smoke_security_private_shared_table_blocked_in_batch)' +# test-group = "stateful-heavy" + +# [[profile.default.overrides]] +# filter = 'test(smoke_security_system_tables_blocked_in_batch)' +# test-group = "stateful-heavy" + +# [[profile.default.overrides]] +# filter = 'test(smoke_storage_custom_templates)' +# test-group = "stateful-heavy" + +# [[profile.default.overrides]] +# filter = 'test(smoke_storage_check_dba_access)' +# test-group = "stateful-heavy" + +# [[profile.default.overrides]] +# filter = 'test(smoke_storage_check_authorization)' +# test-group = "stateful-heavy" + +# [[profile.default.overrides]] +# filter = 'test(test_topic_consume_update_events)' +# test-group = "stateful-heavy" + +# [[profile.default.overrides]] +# filter = 'test(test_topic_consume_offset_persistence)' +# test-group = "stateful-heavy" + +# [[profile.default.overrides]] +# filter = 'test(smoke_export_download_forbidden_for_other_user)' +# test-group = "stateful-heavy" + +# [[profile.default.overrides]] +# filter = 'test(cluster_test_leader_only_flush_jobs)' +# # Uses system.jobs queries — must not race with other tests' flush jobs or background scheduler. +# test-group = "stateful-heavy" + +# [[profile.default.overrides]] +# filter = 'test(cluster_test_jobs_table_consistency)' +# # Queries global system.jobs count — must not race with concurrent tests creating jobs. +# test-group = "stateful-heavy" # [[profile.default.overrides]] # filter = 'package(kalamdb-server)'