diff --git a/Cargo.lock b/Cargo.lock index 11368025c..18695279f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -620,6 +620,21 @@ dependencies = [ "thiserror 2.0.17", ] +[[package]] +name = "cassowary" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df8670b8c7b9dae1793364eafadf7239c40d669904660c5960d74cfd80b46a53" + +[[package]] +name = "castaway" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dec551ab6e7578819132c713a93c022a05d60159dc86e7a7050223577484c55a" +dependencies = [ + "rustversion", +] + [[package]] name = "cc" version = "1.2.39" @@ -821,16 +836,31 @@ version = "0.1.0" dependencies = [ "clap", "clearscreen", + "cmon-common", + "crossterm", "crucible", "crucible-control-client", "crucible-protocol", "crucible-workspace-hack", + "ratatui", + "serde", "serde_json", "strum 0.27.2", - "strum_macros 0.27.2", "tokio", ] +[[package]] +name = "cmon-common" +version = "0.1.0" +dependencies = [ + "clap", + "crucible", + "crucible-workspace-hack", + "serde", + "strum 0.27.2", + "strum_macros 0.27.2", +] + [[package]] name = "cobs" version = "0.2.3" @@ -866,6 +896,20 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "compact_str" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b79c4069c6cad78e2e0cdfcbd26275770669fb39fd308a752dc110e83b9af32" +dependencies = [ + "castaway", + "cfg-if", + "itoa", + "rustversion", + "ryu", + "static_assertions", +] + [[package]] name = "console" version = "0.15.7" @@ -1576,6 +1620,7 @@ dependencies = [ "clap_builder", "crossbeam-epoch", "crossbeam-utils", + "crossterm", "crypto-common", "digest", "dof 0.3.0", @@ -1754,6 +1799,23 @@ dependencies = [ "memchr", ] +[[package]] +name = "ctop" +version = "0.1.0" +dependencies = [ + "clap", + "cmon-common", + "crossterm", + "crucible", + "crucible-protocol", + "crucible-workspace-hack", + "ratatui", + "serde", + "serde_json", + "strum 0.27.2", + "tokio", +] + [[package]] name = "ctr" version = "0.9.2" @@ -3499,6 +3561,15 @@ dependencies = [ "web-time 1.1.0", ] +[[package]] +name = "indoc" +version = "2.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79cf5c93f93228cf8efb3ba362535fb11199ac548a09ce117c9b1adc3030d706" +dependencies = [ + "rustversion", +] + [[package]] name = "ingot" version = "0.1.1" @@ -3547,6 +3618,19 @@ dependencies = [ "generic-array", ] +[[package]] +name = "instability" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "435d80800b936787d62688c927b6490e887c7ef5ff9ce922c6c6050fca75eb9a" +dependencies = [ + "darling 0.20.11", + "indoc", + "proc-macro2", + "quote", + "syn 2.0.110", +] + [[package]] name = "instant" version = "0.1.12" @@ -3924,6 +4008,15 @@ dependencies = [ "tracing-subscriber", ] +[[package]] +name = "lru" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "234cf4f4a04dc1f57e24b96cc0cd600cf2af460d4161ac5ecdd0af8e1f3b2a38" +dependencies = [ + "hashbrown 0.15.2", +] + [[package]] name = "lru-cache" version = "0.1.2" @@ -5935,6 +6028,27 @@ version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "acbbbbea733ec66275512d0b9694f34102e7d5406fdbe2ad8d21b28dce92887c" +[[package]] +name = "ratatui" +version = "0.28.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdef7f9be5c0122f890d58bdf4d964349ba6a6161f705907526d891efabba57d" +dependencies = [ + "bitflags 2.9.4", + "cassowary", + "compact_str", + "crossterm", + "instability", + "itertools 0.13.0", + "lru", + "paste", + "strum 0.26.3", + "strum_macros 0.26.4", + "unicode-segmentation", + "unicode-truncate", + "unicode-width 0.1.14", +] + [[package]] name = "rayon" version = "1.11.0" @@ -8140,6 +8254,17 @@ version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1dd624098567895118886609431a7c3b8f516e41d30e0643f03d94592a147e36" +[[package]] +name = "unicode-truncate" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3644627a5af5fa321c95b9b235a72fd24cd29c648c2c379431e6628655627bf" +dependencies = [ + "itertools 0.13.0", + "unicode-segmentation", + "unicode-width 0.1.14", +] + [[package]] name = "unicode-width" version = "0.1.14" diff --git a/Cargo.toml b/Cargo.toml index 08b7d11e4..0473df6ce 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,8 +8,10 @@ members = [ "common", "control-client", "cmon", + "cmon-common", "crudd", "crutest", + "ctop", "downstairs", "downstairs-api", "downstairs-types", @@ -90,6 +92,7 @@ proptest = "1.9.0" rayon = "1.11.0" rand = { version = "0.9.2", features = [ "small_rng"] } rand_chacha = "0.9.0" +ratatui = "0.28" reedline = "0.43.0" rangemap = "1.7.0" reqwest = { version = "0.12", features = ["default", "blocking", "json", "stream"] } @@ -140,6 +143,7 @@ oximeter = { git = "https://github.com/oxidecomputer/omicron", branch = "main" } oximeter-producer = { git = "https://github.com/oxidecomputer/omicron", branch = "main" } # local path +cmon-common = { path = "./cmon-common" } crucible = { path = "./upstairs" } crucible-agent-api = { path = "./agent-api" } crucible-agent-client = { path = "./agent-client" } diff --git a/cmon-common/Cargo.toml b/cmon-common/Cargo.toml new file mode 100644 index 000000000..cb1de3826 --- /dev/null +++ b/cmon-common/Cargo.toml @@ -0,0 +1,19 @@ +[package] +name = "cmon-common" +version = "0.1.0" +license = "MPL-2.0" +edition = "2024" + +[dependencies] +crucible.workspace = true +serde.workspace = true +strum.workspace = true +strum_macros.workspace = true +crucible-workspace-hack.workspace = true + +# Optional dependency for clap integration +clap = { workspace = true, optional = true } + +[features] +# Enable clap ValueEnum derive for DtraceDisplay +clap = ["dep:clap"] diff --git a/cmon-common/src/lib.rs b/cmon-common/src/lib.rs new file mode 100644 index 000000000..fa1fc190f --- /dev/null +++ b/cmon-common/src/lib.rs @@ -0,0 +1,214 @@ +// Copyright 2026 Oxide Computer Company + +//! Common types and utilities shared between cmon and ctop + +use crucible::DtraceInfo; +use serde::Deserialize; +use std::fmt; +use strum_macros::EnumIter; + +/// Wrapper for DTrace output with PID +#[derive(Debug, Deserialize)] +pub struct DtraceWrapper { + pub pid: u32, + pub status: DtraceInfo, +} + +/// The possible fields we will display when receiving DTrace output. +#[derive(Debug, Copy, Clone, PartialEq, Eq, EnumIter)] +#[cfg_attr(feature = "clap", derive(clap::ValueEnum))] +pub enum DtraceDisplay { + Pid, + Session, + UpstairsId, + State, + IoCount, + IoSummary, + UpCount, + DsCount, + Reconcile, + DsReconciled, + DsReconcileNeeded, + LiveRepair, + Connected, + Replaced, + ExtentLiveRepair, + ExtentLimit, + NextJobId, + JobDelta, + DsDelay, + WriteBytesOut, + RoLrSkipped, + DsIoInProgress, + DsIoDone, + DsIoSkipped, + DsIoError, +} + +impl fmt::Display for DtraceDisplay { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + match self { + DtraceDisplay::Pid => write!(f, "pid"), + DtraceDisplay::Session => write!(f, "session"), + DtraceDisplay::UpstairsId => write!(f, "upstairs_id"), + DtraceDisplay::State => write!(f, "state"), + DtraceDisplay::IoCount => write!(f, "io_count"), + DtraceDisplay::IoSummary => write!(f, "io_summary"), + DtraceDisplay::UpCount => write!(f, "up_count"), + DtraceDisplay::DsCount => write!(f, "ds_count"), + DtraceDisplay::Reconcile => write!(f, "reconcile"), + DtraceDisplay::DsReconciled => write!(f, "ds_reconciled"), + DtraceDisplay::DsReconcileNeeded => { + write!(f, "ds_reconcile_needed") + } + DtraceDisplay::LiveRepair => write!(f, "live_repair"), + DtraceDisplay::Connected => write!(f, "connected"), + DtraceDisplay::Replaced => write!(f, "replaced"), + DtraceDisplay::ExtentLiveRepair => write!(f, "extent_live_repair"), + DtraceDisplay::ExtentLimit => write!(f, "extent_under_repair"), + DtraceDisplay::NextJobId => write!(f, "next_job_id"), + DtraceDisplay::JobDelta => write!(f, "job_delta"), + DtraceDisplay::DsDelay => write!(f, "ds_delay"), + DtraceDisplay::WriteBytesOut => write!(f, "write_bytes_out"), + DtraceDisplay::RoLrSkipped => write!(f, "ro_lr_skipped"), + DtraceDisplay::DsIoInProgress => write!(f, "ds_io_in_progress"), + DtraceDisplay::DsIoDone => write!(f, "ds_io_done"), + DtraceDisplay::DsIoSkipped => write!(f, "ds_io_skipped"), + DtraceDisplay::DsIoError => write!(f, "ds_io_error"), + } + } +} + +/// Translate DsState string into a three letter abbreviation +pub fn short_state(dss: &str) -> String { + match dss { + "Active" => "ACT".to_string(), + "WaitQuorum" => "WQ".to_string(), + "Reconcile" => "REC".to_string(), + "LiveRepairReady" => "LRR".to_string(), + "New" => "NEW".to_string(), + "Faulted" => "FLT".to_string(), + "Offline" => "OFL".to_string(), + "Replaced" => "RPL".to_string(), + "LiveRepair" => "LR".to_string(), + "Replacing" => "RPC".to_string(), + "Disabled" => "DIS".to_string(), + "Deactivated" => "DAV".to_string(), + "NegotiationFailed" => "NF".to_string(), + "Fault" => "FLT".to_string(), + x => x.to_string(), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use strum::IntoEnumIterator; + + #[test] + fn test_short_state_all_known_states() { + // Test all known downstairs states + assert_eq!(short_state("Active"), "ACT"); + assert_eq!(short_state("WaitQuorum"), "WQ"); + assert_eq!(short_state("Reconcile"), "REC"); + assert_eq!(short_state("LiveRepairReady"), "LRR"); + assert_eq!(short_state("New"), "NEW"); + assert_eq!(short_state("Faulted"), "FLT"); + assert_eq!(short_state("Offline"), "OFL"); + assert_eq!(short_state("Replaced"), "RPL"); + assert_eq!(short_state("LiveRepair"), "LR"); + assert_eq!(short_state("Replacing"), "RPC"); + assert_eq!(short_state("Disabled"), "DIS"); + assert_eq!(short_state("Deactivated"), "DAV"); + assert_eq!(short_state("NegotiationFailed"), "NF"); + assert_eq!(short_state("Fault"), "FLT"); + } + + #[test] + fn test_short_state_unknown_state() { + // Unknown states should pass through unchanged + assert_eq!(short_state("UnknownState"), "UnknownState"); + assert_eq!(short_state(""), ""); + assert_eq!(short_state("XYZ"), "XYZ"); + } + + #[test] + fn test_short_state_length() { + // All known states should produce 2-3 character abbreviations + let known_states = vec![ + "Active", + "WaitQuorum", + "Reconcile", + "LiveRepairReady", + "New", + "Faulted", + "Offline", + "Replaced", + "LiveRepair", + "Replacing", + "Disabled", + "Deactivated", + "NegotiationFailed", + "Fault", + ]; + + for state in known_states { + let short = short_state(state); + assert!( + short.len() <= 3, + "State {} abbreviation '{}' is too long", + state, + short + ); + } + } + + #[test] + fn test_dtrace_display_to_string() { + // Test that Display trait works for all variants + assert_eq!(DtraceDisplay::Pid.to_string(), "pid"); + assert_eq!(DtraceDisplay::Session.to_string(), "session"); + assert_eq!(DtraceDisplay::UpstairsId.to_string(), "upstairs_id"); + assert_eq!(DtraceDisplay::State.to_string(), "state"); + assert_eq!(DtraceDisplay::IoCount.to_string(), "io_count"); + assert_eq!(DtraceDisplay::IoSummary.to_string(), "io_summary"); + assert_eq!(DtraceDisplay::NextJobId.to_string(), "next_job_id"); + assert_eq!(DtraceDisplay::JobDelta.to_string(), "job_delta"); + assert_eq!( + DtraceDisplay::ExtentLimit.to_string(), + "extent_under_repair" + ); + } + + #[test] + fn test_dtrace_display_all_variants_have_display() { + // Ensure all enum variants can be displayed without panicking + for variant in DtraceDisplay::iter() { + let display = variant.to_string(); + // Should produce a non-empty string + assert!( + !display.is_empty(), + "Variant {:?} has empty display", + variant + ); + // Should be lowercase with underscores + assert!( + display.chars().all(|c| c.is_lowercase() || c == '_'), + "Variant {:?} display '{}' should be lowercase with underscores", + variant, + display + ); + } + } + + #[test] + fn test_dtrace_display_copy_clone() { + // Verify Copy and Clone work correctly + let display = DtraceDisplay::Pid; + let copied = display; + let cloned = display; // Copy trait, no need for .clone() + + assert_eq!(format!("{:?}", display), format!("{:?}", copied)); + assert_eq!(format!("{:?}", display), format!("{:?}", cloned)); + } +} diff --git a/cmon/Cargo.toml b/cmon/Cargo.toml index 8b30a5ad0..cc65cde54 100644 --- a/cmon/Cargo.toml +++ b/cmon/Cargo.toml @@ -7,11 +7,14 @@ edition = "2024" [dependencies] clap.workspace = true clearscreen.workspace = true +cmon-common = { path = "../cmon-common", features = ["clap"] } +crossterm.workspace = true crucible.workspace = true crucible-control-client.workspace = true crucible-protocol.workspace = true +ratatui.workspace = true +serde.workspace = true serde_json.workspace = true strum.workspace = true -strum_macros.workspace = true tokio.workspace = true crucible-workspace-hack.workspace = true diff --git a/cmon/src/main.rs b/cmon/src/main.rs index bcf3987da..cd946442c 100644 --- a/cmon/src/main.rs +++ b/cmon/src/main.rs @@ -1,15 +1,13 @@ // Copyright 2022 Oxide Computer Company -use clap::{Parser, Subcommand, ValueEnum}; +use clap::{Parser, Subcommand}; +use cmon_common::{DtraceDisplay, DtraceWrapper, short_state}; +use crucible::DtraceInfo; use crucible_control_client::Client; use crucible_protocol::ClientId; -use std::fmt; use std::io::{self, BufRead}; use strum::IntoEnumIterator; -use strum_macros::EnumIter; use tokio::time::{Duration, sleep}; -use crucible::DtraceInfo; - /// Connect to crucible control server #[derive(Parser, Debug)] #[clap(name = "cmon", term_width = 80)] @@ -27,52 +25,12 @@ struct Args { seconds: u64, } -// The possible fields we will display when receiving DTrace output. -#[derive(Debug, Copy, Clone, ValueEnum, EnumIter)] -enum DtraceDisplay { - State, - IoCount, - IoSummary, - UpCount, - DsCount, - Reconcile, - LiveRepair, - Connected, - Replaced, - ExtentLiveRepair, - ExtentLimit, - NextJobId, - JobDelta, - DsDelay, -} - -impl fmt::Display for DtraceDisplay { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - match self { - DtraceDisplay::State => write!(f, "state"), - DtraceDisplay::IoCount => write!(f, "io_count"), - DtraceDisplay::IoSummary => write!(f, "io_summary"), - DtraceDisplay::UpCount => write!(f, "up_count"), - DtraceDisplay::DsCount => write!(f, "ds_count"), - DtraceDisplay::Reconcile => write!(f, "reconcile"), - DtraceDisplay::LiveRepair => write!(f, "live_repair"), - DtraceDisplay::Connected => write!(f, "connected"), - DtraceDisplay::Replaced => write!(f, "replaced"), - DtraceDisplay::ExtentLiveRepair => write!(f, "extent_live_repair"), - DtraceDisplay::ExtentLimit => write!(f, "extent_under_repair"), - DtraceDisplay::NextJobId => write!(f, "next_job_id"), - DtraceDisplay::JobDelta => write!(f, "job_delta"), - DtraceDisplay::DsDelay => write!(f, "ds_delay"), - } - } -} - #[derive(Debug, Subcommand)] enum Action { /// Read from stdin Dtrace { /// Fields to display from dtrace received input - #[clap(short, long, default_value = "io-count")] + #[clap(short, long, value_delimiter = ',', default_values_t = vec![DtraceDisplay::Pid, DtraceDisplay::Session, DtraceDisplay::State, DtraceDisplay::NextJobId, DtraceDisplay::JobDelta, DtraceDisplay::ExtentLimit, DtraceDisplay::DsReconciled, DtraceDisplay::DsReconcileNeeded])] #[arg(value_enum)] output: Vec, }, @@ -84,28 +42,6 @@ enum Action { Repair, } -/// Translate what the default DsState string is (that we are getting from DTrace) -/// into a three letter string for printing. -fn short_state(dss: &str) -> String { - match dss { - "Active" => "ACT".to_string(), - "WaitQuorum" => "WQ".to_string(), - "Reconcile" => "REC".to_string(), - "LiveRepairReady" => "LRR".to_string(), - "New" => "NEW".to_string(), - "Faulted" => "FLT".to_string(), - "Offline" => "OFL".to_string(), - "Replaced" => "RPL".to_string(), - "LiveRepair" => "LR".to_string(), - "Replacing" => "RPC".to_string(), - "Disabled" => "DIS".to_string(), - "Deactivated" => "DAV".to_string(), - "NegotiationFailed" => "NF".to_string(), - "Fault" => "FLT".to_string(), - x => x.to_string(), - } -} - // Show the downstairs work queue async fn show_work_queue(args: Args) { let ca = Client::new(&args.control); @@ -192,6 +128,15 @@ async fn show_repair_stats(args: Args) { fn print_dtrace_header(dd: &[DtraceDisplay]) { for display_item in dd.iter() { match display_item { + DtraceDisplay::Pid => { + print!(" {:>5}", "PID"); + } + DtraceDisplay::Session => { + print!(" {:>8}", "SESSION"); + } + DtraceDisplay::UpstairsId => { + print!(" {:>8}", "UPSTAIRS"); + } DtraceDisplay::State => { print!(" {:>3} {:>3} {:>3}", "DS0", "DS1", "DS2",); } @@ -213,6 +158,12 @@ fn print_dtrace_header(dd: &[DtraceDisplay]) { DtraceDisplay::Reconcile => { print!(" {:>4} {:>4} {:>4}", "REC", "NREC", "AREC"); } + DtraceDisplay::DsReconciled => { + print!(" {:>4}", "ERR"); + } + DtraceDisplay::DsReconcileNeeded => { + print!(" {:>4}", "ERN"); + } DtraceDisplay::LiveRepair => { print!(" {:>4} {:>4} {:>4}", "LRC0", "LRC1", "LRC0"); print!(" {:>4} {:>4} {:>4}", "LRA0", "LRA1", "LRA2"); @@ -237,7 +188,25 @@ fn print_dtrace_header(dd: &[DtraceDisplay]) { print!(" {:>5}", "DELTA"); } DtraceDisplay::DsDelay => { - print!(" {:>5}", "DELAY"); + print!(" {:>5} {:>5} {:>5}", "DLY0", "DLY1", "DLY2"); + } + DtraceDisplay::WriteBytesOut => { + print!(" {:>10}", "WRBYTES"); + } + DtraceDisplay::RoLrSkipped => { + print!(" {:>4} {:>4} {:>4}", "RLS0", "RLS1", "RLS2"); + } + DtraceDisplay::DsIoInProgress => { + print!(" {:>5} {:>5} {:>5}", "IP0", "IP1", "IP2"); + } + DtraceDisplay::DsIoDone => { + print!(" {:>5} {:>5} {:>5}", "D0", "D1", "D2"); + } + DtraceDisplay::DsIoSkipped => { + print!(" {:>5} {:>5} {:>5}", "S0", "S1", "S2"); + } + DtraceDisplay::DsIoError => { + print!(" {:>4} {:>4} {:>4}", "E0", "E1", "E2"); } } } @@ -247,12 +216,26 @@ fn print_dtrace_header(dd: &[DtraceDisplay]) { // Print out the values in the dtrace output based on what the DtraceDisplay // enums are set in the given Vec. fn print_dtrace_row( + pid: u32, d_out: DtraceInfo, dd: &[DtraceDisplay], last_job_id: &mut u64, ) { for display_item in dd.iter() { match display_item { + DtraceDisplay::Pid => { + print!(" {:>5}", pid); + } + DtraceDisplay::Session => { + let session_short = + d_out.session_id.chars().take(8).collect::(); + print!(" {:>8}", session_short); + } + DtraceDisplay::UpstairsId => { + let upstairs_short = + d_out.upstairs_id.chars().take(8).collect::(); + print!(" {:>8}", upstairs_short); + } DtraceDisplay::State => { print!( " {:>3} {:>3} {:>3}", @@ -303,6 +286,12 @@ fn print_dtrace_row( d_out.ds_reconcile_aborted, ); } + DtraceDisplay::DsReconciled => { + print!(" {:>4}", d_out.ds_reconciled); + } + DtraceDisplay::DsReconcileNeeded => { + print!(" {:>4}", d_out.ds_reconcile_needed); + } DtraceDisplay::LiveRepair => { print!( " {:4} {:4} {:4}", @@ -354,12 +343,12 @@ fn print_dtrace_row( print!(" {:>7}", d_out.next_job_id); } DtraceDisplay::JobDelta => { - let delta = if *last_job_id == 0 { - d_out.next_job_id.0 + if *last_job_id == 0 { + print!(" {:>5}", "---"); } else { - d_out.next_job_id.0 - *last_job_id - }; - print!(" {:5}", delta); + let delta = d_out.next_job_id.0 - *last_job_id; + print!(" {:5}", delta); + } *last_job_id = d_out.next_job_id.0; } DtraceDisplay::DsDelay => { @@ -370,6 +359,49 @@ fn print_dtrace_row( d_out.ds_delay_us[2], ); } + DtraceDisplay::WriteBytesOut => { + print!(" {:10}", d_out.write_bytes_out); + } + DtraceDisplay::RoLrSkipped => { + print!( + " {:4} {:4} {:4}", + d_out.ds_ro_lr_skipped[0], + d_out.ds_ro_lr_skipped[1], + d_out.ds_ro_lr_skipped[2], + ); + } + DtraceDisplay::DsIoInProgress => { + print!( + " {:5} {:5} {:5}", + d_out.ds_io_count.in_progress[ClientId::new(0)], + d_out.ds_io_count.in_progress[ClientId::new(1)], + d_out.ds_io_count.in_progress[ClientId::new(2)], + ); + } + DtraceDisplay::DsIoDone => { + print!( + " {:5} {:5} {:5}", + d_out.ds_io_count.done[ClientId::new(0)], + d_out.ds_io_count.done[ClientId::new(1)], + d_out.ds_io_count.done[ClientId::new(2)], + ); + } + DtraceDisplay::DsIoSkipped => { + print!( + " {:5} {:5} {:5}", + d_out.ds_io_count.skipped[ClientId::new(0)], + d_out.ds_io_count.skipped[ClientId::new(1)], + d_out.ds_io_count.skipped[ClientId::new(2)], + ); + } + DtraceDisplay::DsIoError => { + print!( + " {:4} {:4} {:4}", + d_out.ds_io_count.error[ClientId::new(0)], + d_out.ds_io_count.error[ClientId::new(1)], + d_out.ds_io_count.error[ClientId::new(2)], + ); + } } } println!(); @@ -390,15 +422,20 @@ fn dtrace_loop(output: Vec) { print_dtrace_header(&output); } count = (count + 1) % 20; - let d_out: DtraceInfo = match serde_json::from_str(&dtrace_out) - { - Ok(a) => a, - Err(e) => { - println!("Err {:?}", e); - continue; - } - }; - print_dtrace_row(d_out, &output, &mut last_job_id); + let wrapper: DtraceWrapper = + match serde_json::from_str(&dtrace_out) { + Ok(a) => a, + Err(e) => { + println!("Err {:?}", e); + continue; + } + }; + print_dtrace_row( + wrapper.pid, + wrapper.status, + &output, + &mut last_job_id, + ); } Err(e) => { println!("Error: {:?}", e); diff --git a/ctop/Cargo.toml b/ctop/Cargo.toml new file mode 100644 index 000000000..8f9fafaf5 --- /dev/null +++ b/ctop/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "ctop" +version = "0.1.0" +license = "MPL-2.0" +edition = "2024" + +[dependencies] +clap.workspace = true +cmon-common = { path = "../cmon-common" } +crossterm.workspace = true +crucible.workspace = true +crucible-protocol.workspace = true +ratatui.workspace = true +serde.workspace = true +serde_json.workspace = true +strum.workspace = true +tokio.workspace = true +crucible-workspace-hack.workspace = true diff --git a/ctop/TODO.md b/ctop/TODO.md new file mode 100644 index 000000000..87d1feb7a --- /dev/null +++ b/ctop/TODO.md @@ -0,0 +1,35 @@ +# ctop TODO List + +## Feature Enhancements + +### Color enhancement +- [ ] Add color option to output, Make colors for ACT (good) different than + colors for other downstairs state. FLT red, LRR or LR a middle color. + NEW is grey maybe. Include tests for this +- [ ] Can we determine the current foreground/background of the terminal? How + will we handle light or dark mode. Maybe make a selection key to toggle + between the modes. + +### Session Selection and Multi-Session Display +- [ ] Add ability to select multiple sessions (e.g., with Space key) +- [ ] Display detailed graph for all selected sessions simultaneously +- [ ] Show visual indicator for which sessions are selected in the main list + +### Normalization Improvements +- [ ] Change normalization toggle. + - Make the same key rotate between the three options. + - Rotate through + 1: min/max for the current session (or, when finished, all selected sessions) + 2: 0 and the selected session(s) max + 3: 0 and max for all sessions (while just shown the current selections data). + +### Program improvements. +- [ ] Store 500 data points for each session. +- [ ] Add an option to only run for a specific number of seconds then exit. +- [ ] Add an option to keep the last state displayed, or, really, reproduce the + final screen but after we have exited the curses window. +- [ ] allow user to select which possible dtrace probes to display. + - If job delta is not selected, then detailed graphs are not available. +- [ ] Allow some downstairs individual stats to be combined into a "sum", like + connections for each downstairs summed into a single value. Not all + dtrace probes could do this, so give just options for ones we can. diff --git a/ctop/src/main.rs b/ctop/src/main.rs new file mode 100644 index 000000000..1878a73a8 --- /dev/null +++ b/ctop/src/main.rs @@ -0,0 +1,1571 @@ +// Copyright 2026 Oxide Computer Company + +//! Standalone ctop - curses-based top-like display of crucible dtrace data + +use clap::Parser; +use cmon_common::{DtraceDisplay, DtraceWrapper, short_state}; +use crossterm::{ + cursor, + event::{self, Event, KeyCode, KeyEvent, KeyModifiers}, + execute, + terminal::{ + Clear, ClearType, EnterAlternateScreen, LeaveAlternateScreen, + disable_raw_mode, enable_raw_mode, + }, +}; +use crucible::DtraceInfo; +use crucible_protocol::ClientId; +use ratatui::{ + Terminal, + backend::CrosstermBackend, + layout::{Constraint, Layout}, + style::Color, + widgets::canvas::{Canvas, Line, Points}, + widgets::{Block, Borders, Paragraph}, +}; +use std::collections::{HashMap, VecDeque}; +use std::io::{self, Write}; +use std::sync::Arc; +use std::time::{Duration, Instant}; +use tokio::io::{AsyncBufReadExt, BufReader}; +use tokio::process::Command; +use tokio::sync::{Notify, RwLock}; + +/// Default dtrace command - embedded one-liner that matches upstairs_raw.d +/// +/// This command: +/// - Uses -Z to continue even if no probes match +/// - Uses -q for quiet mode (no dtrace header) +/// - Sets strsize=2k for 2KB string buffers +/// - Probes crucible_upstairs*:::up-status +/// - Outputs JSON with pid and status +const DEFAULT_DTRACE_CMD: &str = r#"dtrace -Z -q -x strsize=2k -n 'crucible_upstairs*:::up-status { printf("{\"pid\":%d,\"status\":%s}\n", pid, json(copyinstr(arg1), "ok")); }'"#; + +/// Crucible top - monitor crucible upstairs via dtrace +#[derive(Parser, Debug)] +#[clap(name = "ctop", term_width = 80)] +#[clap(about = "Curses-based crucible monitor", long_about = None)] +struct Args { + /// Command to run to generate dtrace output + #[clap(long, default_value = DEFAULT_DTRACE_CMD)] + dtrace_cmd: String, +} + +const STALE_THRESHOLD_SECS: u64 = 5; +const REMOVE_THRESHOLD_SECS: u64 = 30; +const MAX_DELTA_HISTORY: usize = 100; + +/// Data for a single session +#[derive(Debug, Clone)] +struct SessionData { + pid: u32, + dtrace_info: DtraceInfo, + last_job_id: u64, + last_updated: Instant, + current_delta: Option, + delta_history: VecDeque, +} + +/// Shared state between stdin reader and display tasks +#[derive(Debug, Default)] +struct CtopState { + sessions: HashMap, + selected_index: usize, + scroll_offset: usize, + detail_mode: bool, + normalize_detail: bool, // Use global min/max for detail view scaling +} + +/// Default display fields (same as dtrace command defaults) +fn default_display_fields() -> Vec { + vec![ + DtraceDisplay::Pid, + DtraceDisplay::Session, + DtraceDisplay::State, + DtraceDisplay::NextJobId, + DtraceDisplay::JobDelta, + DtraceDisplay::ExtentLimit, + DtraceDisplay::DsReconciled, + DtraceDisplay::DsReconcileNeeded, + ] +} + +/// Format header line for the given display fields +fn format_header(dd: &[DtraceDisplay]) -> String { + let mut result = String::new(); + for display_item in dd.iter() { + match display_item { + DtraceDisplay::Pid => { + result.push_str(&format!(" {:>5}", "PID")); + } + DtraceDisplay::Session => { + result.push_str(&format!(" {:>8}", "SESSION")); + } + DtraceDisplay::UpstairsId => { + result.push_str(&format!(" {:>8}", "UPSTAIRS")); + } + DtraceDisplay::State => { + result.push_str(&format!( + " {:>3} {:>3} {:>3}", + "DS0", "DS1", "DS2" + )); + } + DtraceDisplay::UpCount => { + result.push_str(&format!(" {:>3}", "UPW")); + } + DtraceDisplay::DsCount => { + result.push_str(&format!(" {:>5}", "DSW")); + } + DtraceDisplay::IoCount | DtraceDisplay::IoSummary => { + result.push_str(&format!( + " {:>5} {:>5} {:>5}", + "IP0", "IP1", "IP2" + )); + result + .push_str(&format!(" {:>5} {:>5} {:>5}", "D0", "D1", "D2")); + result + .push_str(&format!(" {:>5} {:>5} {:>5}", "S0", "S1", "S2")); + + if matches!(display_item, DtraceDisplay::IoCount) { + result.push_str(&format!( + " {:>4} {:>4} {:>4}", + "E0", "E1", "E2" + )); + } + } + DtraceDisplay::Reconcile => { + result.push_str(&format!( + " {:>4} {:>4} {:>4}", + "REC", "NREC", "AREC" + )); + } + DtraceDisplay::DsReconciled => { + result.push_str(&format!(" {:>4}", "ERR")); + } + DtraceDisplay::DsReconcileNeeded => { + result.push_str(&format!(" {:>4}", "ERN")); + } + DtraceDisplay::LiveRepair => { + result.push_str(&format!( + " {:>4} {:>4} {:>4}", + "LRC0", "LRC1", "LRC0" + )); + result.push_str(&format!( + " {:>4} {:>4} {:>4}", + "LRA0", "LRA1", "LRA2" + )); + } + DtraceDisplay::Connected => { + result.push_str(&format!( + " {:>4} {:>4} {:>4}", + "CON0", "CON1", "CON2" + )); + } + DtraceDisplay::Replaced => { + result.push_str(&format!( + " {:>4} {:>4} {:>4}", + "RPL0", "RPL1", "RPL2" + )); + } + DtraceDisplay::ExtentLiveRepair => { + result.push_str(&format!( + " {:>4} {:>4} {:>4}", + "EXR0", "EXR1", "EXR2" + )); + result.push_str(&format!( + " {:>4} {:>4} {:>4}", + "EXC0", "EXC1", "EXC2" + )); + } + DtraceDisplay::ExtentLimit => { + result.push_str(&format!(" {:>4}", "EXTL")); + } + DtraceDisplay::NextJobId => { + result.push_str(&format!(" {:>10}", "NEXTJOB")); + } + DtraceDisplay::JobDelta => { + result.push_str(&format!(" {:>5}", "DELTA")); + } + DtraceDisplay::DsDelay => { + result.push_str(&format!( + " {:>5} {:>5} {:>5}", + "DLY0", "DLY1", "DLY2" + )); + } + DtraceDisplay::WriteBytesOut => { + result.push_str(&format!(" {:>10}", "WRBYTES")); + } + DtraceDisplay::RoLrSkipped => { + result.push_str(&format!( + " {:>4} {:>4} {:>4}", + "RLS0", "RLS1", "RLS2" + )); + } + DtraceDisplay::DsIoInProgress => { + result.push_str(&format!( + " {:>5} {:>5} {:>5}", + "IP0", "IP1", "IP2" + )); + } + DtraceDisplay::DsIoDone => { + result + .push_str(&format!(" {:>5} {:>5} {:>5}", "D0", "D1", "D2")); + } + DtraceDisplay::DsIoSkipped => { + result + .push_str(&format!(" {:>5} {:>5} {:>5}", "S0", "S1", "S2")); + } + DtraceDisplay::DsIoError => { + result + .push_str(&format!(" {:>4} {:>4} {:>4}", "E0", "E1", "E2")); + } + } + } + result +} + +/// Format a row for a single process +fn format_row( + pid: u32, + d_out: &DtraceInfo, + precomputed_delta: Option, + dd: &[DtraceDisplay], +) -> String { + let mut result = String::new(); + + for display_item in dd.iter() { + match display_item { + DtraceDisplay::Pid => { + // Note: stale indicator is now shown in the first column + result.push_str(&format!(" {:>5}", pid)); + } + DtraceDisplay::Session => { + let session_short = + d_out.session_id.chars().take(8).collect::(); + result.push_str(&format!(" {:>8}", session_short)); + } + DtraceDisplay::UpstairsId => { + let upstairs_short = + d_out.upstairs_id.chars().take(8).collect::(); + result.push_str(&format!(" {:>8}", upstairs_short)); + } + DtraceDisplay::State => { + result.push_str(&format!( + " {:>3} {:>3} {:>3}", + short_state(&d_out.ds_state[0]), + short_state(&d_out.ds_state[1]), + short_state(&d_out.ds_state[2]), + )); + } + DtraceDisplay::UpCount => { + result.push_str(&format!(" {:3}", d_out.up_count)); + } + DtraceDisplay::DsCount => { + result.push_str(&format!(" {:5}", d_out.ds_count)); + } + DtraceDisplay::IoCount | DtraceDisplay::IoSummary => { + result.push_str(&format!( + " {:5} {:5} {:5}", + d_out.ds_io_count.in_progress[ClientId::new(0)], + d_out.ds_io_count.in_progress[ClientId::new(1)], + d_out.ds_io_count.in_progress[ClientId::new(2)], + )); + result.push_str(&format!( + " {:5} {:5} {:5}", + d_out.ds_io_count.done[ClientId::new(0)], + d_out.ds_io_count.done[ClientId::new(1)], + d_out.ds_io_count.done[ClientId::new(2)], + )); + result.push_str(&format!( + " {:5} {:5} {:5}", + d_out.ds_io_count.skipped[ClientId::new(0)], + d_out.ds_io_count.skipped[ClientId::new(1)], + d_out.ds_io_count.skipped[ClientId::new(2)], + )); + if matches!(display_item, DtraceDisplay::IoCount) { + result.push_str(&format!( + " {:4} {:4} {:4}", + d_out.ds_io_count.error[ClientId::new(0)], + d_out.ds_io_count.error[ClientId::new(1)], + d_out.ds_io_count.error[ClientId::new(2)], + )); + } + } + DtraceDisplay::Reconcile => { + result.push_str(&format!( + " {:4} {:4} {:4}", + d_out.ds_reconciled, + d_out.ds_reconcile_needed, + d_out.ds_reconcile_aborted, + )); + } + DtraceDisplay::DsReconciled => { + result.push_str(&format!(" {:>4}", d_out.ds_reconciled)); + } + DtraceDisplay::DsReconcileNeeded => { + result.push_str(&format!(" {:>4}", d_out.ds_reconcile_needed)); + } + DtraceDisplay::LiveRepair => { + result.push_str(&format!( + " {:4} {:4} {:4}", + d_out.ds_live_repair_completed[0], + d_out.ds_live_repair_completed[1], + d_out.ds_live_repair_completed[2], + )); + result.push_str(&format!( + " {:4} {:4} {:4}", + d_out.ds_live_repair_aborted[0], + d_out.ds_live_repair_aborted[1], + d_out.ds_live_repair_aborted[2], + )); + } + DtraceDisplay::Connected => { + result.push_str(&format!( + " {:4} {:4} {:4}", + d_out.ds_connected[0], + d_out.ds_connected[1], + d_out.ds_connected[2], + )); + } + DtraceDisplay::Replaced => { + result.push_str(&format!( + " {:4} {:4} {:4}", + d_out.ds_replaced[0], + d_out.ds_replaced[1], + d_out.ds_replaced[2], + )); + } + DtraceDisplay::ExtentLiveRepair => { + result.push_str(&format!( + " {:4} {:4} {:4}", + d_out.ds_extents_repaired[0], + d_out.ds_extents_repaired[1], + d_out.ds_extents_repaired[2], + )); + result.push_str(&format!( + " {:4} {:4} {:4}", + d_out.ds_extents_confirmed[0], + d_out.ds_extents_confirmed[1], + d_out.ds_extents_confirmed[2], + )); + } + DtraceDisplay::ExtentLimit => { + result.push_str(&format!(" {:4}", d_out.ds_extent_limit)); + } + DtraceDisplay::NextJobId => { + result.push_str(&format!(" {:>10}", d_out.next_job_id)); + } + DtraceDisplay::JobDelta => { + if let Some(delta) = precomputed_delta { + result.push_str(&format!(" {:5}", delta)); + } else { + result.push_str(&format!(" {:>5}", "---")); + } + } + DtraceDisplay::DsDelay => { + result.push_str(&format!( + " {:5} {:5} {:5}", + d_out.ds_delay_us[0], + d_out.ds_delay_us[1], + d_out.ds_delay_us[2], + )); + } + DtraceDisplay::WriteBytesOut => { + result.push_str(&format!(" {:10}", d_out.write_bytes_out)); + } + DtraceDisplay::RoLrSkipped => { + result.push_str(&format!( + " {:4} {:4} {:4}", + d_out.ds_ro_lr_skipped[0], + d_out.ds_ro_lr_skipped[1], + d_out.ds_ro_lr_skipped[2], + )); + } + DtraceDisplay::DsIoInProgress => { + result.push_str(&format!( + " {:5} {:5} {:5}", + d_out.ds_io_count.in_progress[ClientId::new(0)], + d_out.ds_io_count.in_progress[ClientId::new(1)], + d_out.ds_io_count.in_progress[ClientId::new(2)], + )); + } + DtraceDisplay::DsIoDone => { + result.push_str(&format!( + " {:5} {:5} {:5}", + d_out.ds_io_count.done[ClientId::new(0)], + d_out.ds_io_count.done[ClientId::new(1)], + d_out.ds_io_count.done[ClientId::new(2)], + )); + } + DtraceDisplay::DsIoSkipped => { + result.push_str(&format!( + " {:5} {:5} {:5}", + d_out.ds_io_count.skipped[ClientId::new(0)], + d_out.ds_io_count.skipped[ClientId::new(1)], + d_out.ds_io_count.skipped[ClientId::new(2)], + )); + } + DtraceDisplay::DsIoError => { + result.push_str(&format!( + " {:4} {:4} {:4}", + d_out.ds_io_count.error[ClientId::new(0)], + d_out.ds_io_count.error[ClientId::new(1)], + d_out.ds_io_count.error[ClientId::new(2)], + )); + } + } + } + result +} + +/// Calculate the number of visible rows for session display +/// +/// Terminal layout (total FIXED_LINES = 7): +/// 1 header (timestamp) +/// 1 blank line +/// 1 column headers +/// N session lines (variable, returned by this function) +/// 1 scroll indicator (↑ more above) +/// 1 scroll indicator (↓ more below) +/// 1 blank line +/// 1 footer (help text) +/// +/// Returns at least 1 visible row to prevent crashes. +fn calculate_visible_rows(terminal_height: u16) -> usize { + const FIXED_LINES: u16 = 7; + if terminal_height > FIXED_LINES { + (terminal_height - FIXED_LINES) as usize + } else { + 1 // Minimum to avoid crashes + } +} + +/// Render a sparkline from delta history +/// Uses Unicode block characters to show trend: ▁▂▃▄▅▆▇█ +/// If global_max is provided, scales relative to that value for +/// cross-session comparison +/// The sparkline is right-aligned: newest value at rightmost column, +/// older values scroll left, padding with spaces on the left if needed +fn render_sparkline( + history: &VecDeque, + width: usize, + global_max: u64, +) -> String { + if history.is_empty() || width == 0 { + return " ".repeat(width); + } + + // Unicode block characters from lowest to highest + const BLOCKS: [char; 8] = ['▁', '▂', '▃', '▄', '▅', '▆', '▇', '█']; + + // Take last 'width' samples, reverse to show oldest->newest (left->right) + let samples: Vec = history + .iter() + .rev() + .take(width) + .copied() + .collect::>() + .into_iter() + .rev() + .collect(); + + if samples.is_empty() { + return " ".repeat(width); + } + + // Use global max for scaling (minimum 1 to avoid division by zero) + let max = global_max.max(1); + + // Map each value to a block character + let sparkline: String = samples + .iter() + .map(|&val| { + if val == 0 { + BLOCKS[0] + } else { + let normalized = (val as f64 / max as f64 * 7.0) as usize; + BLOCKS[normalized.min(7)] + } + }) + .collect(); + + // Right-align: pad left with spaces if we have fewer samples than width + if sparkline.chars().count() < width { + let padding = width - sparkline.chars().count(); + format!("{}{}", " ".repeat(padding), sparkline) + } else { + sparkline + } +} + +/// Subprocess reader task - spawns dtrace command and reads JSON output +async fn subprocess_reader_task( + dtrace_cmd: String, + state: Arc>, + notify: Arc, +) -> Result<(), Box> { + if dtrace_cmd.is_empty() { + return Err("Empty dtrace command".into()); + } + + // Execute the command through a shell to properly handle quoting + let mut child = Command::new("sh") + .arg("-c") + .arg(&dtrace_cmd) + .stdout(std::process::Stdio::piped()) + .stderr(std::process::Stdio::null()) + .spawn()?; + + let stdout = child + .stdout + .take() + .ok_or("Failed to capture subprocess stdout")?; + + let reader = BufReader::new(stdout); + let mut lines = reader.lines(); + + // Read lines from subprocess stdout + while let Some(line) = lines.next_line().await? { + // Parse JSON + let wrapper: DtraceWrapper = match serde_json::from_str(&line) { + Ok(w) => w, + Err(_) => continue, + }; + + // Update state + let mut state_guard = state.write().await; + + let session_data = state_guard + .sessions + .entry(wrapper.status.session_id.clone()) + .or_insert_with(|| SessionData { + pid: wrapper.pid, + dtrace_info: wrapper.status.clone(), + last_job_id: 0, + last_updated: Instant::now(), + current_delta: None, + delta_history: VecDeque::new(), + }); + + // Calculate delta (jobs per second) + let current_job_id = wrapper.status.next_job_id.0; + let delta = if session_data.last_job_id != 0 { + let d = current_job_id.saturating_sub(session_data.last_job_id); + + // Add to history ring buffer + session_data.delta_history.push_back(d); + if session_data.delta_history.len() > MAX_DELTA_HISTORY { + session_data.delta_history.pop_front(); + } + + Some(d) + } else { + None + }; + + // Store current delta and update state + session_data.current_delta = delta; + session_data.last_job_id = current_job_id; + session_data.dtrace_info = wrapper.status; + session_data.last_updated = Instant::now(); + + drop(state_guard); + + // Notify display task + notify.notify_one(); + } + + // Wait for child to exit + let _ = child.wait().await; + + Ok(()) +} + +/// Render full-screen detail view for a selected session +fn render_detail_view( + session_data: &SessionData, + _terminal_size: (u16, u16), + terminal: &mut Terminal>, + global_min: Option, + global_max: Option, + normalize: bool, +) -> io::Result<()> { + // Calculate statistics (oldest on left, newest on right) + let history: Vec = + session_data.delta_history.iter().copied().collect(); + let session_max = history.iter().copied().max().unwrap_or(1); + let session_min = history.iter().copied().min().unwrap_or(0); + let avg = if !history.is_empty() { + history.iter().sum::() / history.len() as u64 + } else { + 0 + }; + let current = session_data.current_delta.unwrap_or(0); + + // Choose min/max based on normalize mode + let (display_min, display_max) = if normalize { + ( + global_min.unwrap_or(session_min), + global_max.unwrap_or(session_max), + ) + } else { + (session_min, session_max) + }; + + // Render using ratatui (terminal is reused, ratatui handles diffing) + terminal.draw(|f| { + let area = f.area(); + + // Split area: 1 line at top for session data, rest for canvas + let chunks = Layout::default() + .constraints([Constraint::Length(1), Constraint::Min(0)]) + .split(area); + + // Format the session data row + let display_fields = default_display_fields(); + let row_data = format_row( + session_data.pid, + &session_data.dtrace_info, + session_data.current_delta, + &display_fields, + ); + + // Render session data at top + let data_paragraph = Paragraph::new(row_data); + f.render_widget(data_paragraph, chunks[0]); + + // Create title + let session_short: String = + session_data.dtrace_info.session_id.chars().take(8).collect(); + let mode_str = if normalize { " [NORMALIZED]" } else { "" }; + let title = format!( + " Delta History - PID {} - Session {}{} ", + session_data.pid, session_short, mode_str + ); + + // Create canvas widget in bottom area (1 line shorter) + let canvas = Canvas::default() + .block( + Block::default() + .borders(Borders::ALL) + .title(title) + .title_bottom(format!( + " Samples: {} | Min: {} | Max: {} | Avg: {} | Current: {} ", + history.len(), + session_min, + session_max, + avg, + current + )) + .title_bottom( + " ['d': Back | 'n': Toggle normalize | 'q': Quit] ", + ), + ) + .x_bounds([0.0, history.len().max(1) as f64]) + .y_bounds([display_min as f64, display_max as f64]) + .paint(|ctx| { + // Draw Y-axis labels (at left edge of graph) + let y_range = display_max as f64 - display_min as f64; + let y_positions = [ + display_max, + display_min + (y_range * 0.75) as u64, + display_min + (y_range * 0.5) as u64, + display_min + (y_range * 0.25) as u64, + display_min, + ]; + + for y_val in &y_positions { + ctx.print( + 0.0, + *y_val as f64, + ratatui::text::Span::styled( + format!("{}", y_val), + ratatui::style::Style::default() + .fg(Color::Gray), + ), + ); + } + + // Draw the line graph + if history.len() > 1 { + for i in 0..history.len() - 1 { + let x1 = i as f64; + let y1 = history[i] as f64; + let x2 = (i + 1) as f64; + let y2 = history[i + 1] as f64; + + ctx.draw(&Line { + x1, + y1, + x2, + y2, + color: Color::Cyan, + }); + } + } + + // Draw points for each sample + for (i, &value) in history.iter().enumerate() { + ctx.draw(&Points { + coords: &[(i as f64, value as f64)], + color: Color::Yellow, + }); + } + }); + + f.render_widget(canvas, chunks[1]); + })?; + + Ok(()) +} + +/// Display task - renders the screen and handles keyboard input +async fn display_task( + state: Arc>, + notify: Arc, +) -> Result<(), Box> { + // Set up terminal + enable_raw_mode()?; + let mut stdout = io::stdout(); + execute!(stdout, EnterAlternateScreen)?; + + // Set up panic handler to restore terminal + let original_hook = std::panic::take_hook(); + std::panic::set_hook(Box::new(move |panic_info| { + let _ = execute!(io::stdout(), LeaveAlternateScreen); + let _ = disable_raw_mode(); + original_hook(panic_info); + })); + + let display_fields = default_display_fields(); + + // Track detail mode and persistent terminal for detail view + let mut was_in_detail_mode = false; + let mut detail_terminal: Option>> = + None; + + loop { + // Wait for notification or timeout + tokio::select! { + _ = notify.notified() => {}, + _ = tokio::time::sleep(Duration::from_millis(100)) => {}, + } + + // Get current time + let now = Instant::now(); + let system_time = std::time::SystemTime::now(); + let duration = system_time + .duration_since(std::time::UNIX_EPOCH) + .unwrap_or_default(); + + // Get terminal size + let terminal_size = crossterm::terminal::size()?; + + // Clean up sessions that haven't been updated in REMOVE_THRESHOLD_SECS + { + let mut state_guard = state.write().await; + let sessions_before = state_guard.sessions.len(); + + state_guard.sessions.retain(|_, session_data| { + now.duration_since(session_data.last_updated) + <= Duration::from_secs(REMOVE_THRESHOLD_SECS) + }); + + let sessions_after = state_guard.sessions.len(); + + // Adjust selected_index if sessions were removed + if sessions_after < sessions_before && sessions_after > 0 { + state_guard.selected_index = + state_guard.selected_index.min(sessions_after - 1); + state_guard.scroll_offset = + state_guard.scroll_offset.min(sessions_after - 1); + } else if sessions_after == 0 { + state_guard.selected_index = 0; + state_guard.scroll_offset = 0; + } + + drop(state_guard); + } + + // Read state to check mode + let state_guard = state.read().await; + let in_detail_mode = state_guard.detail_mode; + let selected_index = state_guard.selected_index; + let normalize_detail = state_guard.normalize_detail; + + // If in detail mode, render detail view and skip table + if in_detail_mode { + // Create terminal on first entry to detail mode + if !was_in_detail_mode { + execute!(stdout, Clear(ClearType::All))?; + // Create a new stdout handle for the Terminal + let detail_stdout = io::stdout(); + let backend = CrosstermBackend::new(detail_stdout); + detail_terminal = Some(Terminal::new(backend)?); + } + + let mut sessions: Vec<&SessionData> = + state_guard.sessions.values().collect(); + sessions.sort_by_key(|s| (s.pid, &s.dtrace_info.session_id)); + + // Calculate global min/max across all sessions for normalization + let global_min = sessions + .iter() + .flat_map(|s| s.delta_history.iter()) + .copied() + .min(); + let global_max = sessions + .iter() + .flat_map(|s| s.delta_history.iter()) + .copied() + .max(); + + if let Some(selected_session) = sessions.get(selected_index) { + // Clone the session data so we can drop the lock + let session_clone = (*selected_session).clone(); + drop(state_guard); + + if let Some(terminal) = detail_terminal.as_mut() { + render_detail_view( + &session_clone, + terminal_size, + terminal, + global_min, + global_max, + normalize_detail, + )?; + } + } else { + drop(state_guard); + } + + was_in_detail_mode = true; + } else { + // Exiting detail mode - drop terminal and redraw table + if was_in_detail_mode { + detail_terminal = None; + execute!(stdout, Clear(ClearType::All))?; + } + was_in_detail_mode = false; + + // Move cursor to top-left + execute!(stdout, cursor::MoveTo(0, 0))?; + // Table mode - render normal view + drop(state_guard); + + // Display header (clear line first to remove artifacts) + execute!(stdout, cursor::MoveTo(0, 0))?; + write!(stdout, "ctop - Unix timestamp: {}", duration.as_secs())?; + execute!(stdout, Clear(ClearType::UntilNewLine))?; + write!(stdout, "\r\n")?; + execute!(stdout, Clear(ClearType::UntilNewLine))?; + write!(stdout, "\r\n")?; + + // Display column headers + write!(stdout, "{}", format_header(&display_fields))?; + execute!(stdout, Clear(ClearType::UntilNewLine))?; + write!(stdout, "\r\n")?; + + let (terminal_width, terminal_height) = terminal_size; + + // Read state and display sessions sorted by PID (then session_id) + let state_guard = state.read().await; + let mut sessions: Vec<&SessionData> = + state_guard.sessions.values().collect(); + sessions.sort_by_key(|s| (s.pid, &s.dtrace_info.session_id)); + + // Calculate global max across all sessions for consistent sparkline + // scaling + let global_max = sessions + .iter() + .flat_map(|s| s.delta_history.iter()) + .copied() + .max() + .unwrap_or(1); + + let selected_index = state_guard.selected_index; + let scroll_offset = state_guard.scroll_offset; + + // Calculate visible window + let total_sessions = sessions.len(); + let has_more_above = scroll_offset > 0; + let visible_rows = calculate_visible_rows(terminal_height); + + let scroll_end = (scroll_offset + visible_rows).min(total_sessions); + let has_more_below = scroll_end < total_sessions; + + // Display scroll indicator if there are sessions above + if has_more_above { + write!(stdout, " ↑ More above")?; + execute!(stdout, Clear(ClearType::UntilNewLine))?; + write!(stdout, "\r\n")?; + } + + // Display visible sessions + for (idx, session_data) in sessions + .iter() + .enumerate() + .skip(scroll_offset) + .take(visible_rows) + { + let is_stale = now.duration_since(session_data.last_updated) + > Duration::from_secs(STALE_THRESHOLD_SECS); + + // Add indicator: > for selected, * for stale, space otherwise + // Selection indicator (>) takes priority over stale indicator (*) + let indicator = if idx == selected_index { + ">" + } else if is_stale { + "*" + } else { + " " + }; + write!(stdout, "{}", indicator)?; + + let row = format_row( + session_data.pid, + &session_data.dtrace_info, + session_data.current_delta, + &display_fields, + ); + write!(stdout, "{}", row)?; + + // Render sparkline right-aligned to fill remaining space to terminal edge + // Account for the indicator character (1 char) + let row_len = row.chars().count() + 1; + if terminal_width > row_len as u16 { + let sparkline_width = terminal_width as usize - row_len; + let sparkline = render_sparkline( + &session_data.delta_history, + sparkline_width, + global_max, + ); + write!(stdout, "{}", sparkline)?; + } + + execute!(stdout, Clear(ClearType::UntilNewLine))?; + write!(stdout, "\r\n")?; + } + + // Display scroll indicator if there are sessions below + if has_more_below { + let num_more = total_sessions - scroll_end; + write!(stdout, " ↓ More below ({} more)", num_more)?; + execute!(stdout, Clear(ClearType::UntilNewLine))?; + write!(stdout, "\r\n")?; + } + + drop(state_guard); + + // Display footer + execute!(stdout, Clear(ClearType::UntilNewLine))?; + write!(stdout, "\r\n")?; + write!( + stdout, + "[↑↓/PgUp/PgDn: Navigate | 'd': Details | 'q': Quit] > = selected, * = stale ({}s)", + STALE_THRESHOLD_SECS + )?; + execute!(stdout, Clear(ClearType::UntilNewLine))?; + write!(stdout, "\r\n")?; + + // Clear from cursor to end of screen (removes any leftover lines) + execute!(stdout, Clear(ClearType::FromCursorDown))?; + + stdout.flush()?; + } // End of table mode rendering + + // Check for keyboard input (non-blocking) + if event::poll(Duration::from_millis(0))? + && let Event::Key(key_event) = event::read()? + { + let mut state_guard = state.write().await; + let num_sessions = state_guard.sessions.len(); + + match key_event { + KeyEvent { + code: KeyCode::Char('q'), + modifiers: KeyModifiers::NONE, + .. + } => break, + KeyEvent { + code: KeyCode::Char('c'), + modifiers: KeyModifiers::CONTROL, + .. + } => break, + KeyEvent { + code: KeyCode::Char('d'), + modifiers: KeyModifiers::NONE, + .. + } => { + // Toggle detail mode + state_guard.detail_mode = !state_guard.detail_mode; + } + KeyEvent { + code: KeyCode::Char('n'), + modifiers: KeyModifiers::NONE, + .. + } => { + // Toggle normalize mode (only affects detail view) + state_guard.normalize_detail = + !state_guard.normalize_detail; + } + KeyEvent { + code: KeyCode::Up, + modifiers: KeyModifiers::NONE, + .. + } => { + // Move selection up (only in table mode) + if !state_guard.detail_mode + && num_sessions > 0 + && state_guard.selected_index > 0 + { + state_guard.selected_index -= 1; + + // Scroll up if selection moves above visible window + if state_guard.selected_index + < state_guard.scroll_offset + { + state_guard.scroll_offset = + state_guard.selected_index; + } + } + } + KeyEvent { + code: KeyCode::Down, + modifiers: KeyModifiers::NONE, + .. + } => { + // Move selection down (only in table mode) + if !state_guard.detail_mode + && num_sessions > 0 + && state_guard.selected_index < num_sessions - 1 + { + state_guard.selected_index += 1; + + // Calculate visible rows to determine scroll behavior + let visible_rows = + calculate_visible_rows(terminal_size.1); + + // Scroll down if selection moves below visible window + let scroll_end = + state_guard.scroll_offset + visible_rows; + if state_guard.selected_index >= scroll_end { + state_guard.scroll_offset += 1; + } + } + } + KeyEvent { + code: KeyCode::PageUp, + modifiers: KeyModifiers::NONE, + .. + } => { + // Page up (only in table mode) + if !state_guard.detail_mode && num_sessions > 0 { + // Calculate visible rows + let visible_rows = + calculate_visible_rows(terminal_size.1); + + // Move up by visible_rows + state_guard.selected_index = state_guard + .selected_index + .saturating_sub(visible_rows); + state_guard.scroll_offset = state_guard + .scroll_offset + .saturating_sub(visible_rows); + } + } + KeyEvent { + code: KeyCode::PageDown, + modifiers: KeyModifiers::NONE, + .. + } => { + // Page down (only in table mode) + if !state_guard.detail_mode && num_sessions > 0 { + // Calculate visible rows + let visible_rows = + calculate_visible_rows(terminal_size.1); + + // Move down by visible_rows + let new_selected = + state_guard.selected_index + visible_rows; + state_guard.selected_index = + new_selected.min(num_sessions - 1); + + let new_scroll = + state_guard.scroll_offset + visible_rows; + state_guard.scroll_offset = new_scroll + .min(num_sessions.saturating_sub(visible_rows)); + } + } + KeyEvent { + code: KeyCode::Esc, + modifiers: KeyModifiers::NONE, + .. + } => { + // Exit detail mode + state_guard.detail_mode = false; + } + _ => {} + } + drop(state_guard); + } + } + + // Clean up terminal + execute!(stdout, LeaveAlternateScreen)?; + disable_raw_mode()?; + + Ok(()) +} + +/// Main entry point for ctop +pub async fn ctop_loop( + dtrace_cmd: String, +) -> Result<(), Box> { + let state = Arc::new(RwLock::new(CtopState::default())); + let notify = Arc::new(Notify::new()); + + let state_reader = Arc::clone(&state); + let notify_reader = Arc::clone(¬ify); + + // Spawn subprocess reader task + let reader_handle = tokio::spawn(async move { + if let Err(e) = + subprocess_reader_task(dtrace_cmd, state_reader, notify_reader) + .await + { + eprintln!("Subprocess reader error: {}", e); + } + }); + + // Run display task (blocks until user quits) + let display_result = display_task(state, notify).await; + + // Wait for reader task to finish (it should exit quickly) + let _ = + tokio::time::timeout(Duration::from_millis(100), reader_handle).await; + + display_result +} + +/// Main entry point +#[tokio::main] +async fn main() { + let args = Args::parse(); + + if let Err(e) = ctop_loop(args.dtrace_cmd).await { + eprintln!("Error running ctop: {}", e); + std::process::exit(1); + } +} + +#[cfg(test)] +mod tests { + //! Unit tests for ctop + //! + //! # Test Coverage Overview + //! + //! These unit tests validate the pure functions and data structures that + //! power ctop's display logic, focusing on areas that don't require async + //! runtime or terminal mocking. + //! + //! ## Current Coverage + //! + //! ### Display Formatting (format_header, format_row) + //! - Column header generation for all DtraceDisplay field types + //! - Multi-column fields (State shows DS0/DS1/DS2) + //! - IO summary fields with multiple columns per downstairs + //! - Empty field handling + //! + //! ### Sparkline Rendering (render_sparkline) + //! - Empty history handling + //! - Zero-width rendering + //! - Single and multiple value rendering + //! - Width limiting (showing only recent samples) + //! - Normalization using global max value + //! - Unicode block character validation (▁▂▃▄▅▆▇█) + //! - Ascending/descending trend visualization + //! + //! ### State Management + //! - Default CtopState initialization + //! - Delta history ring buffer behavior (MAX_DELTA_HISTORY) + //! - Constants validation (STALE_THRESHOLD_SECS, MAX_DELTA_HISTORY) + //! + //! ## What's NOT Tested (See integration_test.rs for full list) + //! + //! - Async tasks (subprocess_reader_task, display_task) + //! - Terminal rendering and keyboard input + //! - Session lifecycle and state updates + //! - DTrace subprocess integration + //! - Multi-session coordination + //! + //! ## Testing Strategy + //! + //! These tests focus on **testable units** - pure functions with no I/O. + //! For components requiring mocking (terminal, subprocess, async runtime), + //! see the testing improvement proposals in integration_test.rs. + //! + //! ## Running Tests + //! + //! ```bash + //! # Run all ctop unit tests + //! cargo test -p ctop --lib + //! + //! # Run specific test + //! cargo test -p ctop --lib test_render_sparkline_normalization + //! ``` + + use super::*; + + // ============================================================================ + // Display Field Configuration Tests + // ============================================================================ + // + // These tests verify the default display configuration that users see when + // they first run ctop. The fields should provide a good overview of upstairs + // state without overwhelming the display. + + #[test] + fn test_default_display_fields() { + let fields = default_display_fields(); + + // Verify we have the expected default fields + assert_eq!(fields.len(), 8); + assert_eq!(fields[0], DtraceDisplay::Pid); + assert_eq!(fields[1], DtraceDisplay::Session); + assert_eq!(fields[2], DtraceDisplay::State); + assert_eq!(fields[3], DtraceDisplay::NextJobId); + assert_eq!(fields[4], DtraceDisplay::JobDelta); + assert_eq!(fields[5], DtraceDisplay::ExtentLimit); + assert_eq!(fields[6], DtraceDisplay::DsReconciled); + assert_eq!(fields[7], DtraceDisplay::DsReconcileNeeded); + } + + // ============================================================================ + // Header Formatting Tests + // ============================================================================ + // + // These tests verify that format_header() generates correct column headers + // for various field types. Some fields (like State, IoSummary) expand into + // multiple columns representing the three downstairs replicas. + + #[test] + fn test_format_header_basic_fields() { + let fields = vec![DtraceDisplay::Pid, DtraceDisplay::Session]; + let header = format_header(&fields); + + // Check that header contains expected column names + assert!(header.contains("PID")); + assert!(header.contains("SESSION")); + } + + #[test] + fn test_format_header_state_field() { + let fields = vec![DtraceDisplay::State]; + let header = format_header(&fields); + + // State field should show three downstairs columns + assert!(header.contains("DS0")); + assert!(header.contains("DS1")); + assert!(header.contains("DS2")); + } + + #[test] + fn test_format_header_io_fields() { + let fields = vec![DtraceDisplay::IoSummary]; + let header = format_header(&fields); + + // Should show in_progress, done, and skipped for each DS + assert!(header.contains("IP0")); + assert!(header.contains("IP1")); + assert!(header.contains("IP2")); + assert!(header.contains("D0")); + assert!(header.contains("D1")); + assert!(header.contains("D2")); + assert!(header.contains("S0")); + assert!(header.contains("S1")); + assert!(header.contains("S2")); + } + + #[test] + fn test_format_header_empty_fields() { + let fields = vec![]; + let header = format_header(&fields); + + // Empty fields should produce empty header + assert_eq!(header, ""); + } + + // ============================================================================ + // Sparkline Rendering Tests + // ============================================================================ + // + // Sparklines provide a compact visualization of job delta trends over time. + // They use Unicode block characters (▁▂▃▄▅▆▇█) to represent values. + // + // Key behaviors tested: + // - Empty history and edge cases (zero width, single values) + // - Width limiting (showing only the most recent N samples) + // - Normalization across sessions (global_max parameter) + // - Unicode character validity + // + // TODO: Consider property-based testing (proptest) to generate random + // histories and verify properties like: + // - Sparkline length <= requested width + // - All characters are valid block characters + // - Normalized values respect global_max + + #[test] + fn test_render_sparkline_empty() { + let history = VecDeque::new(); + let sparkline = render_sparkline(&history, 10, 100); + + // Empty history should return spaces to maintain right alignment + assert_eq!(sparkline, " "); // 10 spaces + } + + #[test] + fn test_render_sparkline_zero_width() { + let mut history = VecDeque::new(); + history.push_back(10); + history.push_back(20); + + let sparkline = render_sparkline(&history, 0, 100); + assert_eq!(sparkline, ""); // Empty string for 0 width + } + + #[test] + fn test_render_sparkline_single_value() { + let mut history = VecDeque::new(); + history.push_back(50); + + let sparkline = render_sparkline(&history, 10, 100); + + // Should have 10 characters total (9 spaces + 1 block, right-aligned) + assert_eq!(sparkline.chars().count(), 10); + // Last character should be the data value + assert_ne!(sparkline.chars().last().unwrap(), ' '); + } + + #[test] + fn test_render_sparkline_ascending_values() { + let mut history = VecDeque::new(); + for i in 0..10 { + history.push_back(i * 10); + } + + let sparkline = render_sparkline(&history, 10, 100); + + // Should have 10 characters (one per value) + assert_eq!(sparkline.chars().count(), 10); + + // First character should be lower than last (oldest on left, newest on right) + let chars: Vec = sparkline.chars().collect(); + assert!(chars[0] < chars[9]); + } + + #[test] + fn test_render_sparkline_width_limit() { + let mut history = VecDeque::new(); + for i in 0..100 { + history.push_back(i); + } + + // Request only last 5 samples + let sparkline = render_sparkline(&history, 5, 100); + + // Should only show 5 characters + assert_eq!(sparkline.chars().count(), 5); + } + + #[test] + fn test_render_sparkline_max_value() { + let mut history = VecDeque::new(); + history.push_back(0); + history.push_back(100); + + // Use width=2 to match data size (no padding) + let sparkline = render_sparkline(&history, 2, 100); + + // Should use valid unicode block characters (no spaces since width matches data) + for c in sparkline.chars() { + assert!( + ['▁', '▂', '▃', '▄', '▅', '▆', '▇', '█'].contains(&c), + "Invalid sparkline character: {}", + c + ); + } + + // Verify correct blocks: 0 maps to lowest, 100 maps to highest + let chars: Vec = sparkline.chars().collect(); + assert_eq!(chars[0], '▁', "Value 0 should map to lowest block"); + assert_eq!( + chars[1], '█', + "Value 100 (max) should map to highest block" + ); + } + + #[test] + fn test_render_sparkline_normalization() { + // This test verifies a critical feature: sparklines are normalized using + // a global_max value computed across ALL sessions. This allows users to + // visually compare activity levels between different upstairs instances. + // + // Without global normalization, each session would auto-scale to its own + // range, making cross-session comparison meaningless. + + let mut history = VecDeque::new(); + history.push_back(50); + history.push_back(100); + + // Test with global max = 200 (should scale differently than 100) + // Use width=2 to avoid padding and test actual data + let sparkline1 = render_sparkline(&history, 2, 200); + let sparkline2 = render_sparkline(&history, 2, 100); + + // With higher global max, the values should appear relatively lower + let chars1: Vec = sparkline1.chars().collect(); + let chars2: Vec = sparkline2.chars().collect(); + + // Sparkline1 (max=200): value 100 is only halfway, so should use lower blocks + // Sparkline2 (max=100): value 100 is the maximum, so should use highest block + // + // For value 100: + // normalized1 = 100/200 * 7 = 3.5 → index 3 = '▄' + // normalized2 = 100/100 * 7 = 7.0 → index 7 = '█' + // + // Therefore chars1[1] should be '▄' and chars2[1] should be '█' + assert!( + chars1[1] < chars2[1], + "Expected normalization to affect block height: \ + sparkline1[1]='{}' should be < sparkline2[1]='{}'", + chars1[1], + chars2[1] + ); + + // Verify the actual characters are as expected + assert_eq!(chars1[1], '▄', "Value 100 with max=200 should be ▄"); + assert_eq!(chars2[1], '█', "Value 100 with max=100 should be █"); + } + + // ============================================================================ + // State Management Tests + // ============================================================================ + // + // These tests verify the data structures that track session state: + // - CtopState: Overall application state (sessions map, selection, mode) + // - SessionData: Per-session data including delta history ring buffer + // - Constants: STALE_THRESHOLD_SECS, MAX_DELTA_HISTORY + // + // Note: These tests only validate initialization and constants. Full + // state lifecycle testing (session updates, transitions) requires async + // mocking infrastructure. + + #[test] + fn test_ctop_state_default() { + let state = CtopState::default(); + + assert_eq!(state.sessions.len(), 0); + assert_eq!(state.selected_index, 0); + assert!(!state.detail_mode); + assert!(!state.normalize_detail); + } + + #[test] + fn test_session_data_delta_history_max_size() { + // Delta history uses a ring buffer (VecDeque) to maintain a sliding window + // of recent job delta values for sparkline rendering. This test verifies + // the ring buffer behavior: old values are evicted when capacity is reached. + // + // In production, subprocess_reader_task maintains this ring buffer by: + // 1. Computing delta = new_job_id - old_job_id + // 2. Pushing delta to back of deque + // 3. Popping from front if len > MAX_DELTA_HISTORY + + let mut delta_history = VecDeque::new(); + + // Simulate adding more than MAX_DELTA_HISTORY items + for i in 0..(MAX_DELTA_HISTORY + 10) { + delta_history.push_back(i as u64); + if delta_history.len() > MAX_DELTA_HISTORY { + delta_history.pop_front(); + } + } + + // Should never exceed MAX_DELTA_HISTORY + assert_eq!(delta_history.len(), MAX_DELTA_HISTORY); + + // Should contain the most recent items (oldest 10 were evicted) + assert_eq!(*delta_history.front().unwrap(), 10); // First item should be item 10 + assert_eq!( + *delta_history.back().unwrap(), + (MAX_DELTA_HISTORY + 9) as u64 + ); // Last item is most recent + } + + // Note: Constant validation tests removed - clippy warns that assertions + // on constants are optimized out. Constants are validated at compile time + // by their usage in the code. + + // ============================================================================ + // Future Testing Opportunities + // ============================================================================ + // + // The following areas could benefit from additional testing infrastructure: + // + // ## 1. Async Task Testing (mockall crate) + // + // Mock the subprocess Command to test subprocess_reader_task: + // - Emit controlled JSON lines + // - Verify state updates (sessions map, delta calculations) + // - Test error handling (invalid JSON, subprocess crashes) + // - Test session expiration and cleanup + // + // ## 2. Terminal UI Testing (ratatui::TestBackend) + // + // Capture terminal output to verify: + // - Header and row formatting in actual terminal context + // - Stale session indicators (*) appear correctly + // - Selection indicators (>) highlight correct row + // - Sparklines render in available terminal width + // - Detail mode layout and graphing + // + // ## 3. Keyboard Input Testing + // + // Mock crossterm events to test: + // - Up/Down arrow navigation (selected_index changes) + // - 'd' toggles detail_mode + // - 'n' toggles normalize_detail + // - 'q' and Ctrl-C exit cleanly + // - Esc exits detail mode + // + // ## 4. Property-Based Testing (proptest) + // + // Generate random inputs to verify invariants: + // - Sparkline width never exceeds requested width + // - format_header produces valid UTF-8 + // - Delta history never exceeds MAX_DELTA_HISTORY + // - selected_index never exceeds sessions.len() + // + // ## 5. Snapshot Testing (insta) + // + // Capture and compare terminal output snapshots: + // - Table view with various session counts + // - Detail view with different data patterns + // - Regression testing for layout changes + // + // ## 6. Performance Testing (criterion) + // + // Benchmark critical paths: + // - render_sparkline with full history + // - format_row with many display fields + // - Session sorting with 100+ sessions + // + // ## 7. End-to-End Testing + // + // Run ctop against real DTrace output: + // - Capture actual crucible DTrace JSON + // - Replay captured output to ctop + // - Verify no parsing errors + // - Compare against expected session data +} diff --git a/ctop/tests/integration_test.rs b/ctop/tests/integration_test.rs new file mode 100644 index 000000000..e9fe15fcb --- /dev/null +++ b/ctop/tests/integration_test.rs @@ -0,0 +1,187 @@ +// Copyright 2026 Oxide Computer Company + +//! Integration tests for ctop +//! +//! # Test Coverage Overview +//! +//! These integration tests focus on validating the data flow from DTrace +//! output through ctop's JSON parsing pipeline. They ensure that ctop can +//! correctly handle the JSON format produced by the DTrace scripts used to +//! monitor Crucible upstairs processes. +//! +//! ## Current Coverage +//! +//! - **JSON Parsing**: Valid, invalid, and incomplete DTrace JSON structures +//! - **Data Format Validation**: Ensures expected fields are present and +//! correctly typed +//! +//! ## What's NOT Tested (Yet) +//! +//! The following areas require more sophisticated testing infrastructure: +//! +//! ### Async Components +//! - Subprocess spawning and management (tokio Command) +//! - State updates from subprocess_reader_task +//! - Async communication between reader and display tasks +//! - Tokio Notify coordination +//! +//! ### Terminal UI +//! - Keyboard input handling (arrow keys, 'd', 'q', etc.) +//! - Screen rendering and layout +//! - Detail mode transitions +//! - Sparkline rendering in actual terminals +//! - Alternate screen buffer management +//! +//! ### State Management +//! - Session creation and updates +//! - Session selection (up/down arrows) +//! - Detail mode toggling +//! - Stale session detection (5s threshold) +//! - Delta history ring buffer behavior +//! +//! ### Edge Cases +//! - Multiple sessions with same PID +//! - Rapidly changing session data +//! - Very long session IDs or upstairs IDs +//! - Terminal resize during operation +//! - Empty or missing DTrace output +//! +//! ## Proposed Testing Improvements +//! +//! Consider these testing strategies for more comprehensive coverage: +//! +//! ### 1. Subprocess Mocking +//! Use a mock subprocess that emits controlled DTrace JSON output to test +//! the full data pipeline without requiring actual DTrace. +//! +//! ### 2. Terminal Mocking +//! Libraries like `ratatui::TestBackend` can capture terminal output for +//! validation without requiring an actual terminal. +//! +//! ### 3. Property-Based Testing (proptest) +//! Generate random but valid DTrace JSON structures to test parsing +//! robustness and find edge cases. +//! +//! ### 4. Snapshot Testing (insta) +//! Capture and verify terminal output snapshots for regression testing +//! of UI layout and formatting. +//! +//! ### 5. End-to-End Tests +//! Run ctop against real DTrace output (captured from actual crucible +//! processes) to validate production behavior. +//! +//! ## Running Tests +//! +//! ```bash +//! # Run all ctop tests (unit + integration) +//! cargo test -p ctop +//! +//! # Run only integration tests +//! cargo test -p ctop --test integration_test +//! +//! # Run specific integration test +//! cargo test -p ctop --test integration_test test_parse_dtrace_json_format +//! ``` + +use cmon_common::DtraceWrapper; + +/// Test that we can parse valid DTrace JSON output +#[test] +fn test_parse_dtrace_json_format() { + // This is the JSON format that dtrace scripts output + // and that ctop needs to parse + let sample_json = r#"{ + "pid": 12345, + "status": { + "upstairs_id": "test-uuid-123", + "session_id": "session-456", + "up_count": 10, + "up_counters": { + "apply": 100, + "action_downstairs": 50, + "action_guest": 30, + "action_deferred_block": 0, + "action_deferred_message": 0, + "action_flush_check": 10, + "action_stat_check": 5, + "action_control_check": 3, + "action_noop": 2 + }, + "next_job_id": 1000, + "ds_count": 15, + "write_bytes_out": 10240, + "ds_state": ["Active", "Active", "Active"], + "ds_io_count": { + "in_progress": [5, 5, 5], + "done": [100, 100, 100], + "skipped": [0, 0, 0], + "error": [0, 0, 0] + }, + "ds_reconciled": 0, + "ds_reconcile_needed": 0, + "ds_reconcile_aborted": 0, + "ds_live_repair_completed": [0, 0, 0], + "ds_live_repair_aborted": [0, 0, 0], + "ds_connected": [1, 1, 1], + "ds_replaced": [0, 0, 0], + "ds_extents_repaired": [0, 0, 0], + "ds_extents_confirmed": [100, 100, 100], + "ds_extent_limit": 0, + "ds_delay_us": [0, 0, 0], + "ds_ro_lr_skipped": [0, 0, 0] + } + }"#; + + // Parse into the actual DtraceWrapper type used in production + // This ensures the test catches real compatibility issues + let result: Result = serde_json::from_str(sample_json); + assert!( + result.is_ok(), + "Sample dtrace JSON should parse into DtraceWrapper: {:?}", + result.err() + ); + + let parsed = result.unwrap(); + assert_eq!(parsed.pid, 12345); + assert_eq!(parsed.status.session_id, "session-456"); + assert_eq!(parsed.status.ds_state[0], "Active"); +} + +/// Test handling of invalid JSON +#[test] +fn test_parse_invalid_dtrace_json() { + let invalid_json = r#"{ "pid": 12345, "status": invalid }"#; + + let result: Result = + serde_json::from_str(invalid_json); + assert!(result.is_err(), "Invalid JSON should fail to parse"); +} + +/// Test handling of partial/incomplete JSON +#[test] +fn test_parse_incomplete_dtrace_json() { + // Missing required fields + let incomplete_json = r#"{ "pid": 12345 }"#; + + let result: Result = + serde_json::from_str(incomplete_json); + assert!(result.is_ok(), "Partial JSON should parse as JSON"); + + let parsed = result.unwrap(); + assert_eq!(parsed["pid"], 12345); + assert!(parsed["status"].is_null()); +} + +// Note: Additional integration tests would ideally cover: +// - Mock dtrace subprocess and verify parsing pipeline +// - Test keyboard input handling (requires terminal mock) +// - Test state transitions (detail mode, session selection) +// - Test multi-session handling +// - Test stale session detection +// - Performance tests with many sessions +// +// These are challenging without a full terminal mocking framework +// and subprocess mocking capabilities. Consider using: +// - mockall crate for mocking +// - proptest for property-based testing +// - criterion for benchmarking diff --git a/tools/dtrace/upstairs_raw.d b/tools/dtrace/upstairs_raw.d index 3552ecc1a..d12f75efa 100755 --- a/tools/dtrace/upstairs_raw.d +++ b/tools/dtrace/upstairs_raw.d @@ -7,6 +7,5 @@ #pragma D option strsize=2k crucible_upstairs*:::up-status { - trace(json(copyinstr(arg1), "ok")); - printf("\n"); + printf("{\"pid\":%d,\"status\":%s}\n", pid, json(copyinstr(arg1), "ok")); } diff --git a/workspace-hack/Cargo.toml b/workspace-hack/Cargo.toml index 23bfc8232..53fe52457 100644 --- a/workspace-hack/Cargo.toml +++ b/workspace-hack/Cargo.toml @@ -27,6 +27,7 @@ clap = { version = "4", features = ["cargo", "derive", "env", "wrap_help"] } clap_builder = { version = "4", default-features = false, features = ["cargo", "color", "env", "std", "suggestions", "usage", "wrap_help"] } crossbeam-epoch = { version = "0.9" } crossbeam-utils = { version = "0.8" } +crossterm = { version = "0.28", features = ["serde"] } crypto-common = { version = "0.1", default-features = false, features = ["getrandom", "std"] } digest = { version = "0.10", features = ["mac", "std"] } either = { version = "1", features = ["use_std"] }