From a590fbc51066f4e5d8ca0773baea9694d85ada54 Mon Sep 17 00:00:00 2001 From: Alan Hanson Date: Thu, 23 Oct 2025 19:56:57 -0700 Subject: [PATCH 01/36] cmon polish Add a PID to upstairs_raw Make default cmon more like get_up_state.sh script, which is what I wanted this to be to begin with. Add more of the dtrace json blob to cmon --- Cargo.lock | 1 + cmon/Cargo.toml | 1 + cmon/src/main.rs | 165 ++++++++++++++++++++++++++++++++---- tools/dtrace/upstairs_raw.d | 3 +- 4 files changed, 152 insertions(+), 18 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a751005cd..696764864 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -849,6 +849,7 @@ dependencies = [ "crucible-control-client", "crucible-protocol", "crucible-workspace-hack", + "serde", "serde_json", "strum 0.27.2", "strum_macros 0.27.2", diff --git a/cmon/Cargo.toml b/cmon/Cargo.toml index adfb21c48..02c8cc76a 100644 --- a/cmon/Cargo.toml +++ b/cmon/Cargo.toml @@ -10,6 +10,7 @@ clearscreen.workspace = true crucible.workspace = true crucible-control-client.workspace = true crucible-protocol.workspace = true +serde.workspace = true serde_json.workspace = true strum.workspace = true strum_macros.workspace = true diff --git a/cmon/src/main.rs b/cmon/src/main.rs index 3d2c44a9d..a2ee18680 100644 --- a/cmon/src/main.rs +++ b/cmon/src/main.rs @@ -2,6 +2,7 @@ use clap::{Parser, Subcommand, ValueEnum}; use crucible_control_client::Client; use crucible_protocol::ClientId; +use serde::Deserialize; use std::fmt; use std::io::{self, BufRead}; use strum::IntoEnumIterator; @@ -10,6 +11,12 @@ use tokio::time::{sleep, Duration}; use crucible::DtraceInfo; +#[derive(Debug, Deserialize)] +struct DtraceWrapper { + pid: u32, + status: DtraceInfo, +} + /// Connect to crucible control server #[derive(Parser, Debug)] #[clap(name = "cmon", term_width = 80)] @@ -30,12 +37,17 @@ struct Args { // The possible fields we will display when receiving DTrace output. #[derive(Debug, Copy, Clone, ValueEnum, EnumIter)] enum DtraceDisplay { + Pid, + Session, + UpstairsId, State, IoCount, IoSummary, UpCount, DsCount, Reconcile, + DsReconciled, + DsReconcileNeeded, LiveRepair, Connected, Replaced, @@ -44,17 +56,31 @@ enum DtraceDisplay { NextJobId, JobDelta, DsDelay, + WriteBytesOut, + RoLrSkipped, + // IOStateCount fields (already partially covered by IoCount/IoSummary) + DsIoInProgress, + DsIoDone, + DsIoSkipped, + DsIoError, } impl fmt::Display for DtraceDisplay { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { match self { + DtraceDisplay::Pid => write!(f, "pid"), + DtraceDisplay::Session => write!(f, "session"), + DtraceDisplay::UpstairsId => write!(f, "upstairs_id"), DtraceDisplay::State => write!(f, "state"), DtraceDisplay::IoCount => write!(f, "io_count"), DtraceDisplay::IoSummary => write!(f, "io_summary"), DtraceDisplay::UpCount => write!(f, "up_count"), DtraceDisplay::DsCount => write!(f, "ds_count"), DtraceDisplay::Reconcile => write!(f, "reconcile"), + DtraceDisplay::DsReconciled => write!(f, "ds_reconciled"), + DtraceDisplay::DsReconcileNeeded => { + write!(f, "ds_reconcile_needed") + } DtraceDisplay::LiveRepair => write!(f, "live_repair"), DtraceDisplay::Connected => write!(f, "connected"), DtraceDisplay::Replaced => write!(f, "replaced"), @@ -63,6 +89,12 @@ impl fmt::Display for DtraceDisplay { DtraceDisplay::NextJobId => write!(f, "next_job_id"), DtraceDisplay::JobDelta => write!(f, "job_delta"), DtraceDisplay::DsDelay => write!(f, "ds_delay"), + DtraceDisplay::WriteBytesOut => write!(f, "write_bytes_out"), + DtraceDisplay::RoLrSkipped => write!(f, "ro_lr_skipped"), + DtraceDisplay::DsIoInProgress => write!(f, "ds_io_in_progress"), + DtraceDisplay::DsIoDone => write!(f, "ds_io_done"), + DtraceDisplay::DsIoSkipped => write!(f, "ds_io_skipped"), + DtraceDisplay::DsIoError => write!(f, "ds_io_error"), } } } @@ -72,7 +104,7 @@ enum Action { /// Read from stdin Dtrace { /// Fields to display from dtrace received input - #[clap(short, long, default_value = "io-count")] + #[clap(short, long, value_delimiter = ',', default_values_t = vec![DtraceDisplay::Pid, DtraceDisplay::Session, DtraceDisplay::State, DtraceDisplay::NextJobId, DtraceDisplay::JobDelta, DtraceDisplay::ExtentLimit, DtraceDisplay::DsReconciled, DtraceDisplay::DsReconcileNeeded])] #[arg(value_enum)] output: Vec, }, @@ -192,6 +224,15 @@ async fn show_repair_stats(args: Args) { fn print_dtrace_header(dd: &[DtraceDisplay]) { for display_item in dd.iter() { match display_item { + DtraceDisplay::Pid => { + print!(" {:>5}", "PID"); + } + DtraceDisplay::Session => { + print!(" {:>8}", "SESSION"); + } + DtraceDisplay::UpstairsId => { + print!(" {:>8}", "UPSTAIRS"); + } DtraceDisplay::State => { print!(" {:>3} {:>3} {:>3}", "DS0", "DS1", "DS2",); } @@ -213,6 +254,12 @@ fn print_dtrace_header(dd: &[DtraceDisplay]) { DtraceDisplay::Reconcile => { print!(" {:>4} {:>4} {:>4}", "REC", "NREC", "AREC"); } + DtraceDisplay::DsReconciled => { + print!(" {:>4}", "ERR"); + } + DtraceDisplay::DsReconcileNeeded => { + print!(" {:>4}", "ERN"); + } DtraceDisplay::LiveRepair => { print!(" {:>4} {:>4} {:>4}", "LRC0", "LRC1", "LRC0"); print!(" {:>4} {:>4} {:>4}", "LRA0", "LRA1", "LRA2"); @@ -237,7 +284,25 @@ fn print_dtrace_header(dd: &[DtraceDisplay]) { print!(" {:>5}", "DELTA"); } DtraceDisplay::DsDelay => { - print!(" {:>5}", "DELAY"); + print!(" {:>5} {:>5} {:>5}", "DLY0", "DLY1", "DLY2"); + } + DtraceDisplay::WriteBytesOut => { + print!(" {:>10}", "WRBYTES"); + } + DtraceDisplay::RoLrSkipped => { + print!(" {:>4} {:>4} {:>4}", "RLS0", "RLS1", "RLS2"); + } + DtraceDisplay::DsIoInProgress => { + print!(" {:>5} {:>5} {:>5}", "IP0", "IP1", "IP2"); + } + DtraceDisplay::DsIoDone => { + print!(" {:>5} {:>5} {:>5}", "D0", "D1", "D2"); + } + DtraceDisplay::DsIoSkipped => { + print!(" {:>5} {:>5} {:>5}", "S0", "S1", "S2"); + } + DtraceDisplay::DsIoError => { + print!(" {:>4} {:>4} {:>4}", "E0", "E1", "E2"); } } } @@ -247,12 +312,26 @@ fn print_dtrace_header(dd: &[DtraceDisplay]) { // Print out the values in the dtrace output based on what the DtraceDisplay // enums are set in the given Vec. fn print_dtrace_row( + pid: u32, d_out: DtraceInfo, dd: &[DtraceDisplay], last_job_id: &mut u64, ) { for display_item in dd.iter() { match display_item { + DtraceDisplay::Pid => { + print!(" {:>5}", pid); + } + DtraceDisplay::Session => { + let session_short = + d_out.session_id.chars().take(8).collect::(); + print!(" {:>8}", session_short); + } + DtraceDisplay::UpstairsId => { + let upstairs_short = + d_out.upstairs_id.chars().take(8).collect::(); + print!(" {:>8}", upstairs_short); + } DtraceDisplay::State => { print!( " {:>3} {:>3} {:>3}", @@ -303,6 +382,12 @@ fn print_dtrace_row( d_out.ds_reconcile_aborted, ); } + DtraceDisplay::DsReconciled => { + print!(" {:>4}", d_out.ds_reconciled); + } + DtraceDisplay::DsReconcileNeeded => { + print!(" {:>4}", d_out.ds_reconcile_needed); + } DtraceDisplay::LiveRepair => { print!( " {:4} {:4} {:4}", @@ -354,12 +439,12 @@ fn print_dtrace_row( print!(" {:>7}", d_out.next_job_id); } DtraceDisplay::JobDelta => { - let delta = if *last_job_id == 0 { - d_out.next_job_id.0 + if *last_job_id == 0 { + print!(" {:>5}", "---"); } else { - d_out.next_job_id.0 - *last_job_id - }; - print!(" {:5}", delta); + let delta = d_out.next_job_id.0 - *last_job_id; + print!(" {:5}", delta); + } *last_job_id = d_out.next_job_id.0; } DtraceDisplay::DsDelay => { @@ -370,6 +455,49 @@ fn print_dtrace_row( d_out.ds_delay_us[2], ); } + DtraceDisplay::WriteBytesOut => { + print!(" {:10}", d_out.write_bytes_out); + } + DtraceDisplay::RoLrSkipped => { + print!( + " {:4} {:4} {:4}", + d_out.ds_ro_lr_skipped[0], + d_out.ds_ro_lr_skipped[1], + d_out.ds_ro_lr_skipped[2], + ); + } + DtraceDisplay::DsIoInProgress => { + print!( + " {:5} {:5} {:5}", + d_out.ds_io_count.in_progress[ClientId::new(0)], + d_out.ds_io_count.in_progress[ClientId::new(1)], + d_out.ds_io_count.in_progress[ClientId::new(2)], + ); + } + DtraceDisplay::DsIoDone => { + print!( + " {:5} {:5} {:5}", + d_out.ds_io_count.done[ClientId::new(0)], + d_out.ds_io_count.done[ClientId::new(1)], + d_out.ds_io_count.done[ClientId::new(2)], + ); + } + DtraceDisplay::DsIoSkipped => { + print!( + " {:5} {:5} {:5}", + d_out.ds_io_count.skipped[ClientId::new(0)], + d_out.ds_io_count.skipped[ClientId::new(1)], + d_out.ds_io_count.skipped[ClientId::new(2)], + ); + } + DtraceDisplay::DsIoError => { + print!( + " {:4} {:4} {:4}", + d_out.ds_io_count.error[ClientId::new(0)], + d_out.ds_io_count.error[ClientId::new(1)], + d_out.ds_io_count.error[ClientId::new(2)], + ); + } } } println!(); @@ -390,15 +518,20 @@ fn dtrace_loop(output: Vec) { print_dtrace_header(&output); } count = (count + 1) % 20; - let d_out: DtraceInfo = match serde_json::from_str(&dtrace_out) - { - Ok(a) => a, - Err(e) => { - println!("Err {:?}", e); - continue; - } - }; - print_dtrace_row(d_out, &output, &mut last_job_id); + let wrapper: DtraceWrapper = + match serde_json::from_str(&dtrace_out) { + Ok(a) => a, + Err(e) => { + println!("Err {:?}", e); + continue; + } + }; + print_dtrace_row( + wrapper.pid, + wrapper.status, + &output, + &mut last_job_id, + ); } Err(e) => { println!("Error: {:?}", e); diff --git a/tools/dtrace/upstairs_raw.d b/tools/dtrace/upstairs_raw.d index 3552ecc1a..d12f75efa 100755 --- a/tools/dtrace/upstairs_raw.d +++ b/tools/dtrace/upstairs_raw.d @@ -7,6 +7,5 @@ #pragma D option strsize=2k crucible_upstairs*:::up-status { - trace(json(copyinstr(arg1), "ok")); - printf("\n"); + printf("{\"pid\":%d,\"status\":%s}\n", pid, json(copyinstr(arg1), "ok")); } From 8b36e4e689c020a98bb057fecfe0516ce6063828 Mon Sep 17 00:00:00 2001 From: Alan Hanson Date: Fri, 24 Oct 2025 15:50:05 -0700 Subject: [PATCH 02/36] framework for curses ctop loop --- Cargo.lock | 1 + cmon/Cargo.toml | 1 + cmon/src/main.rs | 78 +++++++++++++++++++++++++++++++++++++++++++++++- 3 files changed, 79 insertions(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index 696764864..40d86c90e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -845,6 +845,7 @@ version = "0.1.0" dependencies = [ "clap", "clearscreen", + "crossterm", "crucible", "crucible-control-client", "crucible-protocol", diff --git a/cmon/Cargo.toml b/cmon/Cargo.toml index 02c8cc76a..eeb460426 100644 --- a/cmon/Cargo.toml +++ b/cmon/Cargo.toml @@ -7,6 +7,7 @@ edition = "2021" [dependencies] clap.workspace = true clearscreen.workspace = true +crossterm.workspace = true crucible.workspace = true crucible-control-client.workspace = true crucible-protocol.workspace = true diff --git a/cmon/src/main.rs b/cmon/src/main.rs index a2ee18680..5c84a5b2a 100644 --- a/cmon/src/main.rs +++ b/cmon/src/main.rs @@ -1,10 +1,19 @@ // Copyright 2022 Oxide Computer Company use clap::{Parser, Subcommand, ValueEnum}; +use crossterm::{ + cursor, + event::{self, Event, KeyCode, KeyEvent, KeyModifiers}, + execute, + terminal::{ + disable_raw_mode, enable_raw_mode, Clear, ClearType, + EnterAlternateScreen, LeaveAlternateScreen, + }, +}; use crucible_control_client::Client; use crucible_protocol::ClientId; use serde::Deserialize; use std::fmt; -use std::io::{self, BufRead}; +use std::io::{self, BufRead, Write}; use strum::IntoEnumIterator; use strum_macros::EnumIter; use tokio::time::{sleep, Duration}; @@ -114,6 +123,8 @@ enum Action { Jobs, /// Show the status of various LiveRepair stats Repair, + /// Curses-based top-like display of dtrace data + Ctop, } /// Translate what the default DsState string is (that we are getting from DTrace) @@ -540,6 +551,66 @@ fn dtrace_loop(output: Vec) { } } +// Ctop display loop with curses-like interface +fn ctop_loop() -> Result<(), Box> { + // Set up terminal + enable_raw_mode()?; + let mut stdout = io::stdout(); + execute!(stdout, EnterAlternateScreen)?; + + // Set up panic handler to restore terminal + let original_hook = std::panic::take_hook(); + std::panic::set_hook(Box::new(move |panic_info| { + let _ = execute!(io::stdout(), LeaveAlternateScreen); + let _ = disable_raw_mode(); + original_hook(panic_info); + })); + + // Main loop + loop { + // Clear screen + execute!(stdout, Clear(ClearType::All), cursor::MoveTo(0, 0))?; + + // Get current date and time + let now = std::time::SystemTime::now(); + let duration = now + .duration_since(std::time::UNIX_EPOCH) + .unwrap_or_default(); + let date_str = format!("Unix timestamp: {}", duration.as_secs()); + + // Display date + writeln!(stdout, "cmon ctop - {}", date_str)?; + stdout.flush()?; + + // Check for keyboard input with timeout + if event::poll(std::time::Duration::from_millis(100))? { + if let Event::Key(key_event) = event::read()? { + match key_event { + // Exit on 'q' + KeyEvent { + code: KeyCode::Char('q'), + modifiers: KeyModifiers::NONE, + .. + } => break, + // Exit on Ctrl+C + KeyEvent { + code: KeyCode::Char('c'), + modifiers: KeyModifiers::CONTROL, + .. + } => break, + _ => {} + } + } + } + } + + // Clean up terminal + execute!(stdout, LeaveAlternateScreen)?; + disable_raw_mode()?; + + Ok(()) +} + /* * Simple tool to connect to a crucible upstairs control http port * and report back the results from a upstairs_fill_info command. @@ -564,5 +635,10 @@ async fn main() { Action::Repair => { show_repair_stats(args).await; } + Action::Ctop => { + if let Err(e) = ctop_loop() { + eprintln!("Error running ctop: {}", e); + } + } } } From 8037d2c098d654c1f6346ab9e5f00f7b0991ae65 Mon Sep 17 00:00:00 2001 From: Alan Hanson Date: Fri, 24 Oct 2025 16:58:30 -0700 Subject: [PATCH 03/36] ctop taking shape --- cmon/src/ctop.rs | 592 +++++++++++++++++++++++++++++++++++++++++++++++ cmon/src/main.rs | 96 ++------ 2 files changed, 610 insertions(+), 78 deletions(-) create mode 100644 cmon/src/ctop.rs diff --git a/cmon/src/ctop.rs b/cmon/src/ctop.rs new file mode 100644 index 000000000..99377c448 --- /dev/null +++ b/cmon/src/ctop.rs @@ -0,0 +1,592 @@ +// Copyright 2025 Oxide Computer Company + +use crate::{short_state, DtraceDisplay, DtraceWrapper}; +use crossterm::{ + cursor, + event::{self, Event, KeyCode, KeyEvent, KeyModifiers}, + execute, + terminal::{ + disable_raw_mode, enable_raw_mode, Clear, ClearType, + EnterAlternateScreen, LeaveAlternateScreen, + }, +}; +use crucible::DtraceInfo; +use crucible_protocol::ClientId; +use std::collections::HashMap; +use std::io::{self, Write}; +use std::sync::Arc; +use std::time::{Duration, Instant}; +use tokio::io::{AsyncBufReadExt, BufReader}; +use tokio::process::Command; +use tokio::sync::{Notify, RwLock}; + +const STALE_THRESHOLD_SECS: u64 = 10; + +/// Data for a single process +#[derive(Debug, Clone)] +struct ProcessData { + dtrace_info: DtraceInfo, + last_job_id: u64, + last_updated: Instant, +} + +/// Shared state between stdin reader and display tasks +#[derive(Debug, Default)] +struct CtopState { + processes: HashMap, +} + +/// Default display fields (same as dtrace command defaults) +fn default_display_fields() -> Vec { + vec![ + DtraceDisplay::Pid, + DtraceDisplay::Session, + DtraceDisplay::State, + DtraceDisplay::NextJobId, + DtraceDisplay::JobDelta, + DtraceDisplay::ExtentLimit, + DtraceDisplay::DsReconciled, + DtraceDisplay::DsReconcileNeeded, + ] +} + +/// Format header line for the given display fields +fn format_header(dd: &[DtraceDisplay]) -> String { + let mut result = String::new(); + for display_item in dd.iter() { + match display_item { + DtraceDisplay::Pid => { + result.push_str(&format!(" {:>5}", "PID")); + } + DtraceDisplay::Session => { + result.push_str(&format!(" {:>8}", "SESSION")); + } + DtraceDisplay::UpstairsId => { + result.push_str(&format!(" {:>8}", "UPSTAIRS")); + } + DtraceDisplay::State => { + result.push_str(&format!( + " {:>3} {:>3} {:>3}", + "DS0", "DS1", "DS2" + )); + } + DtraceDisplay::UpCount => { + result.push_str(&format!(" {:>3}", "UPW")); + } + DtraceDisplay::DsCount => { + result.push_str(&format!(" {:>5}", "DSW")); + } + DtraceDisplay::IoCount | DtraceDisplay::IoSummary => { + result.push_str(&format!( + " {:>5} {:>5} {:>5}", + "IP0", "IP1", "IP2" + )); + result + .push_str(&format!(" {:>5} {:>5} {:>5}", "D0", "D1", "D2")); + result + .push_str(&format!(" {:>5} {:>5} {:>5}", "S0", "S1", "S2")); + + if matches!(display_item, DtraceDisplay::IoCount) { + result.push_str(&format!( + " {:>4} {:>4} {:>4}", + "E0", "E1", "E2" + )); + } + } + DtraceDisplay::Reconcile => { + result.push_str(&format!( + " {:>4} {:>4} {:>4}", + "REC", "NREC", "AREC" + )); + } + DtraceDisplay::DsReconciled => { + result.push_str(&format!(" {:>4}", "ERR")); + } + DtraceDisplay::DsReconcileNeeded => { + result.push_str(&format!(" {:>4}", "ERN")); + } + DtraceDisplay::LiveRepair => { + result.push_str(&format!( + " {:>4} {:>4} {:>4}", + "LRC0", "LRC1", "LRC0" + )); + result.push_str(&format!( + " {:>4} {:>4} {:>4}", + "LRA0", "LRA1", "LRA2" + )); + } + DtraceDisplay::Connected => { + result.push_str(&format!( + " {:>4} {:>4} {:>4}", + "CON0", "CON1", "CON2" + )); + } + DtraceDisplay::Replaced => { + result.push_str(&format!( + " {:>4} {:>4} {:>4}", + "RPL0", "RPL1", "RPL2" + )); + } + DtraceDisplay::ExtentLiveRepair => { + result.push_str(&format!( + " {:>4} {:>4} {:>4}", + "EXR0", "EXR1", "EXR2" + )); + result.push_str(&format!( + " {:>4} {:>4} {:>4}", + "EXC0", "EXC1", "EXC2" + )); + } + DtraceDisplay::ExtentLimit => { + result.push_str(&format!(" {:>4}", "EXTL")); + } + DtraceDisplay::NextJobId => { + result.push_str(&format!(" {:>7}", "NEXTJOB")); + } + DtraceDisplay::JobDelta => { + result.push_str(&format!(" {:>5}", "DELTA")); + } + DtraceDisplay::DsDelay => { + result.push_str(&format!( + " {:>5} {:>5} {:>5}", + "DLY0", "DLY1", "DLY2" + )); + } + DtraceDisplay::WriteBytesOut => { + result.push_str(&format!(" {:>10}", "WRBYTES")); + } + DtraceDisplay::RoLrSkipped => { + result.push_str(&format!( + " {:>4} {:>4} {:>4}", + "RLS0", "RLS1", "RLS2" + )); + } + DtraceDisplay::DsIoInProgress => { + result.push_str(&format!( + " {:>5} {:>5} {:>5}", + "IP0", "IP1", "IP2" + )); + } + DtraceDisplay::DsIoDone => { + result + .push_str(&format!(" {:>5} {:>5} {:>5}", "D0", "D1", "D2")); + } + DtraceDisplay::DsIoSkipped => { + result + .push_str(&format!(" {:>5} {:>5} {:>5}", "S0", "S1", "S2")); + } + DtraceDisplay::DsIoError => { + result + .push_str(&format!(" {:>4} {:>4} {:>4}", "E0", "E1", "E2")); + } + } + } + result +} + +/// Format a row for a single process +fn format_row( + pid: u32, + d_out: &DtraceInfo, + last_job_id: u64, + dd: &[DtraceDisplay], + is_stale: bool, +) -> String { + let mut result = String::new(); + let mut computed_delta: Option = None; + + for display_item in dd.iter() { + match display_item { + DtraceDisplay::Pid => { + let pid_str = if is_stale { + format!("{pid}*") + } else { + format!("{pid}") + }; + result.push_str(&format!(" {:>5}", pid_str)); + } + DtraceDisplay::Session => { + let session_short = + d_out.session_id.chars().take(8).collect::(); + result.push_str(&format!(" {:>8}", session_short)); + } + DtraceDisplay::UpstairsId => { + let upstairs_short = + d_out.upstairs_id.chars().take(8).collect::(); + result.push_str(&format!(" {:>8}", upstairs_short)); + } + DtraceDisplay::State => { + result.push_str(&format!( + " {:>3} {:>3} {:>3}", + short_state(&d_out.ds_state[0]), + short_state(&d_out.ds_state[1]), + short_state(&d_out.ds_state[2]), + )); + } + DtraceDisplay::UpCount => { + result.push_str(&format!(" {:3}", d_out.up_count)); + } + DtraceDisplay::DsCount => { + result.push_str(&format!(" {:5}", d_out.ds_count)); + } + DtraceDisplay::IoCount | DtraceDisplay::IoSummary => { + result.push_str(&format!( + " {:5} {:5} {:5}", + d_out.ds_io_count.in_progress[ClientId::new(0)], + d_out.ds_io_count.in_progress[ClientId::new(1)], + d_out.ds_io_count.in_progress[ClientId::new(2)], + )); + result.push_str(&format!( + " {:5} {:5} {:5}", + d_out.ds_io_count.done[ClientId::new(0)], + d_out.ds_io_count.done[ClientId::new(1)], + d_out.ds_io_count.done[ClientId::new(2)], + )); + result.push_str(&format!( + " {:5} {:5} {:5}", + d_out.ds_io_count.skipped[ClientId::new(0)], + d_out.ds_io_count.skipped[ClientId::new(1)], + d_out.ds_io_count.skipped[ClientId::new(2)], + )); + if matches!(display_item, DtraceDisplay::IoCount) { + result.push_str(&format!( + " {:4} {:4} {:4}", + d_out.ds_io_count.error[ClientId::new(0)], + d_out.ds_io_count.error[ClientId::new(1)], + d_out.ds_io_count.error[ClientId::new(2)], + )); + } + } + DtraceDisplay::Reconcile => { + result.push_str(&format!( + " {:4} {:4} {:4}", + d_out.ds_reconciled, + d_out.ds_reconcile_needed, + d_out.ds_reconcile_aborted, + )); + } + DtraceDisplay::DsReconciled => { + result.push_str(&format!(" {:>4}", d_out.ds_reconciled)); + } + DtraceDisplay::DsReconcileNeeded => { + result.push_str(&format!(" {:>4}", d_out.ds_reconcile_needed)); + } + DtraceDisplay::LiveRepair => { + result.push_str(&format!( + " {:4} {:4} {:4}", + d_out.ds_live_repair_completed[0], + d_out.ds_live_repair_completed[1], + d_out.ds_live_repair_completed[2], + )); + result.push_str(&format!( + " {:4} {:4} {:4}", + d_out.ds_live_repair_aborted[0], + d_out.ds_live_repair_aborted[1], + d_out.ds_live_repair_aborted[2], + )); + } + DtraceDisplay::Connected => { + result.push_str(&format!( + " {:4} {:4} {:4}", + d_out.ds_connected[0], + d_out.ds_connected[1], + d_out.ds_connected[2], + )); + } + DtraceDisplay::Replaced => { + result.push_str(&format!( + " {:4} {:4} {:4}", + d_out.ds_replaced[0], + d_out.ds_replaced[1], + d_out.ds_replaced[2], + )); + } + DtraceDisplay::ExtentLiveRepair => { + result.push_str(&format!( + " {:4} {:4} {:4}", + d_out.ds_extents_repaired[0], + d_out.ds_extents_repaired[1], + d_out.ds_extents_repaired[2], + )); + result.push_str(&format!( + " {:4} {:4} {:4}", + d_out.ds_extents_confirmed[0], + d_out.ds_extents_confirmed[1], + d_out.ds_extents_confirmed[2], + )); + } + DtraceDisplay::ExtentLimit => { + result.push_str(&format!(" {:4}", d_out.ds_extent_limit)); + } + DtraceDisplay::NextJobId => { + result.push_str(&format!(" {:>7}", d_out.next_job_id)); + } + DtraceDisplay::JobDelta => { + if computed_delta.is_none() { + computed_delta = if last_job_id == 0 { + None + } else { + Some(d_out.next_job_id.0 - last_job_id) + }; + } + if let Some(delta) = computed_delta { + result.push_str(&format!(" {:5}", delta)); + } else { + result.push_str(&format!(" {:>5}", "---")); + } + } + DtraceDisplay::DsDelay => { + result.push_str(&format!( + " {:5} {:5} {:5}", + d_out.ds_delay_us[0], + d_out.ds_delay_us[1], + d_out.ds_delay_us[2], + )); + } + DtraceDisplay::WriteBytesOut => { + result.push_str(&format!(" {:10}", d_out.write_bytes_out)); + } + DtraceDisplay::RoLrSkipped => { + result.push_str(&format!( + " {:4} {:4} {:4}", + d_out.ds_ro_lr_skipped[0], + d_out.ds_ro_lr_skipped[1], + d_out.ds_ro_lr_skipped[2], + )); + } + DtraceDisplay::DsIoInProgress => { + result.push_str(&format!( + " {:5} {:5} {:5}", + d_out.ds_io_count.in_progress[ClientId::new(0)], + d_out.ds_io_count.in_progress[ClientId::new(1)], + d_out.ds_io_count.in_progress[ClientId::new(2)], + )); + } + DtraceDisplay::DsIoDone => { + result.push_str(&format!( + " {:5} {:5} {:5}", + d_out.ds_io_count.done[ClientId::new(0)], + d_out.ds_io_count.done[ClientId::new(1)], + d_out.ds_io_count.done[ClientId::new(2)], + )); + } + DtraceDisplay::DsIoSkipped => { + result.push_str(&format!( + " {:5} {:5} {:5}", + d_out.ds_io_count.skipped[ClientId::new(0)], + d_out.ds_io_count.skipped[ClientId::new(1)], + d_out.ds_io_count.skipped[ClientId::new(2)], + )); + } + DtraceDisplay::DsIoError => { + result.push_str(&format!( + " {:4} {:4} {:4}", + d_out.ds_io_count.error[ClientId::new(0)], + d_out.ds_io_count.error[ClientId::new(1)], + d_out.ds_io_count.error[ClientId::new(2)], + )); + } + } + } + result +} + +/// Subprocess reader task - spawns dtrace command and reads JSON output +async fn subprocess_reader_task( + dtrace_cmd: String, + state: Arc>, + notify: Arc, +) -> Result<(), Box> { + // Parse command string into command and args + let parts: Vec<&str> = dtrace_cmd.split_whitespace().collect(); + if parts.is_empty() { + return Err("Empty dtrace command".into()); + } + + // Spawn the dtrace subprocess + let mut child = Command::new(parts[0]) + .args(&parts[1..]) + .stdout(std::process::Stdio::piped()) + .stderr(std::process::Stdio::null()) + .spawn()?; + + let stdout = child + .stdout + .take() + .ok_or("Failed to capture subprocess stdout")?; + + let reader = BufReader::new(stdout); + let mut lines = reader.lines(); + + // Read lines from subprocess stdout + while let Some(line) = lines.next_line().await? { + // Parse JSON + let wrapper: DtraceWrapper = match serde_json::from_str(&line) { + Ok(w) => w, + Err(_) => continue, + }; + + // Update state + let mut state_guard = state.write().await; + let process_data = state_guard + .processes + .entry(wrapper.pid) + .or_insert_with(|| ProcessData { + dtrace_info: wrapper.status.clone(), + last_job_id: 0, + last_updated: Instant::now(), + }); + + process_data.last_job_id = process_data.dtrace_info.next_job_id.0; + process_data.dtrace_info = wrapper.status; + process_data.last_updated = Instant::now(); + + drop(state_guard); + + // Notify display task + notify.notify_one(); + } + + // Wait for child to exit + let _ = child.wait().await; + + Ok(()) +} + +/// Display task - renders the screen and handles keyboard input +async fn display_task( + state: Arc>, + notify: Arc, +) -> Result<(), Box> { + // Set up terminal + enable_raw_mode()?; + let mut stdout = io::stdout(); + execute!(stdout, EnterAlternateScreen)?; + + // Set up panic handler to restore terminal + let original_hook = std::panic::take_hook(); + std::panic::set_hook(Box::new(move |panic_info| { + let _ = execute!(io::stdout(), LeaveAlternateScreen); + let _ = disable_raw_mode(); + original_hook(panic_info); + })); + + let display_fields = default_display_fields(); + + loop { + // Wait for notification or timeout + tokio::select! { + _ = notify.notified() => {}, + _ = tokio::time::sleep(Duration::from_millis(100)) => {}, + } + + // Clear screen + execute!(stdout, Clear(ClearType::All), cursor::MoveTo(0, 0))?; + + // Get current time + let now = Instant::now(); + let system_time = std::time::SystemTime::now(); + let duration = system_time + .duration_since(std::time::UNIX_EPOCH) + .unwrap_or_default(); + + // Display header + write!( + stdout, + "cmon ctop - Unix timestamp: {}\r\n", + duration.as_secs() + )?; + write!(stdout, "\r\n")?; + + // Display column headers + write!(stdout, "{}\r\n", format_header(&display_fields))?; + + // Read state and display processes sorted by PID + let state_guard = state.read().await; + let mut pids: Vec = + state_guard.processes.keys().copied().collect(); + pids.sort_unstable(); + + for pid in pids { + if let Some(process_data) = state_guard.processes.get(&pid) { + let is_stale = now.duration_since(process_data.last_updated) + > Duration::from_secs(STALE_THRESHOLD_SECS); + + let row = format_row( + pid, + &process_data.dtrace_info, + process_data.last_job_id, + &display_fields, + is_stale, + ); + write!(stdout, "{}\r\n", row)?; + } + } + drop(state_guard); + + // Display footer + write!(stdout, "\r\n")?; + write!( + stdout, + "Press 'q' or Ctrl+C to quit. * = stale (no update in {}s)\r\n", + STALE_THRESHOLD_SECS + )?; + + stdout.flush()?; + + // Check for keyboard input (non-blocking) + if event::poll(Duration::from_millis(0))? { + if let Event::Key(key_event) = event::read()? { + match key_event { + KeyEvent { + code: KeyCode::Char('q'), + modifiers: KeyModifiers::NONE, + .. + } => break, + KeyEvent { + code: KeyCode::Char('c'), + modifiers: KeyModifiers::CONTROL, + .. + } => break, + _ => {} + } + } + } + } + + // Clean up terminal + execute!(stdout, LeaveAlternateScreen)?; + disable_raw_mode()?; + + Ok(()) +} + +/// Main entry point for ctop +pub async fn ctop_loop( + dtrace_cmd: String, +) -> Result<(), Box> { + let state = Arc::new(RwLock::new(CtopState::default())); + let notify = Arc::new(Notify::new()); + + let state_reader = Arc::clone(&state); + let notify_reader = Arc::clone(¬ify); + + // Spawn subprocess reader task + let reader_handle = tokio::spawn(async move { + if let Err(e) = + subprocess_reader_task(dtrace_cmd, state_reader, notify_reader) + .await + { + eprintln!("Subprocess reader error: {}", e); + } + }); + + // Run display task (blocks until user quits) + let display_result = display_task(state, notify).await; + + // Wait for reader task to finish (it should exit quickly) + let _ = + tokio::time::timeout(Duration::from_millis(100), reader_handle).await; + + display_result +} diff --git a/cmon/src/main.rs b/cmon/src/main.rs index 5c84a5b2a..5e9f8f488 100644 --- a/cmon/src/main.rs +++ b/cmon/src/main.rs @@ -1,29 +1,22 @@ // Copyright 2022 Oxide Computer Company use clap::{Parser, Subcommand, ValueEnum}; -use crossterm::{ - cursor, - event::{self, Event, KeyCode, KeyEvent, KeyModifiers}, - execute, - terminal::{ - disable_raw_mode, enable_raw_mode, Clear, ClearType, - EnterAlternateScreen, LeaveAlternateScreen, - }, -}; use crucible_control_client::Client; use crucible_protocol::ClientId; use serde::Deserialize; use std::fmt; -use std::io::{self, BufRead, Write}; +use std::io::{self, BufRead}; use strum::IntoEnumIterator; use strum_macros::EnumIter; use tokio::time::{sleep, Duration}; use crucible::DtraceInfo; +mod ctop; + #[derive(Debug, Deserialize)] -struct DtraceWrapper { - pid: u32, - status: DtraceInfo, +pub struct DtraceWrapper { + pub pid: u32, + pub status: DtraceInfo, } /// Connect to crucible control server @@ -45,7 +38,7 @@ struct Args { // The possible fields we will display when receiving DTrace output. #[derive(Debug, Copy, Clone, ValueEnum, EnumIter)] -enum DtraceDisplay { +pub enum DtraceDisplay { Pid, Session, UpstairsId, @@ -124,12 +117,19 @@ enum Action { /// Show the status of various LiveRepair stats Repair, /// Curses-based top-like display of dtrace data - Ctop, + Ctop { + /// Command to run to generate dtrace output + #[clap( + long, + default_value = "dtrace -s /opt/oxide/crucible_dtrace/upstairs_raw.d" + )] + dtrace_cmd: String, + }, } /// Translate what the default DsState string is (that we are getting from DTrace) /// into a three letter string for printing. -fn short_state(dss: &str) -> String { +pub fn short_state(dss: &str) -> String { match dss { "Active" => "ACT".to_string(), "WaitQuorum" => "WQ".to_string(), @@ -551,66 +551,6 @@ fn dtrace_loop(output: Vec) { } } -// Ctop display loop with curses-like interface -fn ctop_loop() -> Result<(), Box> { - // Set up terminal - enable_raw_mode()?; - let mut stdout = io::stdout(); - execute!(stdout, EnterAlternateScreen)?; - - // Set up panic handler to restore terminal - let original_hook = std::panic::take_hook(); - std::panic::set_hook(Box::new(move |panic_info| { - let _ = execute!(io::stdout(), LeaveAlternateScreen); - let _ = disable_raw_mode(); - original_hook(panic_info); - })); - - // Main loop - loop { - // Clear screen - execute!(stdout, Clear(ClearType::All), cursor::MoveTo(0, 0))?; - - // Get current date and time - let now = std::time::SystemTime::now(); - let duration = now - .duration_since(std::time::UNIX_EPOCH) - .unwrap_or_default(); - let date_str = format!("Unix timestamp: {}", duration.as_secs()); - - // Display date - writeln!(stdout, "cmon ctop - {}", date_str)?; - stdout.flush()?; - - // Check for keyboard input with timeout - if event::poll(std::time::Duration::from_millis(100))? { - if let Event::Key(key_event) = event::read()? { - match key_event { - // Exit on 'q' - KeyEvent { - code: KeyCode::Char('q'), - modifiers: KeyModifiers::NONE, - .. - } => break, - // Exit on Ctrl+C - KeyEvent { - code: KeyCode::Char('c'), - modifiers: KeyModifiers::CONTROL, - .. - } => break, - _ => {} - } - } - } - } - - // Clean up terminal - execute!(stdout, LeaveAlternateScreen)?; - disable_raw_mode()?; - - Ok(()) -} - /* * Simple tool to connect to a crucible upstairs control http port * and report back the results from a upstairs_fill_info command. @@ -635,8 +575,8 @@ async fn main() { Action::Repair => { show_repair_stats(args).await; } - Action::Ctop => { - if let Err(e) = ctop_loop() { + Action::Ctop { dtrace_cmd } => { + if let Err(e) = ctop::ctop_loop(dtrace_cmd).await { eprintln!("Error running ctop: {}", e); } } From 9a7012c5ac3c2473bfbe0d71370f2e03281f834b Mon Sep 17 00:00:00 2001 From: Alan Hanson Date: Fri, 24 Oct 2025 17:18:58 -0700 Subject: [PATCH 04/36] fix bug to handle pid with many sessions --- cmon/src/ctop.rs | 60 +++++++++++++++++++++++++----------------------- 1 file changed, 31 insertions(+), 29 deletions(-) diff --git a/cmon/src/ctop.rs b/cmon/src/ctop.rs index 99377c448..64b158209 100644 --- a/cmon/src/ctop.rs +++ b/cmon/src/ctop.rs @@ -22,9 +22,10 @@ use tokio::sync::{Notify, RwLock}; const STALE_THRESHOLD_SECS: u64 = 10; -/// Data for a single process +/// Data for a single session #[derive(Debug, Clone)] -struct ProcessData { +struct SessionData { + pid: u32, dtrace_info: DtraceInfo, last_job_id: u64, last_updated: Instant, @@ -33,7 +34,7 @@ struct ProcessData { /// Shared state between stdin reader and display tasks #[derive(Debug, Default)] struct CtopState { - processes: HashMap, + sessions: HashMap, } /// Default display fields (same as dtrace command defaults) @@ -428,18 +429,21 @@ async fn subprocess_reader_task( // Update state let mut state_guard = state.write().await; - let process_data = state_guard - .processes - .entry(wrapper.pid) - .or_insert_with(|| ProcessData { + + let session_data = state_guard + .sessions + .entry(wrapper.status.session_id.clone()) + .or_insert_with(|| SessionData { + pid: wrapper.pid, dtrace_info: wrapper.status.clone(), last_job_id: 0, last_updated: Instant::now(), }); - process_data.last_job_id = process_data.dtrace_info.next_job_id.0; - process_data.dtrace_info = wrapper.status; - process_data.last_updated = Instant::now(); + // Save the old job_id before updating + session_data.last_job_id = session_data.dtrace_info.next_job_id.0; + session_data.dtrace_info = wrapper.status; + session_data.last_updated = Instant::now(); drop(state_guard); @@ -501,26 +505,24 @@ async fn display_task( // Display column headers write!(stdout, "{}\r\n", format_header(&display_fields))?; - // Read state and display processes sorted by PID + // Read state and display sessions sorted by PID (then session_id) let state_guard = state.read().await; - let mut pids: Vec = - state_guard.processes.keys().copied().collect(); - pids.sort_unstable(); - - for pid in pids { - if let Some(process_data) = state_guard.processes.get(&pid) { - let is_stale = now.duration_since(process_data.last_updated) - > Duration::from_secs(STALE_THRESHOLD_SECS); - - let row = format_row( - pid, - &process_data.dtrace_info, - process_data.last_job_id, - &display_fields, - is_stale, - ); - write!(stdout, "{}\r\n", row)?; - } + let mut sessions: Vec<&SessionData> = + state_guard.sessions.values().collect(); + sessions.sort_by_key(|s| (s.pid, &s.dtrace_info.session_id)); + + for session_data in sessions { + let is_stale = now.duration_since(session_data.last_updated) + > Duration::from_secs(STALE_THRESHOLD_SECS); + + let row = format_row( + session_data.pid, + &session_data.dtrace_info, + session_data.last_job_id, + &display_fields, + is_stale, + ); + write!(stdout, "{}\r\n", row)?; } drop(state_guard); From f3b4e32bca08f6994ed19ad57942f9c5c9fb1473 Mon Sep 17 00:00:00 2001 From: Alan Hanson Date: Fri, 24 Oct 2025 17:29:37 -0700 Subject: [PATCH 05/36] less flicker --- cmon/src/ctop.rs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/cmon/src/ctop.rs b/cmon/src/ctop.rs index 64b158209..482c2ddfc 100644 --- a/cmon/src/ctop.rs +++ b/cmon/src/ctop.rs @@ -484,8 +484,8 @@ async fn display_task( _ = tokio::time::sleep(Duration::from_millis(100)) => {}, } - // Clear screen - execute!(stdout, Clear(ClearType::All), cursor::MoveTo(0, 0))?; + // Move cursor to top-left (don't clear entire screen) + execute!(stdout, cursor::MoveTo(0, 0))?; // Get current time let now = Instant::now(); @@ -534,6 +534,9 @@ async fn display_task( STALE_THRESHOLD_SECS )?; + // Clear from cursor to end of screen (removes any leftover data) + execute!(stdout, Clear(ClearType::FromCursorDown))?; + stdout.flush()?; // Check for keyboard input (non-blocking) From a8dce869915351f498124d0129df9b4a14d762cc Mon Sep 17 00:00:00 2001 From: Alan Hanson Date: Fri, 24 Oct 2025 18:38:21 -0700 Subject: [PATCH 06/36] fix status line leaving junk behind --- cmon/src/ctop.rs | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/cmon/src/ctop.rs b/cmon/src/ctop.rs index 482c2ddfc..c55a8c264 100644 --- a/cmon/src/ctop.rs +++ b/cmon/src/ctop.rs @@ -494,16 +494,21 @@ async fn display_task( .duration_since(std::time::UNIX_EPOCH) .unwrap_or_default(); - // Display header + // Display header (clear line first to remove artifacts) write!( stdout, - "cmon ctop - Unix timestamp: {}\r\n", + "cmon ctop - Unix timestamp: {}", duration.as_secs() )?; + execute!(stdout, Clear(ClearType::UntilNewLine))?; + write!(stdout, "\r\n")?; + execute!(stdout, Clear(ClearType::UntilNewLine))?; write!(stdout, "\r\n")?; // Display column headers - write!(stdout, "{}\r\n", format_header(&display_fields))?; + write!(stdout, "{}", format_header(&display_fields))?; + execute!(stdout, Clear(ClearType::UntilNewLine))?; + write!(stdout, "\r\n")?; // Read state and display sessions sorted by PID (then session_id) let state_guard = state.read().await; @@ -522,19 +527,24 @@ async fn display_task( &display_fields, is_stale, ); - write!(stdout, "{}\r\n", row)?; + write!(stdout, "{}", row)?; + execute!(stdout, Clear(ClearType::UntilNewLine))?; + write!(stdout, "\r\n")?; } drop(state_guard); // Display footer + execute!(stdout, Clear(ClearType::UntilNewLine))?; write!(stdout, "\r\n")?; write!( stdout, - "Press 'q' or Ctrl+C to quit. * = stale (no update in {}s)\r\n", + "Press 'q' or Ctrl+C to quit. * = stale (no update in {}s)", STALE_THRESHOLD_SECS )?; + execute!(stdout, Clear(ClearType::UntilNewLine))?; + write!(stdout, "\r\n")?; - // Clear from cursor to end of screen (removes any leftover data) + // Clear from cursor to end of screen (removes any leftover lines) execute!(stdout, Clear(ClearType::FromCursorDown))?; stdout.flush()?; From 64d03e97bb2ce8c2bb1c6c8144d8cbc8ae3ed231 Mon Sep 17 00:00:00 2001 From: Alan Hanson Date: Fri, 24 Oct 2025 21:58:35 -0700 Subject: [PATCH 07/36] add sparkline io history --- Cargo.lock | 104 ++++++++++++++++++++++++++++++++++++++++----- Cargo.toml | 1 + cmon/Cargo.toml | 1 + cmon/src/ctop.rs | 107 ++++++++++++++++++++++++++++++++++++++--------- 4 files changed, 184 insertions(+), 29 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 40d86c90e..4ed790bfe 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -405,7 +405,7 @@ dependencies = [ "bitflags 2.9.4", "cexpr", "clang-sys", - "itertools 0.10.5", + "itertools 0.12.1", "lazy_static", "lazycell", "log", @@ -644,6 +644,21 @@ dependencies = [ "thiserror 2.0.16", ] +[[package]] +name = "cassowary" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df8670b8c7b9dae1793364eafadf7239c40d669904660c5960d74cfd80b46a53" + +[[package]] +name = "castaway" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dec551ab6e7578819132c713a93c022a05d60159dc86e7a7050223577484c55a" +dependencies = [ + "rustversion", +] + [[package]] name = "cc" version = "1.2.39" @@ -850,6 +865,7 @@ dependencies = [ "crucible-control-client", "crucible-protocol", "crucible-workspace-hack", + "ratatui", "serde", "serde_json", "strum 0.27.2", @@ -892,6 +908,20 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "compact_str" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b79c4069c6cad78e2e0cdfcbd26275770669fb39fd308a752dc110e83b9af32" +dependencies = [ + "castaway", + "cfg-if", + "itoa", + "rustversion", + "ryu", + "static_assertions", +] + [[package]] name = "console" version = "0.15.7" @@ -3441,6 +3471,15 @@ dependencies = [ "web-time 1.1.0", ] +[[package]] +name = "indoc" +version = "2.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79cf5c93f93228cf8efb3ba362535fb11199ac548a09ce117c9b1adc3030d706" +dependencies = [ + "rustversion", +] + [[package]] name = "ingot" version = "0.1.1" @@ -3489,6 +3528,19 @@ dependencies = [ "generic-array", ] +[[package]] +name = "instability" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "435d80800b936787d62688c927b6490e887c7ef5ff9ce922c6c6050fca75eb9a" +dependencies = [ + "darling 0.20.11", + "indoc", + "proc-macro2", + "quote", + "syn 2.0.106", +] + [[package]] name = "instant" version = "0.1.12" @@ -3602,15 +3654,6 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7655c9839580ee829dfacba1d1278c2b7883e50a277ff7541299489d6bdfdc45" -[[package]] -name = "itertools" -version = "0.10.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" -dependencies = [ - "either", -] - [[package]] name = "itertools" version = "0.12.1" @@ -3886,6 +3929,15 @@ dependencies = [ "tracing-subscriber", ] +[[package]] +name = "lru" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "234cf4f4a04dc1f57e24b96cc0cd600cf2af460d4161ac5ecdd0af8e1f3b2a38" +dependencies = [ + "hashbrown 0.15.2", +] + [[package]] name = "lru-cache" version = "0.1.2" @@ -5919,6 +5971,27 @@ version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f60fcc7d6849342eff22c4350c8b9a989ee8ceabc4b481253e8946b9fe83d684" +[[package]] +name = "ratatui" +version = "0.28.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdef7f9be5c0122f890d58bdf4d964349ba6a6161f705907526d891efabba57d" +dependencies = [ + "bitflags 2.9.4", + "cassowary", + "compact_str", + "crossterm", + "instability", + "itertools 0.13.0", + "lru", + "paste", + "strum 0.26.3", + "strum_macros 0.26.4", + "unicode-segmentation", + "unicode-truncate", + "unicode-width 0.1.14", +] + [[package]] name = "rayon" version = "1.10.0" @@ -8112,6 +8185,17 @@ version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1dd624098567895118886609431a7c3b8f516e41d30e0643f03d94592a147e36" +[[package]] +name = "unicode-truncate" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3644627a5af5fa321c95b9b235a72fd24cd29c648c2c379431e6628655627bf" +dependencies = [ + "itertools 0.13.0", + "unicode-segmentation", + "unicode-width 0.1.14", +] + [[package]] name = "unicode-width" version = "0.1.14" diff --git a/Cargo.toml b/Cargo.toml index f9851f8c0..9410766c1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -88,6 +88,7 @@ proptest = "1.6.0" rayon = "1.10.0" rand = { version = "0.9.1", features = [ "small_rng"] } rand_chacha = "0.9.0" +ratatui = "0.28" reedline = "0.39.0" rangemap = "1.5.1" reqwest = { version = "0.12", features = ["default", "blocking", "json", "stream"] } diff --git a/cmon/Cargo.toml b/cmon/Cargo.toml index eeb460426..1b1b023e0 100644 --- a/cmon/Cargo.toml +++ b/cmon/Cargo.toml @@ -11,6 +11,7 @@ crossterm.workspace = true crucible.workspace = true crucible-control-client.workspace = true crucible-protocol.workspace = true +ratatui.workspace = true serde.workspace = true serde_json.workspace = true strum.workspace = true diff --git a/cmon/src/ctop.rs b/cmon/src/ctop.rs index c55a8c264..6cca47f94 100644 --- a/cmon/src/ctop.rs +++ b/cmon/src/ctop.rs @@ -12,7 +12,7 @@ use crossterm::{ }; use crucible::DtraceInfo; use crucible_protocol::ClientId; -use std::collections::HashMap; +use std::collections::{HashMap, VecDeque}; use std::io::{self, Write}; use std::sync::Arc; use std::time::{Duration, Instant}; @@ -21,6 +21,7 @@ use tokio::process::Command; use tokio::sync::{Notify, RwLock}; const STALE_THRESHOLD_SECS: u64 = 10; +const MAX_DELTA_HISTORY: usize = 100; /// Data for a single session #[derive(Debug, Clone)] @@ -29,6 +30,8 @@ struct SessionData { dtrace_info: DtraceInfo, last_job_id: u64, last_updated: Instant, + current_delta: Option, + delta_history: VecDeque, } /// Shared state between stdin reader and display tasks @@ -189,12 +192,11 @@ fn format_header(dd: &[DtraceDisplay]) -> String { fn format_row( pid: u32, d_out: &DtraceInfo, - last_job_id: u64, + precomputed_delta: Option, dd: &[DtraceDisplay], is_stale: bool, ) -> String { let mut result = String::new(); - let mut computed_delta: Option = None; for display_item in dd.iter() { match display_item { @@ -323,14 +325,7 @@ fn format_row( result.push_str(&format!(" {:>7}", d_out.next_job_id)); } DtraceDisplay::JobDelta => { - if computed_delta.is_none() { - computed_delta = if last_job_id == 0 { - None - } else { - Some(d_out.next_job_id.0 - last_job_id) - }; - } - if let Some(delta) = computed_delta { + if let Some(delta) = precomputed_delta { result.push_str(&format!(" {:5}", delta)); } else { result.push_str(&format!(" {:>5}", "---")); @@ -392,6 +387,47 @@ fn format_row( result } +/// Render a sparkline from delta history +/// Uses Unicode block characters to show trend: ▁▂▃▄▅▆▇█ +fn render_sparkline(history: &VecDeque, width: usize) -> String { + if history.is_empty() || width == 0 { + return String::new(); + } + + // Unicode block characters from lowest to highest + const BLOCKS: [char; 8] = ['▁', '▂', '▃', '▄', '▅', '▆', '▇', '█']; + + // Take last 'width' samples (most recent) + let samples: Vec = history + .iter() + .rev() + .take(width) + .copied() + .collect::>() + .into_iter() + .rev() + .collect(); + + if samples.is_empty() { + return String::new(); + } + + // Find max value for scaling + let max = *samples.iter().max().unwrap_or(&1); + if max == 0 { + return BLOCKS[0].to_string().repeat(samples.len()); + } + + // Map each value to a block character + samples + .iter() + .map(|&val| { + let normalized = (val as f64 / max as f64 * 7.0) as usize; + BLOCKS[normalized.min(7)] + }) + .collect() +} + /// Subprocess reader task - spawns dtrace command and reads JSON output async fn subprocess_reader_task( dtrace_cmd: String, @@ -438,10 +474,29 @@ async fn subprocess_reader_task( dtrace_info: wrapper.status.clone(), last_job_id: 0, last_updated: Instant::now(), + current_delta: None, + delta_history: VecDeque::new(), }); - // Save the old job_id before updating - session_data.last_job_id = session_data.dtrace_info.next_job_id.0; + // Calculate delta (jobs per second) + let current_job_id = wrapper.status.next_job_id.0; + let delta = if session_data.last_job_id != 0 { + let d = current_job_id.saturating_sub(session_data.last_job_id); + + // Add to history ring buffer + session_data.delta_history.push_back(d); + if session_data.delta_history.len() > MAX_DELTA_HISTORY { + session_data.delta_history.pop_front(); + } + + Some(d) + } else { + None + }; + + // Store current delta and update state + session_data.current_delta = delta; + session_data.last_job_id = current_job_id; session_data.dtrace_info = wrapper.status; session_data.last_updated = Instant::now(); @@ -495,11 +550,7 @@ async fn display_task( .unwrap_or_default(); // Display header (clear line first to remove artifacts) - write!( - stdout, - "cmon ctop - Unix timestamp: {}", - duration.as_secs() - )?; + write!(stdout, "cmon ctop - Unix timestamp: {}", duration.as_secs())?; execute!(stdout, Clear(ClearType::UntilNewLine))?; write!(stdout, "\r\n")?; execute!(stdout, Clear(ClearType::UntilNewLine))?; @@ -510,6 +561,9 @@ async fn display_task( execute!(stdout, Clear(ClearType::UntilNewLine))?; write!(stdout, "\r\n")?; + // Get terminal size for sparkline width calculation + let (terminal_width, _) = crossterm::terminal::size()?; + // Read state and display sessions sorted by PID (then session_id) let state_guard = state.read().await; let mut sessions: Vec<&SessionData> = @@ -523,11 +577,26 @@ async fn display_task( let row = format_row( session_data.pid, &session_data.dtrace_info, - session_data.last_job_id, + session_data.current_delta, &display_fields, is_stale, ); write!(stdout, "{}", row)?; + + // Render sparkline in remaining space + let row_len = row.chars().count(); + if terminal_width > row_len as u16 { + let sparkline_width = + (terminal_width as usize - row_len).saturating_sub(1); + if sparkline_width > 0 { + let sparkline = render_sparkline( + &session_data.delta_history, + sparkline_width, + ); + write!(stdout, " {}", sparkline)?; + } + } + execute!(stdout, Clear(ClearType::UntilNewLine))?; write!(stdout, "\r\n")?; } From e253d9ce6004e9a49c7e0ba112fccb6323a7b9c9 Mon Sep 17 00:00:00 2001 From: Alan Hanson Date: Fri, 24 Oct 2025 22:31:19 -0700 Subject: [PATCH 08/36] normalize sparkline --- cmon/src/ctop.rs | 33 +++++++++++++++++++++++++-------- 1 file changed, 25 insertions(+), 8 deletions(-) diff --git a/cmon/src/ctop.rs b/cmon/src/ctop.rs index 6cca47f94..09403c9c9 100644 --- a/cmon/src/ctop.rs +++ b/cmon/src/ctop.rs @@ -389,7 +389,13 @@ fn format_row( /// Render a sparkline from delta history /// Uses Unicode block characters to show trend: ▁▂▃▄▅▆▇█ -fn render_sparkline(history: &VecDeque, width: usize) -> String { +/// If global_max is provided, scales relative to that value for +/// cross-session comparison +fn render_sparkline( + history: &VecDeque, + width: usize, + global_max: u64, +) -> String { if history.is_empty() || width == 0 { return String::new(); } @@ -412,18 +418,19 @@ fn render_sparkline(history: &VecDeque, width: usize) -> String { return String::new(); } - // Find max value for scaling - let max = *samples.iter().max().unwrap_or(&1); - if max == 0 { - return BLOCKS[0].to_string().repeat(samples.len()); - } + // Use global max for scaling (minimum 1 to avoid division by zero) + let max = global_max.max(1); // Map each value to a block character samples .iter() .map(|&val| { - let normalized = (val as f64 / max as f64 * 7.0) as usize; - BLOCKS[normalized.min(7)] + if val == 0 { + BLOCKS[0] + } else { + let normalized = (val as f64 / max as f64 * 7.0) as usize; + BLOCKS[normalized.min(7)] + } }) .collect() } @@ -570,6 +577,15 @@ async fn display_task( state_guard.sessions.values().collect(); sessions.sort_by_key(|s| (s.pid, &s.dtrace_info.session_id)); + // Calculate global max across all sessions for consistent sparkline + // scaling + let global_max = sessions + .iter() + .flat_map(|s| s.delta_history.iter()) + .copied() + .max() + .unwrap_or(1); + for session_data in sessions { let is_stale = now.duration_since(session_data.last_updated) > Duration::from_secs(STALE_THRESHOLD_SECS); @@ -592,6 +608,7 @@ async fn display_task( let sparkline = render_sparkline( &session_data.delta_history, sparkline_width, + global_max, ); write!(stdout, " {}", sparkline)?; } From d273bb64c8190466a265e322ee2afa0c6b513973 Mon Sep 17 00:00:00 2001 From: Alan Hanson Date: Sat, 25 Oct 2025 09:33:48 -0700 Subject: [PATCH 09/36] Add detailed delta graphs --- cmon/src/ctop.rs | 314 +++++++++++++++++++++++++++++++++++++---------- 1 file changed, 246 insertions(+), 68 deletions(-) diff --git a/cmon/src/ctop.rs b/cmon/src/ctop.rs index 09403c9c9..3a9bda96e 100644 --- a/cmon/src/ctop.rs +++ b/cmon/src/ctop.rs @@ -12,6 +12,13 @@ use crossterm::{ }; use crucible::DtraceInfo; use crucible_protocol::ClientId; +use ratatui::{ + backend::CrosstermBackend, + style::Color, + widgets::canvas::{Canvas, Line, Points}, + widgets::{Block, Borders}, + Terminal, +}; use std::collections::{HashMap, VecDeque}; use std::io::{self, Write}; use std::sync::Arc; @@ -38,6 +45,8 @@ struct SessionData { #[derive(Debug, Default)] struct CtopState { sessions: HashMap, + selected_index: usize, + detail_mode: bool, } /// Default display fields (same as dtrace command defaults) @@ -519,6 +528,94 @@ async fn subprocess_reader_task( Ok(()) } +/// Render full-screen detail view for a selected session +fn render_detail_view( + session_data: &SessionData, + _terminal_size: (u16, u16), +) -> io::Result<()> { + // Calculate statistics + let history: Vec = + session_data.delta_history.iter().copied().collect(); + let max = history.iter().copied().max().unwrap_or(1); + let min = history.iter().copied().min().unwrap_or(0); + let avg = if !history.is_empty() { + history.iter().sum::() / history.len() as u64 + } else { + 0 + }; + let current = session_data.current_delta.unwrap_or(0); + + // Render using ratatui + let mut stdout = io::stdout(); + let backend = CrosstermBackend::new(&mut stdout); + let mut terminal = Terminal::new(backend)?; + + // Clear the screen to remove previous frame (ratatui handles this efficiently) + terminal.clear()?; + + terminal.draw(|f| { + let area = f.area(); + + // Create title + let session_short: String = + session_data.dtrace_info.session_id.chars().take(8).collect(); + let title = format!( + " Delta History - PID {} - Session {} ", + session_data.pid, session_short + ); + + // Create canvas widget + let canvas = Canvas::default() + .block( + Block::default() + .borders(Borders::ALL) + .title(title) + .title_bottom(format!( + " Samples: {} | Min: {} | Max: {} | Avg: {} | Current: {} ", + history.len(), + min, + max, + avg, + current + )) + .title_bottom(" ['d': Back to table | 'q': Quit] "), + ) + .x_bounds([0.0, history.len().max(1) as f64]) + .y_bounds([min as f64, max as f64]) + .paint(|ctx| { + // Draw the line graph + if history.len() > 1 { + for i in 0..history.len() - 1 { + let x1 = i as f64; + let y1 = history[i] as f64; + let x2 = (i + 1) as f64; + let y2 = history[i + 1] as f64; + + ctx.draw(&Line { + x1, + y1, + x2, + y2, + color: Color::Cyan, + }); + } + } + + // Draw points for each sample + for (i, &value) in history.iter().enumerate() { + ctx.draw(&Points { + coords: &[(i as f64, value as f64)], + color: Color::Yellow, + }); + } + }); + + f.render_widget(canvas, area); + })?; + + Ok(()) +} + /// Display task - renders the screen and handles keyboard input async fn display_task( state: Arc>, @@ -556,88 +653,129 @@ async fn display_task( .duration_since(std::time::UNIX_EPOCH) .unwrap_or_default(); - // Display header (clear line first to remove artifacts) - write!(stdout, "cmon ctop - Unix timestamp: {}", duration.as_secs())?; - execute!(stdout, Clear(ClearType::UntilNewLine))?; - write!(stdout, "\r\n")?; - execute!(stdout, Clear(ClearType::UntilNewLine))?; - write!(stdout, "\r\n")?; + // Get terminal size + let terminal_size = crossterm::terminal::size()?; - // Display column headers - write!(stdout, "{}", format_header(&display_fields))?; - execute!(stdout, Clear(ClearType::UntilNewLine))?; - write!(stdout, "\r\n")?; + // Read state to check mode + let state_guard = state.read().await; + let in_detail_mode = state_guard.detail_mode; + let selected_index = state_guard.selected_index; + + // If in detail mode, render detail view and skip table + if in_detail_mode { + let mut sessions: Vec<&SessionData> = + state_guard.sessions.values().collect(); + sessions.sort_by_key(|s| (s.pid, &s.dtrace_info.session_id)); + + if let Some(selected_session) = sessions.get(selected_index) { + // Clone the session data so we can drop the lock + let session_clone = (*selected_session).clone(); + drop(state_guard); + render_detail_view(&session_clone, terminal_size)?; + } else { + drop(state_guard); + } + } else { + // Table mode - render normal view + drop(state_guard); + + // Display header (clear line first to remove artifacts) + execute!(stdout, cursor::MoveTo(0, 0))?; + write!( + stdout, + "cmon ctop - Unix timestamp: {}", + duration.as_secs() + )?; + execute!(stdout, Clear(ClearType::UntilNewLine))?; + write!(stdout, "\r\n")?; + execute!(stdout, Clear(ClearType::UntilNewLine))?; + write!(stdout, "\r\n")?; - // Get terminal size for sparkline width calculation - let (terminal_width, _) = crossterm::terminal::size()?; + // Display column headers + write!(stdout, "{}", format_header(&display_fields))?; + execute!(stdout, Clear(ClearType::UntilNewLine))?; + write!(stdout, "\r\n")?; - // Read state and display sessions sorted by PID (then session_id) - let state_guard = state.read().await; - let mut sessions: Vec<&SessionData> = - state_guard.sessions.values().collect(); - sessions.sort_by_key(|s| (s.pid, &s.dtrace_info.session_id)); - - // Calculate global max across all sessions for consistent sparkline - // scaling - let global_max = sessions - .iter() - .flat_map(|s| s.delta_history.iter()) - .copied() - .max() - .unwrap_or(1); - - for session_data in sessions { - let is_stale = now.duration_since(session_data.last_updated) - > Duration::from_secs(STALE_THRESHOLD_SECS); - - let row = format_row( - session_data.pid, - &session_data.dtrace_info, - session_data.current_delta, - &display_fields, - is_stale, - ); - write!(stdout, "{}", row)?; - - // Render sparkline in remaining space - let row_len = row.chars().count(); - if terminal_width > row_len as u16 { - let sparkline_width = - (terminal_width as usize - row_len).saturating_sub(1); - if sparkline_width > 0 { - let sparkline = render_sparkline( - &session_data.delta_history, - sparkline_width, - global_max, - ); - write!(stdout, " {}", sparkline)?; + let (terminal_width, _) = terminal_size; + + // Read state and display sessions sorted by PID (then session_id) + let state_guard = state.read().await; + let mut sessions: Vec<&SessionData> = + state_guard.sessions.values().collect(); + sessions.sort_by_key(|s| (s.pid, &s.dtrace_info.session_id)); + + // Calculate global max across all sessions for consistent sparkline + // scaling + let global_max = sessions + .iter() + .flat_map(|s| s.delta_history.iter()) + .copied() + .max() + .unwrap_or(1); + + let selected_index = state_guard.selected_index; + + for (idx, session_data) in sessions.iter().enumerate() { + let is_stale = now.duration_since(session_data.last_updated) + > Duration::from_secs(STALE_THRESHOLD_SECS); + + // Add selection indicator + let indicator = if idx == selected_index { ">" } else { " " }; + write!(stdout, "{}", indicator)?; + + let row = format_row( + session_data.pid, + &session_data.dtrace_info, + session_data.current_delta, + &display_fields, + is_stale, + ); + write!(stdout, "{}", row)?; + + // Render sparkline in remaining space + // Account for the indicator character (1 char) + let row_len = row.chars().count() + 1; + if terminal_width > row_len as u16 { + let sparkline_width = + (terminal_width as usize - row_len).saturating_sub(1); + if sparkline_width > 0 { + let sparkline = render_sparkline( + &session_data.delta_history, + sparkline_width, + global_max, + ); + write!(stdout, " {}", sparkline)?; + } } + + execute!(stdout, Clear(ClearType::UntilNewLine))?; + write!(stdout, "\r\n")?; } + drop(state_guard); + // Display footer + execute!(stdout, Clear(ClearType::UntilNewLine))?; + write!(stdout, "\r\n")?; + write!( + stdout, + "[↑↓: Select | 'd': Details | 'q': Quit] * = stale ({}s)", + STALE_THRESHOLD_SECS + )?; execute!(stdout, Clear(ClearType::UntilNewLine))?; write!(stdout, "\r\n")?; - } - drop(state_guard); - - // Display footer - execute!(stdout, Clear(ClearType::UntilNewLine))?; - write!(stdout, "\r\n")?; - write!( - stdout, - "Press 'q' or Ctrl+C to quit. * = stale (no update in {}s)", - STALE_THRESHOLD_SECS - )?; - execute!(stdout, Clear(ClearType::UntilNewLine))?; - write!(stdout, "\r\n")?; - // Clear from cursor to end of screen (removes any leftover lines) - execute!(stdout, Clear(ClearType::FromCursorDown))?; + // Clear from cursor to end of screen (removes any leftover lines) + execute!(stdout, Clear(ClearType::FromCursorDown))?; - stdout.flush()?; + stdout.flush()?; + } // End of table mode rendering // Check for keyboard input (non-blocking) if event::poll(Duration::from_millis(0))? { if let Event::Key(key_event) = event::read()? { + let mut state_guard = state.write().await; + let num_sessions = state_guard.sessions.len(); + match key_event { KeyEvent { code: KeyCode::Char('q'), @@ -649,8 +787,48 @@ async fn display_task( modifiers: KeyModifiers::CONTROL, .. } => break, + KeyEvent { + code: KeyCode::Char('d'), + modifiers: KeyModifiers::NONE, + .. + } => { + // Toggle detail mode + state_guard.detail_mode = !state_guard.detail_mode; + } + KeyEvent { + code: KeyCode::Up, + modifiers: KeyModifiers::NONE, + .. + } => { + // Move selection up (only in table mode) + if !state_guard.detail_mode && num_sessions > 0 { + state_guard.selected_index = + state_guard.selected_index.saturating_sub(1); + } + } + KeyEvent { + code: KeyCode::Down, + modifiers: KeyModifiers::NONE, + .. + } => { + // Move selection down (only in table mode) + if !state_guard.detail_mode && num_sessions > 0 { + state_guard.selected_index = + (state_guard.selected_index + 1) + .min(num_sessions.saturating_sub(1)); + } + } + KeyEvent { + code: KeyCode::Esc, + modifiers: KeyModifiers::NONE, + .. + } => { + // Exit detail mode + state_guard.detail_mode = false; + } _ => {} } + drop(state_guard); } } } From d97e65dd312dd815d49d0c5fe9bea5c4c6522ad9 Mon Sep 17 00:00:00 2001 From: Alan Hanson Date: Sat, 25 Oct 2025 11:30:15 -0700 Subject: [PATCH 10/36] polish graphing, norm options --- cmon/src/ctop.rs | 126 ++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 107 insertions(+), 19 deletions(-) diff --git a/cmon/src/ctop.rs b/cmon/src/ctop.rs index 3a9bda96e..31bdefdc2 100644 --- a/cmon/src/ctop.rs +++ b/cmon/src/ctop.rs @@ -47,6 +47,7 @@ struct CtopState { sessions: HashMap, selected_index: usize, detail_mode: bool, + normalize_detail: bool, // Use global min/max for detail view scaling } /// Default display fields (same as dtrace command defaults) @@ -532,12 +533,16 @@ async fn subprocess_reader_task( fn render_detail_view( session_data: &SessionData, _terminal_size: (u16, u16), + terminal: &mut Terminal>, + global_min: Option, + global_max: Option, + normalize: bool, ) -> io::Result<()> { // Calculate statistics let history: Vec = session_data.delta_history.iter().copied().collect(); - let max = history.iter().copied().max().unwrap_or(1); - let min = history.iter().copied().min().unwrap_or(0); + let session_max = history.iter().copied().max().unwrap_or(1); + let session_min = history.iter().copied().min().unwrap_or(0); let avg = if !history.is_empty() { history.iter().sum::() / history.len() as u64 } else { @@ -545,23 +550,27 @@ fn render_detail_view( }; let current = session_data.current_delta.unwrap_or(0); - // Render using ratatui - let mut stdout = io::stdout(); - let backend = CrosstermBackend::new(&mut stdout); - let mut terminal = Terminal::new(backend)?; - - // Clear the screen to remove previous frame (ratatui handles this efficiently) - terminal.clear()?; + // Choose min/max based on normalize mode + let (display_min, display_max) = if normalize { + ( + global_min.unwrap_or(session_min), + global_max.unwrap_or(session_max), + ) + } else { + (session_min, session_max) + }; + // Render using ratatui (terminal is reused, ratatui handles diffing) terminal.draw(|f| { let area = f.area(); // Create title let session_short: String = session_data.dtrace_info.session_id.chars().take(8).collect(); + let mode_str = if normalize { " [NORMALIZED]" } else { "" }; let title = format!( - " Delta History - PID {} - Session {} ", - session_data.pid, session_short + " Delta History - PID {} - Session {}{} ", + session_data.pid, session_short, mode_str ); // Create canvas widget @@ -573,16 +582,40 @@ fn render_detail_view( .title_bottom(format!( " Samples: {} | Min: {} | Max: {} | Avg: {} | Current: {} ", history.len(), - min, - max, + session_min, + session_max, avg, current )) - .title_bottom(" ['d': Back to table | 'q': Quit] "), + .title_bottom( + " ['d': Back | 'n': Toggle normalize | 'q': Quit] ", + ), ) .x_bounds([0.0, history.len().max(1) as f64]) - .y_bounds([min as f64, max as f64]) + .y_bounds([display_min as f64, display_max as f64]) .paint(|ctx| { + // Draw Y-axis labels (at left edge of graph) + let y_range = display_max as f64 - display_min as f64; + let labels = [ + (display_max, "top"), + (display_min + (y_range * 0.75) as u64, "3/4"), + (display_min + (y_range * 0.5) as u64, "1/2"), + (display_min + (y_range * 0.25) as u64, "1/4"), + (display_min, "base"), + ]; + + for (y_val, label) in &labels { + ctx.print( + 0.0, + *y_val as f64, + ratatui::text::Span::styled( + format!("{}: {}", label, y_val), + ratatui::style::Style::default() + .fg(Color::Gray), + ), + ); + } + // Draw the line graph if history.len() > 1 { for i in 0..history.len() - 1 { @@ -636,6 +669,12 @@ async fn display_task( let display_fields = default_display_fields(); + // Track detail mode and persistent terminal for detail view + let mut was_in_detail_mode = false; + let mut detail_terminal: Option< + Terminal>, + > = None; + loop { // Wait for notification or timeout tokio::select! { @@ -643,9 +682,6 @@ async fn display_task( _ = tokio::time::sleep(Duration::from_millis(100)) => {}, } - // Move cursor to top-left (don't clear entire screen) - execute!(stdout, cursor::MoveTo(0, 0))?; - // Get current time let now = Instant::now(); let system_time = std::time::SystemTime::now(); @@ -660,22 +696,65 @@ async fn display_task( let state_guard = state.read().await; let in_detail_mode = state_guard.detail_mode; let selected_index = state_guard.selected_index; + let normalize_detail = state_guard.normalize_detail; // If in detail mode, render detail view and skip table if in_detail_mode { + // Create terminal on first entry to detail mode + if !was_in_detail_mode { + execute!(stdout, Clear(ClearType::All))?; + // Create a new stdout handle for the Terminal + let detail_stdout = io::stdout(); + let backend = CrosstermBackend::new(detail_stdout); + detail_terminal = Some(Terminal::new(backend)?); + } + let mut sessions: Vec<&SessionData> = state_guard.sessions.values().collect(); sessions.sort_by_key(|s| (s.pid, &s.dtrace_info.session_id)); + // Calculate global min/max across all sessions for normalization + let global_min = sessions + .iter() + .flat_map(|s| s.delta_history.iter()) + .copied() + .min(); + let global_max = sessions + .iter() + .flat_map(|s| s.delta_history.iter()) + .copied() + .max(); + if let Some(selected_session) = sessions.get(selected_index) { // Clone the session data so we can drop the lock let session_clone = (*selected_session).clone(); drop(state_guard); - render_detail_view(&session_clone, terminal_size)?; + + if let Some(terminal) = detail_terminal.as_mut() { + render_detail_view( + &session_clone, + terminal_size, + terminal, + global_min, + global_max, + normalize_detail, + )?; + } } else { drop(state_guard); } + + was_in_detail_mode = true; } else { + // Exiting detail mode - drop terminal and redraw table + if was_in_detail_mode { + detail_terminal = None; + execute!(stdout, Clear(ClearType::All))?; + } + was_in_detail_mode = false; + + // Move cursor to top-left + execute!(stdout, cursor::MoveTo(0, 0))?; // Table mode - render normal view drop(state_guard); @@ -795,6 +874,15 @@ async fn display_task( // Toggle detail mode state_guard.detail_mode = !state_guard.detail_mode; } + KeyEvent { + code: KeyCode::Char('n'), + modifiers: KeyModifiers::NONE, + .. + } => { + // Toggle normalize mode (only affects detail view) + state_guard.normalize_detail = + !state_guard.normalize_detail; + } KeyEvent { code: KeyCode::Up, modifiers: KeyModifiers::NONE, From 27be4f229a4c21d68bfe68699bacd19729654e4b Mon Sep 17 00:00:00 2001 From: Alan Hanson Date: Sat, 25 Oct 2025 11:39:40 -0700 Subject: [PATCH 11/36] fix stale function and column width --- cmon/src/ctop.rs | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/cmon/src/ctop.rs b/cmon/src/ctop.rs index 31bdefdc2..1ad403faf 100644 --- a/cmon/src/ctop.rs +++ b/cmon/src/ctop.rs @@ -57,6 +57,9 @@ fn default_display_fields() -> Vec { DtraceDisplay::Session, DtraceDisplay::State, DtraceDisplay::NextJobId, + DtraceDisplay::Connected, + DtraceDisplay::UpCount, + DtraceDisplay::DsCount, DtraceDisplay::JobDelta, DtraceDisplay::ExtentLimit, DtraceDisplay::DsReconciled, @@ -204,19 +207,15 @@ fn format_row( d_out: &DtraceInfo, precomputed_delta: Option, dd: &[DtraceDisplay], - is_stale: bool, + _is_stale: bool, ) -> String { let mut result = String::new(); for display_item in dd.iter() { match display_item { DtraceDisplay::Pid => { - let pid_str = if is_stale { - format!("{pid}*") - } else { - format!("{pid}") - }; - result.push_str(&format!(" {:>5}", pid_str)); + // Note: stale indicator is now shown in the first column + result.push_str(&format!(" {:>5}", pid)); } DtraceDisplay::Session => { let session_short = @@ -798,8 +797,15 @@ async fn display_task( let is_stale = now.duration_since(session_data.last_updated) > Duration::from_secs(STALE_THRESHOLD_SECS); - // Add selection indicator - let indicator = if idx == selected_index { ">" } else { " " }; + // Add indicator: > for selected, * for stale, space otherwise + // Selection indicator (>) takes priority over stale indicator (*) + let indicator = if idx == selected_index { + ">" + } else if is_stale { + "*" + } else { + " " + }; write!(stdout, "{}", indicator)?; let row = format_row( @@ -837,7 +843,7 @@ async fn display_task( write!(stdout, "\r\n")?; write!( stdout, - "[↑↓: Select | 'd': Details | 'q': Quit] * = stale ({}s)", + "[↑↓: Select | 'd': Details | 'q': Quit] > = selected, * = stale ({}s)", STALE_THRESHOLD_SECS )?; execute!(stdout, Clear(ClearType::UntilNewLine))?; From 9721a9e0ba7a0f615f3553f1ff93f947a5332f16 Mon Sep 17 00:00:00 2001 From: Alan Hanson Date: Sat, 25 Oct 2025 11:53:15 -0700 Subject: [PATCH 12/36] fix column width --- cmon/src/ctop.rs | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/cmon/src/ctop.rs b/cmon/src/ctop.rs index 1ad403faf..b1af226b6 100644 --- a/cmon/src/ctop.rs +++ b/cmon/src/ctop.rs @@ -57,9 +57,6 @@ fn default_display_fields() -> Vec { DtraceDisplay::Session, DtraceDisplay::State, DtraceDisplay::NextJobId, - DtraceDisplay::Connected, - DtraceDisplay::UpCount, - DtraceDisplay::DsCount, DtraceDisplay::JobDelta, DtraceDisplay::ExtentLimit, DtraceDisplay::DsReconciled, @@ -158,7 +155,7 @@ fn format_header(dd: &[DtraceDisplay]) -> String { result.push_str(&format!(" {:>4}", "EXTL")); } DtraceDisplay::NextJobId => { - result.push_str(&format!(" {:>7}", "NEXTJOB")); + result.push_str(&format!(" {:>10}", "NEXTJOB")); } DtraceDisplay::JobDelta => { result.push_str(&format!(" {:>5}", "DELTA")); @@ -331,7 +328,7 @@ fn format_row( result.push_str(&format!(" {:4}", d_out.ds_extent_limit)); } DtraceDisplay::NextJobId => { - result.push_str(&format!(" {:>7}", d_out.next_job_id)); + result.push_str(&format!(" {:>10}", d_out.next_job_id)); } DtraceDisplay::JobDelta => { if let Some(delta) = precomputed_delta { From 9a6a3f6e429d0490979d968f71d6ea0d19117593 Mon Sep 17 00:00:00 2001 From: Alan Hanson Date: Sat, 25 Oct 2025 15:05:08 -0700 Subject: [PATCH 13/36] fix header offset --- cmon/src/ctop.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmon/src/ctop.rs b/cmon/src/ctop.rs index b1af226b6..7a6534403 100644 --- a/cmon/src/ctop.rs +++ b/cmon/src/ctop.rs @@ -155,7 +155,7 @@ fn format_header(dd: &[DtraceDisplay]) -> String { result.push_str(&format!(" {:>4}", "EXTL")); } DtraceDisplay::NextJobId => { - result.push_str(&format!(" {:>10}", "NEXTJOB")); + result.push_str(&format!(" {:>10}", "NEXTJOB")); } DtraceDisplay::JobDelta => { result.push_str(&format!(" {:>5}", "DELTA")); From 16fd9fcbd730a43c401972d7cd7c088b44e97252 Mon Sep 17 00:00:00 2001 From: Alan Hanson Date: Sat, 25 Oct 2025 15:34:14 -0700 Subject: [PATCH 14/36] Polis Y axis for detailed graph --- cmon/src/ctop.rs | 47 ++++++++++++++++++++++++++++++++++++----------- 1 file changed, 36 insertions(+), 11 deletions(-) diff --git a/cmon/src/ctop.rs b/cmon/src/ctop.rs index 7a6534403..65f5af3fd 100644 --- a/cmon/src/ctop.rs +++ b/cmon/src/ctop.rs @@ -14,9 +14,10 @@ use crucible::DtraceInfo; use crucible_protocol::ClientId; use ratatui::{ backend::CrosstermBackend, + layout::{Constraint, Layout}, style::Color, widgets::canvas::{Canvas, Line, Points}, - widgets::{Block, Borders}, + widgets::{Block, Borders, Paragraph}, Terminal, }; use std::collections::{HashMap, VecDeque}; @@ -560,6 +561,30 @@ fn render_detail_view( terminal.draw(|f| { let area = f.area(); + // Split area: 1 line at top for session data, rest for canvas + let chunks = Layout::default() + .constraints([Constraint::Length(1), Constraint::Min(0)]) + .split(area); + + // Format the session data row + let display_fields = default_display_fields(); + let is_stale = session_data + .last_updated + .elapsed() + .as_secs() + > STALE_THRESHOLD_SECS; + let row_data = format_row( + session_data.pid, + &session_data.dtrace_info, + session_data.current_delta, + &display_fields, + is_stale, + ); + + // Render session data at top + let data_paragraph = Paragraph::new(row_data); + f.render_widget(data_paragraph, chunks[0]); + // Create title let session_short: String = session_data.dtrace_info.session_id.chars().take(8).collect(); @@ -569,7 +594,7 @@ fn render_detail_view( session_data.pid, session_short, mode_str ); - // Create canvas widget + // Create canvas widget in bottom area (1 line shorter) let canvas = Canvas::default() .block( Block::default() @@ -592,20 +617,20 @@ fn render_detail_view( .paint(|ctx| { // Draw Y-axis labels (at left edge of graph) let y_range = display_max as f64 - display_min as f64; - let labels = [ - (display_max, "top"), - (display_min + (y_range * 0.75) as u64, "3/4"), - (display_min + (y_range * 0.5) as u64, "1/2"), - (display_min + (y_range * 0.25) as u64, "1/4"), - (display_min, "base"), + let y_positions = [ + display_max, + display_min + (y_range * 0.75) as u64, + display_min + (y_range * 0.5) as u64, + display_min + (y_range * 0.25) as u64, + display_min, ]; - for (y_val, label) in &labels { + for y_val in &y_positions { ctx.print( 0.0, *y_val as f64, ratatui::text::Span::styled( - format!("{}: {}", label, y_val), + format!("{}", y_val), ratatui::style::Style::default() .fg(Color::Gray), ), @@ -639,7 +664,7 @@ fn render_detail_view( } }); - f.render_widget(canvas, area); + f.render_widget(canvas, chunks[1]); })?; Ok(()) From 0d4b094423bfd2bec615293999ca0300d7e7a0ea Mon Sep 17 00:00:00 2001 From: Alan Hanson Date: Sun, 26 Oct 2025 11:25:18 -0700 Subject: [PATCH 15/36] break ctop into a different binary --- Cargo.lock | 30 ++ Cargo.toml | 3 + cmon/Cargo.toml | 3 +- cmon/src/ctop.rs | 141 ++++--- cmon/src/main.rs | 104 +---- ctop/Cargo.toml | 18 + ctop/src/main.rs | 1013 ++++++++++++++++++++++++++++++++++++++++++++++ 7 files changed, 1140 insertions(+), 172 deletions(-) create mode 100644 ctop/Cargo.toml create mode 100644 ctop/src/main.rs diff --git a/Cargo.lock b/Cargo.lock index 4ed790bfe..072014f99 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -860,6 +860,7 @@ version = "0.1.0" dependencies = [ "clap", "clearscreen", + "cmon-common", "crossterm", "crucible", "crucible-control-client", @@ -873,6 +874,18 @@ dependencies = [ "tokio", ] +[[package]] +name = "cmon-common" +version = "0.1.0" +dependencies = [ + "clap", + "crucible", + "crucible-workspace-hack", + "serde", + "strum 0.27.2", + "strum_macros 0.27.2", +] + [[package]] name = "cobs" version = "0.2.3" @@ -1771,6 +1784,23 @@ dependencies = [ "memchr", ] +[[package]] +name = "ctop" +version = "0.1.0" +dependencies = [ + "clap", + "cmon-common", + "crossterm", + "crucible", + "crucible-protocol", + "crucible-workspace-hack", + "ratatui", + "serde", + "serde_json", + "strum 0.27.2", + "tokio", +] + [[package]] name = "ctr" version = "0.9.2" diff --git a/Cargo.toml b/Cargo.toml index 9410766c1..81e4191b1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,8 +8,10 @@ members = [ "common", "control-client", "cmon", + "cmon-common", "crudd", "crutest", + "ctop", "downstairs", "downstairs-api", "downstairs-types", @@ -139,6 +141,7 @@ oximeter = { git = "https://github.com/oxidecomputer/omicron", branch = "main" } oximeter-producer = { git = "https://github.com/oxidecomputer/omicron", branch = "main" } # local path +cmon-common = { path = "./cmon-common" } crucible = { path = "./upstairs" } crucible-agent-api = { path = "./agent-api" } crucible-agent-client = { path = "./agent-client" } diff --git a/cmon/Cargo.toml b/cmon/Cargo.toml index 1b1b023e0..447a52e6a 100644 --- a/cmon/Cargo.toml +++ b/cmon/Cargo.toml @@ -2,11 +2,12 @@ name = "cmon" version = "0.1.0" license = "MPL-2.0" -edition = "2021" +edition = "2024" [dependencies] clap.workspace = true clearscreen.workspace = true +cmon-common = { path = "../cmon-common", features = ["clap"] } crossterm.workspace = true crucible.workspace = true crucible-control-client.workspace = true diff --git a/cmon/src/ctop.rs b/cmon/src/ctop.rs index 65f5af3fd..161be4ae8 100644 --- a/cmon/src/ctop.rs +++ b/cmon/src/ctop.rs @@ -1,24 +1,24 @@ // Copyright 2025 Oxide Computer Company -use crate::{short_state, DtraceDisplay, DtraceWrapper}; +use cmon_common::{DtraceDisplay, DtraceWrapper, short_state}; use crossterm::{ cursor, event::{self, Event, KeyCode, KeyEvent, KeyModifiers}, execute, terminal::{ - disable_raw_mode, enable_raw_mode, Clear, ClearType, - EnterAlternateScreen, LeaveAlternateScreen, + Clear, ClearType, EnterAlternateScreen, LeaveAlternateScreen, + disable_raw_mode, enable_raw_mode, }, }; use crucible::DtraceInfo; use crucible_protocol::ClientId; use ratatui::{ + Terminal, backend::CrosstermBackend, layout::{Constraint, Layout}, style::Color, widgets::canvas::{Canvas, Line, Points}, widgets::{Block, Borders, Paragraph}, - Terminal, }; use std::collections::{HashMap, VecDeque}; use std::io::{self, Write}; @@ -692,9 +692,8 @@ async fn display_task( // Track detail mode and persistent terminal for detail view let mut was_in_detail_mode = false; - let mut detail_terminal: Option< - Terminal>, - > = None; + let mut detail_terminal: Option>> = + None; loop { // Wait for notification or timeout @@ -878,74 +877,74 @@ async fn display_task( } // End of table mode rendering // Check for keyboard input (non-blocking) - if event::poll(Duration::from_millis(0))? { - if let Event::Key(key_event) = event::read()? { - let mut state_guard = state.write().await; - let num_sessions = state_guard.sessions.len(); - - match key_event { - KeyEvent { - code: KeyCode::Char('q'), - modifiers: KeyModifiers::NONE, - .. - } => break, - KeyEvent { - code: KeyCode::Char('c'), - modifiers: KeyModifiers::CONTROL, - .. - } => break, - KeyEvent { - code: KeyCode::Char('d'), - modifiers: KeyModifiers::NONE, - .. - } => { - // Toggle detail mode - state_guard.detail_mode = !state_guard.detail_mode; - } - KeyEvent { - code: KeyCode::Char('n'), - modifiers: KeyModifiers::NONE, - .. - } => { - // Toggle normalize mode (only affects detail view) - state_guard.normalize_detail = - !state_guard.normalize_detail; - } - KeyEvent { - code: KeyCode::Up, - modifiers: KeyModifiers::NONE, - .. - } => { - // Move selection up (only in table mode) - if !state_guard.detail_mode && num_sessions > 0 { - state_guard.selected_index = - state_guard.selected_index.saturating_sub(1); - } - } - KeyEvent { - code: KeyCode::Down, - modifiers: KeyModifiers::NONE, - .. - } => { - // Move selection down (only in table mode) - if !state_guard.detail_mode && num_sessions > 0 { - state_guard.selected_index = - (state_guard.selected_index + 1) - .min(num_sessions.saturating_sub(1)); - } + if event::poll(Duration::from_millis(0))? + && let Event::Key(key_event) = event::read()? + { + let mut state_guard = state.write().await; + let num_sessions = state_guard.sessions.len(); + + match key_event { + KeyEvent { + code: KeyCode::Char('q'), + modifiers: KeyModifiers::NONE, + .. + } => break, + KeyEvent { + code: KeyCode::Char('c'), + modifiers: KeyModifiers::CONTROL, + .. + } => break, + KeyEvent { + code: KeyCode::Char('d'), + modifiers: KeyModifiers::NONE, + .. + } => { + // Toggle detail mode + state_guard.detail_mode = !state_guard.detail_mode; + } + KeyEvent { + code: KeyCode::Char('n'), + modifiers: KeyModifiers::NONE, + .. + } => { + // Toggle normalize mode (only affects detail view) + state_guard.normalize_detail = + !state_guard.normalize_detail; + } + KeyEvent { + code: KeyCode::Up, + modifiers: KeyModifiers::NONE, + .. + } => { + // Move selection up (only in table mode) + if !state_guard.detail_mode && num_sessions > 0 { + state_guard.selected_index = + state_guard.selected_index.saturating_sub(1); } - KeyEvent { - code: KeyCode::Esc, - modifiers: KeyModifiers::NONE, - .. - } => { - // Exit detail mode - state_guard.detail_mode = false; + } + KeyEvent { + code: KeyCode::Down, + modifiers: KeyModifiers::NONE, + .. + } => { + // Move selection down (only in table mode) + if !state_guard.detail_mode && num_sessions > 0 { + state_guard.selected_index = + (state_guard.selected_index + 1) + .min(num_sessions.saturating_sub(1)); } - _ => {} } - drop(state_guard); + KeyEvent { + code: KeyCode::Esc, + modifiers: KeyModifiers::NONE, + .. + } => { + // Exit detail mode + state_guard.detail_mode = false; + } + _ => {} } + drop(state_guard); } } diff --git a/cmon/src/main.rs b/cmon/src/main.rs index 5e9f8f488..015f763dd 100644 --- a/cmon/src/main.rs +++ b/cmon/src/main.rs @@ -1,24 +1,15 @@ // Copyright 2022 Oxide Computer Company -use clap::{Parser, Subcommand, ValueEnum}; +use clap::{Parser, Subcommand}; +use cmon_common::{DtraceDisplay, DtraceWrapper, short_state}; +use crucible::DtraceInfo; use crucible_control_client::Client; use crucible_protocol::ClientId; -use serde::Deserialize; -use std::fmt; use std::io::{self, BufRead}; use strum::IntoEnumIterator; -use strum_macros::EnumIter; -use tokio::time::{sleep, Duration}; - -use crucible::DtraceInfo; +use tokio::time::{Duration, sleep}; mod ctop; -#[derive(Debug, Deserialize)] -pub struct DtraceWrapper { - pub pid: u32, - pub status: DtraceInfo, -} - /// Connect to crucible control server #[derive(Parser, Debug)] #[clap(name = "cmon", term_width = 80)] @@ -36,71 +27,6 @@ struct Args { seconds: u64, } -// The possible fields we will display when receiving DTrace output. -#[derive(Debug, Copy, Clone, ValueEnum, EnumIter)] -pub enum DtraceDisplay { - Pid, - Session, - UpstairsId, - State, - IoCount, - IoSummary, - UpCount, - DsCount, - Reconcile, - DsReconciled, - DsReconcileNeeded, - LiveRepair, - Connected, - Replaced, - ExtentLiveRepair, - ExtentLimit, - NextJobId, - JobDelta, - DsDelay, - WriteBytesOut, - RoLrSkipped, - // IOStateCount fields (already partially covered by IoCount/IoSummary) - DsIoInProgress, - DsIoDone, - DsIoSkipped, - DsIoError, -} - -impl fmt::Display for DtraceDisplay { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - match self { - DtraceDisplay::Pid => write!(f, "pid"), - DtraceDisplay::Session => write!(f, "session"), - DtraceDisplay::UpstairsId => write!(f, "upstairs_id"), - DtraceDisplay::State => write!(f, "state"), - DtraceDisplay::IoCount => write!(f, "io_count"), - DtraceDisplay::IoSummary => write!(f, "io_summary"), - DtraceDisplay::UpCount => write!(f, "up_count"), - DtraceDisplay::DsCount => write!(f, "ds_count"), - DtraceDisplay::Reconcile => write!(f, "reconcile"), - DtraceDisplay::DsReconciled => write!(f, "ds_reconciled"), - DtraceDisplay::DsReconcileNeeded => { - write!(f, "ds_reconcile_needed") - } - DtraceDisplay::LiveRepair => write!(f, "live_repair"), - DtraceDisplay::Connected => write!(f, "connected"), - DtraceDisplay::Replaced => write!(f, "replaced"), - DtraceDisplay::ExtentLiveRepair => write!(f, "extent_live_repair"), - DtraceDisplay::ExtentLimit => write!(f, "extent_under_repair"), - DtraceDisplay::NextJobId => write!(f, "next_job_id"), - DtraceDisplay::JobDelta => write!(f, "job_delta"), - DtraceDisplay::DsDelay => write!(f, "ds_delay"), - DtraceDisplay::WriteBytesOut => write!(f, "write_bytes_out"), - DtraceDisplay::RoLrSkipped => write!(f, "ro_lr_skipped"), - DtraceDisplay::DsIoInProgress => write!(f, "ds_io_in_progress"), - DtraceDisplay::DsIoDone => write!(f, "ds_io_done"), - DtraceDisplay::DsIoSkipped => write!(f, "ds_io_skipped"), - DtraceDisplay::DsIoError => write!(f, "ds_io_error"), - } - } -} - #[derive(Debug, Subcommand)] enum Action { /// Read from stdin @@ -127,28 +53,6 @@ enum Action { }, } -/// Translate what the default DsState string is (that we are getting from DTrace) -/// into a three letter string for printing. -pub fn short_state(dss: &str) -> String { - match dss { - "Active" => "ACT".to_string(), - "WaitQuorum" => "WQ".to_string(), - "Reconcile" => "REC".to_string(), - "LiveRepairReady" => "LRR".to_string(), - "New" => "NEW".to_string(), - "Faulted" => "FLT".to_string(), - "Offline" => "OFL".to_string(), - "Replaced" => "RPL".to_string(), - "LiveRepair" => "LR".to_string(), - "Replacing" => "RPC".to_string(), - "Disabled" => "DIS".to_string(), - "Deactivated" => "DAV".to_string(), - "NegotiationFailed" => "NF".to_string(), - "Fault" => "FLT".to_string(), - x => x.to_string(), - } -} - // Show the downstairs work queue async fn show_work_queue(args: Args) { let ca = Client::new(&args.control); diff --git a/ctop/Cargo.toml b/ctop/Cargo.toml new file mode 100644 index 000000000..8f9fafaf5 --- /dev/null +++ b/ctop/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "ctop" +version = "0.1.0" +license = "MPL-2.0" +edition = "2024" + +[dependencies] +clap.workspace = true +cmon-common = { path = "../cmon-common" } +crossterm.workspace = true +crucible.workspace = true +crucible-protocol.workspace = true +ratatui.workspace = true +serde.workspace = true +serde_json.workspace = true +strum.workspace = true +tokio.workspace = true +crucible-workspace-hack.workspace = true diff --git a/ctop/src/main.rs b/ctop/src/main.rs new file mode 100644 index 000000000..82b118579 --- /dev/null +++ b/ctop/src/main.rs @@ -0,0 +1,1013 @@ +// Copyright 2025 Oxide Computer Company + +//! Standalone ctop - curses-based top-like display of crucible dtrace data + +use clap::Parser; +use cmon_common::{DtraceDisplay, DtraceWrapper, short_state}; +use crossterm::{ + cursor, + event::{self, Event, KeyCode, KeyEvent, KeyModifiers}, + execute, + terminal::{ + Clear, ClearType, EnterAlternateScreen, LeaveAlternateScreen, + disable_raw_mode, enable_raw_mode, + }, +}; +use crucible::DtraceInfo; +use crucible_protocol::ClientId; +use ratatui::{ + Terminal, + backend::CrosstermBackend, + layout::{Constraint, Layout}, + style::Color, + widgets::canvas::{Canvas, Line, Points}, + widgets::{Block, Borders, Paragraph}, +}; +use std::collections::{HashMap, VecDeque}; +use std::io::{self, Write}; +use std::sync::Arc; +use std::time::{Duration, Instant}; +use tokio::io::{AsyncBufReadExt, BufReader}; +use tokio::process::Command; +use tokio::sync::{Notify, RwLock}; + +/// Crucible top - monitor crucible upstairs via dtrace +#[derive(Parser, Debug)] +#[clap(name = "ctop", term_width = 80)] +#[clap(about = "Curses-based crucible monitor", long_about = None)] +struct Args { + /// Command to run to generate dtrace output + #[clap( + long, + default_value = "dtrace -s /opt/oxide/crucible_dtrace/upstairs_raw.d" + )] + dtrace_cmd: String, +} + +const STALE_THRESHOLD_SECS: u64 = 10; +const MAX_DELTA_HISTORY: usize = 100; + +/// Data for a single session +#[derive(Debug, Clone)] +struct SessionData { + pid: u32, + dtrace_info: DtraceInfo, + last_job_id: u64, + last_updated: Instant, + current_delta: Option, + delta_history: VecDeque, +} + +/// Shared state between stdin reader and display tasks +#[derive(Debug, Default)] +struct CtopState { + sessions: HashMap, + selected_index: usize, + detail_mode: bool, + normalize_detail: bool, // Use global min/max for detail view scaling +} + +/// Default display fields (same as dtrace command defaults) +fn default_display_fields() -> Vec { + vec![ + DtraceDisplay::Pid, + DtraceDisplay::Session, + DtraceDisplay::State, + DtraceDisplay::NextJobId, + DtraceDisplay::JobDelta, + DtraceDisplay::ExtentLimit, + DtraceDisplay::DsReconciled, + DtraceDisplay::DsReconcileNeeded, + ] +} + +/// Format header line for the given display fields +fn format_header(dd: &[DtraceDisplay]) -> String { + let mut result = String::new(); + for display_item in dd.iter() { + match display_item { + DtraceDisplay::Pid => { + result.push_str(&format!(" {:>5}", "PID")); + } + DtraceDisplay::Session => { + result.push_str(&format!(" {:>8}", "SESSION")); + } + DtraceDisplay::UpstairsId => { + result.push_str(&format!(" {:>8}", "UPSTAIRS")); + } + DtraceDisplay::State => { + result.push_str(&format!( + " {:>3} {:>3} {:>3}", + "DS0", "DS1", "DS2" + )); + } + DtraceDisplay::UpCount => { + result.push_str(&format!(" {:>3}", "UPW")); + } + DtraceDisplay::DsCount => { + result.push_str(&format!(" {:>5}", "DSW")); + } + DtraceDisplay::IoCount | DtraceDisplay::IoSummary => { + result.push_str(&format!( + " {:>5} {:>5} {:>5}", + "IP0", "IP1", "IP2" + )); + result + .push_str(&format!(" {:>5} {:>5} {:>5}", "D0", "D1", "D2")); + result + .push_str(&format!(" {:>5} {:>5} {:>5}", "S0", "S1", "S2")); + + if matches!(display_item, DtraceDisplay::IoCount) { + result.push_str(&format!( + " {:>4} {:>4} {:>4}", + "E0", "E1", "E2" + )); + } + } + DtraceDisplay::Reconcile => { + result.push_str(&format!( + " {:>4} {:>4} {:>4}", + "REC", "NREC", "AREC" + )); + } + DtraceDisplay::DsReconciled => { + result.push_str(&format!(" {:>4}", "ERR")); + } + DtraceDisplay::DsReconcileNeeded => { + result.push_str(&format!(" {:>4}", "ERN")); + } + DtraceDisplay::LiveRepair => { + result.push_str(&format!( + " {:>4} {:>4} {:>4}", + "LRC0", "LRC1", "LRC0" + )); + result.push_str(&format!( + " {:>4} {:>4} {:>4}", + "LRA0", "LRA1", "LRA2" + )); + } + DtraceDisplay::Connected => { + result.push_str(&format!( + " {:>4} {:>4} {:>4}", + "CON0", "CON1", "CON2" + )); + } + DtraceDisplay::Replaced => { + result.push_str(&format!( + " {:>4} {:>4} {:>4}", + "RPL0", "RPL1", "RPL2" + )); + } + DtraceDisplay::ExtentLiveRepair => { + result.push_str(&format!( + " {:>4} {:>4} {:>4}", + "EXR0", "EXR1", "EXR2" + )); + result.push_str(&format!( + " {:>4} {:>4} {:>4}", + "EXC0", "EXC1", "EXC2" + )); + } + DtraceDisplay::ExtentLimit => { + result.push_str(&format!(" {:>4}", "EXTL")); + } + DtraceDisplay::NextJobId => { + result.push_str(&format!(" {:>10}", "NEXTJOB")); + } + DtraceDisplay::JobDelta => { + result.push_str(&format!(" {:>5}", "DELTA")); + } + DtraceDisplay::DsDelay => { + result.push_str(&format!( + " {:>5} {:>5} {:>5}", + "DLY0", "DLY1", "DLY2" + )); + } + DtraceDisplay::WriteBytesOut => { + result.push_str(&format!(" {:>10}", "WRBYTES")); + } + DtraceDisplay::RoLrSkipped => { + result.push_str(&format!( + " {:>4} {:>4} {:>4}", + "RLS0", "RLS1", "RLS2" + )); + } + DtraceDisplay::DsIoInProgress => { + result.push_str(&format!( + " {:>5} {:>5} {:>5}", + "IP0", "IP1", "IP2" + )); + } + DtraceDisplay::DsIoDone => { + result + .push_str(&format!(" {:>5} {:>5} {:>5}", "D0", "D1", "D2")); + } + DtraceDisplay::DsIoSkipped => { + result + .push_str(&format!(" {:>5} {:>5} {:>5}", "S0", "S1", "S2")); + } + DtraceDisplay::DsIoError => { + result + .push_str(&format!(" {:>4} {:>4} {:>4}", "E0", "E1", "E2")); + } + } + } + result +} + +/// Format a row for a single process +fn format_row( + pid: u32, + d_out: &DtraceInfo, + precomputed_delta: Option, + dd: &[DtraceDisplay], + _is_stale: bool, +) -> String { + let mut result = String::new(); + + for display_item in dd.iter() { + match display_item { + DtraceDisplay::Pid => { + // Note: stale indicator is now shown in the first column + result.push_str(&format!(" {:>5}", pid)); + } + DtraceDisplay::Session => { + let session_short = + d_out.session_id.chars().take(8).collect::(); + result.push_str(&format!(" {:>8}", session_short)); + } + DtraceDisplay::UpstairsId => { + let upstairs_short = + d_out.upstairs_id.chars().take(8).collect::(); + result.push_str(&format!(" {:>8}", upstairs_short)); + } + DtraceDisplay::State => { + result.push_str(&format!( + " {:>3} {:>3} {:>3}", + short_state(&d_out.ds_state[0]), + short_state(&d_out.ds_state[1]), + short_state(&d_out.ds_state[2]), + )); + } + DtraceDisplay::UpCount => { + result.push_str(&format!(" {:3}", d_out.up_count)); + } + DtraceDisplay::DsCount => { + result.push_str(&format!(" {:5}", d_out.ds_count)); + } + DtraceDisplay::IoCount | DtraceDisplay::IoSummary => { + result.push_str(&format!( + " {:5} {:5} {:5}", + d_out.ds_io_count.in_progress[ClientId::new(0)], + d_out.ds_io_count.in_progress[ClientId::new(1)], + d_out.ds_io_count.in_progress[ClientId::new(2)], + )); + result.push_str(&format!( + " {:5} {:5} {:5}", + d_out.ds_io_count.done[ClientId::new(0)], + d_out.ds_io_count.done[ClientId::new(1)], + d_out.ds_io_count.done[ClientId::new(2)], + )); + result.push_str(&format!( + " {:5} {:5} {:5}", + d_out.ds_io_count.skipped[ClientId::new(0)], + d_out.ds_io_count.skipped[ClientId::new(1)], + d_out.ds_io_count.skipped[ClientId::new(2)], + )); + if matches!(display_item, DtraceDisplay::IoCount) { + result.push_str(&format!( + " {:4} {:4} {:4}", + d_out.ds_io_count.error[ClientId::new(0)], + d_out.ds_io_count.error[ClientId::new(1)], + d_out.ds_io_count.error[ClientId::new(2)], + )); + } + } + DtraceDisplay::Reconcile => { + result.push_str(&format!( + " {:4} {:4} {:4}", + d_out.ds_reconciled, + d_out.ds_reconcile_needed, + d_out.ds_reconcile_aborted, + )); + } + DtraceDisplay::DsReconciled => { + result.push_str(&format!(" {:>4}", d_out.ds_reconciled)); + } + DtraceDisplay::DsReconcileNeeded => { + result.push_str(&format!(" {:>4}", d_out.ds_reconcile_needed)); + } + DtraceDisplay::LiveRepair => { + result.push_str(&format!( + " {:4} {:4} {:4}", + d_out.ds_live_repair_completed[0], + d_out.ds_live_repair_completed[1], + d_out.ds_live_repair_completed[2], + )); + result.push_str(&format!( + " {:4} {:4} {:4}", + d_out.ds_live_repair_aborted[0], + d_out.ds_live_repair_aborted[1], + d_out.ds_live_repair_aborted[2], + )); + } + DtraceDisplay::Connected => { + result.push_str(&format!( + " {:4} {:4} {:4}", + d_out.ds_connected[0], + d_out.ds_connected[1], + d_out.ds_connected[2], + )); + } + DtraceDisplay::Replaced => { + result.push_str(&format!( + " {:4} {:4} {:4}", + d_out.ds_replaced[0], + d_out.ds_replaced[1], + d_out.ds_replaced[2], + )); + } + DtraceDisplay::ExtentLiveRepair => { + result.push_str(&format!( + " {:4} {:4} {:4}", + d_out.ds_extents_repaired[0], + d_out.ds_extents_repaired[1], + d_out.ds_extents_repaired[2], + )); + result.push_str(&format!( + " {:4} {:4} {:4}", + d_out.ds_extents_confirmed[0], + d_out.ds_extents_confirmed[1], + d_out.ds_extents_confirmed[2], + )); + } + DtraceDisplay::ExtentLimit => { + result.push_str(&format!(" {:4}", d_out.ds_extent_limit)); + } + DtraceDisplay::NextJobId => { + result.push_str(&format!(" {:>10}", d_out.next_job_id)); + } + DtraceDisplay::JobDelta => { + if let Some(delta) = precomputed_delta { + result.push_str(&format!(" {:5}", delta)); + } else { + result.push_str(&format!(" {:>5}", "---")); + } + } + DtraceDisplay::DsDelay => { + result.push_str(&format!( + " {:5} {:5} {:5}", + d_out.ds_delay_us[0], + d_out.ds_delay_us[1], + d_out.ds_delay_us[2], + )); + } + DtraceDisplay::WriteBytesOut => { + result.push_str(&format!(" {:10}", d_out.write_bytes_out)); + } + DtraceDisplay::RoLrSkipped => { + result.push_str(&format!( + " {:4} {:4} {:4}", + d_out.ds_ro_lr_skipped[0], + d_out.ds_ro_lr_skipped[1], + d_out.ds_ro_lr_skipped[2], + )); + } + DtraceDisplay::DsIoInProgress => { + result.push_str(&format!( + " {:5} {:5} {:5}", + d_out.ds_io_count.in_progress[ClientId::new(0)], + d_out.ds_io_count.in_progress[ClientId::new(1)], + d_out.ds_io_count.in_progress[ClientId::new(2)], + )); + } + DtraceDisplay::DsIoDone => { + result.push_str(&format!( + " {:5} {:5} {:5}", + d_out.ds_io_count.done[ClientId::new(0)], + d_out.ds_io_count.done[ClientId::new(1)], + d_out.ds_io_count.done[ClientId::new(2)], + )); + } + DtraceDisplay::DsIoSkipped => { + result.push_str(&format!( + " {:5} {:5} {:5}", + d_out.ds_io_count.skipped[ClientId::new(0)], + d_out.ds_io_count.skipped[ClientId::new(1)], + d_out.ds_io_count.skipped[ClientId::new(2)], + )); + } + DtraceDisplay::DsIoError => { + result.push_str(&format!( + " {:4} {:4} {:4}", + d_out.ds_io_count.error[ClientId::new(0)], + d_out.ds_io_count.error[ClientId::new(1)], + d_out.ds_io_count.error[ClientId::new(2)], + )); + } + } + } + result +} + +/// Render a sparkline from delta history +/// Uses Unicode block characters to show trend: ▁▂▃▄▅▆▇█ +/// If global_max is provided, scales relative to that value for +/// cross-session comparison +fn render_sparkline( + history: &VecDeque, + width: usize, + global_max: u64, +) -> String { + if history.is_empty() || width == 0 { + return String::new(); + } + + // Unicode block characters from lowest to highest + const BLOCKS: [char; 8] = ['▁', '▂', '▃', '▄', '▅', '▆', '▇', '█']; + + // Take last 'width' samples (most recent) + let samples: Vec = history + .iter() + .rev() + .take(width) + .copied() + .collect::>() + .into_iter() + .rev() + .collect(); + + if samples.is_empty() { + return String::new(); + } + + // Use global max for scaling (minimum 1 to avoid division by zero) + let max = global_max.max(1); + + // Map each value to a block character + samples + .iter() + .map(|&val| { + if val == 0 { + BLOCKS[0] + } else { + let normalized = (val as f64 / max as f64 * 7.0) as usize; + BLOCKS[normalized.min(7)] + } + }) + .collect() +} + +/// Subprocess reader task - spawns dtrace command and reads JSON output +async fn subprocess_reader_task( + dtrace_cmd: String, + state: Arc>, + notify: Arc, +) -> Result<(), Box> { + // Parse command string into command and args + let parts: Vec<&str> = dtrace_cmd.split_whitespace().collect(); + if parts.is_empty() { + return Err("Empty dtrace command".into()); + } + + // Spawn the dtrace subprocess + let mut child = Command::new(parts[0]) + .args(&parts[1..]) + .stdout(std::process::Stdio::piped()) + .stderr(std::process::Stdio::null()) + .spawn()?; + + let stdout = child + .stdout + .take() + .ok_or("Failed to capture subprocess stdout")?; + + let reader = BufReader::new(stdout); + let mut lines = reader.lines(); + + // Read lines from subprocess stdout + while let Some(line) = lines.next_line().await? { + // Parse JSON + let wrapper: DtraceWrapper = match serde_json::from_str(&line) { + Ok(w) => w, + Err(_) => continue, + }; + + // Update state + let mut state_guard = state.write().await; + + let session_data = state_guard + .sessions + .entry(wrapper.status.session_id.clone()) + .or_insert_with(|| SessionData { + pid: wrapper.pid, + dtrace_info: wrapper.status.clone(), + last_job_id: 0, + last_updated: Instant::now(), + current_delta: None, + delta_history: VecDeque::new(), + }); + + // Calculate delta (jobs per second) + let current_job_id = wrapper.status.next_job_id.0; + let delta = if session_data.last_job_id != 0 { + let d = current_job_id.saturating_sub(session_data.last_job_id); + + // Add to history ring buffer + session_data.delta_history.push_back(d); + if session_data.delta_history.len() > MAX_DELTA_HISTORY { + session_data.delta_history.pop_front(); + } + + Some(d) + } else { + None + }; + + // Store current delta and update state + session_data.current_delta = delta; + session_data.last_job_id = current_job_id; + session_data.dtrace_info = wrapper.status; + session_data.last_updated = Instant::now(); + + drop(state_guard); + + // Notify display task + notify.notify_one(); + } + + // Wait for child to exit + let _ = child.wait().await; + + Ok(()) +} + +/// Render full-screen detail view for a selected session +fn render_detail_view( + session_data: &SessionData, + _terminal_size: (u16, u16), + terminal: &mut Terminal>, + global_min: Option, + global_max: Option, + normalize: bool, +) -> io::Result<()> { + // Calculate statistics + let history: Vec = + session_data.delta_history.iter().copied().collect(); + let session_max = history.iter().copied().max().unwrap_or(1); + let session_min = history.iter().copied().min().unwrap_or(0); + let avg = if !history.is_empty() { + history.iter().sum::() / history.len() as u64 + } else { + 0 + }; + let current = session_data.current_delta.unwrap_or(0); + + // Choose min/max based on normalize mode + let (display_min, display_max) = if normalize { + ( + global_min.unwrap_or(session_min), + global_max.unwrap_or(session_max), + ) + } else { + (session_min, session_max) + }; + + // Render using ratatui (terminal is reused, ratatui handles diffing) + terminal.draw(|f| { + let area = f.area(); + + // Split area: 1 line at top for session data, rest for canvas + let chunks = Layout::default() + .constraints([Constraint::Length(1), Constraint::Min(0)]) + .split(area); + + // Format the session data row + let display_fields = default_display_fields(); + let is_stale = session_data + .last_updated + .elapsed() + .as_secs() + > STALE_THRESHOLD_SECS; + let row_data = format_row( + session_data.pid, + &session_data.dtrace_info, + session_data.current_delta, + &display_fields, + is_stale, + ); + + // Render session data at top + let data_paragraph = Paragraph::new(row_data); + f.render_widget(data_paragraph, chunks[0]); + + // Create title + let session_short: String = + session_data.dtrace_info.session_id.chars().take(8).collect(); + let mode_str = if normalize { " [NORMALIZED]" } else { "" }; + let title = format!( + " Delta History - PID {} - Session {}{} ", + session_data.pid, session_short, mode_str + ); + + // Create canvas widget in bottom area (1 line shorter) + let canvas = Canvas::default() + .block( + Block::default() + .borders(Borders::ALL) + .title(title) + .title_bottom(format!( + " Samples: {} | Min: {} | Max: {} | Avg: {} | Current: {} ", + history.len(), + session_min, + session_max, + avg, + current + )) + .title_bottom( + " ['d': Back | 'n': Toggle normalize | 'q': Quit] ", + ), + ) + .x_bounds([0.0, history.len().max(1) as f64]) + .y_bounds([display_min as f64, display_max as f64]) + .paint(|ctx| { + // Draw Y-axis labels (at left edge of graph) + let y_range = display_max as f64 - display_min as f64; + let y_positions = [ + display_max, + display_min + (y_range * 0.75) as u64, + display_min + (y_range * 0.5) as u64, + display_min + (y_range * 0.25) as u64, + display_min, + ]; + + for y_val in &y_positions { + ctx.print( + 0.0, + *y_val as f64, + ratatui::text::Span::styled( + format!("{}", y_val), + ratatui::style::Style::default() + .fg(Color::Gray), + ), + ); + } + + // Draw the line graph + if history.len() > 1 { + for i in 0..history.len() - 1 { + let x1 = i as f64; + let y1 = history[i] as f64; + let x2 = (i + 1) as f64; + let y2 = history[i + 1] as f64; + + ctx.draw(&Line { + x1, + y1, + x2, + y2, + color: Color::Cyan, + }); + } + } + + // Draw points for each sample + for (i, &value) in history.iter().enumerate() { + ctx.draw(&Points { + coords: &[(i as f64, value as f64)], + color: Color::Yellow, + }); + } + }); + + f.render_widget(canvas, chunks[1]); + })?; + + Ok(()) +} + +/// Display task - renders the screen and handles keyboard input +async fn display_task( + state: Arc>, + notify: Arc, +) -> Result<(), Box> { + // Set up terminal + enable_raw_mode()?; + let mut stdout = io::stdout(); + execute!(stdout, EnterAlternateScreen)?; + + // Set up panic handler to restore terminal + let original_hook = std::panic::take_hook(); + std::panic::set_hook(Box::new(move |panic_info| { + let _ = execute!(io::stdout(), LeaveAlternateScreen); + let _ = disable_raw_mode(); + original_hook(panic_info); + })); + + let display_fields = default_display_fields(); + + // Track detail mode and persistent terminal for detail view + let mut was_in_detail_mode = false; + let mut detail_terminal: Option>> = + None; + + loop { + // Wait for notification or timeout + tokio::select! { + _ = notify.notified() => {}, + _ = tokio::time::sleep(Duration::from_millis(100)) => {}, + } + + // Get current time + let now = Instant::now(); + let system_time = std::time::SystemTime::now(); + let duration = system_time + .duration_since(std::time::UNIX_EPOCH) + .unwrap_or_default(); + + // Get terminal size + let terminal_size = crossterm::terminal::size()?; + + // Read state to check mode + let state_guard = state.read().await; + let in_detail_mode = state_guard.detail_mode; + let selected_index = state_guard.selected_index; + let normalize_detail = state_guard.normalize_detail; + + // If in detail mode, render detail view and skip table + if in_detail_mode { + // Create terminal on first entry to detail mode + if !was_in_detail_mode { + execute!(stdout, Clear(ClearType::All))?; + // Create a new stdout handle for the Terminal + let detail_stdout = io::stdout(); + let backend = CrosstermBackend::new(detail_stdout); + detail_terminal = Some(Terminal::new(backend)?); + } + + let mut sessions: Vec<&SessionData> = + state_guard.sessions.values().collect(); + sessions.sort_by_key(|s| (s.pid, &s.dtrace_info.session_id)); + + // Calculate global min/max across all sessions for normalization + let global_min = sessions + .iter() + .flat_map(|s| s.delta_history.iter()) + .copied() + .min(); + let global_max = sessions + .iter() + .flat_map(|s| s.delta_history.iter()) + .copied() + .max(); + + if let Some(selected_session) = sessions.get(selected_index) { + // Clone the session data so we can drop the lock + let session_clone = (*selected_session).clone(); + drop(state_guard); + + if let Some(terminal) = detail_terminal.as_mut() { + render_detail_view( + &session_clone, + terminal_size, + terminal, + global_min, + global_max, + normalize_detail, + )?; + } + } else { + drop(state_guard); + } + + was_in_detail_mode = true; + } else { + // Exiting detail mode - drop terminal and redraw table + if was_in_detail_mode { + detail_terminal = None; + execute!(stdout, Clear(ClearType::All))?; + } + was_in_detail_mode = false; + + // Move cursor to top-left + execute!(stdout, cursor::MoveTo(0, 0))?; + // Table mode - render normal view + drop(state_guard); + + // Display header (clear line first to remove artifacts) + execute!(stdout, cursor::MoveTo(0, 0))?; + write!( + stdout, + "cmon ctop - Unix timestamp: {}", + duration.as_secs() + )?; + execute!(stdout, Clear(ClearType::UntilNewLine))?; + write!(stdout, "\r\n")?; + execute!(stdout, Clear(ClearType::UntilNewLine))?; + write!(stdout, "\r\n")?; + + // Display column headers + write!(stdout, "{}", format_header(&display_fields))?; + execute!(stdout, Clear(ClearType::UntilNewLine))?; + write!(stdout, "\r\n")?; + + let (terminal_width, _) = terminal_size; + + // Read state and display sessions sorted by PID (then session_id) + let state_guard = state.read().await; + let mut sessions: Vec<&SessionData> = + state_guard.sessions.values().collect(); + sessions.sort_by_key(|s| (s.pid, &s.dtrace_info.session_id)); + + // Calculate global max across all sessions for consistent sparkline + // scaling + let global_max = sessions + .iter() + .flat_map(|s| s.delta_history.iter()) + .copied() + .max() + .unwrap_or(1); + + let selected_index = state_guard.selected_index; + + for (idx, session_data) in sessions.iter().enumerate() { + let is_stale = now.duration_since(session_data.last_updated) + > Duration::from_secs(STALE_THRESHOLD_SECS); + + // Add indicator: > for selected, * for stale, space otherwise + // Selection indicator (>) takes priority over stale indicator (*) + let indicator = if idx == selected_index { + ">" + } else if is_stale { + "*" + } else { + " " + }; + write!(stdout, "{}", indicator)?; + + let row = format_row( + session_data.pid, + &session_data.dtrace_info, + session_data.current_delta, + &display_fields, + is_stale, + ); + write!(stdout, "{}", row)?; + + // Render sparkline in remaining space + // Account for the indicator character (1 char) + let row_len = row.chars().count() + 1; + if terminal_width > row_len as u16 { + let sparkline_width = + (terminal_width as usize - row_len).saturating_sub(1); + if sparkline_width > 0 { + let sparkline = render_sparkline( + &session_data.delta_history, + sparkline_width, + global_max, + ); + write!(stdout, " {}", sparkline)?; + } + } + + execute!(stdout, Clear(ClearType::UntilNewLine))?; + write!(stdout, "\r\n")?; + } + drop(state_guard); + + // Display footer + execute!(stdout, Clear(ClearType::UntilNewLine))?; + write!(stdout, "\r\n")?; + write!( + stdout, + "[↑↓: Select | 'd': Details | 'q': Quit] > = selected, * = stale ({}s)", + STALE_THRESHOLD_SECS + )?; + execute!(stdout, Clear(ClearType::UntilNewLine))?; + write!(stdout, "\r\n")?; + + // Clear from cursor to end of screen (removes any leftover lines) + execute!(stdout, Clear(ClearType::FromCursorDown))?; + + stdout.flush()?; + } // End of table mode rendering + + // Check for keyboard input (non-blocking) + if event::poll(Duration::from_millis(0))? + && let Event::Key(key_event) = event::read()? + { + let mut state_guard = state.write().await; + let num_sessions = state_guard.sessions.len(); + + match key_event { + KeyEvent { + code: KeyCode::Char('q'), + modifiers: KeyModifiers::NONE, + .. + } => break, + KeyEvent { + code: KeyCode::Char('c'), + modifiers: KeyModifiers::CONTROL, + .. + } => break, + KeyEvent { + code: KeyCode::Char('d'), + modifiers: KeyModifiers::NONE, + .. + } => { + // Toggle detail mode + state_guard.detail_mode = !state_guard.detail_mode; + } + KeyEvent { + code: KeyCode::Char('n'), + modifiers: KeyModifiers::NONE, + .. + } => { + // Toggle normalize mode (only affects detail view) + state_guard.normalize_detail = + !state_guard.normalize_detail; + } + KeyEvent { + code: KeyCode::Up, + modifiers: KeyModifiers::NONE, + .. + } => { + // Move selection up (only in table mode) + if !state_guard.detail_mode && num_sessions > 0 { + state_guard.selected_index = + state_guard.selected_index.saturating_sub(1); + } + } + KeyEvent { + code: KeyCode::Down, + modifiers: KeyModifiers::NONE, + .. + } => { + // Move selection down (only in table mode) + if !state_guard.detail_mode && num_sessions > 0 { + state_guard.selected_index = + (state_guard.selected_index + 1) + .min(num_sessions.saturating_sub(1)); + } + } + KeyEvent { + code: KeyCode::Esc, + modifiers: KeyModifiers::NONE, + .. + } => { + // Exit detail mode + state_guard.detail_mode = false; + } + _ => {} + } + drop(state_guard); + } + } + + // Clean up terminal + execute!(stdout, LeaveAlternateScreen)?; + disable_raw_mode()?; + + Ok(()) +} + +/// Main entry point for ctop +pub async fn ctop_loop( + dtrace_cmd: String, +) -> Result<(), Box> { + let state = Arc::new(RwLock::new(CtopState::default())); + let notify = Arc::new(Notify::new()); + + let state_reader = Arc::clone(&state); + let notify_reader = Arc::clone(¬ify); + + // Spawn subprocess reader task + let reader_handle = tokio::spawn(async move { + if let Err(e) = + subprocess_reader_task(dtrace_cmd, state_reader, notify_reader) + .await + { + eprintln!("Subprocess reader error: {}", e); + } + }); + + // Run display task (blocks until user quits) + let display_result = display_task(state, notify).await; + + // Wait for reader task to finish (it should exit quickly) + let _ = + tokio::time::timeout(Duration::from_millis(100), reader_handle).await; + + display_result +} + +/// Main entry point +#[tokio::main] +async fn main() { + let args = Args::parse(); + + if let Err(e) = ctop_loop(args.dtrace_cmd).await { + eprintln!("Error running ctop: {}", e); + std::process::exit(1); + } +} From cb2378e0848a6d02c2d4b2bab20bb0527e12ed51 Mon Sep 17 00:00:00 2001 From: Alan Hanson Date: Sun, 26 Oct 2025 11:57:09 -0700 Subject: [PATCH 16/36] more breakout of ctop --- cmon-common/Cargo.toml | 19 ++ cmon-common/src/lib.rs | 225 +++++++++++++++++++++++ ctop/src/main.rs | 407 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 651 insertions(+) create mode 100644 cmon-common/Cargo.toml create mode 100644 cmon-common/src/lib.rs diff --git a/cmon-common/Cargo.toml b/cmon-common/Cargo.toml new file mode 100644 index 000000000..cb1de3826 --- /dev/null +++ b/cmon-common/Cargo.toml @@ -0,0 +1,19 @@ +[package] +name = "cmon-common" +version = "0.1.0" +license = "MPL-2.0" +edition = "2024" + +[dependencies] +crucible.workspace = true +serde.workspace = true +strum.workspace = true +strum_macros.workspace = true +crucible-workspace-hack.workspace = true + +# Optional dependency for clap integration +clap = { workspace = true, optional = true } + +[features] +# Enable clap ValueEnum derive for DtraceDisplay +clap = ["dep:clap"] diff --git a/cmon-common/src/lib.rs b/cmon-common/src/lib.rs new file mode 100644 index 000000000..65f588194 --- /dev/null +++ b/cmon-common/src/lib.rs @@ -0,0 +1,225 @@ +// Copyright 2025 Oxide Computer Company + +//! Common types and utilities shared between cmon and ctop + +use crucible::DtraceInfo; +use serde::Deserialize; +use std::fmt; +use strum_macros::EnumIter; + +/// Wrapper for DTrace output with PID +#[derive(Debug, Deserialize)] +pub struct DtraceWrapper { + pub pid: u32, + pub status: DtraceInfo, +} + +/// The possible fields we will display when receiving DTrace output. +#[derive(Debug, Copy, Clone, PartialEq, Eq, EnumIter)] +#[cfg_attr(feature = "clap", derive(clap::ValueEnum))] +pub enum DtraceDisplay { + Pid, + Session, + UpstairsId, + State, + IoCount, + IoSummary, + UpCount, + DsCount, + Reconcile, + DsReconciled, + DsReconcileNeeded, + LiveRepair, + Connected, + Replaced, + ExtentLiveRepair, + ExtentLimit, + NextJobId, + JobDelta, + DsDelay, + WriteBytesOut, + RoLrSkipped, + DsIoInProgress, + DsIoDone, + DsIoSkipped, + DsIoError, +} + +impl fmt::Display for DtraceDisplay { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + match self { + DtraceDisplay::Pid => write!(f, "pid"), + DtraceDisplay::Session => write!(f, "session"), + DtraceDisplay::UpstairsId => write!(f, "upstairs_id"), + DtraceDisplay::State => write!(f, "state"), + DtraceDisplay::IoCount => write!(f, "io_count"), + DtraceDisplay::IoSummary => write!(f, "io_summary"), + DtraceDisplay::UpCount => write!(f, "up_count"), + DtraceDisplay::DsCount => write!(f, "ds_count"), + DtraceDisplay::Reconcile => write!(f, "reconcile"), + DtraceDisplay::DsReconciled => write!(f, "ds_reconciled"), + DtraceDisplay::DsReconcileNeeded => { + write!(f, "ds_reconcile_needed") + } + DtraceDisplay::LiveRepair => write!(f, "live_repair"), + DtraceDisplay::Connected => write!(f, "connected"), + DtraceDisplay::Replaced => write!(f, "replaced"), + DtraceDisplay::ExtentLiveRepair => write!(f, "extent_live_repair"), + DtraceDisplay::ExtentLimit => write!(f, "extent_under_repair"), + DtraceDisplay::NextJobId => write!(f, "next_job_id"), + DtraceDisplay::JobDelta => write!(f, "job_delta"), + DtraceDisplay::DsDelay => write!(f, "ds_delay"), + DtraceDisplay::WriteBytesOut => write!(f, "write_bytes_out"), + DtraceDisplay::RoLrSkipped => write!(f, "ro_lr_skipped"), + DtraceDisplay::DsIoInProgress => write!(f, "ds_io_in_progress"), + DtraceDisplay::DsIoDone => write!(f, "ds_io_done"), + DtraceDisplay::DsIoSkipped => write!(f, "ds_io_skipped"), + DtraceDisplay::DsIoError => write!(f, "ds_io_error"), + } + } +} + +/// Translate DsState string into a three letter abbreviation +pub fn short_state(dss: &str) -> String { + match dss { + "Active" => "ACT".to_string(), + "WaitQuorum" => "WQ".to_string(), + "Reconcile" => "REC".to_string(), + "LiveRepairReady" => "LRR".to_string(), + "New" => "NEW".to_string(), + "Faulted" => "FLT".to_string(), + "Offline" => "OFL".to_string(), + "Replaced" => "RPL".to_string(), + "LiveRepair" => "LR".to_string(), + "Replacing" => "RPC".to_string(), + "Disabled" => "DIS".to_string(), + "Deactivated" => "DAV".to_string(), + "NegotiationFailed" => "NF".to_string(), + "Fault" => "FLT".to_string(), + x => x.to_string(), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use strum::IntoEnumIterator; + + #[test] + fn test_short_state_all_known_states() { + // Test all known downstairs states + assert_eq!(short_state("Active"), "ACT"); + assert_eq!(short_state("WaitQuorum"), "WQ"); + assert_eq!(short_state("Reconcile"), "REC"); + assert_eq!(short_state("LiveRepairReady"), "LRR"); + assert_eq!(short_state("New"), "NEW"); + assert_eq!(short_state("Faulted"), "FLT"); + assert_eq!(short_state("Offline"), "OFL"); + assert_eq!(short_state("Replaced"), "RPL"); + assert_eq!(short_state("LiveRepair"), "LR"); + assert_eq!(short_state("Replacing"), "RPC"); + assert_eq!(short_state("Disabled"), "DIS"); + assert_eq!(short_state("Deactivated"), "DAV"); + assert_eq!(short_state("NegotiationFailed"), "NF"); + assert_eq!(short_state("Fault"), "FLT"); + } + + #[test] + fn test_short_state_unknown_state() { + // Unknown states should pass through unchanged + assert_eq!(short_state("UnknownState"), "UnknownState"); + assert_eq!(short_state(""), ""); + assert_eq!(short_state("XYZ"), "XYZ"); + } + + #[test] + fn test_short_state_length() { + // All known states should produce 2-3 character abbreviations + let known_states = vec![ + "Active", + "WaitQuorum", + "Reconcile", + "LiveRepairReady", + "New", + "Faulted", + "Offline", + "Replaced", + "LiveRepair", + "Replacing", + "Disabled", + "Deactivated", + "NegotiationFailed", + "Fault", + ]; + + for state in known_states { + let short = short_state(state); + assert!( + short.len() <= 3, + "State {} abbreviation '{}' is too long", + state, + short + ); + } + } + + #[test] + fn test_dtrace_display_to_string() { + // Test that Display trait works for all variants + assert_eq!(DtraceDisplay::Pid.to_string(), "pid"); + assert_eq!(DtraceDisplay::Session.to_string(), "session"); + assert_eq!(DtraceDisplay::UpstairsId.to_string(), "upstairs_id"); + assert_eq!(DtraceDisplay::State.to_string(), "state"); + assert_eq!(DtraceDisplay::IoCount.to_string(), "io_count"); + assert_eq!(DtraceDisplay::IoSummary.to_string(), "io_summary"); + assert_eq!(DtraceDisplay::NextJobId.to_string(), "next_job_id"); + assert_eq!(DtraceDisplay::JobDelta.to_string(), "job_delta"); + assert_eq!( + DtraceDisplay::ExtentLimit.to_string(), + "extent_under_repair" + ); + } + + #[test] + fn test_dtrace_display_all_variants_have_display() { + // Ensure all enum variants can be displayed without panicking + for variant in DtraceDisplay::iter() { + let display = variant.to_string(); + // Should produce a non-empty string + assert!( + !display.is_empty(), + "Variant {:?} has empty display", + variant + ); + // Should be lowercase with underscores + assert!( + display.chars().all(|c| c.is_lowercase() || c == '_'), + "Variant {:?} display '{}' should be lowercase with underscores", + variant, + display + ); + } + } + + #[test] + fn test_dtrace_display_copy_clone() { + // Verify Copy and Clone work correctly + let display = DtraceDisplay::Pid; + let copied = display; + let cloned = display.clone(); + + assert_eq!(format!("{:?}", display), format!("{:?}", copied)); + assert_eq!(format!("{:?}", display), format!("{:?}", cloned)); + } + + #[test] + fn test_dtrace_display_enum_count() { + // Verify we have the expected number of variants + let count = DtraceDisplay::iter().count(); + assert_eq!( + count, 25, + "Expected 25 DtraceDisplay variants, found {}", + count + ); + } +} diff --git a/ctop/src/main.rs b/ctop/src/main.rs index 82b118579..fd5b4f689 100644 --- a/ctop/src/main.rs +++ b/ctop/src/main.rs @@ -1011,3 +1011,410 @@ async fn main() { std::process::exit(1); } } + +#[cfg(test)] +mod tests { + //! Unit tests for ctop + //! + //! # Test Coverage Overview + //! + //! These unit tests validate the pure functions and data structures that + //! power ctop's display logic, focusing on areas that don't require async + //! runtime or terminal mocking. + //! + //! ## Current Coverage + //! + //! ### Display Formatting (format_header, format_row) + //! - Column header generation for all DtraceDisplay field types + //! - Multi-column fields (State shows DS0/DS1/DS2) + //! - IO summary fields with multiple columns per downstairs + //! - Empty field handling + //! + //! ### Sparkline Rendering (render_sparkline) + //! - Empty history handling + //! - Zero-width rendering + //! - Single and multiple value rendering + //! - Width limiting (showing only recent samples) + //! - Normalization using global max value + //! - Unicode block character validation (▁▂▃▄▅▆▇█) + //! - Ascending/descending trend visualization + //! + //! ### State Management + //! - Default CtopState initialization + //! - Delta history ring buffer behavior (MAX_DELTA_HISTORY) + //! - Constants validation (STALE_THRESHOLD_SECS, MAX_DELTA_HISTORY) + //! + //! ## What's NOT Tested (See integration_test.rs for full list) + //! + //! - Async tasks (subprocess_reader_task, display_task) + //! - Terminal rendering and keyboard input + //! - Session lifecycle and state updates + //! - DTrace subprocess integration + //! - Multi-session coordination + //! + //! ## Testing Strategy + //! + //! These tests focus on **testable units** - pure functions with no I/O. + //! For components requiring mocking (terminal, subprocess, async runtime), + //! see the testing improvement proposals in integration_test.rs. + //! + //! ## Running Tests + //! + //! ```bash + //! # Run all ctop unit tests + //! cargo test -p ctop --lib + //! + //! # Run specific test + //! cargo test -p ctop --lib test_render_sparkline_normalization + //! ``` + + use super::*; + + // ============================================================================ + // Display Field Configuration Tests + // ============================================================================ + // + // These tests verify the default display configuration that users see when + // they first run ctop. The fields should provide a good overview of upstairs + // state without overwhelming the display. + + #[test] + fn test_default_display_fields() { + let fields = default_display_fields(); + + // Verify we have the expected default fields + assert_eq!(fields.len(), 8); + assert_eq!(fields[0], DtraceDisplay::Pid); + assert_eq!(fields[1], DtraceDisplay::Session); + assert_eq!(fields[2], DtraceDisplay::State); + assert_eq!(fields[3], DtraceDisplay::NextJobId); + assert_eq!(fields[4], DtraceDisplay::JobDelta); + assert_eq!(fields[5], DtraceDisplay::ExtentLimit); + assert_eq!(fields[6], DtraceDisplay::DsReconciled); + assert_eq!(fields[7], DtraceDisplay::DsReconcileNeeded); + } + + // ============================================================================ + // Header Formatting Tests + // ============================================================================ + // + // These tests verify that format_header() generates correct column headers + // for various field types. Some fields (like State, IoSummary) expand into + // multiple columns representing the three downstairs replicas. + + #[test] + fn test_format_header_basic_fields() { + let fields = vec![DtraceDisplay::Pid, DtraceDisplay::Session]; + let header = format_header(&fields); + + // Check that header contains expected column names + assert!(header.contains("PID")); + assert!(header.contains("SESSION")); + } + + #[test] + fn test_format_header_state_field() { + let fields = vec![DtraceDisplay::State]; + let header = format_header(&fields); + + // State field should show three downstairs columns + assert!(header.contains("DS0")); + assert!(header.contains("DS1")); + assert!(header.contains("DS2")); + } + + #[test] + fn test_format_header_io_fields() { + let fields = vec![DtraceDisplay::IoSummary]; + let header = format_header(&fields); + + // Should show in_progress, done, and skipped for each DS + assert!(header.contains("IP0")); + assert!(header.contains("IP1")); + assert!(header.contains("IP2")); + assert!(header.contains("D0")); + assert!(header.contains("D1")); + assert!(header.contains("D2")); + assert!(header.contains("S0")); + assert!(header.contains("S1")); + assert!(header.contains("S2")); + } + + #[test] + fn test_format_header_empty_fields() { + let fields = vec![]; + let header = format_header(&fields); + + // Empty fields should produce empty header + assert_eq!(header, ""); + } + + // ============================================================================ + // Sparkline Rendering Tests + // ============================================================================ + // + // Sparklines provide a compact visualization of job delta trends over time. + // They use Unicode block characters (▁▂▃▄▅▆▇█) to represent values. + // + // Key behaviors tested: + // - Empty history and edge cases (zero width, single values) + // - Width limiting (showing only the most recent N samples) + // - Normalization across sessions (global_max parameter) + // - Unicode character validity + // + // TODO: Consider property-based testing (proptest) to generate random + // histories and verify properties like: + // - Sparkline length <= requested width + // - All characters are valid block characters + // - Normalized values respect global_max + + #[test] + fn test_render_sparkline_empty() { + let history = VecDeque::new(); + let sparkline = render_sparkline(&history, 10, 100); + + assert_eq!(sparkline, ""); + } + + #[test] + fn test_render_sparkline_zero_width() { + let mut history = VecDeque::new(); + history.push_back(10); + history.push_back(20); + + let sparkline = render_sparkline(&history, 0, 100); + assert_eq!(sparkline, ""); + } + + #[test] + fn test_render_sparkline_single_value() { + let mut history = VecDeque::new(); + history.push_back(50); + + let sparkline = render_sparkline(&history, 10, 100); + + // Should have one character + assert_eq!(sparkline.chars().count(), 1); + } + + #[test] + fn test_render_sparkline_ascending_values() { + let mut history = VecDeque::new(); + for i in 0..10 { + history.push_back(i * 10); + } + + let sparkline = render_sparkline(&history, 10, 100); + + // Should have 10 characters (one per value) + assert_eq!(sparkline.chars().count(), 10); + + // First character should be lower than last (ascending trend) + let chars: Vec = sparkline.chars().collect(); + assert!(chars[0] < chars[9]); + } + + #[test] + fn test_render_sparkline_width_limit() { + let mut history = VecDeque::new(); + for i in 0..100 { + history.push_back(i); + } + + // Request only last 5 samples + let sparkline = render_sparkline(&history, 5, 100); + + // Should only show 5 characters + assert_eq!(sparkline.chars().count(), 5); + } + + #[test] + fn test_render_sparkline_max_value() { + let mut history = VecDeque::new(); + history.push_back(0); + history.push_back(100); + + let sparkline = render_sparkline(&history, 10, 100); + + // Should use valid unicode block characters + for c in sparkline.chars() { + assert!( + ['▁', '▂', '▃', '▄', '▅', '▆', '▇', '█'].contains(&c), + "Invalid sparkline character: {}", + c + ); + } + } + + #[test] + fn test_render_sparkline_normalization() { + // This test verifies a critical feature: sparklines are normalized using + // a global_max value computed across ALL sessions. This allows users to + // visually compare activity levels between different upstairs instances. + // + // Without global normalization, each session would auto-scale to its own + // range, making cross-session comparison meaningless. + + let mut history = VecDeque::new(); + history.push_back(50); + history.push_back(100); + + // Test with global max = 200 (should scale differently than 100) + let sparkline1 = render_sparkline(&history, 10, 200); + let sparkline2 = render_sparkline(&history, 10, 100); + + // With higher global max, the values should appear relatively lower + let chars1: Vec = sparkline1.chars().collect(); + let chars2: Vec = sparkline2.chars().collect(); + + // Second sparkline should use higher blocks (value 100 is max in range 0-100, + // but only midpoint in range 0-200) + assert!( + chars1[1] < chars2[1], + "Expected normalization to affect block height" + ); + } + + // ============================================================================ + // State Management Tests + // ============================================================================ + // + // These tests verify the data structures that track session state: + // - CtopState: Overall application state (sessions map, selection, mode) + // - SessionData: Per-session data including delta history ring buffer + // - Constants: STALE_THRESHOLD_SECS, MAX_DELTA_HISTORY + // + // Note: These tests only validate initialization and constants. Full + // state lifecycle testing (session updates, transitions) requires async + // mocking infrastructure. + + #[test] + fn test_ctop_state_default() { + let state = CtopState::default(); + + assert_eq!(state.sessions.len(), 0); + assert_eq!(state.selected_index, 0); + assert!(!state.detail_mode); + assert!(!state.normalize_detail); + } + + #[test] + fn test_session_data_delta_history_max_size() { + // Delta history uses a ring buffer (VecDeque) to maintain a sliding window + // of recent job delta values for sparkline rendering. This test verifies + // the ring buffer behavior: old values are evicted when capacity is reached. + // + // In production, subprocess_reader_task maintains this ring buffer by: + // 1. Computing delta = new_job_id - old_job_id + // 2. Pushing delta to back of deque + // 3. Popping from front if len > MAX_DELTA_HISTORY + + let mut delta_history = VecDeque::new(); + + // Simulate adding more than MAX_DELTA_HISTORY items + for i in 0..(MAX_DELTA_HISTORY + 10) { + delta_history.push_back(i as u64); + if delta_history.len() > MAX_DELTA_HISTORY { + delta_history.pop_front(); + } + } + + // Should never exceed MAX_DELTA_HISTORY + assert_eq!(delta_history.len(), MAX_DELTA_HISTORY); + + // Should contain the most recent items (oldest 10 were evicted) + assert_eq!(*delta_history.front().unwrap(), 10); // First item should be item 10 + assert_eq!( + *delta_history.back().unwrap(), + (MAX_DELTA_HISTORY + 9) as u64 + ); // Last item is most recent + } + + #[test] + fn test_stale_threshold_constant() { + // Verify STALE_THRESHOLD_SECS is reasonable + assert!( + STALE_THRESHOLD_SECS > 0, + "Stale threshold should be positive" + ); + assert!( + STALE_THRESHOLD_SECS <= 60, + "Stale threshold should be under a minute" + ); + } + + #[test] + fn test_max_delta_history_constant() { + // Verify MAX_DELTA_HISTORY is reasonable for sparklines + assert!( + MAX_DELTA_HISTORY >= 50, + "Should store enough history for reasonable sparklines" + ); + assert!( + MAX_DELTA_HISTORY <= 1000, + "History shouldn't be excessively large" + ); + } + + // ============================================================================ + // Future Testing Opportunities + // ============================================================================ + // + // The following areas could benefit from additional testing infrastructure: + // + // ## 1. Async Task Testing (mockall crate) + // + // Mock the subprocess Command to test subprocess_reader_task: + // - Emit controlled JSON lines + // - Verify state updates (sessions map, delta calculations) + // - Test error handling (invalid JSON, subprocess crashes) + // - Test session expiration and cleanup + // + // ## 2. Terminal UI Testing (ratatui::TestBackend) + // + // Capture terminal output to verify: + // - Header and row formatting in actual terminal context + // - Stale session indicators (*) appear correctly + // - Selection indicators (>) highlight correct row + // - Sparklines render in available terminal width + // - Detail mode layout and graphing + // + // ## 3. Keyboard Input Testing + // + // Mock crossterm events to test: + // - Up/Down arrow navigation (selected_index changes) + // - 'd' toggles detail_mode + // - 'n' toggles normalize_detail + // - 'q' and Ctrl-C exit cleanly + // - Esc exits detail mode + // + // ## 4. Property-Based Testing (proptest) + // + // Generate random inputs to verify invariants: + // - Sparkline width never exceeds requested width + // - format_header produces valid UTF-8 + // - Delta history never exceeds MAX_DELTA_HISTORY + // - selected_index never exceeds sessions.len() + // + // ## 5. Snapshot Testing (insta) + // + // Capture and compare terminal output snapshots: + // - Table view with various session counts + // - Detail view with different data patterns + // - Regression testing for layout changes + // + // ## 6. Performance Testing (criterion) + // + // Benchmark critical paths: + // - render_sparkline with full history + // - format_row with many display fields + // - Session sorting with 100+ sessions + // + // ## 7. End-to-End Testing + // + // Run ctop against real DTrace output: + // - Capture actual crucible DTrace JSON + // - Replay captured output to ctop + // - Verify no parsing errors + // - Compare against expected session data +} From 9c94ed4d344c89beb08253f6463473dd2b9c2e75 Mon Sep 17 00:00:00 2001 From: Alan Hanson Date: Sun, 26 Oct 2025 13:44:01 -0700 Subject: [PATCH 17/36] Add tests, include dtrace script Add the missing test directory Include the dtrace default script in the binary. Help polish. --- ctop/tests/integration_test.rs | 187 +++++++++++++++++++++++++++++++++ 1 file changed, 187 insertions(+) create mode 100644 ctop/tests/integration_test.rs diff --git a/ctop/tests/integration_test.rs b/ctop/tests/integration_test.rs new file mode 100644 index 000000000..6ad02eb9b --- /dev/null +++ b/ctop/tests/integration_test.rs @@ -0,0 +1,187 @@ +// Copyright 2025 Oxide Computer Company + +//! Integration tests for ctop +//! +//! # Test Coverage Overview +//! +//! These integration tests focus on validating the data flow from DTrace +//! output through ctop's JSON parsing pipeline. They ensure that ctop can +//! correctly handle the JSON format produced by the DTrace scripts used to +//! monitor Crucible upstairs processes. +//! +//! ## Current Coverage +//! +//! - **JSON Parsing**: Valid, invalid, and incomplete DTrace JSON structures +//! - **Data Format Validation**: Ensures expected fields are present and +//! correctly typed +//! +//! ## What's NOT Tested (Yet) +//! +//! The following areas require more sophisticated testing infrastructure: +//! +//! ### Async Components +//! - Subprocess spawning and management (tokio Command) +//! - State updates from subprocess_reader_task +//! - Async communication between reader and display tasks +//! - Tokio Notify coordination +//! +//! ### Terminal UI +//! - Keyboard input handling (arrow keys, 'd', 'q', etc.) +//! - Screen rendering and layout +//! - Detail mode transitions +//! - Sparkline rendering in actual terminals +//! - Alternate screen buffer management +//! +//! ### State Management +//! - Session creation and updates +//! - Session selection (up/down arrows) +//! - Detail mode toggling +//! - Stale session detection (5s threshold) +//! - Delta history ring buffer behavior +//! +//! ### Edge Cases +//! - Multiple sessions with same PID +//! - Rapidly changing session data +//! - Very long session IDs or upstairs IDs +//! - Terminal resize during operation +//! - Empty or missing DTrace output +//! +//! ## Proposed Testing Improvements +//! +//! Consider these testing strategies for more comprehensive coverage: +//! +//! ### 1. Subprocess Mocking +//! Use a mock subprocess that emits controlled DTrace JSON output to test +//! the full data pipeline without requiring actual DTrace. +//! +//! ### 2. Terminal Mocking +//! Libraries like `ratatui::TestBackend` can capture terminal output for +//! validation without requiring an actual terminal. +//! +//! ### 3. Property-Based Testing (proptest) +//! Generate random but valid DTrace JSON structures to test parsing +//! robustness and find edge cases. +//! +//! ### 4. Snapshot Testing (insta) +//! Capture and verify terminal output snapshots for regression testing +//! of UI layout and formatting. +//! +//! ### 5. End-to-End Tests +//! Run ctop against real DTrace output (captured from actual crucible +//! processes) to validate production behavior. +//! +//! ## Running Tests +//! +//! ```bash +//! # Run all ctop tests (unit + integration) +//! cargo test -p ctop +//! +//! # Run only integration tests +//! cargo test -p ctop --test integration_test +//! +//! # Run specific integration test +//! cargo test -p ctop --test integration_test test_parse_dtrace_json_format +//! ``` + +use serde_json; + +/// Test that we can parse valid DTrace JSON output +#[test] +fn test_parse_dtrace_json_format() { + // This is the JSON format that dtrace scripts output + // and that ctop needs to parse + let sample_json = r#"{ + "pid": 12345, + "status": { + "upstairs_id": "test-uuid-123", + "session_id": "session-456", + "up_count": 10, + "up_counters": { + "apply": 100, + "action_downstairs": 50, + "action_guest": 30, + "action_deferred_block": 0, + "action_deferred_message": 0, + "action_flush_check": 10, + "action_stat_check": 5, + "action_control_check": 3, + "action_noop": 2 + }, + "next_job_id": 1000, + "ds_count": 15, + "write_bytes_out": 10240, + "ds_state": ["Active", "Active", "Active"], + "ds_io_count": { + "in_progress": [5, 5, 5], + "done": [100, 100, 100], + "skipped": [0, 0, 0], + "error": [0, 0, 0] + }, + "ds_reconciled": 0, + "ds_reconcile_needed": 0, + "ds_reconcile_aborted": 0, + "ds_live_repair_completed": [0, 0, 0], + "ds_live_repair_aborted": [0, 0, 0], + "ds_connected": [1, 1, 1], + "ds_replaced": [0, 0, 0], + "ds_extents_repaired": [0, 0, 0], + "ds_extents_confirmed": [100, 100, 100], + "ds_extent_limit": 0, + "ds_delay_us": [0, 0, 0], + "ds_ro_lr_skipped": [0, 0, 0] + } + }"#; + + // Attempt to parse - this validates the JSON structure matches what + // cmon_common::DtraceWrapper expects + let result: Result = + serde_json::from_str(sample_json); + assert!( + result.is_ok(), + "Sample dtrace JSON should parse successfully" + ); + + let parsed = result.unwrap(); + assert_eq!(parsed["pid"], 12345); + assert_eq!(parsed["status"]["session_id"], "session-456"); + assert_eq!(parsed["status"]["ds_state"][0], "Active"); +} + +/// Test handling of invalid JSON +#[test] +fn test_parse_invalid_dtrace_json() { + let invalid_json = r#"{ "pid": 12345, "status": invalid }"#; + + let result: Result = + serde_json::from_str(invalid_json); + assert!(result.is_err(), "Invalid JSON should fail to parse"); +} + +/// Test handling of partial/incomplete JSON +#[test] +fn test_parse_incomplete_dtrace_json() { + // Missing required fields + let incomplete_json = r#"{ "pid": 12345 }"#; + + let result: Result = + serde_json::from_str(incomplete_json); + assert!(result.is_ok(), "Partial JSON should parse as JSON"); + + let parsed = result.unwrap(); + assert_eq!(parsed["pid"], 12345); + assert!(parsed["status"].is_null()); +} + +// Note: Additional integration tests would ideally cover: +// - Mock dtrace subprocess and verify parsing pipeline +// - Test keyboard input handling (requires terminal mock) +// - Test state transitions (detail mode, session selection) +// - Test multi-session handling +// - Test stale session detection +// - Performance tests with many sessions +// +// These are challenging without a full terminal mocking framework +// and subprocess mocking capabilities. Consider using: +// - mockall crate for mocking +// - proptest for property-based testing +// - criterion for benchmarking From a9f4401f63f9e90cc064c9ad124abfaf65d01546 Mon Sep 17 00:00:00 2001 From: Alan Hanson Date: Sun, 26 Oct 2025 14:00:28 -0700 Subject: [PATCH 18/36] remove ctop from cmon --- cmon/src/ctop.rs | 986 ----------------------------------------------- cmon/src/main.rs | 16 - ctop/src/main.rs | 16 +- 3 files changed, 12 insertions(+), 1006 deletions(-) delete mode 100644 cmon/src/ctop.rs diff --git a/cmon/src/ctop.rs b/cmon/src/ctop.rs deleted file mode 100644 index 161be4ae8..000000000 --- a/cmon/src/ctop.rs +++ /dev/null @@ -1,986 +0,0 @@ -// Copyright 2025 Oxide Computer Company - -use cmon_common::{DtraceDisplay, DtraceWrapper, short_state}; -use crossterm::{ - cursor, - event::{self, Event, KeyCode, KeyEvent, KeyModifiers}, - execute, - terminal::{ - Clear, ClearType, EnterAlternateScreen, LeaveAlternateScreen, - disable_raw_mode, enable_raw_mode, - }, -}; -use crucible::DtraceInfo; -use crucible_protocol::ClientId; -use ratatui::{ - Terminal, - backend::CrosstermBackend, - layout::{Constraint, Layout}, - style::Color, - widgets::canvas::{Canvas, Line, Points}, - widgets::{Block, Borders, Paragraph}, -}; -use std::collections::{HashMap, VecDeque}; -use std::io::{self, Write}; -use std::sync::Arc; -use std::time::{Duration, Instant}; -use tokio::io::{AsyncBufReadExt, BufReader}; -use tokio::process::Command; -use tokio::sync::{Notify, RwLock}; - -const STALE_THRESHOLD_SECS: u64 = 10; -const MAX_DELTA_HISTORY: usize = 100; - -/// Data for a single session -#[derive(Debug, Clone)] -struct SessionData { - pid: u32, - dtrace_info: DtraceInfo, - last_job_id: u64, - last_updated: Instant, - current_delta: Option, - delta_history: VecDeque, -} - -/// Shared state between stdin reader and display tasks -#[derive(Debug, Default)] -struct CtopState { - sessions: HashMap, - selected_index: usize, - detail_mode: bool, - normalize_detail: bool, // Use global min/max for detail view scaling -} - -/// Default display fields (same as dtrace command defaults) -fn default_display_fields() -> Vec { - vec![ - DtraceDisplay::Pid, - DtraceDisplay::Session, - DtraceDisplay::State, - DtraceDisplay::NextJobId, - DtraceDisplay::JobDelta, - DtraceDisplay::ExtentLimit, - DtraceDisplay::DsReconciled, - DtraceDisplay::DsReconcileNeeded, - ] -} - -/// Format header line for the given display fields -fn format_header(dd: &[DtraceDisplay]) -> String { - let mut result = String::new(); - for display_item in dd.iter() { - match display_item { - DtraceDisplay::Pid => { - result.push_str(&format!(" {:>5}", "PID")); - } - DtraceDisplay::Session => { - result.push_str(&format!(" {:>8}", "SESSION")); - } - DtraceDisplay::UpstairsId => { - result.push_str(&format!(" {:>8}", "UPSTAIRS")); - } - DtraceDisplay::State => { - result.push_str(&format!( - " {:>3} {:>3} {:>3}", - "DS0", "DS1", "DS2" - )); - } - DtraceDisplay::UpCount => { - result.push_str(&format!(" {:>3}", "UPW")); - } - DtraceDisplay::DsCount => { - result.push_str(&format!(" {:>5}", "DSW")); - } - DtraceDisplay::IoCount | DtraceDisplay::IoSummary => { - result.push_str(&format!( - " {:>5} {:>5} {:>5}", - "IP0", "IP1", "IP2" - )); - result - .push_str(&format!(" {:>5} {:>5} {:>5}", "D0", "D1", "D2")); - result - .push_str(&format!(" {:>5} {:>5} {:>5}", "S0", "S1", "S2")); - - if matches!(display_item, DtraceDisplay::IoCount) { - result.push_str(&format!( - " {:>4} {:>4} {:>4}", - "E0", "E1", "E2" - )); - } - } - DtraceDisplay::Reconcile => { - result.push_str(&format!( - " {:>4} {:>4} {:>4}", - "REC", "NREC", "AREC" - )); - } - DtraceDisplay::DsReconciled => { - result.push_str(&format!(" {:>4}", "ERR")); - } - DtraceDisplay::DsReconcileNeeded => { - result.push_str(&format!(" {:>4}", "ERN")); - } - DtraceDisplay::LiveRepair => { - result.push_str(&format!( - " {:>4} {:>4} {:>4}", - "LRC0", "LRC1", "LRC0" - )); - result.push_str(&format!( - " {:>4} {:>4} {:>4}", - "LRA0", "LRA1", "LRA2" - )); - } - DtraceDisplay::Connected => { - result.push_str(&format!( - " {:>4} {:>4} {:>4}", - "CON0", "CON1", "CON2" - )); - } - DtraceDisplay::Replaced => { - result.push_str(&format!( - " {:>4} {:>4} {:>4}", - "RPL0", "RPL1", "RPL2" - )); - } - DtraceDisplay::ExtentLiveRepair => { - result.push_str(&format!( - " {:>4} {:>4} {:>4}", - "EXR0", "EXR1", "EXR2" - )); - result.push_str(&format!( - " {:>4} {:>4} {:>4}", - "EXC0", "EXC1", "EXC2" - )); - } - DtraceDisplay::ExtentLimit => { - result.push_str(&format!(" {:>4}", "EXTL")); - } - DtraceDisplay::NextJobId => { - result.push_str(&format!(" {:>10}", "NEXTJOB")); - } - DtraceDisplay::JobDelta => { - result.push_str(&format!(" {:>5}", "DELTA")); - } - DtraceDisplay::DsDelay => { - result.push_str(&format!( - " {:>5} {:>5} {:>5}", - "DLY0", "DLY1", "DLY2" - )); - } - DtraceDisplay::WriteBytesOut => { - result.push_str(&format!(" {:>10}", "WRBYTES")); - } - DtraceDisplay::RoLrSkipped => { - result.push_str(&format!( - " {:>4} {:>4} {:>4}", - "RLS0", "RLS1", "RLS2" - )); - } - DtraceDisplay::DsIoInProgress => { - result.push_str(&format!( - " {:>5} {:>5} {:>5}", - "IP0", "IP1", "IP2" - )); - } - DtraceDisplay::DsIoDone => { - result - .push_str(&format!(" {:>5} {:>5} {:>5}", "D0", "D1", "D2")); - } - DtraceDisplay::DsIoSkipped => { - result - .push_str(&format!(" {:>5} {:>5} {:>5}", "S0", "S1", "S2")); - } - DtraceDisplay::DsIoError => { - result - .push_str(&format!(" {:>4} {:>4} {:>4}", "E0", "E1", "E2")); - } - } - } - result -} - -/// Format a row for a single process -fn format_row( - pid: u32, - d_out: &DtraceInfo, - precomputed_delta: Option, - dd: &[DtraceDisplay], - _is_stale: bool, -) -> String { - let mut result = String::new(); - - for display_item in dd.iter() { - match display_item { - DtraceDisplay::Pid => { - // Note: stale indicator is now shown in the first column - result.push_str(&format!(" {:>5}", pid)); - } - DtraceDisplay::Session => { - let session_short = - d_out.session_id.chars().take(8).collect::(); - result.push_str(&format!(" {:>8}", session_short)); - } - DtraceDisplay::UpstairsId => { - let upstairs_short = - d_out.upstairs_id.chars().take(8).collect::(); - result.push_str(&format!(" {:>8}", upstairs_short)); - } - DtraceDisplay::State => { - result.push_str(&format!( - " {:>3} {:>3} {:>3}", - short_state(&d_out.ds_state[0]), - short_state(&d_out.ds_state[1]), - short_state(&d_out.ds_state[2]), - )); - } - DtraceDisplay::UpCount => { - result.push_str(&format!(" {:3}", d_out.up_count)); - } - DtraceDisplay::DsCount => { - result.push_str(&format!(" {:5}", d_out.ds_count)); - } - DtraceDisplay::IoCount | DtraceDisplay::IoSummary => { - result.push_str(&format!( - " {:5} {:5} {:5}", - d_out.ds_io_count.in_progress[ClientId::new(0)], - d_out.ds_io_count.in_progress[ClientId::new(1)], - d_out.ds_io_count.in_progress[ClientId::new(2)], - )); - result.push_str(&format!( - " {:5} {:5} {:5}", - d_out.ds_io_count.done[ClientId::new(0)], - d_out.ds_io_count.done[ClientId::new(1)], - d_out.ds_io_count.done[ClientId::new(2)], - )); - result.push_str(&format!( - " {:5} {:5} {:5}", - d_out.ds_io_count.skipped[ClientId::new(0)], - d_out.ds_io_count.skipped[ClientId::new(1)], - d_out.ds_io_count.skipped[ClientId::new(2)], - )); - if matches!(display_item, DtraceDisplay::IoCount) { - result.push_str(&format!( - " {:4} {:4} {:4}", - d_out.ds_io_count.error[ClientId::new(0)], - d_out.ds_io_count.error[ClientId::new(1)], - d_out.ds_io_count.error[ClientId::new(2)], - )); - } - } - DtraceDisplay::Reconcile => { - result.push_str(&format!( - " {:4} {:4} {:4}", - d_out.ds_reconciled, - d_out.ds_reconcile_needed, - d_out.ds_reconcile_aborted, - )); - } - DtraceDisplay::DsReconciled => { - result.push_str(&format!(" {:>4}", d_out.ds_reconciled)); - } - DtraceDisplay::DsReconcileNeeded => { - result.push_str(&format!(" {:>4}", d_out.ds_reconcile_needed)); - } - DtraceDisplay::LiveRepair => { - result.push_str(&format!( - " {:4} {:4} {:4}", - d_out.ds_live_repair_completed[0], - d_out.ds_live_repair_completed[1], - d_out.ds_live_repair_completed[2], - )); - result.push_str(&format!( - " {:4} {:4} {:4}", - d_out.ds_live_repair_aborted[0], - d_out.ds_live_repair_aborted[1], - d_out.ds_live_repair_aborted[2], - )); - } - DtraceDisplay::Connected => { - result.push_str(&format!( - " {:4} {:4} {:4}", - d_out.ds_connected[0], - d_out.ds_connected[1], - d_out.ds_connected[2], - )); - } - DtraceDisplay::Replaced => { - result.push_str(&format!( - " {:4} {:4} {:4}", - d_out.ds_replaced[0], - d_out.ds_replaced[1], - d_out.ds_replaced[2], - )); - } - DtraceDisplay::ExtentLiveRepair => { - result.push_str(&format!( - " {:4} {:4} {:4}", - d_out.ds_extents_repaired[0], - d_out.ds_extents_repaired[1], - d_out.ds_extents_repaired[2], - )); - result.push_str(&format!( - " {:4} {:4} {:4}", - d_out.ds_extents_confirmed[0], - d_out.ds_extents_confirmed[1], - d_out.ds_extents_confirmed[2], - )); - } - DtraceDisplay::ExtentLimit => { - result.push_str(&format!(" {:4}", d_out.ds_extent_limit)); - } - DtraceDisplay::NextJobId => { - result.push_str(&format!(" {:>10}", d_out.next_job_id)); - } - DtraceDisplay::JobDelta => { - if let Some(delta) = precomputed_delta { - result.push_str(&format!(" {:5}", delta)); - } else { - result.push_str(&format!(" {:>5}", "---")); - } - } - DtraceDisplay::DsDelay => { - result.push_str(&format!( - " {:5} {:5} {:5}", - d_out.ds_delay_us[0], - d_out.ds_delay_us[1], - d_out.ds_delay_us[2], - )); - } - DtraceDisplay::WriteBytesOut => { - result.push_str(&format!(" {:10}", d_out.write_bytes_out)); - } - DtraceDisplay::RoLrSkipped => { - result.push_str(&format!( - " {:4} {:4} {:4}", - d_out.ds_ro_lr_skipped[0], - d_out.ds_ro_lr_skipped[1], - d_out.ds_ro_lr_skipped[2], - )); - } - DtraceDisplay::DsIoInProgress => { - result.push_str(&format!( - " {:5} {:5} {:5}", - d_out.ds_io_count.in_progress[ClientId::new(0)], - d_out.ds_io_count.in_progress[ClientId::new(1)], - d_out.ds_io_count.in_progress[ClientId::new(2)], - )); - } - DtraceDisplay::DsIoDone => { - result.push_str(&format!( - " {:5} {:5} {:5}", - d_out.ds_io_count.done[ClientId::new(0)], - d_out.ds_io_count.done[ClientId::new(1)], - d_out.ds_io_count.done[ClientId::new(2)], - )); - } - DtraceDisplay::DsIoSkipped => { - result.push_str(&format!( - " {:5} {:5} {:5}", - d_out.ds_io_count.skipped[ClientId::new(0)], - d_out.ds_io_count.skipped[ClientId::new(1)], - d_out.ds_io_count.skipped[ClientId::new(2)], - )); - } - DtraceDisplay::DsIoError => { - result.push_str(&format!( - " {:4} {:4} {:4}", - d_out.ds_io_count.error[ClientId::new(0)], - d_out.ds_io_count.error[ClientId::new(1)], - d_out.ds_io_count.error[ClientId::new(2)], - )); - } - } - } - result -} - -/// Render a sparkline from delta history -/// Uses Unicode block characters to show trend: ▁▂▃▄▅▆▇█ -/// If global_max is provided, scales relative to that value for -/// cross-session comparison -fn render_sparkline( - history: &VecDeque, - width: usize, - global_max: u64, -) -> String { - if history.is_empty() || width == 0 { - return String::new(); - } - - // Unicode block characters from lowest to highest - const BLOCKS: [char; 8] = ['▁', '▂', '▃', '▄', '▅', '▆', '▇', '█']; - - // Take last 'width' samples (most recent) - let samples: Vec = history - .iter() - .rev() - .take(width) - .copied() - .collect::>() - .into_iter() - .rev() - .collect(); - - if samples.is_empty() { - return String::new(); - } - - // Use global max for scaling (minimum 1 to avoid division by zero) - let max = global_max.max(1); - - // Map each value to a block character - samples - .iter() - .map(|&val| { - if val == 0 { - BLOCKS[0] - } else { - let normalized = (val as f64 / max as f64 * 7.0) as usize; - BLOCKS[normalized.min(7)] - } - }) - .collect() -} - -/// Subprocess reader task - spawns dtrace command and reads JSON output -async fn subprocess_reader_task( - dtrace_cmd: String, - state: Arc>, - notify: Arc, -) -> Result<(), Box> { - // Parse command string into command and args - let parts: Vec<&str> = dtrace_cmd.split_whitespace().collect(); - if parts.is_empty() { - return Err("Empty dtrace command".into()); - } - - // Spawn the dtrace subprocess - let mut child = Command::new(parts[0]) - .args(&parts[1..]) - .stdout(std::process::Stdio::piped()) - .stderr(std::process::Stdio::null()) - .spawn()?; - - let stdout = child - .stdout - .take() - .ok_or("Failed to capture subprocess stdout")?; - - let reader = BufReader::new(stdout); - let mut lines = reader.lines(); - - // Read lines from subprocess stdout - while let Some(line) = lines.next_line().await? { - // Parse JSON - let wrapper: DtraceWrapper = match serde_json::from_str(&line) { - Ok(w) => w, - Err(_) => continue, - }; - - // Update state - let mut state_guard = state.write().await; - - let session_data = state_guard - .sessions - .entry(wrapper.status.session_id.clone()) - .or_insert_with(|| SessionData { - pid: wrapper.pid, - dtrace_info: wrapper.status.clone(), - last_job_id: 0, - last_updated: Instant::now(), - current_delta: None, - delta_history: VecDeque::new(), - }); - - // Calculate delta (jobs per second) - let current_job_id = wrapper.status.next_job_id.0; - let delta = if session_data.last_job_id != 0 { - let d = current_job_id.saturating_sub(session_data.last_job_id); - - // Add to history ring buffer - session_data.delta_history.push_back(d); - if session_data.delta_history.len() > MAX_DELTA_HISTORY { - session_data.delta_history.pop_front(); - } - - Some(d) - } else { - None - }; - - // Store current delta and update state - session_data.current_delta = delta; - session_data.last_job_id = current_job_id; - session_data.dtrace_info = wrapper.status; - session_data.last_updated = Instant::now(); - - drop(state_guard); - - // Notify display task - notify.notify_one(); - } - - // Wait for child to exit - let _ = child.wait().await; - - Ok(()) -} - -/// Render full-screen detail view for a selected session -fn render_detail_view( - session_data: &SessionData, - _terminal_size: (u16, u16), - terminal: &mut Terminal>, - global_min: Option, - global_max: Option, - normalize: bool, -) -> io::Result<()> { - // Calculate statistics - let history: Vec = - session_data.delta_history.iter().copied().collect(); - let session_max = history.iter().copied().max().unwrap_or(1); - let session_min = history.iter().copied().min().unwrap_or(0); - let avg = if !history.is_empty() { - history.iter().sum::() / history.len() as u64 - } else { - 0 - }; - let current = session_data.current_delta.unwrap_or(0); - - // Choose min/max based on normalize mode - let (display_min, display_max) = if normalize { - ( - global_min.unwrap_or(session_min), - global_max.unwrap_or(session_max), - ) - } else { - (session_min, session_max) - }; - - // Render using ratatui (terminal is reused, ratatui handles diffing) - terminal.draw(|f| { - let area = f.area(); - - // Split area: 1 line at top for session data, rest for canvas - let chunks = Layout::default() - .constraints([Constraint::Length(1), Constraint::Min(0)]) - .split(area); - - // Format the session data row - let display_fields = default_display_fields(); - let is_stale = session_data - .last_updated - .elapsed() - .as_secs() - > STALE_THRESHOLD_SECS; - let row_data = format_row( - session_data.pid, - &session_data.dtrace_info, - session_data.current_delta, - &display_fields, - is_stale, - ); - - // Render session data at top - let data_paragraph = Paragraph::new(row_data); - f.render_widget(data_paragraph, chunks[0]); - - // Create title - let session_short: String = - session_data.dtrace_info.session_id.chars().take(8).collect(); - let mode_str = if normalize { " [NORMALIZED]" } else { "" }; - let title = format!( - " Delta History - PID {} - Session {}{} ", - session_data.pid, session_short, mode_str - ); - - // Create canvas widget in bottom area (1 line shorter) - let canvas = Canvas::default() - .block( - Block::default() - .borders(Borders::ALL) - .title(title) - .title_bottom(format!( - " Samples: {} | Min: {} | Max: {} | Avg: {} | Current: {} ", - history.len(), - session_min, - session_max, - avg, - current - )) - .title_bottom( - " ['d': Back | 'n': Toggle normalize | 'q': Quit] ", - ), - ) - .x_bounds([0.0, history.len().max(1) as f64]) - .y_bounds([display_min as f64, display_max as f64]) - .paint(|ctx| { - // Draw Y-axis labels (at left edge of graph) - let y_range = display_max as f64 - display_min as f64; - let y_positions = [ - display_max, - display_min + (y_range * 0.75) as u64, - display_min + (y_range * 0.5) as u64, - display_min + (y_range * 0.25) as u64, - display_min, - ]; - - for y_val in &y_positions { - ctx.print( - 0.0, - *y_val as f64, - ratatui::text::Span::styled( - format!("{}", y_val), - ratatui::style::Style::default() - .fg(Color::Gray), - ), - ); - } - - // Draw the line graph - if history.len() > 1 { - for i in 0..history.len() - 1 { - let x1 = i as f64; - let y1 = history[i] as f64; - let x2 = (i + 1) as f64; - let y2 = history[i + 1] as f64; - - ctx.draw(&Line { - x1, - y1, - x2, - y2, - color: Color::Cyan, - }); - } - } - - // Draw points for each sample - for (i, &value) in history.iter().enumerate() { - ctx.draw(&Points { - coords: &[(i as f64, value as f64)], - color: Color::Yellow, - }); - } - }); - - f.render_widget(canvas, chunks[1]); - })?; - - Ok(()) -} - -/// Display task - renders the screen and handles keyboard input -async fn display_task( - state: Arc>, - notify: Arc, -) -> Result<(), Box> { - // Set up terminal - enable_raw_mode()?; - let mut stdout = io::stdout(); - execute!(stdout, EnterAlternateScreen)?; - - // Set up panic handler to restore terminal - let original_hook = std::panic::take_hook(); - std::panic::set_hook(Box::new(move |panic_info| { - let _ = execute!(io::stdout(), LeaveAlternateScreen); - let _ = disable_raw_mode(); - original_hook(panic_info); - })); - - let display_fields = default_display_fields(); - - // Track detail mode and persistent terminal for detail view - let mut was_in_detail_mode = false; - let mut detail_terminal: Option>> = - None; - - loop { - // Wait for notification or timeout - tokio::select! { - _ = notify.notified() => {}, - _ = tokio::time::sleep(Duration::from_millis(100)) => {}, - } - - // Get current time - let now = Instant::now(); - let system_time = std::time::SystemTime::now(); - let duration = system_time - .duration_since(std::time::UNIX_EPOCH) - .unwrap_or_default(); - - // Get terminal size - let terminal_size = crossterm::terminal::size()?; - - // Read state to check mode - let state_guard = state.read().await; - let in_detail_mode = state_guard.detail_mode; - let selected_index = state_guard.selected_index; - let normalize_detail = state_guard.normalize_detail; - - // If in detail mode, render detail view and skip table - if in_detail_mode { - // Create terminal on first entry to detail mode - if !was_in_detail_mode { - execute!(stdout, Clear(ClearType::All))?; - // Create a new stdout handle for the Terminal - let detail_stdout = io::stdout(); - let backend = CrosstermBackend::new(detail_stdout); - detail_terminal = Some(Terminal::new(backend)?); - } - - let mut sessions: Vec<&SessionData> = - state_guard.sessions.values().collect(); - sessions.sort_by_key(|s| (s.pid, &s.dtrace_info.session_id)); - - // Calculate global min/max across all sessions for normalization - let global_min = sessions - .iter() - .flat_map(|s| s.delta_history.iter()) - .copied() - .min(); - let global_max = sessions - .iter() - .flat_map(|s| s.delta_history.iter()) - .copied() - .max(); - - if let Some(selected_session) = sessions.get(selected_index) { - // Clone the session data so we can drop the lock - let session_clone = (*selected_session).clone(); - drop(state_guard); - - if let Some(terminal) = detail_terminal.as_mut() { - render_detail_view( - &session_clone, - terminal_size, - terminal, - global_min, - global_max, - normalize_detail, - )?; - } - } else { - drop(state_guard); - } - - was_in_detail_mode = true; - } else { - // Exiting detail mode - drop terminal and redraw table - if was_in_detail_mode { - detail_terminal = None; - execute!(stdout, Clear(ClearType::All))?; - } - was_in_detail_mode = false; - - // Move cursor to top-left - execute!(stdout, cursor::MoveTo(0, 0))?; - // Table mode - render normal view - drop(state_guard); - - // Display header (clear line first to remove artifacts) - execute!(stdout, cursor::MoveTo(0, 0))?; - write!( - stdout, - "cmon ctop - Unix timestamp: {}", - duration.as_secs() - )?; - execute!(stdout, Clear(ClearType::UntilNewLine))?; - write!(stdout, "\r\n")?; - execute!(stdout, Clear(ClearType::UntilNewLine))?; - write!(stdout, "\r\n")?; - - // Display column headers - write!(stdout, "{}", format_header(&display_fields))?; - execute!(stdout, Clear(ClearType::UntilNewLine))?; - write!(stdout, "\r\n")?; - - let (terminal_width, _) = terminal_size; - - // Read state and display sessions sorted by PID (then session_id) - let state_guard = state.read().await; - let mut sessions: Vec<&SessionData> = - state_guard.sessions.values().collect(); - sessions.sort_by_key(|s| (s.pid, &s.dtrace_info.session_id)); - - // Calculate global max across all sessions for consistent sparkline - // scaling - let global_max = sessions - .iter() - .flat_map(|s| s.delta_history.iter()) - .copied() - .max() - .unwrap_or(1); - - let selected_index = state_guard.selected_index; - - for (idx, session_data) in sessions.iter().enumerate() { - let is_stale = now.duration_since(session_data.last_updated) - > Duration::from_secs(STALE_THRESHOLD_SECS); - - // Add indicator: > for selected, * for stale, space otherwise - // Selection indicator (>) takes priority over stale indicator (*) - let indicator = if idx == selected_index { - ">" - } else if is_stale { - "*" - } else { - " " - }; - write!(stdout, "{}", indicator)?; - - let row = format_row( - session_data.pid, - &session_data.dtrace_info, - session_data.current_delta, - &display_fields, - is_stale, - ); - write!(stdout, "{}", row)?; - - // Render sparkline in remaining space - // Account for the indicator character (1 char) - let row_len = row.chars().count() + 1; - if terminal_width > row_len as u16 { - let sparkline_width = - (terminal_width as usize - row_len).saturating_sub(1); - if sparkline_width > 0 { - let sparkline = render_sparkline( - &session_data.delta_history, - sparkline_width, - global_max, - ); - write!(stdout, " {}", sparkline)?; - } - } - - execute!(stdout, Clear(ClearType::UntilNewLine))?; - write!(stdout, "\r\n")?; - } - drop(state_guard); - - // Display footer - execute!(stdout, Clear(ClearType::UntilNewLine))?; - write!(stdout, "\r\n")?; - write!( - stdout, - "[↑↓: Select | 'd': Details | 'q': Quit] > = selected, * = stale ({}s)", - STALE_THRESHOLD_SECS - )?; - execute!(stdout, Clear(ClearType::UntilNewLine))?; - write!(stdout, "\r\n")?; - - // Clear from cursor to end of screen (removes any leftover lines) - execute!(stdout, Clear(ClearType::FromCursorDown))?; - - stdout.flush()?; - } // End of table mode rendering - - // Check for keyboard input (non-blocking) - if event::poll(Duration::from_millis(0))? - && let Event::Key(key_event) = event::read()? - { - let mut state_guard = state.write().await; - let num_sessions = state_guard.sessions.len(); - - match key_event { - KeyEvent { - code: KeyCode::Char('q'), - modifiers: KeyModifiers::NONE, - .. - } => break, - KeyEvent { - code: KeyCode::Char('c'), - modifiers: KeyModifiers::CONTROL, - .. - } => break, - KeyEvent { - code: KeyCode::Char('d'), - modifiers: KeyModifiers::NONE, - .. - } => { - // Toggle detail mode - state_guard.detail_mode = !state_guard.detail_mode; - } - KeyEvent { - code: KeyCode::Char('n'), - modifiers: KeyModifiers::NONE, - .. - } => { - // Toggle normalize mode (only affects detail view) - state_guard.normalize_detail = - !state_guard.normalize_detail; - } - KeyEvent { - code: KeyCode::Up, - modifiers: KeyModifiers::NONE, - .. - } => { - // Move selection up (only in table mode) - if !state_guard.detail_mode && num_sessions > 0 { - state_guard.selected_index = - state_guard.selected_index.saturating_sub(1); - } - } - KeyEvent { - code: KeyCode::Down, - modifiers: KeyModifiers::NONE, - .. - } => { - // Move selection down (only in table mode) - if !state_guard.detail_mode && num_sessions > 0 { - state_guard.selected_index = - (state_guard.selected_index + 1) - .min(num_sessions.saturating_sub(1)); - } - } - KeyEvent { - code: KeyCode::Esc, - modifiers: KeyModifiers::NONE, - .. - } => { - // Exit detail mode - state_guard.detail_mode = false; - } - _ => {} - } - drop(state_guard); - } - } - - // Clean up terminal - execute!(stdout, LeaveAlternateScreen)?; - disable_raw_mode()?; - - Ok(()) -} - -/// Main entry point for ctop -pub async fn ctop_loop( - dtrace_cmd: String, -) -> Result<(), Box> { - let state = Arc::new(RwLock::new(CtopState::default())); - let notify = Arc::new(Notify::new()); - - let state_reader = Arc::clone(&state); - let notify_reader = Arc::clone(¬ify); - - // Spawn subprocess reader task - let reader_handle = tokio::spawn(async move { - if let Err(e) = - subprocess_reader_task(dtrace_cmd, state_reader, notify_reader) - .await - { - eprintln!("Subprocess reader error: {}", e); - } - }); - - // Run display task (blocks until user quits) - let display_result = display_task(state, notify).await; - - // Wait for reader task to finish (it should exit quickly) - let _ = - tokio::time::timeout(Duration::from_millis(100), reader_handle).await; - - display_result -} diff --git a/cmon/src/main.rs b/cmon/src/main.rs index 015f763dd..cd946442c 100644 --- a/cmon/src/main.rs +++ b/cmon/src/main.rs @@ -8,8 +8,6 @@ use std::io::{self, BufRead}; use strum::IntoEnumIterator; use tokio::time::{Duration, sleep}; -mod ctop; - /// Connect to crucible control server #[derive(Parser, Debug)] #[clap(name = "cmon", term_width = 80)] @@ -42,15 +40,6 @@ enum Action { Jobs, /// Show the status of various LiveRepair stats Repair, - /// Curses-based top-like display of dtrace data - Ctop { - /// Command to run to generate dtrace output - #[clap( - long, - default_value = "dtrace -s /opt/oxide/crucible_dtrace/upstairs_raw.d" - )] - dtrace_cmd: String, - }, } // Show the downstairs work queue @@ -479,10 +468,5 @@ async fn main() { Action::Repair => { show_repair_stats(args).await; } - Action::Ctop { dtrace_cmd } => { - if let Err(e) = ctop::ctop_loop(dtrace_cmd).await { - eprintln!("Error running ctop: {}", e); - } - } } } diff --git a/ctop/src/main.rs b/ctop/src/main.rs index fd5b4f689..d8ab636b9 100644 --- a/ctop/src/main.rs +++ b/ctop/src/main.rs @@ -31,16 +31,24 @@ use tokio::io::{AsyncBufReadExt, BufReader}; use tokio::process::Command; use tokio::sync::{Notify, RwLock}; +/// Default dtrace command - embedded one-liner that matches upstairs_raw.d +/// +/// This command: +/// - Uses -Z to continue even if no probes match +/// - Uses -q for quiet mode (no dtrace header) +/// - Sets strsize=2k for 2KB string buffers +/// - Probes crucible_upstairs*:::up-status +/// - Outputs JSON with pid and status +const DEFAULT_DTRACE_CMD: &str = + r#"dtrace -Z -q -x strsize=2k -n 'crucible_upstairs*:::up-status { printf("{\"pid\":%d,\"status\":%s}\n", pid, json(copyinstr(arg1), "ok")); }'"#; + /// Crucible top - monitor crucible upstairs via dtrace #[derive(Parser, Debug)] #[clap(name = "ctop", term_width = 80)] #[clap(about = "Curses-based crucible monitor", long_about = None)] struct Args { /// Command to run to generate dtrace output - #[clap( - long, - default_value = "dtrace -s /opt/oxide/crucible_dtrace/upstairs_raw.d" - )] + #[clap(long, default_value = DEFAULT_DTRACE_CMD)] dtrace_cmd: String, } From 015962b9230ecb8e8ef2e234f7b65bdab92eed0a Mon Sep 17 00:00:00 2001 From: Alan Hanson Date: Sun, 26 Oct 2025 15:02:12 -0700 Subject: [PATCH 19/36] fix embedded dtrace command --- ctop/src/main.rs | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/ctop/src/main.rs b/ctop/src/main.rs index d8ab636b9..7592c15a3 100644 --- a/ctop/src/main.rs +++ b/ctop/src/main.rs @@ -472,15 +472,14 @@ async fn subprocess_reader_task( state: Arc>, notify: Arc, ) -> Result<(), Box> { - // Parse command string into command and args - let parts: Vec<&str> = dtrace_cmd.split_whitespace().collect(); - if parts.is_empty() { + if dtrace_cmd.is_empty() { return Err("Empty dtrace command".into()); } - // Spawn the dtrace subprocess - let mut child = Command::new(parts[0]) - .args(&parts[1..]) + // Execute the command through a shell to properly handle quoting + let mut child = Command::new("sh") + .arg("-c") + .arg(&dtrace_cmd) .stdout(std::process::Stdio::piped()) .stderr(std::process::Stdio::null()) .spawn()?; From 14b6e9495dab5fcb8e37c4014409e4e8a759787d Mon Sep 17 00:00:00 2001 From: Alan Hanson Date: Thu, 30 Oct 2025 11:21:33 -0700 Subject: [PATCH 20/36] Added TODO for ctop --- ctop/TODO.md | 69 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) create mode 100644 ctop/TODO.md diff --git a/ctop/TODO.md b/ctop/TODO.md new file mode 100644 index 000000000..3da839e42 --- /dev/null +++ b/ctop/TODO.md @@ -0,0 +1,69 @@ +# ctop TODO List + +## Feature Enhancements + +### Color enhancement +- [ ] Add color option to output, Make colors for ACT (good) different than + colors for other downstairs state. FLT red, LRR or LR a middle color. + NEW is grey maybe. Include tests for this +- [ ] Can we determine the current foreground/background of the terminal? How + will we handle light or dark mode. Maybe make a selection key to toggle + between the modes. + +### Session Selection and Multi-Session Display +- [ ] Add ability to select multiple sessions (e.g., with Space key) +- [ ] Display detailed graph for all selected sessions simultaneously +- [ ] Show visual indicator for which sessions are selected in the main list + +### Screen Scrolling +- [ ] Add ability to scroll the session list when there are more sessions than + can fit on screen +- [ ] Implement scroll indicators (e.g., "↑ More above", "↓ More below") +- [ ] Consider adding Page Up/Page Down support for faster navigation +- [ ] Remove sessions that have not updated after 60 seconds + +### Normalization Improvements +- [ ] Change normalization toggle. + - Make the same key rotate between the three options. + - Rotate through + 1: min/max for the current session (or, when finished, all selected sessions) + 2: 0 and the selected session(s) max + 3: 0 and max for all sessions (while just shown the current selections data). + +### Program improvements. +- [ ] Add an option to only run for a specific number of seconds then exit. +- [ ] Add an option to keep the last state displayed, or, really, reproduce the + final screen but after we have exited the curses window. +- [ ] allow user to select which possible dtrace probes to display. + - If job delta is not selected, then detailed graphs are not available. +- [ ] Allow some downstairs individual stats to be combined into a "sum", like + connections for each downstairs summed into a single value. Not all + dtrace probes could do this, so give just options for ones we can. + +## Implementation Notes + +### Session Selection +- Maintain a `HashSet` of selected session IDs in `CtopState` +- Space key toggles selection of currently highlighted session +- Selected sessions marked with `[*]` or similar indicator +- Detail view shows all selected sessions' sparklines stacked or overlaid + +### Scrolling +- Track `scroll_offset` in display state +- Calculate visible window based on terminal height +- Adjust rendering to show sessions from `scroll_offset` to + `scroll_offset + visible_rows` +- Update scroll offset on up/down arrow keys + +### Normalization +- Modify `render_detail_view()` and sparkline rendering +- For zero-based normalization: always use `min = 0`, compute `max` from data +- For selection-based normalization: filter sessions to only selected ones + before computing min/max +- Add visual indicator in detail view header showing normalization mode + +## UI/UX Considerations +- Document new keyboard shortcuts in help or header +- Ensure selected sessions are visually distinct +- Consider performance impact of rendering multiple detailed graphs +- Test with many sessions (10+, 50+, 100+) to ensure scrolling performs well From 4ccc04fb90a5c02bcbb591241f5235159c5c929b Mon Sep 17 00:00:00 2001 From: Alan Hanson Date: Thu, 30 Oct 2025 11:22:37 -0700 Subject: [PATCH 21/36] remove reference to cmon --- ctop/src/main.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ctop/src/main.rs b/ctop/src/main.rs index 7592c15a3..2a13ae632 100644 --- a/ctop/src/main.rs +++ b/ctop/src/main.rs @@ -805,7 +805,7 @@ async fn display_task( execute!(stdout, cursor::MoveTo(0, 0))?; write!( stdout, - "cmon ctop - Unix timestamp: {}", + "ctop - Unix timestamp: {}", duration.as_secs() )?; execute!(stdout, Clear(ClearType::UntilNewLine))?; From 529d7590442f700af335597b184cc92a8f516e92 Mon Sep 17 00:00:00 2001 From: Alan Hanson Date: Thu, 30 Oct 2025 13:07:23 -0700 Subject: [PATCH 22/36] Better window size handling, remove old sessions --- ctop/src/main.rs | 193 +++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 177 insertions(+), 16 deletions(-) diff --git a/ctop/src/main.rs b/ctop/src/main.rs index 2a13ae632..53278fe8c 100644 --- a/ctop/src/main.rs +++ b/ctop/src/main.rs @@ -39,8 +39,7 @@ use tokio::sync::{Notify, RwLock}; /// - Sets strsize=2k for 2KB string buffers /// - Probes crucible_upstairs*:::up-status /// - Outputs JSON with pid and status -const DEFAULT_DTRACE_CMD: &str = - r#"dtrace -Z -q -x strsize=2k -n 'crucible_upstairs*:::up-status { printf("{\"pid\":%d,\"status\":%s}\n", pid, json(copyinstr(arg1), "ok")); }'"#; +const DEFAULT_DTRACE_CMD: &str = r#"dtrace -Z -q -x strsize=2k -n 'crucible_upstairs*:::up-status { printf("{\"pid\":%d,\"status\":%s}\n", pid, json(copyinstr(arg1), "ok")); }'"#; /// Crucible top - monitor crucible upstairs via dtrace #[derive(Parser, Debug)] @@ -53,6 +52,7 @@ struct Args { } const STALE_THRESHOLD_SECS: u64 = 10; +const REMOVE_THRESHOLD_SECS: u64 = 60; const MAX_DELTA_HISTORY: usize = 100; /// Data for a single session @@ -71,6 +71,7 @@ struct SessionData { struct CtopState { sessions: HashMap, selected_index: usize, + scroll_offset: usize, detail_mode: bool, normalize_detail: bool, // Use global min/max for detail view scaling } @@ -735,6 +736,32 @@ async fn display_task( // Get terminal size let terminal_size = crossterm::terminal::size()?; + // Clean up sessions that haven't been updated in REMOVE_THRESHOLD_SECS + { + let mut state_guard = state.write().await; + let sessions_before = state_guard.sessions.len(); + + state_guard.sessions.retain(|_, session_data| { + now.duration_since(session_data.last_updated) + <= Duration::from_secs(REMOVE_THRESHOLD_SECS) + }); + + let sessions_after = state_guard.sessions.len(); + + // Adjust selected_index if sessions were removed + if sessions_after < sessions_before && sessions_after > 0 { + state_guard.selected_index = + state_guard.selected_index.min(sessions_after - 1); + state_guard.scroll_offset = + state_guard.scroll_offset.min(sessions_after - 1); + } else if sessions_after == 0 { + state_guard.selected_index = 0; + state_guard.scroll_offset = 0; + } + + drop(state_guard); + } + // Read state to check mode let state_guard = state.read().await; let in_detail_mode = state_guard.detail_mode; @@ -803,11 +830,7 @@ async fn display_task( // Display header (clear line first to remove artifacts) execute!(stdout, cursor::MoveTo(0, 0))?; - write!( - stdout, - "ctop - Unix timestamp: {}", - duration.as_secs() - )?; + write!(stdout, "ctop - Unix timestamp: {}", duration.as_secs())?; execute!(stdout, Clear(ClearType::UntilNewLine))?; write!(stdout, "\r\n")?; execute!(stdout, Clear(ClearType::UntilNewLine))?; @@ -818,7 +841,7 @@ async fn display_task( execute!(stdout, Clear(ClearType::UntilNewLine))?; write!(stdout, "\r\n")?; - let (terminal_width, _) = terminal_size; + let (terminal_width, terminal_height) = terminal_size; // Read state and display sessions sorted by PID (then session_id) let state_guard = state.read().await; @@ -836,8 +859,42 @@ async fn display_task( .unwrap_or(1); let selected_index = state_guard.selected_index; + let scroll_offset = state_guard.scroll_offset; + + // Calculate visible window + // Terminal layout: 1 header + 1 blank + 1 column headers + sessions + 1 blank + 1 footer + // Plus optional scroll indicators (1 line each if present) + let total_sessions = sessions.len(); + let has_more_above = scroll_offset > 0; + let has_more_below_tentative = scroll_offset + 1 < total_sessions; + + let fixed_lines = 5 + + if has_more_above { 1 } else { 0 } + + if has_more_below_tentative { 1 } else { 0 }; + + let visible_rows = if terminal_height > fixed_lines as u16 { + (terminal_height - fixed_lines as u16) as usize + } else { + 1 // Minimum of 1 row to avoid crashes + }; + + let scroll_end = (scroll_offset + visible_rows).min(total_sessions); + let has_more_below = scroll_end < total_sessions; + + // Display scroll indicator if there are sessions above + if has_more_above { + write!(stdout, " ↑ More above")?; + execute!(stdout, Clear(ClearType::UntilNewLine))?; + write!(stdout, "\r\n")?; + } - for (idx, session_data) in sessions.iter().enumerate() { + // Display visible sessions + for (idx, session_data) in sessions + .iter() + .enumerate() + .skip(scroll_offset) + .take(visible_rows) + { let is_stale = now.duration_since(session_data.last_updated) > Duration::from_secs(STALE_THRESHOLD_SECS); @@ -880,6 +937,15 @@ async fn display_task( execute!(stdout, Clear(ClearType::UntilNewLine))?; write!(stdout, "\r\n")?; } + + // Display scroll indicator if there are sessions below + if has_more_below { + let num_more = total_sessions - scroll_end; + write!(stdout, " ↓ More below ({} more)", num_more)?; + execute!(stdout, Clear(ClearType::UntilNewLine))?; + write!(stdout, "\r\n")?; + } + drop(state_guard); // Display footer @@ -887,8 +953,8 @@ async fn display_task( write!(stdout, "\r\n")?; write!( stdout, - "[↑↓: Select | 'd': Details | 'q': Quit] > = selected, * = stale ({}s)", - STALE_THRESHOLD_SECS + "[↑↓/PgUp/PgDn: Navigate | 'd': Details | 'q': Quit] > = selected, * = stale ({}s, removed at {}s)", + STALE_THRESHOLD_SECS, REMOVE_THRESHOLD_SECS )?; execute!(stdout, Clear(ClearType::UntilNewLine))?; write!(stdout, "\r\n")?; @@ -940,9 +1006,19 @@ async fn display_task( .. } => { // Move selection up (only in table mode) - if !state_guard.detail_mode && num_sessions > 0 { - state_guard.selected_index = - state_guard.selected_index.saturating_sub(1); + if !state_guard.detail_mode + && num_sessions > 0 + && state_guard.selected_index > 0 + { + state_guard.selected_index -= 1; + + // Scroll up if selection moves above visible window + if state_guard.selected_index + < state_guard.scroll_offset + { + state_guard.scroll_offset = + state_guard.selected_index; + } } } KeyEvent { @@ -951,10 +1027,95 @@ async fn display_task( .. } => { // Move selection down (only in table mode) + if !state_guard.detail_mode + && num_sessions > 0 + && state_guard.selected_index < num_sessions - 1 + { + state_guard.selected_index += 1; + + // Calculate visible rows to determine scroll behavior + let has_more_above = state_guard.scroll_offset > 0; + let has_more_below_tentative = + state_guard.scroll_offset + 1 < num_sessions; + let fixed_lines = 5 + + if has_more_above { 1 } else { 0 } + + if has_more_below_tentative { 1 } else { 0 }; + let visible_rows = + if terminal_size.1 > fixed_lines as u16 { + (terminal_size.1 - fixed_lines as u16) as usize + } else { + 1 + }; + + // Scroll down if selection moves below visible window + let scroll_end = + state_guard.scroll_offset + visible_rows; + if state_guard.selected_index >= scroll_end { + state_guard.scroll_offset += 1; + } + } + } + KeyEvent { + code: KeyCode::PageUp, + modifiers: KeyModifiers::NONE, + .. + } => { + // Page up (only in table mode) if !state_guard.detail_mode && num_sessions > 0 { + // Calculate visible rows + let has_more_above = state_guard.scroll_offset > 0; + let has_more_below_tentative = + state_guard.scroll_offset + 1 < num_sessions; + let fixed_lines = 5 + + if has_more_above { 1 } else { 0 } + + if has_more_below_tentative { 1 } else { 0 }; + let visible_rows = + if terminal_size.1 > fixed_lines as u16 { + (terminal_size.1 - fixed_lines as u16) as usize + } else { + 1 + }; + + // Move up by visible_rows + state_guard.selected_index = state_guard + .selected_index + .saturating_sub(visible_rows); + state_guard.scroll_offset = state_guard + .scroll_offset + .saturating_sub(visible_rows); + } + } + KeyEvent { + code: KeyCode::PageDown, + modifiers: KeyModifiers::NONE, + .. + } => { + // Page down (only in table mode) + if !state_guard.detail_mode && num_sessions > 0 { + // Calculate visible rows + let has_more_above = state_guard.scroll_offset > 0; + let has_more_below_tentative = + state_guard.scroll_offset + 1 < num_sessions; + let fixed_lines = 5 + + if has_more_above { 1 } else { 0 } + + if has_more_below_tentative { 1 } else { 0 }; + let visible_rows = + if terminal_size.1 > fixed_lines as u16 { + (terminal_size.1 - fixed_lines as u16) as usize + } else { + 1 + }; + + // Move down by visible_rows + let new_selected = + state_guard.selected_index + visible_rows; state_guard.selected_index = - (state_guard.selected_index + 1) - .min(num_sessions.saturating_sub(1)); + new_selected.min(num_sessions - 1); + + let new_scroll = + state_guard.scroll_offset + visible_rows; + state_guard.scroll_offset = new_scroll + .min(num_sessions.saturating_sub(visible_rows)); } } KeyEvent { From b25fa1fa069597a7c0d5613d3bf6089c27a8652b Mon Sep 17 00:00:00 2001 From: Alan Hanson Date: Mon, 10 Nov 2025 12:07:06 -0800 Subject: [PATCH 23/36] job deltas scroll to the right --- ctop/src/main.rs | 20 ++++++-------------- 1 file changed, 6 insertions(+), 14 deletions(-) diff --git a/ctop/src/main.rs b/ctop/src/main.rs index 53278fe8c..973839276 100644 --- a/ctop/src/main.rs +++ b/ctop/src/main.rs @@ -435,16 +435,8 @@ fn render_sparkline( // Unicode block characters from lowest to highest const BLOCKS: [char; 8] = ['▁', '▂', '▃', '▄', '▅', '▆', '▇', '█']; - // Take last 'width' samples (most recent) - let samples: Vec = history - .iter() - .rev() - .take(width) - .copied() - .collect::>() - .into_iter() - .rev() - .collect(); + // Take last 'width' samples (most recent first, newest on left) + let samples: Vec = history.iter().rev().take(width).copied().collect(); if samples.is_empty() { return String::new(); @@ -559,9 +551,9 @@ fn render_detail_view( global_max: Option, normalize: bool, ) -> io::Result<()> { - // Calculate statistics + // Calculate statistics (reversed so newest is on left) let history: Vec = - session_data.delta_history.iter().copied().collect(); + session_data.delta_history.iter().rev().copied().collect(); let session_max = history.iter().copied().max().unwrap_or(1); let session_min = history.iter().copied().min().unwrap_or(0); let avg = if !history.is_empty() { @@ -1377,9 +1369,9 @@ mod tests { // Should have 10 characters (one per value) assert_eq!(sparkline.chars().count(), 10); - // First character should be lower than last (ascending trend) + // First character should be higher than last (newest on left) let chars: Vec = sparkline.chars().collect(); - assert!(chars[0] < chars[9]); + assert!(chars[0] > chars[9]); } #[test] From 62d781bf252a4f2b54b83ac256dc1b8ecf2917e9 Mon Sep 17 00:00:00 2001 From: Alan Hanson Date: Mon, 10 Nov 2025 15:55:11 -0800 Subject: [PATCH 24/36] stale removed at 30s --- ctop/src/main.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ctop/src/main.rs b/ctop/src/main.rs index 973839276..d3a0d1bc9 100644 --- a/ctop/src/main.rs +++ b/ctop/src/main.rs @@ -52,7 +52,7 @@ struct Args { } const STALE_THRESHOLD_SECS: u64 = 10; -const REMOVE_THRESHOLD_SECS: u64 = 60; +const REMOVE_THRESHOLD_SECS: u64 = 30; const MAX_DELTA_HISTORY: usize = 100; /// Data for a single session @@ -945,8 +945,8 @@ async fn display_task( write!(stdout, "\r\n")?; write!( stdout, - "[↑↓/PgUp/PgDn: Navigate | 'd': Details | 'q': Quit] > = selected, * = stale ({}s, removed at {}s)", - STALE_THRESHOLD_SECS, REMOVE_THRESHOLD_SECS + "[↑↓/PgUp/PgDn: Navigate | 'd': Details | 'q': Quit] > = selected, * = stale ({}s)", + STALE_THRESHOLD_SECS )?; execute!(stdout, Clear(ClearType::UntilNewLine))?; write!(stdout, "\r\n")?; From 8401f06d6e19ed87013feb2a56f111ed445c53ae Mon Sep 17 00:00:00 2001 From: Alan Hanson Date: Mon, 10 Nov 2025 15:59:39 -0800 Subject: [PATCH 25/36] Stale at 5 seconds --- ctop/src/main.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ctop/src/main.rs b/ctop/src/main.rs index d3a0d1bc9..14daee562 100644 --- a/ctop/src/main.rs +++ b/ctop/src/main.rs @@ -51,7 +51,7 @@ struct Args { dtrace_cmd: String, } -const STALE_THRESHOLD_SECS: u64 = 10; +const STALE_THRESHOLD_SECS: u64 = 5; const REMOVE_THRESHOLD_SECS: u64 = 30; const MAX_DELTA_HISTORY: usize = 100; From e6e09c51bc8fc09dd0402971e8ba50a244371f61 Mon Sep 17 00:00:00 2001 From: Alan Hanson Date: Tue, 11 Nov 2025 09:26:55 -0800 Subject: [PATCH 26/36] Fix flicker if we have more data than rows --- ctop/src/main.rs | 32 +++++++++----------------------- 1 file changed, 9 insertions(+), 23 deletions(-) diff --git a/ctop/src/main.rs b/ctop/src/main.rs index 14daee562..ebfd7c74e 100644 --- a/ctop/src/main.rs +++ b/ctop/src/main.rs @@ -855,14 +855,12 @@ async fn display_task( // Calculate visible window // Terminal layout: 1 header + 1 blank + 1 column headers + sessions + 1 blank + 1 footer - // Plus optional scroll indicators (1 line each if present) + // Always reserve 2 lines for scroll indicators to prevent flickering let total_sessions = sessions.len(); let has_more_above = scroll_offset > 0; - let has_more_below_tentative = scroll_offset + 1 < total_sessions; - let fixed_lines = 5 - + if has_more_above { 1 } else { 0 } - + if has_more_below_tentative { 1 } else { 0 }; + // Always reserve space for both scroll indicators to prevent layout changes + let fixed_lines = 7; // 5 base lines + 2 for scroll indicators let visible_rows = if terminal_height > fixed_lines as u16 { (terminal_height - fixed_lines as u16) as usize @@ -1026,12 +1024,8 @@ async fn display_task( state_guard.selected_index += 1; // Calculate visible rows to determine scroll behavior - let has_more_above = state_guard.scroll_offset > 0; - let has_more_below_tentative = - state_guard.scroll_offset + 1 < num_sessions; - let fixed_lines = 5 - + if has_more_above { 1 } else { 0 } - + if has_more_below_tentative { 1 } else { 0 }; + // Always use fixed 7 lines to prevent flickering + let fixed_lines = 7; let visible_rows = if terminal_size.1 > fixed_lines as u16 { (terminal_size.1 - fixed_lines as u16) as usize @@ -1055,12 +1049,8 @@ async fn display_task( // Page up (only in table mode) if !state_guard.detail_mode && num_sessions > 0 { // Calculate visible rows - let has_more_above = state_guard.scroll_offset > 0; - let has_more_below_tentative = - state_guard.scroll_offset + 1 < num_sessions; - let fixed_lines = 5 - + if has_more_above { 1 } else { 0 } - + if has_more_below_tentative { 1 } else { 0 }; + // Always use fixed 7 lines to prevent flickering + let fixed_lines = 7; let visible_rows = if terminal_size.1 > fixed_lines as u16 { (terminal_size.1 - fixed_lines as u16) as usize @@ -1085,12 +1075,8 @@ async fn display_task( // Page down (only in table mode) if !state_guard.detail_mode && num_sessions > 0 { // Calculate visible rows - let has_more_above = state_guard.scroll_offset > 0; - let has_more_below_tentative = - state_guard.scroll_offset + 1 < num_sessions; - let fixed_lines = 5 - + if has_more_above { 1 } else { 0 } - + if has_more_below_tentative { 1 } else { 0 }; + // Always use fixed 7 lines to prevent flickering + let fixed_lines = 7; let visible_rows = if terminal_size.1 > fixed_lines as u16 { (terminal_size.1 - fixed_lines as u16) as usize From c02dd55f9284dc0fc83b7034fab6474ca9c9da00 Mon Sep 17 00:00:00 2001 From: Alan Hanson Date: Tue, 11 Nov 2025 10:11:50 -0800 Subject: [PATCH 27/36] flip back the delta graph --- ctop/src/main.rs | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/ctop/src/main.rs b/ctop/src/main.rs index ebfd7c74e..df1cd08e9 100644 --- a/ctop/src/main.rs +++ b/ctop/src/main.rs @@ -435,8 +435,16 @@ fn render_sparkline( // Unicode block characters from lowest to highest const BLOCKS: [char; 8] = ['▁', '▂', '▃', '▄', '▅', '▆', '▇', '█']; - // Take last 'width' samples (most recent first, newest on left) - let samples: Vec = history.iter().rev().take(width).copied().collect(); + // Take last 'width' samples, reverse to show oldest->newest (left->right) + let samples: Vec = history + .iter() + .rev() + .take(width) + .copied() + .collect::>() + .into_iter() + .rev() + .collect(); if samples.is_empty() { return String::new(); @@ -551,9 +559,9 @@ fn render_detail_view( global_max: Option, normalize: bool, ) -> io::Result<()> { - // Calculate statistics (reversed so newest is on left) + // Calculate statistics (oldest on left, newest on right) let history: Vec = - session_data.delta_history.iter().rev().copied().collect(); + session_data.delta_history.iter().copied().collect(); let session_max = history.iter().copied().max().unwrap_or(1); let session_min = history.iter().copied().min().unwrap_or(0); let avg = if !history.is_empty() { @@ -1355,9 +1363,9 @@ mod tests { // Should have 10 characters (one per value) assert_eq!(sparkline.chars().count(), 10); - // First character should be higher than last (newest on left) + // First character should be lower than last (oldest on left, newest on right) let chars: Vec = sparkline.chars().collect(); - assert!(chars[0] > chars[9]); + assert!(chars[0] < chars[9]); } #[test] From fba564ea79f87cb9c66ad799fbb0e0bf013a7587 Mon Sep 17 00:00:00 2001 From: Alan Hanson Date: Tue, 11 Nov 2025 10:45:12 -0800 Subject: [PATCH 28/36] Really fix it, come on --- ctop/src/main.rs | 48 +++++++++++++++++++++++++++++------------------- 1 file changed, 29 insertions(+), 19 deletions(-) diff --git a/ctop/src/main.rs b/ctop/src/main.rs index df1cd08e9..2015d8c8b 100644 --- a/ctop/src/main.rs +++ b/ctop/src/main.rs @@ -423,13 +423,15 @@ fn format_row( /// Uses Unicode block characters to show trend: ▁▂▃▄▅▆▇█ /// If global_max is provided, scales relative to that value for /// cross-session comparison +/// The sparkline is right-aligned: newest value at rightmost column, +/// older values scroll left, padding with spaces on the left if needed fn render_sparkline( history: &VecDeque, width: usize, global_max: u64, ) -> String { if history.is_empty() || width == 0 { - return String::new(); + return " ".repeat(width); } // Unicode block characters from lowest to highest @@ -447,14 +449,14 @@ fn render_sparkline( .collect(); if samples.is_empty() { - return String::new(); + return " ".repeat(width); } // Use global max for scaling (minimum 1 to avoid division by zero) let max = global_max.max(1); // Map each value to a block character - samples + let sparkline: String = samples .iter() .map(|&val| { if val == 0 { @@ -464,7 +466,15 @@ fn render_sparkline( BLOCKS[normalized.min(7)] } }) - .collect() + .collect(); + + // Right-align: pad left with spaces if we have fewer samples than width + if sparkline.chars().count() < width { + let padding = width - sparkline.chars().count(); + format!("{}{}", " ".repeat(padding), sparkline) + } else { + sparkline + } } /// Subprocess reader task - spawns dtrace command and reads JSON output @@ -916,20 +926,17 @@ async fn display_task( ); write!(stdout, "{}", row)?; - // Render sparkline in remaining space + // Render sparkline right-aligned to fill remaining space to terminal edge // Account for the indicator character (1 char) let row_len = row.chars().count() + 1; if terminal_width > row_len as u16 { - let sparkline_width = - (terminal_width as usize - row_len).saturating_sub(1); - if sparkline_width > 0 { - let sparkline = render_sparkline( - &session_data.delta_history, - sparkline_width, - global_max, - ); - write!(stdout, " {}", sparkline)?; - } + let sparkline_width = terminal_width as usize - row_len; + let sparkline = render_sparkline( + &session_data.delta_history, + sparkline_width, + global_max, + ); + write!(stdout, "{}", sparkline)?; } execute!(stdout, Clear(ClearType::UntilNewLine))?; @@ -1327,7 +1334,8 @@ mod tests { let history = VecDeque::new(); let sparkline = render_sparkline(&history, 10, 100); - assert_eq!(sparkline, ""); + // Empty history should return spaces to maintain right alignment + assert_eq!(sparkline, " "); // 10 spaces } #[test] @@ -1337,7 +1345,7 @@ mod tests { history.push_back(20); let sparkline = render_sparkline(&history, 0, 100); - assert_eq!(sparkline, ""); + assert_eq!(sparkline, ""); // Empty string for 0 width } #[test] @@ -1347,8 +1355,10 @@ mod tests { let sparkline = render_sparkline(&history, 10, 100); - // Should have one character - assert_eq!(sparkline.chars().count(), 1); + // Should have 10 characters total (9 spaces + 1 block, right-aligned) + assert_eq!(sparkline.chars().count(), 10); + // Last character should be the data value + assert_ne!(sparkline.chars().last().unwrap(), ' '); } #[test] From 99104acfa545fbaa7fd8068a09a59a2902113d34 Mon Sep 17 00:00:00 2001 From: Alan Hanson Date: Mon, 17 Nov 2025 16:47:03 -0800 Subject: [PATCH 29/36] Update TODO.md --- ctop/TODO.md | 1 + 1 file changed, 1 insertion(+) diff --git a/ctop/TODO.md b/ctop/TODO.md index 3da839e42..cc46bdd54 100644 --- a/ctop/TODO.md +++ b/ctop/TODO.md @@ -31,6 +31,7 @@ 3: 0 and max for all sessions (while just shown the current selections data). ### Program improvements. +- [ ] Store 500 data points for each session. - [ ] Add an option to only run for a specific number of seconds then exit. - [ ] Add an option to keep the last state displayed, or, really, reproduce the final screen but after we have exited the curses window. From 10593d661a0fcc8674b70cf476317411338c7888 Mon Sep 17 00:00:00 2001 From: Alan Hanson Date: Thu, 15 Jan 2026 10:26:17 -0800 Subject: [PATCH 30/36] Polish the ctop/cmon changes Extract calculate_visible_rows() helper function to eliminate duplicated terminal layout calculation that appeared in 4 places. Adds comprehensive documentation of the 7-line terminal layout. Fix integration test to parse into actual DtraceWrapper type instead of generic serde_json::Value, ensuring test catches real compatibility issues with production code. Remove brittle test_dtrace_display_enum_count test that required manual updates whenever enum variants changed Remove unused strum_macros dependency and EnumIter import from cmon --- Cargo.lock | 3 +- cmon-common/src/lib.rs | 13 +------- cmon/Cargo.toml | 1 - cmon/src/main.rs | 1 - ctop/src/main.rs | 60 +++++++++++++++------------------- ctop/tests/integration_test.rs | 19 ++++++----- 6 files changed, 39 insertions(+), 58 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 414c486b7..cc98214f9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -846,7 +846,6 @@ dependencies = [ "serde", "serde_json", "strum 0.27.2", - "strum_macros 0.27.2", "tokio", ] @@ -3628,7 +3627,7 @@ dependencies = [ "indoc", "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.110", ] [[package]] diff --git a/cmon-common/src/lib.rs b/cmon-common/src/lib.rs index 65f588194..11dfeb2a0 100644 --- a/cmon-common/src/lib.rs +++ b/cmon-common/src/lib.rs @@ -1,4 +1,4 @@ -// Copyright 2025 Oxide Computer Company +// Copyright 2026 Oxide Computer Company //! Common types and utilities shared between cmon and ctop @@ -211,15 +211,4 @@ mod tests { assert_eq!(format!("{:?}", display), format!("{:?}", copied)); assert_eq!(format!("{:?}", display), format!("{:?}", cloned)); } - - #[test] - fn test_dtrace_display_enum_count() { - // Verify we have the expected number of variants - let count = DtraceDisplay::iter().count(); - assert_eq!( - count, 25, - "Expected 25 DtraceDisplay variants, found {}", - count - ); - } } diff --git a/cmon/Cargo.toml b/cmon/Cargo.toml index 447a52e6a..cc65cde54 100644 --- a/cmon/Cargo.toml +++ b/cmon/Cargo.toml @@ -16,6 +16,5 @@ ratatui.workspace = true serde.workspace = true serde_json.workspace = true strum.workspace = true -strum_macros.workspace = true tokio.workspace = true crucible-workspace-hack.workspace = true diff --git a/cmon/src/main.rs b/cmon/src/main.rs index c3e6fbfe9..cd946442c 100644 --- a/cmon/src/main.rs +++ b/cmon/src/main.rs @@ -6,7 +6,6 @@ use crucible_control_client::Client; use crucible_protocol::ClientId; use std::io::{self, BufRead}; use strum::IntoEnumIterator; -use strum_macros::EnumIter; use tokio::time::{Duration, sleep}; /// Connect to crucible control server diff --git a/ctop/src/main.rs b/ctop/src/main.rs index 2015d8c8b..849e8eef9 100644 --- a/ctop/src/main.rs +++ b/ctop/src/main.rs @@ -1,4 +1,4 @@ -// Copyright 2025 Oxide Computer Company +// Copyright 2026 Oxide Computer Company //! Standalone ctop - curses-based top-like display of crucible dtrace data @@ -419,6 +419,28 @@ fn format_row( result } +/// Calculate the number of visible rows for session display +/// +/// Terminal layout (total FIXED_LINES = 7): +/// 1 header (timestamp) +/// 1 blank line +/// 1 column headers +/// N session lines (variable, returned by this function) +/// 1 scroll indicator (↑ more above) +/// 1 scroll indicator (↓ more below) +/// 1 blank line +/// 1 footer (help text) +/// +/// Returns at least 1 visible row to prevent crashes. +fn calculate_visible_rows(terminal_height: u16) -> usize { + const FIXED_LINES: u16 = 7; + if terminal_height > FIXED_LINES { + (terminal_height - FIXED_LINES) as usize + } else { + 1 // Minimum to avoid crashes + } +} + /// Render a sparkline from delta history /// Uses Unicode block characters to show trend: ▁▂▃▄▅▆▇█ /// If global_max is provided, scales relative to that value for @@ -872,19 +894,9 @@ async fn display_task( let scroll_offset = state_guard.scroll_offset; // Calculate visible window - // Terminal layout: 1 header + 1 blank + 1 column headers + sessions + 1 blank + 1 footer - // Always reserve 2 lines for scroll indicators to prevent flickering let total_sessions = sessions.len(); let has_more_above = scroll_offset > 0; - - // Always reserve space for both scroll indicators to prevent layout changes - let fixed_lines = 7; // 5 base lines + 2 for scroll indicators - - let visible_rows = if terminal_height > fixed_lines as u16 { - (terminal_height - fixed_lines as u16) as usize - } else { - 1 // Minimum of 1 row to avoid crashes - }; + let visible_rows = calculate_visible_rows(terminal_height); let scroll_end = (scroll_offset + visible_rows).min(total_sessions); let has_more_below = scroll_end < total_sessions; @@ -1039,14 +1051,8 @@ async fn display_task( state_guard.selected_index += 1; // Calculate visible rows to determine scroll behavior - // Always use fixed 7 lines to prevent flickering - let fixed_lines = 7; let visible_rows = - if terminal_size.1 > fixed_lines as u16 { - (terminal_size.1 - fixed_lines as u16) as usize - } else { - 1 - }; + calculate_visible_rows(terminal_size.1); // Scroll down if selection moves below visible window let scroll_end = @@ -1064,14 +1070,8 @@ async fn display_task( // Page up (only in table mode) if !state_guard.detail_mode && num_sessions > 0 { // Calculate visible rows - // Always use fixed 7 lines to prevent flickering - let fixed_lines = 7; let visible_rows = - if terminal_size.1 > fixed_lines as u16 { - (terminal_size.1 - fixed_lines as u16) as usize - } else { - 1 - }; + calculate_visible_rows(terminal_size.1); // Move up by visible_rows state_guard.selected_index = state_guard @@ -1090,14 +1090,8 @@ async fn display_task( // Page down (only in table mode) if !state_guard.detail_mode && num_sessions > 0 { // Calculate visible rows - // Always use fixed 7 lines to prevent flickering - let fixed_lines = 7; let visible_rows = - if terminal_size.1 > fixed_lines as u16 { - (terminal_size.1 - fixed_lines as u16) as usize - } else { - 1 - }; + calculate_visible_rows(terminal_size.1); // Move down by visible_rows let new_selected = diff --git a/ctop/tests/integration_test.rs b/ctop/tests/integration_test.rs index 6ad02eb9b..6c4d848e6 100644 --- a/ctop/tests/integration_test.rs +++ b/ctop/tests/integration_test.rs @@ -1,4 +1,4 @@ -// Copyright 2025 Oxide Computer Company +// Copyright 2026 Oxide Computer Company //! Integration tests for ctop //! @@ -83,6 +83,7 @@ //! cargo test -p ctop --test integration_test test_parse_dtrace_json_format //! ``` +use cmon_common::DtraceWrapper; use serde_json; /// Test that we can parse valid DTrace JSON output @@ -132,19 +133,19 @@ fn test_parse_dtrace_json_format() { } }"#; - // Attempt to parse - this validates the JSON structure matches what - // cmon_common::DtraceWrapper expects - let result: Result = - serde_json::from_str(sample_json); + // Parse into the actual DtraceWrapper type used in production + // This ensures the test catches real compatibility issues + let result: Result = serde_json::from_str(sample_json); assert!( result.is_ok(), - "Sample dtrace JSON should parse successfully" + "Sample dtrace JSON should parse into DtraceWrapper: {:?}", + result.err() ); let parsed = result.unwrap(); - assert_eq!(parsed["pid"], 12345); - assert_eq!(parsed["status"]["session_id"], "session-456"); - assert_eq!(parsed["status"]["ds_state"][0], "Active"); + assert_eq!(parsed.pid, 12345); + assert_eq!(parsed.status.session_id, "session-456"); + assert_eq!(parsed.status.ds_state[0], "Active"); } /// Test handling of invalid JSON From 97f8bbdff18179cde409d902e998ab34df0a986f Mon Sep 17 00:00:00 2001 From: Alan Hanson Date: Mon, 2 Feb 2026 13:55:18 -0800 Subject: [PATCH 31/36] more tests for ctop --- ctop/src/main.rs | 37 ++++++++++++++++++++++++++++++------- 1 file changed, 30 insertions(+), 7 deletions(-) diff --git a/ctop/src/main.rs b/ctop/src/main.rs index 849e8eef9..8202d98d3 100644 --- a/ctop/src/main.rs +++ b/ctop/src/main.rs @@ -1392,9 +1392,10 @@ mod tests { history.push_back(0); history.push_back(100); - let sparkline = render_sparkline(&history, 10, 100); + // Use width=2 to match data size (no padding) + let sparkline = render_sparkline(&history, 2, 100); - // Should use valid unicode block characters + // Should use valid unicode block characters (no spaces since width matches data) for c in sparkline.chars() { assert!( ['▁', '▂', '▃', '▄', '▅', '▆', '▇', '█'].contains(&c), @@ -1402,6 +1403,14 @@ mod tests { c ); } + + // Verify correct blocks: 0 maps to lowest, 100 maps to highest + let chars: Vec = sparkline.chars().collect(); + assert_eq!(chars[0], '▁', "Value 0 should map to lowest block"); + assert_eq!( + chars[1], '█', + "Value 100 (max) should map to highest block" + ); } #[test] @@ -1418,19 +1427,33 @@ mod tests { history.push_back(100); // Test with global max = 200 (should scale differently than 100) - let sparkline1 = render_sparkline(&history, 10, 200); - let sparkline2 = render_sparkline(&history, 10, 100); + // Use width=2 to avoid padding and test actual data + let sparkline1 = render_sparkline(&history, 2, 200); + let sparkline2 = render_sparkline(&history, 2, 100); // With higher global max, the values should appear relatively lower let chars1: Vec = sparkline1.chars().collect(); let chars2: Vec = sparkline2.chars().collect(); - // Second sparkline should use higher blocks (value 100 is max in range 0-100, - // but only midpoint in range 0-200) + // Sparkline1 (max=200): value 100 is only halfway, so should use lower blocks + // Sparkline2 (max=100): value 100 is the maximum, so should use highest block + // + // For value 100: + // normalized1 = 100/200 * 7 = 3.5 → index 3 = '▄' + // normalized2 = 100/100 * 7 = 7.0 → index 7 = '█' + // + // Therefore chars1[1] should be '▄' and chars2[1] should be '█' assert!( chars1[1] < chars2[1], - "Expected normalization to affect block height" + "Expected normalization to affect block height: \ + sparkline1[1]='{}' should be < sparkline2[1]='{}'", + chars1[1], + chars2[1] ); + + // Verify the actual characters are as expected + assert_eq!(chars1[1], '▄', "Value 100 with max=200 should be ▄"); + assert_eq!(chars2[1], '█', "Value 100 with max=100 should be █"); } // ============================================================================ From 8e9a672f735c3457b69e300e42126f60284fdc10 Mon Sep 17 00:00:00 2001 From: Alan Hanson Date: Sat, 14 Feb 2026 15:02:54 -0800 Subject: [PATCH 32/36] Fix clippy --- ctop/src/main.rs | 28 +++------------------------- ctop/tests/integration_test.rs | 1 - 2 files changed, 3 insertions(+), 26 deletions(-) diff --git a/ctop/src/main.rs b/ctop/src/main.rs index 8202d98d3..bf782a343 100644 --- a/ctop/src/main.rs +++ b/ctop/src/main.rs @@ -1511,31 +1511,9 @@ mod tests { ); // Last item is most recent } - #[test] - fn test_stale_threshold_constant() { - // Verify STALE_THRESHOLD_SECS is reasonable - assert!( - STALE_THRESHOLD_SECS > 0, - "Stale threshold should be positive" - ); - assert!( - STALE_THRESHOLD_SECS <= 60, - "Stale threshold should be under a minute" - ); - } - - #[test] - fn test_max_delta_history_constant() { - // Verify MAX_DELTA_HISTORY is reasonable for sparklines - assert!( - MAX_DELTA_HISTORY >= 50, - "Should store enough history for reasonable sparklines" - ); - assert!( - MAX_DELTA_HISTORY <= 1000, - "History shouldn't be excessively large" - ); - } + // Note: Constant validation tests removed - clippy warns that assertions + // on constants are optimized out. Constants are validated at compile time + // by their usage in the code. // ============================================================================ // Future Testing Opportunities diff --git a/ctop/tests/integration_test.rs b/ctop/tests/integration_test.rs index 6c4d848e6..e9fe15fcb 100644 --- a/ctop/tests/integration_test.rs +++ b/ctop/tests/integration_test.rs @@ -84,7 +84,6 @@ //! ``` use cmon_common::DtraceWrapper; -use serde_json; /// Test that we can parse valid DTrace JSON output #[test] From 75794cde2f78f4af776ee949b48403da29782eff Mon Sep 17 00:00:00 2001 From: Alan Hanson Date: Sat, 14 Feb 2026 15:03:12 -0800 Subject: [PATCH 33/36] Fix clippy --- cmon-common/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmon-common/src/lib.rs b/cmon-common/src/lib.rs index 11dfeb2a0..fa1fc190f 100644 --- a/cmon-common/src/lib.rs +++ b/cmon-common/src/lib.rs @@ -206,7 +206,7 @@ mod tests { // Verify Copy and Clone work correctly let display = DtraceDisplay::Pid; let copied = display; - let cloned = display.clone(); + let cloned = display; // Copy trait, no need for .clone() assert_eq!(format!("{:?}", display), format!("{:?}", copied)); assert_eq!(format!("{:?}", display), format!("{:?}", cloned)); From f045a30efc09bbc674945580b01b13c1b1dc5481 Mon Sep 17 00:00:00 2001 From: Alan Hanson Date: Sat, 14 Feb 2026 15:09:43 -0800 Subject: [PATCH 34/36] remove unused _is_stale --- ctop/src/main.rs | 8 -------- 1 file changed, 8 deletions(-) diff --git a/ctop/src/main.rs b/ctop/src/main.rs index bf782a343..1878a73a8 100644 --- a/ctop/src/main.rs +++ b/ctop/src/main.rs @@ -230,7 +230,6 @@ fn format_row( d_out: &DtraceInfo, precomputed_delta: Option, dd: &[DtraceDisplay], - _is_stale: bool, ) -> String { let mut result = String::new(); @@ -624,17 +623,11 @@ fn render_detail_view( // Format the session data row let display_fields = default_display_fields(); - let is_stale = session_data - .last_updated - .elapsed() - .as_secs() - > STALE_THRESHOLD_SECS; let row_data = format_row( session_data.pid, &session_data.dtrace_info, session_data.current_delta, &display_fields, - is_stale, ); // Render session data at top @@ -934,7 +927,6 @@ async fn display_task( &session_data.dtrace_info, session_data.current_delta, &display_fields, - is_stale, ); write!(stdout, "{}", row)?; From defe9bc047d09446896733a4a1f20cd91b1960e6 Mon Sep 17 00:00:00 2001 From: Alan Hanson Date: Sat, 14 Feb 2026 15:31:10 -0800 Subject: [PATCH 35/36] Update TODO --- ctop/TODO.md | 35 ----------------------------------- 1 file changed, 35 deletions(-) diff --git a/ctop/TODO.md b/ctop/TODO.md index cc46bdd54..87d1feb7a 100644 --- a/ctop/TODO.md +++ b/ctop/TODO.md @@ -15,13 +15,6 @@ - [ ] Display detailed graph for all selected sessions simultaneously - [ ] Show visual indicator for which sessions are selected in the main list -### Screen Scrolling -- [ ] Add ability to scroll the session list when there are more sessions than - can fit on screen -- [ ] Implement scroll indicators (e.g., "↑ More above", "↓ More below") -- [ ] Consider adding Page Up/Page Down support for faster navigation -- [ ] Remove sessions that have not updated after 60 seconds - ### Normalization Improvements - [ ] Change normalization toggle. - Make the same key rotate between the three options. @@ -40,31 +33,3 @@ - [ ] Allow some downstairs individual stats to be combined into a "sum", like connections for each downstairs summed into a single value. Not all dtrace probes could do this, so give just options for ones we can. - -## Implementation Notes - -### Session Selection -- Maintain a `HashSet` of selected session IDs in `CtopState` -- Space key toggles selection of currently highlighted session -- Selected sessions marked with `[*]` or similar indicator -- Detail view shows all selected sessions' sparklines stacked or overlaid - -### Scrolling -- Track `scroll_offset` in display state -- Calculate visible window based on terminal height -- Adjust rendering to show sessions from `scroll_offset` to - `scroll_offset + visible_rows` -- Update scroll offset on up/down arrow keys - -### Normalization -- Modify `render_detail_view()` and sparkline rendering -- For zero-based normalization: always use `min = 0`, compute `max` from data -- For selection-based normalization: filter sessions to only selected ones - before computing min/max -- Add visual indicator in detail view header showing normalization mode - -## UI/UX Considerations -- Document new keyboard shortcuts in help or header -- Ensure selected sessions are visually distinct -- Consider performance impact of rendering multiple detailed graphs -- Test with many sessions (10+, 50+, 100+) to ensure scrolling performs well From 59356dbe60c15a36ffe06f282184f10210d934e9 Mon Sep 17 00:00:00 2001 From: Alan Hanson Date: Sat, 14 Feb 2026 23:33:46 +0000 Subject: [PATCH 36/36] Fix cargo hakari --- Cargo.lock | 1 + workspace-hack/Cargo.toml | 1 + 2 files changed, 2 insertions(+) diff --git a/Cargo.lock b/Cargo.lock index cc98214f9..18695279f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1620,6 +1620,7 @@ dependencies = [ "clap_builder", "crossbeam-epoch", "crossbeam-utils", + "crossterm", "crypto-common", "digest", "dof 0.3.0", diff --git a/workspace-hack/Cargo.toml b/workspace-hack/Cargo.toml index 23bfc8232..53fe52457 100644 --- a/workspace-hack/Cargo.toml +++ b/workspace-hack/Cargo.toml @@ -27,6 +27,7 @@ clap = { version = "4", features = ["cargo", "derive", "env", "wrap_help"] } clap_builder = { version = "4", default-features = false, features = ["cargo", "color", "env", "std", "suggestions", "usage", "wrap_help"] } crossbeam-epoch = { version = "0.9" } crossbeam-utils = { version = "0.8" } +crossterm = { version = "0.28", features = ["serde"] } crypto-common = { version = "0.1", default-features = false, features = ["getrandom", "std"] } digest = { version = "0.10", features = ["mac", "std"] } either = { version = "1", features = ["use_std"] }