diff --git a/nexus/db-model/src/fm/case.rs b/nexus/db-model/src/fm/case.rs index efe5d4e5487..1922df26a30 100644 --- a/nexus/db-model/src/fm/case.rs +++ b/nexus/db-model/src/fm/case.rs @@ -58,6 +58,7 @@ impl CaseMetadata { de, comment, alerts_requested: _, + support_bundles_requested: _, ereports: _, } = case; Self { diff --git a/nexus/db-model/src/schema_versions.rs b/nexus/db-model/src/schema_versions.rs index 5e0fdd10b09..d80d50e35bd 100644 --- a/nexus/db-model/src/schema_versions.rs +++ b/nexus/db-model/src/schema_versions.rs @@ -16,7 +16,7 @@ use std::{collections::BTreeMap, sync::LazyLock}; /// /// This must be updated when you change the database schema. Refer to /// schema/crdb/README.adoc in the root of this repository for details. -pub const SCHEMA_VERSION: Version = Version::new(240, 0, 0); +pub const SCHEMA_VERSION: Version = Version::new(241, 0, 0); /// List of all past database schema versions, in *reverse* order /// @@ -28,6 +28,7 @@ static KNOWN_VERSIONS: LazyLock> = LazyLock::new(|| { // | leaving the first copy as an example for the next person. // v // KnownVersion::new(next_int, "unique-dirname-with-the-sql-files"), + KnownVersion::new(241, "fm-support-bundle-request"), KnownVersion::new(240, "multicast-drop-mvlan"), KnownVersion::new(239, "fm-alert-request"), KnownVersion::new(238, "fewer-nullable-columns"), diff --git a/nexus/db-queries/src/db/datastore/ereport.rs b/nexus/db-queries/src/db/datastore/ereport.rs index c2456c3c871..78488f90f65 100644 --- a/nexus/db-queries/src/db/datastore/ereport.rs +++ b/nexus/db-queries/src/db/datastore/ereport.rs @@ -27,6 +27,7 @@ use nexus_db_errors::public_error_from_diesel; use nexus_db_lookup::DbConnection; use nexus_db_schema::schema::ereport::dsl; use nexus_types::fm::ereport as fm; +use nexus_types::fm::ereport::EreportFilters; use nexus_types::fm::ereport::EreportId; use omicron_common::api::external::CreateResult; use omicron_common::api::external::DataPageParams; @@ -48,44 +49,6 @@ pub struct EreporterRestartBySerial { pub ereports: u32, } -/// A set of filters for fetching ereports. -#[derive(Clone, Debug, Default, Eq, PartialEq)] -pub struct EreportFilters { - /// If present, include only ereports that were collected at the specified - /// timestamp or later. - /// - /// If `end_time` is also present, this value *must* be earlier than - /// `end_time`. - pub start_time: Option>, - /// If present, include only ereports that were collected at the specified - /// timestamp or before. - /// - /// If `start_time` is also present, this value *must* be later than - /// `start_time`. - pub end_time: Option>, - /// If this list is non-empty, include only ereports that were reported by - /// systems with the provided serial numbers. - pub only_serials: Vec, - /// If this list is non-empty, include only ereports with the provided class - /// strings. - // TODO(eliza): globbing could be nice to add here eventually... - pub only_classes: Vec, -} - -impl EreportFilters { - fn check_time_range(&self) -> Result<(), Error> { - if let (Some(start), Some(end)) = (self.start_time, self.end_time) { - if start > end { - return Err(Error::invalid_request( - "start time must be before end time", - )); - } - } - - Ok(()) - } -} - impl DataStore { /// Fetch an ereport by its restart ID and ENA. /// diff --git a/nexus/db-queries/src/db/datastore/fm.rs b/nexus/db-queries/src/db/datastore/fm.rs index 81072530b5c..f3334621deb 100644 --- a/nexus/db-queries/src/db/datastore/fm.rs +++ b/nexus/db-queries/src/db/datastore/fm.rs @@ -385,6 +385,7 @@ impl DataStore { comment, ereports, alerts_requested, + support_bundles_requested: iddqd::IdOrdMap::new(), } })); } @@ -1552,6 +1553,7 @@ mod tests { de, ereports, alerts_requested, + support_bundles_requested: _, } = case; let case_id = id; let Some(expected) = this.cases.get(&case_id) else { @@ -1770,6 +1772,7 @@ mod tests { de: fm::DiagnosisEngineKind::PowerShelf, ereports, alerts_requested, + support_bundles_requested: iddqd::IdOrdMap::new(), comment: "my cool case".to_string(), } }; @@ -1802,6 +1805,7 @@ mod tests { de: fm::DiagnosisEngineKind::PowerShelf, ereports, alerts_requested, + support_bundles_requested: iddqd::IdOrdMap::new(), comment: "break in case of emergency".to_string(), } }; diff --git a/nexus/db-queries/src/db/datastore/mod.rs b/nexus/db-queries/src/db/datastore/mod.rs index 37525b020b8..bca532e171d 100644 --- a/nexus/db-queries/src/db/datastore/mod.rs +++ b/nexus/db-queries/src/db/datastore/mod.rs @@ -141,7 +141,6 @@ pub use disk::LocalStorageAllocation; pub use disk::LocalStorageDisk; pub use dns::DataStoreDnsTest; pub use dns::DnsVersionUpdateBuilder; -pub use ereport::EreportFilters; pub use external_ip::FloatingIpAllocation; pub use external_subnet::ExternalSubnetBeginOpResult; pub use external_subnet::ExternalSubnetCompleteOpResult; diff --git a/nexus/db-schema/src/schema.rs b/nexus/db-schema/src/schema.rs index d108372694b..fb162694fdf 100644 --- a/nexus/db-schema/src/schema.rs +++ b/nexus/db-schema/src/schema.rs @@ -1590,6 +1590,7 @@ table! { assigned_nexus -> Nullable, user_comment -> Nullable, + fm_case_id -> Nullable, } } @@ -3183,6 +3184,80 @@ table! { } } +// FM support bundle requests, stored per-sitrep like alert requests. +table! { + fm_support_bundle_request (sitrep_id, id) { + id -> Uuid, + sitrep_id -> Uuid, + requested_sitrep_id -> Uuid, + case_id -> Uuid, + } +} + +// Per-variant data selection tables for fm_support_bundle_request. +// Each table corresponds to a BundleData variant. Row existence means +// "include this category in the bundle." No rows across any variant +// table means "collect everything." + +// BundleData::Reconfigurator (unit variant, no filter columns) +table! { + fm_sb_req_reconfigurator (sitrep_id, request_id) { + sitrep_id -> Uuid, + request_id -> Uuid, + } +} + +// BundleData::SledCubbyInfo (unit variant, no filter columns) +table! { + fm_sb_req_sled_cubby_info (sitrep_id, request_id) { + sitrep_id -> Uuid, + request_id -> Uuid, + } +} + +// BundleData::SpDumps (unit variant, no filter columns) +table! { + fm_sb_req_sp_dumps (sitrep_id, request_id) { + sitrep_id -> Uuid, + request_id -> Uuid, + } +} + +// BundleData::HostInfo(HashSet) +table! { + fm_sb_req_host_info (sitrep_id, request_id) { + sitrep_id -> Uuid, + request_id -> Uuid, + all_sleds -> Bool, + sled_ids -> Array, + } +} + +// BundleData::Ereports(EreportFilters) +table! { + fm_sb_req_ereports (sitrep_id, request_id) { + sitrep_id -> Uuid, + request_id -> Uuid, + start_time -> Nullable, + end_time -> Nullable, + only_serials -> Array, + only_classes -> Array, + } +} + +// The per-variant tables use composite keys (sitrep_id, request_id) +// matching fm_support_bundle_request's (sitrep_id, id), so joinable! +// cannot be used (it requires single-column FKs). Queries must use +// explicit .on() clauses instead. +allow_tables_to_appear_in_same_query!( + fm_support_bundle_request, + fm_sb_req_reconfigurator, + fm_sb_req_sled_cubby_info, + fm_sb_req_sp_dumps, + fm_sb_req_host_info, + fm_sb_req_ereports, +); + table! { trust_quorum_configuration (rack_id, epoch) { rack_id -> Uuid, diff --git a/nexus/fm/src/builder/case.rs b/nexus/fm/src/builder/case.rs index 29d3cc3aa4d..f5b6a2d1419 100644 --- a/nexus/fm/src/builder/case.rs +++ b/nexus/fm/src/builder/case.rs @@ -67,6 +67,7 @@ impl AllCases { comment: String::new(), ereports: Default::default(), alerts_requested: Default::default(), + support_bundles_requested: Default::default(), }; entry.insert(CaseBuilder::new( &self.log, sitrep_id, case, case_rng, diff --git a/nexus/src/app/background/tasks/fm_rendezvous.rs b/nexus/src/app/background/tasks/fm_rendezvous.rs index 4d15465b84c..9e893d83e42 100644 --- a/nexus/src/app/background/tasks/fm_rendezvous.rs +++ b/nexus/src/app/background/tasks/fm_rendezvous.rs @@ -230,6 +230,7 @@ mod tests { de: fm::DiagnosisEngineKind::PowerShelf, alerts_requested: iddqd::IdOrdMap::new(), ereports: iddqd::IdOrdMap::new(), + support_bundles_requested: iddqd::IdOrdMap::new(), comment: "my great case".to_string(), }; case1 @@ -305,6 +306,7 @@ mod tests { de: fm::DiagnosisEngineKind::PowerShelf, alerts_requested: iddqd::IdOrdMap::new(), ereports: iddqd::IdOrdMap::new(), + support_bundles_requested: iddqd::IdOrdMap::new(), comment: "my other great case".to_string(), }; case2 diff --git a/nexus/src/app/background/tasks/support_bundle/request.rs b/nexus/src/app/background/tasks/support_bundle/request.rs index 89d43ecc73c..dac2fb237be 100644 --- a/nexus/src/app/background/tasks/support_bundle/request.rs +++ b/nexus/src/app/background/tasks/support_bundle/request.rs @@ -4,10 +4,12 @@ //! Support bundle request types and data selection -use nexus_db_queries::db::datastore::EreportFilters; +use nexus_types::fm::ereport::EreportFilters; +use nexus_types::support_bundle::{ + BundleData, BundleDataCategory, BundleDataSelection, SledSelection, +}; + use omicron_uuid_kinds::SledUuid; -use std::collections::HashMap; -use std::collections::HashSet; use std::num::NonZeroU64; /// We use "/var/tmp" to use Nexus' filesystem for temporary storage, @@ -18,121 +20,6 @@ pub const TEMPDIR: &str = "/var/tmp"; /// within a single streaming request. pub const CHUNK_SIZE: NonZeroU64 = NonZeroU64::new(1024 * 1024 * 1024).unwrap(); -/// Describes the category of support bundle data. -#[derive(Debug, Clone, Copy, Hash, Eq, PartialEq)] -pub enum BundleDataCategory { - /// Collects reconfigurator state (some of the latest blueprints, - /// information about the target blueprint). - Reconfigurator, - /// Collects info from sled agents, running a handful of - /// diagnostic commands (e.g., zoneadm, dladm, etc). - HostInfo, - /// Collects sled serial numbers, cubby numbers, and UUIDs. - SledCubbyInfo, - /// Saves task dumps from SPs. - SpDumps, - /// Collects ereports. - Ereports, -} - -/// Specifies what data to collect for a bundle data category. -/// -/// Each variant corresponds to a BundleDataCategory. -/// For categories without additional parameters, the variant is a unit variant. -/// For categories that can be filtered or configured, the variant contains -/// that configuration data. -#[derive(Debug, Clone, Eq, PartialEq)] -pub enum BundleData { - Reconfigurator, - HostInfo(HashSet), - SledCubbyInfo, - SpDumps, - Ereports(EreportFilters), -} - -impl BundleData { - fn category(&self) -> BundleDataCategory { - match self { - Self::Reconfigurator => BundleDataCategory::Reconfigurator, - Self::HostInfo(_) => BundleDataCategory::HostInfo, - Self::SledCubbyInfo => BundleDataCategory::SledCubbyInfo, - Self::SpDumps => BundleDataCategory::SpDumps, - Self::Ereports(_) => BundleDataCategory::Ereports, - } - } -} - -/// A collection of bundle data specifications. -/// -/// This wrapper ensures that categories and data always match - you can't -/// insert (BundleDataCategory::Reconfigurator, BundleData::SpDumps) -/// because each BundleData determines its own category. -#[derive(Debug, Clone)] -pub struct BundleDataSelection { - data: HashMap, -} - -impl BundleDataSelection { - pub fn new() -> Self { - Self { data: HashMap::new() } - } - - /// Inserts BundleData to be queried for a particular category within the - /// bundle. - /// - /// Each category of data can only be specified once (e.g., inserting - /// BundleData::HostInfo multiple times will only use the most-recently - /// inserted specification) - pub fn insert(&mut self, bundle_data: BundleData) { - self.data.insert(bundle_data.category(), bundle_data); - } - - pub fn contains(&self, category: BundleDataCategory) -> bool { - self.data.contains_key(&category) - } - - pub fn get(&self, category: BundleDataCategory) -> Option<&BundleData> { - self.data.get(&category) - } -} - -impl FromIterator for BundleDataSelection { - fn from_iter>(iter: T) -> Self { - let mut selection = Self::new(); - for bundle_data in iter { - selection.insert(bundle_data); - } - selection - } -} - -impl Default for BundleDataSelection { - fn default() -> Self { - [ - BundleData::Reconfigurator, - BundleData::HostInfo(HashSet::from([SledSelection::All])), - BundleData::SledCubbyInfo, - BundleData::SpDumps, - BundleData::Ereports(EreportFilters { - start_time: Some(chrono::Utc::now() - chrono::Days::new(7)), - ..EreportFilters::default() - }), - ] - .into_iter() - .collect() - } -} - -/// The set of sleds to include -/// -/// Multiple values of this enum are joined together into a HashSet. -/// Therefore "SledSelection::All" overrides specific sleds. -#[derive(Debug, Clone, Hash, Eq, PartialEq)] -pub enum SledSelection { - All, - Specific(SledUuid), -} - /// Specifies the data to be collected within the Support Bundle. #[derive(Clone)] pub struct BundleRequest { diff --git a/nexus/src/app/background/tasks/support_bundle/steps/ereports.rs b/nexus/src/app/background/tasks/support_bundle/steps/ereports.rs index 24d8272aefb..459be751b2f 100644 --- a/nexus/src/app/background/tasks/support_bundle/steps/ereports.rs +++ b/nexus/src/app/background/tasks/support_bundle/steps/ereports.rs @@ -6,6 +6,7 @@ use crate::app::background::tasks::support_bundle::collection::BundleCollection; use crate::app::background::tasks::support_bundle::step::CollectionStepOutput; +use nexus_types::fm::ereport::EreportFilters; use anyhow::Context; use camino::Utf8Path; @@ -13,7 +14,6 @@ use camino::Utf8PathBuf; use nexus_db_queries::context::OpContext; use nexus_db_queries::db::DataStore; use nexus_db_queries::db::datastore; -use nexus_db_queries::db::datastore::EreportFilters; use nexus_db_queries::db::pagination::Paginator; use nexus_types::fm::Ereport; use nexus_types::internal_api::background::SupportBundleEreportStatus; diff --git a/nexus/src/app/background/tasks/support_bundle_collector.rs b/nexus/src/app/background/tasks/support_bundle_collector.rs index 76794dd6ebe..2d75c63e6b9 100644 --- a/nexus/src/app/background/tasks/support_bundle_collector.rs +++ b/nexus/src/app/background/tasks/support_bundle_collector.rs @@ -441,7 +441,6 @@ mod test { use super::*; use crate::app::background::tasks::support_bundle::perfetto; - use crate::app::background::tasks::support_bundle::request::BundleData; use crate::app::support_bundles::SupportBundleQueryType; use http_body_util::BodyExt; use illumos_utils::zpool::ZpoolHealth; @@ -456,6 +455,7 @@ mod test { use nexus_types::internal_api::background::SupportBundleCollectionStep; use nexus_types::internal_api::background::SupportBundleEreportStatus; use nexus_types::inventory::SpType; + use nexus_types::support_bundle::BundleData; use omicron_common::api::external::ByteCount; use omicron_common::api::internal::shared::DatasetKind; use omicron_common::disk::DatasetConfig; diff --git a/nexus/types/src/fm/case.rs b/nexus/types/src/fm/case.rs index afcefd3dd93..991ce788ba1 100644 --- a/nexus/types/src/fm/case.rs +++ b/nexus/types/src/fm/case.rs @@ -5,8 +5,11 @@ use crate::alert::AlertClass; use crate::fm::DiagnosisEngineKind; use crate::fm::Ereport; +use crate::support_bundle::BundleDataSelection; use iddqd::{IdOrdItem, IdOrdMap}; -use omicron_uuid_kinds::{AlertUuid, CaseEreportUuid, CaseUuid, SitrepUuid}; +use omicron_uuid_kinds::{ + AlertUuid, CaseEreportUuid, CaseUuid, SitrepUuid, SupportBundleUuid, +}; use serde::{Deserialize, Serialize}; use std::fmt; use std::sync::Arc; @@ -21,6 +24,7 @@ pub struct Case { pub ereports: IdOrdMap, pub alerts_requested: IdOrdMap, + pub support_bundles_requested: IdOrdMap, pub comment: String, } @@ -88,6 +92,22 @@ impl iddqd::IdOrdItem for AlertRequest { iddqd::id_upcast!(); } +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +pub struct SupportBundleRequest { + pub id: SupportBundleUuid, + pub requested_sitrep_id: SitrepUuid, + pub data_selection: Option, +} + +impl iddqd::IdOrdItem for SupportBundleRequest { + type Key<'a> = &'a SupportBundleUuid; + fn key(&self) -> Self::Key<'_> { + &self.id + } + + iddqd::id_upcast!(); +} + struct DisplayCase<'a> { case: &'a Case, indent: usize, @@ -120,6 +140,7 @@ impl fmt::Display for DisplayCase<'_> { ereports, comment, alerts_requested, + support_bundles_requested, }, indent, sitrep_id, @@ -234,6 +255,44 @@ impl fmt::Display for DisplayCase<'_> { } } + if !support_bundles_requested.is_empty() { + writeln!(f, "\n{:>indent$}support bundles requested:", "")?; + writeln!(f, "{:>indent$}-------------------------", "")?; + + let indent = indent + 2; + for SupportBundleRequest { + id, + requested_sitrep_id, + data_selection, + } in support_bundles_requested.iter() + { + const REQUESTED_IN: &str = "requested in:"; + const DATA: &str = "data:"; + const WIDTH: usize = const_max_len(&[REQUESTED_IN, DATA]); + + writeln!(f, "{BULLET:>indent$}bundle {id}",)?; + writeln!( + f, + "{:>indent$}{REQUESTED_IN: { + writeln!( + f, + "{:>indent$}{DATA: { + writeln!(f, "{:>indent$}{DATA}", "")?; + writeln!(f, "{}\n", selection.display(indent + 2))?; + } + } + } + } + writeln!(f)?; Ok(()) @@ -244,11 +303,17 @@ impl fmt::Display for DisplayCase<'_> { mod tests { use super::*; use crate::fm::DiagnosisEngineKind; + use crate::fm::ereport::EreportFilters; use crate::inventory::SpType; + use crate::support_bundle::{ + BundleData, BundleDataSelection, SledSelection, + }; use ereport_types::{Ena, EreportId}; use omicron_uuid_kinds::{ AlertUuid, CaseUuid, EreporterRestartUuid, OmicronZoneUuid, SitrepUuid, + SupportBundleUuid, }; + use std::collections::HashSet; use std::str::FromStr; use std::sync::Arc; @@ -276,6 +341,12 @@ mod tests { let alert2_id = AlertUuid::from_str("8a6f88ef-c436-44a9-b4cb-cae91d7306c9") .unwrap(); + let bundle1_id = + SupportBundleUuid::from_str("d1a2b3c4-e5f6-7890-abcd-ef1234567890") + .unwrap(); + let bundle2_id = + SupportBundleUuid::from_str("a9b8c7d6-e5f4-3210-fedc-ba0987654321") + .unwrap(); // Create some ereports let mut ereports = IdOrdMap::new(); @@ -349,6 +420,32 @@ mod tests { }) .unwrap(); + let mut bundle1_data = BundleDataSelection::new(); + bundle1_data.insert(BundleData::Reconfigurator); + bundle1_data.insert(BundleData::SpDumps); + bundle1_data + .insert(BundleData::HostInfo(HashSet::from([SledSelection::All]))); + bundle1_data.insert(BundleData::Ereports(EreportFilters { + only_classes: vec!["hw.pwr.*".to_string()], + ..Default::default() + })); + + let mut support_bundles_requested = IdOrdMap::new(); + support_bundles_requested + .insert_unique(SupportBundleRequest { + id: bundle1_id, + requested_sitrep_id: created_sitrep_id, + data_selection: Some(bundle1_data), + }) + .unwrap(); + support_bundles_requested + .insert_unique(SupportBundleRequest { + id: bundle2_id, + requested_sitrep_id: closed_sitrep_id, + data_selection: None, + }) + .unwrap(); + // Create the case let case = Case { id: case_id, @@ -357,6 +454,7 @@ mod tests { de: DiagnosisEngineKind::PowerShelf, ereports, alerts_requested, + support_bundles_requested, comment: "Power shelf rectifier added and removed here :-)" .to_string(), }; diff --git a/nexus/types/src/fm/ereport.rs b/nexus/types/src/fm/ereport.rs index 1ad263af15f..dd8ebc725f0 100644 --- a/nexus/types/src/fm/ereport.rs +++ b/nexus/types/src/fm/ereport.rs @@ -6,6 +6,7 @@ use crate::inventory::SpType; use chrono::{DateTime, Utc}; +use omicron_common::api::external::Error; use omicron_uuid_kinds::EreporterRestartUuid; use omicron_uuid_kinds::OmicronZoneUuid; use omicron_uuid_kinds::SledUuid; @@ -214,3 +215,141 @@ fn get_sp_metadata_string( } } } + +/// A set of filters for fetching ereports. +#[derive(Clone, Debug, Default, Eq, PartialEq, Serialize, Deserialize)] +pub struct EreportFilters { + /// If present, include only ereports that were collected at the specified + /// timestamp or later. + /// + /// If `end_time` is also present, this value *must* be earlier than + /// `end_time`. + pub start_time: Option>, + /// If present, include only ereports that were collected at the specified + /// timestamp or before. + /// + /// If `start_time` is also present, this value *must* be later than + /// `start_time`. + pub end_time: Option>, + /// If this list is non-empty, include only ereports that were reported by + /// systems with the provided serial numbers. + pub only_serials: Vec, + /// If this list is non-empty, include only ereports with the provided class + /// strings. + // TODO(eliza): globbing could be nice to add here eventually... + pub only_classes: Vec, +} + +/// Displayer for pretty-printing [`EreportFilters`]. +#[must_use = "this struct does nothing unless displayed"] +pub struct DisplayEreportFilters<'a> { + filters: &'a EreportFilters, +} + +impl EreportFilters { + pub fn display(&self) -> DisplayEreportFilters<'_> { + DisplayEreportFilters { filters: self } + } +} + +impl fmt::Display for DisplayEreportFilters<'_> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + use itertools::Itertools; + + let filters = self.filters; + + // Writes a semicolon-separated part to the formatter, tracking whether + // we've written anything yet. + let mut empty = true; + let mut fmt_part = + |f: &mut fmt::Formatter, args: fmt::Arguments| -> fmt::Result { + if !empty { + write!(f, "; ")?; + } + empty = false; + f.write_fmt(args) + }; + + if let Some(start) = &filters.start_time { + fmt_part(f, format_args!("start: {start}"))?; + } + if let Some(end) = &filters.end_time { + fmt_part(f, format_args!("end: {end}"))?; + } + if !filters.only_serials.is_empty() { + fmt_part( + f, + format_args!( + "serials: {}", + filters.only_serials.iter().format(", ") + ), + )?; + } + if !filters.only_classes.is_empty() { + fmt_part( + f, + format_args!( + "classes: {}", + filters.only_classes.iter().format(", ") + ), + )?; + } + + // If no filters are set, display "none" rather than empty output. + if empty { + write!(f, "none")?; + } + Ok(()) + } +} + +impl EreportFilters { + pub fn check_time_range(&self) -> Result<(), Error> { + if let (Some(start), Some(end)) = (self.start_time, self.end_time) { + if start > end { + return Err(Error::invalid_request( + "start time must be before end time", + )); + } + } + + Ok(()) + } +} + +#[cfg(test)] +pub(crate) mod test_utils { + use super::*; + use proptest::prelude::*; + + fn arb_datetime() -> impl Strategy> { + // Generate timestamps in a reasonable range (2020-2030). + (1577836800i64..1893456000i64) + .prop_map(|secs| DateTime::from_timestamp(secs, 0).unwrap()) + } + + impl Arbitrary for EreportFilters { + type Parameters = (); + type Strategy = BoxedStrategy; + + fn arbitrary_with(_: Self::Parameters) -> Self::Strategy { + ( + prop::option::of(arb_datetime()), + prop::option::of(arb_datetime()), + prop::collection::vec(".*", 0..=3), + prop::collection::vec(".*", 0..=3), + ) + .prop_map( + |(start_time, end_time, only_serials, only_classes)| { + EreportFilters { + start_time, + end_time, + only_serials, + only_classes, + } + }, + ) + .boxed() + } + } +} diff --git a/nexus/types/src/lib.rs b/nexus/types/src/lib.rs index 555b74dc988..13978312f5a 100644 --- a/nexus/types/src/lib.rs +++ b/nexus/types/src/lib.rs @@ -41,4 +41,5 @@ pub mod multicast; pub mod quiesce; pub mod saga; pub mod silo; +pub mod support_bundle; pub mod trust_quorum; diff --git a/nexus/types/src/support_bundle.rs b/nexus/types/src/support_bundle.rs new file mode 100644 index 00000000000..71282c069f5 --- /dev/null +++ b/nexus/types/src/support_bundle.rs @@ -0,0 +1,252 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Support bundle data selection types. +//! +//! These types specify what data to collect in a support bundle. +//! They are shared between the support bundle collector and FM case types. + +use crate::fm::ereport::EreportFilters; +use itertools::Itertools; +use omicron_uuid_kinds::SledUuid; +use serde::{Deserialize, Serialize}; +use std::collections::HashMap; +use std::collections::HashSet; +use std::fmt; + +/// Describes the category of support bundle data. +#[derive(Debug, Clone, Copy, Hash, Eq, PartialEq, Serialize, Deserialize)] +#[cfg_attr(test, derive(test_strategy::Arbitrary))] +pub enum BundleDataCategory { + /// Collects reconfigurator state (some of the latest blueprints, + /// information about the target blueprint). + Reconfigurator, + /// Collects info from sled agents, running a handful of + /// diagnostic commands (e.g., zoneadm, dladm, etc). + HostInfo, + /// Collects sled serial numbers, cubby numbers, and UUIDs. + SledCubbyInfo, + /// Saves task dumps from SPs. + SpDumps, + /// Collects ereports. + Ereports, +} + +/// Specifies what data to collect for a bundle data category. +/// +/// Each variant corresponds to a BundleDataCategory. +/// For categories without additional parameters, the variant is a unit variant. +/// For categories that can be filtered or configured, the variant contains +/// that configuration data. +#[derive(Debug, Clone, Eq, PartialEq, Serialize, Deserialize)] +#[cfg_attr(test, derive(test_strategy::Arbitrary))] +pub enum BundleData { + Reconfigurator, + HostInfo(HashSet), + SledCubbyInfo, + SpDumps, + Ereports(EreportFilters), +} + +impl BundleData { + fn category(&self) -> BundleDataCategory { + match self { + Self::Reconfigurator => BundleDataCategory::Reconfigurator, + Self::HostInfo(_) => BundleDataCategory::HostInfo, + Self::SledCubbyInfo => BundleDataCategory::SledCubbyInfo, + Self::SpDumps => BundleDataCategory::SpDumps, + Self::Ereports(_) => BundleDataCategory::Ereports, + } + } +} + +/// Displayer for pretty-printing [`BundleData`]. +#[must_use = "this struct does nothing unless displayed"] +pub struct DisplayBundleData<'a> { + data: &'a BundleData, +} + +impl BundleData { + pub fn display(&self) -> DisplayBundleData<'_> { + DisplayBundleData { data: self } + } +} + +impl fmt::Display for DisplayBundleData<'_> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self.data { + BundleData::Reconfigurator => write!(f, "reconfigurator"), + BundleData::HostInfo(sleds) => { + write!( + f, + "host_info({})", + sleds.iter().format_with(", ", |s, f| f(&s.display())) + ) + } + BundleData::SledCubbyInfo => write!(f, "sled_cubby_info"), + BundleData::SpDumps => write!(f, "sp_dumps"), + BundleData::Ereports(filters) => { + write!(f, "ereports({})", filters.display()) + } + } + } +} + +/// A collection of bundle data specifications. +/// +/// This wrapper ensures that categories and data always match - you can't +/// insert (BundleDataCategory::Reconfigurator, BundleData::SpDumps) +/// because each BundleData determines its own category. +#[derive(Debug, Clone, Eq, PartialEq, Serialize, Deserialize)] +pub struct BundleDataSelection { + data: HashMap, +} + +impl BundleDataSelection { + /// Creates an empty selection with no data categories. + /// + /// This is distinct from [`Self::default`], which returns a selection + /// containing all categories (i.e. "collect everything"). + pub fn new() -> Self { + Self { data: HashMap::new() } + } + + /// Inserts BundleData to be queried for a particular category within the + /// bundle. + /// + /// Each category of data can only be specified once (e.g., inserting + /// BundleData::HostInfo multiple times will only use the most-recently + /// inserted specification) + pub fn insert(&mut self, bundle_data: BundleData) { + self.data.insert(bundle_data.category(), bundle_data); + } + + pub fn contains(&self, category: BundleDataCategory) -> bool { + self.data.contains_key(&category) + } + + pub fn get(&self, category: BundleDataCategory) -> Option<&BundleData> { + self.data.get(&category) + } +} + +/// Displayer for pretty-printing [`BundleDataSelection`]. +#[must_use = "this struct does nothing unless displayed"] +pub struct DisplayBundleDataSelection<'a> { + selection: &'a BundleDataSelection, + indent: usize, +} + +impl BundleDataSelection { + pub fn display(&self, indent: usize) -> DisplayBundleDataSelection<'_> { + DisplayBundleDataSelection { selection: self, indent } + } +} + +impl fmt::Display for DisplayBundleDataSelection<'_> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let indent = self.indent; + for (i, item) in self.selection.data.values().enumerate() { + if i > 0 { + writeln!(f)?; + } + write!(f, "{:>indent$}- {}", "", item.display())?; + } + Ok(()) + } +} + +impl FromIterator for BundleDataSelection { + fn from_iter>(iter: T) -> Self { + let mut selection = Self::new(); + for bundle_data in iter { + selection.insert(bundle_data); + } + selection + } +} + +impl Default for BundleDataSelection { + /// Returns a selection containing all data categories (i.e. "collect + /// everything"). This is distinct from [`Self::new`], which returns an + /// empty selection. + fn default() -> Self { + [ + BundleData::Reconfigurator, + BundleData::HostInfo(HashSet::from([SledSelection::All])), + BundleData::SledCubbyInfo, + BundleData::SpDumps, + BundleData::Ereports(EreportFilters { + start_time: Some(chrono::Utc::now() - chrono::Days::new(7)), + ..EreportFilters::default() + }), + ] + .into_iter() + .collect() + } +} + +/// The set of sleds to include. +/// +/// Multiple values of this enum are joined together into a HashSet. +/// Therefore "SledSelection::All" overrides specific sleds. +#[derive(Debug, Clone, Hash, Eq, PartialEq, Serialize, Deserialize)] +#[cfg_attr(test, derive(test_strategy::Arbitrary))] +pub enum SledSelection { + All, + Specific(SledUuid), +} + +/// Displayer for pretty-printing [`SledSelection`]. +#[must_use = "this struct does nothing unless displayed"] +pub struct DisplaySledSelection<'a> { + selection: &'a SledSelection, +} + +impl SledSelection { + pub fn display(&self) -> DisplaySledSelection<'_> { + DisplaySledSelection { selection: self } + } +} + +impl fmt::Display for DisplaySledSelection<'_> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self.selection { + SledSelection::All => write!(f, "all"), + SledSelection::Specific(id) => write!(f, "{id}"), + } + } +} + +#[cfg(test)] +pub(crate) mod test_utils { + use super::*; + use proptest::prelude::*; + + impl Arbitrary for BundleDataSelection { + type Parameters = (); + type Strategy = BoxedStrategy; + + fn arbitrary_with(_: Self::Parameters) -> Self::Strategy { + prop::collection::vec(any::(), 0..=5) + .prop_map(|data| data.into_iter().collect()) + .boxed() + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use proptest::prelude::*; + use test_strategy::proptest; + + #[proptest] + fn bundle_data_selection_serde_round_trip(selection: BundleDataSelection) { + let json = serde_json::to_string(&selection).unwrap(); + let deserialized: BundleDataSelection = + serde_json::from_str(&json).unwrap(); + prop_assert_eq!(selection, deserialized); + } +} diff --git a/schema/crdb/dbinit.sql b/schema/crdb/dbinit.sql index ae61055b071..566bbf82702 100644 --- a/schema/crdb/dbinit.sql +++ b/schema/crdb/dbinit.sql @@ -3156,7 +3156,10 @@ CREATE TABLE IF NOT EXISTS omicron.public.support_bundle ( -- and later managing its storage. assigned_nexus UUID, - user_comment TEXT + user_comment TEXT, + + -- If this bundle was requested by an FM case, the case UUID. + fm_case_id UUID ); @@ -7440,6 +7443,70 @@ CREATE INDEX IF NOT EXISTS lookup_fm_alert_requests_for_case ON omicron.public.fm_alert_request (sitrep_id, case_id); +CREATE TABLE IF NOT EXISTS omicron.public.fm_support_bundle_request ( + -- Requested support bundle UUID. + id UUID NOT NULL, + -- UUID of the current sitrep that this request record is part of. + -- + -- Note that this is *not* the sitrep in which the bundle was requested. + sitrep_id UUID NOT NULL, + -- UUID of the original sitrep in which the bundle was first requested. + requested_sitrep_id UUID NOT NULL, + -- UUID of the case to which this request belongs. + case_id UUID NOT NULL, + + PRIMARY KEY (sitrep_id, id) +); + +CREATE INDEX IF NOT EXISTS + lookup_fm_support_bundle_requests_for_case +ON omicron.public.fm_support_bundle_request (sitrep_id, case_id); + +-- Per-variant data selection tables for fm_support_bundle_request. +-- Row existence = "include this category in the bundle." +-- No rows in any variant table = "collect everything." + +CREATE TABLE IF NOT EXISTS omicron.public.fm_sb_req_reconfigurator ( + sitrep_id UUID NOT NULL, + request_id UUID NOT NULL, + + PRIMARY KEY (sitrep_id, request_id) +); + +CREATE TABLE IF NOT EXISTS omicron.public.fm_sb_req_sled_cubby_info ( + sitrep_id UUID NOT NULL, + request_id UUID NOT NULL, + + PRIMARY KEY (sitrep_id, request_id) +); + +CREATE TABLE IF NOT EXISTS omicron.public.fm_sb_req_sp_dumps ( + sitrep_id UUID NOT NULL, + request_id UUID NOT NULL, + + PRIMARY KEY (sitrep_id, request_id) +); + +CREATE TABLE IF NOT EXISTS omicron.public.fm_sb_req_host_info ( + sitrep_id UUID NOT NULL, + request_id UUID NOT NULL, + all_sleds BOOL NOT NULL, + sled_ids UUID[] NOT NULL DEFAULT ARRAY[], + + PRIMARY KEY (sitrep_id, request_id) +); + +CREATE TABLE IF NOT EXISTS omicron.public.fm_sb_req_ereports ( + sitrep_id UUID NOT NULL, + request_id UUID NOT NULL, + start_time TIMESTAMPTZ, + end_time TIMESTAMPTZ, + only_serials TEXT[] NOT NULL DEFAULT ARRAY[], + only_classes TEXT[] NOT NULL DEFAULT ARRAY[], + + PRIMARY KEY (sitrep_id, request_id) +); + /* * List of datasets available to be sliced up and passed to VMMs for encrypted * instance local storage. @@ -8270,7 +8337,7 @@ INSERT INTO omicron.public.db_metadata ( version, target_version ) VALUES - (TRUE, NOW(), NOW(), '240.0.0', NULL) + (TRUE, NOW(), NOW(), '241.0.0', NULL) ON CONFLICT DO NOTHING; COMMIT; diff --git a/schema/crdb/fm-support-bundle-request/up1.sql b/schema/crdb/fm-support-bundle-request/up1.sql new file mode 100644 index 00000000000..4ab9fdda7f9 --- /dev/null +++ b/schema/crdb/fm-support-bundle-request/up1.sql @@ -0,0 +1,13 @@ +CREATE TABLE IF NOT EXISTS omicron.public.fm_support_bundle_request ( + -- Requested support bundle UUID. + id UUID NOT NULL, + -- UUID of the current sitrep that this request record is part of. + -- + -- Note that this is *not* the sitrep in which the bundle was requested. + sitrep_id UUID NOT NULL, + -- UUID of the original sitrep in which the bundle was first requested. + requested_sitrep_id UUID NOT NULL, + -- UUID of the case to which this request belongs. + case_id UUID NOT NULL, + PRIMARY KEY (sitrep_id, id) +); diff --git a/schema/crdb/fm-support-bundle-request/up2.sql b/schema/crdb/fm-support-bundle-request/up2.sql new file mode 100644 index 00000000000..2742178f61a --- /dev/null +++ b/schema/crdb/fm-support-bundle-request/up2.sql @@ -0,0 +1,3 @@ +CREATE INDEX IF NOT EXISTS + lookup_fm_support_bundle_requests_for_case +ON omicron.public.fm_support_bundle_request (sitrep_id, case_id); diff --git a/schema/crdb/fm-support-bundle-request/up2.verify.sql b/schema/crdb/fm-support-bundle-request/up2.verify.sql new file mode 100644 index 00000000000..aa1319640d3 --- /dev/null +++ b/schema/crdb/fm-support-bundle-request/up2.verify.sql @@ -0,0 +1,2 @@ +-- DO NOT EDIT. Generated by test_migration_verification_files. +SELECT CAST(IF((SELECT true WHERE EXISTS (SELECT index_name FROM omicron.crdb_internal.table_indexes WHERE descriptor_name = 'fm_support_bundle_request' AND index_name = 'lookup_fm_support_bundle_requests_for_case')),'true','Schema change verification failed: index lookup_fm_support_bundle_requests_for_case on table fm_support_bundle_request does not exist') AS BOOL); diff --git a/schema/crdb/fm-support-bundle-request/up3.sql b/schema/crdb/fm-support-bundle-request/up3.sql new file mode 100644 index 00000000000..0699362b195 --- /dev/null +++ b/schema/crdb/fm-support-bundle-request/up3.sql @@ -0,0 +1,6 @@ +CREATE TABLE IF NOT EXISTS omicron.public.fm_sb_req_reconfigurator ( + sitrep_id UUID NOT NULL, + request_id UUID NOT NULL, + + PRIMARY KEY (sitrep_id, request_id) +); diff --git a/schema/crdb/fm-support-bundle-request/up4.sql b/schema/crdb/fm-support-bundle-request/up4.sql new file mode 100644 index 00000000000..565f4943625 --- /dev/null +++ b/schema/crdb/fm-support-bundle-request/up4.sql @@ -0,0 +1,6 @@ +CREATE TABLE IF NOT EXISTS omicron.public.fm_sb_req_sled_cubby_info ( + sitrep_id UUID NOT NULL, + request_id UUID NOT NULL, + + PRIMARY KEY (sitrep_id, request_id) +); diff --git a/schema/crdb/fm-support-bundle-request/up5.sql b/schema/crdb/fm-support-bundle-request/up5.sql new file mode 100644 index 00000000000..f30f40caaf6 --- /dev/null +++ b/schema/crdb/fm-support-bundle-request/up5.sql @@ -0,0 +1,6 @@ +CREATE TABLE IF NOT EXISTS omicron.public.fm_sb_req_sp_dumps ( + sitrep_id UUID NOT NULL, + request_id UUID NOT NULL, + + PRIMARY KEY (sitrep_id, request_id) +); diff --git a/schema/crdb/fm-support-bundle-request/up6.sql b/schema/crdb/fm-support-bundle-request/up6.sql new file mode 100644 index 00000000000..6cce61c17c2 --- /dev/null +++ b/schema/crdb/fm-support-bundle-request/up6.sql @@ -0,0 +1,8 @@ +CREATE TABLE IF NOT EXISTS omicron.public.fm_sb_req_host_info ( + sitrep_id UUID NOT NULL, + request_id UUID NOT NULL, + all_sleds BOOL NOT NULL, + sled_ids UUID[] NOT NULL DEFAULT ARRAY[], + + PRIMARY KEY (sitrep_id, request_id) +); diff --git a/schema/crdb/fm-support-bundle-request/up7.sql b/schema/crdb/fm-support-bundle-request/up7.sql new file mode 100644 index 00000000000..d64e8010792 --- /dev/null +++ b/schema/crdb/fm-support-bundle-request/up7.sql @@ -0,0 +1,10 @@ +CREATE TABLE IF NOT EXISTS omicron.public.fm_sb_req_ereports ( + sitrep_id UUID NOT NULL, + request_id UUID NOT NULL, + start_time TIMESTAMPTZ, + end_time TIMESTAMPTZ, + only_serials TEXT[] NOT NULL DEFAULT ARRAY[], + only_classes TEXT[] NOT NULL DEFAULT ARRAY[], + + PRIMARY KEY (sitrep_id, request_id) +); diff --git a/schema/crdb/fm-support-bundle-request/up8.sql b/schema/crdb/fm-support-bundle-request/up8.sql new file mode 100644 index 00000000000..199098c0ccb --- /dev/null +++ b/schema/crdb/fm-support-bundle-request/up8.sql @@ -0,0 +1,2 @@ +ALTER TABLE omicron.public.support_bundle + ADD COLUMN IF NOT EXISTS fm_case_id UUID;