diff --git a/transactional_emulator/Cargo.lock b/transactional_emulator/Cargo.lock index fb28d483..e051acc2 100644 --- a/transactional_emulator/Cargo.lock +++ b/transactional_emulator/Cargo.lock @@ -795,7 +795,9 @@ dependencies = [ "proptest", "rand 0.9.2", "runtime", + "serde", "tokio", + "tracing", "trait-variant", "zerocopy", ] @@ -1722,6 +1724,7 @@ dependencies = [ "ramulator", "runtime", "serde", + "serde_json", "sram", "tch", "tokio", diff --git a/transactional_emulator/Cargo.toml b/transactional_emulator/Cargo.toml index c8bbe7ad..76c6c5c0 100644 --- a/transactional_emulator/Cargo.toml +++ b/transactional_emulator/Cargo.toml @@ -15,6 +15,7 @@ tch = { version = "0.20.0", features = ["download-libtorch"] } half = "2" clap = { version = "4", features = ["derive"] } serde = { version = "1.0", features = ["derive"] } +serde_json = "1.0" toml = "0.8" tracing = "0.1" tracing-subscriber = { version = "0.3", features = ["env-filter"] } diff --git a/transactional_emulator/lib/memory/Cargo.toml b/transactional_emulator/lib/memory/Cargo.toml index a123f125..59b69016 100644 --- a/transactional_emulator/lib/memory/Cargo.toml +++ b/transactional_emulator/lib/memory/Cargo.toml @@ -6,11 +6,13 @@ edition = "2024" [dependencies] async-trait = "0.1.88" trait-variant = "0.1.2" +serde = { version = "1", features = ["derive"] } zerocopy = "0.8" runtime = { path = "../runtime" } tokio = { version = "1.45.1", features = ["sync"] } futures-util = "0.3.31" rand = "0.9.2" +tracing = "0.1" [dev-dependencies] tokio = { version = "1.45.1", features = ["sync", "rt-multi-thread", "macros"] } diff --git a/transactional_emulator/lib/memory/src/lib.rs b/transactional_emulator/lib/memory/src/lib.rs index 4b26e866..2c14f9f0 100644 --- a/transactional_emulator/lib/memory/src/lib.rs +++ b/transactional_emulator/lib/memory/src/lib.rs @@ -2,6 +2,7 @@ pub mod chunked; mod frfcfs; mod naive; mod simple; +pub mod streaming; pub mod testutils; use std::mem::ManuallyDrop; @@ -44,12 +45,24 @@ pub trait MemoryModel: Send + Sync { /// Write 64-bytes of memory. async fn write(&self, addr: u64, bytes: [u8; 64]); + + /// Render a human-readable, per-model statistics summary, or `None` if the + /// model does not track statistics. `elapsed_secs` is the simulated wall + /// time used for bandwidth/utilization figures. + /// + /// This is intentionally a non-async default so that `trait_variant` (which + /// only rewrites async methods) passes it through unchanged, letting it be + /// forwarded uniformly via `ErasedMemoryModel`. + fn statistics_summary(&self, _elapsed_secs: f64) -> Option { + None + } } #[async_trait::async_trait] pub trait ErasedMemoryModel: Send + Sync { async fn box_read(&self, addr: u64) -> [u8; 64]; async fn box_write(&self, addr: u64, bytes: [u8; 64]); + fn box_statistics_summary(&self, elapsed_secs: f64) -> Option; } #[async_trait::async_trait] @@ -61,6 +74,10 @@ impl ErasedMemoryModel for T { async fn box_write(&self, addr: u64, bytes: [u8; 64]) { self.write(addr, bytes).await } + + fn box_statistics_summary(&self, elapsed_secs: f64) -> Option { + self.statistics_summary(elapsed_secs) + } } impl MemoryModel for dyn ErasedMemoryModel { @@ -71,6 +88,10 @@ impl MemoryModel for dyn ErasedMemoryModel { async fn write(&self, addr: u64, bytes: [u8; 64]) { self.box_write(addr, bytes).await } + + fn statistics_summary(&self, elapsed_secs: f64) -> Option { + self.box_statistics_summary(elapsed_secs) + } } /// A memory that discards all written data. @@ -206,6 +227,24 @@ impl MemoryModel for WithStats { } self.model.write(addr, bytes).await } + + fn statistics_summary(&self, elapsed_secs: f64) -> Option { + let s = self.statistics(); + let hbm_line = format!( + "HBM Statistics - Bytes read: {:?} | Bytes written: {:?} | Utilization: {:.2e} bytes/sec", + s.total_bytes_read, + s.total_bytes_written, + (s.total_bytes_read + s.total_bytes_written) as f64 / elapsed_secs + ); + // `WithStats` wraps the streaming models (LayerSwapping/HostStream) in the + // runner, so forward the inner model's summary too instead of shadowing it. + // The default HBM path's inner is `WithTiming`, whose summary is `None`, so + // this returns `hbm_line` unchanged and the HBM output stays byte-for-byte. + match self.model.statistics_summary(elapsed_secs) { + Some(inner) => Some(format!("{hbm_line}\n{inner}")), + None => Some(hbm_line), + } + } } #[cfg(test)] diff --git a/transactional_emulator/lib/memory/src/streaming.rs b/transactional_emulator/lib/memory/src/streaming.rs new file mode 100644 index 00000000..f077cb73 --- /dev/null +++ b/transactional_emulator/lib/memory/src/streaming.rs @@ -0,0 +1,526 @@ +use std::sync::Mutex; + +use runtime::Duration; +use tokio::sync::Semaphore; + +use crate::MemoryModel; + +/// Convert a byte count into a transfer time in nanoseconds at a fixed +/// bandwidth. The multiplication is done in `u128` so large layers (where +/// `bytes * 1_000_000_000` would overflow `u64`) compute correctly. +fn transfer_nanos(bytes: u64, bandwidth_bytes_per_sec: u64) -> u64 { + (bytes as u128 * 1_000_000_000u128 / bandwidth_bytes_per_sec as u128) as u64 +} + +/// Classification of a manifest region by the kind of tensor data it holds. +/// +/// Used by the capacity-aware [`CapacityModel`] to decide which regions are +/// pinned in DDR, swapped under a capacity bound, or streamed from the host. +/// Defaults to `Weight` so legacy weight-only manifests (which omit the field) +/// still deserialize with every region treated as a weight region. +#[derive(Debug, Clone, Copy, PartialEq, Default, serde::Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum RegionKind { + #[default] + Weight, + Kv, + Activation, +} + +#[derive(Debug, Clone, serde::Deserialize)] +pub struct WeightRegion { + pub layer_id: u32, + pub offset: u64, + pub size: u64, + #[serde(default)] + pub kind: RegionKind, +} + +#[derive(Debug, Clone, serde::Deserialize)] +pub struct WeightManifest { + pub activation_ceiling: u64, + pub regions: Vec, + /// Optional DDR capacity (bytes) the board exposes. When present it can be + /// used as the capacity for regime auto-selection in place of a CLI flag. + #[serde(default)] + pub ddr_capacity_bytes: Option, +} + +impl WeightManifest { + pub fn is_weight_addr(&self, addr: u64) -> bool { + if addr < self.activation_ceiling { + return false; + } + self.regions + .iter() + .any(|r| addr >= r.offset && addr < r.offset + r.size) + } + + pub fn layer_for_addr(&self, addr: u64) -> Option { + self.regions + .iter() + .find(|r| addr >= r.offset && addr < r.offset + r.size) + .map(|r| r.layer_id) + } + + /// Classify an address by the kind of the region containing it, or `None` + /// if no region covers it. + pub fn region_kind(&self, addr: u64) -> Option { + self.regions + .iter() + .find(|r| addr >= r.offset && addr < r.offset + r.size) + .map(|r| r.kind) + } + + /// Sum region sizes per kind, returning `(weight, kv, activation)` bytes. + pub fn footprint_by_kind(&self) -> (u64, u64, u64) { + let mut weight = 0u64; + let mut kv = 0u64; + let mut activation = 0u64; + for r in &self.regions { + match r.kind { + RegionKind::Weight => weight += r.size, + RegionKind::Kv => kv += r.size, + RegionKind::Activation => activation += r.size, + } + } + (weight, kv, activation) + } +} + +#[derive(Debug, Default)] +pub struct StreamingStats { + pub weight_reads: u64, + pub weight_bytes: u64, + pub activation_reads: u64, + pub activation_bytes: u64, +} + +/// Direct host-to-SRAM streaming memory model. +/// +/// Weight reads bypass DDR3 and model a host transfer at a fixed bandwidth. +/// Activation reads go through the inner memory model (DDR3 timing). +/// Data is always served from the backing store — only timing differs. +pub struct HostStream { + inner: T, + manifest: WeightManifest, + host_transfer_time_per_chunk: Duration, + host_link: Semaphore, + stats: Mutex, +} + +impl HostStream { + pub fn new(inner: T, manifest: WeightManifest, host_bandwidth_bytes_per_sec: u64) -> Self { + let nanos_per_chunk = transfer_nanos(64, host_bandwidth_bytes_per_sec); + Self { + inner, + manifest, + host_transfer_time_per_chunk: Duration::from_nanos(nanos_per_chunk), + host_link: Semaphore::new(1), + stats: Mutex::new(StreamingStats::default()), + } + } + + pub fn statistics(&self) -> StreamingStats { + let guard = self.stats.lock().unwrap(); + StreamingStats { + weight_reads: guard.weight_reads, + weight_bytes: guard.weight_bytes, + activation_reads: guard.activation_reads, + activation_bytes: guard.activation_bytes, + } + } +} + +impl MemoryModel for HostStream { + async fn read(&self, addr: u64) -> [u8; 64] { + if self.manifest.is_weight_addr(addr) { + let _permit = self.host_link.acquire().await.unwrap(); + let rt = runtime::Executor::current(); + rt.resolve_at(self.host_transfer_time_per_chunk).await; + { + let mut stats = self.stats.lock().unwrap(); + stats.weight_reads += 1; + stats.weight_bytes += 64; + } + self.inner.read(addr).await + } else { + { + let mut stats = self.stats.lock().unwrap(); + stats.activation_reads += 1; + stats.activation_bytes += 64; + } + self.inner.read(addr).await + } + } + + async fn write(&self, addr: u64, bytes: [u8; 64]) { + self.inner.write(addr, bytes).await + } + + fn statistics_summary(&self, _elapsed_secs: f64) -> Option { + let s = self.statistics(); + Some(format!( + "Host-stream memory - weight reads: {} ({} bytes) | activation reads: {} ({} bytes)", + s.weight_reads, s.weight_bytes, s.activation_reads, s.activation_bytes + )) + } +} + +#[derive(Debug, Default)] +pub struct LayerSwapStats { + pub total_swaps: u64, + pub total_swap_bytes: u64, + pub total_swap_nanos: u64, +} + +/// Layer-swapping DDR3 memory model. +/// +/// Simulates a capacity-limited DDR3 where only a few layers' weights fit +/// at a time. When a weight read targets a non-resident layer, the model +/// charges a swap penalty (evict + load) based on host bandwidth. +/// Activation reads always go through DDR3 timing. +pub struct LayerSwapping { + inner: T, + manifest: WeightManifest, + capacity: u64, + swap_bandwidth_bytes_per_sec: u64, + state: Mutex, + stats: Mutex, +} + +#[derive(Debug)] +struct LayerSwapState { + resident_layers: Vec, + used_bytes: u64, +} + +impl LayerSwapping { + pub fn new( + inner: T, + manifest: WeightManifest, + capacity: u64, + swap_bandwidth_bytes_per_sec: u64, + ) -> Self { + Self { + inner, + manifest, + capacity, + swap_bandwidth_bytes_per_sec, + state: Mutex::new(LayerSwapState { + resident_layers: Vec::new(), + used_bytes: 0, + }), + stats: Mutex::new(LayerSwapStats::default()), + } + } + + pub fn statistics(&self) -> LayerSwapStats { + let guard = self.stats.lock().unwrap(); + LayerSwapStats { + total_swaps: guard.total_swaps, + total_swap_bytes: guard.total_swap_bytes, + total_swap_nanos: guard.total_swap_nanos, + } + } + + fn layer_size(&self, layer_id: u32) -> u64 { + self.manifest + .regions + .iter() + .filter(|r| r.layer_id == layer_id) + .map(|r| r.size) + .sum() + } + + fn ensure_resident(&self, layer_id: u32) -> Option { + let mut state = self.state.lock().unwrap(); + if state.resident_layers.contains(&layer_id) { + return None; + } + + let needed = self.layer_size(layer_id); + + while state.used_bytes + needed > self.capacity && !state.resident_layers.is_empty() { + let evicted = state.resident_layers.remove(0); + let freed = self.layer_size(evicted); + state.used_bytes = state.used_bytes.saturating_sub(freed); + } + + // A single layer that exceeds total capacity can never stay resident: + // every access reloads it. We still admit/charge it (preserving the + // timing semantics) but surface the condition so it isn't silent. + if needed > self.capacity { + tracing::warn!( + "layer {layer_id} weights ({needed} bytes) exceed DDR3 capacity ({} bytes); modelling as reload on every access", + self.capacity + ); + } + + let load_nanos = transfer_nanos(needed, self.swap_bandwidth_bytes_per_sec); + state.resident_layers.push(layer_id); + state.used_bytes += needed; + + { + let mut stats = self.stats.lock().unwrap(); + stats.total_swaps += 1; + stats.total_swap_bytes += needed; + stats.total_swap_nanos += load_nanos; + } + + Some(Duration::from_nanos(load_nanos)) + } +} + +impl MemoryModel for LayerSwapping { + async fn read(&self, addr: u64) -> [u8; 64] { + if let Some(layer_id) = self.manifest.layer_for_addr(addr) { + if let Some(swap_duration) = self.ensure_resident(layer_id) { + let rt = runtime::Executor::current(); + rt.resolve_at(swap_duration).await; + } + } + self.inner.read(addr).await + } + + async fn write(&self, addr: u64, bytes: [u8; 64]) { + self.inner.write(addr, bytes).await + } + + fn statistics_summary(&self, _elapsed_secs: f64) -> Option { + let s = self.statistics(); + Some(format!( + "Layer-swap memory - swaps: {} | swapped bytes: {} | swap time: {} ns", + s.total_swaps, s.total_swap_bytes, s.total_swap_nanos + )) + } +} + +/// DDR-capacity regime for a fixed-size board. +/// +/// Selected explicitly on the CLI, or automatically by [`choose_regime`] from +/// the model's footprint relative to the board's DDR capacity. +#[derive(Debug, Clone, Copy, PartialEq)] +pub enum Regime { + /// Everything fits in DDR: all kinds pinned resident, pure DDR timing. + Resident, + /// Weights fit in DDR (pinned); KV + activations are swapped under the + /// remaining capacity, charging a host transfer on each miss. + KvSwap, + /// Weights do not fit in DDR: weights stream from the host link per access; + /// KV + activations stay resident in DDR. + WeightStream, +} + +impl Regime { + fn label(self) -> &'static str { + match self { + Regime::Resident => "resident", + Regime::KvSwap => "kv-swap", + Regime::WeightStream => "weight-stream", + } + } +} + +/// Choose a DDR-capacity regime from a model's footprint. +/// +/// * `total <= capacity` → [`Regime::Resident`] (everything is pinned) +/// * else `weight <= capacity` → [`Regime::KvSwap`] (weights pinned, KV swaps) +/// * else → [`Regime::WeightStream`] (weights stream) +pub fn choose_regime(weight_bytes: u64, total_bytes: u64, capacity: u64) -> Regime { + if total_bytes <= capacity { + Regime::Resident + } else if weight_bytes <= capacity { + Regime::KvSwap + } else { + Regime::WeightStream + } +} + +#[derive(Debug, Default)] +pub struct CapacityStats { + /// Bytes served straight from DDR (pinned/resident) without a host transfer. + pub resident_bytes: u64, + /// Bytes charged as KV/activation swap-ins under the capacity bound. + pub kv_swapped_bytes: u64, + /// Number of KV/activation swap-in events (misses). + pub swap_count: u64, + /// Bytes charged as host-streamed weight chunks. + pub weight_streamed_bytes: u64, +} + +/// Swappable-region LRU residency for the KV-swap regime. +/// +/// Keyed on `(offset, size)` regions (not layer ids) so KV and activation +/// regions are admitted/evicted independently of weights. Mirrors +/// [`LayerSwapping`]'s mechanics: a miss charges `transfer_nanos(size, host_bw)` +/// and the least-recently-admitted region is evicted when over capacity. +#[derive(Debug)] +struct SwapState { + /// Resident regions as `(offset, size)`, oldest-first (LRU at index 0). + resident: Vec<(u64, u64)>, + used_bytes: u64, +} + +/// Capacity-aware memory model for a fixed-size DDR board. +/// +/// Wraps a DDR-timed inner model and applies a per-kind pinning policy chosen +/// by [`Regime`]. Data is always served from `inner`; only the extra host +/// transfer timing differs between regimes. +pub struct CapacityModel { + inner: T, + manifest: WeightManifest, + regime: Regime, + /// DDR capacity available to swappable kinds (KV/activation) in `KvSwap`: + /// `ddr_capacity - weight_footprint`, saturating at 0. + swap_capacity: u64, + host_bandwidth_bytes_per_sec: u64, + /// Per-64B host-stream chunk time for the `WeightStream` regime. + host_transfer_time_per_chunk: Duration, + /// 1-permit link serialising host streaming (mirrors `HostStream`). + host_link: Semaphore, + swap_state: Mutex, + stats: Mutex, +} + +impl CapacityModel { + pub fn new( + inner: T, + manifest: WeightManifest, + ddr_capacity_bytes: u64, + host_bandwidth_bytes_per_sec: u64, + regime: Regime, + ) -> Self { + let (weight_footprint, _kv, _act) = manifest.footprint_by_kind(); + let swap_capacity = ddr_capacity_bytes.saturating_sub(weight_footprint); + let nanos_per_chunk = transfer_nanos(64, host_bandwidth_bytes_per_sec); + Self { + inner, + manifest, + regime, + swap_capacity, + host_bandwidth_bytes_per_sec, + host_transfer_time_per_chunk: Duration::from_nanos(nanos_per_chunk), + host_link: Semaphore::new(1), + swap_state: Mutex::new(SwapState { + resident: Vec::new(), + used_bytes: 0, + }), + stats: Mutex::new(CapacityStats::default()), + } + } + + pub fn statistics(&self) -> CapacityStats { + let g = self.stats.lock().unwrap(); + CapacityStats { + resident_bytes: g.resident_bytes, + kv_swapped_bytes: g.kv_swapped_bytes, + swap_count: g.swap_count, + weight_streamed_bytes: g.weight_streamed_bytes, + } + } + + /// The `(offset, size)` of the region containing `addr`, if any. + fn region_bounds(&self, addr: u64) -> Option<(u64, u64)> { + self.manifest + .regions + .iter() + .find(|r| addr >= r.offset && addr < r.offset + r.size) + .map(|r| (r.offset, r.size)) + } + + /// Ensure the swappable region containing `addr` is resident under the + /// capacity bound. Returns the swap-in delay to charge on a miss, or `None` + /// if already resident. + fn ensure_swappable_resident(&self, addr: u64) -> Option { + let (offset, size) = self.region_bounds(addr)?; + let mut state = self.swap_state.lock().unwrap(); + if state.resident.iter().any(|&(o, _)| o == offset) { + return None; + } + + while state.used_bytes + size > self.swap_capacity && !state.resident.is_empty() { + let (_, freed) = state.resident.remove(0); + state.used_bytes = state.used_bytes.saturating_sub(freed); + } + + // A region larger than the whole swap capacity can never stay resident: + // every access reloads it. Admit/charge it anyway (preserving timing). + if size > self.swap_capacity { + tracing::warn!( + "KV/activation region at offset {offset} ({size} bytes) exceeds swap capacity ({} bytes); modelling as reload on every access", + self.swap_capacity + ); + } + + let load_nanos = transfer_nanos(size, self.host_bandwidth_bytes_per_sec); + state.resident.push((offset, size)); + state.used_bytes += size; + + { + let mut stats = self.stats.lock().unwrap(); + stats.swap_count += 1; + stats.kv_swapped_bytes += size; + } + + Some(Duration::from_nanos(load_nanos)) + } +} + +impl MemoryModel for CapacityModel { + async fn read(&self, addr: u64) -> [u8; 64] { + let kind = self.manifest.region_kind(addr); + match self.regime { + // Everything pinned in DDR: pure inner timing. + Regime::Resident => { + self.stats.lock().unwrap().resident_bytes += 64; + self.inner.read(addr).await + } + // Weights pinned; KV + activations swap under the capacity bound. + Regime::KvSwap => match kind { + Some(RegionKind::Kv) | Some(RegionKind::Activation) => { + if let Some(swap_duration) = self.ensure_swappable_resident(addr) { + let rt = runtime::Executor::current(); + rt.resolve_at(swap_duration).await; + } + self.inner.read(addr).await + } + // Weight regions / non-manifest / below ceiling → resident DDR. + _ => { + self.stats.lock().unwrap().resident_bytes += 64; + self.inner.read(addr).await + } + }, + // Weights stream from the host link; KV + activations resident. + Regime::WeightStream => match kind { + Some(RegionKind::Weight) => { + let _permit = self.host_link.acquire().await.unwrap(); + let rt = runtime::Executor::current(); + rt.resolve_at(self.host_transfer_time_per_chunk).await; + self.stats.lock().unwrap().weight_streamed_bytes += 64; + self.inner.read(addr).await + } + _ => { + self.stats.lock().unwrap().resident_bytes += 64; + self.inner.read(addr).await + } + }, + } + } + + async fn write(&self, addr: u64, bytes: [u8; 64]) { + self.inner.write(addr, bytes).await + } + + fn statistics_summary(&self, _elapsed_secs: f64) -> Option { + let s = self.statistics(); + Some(format!( + "Capacity model [{}] - resident bytes: {} | kv swapped bytes: {} (swaps: {}) | weight streamed bytes: {}", + self.regime.label(), + s.resident_bytes, + s.kv_swapped_bytes, + s.swap_count, + s.weight_streamed_bytes + )) + } +} diff --git a/transactional_emulator/src/cli.rs b/transactional_emulator/src/cli.rs index 74774565..1052a073 100644 --- a/transactional_emulator/src/cli.rs +++ b/transactional_emulator/src/cli.rs @@ -106,6 +106,39 @@ fn parse_size(s: &str) -> Result { .ok_or_else(|| format!("size {:?} overflows usize", s)) } +/// Parse a `u64` bandwidth value, rejecting zero. +/// +/// A zero bandwidth would divide-by-zero when converting bytes to a transfer +/// time, so it is rejected at the CLI rather than panicking later. +fn parse_nonzero_u64(s: &str) -> Result { + let value: u64 = s + .trim() + .parse() + .map_err(|_| format!("invalid number {:?}", s))?; + if value == 0 { + return Err("must be > 0".to_string()); + } + Ok(value) +} + +#[derive(Clone, Copy, Debug, ValueEnum)] +pub(crate) enum MemoryModelKind { + /// Unlimited HBM with Ramulator HBM2 timing (current default) + Hbm, + /// DDR3 with layer swapping (host streams layers on demand) + LayerSwap, + /// Direct host-to-SRAM streaming (no DDR3 for weights) + HostStream, + /// Auto-select a capacity regime from the manifest footprint vs DDR capacity + Auto, + /// Capacity regime: everything pinned resident in DDR (pure DDR timing) + Resident, + /// Capacity regime: weights pinned, KV/activations swapped under capacity + KvSwap, + /// Capacity regime: weights stream from the host link, KV/activations resident + WeightStream, +} + #[derive(Parser)] pub(crate) struct Opts { #[arg(long)] @@ -139,6 +172,34 @@ pub(crate) struct Opts { #[arg(long, help_heading = "Logging")] pub(crate) log_file: Option, + #[arg(long, value_enum, default_value = "hbm")] + /// Memory model to simulate. + /// + /// hbm — Unlimited HBM with Ramulator timing (default) + /// layer-swap — DDR3 with layer swapping (requires --weight-manifest) + /// host-stream — Direct host-to-SRAM streaming (requires --weight-manifest) + /// auto — Capacity-aware: pick regime from footprint vs DDR capacity + /// resident — Capacity-aware: all kinds pinned resident in DDR + /// kv-swap — Capacity-aware: weights pinned, KV/activations swap + /// weight-stream — Capacity-aware: weights stream from host, KV/act resident + /// + /// The capacity-aware regimes (auto/resident/kv-swap/weight-stream) require + /// --weight-manifest, and use --ddr3-capacity as the board DDR capacity and + /// --host-bandwidth as the host link bandwidth. + pub(crate) memory_model: MemoryModelKind, + + #[arg(long)] + /// Path to weight manifest JSON (required for layer-swap and host-stream). + pub(crate) weight_manifest: Option, + + #[arg(long, value_parser = parse_size, default_value = "512M")] + /// DDR3 capacity for layer-swap model. + pub(crate) ddr3_capacity: usize, + + #[arg(long, value_parser = parse_nonzero_u64, default_value = "60000000")] + /// Host-to-board bandwidth in bytes/sec (default: 60 MB/s = USB 2.0 bulk). + pub(crate) host_bandwidth: u64, + #[arg(long, value_parser = parse_size)] /// Override HBM allocation size (default: from plena_settings.toml). /// diff --git a/transactional_emulator/src/runner.rs b/transactional_emulator/src/runner.rs index ef522341..baa10629 100644 --- a/transactional_emulator/src/runner.rs +++ b/transactional_emulator/src/runner.rs @@ -129,10 +129,159 @@ pub(crate) async fn run_from_cli() { effective_hbm_size, effective_hbm_size as f64 / (1024.0 * 1024.0 * 1024.0) ); - let hbm = Arc::new(memory::WithStats::new(memory::WithTiming::new( - ManuallyDrop::new(ramulator::Ramulator::hbm2_preset(8).unwrap()), - memory::MemoryBacked::with_capacity(effective_hbm_size), - ))); + // Build and preload the HBM backing store *before* wrapping it in a memory + // model. The streaming models (LayerSwap, HostStream) consume the backing + // store by value, so the preload has to happen up front; the default Hbm + // model preloads here too, which is behaviorally identical to the old + // post-construction `hbm.model().data().with_data(...)` step. + let hbm_backing = memory::MemoryBacked::with_capacity(effective_hbm_size); + let hbm_data = std::fs::read(&opts.hbm) + .unwrap_or_else(|err| panic!("failed to read HBM preload file {:?}: {err}", opts.hbm)); + hbm_backing.with_data(|f| { + f[..hbm_data.len()].copy_from_slice(&hbm_data); + }); + + // Concrete type of the default HBM model. Kept as a typed handle for the + // Hbm case so we can still surface `.statistics()` and dump the backing + // store via `.model().data()` (the `ErasedMemoryModel` trait exposes + // neither). For the streaming models this stays `None`. + type HbmModel = memory::WithStats< + memory::WithTiming, memory::MemoryBacked>, + >; + + // Wrap the preloaded backing store in the selected memory model. `hbm` is + // the erased handle handed to the accelerator; `hbm_concrete` is a typed + // alias of the same allocation, present only for the default Hbm path. + let (hbm, hbm_concrete): (Arc, Option>) = + match opts.memory_model { + cli::MemoryModelKind::Hbm => { + let model: Arc = + Arc::new(memory::WithStats::new(memory::WithTiming::new( + ManuallyDrop::new(ramulator::Ramulator::hbm2_preset(8).unwrap()), + hbm_backing, + ))); + (model.clone(), Some(model)) + } + cli::MemoryModelKind::LayerSwap => { + let manifest_path = opts + .weight_manifest + .as_ref() + .expect("--weight-manifest required for layer-swap memory model"); + let manifest_json = std::fs::read_to_string(manifest_path).unwrap_or_else(|err| { + panic!("failed to read weight manifest {:?}: {err}", manifest_path) + }); + let manifest: memory::streaming::WeightManifest = + serde_json::from_str(&manifest_json).unwrap_or_else(|err| { + panic!("failed to parse weight manifest {:?}: {err}", manifest_path) + }); + let capacity = opts.ddr3_capacity as u64; + let bandwidth = opts.host_bandwidth; + tracing::info!( + "Memory model: layer-swap (DDR3 capacity={} MB, host bandwidth={} MB/s)", + capacity / (1 << 20), + bandwidth / 1_000_000 + ); + // DDR4-2400 preset approximating the board's DDR3-1600 timing. + let ddr3_timing = memory::SimpleTiming::preset_ddr4_2400p(1); + let model = Arc::new(memory::WithStats::new( + memory::streaming::LayerSwapping::new( + memory::WithTiming::new(ddr3_timing, hbm_backing), + manifest, + capacity, + bandwidth, + ), + )); + (model, None) + } + cli::MemoryModelKind::HostStream => { + let manifest_path = opts + .weight_manifest + .as_ref() + .expect("--weight-manifest required for host-stream memory model"); + let manifest_json = std::fs::read_to_string(manifest_path).unwrap_or_else(|err| { + panic!("failed to read weight manifest {:?}: {err}", manifest_path) + }); + let manifest: memory::streaming::WeightManifest = + serde_json::from_str(&manifest_json).unwrap_or_else(|err| { + panic!("failed to parse weight manifest {:?}: {err}", manifest_path) + }); + let bandwidth = opts.host_bandwidth; + tracing::info!( + "Memory model: host-stream (host bandwidth={} MB/s)", + bandwidth / 1_000_000 + ); + // DDR4-2400 preset approximating the board's DDR3-1600 timing. + // Activation reads go through this timing (weight reads bypass it + // via the host link), matching the HostStream struct doc. + let ddr3_timing = memory::SimpleTiming::preset_ddr4_2400p(1); + let model = Arc::new(memory::WithStats::new(memory::streaming::HostStream::new( + memory::WithTiming::new(ddr3_timing, hbm_backing), + manifest, + bandwidth, + ))); + (model, None) + } + cli::MemoryModelKind::Auto + | cli::MemoryModelKind::Resident + | cli::MemoryModelKind::KvSwap + | cli::MemoryModelKind::WeightStream => { + use memory::streaming::Regime; + + let manifest_path = opts + .weight_manifest + .as_ref() + .expect("--weight-manifest required for capacity-aware memory model"); + let manifest_json = std::fs::read_to_string(manifest_path).unwrap_or_else(|err| { + panic!("failed to read weight manifest {:?}: {err}", manifest_path) + }); + let manifest: memory::streaming::WeightManifest = + serde_json::from_str(&manifest_json).unwrap_or_else(|err| { + panic!("failed to parse weight manifest {:?}: {err}", manifest_path) + }); + + // DDR capacity: prefer the manifest's value if present, else the + // CLI --ddr3-capacity flag. + let capacity = manifest + .ddr_capacity_bytes + .unwrap_or(opts.ddr3_capacity as u64); + let bandwidth = opts.host_bandwidth; + + // For Auto, pick the regime from the model footprint; the other + // modes select their regime explicitly. + let regime = match opts.memory_model { + cli::MemoryModelKind::Auto => { + let (weight, kv, activation) = manifest.footprint_by_kind(); + let total = weight + kv + activation; + memory::streaming::choose_regime(weight, total, capacity) + } + cli::MemoryModelKind::Resident => Regime::Resident, + cli::MemoryModelKind::KvSwap => Regime::KvSwap, + cli::MemoryModelKind::WeightStream => Regime::WeightStream, + _ => unreachable!("outer match restricts to capacity-aware kinds"), + }; + + tracing::info!( + "Memory model: capacity-aware [{:?}] (DDR capacity={} MB, host bandwidth={} MB/s)", + regime, + capacity / (1 << 20), + bandwidth / 1_000_000 + ); + + // DDR4-2400 preset approximating the board's DDR3-1600 timing + // (same inner DDR-timed model used by layer-swap). + let ddr3_timing = memory::SimpleTiming::preset_ddr4_2400p(1); + let model = Arc::new(memory::WithStats::new( + memory::streaming::CapacityModel::new( + memory::WithTiming::new(ddr3_timing, hbm_backing), + manifest, + capacity, + bandwidth, + regime, + ), + )); + (model, None) + } + }; let mut accelerator = Accelerator::new(m_machine, v_machine, hbm.clone()); @@ -151,14 +300,6 @@ pub(crate) async fn run_from_cli() { }) .collect(); - // Memory Initialization - // - HBM Preload - let hbm_data = std::fs::read(&opts.hbm) - .unwrap_or_else(|err| panic!("failed to read HBM preload file {:?}: {err}", opts.hbm)); - hbm.model().data().with_data(|f| { - f[..hbm_data.len()].copy_from_slice(&hbm_data); - }); - // Load fpsram and intsram as raw bytes and map to the vector files. // - fpsram Preload let fpsram_data = std::fs::read(&opts.fpsram).unwrap_or_else(|err| { @@ -210,26 +351,34 @@ pub(crate) async fn run_from_cli() { let fpsram_bytes = accelerator.fpsram_dump_bytes(); dump_to_file("fpsram_dump.bin", &fpsram_bytes); - // Dump HBM — skipped unless DEBUG tracing is enabled because HBM_SIZE may - // be 128 GiB+. Tests run with --log-level warn and don't need hbm_dump.bin; - // only manual debug runs dump HBM. - if tracing::enabled!(tracing::Level::DEBUG) { - let hbm_size = effective_hbm_size; - let mut hbm_bytes = vec![0u8; hbm_size]; - hbm.model().data().with_data(|f| { - let len = std::cmp::min(hbm_size, f.len()); - hbm_bytes[..len].copy_from_slice(&f[..len]); - }); - dump_to_file("hbm_dump.bin", &hbm_bytes); + // Dump HBM (DEBUG only) + report statistics. + // + // The DEBUG-only HBM dump reads back the backing store via + // `.model().data()`, a concrete `WithStats>` method that the + // erased `ErasedMemoryModel` trait does not expose. It is therefore still + // gated on the typed `hbm_concrete` handle, which is only populated for the + // default Hbm model. + if let Some(hbm) = &hbm_concrete { + // Dump HBM — skipped unless DEBUG tracing is enabled because + // HBM_SIZE may be 128 GiB+. Tests run with --log-level warn and + // don't need hbm_dump.bin; only manual debug runs dump HBM. + if tracing::enabled!(tracing::Level::DEBUG) { + let hbm_size = effective_hbm_size; + let mut hbm_bytes = vec![0u8; hbm_size]; + hbm.model().data().with_data(|f| { + let len = std::cmp::min(hbm_size, f.len()); + hbm_bytes[..len].copy_from_slice(&f[..len]); + }); + dump_to_file("hbm_dump.bin", &hbm_bytes); + } } - let memory_stats = hbm.statistics(); - let utilization = (memory_stats.total_bytes_read + memory_stats.total_bytes_written) as f64 - / Executor::current().now().to_secs(); - tracing::info!( - "HBM Statistics - Bytes read: {:?} | Bytes written: {:?} | Utilization: {:.2e} bytes/sec", - memory_stats.total_bytes_read, - memory_stats.total_bytes_written, - utilization - ); + // Statistics are surfaced uniformly through the erased memory model. Each + // concrete model renders its own summary (the default Hbm path reproduces + // the historical "HBM Statistics - ..." line byte-for-byte); models that + // don't track statistics return `None`. + let elapsed_secs = Executor::current().now().to_secs(); + if let Some(summary) = hbm.box_statistics_summary(elapsed_secs) { + tracing::info!("{summary}"); + } } diff --git a/transactional_emulator/testbench/aten/compare/isa_analysis.py b/transactional_emulator/testbench/aten/compare/isa_analysis.py index 15b0f95b..3c191be3 100644 --- a/transactional_emulator/testbench/aten/compare/isa_analysis.py +++ b/transactional_emulator/testbench/aten/compare/isa_analysis.py @@ -74,13 +74,26 @@ class SimulatorCycleModel: scalar_fp_sqrt_cycles: int scalar_fp_reci_cycles: int scalar_int_basic_cycles: int + # DDR3 / memory cycle costs. Board configs supply these (memory: section); + # the default 0 keeps H_PREFETCH/H_STORE async (charged lazily by a later + # SRAM access), matching the behaviour-simulator model used everywhere else. + prefetch_v_cycles: int = 0 + prefetch_m_cycles: int = 0 + store_v_cycles: int = 0 @property def description(self) -> str: + if self.prefetch_v_cycles or self.prefetch_m_cycles or self.store_v_cycles: + mem_desc = ( + "H_PREFETCH/H_STORE statically charged " + f"(V={self.prefetch_v_cycles}, M={self.prefetch_m_cycles}, store={self.store_v_cycles} cyc)" + ) + else: + mem_desc = "H_PREFETCH/H_STORE async memory timing not statically charged" return ( f"behavior simulator constants from {self.settings_path} " f"(DC_EN={self.dc_en}, latency_profile={self.latency_profile or 'dc_lib_dis'}; " - "H_PREFETCH/H_STORE async memory timing not statically charged)" + f"{mem_desc})" ) def instruction_cycles(self, opcode: str) -> int: @@ -116,10 +129,14 @@ def instruction_cycles(self, opcode: str) -> int: return self.scalar_int_basic_cycles if opcode in CONTROL_ONE_CYCLE_OPS: return 1 - if opcode.startswith("H_PREFETCH") or opcode.startswith("H_STORE"): - # The simulator starts an async memory transfer here; the wait is - # paid by a later SRAM read/write if the transfer has not resolved. - return 0 + if opcode == "H_PREFETCH_V": + return self.prefetch_v_cycles + if opcode == "H_PREFETCH_M": + return self.prefetch_m_cycles + if opcode.startswith("H_STORE"): + return self.store_v_cycles + if opcode.startswith("H_PREFETCH"): + return self.prefetch_v_cycles return 1 @@ -305,7 +322,7 @@ def analyze_asm( def load_board_config(board: str) -> dict[str, Any]: - """Load a board config YAML by name (e.g. 'nexys_a7', 'v80').""" + """Load a board config YAML by name (e.g. 'nexys_video', 'custom_a7', 'v80').""" import yaml path = _BOARD_CONFIG_DIR / f"{board}.yaml" @@ -319,6 +336,7 @@ def load_board_config(board: str) -> dict[str, Any]: def cycle_model_from_board(board_cfg: dict[str, Any], *, mlen: int = 64, vlen: int = 64) -> SimulatorCycleModel: """Build a SimulatorCycleModel from a board config's `latency:` section.""" lat = board_cfg["latency"] + mem = board_cfg.get("memory", {}) dc_en = 1 if lat.get("dc_lib_en", False) else 0 return SimulatorCycleModel( settings_path=Path(board_cfg.get("name", "board_config")), @@ -338,6 +356,9 @@ def cycle_model_from_board(board_cfg: dict[str, Any], *, mlen: int = 64, vlen: i scalar_fp_sqrt_cycles=lat["scalar_fp_sqrt_cycles"], scalar_fp_reci_cycles=lat["scalar_fp_reci_cycles"], scalar_int_basic_cycles=lat["scalar_int_basic_cycles"], + prefetch_v_cycles=mem.get("prefetch_v_cycles", 0), + prefetch_m_cycles=mem.get("prefetch_m_cycles", 0), + store_v_cycles=mem.get("store_v_cycles", 0), ) @@ -349,7 +370,7 @@ def main() -> None: parser = argparse.ArgumentParser(description="Profile PLENA ASM cycle cost against a board config") parser.add_argument("asm_file", type=Path, help="Path to generated_asm_code.asm") parser.add_argument( - "--board", default="nexys_a7", help=f"Board config name (from board_configs/). Available: {available}" + "--board", default="nexys_video", help=f"Board config name (from board_configs/). Available: {available}" ) parser.add_argument("--mlen", type=int, default=64, help="MLEN for matrix-op cycle cost (default: 64)") parser.add_argument("--clock-mhz", type=float, default=None, help="Override clock (default: board clock_mhz)") diff --git a/transactional_emulator/testbench/aten/configurable.py b/transactional_emulator/testbench/aten/configurable.py index fbd889d5..5efa350f 100644 --- a/transactional_emulator/testbench/aten/configurable.py +++ b/transactional_emulator/testbench/aten/configurable.py @@ -37,7 +37,7 @@ LATENCY_PROFILE_PRESETS: dict[str, dict[str, int]] = { # Measured on current main + khl/addr-pipeline-fix RTL, behavioral mode. # See ~/docs/plena_latency_audit_20260520.md. - "nexys_a7_150mhz": { + "nexys_video_150mhz": { "SCALAR_FP_EXP_CYCLES": 18, "SCALAR_FP_RECI_CYCLES": 6, "VECTOR_ADD_CYCLES": 11, @@ -72,6 +72,56 @@ def apply_latency_profile_config( latency[name][latency_profile] = int(value) +# Board YAML `latency:` keys -> TOML LATENCY entry (uppercased). The TOML carries +# extra entries (VECTOR_BASIC/_LONGEST_OPERATE, SCALAR_FP_LONGEST_OPERATE) that the +# board does not specify; those keep their base defaults. +_BOARD_LATENCY_KEYS: tuple[str, ...] = ( + "systolic_processing_overhead", + "vector_add_cycles", + "vector_mul_cycles", + "vector_exp_cycles", + "vector_reci_cycles", + "vector_max_cycles", + "vector_sum_cycles", + "vector_shift_cycles", + "vector_prefix_scan_cycles", + "scalar_fp_basic_cycles", + "scalar_fp_exp_cycles", + "scalar_fp_sqrt_cycles", + "scalar_fp_reci_cycles", + "scalar_int_basic_cycles", +) + + +def apply_board_latency_config(config: dict[str, Any], board_cfg: dict[str, Any]) -> None: + """Apply a board YAML's `latency:` block to the per-build TOML. + + Sets DC_EN from the board's `dc_lib_en` flag and writes each board per-op + cycle cost into the column DC_EN selects (`dc_lib_en` when enabled, else + `dc_lib_dis`), so the transactional emulator runs on that board's latencies. + The board's `memory:` cycle params (prefetch/ddr3) feed the analytic profiler + (isa_analysis.py), not the emulator's memory model, so they are not applied + here. + """ + lat = board_cfg.get("latency") or {} + if not lat: + return + dc_en = 1 if lat.get("dc_lib_en", False) else 0 + column = "dc_lib_en" if dc_en else "dc_lib_dis" + for mode in ("TRANSACTIONAL", "ANALYTIC"): + if mode not in config: + continue + mode_config = config[mode].setdefault("CONFIG", {}) + mode_config.setdefault("DC_EN", {})["value"] = dc_en + latency = config[mode].setdefault("LATENCY", {}) + for key in _BOARD_LATENCY_KEYS: + if key not in lat: + continue + entry = latency.get(key.upper()) + if entry is not None: + entry[column] = int(lat[key]) + + @dataclass(frozen=True) class HardwareConfig: mlen: int @@ -85,6 +135,7 @@ class HardwareConfig: hbm_v_prefetch_amount: int | None hbm_v_writeback_amount: int | None real_data_ratio: float = DEFAULT_REAL_DATA_RATIO + board_cfg: dict[str, Any] | None = None @classmethod def from_args( @@ -154,6 +205,11 @@ def write_toml(self, build_dir: Path) -> Path: dc_en=self.dc_en, latency_profile=self.latency_profile, ) + # A board config (if given) sets the per-op cycle costs + DC_EN directly + # from its `latency:` block, overriding the base defaults so the emulator + # runs on that board's latencies. + if self.board_cfg: + apply_board_latency_config(config, self.board_cfg) # PlenaCompiler reads from BEHAVIOR.CONFIG — mirror tile dimensions AND # the HBM prefetch/writeback amounts there. The compiler's preload @@ -307,7 +363,7 @@ def add_common_args(parser: argparse.ArgumentParser, *, default_build_dir: Path) parser.add_argument( "--latency-profile", default=None, - help=("Optional per-FPGA latency profile for DC_EN=0. Known preset: nexys_a7_150mhz."), + help=("Optional per-FPGA latency profile for DC_EN=0. Known preset: nexys_video_150mhz."), ) parser.add_argument("--seed", type=int, default=42) parser.add_argument("--no-run", action="store_true") diff --git a/transactional_emulator/testbench/board_configs/custom_a7.yaml b/transactional_emulator/testbench/board_configs/custom_a7.yaml new file mode 100644 index 00000000..a88c947c --- /dev/null +++ b/transactional_emulator/testbench/board_configs/custom_a7.yaml @@ -0,0 +1,81 @@ +# Custom Artix-7 board — Xilinx XC7A200T, 512 MB DDR3, PCIe Gen2 x4 host link. +# Same FPGA fabric / DDR3 as the Nexys Video, but the host link is PCIe Gen2 x4 +# (~1.5 GB/s effective, ~315x the Nexys Video USB2 link) instead of USB. + +name: custom_a7 +description: Custom Artix-7 XC7A200T, 512 MB DDR3, PCIe Gen2 x4 host link +vendor: xilinx +family: artix7 +part: xc7a200t + +clock_mhz: 100 + +memory: + type: ddr3 + spec: DDR3-1600 + chip: MT41K256M16HA + bus_width: x16 + physical_size_bytes: 536870912 # 512 MiB DDR3 + capacity_bytes: 536870912 # usable DDR3 capacity for the capacity-aware memory model + modeled_size_bytes: 137438953472 # 128 GiB (emulator flat-HBM model) + bus_width_bits: 512 + peak_bandwidth_gbps: 1.6 + effective_bandwidth_gbps: 1.0 + matrix_sram_tiles: 4096 + vector_sram_depth: 4194304 + hbm_m_prefetch_amount: 64 + hbm_v_prefetch_amount: 4 + hbm_v_writeback_amount: 4 + ddr3_burst_cycles: 5 + ddr3_row_miss_penalty: 3 + prefetch_v_cycles: 25 + prefetch_m_cycles: 400 + store_v_cycles: 25 + +# Host link for streaming weights/KV that don't fit in DDR3. +# PCIe Gen2 x4: 5 GT/s/lane x 8b/10b = 500 MB/s/lane x4 = 2.0 GB/s theoretical +# (16 Gbps); realistic sustained after TLP overhead ~1.5 GB/s (~12 Gbps). +host_link: + type: pcie_gen2_x4 + bandwidth_bytes_per_sec: 1500000000 # ~1.5 GB/s sustained (theoretical 2.0 GB/s) + theoretical_bandwidth_bytes_per_sec: 2000000000 + +precision: + matrix_sram: {type: fp, sign: true, exponent: 8, mantissa: 7} # BF16 + vector_sram: {type: fp, sign: true, exponent: 8, mantissa: 7} # BF16 + hbm_act: {exponent: 4, mantissa: 3, scale_exponent: 8, block: 8} # MXFP8 E4M3 + hbm_weight: {exponent: 4, mantissa: 3, scale_exponent: 8, block: 8} + +latency: + dc_lib_en: false + # Aligned to RTL pipeline_pkg DC_LIB_DIS block (src/definitions/configuration.svh, FPGA build). + # Prior values under-counted the vector FP pipeline (incl. the 100 MHz timing-closure pipeline regs). + systolic_processing_overhead: 8 # was 0; RTL=8 (systolic feed/drain: 2*COMPUTE_DIM + 8) + vector_add_cycles: 9 # was 2; RTL=9 (+1 fp_cp_adder partition register) + vector_mul_cycles: 7 # was 5; RTL=7 + vector_exp_cycles: 15 # was 6; RTL=15 (+1 fp_cp_exp normalize->cast PIPELINE_STAGE) + vector_reci_cycles: 8 # was 7; RTL=8 (+1 fp_cp_reciprocal) + vector_max_cycles: 4 + vector_sum_cycles: 30 # was 20; RTL=30 (+1/level x5 reduction partition register) + vector_shift_cycles: 1 + vector_prefix_scan_cycles: 9 + scalar_fp_basic_cycles: 1 + scalar_fp_exp_cycles: 2 + scalar_fp_sqrt_cycles: 2 + scalar_fp_reci_cycles: 2 + scalar_int_basic_cycles: 1 + +power: + tdp_watts: 7.5 + io_voltage: 3.3 + +resources: + luts: 134600 + ffs: 269200 + bram_36k: 365 + dsp48: 740 + +tile_hints: + - {mlen: 64, vlen: 64, blen: 4, batch_size: 1, status: proven} + - {mlen: 64, vlen: 64, blen: 16, batch_size: 1, status: proven} + - {mlen: 256, vlen: 256, blen: 64, batch_size: 1, status: exceeds_resources} diff --git a/transactional_emulator/testbench/board_configs/nexys_a7.yaml b/transactional_emulator/testbench/board_configs/nexys_video.yaml similarity index 65% rename from transactional_emulator/testbench/board_configs/nexys_a7.yaml rename to transactional_emulator/testbench/board_configs/nexys_video.yaml index 56ff95a4..b4448494 100644 --- a/transactional_emulator/testbench/board_configs/nexys_a7.yaml +++ b/transactional_emulator/testbench/board_configs/nexys_video.yaml @@ -1,8 +1,9 @@ # Digilent Nexys Video — Xilinx Artix-7 XC7A200T -# FPGA fabric only, no dedicated compute library (dc_lib_dis) +# FPGA fabric only, no dedicated compute library (dc_lib_dis). +# Host link is USB only (FT2232H) — NO PCIe on this board. -name: nexys_a7 -description: Artix-7 XC7A200T on Nexys Video board +name: nexys_video +description: Artix-7 XC7A200T on Digilent Nexys Video (512 MB DDR3, USB host link, no PCIe) vendor: xilinx family: artix7 part: xc7a200t @@ -15,7 +16,8 @@ memory: chip: MT41K256M16HA # single x16 chip on Nexys Video bus_width: x16 # 2 bytes per transfer physical_size_bytes: 536870912 # 512 MiB (Nexys Video onboard) - modeled_size_bytes: 137438953472 # 128 GiB (emulator model) + capacity_bytes: 536870912 # usable DDR3 capacity for the capacity-aware memory model + modeled_size_bytes: 137438953472 # 128 GiB (emulator flat-HBM model) bus_width_bits: 512 # PLENA HBM abstraction layer width peak_bandwidth_gbps: 1.6 # x16 bus: 2B × 800 MT/s effective_bandwidth_gbps: 1.0 # ~60-65% efficiency (matches Vitis AI DPU B1152 avg) @@ -34,7 +36,12 @@ memory: prefetch_v_cycles: 25 # 256B at ~1.0 GB/s effective prefetch_m_cycles: 400 # 4KB at ~1.0 GB/s effective store_v_cycles: 25 # write-back same as prefetch V - usb2_bandwidth_mbps: 38 # FT2232H practical (not theoretical 60) + +# Host link used to stream weights/KV when they do not fit in DDR3. +# Nexys Video has no PCIe — only USB2 via the FT2232H. +host_link: + type: usb2 + bandwidth_bytes_per_sec: 4750000 # FT2232H ~38 Mbps practical (not theoretical 60) = ~4.75 MB/s precision: matrix_sram: {type: fp, sign: true, exponent: 8, mantissa: 7} # BF16 @@ -44,13 +51,15 @@ precision: latency: dc_lib_en: false - systolic_processing_overhead: 0 - vector_add_cycles: 2 - vector_mul_cycles: 5 - vector_exp_cycles: 6 - vector_reci_cycles: 7 + # Aligned to RTL pipeline_pkg DC_LIB_DIS block (src/definitions/configuration.svh, FPGA build). + # Prior values under-counted the vector FP pipeline (incl. the 100 MHz timing-closure pipeline regs). + systolic_processing_overhead: 8 # was 0; RTL=8 (systolic feed/drain: 2*COMPUTE_DIM + 8) + vector_add_cycles: 9 # was 2; RTL=9 (+1 fp_cp_adder partition register) + vector_mul_cycles: 7 # was 5; RTL=7 + vector_exp_cycles: 15 # was 6; RTL=15 (+1 fp_cp_exp normalize->cast PIPELINE_STAGE) + vector_reci_cycles: 8 # was 7; RTL=8 (+1 fp_cp_reciprocal) vector_max_cycles: 4 - vector_sum_cycles: 20 + vector_sum_cycles: 30 # was 20; RTL=30 (+1/level x5 reduction partition register) vector_shift_cycles: 1 vector_prefix_scan_cycles: 9 scalar_fp_basic_cycles: 1 diff --git a/transactional_emulator/testbench/make_weight_manifest.py b/transactional_emulator/testbench/make_weight_manifest.py new file mode 100644 index 00000000..06559f28 --- /dev/null +++ b/transactional_emulator/testbench/make_weight_manifest.py @@ -0,0 +1,365 @@ +#!/usr/bin/env python3 +"""Build a kind-tagged weight manifest from a real PLENA compile. + +The capacity-aware memory model in the emulator (``lib/memory/src/streaming.rs``) +selects a DDR-residency regime (resident / kv-swap / weight-stream) from a +``WeightManifest`` that tags every persistent HBM region as ``weight``, ``kv`` or +``activation`` and carries the board's DDR capacity. This script turns the +artifacts a real compile drops into ```` into exactly that manifest, so +the regimes fire on a real SmolVLM2 / SmolLM2 / clm workload instead of a +synthetic stand-in. + +Inputs (written by ``run_model.py`` right after the ISA, for any --case): + * ``hbm_addrs.json`` name -> HBM base byte offset (REQUIRED) + * ``hbm_sizes.json`` name -> region bytes (authoritative; incl. KV) (preferred) + * ``tensor_layouts.json`` name -> {storage_shape, ...} for INPUT tensors (fallback) + +Sizing precedence per region: + 1. ``hbm_sizes[name]`` - authoritative; covers KV stores. + 2. ``int(prod(storage_shape) * 1.125)`` - exact for inputs/weights; misses KV. + (MXFP8 E4M3, block=8, scale_exponent=8 -> real_data_ratio = 9/8 = 1.125, + matching the compiler: program_tensors.py ``hbm_size = int(size*1.125)``.) + 3. sorted-offset address delta - last resort, contiguous prefix only. + +Classification by region name (see plena_frontend.py emission conventions): + * KV : name contains a ``_stored_`` token preceded by K or V + (K_stored_*, V_stored_*, V_K_stored_*, V_V_stored_*). + * weight : decoder matmul weights W_*; vision attention/FFN weights V_W_*; + vision biases V_B_* (uppercase B: V_B_q/k/v/o/fc1/fc2); vision + layernorm params V_LN1_*/V_LN2_*/V_POST_LN_* (UPPERCASE); and the + patch-embed/connector weights V_PATCH_W, V_PATCH_BIAS_POS, + V_CONNECTOR_W, V_CONNECTOR_B. + * activation : everything else (X, POS, COS, SIN, R_rope, causal_mask, V_PIXELS, + the post-LN output vision_post_ln, and per-layer intermediates like + K_{l}_h{kv}, O_proj_{l}, V_FC1_{l}, V_PATCH_OUT, V_CONNECTOR_OUT). + NB classify KV by the ``_stored_`` token, NOT a leading K/V: bare K_{l}_h{kv} is a + pre-store projection activation, and the V_ prefix is the *vision* namespace. + +Usage: + python testbench/make_weight_manifest.py \ + --build-dir testbench/build/clm60m_native_64x64x16_b1_decoder \ + --board testbench/board_configs/custom_a7.yaml + # optional: --total-layers 30 (extrapolate full-model footprint in the report) + # --ddr-capacity 256M (override board capacity, e.g. to force a regime) + # -o path/to/weight_manifest.json +""" + +from __future__ import annotations + +import argparse +import json +import re +import sys +from math import prod +from pathlib import Path + +# MXFP8 (E4M3, block=8, scale_exponent=8): 1 data byte/elem + 1 scale byte / 8 +# elems = 9/8 bytes per element. Hardcoded in the compiler as real_data_ratio. +REAL_DATA_RATIO = 1.125 + +_KV_RE = re.compile(r"(^|_)[KV]_stored_") +# Weight (persistent parameter) name prefixes. Decoder matmul weights are W_*; +# vision attention/FFN weights V_W_*; vision biases V_B_* (uppercase B, incl. +# V_B_q/k/v/o/fc1/fc2); vision layernorm params are UPPERCASE V_LN1_*/V_LN2_*. +_WEIGHT_PREFIXES = ("W_", "V_W_", "V_B_", "V_LN1_", "V_LN2_", "V_POST_LN_") +# Vision patch-embed + connector weights/biases (exact names: their _OUT / +# _SHUFFLED siblings are VRAM alloc intermediates, never HBM regions). NB the +# bare "vision_post_ln" is the post-LN *output* activation — the params are +# V_POST_LN_weight/bias (covered by the V_POST_LN_ prefix above). +_WEIGHT_EXACT = frozenset({"V_PATCH_W", "V_PATCH_BIAS_POS", "V_CONNECTOR_W", "V_CONNECTOR_B"}) +# Per-head suffix and digit-bearing tokens (vision fc1/fc2/ln1/ln2) that are NOT +# the layer index; stripped/neutralised before extracting the layer id. +_HEAD_SUFFIX_RE = re.compile(r"_h\d+$") +_LAYER_TOKEN_RE = re.compile(r"(?i)(fc|ln)[12]") +_FIRST_INT_RE = re.compile(r"\d+") + +# RegionKind strings the Rust deserializer accepts (#[serde(rename_all="lowercase")]). +KIND_WEIGHT = "weight" +KIND_KV = "kv" +KIND_ACTIVATION = "activation" + + +def classify(name: str) -> str: + """Map a region name to a RegionKind string.""" + if _KV_RE.search(name): # classify KV by the _stored_ token, not a leading K/V + return KIND_KV + if name in _WEIGHT_EXACT or name.startswith(_WEIGHT_PREFIXES): + return KIND_WEIGHT + return KIND_ACTIVATION + + +def layer_id(name: str) -> int: + """Layer index from the name; 0 for global/shared tensors. + + Strips the per-head suffix (_h) and neutralises digit-bearing tokens that + are not the layer (vision fc1/fc2/ln1/ln2 -> fc/ln) so the first remaining + integer is the layer index (e.g. V_W_fc1_4 -> 4, vision_l4_ln1 -> 4). + """ + base = _HEAD_SUFFIX_RE.sub("", name) + base = _LAYER_TOKEN_RE.sub(lambda m: m.group(0)[:-1], base) + m = _FIRST_INT_RE.search(base) + return int(m.group()) if m else 0 + + +def parse_size(text: str) -> int: + """Parse '512M' / '512MiB' / '256Mi' / '1G' / plain bytes -> integer bytes. + + Binary units (K/M/G = 2^10/2^20/2^30), matching the Rust --ddr3-capacity + parser and the board YAML capacity_bytes values. Rejects malformed or + non-positive sizes with a clear error rather than a deep ValueError. + """ + t = str(text).strip().upper() + if t.endswith("B"): + t = t[:-1] + if t.endswith("I"): # MiB/Mi -> M + t = t[:-1] + mult = 1 + if t.endswith("K"): + mult, t = 1 << 10, t[:-1] + elif t.endswith("M"): + mult, t = 1 << 20, t[:-1] + elif t.endswith("G"): + mult, t = 1 << 30, t[:-1] + try: + val = int(float(t) * mult) + except ValueError: + raise SystemExit(f"--ddr-capacity: cannot parse size {text!r}") + if val <= 0: + raise SystemExit(f"--ddr-capacity must be a positive size, got {text!r}") + return val + + +def load_board_capacity(board_path: Path) -> tuple[int, str]: + """Return (capacity_bytes, board_name) from a board YAML's memory section.""" + import yaml + + cfg = yaml.safe_load(board_path.read_text()) + mem = cfg.get("memory", {}) + cap = mem.get("capacity_bytes") or mem.get("physical_size_bytes") + if cap is None: + raise SystemExit(f"board {board_path} has no memory.capacity_bytes / physical_size_bytes") + return int(cap), cfg.get("name", board_path.stem) + + +def tl_size(entry: dict) -> int | None: + """Region bytes from a tensor_layouts entry: int(prod(storage_shape)*1.125).""" + shape = entry.get("storage_shape") + if not shape: + return None + return int(prod(int(d) for d in shape) * REAL_DATA_RATIO) + + +def build_regions(build_dir: Path) -> tuple[list[dict], dict]: + """Build sorted, kind-tagged regions from a build dir's HBM artifacts. + + Returns (regions, diag) where diag records which size source was used. + """ + addrs_path = build_dir / "hbm_addrs.json" + if not addrs_path.exists(): + raise SystemExit( + f"missing {addrs_path}\n" + "Recompile with run_model.py (any case) to emit hbm_addrs.json; e.g.\n" + " python testbench/run_model.py clm60m --config native_64x64x16_b1 " + "--case decoder --layers 1 --compile-only" + ) + addrs: dict[str, int] = json.loads(addrs_path.read_text()) + + sizes_path = build_dir / "hbm_sizes.json" + sizes: dict[str, int | None] = json.loads(sizes_path.read_text()) if sizes_path.exists() else {} + layouts_path = build_dir / "tensor_layouts.json" + layouts: dict[str, dict] = json.loads(layouts_path.read_text()) if layouts_path.exists() else {} + + # Sorted by offset so the delta fallback (and human-readable output) is + # stable. Skip entries without a usable integer offset (e.g. an unmapped + # input with hbm_addr None) rather than crashing the sort. + ordered = sorted( + ((n, o) for n, o in addrs.items() if isinstance(o, int)), + key=lambda kv: kv[1], + ) + dropped_offset = [n for n, o in addrs.items() if not isinstance(o, int)] + src_count = {"hbm_sizes": 0, "tensor_layouts": 0, "delta": 0} + unsized: list[str] = [] + regions: list[dict] = [] + + for idx, (name, offset) in enumerate(ordered): + # Resolve size by precedence; accept a source only if it yields a + # positive int. A None/0 from hbm_sizes must NOT shadow a usable + # tensor_layouts/delta size, so fall through on each miss. + size = None + source = None + cand = sizes.get(name) + if isinstance(cand, int) and cand > 0: + size, source = cand, "hbm_sizes" + if size is None: + entry = layouts.get(name) + cand = tl_size(entry) if entry is not None else None + if isinstance(cand, int) and cand > 0: + size, source = cand, "tensor_layouts" + if size is None and idx + 1 < len(ordered): + # contiguous-prefix address delta: an UPPER bound (stride incl. any + # padding/gap), reliable only for the never-freed prefix. + cand = ordered[idx + 1][1] - offset + if isinstance(cand, int) and cand > 0: + size, source = cand, "delta" + if size is None: + unsized.append(name) + continue + src_count[source] += 1 + regions.append( + { + "layer_id": layer_id(name), + "offset": int(offset), + "size": int(size), + "kind": classify(name), + "_name": name, # diagnostic only; stripped before writing + } + ) + + # Guard overlaps: Rust region_kind()/region_bounds() are first-match-wins on + # offset-sorted regions, so an oversized region would shadow (misclassify) + # the next. Clamp to abut the next region and record the correction. + overlaps: list[tuple] = [] + for i in range(len(regions) - 1): + end = regions[i]["offset"] + regions[i]["size"] + nxt = regions[i + 1]["offset"] + if end > nxt: + overlaps.append((regions[i]["_name"], regions[i]["size"], nxt - regions[i]["offset"])) + regions[i]["size"] = nxt - regions[i]["offset"] + + return regions, { + "src_count": src_count, + "unsized": unsized, + "dropped_offset": dropped_offset, + "overlaps": overlaps, + "n_addrs": len(addrs), + } + + +def footprint(regions: list[dict]) -> dict[str, int]: + f = {KIND_WEIGHT: 0, KIND_KV: 0, KIND_ACTIVATION: 0} + for r in regions: + f[r["kind"]] += r["size"] + return f + + +def choose_regime(weight: int, total: int, capacity: int) -> str: + """Mirror of streaming.rs::choose_regime.""" + if total <= capacity: + return "resident" + if weight <= capacity: + return "kv-swap" + return "weight-stream" + + +def fmt_mb(n: int) -> str: + return f"{n / (1 << 20):.2f} MiB" if n else "0" + + +def main() -> int: + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--build-dir", required=True, type=Path, help="compile output dir with hbm_addrs.json") + ap.add_argument("--board", required=True, type=Path, help="board YAML (supplies memory.capacity_bytes)") + ap.add_argument("--ddr-capacity", type=str, default=None, help="override board capacity, e.g. 256M") + ap.add_argument( + "--total-layers", type=int, default=None, help="extrapolate per-layer footprint to N layers in the report" + ) + ap.add_argument( + "-o", "--output", type=Path, default=None, help="manifest path (default: /weight_manifest.json)" + ) + args = ap.parse_args() + + regions, diag = build_regions(args.build_dir) + if not regions: + raise SystemExit("no sizeable HBM regions found") + + capacity, board_name = load_board_capacity(args.board) + if args.ddr_capacity: + capacity = parse_size(args.ddr_capacity) + + fp = footprint(regions) + weight, kv, act = fp[KIND_WEIGHT], fp[KIND_KV], fp[KIND_ACTIVATION] + total = weight + kv + act + + # activation_ceiling: legacy-only field (consumed by is_weight_addr in the + # host-stream/layer-swap models, NOT by the capacity model). Set it to the + # lowest weight offset so the legacy view still separates weights from the + # activations/embeddings laid out before them. + weight_offsets = [r["offset"] for r in regions if r["kind"] == KIND_WEIGHT] + activation_ceiling = min(weight_offsets) if weight_offsets else 0 + + out_regions = [{k: r[k] for k in ("layer_id", "offset", "size", "kind")} for r in regions] + manifest = { + "activation_ceiling": activation_ceiling, + "regions": out_regions, + "ddr_capacity_bytes": capacity, + } + + out_path = args.output or (args.build_dir / "weight_manifest.json") + out_path.write_text(json.dumps(manifest, indent=2)) + + # ---- report ----------------------------------------------------------- + by_kind = {KIND_WEIGHT: 0, KIND_KV: 0, KIND_ACTIVATION: 0} + for r in regions: + by_kind[r["kind"]] += 1 + print(f"Wrote {out_path}") + print(f" board: {board_name} (DDR capacity {fmt_mb(capacity)})") + print( + f" regions: {len(regions)} ({by_kind[KIND_WEIGHT]} weight, {by_kind[KIND_KV]} kv, {by_kind[KIND_ACTIVATION]} activation)" + ) + print( + f" sizes: {diag['src_count']['hbm_sizes']} from hbm_sizes, " + f"{diag['src_count']['tensor_layouts']} from tensor_layouts, " + f"{diag['src_count']['delta']} from address-delta" + ) + if diag["unsized"]: + print( + f" WARNING: {len(diag['unsized'])} region(s) had no size and were DROPPED (coverage hole): {diag['unsized']}" + ) + if diag["src_count"]["delta"]: + print( + f" WARNING: {diag['src_count']['delta']} region(s) sized by address-delta (upper bound; footprint may be inflated). " + "Recompile with an hbm_sizes-emitting compiler for exact sizes." + ) + if diag.get("overlaps"): + print(f" WARNING: clamped {len(diag['overlaps'])} overlapping region(s) to abut the next: {diag['overlaps']}") + if diag.get("dropped_offset"): + print( + f" WARNING: {len(diag['dropped_offset'])} region(s) had no integer offset and were skipped: {diag['dropped_offset']}" + ) + print( + f" footprint (this build): weight {fmt_mb(weight)} | kv {fmt_mb(kv)} | activation {fmt_mb(act)} | total {fmt_mb(total)}" + ) + print(f" regime @ {fmt_mb(capacity)} (this build): {choose_regime(weight, total, capacity)}") + + if args.total_layers: + n = args.total_layers + # Uniform-layer extrapolation for the REPORT only (the manifest keeps the + # real regions). Per-layer weight/kv = this build's weight/kv divided by + # the number of distinct layers actually present, so a multi-layer + # (--layers N) build extrapolates correctly. Shared globals (embeddings, + # rope tables) and per-layer activation intermediates are NOT modelled, so + # the activation term is a floor, not the full-model activation footprint. + layers_in_build = len({r["layer_id"] for r in regions if r["kind"] in (KIND_WEIGHT, KIND_KV)}) or 1 + per_layer_w = weight / layers_in_build + per_layer_kv = kv / layers_in_build + est_w = int(per_layer_w * n) + est_kv = int(per_layer_kv * n) + est_total = est_w + est_kv + act + print( + f" --- estimated full model ({n} layers; build holds {layers_in_build}; embeddings/lm_head + per-layer activations excluded) ---" + ) + print( + f" footprint (est): weight {fmt_mb(est_w)} | kv {fmt_mb(est_kv)} | activation>={fmt_mb(act)} | total>={fmt_mb(est_total)}" + ) + regime = choose_regime(est_w, est_total, capacity) + print(f" regime @ {fmt_mb(capacity)} (est full model): {regime}") + print( + f" (manifest contains the REAL build regions; to reproduce the full-model regime\n" + f" on this build, run the emulator with --memory-model {regime})" + ) + + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/transactional_emulator/testbench/run_model.py b/transactional_emulator/testbench/run_model.py index b8e4f9dd..9254aec8 100644 --- a/transactional_emulator/testbench/run_model.py +++ b/transactional_emulator/testbench/run_model.py @@ -65,7 +65,23 @@ def _build_dir(nickname: str, config_name: str, case: str) -> Path: return base -def _write_toml(preset: HardwarePreset, build_dir: Path) -> Path: +def _load_board_cfg(name: str | None) -> dict | None: + """Load a board YAML from testbench/board_configs/.yaml (or a path).""" + if not name: + return None + import yaml + + p = Path(name) + if not p.exists(): + p = Path(__file__).parent / "board_configs" / f"{name}.yaml" + if not p.exists(): + avail = sorted(q.stem for q in (Path(__file__).parent / "board_configs").glob("*.yaml")) + raise SystemExit(f"unknown board {name!r}; available: {avail}") + with p.open() as f: + return yaml.safe_load(f) + + +def _write_toml(preset: HardwarePreset, build_dir: Path, board_cfg: dict | None = None) -> Path: hw = HardwareConfig( mlen=preset.mlen, vlen=preset.vlen, @@ -77,9 +93,14 @@ def _write_toml(preset: HardwarePreset, build_dir: Path) -> Path: hbm_m_prefetch_amount=None, hbm_v_prefetch_amount=None, hbm_v_writeback_amount=None, + board_cfg=board_cfg, ) toml_path = hw.write_toml(build_dir) os.environ["PLENA_SETTINGS_TOML"] = str(toml_path) + if board_cfg: + print( + f"Applied board latency: {board_cfg.get('name', '?')} (DC_EN={1 if board_cfg.get('latency', {}).get('dc_lib_en') else 0})" + ) return toml_path @@ -97,7 +118,7 @@ def compile_sliced(mc: ModelConfig, preset: HardwarePreset, args) -> tuple[dict, batch_size = args.batch_size or preset.batch_size build_dir = _build_dir(mc.nickname, args.config or "default", args.case or "decoder-chain") - _write_toml(preset, build_dir) + _write_toml(preset, build_dir, _load_board_cfg(getattr(args, "board", None))) case = args.case or ("decoder-layer" if layers == 1 else "decoder-chain") common = dict( @@ -149,7 +170,7 @@ def compile_native(mc: ModelConfig, preset: HardwarePreset, args) -> tuple[dict, batch_size = args.batch_size or preset.batch_size build_dir = _build_dir(mc.nickname, args.config or "default", case) - _write_toml(preset, build_dir) + _write_toml(preset, build_dir, _load_board_cfg(getattr(args, "board", None))) print(f"Loading model {mc.model_id}...") model = AutoModel.from_pretrained( @@ -243,6 +264,12 @@ def main(): parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) parser.add_argument("nickname", help="Model nickname (e.g. smollm2, llada-8b, smolvlm2)") parser.add_argument("--config", default=None, help="Hardware preset name (e.g. sliced_64x64x16_b1)") + parser.add_argument( + "--board", + default=None, + help="Board config (board_configs/.yaml or a path) whose latency: block " + "is applied to the emulator (per-op cycles + DC_EN). E.g. nexys_video, custom_a7.", + ) parser.add_argument("--compile-only", action="store_true", help="Generate ASM only, skip emulator") parser.add_argument( "--case", @@ -296,6 +323,24 @@ def main(): print(f"\nASM written to: {asm_path}") print(f"ISA lines: {len(result['isa'].splitlines())}") + # Dump the HBM region map so the weight-manifest generator + # (testbench/board_configs/make_weight_manifest.py) can build a kind-tagged + # capacity-model manifest from a real compile. hbm_addrs (name->offset) is + # always present; hbm_sizes (name->bytes, incl. KV stores) is emitted by + # newer compilers and is the authoritative size source; tensor_layouts + # (input tensors only) is the fallback for sizing. + import json as _json + + for _key, _fname in ( + ("hbm_addrs", "hbm_addrs.json"), + ("hbm_sizes", "hbm_sizes.json"), + ("tensor_layouts", "tensor_layouts.json"), + ): + _data = result.get(_key) + if _data is not None: + (build_dir / _fname).write_text(_json.dumps(_data, indent=2, sort_keys=True)) + print(f"{_key} written to: {build_dir / _fname}") + if args.compile_only: print("\n[compile-only] Skipping emulator run.") return