Track memory usage over time

ci-bench-runner/migrations/1_add_memory.sql

@@ -0,0 +1,2 @@
+ALTER TABLE comparison_runs
+    ADD COLUMN memory_scenarios_missing_in_baseline TEXT;

ci-bench-runner/src/bencher_dev.rs

@@ -10,6 +10,7 @@ use bencher_client::{
 };
 use tracing::error;
 
+use crate::job::MemoryDetails;
 use crate::BencherConfig;
 
 /// The Bencher.dev client along with its configuration
@@ -33,6 +34,7 @@ impl BencherDev {
     }
 
     /// Sends the icount and walltime results to bencher.dev for visualization
+    #[expect(clippy::too_many_arguments)]
     pub async fn track_results(
         &self,
         branch: &str,
@@ -41,9 +43,14 @@ impl BencherDev {
         end_time: DateTime,
         icounts: HashMap<String, f64>,
         walltimes: HashMap<String, f64>,
+        memory: HashMap<String, MemoryDetails>,
     ) -> anyhow::Result<()> {
         let mut bmf_map = results_to_bmf(icounts, INSTRUCTIONS_SLUG_STR.parse().unwrap());
-        bmf_map.extend(results_to_bmf(walltimes, LATENCY_SLUG_STR.parse().unwrap()));
+        merge(
+            &mut bmf_map,
+            results_to_bmf(walltimes, LATENCY_SLUG_STR.parse().unwrap()),
+        );
+        memory_results_to_bmf(&mut bmf_map, memory);
 
         let testbed = self.config.testbed_id.clone();
         let report = JsonNewReport {
@@ -78,6 +85,49 @@ impl BencherDev {
     }
 }
 
+fn memory_results_to_bmf(out: &mut JsonResultsMap, results: HashMap<String, MemoryDetails>) {
+    merge(
+        out,
+        results_to_bmf(
+            results
+                .iter()
+                .map(|(k, v)| (k.clone(), v.heap_peak_blocks as f64))
+                .collect(),
+            "heap-peak-blocks".parse().unwrap(),
+        ),
+    );
+    merge(
+        out,
+        results_to_bmf(
+            results
+                .iter()
+                .map(|(k, v)| (k.clone(), v.heap_peak_bytes as f64))
+                .collect(),
+            "heap-peak-bytes".parse().unwrap(),
+        ),
+    );
+    merge(
+        out,
+        results_to_bmf(
+            results
+                .iter()
+                .map(|(k, v)| (k.clone(), v.heap_total_blocks as f64))
+                .collect(),
+            "heap-total-blocks".parse().unwrap(),
+        ),
+    );
+    merge(
+        out,
+        results_to_bmf(
+            results
+                .iter()
+                .map(|(k, v)| (k.clone(), v.heap_total_bytes as f64))
+                .collect(),
+            "heap-total-bytes".parse().unwrap(),
+        ),
+    );
+}
+
 /// Converts the instruction counts map into Bencher Metric Format (BMF)
 ///
 /// See <https://bencher.dev/docs/explanation/adapters#-json> for details
@@ -106,3 +156,19 @@ fn results_to_bmf(results: HashMap<String, f64>, measure: Measure) -> JsonResult
         })
         .collect()
 }
+
+/// Merges `src` into `target.
+///
+/// With both `src` and `target` in the form:
+///
+/// ```json
+/// { "benchmark-name": { "measure": { ... }}}
+/// ```
+///
+/// Ensures multiple measures appear in the end result.  Does
+/// not require that "benchmark-name" exists in `target`.
+fn merge(target: &mut JsonResultsMap, src: JsonResultsMap) {
+    for (bench, contents) in src.into_iter() {
+        target.entry(bench).or_default().extend(contents);
+    }
+}

ci-bench-runner/src/db.rs

@@ -1,3 +1,4 @@
+use std::collections::HashMap;
 use std::ops::DerefMut;
 use std::sync::Arc;
 
@@ -10,6 +11,8 @@ use time::OffsetDateTime;
 use tokio::sync::Mutex;
 use uuid::Uuid;
 
+use crate::job::MemoryDetails;
+
 /// An enqueued GitHub event
 #[derive(Debug)]
 pub struct QueuedEvent {
@@ -84,6 +87,8 @@ pub struct ComparisonResult {
     pub icount: ComparisonSubResult,
     /// Result for the walltime benchmarks
     pub walltime: ComparisonSubResult,
+    /// Result for the memory benchmarks
+    pub memory: ComparisonSubResult,
 }
 
 #[derive(Debug, Clone)]
@@ -92,6 +97,8 @@ pub struct ComparisonSubResult {
     pub diffs: Vec<ScenarioDiff>,
     /// Benchmark scenarios present in the candidate but missing in the baseline
     pub scenarios_missing_in_baseline: Vec<String>,
+    /// Detailed results for memory jobs (name -> (baseline, candidate))
+    pub(crate) memory_details: Option<HashMap<String, (MemoryDetails, MemoryDetails)>>,
 }
 
 /// A diff for a particular scenario, obtained by comparing benchmark results between two versions
@@ -123,12 +130,46 @@ impl ScenarioDiff {
     pub fn diff_ratio(&self) -> f64 {
         self.diff() / self.baseline_result
     }
+
+    pub(crate) fn baseline_memory(
+        &self,
+        details: &Option<HashMap<String, (MemoryDetails, MemoryDetails)>>,
+    ) -> String {
+        let Some(detail) = details.as_ref().and_then(|m| m.get(&self.scenario_name)) else {
+            return String::new();
+        };
+
+        detail.0.to_string()
+    }
+
+    pub(crate) fn candidate_memory(
+        &self,
+        details: &Option<HashMap<String, (MemoryDetails, MemoryDetails)>>,
+    ) -> String {
+        let Some(detail) = details.as_ref().and_then(|m| m.get(&self.scenario_name)) else {
+            return String::new();
+        };
+
+        detail.1.to_string()
+    }
+
+    pub(crate) fn memory_diff(
+        &self,
+        details: &Option<HashMap<String, (MemoryDetails, MemoryDetails)>>,
+    ) -> String {
+        let Some(detail) = details.as_ref().and_then(|m| m.get(&self.scenario_name)) else {
+            return String::new();
+        };
+
+        detail.0.diff_string(detail.1)
+    }
 }
 
 #[derive(Copy, Clone, Debug, PartialEq)]
 pub enum ScenarioKind {
     Icount = 0,
     Walltime = 1,
+    Memory = 2,
 }
 
 impl TryFrom<i64> for ScenarioKind {
@@ -138,6 +179,7 @@ impl TryFrom<i64> for ScenarioKind {
         match value {
             0 => Ok(Self::Icount),
             1 => Ok(Self::Walltime),
+            2 => Ok(Self::Memory),
             kind => bail!("invalid scenario kind: {kind}"),
         }
     }
@@ -382,6 +424,7 @@ impl Db {
         let icount_scenarios_missing = to_json_array(&result.icount.scenarios_missing_in_baseline);
         let walltime_scenarios_missing =
             to_json_array(&result.walltime.scenarios_missing_in_baseline);
+        let memory_scenarios_missing = to_json_array(&result.memory.scenarios_missing_in_baseline);
 
         let mut conn = self.sqlite.lock().await;
         let id = conn.transaction(|t| {
@@ -390,19 +433,20 @@ impl Db {
                 let id = Uuid::new_v4();
                 let now = OffsetDateTime::now_utc();
                 sqlx::query(
-                    "INSERT INTO comparison_runs (id, created_utc, baseline_commit, candidate_commit, icount_scenarios_missing_in_baseline, walltime_scenarios_missing_in_baseline) VALUES (?, ?, ?, ?, ?, ?)",
+                    "INSERT INTO comparison_runs (id, created_utc, baseline_commit, candidate_commit, icount_scenarios_missing_in_baseline, walltime_scenarios_missing_in_baseline, memory_scenarios_missing_in_baseline) VALUES (?, ?, ?, ?, ?, ?, ?)",
                 )
                     .bind(id.as_bytes().as_slice())
                     .bind(now)
                     .bind(baseline_commit)
                     .bind(candidate_commit)
                     .bind(icount_scenarios_missing)
                     .bind(walltime_scenarios_missing)
+                    .bind(memory_scenarios_missing)
                     .execute(t.deref_mut())
                     .await?;
 
                 // Insert the associated diffs
-                for diff in result.icount.diffs.into_iter().chain(result.walltime.diffs) {
+                for diff in result.icount.diffs.into_iter().chain(result.walltime.diffs).chain(result.memory.diffs) {
                     sqlx::query(
                         "INSERT INTO scenario_diffs (comparison_run_id, scenario_name, scenario_kind, baseline_result, candidate_result, significance_threshold, cachegrind_diff) VALUES (?, ?, ?, ?, ?, ?, ?)",
                     )
@@ -442,7 +486,7 @@ impl Db {
         let mut conn = self.sqlite.lock().await;
         let row = sqlx::query(
             r"
-            SELECT id, created_utc, icount_scenarios_missing_in_baseline, walltime_scenarios_missing_in_baseline
+            SELECT id, created_utc, icount_scenarios_missing_in_baseline, walltime_scenarios_missing_in_baseline, memory_scenarios_missing_in_baseline
             FROM comparison_runs
             WHERE baseline_commit = ? AND candidate_commit = ?",
         )
@@ -460,6 +504,8 @@ impl Db {
             from_json_array(row.try_get("icount_scenarios_missing_in_baseline")?)?;
         let walltime_scenarios_missing_in_baseline =
             from_json_array(row.try_get("walltime_scenarios_missing_in_baseline")?)?;
+        let memory_scenarios_missing_in_baseline =
+            from_json_array(row.try_get("memory_scenarios_missing_in_baseline")?)?;
 
         let icount_diffs = sqlx::query_as(
             r"
@@ -478,19 +524,37 @@ impl Db {
             FROM scenario_diffs
             WHERE comparison_run_id = ? AND scenario_kind = ?",
         )
-        .bind(id)
+        .bind(&id)
         .bind(ScenarioKind::Walltime as i64)
         .fetch_all(conn.deref_mut())
         .await?;
 
+        let memory_diffs = sqlx::query_as(
+            r"
+            SELECT *
+            FROM scenario_diffs
+            WHERE comparison_run_id = ? AND scenario_kind = ?",
+        )
+        .bind(id)
+        .bind(ScenarioKind::Memory as i64)
+        .fetch_all(conn.deref_mut())
+        .await?;
+
         Ok(Some(ComparisonResult {
             icount: ComparisonSubResult {
                 scenarios_missing_in_baseline: icount_scenarios_missing_in_baseline,
                 diffs: icount_diffs,
+                memory_details: None,
             },
             walltime: ComparisonSubResult {
                 scenarios_missing_in_baseline: walltime_scenarios_missing_in_baseline,
                 diffs: walltime_diffs,
+                memory_details: None,
+            },
+            memory: ComparisonSubResult {
+                scenarios_missing_in_baseline: memory_scenarios_missing_in_baseline,
+                diffs: memory_diffs,
+                memory_details: None,
             },
         }))
     }
@@ -695,6 +759,7 @@ mod test {
         let candidate_commit = "7faf240afbdbb4e76c47ff5f3f049c7a78c9c843";
         let icount_diffs = make_diffs(ScenarioKind::Icount);
         let walltime_diffs = make_diffs(ScenarioKind::Walltime);
+        let memory_diffs = make_diffs(ScenarioKind::Memory);
 
         db.store_comparison_result(
             baseline_commit.to_string(),
@@ -703,10 +768,17 @@ mod test {
                 icount: ComparisonSubResult {
                     scenarios_missing_in_baseline: Vec::new(),
                     diffs: icount_diffs.clone(),
+                    memory_details: None,
                 },
                 walltime: ComparisonSubResult {
                     scenarios_missing_in_baseline: Vec::new(),
                     diffs: walltime_diffs.clone(),
+                    memory_details: None,
+                },
+                memory: ComparisonSubResult {
+                    scenarios_missing_in_baseline: Vec::new(),
+                    diffs: memory_diffs.clone(),
+                    memory_details: None,
                 },
             },
         )
@@ -771,10 +843,17 @@ mod test {
                 icount: ComparisonSubResult {
                     diffs: diffs.clone(),
                     scenarios_missing_in_baseline: vec!["bar".to_string()],
+                    memory_details: None,
                 },
                 walltime: ComparisonSubResult {
                     diffs: Vec::new(),
                     scenarios_missing_in_baseline: vec!["baz".to_string()],
+                    memory_details: None,
+                },
+                memory: ComparisonSubResult {
+                    diffs: Vec::new(),
+                    scenarios_missing_in_baseline: vec!["quux".to_string()],
+                    memory_details: None,
                 },
             },
         )

ci-bench-runner/src/job/bench_main.rs

@@ -5,7 +5,10 @@ use bencher_client::json::DateTime;
 use tempfile::TempDir;
 use tracing::{trace, warn};
 
-use super::{icounts_path, read_icount_results, read_walltime_results, walltimes_path};
+use super::{
+    icounts_path, maybe_read_memory_results, memory_path, read_icount_results,
+    read_walltime_results, walltimes_path,
+};
 use crate::db::ScenarioKind;
 use crate::event_queue::JobContext;
 use crate::github::api::PushEvent;
@@ -79,6 +82,8 @@ pub async fn bench_main(ctx: JobContext<'_>) -> anyhow::Result<()> {
         .context("failed to read instruction counts from file")?;
     let walltimes = read_walltime_results(&walltimes_path(&ctx.job_output_dir))
         .context("failed to read walltimes from file")?;
+    let memory = maybe_read_memory_results(&memory_path(&ctx.job_output_dir))
+        .context("failed to read memory results from file")?;
 
     // Persist results in the DB and in bencher.dev
     let results = icounts
@@ -89,6 +94,13 @@ pub async fn bench_main(ctx: JobContext<'_>) -> anyhow::Result<()> {
                 .iter()
                 .map(|(scenario, result)| (scenario.clone(), ScenarioKind::Walltime, *result)),
         )
+        .chain(memory.iter().map(|(scenario, result)| {
+            (
+                scenario.clone(),
+                ScenarioKind::Memory,
+                result.comparison_basis() as f64,
+            )
+        }))
         .collect();
     ctx.db
         .store_run_results(results)
@@ -104,6 +116,7 @@ pub async fn bench_main(ctx: JobContext<'_>) -> anyhow::Result<()> {
                 benchmark_run_end,
                 icounts,
                 walltimes,
+                memory,
             )
             .await
             .context("failed to send results to bencher.dev");