feat: pin downloaded binaries with sha256 verification

CONTRIBUTING.md

@@ -35,19 +35,32 @@ cargo release -p exec-harness --execute beta
 
 After releasing `memtrack` or `exec-harness`, you **must** update the version references in the runner code:
 
-1. **For memtrack**: Update `MEMTRACK_CODSPEED_VERSION` in `src/executor/memory/executor.rs`:
+1. **For memtrack**: Update the `MEMTRACK_INSTALLER` pin record in `src/binary_pins.rs` (see [Pinned binary hashes](#pinned-binary-hashes) below).
 
-   ```rust
-   const MEMTRACK_CODSPEED_VERSION: &str = "X.Y.Z"; // Update to new version
-   ```
-
-2. **For exec-harness**: Update `EXEC_HARNESS_VERSION` in `src/executor/orchestrator.rs`:
-   ```rust
-   const EXEC_HARNESS_VERSION: &str = "X.Y.Z"; // Update to new version
-   ```
+2. **For exec-harness**: Update the `EXEC_HARNESS_INSTALLER` pin record in `src/binary_pins.rs`.
 
 These constants are used by the runner to download and install the correct versions of the binaries from GitHub releases.
 
+### Pinned binary hashes
+
+Every binary the runner downloads at install time (the patched valgrind `.deb`, the memtrack installer, the exec-harness installer, the mongo-tracer installer) is SHA-256-pinned. Each artifact keeps its version, URL template, and hash together in `src/binary_pins.rs`.
+
+When you bump a pinned version, regenerate the hash for each affected URL and update the matching pin record:
+
+```bash
+curl -sL '<url>' | sha256sum
+```
+
+For valgrind, that is one hash per supported `(distro_version, arch)` combination. `src/binary_pins.rs` also holds `VALGRIND_CODSPEED_VERSION` (the upstream semver, used to detect an already-installed copy) and `VALGRIND_DEB_REV` (the `.deb` revision suffix); the `.deb` package version is `{VALGRIND_CODSPEED_VERSION}-{VALGRIND_DEB_REV}`. Bump `VALGRIND_CODSPEED_VERSION` for a new upstream release, and `VALGRIND_DEB_REV` when the same upstream is repackaged.
+
+After updating, run the network-bound verification test that downloads every pinned URL and checks the bytes against the declared hash:
+
+```bash
+GITHUB_ACTIONS=true cargo test --lib binary_pins::tests::all_pinned_binaries_match_their_declared_sha256
+```
+
+This is also run in CI, but running it locally before opening the PR avoids a release-time round trip if a hash is wrong.
+
 ### Releasing the Main Runner
 
 The main runner (`codspeed-runner`) should be released after ensuring all dependency versions are correct.
@@ -56,8 +69,10 @@ The main runner (`codspeed-runner`) should be released after ensuring all depend
 
 **Verify binary version references**: Check that version constants in the runner code match the released versions:
 
-- `MEMTRACK_CODSPEED_VERSION` in `src/executor/memory/executor.rs`
-- `EXEC_HARNESS_VERSION` in `src/executor/orchestrator.rs`
+- `MEMTRACK_VERSION` in `src/binary_pins.rs`
+- `EXEC_HARNESS_VERSION` in `src/binary_pins.rs`
+
+Also confirm the SHA-256 entries in the pin records in `src/binary_pins.rs` match the released artifacts.
 
 #### Release Command
 

src/binary_installer/mod.rs

@@ -1,29 +1,26 @@
-use crate::cli::run::helpers::download_file;
+use crate::binary_pins::PinnedBinary;
+use crate::cli::run::helpers::download_pinned_file;
 use crate::prelude::*;
 use semver::Version;
 use std::process::Command;
 use tempfile::NamedTempFile;
-use url::Url;
 
 mod versions;
 
-/// Ensure a binary is installed, or install it from a runner's GitHub release using the installer script.
+/// Ensure a binary is installed, or install it from a `PinnedBinary` installer script.
 ///
 /// This function checks if the binary is already installed with the correct version.
-/// If not, it downloads and executes an installer script from the CodSpeed runner repository.
+/// If not, it downloads and executes the pinned installer script.
 ///
 /// # Arguments
 /// * `binary_name` - The binary command name (e.g., "codspeed-memtrack", "codspeed-exec-harness")
 /// * `version` - The version to install (e.g., "4.4.2-alpha.2")
-/// * `get_installer_url` - A closure that returns the URL to download the installer script.
-pub async fn ensure_binary_installed<F>(
+/// * `installer` - The `PinnedBinary` installer to download.
+pub async fn ensure_binary_installed(
     binary_name: &str,
     version: &str,
-    get_installer_url: F,
-) -> Result<()>
-where
-    F: FnOnce() -> String,
-{
+    installer: PinnedBinary,
+) -> Result<()> {
     if is_command_installed(
         binary_name,
         Version::parse(version).context("Invalid version format")?,
@@ -32,13 +29,11 @@ where
         return Ok(());
     }
 
-    let installer_url = Url::parse(&get_installer_url()).context("Invalid installer URL")?;
+    debug!("Downloading installer for {binary_name}");
 
-    debug!("Downloading installer from: {installer_url}");
-
-    // Download the installer script to a temporary file
+    // Download the installer script to a temporary file (with sha256 verification)
     let temp_file = NamedTempFile::new().context("Failed to create temporary file")?;
-    download_file(&installer_url, temp_file.path()).await?;
+    download_pinned_file(installer, temp_file.path()).await?;
 
     // Execute the installer script
     let output = Command::new("sh")

src/binary_pins.rs

@@ -0,0 +1,236 @@
+// Pinned downloads. Each artifact record keeps the version, URL template, and
+// expected SHA-256 together so bumps happen in one place. See CONTRIBUTING.md
+// for the regeneration workflow.
+
+use semver::Version;
+use std::sync::LazyLock;
+
+/// Upstream valgrind-codspeed version. Single source of truth for the .deb
+/// download (combined with `VALGRIND_DEB_REV`) and for detecting an already
+/// installed copy.
+pub const VALGRIND_CODSPEED_VERSION: Version = Version::new(3, 26, 0);
+/// Suffix appended to `VALGRIND_CODSPEED_VERSION` to form the .deb package version.
+/// Bumps when the .deb is repackaged without a new upstream valgrind release.
+const VALGRIND_DEB_REV: &str = "0codspeed2";
+/// String form of `VALGRIND_CODSPEED_VERSION` as it appears in `valgrind --version`
+/// output, used to identify a CodSpeed build at runtime.
+pub static VALGRIND_CODSPEED_VERSION_STRING: LazyLock<String> =
+    LazyLock::new(|| format!("{VALGRIND_CODSPEED_VERSION}.codspeed"));
+
+#[derive(Debug, Clone, Copy)]
+struct BinaryPin {
+    version: &'static str,
+    url_template: &'static str,
+    sha256: &'static str,
+}
+
+impl BinaryPin {
+    fn url(&self) -> String {
+        self.url_template.replace("{version}", self.version)
+    }
+}
+
+/// Ubuntu release for which CodSpeed publishes a patched valgrind .deb.
+/// Variants double as the value used in the download URL and as the
+/// installation key, so any `ValgrindTarget` constructed in the runner
+/// resolves to a real pin without a runtime fallback.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum DistroVersion {
+    Ubuntu2204,
+    Ubuntu2404,
+}
+
+impl DistroVersion {
+    fn as_str(self) -> &'static str {
+        match self {
+            DistroVersion::Ubuntu2204 => "22.04",
+            DistroVersion::Ubuntu2404 => "24.04",
+        }
+    }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum Arch {
+    Amd64,
+    Arm64,
+}
+
+impl Arch {
+    fn as_str(self) -> &'static str {
+        match self {
+            Arch::Amd64 => "amd64",
+            Arch::Arm64 => "arm64",
+        }
+    }
+}
+
+/// A `(DistroVersion, Arch)` pair for which the runner ships a pinned
+/// valgrind .deb. Both `url()` and `sha256()` are exhaustive matches over
+/// the type, so any value constructible here resolves to a real pin.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub struct ValgrindTarget {
+    pub distro_version: DistroVersion,
+    pub arch: Arch,
+}
+
+static VALGRIND_DEB_VERSION: LazyLock<String> =
+    LazyLock::new(|| format!("{VALGRIND_CODSPEED_VERSION}-{VALGRIND_DEB_REV}"));
+const VALGRIND_DEB_URL_TEMPLATE: &str = "https://github.com/CodSpeedHQ/valgrind-codspeed/releases/download/{version}/valgrind_{version}_ubuntu-{distro_version}_{arch}.deb";
+
+impl ValgrindTarget {
+    fn url(self) -> String {
+        VALGRIND_DEB_URL_TEMPLATE
+            .replace("{version}", &VALGRIND_DEB_VERSION)
+            .replace("{distro_version}", self.distro_version.as_str())
+            .replace("{arch}", self.arch.as_str())
+    }
+
+    fn sha256(self) -> &'static str {
+        match (self.distro_version, self.arch) {
+            (DistroVersion::Ubuntu2204, Arch::Amd64) => {
+                "1c677ae440cc77fac6bedded02b2af38c3515c76ffb72ed1c3258f8b839de560"
+            }
+            (DistroVersion::Ubuntu2404, Arch::Amd64) => {
+                "2e21cf5b1dea52bc7e23156fab6adc8893fac5ff101f33a4b68c0ecdb1716f3f"
+            }
+            (DistroVersion::Ubuntu2204, Arch::Arm64) => {
+                "597391c2f61d238454c84c61f929711fe54bf5eae43e01dc27021cf532a8b653"
+            }
+            (DistroVersion::Ubuntu2404, Arch::Arm64) => {
+                "f2fd8440f991014eef7fbed288eae210b4e1e299e11d75c278b55ead1defa493"
+            }
+        }
+    }
+}
+
+const MEMTRACK_INSTALLER: BinaryPin = BinaryPin {
+    version: "1.2.3",
+    url_template: "https://github.com/CodSpeedHQ/codspeed/releases/download/memtrack-v{version}/memtrack-installer.sh",
+    sha256: "67f30ebe17d5da4246b51d8663394026385d95203ff09e81289772159e969603",
+};
+pub const MEMTRACK_VERSION: &str = MEMTRACK_INSTALLER.version;
+
+const EXEC_HARNESS_INSTALLER: BinaryPin = BinaryPin {
+    version: "1.3.0",
+    url_template: "https://github.com/CodSpeedHQ/codspeed/releases/download/exec-harness-v{version}/exec-harness-installer.sh",
+    sha256: "75cbff4fdaefe98927d24fff43fd600c621eb1263b0c40b0fd32c68fa6d88ebd",
+};
+pub const EXEC_HARNESS_VERSION: &str = EXEC_HARNESS_INSTALLER.version;
+
+const MONGO_TRACER_INSTALLER: BinaryPin = BinaryPin {
+    version: "cs-mongo-tracer-v0.2.0",
+    url_template: "https://codspeed-public-assets.s3.eu-west-1.amazonaws.com/mongo-tracer/{version}/cs-mongo-tracer-installer.sh",
+    sha256: "685f1d540cb24c2aa6f447991958339c6b70ec7664df2dba2713b8b3d77687e7",
+};
+
+/// A binary the runner downloads at install time. The download helper looks
+/// up the URL and SHA-256 via `url()` and `sha256()` and rejects the install
+/// if the bytes don't match.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum PinnedBinary {
+    ValgrindDeb(ValgrindTarget),
+    MemtrackInstaller,
+    ExecHarnessInstaller,
+    MongoTracerInstaller,
+}
+
+impl PinnedBinary {
+    pub fn url(&self) -> String {
+        match self {
+            PinnedBinary::ValgrindDeb(target) => target.url(),
+            PinnedBinary::MemtrackInstaller => MEMTRACK_INSTALLER.url(),
+            PinnedBinary::ExecHarnessInstaller => EXEC_HARNESS_INSTALLER.url(),
+            PinnedBinary::MongoTracerInstaller => MONGO_TRACER_INSTALLER.url(),
+        }
+    }
+
+    pub fn sha256(&self) -> &'static str {
+        match self {
+            PinnedBinary::ValgrindDeb(target) => target.sha256(),
+            PinnedBinary::MemtrackInstaller => MEMTRACK_INSTALLER.sha256,
+            PinnedBinary::ExecHarnessInstaller => EXEC_HARNESS_INSTALLER.sha256,
+            PinnedBinary::MongoTracerInstaller => MONGO_TRACER_INSTALLER.sha256,
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::cli::run::helpers::download_pinned_file;
+    use tempfile::NamedTempFile;
+
+    const INSTALLER_BINARIES: &[PinnedBinary] = &[
+        PinnedBinary::MemtrackInstaller,
+        PinnedBinary::ExecHarnessInstaller,
+        PinnedBinary::MongoTracerInstaller,
+    ];
+
+    const ALL_VALGRIND_TARGETS: &[ValgrindTarget] = &[
+        ValgrindTarget {
+            distro_version: DistroVersion::Ubuntu2204,
+            arch: Arch::Amd64,
+        },
+        ValgrindTarget {
+            distro_version: DistroVersion::Ubuntu2404,
+            arch: Arch::Amd64,
+        },
+        ValgrindTarget {
+            distro_version: DistroVersion::Ubuntu2204,
+            arch: Arch::Arm64,
+        },
+        ValgrindTarget {
+            distro_version: DistroVersion::Ubuntu2404,
+            arch: Arch::Arm64,
+        },
+    ];
+
+    fn assert_installer_variant_is_listed(binary: PinnedBinary) {
+        match binary {
+            PinnedBinary::ValgrindDeb(_) => {}
+            PinnedBinary::MemtrackInstaller
+            | PinnedBinary::ExecHarnessInstaller
+            | PinnedBinary::MongoTracerInstaller => {
+                assert!(INSTALLER_BINARIES.contains(&binary));
+            }
+        }
+    }
+
+    fn all_pinned_binaries() -> impl Iterator<Item = PinnedBinary> {
+        ALL_VALGRIND_TARGETS
+            .iter()
+            .copied()
+            .map(PinnedBinary::ValgrindDeb)
+            .chain(INSTALLER_BINARIES.iter().copied())
+    }
+
+    #[test]
+    fn installer_variant_list_is_exhaustive() {
+        assert_installer_variant_is_listed(PinnedBinary::MemtrackInstaller);
+        assert_installer_variant_is_listed(PinnedBinary::ExecHarnessInstaller);
+        assert_installer_variant_is_listed(PinnedBinary::MongoTracerInstaller);
+    }
+
+    // Network-bound: downloads every pinned URL and asserts its bytes hash to
+    // the declared SHA-256. Skipped locally; CI sets `GITHUB_ACTIONS=true`.
+    // Run after bumping a version to make sure the release won't ship a stale
+    // or mistyped hash.
+    #[test_with::env(GITHUB_ACTIONS)]
+    #[tokio::test(flavor = "multi_thread")]
+    async fn all_pinned_binaries_match_their_declared_sha256() {
+        let results = futures::future::join_all(all_pinned_binaries().map(|binary| async move {
+            let temp = NamedTempFile::new().expect("failed to create temp file");
+            download_pinned_file(binary, temp.path())
+                .await
+                .map_err(|e| format!("{binary:?} ({}): {e}", binary.url()))
+        }))
+        .await;
+
+        let failures: Vec<_> = results.into_iter().filter_map(Result::err).collect();
+        assert!(
+            failures.is_empty(),
+            "pinned binaries failed verification:\n  - {}",
+            failures.join("\n  - "),
+        );
+    }
+}

src/cli/run/helpers/download_file.rs

@@ -1,9 +1,10 @@
+use crate::binary_pins::PinnedBinary;
 use crate::{prelude::*, request_client::REQUEST_CLIENT};
 use std::path::Path;
 
 use url::Url;
 
-pub async fn download_file(url: &Url, path: &Path) -> Result<()> {
+async fn download_file(url: &Url, path: &Path) -> Result<()> {
     debug!("Downloading file: {url}");
     let response = REQUEST_CLIENT
         .get(url.clone())
@@ -23,3 +24,26 @@ pub async fn download_file(url: &Url, path: &Path) -> Result<()> {
         .map_err(|e| anyhow!("Failed to write to file: {}, {}", path.display(), e))?;
     Ok(())
 }
+
+/// Download a `PinnedBinary` and verify its bytes against its pinned
+/// SHA-256. On mismatch the partial file is
+/// removed and an error is returned.
+pub async fn download_pinned_file(binary: PinnedBinary, path: &Path) -> Result<()> {
+    let url_str = binary.url();
+    let url = Url::parse(&url_str).context("failed to parse pinned URL")?;
+    download_file(&url, path).await?;
+
+    let actual = sha256::try_digest(path)
+        .with_context(|| format!("failed to compute sha256 of {}", path.display()))?;
+    let expected = binary.sha256();
+
+    if actual != expected {
+        let _ = std::fs::remove_file(path);
+        bail!(
+            "Hash mismatch for {url_str}: expected {expected}, got {actual}. The downloaded file has been deleted."
+        );
+    }
+
+    debug!("Verified sha256 of {url_str}");
+    Ok(())
+}

src/cli/run/helpers/mod.rs

@@ -5,7 +5,7 @@ mod format_memory;
 mod get_env_var;
 mod parse_git_remote;
 
-pub(crate) use download_file::download_file;
+pub(crate) use download_file::download_pinned_file;
 pub(crate) use find_repository_root::find_repository_root;
 pub(crate) use format_duration::format_duration;
 pub(crate) use format_memory::format_memory;