From dffdd31dfc7937faddf36c0d0c497ef416bda49b Mon Sep 17 00:00:00 2001 From: Jared Lunde Date: Sat, 27 Jun 2026 11:13:15 -0700 Subject: [PATCH] fix(boot-profile): place profiler scratch under [cache].dir, not /tmp MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Follow-up to #85. The boot-set profiler's per-run scratch (write cache + foyer clean cache) was created via `tempfile::TempDir::new()`, i.e. in `std::env::temp_dir()` (`/tmp`). On hosts where `/tmp` is a tmpfs that runs the profiler's disk cache in RAM, and it makes correct placement depend on `$TMPDIR` being exported in the unit — the same operational fragility that caused the original incident. Make placement correct by construction: add `BootProfileOptions.scratch_dir` and thread the daemon's configured `[cache].dir` through `profile_base` into `capture_once`, which now creates its `TempDir` there. `None` preserves the old `std::env::temp_dir()` behavior for short-lived CLI callers (`glidefs profile` / `bless` / the bench bin), whose scratch is reclaimed on process exit anyway. Only the long-lived daemon (`router::start_profile`) sets it, to its `cache_dir`. This complements the #85 fd-close fix (which stops the leak); together the profiler scratch is both bounded and on disk, independent of `$TMPDIR`. Co-Authored-By: Claude Opus 4.8 (1M context) --- glidefs/src/bin/boot_profile.rs | 1 + glidefs/src/block/router.rs | 11 ++++++++++- glidefs/src/cli/bless.rs | 1 + glidefs/src/cli/profile.rs | 2 +- glidefs/src/oci/boot_capture_served.rs | 21 ++++++++++++++++++++- glidefs/src/oci/profile_runner.rs | 5 +++++ 6 files changed, 38 insertions(+), 3 deletions(-) diff --git a/glidefs/src/bin/boot_profile.rs b/glidefs/src/bin/boot_profile.rs index 16e3787..b603b4e 100644 --- a/glidefs/src/bin/boot_profile.rs +++ b/glidefs/src/bin/boot_profile.rs @@ -136,6 +136,7 @@ async fn main() -> anyhow::Result<()> { timeout, static_seed: Vec::new(), max_blocks, + scratch_dir: None, }; let boot_set = capture_boot_blocks_served( Arc::clone(&content_store), diff --git a/glidefs/src/block/router.rs b/glidefs/src/block/router.rs index 47c87b7..3bc7135 100644 --- a/glidefs/src/block/router.rs +++ b/glidefs/src/block/router.rs @@ -2241,6 +2241,9 @@ impl ExportRouter { let router = Arc::clone(self); let content_store = self.content_store_for_prefix(s3_prefix); let name_owned = name.to_string(); + // Profiler scratch lands under the daemon's on-disk cache dir, never a + // tmpfs /tmp (foyer clean-cache regions would otherwise run in RAM). + let scratch_dir = self.cache_dir.clone(); let cleanup_key = key; let _handle = spawn_supervised("profile-bg", async move { // Guard removes the profile_tasks entry on any exit (return, @@ -2275,7 +2278,13 @@ impl ExportRouter { force: params.force, max_blocks: params.max_blocks, }; - match crate::oci::profile_runner::profile_base(content_store, &profile_cfg, spec).await + match crate::oci::profile_runner::profile_base( + content_store, + &profile_cfg, + spec, + Some(scratch_dir), + ) + .await { Ok(crate::oci::profile_runner::ProfileOutcome::UpToDate { .. }) => { info!(name = %name_owned, "profile: boot set already up to date"); diff --git a/glidefs/src/cli/bless.rs b/glidefs/src/cli/bless.rs index 4d2f1e2..ffd6f89 100644 --- a/glidefs/src/cli/bless.rs +++ b/glidefs/src/cli/bless.rs @@ -70,6 +70,7 @@ fn profiling_opts(settings: &Settings, static_seed: Vec) -> Result Result<()> { max_blocks: args.max_blocks, }; - match profile_base(content_store, &pcfg, spec).await? { + match profile_base(content_store, &pcfg, spec, None).await? { ProfileOutcome::UpToDate { fingerprint } => { println!( "Boot set for '{}' is up to date (fingerprint {}); skipping. Use --force to re-profile.", diff --git a/glidefs/src/oci/boot_capture_served.rs b/glidefs/src/oci/boot_capture_served.rs index e37801b..07afd29 100644 --- a/glidefs/src/oci/boot_capture_served.rs +++ b/glidefs/src/oci/boot_capture_served.rs @@ -47,6 +47,16 @@ pub struct BootProfileOptions { pub static_seed: Vec, /// Cap on captured blocks per run. pub max_blocks: usize, + /// Parent directory for the per-run scratch (write cache + foyer clean + /// cache). `None` falls back to `std::env::temp_dir()`. + /// + /// The long-lived daemon MUST set this to a real on-disk path (e.g. the + /// configured `[cache].dir`): `std::env::temp_dir()` is `/tmp`, which is a + /// tmpfs on many hosts, so the profiler's disk cache would otherwise run in + /// RAM. Setting it here makes placement correct by construction, regardless + /// of whether `$TMPDIR` is exported in the unit. Short-lived CLI callers can + /// leave it `None` — their scratch is reclaimed when the process exits. + pub scratch_dir: Option, } /// Rank-merge ordered block lists from multiple boot runs into one ordered union. @@ -196,7 +206,16 @@ async fn capture_once( use crate::oci::sandbox::SandboxSpec; use tokio::sync::Notify; - let tmp = tempfile::TempDir::new().ok()?; + // Scratch (write cache + foyer clean cache) goes under `opts.scratch_dir` + // when set — keeping the profiler's disk cache off a tmpfs `/tmp`. Falls + // back to `std::env::temp_dir()` only for callers that opt out (None). + let tmp = match opts.scratch_dir.as_deref() { + Some(dir) => { + std::fs::create_dir_all(dir).ok()?; + tempfile::TempDir::new_in(dir).ok()? + } + None => tempfile::TempDir::new().ok()?, + }; let rtrace = tmp.path().join("boot.rtrace"); let tracer = Arc::new( WriteTracer::new( diff --git a/glidefs/src/oci/profile_runner.rs b/glidefs/src/oci/profile_runner.rs index ddd94ec..6b90c3e 100644 --- a/glidefs/src/oci/profile_runner.rs +++ b/glidefs/src/oci/profile_runner.rs @@ -78,6 +78,10 @@ pub async fn profile_base( content_store: Arc, profile_cfg: &ProfileConfig, spec: ProfileSpec, + // Parent dir for per-run profiler scratch. The daemon passes its + // configured `[cache].dir` so the scratch lands on disk, not a tmpfs + // `/tmp`; CLI callers pass `None` (process-lifetime scratch). + scratch_dir: Option, ) -> Result { // --- Load the base manifest + its fingerprint (ETag) --- let (data, etag) = content_store @@ -182,6 +186,7 @@ pub async fn profile_base( timeout: spec.timeout.max(Duration::from_secs(1)), static_seed, max_blocks: spec.max_blocks, + scratch_dir, }; let blocks = capture_boot_blocks_served( Arc::clone(&content_store),