From 0d5efd58665f9323f0ca466830fa713c9102bf82 Mon Sep 17 00:00:00 2001
From: Jared Lunde <jared.lunde@gmail.com>
Date: Sat, 27 Jun 2026 09:54:27 -0700
Subject: [PATCH] fix(boot-profile): close foyer clean cache before TempDir
 drop
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The boot-set profiler (`capture_once`) opens a per-run `FoyerBlockCache`
in a `tempfile::TempDir` and never closes it — it only Arc-drops at
function return. foyer's SSD-tier region files stay open until the
storage engine is closed, so the `TempDir`'s `remove_dir_all` runs while
the fds are still held, orphaning them as deleted-but-held files that are
never reclaimed until the daemon process exits.

A long-lived daemon that profiles many images (instd drives a profile run
per rootfs compose/create) leaks these unbounded into `$TMPDIR`. On hosts
where `/tmp` is a tmpfs this fills RAM and ENOSPCs every other process on
the node. Observed in the field: 5472 deleted 16 MiB regions (~85 GiB
logical) pinning a 14 GB tmpfs at 100%; a daemon restart reclaimed it all.

Add `BlockCache::close()` (default no-op; in-memory impls hold no fds),
implement it on `FoyerBlockCache` via foyer's `HybridCache::close()`, and
call it in `capture_once` before the `TempDir` is dropped, on every exit
path (the work is wrapped in a `result` async block so early `None`s still
close). The TempDir cleanup then actually frees the region files.

Note: this fixes the fd leak. Placing the profiler scratch on the
configured `[cache].dir` instead of `std::env::temp_dir()` is a separate
follow-up; operators can currently point `$TMPDIR` at a real-disk path.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
---
 glidefs/src/block/cache.rs             | 14 ++++++++++++++
 glidefs/src/oci/boot_capture_served.rs | 19 +++++++++++++++++++
 2 files changed, 33 insertions(+)

diff --git a/glidefs/src/block/cache.rs b/glidefs/src/block/cache.rs
index 66879b4..76b8dcc 100644
--- a/glidefs/src/block/cache.rs
+++ b/glidefs/src/block/cache.rs
@@ -29,6 +29,14 @@ pub trait BlockCache: Send + Sync {
     fn insert(&self, hash: Blake3Hash, data: Bytes);
     /// Remove a block from the cache.
     fn remove(&self, hash: &Blake3Hash);
+    /// Flush and shut down the cache, releasing any backing device fds.
+    ///
+    /// Must be awaited before a short-lived cache's directory is removed:
+    /// foyer's SSD-tier region files stay open until the storage engine is
+    /// closed, so dropping the dir first orphans them as deleted-but-held fds
+    /// that pin the filesystem for the whole process lifetime. Default no-op
+    /// for in-memory implementations that hold no fds.
+    async fn close(&self) {}
 }
 
 // ============================================================================
@@ -135,6 +143,12 @@ impl BlockCache for FoyerBlockCache {
     fn remove(&self, hash: &Blake3Hash) {
         self.inner.remove(hash);
     }
+
+    async fn close(&self) {
+        if let Err(e) = self.inner.close().await {
+            tracing::warn!(error = %e, "foyer clean cache close failed");
+        }
+    }
 }
 
 // ============================================================================
diff --git a/glidefs/src/oci/boot_capture_served.rs b/glidefs/src/oci/boot_capture_served.rs
index 6c9b1cc..e37801b 100644
--- a/glidefs/src/oci/boot_capture_served.rs
+++ b/glidefs/src/oci/boot_capture_served.rs
@@ -233,6 +233,15 @@ async fn capture_once(
         .await
         .ok()?,
     );
+    // Keep a handle to close the foyer clean cache before `tmp` (the TempDir)
+    // is dropped. foyer's SSD-tier region files stay open until the storage
+    // engine is closed; if the TempDir's `remove_dir_all` runs first, those
+    // files become deleted-but-held fds that never free until the daemon
+    // exits — so a daemon that profiles many images leaks them unbounded into
+    // $TMPDIR (which is tmpfs on many hosts → node-wide ENOSPC).
+    let clean_for_close = Arc::clone(&clean);
+
+    let result = async {
     let pack_index_cache = Arc::new(PackIndexCache::open(tmp.path()).await.ok()?);
     let handler = Arc::new(
         BlockHandler::new(
@@ -301,6 +310,16 @@ async fn capture_once(
         return None;
     }
     Some(blocks)
+    }
+    .await;
+
+    // Close the foyer clean cache (flush + release device fds) BEFORE `tmp` is
+    // dropped, so the TempDir cleanup actually frees the region files instead
+    // of orphaning them. Runs on every exit path (including the early `None`s
+    // above) because the work above is wrapped in the `result` async block.
+    clean_for_close.close().await;
+
+    result
 }
 
 #[cfg(test)]