feat(bootstrap): add container memory limit with auto-detection

Add --memory flag to `openshell gateway start` that caps the gateway
container's memory via Docker HostConfig. When unset, auto-detects 80%
of available memory by querying the Docker daemon (docker info), which
correctly reports the Docker Desktop VM's allocated memory on macOS and
Windows rather than the full host RAM. Docker OOM-kills the container
instead of letting runaway sandbox growth trigger the host kernel OOM
killer.

- parse_memory_limit(): human-readable sizes (80g, 4096m, bytes)
- detect_memory_limit(): async, queries Docker daemon MemTotal
- memory_swap = memory (disables swap inside container)
- OPENSHELL_MEMORY_LIMIT env var supported

Signed-off-by: Brian Taylor <brian.taylor818@gmail.com>

Author: brianwtaylor

crates/openshell-bootstrap/src/docker.rs

@@ -40,6 +40,76 @@ fn env_bool(key: &str) -> Option<bool> {
     })
 }
 
+/// Parse a human-readable memory size string into bytes.
+///
+/// Accepts integers (bytes) or values with `k`/`m`/`g`/`t` suffixes
+/// (case-insensitive, with or without a trailing `b`). Binary units
+/// (`ki`/`mi`/`gi`/`ti`) are also accepted. Examples: `80g`, `4096m`,
+/// `0.5g`, `1073741824`.
+///
+/// Returns an error if the value is empty, uses an unknown suffix, overflows
+/// `i64`, or is below the 4 MiB minimum required by Docker.
+pub fn parse_memory_limit(s: &str) -> Result<i64> {
+    let s = s.trim().to_ascii_lowercase();
+    if s.is_empty() {
+        miette::bail!("empty memory limit string");
+    }
+
+    // Split into numeric part and optional suffix.
+    let (num_str, suffix) = match s.find(|c: char| !c.is_ascii_digit() && c != '.') {
+        Some(idx) => (&s[..idx], s[idx..].trim_end_matches('b')),
+        None => (s.as_str(), ""),
+    };
+
+    let value: f64 = num_str
+        .parse()
+        .into_diagnostic()
+        .wrap_err_with(|| format!("invalid numeric part in memory limit: {num_str}"))?;
+
+    let multiplier: f64 = match suffix {
+        "" => 1.0,
+        "k" | "ki" => 1024.0,
+        "m" | "mi" => 1024.0 * 1024.0,
+        "g" | "gi" => 1024.0 * 1024.0 * 1024.0,
+        "t" | "ti" => 1024.0 * 1024.0 * 1024.0 * 1024.0,
+        other => miette::bail!("unknown memory suffix: {other}"),
+    };
+
+    let raw = value * multiplier;
+    if raw > i64::MAX as f64 {
+        miette::bail!("memory limit too large (exceeds i64::MAX): {s}");
+    }
+    #[allow(clippy::cast_possible_truncation)]
+    let bytes = raw as i64;
+
+    // Docker requires at least ~6 MiB; enforce a 4 MiB floor so users get a
+    // clear error instead of an opaque Docker API rejection.
+    const MIN_MEMORY_BYTES: i64 = 4 * 1024 * 1024;
+    if bytes < MIN_MEMORY_BYTES {
+        miette::bail!("memory limit must be at least 4 MiB, got: {s} ({bytes} bytes)");
+    }
+    Ok(bytes)
+}
+
+/// Detect a safe memory limit for the gateway container.
+///
+/// Queries the Docker daemon for `MemTotal` (via `docker info`) and returns
+/// 80% of that value. On macOS and Windows the daemon runs inside a Linux VM
+/// (Docker Desktop, colima, WSL2), so the reported total reflects the VM's
+/// allocated memory rather than the full host RAM.
+///
+/// Returns `None` if the daemon does not report memory information.
+pub async fn detect_memory_limit(docker: &Docker) -> Option<i64> {
+    let info = docker.info().await.ok()?;
+    let total_bytes = info.mem_total?;
+    if total_bytes <= 0 {
+        return None;
+    }
+    #[allow(clippy::cast_possible_truncation)]
+    let limit = (total_bytes as f64 * 0.8) as i64;
+    Some(limit)
+}
+
 /// Platform information for a Docker daemon host.
 #[derive(Debug, Clone)]
 pub struct HostPlatform {
@@ -512,6 +582,7 @@ pub async fn ensure_container(
     registry_token: Option<&str>,
     gpu: bool,
     is_remote: bool,
+    memory_limit: Option<i64>,
 ) -> Result<()> {
     let container_name = container_name(name);
 
@@ -616,6 +687,15 @@ pub async fn ensure_container(
         }]);
     }
 
+    // Apply memory limit. When set, Docker OOM-kills the container instead of
+    // letting unchecked sandbox growth trigger the host kernel OOM killer.
+    // Setting memory_swap equal to memory disables swap inside the container.
+    if let Some(mem) = memory_limit {
+        host_config.memory = Some(mem);
+        host_config.memory_swap = Some(mem);
+        tracing::info!("Container memory limit: {} MiB", mem / (1024 * 1024),);
+    }
+
     let mut cmd = vec![
         "server".to_string(),
         "--disable=traefik".to_string(),
@@ -1352,4 +1432,77 @@ mod tests {
         let input = "nameserver 8.8.8.8\r\nnameserver 1.1.1.1\r\n";
         assert_eq!(parse_resolv_conf(input), vec!["8.8.8.8", "1.1.1.1"]);
     }
+
+    #[test]
+    fn parse_memory_limit_gigabytes() {
+        assert_eq!(parse_memory_limit("80g").unwrap(), 80 * 1024 * 1024 * 1024);
+        assert_eq!(parse_memory_limit("80G").unwrap(), 80 * 1024 * 1024 * 1024);
+        assert_eq!(parse_memory_limit("80gb").unwrap(), 80 * 1024 * 1024 * 1024);
+    }
+
+    #[test]
+    fn parse_memory_limit_megabytes() {
+        assert_eq!(parse_memory_limit("4096m").unwrap(), 4096 * 1024 * 1024);
+        assert_eq!(parse_memory_limit("4096M").unwrap(), 4096 * 1024 * 1024);
+    }
+
+    #[test]
+    fn parse_memory_limit_bare_bytes() {
+        assert_eq!(parse_memory_limit("1073741824").unwrap(), 1073741824);
+    }
+
+    #[test]
+    fn parse_memory_limit_binary_suffixes() {
+        assert_eq!(parse_memory_limit("1gi").unwrap(), 1024 * 1024 * 1024);
+        assert_eq!(parse_memory_limit("1gib").unwrap(), 1024 * 1024 * 1024);
+    }
+
+    #[test]
+    fn parse_memory_limit_rejects_empty() {
+        assert!(parse_memory_limit("").is_err());
+    }
+
+    #[test]
+    fn parse_memory_limit_rejects_unknown_suffix() {
+        assert!(parse_memory_limit("10x").is_err());
+    }
+
+    #[test]
+    fn parse_memory_limit_fractional() {
+        // 0.5g = 512 MiB
+        assert_eq!(parse_memory_limit("0.5g").unwrap(), 512 * 1024 * 1024);
+    }
+
+    #[test]
+    fn parse_memory_limit_rejects_zero() {
+        assert!(parse_memory_limit("0g").is_err());
+    }
+
+    #[test]
+    fn parse_memory_limit_rejects_negative() {
+        assert!(parse_memory_limit("-1g").is_err());
+    }
+
+    #[test]
+    fn parse_memory_limit_rejects_below_minimum() {
+        // 1 KiB is well below the 4 MiB floor
+        assert!(parse_memory_limit("1k").is_err());
+    }
+
+    #[test]
+    fn parse_memory_limit_rejects_overflow() {
+        // 99999999t exceeds i64::MAX (~9.2 exabytes)
+        assert!(parse_memory_limit("99999999t").is_err());
+    }
+
+    #[test]
+    fn parse_memory_limit_whitespace() {
+        assert_eq!(
+            parse_memory_limit("  80g  ").unwrap(),
+            80 * 1024 * 1024 * 1024
+        );
+    }
+
+    // detect_memory_limit is async and requires a Docker daemon connection,
+    // so it is tested via integration / e2e tests rather than unit tests.
 }

crates/openshell-bootstrap/src/lib.rs

@@ -46,6 +46,7 @@ use crate::runtime::{
 pub use crate::constants::container_name;
 pub use crate::docker::{
     DockerPreflight, ExistingGatewayInfo, check_docker_available, create_ssh_docker_client,
+    detect_memory_limit, parse_memory_limit,
 };
 pub use crate::metadata::{
     GatewayMetadata, clear_active_gateway, extract_host_from_ssh_destination, get_gateway_metadata,
@@ -119,6 +120,11 @@ pub struct DeployOptions {
     /// When false, an existing gateway is left as-is and deployment is
     /// skipped (the caller is responsible for prompting the user first).
     pub recreate: bool,
+    /// Memory limit for the gateway container in bytes. When set, Docker
+    /// enforces the ceiling and OOM-kills the container instead of the host
+    /// kernel OOM-killing unrelated processes. When `None`, auto-detected
+    /// as 80% of available memory via the Docker daemon.
+    pub memory_limit: Option<i64>,
 }
 
 impl DeployOptions {
@@ -135,6 +141,7 @@ impl DeployOptions {
             registry_token: None,
             gpu: false,
             recreate: false,
+            memory_limit: None,
         }
     }
 
@@ -200,6 +207,13 @@ impl DeployOptions {
         self.recreate = recreate;
         self
     }
+
+    /// Set the memory limit for the gateway container in bytes.
+    #[must_use]
+    pub fn with_memory_limit(mut self, limit: i64) -> Self {
+        self.memory_limit = Some(limit);
+        self
+    }
 }
 
 #[derive(Debug, Clone)]
@@ -264,6 +278,7 @@ where
     let registry_token = options.registry_token;
     let gpu = options.gpu;
     let recreate = options.recreate;
+    let explicit_memory_limit = options.memory_limit;
 
     // Wrap on_log in Arc<Mutex<>> so we can share it with pull_remote_image
     // which needs a 'static callback for the bollard streaming pull.
@@ -288,6 +303,14 @@ where
         (preflight.docker, None)
     };
 
+    // Resolve memory limit: explicit value from CLI, or auto-detect from the
+    // Docker daemon. On macOS / Windows this correctly reports the Docker
+    // Desktop VM's memory, not the full host RAM.
+    let memory_limit = match explicit_memory_limit {
+        Some(limit) => Some(limit),
+        None => detect_memory_limit(&target_docker).await,
+    };
+
     // If an existing gateway is found, either tear it down (when recreate is
     // requested) or bail out so the caller can prompt the user / reuse it.
     if let Some(existing) = check_existing_gateway(&target_docker, &name).await? {
@@ -418,6 +441,7 @@ where
             registry_token.as_deref(),
             gpu,
             remote_opts.is_some(),
+            memory_limit,
         )
         .await?;
         start_container(&target_docker, &name).await?;

crates/openshell-cli/src/bootstrap.rs

@@ -179,6 +179,15 @@ pub async fn run_bootstrap(
         options = options.with_gateway_host(host);
     }
     options = options.with_gpu(gpu);
+    // Read memory limit override from environment. The explicit `--memory`
+    // flag is only on `gateway start`; this env var covers the auto-bootstrap
+    // path triggered by `sandbox create`.
+    if let Ok(mem_str) = std::env::var("OPENSHELL_MEMORY_LIMIT")
+        && !mem_str.trim().is_empty()
+    {
+        let limit = openshell_bootstrap::parse_memory_limit(&mem_str)?;
+        options = options.with_memory_limit(limit);
+    }
 
     let handle = deploy_gateway_with_panel(options, &gateway_name, location).await?;
     let server = handle.gateway_endpoint().to_string();

crates/openshell-cli/src/main.rs

@@ -809,6 +809,17 @@ enum GatewayCommands {
         /// NVIDIA Container Toolkit on the host.
         #[arg(long)]
         gpu: bool,
+
+        /// Memory limit for the gateway container.
+        ///
+        /// Accepts human-readable sizes: `80g`, `4096m`, `1073741824` (bytes).
+        /// When unset, defaults to 80% of available memory (auto-detected via
+        /// the Docker daemon). On macOS and Windows this reflects the Docker
+        /// Desktop VM's allocated memory, not the full host RAM. Docker
+        /// OOM-kills the container if it exceeds this limit, preventing
+        /// runaway sandbox growth from triggering the host kernel OOM killer.
+        #[arg(long, env = "OPENSHELL_MEMORY_LIMIT")]
+        memory: Option<String>,
     },
 
     /// Stop the gateway (preserves state).
@@ -1561,7 +1572,12 @@ async fn main() -> Result<()> {
                 registry_username,
                 registry_token,
                 gpu,
+                memory,
             } => {
+                let memory_limit = memory
+                    .as_deref()
+                    .map(openshell_bootstrap::parse_memory_limit)
+                    .transpose()?;
                 run::gateway_admin_deploy(
                     &name,
                     remote.as_deref(),
@@ -1574,6 +1590,7 @@ async fn main() -> Result<()> {
                     registry_username.as_deref(),
                     registry_token.as_deref(),
                     gpu,
+                    memory_limit,
                 )
                 .await?;
             }

crates/openshell-cli/src/run.rs

@@ -1356,6 +1356,7 @@ pub async fn gateway_admin_deploy(
     registry_username: Option<&str>,
     registry_token: Option<&str>,
     gpu: bool,
+    memory_limit: Option<i64>,
 ) -> Result<()> {
     let location = if remote.is_some() { "remote" } else { "local" };
 
@@ -1421,6 +1422,9 @@ pub async fn gateway_admin_deploy(
         .with_disable_gateway_auth(disable_gateway_auth)
         .with_gpu(gpu)
         .with_recreate(should_recreate);
+    if let Some(mem) = memory_limit {
+        options = options.with_memory_limit(mem);
+    }
     if let Some(opts) = remote_opts {
         options = options.with_remote(opts);
     }