From db236abaaa0a7cdd0cfb5e664643bf8f115998e2 Mon Sep 17 00:00:00 2001 From: Sidharth Baskaran <62273881+sidnb13@users.noreply.github.com> Date: Sat, 22 Nov 2025 19:01:31 -0600 Subject: [PATCH] feat: add multi-container site-packages support with isolated package directories --- CLAUDE.md | 49 ++++++++++ src/mlt/mlt/config.py | 74 +++++++++++++++ src/mlt/mlt/lsp_proxy.py | 89 ++++++++++++------- src/mltoolbox/templates/docker-compose.yml.j2 | 4 +- 4 files changed, 180 insertions(+), 36 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 09843e8..ccdc017 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -334,6 +334,55 @@ pip install mlt-toolbox pip install mlt-toolbox==0.1.0 ``` +#### Multi-Container Support + +When working with multiple containers on the same remote host (different projects or branches), mltoolbox uses project-specific site-packages caching to avoid conflicts: + +**How It Works:** + +1. **Isolated Package Directories**: Each container mounts its site-packages to a unique host directory: + ```yaml + volumes: + - ~/.cache/python-packages/${CONTAINER_NAME}:/usr/local/lib/python3.11/dist-packages + ``` + +2. **No Conflicts**: Different containers can have: + - Different Python versions + - Different package versions + - Completely isolated dependencies + +3. **Path Translation**: `mlt` translates both: + - **Project paths**: `/home/ubuntu/projects/X` ↔ `/workspace/X` + - **Library paths**: `~/.cache/python-packages/container-name` ↔ `/usr/local/lib/pythonX.Y/dist-packages` + +**Example Setup:** + +```bash +# Container 1: causalab-main with Python 3.11 +~/.cache/python-packages/causalab-main/ → /usr/local/lib/python3.11/dist-packages + +# Container 2: causalab-feature with Python 3.12 +~/.cache/python-packages/causalab-feature/ → /usr/local/lib/python3.12/dist-packages + +# No conflicts! Each has isolated packages +``` + +**Container Detection:** + +When multiple containers exist, `mlt` automatically detects the correct one based on: +1. Current project directory name +2. CONTAINER_NAME in `.env` file +3. Docker ps filtering by project name + +Each Zed instance runs in its project directory, so `mlt` always connects to the right container. + +**Benefits:** + +- Work on multiple branches simultaneously without package conflicts +- Different Python versions per project +- Clean separation of dependencies +- Zed LSP works correctly for all containers independently + ### Branch Strategy - **Main branch**: Stable releases diff --git a/src/mlt/mlt/config.py b/src/mlt/mlt/config.py index 26985ea..1bd3b87 100644 --- a/src/mlt/mlt/config.py +++ b/src/mlt/mlt/config.py @@ -121,3 +121,77 @@ def get_path_mapping(project_dir: str = ".") -> tuple[str, str] | None: return (host_path, container_path) return None + + +def get_all_path_mappings( + project_dir: str = ".", +) -> tuple[tuple[str, str] | None, tuple[str, str] | None]: + """ + Get both project and library path mappings from docker-compose.yml. + + Args: + project_dir: Project directory containing docker-compose.yml + + Returns: + Tuple of (project_mapping, library_mapping) where each is (host_path, container_path) or None + project_mapping: The main project volume mount (e.g., .:/workspace/project) + library_mapping: The site-packages mount (e.g., ~/.cache/python-packages/X:/usr/local/lib/pythonX.Y/dist-packages) + """ + project_path = Path(project_dir).resolve() + compose_file = project_path / "docker-compose.yml" + + if not compose_file.exists(): + return (None, None) + + # Load environment variables + env_vars = load_env_file(project_dir) + + try: + with open(compose_file) as f: + compose_content = f.read() + # Substitute environment variables + compose_content = substitute_env_vars(compose_content, env_vars) + compose_data = yaml.safe_load(compose_content) + + if not compose_data or "services" not in compose_data: + return (None, None) + + project_mapping = None + library_mapping = None + + for service_name, service_config in compose_data.get("services", {}).items(): + if "volumes" not in service_config: + continue + + for volume in service_config["volumes"]: + if isinstance(volume, str): + # Format: "./path:/container/path" or "~/path:/container/path" + parts = volume.split(":") + if len(parts) >= 2: + host_path = parts[0] + container_path = parts[1] + + # Expand relative paths + if host_path.startswith("."): + host_path = str(project_path / host_path) + elif host_path.startswith("~"): + host_path = os.path.expanduser(host_path) + + # Resolve to absolute path + host_path = str(Path(host_path).resolve()) + + # Identify project mount (contains project_path) + if str(project_path) in host_path: + project_mapping = (host_path, container_path) + + # Identify library mount (contains dist-packages or site-packages) + if ( + "dist-packages" in container_path + or "site-packages" in container_path + ): + library_mapping = (host_path, container_path) + + return (project_mapping, library_mapping) + + except Exception: + return (None, None) diff --git a/src/mlt/mlt/lsp_proxy.py b/src/mlt/mlt/lsp_proxy.py index 1f5436e..2d73cbb 100644 --- a/src/mlt/mlt/lsp_proxy.py +++ b/src/mlt/mlt/lsp_proxy.py @@ -5,7 +5,7 @@ import sys import threading -from mlt.config import get_path_mapping +from mlt.config import get_all_path_mappings def run_lsp_proxy( @@ -22,21 +22,36 @@ def run_lsp_proxy( Returns: Exit code from LSP server """ - # Get path mappings - path_mapping = get_path_mapping(project_dir) - if not path_mapping: + # Get path mappings (both project and library) + project_mapping, library_mapping = get_all_path_mappings(project_dir) + + if not project_mapping: print( - "[mlt] WARNING: Could not determine path mappings from docker-compose.yml", + "[mlt] WARNING: Could not determine project path mapping from docker-compose.yml", file=sys.stderr, ) print( "[mlt] LSP will work but paths may not be translated correctly", file=sys.stderr, ) - host_path, container_path = None, None + + if project_mapping: + proj_host, proj_container = project_mapping + print( + f"[mlt] Project mapping: {proj_host} <-> {proj_container}", + file=sys.stderr, + ) + else: + proj_host, proj_container = None, None + + if library_mapping: + lib_host, lib_container = library_mapping + print( + f"[mlt] Library mapping: {lib_host} <-> {lib_container}", + file=sys.stderr, + ) else: - host_path, container_path = path_mapping - print(f"[mlt] Path mapping: {host_path} <-> {container_path}", file=sys.stderr) + lib_host, lib_container = None, None # Start LSP server in container docker_cmd = ["docker", "exec", "-i", container_name] + lsp_command @@ -58,35 +73,41 @@ def run_lsp_proxy( return 1 def translate_host_to_container(text: str) -> str: - """Translate host paths to container paths.""" - if not host_path or not container_path: - return text - original = text - # Handle file:// URIs - text = text.replace(f"file://{host_path}", f"file://{container_path}") - # Handle plain paths - text = text.replace(host_path, container_path) - if text != original: - print( - f"[mlt] HOST->CONTAINER: {host_path} -> {container_path}", - file=sys.stderr, - ) + """Translate host paths to container paths (both project and library).""" + + # Translate project paths + if proj_host and proj_container: + # Handle file:// URIs + text = text.replace(f"file://{proj_host}", f"file://{proj_container}") + # Handle plain paths + text = text.replace(proj_host, proj_container) + + # Translate library paths + if lib_host and lib_container: + # Handle file:// URIs + text = text.replace(f"file://{lib_host}", f"file://{lib_container}") + # Handle plain paths + text = text.replace(lib_host, lib_container) + return text def translate_container_to_host(text: str) -> str: - """Translate container paths to host paths.""" - if not host_path or not container_path: - return text - original = text - # Handle file:// URIs - text = text.replace(f"file://{container_path}", f"file://{host_path}") - # Handle plain paths - text = text.replace(container_path, host_path) - if text != original: - print( - f"[mlt] CONTAINER->HOST: {container_path} -> {host_path}", - file=sys.stderr, - ) + """Translate container paths to host paths (both project and library).""" + + # Translate library paths first (more specific, e.g., /usr/local/lib/...) + if lib_container and lib_host: + # Handle file:// URIs + text = text.replace(f"file://{lib_container}", f"file://{lib_host}") + # Handle plain paths + text = text.replace(lib_container, lib_host) + + # Translate project paths + if proj_container and proj_host: + # Handle file:// URIs + text = text.replace(f"file://{proj_container}", f"file://{proj_host}") + # Handle plain paths + text = text.replace(proj_container, proj_host) + return text def forward_stdin(): diff --git a/src/mltoolbox/templates/docker-compose.yml.j2 b/src/mltoolbox/templates/docker-compose.yml.j2 index 30468bd..9b01bfb 100644 --- a/src/mltoolbox/templates/docker-compose.yml.j2 +++ b/src/mltoolbox/templates/docker-compose.yml.j2 @@ -22,8 +22,8 @@ services: - ~/.claude:/root/.claude - /var/run/docker.sock:/var/run/docker.sock - /tmp/ray_lockfiles:/root/ray_lockfiles - # Mount Python site-packages for LSP to read library code (read-write to allow package installs) - - /usr/local/lib/python{{ python_version_short }}/dist-packages:/usr/local/lib/python{{ python_version_short }}/dist-packages + # Mount Python site-packages to project-specific cache (isolates packages per container) + - ~/.cache/python-packages/${CONTAINER_NAME}:/usr/local/lib/python{{ python_version_short }}/dist-packages - type: bind source: ~/.ssh target: /root/.ssh