feat(swe-bench): isolate workspace per instance for parallel execution

Add instance_id to workspace path to prevent git conflicts between
parallel instances. Each instance now uses .workspace/{instance_id}/org/repo
instead of sharing .workspace/org/repo.

Performance: 10 instances in ~8 min (vs ~30 min sequential)
Results: 7/10 resolved (70%) with Claude Sonnet
Signed-off-by: Jerry Guan <jerryguan777@gmail.com>

Author: Jerryguan777

examples/evaluation_and_profiling/swe_bench/src/nat_swe_bench/configs/config_iterative.yml

@@ -14,32 +14,28 @@
 # limitations under the License.
 
 llms:
- nim_llm:
-   _type: nim
-   model_name: mistralai/mistral-nemotron
-   temperature: 0.6
-   max_tokens: 4096    
-
-llms:
+  nim_llm:
+    _type: nim
+    model_name: mistralai/mistral-nemotron
+    temperature: 0.0
+    max_tokens: 4096
   claude_sonnet_llm:
     _type: litellm
     model_name: anthropic/claude-sonnet-4-5-20250929
     temperature: 0.0
-    api_key: "${ANTHROPIC_API_KEY}"  # Set this environment variable before running
-
-# llms:
-#   openai_llm:
-#     _type: litellm
-#     model_name: openai/gpt-5.2
-#     temperature: 0.0
-#     api_key: "${OPENAI_API_KEY}"  # Set this environment variable before running
+    api_key: "${ANTHROPIC_API_KEY}"
+  openai_llm:
+    _type: litellm
+    model_name: openai/gpt-5.2
+    temperature: 0.0
+    api_key: "${OPENAI_API_KEY}"
 
 workflow:
   _type: swe_bench
   predictor:
     _type: iterative
-    llm_name: "claude_sonnet_llm"
-    step_limit: 250
+    llm_name: "claude_sonnet_llm"   # "nim_llm" or "claude_sonnet_llm" or "openai_llm"
+    step_limit: 100
     timeout: 60
 
 functions:
@@ -51,7 +47,7 @@ functions:
 eval:
   general:
     output_dir: .tmp/nat/examples/evaluation_and_profiling/swe_bench/iterative/
-    max_concurrency: 1
+    max_concurrency: 5
     dataset:
       _type: parquet
       file_path: hf://datasets/princeton-nlp/SWE-bench_Lite/data/test-00000-of-00001.parquet
@@ -63,15 +59,15 @@ eval:
           field:
             instance_id:
               - sympy__sympy-20590
-              # - sympy__sympy-21055
-              # - sympy__sympy-11400
-              # - astropy__astropy-12907
-              # - astropy__astropy-6938
-              # - django__django-15781
-              # - django__django-11001
-              # - mwaskom__seaborn-3010
-              # - pallets__flask-4045
-              # - psf__requests-1963
+              - sympy__sympy-21055
+              - sympy__sympy-11400
+              - astropy__astropy-12907
+              - django__django-15781
+              - astropy__astropy-6938
+              - django__django-11001
+              - mwaskom__seaborn-3010
+              - pallets__flask-4045
+              - psf__requests-1963
 
   evaluators:
     swe_bench:

examples/evaluation_and_profiling/swe_bench/src/nat_swe_bench/predictors/predict_iterative/predict_iterative.py

@@ -450,12 +450,13 @@ async def predict_fn(self, swebench_input: SWEBenchInput) -> str:
         org, repo = repo_name.split('__')
         repo_url = f"https://github.com/{org}/{repo}"
 
-        # Setup repo
+        # Setup repo with instance_id for workspace isolation
         try:
             repo_path_str = await self.git_tool.arun(json.dumps({
                 "operation": "setup",
                 "repo_url": repo_url,
-                "base_commit": swebench_input.base_commit
+                "base_commit": swebench_input.base_commit,
+                "instance_id": swebench_input.instance_id  # Isolate workspace per instance
             }))
             repo_path = Path(repo_path_str)
             logger.info("Repository setup at %s", repo_path)

examples/evaluation_and_profiling/swe_bench/src/nat_swe_bench/predictors/predict_iterative/tools/git_tool.py

@@ -27,13 +27,9 @@
 class RepoContext:
     """Context manager for repository operations."""
     repo_url: str
-    base_path: Path
+    repo_path: Path  # Actual path where the repo is cloned
     repo: Repo | None = None
 
-    def __post_init__(self):
-        self.repo_name = self.repo_url.split('/')[-1].replace('.git', '')
-        self.repo_path = self.base_path / self.repo_name
-
 
 class RepoManager:
 
@@ -42,9 +38,18 @@ def __init__(self, workspace_dir: str):
         self.workspace.mkdir(parents=True, exist_ok=True)
         self.active_repos = {}
 
-    async def setup_repository(self, repo_url: str, base_commit: str) -> RepoContext:
-        """Setup a repository at a specific commit."""
-        repo_path = get_repo_path(str(self.workspace), repo_url)
+    async def setup_repository(
+        self, repo_url: str, base_commit: str, instance_id: str | None = None
+    ) -> RepoContext:
+        """Setup a repository at a specific commit.
+
+        Args:
+            repo_url: URL of the repository to clone
+            base_commit: Commit hash to checkout
+            instance_id: Optional instance ID for workspace isolation. When provided,
+                         each instance gets its own clean workspace directory.
+        """
+        repo_path = get_repo_path(str(self.workspace), repo_url, instance_id)
 
         if str(repo_path) in self.active_repos:
             context = self.active_repos[str(repo_path)]
@@ -54,7 +59,7 @@ async def setup_repository(self, repo_url: str, base_commit: str) -> RepoContext
         repo = await clone_repository(repo_url, repo_path)
         await checkout_commit(repo, base_commit)
 
-        context = RepoContext(repo_url=repo_url, base_path=self.workspace, repo=repo)
+        context = RepoContext(repo_url=repo_url, repo_path=repo_path, repo=repo)
         self.active_repos[str(repo_path)] = context
         return context
 
@@ -68,13 +73,28 @@ async def cleanup(self):
         self.active_repos.clear()
 
 
-def get_repo_path(workspace_dir: str, repo_url: str) -> Path:
-    """Generate a unique path for the repository."""
+def get_repo_path(workspace_dir: str, repo_url: str, instance_id: str | None = None) -> Path:
+    """Generate a unique path for the repository.
+
+    Args:
+        workspace_dir: Base workspace directory
+        repo_url: URL of the repository
+        instance_id: Optional instance ID for unique workspace isolation
+
+    Returns:
+        Path to the repository. If instance_id is provided, returns
+        workspace_dir/instance_id/org/repo for complete isolation.
+        Otherwise returns workspace_dir/org/repo.
+    """
     parts = repo_url.rstrip('/').split('/')
     repo_name = parts[-1].replace('.git', '')
     org_name = parts[-2]  # Organization name
-    
-    # Return: workspace_dir/org/repo
+
+    # If instance_id is provided, create isolated workspace per instance
+    if instance_id:
+        return Path(workspace_dir) / instance_id / org_name / repo_name
+
+    # Default: workspace_dir/org/repo
     return Path(workspace_dir) / org_name / repo_name
 
 

examples/evaluation_and_profiling/swe_bench/src/nat_swe_bench/predictors/predict_iterative/tools/register.py

@@ -49,7 +49,11 @@ async def git_operations(args_str: str) -> str:
         if operation == "setup":
             if 'repo_url' not in args or 'base_commit' not in args:
                 raise ValueError("setup operation requires 'repo_url' and 'base_commit'")
-            context = await repo_manager.setup_repository(args['repo_url'], args['base_commit'])
+            # instance_id is optional - when provided, creates isolated workspace per instance
+            instance_id = args.get('instance_id')
+            context = await repo_manager.setup_repository(
+                args['repo_url'], args['base_commit'], instance_id
+            )
             return str(context.repo_path)
 
         if operation == "cleanup":