Skip to content

Commit 8ac6756

Browse files
christsoclaude
christso
and
claude
authored
feat(workspace): per-repo materialization in static workspace mode (#962)
* feat(workspace): per-repo materialization in static workspace mode Static workspaces with YAML-configured paths now check each repo's target directory individually. Existing repos are reused as-is; only missing repos are cloned. This enables migrating from type: local to type: git sources for deps scanner discovery without losing local folder reuse. CLI-provided --workspace-path continues to skip all repo operations (user-managed directory). Closes #961 Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com> * docs(workspace): update static mode docs for per-repo materialization The static workspace section described populated directories as "reused as-is" with clone "bypassed entirely". Updated to reflect per-repo behavior: existing repos reused, missing repos cloned. Also clarified the CLI --workspace-path vs YAML workspace.path distinction. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>
1 parent 4837a4c commit 8ac6756

3 files changed

Lines changed: 146 additions & 12 deletions

File tree

‎apps/web/src/content/docs/docs/guides/workspace-pool.mdx‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -179,9 +179,11 @@ For workspaces you manage outside AgentV, use static mode:
179179
agentv eval evals/my-eval.yaml --workspace-mode static --workspace-path /path/to/my-workspace
180180
```
181181

182-
**Auto-materialisation:** When `workspace.path` points to an empty or missing directory, AgentV automatically copies the template and clones repos into it. If the directory already exists and is populated, it is reused as-is. This makes static mode convenient for first-run bootstrap without manual workspace preparation.
182+
**Auto-materialisation:** When `workspace.path` points to an empty or missing directory, AgentV automatically copies the template and clones repos into it. If the directory already exists and is populated, AgentV checks each repo individually — existing repos are reused as-is, and only missing repos are cloned. This makes static mode convenient for both first-run bootstrap and incremental setup.
183183

184-
When the directory is already populated, clone, copy, and pool are bypassed entirely. AgentV never deletes a user-provided workspace. Lifecycle hooks still execute (unless `hooks.enabled: false`). This is useful for local development where you already have the repo checked out.
184+
AgentV never deletes a user-provided workspace. Lifecycle hooks still execute (unless `hooks.enabled: false`). This is useful for local development where you already have repos checked out.
185+
186+
**Note:** When using `--workspace-path` (CLI flag) instead of `workspace.path` (YAML), the directory is always used as-is with no auto-materialisation or repo cloning.
185187

186188
**Precedence:** `workspace.mode` / `--workspace-mode` first, then default pooled behavior for shared repo workspaces.
187189

@@ -199,7 +201,7 @@ CLI flags `--retain-on-success` / `--retain-on-failure` control temporary eval-r
199201
|------|-----------|-----------|--------------------------|-------------------|
200202
| **Pooled** (default) | First run only; reset on reuse | Yes | Yes (`.gitignore`d files) | Yes (slot per worker) |
201203
| **Temp** (`mode: temp`) | Full clone + checkout every run | No | No | Sequential only |
202-
| **Static** (`mode: static`) | None if populated; auto-materialised if empty/missing | Yes | User-managed | Sequential only |
204+
| **Static** (`mode: static`) | Per-repo: clones only missing repos; auto-materialises if empty | Yes | User-managed | Sequential only |
203205

204206
## When to disable pooling
205207

‎packages/core/src/evaluation/orchestrator.ts‎

Lines changed: 33 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
import { createHash, randomUUID } from 'node:crypto';
2+
import { existsSync } from 'node:fs';
23
import { copyFile, mkdir, readdir, stat } from 'node:fs/promises';
34
import path from 'node:path';
45
import micromatch from 'micromatch';
@@ -629,12 +630,14 @@ export async function runEvaluation(
629630

630631
// Track whether a static workspace was freshly materialised (needs repo clone + hooks)
631632
let staticMaterialised = false;
633+
// YAML-configured static paths support auto-materialisation and per-repo checks.
634+
// CLI-provided paths (--workspace-path) always reuse the directory as-is.
635+
const isYamlConfiguredPath = !cliWorkspacePath && !!yamlWorkspacePath;
632636

633637
// Static workspace: auto-materialise if path is empty or missing, reuse if populated.
634638
// Auto-materialisation only applies to YAML-configured paths (workspace.path), not CLI flags
635639
// (--workspace / --workspace-path), which always reuse the directory as-is.
636640
if (useStaticWorkspace && configuredStaticPath) {
637-
const isYamlConfiguredPath = !cliWorkspacePath && !!yamlWorkspacePath;
638641
const dirExists = await stat(configuredStaticPath).then(
639642
(s) => s.isDirectory(),
640643
() => false,
@@ -714,16 +717,37 @@ export async function runEvaluation(
714717
}
715718
}
716719

717-
// Materialize repos into shared workspace (skip for per_test, pool, and existing static workspace)
720+
// Materialize repos into shared workspace (skip for per_test and pool modes).
721+
// For static workspaces: materialize only repos whose target path is missing (per-repo reuse).
722+
// For non-static workspaces: materialize all repos when freshly created.
723+
const hasReposToMaterialize =
724+
!!suiteWorkspace?.repos?.length && !usePool && !isPerTestIsolation;
718725
const needsRepoMaterialisation =
719-
!!suiteWorkspace?.repos?.length && !usePool && (!useStaticWorkspace || staticMaterialised);
720-
const repoManager = needsRepoMaterialisation ? new RepoManager(verbose) : undefined;
721-
if (repoManager && sharedWorkspacePath && suiteWorkspace?.repos && !isPerTestIsolation) {
722-
setupLog(
723-
`materializing ${suiteWorkspace.repos.length} shared repo(s) into ${sharedWorkspacePath}`,
724-
);
726+
hasReposToMaterialize && (!useStaticWorkspace || staticMaterialised);
727+
const needsPerRepoCheck =
728+
hasReposToMaterialize && useStaticWorkspace && !staticMaterialised && isYamlConfiguredPath;
729+
const repoManager =
730+
needsRepoMaterialisation || needsPerRepoCheck ? new RepoManager(verbose) : undefined;
731+
732+
if (repoManager && sharedWorkspacePath && suiteWorkspace?.repos) {
725733
try {
726-
await repoManager.materializeAll(suiteWorkspace.repos, sharedWorkspacePath);
734+
if (needsPerRepoCheck) {
735+
// Static workspace with existing content: materialize only missing repos
736+
for (const repo of suiteWorkspace.repos) {
737+
const targetDir = path.join(sharedWorkspacePath, repo.path);
738+
if (existsSync(targetDir)) {
739+
setupLog(`reusing existing repo at: ${targetDir}`);
740+
continue;
741+
}
742+
setupLog(`materializing missing repo: ${repo.path}`);
743+
await repoManager.materialize(repo, sharedWorkspacePath);
744+
}
745+
} else {
746+
setupLog(
747+
`materializing ${suiteWorkspace.repos.length} shared repo(s) into ${sharedWorkspacePath}`,
748+
);
749+
await repoManager.materializeAll(suiteWorkspace.repos, sharedWorkspacePath);
750+
}
727751
setupLog('shared repo materialization complete');
728752
} catch (error) {
729753
const message = error instanceof Error ? error.message : String(error);

‎packages/core/test/evaluation/orchestrator.test.ts‎

Lines changed: 108 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2755,6 +2755,114 @@ describe('--workspace flag', () => {
27552755
expect(results[0].error).toBeUndefined();
27562756
});
27572757

2758+
it('materializes only missing repos in YAML-configured static workspace', async () => {
2759+
const {
2760+
mkdtemp,
2761+
mkdir: fsMkdir,
2762+
writeFile,
2763+
access: fsAccess,
2764+
} = await import('node:fs/promises');
2765+
testDir = await mkdtemp(path.join(tmpdir(), 'agentv-ws-static-'));
2766+
2767+
// Pre-create repo-a to simulate an existing local checkout
2768+
const repoADir = path.join(testDir, 'repo-a');
2769+
await fsMkdir(repoADir, { recursive: true });
2770+
await writeFile(path.join(repoADir, 'marker.txt'), 'pre-existing');
2771+
2772+
const provider = new SequenceProvider('mock', {
2773+
responses: [{ output: [{ role: 'assistant', content: [{ type: 'text', text: 'answer' }] }] }],
2774+
});
2775+
2776+
// Use YAML workspace.path (not CLI --workspace) with type: git repos.
2777+
// repo-a exists → should be reused. repo-b is missing but uses a fake URL → should fail clone.
2778+
// Since repo-a is reused (skipped) and repo-b clone fails, this proves per-repo logic works.
2779+
const evalCase: EvalTest = {
2780+
...baseTestCase,
2781+
workspace: {
2782+
mode: 'static',
2783+
path: testDir,
2784+
repos: [
2785+
{
2786+
path: 'repo-a',
2787+
source: { type: 'git', url: 'https://github.com/example/repo-a.git' },
2788+
checkout: { ref: 'main' },
2789+
},
2790+
{
2791+
path: 'repo-b',
2792+
source: { type: 'git', url: 'https://github.com/example/repo-b.git' },
2793+
checkout: { ref: 'main' },
2794+
},
2795+
],
2796+
},
2797+
};
2798+
2799+
// repo-b clone will fail (fake URL), which proves repo-a was skipped (per-repo check)
2800+
// and only repo-b was attempted
2801+
await expect(
2802+
runEvaluation({
2803+
testFilePath: 'in-memory.yaml',
2804+
repoRoot: 'in-memory',
2805+
target: baseTarget,
2806+
providerFactory: () => provider,
2807+
evaluators: evaluatorRegistry,
2808+
evalCases: [evalCase],
2809+
keepWorkspaces: true,
2810+
}),
2811+
).rejects.toThrow('Failed to materialize repos');
2812+
2813+
// repo-a marker should still exist (not deleted by static workspace cleanup)
2814+
await fsAccess(path.join(repoADir, 'marker.txt'));
2815+
});
2816+
2817+
it('skips all repos when all exist in YAML-configured static workspace', async () => {
2818+
const { mkdtemp, mkdir: fsMkdir, writeFile } = await import('node:fs/promises');
2819+
testDir = await mkdtemp(path.join(tmpdir(), 'agentv-ws-static-'));
2820+
2821+
// Pre-create both repos
2822+
await fsMkdir(path.join(testDir, 'repo-a'), { recursive: true });
2823+
await writeFile(path.join(testDir, 'repo-a', 'file.txt'), 'a');
2824+
await fsMkdir(path.join(testDir, 'repo-b'), { recursive: true });
2825+
await writeFile(path.join(testDir, 'repo-b', 'file.txt'), 'b');
2826+
2827+
const provider = new SequenceProvider('mock', {
2828+
responses: [{ output: [{ role: 'assistant', content: [{ type: 'text', text: 'answer' }] }] }],
2829+
});
2830+
2831+
// Both repos exist → no clone attempts → should succeed without network
2832+
const evalCase: EvalTest = {
2833+
...baseTestCase,
2834+
workspace: {
2835+
mode: 'static',
2836+
path: testDir,
2837+
repos: [
2838+
{
2839+
path: 'repo-a',
2840+
source: { type: 'git', url: 'https://github.com/example/repo-a.git' },
2841+
checkout: { ref: 'main' },
2842+
},
2843+
{
2844+
path: 'repo-b',
2845+
source: { type: 'git', url: 'https://github.com/example/repo-b.git' },
2846+
checkout: { ref: 'main' },
2847+
},
2848+
],
2849+
},
2850+
};
2851+
2852+
const results = await runEvaluation({
2853+
testFilePath: 'in-memory.yaml',
2854+
repoRoot: 'in-memory',
2855+
target: baseTarget,
2856+
providerFactory: () => provider,
2857+
evaluators: evaluatorRegistry,
2858+
evalCases: [evalCase],
2859+
keepWorkspaces: true,
2860+
});
2861+
2862+
expect(results).toHaveLength(1);
2863+
expect(results[0].error).toBeUndefined();
2864+
});
2865+
27582866
it('errors when workspaceMode is static without workspace path', async () => {
27592867
const provider = new SequenceProvider('mock', {
27602868
responses: [{ output: [{ role: 'assistant', content: [{ type: 'text', text: 'answer' }] }] }],

0 commit comments

Comments
 (0)