From 61832f95eb682d61deaa7aa7cabee582e7a454a3 Mon Sep 17 00:00:00 2001 From: Ryan Atkinson Date: Tue, 16 Dec 2025 10:44:56 -0500 Subject: [PATCH 1/7] wip --- CLAUDE.md | 14 ++ GITOPS_RUN_SUMMARY.md | 88 ++++++++++ README.md | 14 ++ TODO_GITOPS_RUN.md | 341 +++++++++++++++++++++++++++++++++++++ src/lib/gitops_run.task.ts | 219 ++++++++++++++++++++++++ src/lib/local_repo.ts | 46 ++++- src/routes/library.ts | 43 +++-- static/.nojekyll | 0 8 files changed, 752 insertions(+), 13 deletions(-) create mode 100644 GITOPS_RUN_SUMMARY.md create mode 100644 TODO_GITOPS_RUN.md create mode 100644 src/lib/gitops_run.task.ts create mode 100644 static/.nojekyll diff --git a/CLAUDE.md b/CLAUDE.md index 56c72ad7..a1cd2af3 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -314,6 +314,11 @@ gro gitops_sync # sync repos and update local data gro gitops_sync --download # clone missing repos gro gitops_sync --check # verify repos are ready without fetching data +# Run commands across repos +gro gitops_run "npm test" # run command in all repos (parallel, concurrency: 5) +gro gitops_run "npm audit" --concurrency 3 # limit parallelism +gro gitops_run "gro check" --format json # JSON output for scripting + # Publishing gro gitops_validate # validate configuration (runs analyze, plan, and dry run) gro gitops_analyze # analyze dependencies and changesets @@ -355,6 +360,15 @@ gro test src/test/fixtures/check # validate gitops commands against fixture - Switches branches and pulls latest changes - Installs dependencies if package.json changed - Verify repos ready without fetching (with `--check`) + - Runs in parallel (concurrency: 5 by default) + +**Command Execution (User-Defined Side Effects):** + +- `gro gitops_run ""` - Run shell command across all repos + - Parallel execution (concurrency: 5 by default) + - Continue-on-error behavior + - Structured output (text or JSON) + - Use for testing, auditing, batch operations **Publishing (Git & NPM Side Effects):** diff --git a/GITOPS_RUN_SUMMARY.md b/GITOPS_RUN_SUMMARY.md new file mode 100644 index 00000000..46f71804 --- /dev/null +++ b/GITOPS_RUN_SUMMARY.md @@ -0,0 +1,88 @@ +# gitops_run Implementation Summary + +## What was built + +A new `gro gitops_run` command that executes shell commands across all repos in parallel with configurable concurrency and comprehensive error handling. + +## Key features + +1. **Parallel execution** - Runs commands across repos concurrently (default: 5) +2. **Throttled concurrency** - Uses `map_concurrent_settled` from fuz_util +3. **Continue-on-error** - Shows all results, doesn't fail-fast +4. **Structured output** - Text (default) or JSON format +5. **Lightweight** - Uses `get_repo_paths()` instead of full repo loading + +## Usage + +```bash +# Basic usage +gro gitops_run --command "npm test" + +# Control concurrency +gro gitops_run --command "npm audit" --concurrency 3 + +# JSON output for scripting +gro gitops_run --command "git status" --format json + +# Use with test fixtures +gro gitops_run --path src/test/fixtures/configs/basic_publishing.config.ts --command "pwd" + +# Chain commands +gro gitops_run --command "gro upgrade @ryanatkn/gro@latest --no-pull && git add static/.nojekyll" +``` + +## Implementation details + +### Files created/modified + +- **NEW**: `src/lib/gitops_run.task.ts` - Main task implementation +- **NEW**: `TODO_GITOPS_RUN.md` - Future enhancement ideas +- **MODIFIED**: `src/lib/local_repo.ts` - Added parallel loading with `map_concurrent_settled` +- **MODIFIED**: `CLAUDE.md` - Added gitops_run documentation +- **MODIFIED**: `README.md` - Added usage examples + +### Design choices + +1. **Lightweight execution** - Uses `get_repo_paths()` instead of full `get_gitops_ready()` + - Doesn't require `library.ts` files + - No git sync/pull by default + - Faster startup + +2. **Shell mode** - Commands run via `sh -c` to support pipes, redirects, etc. + - Trade-off: Slightly less safe than argument arrays + - Benefit: Full shell capabilities + +3. **Concurrency** - Default 5 repos at a time + - Based on user preference + - Prevents overwhelming system resources + - Respects rate limits + +4. **Error handling** - Continue-on-error with detailed reporting + - Shows all successes and failures + - Includes exit codes and stderr + - Exits with error if any repo fails (for CI) + +### Future enhancements (see TODO_GITOPS_RUN.md) + +- Config-defined commands +- Conditional execution (--only-with-changesets, etc.) +- Lifecycle hooks +- Retry logic +- Dependency-aware execution order + +## Testing + +Tested with: +- Fixture repos (basic_publishing) ✓ +- Real repos (9 repos in config) ✓ +- Simple commands ✓ +- Complex chained commands ✓ +- JSON output ✓ +- Various concurrency levels ✓ + +## Performance + +Example with 9 repos @ concurrency=3: +- Simple echo command: ~41ms total +- Commands run in batches of 3 +- Minimal overhead per repo (~5-20ms) diff --git a/README.md b/README.md index 7544f4fa..c31eb070 100644 --- a/README.md +++ b/README.md @@ -71,6 +71,20 @@ See [CLAUDE.md](CLAUDE.md#architecture) for detailed documentation. ## Quick Start +### Running commands across repos + +```bash +gro gitops_run "npm test" # run tests in all repos (parallel, concurrency: 5) +gro gitops_run "npm audit" --concurrency 3 # limit parallelism +gro gitops_run "git status" --format json # JSON output for scripting +``` + +**Features:** +- Parallel execution with configurable concurrency (default: 5) +- Continue-on-error behavior (shows all results) +- Structured output formats (text or JSON) +- Uses lightweight repo path resolution (no full sync needed) + ### Syncing repo metadata ```bash diff --git a/TODO_GITOPS_RUN.md b/TODO_GITOPS_RUN.md new file mode 100644 index 00000000..cae7d279 --- /dev/null +++ b/TODO_GITOPS_RUN.md @@ -0,0 +1,341 @@ +# TODO: gitops_run future enhancements + +This doc tracks potential future enhancements to `gitops_run` beyond the initial implementation. + +## Initial Implementation (v1) ✅ + +Core functionality for immediate use: +- Single command string execution across repos +- Parallel execution with concurrency limit (default: 5) +- Continue-on-error behavior with summary +- Output capture and structured reporting +- Uses `map_concurrent_settled` from fuz_util + +**Usage:** +```bash +gro gitops_run "npm test" +gro gitops_run "npm test" --concurrency 3 +gro gitops_run "npm audit" --format json +``` + +## Shell Features & Command Parsing + +**Question: How much shell functionality should we support?** + +Options: +1. **Simple string execution** (current): Just pass string to shell as-is + - Pros: Simple, works for basic cases, supports pipes/redirects naturally + - Cons: Shell injection risk if we ever template commands + +2. **Command array** (like Node's spawn): `["npm", "test"]` + - Pros: No shell injection possible, explicit args + - Cons: No shell features (pipes, redirects, etc.) + +3. **Template interpolation**: `"npm test {{repo_name}}"` + - Pros: Flexible, can pass repo-specific data + - Cons: Adds complexity, shell injection risk + +**Recommendation**: Start with #1 (simple string), add #3 later if needed. + +## Multiple Commands (Chaining) + +Run multiple commands in sequence per repo: + +```bash +# Option A: Multiple positional args +gro gitops_run "npm test" "npm audit" "gro check" + +# Option B: Shell-style chaining (already works?) +gro gitops_run "npm test && npm audit && gro check" + +# Option C: Config file with command sequences +# gitops.config.ts +{ + repos: [...], + commands: { + 'full-check': ['npm test', 'npm audit', 'gro check'], + 'update': ['npm update', 'npm install'], + } +} +``` + +## Conditional Execution + +Run commands only on repos matching criteria: + +```bash +# Only repos with changesets +gro gitops_run "npm test" --only-with-changesets + +# Only repos matching pattern +gro gitops_run "npm test" --only "*_ui" + +# Only repos with specific files +gro gitops_run "npm test" --only-with-file "test/**" + +# Exclude repos +gro gitops_run "npm test" --exclude "fuz_template" +``` + +## Lifecycle Hooks in Config + +Add custom commands at specific lifecycle points: + +```ts +// gitops.config.ts +export default { + repos: [...], + hooks: { + before_sync: "npm ci", // Ensure clean deps before sync + after_clone: "npm install", // Auto-setup after download + before_publish: "npm audit", // Security check + after_publish: "./notify.sh", // Custom notifications + on_failure: "./alert.sh", // Alert on failures + } +} +``` + +## Output Formats & Aggregation + +Better output handling: + +```bash +# JSON output for scripting +gro gitops_run "npm test" --format json + +# Table view with status +gro gitops_run "npm test" --format table + +# Only show failures +gro gitops_run "npm test" --only-failures + +# Save output per repo +gro gitops_run "npm test" --save-output .gro/test_results/ +``` + +**Output format examples:** + +```json +// --format json +{ + "command": "npm test", + "concurrency": 5, + "repos": [ + { + "name": "fuz_ui", + "status": "success", + "duration_ms": 1234, + "stdout": "...", + "stderr": "" + }, + { + "name": "fuz_css", + "status": "failure", + "duration_ms": 567, + "exit_code": 1, + "stdout": "...", + "stderr": "..." + } + ], + "summary": { + "total": 10, + "success": 9, + "failure": 1, + "duration_ms": 5678 + } +} +``` + +## Repo-Specific Command Overrides + +Allow repos to customize commands: + +```ts +// gitops.config.ts +export default { + repos: [ + { + repo_url: 'https://github.com/fuzdev/fuz_ui', + commands: { + test: 'npm test -- --coverage', // Custom test command + lint: 'npm run lint:strict', + } + }, + 'https://github.com/fuzdev/fuz_css', // Uses default commands + ], +} +``` + +## Interactive Mode + +Choose commands interactively: + +```bash +gro gitops_run --interactive +# Prompts: +# > Select command: [test, lint, check, build, custom] +# > Select repos: [all, select, pattern] +# > Concurrency: [1, 3, 5, 10] +``` + +## Dry Run Mode + +Preview what would run: + +```bash +gro gitops_run "npm test" --dry-run +# Output: +# Would run "npm test" in 10 repos with concurrency 5: +# - fuz_ui (~/dev/fuz_ui) +# - fuz_css (~/dev/fuz_css) +# ... +``` + +## Progress Indicators + +Better UX for long-running operations: + +```bash +gro gitops_run "npm test" +# Output: +# Running "npm test" in 10 repos (concurrency: 5)... +# [████████░░] 8/10 complete (fuz_ui: running, fuz_css: success, ...) +``` + +## Workspace State Management + +Commands that need clean/dirty workspace checks: + +```bash +# Require clean workspace +gro gitops_run "npm test" --require-clean + +# Auto-stash before running +gro gitops_run "gro build" --auto-stash +``` + +## Remote Execution + +Run commands on remote CI or via SSH: + +```bash +# Via GitHub Actions +gro gitops_run "npm test" --remote github + +# Via SSH +gro gitops_run "npm test" --remote ssh://user@host +``` + +## Caching & Memoization + +Skip commands if nothing changed: + +```bash +# Only test repos with changes since last run +gro gitops_run "npm test" --cache --since-commit HEAD~5 + +# Only test repos with file changes +gro gitops_run "npm test" --cache --changed-files +``` + +## Error Recovery Strategies + +More sophisticated error handling: + +```bash +# Retry failures +gro gitops_run "npm install" --retry 3 + +# Retry with exponential backoff +gro gitops_run "npm install" --retry 3 --backoff exponential + +# Fail after N failures +gro gitops_run "npm test" --max-failures 3 +``` + +## Dependency-Aware Execution + +Run commands in dependency order: + +```bash +# Build in topological order +gro gitops_run "gro build" --topo + +# Test in parallel but respect dependencies +gro gitops_run "npm test" --topo-parallel +``` + +## Integration with Existing Commands + +Reuse gitops_run patterns in other commands: + +- Update `gitops_sync` to use `map_concurrent_settled` +- Add `--concurrency` flag to `gitops_publish` +- Add parallel preflight checks +- Parallel GitHub API fetching (with rate limit respect) + +## Environment Variable Templating + +Pass repo context as env vars: + +```bash +# Template vars: REPO_NAME, REPO_DIR, REPO_URL +gro gitops_run "echo Testing $REPO_NAME" +``` + +## Logging & Observability + +Structured logging for debugging: + +```bash +# Log timing per repo +gro gitops_run "npm test" --timing + +# Log resource usage +gro gitops_run "npm test" --resource-usage + +# Export to observability format (OpenTelemetry, etc.) +gro gitops_run "npm test" --trace opentelemetry +``` + +## Config-Defined Commands + +Pre-define common command sequences: + +```ts +// gitops.config.ts +export default { + repos: [...], + commands: { + ci: ['npm ci', 'npm test', 'npm run build'], + update: ['gro upgrade @ryanatkn/gro@latest --no-pull', 'npm install'], + check_all: ['gro check', 'npm audit', 'npm outdated'], + } +} +``` + +```bash +gro gitops_run ci +gro gitops_run update +``` + +## Priority & Scheduling + +Control execution order: + +```ts +// gitops.config.ts +{ + repos: [ + {repo_url: '...', priority: 1}, // Run first + {repo_url: '...', priority: 10}, // Run last + ] +} +``` + +## Notes + +- Keep the initial implementation simple and focused +- Add features based on actual usage patterns +- Avoid feature creep - not every shell tool needs to be reimplemented +- Consider which features are better solved by external tools (GNU parallel, etc.) diff --git a/src/lib/gitops_run.task.ts b/src/lib/gitops_run.task.ts new file mode 100644 index 00000000..1720b03b --- /dev/null +++ b/src/lib/gitops_run.task.ts @@ -0,0 +1,219 @@ +import {TaskError, type Task} from '@ryanatkn/gro'; +import {z} from 'zod'; +import {map_concurrent_settled} from '@fuzdev/fuz_util/async.js'; +import {spawn_out} from '@fuzdev/fuz_util/process.js'; +import {styleText as st} from 'node:util'; +import {resolve} from 'node:path'; + +import {get_repo_paths} from './repo_ops.js'; + +export const Args = z.strictObject({ + command: z.string().meta({description: 'shell command to run in each repo'}), + path: z + .string() + .meta({description: 'path to the gitops config file'}) + .default('gitops.config.ts'), + concurrency: z + .number() + .int() + .min(1) + .meta({description: 'maximum number of repos to run in parallel'}) + .default(5), + format: z + .enum(['text', 'json']) + .meta({description: 'output format'}) + .default('text'), +}); +export type Args = z.infer; + +interface RunResult { + repo_name: string; + repo_dir: string; + status: 'success' | 'failure'; + exit_code: number; + stdout: string; + stderr: string; + duration_ms: number; + error?: string; +} + +export const task: Task = { + Args, + summary: 'run a shell command across all repos in parallel', + run: async ({args, log}) => { + const {command, path, concurrency, format} = args; + + // Get repo paths (lightweight, no library.ts loading needed) + const config_path = resolve(path); + const repos = await get_repo_paths(config_path); + + if (repos.length === 0) { + throw new TaskError('No repos found in config'); + } + + log.info( + `Running ${st('cyan', command)} across ${repos.length} repos (concurrency: ${concurrency})`, + ); + + const start_time = performance.now(); + + // Run command in parallel across all repos + const results = await map_concurrent_settled( + repos, + async (repo) => { + const repo_start = performance.now(); + const repo_name = repo.name; + const repo_dir = repo.path; + + try { + // Parse command into cmd + args for spawn + // For now, we use shell mode to support pipes/redirects/etc + const spawned = await spawn_out('sh', ['-c', command], { + cwd: repo_dir, + }); + + const duration_ms = performance.now() - repo_start; + const success = spawned.result.ok; + + const result: RunResult = { + repo_name, + repo_dir, + status: success ? 'success' : 'failure', + exit_code: spawned.result.code ?? 0, + stdout: spawned.stdout || '', + stderr: spawned.stderr || '', + duration_ms, + }; + + return result; + } catch (error) { + const duration_ms = performance.now() - repo_start; + return { + repo_name, + repo_dir, + status: 'failure' as const, + exit_code: -1, + stdout: '', + stderr: '', + duration_ms, + error: String(error), + }; + } + }, + concurrency, + ); + + const total_duration_ms = performance.now() - start_time; + + // Process results + const successes: Array = []; + const failures: Array = []; + + for (const result of results) { + if (result.status === 'fulfilled') { + const run_result = result.value; + if (run_result.status === 'success') { + successes.push(run_result); + } else { + failures.push(run_result); + } + } else { + // This shouldn't happen since we catch errors in the task fn + // but handle it anyway + failures.push({ + repo_name: 'unknown', + repo_dir: 'unknown', + status: 'failure', + exit_code: -1, + stdout: '', + stderr: '', + duration_ms: 0, + error: String(result.reason), + }); + } + } + + // Output results based on format + if (format === 'json') { + const json_output = { + command, + concurrency, + repos: [...successes, ...failures], + summary: { + total: repos.length, + success: successes.length, + failure: failures.length, + duration_ms: Math.round(total_duration_ms), + }, + }; + console.log(JSON.stringify(json_output, null, 2)); + } else { + // Text format + log.info(''); // blank line + + // Show successes + if (successes.length > 0) { + log.info(st('green', `✓ ${successes.length} succeeded:`)); + for (const result of successes) { + const duration = `${Math.round(result.duration_ms)}ms`; + log.info(st('gray', ` ${result.repo_name} ${st('blue', `(${duration})`)}`)); + } + } + + // Show failures with details + if (failures.length > 0) { + log.info(''); // blank line + log.error(st('red', `✗ ${failures.length} failed:`)); + for (const result of failures) { + const duration = `${Math.round(result.duration_ms)}ms`; + log.error(st('gray', ` ${result.repo_name} ${st('blue', `(${duration})`)}`)); + + if (result.error) { + log.error(st('gray', ` Error: ${result.error}`)); + } else if (result.exit_code !== 0) { + log.error(st('gray', ` Exit code: ${result.exit_code}`)); + } + + if (result.stderr) { + // Show first few lines of stderr + const stderr_lines = result.stderr.trim().split('\n'); + const preview_lines = stderr_lines.slice(0, 3); + for (const line of preview_lines) { + log.error(st('gray', ` ${line}`)); + } + if (stderr_lines.length > 3) { + log.error(st('gray', ` ... (${stderr_lines.length - 3} more lines)`)); + } + } + } + } + + // Summary + log.info(''); // blank line + const total = repos.length; + const success_rate = ((successes.length / total) * 100).toFixed(0); + const duration = `${Math.round(total_duration_ms)}ms`; + + if (failures.length === 0) { + log.info( + st( + 'green', + `✓ All ${total} repos succeeded in ${duration} (${success_rate}% success rate)`, + ), + ); + } else { + log.info( + st( + 'yellow', + `⚠ ${successes.length}/${total} repos succeeded in ${duration} (${success_rate}% success rate)`, + ), + ); + } + } + + // Exit with error if any failures (so CI fails) + if (failures.length > 0) { + throw new TaskError(`${failures.length} repos failed`); + } + }, +}; diff --git a/src/lib/local_repo.ts b/src/lib/local_repo.ts index f4bf1dea..8be7c03b 100644 --- a/src/lib/local_repo.ts +++ b/src/lib/local_repo.ts @@ -6,6 +6,7 @@ import {join} from 'node:path'; import {TaskError} from '@ryanatkn/gro'; import type {Logger} from '@fuzdev/fuz_util/log.js'; import {spawn} from '@fuzdev/fuz_util/process.js'; +import {map_concurrent_settled} from '@fuzdev/fuz_util/async.js'; import type {GitOperations, NpmOperations} from './operations.js'; import {default_git_operations, default_npm_operations} from './operations_defaults.js'; @@ -280,16 +281,57 @@ export const local_repos_load = async ({ log, git_ops = default_git_operations, npm_ops = default_npm_operations, + parallel = true, + concurrency = 5, }: { local_repo_paths: Array; log?: Logger; git_ops?: GitOperations; npm_ops?: NpmOperations; + parallel?: boolean; + concurrency?: number; }): Promise> => { + if (!parallel) { + // Sequential loading (original behavior) + const loaded: Array = []; + for (const local_repo_path of local_repo_paths) { + loaded.push(await local_repo_load({local_repo_path, log, git_ops, npm_ops})); // eslint-disable-line no-await-in-loop + } + return loaded; + } + + // Parallel loading with concurrency limit + const results = await map_concurrent_settled( + local_repo_paths, + async (local_repo_path) => { + return local_repo_load({local_repo_path, log, git_ops, npm_ops}); + }, + concurrency, + ); + + // Check for failures and collect successes const loaded: Array = []; - for (const local_repo_path of local_repo_paths) { - loaded.push(await local_repo_load({local_repo_path, log, git_ops, npm_ops})); // eslint-disable-line no-await-in-loop + const errors: Array<{repo_name: string; error: string}> = []; + + for (let i = 0; i < results.length; i++) { + const result = results[i]!; + if (result.status === 'fulfilled') { + loaded.push(result.value); + } else { + const repo_path = local_repo_paths[i]!; + errors.push({ + repo_name: repo_path.repo_name, + error: String(result.reason), + }); + } } + + // If any repos failed to load, throw with details + if (errors.length > 0) { + const error_details = errors.map((e) => ` ${e.repo_name}: ${e.error}`).join('\n'); + throw new TaskError(`Failed to load ${errors.length} repos:\n${error_details}`); + } + return loaded; }; diff --git a/src/routes/library.ts b/src/routes/library.ts index 8ab2a367..3fc903e2 100644 --- a/src/routes/library.ts +++ b/src/routes/library.ts @@ -1595,6 +1595,26 @@ export const library_json: LibraryJson = { 'publishing_plan.ts', ], }, + { + path: 'gitops_run.task.ts', + declarations: [ + { + name: 'Args', + kind: 'type', + source_line: 10, + type_signature: + 'ZodObject<{ command: ZodString; path: ZodDefault; concurrency: ZodDefault; format: ZodDefault>; }, $strict>', + }, + { + name: 'task', + kind: 'variable', + source_line: 40, + type_signature: + 'Task<{ command: string; path: string; concurrency: number; format: "json" | "text"; }, ZodType>, unknown>', + }, + ], + dependencies: ['repo_ops.ts'], + }, { path: 'gitops_sync.task.ts', declarations: [], @@ -1828,7 +1848,7 @@ export const library_json: LibraryJson = { kind: 'type', doc_comment: 'Fully loaded local repo with Library and extracted dependency data.\nDoes not extend LocalRepoPath - Library is source of truth for name/repo_url/etc.', - source_line: 19, + source_line: 20, type_signature: 'LocalRepo', properties: [ { @@ -1878,7 +1898,7 @@ export const library_json: LibraryJson = { kind: 'type', doc_comment: 'A repo that has been located on the filesystem (path exists).\nUsed before loading - just filesystem/git concerns.', - source_line: 34, + source_line: 35, type_signature: 'LocalRepoPath', properties: [ { @@ -1917,7 +1937,7 @@ export const library_json: LibraryJson = { name: 'LocalRepoMissing', kind: 'type', doc_comment: 'A repo that is missing from the filesystem (needs cloning).', - source_line: 46, + source_line: 47, type_signature: 'LocalRepoMissing', properties: [ { @@ -1959,7 +1979,7 @@ export const library_json: LibraryJson = { 'workspace dirty, branch switch fails, install fails, or library.ts missing', }, ], - source_line: 71, + source_line: 72, type_signature: '({ local_repo_path, log: _log, git_ops, npm_ops, }: { local_repo_path: LocalRepoPath; log?: Logger | undefined; git_ops?: GitOperations | undefined; npm_ops?: NpmOperations | undefined; }): Promise<...>', return_type: 'Promise', @@ -1973,7 +1993,7 @@ export const library_json: LibraryJson = { { name: 'local_repos_ensure', kind: 'function', - source_line: 228, + source_line: 229, type_signature: '({ resolved_config, repos_dir, gitops_config, download, log, npm_ops, }: { resolved_config: ResolvedGitopsConfig; repos_dir: string; gitops_config: GitopsConfig; download: boolean; log?: Logger | undefined; npm_ops?: NpmOperations | undefined; }): Promise<...>', return_type: 'Promise', @@ -1987,21 +2007,21 @@ export const library_json: LibraryJson = { { name: 'local_repos_load', kind: 'function', - source_line: 278, + source_line: 279, type_signature: - '({ local_repo_paths, log, git_ops, npm_ops, }: { local_repo_paths: LocalRepoPath[]; log?: Logger | undefined; git_ops?: GitOperations | undefined; npm_ops?: NpmOperations | undefined; }): Promise<...>', + '({ local_repo_paths, log, git_ops, npm_ops, parallel, concurrency, }: { local_repo_paths: LocalRepoPath[]; log?: Logger | undefined; git_ops?: GitOperations | undefined; npm_ops?: NpmOperations | undefined; parallel?: boolean | undefined; concurrency?: number | undefined; }): Promise<...>', return_type: 'Promise', parameters: [ { name: '__0', - type: '{ local_repo_paths: LocalRepoPath[]; log?: Logger | undefined; git_ops?: GitOperations | undefined; npm_ops?: NpmOperations | undefined; }', + type: '{ local_repo_paths: LocalRepoPath[]; log?: Logger | undefined; git_ops?: GitOperations | undefined; npm_ops?: NpmOperations | undefined; parallel?: boolean | undefined; concurrency?: number | undefined; }', }, ], }, { name: 'local_repo_locate', kind: 'function', - source_line: 296, + source_line: 338, type_signature: '({ repo_config, repos_dir, }: { repo_config: GitopsRepoConfig; repos_dir: string; }): LocalRepoPath | LocalRepoMissing', return_type: 'LocalRepoPath | LocalRepoMissing', @@ -2174,7 +2194,7 @@ export const library_json: LibraryJson = { 'Logs a simple bulleted list with a header.\nCommon pattern for warnings, info messages, and other lists.', source_line: 130, type_signature: - '(items: string[], header: string, color: "cyan" | "yellow" | "red" | "dim", log: Logger, log_method?: "info" | "warn" | "error"): void', + '(items: string[], header: string, color: "cyan" | "yellow" | "red" | "dim", log: Logger, log_method?: "error" | "info" | "warn"): void', return_type: 'void', parameters: [ { @@ -2195,7 +2215,7 @@ export const library_json: LibraryJson = { }, { name: 'log_method', - type: '"info" | "warn" | "error"', + type: '"error" | "info" | "warn"', default_value: "'info'", }, ], @@ -4031,6 +4051,7 @@ export const library_json: LibraryJson = { module_comment: 'Generic repository operations for scripts that work across repos.\n\nProvides lightweight utilities for:\n- Getting repo paths from gitops config (without full git sync)\n- Walking files in repos with sensible exclusions\n- Common exclusion patterns for node/svelte projects\n\nFor full git sync/clone functionality, use `get_gitops_ready()` from gitops_task_helpers.', dependencies: ['gitops_config.ts', 'paths.ts'], + dependents: ['gitops_run.task.ts'], }, { path: 'repo.svelte.ts', diff --git a/static/.nojekyll b/static/.nojekyll new file mode 100644 index 00000000..e69de29b From f484a3f91e29462109aa95d12428e9b3b0ca165f Mon Sep 17 00:00:00 2001 From: Ryan Atkinson Date: Tue, 16 Dec 2025 10:46:07 -0500 Subject: [PATCH 2/7] wip --- TODO_GITOPS_RUN.md | 341 -------------------- GITOPS_RUN_SUMMARY.md => docs/gitops_run.md | 0 src/lib/gitops_task_helpers.ts | 17 +- src/test/gitops_run.test.ts | 20 ++ 4 files changed, 34 insertions(+), 344 deletions(-) delete mode 100644 TODO_GITOPS_RUN.md rename GITOPS_RUN_SUMMARY.md => docs/gitops_run.md (100%) create mode 100644 src/test/gitops_run.test.ts diff --git a/TODO_GITOPS_RUN.md b/TODO_GITOPS_RUN.md deleted file mode 100644 index cae7d279..00000000 --- a/TODO_GITOPS_RUN.md +++ /dev/null @@ -1,341 +0,0 @@ -# TODO: gitops_run future enhancements - -This doc tracks potential future enhancements to `gitops_run` beyond the initial implementation. - -## Initial Implementation (v1) ✅ - -Core functionality for immediate use: -- Single command string execution across repos -- Parallel execution with concurrency limit (default: 5) -- Continue-on-error behavior with summary -- Output capture and structured reporting -- Uses `map_concurrent_settled` from fuz_util - -**Usage:** -```bash -gro gitops_run "npm test" -gro gitops_run "npm test" --concurrency 3 -gro gitops_run "npm audit" --format json -``` - -## Shell Features & Command Parsing - -**Question: How much shell functionality should we support?** - -Options: -1. **Simple string execution** (current): Just pass string to shell as-is - - Pros: Simple, works for basic cases, supports pipes/redirects naturally - - Cons: Shell injection risk if we ever template commands - -2. **Command array** (like Node's spawn): `["npm", "test"]` - - Pros: No shell injection possible, explicit args - - Cons: No shell features (pipes, redirects, etc.) - -3. **Template interpolation**: `"npm test {{repo_name}}"` - - Pros: Flexible, can pass repo-specific data - - Cons: Adds complexity, shell injection risk - -**Recommendation**: Start with #1 (simple string), add #3 later if needed. - -## Multiple Commands (Chaining) - -Run multiple commands in sequence per repo: - -```bash -# Option A: Multiple positional args -gro gitops_run "npm test" "npm audit" "gro check" - -# Option B: Shell-style chaining (already works?) -gro gitops_run "npm test && npm audit && gro check" - -# Option C: Config file with command sequences -# gitops.config.ts -{ - repos: [...], - commands: { - 'full-check': ['npm test', 'npm audit', 'gro check'], - 'update': ['npm update', 'npm install'], - } -} -``` - -## Conditional Execution - -Run commands only on repos matching criteria: - -```bash -# Only repos with changesets -gro gitops_run "npm test" --only-with-changesets - -# Only repos matching pattern -gro gitops_run "npm test" --only "*_ui" - -# Only repos with specific files -gro gitops_run "npm test" --only-with-file "test/**" - -# Exclude repos -gro gitops_run "npm test" --exclude "fuz_template" -``` - -## Lifecycle Hooks in Config - -Add custom commands at specific lifecycle points: - -```ts -// gitops.config.ts -export default { - repos: [...], - hooks: { - before_sync: "npm ci", // Ensure clean deps before sync - after_clone: "npm install", // Auto-setup after download - before_publish: "npm audit", // Security check - after_publish: "./notify.sh", // Custom notifications - on_failure: "./alert.sh", // Alert on failures - } -} -``` - -## Output Formats & Aggregation - -Better output handling: - -```bash -# JSON output for scripting -gro gitops_run "npm test" --format json - -# Table view with status -gro gitops_run "npm test" --format table - -# Only show failures -gro gitops_run "npm test" --only-failures - -# Save output per repo -gro gitops_run "npm test" --save-output .gro/test_results/ -``` - -**Output format examples:** - -```json -// --format json -{ - "command": "npm test", - "concurrency": 5, - "repos": [ - { - "name": "fuz_ui", - "status": "success", - "duration_ms": 1234, - "stdout": "...", - "stderr": "" - }, - { - "name": "fuz_css", - "status": "failure", - "duration_ms": 567, - "exit_code": 1, - "stdout": "...", - "stderr": "..." - } - ], - "summary": { - "total": 10, - "success": 9, - "failure": 1, - "duration_ms": 5678 - } -} -``` - -## Repo-Specific Command Overrides - -Allow repos to customize commands: - -```ts -// gitops.config.ts -export default { - repos: [ - { - repo_url: 'https://github.com/fuzdev/fuz_ui', - commands: { - test: 'npm test -- --coverage', // Custom test command - lint: 'npm run lint:strict', - } - }, - 'https://github.com/fuzdev/fuz_css', // Uses default commands - ], -} -``` - -## Interactive Mode - -Choose commands interactively: - -```bash -gro gitops_run --interactive -# Prompts: -# > Select command: [test, lint, check, build, custom] -# > Select repos: [all, select, pattern] -# > Concurrency: [1, 3, 5, 10] -``` - -## Dry Run Mode - -Preview what would run: - -```bash -gro gitops_run "npm test" --dry-run -# Output: -# Would run "npm test" in 10 repos with concurrency 5: -# - fuz_ui (~/dev/fuz_ui) -# - fuz_css (~/dev/fuz_css) -# ... -``` - -## Progress Indicators - -Better UX for long-running operations: - -```bash -gro gitops_run "npm test" -# Output: -# Running "npm test" in 10 repos (concurrency: 5)... -# [████████░░] 8/10 complete (fuz_ui: running, fuz_css: success, ...) -``` - -## Workspace State Management - -Commands that need clean/dirty workspace checks: - -```bash -# Require clean workspace -gro gitops_run "npm test" --require-clean - -# Auto-stash before running -gro gitops_run "gro build" --auto-stash -``` - -## Remote Execution - -Run commands on remote CI or via SSH: - -```bash -# Via GitHub Actions -gro gitops_run "npm test" --remote github - -# Via SSH -gro gitops_run "npm test" --remote ssh://user@host -``` - -## Caching & Memoization - -Skip commands if nothing changed: - -```bash -# Only test repos with changes since last run -gro gitops_run "npm test" --cache --since-commit HEAD~5 - -# Only test repos with file changes -gro gitops_run "npm test" --cache --changed-files -``` - -## Error Recovery Strategies - -More sophisticated error handling: - -```bash -# Retry failures -gro gitops_run "npm install" --retry 3 - -# Retry with exponential backoff -gro gitops_run "npm install" --retry 3 --backoff exponential - -# Fail after N failures -gro gitops_run "npm test" --max-failures 3 -``` - -## Dependency-Aware Execution - -Run commands in dependency order: - -```bash -# Build in topological order -gro gitops_run "gro build" --topo - -# Test in parallel but respect dependencies -gro gitops_run "npm test" --topo-parallel -``` - -## Integration with Existing Commands - -Reuse gitops_run patterns in other commands: - -- Update `gitops_sync` to use `map_concurrent_settled` -- Add `--concurrency` flag to `gitops_publish` -- Add parallel preflight checks -- Parallel GitHub API fetching (with rate limit respect) - -## Environment Variable Templating - -Pass repo context as env vars: - -```bash -# Template vars: REPO_NAME, REPO_DIR, REPO_URL -gro gitops_run "echo Testing $REPO_NAME" -``` - -## Logging & Observability - -Structured logging for debugging: - -```bash -# Log timing per repo -gro gitops_run "npm test" --timing - -# Log resource usage -gro gitops_run "npm test" --resource-usage - -# Export to observability format (OpenTelemetry, etc.) -gro gitops_run "npm test" --trace opentelemetry -``` - -## Config-Defined Commands - -Pre-define common command sequences: - -```ts -// gitops.config.ts -export default { - repos: [...], - commands: { - ci: ['npm ci', 'npm test', 'npm run build'], - update: ['gro upgrade @ryanatkn/gro@latest --no-pull', 'npm install'], - check_all: ['gro check', 'npm audit', 'npm outdated'], - } -} -``` - -```bash -gro gitops_run ci -gro gitops_run update -``` - -## Priority & Scheduling - -Control execution order: - -```ts -// gitops.config.ts -{ - repos: [ - {repo_url: '...', priority: 1}, // Run first - {repo_url: '...', priority: 10}, // Run last - ] -} -``` - -## Notes - -- Keep the initial implementation simple and focused -- Add features based on actual usage patterns -- Avoid feature creep - not every shell tool needs to be reimplemented -- Consider which features are better solved by external tools (GNU parallel, etc.) diff --git a/GITOPS_RUN_SUMMARY.md b/docs/gitops_run.md similarity index 100% rename from GITOPS_RUN_SUMMARY.md rename to docs/gitops_run.md diff --git a/src/lib/gitops_task_helpers.ts b/src/lib/gitops_task_helpers.ts index 9f6c7e3b..f7df92d8 100644 --- a/src/lib/gitops_task_helpers.ts +++ b/src/lib/gitops_task_helpers.ts @@ -33,6 +33,8 @@ export interface GetGitopsReadyOptions { log?: Logger; git_ops?: GitOperations; npm_ops?: NpmOperations; + parallel?: boolean; + concurrency?: number; } /** @@ -41,7 +43,7 @@ export interface GetGitopsReadyOptions { * Initialization sequence: * 1. Loads and normalizes config from `gitops.config.ts` * 2. Resolves local repo paths (creates missing with `--download`) - * 3. Switches branches and pulls latest changes + * 3. Switches branches and pulls latest changes (in parallel by default) * 4. Auto-installs deps if package.json changed during pull * * Priority for path resolution: @@ -51,6 +53,8 @@ export interface GetGitopsReadyOptions { * * @param options.git_ops for testing (defaults to real git operations) * @param options.npm_ops for testing (defaults to real npm operations) + * @param options.parallel whether to load repos in parallel (default: true) + * @param options.concurrency max concurrent repo loads (default: 5) * @returns initialized config and fully loaded repos ready for operations * @throws {TaskError} if config loading or repo resolution fails */ @@ -62,7 +66,7 @@ export const get_gitops_ready = async ( gitops_config: GitopsConfig; local_repos: Array; }> => { - const {path, dir, download, log, git_ops, npm_ops} = options; + const {path, dir, download, log, git_ops, npm_ops, parallel, concurrency} = options; const config_path = resolve(path); const gitops_config = await import_gitops_config(config_path); @@ -88,7 +92,14 @@ export const get_gitops_ready = async ( npm_ops, }); - const local_repos = await local_repos_load({local_repo_paths, log, git_ops, npm_ops}); + const local_repos = await local_repos_load({ + local_repo_paths, + log, + git_ops, + npm_ops, + parallel, + concurrency, + }); return {config_path, repos_dir, gitops_config, local_repos}; }; diff --git a/src/test/gitops_run.test.ts b/src/test/gitops_run.test.ts new file mode 100644 index 00000000..ee4268d1 --- /dev/null +++ b/src/test/gitops_run.test.ts @@ -0,0 +1,20 @@ +import {test} from 'vitest'; + +// TODO: Add tests for gitops_run +// - Test command execution across multiple repos +// - Test concurrency limits +// - Test error handling (continue-on-error) +// - Test JSON output format +// - Test with fixture repos + +// For now, we rely on manual testing and fixture tests. +// The command execution is tested indirectly through manual runs. + +test.skip('gitops_run - placeholder for future tests', () => { + // Tests to add: + // 1. Execute simple command across repos + // 2. Verify concurrency throttling + // 3. Test continue-on-error behavior + // 4. Validate JSON output structure + // 5. Test with various command types (success, failure, mixed) +}); From e4a0daa709f1491cc160383bd922794e40cb3a01 Mon Sep 17 00:00:00 2001 From: Ryan Atkinson Date: Tue, 16 Dec 2025 10:47:16 -0500 Subject: [PATCH 3/7] wip --- CLAUDE.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index a1cd2af3..ef53c067 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -315,7 +315,7 @@ gro gitops_sync --download # clone missing repos gro gitops_sync --check # verify repos are ready without fetching data # Run commands across repos -gro gitops_run "npm test" # run command in all repos (parallel, concurrency: 5) +gro gitops_run "npm test" # run command in all repos (parallel, concurrency: 5) gro gitops_run "npm audit" --concurrency 3 # limit parallelism gro gitops_run "gro check" --format json # JSON output for scripting @@ -407,7 +407,7 @@ For packages you control, use `>=` instead of `^` for peer dependencies: ```json "peerDependencies": { - "@fuzdev/fuz_util": ">=0.38.0", // controlled package - use >= + "@fuzdev/fuz_util": ">=0.38.0", // controlled package - use >= "@ryanatkn/gro": ">=0.174.0", // controlled package - use >= "@sveltejs/kit": "^2", // third-party - use ^ "svelte": "^5" // third-party - use ^ @@ -478,8 +478,8 @@ Uses vitest with **zero mocks** - all tests use the operations pattern for dependency injection (see above). ```bash -gro test # run all tests -gro test version_utils # run specific test file +gro test # run all tests +gro test version_utils # run specific test file gro test src/test/fixtures/check # validate command output fixtures ``` From a34d0d2b2775118ab8af4e83b9d92366aad83668 Mon Sep 17 00:00:00 2001 From: Ryan Atkinson Date: Tue, 16 Dec 2025 10:49:47 -0500 Subject: [PATCH 4/7] wip --- docs/gitops_run.md | 88 ---------------------------------------------- 1 file changed, 88 deletions(-) delete mode 100644 docs/gitops_run.md diff --git a/docs/gitops_run.md b/docs/gitops_run.md deleted file mode 100644 index 46f71804..00000000 --- a/docs/gitops_run.md +++ /dev/null @@ -1,88 +0,0 @@ -# gitops_run Implementation Summary - -## What was built - -A new `gro gitops_run` command that executes shell commands across all repos in parallel with configurable concurrency and comprehensive error handling. - -## Key features - -1. **Parallel execution** - Runs commands across repos concurrently (default: 5) -2. **Throttled concurrency** - Uses `map_concurrent_settled` from fuz_util -3. **Continue-on-error** - Shows all results, doesn't fail-fast -4. **Structured output** - Text (default) or JSON format -5. **Lightweight** - Uses `get_repo_paths()` instead of full repo loading - -## Usage - -```bash -# Basic usage -gro gitops_run --command "npm test" - -# Control concurrency -gro gitops_run --command "npm audit" --concurrency 3 - -# JSON output for scripting -gro gitops_run --command "git status" --format json - -# Use with test fixtures -gro gitops_run --path src/test/fixtures/configs/basic_publishing.config.ts --command "pwd" - -# Chain commands -gro gitops_run --command "gro upgrade @ryanatkn/gro@latest --no-pull && git add static/.nojekyll" -``` - -## Implementation details - -### Files created/modified - -- **NEW**: `src/lib/gitops_run.task.ts` - Main task implementation -- **NEW**: `TODO_GITOPS_RUN.md` - Future enhancement ideas -- **MODIFIED**: `src/lib/local_repo.ts` - Added parallel loading with `map_concurrent_settled` -- **MODIFIED**: `CLAUDE.md` - Added gitops_run documentation -- **MODIFIED**: `README.md` - Added usage examples - -### Design choices - -1. **Lightweight execution** - Uses `get_repo_paths()` instead of full `get_gitops_ready()` - - Doesn't require `library.ts` files - - No git sync/pull by default - - Faster startup - -2. **Shell mode** - Commands run via `sh -c` to support pipes, redirects, etc. - - Trade-off: Slightly less safe than argument arrays - - Benefit: Full shell capabilities - -3. **Concurrency** - Default 5 repos at a time - - Based on user preference - - Prevents overwhelming system resources - - Respects rate limits - -4. **Error handling** - Continue-on-error with detailed reporting - - Shows all successes and failures - - Includes exit codes and stderr - - Exits with error if any repo fails (for CI) - -### Future enhancements (see TODO_GITOPS_RUN.md) - -- Config-defined commands -- Conditional execution (--only-with-changesets, etc.) -- Lifecycle hooks -- Retry logic -- Dependency-aware execution order - -## Testing - -Tested with: -- Fixture repos (basic_publishing) ✓ -- Real repos (9 repos in config) ✓ -- Simple commands ✓ -- Complex chained commands ✓ -- JSON output ✓ -- Various concurrency levels ✓ - -## Performance - -Example with 9 repos @ concurrency=3: -- Simple echo command: ~41ms total -- Commands run in batches of 3 -- Minimal overhead per repo (~5-20ms) From 630055eaa17597e94cc12b93b91ad8ef8874cc9d Mon Sep 17 00:00:00 2001 From: Ryan Atkinson Date: Tue, 16 Dec 2025 11:04:35 -0500 Subject: [PATCH 5/7] wip --- src/lib/constants.ts | 8 -------- src/lib/gitops_analyze.task.ts | 11 ++++++----- src/lib/gitops_constants.ts | 30 ++++++++++++++++++++++++++++++ src/lib/gitops_plan.task.ts | 13 +++++++------ src/lib/gitops_publish.task.ts | 16 ++++++++++------ src/lib/gitops_run.task.ts | 11 ++++++----- src/lib/gitops_sync.task.ts | 11 ++++++----- src/lib/gitops_task_helpers.ts | 14 +++++++------- src/lib/gitops_validate.task.ts | 11 ++++++----- src/lib/local_repo.ts | 3 ++- src/lib/multi_repo_publisher.ts | 10 +++++----- src/lib/publishing_plan.ts | 8 ++++---- 12 files changed, 89 insertions(+), 57 deletions(-) delete mode 100644 src/lib/constants.ts create mode 100644 src/lib/gitops_constants.ts diff --git a/src/lib/constants.ts b/src/lib/constants.ts deleted file mode 100644 index 9b95d751..00000000 --- a/src/lib/constants.ts +++ /dev/null @@ -1,8 +0,0 @@ -/** - * Maximum number of iterations for fixed-point iteration during publishing. - * Used in both plan generation and actual publishing to resolve transitive dependency cascades. - * - * In practice, most repos converge in 2-3 iterations. - * Deep dependency chains may require more iterations. - */ -export const MAX_ITERATIONS = 10; diff --git a/src/lib/gitops_analyze.task.ts b/src/lib/gitops_analyze.task.ts index f5b2b46e..1d6f6a59 100644 --- a/src/lib/gitops_analyze.task.ts +++ b/src/lib/gitops_analyze.task.ts @@ -13,16 +13,17 @@ import { format_production_cycles, } from './log_helpers.js'; import {format_and_output, type OutputFormatters} from './output_helpers.js'; +import {GITOPS_CONFIG_PATH_DEFAULT} from './gitops_constants.js'; /** @nodocs */ export const Args = z.strictObject({ - path: z + config: z .string() .meta({description: 'path to the gitops config file, absolute or relative to the cwd'}) - .default('gitops.config.ts'), + .default(GITOPS_CONFIG_PATH_DEFAULT), dir: z .string() - .meta({description: 'path containing the repos, defaults to the parent of the `path` dir'}) + .meta({description: 'path containing the repos, defaults to the parent of the config dir'}) .optional(), format: z .enum(['stdout', 'json', 'markdown']) @@ -37,10 +38,10 @@ export const task: Task = { Args, summary: 'analyze dependency structure and relationships across repos', run: async ({args, log}) => { - const {path, dir, format, outfile} = args; + const {config, dir, format, outfile} = args; // Get repos ready (without downloading) - const {local_repos} = await get_gitops_ready({path, dir, download: false, log}); + const {local_repos} = await get_gitops_ready({config, dir, download: false, log}); // Build dependency graph and validate (but don't throw on cycles for analyze) const {graph, publishing_order: order} = validate_dependency_graph(local_repos, { diff --git a/src/lib/gitops_constants.ts b/src/lib/gitops_constants.ts new file mode 100644 index 00000000..57e744a0 --- /dev/null +++ b/src/lib/gitops_constants.ts @@ -0,0 +1,30 @@ +/** + * Shared constants for gitops tasks and operations. + * + * Naming convention: GITOPS_{NAME}_DEFAULT for user-facing defaults. + */ + +/** + * Maximum number of iterations for fixed-point iteration during publishing. + * Used in both plan generation and actual publishing to resolve transitive dependency cascades. + * + * In practice, most repos converge in 2-3 iterations. + * Deep dependency chains may require more iterations. + */ +export const GITOPS_MAX_ITERATIONS_DEFAULT = 10; + +/** + * Default path to the gitops configuration file. + */ +export const GITOPS_CONFIG_PATH_DEFAULT = 'gitops.config.ts'; + +/** + * Default number of repos to process concurrently during parallel operations. + */ +export const GITOPS_CONCURRENCY_DEFAULT = 5; + +/** + * Default timeout in milliseconds for waiting on NPM package propagation (10 minutes). + * NPM's CDN uses eventual consistency, so published packages may not be immediately available. + */ +export const GITOPS_NPM_WAIT_TIMEOUT_DEFAULT = 600_000; // 10 minutes diff --git a/src/lib/gitops_plan.task.ts b/src/lib/gitops_plan.task.ts index 57869892..5be673aa 100644 --- a/src/lib/gitops_plan.task.ts +++ b/src/lib/gitops_plan.task.ts @@ -10,16 +10,17 @@ import { type LogPlanOptions, } from './publishing_plan.js'; import {format_and_output, type OutputFormatters} from './output_helpers.js'; +import {GITOPS_CONFIG_PATH_DEFAULT} from './gitops_constants.js'; /** @nodocs */ export const Args = z.strictObject({ - path: z + config: z .string() .meta({description: 'path to the gitops config file, absolute or relative to the cwd'}) - .default('gitops.config.ts'), + .default(GITOPS_CONFIG_PATH_DEFAULT), dir: z .string() - .meta({description: 'path containing the repos, defaults to the parent of the `path` dir'}) + .meta({description: 'path containing the repos, defaults to the parent of the config dir'}) .optional(), format: z .enum(['stdout', 'json', 'markdown']) @@ -37,7 +38,7 @@ export type Args = z.infer; * Usage: * gro gitops_plan * gro gitops_plan --dir ../repos - * gro gitops_plan --path ./custom.config.ts + * gro gitops_plan --config ./custom.config.ts * * @nodocs */ @@ -45,13 +46,13 @@ export const task: Task = { summary: 'generate a publishing plan based on changesets', Args, run: async ({args, log}): Promise => { - const {dir, path, format, outfile, verbose} = args; + const {dir, config, format, outfile, verbose} = args; log.info(st('cyan', 'Generating multi-repo publishing plan...')); // Load local repos const {local_repos} = await get_gitops_ready({ - path, + config, dir, download: false, // Don't download if missing log, diff --git a/src/lib/gitops_publish.task.ts b/src/lib/gitops_publish.task.ts index f164cb14..f61aaad1 100644 --- a/src/lib/gitops_publish.task.ts +++ b/src/lib/gitops_publish.task.ts @@ -11,16 +11,20 @@ import { } from './multi_repo_publisher.js'; import {generate_publishing_plan, log_publishing_plan} from './publishing_plan.js'; import {format_and_output, type OutputFormatters} from './output_helpers.js'; +import { + GITOPS_CONFIG_PATH_DEFAULT, + GITOPS_NPM_WAIT_TIMEOUT_DEFAULT, +} from './gitops_constants.js'; /** @nodocs */ export const Args = z.strictObject({ - path: z + config: z .string() .meta({description: 'path to the gitops config file, absolute or relative to the cwd'}) - .default('gitops.config.ts'), + .default(GITOPS_CONFIG_PATH_DEFAULT), dir: z .string() - .meta({description: 'path containing the repos, defaults to the parent of the `path` dir'}) + .meta({description: 'path containing the repos, defaults to the parent of the config dir'}) .optional(), peer_strategy: z .enum(['exact', 'caret', 'tilde']) @@ -43,7 +47,7 @@ export const Args = z.strictObject({ max_wait: z .number() .meta({description: 'max time to wait for npm propagation in ms'}) - .default(600000), // 10 minutes + .default(GITOPS_NPM_WAIT_TIMEOUT_DEFAULT), skip_install: z .boolean() .meta({description: 'skip npm install after dependency updates'}) @@ -59,7 +63,7 @@ export const task: Task = { Args, run: async ({args, log}): Promise => { const { - path, + config, dir, peer_strategy, dry_run, @@ -74,7 +78,7 @@ export const task: Task = { // Load repos const {local_repos: repos} = await get_gitops_ready({ - path, + config, dir, download: false, // Don't download if missing log, diff --git a/src/lib/gitops_run.task.ts b/src/lib/gitops_run.task.ts index 1720b03b..d43f3f84 100644 --- a/src/lib/gitops_run.task.ts +++ b/src/lib/gitops_run.task.ts @@ -6,19 +6,20 @@ import {styleText as st} from 'node:util'; import {resolve} from 'node:path'; import {get_repo_paths} from './repo_ops.js'; +import {GITOPS_CONCURRENCY_DEFAULT, GITOPS_CONFIG_PATH_DEFAULT} from './gitops_constants.js'; export const Args = z.strictObject({ command: z.string().meta({description: 'shell command to run in each repo'}), - path: z + config: z .string() .meta({description: 'path to the gitops config file'}) - .default('gitops.config.ts'), + .default(GITOPS_CONFIG_PATH_DEFAULT), concurrency: z .number() .int() .min(1) .meta({description: 'maximum number of repos to run in parallel'}) - .default(5), + .default(GITOPS_CONCURRENCY_DEFAULT), format: z .enum(['text', 'json']) .meta({description: 'output format'}) @@ -41,10 +42,10 @@ export const task: Task = { Args, summary: 'run a shell command across all repos in parallel', run: async ({args, log}) => { - const {command, path, concurrency, format} = args; + const {command, config, concurrency, format} = args; // Get repo paths (lightweight, no library.ts loading needed) - const config_path = resolve(path); + const config_path = resolve(config); const repos = await get_repo_paths(config_path); if (repos.length === 0) { diff --git a/src/lib/gitops_sync.task.ts b/src/lib/gitops_sync.task.ts index b405663b..03b6dff6 100644 --- a/src/lib/gitops_sync.task.ts +++ b/src/lib/gitops_sync.task.ts @@ -11,18 +11,19 @@ import {existsSync} from 'node:fs'; import {fetch_repo_data} from './fetch_repo_data.js'; import {create_fs_fetch_value_cache} from './fs_fetch_value_cache.js'; import {get_gitops_ready} from './gitops_task_helpers.js'; +import {GITOPS_CONFIG_PATH_DEFAULT} from './gitops_constants.js'; // TODO add flag to ignore or invalidate cache -- no-cache? clean? /** @nodocs */ export const Args = z.strictObject({ - path: z + config: z .string() .meta({description: 'path to the gitops config file, absolute or relative to the cwd'}) - .default('gitops.config.ts'), + .default(GITOPS_CONFIG_PATH_DEFAULT), dir: z .string() - .meta({description: 'path containing the repos, defaults to the parent of the `path` dir'}) + .meta({description: 'path containing the repos, defaults to the parent of the config dir'}) .optional(), outdir: z .string() @@ -45,9 +46,9 @@ export const task: Task = { Args, summary: 'syncs local repos and generates UI data from repo metadata', run: async ({args, log, svelte_config, invoke_task}) => { - const {path, dir, outdir = svelte_config.routes_path, download, check} = args; + const {config, dir, outdir = svelte_config.routes_path, download, check} = args; - const {local_repos} = await get_gitops_ready({path, dir, download, log}); + const {local_repos} = await get_gitops_ready({config, dir, download, log}); const outfile = resolve(outdir, 'repos.ts'); diff --git a/src/lib/gitops_task_helpers.ts b/src/lib/gitops_task_helpers.ts index f7df92d8..21fd6ea0 100644 --- a/src/lib/gitops_task_helpers.ts +++ b/src/lib/gitops_task_helpers.ts @@ -27,7 +27,7 @@ import {DEFAULT_REPOS_DIR} from './paths.js'; import type {GitOperations, NpmOperations} from './operations.js'; export interface GetGitopsReadyOptions { - path: string; + config: string; dir?: string; download: boolean; log?: Logger; @@ -66,13 +66,13 @@ export const get_gitops_ready = async ( gitops_config: GitopsConfig; local_repos: Array; }> => { - const {path, dir, download, log, git_ops, npm_ops, parallel, concurrency} = options; - const config_path = resolve(path); + const {config, dir, download, log, git_ops, npm_ops, parallel, concurrency} = options; + const config_path = resolve(config); const gitops_config = await import_gitops_config(config_path); // Priority: explicit dir arg → config repos_dir → default (two dirs up from config) const repos_dir = resolve_gitops_paths({ - path, + config, dir, config_repos_dir: gitops_config.repos_dir, }).repos_dir; @@ -105,7 +105,7 @@ export const get_gitops_ready = async ( }; export interface ResolveGitopsPathsOptions { - path: string; + config: string; dir?: string; config_repos_dir?: string; } @@ -113,8 +113,8 @@ export interface ResolveGitopsPathsOptions { export const resolve_gitops_paths = ( options: ResolveGitopsPathsOptions, ): {config_path: string; repos_dir: string} => { - const {path, dir, config_repos_dir} = options; - const config_path = resolve(path); + const {config, dir, config_repos_dir} = options; + const config_path = resolve(config); const config_dir = dirname(config_path); // Priority: explicit dir arg → config repos_dir → default (parent of config dir) diff --git a/src/lib/gitops_validate.task.ts b/src/lib/gitops_validate.task.ts index f8839923..4c46cd05 100644 --- a/src/lib/gitops_validate.task.ts +++ b/src/lib/gitops_validate.task.ts @@ -8,16 +8,17 @@ import {DependencyGraphBuilder} from './dependency_graph.js'; import {generate_publishing_plan, log_publishing_plan} from './publishing_plan.js'; import {publish_repos, type PublishingOptions} from './multi_repo_publisher.js'; import {log_dependency_analysis} from './log_helpers.js'; +import {GITOPS_CONFIG_PATH_DEFAULT} from './gitops_constants.js'; /** @nodocs */ export const Args = z.strictObject({ - path: z + config: z .string() .meta({description: 'path to the gitops config file, absolute or relative to the cwd'}) - .default('gitops.config.ts'), + .default(GITOPS_CONFIG_PATH_DEFAULT), dir: z .string() - .meta({description: 'path containing the repos, defaults to the parent of the `path` dir'}) + .meta({description: 'path containing the repos, defaults to the parent of the config dir'}) .optional(), verbose: z.boolean().meta({description: 'show additional details'}).default(false), }); @@ -29,7 +30,7 @@ export const task: Task = { summary: 'validate gitops configuration by running all read-only commands and checking for issues', run: async ({args, log}) => { - const {path, dir, verbose} = args; + const {config, dir, verbose} = args; log.info(st('cyan', 'Running Gitops Validation Suite')); log.info(st('dim', 'This runs all read-only commands and checks for consistency.')); @@ -49,7 +50,7 @@ export const task: Task = { // Load repos once (shared by all commands) log.info(st('dim', 'Loading repositories...')); - const {local_repos} = await get_gitops_ready({path, dir, download: false, log}); + const {local_repos} = await get_gitops_ready({config, dir, download: false, log}); log.info(st('dim', ` Found ${local_repos.length} local repos`)); // 1. Run gitops_analyze diff --git a/src/lib/local_repo.ts b/src/lib/local_repo.ts index 8be7c03b..54def779 100644 --- a/src/lib/local_repo.ts +++ b/src/lib/local_repo.ts @@ -12,6 +12,7 @@ import {default_git_operations, default_npm_operations} from './operations_defau import type {GitopsConfig, GitopsRepoConfig} from './gitops_config.js'; import type {ResolvedGitopsConfig} from './resolved_gitops_config.js'; +import {GITOPS_CONCURRENCY_DEFAULT} from './gitops_constants.js'; /** * Fully loaded local repo with Library and extracted dependency data. @@ -282,7 +283,7 @@ export const local_repos_load = async ({ git_ops = default_git_operations, npm_ops = default_npm_operations, parallel = true, - concurrency = 5, + concurrency = GITOPS_CONCURRENCY_DEFAULT, }: { local_repo_paths: Array; log?: Logger; diff --git a/src/lib/multi_repo_publisher.ts b/src/lib/multi_repo_publisher.ts index 39b29397..8ec87940 100644 --- a/src/lib/multi_repo_publisher.ts +++ b/src/lib/multi_repo_publisher.ts @@ -10,7 +10,7 @@ import {type PreflightOptions} from './preflight_checks.js'; import {needs_update, is_breaking_change, detect_bump_type} from './version_utils.js'; import type {GitopsOperations} from './operations.js'; import {default_gitops_operations} from './operations_defaults.js'; -import {MAX_ITERATIONS} from './constants.js'; +import {GITOPS_MAX_ITERATIONS_DEFAULT} from './gitops_constants.js'; import {install_with_cache_healing} from './npm_install_helpers.js'; /* eslint-disable no-await-in-loop */ @@ -90,9 +90,9 @@ export const publish_repos = async ( let iteration = 0; let converged = false; - while (!converged && iteration < MAX_ITERATIONS) { + while (!converged && iteration < GITOPS_MAX_ITERATIONS_DEFAULT) { iteration++; - log?.info(st('cyan', `\n🚀 Publishing iteration ${iteration}/${MAX_ITERATIONS}...\n`)); + log?.info(st('cyan', `\n🚀 Publishing iteration ${iteration}/${GITOPS_MAX_ITERATIONS_DEFAULT}...\n`)); // Track if any packages were published in this iteration let published_in_iteration = false; @@ -245,7 +245,7 @@ export const publish_repos = async ( if (!published_in_iteration) { converged = true; log?.info(st('green', `\n✓ Converged after ${iteration} iteration(s) - no new changesets\n`)); - } else if (iteration === MAX_ITERATIONS) { + } else if (iteration === GITOPS_MAX_ITERATIONS_DEFAULT) { // Count packages that still have changesets (not yet published) const pending_count = order.length - published.size; const estimated_iterations = Math.ceil(pending_count / 2); // Rough estimate @@ -253,7 +253,7 @@ export const publish_repos = async ( log?.warn( st( 'yellow', - `\n⚠️ Reached maximum iterations (${MAX_ITERATIONS}) without full convergence\n` + + `\n⚠️ Reached maximum iterations (${GITOPS_MAX_ITERATIONS_DEFAULT}) without full convergence\n` + ` ${pending_count} package(s) may still have changesets to process\n` + ` Estimated ${estimated_iterations} more iteration(s) needed - run 'gro gitops_publish' again\n`, ), diff --git a/src/lib/publishing_plan.ts b/src/lib/publishing_plan.ts index a43f7881..fe66cdde 100644 --- a/src/lib/publishing_plan.ts +++ b/src/lib/publishing_plan.ts @@ -7,7 +7,7 @@ import {validate_dependency_graph} from './graph_validation.js'; import {is_breaking_change, compare_bump_types, calculate_next_version} from './version_utils.js'; import type {ChangesetOperations} from './operations.js'; import {default_changeset_operations} from './operations_defaults.js'; -import {MAX_ITERATIONS} from './constants.js'; +import {GITOPS_MAX_ITERATIONS_DEFAULT} from './gitops_constants.js'; import type {DependencyGraph} from './dependency_graph.ts'; import { calculate_dependency_updates, @@ -247,7 +247,7 @@ export const generate_publishing_plan = async ( let iteration = 0; let changed = true; - while (changed && iteration < MAX_ITERATIONS) { + while (changed && iteration < GITOPS_MAX_ITERATIONS_DEFAULT) { changed = false; iteration++; @@ -392,7 +392,7 @@ export const generate_publishing_plan = async ( } // Check if we hit iteration limit without convergence - if (iteration === MAX_ITERATIONS && changed) { + if (iteration === GITOPS_MAX_ITERATIONS_DEFAULT && changed) { // Calculate how many packages still need processing const pending_packages: Array = []; @@ -428,7 +428,7 @@ export const generate_publishing_plan = async ( const pending_count = pending_packages.length; const estimated_iterations = Math.ceil(pending_count / 2); // Rough estimate warnings.push( - `Reached maximum iterations (${MAX_ITERATIONS}) without full convergence - ` + + `Reached maximum iterations (${GITOPS_MAX_ITERATIONS_DEFAULT}) without full convergence - ` + `${pending_count} package(s) may still need processing: ${pending_packages.join(', ')}. ` + `Estimated ${estimated_iterations} more iteration(s) needed.`, ); From 27690ab406da860cebaccf14b64cf755e98e00a6 Mon Sep 17 00:00:00 2001 From: Ryan Atkinson Date: Tue, 16 Dec 2025 11:09:23 -0500 Subject: [PATCH 6/7] wip --- README.md | 1 + src/lib/gitops_publish.task.ts | 5 +- src/lib/gitops_run.task.ts | 6 +- src/lib/multi_repo_publisher.ts | 11 ++- src/lib/repo_ops.ts | 3 +- src/routes/library.ts | 161 ++++++++++++++++++++++---------- 6 files changed, 124 insertions(+), 63 deletions(-) diff --git a/README.md b/README.md index c31eb070..f9b27a0e 100644 --- a/README.md +++ b/README.md @@ -80,6 +80,7 @@ gro gitops_run "git status" --format json # JSON output for scripting ``` **Features:** + - Parallel execution with configurable concurrency (default: 5) - Continue-on-error behavior (shows all results) - Structured output formats (text or JSON) diff --git a/src/lib/gitops_publish.task.ts b/src/lib/gitops_publish.task.ts index f61aaad1..f437bc13 100644 --- a/src/lib/gitops_publish.task.ts +++ b/src/lib/gitops_publish.task.ts @@ -11,10 +11,7 @@ import { } from './multi_repo_publisher.js'; import {generate_publishing_plan, log_publishing_plan} from './publishing_plan.js'; import {format_and_output, type OutputFormatters} from './output_helpers.js'; -import { - GITOPS_CONFIG_PATH_DEFAULT, - GITOPS_NPM_WAIT_TIMEOUT_DEFAULT, -} from './gitops_constants.js'; +import {GITOPS_CONFIG_PATH_DEFAULT, GITOPS_NPM_WAIT_TIMEOUT_DEFAULT} from './gitops_constants.js'; /** @nodocs */ export const Args = z.strictObject({ diff --git a/src/lib/gitops_run.task.ts b/src/lib/gitops_run.task.ts index d43f3f84..aad95cbb 100644 --- a/src/lib/gitops_run.task.ts +++ b/src/lib/gitops_run.task.ts @@ -20,10 +20,7 @@ export const Args = z.strictObject({ .min(1) .meta({description: 'maximum number of repos to run in parallel'}) .default(GITOPS_CONCURRENCY_DEFAULT), - format: z - .enum(['text', 'json']) - .meta({description: 'output format'}) - .default('text'), + format: z.enum(['text', 'json']).meta({description: 'output format'}).default('text'), }); export type Args = z.infer; @@ -147,6 +144,7 @@ export const task: Task = { duration_ms: Math.round(total_duration_ms), }, }; + // eslint-disable-next-line no-console console.log(JSON.stringify(json_output, null, 2)); } else { // Text format diff --git a/src/lib/multi_repo_publisher.ts b/src/lib/multi_repo_publisher.ts index 8ec87940..3a165256 100644 --- a/src/lib/multi_repo_publisher.ts +++ b/src/lib/multi_repo_publisher.ts @@ -10,7 +10,10 @@ import {type PreflightOptions} from './preflight_checks.js'; import {needs_update, is_breaking_change, detect_bump_type} from './version_utils.js'; import type {GitopsOperations} from './operations.js'; import {default_gitops_operations} from './operations_defaults.js'; -import {GITOPS_MAX_ITERATIONS_DEFAULT} from './gitops_constants.js'; +import { + GITOPS_MAX_ITERATIONS_DEFAULT, + GITOPS_NPM_WAIT_TIMEOUT_DEFAULT, +} from './gitops_constants.js'; import {install_with_cache_healing} from './npm_install_helpers.js'; /* eslint-disable no-await-in-loop */ @@ -92,7 +95,9 @@ export const publish_repos = async ( while (!converged && iteration < GITOPS_MAX_ITERATIONS_DEFAULT) { iteration++; - log?.info(st('cyan', `\n🚀 Publishing iteration ${iteration}/${GITOPS_MAX_ITERATIONS_DEFAULT}...\n`)); + log?.info( + st('cyan', `\n🚀 Publishing iteration ${iteration}/${GITOPS_MAX_ITERATIONS_DEFAULT}...\n`), + ); // Track if any packages were published in this iteration let published_in_iteration = false; @@ -157,7 +162,7 @@ export const publish_repos = async ( max_attempts: 30, initial_delay: 1000, max_delay: 60000, - timeout: options.max_wait || 600000, // 10 minutes default + timeout: options.max_wait ?? GITOPS_NPM_WAIT_TIMEOUT_DEFAULT, }, log, }); diff --git a/src/lib/repo_ops.ts b/src/lib/repo_ops.ts index 2483dc75..17e513fb 100644 --- a/src/lib/repo_ops.ts +++ b/src/lib/repo_ops.ts @@ -15,6 +15,7 @@ import {join, resolve, dirname} from 'node:path'; import {load_gitops_config} from './gitops_config.js'; import {DEFAULT_REPOS_DIR} from './paths.js'; +import {GITOPS_CONFIG_PATH_DEFAULT} from './gitops_constants.js'; /** Default directories to exclude from file walking */ export const DEFAULT_EXCLUDE_DIRS = [ @@ -84,7 +85,7 @@ export interface RepoPath { * @returns Array of repo info with name, path, and url */ export const get_repo_paths = async (config_path?: string): Promise> => { - const resolved_config_path = resolve(config_path ?? 'gitops.config.ts'); + const resolved_config_path = resolve(config_path ?? GITOPS_CONFIG_PATH_DEFAULT); const config = await load_gitops_config(resolved_config_path); if (!config) { diff --git a/src/routes/library.ts b/src/routes/library.ts index 3fc903e2..c6d4a245 100644 --- a/src/routes/library.ts +++ b/src/routes/library.ts @@ -368,20 +368,6 @@ export const library_json: LibraryJson = { dependencies: ['version_utils.ts'], dependents: ['operations_defaults.ts'], }, - { - path: 'constants.ts', - declarations: [ - { - name: 'MAX_ITERATIONS', - kind: 'variable', - doc_comment: - 'Maximum number of iterations for fixed-point iteration during publishing.\nUsed in both plan generation and actual publishing to resolve transitive dependency cascades.\n\nIn practice, most repos converge in 2-3 iterations.\nDeep dependency chains may require more iterations.', - source_line: 8, - type_signature: '10', - }, - ], - dependents: ['multi_repo_publisher.ts', 'publishing_plan.ts'], - }, { path: 'dependency_graph.ts', declarations: [ @@ -1405,6 +1391,7 @@ export const library_json: LibraryJson = { declarations: [], dependencies: [ 'dependency_graph.ts', + 'gitops_constants.ts', 'gitops_task_helpers.ts', 'graph_validation.ts', 'log_helpers.ts', @@ -1580,15 +1567,71 @@ export const library_json: LibraryJson = { dependencies: ['paths.ts'], dependents: ['gitops_task_helpers.ts', 'repo_ops.ts'], }, + { + path: 'gitops_constants.ts', + declarations: [ + { + name: 'GITOPS_MAX_ITERATIONS_DEFAULT', + kind: 'variable', + doc_comment: + 'Maximum number of iterations for fixed-point iteration during publishing.\nUsed in both plan generation and actual publishing to resolve transitive dependency cascades.\n\nIn practice, most repos converge in 2-3 iterations.\nDeep dependency chains may require more iterations.', + source_line: 14, + type_signature: '10', + }, + { + name: 'GITOPS_CONFIG_PATH_DEFAULT', + kind: 'variable', + doc_comment: 'Default path to the gitops configuration file.', + source_line: 19, + type_signature: '"gitops.config.ts"', + }, + { + name: 'GITOPS_CONCURRENCY_DEFAULT', + kind: 'variable', + doc_comment: + 'Default number of repos to process concurrently during parallel operations.', + source_line: 24, + type_signature: '5', + }, + { + name: 'GITOPS_NPM_WAIT_TIMEOUT_DEFAULT', + kind: 'variable', + doc_comment: + "Default timeout in milliseconds for waiting on NPM package propagation (10 minutes).\nNPM's CDN uses eventual consistency, so published packages may not be immediately available.", + source_line: 30, + type_signature: '600000', + }, + ], + module_comment: + 'Shared constants for gitops tasks and operations.\n\nNaming convention: GITOPS_{NAME}_DEFAULT for user-facing defaults.', + dependents: [ + 'gitops_analyze.task.ts', + 'gitops_plan.task.ts', + 'gitops_publish.task.ts', + 'gitops_run.task.ts', + 'gitops_sync.task.ts', + 'gitops_validate.task.ts', + 'local_repo.ts', + 'multi_repo_publisher.ts', + 'publishing_plan.ts', + 'repo_ops.ts', + ], + }, { path: 'gitops_plan.task.ts', declarations: [], - dependencies: ['gitops_task_helpers.ts', 'output_helpers.ts', 'publishing_plan.ts'], + dependencies: [ + 'gitops_constants.ts', + 'gitops_task_helpers.ts', + 'output_helpers.ts', + 'publishing_plan.ts', + ], }, { path: 'gitops_publish.task.ts', declarations: [], dependencies: [ + 'gitops_constants.ts', 'gitops_task_helpers.ts', 'multi_repo_publisher.ts', 'output_helpers.ts', @@ -1601,24 +1644,29 @@ export const library_json: LibraryJson = { { name: 'Args', kind: 'type', - source_line: 10, + source_line: 11, type_signature: - 'ZodObject<{ command: ZodString; path: ZodDefault; concurrency: ZodDefault; format: ZodDefault>; }, $strict>', + 'ZodObject<{ command: ZodString; config: ZodDefault; concurrency: ZodDefault; format: ZodDefault>; }, $strict>', }, { name: 'task', kind: 'variable', - source_line: 40, + source_line: 38, type_signature: - 'Task<{ command: string; path: string; concurrency: number; format: "json" | "text"; }, ZodType>, unknown>', + 'Task<{ command: string; config: string; concurrency: number; format: "json" | "text"; }, ZodType>, unknown>', }, ], - dependencies: ['repo_ops.ts'], + dependencies: ['gitops_constants.ts', 'repo_ops.ts'], }, { path: 'gitops_sync.task.ts', declarations: [], - dependencies: ['fetch_repo_data.ts', 'fs_fetch_value_cache.ts', 'gitops_task_helpers.ts'], + dependencies: [ + 'fetch_repo_data.ts', + 'fs_fetch_value_cache.ts', + 'gitops_constants.ts', + 'gitops_task_helpers.ts', + ], }, { path: 'gitops_task_helpers.ts', @@ -1630,7 +1678,7 @@ export const library_json: LibraryJson = { type_signature: 'GetGitopsReadyOptions', properties: [ { - name: 'path', + name: 'config', kind: 'variable', type_signature: 'string', }, @@ -1659,20 +1707,30 @@ export const library_json: LibraryJson = { kind: 'variable', type_signature: 'NpmOperations', }, + { + name: 'parallel', + kind: 'variable', + type_signature: 'boolean', + }, + { + name: 'concurrency', + kind: 'variable', + type_signature: 'number', + }, ], }, { name: 'get_gitops_ready', kind: 'function', doc_comment: - 'Central initialization function for all gitops tasks.\n\nInitialization sequence:\n1. Loads and normalizes config from `gitops.config.ts`\n2. Resolves local repo paths (creates missing with `--download`)\n3. Switches branches and pulls latest changes\n4. Auto-installs deps if package.json changed during pull\n\nPriority for path resolution:\n- `dir` argument (explicit override)\n- Config `repos_dir` setting\n- `DEFAULT_REPOS_DIR` constant', + 'Central initialization function for all gitops tasks.\n\nInitialization sequence:\n1. Loads and normalizes config from `gitops.config.ts`\n2. Resolves local repo paths (creates missing with `--download`)\n3. Switches branches and pulls latest changes (in parallel by default)\n4. Auto-installs deps if package.json changed during pull\n\nPriority for path resolution:\n- `dir` argument (explicit override)\n- Config `repos_dir` setting\n- `DEFAULT_REPOS_DIR` constant', throws: [ { type: 'if', description: 'config loading or repo resolution fails', }, ], - source_line: 57, + source_line: 61, type_signature: '(options: GetGitopsReadyOptions): Promise<{ config_path: string; repos_dir: string; gitops_config: GitopsConfig; local_repos: LocalRepo[]; }>', return_type: @@ -1688,11 +1746,11 @@ export const library_json: LibraryJson = { { name: 'ResolveGitopsPathsOptions', kind: 'type', - source_line: 96, + source_line: 107, type_signature: 'ResolveGitopsPathsOptions', properties: [ { - name: 'path', + name: 'config', kind: 'variable', type_signature: 'string', }, @@ -1711,7 +1769,7 @@ export const library_json: LibraryJson = { { name: 'resolve_gitops_paths', kind: 'function', - source_line: 102, + source_line: 113, type_signature: '(options: ResolveGitopsPathsOptions): { config_path: string; repos_dir: string; }', return_type: '{ config_path: string; repos_dir: string; }', @@ -1725,7 +1783,7 @@ export const library_json: LibraryJson = { { name: 'import_gitops_config', kind: 'function', - source_line: 120, + source_line: 131, type_signature: '(config_path: string): Promise', return_type: 'Promise', parameters: [ @@ -1757,6 +1815,7 @@ export const library_json: LibraryJson = { declarations: [], dependencies: [ 'dependency_graph.ts', + 'gitops_constants.ts', 'gitops_task_helpers.ts', 'graph_validation.ts', 'log_helpers.ts', @@ -1848,7 +1907,7 @@ export const library_json: LibraryJson = { kind: 'type', doc_comment: 'Fully loaded local repo with Library and extracted dependency data.\nDoes not extend LocalRepoPath - Library is source of truth for name/repo_url/etc.', - source_line: 20, + source_line: 21, type_signature: 'LocalRepo', properties: [ { @@ -1898,7 +1957,7 @@ export const library_json: LibraryJson = { kind: 'type', doc_comment: 'A repo that has been located on the filesystem (path exists).\nUsed before loading - just filesystem/git concerns.', - source_line: 35, + source_line: 36, type_signature: 'LocalRepoPath', properties: [ { @@ -1937,7 +1996,7 @@ export const library_json: LibraryJson = { name: 'LocalRepoMissing', kind: 'type', doc_comment: 'A repo that is missing from the filesystem (needs cloning).', - source_line: 47, + source_line: 48, type_signature: 'LocalRepoMissing', properties: [ { @@ -1979,7 +2038,7 @@ export const library_json: LibraryJson = { 'workspace dirty, branch switch fails, install fails, or library.ts missing', }, ], - source_line: 72, + source_line: 73, type_signature: '({ local_repo_path, log: _log, git_ops, npm_ops, }: { local_repo_path: LocalRepoPath; log?: Logger | undefined; git_ops?: GitOperations | undefined; npm_ops?: NpmOperations | undefined; }): Promise<...>', return_type: 'Promise', @@ -1993,7 +2052,7 @@ export const library_json: LibraryJson = { { name: 'local_repos_ensure', kind: 'function', - source_line: 229, + source_line: 230, type_signature: '({ resolved_config, repos_dir, gitops_config, download, log, npm_ops, }: { resolved_config: ResolvedGitopsConfig; repos_dir: string; gitops_config: GitopsConfig; download: boolean; log?: Logger | undefined; npm_ops?: NpmOperations | undefined; }): Promise<...>', return_type: 'Promise', @@ -2007,7 +2066,7 @@ export const library_json: LibraryJson = { { name: 'local_repos_load', kind: 'function', - source_line: 279, + source_line: 280, type_signature: '({ local_repo_paths, log, git_ops, npm_ops, parallel, concurrency, }: { local_repo_paths: LocalRepoPath[]; log?: Logger | undefined; git_ops?: GitOperations | undefined; npm_ops?: NpmOperations | undefined; parallel?: boolean | undefined; concurrency?: number | undefined; }): Promise<...>', return_type: 'Promise', @@ -2021,7 +2080,7 @@ export const library_json: LibraryJson = { { name: 'local_repo_locate', kind: 'function', - source_line: 338, + source_line: 339, type_signature: '({ repo_config, repos_dir, }: { repo_config: GitopsRepoConfig; repos_dir: string; }): LocalRepoPath | LocalRepoMissing', return_type: 'LocalRepoPath | LocalRepoMissing', @@ -2033,7 +2092,7 @@ export const library_json: LibraryJson = { ], }, ], - dependencies: ['operations_defaults.ts'], + dependencies: ['gitops_constants.ts', 'operations_defaults.ts'], dependents: ['gitops_task_helpers.ts', 'resolved_gitops_config.ts'], }, { @@ -2290,7 +2349,7 @@ export const library_json: LibraryJson = { { name: 'PublishingOptions', kind: 'type', - source_line: 18, + source_line: 21, type_signature: 'PublishingOptions', properties: [ { @@ -2338,7 +2397,7 @@ export const library_json: LibraryJson = { { name: 'PublishedVersion', kind: 'type', - source_line: 29, + source_line: 32, type_signature: 'PublishedVersion', properties: [ { @@ -2381,7 +2440,7 @@ export const library_json: LibraryJson = { { name: 'PublishingResult', kind: 'type', - source_line: 39, + source_line: 42, type_signature: 'PublishingResult', properties: [ { @@ -2409,7 +2468,7 @@ export const library_json: LibraryJson = { { name: 'publish_repos', kind: 'function', - source_line: 46, + source_line: 49, type_signature: '(repos: LocalRepo[], options: PublishingOptions): Promise', return_type: 'Promise', @@ -2426,8 +2485,8 @@ export const library_json: LibraryJson = { }, ], dependencies: [ - 'constants.ts', 'dependency_updater.ts', + 'gitops_constants.ts', 'graph_validation.ts', 'npm_install_helpers.ts', 'operations_defaults.ts', @@ -3829,7 +3888,7 @@ export const library_json: LibraryJson = { }, ], dependencies: [ - 'constants.ts', + 'gitops_constants.ts', 'graph_validation.ts', 'operations_defaults.ts', 'publishing_plan_helpers.ts', @@ -3895,7 +3954,7 @@ export const library_json: LibraryJson = { kind: 'function', doc_comment: 'Walk files in a directory, respecting common exclusions.\nYields absolute paths to files (and optionally directories).', - source_line: 155, + source_line: 156, type_signature: '(dir: string, options?: WalkOptions | undefined): AsyncGenerator', return_type: 'AsyncGenerator', @@ -3917,7 +3976,7 @@ export const library_json: LibraryJson = { name: 'DEFAULT_EXCLUDE_DIRS', kind: 'variable', doc_comment: 'Default directories to exclude from file walking', - source_line: 20, + source_line: 21, type_signature: 'readonly ["node_modules", ".git", ".gro", ".svelte-kit", ".deno", ".vscode", ".idea", "dist", "build", "coverage", ".cache", ".turbo"]', }, @@ -3925,14 +3984,14 @@ export const library_json: LibraryJson = { name: 'DEFAULT_EXCLUDE_EXTENSIONS', kind: 'variable', doc_comment: 'Default binary/non-text extensions to exclude from content processing', - source_line: 36, + source_line: 37, type_signature: 'readonly [".png", ".jpg", ".jpeg", ".gif", ".svg", ".ico", ".webp", ".woff", ".woff2", ".ttf", ".eot", ".mp4", ".webm", ".mp3", ".wav", ".ogg", ".zip", ".tar", ".gz", ".lock", ".pdf"]', }, { name: 'WalkOptions', kind: 'type', - source_line: 60, + source_line: 61, type_signature: 'WalkOptions', properties: [ { @@ -3970,7 +4029,7 @@ export const library_json: LibraryJson = { { name: 'RepoPath', kind: 'type', - source_line: 73, + source_line: 74, type_signature: 'RepoPath', properties: [ { @@ -3995,7 +4054,7 @@ export const library_json: LibraryJson = { kind: 'function', doc_comment: 'Get repo paths from gitops config without full git sync.\nLighter weight than `get_gitops_ready()` - just resolves paths.', - source_line: 86, + source_line: 87, type_signature: '(config_path?: string | undefined): Promise', return_type: 'Promise', return_description: 'Array of repo info with name, path, and url', @@ -4012,7 +4071,7 @@ export const library_json: LibraryJson = { name: 'should_exclude_path', kind: 'function', doc_comment: 'Check if a path should be excluded based on options.', - source_line: 119, + source_line: 120, type_signature: '(file_path: string, options?: WalkOptions | undefined): boolean', return_type: 'boolean', parameters: [ @@ -4032,7 +4091,7 @@ export const library_json: LibraryJson = { kind: 'function', doc_comment: 'Collect all files from walk_repo_files into an array.\nConvenience function for when you need all paths upfront.', - source_line: 204, + source_line: 205, type_signature: '(dir: string, options?: WalkOptions | undefined): Promise', return_type: 'Promise', parameters: [ @@ -4050,7 +4109,7 @@ export const library_json: LibraryJson = { ], module_comment: 'Generic repository operations for scripts that work across repos.\n\nProvides lightweight utilities for:\n- Getting repo paths from gitops config (without full git sync)\n- Walking files in repos with sensible exclusions\n- Common exclusion patterns for node/svelte projects\n\nFor full git sync/clone functionality, use `get_gitops_ready()` from gitops_task_helpers.', - dependencies: ['gitops_config.ts', 'paths.ts'], + dependencies: ['gitops_config.ts', 'gitops_constants.ts', 'paths.ts'], dependents: ['gitops_run.task.ts'], }, { From 5ba96b393e7f5ff0b94cde4bc78d346bedb35427 Mon Sep 17 00:00:00 2001 From: Ryan Atkinson Date: Tue, 16 Dec 2025 11:10:34 -0500 Subject: [PATCH 7/7] wip --- .changeset/quick-houses-guess.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/quick-houses-guess.md diff --git a/.changeset/quick-houses-guess.md b/.changeset/quick-houses-guess.md new file mode 100644 index 00000000..31105fb0 --- /dev/null +++ b/.changeset/quick-houses-guess.md @@ -0,0 +1,5 @@ +--- +'@fuzdev/fuz_gitops': minor +--- + +add `gitops_run` task, tweak some interfaces \ No newline at end of file