diff --git a/.changeset/quick-houses-guess.md b/.changeset/quick-houses-guess.md new file mode 100644 index 00000000..31105fb0 --- /dev/null +++ b/.changeset/quick-houses-guess.md @@ -0,0 +1,5 @@ +--- +'@fuzdev/fuz_gitops': minor +--- + +add `gitops_run` task, tweak some interfaces \ No newline at end of file diff --git a/CLAUDE.md b/CLAUDE.md index 56c72ad7..ef53c067 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -314,6 +314,11 @@ gro gitops_sync # sync repos and update local data gro gitops_sync --download # clone missing repos gro gitops_sync --check # verify repos are ready without fetching data +# Run commands across repos +gro gitops_run "npm test" # run command in all repos (parallel, concurrency: 5) +gro gitops_run "npm audit" --concurrency 3 # limit parallelism +gro gitops_run "gro check" --format json # JSON output for scripting + # Publishing gro gitops_validate # validate configuration (runs analyze, plan, and dry run) gro gitops_analyze # analyze dependencies and changesets @@ -355,6 +360,15 @@ gro test src/test/fixtures/check # validate gitops commands against fixture - Switches branches and pulls latest changes - Installs dependencies if package.json changed - Verify repos ready without fetching (with `--check`) + - Runs in parallel (concurrency: 5 by default) + +**Command Execution (User-Defined Side Effects):** + +- `gro gitops_run ""` - Run shell command across all repos + - Parallel execution (concurrency: 5 by default) + - Continue-on-error behavior + - Structured output (text or JSON) + - Use for testing, auditing, batch operations **Publishing (Git & NPM Side Effects):** @@ -393,7 +407,7 @@ For packages you control, use `>=` instead of `^` for peer dependencies: ```json "peerDependencies": { - "@fuzdev/fuz_util": ">=0.38.0", // controlled package - use >= + "@fuzdev/fuz_util": ">=0.38.0", // controlled package - use >= "@ryanatkn/gro": ">=0.174.0", // controlled package - use >= "@sveltejs/kit": "^2", // third-party - use ^ "svelte": "^5" // third-party - use ^ @@ -464,8 +478,8 @@ Uses vitest with **zero mocks** - all tests use the operations pattern for dependency injection (see above). ```bash -gro test # run all tests -gro test version_utils # run specific test file +gro test # run all tests +gro test version_utils # run specific test file gro test src/test/fixtures/check # validate command output fixtures ``` diff --git a/README.md b/README.md index 7544f4fa..f9b27a0e 100644 --- a/README.md +++ b/README.md @@ -71,6 +71,21 @@ See [CLAUDE.md](CLAUDE.md#architecture) for detailed documentation. ## Quick Start +### Running commands across repos + +```bash +gro gitops_run "npm test" # run tests in all repos (parallel, concurrency: 5) +gro gitops_run "npm audit" --concurrency 3 # limit parallelism +gro gitops_run "git status" --format json # JSON output for scripting +``` + +**Features:** + +- Parallel execution with configurable concurrency (default: 5) +- Continue-on-error behavior (shows all results) +- Structured output formats (text or JSON) +- Uses lightweight repo path resolution (no full sync needed) + ### Syncing repo metadata ```bash diff --git a/src/lib/constants.ts b/src/lib/constants.ts deleted file mode 100644 index 9b95d751..00000000 --- a/src/lib/constants.ts +++ /dev/null @@ -1,8 +0,0 @@ -/** - * Maximum number of iterations for fixed-point iteration during publishing. - * Used in both plan generation and actual publishing to resolve transitive dependency cascades. - * - * In practice, most repos converge in 2-3 iterations. - * Deep dependency chains may require more iterations. - */ -export const MAX_ITERATIONS = 10; diff --git a/src/lib/gitops_analyze.task.ts b/src/lib/gitops_analyze.task.ts index f5b2b46e..1d6f6a59 100644 --- a/src/lib/gitops_analyze.task.ts +++ b/src/lib/gitops_analyze.task.ts @@ -13,16 +13,17 @@ import { format_production_cycles, } from './log_helpers.js'; import {format_and_output, type OutputFormatters} from './output_helpers.js'; +import {GITOPS_CONFIG_PATH_DEFAULT} from './gitops_constants.js'; /** @nodocs */ export const Args = z.strictObject({ - path: z + config: z .string() .meta({description: 'path to the gitops config file, absolute or relative to the cwd'}) - .default('gitops.config.ts'), + .default(GITOPS_CONFIG_PATH_DEFAULT), dir: z .string() - .meta({description: 'path containing the repos, defaults to the parent of the `path` dir'}) + .meta({description: 'path containing the repos, defaults to the parent of the config dir'}) .optional(), format: z .enum(['stdout', 'json', 'markdown']) @@ -37,10 +38,10 @@ export const task: Task = { Args, summary: 'analyze dependency structure and relationships across repos', run: async ({args, log}) => { - const {path, dir, format, outfile} = args; + const {config, dir, format, outfile} = args; // Get repos ready (without downloading) - const {local_repos} = await get_gitops_ready({path, dir, download: false, log}); + const {local_repos} = await get_gitops_ready({config, dir, download: false, log}); // Build dependency graph and validate (but don't throw on cycles for analyze) const {graph, publishing_order: order} = validate_dependency_graph(local_repos, { diff --git a/src/lib/gitops_constants.ts b/src/lib/gitops_constants.ts new file mode 100644 index 00000000..57e744a0 --- /dev/null +++ b/src/lib/gitops_constants.ts @@ -0,0 +1,30 @@ +/** + * Shared constants for gitops tasks and operations. + * + * Naming convention: GITOPS_{NAME}_DEFAULT for user-facing defaults. + */ + +/** + * Maximum number of iterations for fixed-point iteration during publishing. + * Used in both plan generation and actual publishing to resolve transitive dependency cascades. + * + * In practice, most repos converge in 2-3 iterations. + * Deep dependency chains may require more iterations. + */ +export const GITOPS_MAX_ITERATIONS_DEFAULT = 10; + +/** + * Default path to the gitops configuration file. + */ +export const GITOPS_CONFIG_PATH_DEFAULT = 'gitops.config.ts'; + +/** + * Default number of repos to process concurrently during parallel operations. + */ +export const GITOPS_CONCURRENCY_DEFAULT = 5; + +/** + * Default timeout in milliseconds for waiting on NPM package propagation (10 minutes). + * NPM's CDN uses eventual consistency, so published packages may not be immediately available. + */ +export const GITOPS_NPM_WAIT_TIMEOUT_DEFAULT = 600_000; // 10 minutes diff --git a/src/lib/gitops_plan.task.ts b/src/lib/gitops_plan.task.ts index 57869892..5be673aa 100644 --- a/src/lib/gitops_plan.task.ts +++ b/src/lib/gitops_plan.task.ts @@ -10,16 +10,17 @@ import { type LogPlanOptions, } from './publishing_plan.js'; import {format_and_output, type OutputFormatters} from './output_helpers.js'; +import {GITOPS_CONFIG_PATH_DEFAULT} from './gitops_constants.js'; /** @nodocs */ export const Args = z.strictObject({ - path: z + config: z .string() .meta({description: 'path to the gitops config file, absolute or relative to the cwd'}) - .default('gitops.config.ts'), + .default(GITOPS_CONFIG_PATH_DEFAULT), dir: z .string() - .meta({description: 'path containing the repos, defaults to the parent of the `path` dir'}) + .meta({description: 'path containing the repos, defaults to the parent of the config dir'}) .optional(), format: z .enum(['stdout', 'json', 'markdown']) @@ -37,7 +38,7 @@ export type Args = z.infer; * Usage: * gro gitops_plan * gro gitops_plan --dir ../repos - * gro gitops_plan --path ./custom.config.ts + * gro gitops_plan --config ./custom.config.ts * * @nodocs */ @@ -45,13 +46,13 @@ export const task: Task = { summary: 'generate a publishing plan based on changesets', Args, run: async ({args, log}): Promise => { - const {dir, path, format, outfile, verbose} = args; + const {dir, config, format, outfile, verbose} = args; log.info(st('cyan', 'Generating multi-repo publishing plan...')); // Load local repos const {local_repos} = await get_gitops_ready({ - path, + config, dir, download: false, // Don't download if missing log, diff --git a/src/lib/gitops_publish.task.ts b/src/lib/gitops_publish.task.ts index f164cb14..f437bc13 100644 --- a/src/lib/gitops_publish.task.ts +++ b/src/lib/gitops_publish.task.ts @@ -11,16 +11,17 @@ import { } from './multi_repo_publisher.js'; import {generate_publishing_plan, log_publishing_plan} from './publishing_plan.js'; import {format_and_output, type OutputFormatters} from './output_helpers.js'; +import {GITOPS_CONFIG_PATH_DEFAULT, GITOPS_NPM_WAIT_TIMEOUT_DEFAULT} from './gitops_constants.js'; /** @nodocs */ export const Args = z.strictObject({ - path: z + config: z .string() .meta({description: 'path to the gitops config file, absolute or relative to the cwd'}) - .default('gitops.config.ts'), + .default(GITOPS_CONFIG_PATH_DEFAULT), dir: z .string() - .meta({description: 'path containing the repos, defaults to the parent of the `path` dir'}) + .meta({description: 'path containing the repos, defaults to the parent of the config dir'}) .optional(), peer_strategy: z .enum(['exact', 'caret', 'tilde']) @@ -43,7 +44,7 @@ export const Args = z.strictObject({ max_wait: z .number() .meta({description: 'max time to wait for npm propagation in ms'}) - .default(600000), // 10 minutes + .default(GITOPS_NPM_WAIT_TIMEOUT_DEFAULT), skip_install: z .boolean() .meta({description: 'skip npm install after dependency updates'}) @@ -59,7 +60,7 @@ export const task: Task = { Args, run: async ({args, log}): Promise => { const { - path, + config, dir, peer_strategy, dry_run, @@ -74,7 +75,7 @@ export const task: Task = { // Load repos const {local_repos: repos} = await get_gitops_ready({ - path, + config, dir, download: false, // Don't download if missing log, diff --git a/src/lib/gitops_run.task.ts b/src/lib/gitops_run.task.ts new file mode 100644 index 00000000..aad95cbb --- /dev/null +++ b/src/lib/gitops_run.task.ts @@ -0,0 +1,218 @@ +import {TaskError, type Task} from '@ryanatkn/gro'; +import {z} from 'zod'; +import {map_concurrent_settled} from '@fuzdev/fuz_util/async.js'; +import {spawn_out} from '@fuzdev/fuz_util/process.js'; +import {styleText as st} from 'node:util'; +import {resolve} from 'node:path'; + +import {get_repo_paths} from './repo_ops.js'; +import {GITOPS_CONCURRENCY_DEFAULT, GITOPS_CONFIG_PATH_DEFAULT} from './gitops_constants.js'; + +export const Args = z.strictObject({ + command: z.string().meta({description: 'shell command to run in each repo'}), + config: z + .string() + .meta({description: 'path to the gitops config file'}) + .default(GITOPS_CONFIG_PATH_DEFAULT), + concurrency: z + .number() + .int() + .min(1) + .meta({description: 'maximum number of repos to run in parallel'}) + .default(GITOPS_CONCURRENCY_DEFAULT), + format: z.enum(['text', 'json']).meta({description: 'output format'}).default('text'), +}); +export type Args = z.infer; + +interface RunResult { + repo_name: string; + repo_dir: string; + status: 'success' | 'failure'; + exit_code: number; + stdout: string; + stderr: string; + duration_ms: number; + error?: string; +} + +export const task: Task = { + Args, + summary: 'run a shell command across all repos in parallel', + run: async ({args, log}) => { + const {command, config, concurrency, format} = args; + + // Get repo paths (lightweight, no library.ts loading needed) + const config_path = resolve(config); + const repos = await get_repo_paths(config_path); + + if (repos.length === 0) { + throw new TaskError('No repos found in config'); + } + + log.info( + `Running ${st('cyan', command)} across ${repos.length} repos (concurrency: ${concurrency})`, + ); + + const start_time = performance.now(); + + // Run command in parallel across all repos + const results = await map_concurrent_settled( + repos, + async (repo) => { + const repo_start = performance.now(); + const repo_name = repo.name; + const repo_dir = repo.path; + + try { + // Parse command into cmd + args for spawn + // For now, we use shell mode to support pipes/redirects/etc + const spawned = await spawn_out('sh', ['-c', command], { + cwd: repo_dir, + }); + + const duration_ms = performance.now() - repo_start; + const success = spawned.result.ok; + + const result: RunResult = { + repo_name, + repo_dir, + status: success ? 'success' : 'failure', + exit_code: spawned.result.code ?? 0, + stdout: spawned.stdout || '', + stderr: spawned.stderr || '', + duration_ms, + }; + + return result; + } catch (error) { + const duration_ms = performance.now() - repo_start; + return { + repo_name, + repo_dir, + status: 'failure' as const, + exit_code: -1, + stdout: '', + stderr: '', + duration_ms, + error: String(error), + }; + } + }, + concurrency, + ); + + const total_duration_ms = performance.now() - start_time; + + // Process results + const successes: Array = []; + const failures: Array = []; + + for (const result of results) { + if (result.status === 'fulfilled') { + const run_result = result.value; + if (run_result.status === 'success') { + successes.push(run_result); + } else { + failures.push(run_result); + } + } else { + // This shouldn't happen since we catch errors in the task fn + // but handle it anyway + failures.push({ + repo_name: 'unknown', + repo_dir: 'unknown', + status: 'failure', + exit_code: -1, + stdout: '', + stderr: '', + duration_ms: 0, + error: String(result.reason), + }); + } + } + + // Output results based on format + if (format === 'json') { + const json_output = { + command, + concurrency, + repos: [...successes, ...failures], + summary: { + total: repos.length, + success: successes.length, + failure: failures.length, + duration_ms: Math.round(total_duration_ms), + }, + }; + // eslint-disable-next-line no-console + console.log(JSON.stringify(json_output, null, 2)); + } else { + // Text format + log.info(''); // blank line + + // Show successes + if (successes.length > 0) { + log.info(st('green', `✓ ${successes.length} succeeded:`)); + for (const result of successes) { + const duration = `${Math.round(result.duration_ms)}ms`; + log.info(st('gray', ` ${result.repo_name} ${st('blue', `(${duration})`)}`)); + } + } + + // Show failures with details + if (failures.length > 0) { + log.info(''); // blank line + log.error(st('red', `✗ ${failures.length} failed:`)); + for (const result of failures) { + const duration = `${Math.round(result.duration_ms)}ms`; + log.error(st('gray', ` ${result.repo_name} ${st('blue', `(${duration})`)}`)); + + if (result.error) { + log.error(st('gray', ` Error: ${result.error}`)); + } else if (result.exit_code !== 0) { + log.error(st('gray', ` Exit code: ${result.exit_code}`)); + } + + if (result.stderr) { + // Show first few lines of stderr + const stderr_lines = result.stderr.trim().split('\n'); + const preview_lines = stderr_lines.slice(0, 3); + for (const line of preview_lines) { + log.error(st('gray', ` ${line}`)); + } + if (stderr_lines.length > 3) { + log.error(st('gray', ` ... (${stderr_lines.length - 3} more lines)`)); + } + } + } + } + + // Summary + log.info(''); // blank line + const total = repos.length; + const success_rate = ((successes.length / total) * 100).toFixed(0); + const duration = `${Math.round(total_duration_ms)}ms`; + + if (failures.length === 0) { + log.info( + st( + 'green', + `✓ All ${total} repos succeeded in ${duration} (${success_rate}% success rate)`, + ), + ); + } else { + log.info( + st( + 'yellow', + `⚠ ${successes.length}/${total} repos succeeded in ${duration} (${success_rate}% success rate)`, + ), + ); + } + } + + // Exit with error if any failures (so CI fails) + if (failures.length > 0) { + throw new TaskError(`${failures.length} repos failed`); + } + }, +}; diff --git a/src/lib/gitops_sync.task.ts b/src/lib/gitops_sync.task.ts index b405663b..03b6dff6 100644 --- a/src/lib/gitops_sync.task.ts +++ b/src/lib/gitops_sync.task.ts @@ -11,18 +11,19 @@ import {existsSync} from 'node:fs'; import {fetch_repo_data} from './fetch_repo_data.js'; import {create_fs_fetch_value_cache} from './fs_fetch_value_cache.js'; import {get_gitops_ready} from './gitops_task_helpers.js'; +import {GITOPS_CONFIG_PATH_DEFAULT} from './gitops_constants.js'; // TODO add flag to ignore or invalidate cache -- no-cache? clean? /** @nodocs */ export const Args = z.strictObject({ - path: z + config: z .string() .meta({description: 'path to the gitops config file, absolute or relative to the cwd'}) - .default('gitops.config.ts'), + .default(GITOPS_CONFIG_PATH_DEFAULT), dir: z .string() - .meta({description: 'path containing the repos, defaults to the parent of the `path` dir'}) + .meta({description: 'path containing the repos, defaults to the parent of the config dir'}) .optional(), outdir: z .string() @@ -45,9 +46,9 @@ export const task: Task = { Args, summary: 'syncs local repos and generates UI data from repo metadata', run: async ({args, log, svelte_config, invoke_task}) => { - const {path, dir, outdir = svelte_config.routes_path, download, check} = args; + const {config, dir, outdir = svelte_config.routes_path, download, check} = args; - const {local_repos} = await get_gitops_ready({path, dir, download, log}); + const {local_repos} = await get_gitops_ready({config, dir, download, log}); const outfile = resolve(outdir, 'repos.ts'); diff --git a/src/lib/gitops_task_helpers.ts b/src/lib/gitops_task_helpers.ts index 9f6c7e3b..21fd6ea0 100644 --- a/src/lib/gitops_task_helpers.ts +++ b/src/lib/gitops_task_helpers.ts @@ -27,12 +27,14 @@ import {DEFAULT_REPOS_DIR} from './paths.js'; import type {GitOperations, NpmOperations} from './operations.js'; export interface GetGitopsReadyOptions { - path: string; + config: string; dir?: string; download: boolean; log?: Logger; git_ops?: GitOperations; npm_ops?: NpmOperations; + parallel?: boolean; + concurrency?: number; } /** @@ -41,7 +43,7 @@ export interface GetGitopsReadyOptions { * Initialization sequence: * 1. Loads and normalizes config from `gitops.config.ts` * 2. Resolves local repo paths (creates missing with `--download`) - * 3. Switches branches and pulls latest changes + * 3. Switches branches and pulls latest changes (in parallel by default) * 4. Auto-installs deps if package.json changed during pull * * Priority for path resolution: @@ -51,6 +53,8 @@ export interface GetGitopsReadyOptions { * * @param options.git_ops for testing (defaults to real git operations) * @param options.npm_ops for testing (defaults to real npm operations) + * @param options.parallel whether to load repos in parallel (default: true) + * @param options.concurrency max concurrent repo loads (default: 5) * @returns initialized config and fully loaded repos ready for operations * @throws {TaskError} if config loading or repo resolution fails */ @@ -62,13 +66,13 @@ export const get_gitops_ready = async ( gitops_config: GitopsConfig; local_repos: Array; }> => { - const {path, dir, download, log, git_ops, npm_ops} = options; - const config_path = resolve(path); + const {config, dir, download, log, git_ops, npm_ops, parallel, concurrency} = options; + const config_path = resolve(config); const gitops_config = await import_gitops_config(config_path); // Priority: explicit dir arg → config repos_dir → default (two dirs up from config) const repos_dir = resolve_gitops_paths({ - path, + config, dir, config_repos_dir: gitops_config.repos_dir, }).repos_dir; @@ -88,13 +92,20 @@ export const get_gitops_ready = async ( npm_ops, }); - const local_repos = await local_repos_load({local_repo_paths, log, git_ops, npm_ops}); + const local_repos = await local_repos_load({ + local_repo_paths, + log, + git_ops, + npm_ops, + parallel, + concurrency, + }); return {config_path, repos_dir, gitops_config, local_repos}; }; export interface ResolveGitopsPathsOptions { - path: string; + config: string; dir?: string; config_repos_dir?: string; } @@ -102,8 +113,8 @@ export interface ResolveGitopsPathsOptions { export const resolve_gitops_paths = ( options: ResolveGitopsPathsOptions, ): {config_path: string; repos_dir: string} => { - const {path, dir, config_repos_dir} = options; - const config_path = resolve(path); + const {config, dir, config_repos_dir} = options; + const config_path = resolve(config); const config_dir = dirname(config_path); // Priority: explicit dir arg → config repos_dir → default (parent of config dir) diff --git a/src/lib/gitops_validate.task.ts b/src/lib/gitops_validate.task.ts index f8839923..4c46cd05 100644 --- a/src/lib/gitops_validate.task.ts +++ b/src/lib/gitops_validate.task.ts @@ -8,16 +8,17 @@ import {DependencyGraphBuilder} from './dependency_graph.js'; import {generate_publishing_plan, log_publishing_plan} from './publishing_plan.js'; import {publish_repos, type PublishingOptions} from './multi_repo_publisher.js'; import {log_dependency_analysis} from './log_helpers.js'; +import {GITOPS_CONFIG_PATH_DEFAULT} from './gitops_constants.js'; /** @nodocs */ export const Args = z.strictObject({ - path: z + config: z .string() .meta({description: 'path to the gitops config file, absolute or relative to the cwd'}) - .default('gitops.config.ts'), + .default(GITOPS_CONFIG_PATH_DEFAULT), dir: z .string() - .meta({description: 'path containing the repos, defaults to the parent of the `path` dir'}) + .meta({description: 'path containing the repos, defaults to the parent of the config dir'}) .optional(), verbose: z.boolean().meta({description: 'show additional details'}).default(false), }); @@ -29,7 +30,7 @@ export const task: Task = { summary: 'validate gitops configuration by running all read-only commands and checking for issues', run: async ({args, log}) => { - const {path, dir, verbose} = args; + const {config, dir, verbose} = args; log.info(st('cyan', 'Running Gitops Validation Suite')); log.info(st('dim', 'This runs all read-only commands and checks for consistency.')); @@ -49,7 +50,7 @@ export const task: Task = { // Load repos once (shared by all commands) log.info(st('dim', 'Loading repositories...')); - const {local_repos} = await get_gitops_ready({path, dir, download: false, log}); + const {local_repos} = await get_gitops_ready({config, dir, download: false, log}); log.info(st('dim', ` Found ${local_repos.length} local repos`)); // 1. Run gitops_analyze diff --git a/src/lib/local_repo.ts b/src/lib/local_repo.ts index f4bf1dea..54def779 100644 --- a/src/lib/local_repo.ts +++ b/src/lib/local_repo.ts @@ -6,11 +6,13 @@ import {join} from 'node:path'; import {TaskError} from '@ryanatkn/gro'; import type {Logger} from '@fuzdev/fuz_util/log.js'; import {spawn} from '@fuzdev/fuz_util/process.js'; +import {map_concurrent_settled} from '@fuzdev/fuz_util/async.js'; import type {GitOperations, NpmOperations} from './operations.js'; import {default_git_operations, default_npm_operations} from './operations_defaults.js'; import type {GitopsConfig, GitopsRepoConfig} from './gitops_config.js'; import type {ResolvedGitopsConfig} from './resolved_gitops_config.js'; +import {GITOPS_CONCURRENCY_DEFAULT} from './gitops_constants.js'; /** * Fully loaded local repo with Library and extracted dependency data. @@ -280,16 +282,57 @@ export const local_repos_load = async ({ log, git_ops = default_git_operations, npm_ops = default_npm_operations, + parallel = true, + concurrency = GITOPS_CONCURRENCY_DEFAULT, }: { local_repo_paths: Array; log?: Logger; git_ops?: GitOperations; npm_ops?: NpmOperations; + parallel?: boolean; + concurrency?: number; }): Promise> => { + if (!parallel) { + // Sequential loading (original behavior) + const loaded: Array = []; + for (const local_repo_path of local_repo_paths) { + loaded.push(await local_repo_load({local_repo_path, log, git_ops, npm_ops})); // eslint-disable-line no-await-in-loop + } + return loaded; + } + + // Parallel loading with concurrency limit + const results = await map_concurrent_settled( + local_repo_paths, + async (local_repo_path) => { + return local_repo_load({local_repo_path, log, git_ops, npm_ops}); + }, + concurrency, + ); + + // Check for failures and collect successes const loaded: Array = []; - for (const local_repo_path of local_repo_paths) { - loaded.push(await local_repo_load({local_repo_path, log, git_ops, npm_ops})); // eslint-disable-line no-await-in-loop + const errors: Array<{repo_name: string; error: string}> = []; + + for (let i = 0; i < results.length; i++) { + const result = results[i]!; + if (result.status === 'fulfilled') { + loaded.push(result.value); + } else { + const repo_path = local_repo_paths[i]!; + errors.push({ + repo_name: repo_path.repo_name, + error: String(result.reason), + }); + } } + + // If any repos failed to load, throw with details + if (errors.length > 0) { + const error_details = errors.map((e) => ` ${e.repo_name}: ${e.error}`).join('\n'); + throw new TaskError(`Failed to load ${errors.length} repos:\n${error_details}`); + } + return loaded; }; diff --git a/src/lib/multi_repo_publisher.ts b/src/lib/multi_repo_publisher.ts index 39b29397..3a165256 100644 --- a/src/lib/multi_repo_publisher.ts +++ b/src/lib/multi_repo_publisher.ts @@ -10,7 +10,10 @@ import {type PreflightOptions} from './preflight_checks.js'; import {needs_update, is_breaking_change, detect_bump_type} from './version_utils.js'; import type {GitopsOperations} from './operations.js'; import {default_gitops_operations} from './operations_defaults.js'; -import {MAX_ITERATIONS} from './constants.js'; +import { + GITOPS_MAX_ITERATIONS_DEFAULT, + GITOPS_NPM_WAIT_TIMEOUT_DEFAULT, +} from './gitops_constants.js'; import {install_with_cache_healing} from './npm_install_helpers.js'; /* eslint-disable no-await-in-loop */ @@ -90,9 +93,11 @@ export const publish_repos = async ( let iteration = 0; let converged = false; - while (!converged && iteration < MAX_ITERATIONS) { + while (!converged && iteration < GITOPS_MAX_ITERATIONS_DEFAULT) { iteration++; - log?.info(st('cyan', `\n🚀 Publishing iteration ${iteration}/${MAX_ITERATIONS}...\n`)); + log?.info( + st('cyan', `\n🚀 Publishing iteration ${iteration}/${GITOPS_MAX_ITERATIONS_DEFAULT}...\n`), + ); // Track if any packages were published in this iteration let published_in_iteration = false; @@ -157,7 +162,7 @@ export const publish_repos = async ( max_attempts: 30, initial_delay: 1000, max_delay: 60000, - timeout: options.max_wait || 600000, // 10 minutes default + timeout: options.max_wait ?? GITOPS_NPM_WAIT_TIMEOUT_DEFAULT, }, log, }); @@ -245,7 +250,7 @@ export const publish_repos = async ( if (!published_in_iteration) { converged = true; log?.info(st('green', `\n✓ Converged after ${iteration} iteration(s) - no new changesets\n`)); - } else if (iteration === MAX_ITERATIONS) { + } else if (iteration === GITOPS_MAX_ITERATIONS_DEFAULT) { // Count packages that still have changesets (not yet published) const pending_count = order.length - published.size; const estimated_iterations = Math.ceil(pending_count / 2); // Rough estimate @@ -253,7 +258,7 @@ export const publish_repos = async ( log?.warn( st( 'yellow', - `\n⚠️ Reached maximum iterations (${MAX_ITERATIONS}) without full convergence\n` + + `\n⚠️ Reached maximum iterations (${GITOPS_MAX_ITERATIONS_DEFAULT}) without full convergence\n` + ` ${pending_count} package(s) may still have changesets to process\n` + ` Estimated ${estimated_iterations} more iteration(s) needed - run 'gro gitops_publish' again\n`, ), diff --git a/src/lib/publishing_plan.ts b/src/lib/publishing_plan.ts index a43f7881..fe66cdde 100644 --- a/src/lib/publishing_plan.ts +++ b/src/lib/publishing_plan.ts @@ -7,7 +7,7 @@ import {validate_dependency_graph} from './graph_validation.js'; import {is_breaking_change, compare_bump_types, calculate_next_version} from './version_utils.js'; import type {ChangesetOperations} from './operations.js'; import {default_changeset_operations} from './operations_defaults.js'; -import {MAX_ITERATIONS} from './constants.js'; +import {GITOPS_MAX_ITERATIONS_DEFAULT} from './gitops_constants.js'; import type {DependencyGraph} from './dependency_graph.ts'; import { calculate_dependency_updates, @@ -247,7 +247,7 @@ export const generate_publishing_plan = async ( let iteration = 0; let changed = true; - while (changed && iteration < MAX_ITERATIONS) { + while (changed && iteration < GITOPS_MAX_ITERATIONS_DEFAULT) { changed = false; iteration++; @@ -392,7 +392,7 @@ export const generate_publishing_plan = async ( } // Check if we hit iteration limit without convergence - if (iteration === MAX_ITERATIONS && changed) { + if (iteration === GITOPS_MAX_ITERATIONS_DEFAULT && changed) { // Calculate how many packages still need processing const pending_packages: Array = []; @@ -428,7 +428,7 @@ export const generate_publishing_plan = async ( const pending_count = pending_packages.length; const estimated_iterations = Math.ceil(pending_count / 2); // Rough estimate warnings.push( - `Reached maximum iterations (${MAX_ITERATIONS}) without full convergence - ` + + `Reached maximum iterations (${GITOPS_MAX_ITERATIONS_DEFAULT}) without full convergence - ` + `${pending_count} package(s) may still need processing: ${pending_packages.join(', ')}. ` + `Estimated ${estimated_iterations} more iteration(s) needed.`, ); diff --git a/src/lib/repo_ops.ts b/src/lib/repo_ops.ts index 2483dc75..17e513fb 100644 --- a/src/lib/repo_ops.ts +++ b/src/lib/repo_ops.ts @@ -15,6 +15,7 @@ import {join, resolve, dirname} from 'node:path'; import {load_gitops_config} from './gitops_config.js'; import {DEFAULT_REPOS_DIR} from './paths.js'; +import {GITOPS_CONFIG_PATH_DEFAULT} from './gitops_constants.js'; /** Default directories to exclude from file walking */ export const DEFAULT_EXCLUDE_DIRS = [ @@ -84,7 +85,7 @@ export interface RepoPath { * @returns Array of repo info with name, path, and url */ export const get_repo_paths = async (config_path?: string): Promise> => { - const resolved_config_path = resolve(config_path ?? 'gitops.config.ts'); + const resolved_config_path = resolve(config_path ?? GITOPS_CONFIG_PATH_DEFAULT); const config = await load_gitops_config(resolved_config_path); if (!config) { diff --git a/src/routes/library.ts b/src/routes/library.ts index 8ab2a367..c6d4a245 100644 --- a/src/routes/library.ts +++ b/src/routes/library.ts @@ -368,20 +368,6 @@ export const library_json: LibraryJson = { dependencies: ['version_utils.ts'], dependents: ['operations_defaults.ts'], }, - { - path: 'constants.ts', - declarations: [ - { - name: 'MAX_ITERATIONS', - kind: 'variable', - doc_comment: - 'Maximum number of iterations for fixed-point iteration during publishing.\nUsed in both plan generation and actual publishing to resolve transitive dependency cascades.\n\nIn practice, most repos converge in 2-3 iterations.\nDeep dependency chains may require more iterations.', - source_line: 8, - type_signature: '10', - }, - ], - dependents: ['multi_repo_publisher.ts', 'publishing_plan.ts'], - }, { path: 'dependency_graph.ts', declarations: [ @@ -1405,6 +1391,7 @@ export const library_json: LibraryJson = { declarations: [], dependencies: [ 'dependency_graph.ts', + 'gitops_constants.ts', 'gitops_task_helpers.ts', 'graph_validation.ts', 'log_helpers.ts', @@ -1580,25 +1567,106 @@ export const library_json: LibraryJson = { dependencies: ['paths.ts'], dependents: ['gitops_task_helpers.ts', 'repo_ops.ts'], }, + { + path: 'gitops_constants.ts', + declarations: [ + { + name: 'GITOPS_MAX_ITERATIONS_DEFAULT', + kind: 'variable', + doc_comment: + 'Maximum number of iterations for fixed-point iteration during publishing.\nUsed in both plan generation and actual publishing to resolve transitive dependency cascades.\n\nIn practice, most repos converge in 2-3 iterations.\nDeep dependency chains may require more iterations.', + source_line: 14, + type_signature: '10', + }, + { + name: 'GITOPS_CONFIG_PATH_DEFAULT', + kind: 'variable', + doc_comment: 'Default path to the gitops configuration file.', + source_line: 19, + type_signature: '"gitops.config.ts"', + }, + { + name: 'GITOPS_CONCURRENCY_DEFAULT', + kind: 'variable', + doc_comment: + 'Default number of repos to process concurrently during parallel operations.', + source_line: 24, + type_signature: '5', + }, + { + name: 'GITOPS_NPM_WAIT_TIMEOUT_DEFAULT', + kind: 'variable', + doc_comment: + "Default timeout in milliseconds for waiting on NPM package propagation (10 minutes).\nNPM's CDN uses eventual consistency, so published packages may not be immediately available.", + source_line: 30, + type_signature: '600000', + }, + ], + module_comment: + 'Shared constants for gitops tasks and operations.\n\nNaming convention: GITOPS_{NAME}_DEFAULT for user-facing defaults.', + dependents: [ + 'gitops_analyze.task.ts', + 'gitops_plan.task.ts', + 'gitops_publish.task.ts', + 'gitops_run.task.ts', + 'gitops_sync.task.ts', + 'gitops_validate.task.ts', + 'local_repo.ts', + 'multi_repo_publisher.ts', + 'publishing_plan.ts', + 'repo_ops.ts', + ], + }, { path: 'gitops_plan.task.ts', declarations: [], - dependencies: ['gitops_task_helpers.ts', 'output_helpers.ts', 'publishing_plan.ts'], + dependencies: [ + 'gitops_constants.ts', + 'gitops_task_helpers.ts', + 'output_helpers.ts', + 'publishing_plan.ts', + ], }, { path: 'gitops_publish.task.ts', declarations: [], dependencies: [ + 'gitops_constants.ts', 'gitops_task_helpers.ts', 'multi_repo_publisher.ts', 'output_helpers.ts', 'publishing_plan.ts', ], }, + { + path: 'gitops_run.task.ts', + declarations: [ + { + name: 'Args', + kind: 'type', + source_line: 11, + type_signature: + 'ZodObject<{ command: ZodString; config: ZodDefault; concurrency: ZodDefault; format: ZodDefault>; }, $strict>', + }, + { + name: 'task', + kind: 'variable', + source_line: 38, + type_signature: + 'Task<{ command: string; config: string; concurrency: number; format: "json" | "text"; }, ZodType>, unknown>', + }, + ], + dependencies: ['gitops_constants.ts', 'repo_ops.ts'], + }, { path: 'gitops_sync.task.ts', declarations: [], - dependencies: ['fetch_repo_data.ts', 'fs_fetch_value_cache.ts', 'gitops_task_helpers.ts'], + dependencies: [ + 'fetch_repo_data.ts', + 'fs_fetch_value_cache.ts', + 'gitops_constants.ts', + 'gitops_task_helpers.ts', + ], }, { path: 'gitops_task_helpers.ts', @@ -1610,7 +1678,7 @@ export const library_json: LibraryJson = { type_signature: 'GetGitopsReadyOptions', properties: [ { - name: 'path', + name: 'config', kind: 'variable', type_signature: 'string', }, @@ -1639,20 +1707,30 @@ export const library_json: LibraryJson = { kind: 'variable', type_signature: 'NpmOperations', }, + { + name: 'parallel', + kind: 'variable', + type_signature: 'boolean', + }, + { + name: 'concurrency', + kind: 'variable', + type_signature: 'number', + }, ], }, { name: 'get_gitops_ready', kind: 'function', doc_comment: - 'Central initialization function for all gitops tasks.\n\nInitialization sequence:\n1. Loads and normalizes config from `gitops.config.ts`\n2. Resolves local repo paths (creates missing with `--download`)\n3. Switches branches and pulls latest changes\n4. Auto-installs deps if package.json changed during pull\n\nPriority for path resolution:\n- `dir` argument (explicit override)\n- Config `repos_dir` setting\n- `DEFAULT_REPOS_DIR` constant', + 'Central initialization function for all gitops tasks.\n\nInitialization sequence:\n1. Loads and normalizes config from `gitops.config.ts`\n2. Resolves local repo paths (creates missing with `--download`)\n3. Switches branches and pulls latest changes (in parallel by default)\n4. Auto-installs deps if package.json changed during pull\n\nPriority for path resolution:\n- `dir` argument (explicit override)\n- Config `repos_dir` setting\n- `DEFAULT_REPOS_DIR` constant', throws: [ { type: 'if', description: 'config loading or repo resolution fails', }, ], - source_line: 57, + source_line: 61, type_signature: '(options: GetGitopsReadyOptions): Promise<{ config_path: string; repos_dir: string; gitops_config: GitopsConfig; local_repos: LocalRepo[]; }>', return_type: @@ -1668,11 +1746,11 @@ export const library_json: LibraryJson = { { name: 'ResolveGitopsPathsOptions', kind: 'type', - source_line: 96, + source_line: 107, type_signature: 'ResolveGitopsPathsOptions', properties: [ { - name: 'path', + name: 'config', kind: 'variable', type_signature: 'string', }, @@ -1691,7 +1769,7 @@ export const library_json: LibraryJson = { { name: 'resolve_gitops_paths', kind: 'function', - source_line: 102, + source_line: 113, type_signature: '(options: ResolveGitopsPathsOptions): { config_path: string; repos_dir: string; }', return_type: '{ config_path: string; repos_dir: string; }', @@ -1705,7 +1783,7 @@ export const library_json: LibraryJson = { { name: 'import_gitops_config', kind: 'function', - source_line: 120, + source_line: 131, type_signature: '(config_path: string): Promise', return_type: 'Promise', parameters: [ @@ -1737,6 +1815,7 @@ export const library_json: LibraryJson = { declarations: [], dependencies: [ 'dependency_graph.ts', + 'gitops_constants.ts', 'gitops_task_helpers.ts', 'graph_validation.ts', 'log_helpers.ts', @@ -1828,7 +1907,7 @@ export const library_json: LibraryJson = { kind: 'type', doc_comment: 'Fully loaded local repo with Library and extracted dependency data.\nDoes not extend LocalRepoPath - Library is source of truth for name/repo_url/etc.', - source_line: 19, + source_line: 21, type_signature: 'LocalRepo', properties: [ { @@ -1878,7 +1957,7 @@ export const library_json: LibraryJson = { kind: 'type', doc_comment: 'A repo that has been located on the filesystem (path exists).\nUsed before loading - just filesystem/git concerns.', - source_line: 34, + source_line: 36, type_signature: 'LocalRepoPath', properties: [ { @@ -1917,7 +1996,7 @@ export const library_json: LibraryJson = { name: 'LocalRepoMissing', kind: 'type', doc_comment: 'A repo that is missing from the filesystem (needs cloning).', - source_line: 46, + source_line: 48, type_signature: 'LocalRepoMissing', properties: [ { @@ -1959,7 +2038,7 @@ export const library_json: LibraryJson = { 'workspace dirty, branch switch fails, install fails, or library.ts missing', }, ], - source_line: 71, + source_line: 73, type_signature: '({ local_repo_path, log: _log, git_ops, npm_ops, }: { local_repo_path: LocalRepoPath; log?: Logger | undefined; git_ops?: GitOperations | undefined; npm_ops?: NpmOperations | undefined; }): Promise<...>', return_type: 'Promise', @@ -1973,7 +2052,7 @@ export const library_json: LibraryJson = { { name: 'local_repos_ensure', kind: 'function', - source_line: 228, + source_line: 230, type_signature: '({ resolved_config, repos_dir, gitops_config, download, log, npm_ops, }: { resolved_config: ResolvedGitopsConfig; repos_dir: string; gitops_config: GitopsConfig; download: boolean; log?: Logger | undefined; npm_ops?: NpmOperations | undefined; }): Promise<...>', return_type: 'Promise', @@ -1987,21 +2066,21 @@ export const library_json: LibraryJson = { { name: 'local_repos_load', kind: 'function', - source_line: 278, + source_line: 280, type_signature: - '({ local_repo_paths, log, git_ops, npm_ops, }: { local_repo_paths: LocalRepoPath[]; log?: Logger | undefined; git_ops?: GitOperations | undefined; npm_ops?: NpmOperations | undefined; }): Promise<...>', + '({ local_repo_paths, log, git_ops, npm_ops, parallel, concurrency, }: { local_repo_paths: LocalRepoPath[]; log?: Logger | undefined; git_ops?: GitOperations | undefined; npm_ops?: NpmOperations | undefined; parallel?: boolean | undefined; concurrency?: number | undefined; }): Promise<...>', return_type: 'Promise', parameters: [ { name: '__0', - type: '{ local_repo_paths: LocalRepoPath[]; log?: Logger | undefined; git_ops?: GitOperations | undefined; npm_ops?: NpmOperations | undefined; }', + type: '{ local_repo_paths: LocalRepoPath[]; log?: Logger | undefined; git_ops?: GitOperations | undefined; npm_ops?: NpmOperations | undefined; parallel?: boolean | undefined; concurrency?: number | undefined; }', }, ], }, { name: 'local_repo_locate', kind: 'function', - source_line: 296, + source_line: 339, type_signature: '({ repo_config, repos_dir, }: { repo_config: GitopsRepoConfig; repos_dir: string; }): LocalRepoPath | LocalRepoMissing', return_type: 'LocalRepoPath | LocalRepoMissing', @@ -2013,7 +2092,7 @@ export const library_json: LibraryJson = { ], }, ], - dependencies: ['operations_defaults.ts'], + dependencies: ['gitops_constants.ts', 'operations_defaults.ts'], dependents: ['gitops_task_helpers.ts', 'resolved_gitops_config.ts'], }, { @@ -2174,7 +2253,7 @@ export const library_json: LibraryJson = { 'Logs a simple bulleted list with a header.\nCommon pattern for warnings, info messages, and other lists.', source_line: 130, type_signature: - '(items: string[], header: string, color: "cyan" | "yellow" | "red" | "dim", log: Logger, log_method?: "info" | "warn" | "error"): void', + '(items: string[], header: string, color: "cyan" | "yellow" | "red" | "dim", log: Logger, log_method?: "error" | "info" | "warn"): void', return_type: 'void', parameters: [ { @@ -2195,7 +2274,7 @@ export const library_json: LibraryJson = { }, { name: 'log_method', - type: '"info" | "warn" | "error"', + type: '"error" | "info" | "warn"', default_value: "'info'", }, ], @@ -2270,7 +2349,7 @@ export const library_json: LibraryJson = { { name: 'PublishingOptions', kind: 'type', - source_line: 18, + source_line: 21, type_signature: 'PublishingOptions', properties: [ { @@ -2318,7 +2397,7 @@ export const library_json: LibraryJson = { { name: 'PublishedVersion', kind: 'type', - source_line: 29, + source_line: 32, type_signature: 'PublishedVersion', properties: [ { @@ -2361,7 +2440,7 @@ export const library_json: LibraryJson = { { name: 'PublishingResult', kind: 'type', - source_line: 39, + source_line: 42, type_signature: 'PublishingResult', properties: [ { @@ -2389,7 +2468,7 @@ export const library_json: LibraryJson = { { name: 'publish_repos', kind: 'function', - source_line: 46, + source_line: 49, type_signature: '(repos: LocalRepo[], options: PublishingOptions): Promise', return_type: 'Promise', @@ -2406,8 +2485,8 @@ export const library_json: LibraryJson = { }, ], dependencies: [ - 'constants.ts', 'dependency_updater.ts', + 'gitops_constants.ts', 'graph_validation.ts', 'npm_install_helpers.ts', 'operations_defaults.ts', @@ -3809,7 +3888,7 @@ export const library_json: LibraryJson = { }, ], dependencies: [ - 'constants.ts', + 'gitops_constants.ts', 'graph_validation.ts', 'operations_defaults.ts', 'publishing_plan_helpers.ts', @@ -3875,7 +3954,7 @@ export const library_json: LibraryJson = { kind: 'function', doc_comment: 'Walk files in a directory, respecting common exclusions.\nYields absolute paths to files (and optionally directories).', - source_line: 155, + source_line: 156, type_signature: '(dir: string, options?: WalkOptions | undefined): AsyncGenerator', return_type: 'AsyncGenerator', @@ -3897,7 +3976,7 @@ export const library_json: LibraryJson = { name: 'DEFAULT_EXCLUDE_DIRS', kind: 'variable', doc_comment: 'Default directories to exclude from file walking', - source_line: 20, + source_line: 21, type_signature: 'readonly ["node_modules", ".git", ".gro", ".svelte-kit", ".deno", ".vscode", ".idea", "dist", "build", "coverage", ".cache", ".turbo"]', }, @@ -3905,14 +3984,14 @@ export const library_json: LibraryJson = { name: 'DEFAULT_EXCLUDE_EXTENSIONS', kind: 'variable', doc_comment: 'Default binary/non-text extensions to exclude from content processing', - source_line: 36, + source_line: 37, type_signature: 'readonly [".png", ".jpg", ".jpeg", ".gif", ".svg", ".ico", ".webp", ".woff", ".woff2", ".ttf", ".eot", ".mp4", ".webm", ".mp3", ".wav", ".ogg", ".zip", ".tar", ".gz", ".lock", ".pdf"]', }, { name: 'WalkOptions', kind: 'type', - source_line: 60, + source_line: 61, type_signature: 'WalkOptions', properties: [ { @@ -3950,7 +4029,7 @@ export const library_json: LibraryJson = { { name: 'RepoPath', kind: 'type', - source_line: 73, + source_line: 74, type_signature: 'RepoPath', properties: [ { @@ -3975,7 +4054,7 @@ export const library_json: LibraryJson = { kind: 'function', doc_comment: 'Get repo paths from gitops config without full git sync.\nLighter weight than `get_gitops_ready()` - just resolves paths.', - source_line: 86, + source_line: 87, type_signature: '(config_path?: string | undefined): Promise', return_type: 'Promise', return_description: 'Array of repo info with name, path, and url', @@ -3992,7 +4071,7 @@ export const library_json: LibraryJson = { name: 'should_exclude_path', kind: 'function', doc_comment: 'Check if a path should be excluded based on options.', - source_line: 119, + source_line: 120, type_signature: '(file_path: string, options?: WalkOptions | undefined): boolean', return_type: 'boolean', parameters: [ @@ -4012,7 +4091,7 @@ export const library_json: LibraryJson = { kind: 'function', doc_comment: 'Collect all files from walk_repo_files into an array.\nConvenience function for when you need all paths upfront.', - source_line: 204, + source_line: 205, type_signature: '(dir: string, options?: WalkOptions | undefined): Promise', return_type: 'Promise', parameters: [ @@ -4030,7 +4109,8 @@ export const library_json: LibraryJson = { ], module_comment: 'Generic repository operations for scripts that work across repos.\n\nProvides lightweight utilities for:\n- Getting repo paths from gitops config (without full git sync)\n- Walking files in repos with sensible exclusions\n- Common exclusion patterns for node/svelte projects\n\nFor full git sync/clone functionality, use `get_gitops_ready()` from gitops_task_helpers.', - dependencies: ['gitops_config.ts', 'paths.ts'], + dependencies: ['gitops_config.ts', 'gitops_constants.ts', 'paths.ts'], + dependents: ['gitops_run.task.ts'], }, { path: 'repo.svelte.ts', diff --git a/src/test/gitops_run.test.ts b/src/test/gitops_run.test.ts new file mode 100644 index 00000000..ee4268d1 --- /dev/null +++ b/src/test/gitops_run.test.ts @@ -0,0 +1,20 @@ +import {test} from 'vitest'; + +// TODO: Add tests for gitops_run +// - Test command execution across multiple repos +// - Test concurrency limits +// - Test error handling (continue-on-error) +// - Test JSON output format +// - Test with fixture repos + +// For now, we rely on manual testing and fixture tests. +// The command execution is tested indirectly through manual runs. + +test.skip('gitops_run - placeholder for future tests', () => { + // Tests to add: + // 1. Execute simple command across repos + // 2. Verify concurrency throttling + // 3. Test continue-on-error behavior + // 4. Validate JSON output structure + // 5. Test with various command types (success, failure, mixed) +}); diff --git a/static/.nojekyll b/static/.nojekyll new file mode 100644 index 00000000..e69de29b