diff --git a/src/components/WorkflowEditor.tsx b/src/components/WorkflowEditor.tsx index e3e258e..8b9818c 100644 --- a/src/components/WorkflowEditor.tsx +++ b/src/components/WorkflowEditor.tsx @@ -17,6 +17,7 @@ export function WorkflowEditor(props: WorkflowEditorProps) { const importFromConfig = useWorkflowStore((s) => s.importFromConfig); const exportToConfig = useWorkflowStore((s) => s.exportToConfig); const exportToFileMap = useWorkflowStore((s) => s.exportToFileMap); + const exportMainFile = useWorkflowStore((s) => s.exportMainFileYaml); const addToast = useWorkflowStore((s) => s.addToast); const sourceMap = useWorkflowStore((s) => s.sourceMap); const setTestResults = useWorkflowStore((s) => s.setTestResults); @@ -72,12 +73,17 @@ export function WorkflowEditor(props: WorkflowEditorProps) { if (!onChange) return; const unsub = useWorkflowStore.subscribe(() => { if (importingRef.current) return; - const config = exportToConfig(); - const yaml = configToYaml(config); - onChange(yaml); + if (hasMultiFileRef.current) { + // In multi-file mode emit only the main file content (with imports: references) + // rather than the fully merged YAML, to prevent the host from inlining all files. + // Use the cheaper exportMainFile() which avoids serialising every imported file. + onChange(exportMainFile()); + } else { + onChange(configToYaml(exportToConfig())); + } }); return unsub; - }, [onChange, exportToConfig]); + }, [onChange, exportToConfig, exportMainFile]); // Sync testResults prop into the store useEffect(() => { diff --git a/src/stores/workflowStore.ts b/src/stores/workflowStore.ts index 3244dce..2b08be6 100644 --- a/src/stores/workflowStore.ts +++ b/src/stores/workflowStore.ts @@ -13,7 +13,7 @@ import { import type { WorkflowConfig, WorkflowTab, CrossWorkflowLink } from '../types/workflow.ts'; import { MODULE_TYPE_MAP as STATIC_MODULE_TYPE_MAP } from '../types/workflow.ts'; import useModuleSchemaStore from './moduleSchemaStore.ts'; -import { nodesToConfig, configToNodes, nodeComponentType, exportToFiles } from '../utils/serialization.ts'; +import { nodesToConfig, configToNodes, nodeComponentType, exportToFiles, exportMainFileYaml } from '../utils/serialization.ts'; import { layoutNodes } from '../utils/autoLayout.ts'; import { exportLayout, importLayout, type LayoutData } from '../utils/layout-sidecar.ts'; import { autoGroupOrphanedNodes } from '../utils/grouping.ts'; @@ -107,6 +107,8 @@ interface WorkflowStore { exportToConfig: () => WorkflowConfig; exportToFileMap: () => Map; + /** Cheaply produce only the main-file YAML (null key) without serialising all imported files. */ + exportMainFileYaml: () => string; importFromConfig: (config: WorkflowConfig, sourceMap?: Map) => void; clearCanvas: () => void; exportLayout: () => LayoutData; @@ -468,6 +470,12 @@ const useWorkflowStore = create()( return exportToFiles(config, sourceMap); }, + exportMainFileYaml: () => { + const config = get().exportToConfig(); + const { sourceMap } = get(); + return exportMainFileYaml(config, sourceMap); + }, + importFromConfig: (config, sourceMap) => { get().pushHistory(); const moduleTypeMap = useModuleSchemaStore.getState().moduleTypeMap; diff --git a/src/utils/index.ts b/src/utils/index.ts index 98b64ee..b1ec8bd 100644 --- a/src/utils/index.ts +++ b/src/utils/index.ts @@ -10,6 +10,7 @@ export { nodesToMultiConfig, resolveImports, exportToFiles, + exportMainFileYaml, hasFileReferences, } from './serialization'; export { layoutNodes } from './autoLayout'; diff --git a/src/utils/serialization-multifile.test.ts b/src/utils/serialization-multifile.test.ts index af31107..56e8dbe 100644 --- a/src/utils/serialization-multifile.test.ts +++ b/src/utils/serialization-multifile.test.ts @@ -1,5 +1,5 @@ import { describe, it, expect } from 'vitest'; -import { configToNodes, exportToFiles } from './serialization.ts'; +import { configToNodes, exportToFiles, resolveImports } from './serialization.ts'; import { MODULE_TYPE_MAP } from '../types/workflow.ts'; import type { WorkflowConfig } from '../types/workflow.ts'; @@ -24,7 +24,7 @@ describe('partial config with sourceMap renders all nodes', () => { const sourceMap = new Map([ ['http-server', 'config/modules.yaml'], ['router', 'config/modules.yaml'], - ['auth-register', 'pipelines/auth.yaml'], + ['pipeline:auth-register', 'pipelines/auth.yaml'], ]); const { nodes } = configToNodes(config, MODULE_TYPE_MAP, sourceMap); @@ -107,7 +107,7 @@ describe('exportToFiles splits config by sourceMap', () => { }, }; const sourceMap = new Map([ - ['auth-register', 'pipelines/auth.yaml'], + ['pipeline:auth-register', 'pipelines/auth.yaml'], // user-update has no entry -> goes to main ]); @@ -240,3 +240,256 @@ describe('name and version preserved in multi-file export', () => { expect(importedYaml).not.toMatch(/^version:/m); }); }); + +// --------------------------------------------------------------------------- +// resolveImports — complex multi-file fixture scenarios +// --------------------------------------------------------------------------- + +import { readFileSync } from 'fs'; +import { resolve as resolveFsPath } from 'path'; + +/** + * Load one of the multifile fixture files from test-fixtures/multifile/. + * Using on-disk fixtures avoids duplicating YAML content in the test file. + */ +function loadFixture(name: string): string { + return readFileSync( + resolveFsPath(__dirname, '../../test-fixtures/multifile', name), + 'utf-8', + ); +} + +const FIXTURE_MAIN = loadFixture('main.yaml'); +const FIXTURE_BASE = loadFixture('base.yaml'); +const FIXTURE_DATABASE = loadFixture('database.yaml'); +const FIXTURE_API = loadFixture('api.yaml'); + +/** Build a simple in-memory resolver from a path→content map. */ +function makeResolver(files: Record) { + return async (path: string): Promise => files[path] ?? null; +} + +describe('resolveImports — complex nested multi-file scenario', () => { + const resolver = makeResolver({ + 'base.yaml': FIXTURE_BASE, + 'database.yaml': FIXTURE_DATABASE, + 'api.yaml': FIXTURE_API, + }); + + it('resolves all modules across three levels of nesting', async () => { + const { config, error } = await resolveImports(FIXTURE_MAIN, resolver); + expect(error).toBeUndefined(); + const names = config.modules.map((m) => m.name); + expect(names).toContain('cache'); // from base.yaml + expect(names).toContain('db'); // from database.yaml (nested import inside base.yaml) + expect(names).toContain('http-server'); // from api.yaml + expect(names).toContain('router'); // from api.yaml + }); + + it('assigns correct sourceFile for every module in sourceMap', async () => { + const { sourceMap } = await resolveImports(FIXTURE_MAIN, resolver); + expect(sourceMap.get('cache')).toBe('base.yaml'); + expect(sourceMap.get('db')).toBe('database.yaml'); + expect(sourceMap.get('http-server')).toBe('api.yaml'); + expect(sourceMap.get('router')).toBe('api.yaml'); + }); + + it('tracks pipelines in sourceMap so they round-trip to their source file', async () => { + const { sourceMap } = await resolveImports(FIXTURE_MAIN, resolver); + expect(sourceMap.get('pipeline:user-create')).toBe('api.yaml'); + expect(sourceMap.get('pipeline:user-get')).toBe('api.yaml'); + }); + + it('merges workflows from imported files', async () => { + const { config } = await resolveImports(FIXTURE_MAIN, resolver); + expect(config.workflows).toHaveProperty('http'); + }); + + it('preserves application name and version from application: section', async () => { + const { config } = await resolveImports(FIXTURE_MAIN, resolver); + expect(config.name).toBe('my-platform'); + expect(config.version).toBe('2.0.0'); + }); + + it('does not duplicate modules that appear in multiple imports', async () => { + const { config } = await resolveImports(FIXTURE_MAIN, resolver); + const names = config.modules.map((m) => m.name); + const unique = new Set(names); + expect(names.length).toBe(unique.size); + }); +}); + +describe('resolveImports — pipeline sourceMap enables correct round-trip export', () => { + it('pipelines round-trip to their source file via exportToFiles', async () => { + const resolver = makeResolver({ + 'api.yaml': FIXTURE_API, + 'base.yaml': FIXTURE_BASE, + 'database.yaml': FIXTURE_DATABASE, + }); + const { config, sourceMap } = await resolveImports(FIXTURE_MAIN, resolver); + + const fileMap = exportToFiles(config, sourceMap); + + // Pipelines belong to api.yaml — they must NOT appear as a pipelines: block in the main file + const mainYaml = fileMap.get(null)!; + expect(mainYaml).not.toMatch(/^pipelines:/m); + + // Pipelines must appear in api.yaml's output + const apiYaml = fileMap.get('api.yaml')!; + expect(apiYaml).toMatch(/^pipelines:/m); + expect(apiYaml).toContain('user-create'); + expect(apiYaml).toContain('user-get'); + }); + + it('modules round-trip to their source file via exportToFiles', async () => { + const resolver = makeResolver({ + 'api.yaml': FIXTURE_API, + 'base.yaml': FIXTURE_BASE, + 'database.yaml': FIXTURE_DATABASE, + }); + const { config, sourceMap } = await resolveImports(FIXTURE_MAIN, resolver); + + const fileMap = exportToFiles(config, sourceMap); + + // Main file modules list must be empty (all modules belong to imported files) + const mainYaml = fileMap.get(null)!; + expect(mainYaml).toMatch(/^modules:\s*\[\]/m); + + // Modules appear in their respective source files + expect(fileMap.get('api.yaml')).toContain('http-server'); + expect(fileMap.get('base.yaml')).toContain('cache'); + expect(fileMap.get('database.yaml')).toContain('db'); + }); + + it('main file references imported files rather than inlining them', async () => { + const resolver = makeResolver({ + 'api.yaml': FIXTURE_API, + 'base.yaml': FIXTURE_BASE, + 'database.yaml': FIXTURE_DATABASE, + }); + const { config, sourceMap } = await resolveImports(FIXTURE_MAIN, resolver); + + const fileMap = exportToFiles(config, sourceMap); + const mainYaml = fileMap.get(null)!; + + // Main file should reference imported files, not inline their content + expect(mainYaml).toContain('imports:'); + // Each imported file path must appear as a reference + const importedFiles = ['api.yaml', 'base.yaml', 'database.yaml']; + const hasAtLeastOneRef = importedFiles.some((f) => mainYaml.includes(f)); + expect(hasAtLeastOneRef).toBe(true); + }); +}); + +describe('resolveImports — editing a node routes the change to the correct file', () => { + it('after renaming a module, exportToFiles puts it in the same source file', async () => { + const resolver = makeResolver({ + 'api.yaml': FIXTURE_API, + 'base.yaml': FIXTURE_BASE, + 'database.yaml': FIXTURE_DATABASE, + }); + const { config, sourceMap } = await resolveImports(FIXTURE_MAIN, resolver); + + // Simulate renaming 'cache' (from base.yaml) to 'cache-v2' + const updatedConfig: WorkflowConfig = { + ...config, + modules: config.modules.map((m) => + m.name === 'cache' ? { ...m, name: 'cache-v2' } : m, + ), + }; + // Update sourceMap to reflect the rename + const updatedSourceMap = new Map(sourceMap); + updatedSourceMap.delete('cache'); + updatedSourceMap.set('cache-v2', 'base.yaml'); + + const fileMap = exportToFiles(updatedConfig, updatedSourceMap); + + // The renamed module should appear in base.yaml, not in main + expect(fileMap.get('base.yaml')).toContain('cache-v2'); + expect(fileMap.get(null)).not.toContain('cache-v2'); + }); + + it('after modifying a pipeline step, exportToFiles keeps it in the source file', async () => { + const resolver = makeResolver({ + 'api.yaml': FIXTURE_API, + 'base.yaml': FIXTURE_BASE, + 'database.yaml': FIXTURE_DATABASE, + }); + const { config, sourceMap } = await resolveImports(FIXTURE_MAIN, resolver); + + // Simulate adding a step to the user-create pipeline + const updatedConfig: WorkflowConfig = { + ...config, + pipelines: { + ...config.pipelines, + 'user-create': { + steps: [ + { name: 'validate', type: 'step.validate' }, + { name: 'auth-check', type: 'step.set' }, + { name: 'insert', type: 'step.db_exec' }, + ], + }, + }, + }; + + const fileMap = exportToFiles(updatedConfig, sourceMap); + + // Modified pipeline must stay in api.yaml + const apiYaml = fileMap.get('api.yaml')!; + expect(apiYaml).toContain('auth-check'); + // Must NOT bleed into main file + expect(fileMap.get(null)).not.toContain('auth-check'); + }); +}); + +describe('resolveImports — cycle detection and error handling', () => { + it('handles circular imports gracefully without infinite loop', async () => { + const circularResolver = makeResolver({ + 'a.yaml': 'imports:\n - b.yaml\nmodules:\n - name: mod-a\n type: http.server\n config: {}', + 'b.yaml': 'imports:\n - a.yaml\nmodules:\n - name: mod-b\n type: database.postgres\n config: {}', + }); + const mainWithCircular = 'imports:\n - a.yaml'; + // Should not hang or throw + const { config } = await resolveImports(mainWithCircular, circularResolver); + const names = config.modules.map((m) => m.name); + expect(names).toContain('mod-a'); + expect(names).toContain('mod-b'); + }); + + it('reports missing files as errors but still merges what is available', async () => { + const partialResolver = makeResolver({ 'api.yaml': FIXTURE_API }); + const { config, error } = await resolveImports(FIXTURE_MAIN, partialResolver); + // Some error about missing files + expect(error).toBeTruthy(); + // But api.yaml modules/pipelines are still present + const names = config.modules.map((m) => m.name); + expect(names).toContain('http-server'); + }); +}); + +describe('resolveImports — imports: path also tracks pipelines in sourceMap', () => { + it('pipelines from an imports: file get correct sourceMap entries', async () => { + const mainYaml = ` +imports: + - feature-a.yaml +`.trim(); + const featureAYaml = ` +modules: + - name: svc-a + type: http.server + config: + port: 9000 +pipelines: + pipeline-a: + steps: + - name: step1 + type: step.validate +`.trim(); + const { sourceMap } = await resolveImports( + mainYaml, + makeResolver({ 'feature-a.yaml': featureAYaml }), + ); + expect(sourceMap.get('svc-a')).toBe('feature-a.yaml'); + expect(sourceMap.get('pipeline:pipeline-a')).toBe('feature-a.yaml'); + }); +}); diff --git a/src/utils/serialization.ts b/src/utils/serialization.ts index 6bd6482..b134627 100644 --- a/src/utils/serialization.ts +++ b/src/utils/serialization.ts @@ -730,7 +730,7 @@ export function configToNodes( const pipeline = pipelineValue as { steps?: Array<{ name: string; type: string; config?: Record }> }; if (!pipeline?.steps || pipeline.steps.length === 0) continue; - const pipelineSourceFile = sourceMap?.get(pipelineName); + const pipelineSourceFile = sourceMap?.get(pipelineKey(pipelineName)); let prevNodeId: string | null = null; for (let si = 0; si < pipeline.steps.length; si++) { @@ -955,11 +955,54 @@ export function extractStateMachineBranches( return { nodes: newNodes, edges: newEdges }; } +/** + * Resolve a relative path against a base file path. + * Examples: + * resolvePath('base.yaml', 'database.yaml') => 'database.yaml' + * resolvePath('services/base.yaml', '../db.yaml') => 'db.yaml' + * resolvePath('services/base.yaml', 'cache.yaml') => 'services/cache.yaml' + */ +function resolvePath(basePath: string, relPath: string): string { + if (relPath.startsWith('/')) return relPath; + const baseDir = basePath.includes('/') ? basePath.substring(0, basePath.lastIndexOf('/') + 1) : ''; + const combined = baseDir + relPath; + const parts = combined.split('/'); + const resolved: string[] = []; + for (const part of parts) { + if (part === '..') { + // Only go up if there is a segment to remove; otherwise keep the '..' to avoid + // silently discarding path components that exceed the base directory depth. + if (resolved.length > 0) { + resolved.pop(); + } else { + resolved.push(part); + } + } else if (part !== '.') { + resolved.push(part); + } + } + return resolved.join('/'); +} + +/** + * Namespace prefix used to key pipeline names in sourceMap, preventing collision + * with module names that happen to share the same string. + */ +const PIPELINE_KEY_PREFIX = 'pipeline:'; + +/** Return the sourceMap key for a pipeline name. */ +function pipelineKey(name: string): string { + return `${PIPELINE_KEY_PREFIX}${name}`; +} + /** * Given a parsed YAML config, detect `imports:` array and `application.workflows[].file:` entries. * For each reference, call the resolver to get file contents, parse them, and merge into the config. - * Track which modules came from which source file. - * Returns { config: merged WorkflowConfig, sourceMap: Map } where sourceMap maps module name to source file path. + * Track which modules and pipelines came from which source file. + * Supports nested file references: imported files may themselves declare `imports:` or + * `application.workflows[].file:` entries, which are resolved recursively (depth-first). + * Returns { config: merged WorkflowConfig, sourceMap: Map } where sourceMap maps + * module name or `pipeline:` to source file path. */ export async function resolveImports( yamlText: string, @@ -979,7 +1022,6 @@ export async function resolveImports( const mainModules = (parsed.modules ?? []) as ModuleConfig[]; const mainModuleNames = new Set(mainModules.map((m) => m.name)); - // Track main file modules in sourceMap as having no source (they belong to the main file) let mergedModules = [...mainModules]; let mergedWorkflows = { ...((parsed.workflows ?? {}) as Record) }; @@ -987,49 +1029,134 @@ export async function resolveImports( let mergedPipelines = parsed.pipelines ? { ...(parsed.pipelines as Record) } : undefined; const errors: string[] = []; - // Handle `imports:` directive — main file wins on duplicate module names - const imports = parsed.imports as string[] | undefined; - if (Array.isArray(imports)) { - for (const importPath of imports) { - const content = await resolver(importPath); - if (content === null) { - errors.push(`Import not found: ${importPath}`); + // `inProgress`: paths currently being fetched/parsed (cycle detection during recursion). + // `completed`: paths that have been successfully loaded and merged (deduplication). + // Keeping them separate ensures a file that failed to load is not silently skipped + // when later referenced via a stricter call site (e.g. application.workflows[].file:). + const inProgress = new Set(); + const completed = new Set(); + + /** + * Load and recursively merge a single file. + * @param resolvedPath The path passed to the resolver (already resolved against parent). + * @param strictConflicts When true, duplicate module/workflow/pipeline names are errors + * (used for `application.workflows[].file:` references). + * When false, duplicates are silently skipped ("first-wins"). + */ + async function mergeFile(resolvedPath: string, strictConflicts: boolean): Promise { + if (inProgress.has(resolvedPath)) return; // cycle — currently being processed up the call stack + if (completed.has(resolvedPath)) return; // already fully merged + + inProgress.add(resolvedPath); + + const content = await resolver(resolvedPath); + if (content === null) { + errors.push(strictConflicts ? `Workflow file not found: ${resolvedPath}` : `Import not found: ${resolvedPath}`); + inProgress.delete(resolvedPath); + return; + } + + let fileParsed: Record; + try { + fileParsed = yaml.load(content) as Record; + if (!fileParsed || typeof fileParsed !== 'object') { + inProgress.delete(resolvedPath); + return; + } + } catch (e) { + errors.push(`Error parsing ${resolvedPath}: ${(e as Error).message}`); + inProgress.delete(resolvedPath); + return; + } + + // Recursively process this file's own `imports:` entries first (depth-first). + // Sub-imports use "first-wins, no error" semantics. + const subImports = fileParsed.imports as string[] | undefined; + if (Array.isArray(subImports)) { + for (const subImportPath of subImports) { + await mergeFile(resolvePath(resolvedPath, subImportPath), false); + } + } + + // Also recurse into any application.workflows[].file: entries in the sub-file. + // These are always strict (conflicts are errors), matching top-level behaviour. + const subApp = fileParsed.application as Record | undefined; + if (subApp && Array.isArray(subApp.workflows)) { + for (const entry of subApp.workflows as Array>) { + if (typeof entry.file === 'string') { + await mergeFile(resolvePath(resolvedPath, entry.file), true); + } + } + } + + // Merge modules (tracked in sourceMap by module name) + const fileModules = (fileParsed.modules ?? []) as ModuleConfig[]; + for (const mod of fileModules) { + if (mainModuleNames.has(mod.name)) { + if (strictConflicts) { + errors.push(`Conflict: module "${mod.name}" in ${resolvedPath} conflicts with existing module`); + } continue; } - try { - const importedParsed = yaml.load(content) as Record; - if (!importedParsed || typeof importedParsed !== 'object') continue; - - const importedModules = (importedParsed.modules ?? []) as ModuleConfig[]; - for (const mod of importedModules) { - if (!mainModuleNames.has(mod.name)) { - mergedModules.push(mod); - sourceMap.set(mod.name, importPath); - mainModuleNames.add(mod.name); - } - // Main wins on duplicates — skip if already present + mergedModules.push(mod); + sourceMap.set(mod.name, resolvedPath); + mainModuleNames.add(mod.name); + } + + // Merge workflows + const fileWorkflows = (fileParsed.workflows ?? {}) as Record; + for (const [key, value] of Object.entries(fileWorkflows)) { + if (key in mergedWorkflows) { + if (strictConflicts) { + errors.push(`Conflict: workflow "${key}" in ${resolvedPath} conflicts with existing workflow`); } + continue; + } + mergedWorkflows[key] = value; + } - // Merge workflows/triggers (imported ones don't override main) - const importedWorkflows = (importedParsed.workflows ?? {}) as Record; - for (const [key, value] of Object.entries(importedWorkflows)) { - if (!(key in mergedWorkflows)) { - mergedWorkflows[key] = value; - } + // Merge triggers + const fileTriggers = (fileParsed.triggers ?? {}) as Record; + for (const [key, value] of Object.entries(fileTriggers)) { + if (key in mergedTriggers) { + if (strictConflicts) { + errors.push(`Conflict: trigger "${key}" in ${resolvedPath} conflicts with existing trigger`); } - const importedTriggers = (importedParsed.triggers ?? {}) as Record; - for (const [key, value] of Object.entries(importedTriggers)) { - if (!(key in mergedTriggers)) { - mergedTriggers[key] = value; + continue; + } + mergedTriggers[key] = value; + } + + // Merge pipelines (tracked in sourceMap under namespaced keys so they never + // collide with module names that share the same string). + if (fileParsed.pipelines) { + if (!mergedPipelines) mergedPipelines = {}; + const filePipelines = fileParsed.pipelines as Record; + for (const [key, value] of Object.entries(filePipelines)) { + if (key in mergedPipelines) { + if (strictConflicts) { + errors.push(`Conflict: pipeline "${key}" in ${resolvedPath} conflicts with existing pipeline`); } + continue; } - } catch (e) { - errors.push(`Error parsing ${importPath}: ${(e as Error).message}`); + mergedPipelines[key] = value; + sourceMap.set(pipelineKey(key), resolvedPath); } } + + completed.add(resolvedPath); + inProgress.delete(resolvedPath); + } + + // Handle `imports:` directive — main file wins on duplicate names (no conflict errors) + const imports = parsed.imports as string[] | undefined; + if (Array.isArray(imports)) { + for (const importPath of imports) { + await mergeFile(importPath, false); + } } - // Handle `application.workflows[].file:` directive + // Handle `application.workflows[].file:` directive — conflicts are reported as errors const application = parsed.application as Record | undefined; if (application && typeof application === 'object') { const appWorkflows = (application.workflows ?? []) as Array>; @@ -1037,56 +1164,7 @@ export async function resolveImports( for (const entry of appWorkflows) { const filePath = entry.file as string | undefined; if (!filePath) continue; - const content = await resolver(filePath); - if (content === null) { - errors.push(`Workflow file not found: ${filePath}`); - continue; - } - try { - const fileParsed = yaml.load(content) as Record; - if (!fileParsed || typeof fileParsed !== 'object') continue; - - const fileModules = (fileParsed.modules ?? []) as ModuleConfig[]; - for (const mod of fileModules) { - if (mainModuleNames.has(mod.name)) { - errors.push(`Conflict: module "${mod.name}" in ${filePath} conflicts with existing module`); - continue; - } - mergedModules.push(mod); - sourceMap.set(mod.name, filePath); - mainModuleNames.add(mod.name); - } - - const fileWorkflows = (fileParsed.workflows ?? {}) as Record; - for (const [key, value] of Object.entries(fileWorkflows)) { - if (key in mergedWorkflows) { - errors.push(`Conflict: workflow "${key}" in ${filePath} conflicts with existing workflow`); - continue; - } - mergedWorkflows[key] = value; - } - - const fileTriggers = (fileParsed.triggers ?? {}) as Record; - for (const [key, value] of Object.entries(fileTriggers)) { - if (key in mergedTriggers) { - errors.push(`Conflict: trigger "${key}" in ${filePath} conflicts with existing trigger`); - continue; - } - mergedTriggers[key] = value; - } - - if (fileParsed.pipelines) { - if (!mergedPipelines) mergedPipelines = {}; - const filePipelines = fileParsed.pipelines as Record; - for (const [key, value] of Object.entries(filePipelines)) { - if (!(key in mergedPipelines)) { - mergedPipelines[key] = value; - } - } - } - } catch (e) { - errors.push(`Error parsing ${filePath}: ${(e as Error).message}`); - } + await mergeFile(filePath, true); } } } @@ -1099,6 +1177,12 @@ export async function resolveImports( if (mergedPipelines) { config.pipelines = mergedPipelines; } + // Preserve name/version from the main file — check both top-level fields and the + // application: section (common in application.workflows[].file: configs). + const configName = (parsed.name ?? application?.name) as string | undefined; + const configVersion = (parsed.version ?? application?.version) as string | undefined; + if (configName) config.name = configName; + if (configVersion) config.version = configVersion; return { config, @@ -1126,10 +1210,11 @@ export function exportToFiles( fileModules.get(file)!.push(mod); } - // Split pipelines by source file + // Split pipelines by source file; pipeline names are stored under the + // namespaced key `pipeline:` to avoid collisions with module names. if (config.pipelines) { for (const [name, value] of Object.entries(config.pipelines)) { - const file = sourceMap.get(name) ?? null; + const file = sourceMap.get(pipelineKey(name)) ?? null; if (!filePipelines.has(file)) filePipelines.set(file, {}); filePipelines.get(file)![name] = value; } @@ -1137,7 +1222,60 @@ export function exportToFiles( const result = new Map(); - // Main file gets its modules + workflows/triggers/pipelines + // Compute the main-file content and collect the list of imported file paths. + const { yaml: mainYaml, importedFiles } = buildMainFileContent(config, fileModules, filePipelines); + result.set(null, mainYaml); + + // Each imported file gets its modules and/or pipelines + for (const file of importedFiles) { + const fileConfig: Record = {}; + const modules = fileModules.get(file); + if (modules && modules.length > 0) fileConfig.modules = modules; + const pipelines = filePipelines.get(file); + if (pipelines && Object.keys(pipelines).length > 0) fileConfig.pipelines = pipelines; + result.set(file, yaml.dump(fileConfig, { lineWidth: -1, noRefs: true, sortKeys: false })); + } + + return result; +} + +/** + * Produce only the main-file YAML (the `null` entry) without serialising the + * content of every imported file. Use this for cheap `onChange` notifications + * in multi-file mode where only the main file needs to be communicated. + */ +export function exportMainFileYaml( + config: WorkflowConfig, + sourceMap: Map, +): string { + const fileModules = new Map(); + const filePipelines = new Map>(); + + for (const mod of config.modules) { + const file = sourceMap.get(mod.name) ?? null; + if (!fileModules.has(file)) fileModules.set(file, []); + fileModules.get(file)!.push(mod); + } + + if (config.pipelines) { + for (const [name, value] of Object.entries(config.pipelines)) { + const file = sourceMap.get(pipelineKey(name)) ?? null; + if (!filePipelines.has(file)) filePipelines.set(file, {}); + filePipelines.get(file)![name] = value; + } + } + + return buildMainFileContent(config, fileModules, filePipelines).yaml; +} + +/** + * Internal helper: build the main-file YAML string and collect imported file paths. + */ +function buildMainFileContent( + config: WorkflowConfig, + fileModules: Map, + filePipelines: Map>, +): { yaml: string; importedFiles: string[] } { const mainModules = fileModules.get(null) ?? []; const mainConfig: Record = {}; if (config.name !== undefined) mainConfig.name = config.name; @@ -1165,19 +1303,7 @@ export function exportToFiles( mainConfig.imports = importedFiles; } - result.set(null, yaml.dump(mainConfig, { lineWidth: -1, noRefs: true, sortKeys: false })); - - // Each imported file gets its modules and/or pipelines - for (const file of importedFiles) { - const fileConfig: Record = {}; - const modules = fileModules.get(file); - if (modules && modules.length > 0) fileConfig.modules = modules; - const pipelines = filePipelines.get(file); - if (pipelines && Object.keys(pipelines).length > 0) fileConfig.pipelines = pipelines; - result.set(file, yaml.dump(fileConfig, { lineWidth: -1, noRefs: true, sortKeys: false })); - } - - return result; + return { yaml: yaml.dump(mainConfig, { lineWidth: -1, noRefs: true, sortKeys: false }), importedFiles }; } /** diff --git a/test-fixtures/multifile/api.yaml b/test-fixtures/multifile/api.yaml new file mode 100644 index 0000000..85017ab --- /dev/null +++ b/test-fixtures/multifile/api.yaml @@ -0,0 +1,32 @@ +modules: + - name: http-server + type: http.server + config: + port: 8080 + - name: router + type: http.router + config: {} + +workflows: + http: + server: http-server + router: router + routes: + - method: POST + path: /api/users + handler: user-create + - method: GET + path: /api/users/:id + handler: user-get + +pipelines: + user-create: + steps: + - name: validate + type: step.validate + - name: insert + type: step.db_exec + user-get: + steps: + - name: fetch + type: step.db_exec diff --git a/test-fixtures/multifile/base.yaml b/test-fixtures/multifile/base.yaml new file mode 100644 index 0000000..1014cae --- /dev/null +++ b/test-fixtures/multifile/base.yaml @@ -0,0 +1,10 @@ +# Base infrastructure — embeds the database layer +imports: + - database.yaml + +modules: + - name: cache + type: nosql.redis + config: + host: localhost + port: 6379 diff --git a/test-fixtures/multifile/database.yaml b/test-fixtures/multifile/database.yaml new file mode 100644 index 0000000..e0508a0 --- /dev/null +++ b/test-fixtures/multifile/database.yaml @@ -0,0 +1,7 @@ +modules: + - name: db + type: database.postgres + config: + host: localhost + port: 5432 + database: myapp diff --git a/test-fixtures/multifile/main.yaml b/test-fixtures/multifile/main.yaml new file mode 100644 index 0000000..62f03b7 --- /dev/null +++ b/test-fixtures/multifile/main.yaml @@ -0,0 +1,6 @@ +application: + name: my-platform + version: 2.0.0 + workflows: + - file: base.yaml + - file: api.yaml