Fix multi-file workflow: pipeline sourceMap tracking, recursive imports, prevent inline rebuild on onChange

src/components/WorkflowEditor.tsx

@@ -17,6 +17,7 @@ export function WorkflowEditor(props: WorkflowEditorProps) {
   const importFromConfig = useWorkflowStore((s) => s.importFromConfig);
   const exportToConfig = useWorkflowStore((s) => s.exportToConfig);
   const exportToFileMap = useWorkflowStore((s) => s.exportToFileMap);
+  const exportMainFile = useWorkflowStore((s) => s.exportMainFileYaml);
   const addToast = useWorkflowStore((s) => s.addToast);
   const sourceMap = useWorkflowStore((s) => s.sourceMap);
   const setTestResults = useWorkflowStore((s) => s.setTestResults);
@@ -72,12 +73,17 @@ export function WorkflowEditor(props: WorkflowEditorProps) {
     if (!onChange) return;
     const unsub = useWorkflowStore.subscribe(() => {
       if (importingRef.current) return;
-      const config = exportToConfig();
-      const yaml = configToYaml(config);
-      onChange(yaml);
+      if (hasMultiFileRef.current) {
+        // In multi-file mode emit only the main file content (with imports: references)
+        // rather than the fully merged YAML, to prevent the host from inlining all files.
+        // Use the cheaper exportMainFile() which avoids serialising every imported file.
+        onChange(exportMainFile());
+      } else {
+        onChange(configToYaml(exportToConfig()));
+      }
     });
     return unsub;
-  }, [onChange, exportToConfig]);
+  }, [onChange, exportToConfig, exportMainFile]);
 
   // Sync testResults prop into the store
   useEffect(() => {

src/stores/workflowStore.ts

@@ -13,7 +13,7 @@ import {
 import type { WorkflowConfig, WorkflowTab, CrossWorkflowLink } from '../types/workflow.ts';
 import { MODULE_TYPE_MAP as STATIC_MODULE_TYPE_MAP } from '../types/workflow.ts';
 import useModuleSchemaStore from './moduleSchemaStore.ts';
-import { nodesToConfig, configToNodes, nodeComponentType, exportToFiles } from '../utils/serialization.ts';
+import { nodesToConfig, configToNodes, nodeComponentType, exportToFiles, exportMainFileYaml } from '../utils/serialization.ts';
 import { layoutNodes } from '../utils/autoLayout.ts';
 import { exportLayout, importLayout, type LayoutData } from '../utils/layout-sidecar.ts';
 import { autoGroupOrphanedNodes } from '../utils/grouping.ts';
@@ -107,6 +107,8 @@ interface WorkflowStore {
 
   exportToConfig: () => WorkflowConfig;
   exportToFileMap: () => Map<string | null, string>;
+  /** Cheaply produce only the main-file YAML (null key) without serialising all imported files. */
+  exportMainFileYaml: () => string;
   importFromConfig: (config: WorkflowConfig, sourceMap?: Map<string, string>) => void;
   clearCanvas: () => void;
   exportLayout: () => LayoutData;
@@ -468,6 +470,12 @@ const useWorkflowStore = create<WorkflowStore>()(
     return exportToFiles(config, sourceMap);
   },
 
+  exportMainFileYaml: () => {
+    const config = get().exportToConfig();
+    const { sourceMap } = get();
+    return exportMainFileYaml(config, sourceMap);
+  },
+
   importFromConfig: (config, sourceMap) => {
     get().pushHistory();
     const moduleTypeMap = useModuleSchemaStore.getState().moduleTypeMap;

src/utils/index.ts

@@ -10,6 +10,7 @@ export {
   nodesToMultiConfig,
   resolveImports,
   exportToFiles,
+  exportMainFileYaml,
   hasFileReferences,
 } from './serialization';
 export { layoutNodes } from './autoLayout';

src/utils/serialization-multifile.test.ts

@@ -1,5 +1,5 @@
 import { describe, it, expect } from 'vitest';
-import { configToNodes, exportToFiles } from './serialization.ts';
+import { configToNodes, exportToFiles, resolveImports } from './serialization.ts';
 import { MODULE_TYPE_MAP } from '../types/workflow.ts';
 import type { WorkflowConfig } from '../types/workflow.ts';
 
@@ -24,7 +24,7 @@ describe('partial config with sourceMap renders all nodes', () => {
     const sourceMap = new Map([
       ['http-server', 'config/modules.yaml'],
       ['router', 'config/modules.yaml'],
-      ['auth-register', 'pipelines/auth.yaml'],
+      ['pipeline:auth-register', 'pipelines/auth.yaml'],
     ]);
 
     const { nodes } = configToNodes(config, MODULE_TYPE_MAP, sourceMap);
@@ -107,7 +107,7 @@ describe('exportToFiles splits config by sourceMap', () => {
       },
     };
     const sourceMap = new Map([
-      ['auth-register', 'pipelines/auth.yaml'],
+      ['pipeline:auth-register', 'pipelines/auth.yaml'],
       // user-update has no entry -> goes to main
     ]);
 
@@ -240,3 +240,256 @@ describe('name and version preserved in multi-file export', () => {
     expect(importedYaml).not.toMatch(/^version:/m);
   });
 });
+
+// ---------------------------------------------------------------------------
+// resolveImports — complex multi-file fixture scenarios
+// ---------------------------------------------------------------------------
+
+import { readFileSync } from 'fs';
+import { resolve as resolveFsPath } from 'path';
+
+/**
+ * Load one of the multifile fixture files from test-fixtures/multifile/.
+ * Using on-disk fixtures avoids duplicating YAML content in the test file.
+ */
+function loadFixture(name: string): string {
+  return readFileSync(
+    resolveFsPath(__dirname, '../../test-fixtures/multifile', name),
+    'utf-8',
+  );
+}
+
+const FIXTURE_MAIN = loadFixture('main.yaml');
+const FIXTURE_BASE = loadFixture('base.yaml');
+const FIXTURE_DATABASE = loadFixture('database.yaml');
+const FIXTURE_API = loadFixture('api.yaml');
+
+/** Build a simple in-memory resolver from a path→content map. */
+function makeResolver(files: Record<string, string>) {
+  return async (path: string): Promise<string | null> => files[path] ?? null;
+}
+
+describe('resolveImports — complex nested multi-file scenario', () => {
+  const resolver = makeResolver({
+    'base.yaml': FIXTURE_BASE,
+    'database.yaml': FIXTURE_DATABASE,
+    'api.yaml': FIXTURE_API,
+  });
+
+  it('resolves all modules across three levels of nesting', async () => {
+    const { config, error } = await resolveImports(FIXTURE_MAIN, resolver);
+    expect(error).toBeUndefined();
+    const names = config.modules.map((m) => m.name);
+    expect(names).toContain('cache');       // from base.yaml
+    expect(names).toContain('db');          // from database.yaml (nested import inside base.yaml)
+    expect(names).toContain('http-server'); // from api.yaml
+    expect(names).toContain('router');      // from api.yaml
+  });
+
+  it('assigns correct sourceFile for every module in sourceMap', async () => {
+    const { sourceMap } = await resolveImports(FIXTURE_MAIN, resolver);
+    expect(sourceMap.get('cache')).toBe('base.yaml');
+    expect(sourceMap.get('db')).toBe('database.yaml');
+    expect(sourceMap.get('http-server')).toBe('api.yaml');
+    expect(sourceMap.get('router')).toBe('api.yaml');
+  });
+
+  it('tracks pipelines in sourceMap so they round-trip to their source file', async () => {
+    const { sourceMap } = await resolveImports(FIXTURE_MAIN, resolver);
+    expect(sourceMap.get('pipeline:user-create')).toBe('api.yaml');
+    expect(sourceMap.get('pipeline:user-get')).toBe('api.yaml');
+  });
+
+  it('merges workflows from imported files', async () => {
+    const { config } = await resolveImports(FIXTURE_MAIN, resolver);
+    expect(config.workflows).toHaveProperty('http');
+  });
+
+  it('preserves application name and version from application: section', async () => {
+    const { config } = await resolveImports(FIXTURE_MAIN, resolver);
+    expect(config.name).toBe('my-platform');
+    expect(config.version).toBe('2.0.0');
+  });
+
+  it('does not duplicate modules that appear in multiple imports', async () => {
+    const { config } = await resolveImports(FIXTURE_MAIN, resolver);
+    const names = config.modules.map((m) => m.name);
+    const unique = new Set(names);
+    expect(names.length).toBe(unique.size);
+  });
+});
+
+describe('resolveImports — pipeline sourceMap enables correct round-trip export', () => {
+  it('pipelines round-trip to their source file via exportToFiles', async () => {
+    const resolver = makeResolver({
+      'api.yaml': FIXTURE_API,
+      'base.yaml': FIXTURE_BASE,
+      'database.yaml': FIXTURE_DATABASE,
+    });
+    const { config, sourceMap } = await resolveImports(FIXTURE_MAIN, resolver);
+
+    const fileMap = exportToFiles(config, sourceMap);
+
+    // Pipelines belong to api.yaml — they must NOT appear as a pipelines: block in the main file
+    const mainYaml = fileMap.get(null)!;
+    expect(mainYaml).not.toMatch(/^pipelines:/m);
+
+    // Pipelines must appear in api.yaml's output
+    const apiYaml = fileMap.get('api.yaml')!;
+    expect(apiYaml).toMatch(/^pipelines:/m);
+    expect(apiYaml).toContain('user-create');
+    expect(apiYaml).toContain('user-get');
+  });
+
+  it('modules round-trip to their source file via exportToFiles', async () => {
+    const resolver = makeResolver({
+      'api.yaml': FIXTURE_API,
+      'base.yaml': FIXTURE_BASE,
+      'database.yaml': FIXTURE_DATABASE,
+    });
+    const { config, sourceMap } = await resolveImports(FIXTURE_MAIN, resolver);
+
+    const fileMap = exportToFiles(config, sourceMap);
+
+    // Main file modules list must be empty (all modules belong to imported files)
+    const mainYaml = fileMap.get(null)!;
+    expect(mainYaml).toMatch(/^modules:\s*\[\]/m);
+
+    // Modules appear in their respective source files
+    expect(fileMap.get('api.yaml')).toContain('http-server');
+    expect(fileMap.get('base.yaml')).toContain('cache');
+    expect(fileMap.get('database.yaml')).toContain('db');
+  });
+
+  it('main file references imported files rather than inlining them', async () => {
+    const resolver = makeResolver({
+      'api.yaml': FIXTURE_API,
+      'base.yaml': FIXTURE_BASE,
+      'database.yaml': FIXTURE_DATABASE,
+    });
+    const { config, sourceMap } = await resolveImports(FIXTURE_MAIN, resolver);
+
+    const fileMap = exportToFiles(config, sourceMap);
+    const mainYaml = fileMap.get(null)!;
+
+    // Main file should reference imported files, not inline their content
+    expect(mainYaml).toContain('imports:');
+    // Each imported file path must appear as a reference
+    const importedFiles = ['api.yaml', 'base.yaml', 'database.yaml'];
+    const hasAtLeastOneRef = importedFiles.some((f) => mainYaml.includes(f));
+    expect(hasAtLeastOneRef).toBe(true);
+  });
+});
+
+describe('resolveImports — editing a node routes the change to the correct file', () => {
+  it('after renaming a module, exportToFiles puts it in the same source file', async () => {
+    const resolver = makeResolver({
+      'api.yaml': FIXTURE_API,
+      'base.yaml': FIXTURE_BASE,
+      'database.yaml': FIXTURE_DATABASE,
+    });
+    const { config, sourceMap } = await resolveImports(FIXTURE_MAIN, resolver);
+
+    // Simulate renaming 'cache' (from base.yaml) to 'cache-v2'
+    const updatedConfig: WorkflowConfig = {
+      ...config,
+      modules: config.modules.map((m) =>
+        m.name === 'cache' ? { ...m, name: 'cache-v2' } : m,
+      ),
+    };
+    // Update sourceMap to reflect the rename
+    const updatedSourceMap = new Map(sourceMap);
+    updatedSourceMap.delete('cache');
+    updatedSourceMap.set('cache-v2', 'base.yaml');
+
+    const fileMap = exportToFiles(updatedConfig, updatedSourceMap);
+
+    // The renamed module should appear in base.yaml, not in main
+    expect(fileMap.get('base.yaml')).toContain('cache-v2');
+    expect(fileMap.get(null)).not.toContain('cache-v2');
+  });
+
+  it('after modifying a pipeline step, exportToFiles keeps it in the source file', async () => {
+    const resolver = makeResolver({
+      'api.yaml': FIXTURE_API,
+      'base.yaml': FIXTURE_BASE,
+      'database.yaml': FIXTURE_DATABASE,
+    });
+    const { config, sourceMap } = await resolveImports(FIXTURE_MAIN, resolver);
+
+    // Simulate adding a step to the user-create pipeline
+    const updatedConfig: WorkflowConfig = {
+      ...config,
+      pipelines: {
+        ...config.pipelines,
+        'user-create': {
+          steps: [
+            { name: 'validate', type: 'step.validate' },
+            { name: 'auth-check', type: 'step.set' },
+            { name: 'insert', type: 'step.db_exec' },
+          ],
+        },
+      },
+    };
+
+    const fileMap = exportToFiles(updatedConfig, sourceMap);
+
+    // Modified pipeline must stay in api.yaml
+    const apiYaml = fileMap.get('api.yaml')!;
+    expect(apiYaml).toContain('auth-check');
+    // Must NOT bleed into main file
+    expect(fileMap.get(null)).not.toContain('auth-check');
+  });
+});
+
+describe('resolveImports — cycle detection and error handling', () => {
+  it('handles circular imports gracefully without infinite loop', async () => {
+    const circularResolver = makeResolver({
+      'a.yaml': 'imports:\n  - b.yaml\nmodules:\n  - name: mod-a\n    type: http.server\n    config: {}',
+      'b.yaml': 'imports:\n  - a.yaml\nmodules:\n  - name: mod-b\n    type: database.postgres\n    config: {}',
+    });
+    const mainWithCircular = 'imports:\n  - a.yaml';
+    // Should not hang or throw
+    const { config } = await resolveImports(mainWithCircular, circularResolver);
+    const names = config.modules.map((m) => m.name);
+    expect(names).toContain('mod-a');
+    expect(names).toContain('mod-b');
+  });
+
+  it('reports missing files as errors but still merges what is available', async () => {
+    const partialResolver = makeResolver({ 'api.yaml': FIXTURE_API });
+    const { config, error } = await resolveImports(FIXTURE_MAIN, partialResolver);
+    // Some error about missing files
+    expect(error).toBeTruthy();
+    // But api.yaml modules/pipelines are still present
+    const names = config.modules.map((m) => m.name);
+    expect(names).toContain('http-server');
+  });
+});
+
+describe('resolveImports — imports: path also tracks pipelines in sourceMap', () => {
+  it('pipelines from an imports: file get correct sourceMap entries', async () => {
+    const mainYaml = `
+imports:
+  - feature-a.yaml
+`.trim();
+    const featureAYaml = `
+modules:
+  - name: svc-a
+    type: http.server
+    config:
+      port: 9000
+pipelines:
+  pipeline-a:
+    steps:
+      - name: step1
+        type: step.validate
+`.trim();
+    const { sourceMap } = await resolveImports(
+      mainYaml,
+      makeResolver({ 'feature-a.yaml': featureAYaml }),
+    );
+    expect(sourceMap.get('svc-a')).toBe('feature-a.yaml');
+    expect(sourceMap.get('pipeline:pipeline-a')).toBe('feature-a.yaml');
+  });
+});