fix(cli): optimize protobuf buf generate to reuse single working dir

packages/cli/cli/versions.yml

@@ -1,4 +1,17 @@
 # yaml-language-server: $schema=../../../fern-versions-yml.schema.json
+- version: 4.37.1
+  changelogEntry:
+    - summary: |
+        Optimize protobuf buf generate to reuse a single working directory
+        across all proto files. Instead of creating a new temp dir, copying
+        the proto root, checking binaries, and resolving deps for every file,
+        setup is done once and only `buf generate <target>` runs per file.
+        The two buf generate passes (protoc-gen-openapi and protoc-gen-fern)
+        also now run concurrently.
+      type: fix
+  createdAt: "2026-03-18"
+  irVersion: 65
+
 - version: 4.37.0
   changelogEntry:
     - summary: |

packages/cli/workspace/lazy-fern-workspace/src/OSSWorkspace.ts

@@ -30,7 +30,6 @@ import { v4 as uuidv4 } from "uuid";
 import { loadOpenRpc } from "./loaders/index.js";
 import { OpenAPILoader } from "./loaders/OpenAPILoader.js";
 import { ProtobufIRGenerator } from "./protobuf/ProtobufIRGenerator.js";
-import { MaybeValid } from "./protobuf/utils.js";
 import { getAllOpenAPISpecs } from "./utils/getAllOpenAPISpecs.js";
 
 export declare namespace OSSWorkspace {
@@ -95,6 +94,11 @@ export class OSSWorkspace extends BaseOpenAPIWorkspace {
     private graphqlOperations: Record<FdrAPI.GraphQlOperationId, FdrAPI.api.v1.register.GraphQlOperation> = {};
     private graphqlTypes: Record<FdrAPI.TypeId, FdrAPI.api.v1.register.TypeDefinition> = {};
 
+    // Cache for protobuf → OpenAPI generation results, keyed by relativePathToDependency.
+    // This avoids running buf generate twice when both toFernWorkspace() and
+    // validateOSSWorkspace() need the same OpenAPI specs.
+    private openApiSpecsCache: Map<string, Promise<OpenAPISpec[]>> = new Map();
+
     constructor({ allSpecs, specs, ...superArgs }: OSSWorkspace.Args) {
         const openapiSpecs = specs.filter((spec) => spec.type === "openapi" && spec.source.type === "openapi");
         super({
@@ -215,6 +219,28 @@ export class OSSWorkspace extends BaseOpenAPIWorkspace {
         }
     }
 
+    /**
+     * Returns cached OpenAPI specs (including protobuf → OpenAPI conversions).
+     * The expensive buf generate work is performed once per unique
+     * relativePathToDependency and reused across getOpenAPIIr(),
+     * getIntermediateRepresentation(), and workspace validation.
+     */
+    public getOpenAPISpecsCached({
+        context,
+        relativePathToDependency
+    }: {
+        context: TaskContext;
+        relativePathToDependency?: RelativeFilePath;
+    }): Promise<OpenAPISpec[]> {
+        const key = relativePathToDependency ?? "";
+        let cached = this.openApiSpecsCache.get(key);
+        if (cached == null) {
+            cached = getAllOpenAPISpecs({ context, specs: this.specs, relativePathToDependency });
+            this.openApiSpecsCache.set(key, cached);
+        }
+        return cached;
+    }
+
     public async getOpenAPIIr(
         {
             context,
@@ -227,7 +253,7 @@ export class OSSWorkspace extends BaseOpenAPIWorkspace {
         },
         settings?: OSSWorkspace.Settings
     ): Promise<OpenApiIntermediateRepresentation> {
-        const openApiSpecs = await getAllOpenAPISpecs({ context, specs: this.specs, relativePathToDependency });
+        const openApiSpecs = await this.getOpenAPISpecsCached({ context, relativePathToDependency });
         return parse({
             context,
             documents: await this.loader.loadDocuments({
@@ -259,7 +285,10 @@ export class OSSWorkspace extends BaseOpenAPIWorkspace {
         generateV1Examples: boolean;
         logWarnings: boolean;
     }): Promise<IntermediateRepresentation> {
-        const specs = await getAllOpenAPISpecs({ context, specs: this.specs });
+        // Start protobuf IR generation in parallel with OpenAPI processing
+        const protobufIRResultsPromise = this.generateAllProtobufIRs({ context });
+
+        const specs = await this.getOpenAPISpecsCached({ context });
         const documents = await this.loader.loadDocuments({ context, specs });
 
         const authOverrides =
@@ -409,54 +438,21 @@ export class OSSWorkspace extends BaseOpenAPIWorkspace {
                             ? result
                             : mergeIntermediateRepresentation(mergedIr, result, casingsGenerator);
                 }
-            } else if (spec.type === "protobuf") {
-                // Handle protobuf specs by calling buf generate with protoc-gen-fern
-                try {
-                    const protobufIRGenerator = new ProtobufIRGenerator({ context });
-                    const protobufIRFilepath = await protobufIRGenerator.generate({
-                        absoluteFilepathToProtobufRoot: spec.absoluteFilepathToProtobufRoot,
-                        absoluteFilepathToProtobufTarget: spec.absoluteFilepathToProtobufTarget,
-                        local: true,
-                        deps: spec.dependencies
-                    });
-
-                    const result = await readFile(protobufIRFilepath, "utf-8");
-
-                    const casingsGenerator = constructCasingsGenerator({
-                        generationLanguage: "typescript",
-                        keywords: undefined,
-                        smartCasing: false
-                    });
-
-                    if (result != null) {
-                        let serializedIr: MaybeValid<IntermediateRepresentation>;
-                        try {
-                            serializedIr = serialization.IntermediateRepresentation.parse(JSON.parse(result), {
-                                allowUnrecognizedEnumValues: true,
-                                skipValidation: true
-                            });
-                            if (serializedIr.ok) {
-                                mergedIr =
-                                    mergedIr === undefined
-                                        ? serializedIr.value
-                                        : mergeIntermediateRepresentation(
-                                              mergedIr,
-                                              serializedIr.value,
-                                              casingsGenerator
-                                          );
-                            } else {
-                                throw new Error();
-                            }
-                        } catch (error) {
-                            context.logger.log("error", "Failed to parse protobuf IR: ");
-                        }
-                    }
-                } catch (error) {
-                    context.logger.log("warn", "Failed to parse protobuf IR: " + error);
-                }
             }
         }
 
+        // Await and merge protobuf IR results (generated in parallel with OpenAPI processing)
+        const protobufIRResults = await protobufIRResultsPromise;
+        const protobufCasingsGenerator = constructCasingsGenerator({
+            generationLanguage: "typescript",
+            keywords: undefined,
+            smartCasing: false
+        });
+        for (const ir of protobufIRResults) {
+            mergedIr =
+                mergedIr === undefined ? ir : mergeIntermediateRepresentation(mergedIr, ir, protobufCasingsGenerator);
+        }
+
         for (const errorCollector of errorCollectors) {
             if (errorCollector.hasErrors()) {
                 const errorStats = errorCollector.getErrorStats();
@@ -486,6 +482,45 @@ export class OSSWorkspace extends BaseOpenAPIWorkspace {
         return mergedIr;
     }
 
+    private async generateAllProtobufIRs({ context }: { context: TaskContext }): Promise<IntermediateRepresentation[]> {
+        const protobufSpecs = this.allSpecs.filter((spec): spec is ProtobufSpec => spec.type === "protobuf");
+        if (protobufSpecs.length === 0) {
+            return [];
+        }
+
+        // Process specs sequentially because ProtobufIRGenerator.generate()
+        // runs `npm install -g` which is not safe to invoke concurrently.
+        const results: IntermediateRepresentation[] = [];
+        for (const spec of protobufSpecs) {
+            try {
+                const protobufIRGenerator = new ProtobufIRGenerator({ context });
+                const protobufIRFilepath = await protobufIRGenerator.generate({
+                    absoluteFilepathToProtobufRoot: spec.absoluteFilepathToProtobufRoot,
+                    absoluteFilepathToProtobufTarget: spec.absoluteFilepathToProtobufTarget,
+                    local: true,
+                    deps: spec.dependencies
+                });
+
+                const result = await readFile(protobufIRFilepath, "utf-8");
+                if (result != null) {
+                    const serializedIr = serialization.IntermediateRepresentation.parse(JSON.parse(result), {
+                        allowUnrecognizedEnumValues: true,
+                        skipValidation: true
+                    });
+                    if (serializedIr.ok) {
+                        results.push(serializedIr.value);
+                        continue;
+                    }
+                    context.logger.log("error", "Failed to parse protobuf IR");
+                }
+            } catch (error) {
+                context.logger.log("warn", "Failed to parse protobuf IR: " + error);
+            }
+        }
+
+        return results;
+    }
+
     public async toFernWorkspace(
         { context }: { context: TaskContext },
         settings?: OSSWorkspace.Settings,

packages/cli/workspace/lazy-fern-workspace/src/protobuf/ProtobufOpenAPIGenerator.ts

@@ -1,7 +1,7 @@
 import { AbsoluteFilePath, join, RelativeFilePath, relative } from "@fern-api/fs-utils";
 import { createLoggingExecutable } from "@fern-api/logging-execa";
 import { TaskContext } from "@fern-api/task-context";
-import { access, cp, readFile, unlink, writeFile } from "fs/promises";
+import { access, cp, readFile, rename, unlink, writeFile } from "fs/promises";
 import path from "path";
 import tmp from "tmp-promise";
 import { resolveProtocGenOpenAPI } from "./ProtocGenOpenAPIDownloader.js";
@@ -11,6 +11,15 @@ const PROTOBUF_GENERATOR_CONFIG_FILENAME = "buf.gen.yaml";
 const PROTOBUF_GENERATOR_OUTPUT_PATH = "output";
 const PROTOBUF_GENERATOR_OUTPUT_FILEPATH = `${PROTOBUF_GENERATOR_OUTPUT_PATH}/openapi.yaml`;
 
+/**
+ * Prepared working directory for running buf generate multiple times
+ * without repeating setup (copy, which checks, dep resolution).
+ */
+interface PreparedWorkingDir {
+    cwd: AbsoluteFilePath;
+    envOverride: Record<string, string> | undefined;
+}
+
 export class ProtobufOpenAPIGenerator {
     private context: TaskContext;
     private isAirGapped: boolean | undefined;
@@ -49,6 +58,124 @@ export class ProtobufOpenAPIGenerator {
         return this.generateRemote();
     }
 
+    /**
+     * Prepares a reusable working directory: copies the proto root once,
+     * writes buf config files, checks for required binaries, and resolves
+     * dependencies. Returns a handle that can be passed to generateFromPrepared()
+     * for each target file — no repeated setup.
+     */
+    public async prepare({
+        absoluteFilepathToProtobufRoot,
+        relativeFilepathToProtobufRoot,
+        local,
+        deps
+    }: {
+        absoluteFilepathToProtobufRoot: AbsoluteFilePath;
+        relativeFilepathToProtobufRoot: RelativeFilePath;
+        local: boolean;
+        deps: string[];
+    }): Promise<PreparedWorkingDir> {
+        if (!local) {
+            this.context.failAndThrow("Remote Protobuf generation is unimplemented.");
+        }
+
+        // Resolve buf and protoc-gen-openapi binaries once
+        await this.ensureBufResolved();
+        await this.ensureProtocGenOpenAPIResolved();
+
+        if (deps.length > 0 && this.isAirGapped === undefined) {
+            this.isAirGapped = await detectAirGappedModeForProtobuf(
+                absoluteFilepathToProtobufRoot,
+                this.context.logger,
+                this.resolvedBufCommand
+            );
+        }
+
+        const cwd = AbsoluteFilePath.of((await tmp.dir()).path);
+        await cp(absoluteFilepathToProtobufRoot, cwd, { recursive: true });
+        await writeFile(
+            join(cwd, RelativeFilePath.of(PROTOBUF_GENERATOR_CONFIG_FILENAME)),
+            getProtobufGeneratorConfig({ relativeFilepathToProtobufRoot })
+        );
+
+        // If we downloaded protoc-gen-openapi, prepend its directory to PATH so buf can find it
+        const envOverride =
+            this.protocGenOpenAPIBinDir != null
+                ? { PATH: `${this.protocGenOpenAPIBinDir}${path.delimiter}${process.env.PATH ?? ""}` }
+                : undefined;
+
+        // Write buf.yaml and resolve dependencies once
+        const bufYamlPath = join(cwd, RelativeFilePath.of("buf.yaml"));
+        const bufLockPath = join(cwd, RelativeFilePath.of("buf.lock"));
+        await writeFile(bufYamlPath, getProtobufYamlV1(deps));
+
+        if (deps.length > 0) {
+            if (this.isAirGapped) {
+                this.context.logger.debug("Air-gapped mode: skipping buf dep update");
+                try {
+                    await access(bufLockPath);
+                } catch {
+                    this.context.failAndThrow(
+                        "Air-gapped mode requires a pre-cached buf.lock file. Please run 'buf dep update' at build time to cache dependencies."
+                    );
+                }
+            } else {
+                const bufCommand = this.resolvedBufCommand ?? "buf";
+                const buf = createLoggingExecutable(bufCommand, {
+                    cwd,
+                    logger: this.context.logger,
+                    stdout: "ignore",
+                    stderr: "pipe",
+                    ...(envOverride != null ? { env: { ...process.env, ...envOverride } } : {})
+                });
+                await buf(["dep", "update"]);
+            }
+        }
+
+        return { cwd, envOverride };
+    }
+
+    /**
+     * Generates OpenAPI output for a single proto target using a previously
+     * prepared working directory. Each call only runs `buf generate <target>`
+     * — all heavy setup was done once in prepare().
+     *
+     * Because protoc-gen-openapi writes to a fixed output path, each call
+     * renames the output to a unique temp file to avoid conflicts when called
+     * sequentially for multiple targets.
+     */
+    public async generateFromPrepared({
+        preparedDir,
+        absoluteFilepathToProtobufRoot,
+        absoluteFilepathToProtobufTarget
+    }: {
+        preparedDir: PreparedWorkingDir;
+        absoluteFilepathToProtobufRoot: AbsoluteFilePath;
+        absoluteFilepathToProtobufTarget: AbsoluteFilePath;
+    }): Promise<{ absoluteFilepath: AbsoluteFilePath }> {
+        const target = relative(absoluteFilepathToProtobufRoot, absoluteFilepathToProtobufTarget);
+        const bufCommand = this.resolvedBufCommand ?? "buf";
+        const buf = createLoggingExecutable(bufCommand, {
+            cwd: preparedDir.cwd,
+            logger: this.context.logger,
+            stdout: "ignore",
+            stderr: "pipe",
+            ...(preparedDir.envOverride != null ? { env: { ...process.env, ...preparedDir.envOverride } } : {})
+        });
+
+        const bufGenerateResult = await buf(["generate", target.toString()]);
+        if (bufGenerateResult.exitCode !== 0) {
+            this.context.failAndThrow(bufGenerateResult.stderr);
+        }
+
+        // Move output to a unique temp file so the next call doesn't overwrite it
+        const outputPath = join(preparedDir.cwd, RelativeFilePath.of(PROTOBUF_GENERATOR_OUTPUT_FILEPATH));
+        const uniqueOutput = AbsoluteFilePath.of((await tmp.file({ postfix: ".yaml" })).path);
+        await rename(outputPath, uniqueOutput);
+
+        return { absoluteFilepath: uniqueOutput };
+    }
+
     private async generateLocal({
         absoluteFilepathToProtobufRoot,
         absoluteFilepathToProtobufTarget,
@@ -201,7 +328,13 @@ export class ProtobufOpenAPIGenerator {
         });
 
         try {
-            await which(["protoc-gen-openapi"]);
+            const result = await which(["protoc-gen-openapi"]);
+            const resolvedPath = result.stdout?.trim();
+            if (resolvedPath) {
+                this.context.logger.debug(`Found protoc-gen-openapi on PATH: ${resolvedPath}`);
+            } else {
+                this.context.logger.debug("Found protoc-gen-openapi on PATH");
+            }
             this.protocGenOpenAPIResolved = true;
         } catch {
             this.context.logger.debug("protoc-gen-openapi not found on PATH, attempting auto-download");

packages/cli/workspace/lazy-fern-workspace/src/protobuf/utils.ts

@@ -153,12 +153,19 @@ export async function ensureBufCommand(logger: Logger): Promise<string> {
     });
 
     try {
-        await which(["buf"]);
+        const result = await which(["buf"]);
+        const bufPath = result.stdout?.trim();
+        if (bufPath) {
+            logger.debug(`Found buf on PATH: ${bufPath}`);
+        } else {
+            logger.debug("Found buf on PATH");
+        }
         return "buf";
     } catch {
         logger.debug("buf not found on PATH, attempting auto-download");
         const downloadedBufPath = await resolveBuf(logger);
         if (downloadedBufPath != null) {
+            logger.debug(`Using auto-downloaded buf: ${downloadedBufPath}`);
             return downloadedBufPath;
         }
         throw new Error("Missing required dependency; please install 'buf' to continue (e.g. 'brew install buf').");