chore: sync upstream firecrawl/main (2026-05-11)

.github/workflows/npm-audit-claude-remediation.yml

@@ -0,0 +1,106 @@
+name: Audit NPM Claude Remediation
+
+on:
+  workflow_run:
+    workflows: ["Audit NPM Packages"]
+    types:
+      - completed
+  workflow_dispatch:
+
+permissions:
+  actions: read
+  contents: write
+  id-token: write
+  issues: write
+  pull-requests: write
+
+concurrency:
+  group: audit-npm-claude-remediation-${{ github.event.repository.default_branch }}
+  cancel-in-progress: false
+
+jobs:
+  remediate:
+    if: ${{ github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'failure' }}
+    runs-on: blacksmith-2vcpu-ubuntu-2404
+    timeout-minutes: 60
+
+    steps:
+      - name: Checkout default branch
+        uses: actions/checkout@v5
+        with:
+          ref: ${{ github.event.repository.default_branch }}
+
+      - name: Install pnpm
+        uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v4
+        with:
+          version: 10
+
+      - name: Scan default branch audit failures
+        id: scan
+        env:
+          GH_TOKEN: ${{ github.token }}
+          GITHUB_TOKEN: ${{ github.token }}
+        run: node .github/scripts/audit-ci-vuln-scan.mjs
+
+      - name: Skip when all vulnerabilities are covered
+        if: ${{ steps.scan.outputs.has_uncovered != 'true' }}
+        run: |
+          echo "No uncovered audit-ci vulnerabilities on ${GITHUB_REF_NAME}; skipping Claude."
+
+      - name: Run Claude remediation
+        id: claude
+        if: ${{ steps.scan.outputs.has_uncovered == 'true' }}
+        uses: anthropics/claude-code-action@2cc1ac1331eac7a6a96d716dd204dd2888d0fcd2 # v1
+        with:
+          anthropic_api_key: ${{ secrets.NPM_AUDIT_CLAUDE_ANTHROPIC_API_KEY }}
+          base_branch: ${{ github.event.repository.default_branch }}
+          branch_prefix: claude/audit-ci/
+          prompt: ${{ steps.scan.outputs.prompt }}
+          claude_args: |
+            --max-turns 40
+            --allowedTools "Read,Edit,MultiEdit,Write,Glob,Grep,LS,WebFetch,WebSearch,TodoWrite,Bash(pnpm:*),Bash(npm:*),Bash(node:*),Bash(git:*),Bash(gh:*),Bash(jq:*),Bash(rg:*),Bash(curl:*),Bash(date:*)"
+
+      - name: Ensure remediation PR marker
+        if: ${{ steps.scan.outputs.has_uncovered == 'true' && steps.claude.outputs.branch_name != '' }}
+        env:
+          GH_TOKEN: ${{ github.token }}
+          BRANCH_NAME: ${{ steps.claude.outputs.branch_name }}
+          MARKER: ${{ steps.scan.outputs.marker }}
+        run: |
+          set -euo pipefail
+
+          PR_NUMBER="$(gh pr list --head "$BRANCH_NAME" --state open --json number --jq '.[0].number // empty')"
+          if [ -z "$PR_NUMBER" ]; then
+            echo "::warning::Claude did not leave an open PR for branch ${BRANCH_NAME}; marker could not be enforced."
+            exit 0
+          fi
+
+          gh label create audit-ci-remediation \
+            --description "Automated audit-ci vulnerability remediation" \
+            --color "B60205" \
+            2>/dev/null || true
+          gh pr edit "$PR_NUMBER" --add-label audit-ci-remediation
+
+          BODY_FILE="$(mktemp)"
+          UPDATED_BODY_FILE="$(mktemp)"
+          gh pr view "$PR_NUMBER" --json body --jq '.body // ""' > "$BODY_FILE"
+
+          node - "$BODY_FILE" "$UPDATED_BODY_FILE" "$MARKER" <<'NODE'
+          const { readFileSync, writeFileSync } = require("node:fs");
+
+          const [, , inputPath, outputPath, marker] = process.argv;
+          const markerRegex = /<!--\s*audit-ci-vuln-keys:\s*\[[\s\S]*?\]\s*-->/;
+          let body = readFileSync(inputPath, "utf8");
+
+          if (!markerRegex.test(body)) {
+            if (/^## Summary\b.*$/m.test(body)) {
+              body = body.replace(/^## Summary\b.*$/m, (heading) => `${heading}\n\n${marker}`);
+            } else {
+              body = `## Summary\n\n${marker}\n\n${body.trim()}\n`;
+            }
+          }
+
+          writeFileSync(outputPath, body);
+          NODE
+
+          gh pr edit "$PR_NUMBER" --body-file "$UPDATED_BODY_FILE"

.github/workflows/test-php-sdk.yml

@@ -18,6 +18,8 @@ jobs:
   build-and-test:
     name: Build and Test
     runs-on: blacksmith-4vcpu-ubuntu-2404
+    env:
+      HAS_API_KEY: ${{ secrets.FIRECRAWL_API_KEY != '' }}
     if: >-
       github.event_name == 'workflow_dispatch' ||
       github.event_name == 'pull_request' ||
@@ -62,7 +64,7 @@ jobs:
         run: vendor/bin/pest --ci
 
       - name: Run E2E tests
-        if: ${{ secrets.FIRECRAWL_API_KEY != '' }}
+        if: env.HAS_API_KEY == 'true'
         working-directory: ./apps/php-sdk
         env:
           FIRECRAWL_API_KEY: ${{ secrets.FIRECRAWL_API_KEY }}

apps/api/.env.example

@@ -84,6 +84,11 @@ SELF_HOSTED_WEBHOOK_URL=
 # Set this to the HMAC secret of your webhook when using the self-hosted version of FireCrawl
 SELF_HOSTED_WEBHOOK_HMAC_SECRET=
 
+# Support Agent service URL for /v2/support/* proxy
+SUPPORT_AGENT_URL=
+# Vercel Deployment Protection bypass secret for the support agent
+SUPPORT_AGENT_VERCEL_BYPASS_SECRET=
+
 # Resend API Key for transactional emails
 RESEND_API_KEY=
 

apps/api/audit-ci.jsonc

@@ -2,6 +2,7 @@
     "$schema": "https://github.com/IBM/audit-ci/raw/main/docs/schema.json",
     "low": true,
     "allowlist": [
-        "GHSA-w5hq-g745-h8pq"
+        "GHSA-w5hq-g745-h8pq",
+        "GHSA-v2v4-37r5-5v8g"
     ]
 }

apps/api/package.json

@@ -103,7 +103,7 @@
     "amqplib": "^0.10.9",
     "async-mutex": "^0.5.0",
     "autumn-js": "1.2.13",
-    "axios": "^1.15.0",
+    "axios": "^1.15.2",
     "body-parser": "^1.20.3",
     "bullmq": "^5.56.7",
     "cacheable-lookup": "^6.1.0",

apps/api/src/__tests__/snips/lib.ts

@@ -23,6 +23,7 @@ export const TEST_PRODUCTION = !TEST_SELF_HOST;
 
 // TODO: do we want to run AI tests when users run this command locally? It may lead to increased spending for them, depending on configuration
 export const HAS_AI = !!(config.OPENAI_API_KEY || config.OLLAMA_BASE_URL);
+export const HAS_FIREWORKS = !!process.env.FIREWORKS_API_KEY;
 export const HAS_FIRE_ENGINE = !!config.FIRE_ENGINE_BETA_URL;
 export const HAS_PLAYWRIGHT = !!config.PLAYWRIGHT_MICROSERVICE_URL;
 export const HAS_PROXY = !!config.PROXY_SERVER;

apps/api/src/__tests__/snips/v1/deprecation.test.ts

@@ -0,0 +1,93 @@
+import { describe, it, expect, beforeAll } from "@jest/globals";
+import request from "supertest";
+import { TEST_API_URL } from "../lib";
+import { idmux, Identity } from "./lib";
+
+let identity: Identity;
+
+beforeAll(async () => {
+  identity = await idmux({
+    name: "deprecation",
+    concurrency: 10,
+    credits: 1000,
+  });
+}, 10000);
+
+describe("Deprecation warnings on legacy endpoints", () => {
+  it("POST /v1/llmstxt enqueues with Deprecation header and warnings in body", async () => {
+    const res = await request(TEST_API_URL)
+      .post("/v1/llmstxt")
+      .set("Authorization", `Bearer ${identity.apiKey}`)
+      .set("Content-Type", "application/json")
+      .send({ url: "https://firecrawl.dev" });
+
+    expect(res.statusCode).toBe(200);
+    expect(res.body.success).toBe(true);
+    expect(res.headers["deprecation"]).toBe("true");
+    expect(res.headers["warning"]).toMatch(/^299 - "/);
+    expect(res.headers["warning"]).toMatch(/llmstxt/i);
+    expect(Array.isArray(res.body.warnings)).toBe(true);
+    expect(res.body.warnings.some((w: string) => /llmstxt/i.test(w))).toBe(
+      true,
+    );
+    expect(res.body.warnings.some((w: string) => /deprecated/i.test(w))).toBe(
+      true,
+    );
+    expect(res.body.replacement).toBeUndefined();
+    expect(res.headers["link"]).toBeUndefined();
+  }, 30000);
+
+  it("GET /v1/llmstxt/:jobId still emits warnings on 404", async () => {
+    const res = await request(TEST_API_URL)
+      .get(`/v1/llmstxt/${crypto.randomUUID()}`)
+      .set("Authorization", `Bearer ${identity.apiKey}`);
+
+    expect(res.statusCode).toBe(404);
+    expect(res.headers["deprecation"]).toBe("true");
+    expect(res.headers["warning"]).toMatch(/deprecated/i);
+    expect(Array.isArray(res.body.warnings)).toBe(true);
+    expect(res.body.warnings.some((w: string) => /deprecated/i.test(w))).toBe(
+      true,
+    );
+  }, 30000);
+
+  it("POST /v1/deep-research returns warnings and successor-version Link", async () => {
+    const res = await request(TEST_API_URL)
+      .post("/v1/deep-research")
+      .set("Authorization", `Bearer ${identity.apiKey}`)
+      .set("Content-Type", "application/json")
+      .send({
+        query: "what is firecrawl",
+        maxDepth: 1,
+        maxUrls: 1,
+        timeLimit: 60,
+      });
+
+    expect(res.statusCode).toBe(200);
+    expect(res.headers["deprecation"]).toBe("true");
+    expect(res.headers["warning"]).toMatch(/deep-research/i);
+    expect(res.headers["link"]).toContain(
+      '</v2/search>; rel="successor-version"',
+    );
+    expect(Array.isArray(res.body.warnings)).toBe(true);
+    expect(
+      res.body.warnings.some((w: string) => /deep-research/i.test(w)),
+    ).toBe(true);
+    expect(res.body.replacement).toBe("/v2/search");
+  }, 30000);
+
+  it("non-deprecated endpoints do not emit Deprecation header or warnings", async () => {
+    const res = await request(TEST_API_URL)
+      .post("/v1/scrape")
+      .set("Authorization", `Bearer ${identity.apiKey}`)
+      .set("Content-Type", "application/json")
+      .send({ url: "https://firecrawl.dev" });
+
+    expect(res.headers["deprecation"]).toBeUndefined();
+    expect(res.headers["warning"]).toBeUndefined();
+    if (res.body && typeof res.body === "object") {
+      expect(res.body.warnings).toBeUndefined();
+      expect(res.body.replacement).toBeUndefined();
+    }
+  }, 60000);
+});

apps/api/src/__tests__/snips/v2/crawl.test.ts

@@ -437,7 +437,7 @@ describe("Crawl tests", () => {
         async () => {
           const res = await crawl(
             {
-              url: base,
+              url: new URL("/blog", base).href,
               prompt:
                 "Crawl everything including external links and subdomains",
               // Explicit options that should override the prompt