chore: sync fork with upstream/main while preserving custom overlays

.github/CODEOWNERS

@@ -112,5 +112,9 @@
 # Tests
 /apps/api/src/__tests__/snips/*                 @mogery
 
+# Branding
+**/branding*                                    @abimaelmartell
+**/branding-script*                             @abimaelmartell
+
 # Examples
 /examples/*                                     @ericciarla @nickscamara

.github/archive/js-sdk.yml

@@ -21,7 +21,7 @@ env:
 
 jobs:  
   build:  
-    runs-on: ubuntu-latest  
+    runs-on: blacksmith-4vcpu-ubuntu-2404  
     services:
       redis:
         image: redis

.github/archive/publish-rust-sdk.yml

@@ -7,7 +7,7 @@ env:
 
 jobs:
   build-and-publish:
-    runs-on: ubuntu-latest
+    runs-on: blacksmith-4vcpu-ubuntu-2404
 
     steps:
       - name: Checkout repository

.github/archive/python-sdk.yml

@@ -21,7 +21,7 @@ env:
 
 jobs:  
   build:  
-    runs-on: ubuntu-latest  
+    runs-on: blacksmith-4vcpu-ubuntu-2404  
     strategy:  
       matrix:  
         python-version: ["3.10"]

.github/archive/rust-sdk.yml

@@ -22,7 +22,7 @@ env:
 
 jobs:
   build:
-    runs-on: ubuntu-latest
+    runs-on: blacksmith-4vcpu-ubuntu-2404
     services:
       redis:
         image: redis

.github/scripts/check_version_has_incremented.py

@@ -8,11 +8,16 @@
 Published version: 0.0.21
 true
 
-python .github/scripts/check_version_has_incremented.py python ./apps/python-sdk/firecrawl firecrawl-py 
+python .github/scripts/check_version_has_incremented.py python ./apps/python-sdk/firecrawl firecrawl-py
 Local version: 0.0.11
 Published version: 0.0.11
 false
 
+python .github/scripts/check_version_has_incremented.py java ./apps/java-sdk com.firecrawl:firecrawl-java
+Local version: 1.0.0
+Published version: 0.0.0  (0.0.0 means not yet published on Maven Central)
+true
+
 """
 import json
 import os
@@ -53,6 +58,30 @@ def get_npm_version(package_name: str) -> str:
     version = response.json()['version']
     return version.strip()
 
+def get_gradle_version(file_path: str) -> str:
+    """Extract version string from build.gradle.kts."""
+    build_file = Path(file_path).read_text()
+    version_match = re.search(r'^version\s*=\s*["\']([^"\']*)["\']', build_file, re.M)
+    if version_match:
+        return version_match.group(1).strip()
+    raise RuntimeError("Unable to find version string in build.gradle.kts.")
+
+def get_maven_central_version(package_name: str) -> str:
+    """Get latest version of Java package from Maven Central. package_name should be groupId:artifactId."""
+    group_id, artifact_id = package_name.split(":")
+    group_path = group_id.replace(".", "/")
+    url = f"https://repo1.maven.org/maven2/{group_path}/{artifact_id}/maven-metadata.xml"
+    response = requests.get(url)
+    if response.status_code == 404:
+        return "0.0.0"
+    response.raise_for_status()
+    version_match = re.search(r"<release>(.*?)</release>", response.text)
+    if not version_match:
+        version_match = re.search(r"<latest>(.*?)</latest>", response.text)
+    if version_match:
+        return version_match.group(1).strip()
+    return "0.0.0"
+
 # def get_rust_version(file_path: str) -> str:
 #     """Extract version string from Cargo.toml."""
 #     cargo_toml = toml.load(file_path)
@@ -87,14 +116,19 @@ def is_version_incremented(local_version: str, published_version: str) -> bool:
         current_version = get_js_version(os.path.join(package_path, 'package.json'))
         # Get published version from npm
         published_version = get_npm_version(package_name)
+    elif package_type == "java":
+        # Get current version from build.gradle.kts
+        current_version = get_gradle_version(os.path.join(package_path, 'build.gradle.kts'))
+        # Get published version from Maven Central
+        published_version = get_maven_central_version(package_name)
     # if package_type == "rust":
     #     # Get current version from Cargo.toml
     #     current_version = get_rust_version(os.path.join(package_path, 'Cargo.toml'))
     #     # Get published version from crates.io
     #     published_version = get_crates_version(package_name)
 
     else:
-        raise ValueError("Invalid package type. Use 'python' or 'js'.")
+        raise ValueError("Invalid package type. Use 'python', 'js', or 'java'.")
 
     # Print versions for debugging
     # print(f"Local version: {current_version}")

.github/workflows/deploy-go-service.yaml

@@ -14,16 +14,16 @@ on:
 
 jobs:
   push-app-image:
-    runs-on: ubuntu-latest
+    runs-on: blacksmith-4vcpu-ubuntu-2404
     defaults:
       run:
         working-directory: "./apps/go-html-to-md-service"
     steps:
       - name: "Checkout GitHub Action"
         uses: actions/checkout@main
 
-      - name: "Set up Docker Buildx"
-        uses: docker/setup-buildx-action@v3
+      - name: Setup Blacksmith Builder
+        uses: useblacksmith/setup-docker-builder@v1
 
       - name: "Login to GitHub Container Registry"
         uses: docker/login-action@v3
@@ -33,10 +33,8 @@ jobs:
           password: ${{secrets.GITHUB_TOKEN}}
 
       - name: "Build and Push Image"
-        uses: docker/build-push-action@v6
+        uses: useblacksmith/build-push-action@v2
         with:
           context: ./apps/go-html-to-md-service
           push: true
           tags: ghcr.io/firecrawl/go-html-to-md-service:latest
-          cache-from: type=registry,ref=ghcr.io/firecrawl/go-html-to-md-service:buildcache
-          cache-to: type=registry,ref=ghcr.io/firecrawl/go-html-to-md-service:buildcache,mode=max

.github/workflows/deploy-image-staging.yml

@@ -8,7 +8,7 @@ on:
 
 jobs:
   push-app-image:
-    runs-on: ubuntu-latest
+    runs-on: blacksmith-4vcpu-ubuntu-2404
     defaults:
       run:
         working-directory: './apps/api'
@@ -26,4 +26,4 @@ jobs:
       - name: 'Build Inventory Image'
         run: |
           docker build . --tag ghcr.io/firecrawl/firecrawl-staging:latest
-          docker push ghcr.io/firecrawl/firecrawl-staging:latest
+          docker push ghcr.io/firecrawl/firecrawl-staging:latest

.github/workflows/deploy-image.yml

@@ -1,41 +1,81 @@
 name: Deploy Images to GHCR
 
-env:
-  DOTNET_VERSION: '6.0.x'
-
 on:
   push:
     branches:
       - main
     paths:
       - apps/api/**
+      - .github/workflows/deploy-image.yml
   workflow_dispatch:
 
 jobs:
-      push-app-image:
-        runs-on: ubuntu-latest
-        defaults:
-          run:
-            working-directory: './apps/api'
-        steps:
-          - name: 'Checkout GitHub Action'
-            uses: actions/checkout@main
-
-          - name: 'Set up Docker Buildx'
-            uses: docker/setup-buildx-action@v3
-
-          - name: 'Login to GitHub Container Registry'
-            uses: docker/login-action@v3
-            with:
-              registry: ghcr.io
-              username: ${{github.actor}}
-              password: ${{secrets.GITHUB_TOKEN}}
-
-          - name: 'Build and Push Image'
-            uses: docker/build-push-action@v6
-            with:
-              context: ./apps/api
-              push: true
-              tags: ghcr.io/firecrawl/firecrawl:latest
-              cache-from: type=registry,ref=ghcr.io/firecrawl/firecrawl:buildcache
-              cache-to: type=registry,ref=ghcr.io/firecrawl/firecrawl:buildcache,mode=max
+  build:
+    runs-on: ${{ matrix.runner }}
+    strategy:
+      matrix:
+        include:
+          - platform: linux/amd64
+            runner: blacksmith-4vcpu-ubuntu-2404
+          - platform: linux/arm64
+            runner: blacksmith-4vcpu-ubuntu-2404-arm
+    defaults:
+      run:
+        working-directory: './apps/api'
+    steps:
+      - name: 'Checkout GitHub Action'
+        uses: actions/checkout@main
+
+      - name: Setup Blacksmith Builder
+        uses: useblacksmith/setup-docker-builder@v1
+
+      - name: 'Login to GitHub Container Registry'
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{github.actor}}
+          password: ${{secrets.GITHUB_TOKEN}}
+
+      - name: Lowercase Repo Owner
+        run: |
+          echo "REPO_OWNER=${GITHUB_REPOSITORY_OWNER,,}" >>${GITHUB_ENV}
+        env:
+          GITHUB_REPOSITORY_OWNER: '${{ github.repository_owner }}'
+
+      - name: Extract platform suffix
+        run: |
+          platform=${{ matrix.platform }}
+          echo "PLATFORM_SUFFIX=${platform//\//-}" >> $GITHUB_ENV
+
+      - name: 'Build and Push Image'
+        uses: useblacksmith/build-push-action@v2
+        with:
+          context: ./apps/api
+          push: true
+          tags: ghcr.io/${{ env.REPO_OWNER }}/firecrawl:${{ env.PLATFORM_SUFFIX }}
+          platforms: ${{ matrix.platform }}
+          provenance: false
+
+  manifest:
+    runs-on: blacksmith-2vcpu-ubuntu-2404
+    needs: build
+    steps:
+      - name: 'Login to GitHub Container Registry'
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{github.actor}}
+          password: ${{secrets.GITHUB_TOKEN}}
+
+      - name: Lowercase Repo Owner
+        run: |
+          echo "REPO_OWNER=${GITHUB_REPOSITORY_OWNER,,}" >>${GITHUB_ENV}
+        env:
+          GITHUB_REPOSITORY_OWNER: '${{ github.repository_owner }}'
+
+      - name: 'Create and Push Multi-Arch Manifest'
+        run: |
+          docker manifest create ghcr.io/${{ env.REPO_OWNER }}/firecrawl:latest \
+            ghcr.io/${{ env.REPO_OWNER }}/firecrawl:linux-amd64 \
+            ghcr.io/${{ env.REPO_OWNER }}/firecrawl:linux-arm64
+          docker manifest push ghcr.io/${{ env.REPO_OWNER }}/firecrawl:latest

.github/workflows/deploy-nuq-postgres.yml

@@ -10,7 +10,7 @@ on:
 
 jobs:
   push-app-image:
-    runs-on: ubuntu-latest
+    runs-on: blacksmith-4vcpu-ubuntu-2404
     defaults:
       run:
         working-directory: './apps/nuq-postgres'

.github/workflows/deploy-playwright.yml

@@ -1,34 +1,81 @@
 name: Deploy Playwright to GHCR
 
-env:
-  DOTNET_VERSION: '6.0.x'
-
 on:
   push:
     branches:
       - main
     paths:
       - apps/playwright-service-ts/**
+      - .github/workflows/deploy-playwright.yml
   workflow_dispatch:
 
 jobs:
-      push-app-image:
-        runs-on: ubuntu-latest
-        defaults:
-          run:
-            working-directory: './apps/playwright-service-ts'
-        steps:
-          - name: 'Checkout GitHub Action'
-            uses: actions/checkout@main
-
-          - name: 'Login to GitHub Container Registry'
-            uses: docker/login-action@v3
-            with:
-              registry: ghcr.io
-              username: ${{github.actor}}
-              password: ${{secrets.GITHUB_TOKEN}}
-
-          - name: 'Build Inventory Image'
-            run: |
-              docker build . --tag ghcr.io/firecrawl/playwright-service:latest
-              docker push ghcr.io/firecrawl/playwright-service:latest
+  build:
+    runs-on: ${{ matrix.runner }}
+    strategy:
+      matrix:
+        include:
+          - platform: linux/amd64
+            runner: blacksmith-4vcpu-ubuntu-2404
+          - platform: linux/arm64
+            runner: blacksmith-4vcpu-ubuntu-2404-arm
+    defaults:
+      run:
+        working-directory: './apps/playwright-service-ts'
+    steps:
+      - name: 'Checkout GitHub Action'
+        uses: actions/checkout@main
+
+      - name: Setup Blacksmith Builder
+        uses: useblacksmith/setup-docker-builder@v1
+
+      - name: 'Login to GitHub Container Registry'
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{github.actor}}
+          password: ${{secrets.GITHUB_TOKEN}}
+
+      - name: Lowercase Repo Owner
+        run: |
+          echo "REPO_OWNER=${GITHUB_REPOSITORY_OWNER,,}" >>${GITHUB_ENV}
+        env:
+          GITHUB_REPOSITORY_OWNER: '${{ github.repository_owner }}'
+
+      - name: Extract platform suffix
+        run: |
+          platform=${{ matrix.platform }}
+          echo "PLATFORM_SUFFIX=${platform//\//-}" >> $GITHUB_ENV
+
+      - name: 'Build and Push Image'
+        uses: useblacksmith/build-push-action@v2
+        with:
+          context: ./apps/playwright-service-ts
+          push: true
+          tags: ghcr.io/${{ env.REPO_OWNER }}/playwright-service:${{ env.PLATFORM_SUFFIX }}
+          platforms: ${{ matrix.platform }}
+          provenance: false
+
+  manifest:
+    runs-on: blacksmith-2vcpu-ubuntu-2404
+    needs: build
+    steps:
+      - name: 'Login to GitHub Container Registry'
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{github.actor}}
+          password: ${{secrets.GITHUB_TOKEN}}
+
+      - name: Lowercase Repo Owner
+        run: |
+          echo "REPO_OWNER=${GITHUB_REPOSITORY_OWNER,,}" >>${GITHUB_ENV}
+        env:
+          GITHUB_REPOSITORY_OWNER: '${{ github.repository_owner }}'
+
+      - name: 'Create and Push Multi-Arch Manifest'
+        run: |
+          docker manifest create ghcr.io/${{ env.REPO_OWNER }}/playwright-service:latest \
+            ghcr.io/${{ env.REPO_OWNER }}/playwright-service:linux-amd64 \
+            ghcr.io/${{ env.REPO_OWNER }}/playwright-service:linux-arm64
+          docker manifest push ghcr.io/${{ env.REPO_OWNER }}/playwright-service:latest