Fix sync-content workflow for apache/pulsar gradle build

.github/actions/sync-content/action.yml

@@ -31,27 +31,22 @@ runs:
       with:
         python-version: '3.12'
         cache: 'poetry'
-    - name: Cache local Maven repository
-      uses: actions/cache@v5
-      with:
-        path: |
-          ~/.m2/repository/*/*/*
-          !~/.m2/repository/org/apache/pulsar
-        key: ${{ runner.os }}-m2-dependencies-website-${{ hashFiles('**/pom.xml') }}
-        restore-keys: |
-          ${{ runner.os }}-m2-dependencies-all-${{ hashFiles('**/pom.xml') }}
-          ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }}
-          ${{ runner.os }}-m2-dependencies-core-modules-
     - name: Set up JDK 21
       uses: actions/setup-java@v5
       with:
         distribution: corretto
         java-version: 21
-    - name: Run install by skip tests
+    - name: Set up Gradle
+      uses: gradle/actions/setup-gradle@39e147cb9de83bb9910b8ef8bd7fff0ee20fcd6f # v5
+    - name: Build pulsar artifacts and export classpath
       working-directory: tmp/pulsar
-      env:
-        MAVEN_OPTS: -Xss1500k -Xmx1500m -Daether.connector.http.reuseConnections=false -Daether.connector.requestTimeout=60000 -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 -Dmaven.wagon.http.retryHandler.requestSentEnabled=true -Dmaven.wagon.http.serviceUnavailableRetryStrategy.class=standard -Dmaven.wagon.rto=60000
-      run: mvn -B -ntp install -Pcore-modules,swagger,-main -DskipTests -DskipSourceReleaseAssembly=true -Dspotbugs.skip=true -Dlicense.skip=true
+      run: |
+        ./gradlew \
+          assemble \
+          :distribution:pulsar-server-distribution:exportClasspath \
+          :distribution:pulsar-shell-distribution:exportClasspath \
+          --no-configuration-cache \
+          --no-daemon
       shell: bash
     - name: Update generated docs
       working-directory: tools/pytools

.github/workflows/ci-dummy-sync.yml

@@ -33,6 +33,8 @@ jobs:
     name: Check synchronize content
     runs-on: ubuntu-latest
     timeout-minutes: 180
+    permissions:
+      contents: read
     steps:
       - uses: actions/checkout@v6
       - name: Sync content without push

.github/workflows/ci-sync-content.yml

@@ -27,9 +27,9 @@ jobs:
     name: Synchronize Content from Main Repo
     runs-on: ubuntu-latest
     timeout-minutes: 180
+    permissions:
+      contents: write
     steps:
       - uses: actions/checkout@v6
-        with:
-          token: ${{ secrets.PULSARBOT_TOKEN }}
       - name: Sync content and push
         uses: ./.github/actions/sync-content

tools/pytools/lib/execute/config_doc_generator.py

@@ -22,6 +22,7 @@
 import semver
 from command import find_command, run
 from constant import site_path
+from execute import pulsar_build
 
 
 @dataclass
@@ -35,9 +36,11 @@ class Settings:
 def execute(master: Path, version: str):
     java = find_command('java', msg='java is required')
 
+    build = pulsar_build.detect(master)
+    pulsar_build.ensure_built(master, build)
+
     reference = site_path() / 'static' / 'reference' / version
-    classpath = master / 'distribution' / 'server' / 'target' / 'classpath.txt'
-    classpath = classpath.read_text()
+    classpath = pulsar_build.server_classpath_file(master, build).read_text()
 
     broker_doc_generator = 'org.apache.pulsar.proxy.util.CmdGenerateDocumentation'
     client_doc_generator = 'org.apache.pulsar.proxy.util.CmdGenerateDocumentation'

tools/pytools/lib/execute/pulsar_admin_clidoc_generator.py

@@ -20,9 +20,13 @@
 
 from command import run
 from constant import site_path
+from execute import pulsar_build
 
 
 def execute(basedir: Path, version: str):
+    build = pulsar_build.detect(basedir)
+    pulsar_build.ensure_built(basedir, build)
+
     admin = basedir / 'bin' / 'pulsar-admin'
     reference = site_path() / 'static' / 'reference' / version / 'pulsar-admin'
 

tools/pytools/lib/execute/pulsar_build.py

@@ -0,0 +1,81 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import enum
+from pathlib import Path
+
+from command import run
+
+
+class BuildSystem(enum.Enum):
+    gradle = 'gradle'
+    maven = 'maven'
+
+
+# Apache Pulsar master switched to a Gradle build (no more `pom.xml` at the
+# repo root, classpath/jars land under `build/`). Maintenance branches keep
+# the legacy Maven layout (`pom.xml`, `target/`). Detect once per call site so
+# the same generator code path works for both checkouts.
+def detect(master: Path) -> BuildSystem:
+    if (master / 'gradlew').exists() and (
+        (master / 'build.gradle.kts').exists() or (master / 'build.gradle').exists()
+    ):
+        return BuildSystem.gradle
+    if (master / 'pom.xml').exists():
+        return BuildSystem.maven
+    raise RuntimeError(
+        f'Cannot determine pulsar build system in {master}: '
+        f'no gradlew/build.gradle(.kts) or pom.xml found.'
+    )
+
+
+def server_classpath_file(master: Path, build: BuildSystem) -> Path:
+    if build == BuildSystem.gradle:
+        return master / 'distribution' / 'server' / 'build' / 'classpath.txt'
+    return master / 'distribution' / 'server' / 'target' / 'classpath.txt'
+
+
+def swagger_output_dir(master: Path, build: BuildSystem) -> Path:
+    if build == BuildSystem.gradle:
+        return master / 'pulsar-broker' / 'build' / 'docs'
+    return master / 'pulsar-broker' / 'target' / 'docs'
+
+
+# Pre-build the artifacts needed by the reference/CLI doc generators when the
+# checkout uses Gradle. Maven branches are still primed by the GitHub Action
+# step that runs `mvn install`, so this is a no-op there.
+def ensure_built(master: Path, build: BuildSystem) -> None:
+    if build != BuildSystem.gradle:
+        return
+
+    classpath_file = server_classpath_file(master, build)
+    if classpath_file.exists():
+        return
+
+    gradlew = master / 'gradlew'
+    if not gradlew.exists():
+        raise RuntimeError(f'gradlew not found at {gradlew}')
+
+    run(
+        str(gradlew.absolute()),
+        'assemble',
+        ':distribution:pulsar-server-distribution:exportClasspath',
+        ':distribution:pulsar-shell-distribution:exportClasspath',
+        '--no-configuration-cache',
+        '--no-daemon',
+        cwd=master,
+    )

tools/pytools/lib/execute/pulsar_clidoc_generator.py

@@ -20,9 +20,13 @@
 
 from command import run
 from constant import site_path
+from execute import pulsar_build
 
 
 def execute(basedir: Path, version: str):
+    build = pulsar_build.detect(basedir)
+    pulsar_build.ensure_built(basedir, build)
+
     pulsar = basedir / 'bin' / 'pulsar'
     reference = site_path() / 'static' / 'reference' / version / 'pulsar'
 

tools/pytools/lib/execute/pulsar_client_clidoc_generator.py

@@ -20,9 +20,13 @@
 
 from command import run
 from constant import site_path
+from execute import pulsar_build
 
 
 def execute(basedir: Path, version: str):
+    build = pulsar_build.detect(basedir)
+    pulsar_build.ensure_built(basedir, build)
+
     client = basedir / 'bin' / 'pulsar-client'
     reference = site_path() / 'static' / 'reference' / version / 'pulsar-client'
 

tools/pytools/lib/execute/pulsar_perf_clidoc_generator.py

@@ -20,9 +20,13 @@
 
 from command import run
 from constant import site_path
+from execute import pulsar_build
 
 
 def execute(basedir: Path, version: str):
+    build = pulsar_build.detect(basedir)
+    pulsar_build.ensure_built(basedir, build)
+
     perf = basedir / 'bin' / 'pulsar-perf'
     reference = site_path() / 'static' / 'reference' / version / 'pulsar-perf'
 

tools/pytools/lib/execute/site_uploader.py

@@ -21,6 +21,7 @@
 import os
 import tempfile
 from pathlib import Path
+from typing import Optional
 
 from command import run, find_command, run_pipe
 
@@ -44,15 +45,19 @@ def _should_push(mode: Mode) -> bool:
     return result
 
 
-def _do_push(msg: str, site: Path, branch: str, head_sha: str):
+def _do_push(msg: str, site: Path, branch: str, head_sha: Optional[str]):
     git = find_command('git', msg="git is required")
 
     # Persist the source-repo SHA we just published so the next run can compute
     # the changed-files set against this point. Written before `git add -A` so
     # it lands in the same commit as the published content. If `git push` later
     # fails, this local file is discarded along with the unpushed commit — the
     # next CI run re-clones asf-site-next and reads the previous .publish-ref.
-    (site / '.publish-ref').write_text(head_sha + '\n')
+    # Only relevant when publishing to the build output branch (asf-site-next);
+    # callers that push generated content into other branches (e.g. site-updater
+    # syncing into `main`) pass head_sha=None to skip the marker.
+    if head_sha is not None:
+        (site / '.publish-ref').write_text(head_sha + '\n')
 
     run(git, 'add', '-A', '.', cwd=site)
     changed = run(git, 'diff-index', '--quiet', 'HEAD', codes={0, 1}, cwd=site).returncode
@@ -75,7 +80,7 @@ def _do_push(msg: str, site: Path, branch: str, head_sha: str):
         run(git, 'push', 'origin', branch, cwd=site)
 
 
-def execute(mode: Mode, msg: str, site: Path, branch: str, head_sha: str):
+def execute(mode: Mode, msg: str, site: Path, branch: str, head_sha: Optional[str] = None):
     if _should_push(mode):
         _do_push(msg, site, branch, head_sha)
     else:  # show changes

tools/pytools/lib/execute/swagger_generator.py

@@ -17,18 +17,33 @@
 
 import json
 import os
+import sys
 from pathlib import Path
 
 from command import find_command, run
 from constant import site_path
+from execute import pulsar_build
 
 
 def execute(master: Path, version: str):
-    master_swaggers = master / 'pulsar-broker' / 'target' / 'docs'
+    build = pulsar_build.detect(master)
+    master_swaggers = pulsar_build.swagger_output_dir(master, build)
 
-    if not master_swaggers.exists():  # generate master swaggers
-        mvn = find_command('mvn', msg="mvn is required")
-        run(mvn, '-pl', 'pulsar-broker', 'install', '-DskipTests', '-Pswagger', cwd=master)
+    if not master_swaggers.exists():
+        if build == pulsar_build.BuildSystem.maven:
+            mvn = find_command('mvn', msg="mvn is required")
+            run(mvn, '-pl', 'pulsar-broker', 'install', '-DskipTests', '-Pswagger', cwd=master)
+        else:
+            # Gradle build on apache/pulsar master does not yet have a task
+            # that regenerates the Swagger JSONs (the old `mvn -Pswagger`
+            # invocation has no Gradle equivalent). Skip rather than fail so
+            # the rest of the docs sync still produces useful output.
+            print(
+                f'[swagger_generator] Skipping Swagger generation: Gradle build at {master} '
+                f'has no swagger task; expected output dir {master_swaggers} is missing.',
+                file=sys.stderr,
+            )
+            return
 
     os.makedirs(site_path() / 'static' / 'swagger' / version, exist_ok=True)
     for f in master_swaggers.glob('*.json'):