From d931aad291351e6841d255a232ec72cb1cd08dd7 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Mon, 18 May 2026 10:32:05 +0300 Subject: [PATCH 1/4] Fix sync-content workflow for apache/pulsar gradle build apache/pulsar master switched to a Gradle build, so the previous Maven install in the sync-content composite action no longer produces the jars, classpath, and bin scripts that the docs generators need. Maintenance branches still ship Maven, and the same Python entry points are reused for release-time doc generation there. - Add a pulsar_build helper that detects gradle vs maven from the pulsar checkout layout, exposes the build-system-specific classpath and swagger output dirs, and runs gradle assemble + :distribution:pulsar-{server,shell}-distribution:exportClasspath when the artifacts are missing on a gradle checkout. - Route all five reference/CLI doc generators and swagger_generator through the detector so the same scripts work on master (gradle) and maintenance branches (maven, unchanged). - swagger_generator skips with a warning on gradle since apache/pulsar master has no swagger task yet; the maven mvn -Pswagger fallback is preserved for maintenance branches. - Make site_uploader.execute()'s head_sha optional. site-updater.py never passed it after the .publish-ref change, which left the sync workflow latently broken; head_sha=None now skips the .publish-ref marker that is only meaningful for the asf-site-next branch. - Rewrite .github/actions/sync-content/action.yml to use gradle/actions/setup-gradle and ./gradlew assemble exportClasspath in place of the old mvn install step. - Drop PULSARBOT_TOKEN from .github/workflows/ci-sync-content.yml and grant the default GITHUB_TOKEN contents:write so the workflow can push to main. --- .github/actions/sync-content/action.yml | 27 +++---- .github/workflows/ci-sync-content.yml | 4 +- .../lib/execute/config_doc_generator.py | 7 +- .../execute/pulsar_admin_clidoc_generator.py | 4 + tools/pytools/lib/execute/pulsar_build.py | 81 +++++++++++++++++++ .../lib/execute/pulsar_clidoc_generator.py | 4 + .../execute/pulsar_client_clidoc_generator.py | 4 + .../execute/pulsar_perf_clidoc_generator.py | 4 + tools/pytools/lib/execute/site_uploader.py | 11 ++- .../pytools/lib/execute/swagger_generator.py | 23 +++++- 10 files changed, 143 insertions(+), 26 deletions(-) create mode 100644 tools/pytools/lib/execute/pulsar_build.py diff --git a/.github/actions/sync-content/action.yml b/.github/actions/sync-content/action.yml index 7f02101a01e9..0c1b602c771d 100644 --- a/.github/actions/sync-content/action.yml +++ b/.github/actions/sync-content/action.yml @@ -31,27 +31,24 @@ runs: with: python-version: '3.12' cache: 'poetry' - - name: Cache local Maven repository - uses: actions/cache@v5 - with: - path: | - ~/.m2/repository/*/*/* - !~/.m2/repository/org/apache/pulsar - key: ${{ runner.os }}-m2-dependencies-website-${{ hashFiles('**/pom.xml') }} - restore-keys: | - ${{ runner.os }}-m2-dependencies-all-${{ hashFiles('**/pom.xml') }} - ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }} - ${{ runner.os }}-m2-dependencies-core-modules- - name: Set up JDK 21 uses: actions/setup-java@v5 with: distribution: corretto java-version: 21 - - name: Run install by skip tests + - name: Set up Gradle + uses: gradle/actions/setup-gradle@v5 + with: + build-root-directory: tmp/pulsar + - name: Build pulsar artifacts and export classpath working-directory: tmp/pulsar - env: - MAVEN_OPTS: -Xss1500k -Xmx1500m -Daether.connector.http.reuseConnections=false -Daether.connector.requestTimeout=60000 -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 -Dmaven.wagon.http.retryHandler.requestSentEnabled=true -Dmaven.wagon.http.serviceUnavailableRetryStrategy.class=standard -Dmaven.wagon.rto=60000 - run: mvn -B -ntp install -Pcore-modules,swagger,-main -DskipTests -DskipSourceReleaseAssembly=true -Dspotbugs.skip=true -Dlicense.skip=true + run: | + ./gradlew \ + assemble \ + :distribution:pulsar-server-distribution:exportClasspath \ + :distribution:pulsar-shell-distribution:exportClasspath \ + --no-configuration-cache \ + --no-daemon shell: bash - name: Update generated docs working-directory: tools/pytools diff --git a/.github/workflows/ci-sync-content.yml b/.github/workflows/ci-sync-content.yml index cf04526e6d68..d53bf3616b73 100644 --- a/.github/workflows/ci-sync-content.yml +++ b/.github/workflows/ci-sync-content.yml @@ -27,9 +27,9 @@ jobs: name: Synchronize Content from Main Repo runs-on: ubuntu-latest timeout-minutes: 180 + permissions: + contents: write steps: - uses: actions/checkout@v6 - with: - token: ${{ secrets.PULSARBOT_TOKEN }} - name: Sync content and push uses: ./.github/actions/sync-content diff --git a/tools/pytools/lib/execute/config_doc_generator.py b/tools/pytools/lib/execute/config_doc_generator.py index d34497af4551..a83ef5de7ba4 100644 --- a/tools/pytools/lib/execute/config_doc_generator.py +++ b/tools/pytools/lib/execute/config_doc_generator.py @@ -22,6 +22,7 @@ import semver from command import find_command, run from constant import site_path +from execute import pulsar_build @dataclass @@ -35,9 +36,11 @@ class Settings: def execute(master: Path, version: str): java = find_command('java', msg='java is required') + build = pulsar_build.detect(master) + pulsar_build.ensure_built(master, build) + reference = site_path() / 'static' / 'reference' / version - classpath = master / 'distribution' / 'server' / 'target' / 'classpath.txt' - classpath = classpath.read_text() + classpath = pulsar_build.server_classpath_file(master, build).read_text() broker_doc_generator = 'org.apache.pulsar.proxy.util.CmdGenerateDocumentation' client_doc_generator = 'org.apache.pulsar.proxy.util.CmdGenerateDocumentation' diff --git a/tools/pytools/lib/execute/pulsar_admin_clidoc_generator.py b/tools/pytools/lib/execute/pulsar_admin_clidoc_generator.py index ba61cf99498f..533919e7c536 100644 --- a/tools/pytools/lib/execute/pulsar_admin_clidoc_generator.py +++ b/tools/pytools/lib/execute/pulsar_admin_clidoc_generator.py @@ -20,9 +20,13 @@ from command import run from constant import site_path +from execute import pulsar_build def execute(basedir: Path, version: str): + build = pulsar_build.detect(basedir) + pulsar_build.ensure_built(basedir, build) + admin = basedir / 'bin' / 'pulsar-admin' reference = site_path() / 'static' / 'reference' / version / 'pulsar-admin' diff --git a/tools/pytools/lib/execute/pulsar_build.py b/tools/pytools/lib/execute/pulsar_build.py new file mode 100644 index 000000000000..24c1f6f90f62 --- /dev/null +++ b/tools/pytools/lib/execute/pulsar_build.py @@ -0,0 +1,81 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import enum +from pathlib import Path + +from command import run + + +class BuildSystem(enum.Enum): + gradle = 'gradle' + maven = 'maven' + + +# Apache Pulsar master switched to a Gradle build (no more `pom.xml` at the +# repo root, classpath/jars land under `build/`). Maintenance branches keep +# the legacy Maven layout (`pom.xml`, `target/`). Detect once per call site so +# the same generator code path works for both checkouts. +def detect(master: Path) -> BuildSystem: + if (master / 'gradlew').exists() and ( + (master / 'build.gradle.kts').exists() or (master / 'build.gradle').exists() + ): + return BuildSystem.gradle + if (master / 'pom.xml').exists(): + return BuildSystem.maven + raise RuntimeError( + f'Cannot determine pulsar build system in {master}: ' + f'no gradlew/build.gradle(.kts) or pom.xml found.' + ) + + +def server_classpath_file(master: Path, build: BuildSystem) -> Path: + if build == BuildSystem.gradle: + return master / 'distribution' / 'server' / 'build' / 'classpath.txt' + return master / 'distribution' / 'server' / 'target' / 'classpath.txt' + + +def swagger_output_dir(master: Path, build: BuildSystem) -> Path: + if build == BuildSystem.gradle: + return master / 'pulsar-broker' / 'build' / 'docs' + return master / 'pulsar-broker' / 'target' / 'docs' + + +# Pre-build the artifacts needed by the reference/CLI doc generators when the +# checkout uses Gradle. Maven branches are still primed by the GitHub Action +# step that runs `mvn install`, so this is a no-op there. +def ensure_built(master: Path, build: BuildSystem) -> None: + if build != BuildSystem.gradle: + return + + classpath_file = server_classpath_file(master, build) + if classpath_file.exists(): + return + + gradlew = master / 'gradlew' + if not gradlew.exists(): + raise RuntimeError(f'gradlew not found at {gradlew}') + + run( + str(gradlew.absolute()), + 'assemble', + ':distribution:pulsar-server-distribution:exportClasspath', + ':distribution:pulsar-shell-distribution:exportClasspath', + '--no-configuration-cache', + '--no-daemon', + cwd=master, + ) diff --git a/tools/pytools/lib/execute/pulsar_clidoc_generator.py b/tools/pytools/lib/execute/pulsar_clidoc_generator.py index e76c4bc9fb6b..74efa5f3e437 100644 --- a/tools/pytools/lib/execute/pulsar_clidoc_generator.py +++ b/tools/pytools/lib/execute/pulsar_clidoc_generator.py @@ -20,9 +20,13 @@ from command import run from constant import site_path +from execute import pulsar_build def execute(basedir: Path, version: str): + build = pulsar_build.detect(basedir) + pulsar_build.ensure_built(basedir, build) + pulsar = basedir / 'bin' / 'pulsar' reference = site_path() / 'static' / 'reference' / version / 'pulsar' diff --git a/tools/pytools/lib/execute/pulsar_client_clidoc_generator.py b/tools/pytools/lib/execute/pulsar_client_clidoc_generator.py index c1621f829db7..10df45134952 100644 --- a/tools/pytools/lib/execute/pulsar_client_clidoc_generator.py +++ b/tools/pytools/lib/execute/pulsar_client_clidoc_generator.py @@ -20,9 +20,13 @@ from command import run from constant import site_path +from execute import pulsar_build def execute(basedir: Path, version: str): + build = pulsar_build.detect(basedir) + pulsar_build.ensure_built(basedir, build) + client = basedir / 'bin' / 'pulsar-client' reference = site_path() / 'static' / 'reference' / version / 'pulsar-client' diff --git a/tools/pytools/lib/execute/pulsar_perf_clidoc_generator.py b/tools/pytools/lib/execute/pulsar_perf_clidoc_generator.py index 15f25b615db8..f85824ff7677 100644 --- a/tools/pytools/lib/execute/pulsar_perf_clidoc_generator.py +++ b/tools/pytools/lib/execute/pulsar_perf_clidoc_generator.py @@ -20,9 +20,13 @@ from command import run from constant import site_path +from execute import pulsar_build def execute(basedir: Path, version: str): + build = pulsar_build.detect(basedir) + pulsar_build.ensure_built(basedir, build) + perf = basedir / 'bin' / 'pulsar-perf' reference = site_path() / 'static' / 'reference' / version / 'pulsar-perf' diff --git a/tools/pytools/lib/execute/site_uploader.py b/tools/pytools/lib/execute/site_uploader.py index 692e44cd19ea..0ce733b31209 100755 --- a/tools/pytools/lib/execute/site_uploader.py +++ b/tools/pytools/lib/execute/site_uploader.py @@ -21,6 +21,7 @@ import os import tempfile from pathlib import Path +from typing import Optional from command import run, find_command, run_pipe @@ -44,7 +45,7 @@ def _should_push(mode: Mode) -> bool: return result -def _do_push(msg: str, site: Path, branch: str, head_sha: str): +def _do_push(msg: str, site: Path, branch: str, head_sha: Optional[str]): git = find_command('git', msg="git is required") # Persist the source-repo SHA we just published so the next run can compute @@ -52,7 +53,11 @@ def _do_push(msg: str, site: Path, branch: str, head_sha: str): # it lands in the same commit as the published content. If `git push` later # fails, this local file is discarded along with the unpushed commit — the # next CI run re-clones asf-site-next and reads the previous .publish-ref. - (site / '.publish-ref').write_text(head_sha + '\n') + # Only relevant when publishing to the build output branch (asf-site-next); + # callers that push generated content into other branches (e.g. site-updater + # syncing into `main`) pass head_sha=None to skip the marker. + if head_sha is not None: + (site / '.publish-ref').write_text(head_sha + '\n') run(git, 'add', '-A', '.', cwd=site) changed = run(git, 'diff-index', '--quiet', 'HEAD', codes={0, 1}, cwd=site).returncode @@ -75,7 +80,7 @@ def _do_push(msg: str, site: Path, branch: str, head_sha: str): run(git, 'push', 'origin', branch, cwd=site) -def execute(mode: Mode, msg: str, site: Path, branch: str, head_sha: str): +def execute(mode: Mode, msg: str, site: Path, branch: str, head_sha: Optional[str] = None): if _should_push(mode): _do_push(msg, site, branch, head_sha) else: # show changes diff --git a/tools/pytools/lib/execute/swagger_generator.py b/tools/pytools/lib/execute/swagger_generator.py index b813c80dc5e2..6ec002bfd577 100644 --- a/tools/pytools/lib/execute/swagger_generator.py +++ b/tools/pytools/lib/execute/swagger_generator.py @@ -17,18 +17,33 @@ import json import os +import sys from pathlib import Path from command import find_command, run from constant import site_path +from execute import pulsar_build def execute(master: Path, version: str): - master_swaggers = master / 'pulsar-broker' / 'target' / 'docs' + build = pulsar_build.detect(master) + master_swaggers = pulsar_build.swagger_output_dir(master, build) - if not master_swaggers.exists(): # generate master swaggers - mvn = find_command('mvn', msg="mvn is required") - run(mvn, '-pl', 'pulsar-broker', 'install', '-DskipTests', '-Pswagger', cwd=master) + if not master_swaggers.exists(): + if build == pulsar_build.BuildSystem.maven: + mvn = find_command('mvn', msg="mvn is required") + run(mvn, '-pl', 'pulsar-broker', 'install', '-DskipTests', '-Pswagger', cwd=master) + else: + # Gradle build on apache/pulsar master does not yet have a task + # that regenerates the Swagger JSONs (the old `mvn -Pswagger` + # invocation has no Gradle equivalent). Skip rather than fail so + # the rest of the docs sync still produces useful output. + print( + f'[swagger_generator] Skipping Swagger generation: Gradle build at {master} ' + f'has no swagger task; expected output dir {master_swaggers} is missing.', + file=sys.stderr, + ) + return os.makedirs(site_path() / 'static' / 'swagger' / version, exist_ok=True) for f in master_swaggers.glob('*.json'): From 585868e00004d52300acae246598e03862a4c3c5 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Mon, 18 May 2026 10:34:18 +0300 Subject: [PATCH 2/4] Pin gradle/actions/setup-gradle to SHA --- .github/actions/sync-content/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/sync-content/action.yml b/.github/actions/sync-content/action.yml index 0c1b602c771d..bf982d81ec41 100644 --- a/.github/actions/sync-content/action.yml +++ b/.github/actions/sync-content/action.yml @@ -37,7 +37,7 @@ runs: distribution: corretto java-version: 21 - name: Set up Gradle - uses: gradle/actions/setup-gradle@v5 + uses: gradle/actions/setup-gradle@39e147cb9de83bb9910b8ef8bd7fff0ee20fcd6f # v5 with: build-root-directory: tmp/pulsar - name: Build pulsar artifacts and export classpath From e5eb2ed4a8e7e000011e512560cca9866dd3ec07 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Mon, 18 May 2026 10:41:38 +0300 Subject: [PATCH 3/4] Add explicit contents:read permission to ci-dummy-sync --- .github/workflows/ci-dummy-sync.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ci-dummy-sync.yml b/.github/workflows/ci-dummy-sync.yml index d4653bcb4dd7..d3ca902f4365 100644 --- a/.github/workflows/ci-dummy-sync.yml +++ b/.github/workflows/ci-dummy-sync.yml @@ -33,6 +33,8 @@ jobs: name: Check synchronize content runs-on: ubuntu-latest timeout-minutes: 180 + permissions: + contents: read steps: - uses: actions/checkout@v6 - name: Sync content without push From d1ceb087b575d7a9e6fb5f7b57191ebc608f456c Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Mon, 18 May 2026 11:10:57 +0300 Subject: [PATCH 4/4] Drop invalid build-root-directory input from setup-gradle --- .github/actions/sync-content/action.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/actions/sync-content/action.yml b/.github/actions/sync-content/action.yml index bf982d81ec41..0987917ee184 100644 --- a/.github/actions/sync-content/action.yml +++ b/.github/actions/sync-content/action.yml @@ -38,8 +38,6 @@ runs: java-version: 21 - name: Set up Gradle uses: gradle/actions/setup-gradle@39e147cb9de83bb9910b8ef8bd7fff0ee20fcd6f # v5 - with: - build-root-directory: tmp/pulsar - name: Build pulsar artifacts and export classpath working-directory: tmp/pulsar run: |