diff --git a/.github/actions/sync-content/action.yml b/.github/actions/sync-content/action.yml index 7f02101a01e9..0987917ee184 100644 --- a/.github/actions/sync-content/action.yml +++ b/.github/actions/sync-content/action.yml @@ -31,27 +31,22 @@ runs: with: python-version: '3.12' cache: 'poetry' - - name: Cache local Maven repository - uses: actions/cache@v5 - with: - path: | - ~/.m2/repository/*/*/* - !~/.m2/repository/org/apache/pulsar - key: ${{ runner.os }}-m2-dependencies-website-${{ hashFiles('**/pom.xml') }} - restore-keys: | - ${{ runner.os }}-m2-dependencies-all-${{ hashFiles('**/pom.xml') }} - ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }} - ${{ runner.os }}-m2-dependencies-core-modules- - name: Set up JDK 21 uses: actions/setup-java@v5 with: distribution: corretto java-version: 21 - - name: Run install by skip tests + - name: Set up Gradle + uses: gradle/actions/setup-gradle@39e147cb9de83bb9910b8ef8bd7fff0ee20fcd6f # v5 + - name: Build pulsar artifacts and export classpath working-directory: tmp/pulsar - env: - MAVEN_OPTS: -Xss1500k -Xmx1500m -Daether.connector.http.reuseConnections=false -Daether.connector.requestTimeout=60000 -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 -Dmaven.wagon.http.retryHandler.requestSentEnabled=true -Dmaven.wagon.http.serviceUnavailableRetryStrategy.class=standard -Dmaven.wagon.rto=60000 - run: mvn -B -ntp install -Pcore-modules,swagger,-main -DskipTests -DskipSourceReleaseAssembly=true -Dspotbugs.skip=true -Dlicense.skip=true + run: | + ./gradlew \ + assemble \ + :distribution:pulsar-server-distribution:exportClasspath \ + :distribution:pulsar-shell-distribution:exportClasspath \ + --no-configuration-cache \ + --no-daemon shell: bash - name: Update generated docs working-directory: tools/pytools diff --git a/.github/workflows/ci-dummy-sync.yml b/.github/workflows/ci-dummy-sync.yml index d4653bcb4dd7..d3ca902f4365 100644 --- a/.github/workflows/ci-dummy-sync.yml +++ b/.github/workflows/ci-dummy-sync.yml @@ -33,6 +33,8 @@ jobs: name: Check synchronize content runs-on: ubuntu-latest timeout-minutes: 180 + permissions: + contents: read steps: - uses: actions/checkout@v6 - name: Sync content without push diff --git a/.github/workflows/ci-sync-content.yml b/.github/workflows/ci-sync-content.yml index cf04526e6d68..d53bf3616b73 100644 --- a/.github/workflows/ci-sync-content.yml +++ b/.github/workflows/ci-sync-content.yml @@ -27,9 +27,9 @@ jobs: name: Synchronize Content from Main Repo runs-on: ubuntu-latest timeout-minutes: 180 + permissions: + contents: write steps: - uses: actions/checkout@v6 - with: - token: ${{ secrets.PULSARBOT_TOKEN }} - name: Sync content and push uses: ./.github/actions/sync-content diff --git a/tools/pytools/lib/execute/config_doc_generator.py b/tools/pytools/lib/execute/config_doc_generator.py index d34497af4551..a83ef5de7ba4 100644 --- a/tools/pytools/lib/execute/config_doc_generator.py +++ b/tools/pytools/lib/execute/config_doc_generator.py @@ -22,6 +22,7 @@ import semver from command import find_command, run from constant import site_path +from execute import pulsar_build @dataclass @@ -35,9 +36,11 @@ class Settings: def execute(master: Path, version: str): java = find_command('java', msg='java is required') + build = pulsar_build.detect(master) + pulsar_build.ensure_built(master, build) + reference = site_path() / 'static' / 'reference' / version - classpath = master / 'distribution' / 'server' / 'target' / 'classpath.txt' - classpath = classpath.read_text() + classpath = pulsar_build.server_classpath_file(master, build).read_text() broker_doc_generator = 'org.apache.pulsar.proxy.util.CmdGenerateDocumentation' client_doc_generator = 'org.apache.pulsar.proxy.util.CmdGenerateDocumentation' diff --git a/tools/pytools/lib/execute/pulsar_admin_clidoc_generator.py b/tools/pytools/lib/execute/pulsar_admin_clidoc_generator.py index ba61cf99498f..533919e7c536 100644 --- a/tools/pytools/lib/execute/pulsar_admin_clidoc_generator.py +++ b/tools/pytools/lib/execute/pulsar_admin_clidoc_generator.py @@ -20,9 +20,13 @@ from command import run from constant import site_path +from execute import pulsar_build def execute(basedir: Path, version: str): + build = pulsar_build.detect(basedir) + pulsar_build.ensure_built(basedir, build) + admin = basedir / 'bin' / 'pulsar-admin' reference = site_path() / 'static' / 'reference' / version / 'pulsar-admin' diff --git a/tools/pytools/lib/execute/pulsar_build.py b/tools/pytools/lib/execute/pulsar_build.py new file mode 100644 index 000000000000..24c1f6f90f62 --- /dev/null +++ b/tools/pytools/lib/execute/pulsar_build.py @@ -0,0 +1,81 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import enum +from pathlib import Path + +from command import run + + +class BuildSystem(enum.Enum): + gradle = 'gradle' + maven = 'maven' + + +# Apache Pulsar master switched to a Gradle build (no more `pom.xml` at the +# repo root, classpath/jars land under `build/`). Maintenance branches keep +# the legacy Maven layout (`pom.xml`, `target/`). Detect once per call site so +# the same generator code path works for both checkouts. +def detect(master: Path) -> BuildSystem: + if (master / 'gradlew').exists() and ( + (master / 'build.gradle.kts').exists() or (master / 'build.gradle').exists() + ): + return BuildSystem.gradle + if (master / 'pom.xml').exists(): + return BuildSystem.maven + raise RuntimeError( + f'Cannot determine pulsar build system in {master}: ' + f'no gradlew/build.gradle(.kts) or pom.xml found.' + ) + + +def server_classpath_file(master: Path, build: BuildSystem) -> Path: + if build == BuildSystem.gradle: + return master / 'distribution' / 'server' / 'build' / 'classpath.txt' + return master / 'distribution' / 'server' / 'target' / 'classpath.txt' + + +def swagger_output_dir(master: Path, build: BuildSystem) -> Path: + if build == BuildSystem.gradle: + return master / 'pulsar-broker' / 'build' / 'docs' + return master / 'pulsar-broker' / 'target' / 'docs' + + +# Pre-build the artifacts needed by the reference/CLI doc generators when the +# checkout uses Gradle. Maven branches are still primed by the GitHub Action +# step that runs `mvn install`, so this is a no-op there. +def ensure_built(master: Path, build: BuildSystem) -> None: + if build != BuildSystem.gradle: + return + + classpath_file = server_classpath_file(master, build) + if classpath_file.exists(): + return + + gradlew = master / 'gradlew' + if not gradlew.exists(): + raise RuntimeError(f'gradlew not found at {gradlew}') + + run( + str(gradlew.absolute()), + 'assemble', + ':distribution:pulsar-server-distribution:exportClasspath', + ':distribution:pulsar-shell-distribution:exportClasspath', + '--no-configuration-cache', + '--no-daemon', + cwd=master, + ) diff --git a/tools/pytools/lib/execute/pulsar_clidoc_generator.py b/tools/pytools/lib/execute/pulsar_clidoc_generator.py index e76c4bc9fb6b..74efa5f3e437 100644 --- a/tools/pytools/lib/execute/pulsar_clidoc_generator.py +++ b/tools/pytools/lib/execute/pulsar_clidoc_generator.py @@ -20,9 +20,13 @@ from command import run from constant import site_path +from execute import pulsar_build def execute(basedir: Path, version: str): + build = pulsar_build.detect(basedir) + pulsar_build.ensure_built(basedir, build) + pulsar = basedir / 'bin' / 'pulsar' reference = site_path() / 'static' / 'reference' / version / 'pulsar' diff --git a/tools/pytools/lib/execute/pulsar_client_clidoc_generator.py b/tools/pytools/lib/execute/pulsar_client_clidoc_generator.py index c1621f829db7..10df45134952 100644 --- a/tools/pytools/lib/execute/pulsar_client_clidoc_generator.py +++ b/tools/pytools/lib/execute/pulsar_client_clidoc_generator.py @@ -20,9 +20,13 @@ from command import run from constant import site_path +from execute import pulsar_build def execute(basedir: Path, version: str): + build = pulsar_build.detect(basedir) + pulsar_build.ensure_built(basedir, build) + client = basedir / 'bin' / 'pulsar-client' reference = site_path() / 'static' / 'reference' / version / 'pulsar-client' diff --git a/tools/pytools/lib/execute/pulsar_perf_clidoc_generator.py b/tools/pytools/lib/execute/pulsar_perf_clidoc_generator.py index 15f25b615db8..f85824ff7677 100644 --- a/tools/pytools/lib/execute/pulsar_perf_clidoc_generator.py +++ b/tools/pytools/lib/execute/pulsar_perf_clidoc_generator.py @@ -20,9 +20,13 @@ from command import run from constant import site_path +from execute import pulsar_build def execute(basedir: Path, version: str): + build = pulsar_build.detect(basedir) + pulsar_build.ensure_built(basedir, build) + perf = basedir / 'bin' / 'pulsar-perf' reference = site_path() / 'static' / 'reference' / version / 'pulsar-perf' diff --git a/tools/pytools/lib/execute/site_uploader.py b/tools/pytools/lib/execute/site_uploader.py index 692e44cd19ea..0ce733b31209 100755 --- a/tools/pytools/lib/execute/site_uploader.py +++ b/tools/pytools/lib/execute/site_uploader.py @@ -21,6 +21,7 @@ import os import tempfile from pathlib import Path +from typing import Optional from command import run, find_command, run_pipe @@ -44,7 +45,7 @@ def _should_push(mode: Mode) -> bool: return result -def _do_push(msg: str, site: Path, branch: str, head_sha: str): +def _do_push(msg: str, site: Path, branch: str, head_sha: Optional[str]): git = find_command('git', msg="git is required") # Persist the source-repo SHA we just published so the next run can compute @@ -52,7 +53,11 @@ def _do_push(msg: str, site: Path, branch: str, head_sha: str): # it lands in the same commit as the published content. If `git push` later # fails, this local file is discarded along with the unpushed commit — the # next CI run re-clones asf-site-next and reads the previous .publish-ref. - (site / '.publish-ref').write_text(head_sha + '\n') + # Only relevant when publishing to the build output branch (asf-site-next); + # callers that push generated content into other branches (e.g. site-updater + # syncing into `main`) pass head_sha=None to skip the marker. + if head_sha is not None: + (site / '.publish-ref').write_text(head_sha + '\n') run(git, 'add', '-A', '.', cwd=site) changed = run(git, 'diff-index', '--quiet', 'HEAD', codes={0, 1}, cwd=site).returncode @@ -75,7 +80,7 @@ def _do_push(msg: str, site: Path, branch: str, head_sha: str): run(git, 'push', 'origin', branch, cwd=site) -def execute(mode: Mode, msg: str, site: Path, branch: str, head_sha: str): +def execute(mode: Mode, msg: str, site: Path, branch: str, head_sha: Optional[str] = None): if _should_push(mode): _do_push(msg, site, branch, head_sha) else: # show changes diff --git a/tools/pytools/lib/execute/swagger_generator.py b/tools/pytools/lib/execute/swagger_generator.py index b813c80dc5e2..6ec002bfd577 100644 --- a/tools/pytools/lib/execute/swagger_generator.py +++ b/tools/pytools/lib/execute/swagger_generator.py @@ -17,18 +17,33 @@ import json import os +import sys from pathlib import Path from command import find_command, run from constant import site_path +from execute import pulsar_build def execute(master: Path, version: str): - master_swaggers = master / 'pulsar-broker' / 'target' / 'docs' + build = pulsar_build.detect(master) + master_swaggers = pulsar_build.swagger_output_dir(master, build) - if not master_swaggers.exists(): # generate master swaggers - mvn = find_command('mvn', msg="mvn is required") - run(mvn, '-pl', 'pulsar-broker', 'install', '-DskipTests', '-Pswagger', cwd=master) + if not master_swaggers.exists(): + if build == pulsar_build.BuildSystem.maven: + mvn = find_command('mvn', msg="mvn is required") + run(mvn, '-pl', 'pulsar-broker', 'install', '-DskipTests', '-Pswagger', cwd=master) + else: + # Gradle build on apache/pulsar master does not yet have a task + # that regenerates the Swagger JSONs (the old `mvn -Pswagger` + # invocation has no Gradle equivalent). Skip rather than fail so + # the rest of the docs sync still produces useful output. + print( + f'[swagger_generator] Skipping Swagger generation: Gradle build at {master} ' + f'has no swagger task; expected output dir {master_swaggers} is missing.', + file=sys.stderr, + ) + return os.makedirs(site_path() / 'static' / 'swagger' / version, exist_ok=True) for f in master_swaggers.glob('*.json'):