From ec03ae7de3e2d244fc5afc35cc25c8b986ad0db5 Mon Sep 17 00:00:00 2001 From: "majin.nathan" Date: Thu, 5 Mar 2026 15:49:54 +0800 Subject: [PATCH 1/3] [AMORO-3770] Improve the release procedure This PR improves the release procedure based on feedback from 0.8.1-incubating release voting: 1. Exclude AppleDouble files (._*) from source tarball - Add --exclude '._*' and --exclude '*/._*' to rsync command - These macOS metadata files should not be in Apache releases 2. Include pre-generated git.properties in source tarball - Generate git.properties before excluding .git directory - Copy the file to tarball since target/ is excluded - This ensures version info is available when building from source 3. Build binary releases from source tarball for reproducibility - create_binary_release.sh now requires source tarball to exist - Extracts and builds from the source tarball - Ensures binary releases match the source release 4. Update release guide with correct build order and verification steps --- .../content/community/release-guide.md | 26 +++++++++++--- tools/releasing/create_binary_release.sh | 34 ++++++++++++++++--- tools/releasing/create_source_release.sh | 11 ++++++ 3 files changed, 63 insertions(+), 8 deletions(-) diff --git a/site/amoro-site/content/community/release-guide.md b/site/amoro-site/content/community/release-guide.md index f936eeb637..7957343c2f 100644 --- a/site/amoro-site/content/community/release-guide.md +++ b/site/amoro-site/content/community/release-guide.md @@ -290,18 +290,36 @@ $ git push apache v0.8.0-rc1 ### Build binary and source release -Build Amoro binary release with scripts: +First, build the source release: ```shell $ cd ${AMORO_SOURCE_HOME}/tools -$ RELEASE_VERSION=0.8.0-incubating bash ./releasing/create_binary_release.sh +$ RELEASE_VERSION=0.8.0-incubating bash ./releasing/create_source_release.sh ``` -Then build source release with scripts: +Then build the binary release (this will use the source tarball): ```shell $ cd ${AMORO_SOURCE_HOME}/tools -$ RELEASE_VERSION=0.8.0-incubating bash ./releasing/create_source_release.sh +$ RELEASE_VERSION=0.8.0-incubating bash ./releasing/create_binary_release.sh +``` + +{{< hint warning >}} +**Important**: The binary release must be built AFTER the source release. The `create_binary_release.sh` script extracts and builds from the source tarball to ensure reproducibility. +{{< /hint >}} + +#### Verify the release packages + +Before publishing, verify the packages: + +```shell +# Check that no AppleDouble files (._*) are included in the source tarball +$ tar tzf ${AMORO_SOURCE_HOME}/tools/releasing/release/apache-amoro-0.8.0-incubating-src.tar.gz | grep "^\./\._" +# The command should return nothing (no AppleDouble files) + +# Verify git.properties is included in the source tarball +$ tar tzf ${AMORO_SOURCE_HOME}/tools/releasing/release/apache-amoro-0.8.0-incubating-src.tar.gz | grep "git.properties" +# Should show: amoro-0.8.0-incubating/amoro-ams/target/classes/amoro/git.properties ``` Validate the source and binary packages according to the [How to validate a new release](../validate-release/) guides. diff --git a/tools/releasing/create_binary_release.sh b/tools/releasing/create_binary_release.sh index d1ec122fc2..f9136d13bf 100644 --- a/tools/releasing/create_binary_release.sh +++ b/tools/releasing/create_binary_release.sh @@ -54,6 +54,27 @@ AMORO_DIR=`pwd` RELEASE_DIR=${AMORO_DIR}/tools/releasing/release mkdir -p ${RELEASE_DIR} +# Check if source tarball exists and extract it +SOURCE_TARBALL="${RELEASE_DIR}/apache-amoro-${RELEASE_VERSION}-src.tar.gz" +if [ ! -f "$SOURCE_TARBALL" ]; then + echo "ERROR: Source tarball not found at ${SOURCE_TARBALL}" + echo "Please run create_source_release.sh first to generate the source tarball." + echo "This ensures binary releases are built from the source tarball for reproducibility." + exit 1 +fi + +EXTRACT_DIR=${RELEASE_DIR}/amoro-src-extract +rm -rf ${EXTRACT_DIR} +mkdir -p ${EXTRACT_DIR} +cd ${EXTRACT_DIR} + +echo "Extracting source tarball to build binary release" +tar xzf ${SOURCE_TARBALL} +cd amoro-${RELEASE_VERSION} + +# Update AMORO_DIR to point to extracted source +AMORO_DIR=`pwd` + ########################### # build maven package, create Flink distribution, generate signature @@ -64,10 +85,11 @@ make_binary_release() { HADOOP_VERSION=$1 HADOOP_PROFILE="-P$1" fi - echo "Creating ${HADOOP_VERSION} binary release" + echo "Creating ${HADOOP_VERSION} binary release from source tarball" - # enable release profile here (to check for the maven version) - $MVN clean package ${HADOOP_PROFILE} -Pno-extended-disk-storage -Pfail-on-no-git-dir -Pno-plugin-bin -pl ':dist' -am -Dgpg.skip -Dcheckstyle.skip=true -DskipTests + # Build from source tarball (git.properties is already generated) + # Note: We don't use -Pfail-on-no-git-dir since .git is not in the tarball + $MVN clean package ${HADOOP_PROFILE} -Pno-extended-disk-storage -Pno-plugin-bin -pl ':dist' -am -Dgpg.skip -Dcheckstyle.skip=true -DskipTests local TARGET_FILE="apache-amoro-${RELEASE_VERSION}-bin-${HADOOP_VERSION}.tar.gz" cp dist/target/apache-amoro-${RELEASE_VERSION}-bin.tar.gz ${RELEASE_DIR}/${TARGET_FILE} @@ -83,4 +105,8 @@ make_binary_release() { } make_binary_release "hadoop3" -make_binary_release "hadoop2" \ No newline at end of file +make_binary_release "hadoop2" + +# Cleanup extracted directory +cd ${CURR_DIR} +rm -rf ${EXTRACT_DIR} \ No newline at end of file diff --git a/tools/releasing/create_source_release.sh b/tools/releasing/create_source_release.sh index 9ce6ef498f..5495e161aa 100644 --- a/tools/releasing/create_source_release.sh +++ b/tools/releasing/create_source_release.sh @@ -62,13 +62,24 @@ mkdir -p ${RELEASE_DIR} git clone ${AMORO_DIR} ${CLONE_DIR} cd ${CLONE_DIR} +# Generate git.properties before excluding .git directory +# This ensures version information is available when building from source tarball +echo "Generating git.properties for version ${RELEASE_VERSION}" +${MVN} initialize -pl :amoro-ams -Dgpg.skip -Dcheckstyle.skip=true -DskipTests + rsync -a \ --exclude ".git" --exclude ".gitignore" \ --exclude ".github" --exclude "target" \ --exclude ".idea" --exclude "*.iml" --exclude ".DS_Store" \ + --exclude "._*" --exclude "*/._*" \ --exclude "*/dependency-reduced-pom.xml" \ . amoro-$RELEASE_VERSION +# Copy pre-generated git.properties to the source tarball +# This file is needed for version information when building from source +mkdir -p amoro-$RELEASE_VERSION/amoro-ams/target/classes/amoro +cp amoro-ams/target/classes/amoro/git.properties amoro-$RELEASE_VERSION/amoro-ams/target/classes/amoro/ + tar czf ${RELEASE_DIR}/apache-amoro-${RELEASE_VERSION}-src.tar.gz amoro-$RELEASE_VERSION gpg --armor --detach-sig ${RELEASE_DIR}/apache-amoro-$RELEASE_VERSION-src.tar.gz cd ${RELEASE_DIR} From fe88643d239b0e1161276506b54e086b90688271 Mon Sep 17 00:00:00 2001 From: "majin.nathan" Date: Thu, 5 Mar 2026 21:14:08 +0800 Subject: [PATCH 2/3] Add GitHub Actions workflow for release automation - Add release.yml workflow for manual release builds - Support skip_gpg and skip_deploy parameters for testing - Update create_source_release.sh to support SKIP_GPG env variable --- .github/workflows/release.yml | 147 +++++++++++++++++++++++ tools/releasing/create_source_release.sh | 8 +- 2 files changed, 154 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/release.yml diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000000..ea0b4017d2 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,147 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +name: Release Build + +on: + workflow_dispatch: + inputs: + release_version: + description: 'Release version (e.g., 0.9.0)' + required: true + type: string + skip_gpg: + description: 'Skip GPG signing (for testing)' + required: false + default: false + type: boolean + skip_deploy: + description: 'Skip Maven deploy (for testing)' + required: false + default: false + type: boolean + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + build-source: + name: Build Source Release + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Set up JDK 11 + uses: actions/setup-java@v3 + with: + java-version: '11' + distribution: 'temurin' + cache: maven + + - name: Setup GPG (if not skipped) + if: ${{ inputs.skip_gpg != true }} + run: | + echo "${{ secrets.GPG_PRIVATE_KEY }}" | gpg --import --batch + echo "GPG_PASSPHRASE=${{ secrets.GPG_PASSPHRASE }}" >> $GITHUB_ENV + + - name: Build source release + run: | + cd tools + RELEASE_VERSION=${{ inputs.release_version }} \ + SKIP_GPG=${{ inputs.skip_gpg }} \ + bash ./releasing/create_source_release.sh + + - name: Upload source artifacts + uses: actions/upload-artifact@v4 + with: + name: source-release + path: tools/releasing/release/apache-amoro-*-src.tar.gz* + retention-days: 90 + + build-binary: + name: Build Binary Release (${{ matrix.hadoop }}) + needs: build-source + runs-on: ubuntu-latest + strategy: + matrix: + hadoop: [hadoop2, hadoop3] + steps: + - uses: actions/checkout@v4 + + - name: Set up JDK 11 + uses: actions/setup-java@v3 + with: + java-version: '11' + distribution: 'temurin' + cache: maven + + - name: Download source artifacts + uses: actions/download-artifact@v4 + with: + name: source-release + path: tools/releasing/release + + - name: Setup GPG (if not skipped) + if: ${{ inputs.skip_gpg != true }} + run: | + echo "${{ secrets.GPG_PRIVATE_KEY }}" | gpg --import --batch + echo "GPG_PASSPHRASE=${{ secrets.GPG_PASSPHRASE }}" >> $GITHUB_ENV + + - name: Build binary release + run: | + cd tools + RELEASE_VERSION=${{ inputs.release_version }} \ + SKIP_GPG=${{ inputs.skip_gpg }} \ + HADOOP_PROFILE=${{ matrix.hadoop }} \ + bash ./releasing/create_binary_release.sh + + - name: Upload binary artifacts + uses: actions/upload-artifact@v4 + with: + name: binary-release-${{ matrix.hadoop }} + path: tools/releasing/release/apache-amoro-*-bin-${{ matrix.hadoop }}.tar.gz* + retention-days: 90 + + deploy-maven: + name: Deploy to Maven Staging + needs: build-source + runs-on: ubuntu-latest + if: ${{ inputs.skip_deploy != true && startsWith(github.repository, 'apache/') }} + steps: + - uses: actions/checkout@v4 + + - name: Set up JDK 11 + uses: actions/setup-java@v3 + with: + java-version: '11' + distribution: 'temurin' + cache: maven + gpg-private-key: ${{ secrets.GPG_PRIVATE_KEY }} + gpg-passphrase: ${{ secrets.GPG_PASSPHRASE }} + server-id: apache.releases.https + server-username: NEXUS_USER + server-password: NEXUS_PW + + - name: Deploy to staging + env: + NEXUS_USER: ${{ secrets.NEXUS_USER }} + NEXUS_PW: ${{ secrets.NEXUS_PASSWORD }} + run: | + cd tools + RELEASE_VERSION=${{ inputs.release_version }} \ + bash ./releasing/deploy_staging_jars.sh diff --git a/tools/releasing/create_source_release.sh b/tools/releasing/create_source_release.sh index 5495e161aa..dbaff2e574 100644 --- a/tools/releasing/create_source_release.sh +++ b/tools/releasing/create_source_release.sh @@ -22,6 +22,7 @@ ## PROJECT_HOME=$(cd "$(dirname "$0")"/../.. || exit; pwd) MVN="${PROJECT_HOME}/mvnw" +SKIP_GPG=${SKIP_GPG:-false} if [ -z "${RELEASE_VERSION:-}" ]; then echo "RELEASE_VERSION was not set." @@ -81,7 +82,12 @@ mkdir -p amoro-$RELEASE_VERSION/amoro-ams/target/classes/amoro cp amoro-ams/target/classes/amoro/git.properties amoro-$RELEASE_VERSION/amoro-ams/target/classes/amoro/ tar czf ${RELEASE_DIR}/apache-amoro-${RELEASE_VERSION}-src.tar.gz amoro-$RELEASE_VERSION -gpg --armor --detach-sig ${RELEASE_DIR}/apache-amoro-$RELEASE_VERSION-src.tar.gz + +# Sign the tarball if GPG is not skipped +if [ "$SKIP_GPG" == "false" ] ; then + gpg --armor --detach-sig ${RELEASE_DIR}/apache-amoro-$RELEASE_VERSION-src.tar.gz +fi + cd ${RELEASE_DIR} $SHASUM apache-amoro-$RELEASE_VERSION-src.tar.gz > apache-amoro-$RELEASE_VERSION-src.tar.gz.sha512 From 5dfe1717e2b5f21defbd449caae66d6dca385eea Mon Sep 17 00:00:00 2001 From: "majin.nathan" Date: Thu, 5 Mar 2026 21:22:42 +0800 Subject: [PATCH 3/3] Update release guide to use GitHub Actions for build - Remove manual build commands (now handled by GitHub Actions) - Add GitHub Actions workflow usage instructions - Add secrets configuration requirements - Simplify the build release section --- .../content/community/release-guide.md | 69 ++++++++----------- 1 file changed, 27 insertions(+), 42 deletions(-) diff --git a/site/amoro-site/content/community/release-guide.md b/site/amoro-site/content/community/release-guide.md index 7957343c2f..87c5fd5a64 100644 --- a/site/amoro-site/content/community/release-guide.md +++ b/site/amoro-site/content/community/release-guide.md @@ -254,21 +254,6 @@ $ git checkout -b 0.8.0-branch ``` Update the version in the new branch using the tool `tools/change-version.sh`: -```bash -OLD="0.8-SNAPSHOT" -NEW="0.8.0-incubating" - -HERE=` basename "$PWD"` -if [[ "$HERE" != "tools" ]]; then - echo "Please only execute in the tools/ directory"; - exit 1; -fi - -# change version in all pom files -find .. -name 'pom.xml' -type f -exec perl -pi -e 's#'"$OLD"'#'"$NEW"'#' {} \; -``` - -Then run the scripts and commit the change: ```shell $ cd tools @@ -288,25 +273,36 @@ $ git tag -a v0.8.0-rc1 -m "Release Apache Amoro 0.8.0 rc1" $ git push apache v0.8.0-rc1 ``` -### Build binary and source release +### Build release packages via GitHub Actions -First, build the source release: +The release packages (source and binary) are built using GitHub Actions. -```shell -$ cd ${AMORO_SOURCE_HOME}/tools -$ RELEASE_VERSION=0.8.0-incubating bash ./releasing/create_source_release.sh -``` +#### Configure GitHub Secrets -Then build the binary release (this will use the source tarball): +Before using the release workflow, ensure the following secrets are configured in the repository settings: -```shell -$ cd ${AMORO_SOURCE_HOME}/tools -$ RELEASE_VERSION=0.8.0-incubating bash ./releasing/create_binary_release.sh -``` +| Secret | Description | +|--------|-------------| +| `GPG_PRIVATE_KEY` | GPG private key (ASCII armored format) | +| `GPG_PASSPHRASE` | GPG key passphrase | +| `NEXUS_USER` | Apache Nexus username | +| `NEXUS_PASSWORD` | Apache Nexus password | -{{< hint warning >}} -**Important**: The binary release must be built AFTER the source release. The `create_binary_release.sh` script extracts and builds from the source tarball to ensure reproducibility. -{{< /hint >}} +#### Trigger the Release Build + +1. Go to the [Actions](https://github.com/apache/amoro/actions) page +2. Select **Release Build** workflow +3. Click **Run workflow** +4. Enter the release version (e.g., `0.8.0-incubating`) +5. Click **Run workflow** + +#### Download Artifacts + +After the workflow completes, download the artifacts from the workflow run page. The artifacts include: +- `apache-amoro-{version}-src.tar.gz` - Source release +- `apache-amoro-{version}-bin-hadoop2.tar.gz` - Binary release for Hadoop 2 +- `apache-amoro-{version}-bin-hadoop3.tar.gz` - Binary release for Hadoop 3 +- Corresponding `.sha512` and `.asc` signature files #### Verify the release packages @@ -314,11 +310,11 @@ Before publishing, verify the packages: ```shell # Check that no AppleDouble files (._*) are included in the source tarball -$ tar tzf ${AMORO_SOURCE_HOME}/tools/releasing/release/apache-amoro-0.8.0-incubating-src.tar.gz | grep "^\./\._" +$ tar tzf apache-amoro-0.8.0-incubating-src.tar.gz | grep "^\./\._" # The command should return nothing (no AppleDouble files) # Verify git.properties is included in the source tarball -$ tar tzf ${AMORO_SOURCE_HOME}/tools/releasing/release/apache-amoro-0.8.0-incubating-src.tar.gz | grep "git.properties" +$ tar tzf apache-amoro-0.8.0-incubating-src.tar.gz | grep "git.properties" # Should show: amoro-0.8.0-incubating/amoro-ams/target/classes/amoro/git.properties ``` @@ -334,17 +330,6 @@ $ svn commit -m "Release Apache Amoro 0.8.0 rc1" ``` -### Release Apache Nexus - -Next, we will publish the required JAR files to the ​Apache Nexus​ repository to achieve the final goal of releasing them to the ​Maven Central Repository. - -```shell -$ cd ${$AMORO_SOURCE_HOME}/tools -$ RELEASE_VERSION=0.8.0-incubating bash ./releasing/deploy_staging_jars.sh -``` - -You can visit https://repository.apache.org/ and log in to check the publishment status. You can find the publishment process in the `Staging Repositories` section. You need to close the process when all jars are publised. - ## Vote for the new release Next, vote for the new release via email. First complete the vote within the Amoro community, and upon approval, complete the vote within the Amoro community in the Incubator community. For detailed voting guidelines, please refer to [voting process](https://www.apache.org/foundation/voting.html).