diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000000..ea0b4017d2 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,147 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +name: Release Build + +on: + workflow_dispatch: + inputs: + release_version: + description: 'Release version (e.g., 0.9.0)' + required: true + type: string + skip_gpg: + description: 'Skip GPG signing (for testing)' + required: false + default: false + type: boolean + skip_deploy: + description: 'Skip Maven deploy (for testing)' + required: false + default: false + type: boolean + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + build-source: + name: Build Source Release + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Set up JDK 11 + uses: actions/setup-java@v3 + with: + java-version: '11' + distribution: 'temurin' + cache: maven + + - name: Setup GPG (if not skipped) + if: ${{ inputs.skip_gpg != true }} + run: | + echo "${{ secrets.GPG_PRIVATE_KEY }}" | gpg --import --batch + echo "GPG_PASSPHRASE=${{ secrets.GPG_PASSPHRASE }}" >> $GITHUB_ENV + + - name: Build source release + run: | + cd tools + RELEASE_VERSION=${{ inputs.release_version }} \ + SKIP_GPG=${{ inputs.skip_gpg }} \ + bash ./releasing/create_source_release.sh + + - name: Upload source artifacts + uses: actions/upload-artifact@v4 + with: + name: source-release + path: tools/releasing/release/apache-amoro-*-src.tar.gz* + retention-days: 90 + + build-binary: + name: Build Binary Release (${{ matrix.hadoop }}) + needs: build-source + runs-on: ubuntu-latest + strategy: + matrix: + hadoop: [hadoop2, hadoop3] + steps: + - uses: actions/checkout@v4 + + - name: Set up JDK 11 + uses: actions/setup-java@v3 + with: + java-version: '11' + distribution: 'temurin' + cache: maven + + - name: Download source artifacts + uses: actions/download-artifact@v4 + with: + name: source-release + path: tools/releasing/release + + - name: Setup GPG (if not skipped) + if: ${{ inputs.skip_gpg != true }} + run: | + echo "${{ secrets.GPG_PRIVATE_KEY }}" | gpg --import --batch + echo "GPG_PASSPHRASE=${{ secrets.GPG_PASSPHRASE }}" >> $GITHUB_ENV + + - name: Build binary release + run: | + cd tools + RELEASE_VERSION=${{ inputs.release_version }} \ + SKIP_GPG=${{ inputs.skip_gpg }} \ + HADOOP_PROFILE=${{ matrix.hadoop }} \ + bash ./releasing/create_binary_release.sh + + - name: Upload binary artifacts + uses: actions/upload-artifact@v4 + with: + name: binary-release-${{ matrix.hadoop }} + path: tools/releasing/release/apache-amoro-*-bin-${{ matrix.hadoop }}.tar.gz* + retention-days: 90 + + deploy-maven: + name: Deploy to Maven Staging + needs: build-source + runs-on: ubuntu-latest + if: ${{ inputs.skip_deploy != true && startsWith(github.repository, 'apache/') }} + steps: + - uses: actions/checkout@v4 + + - name: Set up JDK 11 + uses: actions/setup-java@v3 + with: + java-version: '11' + distribution: 'temurin' + cache: maven + gpg-private-key: ${{ secrets.GPG_PRIVATE_KEY }} + gpg-passphrase: ${{ secrets.GPG_PASSPHRASE }} + server-id: apache.releases.https + server-username: NEXUS_USER + server-password: NEXUS_PW + + - name: Deploy to staging + env: + NEXUS_USER: ${{ secrets.NEXUS_USER }} + NEXUS_PW: ${{ secrets.NEXUS_PASSWORD }} + run: | + cd tools + RELEASE_VERSION=${{ inputs.release_version }} \ + bash ./releasing/deploy_staging_jars.sh diff --git a/site/amoro-site/content/community/release-guide.md b/site/amoro-site/content/community/release-guide.md index f936eeb637..87c5fd5a64 100644 --- a/site/amoro-site/content/community/release-guide.md +++ b/site/amoro-site/content/community/release-guide.md @@ -254,21 +254,6 @@ $ git checkout -b 0.8.0-branch ``` Update the version in the new branch using the tool `tools/change-version.sh`: -```bash -OLD="0.8-SNAPSHOT" -NEW="0.8.0-incubating" - -HERE=` basename "$PWD"` -if [[ "$HERE" != "tools" ]]; then - echo "Please only execute in the tools/ directory"; - exit 1; -fi - -# change version in all pom files -find .. -name 'pom.xml' -type f -exec perl -pi -e 's#'"$OLD"'#'"$NEW"'#' {} \; -``` - -Then run the scripts and commit the change: ```shell $ cd tools @@ -288,20 +273,49 @@ $ git tag -a v0.8.0-rc1 -m "Release Apache Amoro 0.8.0 rc1" $ git push apache v0.8.0-rc1 ``` -### Build binary and source release +### Build release packages via GitHub Actions -Build Amoro binary release with scripts: +The release packages (source and binary) are built using GitHub Actions. -```shell -$ cd ${AMORO_SOURCE_HOME}/tools -$ RELEASE_VERSION=0.8.0-incubating bash ./releasing/create_binary_release.sh -``` +#### Configure GitHub Secrets + +Before using the release workflow, ensure the following secrets are configured in the repository settings: + +| Secret | Description | +|--------|-------------| +| `GPG_PRIVATE_KEY` | GPG private key (ASCII armored format) | +| `GPG_PASSPHRASE` | GPG key passphrase | +| `NEXUS_USER` | Apache Nexus username | +| `NEXUS_PASSWORD` | Apache Nexus password | + +#### Trigger the Release Build + +1. Go to the [Actions](https://github.com/apache/amoro/actions) page +2. Select **Release Build** workflow +3. Click **Run workflow** +4. Enter the release version (e.g., `0.8.0-incubating`) +5. Click **Run workflow** + +#### Download Artifacts -Then build source release with scripts: +After the workflow completes, download the artifacts from the workflow run page. The artifacts include: +- `apache-amoro-{version}-src.tar.gz` - Source release +- `apache-amoro-{version}-bin-hadoop2.tar.gz` - Binary release for Hadoop 2 +- `apache-amoro-{version}-bin-hadoop3.tar.gz` - Binary release for Hadoop 3 +- Corresponding `.sha512` and `.asc` signature files + +#### Verify the release packages + +Before publishing, verify the packages: ```shell -$ cd ${AMORO_SOURCE_HOME}/tools -$ RELEASE_VERSION=0.8.0-incubating bash ./releasing/create_source_release.sh +# Check that no AppleDouble files (._*) are included in the source tarball +$ tar tzf apache-amoro-0.8.0-incubating-src.tar.gz | grep "^\./\._" +# The command should return nothing (no AppleDouble files) + +# Verify git.properties is included in the source tarball +$ tar tzf apache-amoro-0.8.0-incubating-src.tar.gz | grep "git.properties" +# Should show: amoro-0.8.0-incubating/amoro-ams/target/classes/amoro/git.properties ``` Validate the source and binary packages according to the [How to validate a new release](../validate-release/) guides. @@ -316,17 +330,6 @@ $ svn commit -m "Release Apache Amoro 0.8.0 rc1" ``` -### Release Apache Nexus - -Next, we will publish the required JAR files to the ​Apache Nexus​ repository to achieve the final goal of releasing them to the ​Maven Central Repository. - -```shell -$ cd ${$AMORO_SOURCE_HOME}/tools -$ RELEASE_VERSION=0.8.0-incubating bash ./releasing/deploy_staging_jars.sh -``` - -You can visit https://repository.apache.org/ and log in to check the publishment status. You can find the publishment process in the `Staging Repositories` section. You need to close the process when all jars are publised. - ## Vote for the new release Next, vote for the new release via email. First complete the vote within the Amoro community, and upon approval, complete the vote within the Amoro community in the Incubator community. For detailed voting guidelines, please refer to [voting process](https://www.apache.org/foundation/voting.html). diff --git a/tools/releasing/create_binary_release.sh b/tools/releasing/create_binary_release.sh index d1ec122fc2..f9136d13bf 100644 --- a/tools/releasing/create_binary_release.sh +++ b/tools/releasing/create_binary_release.sh @@ -54,6 +54,27 @@ AMORO_DIR=`pwd` RELEASE_DIR=${AMORO_DIR}/tools/releasing/release mkdir -p ${RELEASE_DIR} +# Check if source tarball exists and extract it +SOURCE_TARBALL="${RELEASE_DIR}/apache-amoro-${RELEASE_VERSION}-src.tar.gz" +if [ ! -f "$SOURCE_TARBALL" ]; then + echo "ERROR: Source tarball not found at ${SOURCE_TARBALL}" + echo "Please run create_source_release.sh first to generate the source tarball." + echo "This ensures binary releases are built from the source tarball for reproducibility." + exit 1 +fi + +EXTRACT_DIR=${RELEASE_DIR}/amoro-src-extract +rm -rf ${EXTRACT_DIR} +mkdir -p ${EXTRACT_DIR} +cd ${EXTRACT_DIR} + +echo "Extracting source tarball to build binary release" +tar xzf ${SOURCE_TARBALL} +cd amoro-${RELEASE_VERSION} + +# Update AMORO_DIR to point to extracted source +AMORO_DIR=`pwd` + ########################### # build maven package, create Flink distribution, generate signature @@ -64,10 +85,11 @@ make_binary_release() { HADOOP_VERSION=$1 HADOOP_PROFILE="-P$1" fi - echo "Creating ${HADOOP_VERSION} binary release" + echo "Creating ${HADOOP_VERSION} binary release from source tarball" - # enable release profile here (to check for the maven version) - $MVN clean package ${HADOOP_PROFILE} -Pno-extended-disk-storage -Pfail-on-no-git-dir -Pno-plugin-bin -pl ':dist' -am -Dgpg.skip -Dcheckstyle.skip=true -DskipTests + # Build from source tarball (git.properties is already generated) + # Note: We don't use -Pfail-on-no-git-dir since .git is not in the tarball + $MVN clean package ${HADOOP_PROFILE} -Pno-extended-disk-storage -Pno-plugin-bin -pl ':dist' -am -Dgpg.skip -Dcheckstyle.skip=true -DskipTests local TARGET_FILE="apache-amoro-${RELEASE_VERSION}-bin-${HADOOP_VERSION}.tar.gz" cp dist/target/apache-amoro-${RELEASE_VERSION}-bin.tar.gz ${RELEASE_DIR}/${TARGET_FILE} @@ -83,4 +105,8 @@ make_binary_release() { } make_binary_release "hadoop3" -make_binary_release "hadoop2" \ No newline at end of file +make_binary_release "hadoop2" + +# Cleanup extracted directory +cd ${CURR_DIR} +rm -rf ${EXTRACT_DIR} \ No newline at end of file diff --git a/tools/releasing/create_source_release.sh b/tools/releasing/create_source_release.sh index 9ce6ef498f..dbaff2e574 100644 --- a/tools/releasing/create_source_release.sh +++ b/tools/releasing/create_source_release.sh @@ -22,6 +22,7 @@ ## PROJECT_HOME=$(cd "$(dirname "$0")"/../.. || exit; pwd) MVN="${PROJECT_HOME}/mvnw" +SKIP_GPG=${SKIP_GPG:-false} if [ -z "${RELEASE_VERSION:-}" ]; then echo "RELEASE_VERSION was not set." @@ -62,15 +63,31 @@ mkdir -p ${RELEASE_DIR} git clone ${AMORO_DIR} ${CLONE_DIR} cd ${CLONE_DIR} +# Generate git.properties before excluding .git directory +# This ensures version information is available when building from source tarball +echo "Generating git.properties for version ${RELEASE_VERSION}" +${MVN} initialize -pl :amoro-ams -Dgpg.skip -Dcheckstyle.skip=true -DskipTests + rsync -a \ --exclude ".git" --exclude ".gitignore" \ --exclude ".github" --exclude "target" \ --exclude ".idea" --exclude "*.iml" --exclude ".DS_Store" \ + --exclude "._*" --exclude "*/._*" \ --exclude "*/dependency-reduced-pom.xml" \ . amoro-$RELEASE_VERSION +# Copy pre-generated git.properties to the source tarball +# This file is needed for version information when building from source +mkdir -p amoro-$RELEASE_VERSION/amoro-ams/target/classes/amoro +cp amoro-ams/target/classes/amoro/git.properties amoro-$RELEASE_VERSION/amoro-ams/target/classes/amoro/ + tar czf ${RELEASE_DIR}/apache-amoro-${RELEASE_VERSION}-src.tar.gz amoro-$RELEASE_VERSION -gpg --armor --detach-sig ${RELEASE_DIR}/apache-amoro-$RELEASE_VERSION-src.tar.gz + +# Sign the tarball if GPG is not skipped +if [ "$SKIP_GPG" == "false" ] ; then + gpg --armor --detach-sig ${RELEASE_DIR}/apache-amoro-$RELEASE_VERSION-src.tar.gz +fi + cd ${RELEASE_DIR} $SHASUM apache-amoro-$RELEASE_VERSION-src.tar.gz > apache-amoro-$RELEASE_VERSION-src.tar.gz.sha512