diff --git a/.github/workflows/build-ratis.yml b/.github/workflows/build-ratis.yml new file mode 100644 index 000000000000..7c695de9541d --- /dev/null +++ b/.github/workflows/build-ratis.yml @@ -0,0 +1,137 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This workflow can be called by other workflows to build Ratis. +# +# Inputs: +# - Ratis repo +# - the commit to build +# Outputs: +# - various version numbers that need to be provided to the Ozone build process. +# - Ratis repository is uploaded as an artifact named `ratis-jars` +# +# See `intermittent-test-check.yml` as an example use of this workflow. + +name: build-ratis +on: + workflow_call: + inputs: + repo: + description: Ratis repository + default: apache/ratis + required: false + type: string + ref: + description: Ratis ref (branch, tag or commit SHA) + default: master + required: false + type: string + outputs: + ratis-version: + description: "Ratis Version" + value: ${{ jobs.ratis.outputs.ratis-version }} + thirdparty-version: + description: "Ratis Third-Party Version" + value: ${{ jobs.ratis.outputs.thirdparty-version }} + grpc-version: + description: "gRPC Version" + value: ${{ jobs.ratis-thirdparty.outputs.grpc-version }} + netty-version: + description: "Netty Version" + value: ${{ jobs.ratis-thirdparty.outputs.netty-version }} + protobuf-version: + description: "Protobuf Version" + value: ${{ jobs.ratis-thirdparty.outputs.protobuf-version }} +env: + MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 +jobs: + ratis: + runs-on: ubuntu-24.04 + timeout-minutes: 60 + outputs: + ratis-version: ${{ steps.versions.outputs.ratis }} + thirdparty-version: ${{ steps.versions.outputs.thirdparty }} + steps: + - name: Checkout project + uses: actions/checkout@v4 + with: + repository: ${{ inputs.repo }} + ref: ${{ inputs.ref }} + - name: Cache for maven dependencies + uses: actions/cache@v4 + with: + path: | + ~/.m2/repository + !~/.m2/repository/org/apache/ratis + key: ratis-dependencies-${{ hashFiles('**/pom.xml') }} + - name: Setup java + uses: actions/setup-java@v4 + with: + distribution: 'temurin' + java-version: 8 + - name: Get component versions + id: versions + run: | + thirdparty_version="$(mvn help:evaluate -N -q -DforceStdout -Dscan=false -Dexpression=ratis.thirdparty.version)" + echo "thirdparty=${thirdparty_version}" >> $GITHUB_OUTPUT + + ratis_sha=$(git rev-parse --short HEAD) + ratis_version="$(mvn help:evaluate -N -q -DforceStdout -Dscan=false -Dexpression=project.version | sed -e "s/-SNAPSHOT/-${ratis_sha}-SNAPSHOT/")" + echo "ratis=${ratis_version}" >> $GITHUB_OUTPUT + - name: Run a full build + run: | + mvn -B --no-transfer-progress -Dscan=false versions:set -DnewVersion=${{ steps.versions.outputs.ratis }} + dev-support/checks/build.sh + - name: Store Maven repo for tests + uses: actions/upload-artifact@v4 + with: + name: ratis-jars + path: | + ~/.m2/repository/org/apache/ratis + retention-days: 1 + ratis-thirdparty: + runs-on: ubuntu-24.04 + needs: + - ratis + timeout-minutes: 30 + outputs: + grpc-version: ${{ steps.versions.outputs.grpc }} + netty-version: ${{ steps.versions.outputs.netty }} + protobuf-version: ${{ steps.versions.outputs.protobuf }} + steps: + - name: Checkout project + uses: actions/checkout@v4 + with: + repository: apache/ratis-thirdparty + ref: ${{ needs.ratis.outputs.thirdparty-version }} + - name: Get component versions + id: versions + run: | + echo "grpc=$(mvn help:evaluate -N -q -DforceStdout -Dscan=false -Dexpression=shaded.grpc.version)" >> $GITHUB_OUTPUT + echo "netty=$(mvn help:evaluate -N -q -DforceStdout -Dscan=false -Dexpression=shaded.netty.version)" >> $GITHUB_OUTPUT + echo "protobuf=$(mvn help:evaluate -N -q -DforceStdout -Dscan=false -Dexpression=shaded.protobuf.version)" >> $GITHUB_OUTPUT + debug: + runs-on: ubuntu-24.04 + needs: + - ratis + - ratis-thirdparty + steps: + - name: Print versions + run: | + echo ${{ needs.ratis.outputs.ratis-version }} + echo ${{ needs.ratis.outputs.thirdparty-version }} + echo ${{ needs.ratis-thirdparty.outputs.grpc-version }} + echo ${{ needs.ratis-thirdparty.outputs.netty-version }} + echo ${{ needs.ratis-thirdparty.outputs.protobuf-version }} diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml new file mode 100644 index 000000000000..78b74f6cdd9a --- /dev/null +++ b/.github/workflows/check.yml @@ -0,0 +1,295 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This reusable workflow executes a single check from `hadoop-ozone/dev-support/checks/`. +# Before and after the check, it performs various steps based on workflow inputs. + +name: ci-check + +on: + workflow_call: + inputs: + # REQUIRED + script: + type: string + description: "Test script to run from hadoop-ozone/dev-support/checks, without .sh extension" + required: true + + sha: + type: string + description: "Commit SHA to test" + required: true + + # OPTIONAL (ordered alphabetically) + checkout-fetch-depth: + type: number + description: "Fetch depth for checking out the repo (default: no history)" + default: 1 + required: false + + java-version: + type: string + description: "Java version to set up (default: none)" + default: '' + required: false + + needs-maven-cache: + type: boolean + description: "Whether to restore Maven cache before run (default: yes)" + default: true + required: false + + needs-npm-cache: + type: boolean + description: "Whether to restore NPM cache before run (default: no)" + default: false + required: false + + needs-ozone-binary-tarball: + type: boolean + description: "Whether to download Ozone binary tarball created by build (default: no)" + default: false + required: false + + needs-ozone-repo: + type: boolean + description: "Whether to download Ozone jars created by build (default: no)" + default: false + required: false + + needs-ozone-source-tarball: + type: boolean + description: "Whether to download Ozone source tarball created by build (default: no)" + default: false + required: false + + pre-script: + type: string + description: "Command to execute before the test script (default: none)" + default: '' + required: false + + post-failure: + type: string + description: "Command to execute after the test script, if it failed (default: none)" + default: '' + required: false + + post-success: + type: string + description: "Command to execute after the test script, if it succeeded (default: none)" + default: '' + required: false + + ratis-args: + type: string + description: "Version overrides from custom Ratis build (default: none)" + default: '' + required: false + + runner: + type: string + description: "GitHub Actions runner to use" + default: 'ubuntu-24.04' + required: false + + script-args: + type: string + description: "Arguments for the test script, ratis-args are appended" + default: '' + required: false + + split: + type: string + description: "Name of split for matrix jobs, only used in display name" + default: '' + required: false + + timeout-minutes: + type: number + description: "Job timeout in minutes (default: 30)" + default: 30 + required: false + + with-coverage: + type: boolean + description: "The value of OZONE_WITH_COVERAGE to set" + default: true + required: false + +permissions: + contents: read + packages: read + +env: + HADOOP_IMAGE: ghcr.io/apache/hadoop + MAVEN_ARGS: --batch-mode --settings ${{ github.workspace }}/dev-support/ci/maven-settings.xml + MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 + OZONE_IMAGE: ghcr.io/apache/ozone + OZONE_RUNNER_IMAGE: ghcr.io/apache/ozone-runner + OZONE_VOLUME_OWNER: 1000 + +jobs: + check: + name: ${{ (inputs.split && format('{0} ({1})', inputs.script, inputs.split)) || inputs.script }} + runs-on: ${{ inputs.runner }} + timeout-minutes: ${{ inputs.timeout-minutes }} + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + steps: + - name: Checkout project + if: ${{ !inputs.needs-ozone-source-tarball }} + uses: actions/checkout@v4 + with: + ref: ${{ inputs.sha }} + fetch-depth: ${{ inputs.checkout-fetch-depth }} + + - name: Download Ozone source tarball + if: ${{ inputs.needs-ozone-source-tarball }} + uses: actions/download-artifact@v4 + with: + name: ozone-src + + - name: Extract source tarball + if: ${{ inputs.needs-ozone-source-tarball }} + run: | + tar --strip-components 1 -xzvf ozone*-src.tar.gz + + - name: Cache for NPM dependencies + if: ${{ inputs.needs-npm-cache }} + uses: actions/cache@v4 + with: + path: | + ~/.pnpm-store + key: ${{ runner.os }}-pnpm-${{ hashFiles('**/pnpm-lock.yaml') }} + restore-keys: | + ${{ runner.os }}-pnpm- + + - name: Cache for Maven dependencies + if: ${{ inputs.needs-maven-cache }} + uses: actions/cache/restore@v4 + with: + path: | + ~/.m2/repository/*/*/* + !~/.m2/repository/org/apache/ozone + key: maven-repo-${{ hashFiles('**/pom.xml') }} + restore-keys: | + maven-repo- + + - name: Download Ozone repo + id: download-ozone-repo + if: ${{ inputs.needs-ozone-repo }} + uses: actions/download-artifact@v4 + with: + name: ozone-repo + path: | + ~/.m2/repository/org/apache/ozone + + - name: Download Ratis repo + if: ${{ inputs.ratis-args != '' }} + uses: actions/download-artifact@v4 + with: + name: ratis-jars + path: | + ~/.m2/repository/org/apache/ratis + + - name: Download Ozone binary tarball + if: ${{ inputs.needs-ozone-binary-tarball }} + uses: actions/download-artifact@v4 + with: + name: ozone-bin + + - name: Extract binary tarball + if: ${{ inputs.needs-ozone-binary-tarball }} + run: | + mkdir -p hadoop-ozone/dist/target + tar xzvf ozone*.tar.gz -C hadoop-ozone/dist/target + rm ozone*.tar.gz + + - name: Setup java ${{ inputs.java-version }} + if: ${{ inputs.java-version }} + uses: actions/setup-java@v4 + with: + distribution: 'temurin' + java-version: ${{ inputs.java-version }} + + - name: Execute pre-test steps + if: ${{ inputs.pre-script }} + run: | + ${{ inputs.pre-script }} + + - name: Execute tests + run: | + hadoop-ozone/dev-support/checks/${{ inputs.script }}.sh ${{ inputs.script-args }} ${{ inputs.ratis-args }} + env: + DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }} + OZONE_WITH_COVERAGE: ${{ inputs.with-coverage }} + + - name: Execute post-failure steps + if: ${{ failure() && inputs.post-failure }} + run: | + ${{ inputs.post-failure }} + + - name: Execute post-success steps + if: ${{ !failure() && inputs.post-success }} + run: | + ${{ inputs.post-success }} + + - name: Summary of failures + if: ${{ failure() }} + run: | + if [[ -s "target/${{ inputs.script }}/summary.md" ]]; then + cat target/${{ inputs.script }}/summary.md >> $GITHUB_STEP_SUMMARY + fi + hadoop-ozone/dev-support/checks/_summary.sh target/${{ inputs.script }}/summary.txt + + - name: Archive build results + if: ${{ !cancelled() }} + uses: actions/upload-artifact@v4 + with: + name: ${{ (inputs.split && format('{0}-{1}', inputs.script, inputs.split)) || inputs.script }} + # please keep path as a single item; move to that directory all files needed in the artifact + path: target/${{ inputs.script }} + continue-on-error: true + + # The following steps are hard-coded to be run only for 'build' check, + # to avoid the need for 3 more inputs. + - name: Store binaries for tests + if: ${{ inputs.script == 'build' && !cancelled() }} + uses: actions/upload-artifact@v4 + with: + name: ozone-bin + path: | + hadoop-ozone/dist/target/ozone-*.tar.gz + !hadoop-ozone/dist/target/ozone-*-src.tar.gz + retention-days: 1 + + - name: Store source tarball for compilation + if: ${{ inputs.script == 'build' && !cancelled() }} + uses: actions/upload-artifact@v4 + with: + name: ozone-src + path: | + hadoop-ozone/dist/target/ozone-*-src.tar.gz + retention-days: 1 + + - name: Store Maven repo for tests + if: ${{ inputs.script == 'build' && !cancelled() }} + uses: actions/upload-artifact@v4 + with: + name: ozone-repo + path: | + ~/.m2/repository/org/apache/ozone + retention-days: 1 diff --git a/.github/workflows/ci-with-ratis.yml b/.github/workflows/ci-with-ratis.yml new file mode 100644 index 000000000000..ffe1f5323abd --- /dev/null +++ b/.github/workflows/ci-with-ratis.yml @@ -0,0 +1,58 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This workflow tests Ozone with custom build of Ratis. +# +# Inputs: +# - Ozone commit to build +# - Ratis repo +# - Ratis commit to build + +name: ci-with-ratis +on: + workflow_dispatch: + inputs: + ref: + description: Ozone ref (branch, tag or commit SHA) + default: master + required: true + ratis-repo: + description: Ratis repository + default: 'apache/ratis' + required: true + ratis-ref: + description: Ratis ref (branch, tag or commit SHA) + default: 'master' + required: true +run-name: Test Ozone ${{ inputs.ref }} with Ratis ${{ inputs.ratis-repo }} @ ${{ inputs.ratis-ref }} + +permissions: + contents: read + packages: read + +jobs: + ratis: + uses: ./.github/workflows/build-ratis.yml + with: + repo: ${{ github.event.inputs.ratis-repo || format('{0}/ratis', github.repository_owner) }} + ref: ${{ github.event.inputs.ratis-ref }} + ozone: + needs: + - ratis + uses: ./.github/workflows/ci.yml + secrets: inherit + with: + ratis_args: "-Dratis.version=${{ needs.ratis.outputs.ratis-version }} -Dratis.thirdparty.version=${{ needs.ratis.outputs.thirdparty-version }} -Dio.grpc.version=${{ needs.ratis.outputs.grpc-version }} -Dnetty.version=${{ needs.ratis.outputs.netty-version }} -Dgrpc.protobuf-compile.version=${{ needs.ratis.outputs.protobuf-version }}" + ref: ${{ github.event.inputs.ref }} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1a20eff80c2c..28bce29045c9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,113 +12,371 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - -name: CI - +name: full-ci on: - push: - branches: - - '**' - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true + workflow_call: + inputs: + ratis_args: + type: string + description: Version overrides from custom Ratis build + default: '' + required: false + ref: + type: string + description: Ozone ref (branch, tag or commit SHA) + default: '' + required: false permissions: contents: read packages: read env: - MAVEN_ARGS: --batch-mode --show-version --errors --no-transfer-progress --settings ${{ github.workspace }}/dev-support/ci/maven-settings.xml + BUILD_ARGS: "-Pdist -Psrc -Dmaven.javadoc.skip=true -Drocks_tools_native" + # Minimum required Java version for running Ozone is defined in pom.xml (javac.version). + TEST_JAVA_VERSION: 21 # JDK version used by CI build and tests; should match the JDK version in apache/ozone-runner image + # MAVEN_ARGS and MAVEN_OPTS are duplicated in check.yml, please keep in sync + MAVEN_ARGS: --batch-mode --settings ${{ github.workspace }}/dev-support/ci/maven-settings.xml MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 - OZONE_BUILD_ARGS: -Pdist -Psrc -Dmaven.javadoc.skip=true -Drocks_tools_native - OZONE_TEST_ARGS: -Drocks_tools_native + OZONE_WITH_COVERAGE: ${{ github.event_name == 'push' }} jobs: - build: + build-info: runs-on: ubuntu-24.04 - timeout-minutes: 90 env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_CONTEXT: ${{ toJson(github) }} + outputs: + acceptance-suites: ${{ steps.acceptance-suites.outputs.suites }} + integration-suites: ${{ steps.integration-suites.outputs.suites }} + needs-basic-check: ${{ steps.categorize-basic-checks.outputs.needs-basic-check }} + basic-checks: ${{ steps.categorize-basic-checks.outputs.basic-checks }} + needs-build: ${{ steps.selective-checks.outputs.needs-build }} + needs-compile: ${{ steps.selective-checks.outputs.needs-compile }} + needs-compose-tests: ${{ steps.selective-checks.outputs.needs-compose-tests }} + needs-integration-tests: ${{ steps.selective-checks.outputs.needs-integration-tests }} + needs-kubernetes-tests: ${{ steps.selective-checks.outputs.needs-kubernetes-tests }} + sha: ${{ steps.get-sha.outputs.sha }} + # `env` context cannot be used when calling reusable workflow, so we need to convert these to `outputs` + build-args: ${{ env.BUILD_ARGS }} + java-version: ${{ env.TEST_JAVA_VERSION }} + with-coverage: ${{ env.OZONE_WITH_COVERAGE }} steps: - - name: Checkout project + - name: "Checkout ${{ github.ref }} / ${{ github.sha }} (push)" uses: actions/checkout@v4 - - - name: Setup JDK 21 - uses: actions/setup-java@v4 with: - distribution: temurin - java-version: 21 - cache: maven - server-id: arenadata - server-username: GITHUB_ACTOR - server-password: GITHUB_TOKEN - - - name: Cache NPM dependencies - uses: actions/cache@v4 + persist-credentials: false + if: github.event_name == 'push' + - name: "Checkout ${{ github.sha }} with its parent (pull request)" + uses: actions/checkout@v4 with: - path: | - ~/.pnpm-store - **/node_modules - key: ${{ runner.os }}-pnpm-${{ hashFiles('**/pnpm-lock.yaml') }} - restore-keys: | - ${{ runner.os }}-pnpm- - - - name: Build - run: hadoop-ozone/dev-support/checks/build.sh ${{ env.OZONE_BUILD_ARGS }} - - - name: Summarize build failures - if: failure() + ref: ${{ github.sha }} + fetch-depth: 2 + persist-credentials: false + if: github.event_name == 'pull_request' + - name: "Checkout ${{ inputs.ref }} given in workflow input (manual dispatch)" + uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref }} + persist-credentials: false + if: github.event_name == 'workflow_dispatch' + - name: Get SHA of ${{ inputs.ref || github.ref }} + id: get-sha run: | - if [[ -s target/build/summary.md ]]; then - cat target/build/summary.md >> "${GITHUB_STEP_SUMMARY}" + if [[ "${GITHUB_EVENT_NAME}" == "workflow_dispatch" ]]; then + sha="$(git rev-parse --verify HEAD)" + else + sha="${GITHUB_SHA}" fi - hadoop-ozone/dev-support/checks/_summary.sh target/build/summary.txt + echo "sha=$sha" >> $GITHUB_OUTPUT + - name: Selective checks + id: selective-checks + env: + PR_LABELS: "${{ toJSON(github.event.pull_request.labels.*.name) }}" + PR_DRAFT: "${{ github.event.pull_request.draft }}" + run: | + if [[ "${GITHUB_EVENT_NAME}" == "pull_request" ]]; then + # Run selective checks + dev-support/ci/selective_ci_checks.sh "${GITHUB_SHA}" + else + # Run all checks + dev-support/ci/selective_ci_checks.sh + fi + - name: Acceptance suites + id: acceptance-suites + run: dev-support/ci/acceptance_suites.sh + - name: Integration suites + id: integration-suites + run: dev-support/ci/integration_suites.sh + - name: Categorize Basic Checks + id: categorize-basic-checks + env: + ALL_BASIC_CHECKS: "${{ steps.selective-checks.outputs.basic-checks }}" + run: dev-support/ci/categorize_basic_checks.sh - - name: Archive build report - if: always() - uses: actions/upload-artifact@v4 - with: - name: build-report - path: target/build - retention-days: 7 + build: + needs: + - build-info + if: needs.build-info.outputs.needs-build == 'true' || needs.build-info.outputs.needs-integration-tests == 'true' + uses: ./.github/workflows/check.yml + with: + java-version: ${{ needs.build-info.outputs.java-version }} + needs-npm-cache: true + ratis-args: ${{ inputs.ratis_args }} + script: build + script-args: ${{ needs.build-info.outputs.build-args }} + sha: ${{ needs.build-info.outputs.sha }} + timeout-minutes: 60 + with-coverage: ${{ fromJSON(needs.build-info.outputs.with-coverage) }} + secrets: inherit - test: - runs-on: ubuntu-24.04 - timeout-minutes: 120 + compile: + needs: + - build-info + - build + - basic + - dependency + - license + if: needs.build-info.outputs.needs-compile == 'true' + strategy: + matrix: + include: + - java: 11 + os: ubuntu-24.04 + args: "-Pdist -DskipRecon -Dmaven.javadoc.failOnWarnings=true -Djavac.version=11" + - java: 17 + os: ubuntu-24.04 + args: "-Pdist -DskipRecon -Dmaven.javadoc.failOnWarnings=true -Djavac.version=17" + - java: 21 + os: ubuntu-24.04 + args: "-Pdist -DskipRecon -Dmaven.javadoc.failOnWarnings=true -Djavac.version=21" + fail-fast: false + uses: ./.github/workflows/check.yml + with: + java-version: ${{ matrix.java }} + needs-ozone-source-tarball: true + ratis-args: ${{ inputs.ratis_args }} + runner: ${{ matrix.os }} + script: compile + script-args: ${{ matrix.args }} + sha: ${{ needs.build-info.outputs.sha }} + split: ${{ matrix.java }} + timeout-minutes: 45 + with-coverage: false + secrets: inherit + + basic: + needs: + - build-info + if: needs.build-info.outputs.needs-basic-check == 'true' + uses: ./.github/workflows/check.yml + with: + checkout-fetch-depth: ${{ matrix.check != 'bats' && 1 || 0 }} + java-version: 11 + needs-maven-cache: ${{ !contains('author,bats', matrix.check) }} + ratis-args: ${{ inputs.ratis_args }} + script: ${{ matrix.check }} + sha: ${{ needs.build-info.outputs.sha }} + timeout-minutes: 30 + secrets: inherit + strategy: + matrix: + check: ${{ fromJson(needs.build-info.outputs.basic-checks) }} + fail-fast: false + + dependency: + needs: + - build-info + - build + uses: ./.github/workflows/check.yml + secrets: inherit + with: + java-version: ${{ needs.build-info.outputs.java-version }} + needs-ozone-binary-tarball: true + script: dependency + sha: ${{ needs.build-info.outputs.sha }} + timeout-minutes: 5 + + license: + needs: + - build-info + - build + uses: ./.github/workflows/check.yml + secrets: inherit + with: + java-version: ${{ needs.build-info.outputs.java-version }} + needs-ozone-repo: true + script: license + sha: ${{ needs.build-info.outputs.sha }} + timeout-minutes: 15 + + javadoc: + needs: + - build-info + - build + uses: ./.github/workflows/check.yml + secrets: inherit + with: + java-version: ${{ needs.build-info.outputs.java-version }} + needs-ozone-repo: true + script: javadoc + sha: ${{ needs.build-info.outputs.sha }} + timeout-minutes: 30 + + repro: + needs: + - build-info + - build + - basic + - dependency + - license + uses: ./.github/workflows/check.yml + secrets: inherit + with: + java-version: ${{ needs.build-info.outputs.java-version }} + needs-ozone-repo: true + ratis-args: ${{ inputs.ratis_args }} + script: repro + script-args: ${{ needs.build-info.outputs.build-args }} + post-failure: hadoop-ozone/dev-support/checks/_diffoscope.sh + sha: ${{ needs.build-info.outputs.sha }} + timeout-minutes: 30 + with-coverage: ${{ fromJSON(needs.build-info.outputs.with-coverage) }} + + unit: + needs: + - build-info + - build + if: needs.build-info.outputs.needs-build == 'true' || needs.build-info.outputs.needs-integration-tests == 'true' + uses: ./.github/workflows/check.yml + secrets: inherit + with: + java-version: ${{ needs.build-info.outputs.java-version }} + ratis-args: ${{ inputs.ratis_args }} + script: unit + script-args: -Drocks_tools_native + sha: ${{ needs.build-info.outputs.sha }} + timeout-minutes: 120 + with-coverage: ${{ fromJSON(needs.build-info.outputs.with-coverage) }} + + acceptance: + needs: + - build-info + - build + - basic + - dependency + - license + if: needs.build-info.outputs.needs-compose-tests == 'true' + uses: ./.github/workflows/check.yml + secrets: inherit + with: + java-version: 11 # Hadoop may not work with newer Java + needs-ozone-binary-tarball: true + ratis-args: ${{ inputs.ratis_args }} + script: acceptance + script-args: ${{ matrix.suite }} + sha: ${{ needs.build-info.outputs.sha }} + split: ${{ matrix.suite }} + timeout-minutes: 150 + with-coverage: ${{ fromJSON(needs.build-info.outputs.with-coverage) }} + strategy: + matrix: + suite: ${{ fromJson(needs.build-info.outputs.acceptance-suites) }} + fail-fast: false + + kubernetes: + needs: + - build-info + - build + - basic + - dependency + - license + if: needs.build-info.outputs.needs-kubernetes-tests == 'true' + uses: ./.github/workflows/check.yml + secrets: inherit + with: + needs-ozone-binary-tarball: true + ratis-args: ${{ inputs.ratis_args }} + script: kubernetes + sha: ${{ needs.build-info.outputs.sha }} + timeout-minutes: 60 + with-coverage: ${{ fromJSON(needs.build-info.outputs.with-coverage) }} + + integration: needs: + - build-info - build + - basic + - dependency + - license + if: needs.build-info.outputs.needs-integration-tests == 'true' + uses: ./.github/workflows/check.yml + secrets: inherit + with: + java-version: ${{ needs.build-info.outputs.java-version }} + pre-script: sudo hostname localhost + ratis-args: ${{ inputs.ratis_args }} + script: integration + script-args: -Ptest-${{ matrix.profile }} -Drocks_tools_native + sha: ${{ needs.build-info.outputs.sha }} + split: ${{ matrix.profile }} + timeout-minutes: 90 + with-coverage: ${{ fromJSON(needs.build-info.outputs.with-coverage) }} + strategy: + matrix: + profile: ${{ fromJson(needs.build-info.outputs.integration-suites) }} + fail-fast: false + + coverage: + runs-on: ubuntu-24.04 + timeout-minutes: 30 + if: github.event_name == 'push' + needs: + - build-info + - unit + - acceptance + - integration env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} steps: - name: Checkout project uses: actions/checkout@v4 - - - name: Setup JDK 21 - uses: actions/setup-java@v4 with: - distribution: temurin - java-version: 21 - cache: maven - server-id: arenadata - server-username: GITHUB_ACTOR - server-password: GITHUB_TOKEN - - - name: Run unit tests - run: hadoop-ozone/dev-support/checks/unit.sh ${{ env.OZONE_TEST_ARGS }} - - - name: Summarize test failures - if: failure() + fetch-depth: 0 + ref: ${{ needs.build-info.outputs.sha }} + - name: Cache for maven dependencies + uses: actions/cache/restore@v4 + with: + path: | + ~/.m2/repository/*/*/* + !~/.m2/repository/org/apache/ozone + key: maven-repo-${{ hashFiles('**/pom.xml') }} + restore-keys: | + maven-repo- + - name: Download artifacts + uses: actions/download-artifact@v4 + with: + path: target/artifacts + - name: Untar binaries run: | - if [[ -s target/unit/summary.md ]]; then - cat target/unit/summary.md >> "${GITHUB_STEP_SUMMARY}" - fi - hadoop-ozone/dev-support/checks/_summary.sh target/unit/summary.txt - - - name: Archive test report - if: always() + mkdir -p hadoop-ozone/dist/target + tar xzvf target/artifacts/ozone-bin/ozone*.tar.gz -C hadoop-ozone/dist/target + - name: Setup java ${{ env.TEST_JAVA_VERSION }} + uses: actions/setup-java@v4 + with: + distribution: 'temurin' + java-version: ${{ env.TEST_JAVA_VERSION }} + - name: Calculate combined coverage + run: ./hadoop-ozone/dev-support/checks/coverage.sh + - name: Upload coverage to Sonar + run: ./hadoop-ozone/dev-support/checks/sonar.sh + if: github.repository == 'apache/ozone' + env: + SONAR_TOKEN: ${{ secrets.SONARCLOUD_TOKEN }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }} + - name: Archive build results uses: actions/upload-artifact@v4 with: - name: unit-test-report - path: target/unit - retention-days: 7 + name: coverage + path: target/coverage + continue-on-error: true diff --git a/.github/workflows/intermittent-test-check.yml b/.github/workflows/intermittent-test-check.yml new file mode 100644 index 000000000000..26c961473a6b --- /dev/null +++ b/.github/workflows/intermittent-test-check.yml @@ -0,0 +1,261 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +name: flaky-test-check +on: + workflow_dispatch: + inputs: + ref: + description: Git Ref (Branch/Commit_ID/Tag) + default: master + required: true + test-class: + description: Test Class + default: NA + required: true + test-name: + description: Test Name + default: ALL + required: false + submodule: + description: Submodule + default: 'ozone-integration-test' + required: true + iterations: + description: Number of Iterations per split + default: 10 + required: true + splits: + description: Number of splits + default: 10 + required: true + fail-fast: + description: Stop after first failure + default: false + required: true + ratis-repo: + description: Ratis repository + default: '' + required: false + ratis-ref: + description: Ratis ref (branch, tag or commit SHA) + default: '' + required: false + java-version: + description: Java version to use + default: '21' + required: true + +permissions: + contents: read + packages: read + +env: + MAVEN_ARGS: --batch-mode --settings ${{ github.workspace }}/dev-support/ci/maven-settings.xml + MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 + TEST_CLASS: ${{ github.event.inputs.test-class}} + TEST_METHOD: ${{ github.event.inputs.test-name }} + ITERATIONS: ${{ github.event.inputs.iterations }} + FAIL_FAST: ${{ github.event.inputs.fail-fast }} + RATIS_REPO: ${{ github.event.inputs.ratis-repo }} + RATIS_VERSION: ${{ github.event.inputs.ratis-ref }} + JAVA_VERSION: ${{ github.event.inputs.java-version }} + # Surefire 3.0.0-M4 is used because newer versions do not reliably kill the fork on timeout + # SUREFIRE-1722, SUREFIRE-1815 + SUREFIRE_VERSION: 3.0.0-M4 +run-name: ${{ github.event_name == 'workflow_dispatch' && format('{0}#{1}[{2}]-{3}x{4}-java{5}', inputs.test-class, inputs.test-name, inputs.ref, inputs.splits, inputs.iterations, inputs.java-version) || '' }} +jobs: + prepare-job: + runs-on: ubuntu-24.04 + outputs: + matrix: ${{steps.generate.outputs.matrix}} + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ github.event.inputs.ref }} + - id: generate + name: Generate test matrix + run: | + splits=() + for ((i = 1; i <= ${{ github.event.inputs.splits }}; i++)); do + splits+=("$i") + done + printf -v x "%s," "${splits[@]}" + split_matrix="[${x%,}]" + echo "matrix=$split_matrix" >> $GITHUB_OUTPUT + ratis: + uses: ./.github/workflows/build-ratis.yml + if: ${{ github.event.inputs.ratis-ref != '' }} + with: + repo: ${{ github.event.inputs.ratis-repo || format('{0}/ratis', github.repository_owner) }} + ref: ${{ github.event.inputs.ratis-ref }} + build: + if: ${{ always() }} + needs: + - prepare-job + - ratis + runs-on: ubuntu-24.04 + timeout-minutes: 60 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + steps: + - name: Checkout project + uses: actions/checkout@v4 + with: + ref: ${{ github.event.inputs.ref }} + - name: Cache for maven dependencies + uses: actions/cache/restore@v4 + with: + path: | + ~/.m2/repository/*/*/* + !~/.m2/repository/org/apache/ozone + key: maven-repo-${{ hashFiles('**/pom.xml') }} + restore-keys: | + maven-repo- + - name: Download Ratis repo + if: ${{ github.event.inputs.ratis-ref != '' }} + uses: actions/download-artifact@v4 + with: + name: ratis-jars + path: | + ~/.m2/repository/org/apache/ratis + - name: Setup java + uses: actions/setup-java@v4 + with: + distribution: 'temurin' + java-version: ${{ github.event.inputs.java-version }} + - name: Build (most) of Ozone + run: | + args="-DskipRecon -DskipShade -Dmaven.javadoc.skip=true" + if [[ "${{ github.event.inputs.ratis-ref }}" != "" ]]; then + args="$args -Dratis.version=${{ needs.ratis.outputs.ratis-version }}" + args="$args -Dratis.thirdparty.version=${{ needs.ratis.outputs.thirdparty-version }}" + args="$args -Dio.grpc.version=${{ needs.ratis.outputs.grpc-version }}" + args="$args -Dnetty.version=${{ needs.ratis.outputs.netty-version }}" + args="$args -Dgrpc.protobuf-compile.version=${{ needs.ratis.outputs.protobuf-version }}" + fi + + args="$args -am -pl :${{ github.event.inputs.submodule }}" + + hadoop-ozone/dev-support/checks/build.sh $args + - name: Store Maven repo for tests + uses: actions/upload-artifact@v4 + with: + name: ozone-repo + path: | + ~/.m2/repository/org/apache/ozone + retention-days: 1 + run-test: + if: ${{ always() }} + needs: + - prepare-job + - ratis + - build + name: Run-Split + runs-on: ubuntu-24.04 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + strategy: + matrix: + split: ${{fromJson(needs.prepare-job.outputs.matrix)}} # Define splits + fail-fast: ${{ fromJson(github.event.inputs.fail-fast) }} + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ github.event.inputs.ref }} + - name: Cache for maven dependencies + uses: actions/cache/restore@v4 + with: + path: | + ~/.m2/repository/*/*/* + !~/.m2/repository/org/apache/ozone + key: maven-repo-${{ hashFiles('**/pom.xml') }} + restore-keys: | + maven-repo- + - name: Download Ratis repo + if: ${{ github.event.inputs.ratis-ref != '' }} + uses: actions/download-artifact@v4 + with: + name: ratis-jars + path: | + ~/.m2/repository/org/apache/ratis + - name: Download Ozone repo + id: download-ozone-repo + uses: actions/download-artifact@v4 + with: + name: ozone-repo + path: | + ~/.m2/repository/org/apache/ozone + continue-on-error: true + - name: Setup java + uses: actions/setup-java@v4 + with: + distribution: 'temurin' + java-version: ${{ github.event.inputs.java-version }} + - name: Execute tests + run: | + if [[ -e "${{ steps.download-ozone-repo.outputs.download-path }}" ]]; then + export OZONE_REPO_CACHED=true + fi + + args="-DexcludedGroups=native|slow|unhealthy -DskipShade" + if [[ "${{ github.event.inputs.ratis-ref }}" != "" ]]; then + args="$args -Dratis.version=${{ needs.ratis.outputs.ratis-version }}" + args="$args -Dratis.thirdparty.version=${{ needs.ratis.outputs.thirdparty-version }}" + args="$args -Dio.grpc.version=${{ needs.ratis.outputs.grpc-version }}" + args="$args -Dnetty.version=${{ needs.ratis.outputs.netty-version }}" + args="$args -Dgrpc.protobuf-compile.version=${{ needs.ratis.outputs.protobuf-version }}" + fi + + args="$args -pl :${{ github.event.inputs.submodule }}" + + if [ "$TEST_METHOD" = "ALL" ]; then + echo "Running all tests from $TEST_CLASS" + set -x + hadoop-ozone/dev-support/checks/junit.sh $args -Dtest="$TEST_CLASS,Abstract*Test*\$*" + else + echo "Running test: $TEST_METHOD from $TEST_CLASS" + set -x + hadoop-ozone/dev-support/checks/junit.sh $args -Dtest="$TEST_CLASS#$TEST_METHOD,Abstract*Test*\$*" + fi + continue-on-error: true + env: + DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }} + - name: Summary of failures + run: hadoop-ozone/dev-support/checks/_summary.sh target/unit/summary.txt + if: ${{ !cancelled() }} + - name: Archive build results + uses: actions/upload-artifact@v4 + if: ${{ failure() }} + with: + name: result-${{ github.run_number }}-${{ github.run_id }}-split-${{ matrix.split }} + path: target/unit + count-failures: + if: ${{ failure() }} + needs: run-test + runs-on: ubuntu-24.04 + steps: + - name: Download build results + uses: actions/download-artifact@v4 + - name: Count failures + run: | + failures=$(find . -name 'summary.txt' | grep --text -v 'iteration' | xargs grep --text -v 'exit code: 0' | wc -l) + echo "Total failures: $failures" + if [[ $failures -gt 0 ]]; then + echo "" + echo "Failed runs:" + grep --text 'exit code: 1' */summary.txt | grep --text -o 'split.*teration [0-9]*' | sed -e 's/.summary.txt:/ /' -e 's/-/ /' | sort -g -k2 -k4 + echo "" + exit 1 + fi diff --git a/.github/workflows/post-commit.yml b/.github/workflows/post-commit.yml new file mode 100644 index 000000000000..60529a5b44ce --- /dev/null +++ b/.github/workflows/post-commit.yml @@ -0,0 +1,32 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +name: build-branch +on: + pull_request: + types: [opened, ready_for_review, synchronize] + push: +concurrency: + group: ci-${{ github.event.pull_request.number || github.sha }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +permissions: + contents: read + packages: read + +jobs: + CI: + if: github.event_name == 'pull_request' || !startsWith(github.ref_name, 'dependabot') + uses: ./.github/workflows/ci.yml + secrets: inherit diff --git a/.github/workflows/repeat-acceptance.yml b/.github/workflows/repeat-acceptance.yml new file mode 100644 index 000000000000..188c4123cac8 --- /dev/null +++ b/.github/workflows/repeat-acceptance.yml @@ -0,0 +1,171 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This workflow runs some acceptance test(s) multiple times concurrently. +# Specify the test by either: +# * "Test Suite", which should be ones of the existing suites from regular CI, +# e.g. "cert-rotation", or +# * "Test Filter", which is a regex pattern applied to filter test script's path +# (examples: "ozone-csi", "test-vault.sh", "ozone/test-ec.sh", "test-.*-rotation.sh") + +name: repeat-acceptance-test +on: + workflow_dispatch: + inputs: + ref: + description: Git Ref (Branch/Commit_ID/Tag) + default: master + required: true + test-suite: + description: Test Suite + required: false + test-filter: + description: Test Filter + required: false + splits: + description: Number of splits + default: 10 + required: true + fail-fast: + description: Stop after first failure + default: false + required: true + +permissions: + contents: read + packages: read + +env: + MAVEN_ARGS: --batch-mode --settings ${{ github.workspace }}/dev-support/ci/maven-settings.xml + MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 + OZONE_ACCEPTANCE_SUITE: ${{ github.event.inputs.test-suite}} + OZONE_TEST_SELECTOR: ${{ github.event.inputs.test-filter }} + FAIL_FAST: ${{ github.event.inputs.fail-fast }} + JAVA_VERSION: 8 +run-name: ${{ github.event_name == 'workflow_dispatch' && format('{0}[{1}]-{2}', inputs.test-suite || inputs.test-filter, inputs.ref, inputs.splits) || '' }} +jobs: + prepare-job: + runs-on: ubuntu-24.04 + outputs: + matrix: ${{steps.generate.outputs.matrix}} + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ github.event.inputs.ref }} + - name: Verify Test Filter + run: | + cd hadoop-ozone/dist/src/main/compose + source testlib.sh + find_tests + - id: generate + name: Generate test matrix + run: | + splits=() + for ((i = 1; i <= ${{ github.event.inputs.splits }}; i++)); do + splits+=("$i") + done + printf -v x "%s," "${splits[@]}" + split_matrix="[${x%,}]" + echo "matrix=$split_matrix" >> $GITHUB_OUTPUT + build: + needs: + - prepare-job + runs-on: ubuntu-24.04 + timeout-minutes: 60 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + steps: + - name: Checkout project + uses: actions/checkout@v4 + with: + ref: ${{ github.event.inputs.ref }} + - name: Cache for npm dependencies + uses: actions/cache@v4 + with: + path: | + ~/.pnpm-store + **/node_modules + key: ${{ runner.os }}-pnpm-${{ hashFiles('**/pnpm-lock.yaml') }} + restore-keys: | + ${{ runner.os }}-pnpm- + - name: Cache for maven dependencies + uses: actions/cache/restore@v4 + with: + path: | + ~/.m2/repository/*/*/* + !~/.m2/repository/org/apache/ozone + key: maven-repo-${{ hashFiles('**/pom.xml') }}-${{ env.JAVA_VERSION }} + restore-keys: | + maven-repo-${{ hashFiles('**/pom.xml') }} + maven-repo- + - name: Setup java + uses: actions/setup-java@v4 + with: + distribution: 'temurin' + java-version: ${{ env.JAVA_VERSION }} + - name: Run a full build + run: hadoop-ozone/dev-support/checks/build.sh -Pdist -Psrc -Dmaven.javadoc.skip=true + env: + DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }} + - name: Store binaries for tests + uses: actions/upload-artifact@v4 + with: + name: ozone-bin + path: | + hadoop-ozone/dist/target/ozone-*.tar.gz + !hadoop-ozone/dist/target/ozone-*-src.tar.gz + retention-days: 1 + acceptance: + needs: + - prepare-job + - build + name: Run-Split + runs-on: ubuntu-24.04 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + strategy: + matrix: + split: ${{ fromJson(needs.prepare-job.outputs.matrix) }} + fail-fast: ${{ fromJson(github.event.inputs.fail-fast) }} + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ github.event.inputs.ref }} + - name: Download compiled Ozone binaries + uses: actions/download-artifact@v4 + with: + name: ozone-bin + - name: Untar binaries + run: | + mkdir -p hadoop-ozone/dist/target + tar xzvf ozone*.tar.gz -C hadoop-ozone/dist/target + rm ozone*.tar.gz + sudo chmod -R a+rwX hadoop-ozone/dist/target + - name: Execute tests + run: | + ./hadoop-ozone/dev-support/checks/acceptance.sh + env: + KEEP_IMAGE: false + continue-on-error: true + - name: Summary of failures + run: hadoop-ozone/dev-support/checks/_summary.sh target/${{ github.job }}/summary.txt + if: ${{ !cancelled() }} + - name: Archive build results + uses: actions/upload-artifact@v4 + if: always() + with: + name: acceptance-${{ matrix.split }} + path: target/acceptance + continue-on-error: true diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index a0dd35e96118..adcc0b76d2f8 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -99,7 +99,7 @@ For large feature development changes, we use a process called "Ozone Enhancemen We use GitHub pull requests for contributing changes to the repository. The main workflow is as follows: 1. Fork [`apache/ozone`](https://github.com/apache/ozone) repository (first time) and clone it to your local machine - 2. Enable the `CI` GitHub Actions workflow (defined in `.github/workflows/ci.yml`) in your fork + 2. Enable the `build-branch` GitHub Actions workflow (defined in `.github/workflows/post-commit.yml`) in your fork 3. Ensure a Jira issue corresponding to the change exists in the [HDDS project](https://issues.apache.org/jira/projects/HDDS/) (eg. HDDS-1234) * Please search Jira before creating a new issue, someone might have already reported the same. * If this is your first issue, you might not be able to assign it to yourself. If so, please make a comment in the issue, indicating that your are working on it. @@ -107,7 +107,7 @@ We use GitHub pull requests for contributing changes to the repository. The main 5. Make your changes locally. * For complex changes, committing each logical part is recommended. 6. Push your changes to your fork of Ozone - 7. Wait for the `CI` workflow to complete successfully for your commit. + 7. Wait for the `build-branch` workflow to complete successfully for your commit. 8. Create a pull request for your changes * Please include the Jira link, problem description and testing instruction (follow the [template](https://github.com/apache/ozone/blob/master/.github/pull_request_template.md)) 9. Set the Jira issue to "Patch Available" state diff --git a/dev-support/pom.xml b/dev-support/pom.xml index 915ffe947b98..61b7811fa212 100644 --- a/dev-support/pom.xml +++ b/dev-support/pom.xml @@ -17,7 +17,7 @@ org.apache.ozone ozone-main - 2.1.0.1-4.3.0-3 + 2.1.0.1-4.3.0-4 ozone-dev-support Apache Ozone Dev Support diff --git a/hadoop-hdds/annotations/pom.xml b/hadoop-hdds/annotations/pom.xml index eaf4d94e3cea..821af03797b2 100644 --- a/hadoop-hdds/annotations/pom.xml +++ b/hadoop-hdds/annotations/pom.xml @@ -17,11 +17,11 @@ org.apache.ozone hdds - 2.1.0.1-4.3.0-3 + 2.1.0.1-4.3.0-4 hdds-annotation-processing - 2.1.0.1-4.3.0-3 + 2.1.0.1-4.3.0-4 jar Apache Ozone Annotation Processing Apache Ozone annotation processing tools for validating custom diff --git a/hadoop-hdds/client/pom.xml b/hadoop-hdds/client/pom.xml index b47585ad4dcc..5270903f8c1b 100644 --- a/hadoop-hdds/client/pom.xml +++ b/hadoop-hdds/client/pom.xml @@ -17,12 +17,12 @@ org.apache.ozone hdds-hadoop-dependency-client - 2.1.0.1-4.3.0-3 + 2.1.0.1-4.3.0-4 ../hadoop-dependency-client hdds-client - 2.1.0.1-4.3.0-3 + 2.1.0.1-4.3.0-4 jar Apache Ozone HDDS Client Apache Ozone Distributed Data Store Client Library diff --git a/hadoop-hdds/common/pom.xml b/hadoop-hdds/common/pom.xml index b19f3b7cc129..02b1cef14db3 100644 --- a/hadoop-hdds/common/pom.xml +++ b/hadoop-hdds/common/pom.xml @@ -17,11 +17,11 @@ org.apache.ozone hdds-hadoop-dependency-client - 2.1.0.1-4.3.0-3 + 2.1.0.1-4.3.0-4 ../hadoop-dependency-client hdds-common - 2.1.0.1-4.3.0-3 + 2.1.0.1-4.3.0-4 jar Apache Ozone HDDS Common Apache Ozone Distributed Data Store Common diff --git a/hadoop-hdds/config/pom.xml b/hadoop-hdds/config/pom.xml index 6027d1120557..88b0c3c1a4bd 100644 --- a/hadoop-hdds/config/pom.xml +++ b/hadoop-hdds/config/pom.xml @@ -17,10 +17,10 @@ org.apache.ozone hdds - 2.1.0.1-4.3.0-3 + 2.1.0.1-4.3.0-4 hdds-config - 2.1.0.1-4.3.0-3 + 2.1.0.1-4.3.0-4 jar Apache Ozone HDDS Config Apache Ozone Distributed Data Store Config Tools diff --git a/hadoop-hdds/container-service/pom.xml b/hadoop-hdds/container-service/pom.xml index 489a4c5a1aad..8cabbf1c2ad2 100644 --- a/hadoop-hdds/container-service/pom.xml +++ b/hadoop-hdds/container-service/pom.xml @@ -17,10 +17,10 @@ org.apache.ozone hdds - 2.1.0.1-4.3.0-3 + 2.1.0.1-4.3.0-4 hdds-container-service - 2.1.0.1-4.3.0-3 + 2.1.0.1-4.3.0-4 jar Apache Ozone HDDS Container Service Apache Ozone Distributed Data Store Container Service diff --git a/hadoop-hdds/crypto-api/pom.xml b/hadoop-hdds/crypto-api/pom.xml index d0cb7eef0ef6..79049d6a4b4a 100644 --- a/hadoop-hdds/crypto-api/pom.xml +++ b/hadoop-hdds/crypto-api/pom.xml @@ -17,11 +17,11 @@ org.apache.ozone hdds - 2.1.0.1-4.3.0-3 + 2.1.0.1-4.3.0-4 hdds-crypto-api - 2.1.0.1-4.3.0-3 + 2.1.0.1-4.3.0-4 Apache Ozone HDDS Crypto Apache Ozone Distributed Data Store cryptographic functions diff --git a/hadoop-hdds/crypto-default/pom.xml b/hadoop-hdds/crypto-default/pom.xml index fa24cedc2515..c7a3e76b386f 100644 --- a/hadoop-hdds/crypto-default/pom.xml +++ b/hadoop-hdds/crypto-default/pom.xml @@ -17,11 +17,11 @@ org.apache.ozone hdds - 2.1.0.1-4.3.0-3 + 2.1.0.1-4.3.0-4 hdds-crypto-default - 2.1.0.1-4.3.0-3 + 2.1.0.1-4.3.0-4 Apache Ozone HDDS Crypto - Default Default implementation of Apache Ozone Distributed Data Store's cryptographic functions diff --git a/hadoop-hdds/docs/content/feature/Quota.md b/hadoop-hdds/docs/content/feature/Quota.md index 53c196307fa3..47dba21792f7 100644 --- a/hadoop-hdds/docs/content/feature/Quota.md +++ b/hadoop-hdds/docs/content/feature/Quota.md @@ -7,7 +7,6 @@ icon: user menu: main: parent: Features -summary: Introduction to Ozone Quota --- ghcr.io/apache/hadoop - - -lean + + 20250625-2-jdk21 ghcr.io/apache/ozone-testkrb5:20241129-1 apache/ozone @@ -37,6 +36,9 @@ UTF-8 true + + 2.6.0 noarch diff --git a/hadoop-ozone/dist/src/main/compose/common/hadoop-test.sh b/hadoop-ozone/dist/src/main/compose/common/hadoop-test.sh index d08198f428ad..a2c590ba6b33 100755 --- a/hadoop-ozone/dist/src/main/compose/common/hadoop-test.sh +++ b/hadoop-ozone/dist/src/main/compose/common/hadoop-test.sh @@ -28,11 +28,11 @@ if [[ -z "${HADOOP_TEST_IMAGES}" ]]; then # hadoop2 image is only available from Docker Hub HADOOP_TEST_IMAGES="${HADOOP_TEST_IMAGES} apache/hadoop:${hadoop2.version}" HADOOP_TEST_IMAGES="${HADOOP_TEST_IMAGES} ${HADOOP_IMAGE}:3.3.6" - HADOOP_TEST_IMAGES="${HADOOP_TEST_IMAGES} ${HADOOP_IMAGE}:${hadoop.version}${docker.hadoop.image.flavor}" + HADOOP_TEST_IMAGES="${HADOOP_TEST_IMAGES} ${HADOOP_IMAGE}:${hadoop.acceptance.version}${docker.hadoop.image.flavor}" fi export HADOOP_MAJOR_VERSION=3 -export HADOOP_TEST_IMAGE="${HADOOP_IMAGE}:${hadoop.version}" +export HADOOP_TEST_IMAGE="${HADOOP_IMAGE}:${hadoop.acceptance.version}${docker.hadoop.image.flavor}" export OZONE_REPLICATION_FACTOR=3 # shellcheck source=/dev/null diff --git a/hadoop-ozone/dist/src/main/compose/compatibility/docker-config b/hadoop-ozone/dist/src/main/compose/compatibility/docker-config index cae129e8e455..b2f4a7040428 100644 --- a/hadoop-ozone/dist/src/main/compose/compatibility/docker-config +++ b/hadoop-ozone/dist/src/main/compose/compatibility/docker-config @@ -28,6 +28,7 @@ OZONE-SITE.XML_ozone.recon.db.dir=/data/metadata/recon OZONE-SITE.XML_ozone.scm.client.address=scm OZONE-SITE.XML_hdds.datanode.dir=/data/hdds OZONE-SITE.XML_hdds.datanode.volume.min.free.space=100MB +OZONE-SITE.XML_hdds.datanode.volume.min.free.space.percent=0 OZONE-SITE.XML_ozone.recon.address=recon:9891 OZONE-SITE.XML_hdds.scmclient.max.retry.timeout=30s OZONE-SITE.XML_ozone.http.basedir=/tmp/ozone_http diff --git a/hadoop-ozone/dist/src/main/compose/ozone-csi/docker-config b/hadoop-ozone/dist/src/main/compose/ozone-csi/docker-config index ba4d80a9d052..88d069ebfbbd 100644 --- a/hadoop-ozone/dist/src/main/compose/ozone-csi/docker-config +++ b/hadoop-ozone/dist/src/main/compose/ozone-csi/docker-config @@ -34,6 +34,7 @@ OZONE-SITE.XML_ozone.recon.db.dir=/data/metadata/recon OZONE-SITE.XML_ozone.scm.client.address=scm OZONE-SITE.XML_hdds.datanode.dir=/data/hdds OZONE-SITE.XML_hdds.datanode.volume.min.free.space=100MB +OZONE-SITE.XML_hdds.datanode.volume.min.free.space.percent=0 OZONE-SITE.XML_hdds.scmclient.max.retry.timeout=30s OZONE-SITE.XML_ozone.http.basedir=/tmp/ozone_http diff --git a/hadoop-ozone/dist/src/main/compose/ozone-ha/docker-config b/hadoop-ozone/dist/src/main/compose/ozone-ha/docker-config index 7317b2c1aa9f..b2479944e9e2 100644 --- a/hadoop-ozone/dist/src/main/compose/ozone-ha/docker-config +++ b/hadoop-ozone/dist/src/main/compose/ozone-ha/docker-config @@ -35,9 +35,11 @@ OZONE-SITE.XML_ozone.scm.address.scmservice.scm3=scm3 OZONE-SITE.XML_ozone.scm.datanode.id.dir=/data/metadata OZONE-SITE.XML_ozone.scm.container.size=1GB OZONE-SITE.XML_ozone.scm.datanode.ratis.volume.free-space.min=10MB +OZONE-SITE.XML_ozone.scm.ec.pipeline.minimum=1 OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata OZONE-SITE.XML_hdds.datanode.dir=/data/hdds OZONE-SITE.XML_hdds.datanode.volume.min.free.space=100MB +OZONE-SITE.XML_hdds.datanode.volume.min.free.space.percent=0 OZONE-SITE.XML_ozone.datanode.pipeline.limit=1 OZONE-SITE.XML_hdds.scmclient.max.retry.timeout=30s OZONE-SITE.XML_ozone.scm.primordial.node.id=scm1 diff --git a/hadoop-ozone/dist/src/main/compose/ozone-om-ha/docker-compose.yaml b/hadoop-ozone/dist/src/main/compose/ozone-om-ha/docker-compose.yaml index 097d99c6b7a1..273cedb4ad81 100644 --- a/hadoop-ozone/dist/src/main/compose/ozone-om-ha/docker-compose.yaml +++ b/hadoop-ozone/dist/src/main/compose/ozone-om-ha/docker-compose.yaml @@ -21,7 +21,6 @@ services: args: - OZONE_RUNNER_IMAGE - OZONE_RUNNER_VERSION - image: ozone-runner-om-ha:${OZONE_RUNNER_VERSION} privileged: true #required by the profiler volumes: - ../..:/opt/hadoop @@ -36,7 +35,6 @@ services: args: - OZONE_RUNNER_IMAGE - OZONE_RUNNER_VERSION - image: ozone-runner-om-ha:${OZONE_RUNNER_VERSION} privileged: true #required by the profiler volumes: - ../..:/opt/hadoop @@ -54,7 +52,6 @@ services: args: - OZONE_RUNNER_IMAGE - OZONE_RUNNER_VERSION - image: ozone-runner-om-ha:${OZONE_RUNNER_VERSION} privileged: true #required by the profiler volumes: - ../..:/opt/hadoop @@ -72,7 +69,6 @@ services: args: - OZONE_RUNNER_IMAGE - OZONE_RUNNER_VERSION - image: ozone-runner-om-ha:${OZONE_RUNNER_VERSION} privileged: true #required by the profiler volumes: - ../..:/opt/hadoop @@ -90,7 +86,6 @@ services: args: - OZONE_RUNNER_IMAGE - OZONE_RUNNER_VERSION - image: ozone-runner-om-ha:${OZONE_RUNNER_VERSION} privileged: true #required by the profiler volumes: - ../..:/opt/hadoop diff --git a/hadoop-ozone/dist/src/main/compose/ozone-om-ha/docker-config b/hadoop-ozone/dist/src/main/compose/ozone-om-ha/docker-config index b0ebb395f9a0..2e2b69b6c57e 100644 --- a/hadoop-ozone/dist/src/main/compose/ozone-om-ha/docker-config +++ b/hadoop-ozone/dist/src/main/compose/ozone-om-ha/docker-config @@ -33,6 +33,7 @@ OZONE-SITE.XML_ozone.server.default.replication=1 OZONE-SITE.XML_ozone.client.failover.max.attempts=6 OZONE-SITE.XML_hdds.datanode.dir=/data/hdds OZONE-SITE.XML_hdds.datanode.volume.min.free.space=100MB +OZONE-SITE.XML_hdds.datanode.volume.min.free.space.percent=0 OZONE-SITE.XML_hdds.profiler.endpoint.enabled=true OZONE-SITE.XML_hdds.scmclient.max.retry.timeout=30s OZONE-SITE.XML_hdds.container.report.interval=60s diff --git a/hadoop-ozone/dist/src/main/compose/ozone-om-prepare/docker-config b/hadoop-ozone/dist/src/main/compose/ozone-om-prepare/docker-config index 8550e618501e..b197e9a7f7e4 100644 --- a/hadoop-ozone/dist/src/main/compose/ozone-om-prepare/docker-config +++ b/hadoop-ozone/dist/src/main/compose/ozone-om-prepare/docker-config @@ -31,6 +31,7 @@ OZONE-SITE.XML_ozone.scm.client.address=scm OZONE-SITE.XML_ozone.client.failover.max.attempts=6 OZONE-SITE.XML_hdds.datanode.dir=/data/hdds OZONE-SITE.XML_hdds.datanode.volume.min.free.space=100MB +OZONE-SITE.XML_hdds.datanode.volume.min.free.space.percent=0 OZONE-SITE.XML_ozone.http.basedir=/tmp/ozone_http no_proxy=om1,om2,om3,scm,s3g,recon,kdc,localhost,127.0.0.1 diff --git a/hadoop-ozone/dist/src/main/compose/ozone-topology/docker-config b/hadoop-ozone/dist/src/main/compose/ozone-topology/docker-config index 59b1fcf8cab6..893180ded89c 100644 --- a/hadoop-ozone/dist/src/main/compose/ozone-topology/docker-config +++ b/hadoop-ozone/dist/src/main/compose/ozone-topology/docker-config @@ -23,6 +23,7 @@ OZONE-SITE.XML_ozone.scm.datanode.ratis.volume.free-space.min=10MB OZONE-SITE.XML_ozone.ozone.scm.block.size=64MB OZONE-SITE.XML_ozone.scm.pipeline.creation.interval=30s OZONE-SITE.XML_ozone.scm.pipeline.owner.container.count=1 +OZONE-SITE.XML_ozone.scm.ec.pipeline.minimum=1 OZONE-SITE.XML_ozone.scm.names=scm OZONE-SITE.XML_ozone.scm.datanode.id.dir=/data/metadata OZONE-SITE.XML_ozone.scm.block.client.address=scm @@ -33,6 +34,7 @@ OZONE-SITE.XML_ozone.scm.client.address=scm OZONE-SITE.XML_ozone.server.default.replication=3 OZONE-SITE.XML_hdds.datanode.dir=/data/hdds OZONE-SITE.XML_hdds.datanode.volume.min.free.space=100MB +OZONE-SITE.XML_hdds.datanode.volume.min.free.space.percent=0 OZONE-SITE.XML_ozone.recon.address=recon:9891 OZONE-SITE.XML_ozone.recon.http-address=0.0.0.0:9888 OZONE-SITE.XML_ozone.recon.https-address=0.0.0.0:9889 diff --git a/hadoop-ozone/dist/src/main/compose/ozone/docker-config b/hadoop-ozone/dist/src/main/compose/ozone/docker-config index f2a9e0447932..738495873bfa 100644 --- a/hadoop-ozone/dist/src/main/compose/ozone/docker-config +++ b/hadoop-ozone/dist/src/main/compose/ozone/docker-config @@ -28,6 +28,7 @@ OZONE-SITE.XML_ozone.scm.block.size=1MB OZONE-SITE.XML_ozone.scm.datanode.ratis.volume.free-space.min=10MB OZONE-SITE.XML_ozone.scm.pipeline.creation.interval=30s OZONE-SITE.XML_ozone.scm.pipeline.owner.container.count=1 +OZONE-SITE.XML_ozone.scm.ec.pipeline.minimum=1 OZONE-SITE.XML_ozone.scm.names=scm OZONE-SITE.XML_ozone.scm.datanode.id.dir=/data/metadata OZONE-SITE.XML_ozone.scm.block.client.address=scm @@ -36,6 +37,7 @@ OZONE-SITE.XML_ozone.recon.db.dir=/data/metadata/recon OZONE-SITE.XML_ozone.scm.client.address=scm OZONE-SITE.XML_hdds.datanode.dir=/data/hdds OZONE-SITE.XML_hdds.datanode.volume.min.free.space=100MB +OZONE-SITE.XML_hdds.datanode.volume.min.free.space.percent=0 OZONE-SITE.XML_ozone.recon.address=recon:9891 OZONE-SITE.XML_ozone.recon.http-address=0.0.0.0:9888 OZONE-SITE.XML_ozone.recon.https-address=0.0.0.0:9889 diff --git a/hadoop-ozone/dist/src/main/compose/ozoneblockade/docker-config b/hadoop-ozone/dist/src/main/compose/ozoneblockade/docker-config index 87b0cb505378..3fd664f8d5c7 100644 --- a/hadoop-ozone/dist/src/main/compose/ozoneblockade/docker-config +++ b/hadoop-ozone/dist/src/main/compose/ozoneblockade/docker-config @@ -30,6 +30,7 @@ OZONE-SITE.XML_ozone.scm.dead.node.interval=5m OZONE-SITE.XML_ozone.server.default.replication=1 OZONE-SITE.XML_hdds.datanode.dir=/data/hdds OZONE-SITE.XML_hdds.datanode.volume.min.free.space=100MB +OZONE-SITE.XML_hdds.datanode.volume.min.free.space.percent=0 OZONE-SITE.XML_ozone.scm.pipeline.owner.container.count=1 OZONE-SITE.XML_ozone.scm.pipeline.destroy.timeout=15s OZONE-SITE.XML_hdds.heartbeat.interval=2s diff --git a/hadoop-ozone/dist/src/main/compose/ozonescripts/docker-compose.yaml b/hadoop-ozone/dist/src/main/compose/ozonescripts/docker-compose.yaml index 9176739471cd..4f56f39ed5fc 100644 --- a/hadoop-ozone/dist/src/main/compose/ozonescripts/docker-compose.yaml +++ b/hadoop-ozone/dist/src/main/compose/ozonescripts/docker-compose.yaml @@ -21,7 +21,6 @@ services: args: - OZONE_RUNNER_IMAGE - OZONE_RUNNER_VERSION - image: ozone-runner-scripts:${OZONE_RUNNER_VERSION} volumes: - ../..:/opt/hadoop ports: @@ -34,7 +33,6 @@ services: args: - OZONE_RUNNER_IMAGE - OZONE_RUNNER_VERSION - image: ozone-runner-scripts:${OZONE_RUNNER_VERSION} volumes: - ../..:/opt/hadoop ports: @@ -48,7 +46,6 @@ services: args: - OZONE_RUNNER_IMAGE - OZONE_RUNNER_VERSION - image: ozone-runner-scripts:${OZONE_RUNNER_VERSION} volumes: - ../..:/opt/hadoop ports: diff --git a/hadoop-ozone/dist/src/main/compose/ozonescripts/docker-config b/hadoop-ozone/dist/src/main/compose/ozonescripts/docker-config index 6c0262870aac..1be54971bb8a 100644 --- a/hadoop-ozone/dist/src/main/compose/ozonescripts/docker-config +++ b/hadoop-ozone/dist/src/main/compose/ozonescripts/docker-config @@ -27,6 +27,7 @@ OZONE-SITE.XML_ozone.server.default.replication=1 OZONE-SITE.XML_hdds.scm.safemode.min.datanode=1 OZONE-SITE.XML_hdds.datanode.dir=/data/hdds OZONE-SITE.XML_hdds.datanode.volume.min.free.space=100MB +OZONE-SITE.XML_hdds.datanode.volume.min.free.space.percent=0 OZONE-SITE.XML_hdds.scmclient.max.retry.timeout=30s OZONE-SITE.XML_ozone.http.basedir=/tmp/ozone_http diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/.env b/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/.env index 1571c3aa8249..1ba18a9178c0 100644 --- a/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/.env +++ b/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/.env @@ -16,7 +16,7 @@ HDDS_VERSION=${hdds.version} HADOOP_IMAGE=${docker.hadoop.image} -HADOOP_VERSION=${hadoop.version}${docker.hadoop.image.flavor} +HADOOP_VERSION=${hadoop.acceptance.version}${docker.hadoop.image.flavor} OZONE_RUNNER_VERSION=${docker.ozone-runner.version} OZONE_RUNNER_IMAGE=apache/ozone-runner OZONE_TESTKRB5_IMAGE=${docker.ozone-testkr5b.image} @@ -25,6 +25,6 @@ OZONE_OPTS= RANGER_DB_IMAGE=postgres RANGER_DB_IMAGE_VERSION=12 RANGER_IMAGE=ghcr.io/adoroszlai/ranger-admin -RANGER_IMAGE_VERSION=0ae34250d3af672776fca6a53047699adf3afce5-${ranger.version}-8 -RANGER_VERSION=${ranger.version} +RANGER_IMAGE_VERSION=0ae34250d3af672776fca6a53047699adf3afce5-${ranger.acceptance.version}-8 +RANGER_VERSION=${ranger.acceptance.version} WAITFOR_TIMEOUT=3000 diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/docker-config b/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/docker-config index 4fabf7b8cc77..8133eb1073e6 100644 --- a/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/docker-config +++ b/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/docker-config @@ -45,6 +45,7 @@ OZONE-SITE.XML_ozone.scm.container.size=1GB OZONE-SITE.XML_ozone.scm.datanode.ratis.volume.free-space.min=10MB OZONE-SITE.XML_ozone.scm.pipeline.creation.interval=30s OZONE-SITE.XML_ozone.scm.pipeline.owner.container.count=1 +OZONE-SITE.XML_ozone.scm.ec.pipeline.minimum=1 OZONE-SITE.XML_ozone.scm.datanode.id.dir=/data/metadata OZONE-SITE.XML_ozone.scm.block.client.address=scm OZONE-SITE.XML_ozone.scm.stale.node.interval=30s @@ -73,6 +74,7 @@ OZONE-SITE.XML_ozone.s3.administrators="testuser,s3g" OZONE-SITE.XML_hdds.datanode.dir=/data/hdds OZONE-SITE.XML_hdds.datanode.volume.min.free.space=100MB +OZONE-SITE.XML_hdds.datanode.volume.min.free.space.percent=0 HDFS-SITE.XML_dfs.datanode.address=0.0.0.0:1019 HDFS-SITE.XML_dfs.datanode.http.address=0.0.0.0:1012 CORE-SITE.XML_dfs.data.transfer.protection=authentication diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/test-repair-tools.sh b/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/test-repair-tools.sh index da54e913debb..44179a3bad90 100644 --- a/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/test-repair-tools.sh +++ b/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/test-repair-tools.sh @@ -82,6 +82,7 @@ execute_robot_test ${SCM} repair/ratis-transaction-repair.robot repair_and_restart_om "ozonesecure-ha-om1-1" "om1" repair_and_restart_om "ozonesecure-ha-om2-1" "om2" repair_and_restart_om "ozonesecure-ha-om3-1" "om3" +wait_for_om_leader if ! execute_command_in_container scm1.org timeout 15s ozone sh volume list 1>/dev/null; then echo "Command timed out or failed => OMs are not running as expected. Test for repairing ratis transaction failed." exit 1 diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure-mr/.env b/hadoop-ozone/dist/src/main/compose/ozonesecure-mr/.env index c260913a2f5f..cde8c5ccb521 100644 --- a/hadoop-ozone/dist/src/main/compose/ozonesecure-mr/.env +++ b/hadoop-ozone/dist/src/main/compose/ozonesecure-mr/.env @@ -16,7 +16,7 @@ HDDS_VERSION=${hdds.version} HADOOP_IMAGE=${docker.hadoop.image} -HADOOP_VERSION=${hadoop.version}${docker.hadoop.image.flavor} +HADOOP_VERSION=${hadoop.acceptance.version}${docker.hadoop.image.flavor} OZONE_RUNNER_VERSION=${docker.ozone-runner.version} OZONE_RUNNER_IMAGE=apache/ozone-runner OZONE_TESTKRB5_IMAGE=${docker.ozone-testkr5b.image} diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure-mr/docker-config b/hadoop-ozone/dist/src/main/compose/ozonesecure-mr/docker-config index 51ecc7e8adcf..f475144c2d93 100644 --- a/hadoop-ozone/dist/src/main/compose/ozonesecure-mr/docker-config +++ b/hadoop-ozone/dist/src/main/compose/ozonesecure-mr/docker-config @@ -21,6 +21,7 @@ OZONE-SITE.XML_ozone.scm.container.size=1GB OZONE-SITE.XML_ozone.scm.datanode.ratis.volume.free-space.min=10MB OZONE-SITE.XML_ozone.scm.pipeline.creation.interval=30s OZONE-SITE.XML_ozone.scm.pipeline.owner.container.count=1 +OZONE-SITE.XML_ozone.scm.ec.pipeline.minimum=1 OZONE-SITE.XML_ozone.scm.names=scm OZONE-SITE.XML_ozone.scm.datanode.id.dir=/data/metadata OZONE-SITE.XML_ozone.scm.block.client.address=scm @@ -61,6 +62,7 @@ OZONE-SITE.XML_hdds.datanode.kerberos.keytab.file=/etc/security/keytabs/dn.keyta OZONE-SITE.XML_hdds.datanode.dir=/data/hdds OZONE-SITE.XML_hdds.datanode.volume.min.free.space=100MB +OZONE-SITE.XML_hdds.datanode.volume.min.free.space.percent=0 CORE-SITE.XML_dfs.data.transfer.protection=authentication CORE-SITE.XML_hadoop.security.authentication=kerberos diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure/.env b/hadoop-ozone/dist/src/main/compose/ozonesecure/.env index c260913a2f5f..cde8c5ccb521 100644 --- a/hadoop-ozone/dist/src/main/compose/ozonesecure/.env +++ b/hadoop-ozone/dist/src/main/compose/ozonesecure/.env @@ -16,7 +16,7 @@ HDDS_VERSION=${hdds.version} HADOOP_IMAGE=${docker.hadoop.image} -HADOOP_VERSION=${hadoop.version}${docker.hadoop.image.flavor} +HADOOP_VERSION=${hadoop.acceptance.version}${docker.hadoop.image.flavor} OZONE_RUNNER_VERSION=${docker.ozone-runner.version} OZONE_RUNNER_IMAGE=apache/ozone-runner OZONE_TESTKRB5_IMAGE=${docker.ozone-testkr5b.image} diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-config b/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-config index 5daf6c11fc9b..ba22cc08f856 100644 --- a/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-config +++ b/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-config @@ -26,6 +26,7 @@ OZONE-SITE.XML_ozone.scm.http-address=scm:9876 OZONE-SITE.XML_ozone.scm.container.size=1GB OZONE-SITE.XML_ozone.scm.pipeline.creation.interval=30s OZONE-SITE.XML_ozone.scm.pipeline.owner.container.count=1 +OZONE-SITE.XML_ozone.scm.ec.pipeline.minimum=1 OZONE-SITE.XML_ozone.scm.names=scm OZONE-SITE.XML_ozone.scm.datanode.id.dir=/data/metadata OZONE-SITE.XML_ozone.scm.block.client.address=scm @@ -55,6 +56,7 @@ OZONE-SITE.XML_ozone.s3.administrators="testuser,s3g" OZONE-SITE.XML_hdds.datanode.dir=/data/hdds OZONE-SITE.XML_hdds.datanode.volume.min.free.space=100MB +OZONE-SITE.XML_hdds.datanode.volume.min.free.space.percent=0 HDFS-SITE.XML_dfs.datanode.address=0.0.0.0:1019 HDFS-SITE.XML_dfs.datanode.http.address=0.0.0.0:1012 CORE-SITE.XML_dfs.data.transfer.protection=authentication diff --git a/hadoop-ozone/dist/src/main/compose/restart/docker-config b/hadoop-ozone/dist/src/main/compose/restart/docker-config index 852eb6647c3d..2402e51aa86a 100644 --- a/hadoop-ozone/dist/src/main/compose/restart/docker-config +++ b/hadoop-ozone/dist/src/main/compose/restart/docker-config @@ -28,6 +28,7 @@ OZONE-SITE.XML_ozone.recon.db.dir=/data/metadata/recon OZONE-SITE.XML_ozone.scm.client.address=scm OZONE-SITE.XML_hdds.datanode.dir=/data/hdds OZONE-SITE.XML_hdds.datanode.volume.min.free.space=100MB +OZONE-SITE.XML_hdds.datanode.volume.min.free.space.percent=0 OZONE-SITE.XML_ozone.recon.address=recon:9891 OZONE-SITE.XML_ozone.recon.om.snapshot.task.interval.delay=1m OZONE-SITE.XML_hdds.scmclient.max.retry.timeout=30s diff --git a/hadoop-ozone/dist/src/main/compose/upgrade/compose/ha/.env b/hadoop-ozone/dist/src/main/compose/upgrade/compose/ha/.env index d7c817854ba1..3075289550c4 100644 --- a/hadoop-ozone/dist/src/main/compose/upgrade/compose/ha/.env +++ b/hadoop-ozone/dist/src/main/compose/upgrade/compose/ha/.env @@ -15,7 +15,7 @@ # limitations under the License. HADOOP_IMAGE=${docker.hadoop.image} -HADOOP_VERSION=${hadoop.version}${docker.hadoop.image.flavor} +HADOOP_VERSION=${hadoop.acceptance.version}${docker.hadoop.image.flavor} HDDS_VERSION=${hdds.version} OZONE_RUNNER_VERSION=${docker.ozone-runner.version} OZONE_RUNNER_IMAGE=apache/ozone-runner diff --git a/hadoop-ozone/dist/src/main/compose/upgrade/compose/ha/docker-config b/hadoop-ozone/dist/src/main/compose/upgrade/compose/ha/docker-config index 10426b8610bb..adc18d7c8f53 100644 --- a/hadoop-ozone/dist/src/main/compose/upgrade/compose/ha/docker-config +++ b/hadoop-ozone/dist/src/main/compose/upgrade/compose/ha/docker-config @@ -33,10 +33,12 @@ OZONE-SITE.XML_ozone.scm.primordial.node.id=scm1 OZONE-SITE.XML_ozone.scm.pipeline.creation.interval=30s OZONE-SITE.XML_ozone.scm.pipeline.owner.container.count=1 +OZONE-SITE.XML_ozone.scm.ec.pipeline.minimum=1 OZONE-SITE.XML_ozone.scm.datanode.id.dir=/data/metadata OZONE-SITE.XML_ozone.scm.container.size=1GB OZONE-SITE.XML_hdds.datanode.dir=/data/hdds OZONE-SITE.XML_hdds.datanode.volume.min.free.space=100MB +OZONE-SITE.XML_hdds.datanode.volume.min.free.space.percent=0 OZONE-SITE.XML_ozone.http.basedir=/tmp/ozone_http OZONE-SITE.XML_ozone.client.hbase.enhancements.allowed=true OZONE-SITE.XML_ozone.fs.hsync.enabled=true diff --git a/hadoop-ozone/dist/src/main/compose/upgrade/compose/non-ha/docker-config b/hadoop-ozone/dist/src/main/compose/upgrade/compose/non-ha/docker-config index 8a5380b8347a..e09c0ab86492 100644 --- a/hadoop-ozone/dist/src/main/compose/upgrade/compose/non-ha/docker-config +++ b/hadoop-ozone/dist/src/main/compose/upgrade/compose/non-ha/docker-config @@ -25,6 +25,7 @@ OZONE-SITE.XML_ozone.scm.container.size=1GB OZONE-SITE.XML_ozone.scm.datanode.ratis.volume.free-space.min=10MB OZONE-SITE.XML_ozone.scm.pipeline.creation.interval=30s OZONE-SITE.XML_ozone.scm.pipeline.owner.container.count=1 +OZONE-SITE.XML_ozone.scm.ec.pipeline.minimum=1 OZONE-SITE.XML_ozone.scm.names=scm OZONE-SITE.XML_ozone.scm.datanode.id.dir=/data/metadata OZONE-SITE.XML_ozone.scm.block.client.address=scm @@ -33,6 +34,7 @@ OZONE-SITE.XML_ozone.scm.client.address=scm OZONE-SITE.XML_hdds.datanode.dir=/data/hdds OZONE-SITE.XML_hdds.datanode.volume.min.free.space=100MB +OZONE-SITE.XML_hdds.datanode.volume.min.free.space.percent=0 OZONE-SITE.XML_ozone.recon.db.dir=/data/metadata/recon OZONE-SITE.XML_ozone.recon.om.snapshot.task.interval.delay=1m diff --git a/hadoop-ozone/dist/src/main/compose/xcompat/.env b/hadoop-ozone/dist/src/main/compose/xcompat/.env index b99900a58527..9e6230a4d4fe 100644 --- a/hadoop-ozone/dist/src/main/compose/xcompat/.env +++ b/hadoop-ozone/dist/src/main/compose/xcompat/.env @@ -18,7 +18,7 @@ HDDS_VERSION=${hdds.version} OZONE_RUNNER_VERSION=${docker.ozone-runner.version} OZONE_RUNNER_IMAGE=apache/ozone-runner HADOOP_IMAGE=${docker.hadoop.image} -HADOOP_VERSION=${hadoop.version}${docker.hadoop.image.flavor} +HADOOP_VERSION=${hadoop.acceptance.version}${docker.hadoop.image.flavor} OZONE_TESTKRB5_IMAGE=${docker.ozone-testkr5b.image} OZONE_IMAGE=${docker.ozone.image} OZONE_IMAGE_FLAVOR="${docker.ozone.image.flavor}" diff --git a/hadoop-ozone/dist/src/main/compose/xcompat/docker-config b/hadoop-ozone/dist/src/main/compose/xcompat/docker-config index 13d6e11057a6..f128934072ad 100644 --- a/hadoop-ozone/dist/src/main/compose/xcompat/docker-config +++ b/hadoop-ozone/dist/src/main/compose/xcompat/docker-config @@ -36,6 +36,7 @@ OZONE-SITE.XML_ozone.scm.datanode.id.dir=/data/metadata OZONE-SITE.XML_ozone.scm.names=scm OZONE-SITE.XML_ozone.scm.pipeline.creation.interval=30s OZONE-SITE.XML_ozone.scm.pipeline.owner.container.count=1 +OZONE-SITE.XML_ozone.scm.ec.pipeline.minimum=1 OZONE-SITE.XML_ozone.datanode.pipeline.limit=1 OZONE-SITE.XML_recon.om.snapshot.task.interval.delay=1m OZONE-SITE.XML_hdds.scmclient.max.retry.timeout=30s diff --git a/hadoop-ozone/dist/src/main/compose/xcompat/new-cluster.yaml b/hadoop-ozone/dist/src/main/compose/xcompat/new-cluster.yaml index 275338d7e709..3de29df026d4 100644 --- a/hadoop-ozone/dist/src/main/compose/xcompat/new-cluster.yaml +++ b/hadoop-ozone/dist/src/main/compose/xcompat/new-cluster.yaml @@ -56,6 +56,7 @@ services: - 9882 environment: OZONE_OPTS: + OZONE-SITE.XML_hdds.datanode.volume.min.free.space.percent: 0 command: ["ozone","datanode"] om: <<: *new-config diff --git a/hadoop-ozone/dist/src/main/k8s/definitions/ozone/config.yaml b/hadoop-ozone/dist/src/main/k8s/definitions/ozone/config.yaml index e646b1c0af88..87ff48038a7d 100644 --- a/hadoop-ozone/dist/src/main/k8s/definitions/ozone/config.yaml +++ b/hadoop-ozone/dist/src/main/k8s/definitions/ozone/config.yaml @@ -38,7 +38,10 @@ data: OZONE-SITE.XML_ozone.scm.client.address: "scm-0.scm" OZONE-SITE.XML_ozone.scm.datanode.id.dir: "/data/metadata" OZONE-SITE.XML_ozone.scm.names: "scm-0.scm" - LOG4J.PROPERTIES_log4j.rootLogger: "INFO, stdout" - LOG4J.PROPERTIES_log4j.appender.stdout: "org.apache.log4j.ConsoleAppender" - LOG4J.PROPERTIES_log4j.appender.stdout.layout: "org.apache.log4j.PatternLayout" - LOG4J.PROPERTIES_log4j.appender.stdout.layout.ConversionPattern: "%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n" + OZONE-SITE.XML_ozone.client.max.retries: "15" + OZONE-SITE.XML_ozone.client.retry.interval: "1" + LOG4J.PROPERTIES_hadoop.root.logger: "INFO,console" + LOG4J.PROPERTIES_log4j.rootLogger: "${hadoop.root.logger}" + LOG4J.PROPERTIES_log4j.appender.console: "org.apache.log4j.ConsoleAppender" + LOG4J.PROPERTIES_log4j.appender.console.layout: "org.apache.log4j.PatternLayout" + LOG4J.PROPERTIES_log4j.appender.console.layout.ConversionPattern: "%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n" diff --git a/hadoop-ozone/dist/src/main/k8s/examples/ozone/test.sh b/hadoop-ozone/dist/src/main/k8s/examples/ozone/test.sh index d1006a3fc862..91a352d7b9a4 100755 --- a/hadoop-ozone/dist/src/main/k8s/examples/ozone/test.sh +++ b/hadoop-ozone/dist/src/main/k8s/examples/ozone/test.sh @@ -35,6 +35,7 @@ execute_robot_test ${SCM} -v PREFIX:pre smoketest/freon/validate.robot kubectl delete pod datanode-0 datanode-1 datanode-2 wait_for_startup +wait_for_pipeline execute_robot_test ${SCM} -v PREFIX:pre smoketest/freon/validate.robot execute_robot_test ${SCM} -v PREFIX:post smoketest/freon/generate.robot diff --git a/hadoop-ozone/dist/src/main/k8s/examples/testlib.sh b/hadoop-ozone/dist/src/main/k8s/examples/testlib.sh index 1ae4dd98485f..5d03235a2f03 100644 --- a/hadoop-ozone/dist/src/main/k8s/examples/testlib.sh +++ b/hadoop-ozone/dist/src/main/k8s/examples/testlib.sh @@ -50,6 +50,23 @@ wait_for_startup(){ fi } +wait_for_pipeline() { + retry assert_pipeline_exists +} + +execute_command_in_container() { + CONTAINER="$1" + shift 1 #Remove first argument which was the container name + + kubectl exec "${CONTAINER}" -- "$@" +} + +assert_pipeline_exists() { + local count + count=$(execute_command_in_container scm-0 ozone admin pipeline list --state OPEN --filter-by-factor THREE --json | jq -r 'length') + [[ $count -gt 0 ]] +} + all_pods_are_running() { local -i running=$(kubectl get pod --field-selector status.phase=Running | grep -v 'STATUS' | wc -l) local -i all=$(kubectl get pod | grep -v 'STATUS' | wc -l) diff --git a/hadoop-ozone/dist/src/main/license/bin/LICENSE.txt b/hadoop-ozone/dist/src/main/license/bin/LICENSE.txt index 8f989b7f7039..977b106598e3 100644 --- a/hadoop-ozone/dist/src/main/license/bin/LICENSE.txt +++ b/hadoop-ozone/dist/src/main/license/bin/LICENSE.txt @@ -259,6 +259,7 @@ CDDL 1.1 + GPLv2 with classpath exception javax.servlet:javax.servlet-api javax.servlet.jsp:jsp-api javax.ws.rs:jsr311-api + javax.xml.bind:jaxb-api org.glassfish.jaxb:jaxb-runtime org.glassfish.jaxb:txw2 diff --git a/hadoop-ozone/dist/src/main/license/jar-report.txt b/hadoop-ozone/dist/src/main/license/jar-report.txt index 79d6b71c4b50..52e6139e39ec 100644 --- a/hadoop-ozone/dist/src/main/license/jar-report.txt +++ b/hadoop-ozone/dist/src/main/license/jar-report.txt @@ -139,6 +139,7 @@ share/ozone/lib/javax.el-api.jar share/ozone/lib/javax.inject.jar share/ozone/lib/javax.interceptor-api.jar share/ozone/lib/javax.servlet-api.jar +share/ozone/lib/jaxb-api.jar share/ozone/lib/jaxb-runtime.jar share/ozone/lib/jcip-annotations.jar share/ozone/lib/jcl-over-slf4j.jar diff --git a/hadoop-ozone/fault-injection-test/mini-chaos-tests/pom.xml b/hadoop-ozone/fault-injection-test/mini-chaos-tests/pom.xml index b1db75d37937..7178eb6e2d97 100644 --- a/hadoop-ozone/fault-injection-test/mini-chaos-tests/pom.xml +++ b/hadoop-ozone/fault-injection-test/mini-chaos-tests/pom.xml @@ -17,11 +17,11 @@ org.apache.ozone ozone-fault-injection-test - 2.1.0.1-4.3.0-3 + 2.1.0.1-4.3.0-4 mini-chaos-tests - 2.1.0.1-4.3.0-3 + 2.1.0.1-4.3.0-4 Apache Ozone Mini Ozone Chaos Tests Apache Ozone Mini Ozone Chaos Tests diff --git a/hadoop-ozone/fault-injection-test/network-tests/pom.xml b/hadoop-ozone/fault-injection-test/network-tests/pom.xml index 1ffdb30a3146..ce061a776a34 100644 --- a/hadoop-ozone/fault-injection-test/network-tests/pom.xml +++ b/hadoop-ozone/fault-injection-test/network-tests/pom.xml @@ -17,7 +17,7 @@ org.apache.ozone ozone-fault-injection-test - 2.1.0.1-4.3.0-3 + 2.1.0.1-4.3.0-4 ozone-network-tests jar diff --git a/hadoop-ozone/fault-injection-test/pom.xml b/hadoop-ozone/fault-injection-test/pom.xml index 9a7ae3b3e4fa..1298c69fb27f 100644 --- a/hadoop-ozone/fault-injection-test/pom.xml +++ b/hadoop-ozone/fault-injection-test/pom.xml @@ -17,10 +17,10 @@ org.apache.ozone ozone - 2.1.0.1-4.3.0-3 + 2.1.0.1-4.3.0-4 ozone-fault-injection-test - 2.1.0.1-4.3.0-3 + 2.1.0.1-4.3.0-4 pom Apache Ozone Fault Injection Tests Apache Ozone Fault Injection Tests diff --git a/hadoop-ozone/freon/pom.xml b/hadoop-ozone/freon/pom.xml index 2310313dd76a..9237f4a1e77c 100644 --- a/hadoop-ozone/freon/pom.xml +++ b/hadoop-ozone/freon/pom.xml @@ -17,10 +17,10 @@ org.apache.ozone ozone - 2.1.0.1-4.3.0-3 + 2.1.0.1-4.3.0-4 ozone-freon - 2.1.0.1-4.3.0-3 + 2.1.0.1-4.3.0-4 jar Apache Ozone Freon Apache Ozone Freon diff --git a/hadoop-ozone/httpfsgateway/pom.xml b/hadoop-ozone/httpfsgateway/pom.xml index 796c3cbbce92..0bfae18e52dd 100644 --- a/hadoop-ozone/httpfsgateway/pom.xml +++ b/hadoop-ozone/httpfsgateway/pom.xml @@ -19,10 +19,10 @@ org.apache.ozone ozone - 2.1.0.1-4.3.0-3 + 2.1.0.1-4.3.0-4 ozone-httpfsgateway - 2.1.0.1-4.3.0-3 + 2.1.0.1-4.3.0-4 jar Apache Ozone HttpFS diff --git a/hadoop-ozone/insight/pom.xml b/hadoop-ozone/insight/pom.xml index a0efcecdfec6..ad53577f8a2d 100644 --- a/hadoop-ozone/insight/pom.xml +++ b/hadoop-ozone/insight/pom.xml @@ -17,10 +17,10 @@ org.apache.ozone ozone - 2.1.0.1-4.3.0-3 + 2.1.0.1-4.3.0-4 ozone-insight - 2.1.0.1-4.3.0-3 + 2.1.0.1-4.3.0-4 jar Apache Ozone Insight Tool Apache Ozone Insight Tool diff --git a/hadoop-ozone/integration-test-recon/pom.xml b/hadoop-ozone/integration-test-recon/pom.xml index 2d9e788c376d..cfeeaa952618 100644 --- a/hadoop-ozone/integration-test-recon/pom.xml +++ b/hadoop-ozone/integration-test-recon/pom.xml @@ -17,10 +17,10 @@ org.apache.ozone ozone - 2.1.0.1-4.3.0-3 + 2.1.0.1-4.3.0-4 ozone-integration-test-recon - 2.1.0.1-4.3.0-3 + 2.1.0.1-4.3.0-4 jar Apache Ozone Recon Integration Tests Apache Ozone Integration Tests with Recon diff --git a/hadoop-ozone/integration-test-s3/pom.xml b/hadoop-ozone/integration-test-s3/pom.xml index ee409d78576b..438d1b9f3736 100644 --- a/hadoop-ozone/integration-test-s3/pom.xml +++ b/hadoop-ozone/integration-test-s3/pom.xml @@ -17,10 +17,10 @@ org.apache.ozone ozone - 2.1.0.1-4.3.0-3 + 2.1.0.1-4.3.0-4 ozone-integration-test-s3 - 2.1.0.1-4.3.0-3 + 2.1.0.1-4.3.0-4 jar Apache Ozone S3 Integration Tests Apache Ozone Integration Tests with S3 Gateway diff --git a/hadoop-ozone/integration-test/pom.xml b/hadoop-ozone/integration-test/pom.xml index 45283e91ee8d..5f34e07f0727 100644 --- a/hadoop-ozone/integration-test/pom.xml +++ b/hadoop-ozone/integration-test/pom.xml @@ -17,10 +17,10 @@ org.apache.ozone ozone - 2.1.0.1-4.3.0-3 + 2.1.0.1-4.3.0-4 ozone-integration-test - 2.1.0.1-4.3.0-3 + 2.1.0.1-4.3.0-4 jar Apache Ozone Integration Tests Apache Ozone Integration Tests diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerHAWithStoppedNodes.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerHAWithStoppedNodes.java index eed911223dc6..ab8863a6d74d 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerHAWithStoppedNodes.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerHAWithStoppedNodes.java @@ -70,7 +70,6 @@ import org.apache.hadoop.security.UserGroupInformation; import org.apache.log4j.Logger; import org.apache.ozone.test.GenericTestUtils; -import org.apache.ozone.test.tag.Flaky; import org.apache.ratis.client.RaftClient; import org.apache.ratis.conf.RaftProperties; import org.apache.ratis.protocol.ClientId; @@ -501,7 +500,6 @@ void testIncrementalWaitTimeWithSameNodeFailover() throws Exception { omFailoverProxyProvider.getWaitTime()); } - @Flaky("HDDS-11353") @Test void testOMHAMetrics() throws Exception { // Get leader OM @@ -512,7 +510,7 @@ void testOMHAMetrics() throws Exception { // Get a list of all OMs List omList = getCluster().getOzoneManagersList(); // Check metrics for all OMs - checkOMHAMetricsForAllOMs(omList, leaderOMId); + waitForOMHAMetricsForAllOMs(omList, leaderOMId); // Restart leader OM getCluster().shutdownOzoneManager(leaderOM); @@ -524,19 +522,33 @@ void testOMHAMetrics() throws Exception { String newLeaderOMId = newLeaderOM.getOMNodeId(); // Get a list of all OMs again omList = getCluster().getOzoneManagersList(); - // New state for the old leader - int newState = leaderOMId.equals(newLeaderOMId) ? 1 : 0; - // Get old leader - OzoneManager oldLeader = getCluster().getOzoneManager(leaderOMId); - // Get old leader's metrics - OMHAMetrics omhaMetrics = oldLeader.getOmhaMetrics(); + // Check that metrics for all OMs have been updated + waitForOMHAMetricsForAllOMs(omList, newLeaderOMId); + } - assertEquals(newState, - omhaMetrics.getOmhaInfoOzoneManagerHALeaderState()); + private void waitForOMHAMetricsForAllOMs(List omList, + String leaderOMId) throws Exception { + GenericTestUtils.waitFor( + () -> areOMHAMetricsUpdatedForAllOMs(omList, leaderOMId), + 100, 30000); + checkOMHAMetricsForAllOMs(omList, leaderOMId); + } - // Check that metrics for all OMs have been updated - checkOMHAMetricsForAllOMs(omList, newLeaderOMId); + private boolean areOMHAMetricsUpdatedForAllOMs(List omList, + String leaderOMId) { + for (OzoneManager om : omList) { + OMHAMetrics omhaMetrics = om.getOmhaMetrics(); + String nodeId = om.getOMNodeId(); + int expectedState = nodeId.equals(leaderOMId) ? 1 : 0; + if (omhaMetrics == null + || !nodeId.equals(omhaMetrics.getOmhaInfoNodeId()) + || expectedState + != omhaMetrics.getOmhaInfoOzoneManagerHALeaderState()) { + return false; + } + } + return true; } private void checkOMHAMetricsForAllOMs(List omList, diff --git a/hadoop-ozone/interface-client/pom.xml b/hadoop-ozone/interface-client/pom.xml index ae997115d62d..3b1cfe7073b5 100644 --- a/hadoop-ozone/interface-client/pom.xml +++ b/hadoop-ozone/interface-client/pom.xml @@ -17,10 +17,10 @@ org.apache.ozone ozone - 2.1.0.1-4.3.0-3 + 2.1.0.1-4.3.0-4 ozone-interface-client - 2.1.0.1-4.3.0-3 + 2.1.0.1-4.3.0-4 jar Apache Ozone Client Interface Apache Ozone Client interface diff --git a/hadoop-ozone/interface-storage/pom.xml b/hadoop-ozone/interface-storage/pom.xml index 9cb997622a55..e9c0e209fff4 100644 --- a/hadoop-ozone/interface-storage/pom.xml +++ b/hadoop-ozone/interface-storage/pom.xml @@ -17,10 +17,10 @@ org.apache.ozone ozone - 2.1.0.1-4.3.0-3 + 2.1.0.1-4.3.0-4 ozone-interface-storage - 2.1.0.1-4.3.0-3 + 2.1.0.1-4.3.0-4 jar Apache Ozone Storage Interface Apache Ozone Storage Interface diff --git a/hadoop-ozone/mini-cluster/pom.xml b/hadoop-ozone/mini-cluster/pom.xml index a3af0d190e00..f0c9b0c6155a 100644 --- a/hadoop-ozone/mini-cluster/pom.xml +++ b/hadoop-ozone/mini-cluster/pom.xml @@ -17,10 +17,10 @@ org.apache.ozone ozone - 2.1.0.1-4.3.0-3 + 2.1.0.1-4.3.0-4 ozone-mini-cluster - 2.1.0.1-4.3.0-3 + 2.1.0.1-4.3.0-4 jar Apache Ozone Mini Cluster Apache Ozone Mini Cluster for Integration Tests diff --git a/hadoop-ozone/multitenancy-ranger/pom.xml b/hadoop-ozone/multitenancy-ranger/pom.xml index 7f8a3f17c142..8571203f45c7 100644 --- a/hadoop-ozone/multitenancy-ranger/pom.xml +++ b/hadoop-ozone/multitenancy-ranger/pom.xml @@ -17,10 +17,10 @@ org.apache.ozone ozone - 2.1.0.1-4.3.0-3 + 2.1.0.1-4.3.0-4 ozone-multitenancy-ranger - 2.1.0.1-4.3.0-3 + 2.1.0.1-4.3.0-4 jar Apache Ozone Multitenancy with Ranger Implementation of multitenancy for Apache Ozone Manager Server using Apache Ranger diff --git a/hadoop-ozone/ozone-manager/pom.xml b/hadoop-ozone/ozone-manager/pom.xml index 97b9f60c92a2..b65af6b1e3e7 100644 --- a/hadoop-ozone/ozone-manager/pom.xml +++ b/hadoop-ozone/ozone-manager/pom.xml @@ -17,10 +17,10 @@ org.apache.ozone ozone - 2.1.0.1-4.3.0-3 + 2.1.0.1-4.3.0-4 ozone-manager - 2.1.0.1-4.3.0-3 + 2.1.0.1-4.3.0-4 jar Apache Ozone Manager Server Apache Ozone Manager Server diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java index 4b097fa4d334..74d69d7e3873 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java @@ -3845,24 +3845,32 @@ public OmMultipartUploadListParts listParts(final String volumeName, final String bucketName, String keyName, String uploadID, int partNumberMarker, int maxParts) throws IOException { - ResolvedBucket bucket = resolveBucketLink(Pair.of(volumeName, bucketName)); + final ResolvedBucket bucket = resolveBucketLink(Pair.of(volumeName, bucketName)); + final String realVolumeName = bucket.realVolume(); + final String realBucketName = bucket.realBucket(); - Map auditMap = bucket.audit(); - auditMap.put(OzoneConsts.KEY, keyName); - auditMap.put(OzoneConsts.UPLOAD_ID, uploadID); - auditMap.put(OzoneConsts.PART_NUMBER_MARKER, - Integer.toString(partNumberMarker)); - auditMap.put(OzoneConsts.MAX_PARTS, Integer.toString(maxParts)); - - metrics.incNumListMultipartUploadParts(); + final Map auditMap = bucket.audit(); try { - OmMultipartUploadListParts omMultipartUploadListParts = - keyManager.listParts(bucket.realVolume(), bucket.realBucket(), - keyName, uploadID, partNumberMarker, maxParts); + auditMap.put(OzoneConsts.KEY, keyName); + auditMap.put(OzoneConsts.UPLOAD_ID, uploadID); + auditMap.put(OzoneConsts.PART_NUMBER_MARKER, + Integer.toString(partNumberMarker)); + auditMap.put(OzoneConsts.MAX_PARTS, Integer.toString(maxParts)); + + if (getAclsEnabled()) { + omMetadataReader.checkAcls( + ResourceType.BUCKET, StoreType.OZONE, ACLType.READ, realVolumeName, realBucketName, null); + omMetadataReader.checkAcls( + ResourceType.KEY, StoreType.OZONE, ACLType.READ, realVolumeName, realBucketName, keyName); + } + + metrics.incNumListMultipartUploadParts(); + final OmMultipartUploadListParts omMultipartUploadListParts = keyManager.listParts( + realVolumeName, realBucketName, keyName, uploadID, partNumberMarker, maxParts); AUDIT.logReadSuccess(buildAuditMessageForSuccess(OMAction .LIST_MULTIPART_UPLOAD_PARTS, auditMap)); return omMultipartUploadListParts; - } catch (IOException ex) { + } catch (Exception ex) { metrics.incNumListMultipartUploadPartFails(); AUDIT.logReadFailure(buildAuditMessageForFailure(OMAction .LIST_MULTIPART_UPLOAD_PARTS, auditMap, ex)); @@ -3876,15 +3884,24 @@ public OmMultipartUploadList listMultipartUploads(String volumeName, String prefix, String keyMarker, String uploadIdMarker, int maxUploads, boolean withPagination) throws IOException { - ResolvedBucket bucket = resolveBucketLink(Pair.of(volumeName, bucketName)); + final ResolvedBucket bucket = resolveBucketLink(Pair.of(volumeName, bucketName)); + final String realVolumeName = bucket.realVolume(); + final String realBucketName = bucket.realBucket(); - Map auditMap = bucket.audit(); + final Map auditMap = bucket.audit(); auditMap.put(OzoneConsts.PREFIX, prefix); - metrics.incNumListMultipartUploads(); try { - OmMultipartUploadList omMultipartUploadList = keyManager.listMultipartUploads(bucket.realVolume(), - bucket.realBucket(), prefix, keyMarker, uploadIdMarker, maxUploads, withPagination); + if (getAclsEnabled()) { + omMetadataReader.checkAcls( + ResourceType.BUCKET, StoreType.OZONE, ACLType.READ, realVolumeName, realBucketName, null); + omMetadataReader.checkAcls( + ResourceType.BUCKET, StoreType.OZONE, ACLType.LIST, realVolumeName, realBucketName, null); + } + + metrics.incNumListMultipartUploads(); + final OmMultipartUploadList omMultipartUploadList = keyManager.listMultipartUploads( + realVolumeName, realBucketName, prefix, keyMarker, uploadIdMarker, maxUploads, withPagination); AUDIT.logReadSuccess(buildAuditMessageForSuccess(OMAction.LIST_MULTIPART_UPLOADS, auditMap)); return omMultipartUploadList; diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ha/OMHAMetrics.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ha/OMHAMetrics.java index b13a8e0b607a..7ff6dfdf7dd3 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ha/OMHAMetrics.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ha/OMHAMetrics.java @@ -109,7 +109,7 @@ public synchronized void getMetrics(MetricsCollector collector, boolean all) { MetricsRecordBuilder recordBuilder = collector.addRecord(SOURCE_NAME); // Check current node state (1 leader, 0 follower) - int state = currNodeId.equals(leaderId) ? 1 : 0; + int state = getOzoneManagerHALeaderState(); omhaMetricsInfo.setNodeId(currNodeId); omhaMetricsInfo.setOzoneManagerHALeaderState(state); @@ -122,11 +122,15 @@ public synchronized void getMetrics(MetricsCollector collector, boolean all) { @VisibleForTesting public String getOmhaInfoNodeId() { - return omhaMetricsInfo.getNodeId(); + return currNodeId; } @VisibleForTesting public int getOmhaInfoOzoneManagerHALeaderState() { - return omhaMetricsInfo.getOzoneManagerHALeaderState(); + return getOzoneManagerHALeaderState(); + } + + private int getOzoneManagerHALeaderState() { + return currNodeId.equals(leaderId) ? 1 : 0; } } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerListMultipartUploadsAcls.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerListMultipartUploadsAcls.java new file mode 100644 index 000000000000..d23e761a9fd5 --- /dev/null +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerListMultipartUploadsAcls.java @@ -0,0 +1,196 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.om; + +import static java.util.Collections.emptyList; +import static org.apache.hadoop.ozone.security.acl.IAccessAuthorizer.ACLType.LIST; +import static org.apache.hadoop.ozone.security.acl.IAccessAuthorizer.ACLType.READ; +import static org.apache.hadoop.ozone.security.acl.OzoneObj.ResourceType.BUCKET; +import static org.apache.hadoop.ozone.security.acl.OzoneObj.StoreType.OZONE; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyBoolean; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.anyMap; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.doNothing; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.inOrder; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.io.File; +import org.apache.commons.lang3.tuple.Pair; +import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.hdds.scm.HddsWhiteboxTestUtils; +import org.apache.hadoop.hdds.server.ServerUtils; +import org.apache.hadoop.ozone.audit.AuditMessage; +import org.apache.hadoop.ozone.om.exceptions.OMException; +import org.apache.hadoop.ozone.om.helpers.OmMultipartUploadList; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.S3Authentication; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInstance; +import org.junit.jupiter.api.io.TempDir; +import org.mockito.InOrder; + +/** + * Unit tests for ACL checks on {@link OzoneManager#listMultipartUploads}. + */ +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +public class TestOzoneManagerListMultipartUploadsAcls { + + private OmTestManagers omTestManagers; + private OzoneManager om; + + private OzoneManager omSpy; + private OmMetadataReader omMetadataReader; + private KeyManager keyManager; + private OMMetrics metrics; + + private static final String REQUESTED_VOLUME = "requestedVolume"; + private static final String REQUESTED_BUCKET = "requestedBucket"; + private static final String REAL_VOLUME = "realVolume"; + private static final String REAL_BUCKET = "realBucket"; + private static final String PREFIX = "prefix"; + + @BeforeAll + void setup(@TempDir File folder) throws Exception { + final OzoneConfiguration conf = new OzoneConfiguration(); + ServerUtils.setOzoneMetaDirPath(conf, folder.toString()); + omTestManagers = new OmTestManagers(conf); + om = omTestManagers.getOzoneManager(); + } + + @AfterAll + void cleanup() { + if (omTestManagers != null) { + omTestManagers.stop(); + } + } + + @BeforeEach + void init() throws Exception { + omSpy = spy(om); + omMetadataReader = mock(OmMetadataReader.class); + keyManager = mock(KeyManager.class); + metrics = mock(OMMetrics.class); + + HddsWhiteboxTestUtils.setInternalState(omSpy, "omMetadataReader", omMetadataReader); + HddsWhiteboxTestUtils.setInternalState(omSpy, "keyManager", keyManager); + HddsWhiteboxTestUtils.setInternalState(omSpy, "metrics", metrics); + + doReturn(new ResolvedBucket(REQUESTED_VOLUME, REQUESTED_BUCKET, REAL_VOLUME, REAL_BUCKET, "owner", null)) + .when(omSpy).resolveBucketLink(Pair.of(REQUESTED_VOLUME, REQUESTED_BUCKET)); + + final AuditMessage mockAuditMessage = mock(AuditMessage.class); + when(mockAuditMessage.getOp()).thenReturn("LIST_MULTIPART_UPLOADS"); + doReturn(mockAuditMessage).when(omSpy).buildAuditMessageForSuccess(any(), anyMap()); + doReturn(mockAuditMessage).when(omSpy).buildAuditMessageForFailure(any(), anyMap(), any(Throwable.class)); + + when( + keyManager.listMultipartUploads( + anyString(), anyString(), anyString(), anyString(), anyString(), anyInt(), anyBoolean())) + .thenReturn(OmMultipartUploadList.newBuilder().setUploads(emptyList()).build()); + } + + @AfterEach + void tearDown() { + OzoneManager.setS3Auth(null); + } + + @Test + void testSkipsAclChecksWhenAclsAreDisabled() throws Exception { + setupS3Request(); + when(omSpy.getAclsEnabled()).thenReturn(false); + + omSpy.listMultipartUploads(REQUESTED_VOLUME, REQUESTED_BUCKET, PREFIX, "", "", 10, false); + + verify(omMetadataReader, never()).checkAcls(any(), any(), any(), any(), any(), any()); + verify(keyManager).listMultipartUploads( + eq(REAL_VOLUME), eq(REAL_BUCKET), eq(PREFIX), eq(""), eq(""), eq(10), eq(false)); + } + + @Test + void testAclsEnabledChecksBucketReadThenListUsingResolvedNames() throws Exception { + setupS3Request(); + when(omSpy.getAclsEnabled()).thenReturn(true); + + omSpy.listMultipartUploads(REQUESTED_VOLUME, REQUESTED_BUCKET, PREFIX, "", "", 10, false); + + final InOrder inOrder = inOrder(omMetadataReader); + inOrder.verify(omMetadataReader).checkAcls( + BUCKET, OZONE, READ, REAL_VOLUME, REAL_BUCKET, null); + inOrder.verify(omMetadataReader).checkAcls( + BUCKET, OZONE, LIST, REAL_VOLUME, REAL_BUCKET, null); + verify(keyManager).listMultipartUploads( + eq(REAL_VOLUME), eq(REAL_BUCKET), eq(PREFIX), eq(""), eq(""), eq(10), eq(false)); + verify(metrics).incNumListMultipartUploads(); + verify(metrics, never()).incNumListMultipartUploadFails(); + } + + @Test + void testReadAclAccessDeniedSkipsKeyManagerAndListAclChecksAndIncrementsFailMetric() throws Exception { + setupS3Request(); + when(omSpy.getAclsEnabled()).thenReturn(true); + + doThrow(new OMException("denied", OMException.ResultCodes.PERMISSION_DENIED)) + .when(omMetadataReader).checkAcls(BUCKET, OZONE, READ, REAL_VOLUME, REAL_BUCKET, null); + + assertThrows( + OMException.class, () -> omSpy.listMultipartUploads( + REQUESTED_VOLUME, REQUESTED_BUCKET, PREFIX, "", "", 10, false)); + + verify(keyManager, never()).listMultipartUploads( + anyString(), anyString(), anyString(), anyString(), anyString(), anyInt(), anyBoolean()); + verify(metrics).incNumListMultipartUploadFails(); + verify(metrics, never()).incNumListMultipartUploads(); + verify(omMetadataReader, never()).checkAcls(BUCKET, OZONE, LIST, REAL_VOLUME, REAL_BUCKET, null); + } + + @Test + void testListAclAccessDeniedSkipsKeyManagerAndIncrementsFailMetric() throws Exception { + setupS3Request(); + when(omSpy.getAclsEnabled()).thenReturn(true); + + doNothing().when(omMetadataReader).checkAcls(BUCKET, OZONE, READ, REAL_VOLUME, REAL_BUCKET, null); + doThrow(new OMException("denied", OMException.ResultCodes.PERMISSION_DENIED)) + .when(omMetadataReader).checkAcls(BUCKET, OZONE, LIST, REAL_VOLUME, REAL_BUCKET, null); + + assertThrows( + OMException.class, () -> omSpy.listMultipartUploads( + REQUESTED_VOLUME, REQUESTED_BUCKET, PREFIX, "", "", 10, false)); + + verify(keyManager, never()).listMultipartUploads( + anyString(), anyString(), anyString(), anyString(), anyString(), anyInt(), anyBoolean()); + verify(metrics).incNumListMultipartUploadFails(); + verify(metrics, never()).incNumListMultipartUploads(); + } + + private void setupS3Request() { + OzoneManager.setS3Auth(S3Authentication.newBuilder().setAccessId("AKIAJWFJK62WUTKNFJJA").build()); + } +} diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerListPartsAcls.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerListPartsAcls.java new file mode 100644 index 000000000000..537133eae1e0 --- /dev/null +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerListPartsAcls.java @@ -0,0 +1,196 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.om; + +import static org.apache.hadoop.ozone.security.acl.IAccessAuthorizer.ACLType.READ; +import static org.apache.hadoop.ozone.security.acl.OzoneObj.ResourceType.BUCKET; +import static org.apache.hadoop.ozone.security.acl.OzoneObj.ResourceType.KEY; +import static org.apache.hadoop.ozone.security.acl.OzoneObj.StoreType.OZONE; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.anyMap; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.doNothing; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.inOrder; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.io.File; +import org.apache.commons.lang3.tuple.Pair; +import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.hdds.scm.HddsWhiteboxTestUtils; +import org.apache.hadoop.hdds.server.ServerUtils; +import org.apache.hadoop.ozone.audit.AuditMessage; +import org.apache.hadoop.ozone.om.exceptions.OMException; +import org.apache.hadoop.ozone.om.helpers.OmMultipartUploadListParts; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.S3Authentication; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInstance; +import org.junit.jupiter.api.io.TempDir; +import org.mockito.InOrder; + + +/** + * Unit tests for ACL checks on {@link OzoneManager#listParts}. + */ +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +public class TestOzoneManagerListPartsAcls { + + private OmTestManagers omTestManagers; + private OzoneManager om; + + private OzoneManager omSpy; + private OmMetadataReader omMetadataReader; + private KeyManager keyManager; + private OMMetrics metrics; + + private static final String REQUESTED_VOLUME = "requestedVolume"; + private static final String REQUESTED_BUCKET = "requestedBucket"; + private static final String REAL_VOLUME = "realVolume"; + private static final String REAL_BUCKET = "realBucket"; + private static final String KEY_NAME = "object/key"; + private static final String UPLOAD_ID = "uploadId"; + + @BeforeAll + void setup(@TempDir File folder) throws Exception { + final OzoneConfiguration conf = new OzoneConfiguration(); + ServerUtils.setOzoneMetaDirPath(conf, folder.toString()); + omTestManagers = new OmTestManagers(conf); + om = omTestManagers.getOzoneManager(); + } + + @AfterAll + void cleanup() { + if (omTestManagers != null) { + omTestManagers.stop(); + } + } + + @BeforeEach + void init() throws Exception { + omSpy = spy(om); + omMetadataReader = mock(OmMetadataReader.class); + keyManager = mock(KeyManager.class); + metrics = mock(OMMetrics.class); + + HddsWhiteboxTestUtils.setInternalState(omSpy, "omMetadataReader", omMetadataReader); + HddsWhiteboxTestUtils.setInternalState(omSpy, "keyManager", keyManager); + HddsWhiteboxTestUtils.setInternalState(omSpy, "metrics", metrics); + + doReturn(new ResolvedBucket(REQUESTED_VOLUME, REQUESTED_BUCKET, REAL_VOLUME, REAL_BUCKET, "owner", null)) + .when(omSpy).resolveBucketLink(Pair.of(REQUESTED_VOLUME, REQUESTED_BUCKET)); + + final AuditMessage mockAuditMessage = mock(AuditMessage.class); + when(mockAuditMessage.getOp()).thenReturn("LIST_MULTIPART_UPLOAD_PARTS"); + doReturn(mockAuditMessage).when(omSpy).buildAuditMessageForSuccess(any(), anyMap()); + doReturn(mockAuditMessage).when(omSpy).buildAuditMessageForFailure(any(), anyMap(), any(Throwable.class)); + + when(keyManager.listParts(anyString(), anyString(), anyString(), anyString(), anyInt(), anyInt())) + .thenReturn(mock(OmMultipartUploadListParts.class)); + } + + @AfterEach + void tearDown() { + OzoneManager.setS3Auth(null); + } + + @Test + void testSkipsAclChecksWhenAclsAreDisabled() throws Exception { + setupS3Request(); + when(omSpy.getAclsEnabled()).thenReturn(false); + + omSpy.listParts(REQUESTED_VOLUME, REQUESTED_BUCKET, KEY_NAME, UPLOAD_ID, 0, 10); + + verify(omMetadataReader, never()).checkAcls(any(), any(), any(), any(), any(), any()); + verify(keyManager).listParts( + eq(REAL_VOLUME), eq(REAL_BUCKET), eq(KEY_NAME), eq(UPLOAD_ID), eq(0), eq(10)); + verify(metrics).incNumListMultipartUploadParts(); + verify(metrics, never()).incNumListMultipartUploadPartFails(); + } + + @Test + void testAclsEnabledChecksBucketReadThenKeyReadUsingResolvedNames() throws Exception { + setupS3Request(); + when(omSpy.getAclsEnabled()).thenReturn(true); + + omSpy.listParts(REQUESTED_VOLUME, REQUESTED_BUCKET, KEY_NAME, UPLOAD_ID, 0, 10); + + final InOrder inOrder = inOrder(omMetadataReader); + inOrder.verify(omMetadataReader).checkAcls(BUCKET, OZONE, READ, REAL_VOLUME, REAL_BUCKET, null); + inOrder.verify(omMetadataReader).checkAcls(KEY, OZONE, READ, REAL_VOLUME, REAL_BUCKET, KEY_NAME); + verify(keyManager).listParts( + eq(REAL_VOLUME), eq(REAL_BUCKET), eq(KEY_NAME), eq(UPLOAD_ID), eq(0), eq(10)); + verify(metrics).incNumListMultipartUploadParts(); + verify(metrics, never()).incNumListMultipartUploadPartFails(); + } + + @Test + void testReadAclAccessDeniedSkipsKeyManagerAndKeyReadAclChecksAndRecordsFailure() throws Exception { + setupS3Request(); + when(omSpy.getAclsEnabled()).thenReturn(true); + + doThrow(new OMException("denied", OMException.ResultCodes.PERMISSION_DENIED)) + .when(omMetadataReader).checkAcls(BUCKET, OZONE, READ, REAL_VOLUME, REAL_BUCKET, null); + + assertThrows( + OMException.class, () -> omSpy.listParts( + REQUESTED_VOLUME, REQUESTED_BUCKET, KEY_NAME, UPLOAD_ID, 0, 10)); + + verify(keyManager, never()).listParts( + anyString(), anyString(), anyString(), anyString(), anyInt(), anyInt()); + verify(metrics, never()).incNumListMultipartUploadParts(); + verify(metrics).incNumListMultipartUploadPartFails(); + verify(omSpy).buildAuditMessageForFailure(any(), anyMap(), any(Throwable.class)); + verify(omMetadataReader, never()).checkAcls(KEY, OZONE, READ, REAL_VOLUME, REAL_BUCKET, KEY_NAME); + } + + @Test + void testKeyReadAclAccessDeniedSkipsKeyManagerAndRecordsFailure() throws Exception { + setupS3Request(); + when(omSpy.getAclsEnabled()).thenReturn(true); + + doNothing().when(omMetadataReader).checkAcls(BUCKET, OZONE, READ, REAL_VOLUME, REAL_BUCKET, null); + doThrow(new OMException("denied", OMException.ResultCodes.PERMISSION_DENIED)) + .when(omMetadataReader).checkAcls(KEY, OZONE, READ, REAL_VOLUME, REAL_BUCKET, KEY_NAME); + + assertThrows( + OMException.class, () -> omSpy.listParts( + REQUESTED_VOLUME, REQUESTED_BUCKET, KEY_NAME, UPLOAD_ID, 0, 10)); + + verify(keyManager, never()).listParts( + anyString(), anyString(), anyString(), anyString(), anyInt(), anyInt()); + verify(metrics, never()).incNumListMultipartUploadParts(); + verify(metrics).incNumListMultipartUploadPartFails(); + verify(omSpy).buildAuditMessageForFailure(any(), anyMap(), any(Throwable.class)); + } + + private void setupS3Request() { + OzoneManager.setS3Auth(S3Authentication.newBuilder().setAccessId("AKIAJWFJK62WUTKNFJJA").build()); + } +} diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/ha/TestOMHAMetrics.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/ha/TestOMHAMetrics.java index f00ff60c8ea0..b2a0f3aee90b 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/ha/TestOMHAMetrics.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/ha/TestOMHAMetrics.java @@ -55,4 +55,20 @@ public void testGetMetricsWithFollower() { omhaMetrics.getMetrics(METRICS_COLLECTOR, true); assertEquals(0, omhaMetrics.getOmhaInfoOzoneManagerHALeaderState()); } + + @Test + public void testGettersDoNotDependOnMetricsCollection() { + OMHAMetrics leaderMetrics = OMHAMetrics.create(NODE_ID, NODE_ID); + + assertEquals(NODE_ID, leaderMetrics.getOmhaInfoNodeId()); + assertEquals(1, leaderMetrics.getOmhaInfoOzoneManagerHALeaderState()); + + OMHAMetrics.unRegister(); + + String leaderId = "om" + RandomStringUtils.secure().nextNumeric(5); + OMHAMetrics followerMetrics = OMHAMetrics.create(NODE_ID, leaderId); + + assertEquals(NODE_ID, followerMetrics.getOmhaInfoNodeId()); + assertEquals(0, followerMetrics.getOmhaInfoOzoneManagerHALeaderState()); + } } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestKeyDeletingService.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestKeyDeletingService.java index dc1da6c3866c..f3f4468b7e63 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestKeyDeletingService.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestKeyDeletingService.java @@ -1081,7 +1081,6 @@ void testPurgeKeysRequestBatching() throws Exception { // The actual byte size of protobuf messages depends on content. // A small value like 1KB or 2KB should ensure batching for ~10-20 keys. final int actualRatisLimitBytes = 1138; - final int testRatisLimitBytes = 1024; // 2 KB to encourage multiple batches, 90% of the actualRatisLimitBytes. // Create a fresh configuration for this test to control the Ratis limit OzoneConfiguration testConf = new OzoneConfiguration(); @@ -1091,7 +1090,7 @@ void testPurgeKeysRequestBatching() throws Exception { testConf.setTimeDuration(OZONE_BLOCK_DELETING_SERVICE_INTERVAL, 100, TimeUnit.MILLISECONDS); // Set the specific Ratis limit for this test testConf.setStorageSize(OMConfigKeys.OZONE_OM_RATIS_LOG_APPENDER_QUEUE_BYTE_LIMIT, - testRatisLimitBytes, StorageUnit.BYTES); + actualRatisLimitBytes, StorageUnit.BYTES); testConf.setQuietMode(false); ScmBlockLocationTestingClient testScmBlockTestingClient = new ScmBlockLocationTestingClient(null, null, 0); diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/security/acl/TestOzoneNativeAuthorizer.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/security/acl/TestOzoneNativeAuthorizer.java index fbee5cdad08c..a7b2fa42a8dc 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/security/acl/TestOzoneNativeAuthorizer.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/security/acl/TestOzoneNativeAuthorizer.java @@ -335,7 +335,7 @@ private void resetAclsAndValidateAccess( String group = (!testUgi.getGroups().isEmpty()) ? testUgi.getGroups().get(0) : ""; - RequestContext.Builder builder = new RequestContext.Builder() + RequestContext.Builder builder = RequestContext.newBuilder() .setClientUgi(testUgi) .setAclType(accessType); diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/security/acl/TestParentAcl.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/security/acl/TestParentAcl.java index ed47b9578263..8b0556de4c62 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/security/acl/TestParentAcl.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/security/acl/TestParentAcl.java @@ -210,10 +210,11 @@ private void resetAcl(String vol, List volAcls, private void testParentChild(OzoneObj child, ACLType parentAclType, ACLType childAclType) throws IOException { - RequestContext requestContext = new RequestContext.Builder() + RequestContext requestContext = RequestContext.newBuilder() .setClientUgi(testUgi1) .setAclType(USER) - .setAclRights(childAclType).build(); + .setAclRights(childAclType) + .build(); OzoneAcl childAcl = OzoneAcl.of(USER, testUgi1.getUserName(), ACCESS, childAclType); diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/security/acl/TestRequestContext.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/security/acl/TestRequestContext.java index cb05c2ef6260..45272946dac0 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/security/acl/TestRequestContext.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/security/acl/TestRequestContext.java @@ -22,7 +22,6 @@ import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertTrue; -import java.io.IOException; import org.apache.hadoop.security.UserGroupInformation; import org.junit.jupiter.api.Test; @@ -32,103 +31,110 @@ public class TestRequestContext { @Test - public void testRecursiveAccessFlag() throws IOException { - RequestContext context = getUserRequestContext("om", - IAccessAuthorizer.ACLType.CREATE, false, "volume1", - true); - assertTrue(context.isRecursiveAccessCheck(), - "Wrongly sets recursiveAccessCheck flag value"); - - context = getUserRequestContext("om", - IAccessAuthorizer.ACLType.CREATE, false, "volume1", - false); - assertFalse(context.isRecursiveAccessCheck(), - "Wrongly sets recursiveAccessCheck flag value"); - - context = getUserRequestContext( - "user1", IAccessAuthorizer.ACLType.CREATE, - true, "volume1"); - assertFalse(context.isRecursiveAccessCheck(), - "Wrongly sets recursiveAccessCheck flag value"); - - RequestContext.Builder builder = new RequestContext.Builder(); - - assertFalse(builder.build().isRecursiveAccessCheck(), - "Wrongly sets recursive flag value"); + void testRecursiveAccessFlag() { + RequestContext.Builder builder = RequestContext.newBuilder(); + + assertFalse(builder.build().isRecursiveAccessCheck(), "default value"); builder.setRecursiveAccessCheck(true); - assertTrue(builder.build().isRecursiveAccessCheck(), - "Wrongly sets recursive flag value"); - - context = new RequestContext("host", null, - null, "serviceId", - IAccessAuthorizer.ACLIdentityType.GROUP, - IAccessAuthorizer.ACLType.CREATE, "owner"); - assertFalse(context.isRecursiveAccessCheck(), - "Wrongly sets recursive flag value"); - - context = new RequestContext("host", null, - null, "serviceId", - IAccessAuthorizer.ACLIdentityType.GROUP, - IAccessAuthorizer.ACLType.CREATE, "owner", false); - assertFalse(context.isRecursiveAccessCheck(), - "Wrongly sets recursive flag value"); - - context = new RequestContext("host", null, - null, "serviceId", - IAccessAuthorizer.ACLIdentityType.GROUP, - IAccessAuthorizer.ACLType.CREATE, "owner", true); - assertTrue(context.isRecursiveAccessCheck(), - "Wrongly sets recursive flag value"); + assertTrue(builder.build().isRecursiveAccessCheck()); + + builder.setRecursiveAccessCheck(false); + assertFalse(builder.build().isRecursiveAccessCheck()); } @Test - public void testSessionPolicy() { - final RequestContext.Builder builder = new RequestContext.Builder(); - RequestContext context = builder.build(); - assertNull(context.getSessionPolicy(), "sessionPolicy should default to null"); + void testSessionPolicy() { + RequestContext.Builder builder = RequestContext.newBuilder(); + assertNull(builder.build().getSessionPolicy(), "default value"); final String policy = "{\"Statement\":[]}"; - context = new RequestContext.Builder() - .setSessionPolicy(policy) - .build(); - assertEquals(policy, context.getSessionPolicy(), "sessionPolicy should be set via builder"); - - context = new RequestContext( - "host", null, null, "serviceId", IAccessAuthorizer.ACLIdentityType.GROUP, - IAccessAuthorizer.ACLType.CREATE, "owner", true, policy); - assertTrue(context.isRecursiveAccessCheck(), "recursiveAccessCheck should be true"); - assertEquals(policy, context.getSessionPolicy(), "sessionPolicy should be set via constructor"); - - context = RequestContext.getBuilder( - UserGroupInformation.createRemoteUser("user1"), null, null, - IAccessAuthorizer.ACLType.CREATE, "volume1", true) - .setSessionPolicy(policy) - .build(); - assertEquals(policy, context.getSessionPolicy(), "sessionPolicy should be set via getBuilder + builder"); + builder.setSessionPolicy(policy); + assertEquals(policy, builder.build().getSessionPolicy()); + } - context = RequestContext.getBuilder( - UserGroupInformation.createRemoteUser("user1"), null, null, - IAccessAuthorizer.ACLType.CREATE, "volume1", true, policy) + @Test + public void testToBuilderWithNoModifications() { + // Create a RequestContext with all fields set + final UserGroupInformation ugi = UserGroupInformation.createRemoteUser("testUser"); + final String host = "testHost"; + final String serviceId = "testServiceId"; + final String ownerName = "testOwner"; + final String sessionPolicy = "{\"Statement\":[{\"Effect\":\"Allow\"}]}"; + final String s3Action = "GetObject"; + + final RequestContext original = RequestContext.newBuilder() + .setHost(host) + .setClientUgi(ugi) + .setServiceId(serviceId) + .setAclType(IAccessAuthorizer.ACLIdentityType.USER) + .setAclRights(IAccessAuthorizer.ACLType.READ) + .setOwnerName(ownerName) + .setRecursiveAccessCheck(true) + .setSessionPolicy(sessionPolicy) + .setS3Action(s3Action) .build(); - assertEquals( - policy, context.getSessionPolicy(), - "sessionPolicy should be set via getBuilder (all params) + builder"); + + // Use toBuilder to create a new builder + final RequestContext.Builder builder = original.toBuilder(); + final RequestContext requestCtxFromToBuilder = builder.build(); + + // Verify all fields are preserved + assertEquals(original.getHost(), requestCtxFromToBuilder.getHost(), "Host should be preserved"); + assertNull(original.getIp(), "IP should be preserved"); + assertEquals(original.getClientUgi(), requestCtxFromToBuilder.getClientUgi(), "ClientUgi should be preserved"); + assertEquals(original.getServiceId(), requestCtxFromToBuilder.getServiceId(), "ServiceId should be preserved"); + assertEquals(original.getAclType(), requestCtxFromToBuilder.getAclType(), "AclType should be preserved"); + assertEquals(original.getAclRights(), requestCtxFromToBuilder.getAclRights(), "AclRights should be preserved"); + assertEquals(original.getOwnerName(), requestCtxFromToBuilder.getOwnerName(), "OwnerName should be preserved"); + assertTrue(original.isRecursiveAccessCheck(), "RecursiveAccessCheck should be preserved"); + assertEquals(original.getSessionPolicy(), requestCtxFromToBuilder.getSessionPolicy(), + "SessionPolicy should be preserved"); + assertEquals(original.getS3Action(), requestCtxFromToBuilder.getS3Action(), "S3 action should be preserved"); } - private RequestContext getUserRequestContext(String username, - IAccessAuthorizer.ACLType type, boolean isOwner, String ownerName, - boolean recursiveAccessCheck) throws IOException { + @Test + public void testToBuilderWithModifications() { + // Create an original RequestContext + final UserGroupInformation originalUgi = UserGroupInformation.createRemoteUser("user1"); + final RequestContext original = RequestContext.newBuilder() + .setHost("host1") + .setClientUgi(originalUgi) + .setServiceId("service1") + .setAclType(IAccessAuthorizer.ACLIdentityType.USER) + .setAclRights(IAccessAuthorizer.ACLType.READ) + .setOwnerName("owner1") + .setRecursiveAccessCheck(false) + .build(); - return RequestContext.getBuilder( - UserGroupInformation.createRemoteUser(username), null, null, - type, ownerName, recursiveAccessCheck).build(); - } + // Use toBuilder and modify some fields + final UserGroupInformation newUgi = UserGroupInformation.createRemoteUser("user2"); + final RequestContext modified = original.toBuilder() + .setHost("host2") + .setClientUgi(newUgi) + .setAclRights(IAccessAuthorizer.ACLType.WRITE) + .setOwnerName("owner2") + .setRecursiveAccessCheck(true) + .setSessionPolicy("{\"Statement\":[]}") + .setS3Action("DeleteObject") + .build(); - private RequestContext getUserRequestContext(String username, - IAccessAuthorizer.ACLType type, boolean isOwner, String ownerName) { - return RequestContext.getBuilder( - UserGroupInformation.createRemoteUser(username), null, null, - type, ownerName).build(); + // Verify original is unchanged + assertEquals("host1", original.getHost(), "Original should be unchanged"); + assertEquals(originalUgi, original.getClientUgi(), "Original UGI should be unchanged"); + assertEquals(IAccessAuthorizer.ACLType.READ, original.getAclRights(), "Original ACL rights should be unchanged"); + assertEquals("owner1", original.getOwnerName(), "Original owner name should be unchanged"); + assertFalse(original.isRecursiveAccessCheck(), "Original recursive flag should be unchanged"); + assertNull(original.getSessionPolicy(), "Original session policy should be unchanged"); + assertNull(original.getS3Action(), "Original S3 action should be unchanged"); + + // Verify modified has new values + assertEquals("host2", modified.getHost(), "Modified host should be updated"); + assertEquals(newUgi, modified.getClientUgi(), "Modified UGI should be updated"); + assertEquals(IAccessAuthorizer.ACLType.WRITE, modified.getAclRights(), "Modified ACL rights should be updated"); + assertEquals("owner2", modified.getOwnerName(), "Modified owner should be updated"); + assertTrue(modified.isRecursiveAccessCheck(), "Modified recursive flag should be updated"); + assertEquals("{\"Statement\":[]}", modified.getSessionPolicy(), "Modified session policy should be updated"); + assertEquals("DeleteObject", modified.getS3Action(), "Modified S3 action should be updated"); } } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/security/acl/TestVolumeOwner.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/security/acl/TestVolumeOwner.java index 43c0d0a39ac4..1fad71c058e7 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/security/acl/TestVolumeOwner.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/security/acl/TestVolumeOwner.java @@ -245,9 +245,12 @@ public void testKeyOps() throws Exception { private RequestContext getUserRequestContext(String username, IAccessAuthorizer.ACLType type, boolean isOwner, String ownerName) { - return RequestContext.getBuilder( - UserGroupInformation.createRemoteUser(username), null, null, - type, ownerName).build(); + return RequestContext.newBuilder() + .setClientUgi(UserGroupInformation.createRemoteUser(username)) + .setAclType(IAccessAuthorizer.ACLIdentityType.USER) + .setAclRights(type) + .setOwnerName(ownerName) + .build(); } private static String getTestVolumeName(int index) { diff --git a/hadoop-ozone/ozonefs-common/pom.xml b/hadoop-ozone/ozonefs-common/pom.xml index b55f5c6da6d8..44ed44f1170d 100644 --- a/hadoop-ozone/ozonefs-common/pom.xml +++ b/hadoop-ozone/ozonefs-common/pom.xml @@ -17,11 +17,11 @@ org.apache.ozone hdds-hadoop-dependency-client - 2.1.0.1-4.3.0-3 + 2.1.0.1-4.3.0-4 ../../hadoop-hdds/hadoop-dependency-client ozone-filesystem-common - 2.1.0.1-4.3.0-3 + 2.1.0.1-4.3.0-4 jar Apache Ozone FileSystem Common diff --git a/hadoop-ozone/ozonefs-hadoop2/pom.xml b/hadoop-ozone/ozonefs-hadoop2/pom.xml index c1a51655a103..6f336b918611 100644 --- a/hadoop-ozone/ozonefs-hadoop2/pom.xml +++ b/hadoop-ozone/ozonefs-hadoop2/pom.xml @@ -17,10 +17,10 @@ org.apache.ozone ozone - 2.1.0.1-4.3.0-3 + 2.1.0.1-4.3.0-4 ozone-filesystem-hadoop2 - 2.1.0.1-4.3.0-3 + 2.1.0.1-4.3.0-4 jar Apache Ozone FS Hadoop 2.x compatibility diff --git a/hadoop-ozone/ozonefs-hadoop3/pom.xml b/hadoop-ozone/ozonefs-hadoop3/pom.xml index cb0a111992de..9eeb643bdc71 100644 --- a/hadoop-ozone/ozonefs-hadoop3/pom.xml +++ b/hadoop-ozone/ozonefs-hadoop3/pom.xml @@ -17,10 +17,10 @@ org.apache.ozone ozone - 2.1.0.1-4.3.0-3 + 2.1.0.1-4.3.0-4 ozone-filesystem-hadoop3 - 2.1.0.1-4.3.0-3 + 2.1.0.1-4.3.0-4 jar Apache Ozone FS Hadoop 3.x compatibility diff --git a/hadoop-ozone/ozonefs-shaded/pom.xml b/hadoop-ozone/ozonefs-shaded/pom.xml index 59d87920db41..5c9db45fcc45 100644 --- a/hadoop-ozone/ozonefs-shaded/pom.xml +++ b/hadoop-ozone/ozonefs-shaded/pom.xml @@ -17,10 +17,10 @@ org.apache.ozone ozone - 2.1.0.1-4.3.0-3 + 2.1.0.1-4.3.0-4 ozone-filesystem-shaded - 2.1.0.1-4.3.0-3 + 2.1.0.1-4.3.0-4 jar Apache Ozone FileSystem Shaded @@ -180,10 +180,9 @@ - META-INF/BC1024KE.DSA - META-INF/BC2048KE.DSA - META-INF/BC1024KE.SF - META-INF/BC2048KE.SF + .DSA + .RSA + .SF diff --git a/hadoop-ozone/ozonefs/pom.xml b/hadoop-ozone/ozonefs/pom.xml index 9925db6fe52f..b29411b6637b 100644 --- a/hadoop-ozone/ozonefs/pom.xml +++ b/hadoop-ozone/ozonefs/pom.xml @@ -17,11 +17,11 @@ org.apache.ozone hdds-hadoop-dependency-client - 2.1.0.1-4.3.0-3 + 2.1.0.1-4.3.0-4 ../../hadoop-hdds/hadoop-dependency-client ozone-filesystem - 2.1.0.1-4.3.0-3 + 2.1.0.1-4.3.0-4 jar Apache Ozone FileSystem diff --git a/hadoop-ozone/pom.xml b/hadoop-ozone/pom.xml index af37724b4b32..d2b152a00dae 100644 --- a/hadoop-ozone/pom.xml +++ b/hadoop-ozone/pom.xml @@ -17,10 +17,10 @@ org.apache.ozone ozone-main - 2.1.0.1-4.3.0-3 + 2.1.0.1-4.3.0-4 ozone - 2.1.0.1-4.3.0-3 + 2.1.0.1-4.3.0-4 pom Apache Ozone Apache Ozone Project diff --git a/hadoop-ozone/recon-codegen/pom.xml b/hadoop-ozone/recon-codegen/pom.xml index 831a2358d6bd..2c82e65081f7 100644 --- a/hadoop-ozone/recon-codegen/pom.xml +++ b/hadoop-ozone/recon-codegen/pom.xml @@ -17,7 +17,7 @@ org.apache.ozone ozone - 2.1.0.1-4.3.0-3 + 2.1.0.1-4.3.0-4 ozone-reconcodegen Apache Ozone Recon CodeGen diff --git a/hadoop-ozone/recon/pom.xml b/hadoop-ozone/recon/pom.xml index a3c7f834863e..20b5b7688018 100644 --- a/hadoop-ozone/recon/pom.xml +++ b/hadoop-ozone/recon/pom.xml @@ -17,7 +17,7 @@ org.apache.ozone ozone - 2.1.0.1-4.3.0-3 + 2.1.0.1-4.3.0-4 ozone-recon Apache Ozone Recon diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/ContainerEndpoint.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/ContainerEndpoint.java index 49fab38fcf25..c1818b84fae5 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/ContainerEndpoint.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/ContainerEndpoint.java @@ -659,11 +659,11 @@ public Response getContainerMisMatchInsights( } } - List pipelines = new ArrayList<>(); nonOMContainers.forEach(containerInfo -> { ContainerDiscrepancyInfo containerDiscrepancyInfo = new ContainerDiscrepancyInfo(); containerDiscrepancyInfo.setContainerID(containerInfo.getContainerID()); containerDiscrepancyInfo.setNumberOfKeys(0); + List pipelines = new ArrayList<>(); PipelineID pipelineID = null; try { pipelineID = containerInfo.getPipelineID(); diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestContainerEndpoint.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestContainerEndpoint.java index 4794ecf1f309..a0843a7de847 100644 --- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestContainerEndpoint.java +++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestContainerEndpoint.java @@ -26,8 +26,10 @@ import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.writeKeyToOm; import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNotSame; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -1569,6 +1571,102 @@ public void testContainerMissingFilter() assertThat(missingContainerIdsSCM).contains(2L); } + /** + * Helper to create a container in SCM with a specific pipeline. + */ + private void createContainerInSCM(long containerId, Pipeline targetPipeline) + throws IOException, TimeoutException { + ContainerInfo containerInfo = new ContainerInfo.Builder() + .setContainerID(containerId) + .setReplicationConfig( + RatisReplicationConfig.getInstance(ReplicationFactor.THREE)) + .setState(HddsProtos.LifeCycleState.OPEN) + .setOwner("owner" + containerId) + .setNumberOfKeys(0) + .setPipelineID(targetPipeline.getId()) + .build(); + reconContainerManager.addNewContainer( + new ContainerWithPipeline(containerInfo, targetPipeline)); + } + + /** + * Helper to verify pipeline isolation for a container missing in OM. + */ + private void verifyPipelineIsolation(ContainerDiscrepancyInfo info, + long containerId, PipelineID expectedPipelineId, + List> otherPipelineLists) { + List pipelines = info.getPipelines(); + assertNotNull(pipelines); + assertEquals(1, pipelines.size(), + "Container " + containerId + " should have exactly 1 pipeline"); + assertEquals(expectedPipelineId, pipelines.get(0).getId(), + "Container " + containerId + " should have correct pipeline"); + assertEquals("SCM", info.getExistsAt()); + assertEquals(0, info.getNumberOfKeys()); + // Verify list isolation + for (List other : otherPipelineLists) { + if (other != null) { + assertNotSame(pipelines, other, + "Container " + containerId + " should have independent list"); + } + } + } + + @Test + public void testGetContainerInsightsNonOMContainersPipelineIsolation() + throws IOException, TimeoutException { + // Verifies fix for pipeline accumulation bug: containers missing in OM + // should each have their own isolated pipeline list, not shared. + + // Create 3 different pipelines + Pipeline[] pipelines = { + getRandomPipeline(), getRandomPipeline(), getRandomPipeline() + }; + for (Pipeline p : pipelines) { + reconPipelineManager.addPipeline(p); + } + + // Create 3 containers in SCM with different pipelines (not in OM) + long[] containerIds = {501L, 502L, 503L}; + for (int i = 0; i < containerIds.length; i++) { + createContainerInSCM(containerIds[i], pipelines[i]); + assertFalse(reconContainerMetadataManager.doesContainerExists(containerIds[i]), + "Container " + containerIds[i] + " should NOT exist in OM"); + } + + // Call API + Response response = containerEndpoint.getContainerMisMatchInsights(10, 500, "OM"); + assertEquals(Response.Status.OK.getStatusCode(), response.getStatus()); + + Map responseMap = (Map) response.getEntity(); + List discrepancies = + (List) responseMap.get("containerDiscrepancyInfo"); + assertNotNull(discrepancies); + + // Find containers in response + ContainerDiscrepancyInfo[] infos = new ContainerDiscrepancyInfo[3]; + for (int i = 0; i < containerIds.length; i++) { + final long id = containerIds[i]; + infos[i] = discrepancies.stream() + .filter(d -> d.getContainerID() == id) + .findFirst() + .orElseThrow(() -> new AssertionError( + "Container " + id + " not found in mismatch list")); + } + + // Verify pipeline isolation for each container + List pipelineList1 = infos[0].getPipelines(); + List pipelineList2 = infos[1].getPipelines(); + List pipelineList3 = infos[2].getPipelines(); + + verifyPipelineIsolation(infos[0], containerIds[0], pipelines[0].getId(), + Arrays.asList(pipelineList2, pipelineList3)); + verifyPipelineIsolation(infos[1], containerIds[1], pipelines[1].getId(), + Arrays.asList(pipelineList1, pipelineList3)); + verifyPipelineIsolation(infos[2], containerIds[2], pipelines[2].getId(), + Arrays.asList(pipelineList1, pipelineList2)); + } + @Test public void testGetOmContainersDeletedInSCM() throws Exception { Map omContainers = diff --git a/hadoop-ozone/s3-secret-store/pom.xml b/hadoop-ozone/s3-secret-store/pom.xml index b9ef56cd91d0..fffbdc82bc3a 100644 --- a/hadoop-ozone/s3-secret-store/pom.xml +++ b/hadoop-ozone/s3-secret-store/pom.xml @@ -17,10 +17,10 @@ org.apache.ozone ozone - 2.1.0.1-4.3.0-3 + 2.1.0.1-4.3.0-4 ozone-s3-secret-store - 2.1.0.1-4.3.0-3 + 2.1.0.1-4.3.0-4 jar Apache Ozone S3 Secret Store diff --git a/hadoop-ozone/s3gateway/pom.xml b/hadoop-ozone/s3gateway/pom.xml index 14664961637b..915284613f0e 100644 --- a/hadoop-ozone/s3gateway/pom.xml +++ b/hadoop-ozone/s3gateway/pom.xml @@ -17,10 +17,10 @@ org.apache.ozone ozone - 2.1.0.1-4.3.0-3 + 2.1.0.1-4.3.0-4 ozone-s3gateway - 2.1.0.1-4.3.0-3 + 2.1.0.1-4.3.0-4 jar Apache Ozone S3 Gateway @@ -41,6 +41,13 @@ com.fasterxml.jackson.module jackson-module-jaxb-annotations + + + + javax.xml.bind + jaxb-api + + com.github.ben-manes.caffeine diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/Gateway.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/Gateway.java index 9126d8ed4a2b..e1951126aaad 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/Gateway.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/Gateway.java @@ -33,6 +33,7 @@ import org.apache.hadoop.hdds.server.http.BaseHttpServer; import org.apache.hadoop.hdds.tracing.TracingUtil; import org.apache.hadoop.hdds.utils.HddsServerUtil; +import org.apache.hadoop.hdds.utils.IOUtils; import org.apache.hadoop.ozone.OzoneSecurityUtil; import org.apache.hadoop.ozone.s3.metrics.S3GatewayMetrics; import org.apache.hadoop.ozone.util.OzoneNetUtils; @@ -108,8 +109,7 @@ public void start() throws IOException { public void stop() throws Exception { LOG.info("Stopping Ozone S3 gateway"); - httpServer.stop(); - contentServer.stop(); + IOUtils.closeQuietly(httpServer, contentServer); jvmPauseMonitor.stop(); S3GatewayMetrics.unRegister(); } diff --git a/hadoop-ozone/tools/pom.xml b/hadoop-ozone/tools/pom.xml index 04bb0551f05c..bad65d137114 100644 --- a/hadoop-ozone/tools/pom.xml +++ b/hadoop-ozone/tools/pom.xml @@ -17,10 +17,10 @@ org.apache.ozone ozone - 2.1.0.1-4.3.0-3 + 2.1.0.1-4.3.0-4 ozone-tools - 2.1.0.1-4.3.0-3 + 2.1.0.1-4.3.0-4 jar Apache Ozone Tools Apache Ozone Tools diff --git a/pom.xml b/pom.xml index 9defd5b13f5c..407dc90c95c1 100644 --- a/pom.xml +++ b/pom.xml @@ -16,7 +16,7 @@ 4.0.0 org.apache.ozone ozone-main - 2.1.0.1-4.3.0-3 + 2.1.0.1-4.3.0-4 pom Apache Ozone Main Apache Ozone Main @@ -41,7 +41,7 @@ 1.12.788 2.35.5 0.8.0.RELEASE - 1.81 + 1.84 3.6.1 2.9.3 2.0 @@ -66,7 +66,7 @@ 1.10.0 0.23.0 1.0.1 - 4.2.0 + 5.9.0.1-4.3.0-0 2.9.1 @@ -97,6 +97,9 @@ 33.5.0-jre 6.0.0 3.4.2 + + 3.4.3 3.4.3 3.4.3.1-4.3.0-1 @@ -107,8 +110,8 @@ 2.6.1 4.5.14 4.4.16 - 1.71.0 - 2.16.2 + 1.77.1 + 2.18.6 0.8.13 2.1.1 2.6.1 @@ -141,7 +144,7 @@ 1.0.1 1.9.25 2.7.0 - 2.25.0 + 2.25.3 1.0-beta-1 3.1.0 3.7.1 @@ -159,7 +162,7 @@ 3.3.0 1.5.3 3.3.0 - 3.6.0 + 3.6.1 3.21.0 3.3.1 -Xmx8192m -XX:+HeapDumpOnOutOfMemoryError @@ -176,7 +179,7 @@ - 4.1.119.Final + 4.1.133.Final 16.14.2 4.12.0 @@ -185,7 +188,7 @@ Joshua Tree org_apache_ozone_shaded org.apache.ozone.shaded - 2.1.0.1-4.3.0-3 + 2.1.0.1-4.3.0-4 4.7.7 4.2.2 3.26.0 @@ -198,8 +201,8 @@ 1.2.1 1.0.7 0.6.1 - 3.25.8 - 2.6.0.4-2.2.0-1 + 3.25.9 + 2.6.0.7-2.2.0-2 1.71.0 4.1.119.Final @@ -237,8 +240,8 @@ 5.1.0 3.1.9.Final 5.4.0 - 3.8.4 - 1.5.7-5 + 3.9.5.1-4.3.0-0 + 1.5.7-7