From d9095bd0ec4e6bc8539a5a86fb68edbd7e873564 Mon Sep 17 00:00:00 2001 From: CSR2017 Date: Fri, 21 Jul 2023 15:06:32 -0500 Subject: [PATCH 01/18] Automate docker build --- .github/workflows/on-prerelease.yml | 71 +++++++++++++++++++++++++++-- Docker/api-pgsql.Dockerfile | 6 ++- 2 files changed, 71 insertions(+), 6 deletions(-) diff --git a/.github/workflows/on-prerelease.yml b/.github/workflows/on-prerelease.yml index 0855a14d6..c9ec6c4c3 100644 --- a/.github/workflows/on-prerelease.yml +++ b/.github/workflows/on-prerelease.yml @@ -16,6 +16,11 @@ env: VSS_NUGET_EXTERNAL_FEED_ENDPOINTS: '{"endpointCredentials": [{"endpoint": "${{ secrets.AZURE_ARTIFACTS_FEED_URL }}","password": "${{ secrets.AZURE_ARTIFACTS_PERSONAL_ACCESS_TOKEN }}"}]}' MANIFEST_FILE: "_manifest/spdx_2.2/manifest.spdx.json" PACKAGE_NAME: "AdminApi" + IMAGE_NAME: ${{ vars.IMAGE_NAME }} + DOCKER_USERNAME: ${{ vars.DOCKER_USERNAME }} + DOCKER_HUB_TOKEN: ${{ secrets.DOCKER_HUB_TOKEN }} + REF: ${{ github.ref_name }} + jobs: pack: name: Build and Pack @@ -235,7 +240,65 @@ jobs: NuGetApiKey = "${{ env.ARTIFACTS_API_KEY }}" } - $artifact | ForEach-Object { - $arguments.PackageFile = $_ - ./build.ps1 Push @arguments - } + # $artifact | ForEach-Object { + # $arguments.PackageFile = $_ + # ./build.ps1 Push @arguments + # } + docker-publish: + name: Publish to Docker Hub + runs-on: ubuntu-latest + needs: + - publish-package + steps: + - name: Wait 20s + # Give Azure Artifacts caching a moment to catch up + run: sleep 20 + + - name: Prepare Tags + id: prepare-tags + run: | + BASE="${{ env.IMAGE_NAME }}" + + REF="${{ env.REF }}" + + if [[ $REF =~ "pre" ]] + then + # Pre-releases get the version and the tag "pre" + TAGS="${{ env.IMAGE_NAME }}:${REF},${{ env.IMAGE_NAME }}:pre" + else + # Releases get the version, plus shortened form for minor release. + # We are not using shortened form for major or using "latest" + # because they are too imprecise. + MINOR=`echo ${REF} | awk -F"." '{print $1"."$2}'` + TAGS="${{ env.IMAGE_NAME }}:${REF},${{ env.IMAGE_NAME }}:${MINOR}" + fi + + echo "TAGS=$TAGS" >> $GITHUB_OUTPUT + echo "VERSION=0.0.0-alpha.0.65" >> $GITHUB_OUTPUT + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@4b4e9c3e2d4531116a6f8ba8e71fc6e2cb6e6c8c # v2.5.0 + + - name: Log in to Docker Hub + uses: docker/login-action@f4ef78c080cd8ba55a85445d5b36e214a81df20a # v2.1.0 + with: + username: ${{ env.DOCKER_USERNAME }} + password: ${{ env.DOCKER_HUB_TOKEN }} + + - name: Extract metadata (tags, labels) for Docker + id: meta + uses: docker/metadata-action@507c2f2dc502c992ad446e3d7a5dfbe311567a96 # v4.3.0 + with: + images: ${{ env.IMAGE_NAME }} + + - name: Build and push + uses: docker/build-push-action@3b5e8027fcad23fda98b2e3ac259d8d67585f671 # v4.0.0 + with: + context: "{{defaultContext}}:Docker" + cache-from: type=registry,ref=${{ env.IMAGE_NAME }}:pre + cache-to: type=inline + build-args: ML_VERSION=${{ steps.prepare-tags.outputs.VERSION }} + file: api-pgsql.Dockerfile + tags: ${{ steps.prepare-tags.outputs.TAGS }} + labels: ${{ steps.meta.outputs.labels }} + push: true \ No newline at end of file diff --git a/Docker/api-pgsql.Dockerfile b/Docker/api-pgsql.Dockerfile index 9123d105d..2509d65d3 100644 --- a/Docker/api-pgsql.Dockerfile +++ b/Docker/api-pgsql.Dockerfile @@ -6,7 +6,9 @@ #tag 6.0-alpine FROM mcr.microsoft.com/dotnet/aspnet@sha256:201cedd60cb295b2ebea7184561a45c5c0ee337e37300ea0f25cff5a2c762538 LABEL maintainer="Ed-Fi Alliance, LLC and Contributors " -ENV VERSION="0.0.0-alpha.0.65" +#ENV VERSION="0.0.0-alpha.0.65" + +ARG ML_VERSION=latest # Alpine image does not contain Globalization Cultures library so we need to install ICU library to get for LINQ expression to work # Disable the globaliztion invariant mode (set in base image) @@ -19,7 +21,7 @@ COPY Settings/pgsql/run.sh /app/run.sh COPY Settings/pgsql/log4net.config /app/log4net.txt RUN apk --no-cache add curl=~8 unzip=~6 dos2unix=~7 bash=~5 gettext=~0 postgresql13-client=~13 jq=~1 icu=~72 gcompat && \ - wget -O /app/AdminApi.zip https://pkgs.dev.azure.com/ed-fi-alliance/Ed-Fi-Alliance-OSS/_apis/packaging/feeds/EdFi/nuget/packages/EdFi.Suite3.ODS.AdminApi/versions/${VERSION}/content && \ + wget -O /app/AdminApi.zip https://pkgs.dev.azure.com/ed-fi-alliance/Ed-Fi-Alliance-OSS/_apis/packaging/feeds/EdFi/nuget/packages/EdFi.Suite3.ODS.AdminApi/versions/${ML_VERSION}/content && \ unzip /app/AdminApi.zip AdminApi/* -d /app/ && \ cp -r /app/AdminApi/. /app/ && \ rm -f /app/AdminApi.zip && \ From 2bd9f569bcf97450bc5faa8ff5d8a3efbb75e064 Mon Sep 17 00:00:00 2001 From: CSR2017 Date: Fri, 21 Jul 2023 15:39:19 -0500 Subject: [PATCH 02/18] Comment the pre-release tag creation temporarily --- .github/workflows/on-merge-or-tag.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/on-merge-or-tag.yml b/.github/workflows/on-merge-or-tag.yml index 4f239c78a..ef2d3d4a6 100644 --- a/.github/workflows/on-merge-or-tag.yml +++ b/.github/workflows/on-merge-or-tag.yml @@ -64,4 +64,4 @@ jobs: Authorization = "Bearer ${{ env.TOKEN }}" } - Invoke-RestMethod -Method POST -Uri ${{ env.API_URL }}/releases -Body $body -Headers $headers + #Invoke-RestMethod -Method POST -Uri ${{ env.API_URL }}/releases -Body $body -Headers $headers From d5aa69c82d1bb21661762d1b260f50bb58c5e035 Mon Sep 17 00:00:00 2001 From: CSR2017 Date: Fri, 21 Jul 2023 18:32:14 -0500 Subject: [PATCH 03/18] Add build step for admin api db image creation --- .github/workflows/on-prerelease.yml | 46 ++++++++++++++++++----- Docker/Settings/DB-Admin/pgsql/Dockerfile | 2 +- Docker/api-mssql.Dockerfile | 2 +- Docker/api-pgsql.Dockerfile | 5 +-- 4 files changed, 40 insertions(+), 15 deletions(-) diff --git a/.github/workflows/on-prerelease.yml b/.github/workflows/on-prerelease.yml index c9ec6c4c3..4f2b1d66c 100644 --- a/.github/workflows/on-prerelease.yml +++ b/.github/workflows/on-prerelease.yml @@ -17,6 +17,7 @@ env: MANIFEST_FILE: "_manifest/spdx_2.2/manifest.spdx.json" PACKAGE_NAME: "AdminApi" IMAGE_NAME: ${{ vars.IMAGE_NAME }} + DATABASE_IMAGE_NAME: $${{ vars.DATABASE_IMAGE_NAME }} DOCKER_USERNAME: ${{ vars.DOCKER_USERNAME }} DOCKER_HUB_TOKEN: ${{ secrets.DOCKER_HUB_TOKEN }} REF: ${{ github.ref_name }} @@ -259,22 +260,33 @@ jobs: run: | BASE="${{ env.IMAGE_NAME }}" - REF="${{ env.REF }}" + REF="${{ env.REF }}" - if [[ $REF =~ "pre" ]] + if [[ $REF =~ "Pre-Release" ]] then - # Pre-releases get the version and the tag "pre" - TAGS="${{ env.IMAGE_NAME }}:${REF},${{ env.IMAGE_NAME }}:pre" + # Remove Pre-Release prefix + PREFIX="Pre-Release-" + PACKAGEVERSION=${REF#"$PREFIX"} + echo "${PACKAGEVERSION}" + fi + + if [[ $PACKAGEVERSION =~ "alpha" ]] + then + # Pre-releases get the tag "pre" + TAGS="${{ env.IMAGE_NAME }}:pre" else # Releases get the version, plus shortened form for minor release. # We are not using shortened form for major or using "latest" # because they are too imprecise. - MINOR=`echo ${REF} | awk -F"." '{print $1"."$2}'` - TAGS="${{ env.IMAGE_NAME }}:${REF},${{ env.IMAGE_NAME }}:${MINOR}" + MINOR=`echo ${PACKAGEVERSION} | awk -F"." '{print $1"."$2}'` + TAGS="${{ env.IMAGE_NAME }}:${PACKAGEVERSION},${{ env.IMAGE_NAME }}:${MINOR}" fi + SEMVERSION=${PACKAGEVERSION:1} # strip off the leading 'v' echo "TAGS=$TAGS" >> $GITHUB_OUTPUT - echo "VERSION=0.0.0-alpha.0.65" >> $GITHUB_OUTPUT + # echo "VERSION=$SEMVERSION" >> $GITHUB_OUTPUT + echo "${SEMVERSION}" + echo "VERSION=0.0.0-alpha.0.67" >> $GITHUB_OUTPUT - name: Set up Docker Buildx uses: docker/setup-buildx-action@4b4e9c3e2d4531116a6f8ba8e71fc6e2cb6e6c8c # v2.5.0 @@ -289,16 +301,30 @@ jobs: id: meta uses: docker/metadata-action@507c2f2dc502c992ad446e3d7a5dfbe311567a96 # v4.3.0 with: - images: ${{ env.IMAGE_NAME }} + images: | + ${{ env.IMAGE_NAME }} + ${{ env.DATABASE_IMAGE_NAME }} - - name: Build and push + - name: Build and push admin api image uses: docker/build-push-action@3b5e8027fcad23fda98b2e3ac259d8d67585f671 # v4.0.0 with: context: "{{defaultContext}}:Docker" cache-from: type=registry,ref=${{ env.IMAGE_NAME }}:pre cache-to: type=inline - build-args: ML_VERSION=${{ steps.prepare-tags.outputs.VERSION }} + build-args: VERSION=${{ steps.prepare-tags.outputs.VERSION }} file: api-pgsql.Dockerfile tags: ${{ steps.prepare-tags.outputs.TAGS }} labels: ${{ steps.meta.outputs.labels }} + push: true + + - name: Build and push admin api database image + uses: docker/build-push-action@3b5e8027fcad23fda98b2e3ac259d8d67585f671 # v4.0.0 + with: + context: "{{defaultContext}}:Docker/Settings/DB-Admin/pgsql" + cache-from: type=registry,ref=${{ env.DATABASE_IMAGE_NAME }}:pre + cache-to: type=inline + build-args: VERSION=${{ steps.prepare-tags.outputs.VERSION }} + file: Dockerfile + tags: ${{ steps.prepare-tags.outputs.TAGS }} + labels: ${{ steps.meta.outputs.labels }} push: true \ No newline at end of file diff --git a/Docker/Settings/DB-Admin/pgsql/Dockerfile b/Docker/Settings/DB-Admin/pgsql/Dockerfile index 3da7c2bfb..82bbae9d4 100644 --- a/Docker/Settings/DB-Admin/pgsql/Dockerfile +++ b/Docker/Settings/DB-Admin/pgsql/Dockerfile @@ -10,7 +10,7 @@ ENV POSTGRES_USER=${POSTGRES_USER} ENV POSTGRES_PASSWORD=${POSTGRES_PASSWORD} ENV POSTGRES_DB=postgres -ENV VERSION="0.0.0-alpha.0.65" +ARG VERSION=latest COPY run-adminapi-migrations.sh /docker-entrypoint-initdb.d/3-run-adminapi-migrations.sh diff --git a/Docker/api-mssql.Dockerfile b/Docker/api-mssql.Dockerfile index 3d14ed5b3..270c72fbe 100644 --- a/Docker/api-mssql.Dockerfile +++ b/Docker/api-mssql.Dockerfile @@ -6,7 +6,7 @@ #tag 6.0-alpine FROM mcr.microsoft.com/dotnet/aspnet@sha256:201cedd60cb295b2ebea7184561a45c5c0ee337e37300ea0f25cff5a2c762538 LABEL maintainer="Ed-Fi Alliance, LLC and Contributors " -ENV VERSION="0.0.0-alpha.0.65" +ARG VERSION=latest # Alpine image does not contain Globalization Cultures library so we need to install ICU library to get for LINQ expression to work # Disable the globaliztion invariant mode (set in base image) diff --git a/Docker/api-pgsql.Dockerfile b/Docker/api-pgsql.Dockerfile index 2509d65d3..faad61ddd 100644 --- a/Docker/api-pgsql.Dockerfile +++ b/Docker/api-pgsql.Dockerfile @@ -6,9 +6,8 @@ #tag 6.0-alpine FROM mcr.microsoft.com/dotnet/aspnet@sha256:201cedd60cb295b2ebea7184561a45c5c0ee337e37300ea0f25cff5a2c762538 LABEL maintainer="Ed-Fi Alliance, LLC and Contributors " -#ENV VERSION="0.0.0-alpha.0.65" -ARG ML_VERSION=latest +ARG VERSION=latest # Alpine image does not contain Globalization Cultures library so we need to install ICU library to get for LINQ expression to work # Disable the globaliztion invariant mode (set in base image) @@ -21,7 +20,7 @@ COPY Settings/pgsql/run.sh /app/run.sh COPY Settings/pgsql/log4net.config /app/log4net.txt RUN apk --no-cache add curl=~8 unzip=~6 dos2unix=~7 bash=~5 gettext=~0 postgresql13-client=~13 jq=~1 icu=~72 gcompat && \ - wget -O /app/AdminApi.zip https://pkgs.dev.azure.com/ed-fi-alliance/Ed-Fi-Alliance-OSS/_apis/packaging/feeds/EdFi/nuget/packages/EdFi.Suite3.ODS.AdminApi/versions/${ML_VERSION}/content && \ + wget -O /app/AdminApi.zip https://pkgs.dev.azure.com/ed-fi-alliance/Ed-Fi-Alliance-OSS/_apis/packaging/feeds/EdFi/nuget/packages/EdFi.Suite3.ODS.AdminApi/versions/${VERSION}/content && \ unzip /app/AdminApi.zip AdminApi/* -d /app/ && \ cp -r /app/AdminApi/. /app/ && \ rm -f /app/AdminApi.zip && \ From 3694c3b698d8f9362e49185333d0483131273e98 Mon Sep 17 00:00:00 2001 From: CSR2017 Date: Fri, 21 Jul 2023 18:51:18 -0500 Subject: [PATCH 04/18] Remove db image build --- .github/workflows/on-prerelease.yml | 26 ++++++++++++-------------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/.github/workflows/on-prerelease.yml b/.github/workflows/on-prerelease.yml index 4f2b1d66c..28e3c2fb4 100644 --- a/.github/workflows/on-prerelease.yml +++ b/.github/workflows/on-prerelease.yml @@ -267,7 +267,6 @@ jobs: # Remove Pre-Release prefix PREFIX="Pre-Release-" PACKAGEVERSION=${REF#"$PREFIX"} - echo "${PACKAGEVERSION}" fi if [[ $PACKAGEVERSION =~ "alpha" ]] @@ -285,7 +284,6 @@ jobs: SEMVERSION=${PACKAGEVERSION:1} # strip off the leading 'v' echo "TAGS=$TAGS" >> $GITHUB_OUTPUT # echo "VERSION=$SEMVERSION" >> $GITHUB_OUTPUT - echo "${SEMVERSION}" echo "VERSION=0.0.0-alpha.0.67" >> $GITHUB_OUTPUT - name: Set up Docker Buildx @@ -303,7 +301,7 @@ jobs: with: images: | ${{ env.IMAGE_NAME }} - ${{ env.DATABASE_IMAGE_NAME }} + # ${{ env.DATABASE_IMAGE_NAME }} - name: Build and push admin api image uses: docker/build-push-action@3b5e8027fcad23fda98b2e3ac259d8d67585f671 # v4.0.0 @@ -317,14 +315,14 @@ jobs: labels: ${{ steps.meta.outputs.labels }} push: true - - name: Build and push admin api database image - uses: docker/build-push-action@3b5e8027fcad23fda98b2e3ac259d8d67585f671 # v4.0.0 - with: - context: "{{defaultContext}}:Docker/Settings/DB-Admin/pgsql" - cache-from: type=registry,ref=${{ env.DATABASE_IMAGE_NAME }}:pre - cache-to: type=inline - build-args: VERSION=${{ steps.prepare-tags.outputs.VERSION }} - file: Dockerfile - tags: ${{ steps.prepare-tags.outputs.TAGS }} - labels: ${{ steps.meta.outputs.labels }} - push: true \ No newline at end of file + # - name: Build and push admin api database image + # uses: docker/build-push-action@3b5e8027fcad23fda98b2e3ac259d8d67585f671 # v4.0.0 + # with: + # context: "{{defaultContext}}:Docker/Settings/DB-Admin/pgsql" + # cache-from: type=registry,ref=${{ env.DATABASE_IMAGE_NAME }}:pre + # cache-to: type=inline + # build-args: VERSION=${{ steps.prepare-tags.outputs.VERSION }} + # file: Dockerfile + # tags: ${{ steps.prepare-tags.outputs.TAGS }} + # labels: ${{ steps.meta.outputs.labels }} + # push: true \ No newline at end of file From 874f540005e361cd5456a7e621c7201a1bf82486 Mon Sep 17 00:00:00 2001 From: CSR2017 Date: Mon, 24 Jul 2023 10:17:41 -0500 Subject: [PATCH 05/18] try pushing multiple images --- .github/workflows/on-prerelease.yml | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/.github/workflows/on-prerelease.yml b/.github/workflows/on-prerelease.yml index 28e3c2fb4..6684661aa 100644 --- a/.github/workflows/on-prerelease.yml +++ b/.github/workflows/on-prerelease.yml @@ -248,6 +248,14 @@ jobs: docker-publish: name: Publish to Docker Hub runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: + - dockerfile: ./Docker/api-pgsql.Dockerfile + image: ${{ env.IMAGE_NAME }} + - dockerfile: ./Docker/Settings/DB-Admin/pgsql/Dockerfile + image: ${{ env.DATABASE_IMAGE_NAME }} needs: - publish-package steps: @@ -258,8 +266,6 @@ jobs: - name: Prepare Tags id: prepare-tags run: | - BASE="${{ env.IMAGE_NAME }}" - REF="${{ env.REF }}" if [[ $REF =~ "Pre-Release" ]] @@ -299,18 +305,19 @@ jobs: id: meta uses: docker/metadata-action@507c2f2dc502c992ad446e3d7a5dfbe311567a96 # v4.3.0 with: - images: | - ${{ env.IMAGE_NAME }} + images: ${{ matrix.image }} + # ${{ env.IMAGE_NAME }} # ${{ env.DATABASE_IMAGE_NAME }} - name: Build and push admin api image uses: docker/build-push-action@3b5e8027fcad23fda98b2e3ac259d8d67585f671 # v4.0.0 with: - context: "{{defaultContext}}:Docker" + context: "{{defaultContext}}" cache-from: type=registry,ref=${{ env.IMAGE_NAME }}:pre cache-to: type=inline build-args: VERSION=${{ steps.prepare-tags.outputs.VERSION }} - file: api-pgsql.Dockerfile + # file: api-pgsql.Dockerfile + file: ${{ matrix.dockerfile }} tags: ${{ steps.prepare-tags.outputs.TAGS }} labels: ${{ steps.meta.outputs.labels }} push: true From 618ca417bc721add01fd524eba965cbac662f2bb Mon Sep 17 00:00:00 2001 From: CSR2017 Date: Mon, 24 Jul 2023 11:18:00 -0500 Subject: [PATCH 06/18] Have duplication build steps for database image --- .github/workflows/on-prerelease.yml | 51 +++++++++++++---------------- 1 file changed, 23 insertions(+), 28 deletions(-) diff --git a/.github/workflows/on-prerelease.yml b/.github/workflows/on-prerelease.yml index 6684661aa..cefd035dc 100644 --- a/.github/workflows/on-prerelease.yml +++ b/.github/workflows/on-prerelease.yml @@ -248,14 +248,6 @@ jobs: docker-publish: name: Publish to Docker Hub runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - include: - - dockerfile: ./Docker/api-pgsql.Dockerfile - image: ${{ env.IMAGE_NAME }} - - dockerfile: ./Docker/Settings/DB-Admin/pgsql/Dockerfile - image: ${{ env.DATABASE_IMAGE_NAME }} needs: - publish-package steps: @@ -301,35 +293,38 @@ jobs: username: ${{ env.DOCKER_USERNAME }} password: ${{ env.DOCKER_HUB_TOKEN }} - - name: Extract metadata (tags, labels) for Docker - id: meta + - name: Extract metadata (tags, labels) for admin api image + id: metaapi uses: docker/metadata-action@507c2f2dc502c992ad446e3d7a5dfbe311567a96 # v4.3.0 with: - images: ${{ matrix.image }} - # ${{ env.IMAGE_NAME }} - # ${{ env.DATABASE_IMAGE_NAME }} + images: ${{ env.IMAGE_NAME }} - name: Build and push admin api image uses: docker/build-push-action@3b5e8027fcad23fda98b2e3ac259d8d67585f671 # v4.0.0 with: - context: "{{defaultContext}}" + context: "{{defaultContext}}:Docker" cache-from: type=registry,ref=${{ env.IMAGE_NAME }}:pre cache-to: type=inline build-args: VERSION=${{ steps.prepare-tags.outputs.VERSION }} - # file: api-pgsql.Dockerfile - file: ${{ matrix.dockerfile }} + file: api-pgsql.Dockerfile tags: ${{ steps.prepare-tags.outputs.TAGS }} - labels: ${{ steps.meta.outputs.labels }} + labels: ${{ steps.metaapi.outputs.labels }} push: true - # - name: Build and push admin api database image - # uses: docker/build-push-action@3b5e8027fcad23fda98b2e3ac259d8d67585f671 # v4.0.0 - # with: - # context: "{{defaultContext}}:Docker/Settings/DB-Admin/pgsql" - # cache-from: type=registry,ref=${{ env.DATABASE_IMAGE_NAME }}:pre - # cache-to: type=inline - # build-args: VERSION=${{ steps.prepare-tags.outputs.VERSION }} - # file: Dockerfile - # tags: ${{ steps.prepare-tags.outputs.TAGS }} - # labels: ${{ steps.meta.outputs.labels }} - # push: true \ No newline at end of file + - name: Extract metadata (tags, labels) for admin api database image + id: metadatabase + uses: docker/metadata-action@507c2f2dc502c992ad446e3d7a5dfbe311567a96 # v4.3.0 + with: + images: ${{ env.DATABASE_IMAGE_NAME }} + + - name: Build and push admin api database image + uses: docker/build-push-action@3b5e8027fcad23fda98b2e3ac259d8d67585f671 # v4.0.0 + with: + context: "{{defaultContext}}:Docker/Settings/DB-Admin/pgsql" + cache-from: type=registry,ref=${{ env.DATABASE_IMAGE_NAME }}:pre + cache-to: type=inline + build-args: VERSION=${{ steps.prepare-tags.outputs.VERSION }} + file: Dockerfile + tags: ${{ steps.prepare-tags.outputs.TAGS }} + labels: ${{ steps.metadatabase.outputs.labels }} + push: true \ No newline at end of file From 7ad47d1cbca23076ac80a1f94a4359dd501ec68a Mon Sep 17 00:00:00 2001 From: CSR2017 Date: Mon, 24 Jul 2023 12:25:14 -0500 Subject: [PATCH 07/18] Add seperate build steo abd tags for db --- .github/workflows/on-prerelease.yml | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/.github/workflows/on-prerelease.yml b/.github/workflows/on-prerelease.yml index cefd035dc..a9eb56893 100644 --- a/.github/workflows/on-prerelease.yml +++ b/.github/workflows/on-prerelease.yml @@ -270,17 +270,20 @@ jobs: if [[ $PACKAGEVERSION =~ "alpha" ]] then # Pre-releases get the tag "pre" - TAGS="${{ env.IMAGE_NAME }}:pre" + APITAGS="${{ env.IMAGE_NAME }}:pre" + DBTAGS="${{ env.DATABASE_IMAGE_NAME }}:pre" else # Releases get the version, plus shortened form for minor release. # We are not using shortened form for major or using "latest" # because they are too imprecise. MINOR=`echo ${PACKAGEVERSION} | awk -F"." '{print $1"."$2}'` - TAGS="${{ env.IMAGE_NAME }}:${PACKAGEVERSION},${{ env.IMAGE_NAME }}:${MINOR}" + APITAGS="${{ env.IMAGE_NAME }}:${PACKAGEVERSION},${{ env.IMAGE_NAME }}:${MINOR}" + DBTAGS="${{ env.DATABASE_IMAGE_NAME }}:${PACKAGEVERSION},${{ env.DATABASE_IMAGE_NAME }}:${MINOR}" fi SEMVERSION=${PACKAGEVERSION:1} # strip off the leading 'v' - echo "TAGS=$TAGS" >> $GITHUB_OUTPUT + echo "APITAGS=$APITAGS" >> $GITHUB_OUTPUT + echo "DBTAGS=$DBTAGS" >> $GITHUB_OUTPUT # echo "VERSION=$SEMVERSION" >> $GITHUB_OUTPUT echo "VERSION=0.0.0-alpha.0.67" >> $GITHUB_OUTPUT @@ -307,7 +310,7 @@ jobs: cache-to: type=inline build-args: VERSION=${{ steps.prepare-tags.outputs.VERSION }} file: api-pgsql.Dockerfile - tags: ${{ steps.prepare-tags.outputs.TAGS }} + tags: ${{ steps.prepare-tags.outputs.APITAGS }} labels: ${{ steps.metaapi.outputs.labels }} push: true @@ -325,6 +328,6 @@ jobs: cache-to: type=inline build-args: VERSION=${{ steps.prepare-tags.outputs.VERSION }} file: Dockerfile - tags: ${{ steps.prepare-tags.outputs.TAGS }} + tags: ${{ steps.prepare-tags.outputs.DBTAGS }} labels: ${{ steps.metadatabase.outputs.labels }} push: true \ No newline at end of file From 71402a8a0feb4e2452b131523a0b1656dfa82026 Mon Sep 17 00:00:00 2001 From: CSR2017 Date: Mon, 24 Jul 2023 15:20:29 -0500 Subject: [PATCH 08/18] Update env value for db --- .github/workflows/on-prerelease.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/on-prerelease.yml b/.github/workflows/on-prerelease.yml index a9eb56893..abf4b28ac 100644 --- a/.github/workflows/on-prerelease.yml +++ b/.github/workflows/on-prerelease.yml @@ -17,7 +17,7 @@ env: MANIFEST_FILE: "_manifest/spdx_2.2/manifest.spdx.json" PACKAGE_NAME: "AdminApi" IMAGE_NAME: ${{ vars.IMAGE_NAME }} - DATABASE_IMAGE_NAME: $${{ vars.DATABASE_IMAGE_NAME }} + DATABASE_IMAGE_NAME: ${{ vars.DATABASE_IMAGE_NAME }} DOCKER_USERNAME: ${{ vars.DOCKER_USERNAME }} DOCKER_HUB_TOKEN: ${{ secrets.DOCKER_HUB_TOKEN }} REF: ${{ github.ref_name }} From 9f48f0ee84f676d290cb5af936e7ab1f2e6fcb8c Mon Sep 17 00:00:00 2001 From: CSR2017 Date: Tue, 25 Jul 2023 14:54:56 -0500 Subject: [PATCH 09/18] Add cve analysis --- .github/workflows/on-prerelease.yml | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/.github/workflows/on-prerelease.yml b/.github/workflows/on-prerelease.yml index abf4b28ac..fae29d187 100644 --- a/.github/workflows/on-prerelease.yml +++ b/.github/workflows/on-prerelease.yml @@ -330,4 +330,19 @@ jobs: file: Dockerfile tags: ${{ steps.prepare-tags.outputs.DBTAGS }} labels: ${{ steps.metadatabase.outputs.labels }} - push: true \ No newline at end of file + push: true + + - name: Analyze for critical and high CVEs + id: docker-scout-cves + uses: docker/scout-action@v0.18 + with: + command: cves + image: ${{ steps.metaapi.outputs.tags }} + sarif-file: sarif.output.json + summary: true + + - name: Upload SARIF result + id: upload-sarif + uses: github/codeql-action/upload-sarif@v2 + with: + sarif_file: sarif.output.json \ No newline at end of file From 0b0ae0a7628bcb5ad94b61fbc3a371ae6e4faa45 Mon Sep 17 00:00:00 2001 From: CSR2017 Date: Tue, 25 Jul 2023 15:03:45 -0500 Subject: [PATCH 10/18] Update version --- .github/workflows/on-prerelease.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/on-prerelease.yml b/.github/workflows/on-prerelease.yml index fae29d187..c399b1829 100644 --- a/.github/workflows/on-prerelease.yml +++ b/.github/workflows/on-prerelease.yml @@ -334,7 +334,7 @@ jobs: - name: Analyze for critical and high CVEs id: docker-scout-cves - uses: docker/scout-action@v0.18 + uses: docker/scout-action@v0.20.0 with: command: cves image: ${{ steps.metaapi.outputs.tags }} From 97ff2b187499454a95cc750e0283aebf3bf12b3a Mon Sep 17 00:00:00 2001 From: CSR2017 Date: Tue, 25 Jul 2023 15:29:29 -0500 Subject: [PATCH 11/18] Set the image version for CVE --- .github/workflows/on-prerelease.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/on-prerelease.yml b/.github/workflows/on-prerelease.yml index c399b1829..a0b96ca3a 100644 --- a/.github/workflows/on-prerelease.yml +++ b/.github/workflows/on-prerelease.yml @@ -337,7 +337,7 @@ jobs: uses: docker/scout-action@v0.20.0 with: command: cves - image: ${{ steps.metaapi.outputs.tags }} + image: ${{ steps.prepare-tags.outputs.APITAGS }} sarif-file: sarif.output.json summary: true From 0cb7c8a29042fae921f263f74f592d3b4b135662 Mon Sep 17 00:00:00 2001 From: CSR2017 Date: Tue, 25 Jul 2023 15:57:01 -0500 Subject: [PATCH 12/18] enable permissions --- .github/workflows/on-prerelease.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/on-prerelease.yml b/.github/workflows/on-prerelease.yml index a0b96ca3a..5bdf83fda 100644 --- a/.github/workflows/on-prerelease.yml +++ b/.github/workflows/on-prerelease.yml @@ -250,6 +250,8 @@ jobs: runs-on: ubuntu-latest needs: - publish-package + permissions: + security-events: write steps: - name: Wait 20s # Give Azure Artifacts caching a moment to catch up From 5a8107d3c0fafbc18e45bf020585ae0420bdf1de Mon Sep 17 00:00:00 2001 From: CSR2017 Date: Tue, 25 Jul 2023 16:08:03 -0500 Subject: [PATCH 13/18] Delete upload process --- .github/workflows/on-prerelease.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/on-prerelease.yml b/.github/workflows/on-prerelease.yml index 5bdf83fda..699af32ab 100644 --- a/.github/workflows/on-prerelease.yml +++ b/.github/workflows/on-prerelease.yml @@ -343,8 +343,8 @@ jobs: sarif-file: sarif.output.json summary: true - - name: Upload SARIF result - id: upload-sarif - uses: github/codeql-action/upload-sarif@v2 - with: - sarif_file: sarif.output.json \ No newline at end of file + # - name: Upload SARIF result + # id: upload-sarif + # uses: github/codeql-action/upload-sarif@v2 + # with: + # sarif_file: sarif.output.json \ No newline at end of file From f4414adf22fadd7562b645559ff690394c0c807d Mon Sep 17 00:00:00 2001 From: CSR2017 Date: Tue, 25 Jul 2023 17:44:23 -0500 Subject: [PATCH 14/18] print api tags --- .github/workflows/on-prerelease.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/on-prerelease.yml b/.github/workflows/on-prerelease.yml index 699af32ab..9f31383f1 100644 --- a/.github/workflows/on-prerelease.yml +++ b/.github/workflows/on-prerelease.yml @@ -288,6 +288,7 @@ jobs: echo "DBTAGS=$DBTAGS" >> $GITHUB_OUTPUT # echo "VERSION=$SEMVERSION" >> $GITHUB_OUTPUT echo "VERSION=0.0.0-alpha.0.67" >> $GITHUB_OUTPUT + echo "${APITAGS}" - name: Set up Docker Buildx uses: docker/setup-buildx-action@4b4e9c3e2d4531116a6f8ba8e71fc6e2cb6e6c8c # v2.5.0 From b0827f8780276607d1e8f610f7853a42294e806e Mon Sep 17 00:00:00 2001 From: CSR2017 Date: Tue, 25 Jul 2023 17:56:55 -0500 Subject: [PATCH 15/18] Update package version --- .github/workflows/on-prerelease.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/on-prerelease.yml b/.github/workflows/on-prerelease.yml index 9f31383f1..e5c38ac96 100644 --- a/.github/workflows/on-prerelease.yml +++ b/.github/workflows/on-prerelease.yml @@ -267,6 +267,8 @@ jobs: # Remove Pre-Release prefix PREFIX="Pre-Release-" PACKAGEVERSION=${REF#"$PREFIX"} + else + PACKAGEVERSION=${REF} fi if [[ $PACKAGEVERSION =~ "alpha" ]] @@ -278,6 +280,7 @@ jobs: # Releases get the version, plus shortened form for minor release. # We are not using shortened form for major or using "latest" # because they are too imprecise. + echo "${PACKAGEVERSION}" MINOR=`echo ${PACKAGEVERSION} | awk -F"." '{print $1"."$2}'` APITAGS="${{ env.IMAGE_NAME }}:${PACKAGEVERSION},${{ env.IMAGE_NAME }}:${MINOR}" DBTAGS="${{ env.DATABASE_IMAGE_NAME }}:${PACKAGEVERSION},${{ env.DATABASE_IMAGE_NAME }}:${MINOR}" From a3a51ed5411d3041e645c715a132702932047f0a Mon Sep 17 00:00:00 2001 From: CSR2017 Date: Wed, 26 Jul 2023 16:20:28 -0500 Subject: [PATCH 16/18] Add docker file linting --- .../workflows/on-pullrequest-dockerfile.yml | 39 +++++++++++++++++++ Docker/dev.Dockerfile | 2 + 2 files changed, 41 insertions(+) create mode 100644 .github/workflows/on-pullrequest-dockerfile.yml diff --git a/.github/workflows/on-pullrequest-dockerfile.yml b/.github/workflows/on-pullrequest-dockerfile.yml new file mode 100644 index 000000000..fe375c139 --- /dev/null +++ b/.github/workflows/on-pullrequest-dockerfile.yml @@ -0,0 +1,39 @@ +name: On Pull Request - Dockerfile + +on: + push: + branches: + - main + paths: + - ".github/workflows/on-pullrequest-dockerfile.yml" + - "Docker/*" + pull_request: + branches: + - main + paths: + - ".github/workflows/on-pullrequest-dockerfile.yml" + - "Docker/*" + workflow_dispatch: + +jobs: + docker-testing: + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0 + + - uses: hadolint/hadolint-action@54c9adbab1582c2ef04b2016b760714a4bfde3cf # v3.1.0 + name: Run Linter on Dockerfiles + with: + dockerfile: "Docker/api-mssql.Dockerfile Docker/api-pgsql.Dockerfile Docker/dbadmin.Dockerfile Docker/dev.Dockerfile" + + # - uses: hadolint/hadolint-action@54c9adbab1582c2ef04b2016b760714a4bfde3cf # v3.1.0 + # name: Run Linter on NPM Dockerfile + # with: + # dockerfile: docker/Dockerfile + + # - name: Build the NPM Docker image + # # To confirm no errors occur. The "local" one will be built by the e2e + # # testing process. + # run: docker build --build-arg ML_VERSION=pre . + # working-directory: docker diff --git a/Docker/dev.Dockerfile b/Docker/dev.Dockerfile index 70a33731b..5366be20e 100644 --- a/Docker/dev.Dockerfile +++ b/Docker/dev.Dockerfile @@ -44,4 +44,6 @@ RUN apk --no-cache add curl=~8 dos2unix=~7 bash=~5 gettext=~0 icu=~72 gcompat && EXPOSE 443 WORKDIR /app +# dummy change for testing lint + ENTRYPOINT ["/app/run.sh"] From 143478c7aac9696fcc7136928f9c7ea2eeb463d1 Mon Sep 17 00:00:00 2001 From: CSR2017 Date: Wed, 26 Jul 2023 16:25:34 -0500 Subject: [PATCH 17/18] restrict to single file --- .github/workflows/on-pullrequest-dockerfile.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/on-pullrequest-dockerfile.yml b/.github/workflows/on-pullrequest-dockerfile.yml index fe375c139..88b8f9def 100644 --- a/.github/workflows/on-pullrequest-dockerfile.yml +++ b/.github/workflows/on-pullrequest-dockerfile.yml @@ -25,7 +25,8 @@ jobs: - uses: hadolint/hadolint-action@54c9adbab1582c2ef04b2016b760714a4bfde3cf # v3.1.0 name: Run Linter on Dockerfiles with: - dockerfile: "Docker/api-mssql.Dockerfile Docker/api-pgsql.Dockerfile Docker/dbadmin.Dockerfile Docker/dev.Dockerfile" + #dockerfile: "Docker/api-mssql.Dockerfile Docker/api-pgsql.Dockerfile Docker/dbadmin.Dockerfile Docker/dev.Dockerfile" + dockerfile: "Docker/api-pgsql.Dockerfile" # - uses: hadolint/hadolint-action@54c9adbab1582c2ef04b2016b760714a4bfde3cf # v3.1.0 # name: Run Linter on NPM Dockerfile From 4d102a3a24060a3a520cae605c11d57d5860c2a2 Mon Sep 17 00:00:00 2001 From: CSR2017 Date: Thu, 27 Jul 2023 17:28:55 -0500 Subject: [PATCH 18/18] Limit to errors --- .../workflows/on-pullrequest-dockerfile.yml | 29 ++++++++++++------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/.github/workflows/on-pullrequest-dockerfile.yml b/.github/workflows/on-pullrequest-dockerfile.yml index 88b8f9def..cde02ac6a 100644 --- a/.github/workflows/on-pullrequest-dockerfile.yml +++ b/.github/workflows/on-pullrequest-dockerfile.yml @@ -23,18 +23,25 @@ jobs: uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0 - uses: hadolint/hadolint-action@54c9adbab1582c2ef04b2016b760714a4bfde3cf # v3.1.0 - name: Run Linter on Dockerfiles + name: Run Linter on Postgres Dockerfile with: - #dockerfile: "Docker/api-mssql.Dockerfile Docker/api-pgsql.Dockerfile Docker/dbadmin.Dockerfile Docker/dev.Dockerfile" dockerfile: "Docker/api-pgsql.Dockerfile" + failure-threshold: error + + - uses: hadolint/hadolint-action@54c9adbab1582c2ef04b2016b760714a4bfde3cf # v3.1.0 + name: Run Linter on Sql Dockerfile + with: + dockerfile: "Docker/api-mssql.Dockerfile" + failure-threshold: error - # - uses: hadolint/hadolint-action@54c9adbab1582c2ef04b2016b760714a4bfde3cf # v3.1.0 - # name: Run Linter on NPM Dockerfile - # with: - # dockerfile: docker/Dockerfile + - uses: hadolint/hadolint-action@54c9adbab1582c2ef04b2016b760714a4bfde3cf # v3.1.0 + name: Run Linter on Database Dockerfile + with: + dockerfile: "Docker/dbadmin.Dockerfile" + failure-threshold: error - # - name: Build the NPM Docker image - # # To confirm no errors occur. The "local" one will be built by the e2e - # # testing process. - # run: docker build --build-arg ML_VERSION=pre . - # working-directory: docker + - uses: hadolint/hadolint-action@54c9adbab1582c2ef04b2016b760714a4bfde3cf # v3.1.0 + name: Run Linter on Development Dockerfile + with: + dockerfile: "Docker/dev.Dockerfile" + failure-threshold: error