From d2e3def1d126e1968073a9c809f1434a8ec237c7 Mon Sep 17 00:00:00 2001 From: rob Date: Thu, 26 Mar 2026 13:50:40 +0000 Subject: [PATCH 1/7] Allow building of custom docker images --- .../sync_service_dockerhub_image.yml | 49 ++++++++++++++++--- 1 file changed, 43 insertions(+), 6 deletions(-) diff --git a/.github/workflows/sync_service_dockerhub_image.yml b/.github/workflows/sync_service_dockerhub_image.yml index 1fdae46b40..85747ddcd6 100644 --- a/.github/workflows/sync_service_dockerhub_image.yml +++ b/.github/workflows/sync_service_dockerhub_image.yml @@ -19,7 +19,15 @@ on: inputs: release_tag: description: 'The @core/sync-service@... tag to run the workflow for (e.g. @core/sync-service@1.2.10)' - required: true + required: false + type: string + custom_ref: + description: 'Git ref to build from (branch, tag, or commit SHA). Use with custom_tag.' + required: false + type: string + custom_tag: + description: 'Docker Hub tag to publish as (e.g. subqueries-alpha). Use with custom_ref.' + required: false type: string env: @@ -33,6 +41,8 @@ jobs: outputs: git_ref: ${{ steps.git_ref.outputs.git_ref }} is_release: ${{ steps.git_ref.outputs.is_release }} + is_custom: ${{ steps.git_ref.outputs.is_custom }} + custom_tag: ${{ steps.git_ref.outputs.custom_tag }} short_commit_sha: ${{ steps.vars.outputs.short_commit_sha }} electric_version: ${{ steps.vars.outputs.electric_version }} @@ -41,22 +51,41 @@ jobs: id: git_ref env: INPUT_RELEASE_TAG: ${{ inputs.release_tag }} + INPUT_CUSTOM_REF: ${{ inputs.custom_ref }} + INPUT_CUSTOM_TAG: ${{ inputs.custom_tag }} EVENT_RELEASE_TAG: ${{ github.event.release.tag_name }} COMMIT_SHA: ${{ github.sha }} run: | - if [ -n "$INPUT_RELEASE_TAG" ]; then + if [ -n "$INPUT_CUSTOM_REF" ]; then + if [ -z "$INPUT_CUSTOM_TAG" ]; then + echo "::error::custom_ref requires custom_tag to be set" + exit 1 + fi + ref="$INPUT_CUSTOM_REF" + is_release=false + is_custom=true + custom_tag="$INPUT_CUSTOM_TAG" + elif [ -n "$INPUT_RELEASE_TAG" ]; then ref="refs/tags/$INPUT_RELEASE_TAG" is_release=true + is_custom=false + custom_tag="" elif [ -n "$EVENT_RELEASE_TAG" ]; then ref="refs/tags/$EVENT_RELEASE_TAG" is_release=true + is_custom=false + custom_tag="" else ref="$COMMIT_SHA" is_release=false + is_custom=false + custom_tag="" fi echo "git_ref=$ref" >> $GITHUB_OUTPUT echo "is_release=$is_release" >> $GITHUB_OUTPUT + echo "is_custom=$is_custom" >> $GITHUB_OUTPUT + echo "custom_tag=$custom_tag" >> $GITHUB_OUTPUT - uses: actions/checkout@v4 with: @@ -83,9 +112,13 @@ jobs: git rev-parse --short HEAD )" >> $GITHUB_OUTPUT - echo "electric_version=$( - git describe --abbrev=7 --tags --always --first-parent --match '@core/sync-service@*' | sed -En 's|^@core/sync-service@||p' - )" >> $GITHUB_OUTPUT + if [ "${{ steps.git_ref.outputs.is_custom }}" = "true" ]; then + echo "electric_version=${{ steps.git_ref.outputs.custom_tag }}" >> $GITHUB_OUTPUT + else + echo "electric_version=$( + git describe --abbrev=7 --tags --always --first-parent --match '@core/sync-service@*' | sed -En 's|^@core/sync-service@||p' + )" >> $GITHUB_OUTPUT + fi build_and_push_image: strategy: @@ -160,7 +193,11 @@ jobs: - name: Derive image tags from the GitHub Actions event run: | - if [ "${{ needs.derive_build_vars.outputs.is_release }}" = "true" ]; then + if [ "${{ needs.derive_build_vars.outputs.is_custom }}" = "true" ]; then + # A custom build publishes only the requested tag + echo "ELECTRIC_TAGS=-t $DOCKERHUB_REPO:${{ needs.derive_build_vars.outputs.custom_tag }}" >> $GITHUB_ENV + echo "ELECTRIC_CANARY_TAGS=" >> $GITHUB_ENV + elif [ "${{ needs.derive_build_vars.outputs.is_release }}" = "true" ]; then # A release triggers official release image publishing echo "ELECTRIC_TAGS=-t $DOCKERHUB_REPO:latest -t $DOCKERHUB_REPO:${{ needs.derive_build_vars.outputs.electric_version }}" >> $GITHUB_ENV echo "ELECTRIC_CANARY_TAGS=" >> $GITHUB_ENV From 523e9c6cb5d432cae079655dc6e7aaca4c775899 Mon Sep 17 00:00:00 2001 From: rob Date: Thu, 26 Mar 2026 13:51:07 +0000 Subject: [PATCH 2/7] Separate into two workflows --- .../_build_and_publish_electric_image.yml | 125 +++++++++++++ .../workflows/sync_service_custom_image.yml | 28 +++ .../sync_service_dockerhub_image.yml | 165 +++--------------- 3 files changed, 178 insertions(+), 140 deletions(-) create mode 100644 .github/workflows/_build_and_publish_electric_image.yml create mode 100644 .github/workflows/sync_service_custom_image.yml diff --git a/.github/workflows/_build_and_publish_electric_image.yml b/.github/workflows/_build_and_publish_electric_image.yml new file mode 100644 index 0000000000..2d155ee3cc --- /dev/null +++ b/.github/workflows/_build_and_publish_electric_image.yml @@ -0,0 +1,125 @@ +name: Build and publish Electric image + +# Reusable workflow that builds a multi-arch Docker image and publishes it to Docker Hub. +# Called by the release and custom-build workflows. +on: + workflow_call: + inputs: + git_ref: + description: 'Git ref to check out (branch, tag, or commit SHA)' + required: true + type: string + electric_version: + description: 'Version string baked into the binary via ELECTRIC_VERSION build arg' + required: true + type: string + electric_tags: + description: 'Docker tags for electricsql/electric (e.g. -t electricsql/electric:latest)' + required: true + type: string + canary_tags: + description: 'Docker tags for electricsql/electric-canary (empty to skip)' + required: false + default: '' + type: string + +env: + DOCKERHUB_REPO: electricsql/electric + DOCKERHUB_CANARY_REPO: electricsql/electric-canary + +jobs: + build_and_push_image: + strategy: + matrix: + include: + - platform: linux/amd64 + platform_id: amd64 + runner: blacksmith-4vcpu-ubuntu-2404 + - platform: linux/arm64/v8 + platform_id: arm64 + runner: blacksmith-4vcpu-ubuntu-2404-arm + runs-on: ${{ matrix.runner }} + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ inputs.git_ref }} + + - uses: useblacksmith/setup-docker-builder@v1 + + - uses: docker/login-action@v3 + with: + registry: docker.io + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Build and push by digest + id: build + uses: useblacksmith/build-push-action@v2 + with: + context: packages/sync-service + build-contexts: | + electric-telemetry=packages/electric-telemetry + build-args: | + ELECTRIC_VERSION=${{ inputs.electric_version }} + platforms: ${{ matrix.platform }} + push: true + # push an untagged image and export its digest + # the subsequent merge job will assemble the manifest list and apply tags + outputs: type=image,push-by-digest=true,name-canonical=true,push=true + tags: | + ${{ env.DOCKERHUB_REPO }} + ${{ env.DOCKERHUB_CANARY_REPO }} + + # Save the digest so the merge job can find both platform images + - name: Export digest + run: | + mkdir -p /tmp/digests + echo "${{ steps.build.outputs.digest }}" > "/tmp/digests/${{ matrix.platform_id }}.digest" + + - uses: actions/upload-artifact@v4 + with: + name: digests-${{ matrix.platform_id }} + path: /tmp/digests/* + + publish_tagged_image: + needs: [build_and_push_image] + runs-on: blacksmith-2vcpu-ubuntu-2404 + steps: + - uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - uses: useblacksmith/setup-docker-builder@v1 + + - uses: actions/download-artifact@v4 + with: + pattern: digests-* + merge-multiple: true + path: /tmp/digests + + - name: Create multi-arch manifest list + run: | + set -euo pipefail + + # Build a list of $DOCKERHUB_REPO@sha256:... source images + ELECTRIC_IMAGES=$( + for f in /tmp/digests/*.digest; do + echo $DOCKERHUB_REPO@$(cat $f) + done + ) + + # Create a manifest list for $DOCKERHUB_REPO that includes both platforms + docker buildx imagetools create ${{ inputs.electric_tags }} $ELECTRIC_IMAGES + + if [ -n "${{ inputs.canary_tags }}" ]; then + # Build a list of $DOCKERHUB_CANARY_REPO@sha256:... source images + ELECTRIC_CANARY_IMAGES=$( + for f in /tmp/digests/*.digest; do + echo $DOCKERHUB_CANARY_REPO@$(cat $f) + done + ) + + # Create a manifest list for $DOCKERHUB_CANARY_REPO that includes both platforms + docker buildx imagetools create ${{ inputs.canary_tags }} $ELECTRIC_CANARY_IMAGES + fi diff --git a/.github/workflows/sync_service_custom_image.yml b/.github/workflows/sync_service_custom_image.yml new file mode 100644 index 0000000000..60267ec21d --- /dev/null +++ b/.github/workflows/sync_service_custom_image.yml @@ -0,0 +1,28 @@ +name: Publish custom Electric image to Docker Hub + +# Build and publish a Docker image from any branch, tag, or commit SHA +# with a custom Docker Hub tag. Useful for testing pre-release builds. +on: + workflow_dispatch: + inputs: + custom_ref: + description: 'Git ref to build from (branch name, tag, or commit SHA)' + required: true + type: string + custom_tag: + description: 'Docker Hub tag to publish as (e.g. subqueries-alpha)' + required: true + type: string + +env: + DOCKERHUB_REPO: electricsql/electric + +jobs: + build_and_publish: + uses: ./.github/workflows/_build_and_publish_electric_image.yml + secrets: inherit + with: + git_ref: ${{ inputs.custom_ref }} + electric_version: ${{ inputs.custom_tag }} + electric_tags: -t electricsql/electric:${{ inputs.custom_tag }} + canary_tags: '' diff --git a/.github/workflows/sync_service_dockerhub_image.yml b/.github/workflows/sync_service_dockerhub_image.yml index 85747ddcd6..cccd8d4c69 100644 --- a/.github/workflows/sync_service_dockerhub_image.yml +++ b/.github/workflows/sync_service_dockerhub_image.yml @@ -19,15 +19,7 @@ on: inputs: release_tag: description: 'The @core/sync-service@... tag to run the workflow for (e.g. @core/sync-service@1.2.10)' - required: false - type: string - custom_ref: - description: 'Git ref to build from (branch, tag, or commit SHA). Use with custom_tag.' - required: false - type: string - custom_tag: - description: 'Docker Hub tag to publish as (e.g. subqueries-alpha). Use with custom_ref.' - required: false + required: true type: string env: @@ -41,8 +33,6 @@ jobs: outputs: git_ref: ${{ steps.git_ref.outputs.git_ref }} is_release: ${{ steps.git_ref.outputs.is_release }} - is_custom: ${{ steps.git_ref.outputs.is_custom }} - custom_tag: ${{ steps.git_ref.outputs.custom_tag }} short_commit_sha: ${{ steps.vars.outputs.short_commit_sha }} electric_version: ${{ steps.vars.outputs.electric_version }} @@ -51,41 +41,22 @@ jobs: id: git_ref env: INPUT_RELEASE_TAG: ${{ inputs.release_tag }} - INPUT_CUSTOM_REF: ${{ inputs.custom_ref }} - INPUT_CUSTOM_TAG: ${{ inputs.custom_tag }} EVENT_RELEASE_TAG: ${{ github.event.release.tag_name }} COMMIT_SHA: ${{ github.sha }} run: | - if [ -n "$INPUT_CUSTOM_REF" ]; then - if [ -z "$INPUT_CUSTOM_TAG" ]; then - echo "::error::custom_ref requires custom_tag to be set" - exit 1 - fi - ref="$INPUT_CUSTOM_REF" - is_release=false - is_custom=true - custom_tag="$INPUT_CUSTOM_TAG" - elif [ -n "$INPUT_RELEASE_TAG" ]; then + if [ -n "$INPUT_RELEASE_TAG" ]; then ref="refs/tags/$INPUT_RELEASE_TAG" is_release=true - is_custom=false - custom_tag="" elif [ -n "$EVENT_RELEASE_TAG" ]; then ref="refs/tags/$EVENT_RELEASE_TAG" is_release=true - is_custom=false - custom_tag="" else ref="$COMMIT_SHA" is_release=false - is_custom=false - custom_tag="" fi echo "git_ref=$ref" >> $GITHUB_OUTPUT echo "is_release=$is_release" >> $GITHUB_OUTPUT - echo "is_custom=$is_custom" >> $GITHUB_OUTPUT - echo "custom_tag=$custom_tag" >> $GITHUB_OUTPUT - uses: actions/checkout@v4 with: @@ -112,123 +83,37 @@ jobs: git rev-parse --short HEAD )" >> $GITHUB_OUTPUT - if [ "${{ steps.git_ref.outputs.is_custom }}" = "true" ]; then - echo "electric_version=${{ steps.git_ref.outputs.custom_tag }}" >> $GITHUB_OUTPUT - else - echo "electric_version=$( - git describe --abbrev=7 --tags --always --first-parent --match '@core/sync-service@*' | sed -En 's|^@core/sync-service@||p' - )" >> $GITHUB_OUTPUT - fi + echo "electric_version=$( + git describe --abbrev=7 --tags --always --first-parent --match '@core/sync-service@*' | sed -En 's|^@core/sync-service@||p' + )" >> $GITHUB_OUTPUT - build_and_push_image: - strategy: - matrix: - include: - - platform: linux/amd64 - platform_id: amd64 - runner: blacksmith-4vcpu-ubuntu-2404 - - platform: linux/arm64/v8 - platform_id: arm64 - runner: blacksmith-4vcpu-ubuntu-2404-arm - runs-on: ${{ matrix.runner }} + derive_tags: + name: Derive Docker image tags needs: [derive_build_vars] - steps: - - uses: actions/checkout@v4 - with: - ref: ${{ needs.derive_build_vars.outputs.git_ref }} - - - uses: useblacksmith/setup-docker-builder@v1 - - - uses: docker/login-action@v3 - with: - registry: docker.io - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - - name: Build and push by digest - id: build - uses: useblacksmith/build-push-action@v2 - with: - context: packages/sync-service - build-contexts: | - electric-telemetry=packages/electric-telemetry - build-args: | - ELECTRIC_VERSION=${{ needs.derive_build_vars.outputs.electric_version }} - platforms: ${{ matrix.platform }} - push: true - # push an untagged image and export its digest - # the subsequent merge job will assemble the manifest list and apply tags - outputs: type=image,push-by-digest=true,name-canonical=true,push=true - tags: | - ${{ env.DOCKERHUB_REPO }} - ${{ env.DOCKERHUB_CANARY_REPO }} - - # Save the digest so the merge job can find both platform images - - name: Export digest - run: | - mkdir -p /tmp/digests - echo "${{ steps.build.outputs.digest }}" > "/tmp/digests/${{ matrix.platform_id }}.digest" - - - uses: actions/upload-artifact@v4 - with: - name: digests-${{ matrix.platform_id }} - path: /tmp/digests/* - - publish_tagged_image: - needs: [derive_build_vars, build_and_push_image] runs-on: blacksmith-2vcpu-ubuntu-2404 + outputs: + electric_tags: ${{ steps.tags.outputs.electric_tags }} + canary_tags: ${{ steps.tags.outputs.canary_tags }} steps: - - uses: docker/login-action@v3 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - - uses: useblacksmith/setup-docker-builder@v1 - - - uses: actions/download-artifact@v4 - with: - pattern: digests-* - merge-multiple: true - path: /tmp/digests - - name: Derive image tags from the GitHub Actions event + id: tags run: | - if [ "${{ needs.derive_build_vars.outputs.is_custom }}" = "true" ]; then - # A custom build publishes only the requested tag - echo "ELECTRIC_TAGS=-t $DOCKERHUB_REPO:${{ needs.derive_build_vars.outputs.custom_tag }}" >> $GITHUB_ENV - echo "ELECTRIC_CANARY_TAGS=" >> $GITHUB_ENV - elif [ "${{ needs.derive_build_vars.outputs.is_release }}" = "true" ]; then + if [ "${{ needs.derive_build_vars.outputs.is_release }}" = "true" ]; then # A release triggers official release image publishing - echo "ELECTRIC_TAGS=-t $DOCKERHUB_REPO:latest -t $DOCKERHUB_REPO:${{ needs.derive_build_vars.outputs.electric_version }}" >> $GITHUB_ENV - echo "ELECTRIC_CANARY_TAGS=" >> $GITHUB_ENV + echo "electric_tags=-t $DOCKERHUB_REPO:latest -t $DOCKERHUB_REPO:${{ needs.derive_build_vars.outputs.electric_version }}" >> $GITHUB_OUTPUT + echo "canary_tags=" >> $GITHUB_OUTPUT else # A regular push to the main branch triggers canary image publishing - echo "ELECTRIC_TAGS=-t $DOCKERHUB_REPO:canary" >> $GITHUB_ENV - echo "ELECTRIC_CANARY_TAGS=-t $DOCKERHUB_CANARY_REPO:latest -t $DOCKERHUB_CANARY_REPO:${{ needs.derive_build_vars.outputs.short_commit_sha }}" >> $GITHUB_ENV + echo "electric_tags=-t $DOCKERHUB_REPO:canary" >> $GITHUB_OUTPUT + echo "canary_tags=-t $DOCKERHUB_CANARY_REPO:latest -t $DOCKERHUB_CANARY_REPO:${{ needs.derive_build_vars.outputs.short_commit_sha }}" >> $GITHUB_OUTPUT fi - - name: Create multi-arch manifest list - run: | - set -euo pipefail - - # Build a list of $DOCKERHUB_REPO@sha256:... source images - ELECTRIC_IMAGES=$( - for f in /tmp/digests/*.digest; do - echo $DOCKERHUB_REPO@$(cat $f) - done - ) - - # Create a manifest list for $DOCKERHUB_REPO:canary that includes both platforms - docker buildx imagetools create $ELECTRIC_TAGS $ELECTRIC_IMAGES - - if [ -n "$ELECTRIC_CANARY_TAGS" ]; then - # Build a list of $DOCKERHUB_CANARY_REPO@sha256:... source images - ELECTRIC_CANARY_IMAGES=$( - for f in /tmp/digests/*.digest; do - echo $DOCKERHUB_CANARY_REPO@$(cat $f) - done - ) - - # Create a manifest list for $DOCKERHUB_CANARY_REPO:... that includes both platforms - docker buildx imagetools create $ELECTRIC_CANARY_TAGS $ELECTRIC_CANARY_IMAGES - fi + build_and_publish: + needs: [derive_build_vars, derive_tags] + uses: ./.github/workflows/_build_and_publish_electric_image.yml + secrets: inherit + with: + git_ref: ${{ needs.derive_build_vars.outputs.git_ref }} + electric_version: ${{ needs.derive_build_vars.outputs.electric_version }} + electric_tags: ${{ needs.derive_tags.outputs.electric_tags }} + canary_tags: ${{ needs.derive_tags.outputs.canary_tags }} From 5cd829bb5c38e888df04123eb6ec5a3d96a78dac Mon Sep 17 00:00:00 2001 From: rob Date: Thu, 26 Mar 2026 14:25:02 +0000 Subject: [PATCH 3/7] Simplify shared workflow to build-only, keep tagging in callers The shared _build_electric_image.yml now only builds multi-arch images and exports digests. Each caller (release and custom) handles its own manifest creation and tagging logic. Co-Authored-By: Claude Opus 4.6 (1M context) --- .../_build_and_publish_electric_image.yml | 125 ------------------ .github/workflows/_build_electric_image.yml | 72 ++++++++++ .../workflows/sync_service_custom_image.yml | 37 +++++- .../sync_service_dockerhub_image.yml | 70 +++++++--- 4 files changed, 156 insertions(+), 148 deletions(-) delete mode 100644 .github/workflows/_build_and_publish_electric_image.yml create mode 100644 .github/workflows/_build_electric_image.yml diff --git a/.github/workflows/_build_and_publish_electric_image.yml b/.github/workflows/_build_and_publish_electric_image.yml deleted file mode 100644 index 2d155ee3cc..0000000000 --- a/.github/workflows/_build_and_publish_electric_image.yml +++ /dev/null @@ -1,125 +0,0 @@ -name: Build and publish Electric image - -# Reusable workflow that builds a multi-arch Docker image and publishes it to Docker Hub. -# Called by the release and custom-build workflows. -on: - workflow_call: - inputs: - git_ref: - description: 'Git ref to check out (branch, tag, or commit SHA)' - required: true - type: string - electric_version: - description: 'Version string baked into the binary via ELECTRIC_VERSION build arg' - required: true - type: string - electric_tags: - description: 'Docker tags for electricsql/electric (e.g. -t electricsql/electric:latest)' - required: true - type: string - canary_tags: - description: 'Docker tags for electricsql/electric-canary (empty to skip)' - required: false - default: '' - type: string - -env: - DOCKERHUB_REPO: electricsql/electric - DOCKERHUB_CANARY_REPO: electricsql/electric-canary - -jobs: - build_and_push_image: - strategy: - matrix: - include: - - platform: linux/amd64 - platform_id: amd64 - runner: blacksmith-4vcpu-ubuntu-2404 - - platform: linux/arm64/v8 - platform_id: arm64 - runner: blacksmith-4vcpu-ubuntu-2404-arm - runs-on: ${{ matrix.runner }} - steps: - - uses: actions/checkout@v4 - with: - ref: ${{ inputs.git_ref }} - - - uses: useblacksmith/setup-docker-builder@v1 - - - uses: docker/login-action@v3 - with: - registry: docker.io - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - - name: Build and push by digest - id: build - uses: useblacksmith/build-push-action@v2 - with: - context: packages/sync-service - build-contexts: | - electric-telemetry=packages/electric-telemetry - build-args: | - ELECTRIC_VERSION=${{ inputs.electric_version }} - platforms: ${{ matrix.platform }} - push: true - # push an untagged image and export its digest - # the subsequent merge job will assemble the manifest list and apply tags - outputs: type=image,push-by-digest=true,name-canonical=true,push=true - tags: | - ${{ env.DOCKERHUB_REPO }} - ${{ env.DOCKERHUB_CANARY_REPO }} - - # Save the digest so the merge job can find both platform images - - name: Export digest - run: | - mkdir -p /tmp/digests - echo "${{ steps.build.outputs.digest }}" > "/tmp/digests/${{ matrix.platform_id }}.digest" - - - uses: actions/upload-artifact@v4 - with: - name: digests-${{ matrix.platform_id }} - path: /tmp/digests/* - - publish_tagged_image: - needs: [build_and_push_image] - runs-on: blacksmith-2vcpu-ubuntu-2404 - steps: - - uses: docker/login-action@v3 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - - uses: useblacksmith/setup-docker-builder@v1 - - - uses: actions/download-artifact@v4 - with: - pattern: digests-* - merge-multiple: true - path: /tmp/digests - - - name: Create multi-arch manifest list - run: | - set -euo pipefail - - # Build a list of $DOCKERHUB_REPO@sha256:... source images - ELECTRIC_IMAGES=$( - for f in /tmp/digests/*.digest; do - echo $DOCKERHUB_REPO@$(cat $f) - done - ) - - # Create a manifest list for $DOCKERHUB_REPO that includes both platforms - docker buildx imagetools create ${{ inputs.electric_tags }} $ELECTRIC_IMAGES - - if [ -n "${{ inputs.canary_tags }}" ]; then - # Build a list of $DOCKERHUB_CANARY_REPO@sha256:... source images - ELECTRIC_CANARY_IMAGES=$( - for f in /tmp/digests/*.digest; do - echo $DOCKERHUB_CANARY_REPO@$(cat $f) - done - ) - - # Create a manifest list for $DOCKERHUB_CANARY_REPO that includes both platforms - docker buildx imagetools create ${{ inputs.canary_tags }} $ELECTRIC_CANARY_IMAGES - fi diff --git a/.github/workflows/_build_electric_image.yml b/.github/workflows/_build_electric_image.yml new file mode 100644 index 0000000000..a5ac42b594 --- /dev/null +++ b/.github/workflows/_build_electric_image.yml @@ -0,0 +1,72 @@ +name: Build Electric image + +# Reusable workflow that builds multi-arch Docker images and exports their digests. +# Callers are responsible for creating manifest lists and applying tags. +on: + workflow_call: + inputs: + git_ref: + description: 'Git ref to check out (branch, tag, or commit SHA)' + required: true + type: string + electric_version: + description: 'Version string baked into the binary via ELECTRIC_VERSION build arg' + required: true + type: string + +env: + DOCKERHUB_REPO: electricsql/electric + DOCKERHUB_CANARY_REPO: electricsql/electric-canary + +jobs: + build: + strategy: + matrix: + include: + - platform: linux/amd64 + platform_id: amd64 + runner: blacksmith-4vcpu-ubuntu-2404 + - platform: linux/arm64/v8 + platform_id: arm64 + runner: blacksmith-4vcpu-ubuntu-2404-arm + runs-on: ${{ matrix.runner }} + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ inputs.git_ref }} + + - uses: useblacksmith/setup-docker-builder@v1 + + - uses: docker/login-action@v3 + with: + registry: docker.io + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Build and push by digest + id: build + uses: useblacksmith/build-push-action@v2 + with: + context: packages/sync-service + build-contexts: | + electric-telemetry=packages/electric-telemetry + build-args: | + ELECTRIC_VERSION=${{ inputs.electric_version }} + platforms: ${{ matrix.platform }} + push: true + # push an untagged image and export its digest + # the caller will assemble the manifest list and apply tags + outputs: type=image,push-by-digest=true,name-canonical=true,push=true + tags: | + ${{ env.DOCKERHUB_REPO }} + ${{ env.DOCKERHUB_CANARY_REPO }} + + - name: Export digest + run: | + mkdir -p /tmp/digests + echo "${{ steps.build.outputs.digest }}" > "/tmp/digests/${{ matrix.platform_id }}.digest" + + - uses: actions/upload-artifact@v4 + with: + name: digests-${{ matrix.platform_id }} + path: /tmp/digests/* diff --git a/.github/workflows/sync_service_custom_image.yml b/.github/workflows/sync_service_custom_image.yml index 60267ec21d..7665725082 100644 --- a/.github/workflows/sync_service_custom_image.yml +++ b/.github/workflows/sync_service_custom_image.yml @@ -18,11 +18,40 @@ env: DOCKERHUB_REPO: electricsql/electric jobs: - build_and_publish: - uses: ./.github/workflows/_build_and_publish_electric_image.yml + build: + uses: ./.github/workflows/_build_electric_image.yml secrets: inherit with: git_ref: ${{ inputs.custom_ref }} electric_version: ${{ inputs.custom_tag }} - electric_tags: -t electricsql/electric:${{ inputs.custom_tag }} - canary_tags: '' + + publish: + needs: [build] + runs-on: blacksmith-2vcpu-ubuntu-2404 + steps: + - uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - uses: useblacksmith/setup-docker-builder@v1 + + - uses: actions/download-artifact@v4 + with: + pattern: digests-* + merge-multiple: true + path: /tmp/digests + + - name: Create multi-arch manifest list + run: | + set -euo pipefail + + ELECTRIC_IMAGES=$( + for f in /tmp/digests/*.digest; do + echo $DOCKERHUB_REPO@$(cat $f) + done + ) + + docker buildx imagetools create \ + -t $DOCKERHUB_REPO:${{ inputs.custom_tag }} \ + $ELECTRIC_IMAGES diff --git a/.github/workflows/sync_service_dockerhub_image.yml b/.github/workflows/sync_service_dockerhub_image.yml index cccd8d4c69..e806da6d30 100644 --- a/.github/workflows/sync_service_dockerhub_image.yml +++ b/.github/workflows/sync_service_dockerhub_image.yml @@ -87,33 +87,65 @@ jobs: git describe --abbrev=7 --tags --always --first-parent --match '@core/sync-service@*' | sed -En 's|^@core/sync-service@||p' )" >> $GITHUB_OUTPUT - derive_tags: - name: Derive Docker image tags + build: needs: [derive_build_vars] + uses: ./.github/workflows/_build_electric_image.yml + secrets: inherit + with: + git_ref: ${{ needs.derive_build_vars.outputs.git_ref }} + electric_version: ${{ needs.derive_build_vars.outputs.electric_version }} + + publish_tagged_image: + needs: [derive_build_vars, build] runs-on: blacksmith-2vcpu-ubuntu-2404 - outputs: - electric_tags: ${{ steps.tags.outputs.electric_tags }} - canary_tags: ${{ steps.tags.outputs.canary_tags }} steps: + - uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - uses: useblacksmith/setup-docker-builder@v1 + + - uses: actions/download-artifact@v4 + with: + pattern: digests-* + merge-multiple: true + path: /tmp/digests + - name: Derive image tags from the GitHub Actions event - id: tags run: | if [ "${{ needs.derive_build_vars.outputs.is_release }}" = "true" ]; then # A release triggers official release image publishing - echo "electric_tags=-t $DOCKERHUB_REPO:latest -t $DOCKERHUB_REPO:${{ needs.derive_build_vars.outputs.electric_version }}" >> $GITHUB_OUTPUT - echo "canary_tags=" >> $GITHUB_OUTPUT + echo "ELECTRIC_TAGS=-t $DOCKERHUB_REPO:latest -t $DOCKERHUB_REPO:${{ needs.derive_build_vars.outputs.electric_version }}" >> $GITHUB_ENV + echo "ELECTRIC_CANARY_TAGS=" >> $GITHUB_ENV else # A regular push to the main branch triggers canary image publishing - echo "electric_tags=-t $DOCKERHUB_REPO:canary" >> $GITHUB_OUTPUT - echo "canary_tags=-t $DOCKERHUB_CANARY_REPO:latest -t $DOCKERHUB_CANARY_REPO:${{ needs.derive_build_vars.outputs.short_commit_sha }}" >> $GITHUB_OUTPUT + echo "ELECTRIC_TAGS=-t $DOCKERHUB_REPO:canary" >> $GITHUB_ENV + echo "ELECTRIC_CANARY_TAGS=-t $DOCKERHUB_CANARY_REPO:latest -t $DOCKERHUB_CANARY_REPO:${{ needs.derive_build_vars.outputs.short_commit_sha }}" >> $GITHUB_ENV fi - build_and_publish: - needs: [derive_build_vars, derive_tags] - uses: ./.github/workflows/_build_and_publish_electric_image.yml - secrets: inherit - with: - git_ref: ${{ needs.derive_build_vars.outputs.git_ref }} - electric_version: ${{ needs.derive_build_vars.outputs.electric_version }} - electric_tags: ${{ needs.derive_tags.outputs.electric_tags }} - canary_tags: ${{ needs.derive_tags.outputs.canary_tags }} + - name: Create multi-arch manifest list + run: | + set -euo pipefail + + # Build a list of $DOCKERHUB_REPO@sha256:... source images + ELECTRIC_IMAGES=$( + for f in /tmp/digests/*.digest; do + echo $DOCKERHUB_REPO@$(cat $f) + done + ) + + # Create a manifest list for $DOCKERHUB_REPO:canary that includes both platforms + docker buildx imagetools create $ELECTRIC_TAGS $ELECTRIC_IMAGES + + if [ -n "$ELECTRIC_CANARY_TAGS" ]; then + # Build a list of $DOCKERHUB_CANARY_REPO@sha256:... source images + ELECTRIC_CANARY_IMAGES=$( + for f in /tmp/digests/*.digest; do + echo $DOCKERHUB_CANARY_REPO@$(cat $f) + done + ) + + # Create a manifest list for $DOCKERHUB_CANARY_REPO:... that includes both platforms + docker buildx imagetools create $ELECTRIC_CANARY_TAGS $ELECTRIC_CANARY_IMAGES + fi From 71370ae5e8397266f32b1382fde6e38961660db1 Mon Sep 17 00:00:00 2001 From: rob Date: Thu, 26 Mar 2026 14:40:45 +0000 Subject: [PATCH 4/7] Add standalone workflow for custom Docker image publishing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Drop the shared workflow abstraction — just add a new self-contained workflow alongside the unchanged release workflow. Simpler and no risk to the existing release pipeline. Co-Authored-By: Claude Opus 4.6 (1M context) --- .github/workflows/_build_electric_image.yml | 72 ------------------- .../workflows/sync_service_custom_image.yml | 52 ++++++++++++-- .../sync_service_dockerhub_image.yml | 60 ++++++++++++++-- 3 files changed, 100 insertions(+), 84 deletions(-) delete mode 100644 .github/workflows/_build_electric_image.yml diff --git a/.github/workflows/_build_electric_image.yml b/.github/workflows/_build_electric_image.yml deleted file mode 100644 index a5ac42b594..0000000000 --- a/.github/workflows/_build_electric_image.yml +++ /dev/null @@ -1,72 +0,0 @@ -name: Build Electric image - -# Reusable workflow that builds multi-arch Docker images and exports their digests. -# Callers are responsible for creating manifest lists and applying tags. -on: - workflow_call: - inputs: - git_ref: - description: 'Git ref to check out (branch, tag, or commit SHA)' - required: true - type: string - electric_version: - description: 'Version string baked into the binary via ELECTRIC_VERSION build arg' - required: true - type: string - -env: - DOCKERHUB_REPO: electricsql/electric - DOCKERHUB_CANARY_REPO: electricsql/electric-canary - -jobs: - build: - strategy: - matrix: - include: - - platform: linux/amd64 - platform_id: amd64 - runner: blacksmith-4vcpu-ubuntu-2404 - - platform: linux/arm64/v8 - platform_id: arm64 - runner: blacksmith-4vcpu-ubuntu-2404-arm - runs-on: ${{ matrix.runner }} - steps: - - uses: actions/checkout@v4 - with: - ref: ${{ inputs.git_ref }} - - - uses: useblacksmith/setup-docker-builder@v1 - - - uses: docker/login-action@v3 - with: - registry: docker.io - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - - name: Build and push by digest - id: build - uses: useblacksmith/build-push-action@v2 - with: - context: packages/sync-service - build-contexts: | - electric-telemetry=packages/electric-telemetry - build-args: | - ELECTRIC_VERSION=${{ inputs.electric_version }} - platforms: ${{ matrix.platform }} - push: true - # push an untagged image and export its digest - # the caller will assemble the manifest list and apply tags - outputs: type=image,push-by-digest=true,name-canonical=true,push=true - tags: | - ${{ env.DOCKERHUB_REPO }} - ${{ env.DOCKERHUB_CANARY_REPO }} - - - name: Export digest - run: | - mkdir -p /tmp/digests - echo "${{ steps.build.outputs.digest }}" > "/tmp/digests/${{ matrix.platform_id }}.digest" - - - uses: actions/upload-artifact@v4 - with: - name: digests-${{ matrix.platform_id }} - path: /tmp/digests/* diff --git a/.github/workflows/sync_service_custom_image.yml b/.github/workflows/sync_service_custom_image.yml index 7665725082..8149470866 100644 --- a/.github/workflows/sync_service_custom_image.yml +++ b/.github/workflows/sync_service_custom_image.yml @@ -19,11 +19,53 @@ env: jobs: build: - uses: ./.github/workflows/_build_electric_image.yml - secrets: inherit - with: - git_ref: ${{ inputs.custom_ref }} - electric_version: ${{ inputs.custom_tag }} + strategy: + matrix: + include: + - platform: linux/amd64 + platform_id: amd64 + runner: blacksmith-4vcpu-ubuntu-2404 + - platform: linux/arm64/v8 + platform_id: arm64 + runner: blacksmith-4vcpu-ubuntu-2404-arm + runs-on: ${{ matrix.runner }} + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ inputs.custom_ref }} + + - uses: useblacksmith/setup-docker-builder@v1 + + - uses: docker/login-action@v3 + with: + registry: docker.io + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Build and push by digest + id: build + uses: useblacksmith/build-push-action@v2 + with: + context: packages/sync-service + build-contexts: | + electric-telemetry=packages/electric-telemetry + build-args: | + ELECTRIC_VERSION=${{ inputs.custom_tag }} + platforms: ${{ matrix.platform }} + push: true + outputs: type=image,push-by-digest=true,name-canonical=true,push=true + tags: | + ${{ env.DOCKERHUB_REPO }} + + - name: Export digest + run: | + mkdir -p /tmp/digests + echo "${{ steps.build.outputs.digest }}" > "/tmp/digests/${{ matrix.platform_id }}.digest" + + - uses: actions/upload-artifact@v4 + with: + name: digests-${{ matrix.platform_id }} + path: /tmp/digests/* publish: needs: [build] diff --git a/.github/workflows/sync_service_dockerhub_image.yml b/.github/workflows/sync_service_dockerhub_image.yml index e806da6d30..1fdae46b40 100644 --- a/.github/workflows/sync_service_dockerhub_image.yml +++ b/.github/workflows/sync_service_dockerhub_image.yml @@ -87,16 +87,62 @@ jobs: git describe --abbrev=7 --tags --always --first-parent --match '@core/sync-service@*' | sed -En 's|^@core/sync-service@||p' )" >> $GITHUB_OUTPUT - build: + build_and_push_image: + strategy: + matrix: + include: + - platform: linux/amd64 + platform_id: amd64 + runner: blacksmith-4vcpu-ubuntu-2404 + - platform: linux/arm64/v8 + platform_id: arm64 + runner: blacksmith-4vcpu-ubuntu-2404-arm + runs-on: ${{ matrix.runner }} needs: [derive_build_vars] - uses: ./.github/workflows/_build_electric_image.yml - secrets: inherit - with: - git_ref: ${{ needs.derive_build_vars.outputs.git_ref }} - electric_version: ${{ needs.derive_build_vars.outputs.electric_version }} + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ needs.derive_build_vars.outputs.git_ref }} + + - uses: useblacksmith/setup-docker-builder@v1 + + - uses: docker/login-action@v3 + with: + registry: docker.io + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Build and push by digest + id: build + uses: useblacksmith/build-push-action@v2 + with: + context: packages/sync-service + build-contexts: | + electric-telemetry=packages/electric-telemetry + build-args: | + ELECTRIC_VERSION=${{ needs.derive_build_vars.outputs.electric_version }} + platforms: ${{ matrix.platform }} + push: true + # push an untagged image and export its digest + # the subsequent merge job will assemble the manifest list and apply tags + outputs: type=image,push-by-digest=true,name-canonical=true,push=true + tags: | + ${{ env.DOCKERHUB_REPO }} + ${{ env.DOCKERHUB_CANARY_REPO }} + + # Save the digest so the merge job can find both platform images + - name: Export digest + run: | + mkdir -p /tmp/digests + echo "${{ steps.build.outputs.digest }}" > "/tmp/digests/${{ matrix.platform_id }}.digest" + + - uses: actions/upload-artifact@v4 + with: + name: digests-${{ matrix.platform_id }} + path: /tmp/digests/* publish_tagged_image: - needs: [derive_build_vars, build] + needs: [derive_build_vars, build_and_push_image] runs-on: blacksmith-2vcpu-ubuntu-2404 steps: - uses: docker/login-action@v3 From 7b5f43e2d7570ebac77a1ae08049553f6300bb26 Mon Sep 17 00:00:00 2001 From: rob Date: Mon, 30 Mar 2026 18:15:17 +0100 Subject: [PATCH 5/7] Use a single workflow, but with two inputs --- .github/workflows/changesets_release.yml | 2 +- .../workflows/sync_service_custom_image.yml | 99 ----------------- .../sync_service_dockerhub_image.yml | 101 +++++++++++------- 3 files changed, 63 insertions(+), 139 deletions(-) delete mode 100644 .github/workflows/sync_service_custom_image.yml diff --git a/.github/workflows/changesets_release.yml b/.github/workflows/changesets_release.yml index 82345cb367..584cccf45a 100644 --- a/.github/workflows/changesets_release.yml +++ b/.github/workflows/changesets_release.yml @@ -71,7 +71,7 @@ jobs: uses: ./.github/workflows/sync_service_dockerhub_image.yml secrets: inherit with: - release_tag: ${{ needs.changesets.outputs.sync_service_release_tag }} + git_ref: ${{ needs.changesets.outputs.sync_service_release_tag }} update-cloud: name: Update Electric version used by Cloud diff --git a/.github/workflows/sync_service_custom_image.yml b/.github/workflows/sync_service_custom_image.yml deleted file mode 100644 index 8149470866..0000000000 --- a/.github/workflows/sync_service_custom_image.yml +++ /dev/null @@ -1,99 +0,0 @@ -name: Publish custom Electric image to Docker Hub - -# Build and publish a Docker image from any branch, tag, or commit SHA -# with a custom Docker Hub tag. Useful for testing pre-release builds. -on: - workflow_dispatch: - inputs: - custom_ref: - description: 'Git ref to build from (branch name, tag, or commit SHA)' - required: true - type: string - custom_tag: - description: 'Docker Hub tag to publish as (e.g. subqueries-alpha)' - required: true - type: string - -env: - DOCKERHUB_REPO: electricsql/electric - -jobs: - build: - strategy: - matrix: - include: - - platform: linux/amd64 - platform_id: amd64 - runner: blacksmith-4vcpu-ubuntu-2404 - - platform: linux/arm64/v8 - platform_id: arm64 - runner: blacksmith-4vcpu-ubuntu-2404-arm - runs-on: ${{ matrix.runner }} - steps: - - uses: actions/checkout@v4 - with: - ref: ${{ inputs.custom_ref }} - - - uses: useblacksmith/setup-docker-builder@v1 - - - uses: docker/login-action@v3 - with: - registry: docker.io - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - - name: Build and push by digest - id: build - uses: useblacksmith/build-push-action@v2 - with: - context: packages/sync-service - build-contexts: | - electric-telemetry=packages/electric-telemetry - build-args: | - ELECTRIC_VERSION=${{ inputs.custom_tag }} - platforms: ${{ matrix.platform }} - push: true - outputs: type=image,push-by-digest=true,name-canonical=true,push=true - tags: | - ${{ env.DOCKERHUB_REPO }} - - - name: Export digest - run: | - mkdir -p /tmp/digests - echo "${{ steps.build.outputs.digest }}" > "/tmp/digests/${{ matrix.platform_id }}.digest" - - - uses: actions/upload-artifact@v4 - with: - name: digests-${{ matrix.platform_id }} - path: /tmp/digests/* - - publish: - needs: [build] - runs-on: blacksmith-2vcpu-ubuntu-2404 - steps: - - uses: docker/login-action@v3 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - - uses: useblacksmith/setup-docker-builder@v1 - - - uses: actions/download-artifact@v4 - with: - pattern: digests-* - merge-multiple: true - path: /tmp/digests - - - name: Create multi-arch manifest list - run: | - set -euo pipefail - - ELECTRIC_IMAGES=$( - for f in /tmp/digests/*.digest; do - echo $DOCKERHUB_REPO@$(cat $f) - done - ) - - docker buildx imagetools create \ - -t $DOCKERHUB_REPO:${{ inputs.custom_tag }} \ - $ELECTRIC_IMAGES diff --git a/.github/workflows/sync_service_dockerhub_image.yml b/.github/workflows/sync_service_dockerhub_image.yml index 1fdae46b40..15e461bf5c 100644 --- a/.github/workflows/sync_service_dockerhub_image.yml +++ b/.github/workflows/sync_service_dockerhub_image.yml @@ -10,17 +10,25 @@ on: # Allows the workflow to be called by the Changesets workflow workflow_call: inputs: - release_tag: - description: 'The @core/sync-service@... tag passed from caller' + git_ref: + description: 'Git ref to build from (branch name, tag, or commit SHA)' required: true type: string + docker_tag: + description: 'Optional Docker Hub tag override (derived from git_ref if omitted)' + required: false + type: string # Allows the workflow to be triggered manually from the UI workflow_dispatch: inputs: - release_tag: - description: 'The @core/sync-service@... tag to run the workflow for (e.g. @core/sync-service@1.2.10)' + git_ref: + description: 'Git ref to build from (branch name, tag, or commit SHA)' required: true type: string + docker_tag: + description: 'Optional Docker Hub tag override (derived from git_ref if omitted)' + required: false + type: string env: DOCKERHUB_REPO: electricsql/electric @@ -32,60 +40,73 @@ jobs: runs-on: blacksmith-2vcpu-ubuntu-2404 outputs: git_ref: ${{ steps.git_ref.outputs.git_ref }} - is_release: ${{ steps.git_ref.outputs.is_release }} + build_mode: ${{ steps.git_ref.outputs.build_mode }} short_commit_sha: ${{ steps.vars.outputs.short_commit_sha }} electric_version: ${{ steps.vars.outputs.electric_version }} + docker_tag: ${{ steps.vars.outputs.docker_tag }} steps: - name: Determine the ref to check out id: git_ref env: - INPUT_RELEASE_TAG: ${{ inputs.release_tag }} + INPUT_GIT_REF: ${{ inputs.git_ref }} + INPUT_DOCKER_TAG: ${{ inputs.docker_tag }} EVENT_RELEASE_TAG: ${{ github.event.release.tag_name }} COMMIT_SHA: ${{ github.sha }} run: | - if [ -n "$INPUT_RELEASE_TAG" ]; then - ref="refs/tags/$INPUT_RELEASE_TAG" - is_release=true + if [ -n "$INPUT_DOCKER_TAG" ]; then + # Custom docker tag provided — this is a custom build + ref="${INPUT_GIT_REF}" + build_mode=custom + elif [ -n "$INPUT_GIT_REF" ]; then + # git_ref provided without docker_tag — treat as a release tag + ref="refs/tags/$INPUT_GIT_REF" + build_mode=release elif [ -n "$EVENT_RELEASE_TAG" ]; then ref="refs/tags/$EVENT_RELEASE_TAG" - is_release=true + build_mode=release else ref="$COMMIT_SHA" - is_release=false + build_mode=canary fi echo "git_ref=$ref" >> $GITHUB_OUTPUT - echo "is_release=$is_release" >> $GITHUB_OUTPUT + echo "build_mode=$build_mode" >> $GITHUB_OUTPUT - uses: actions/checkout@v4 with: - # The checked out commit influences the value of the ELECTRIC_VERSION variable - # that is baked into the Docker image. - # - # For regular pushes to main, we check out the HEAD commit and publish canary images. - # - # For releases we check out the tag corresponding to the release, e.g. - # @core/sync-service@v1.2.10. - # - # For manual triggers via workflow_dispatch, we check out the tag specified manually - # by the actor. ref: ${{ steps.git_ref.outputs.git_ref }} - # Also important to fetch the whole history since otherwise we won't get that tags - # that are required to determine the correct ELECTRIC_VERSION. fetch-depth: 0 fetch-tags: true - - name: Determine short_commit_sha and electric_version to use in the build step + - name: Determine short_commit_sha, electric_version, and docker_tag id: vars + env: + BUILD_MODE: ${{ steps.git_ref.outputs.build_mode }} + INPUT_DOCKER_TAG: ${{ inputs.docker_tag }} run: | echo "short_commit_sha=$( git rev-parse --short HEAD )" >> $GITHUB_OUTPUT - echo "electric_version=$( + electric_version=$( git describe --abbrev=7 --tags --always --first-parent --match '@core/sync-service@*' | sed -En 's|^@core/sync-service@||p' - )" >> $GITHUB_OUTPUT + ) + # Fall back to short SHA if no matching tag exists (e.g. custom builds from old branches) + if [ -z "$electric_version" ]; then + electric_version=$(git rev-parse --short HEAD) + fi + echo "electric_version=$electric_version" >> $GITHUB_OUTPUT + + # Determine the docker tag to use + if [ -n "$INPUT_DOCKER_TAG" ]; then + docker_tag="$INPUT_DOCKER_TAG" + elif [ "$BUILD_MODE" = "release" ]; then + docker_tag="$electric_version" + else + docker_tag="canary" + fi + echo "docker_tag=$docker_tag" >> $GITHUB_OUTPUT build_and_push_image: strategy: @@ -120,7 +141,7 @@ jobs: build-contexts: | electric-telemetry=packages/electric-telemetry build-args: | - ELECTRIC_VERSION=${{ needs.derive_build_vars.outputs.electric_version }} + ELECTRIC_VERSION=${{ needs.derive_build_vars.outputs.build_mode == 'custom' && needs.derive_build_vars.outputs.docker_tag || needs.derive_build_vars.outputs.electric_version }} platforms: ${{ matrix.platform }} push: true # push an untagged image and export its digest @@ -128,7 +149,7 @@ jobs: outputs: type=image,push-by-digest=true,name-canonical=true,push=true tags: | ${{ env.DOCKERHUB_REPO }} - ${{ env.DOCKERHUB_CANARY_REPO }} + ${{ needs.derive_build_vars.outputs.build_mode == 'canary' && env.DOCKERHUB_CANARY_REPO || '' }} # Save the digest so the merge job can find both platform images - name: Export digest @@ -158,40 +179,42 @@ jobs: merge-multiple: true path: /tmp/digests - - name: Derive image tags from the GitHub Actions event + - name: Derive image tags from the build mode + env: + BUILD_MODE: ${{ needs.derive_build_vars.outputs.build_mode }} + DOCKER_TAG: ${{ needs.derive_build_vars.outputs.docker_tag }} + ELECTRIC_VERSION: ${{ needs.derive_build_vars.outputs.electric_version }} + SHORT_SHA: ${{ needs.derive_build_vars.outputs.short_commit_sha }} run: | - if [ "${{ needs.derive_build_vars.outputs.is_release }}" = "true" ]; then - # A release triggers official release image publishing - echo "ELECTRIC_TAGS=-t $DOCKERHUB_REPO:latest -t $DOCKERHUB_REPO:${{ needs.derive_build_vars.outputs.electric_version }}" >> $GITHUB_ENV + if [ "$BUILD_MODE" = "release" ]; then + echo "ELECTRIC_TAGS=-t $DOCKERHUB_REPO:latest -t $DOCKERHUB_REPO:$ELECTRIC_VERSION" >> $GITHUB_ENV + echo "ELECTRIC_CANARY_TAGS=" >> $GITHUB_ENV + elif [ "$BUILD_MODE" = "custom" ]; then + echo "ELECTRIC_TAGS=-t $DOCKERHUB_REPO:$DOCKER_TAG" >> $GITHUB_ENV echo "ELECTRIC_CANARY_TAGS=" >> $GITHUB_ENV else - # A regular push to the main branch triggers canary image publishing echo "ELECTRIC_TAGS=-t $DOCKERHUB_REPO:canary" >> $GITHUB_ENV - echo "ELECTRIC_CANARY_TAGS=-t $DOCKERHUB_CANARY_REPO:latest -t $DOCKERHUB_CANARY_REPO:${{ needs.derive_build_vars.outputs.short_commit_sha }}" >> $GITHUB_ENV + echo "ELECTRIC_CANARY_TAGS=-t $DOCKERHUB_CANARY_REPO:latest -t $DOCKERHUB_CANARY_REPO:$SHORT_SHA" >> $GITHUB_ENV fi - name: Create multi-arch manifest list run: | set -euo pipefail - # Build a list of $DOCKERHUB_REPO@sha256:... source images ELECTRIC_IMAGES=$( for f in /tmp/digests/*.digest; do echo $DOCKERHUB_REPO@$(cat $f) done ) - # Create a manifest list for $DOCKERHUB_REPO:canary that includes both platforms docker buildx imagetools create $ELECTRIC_TAGS $ELECTRIC_IMAGES if [ -n "$ELECTRIC_CANARY_TAGS" ]; then - # Build a list of $DOCKERHUB_CANARY_REPO@sha256:... source images ELECTRIC_CANARY_IMAGES=$( for f in /tmp/digests/*.digest; do echo $DOCKERHUB_CANARY_REPO@$(cat $f) done ) - # Create a manifest list for $DOCKERHUB_CANARY_REPO:... that includes both platforms docker buildx imagetools create $ELECTRIC_CANARY_TAGS $ELECTRIC_CANARY_IMAGES fi From 3eb4b98074df9047ec33b7582360c498f9bf924f Mon Sep 17 00:00:00 2001 From: rob Date: Mon, 30 Mar 2026 18:26:34 +0100 Subject: [PATCH 6/7] Restore comments and simplify build tags in Docker workflow MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Keep both repo tags in the build step (matching main's approach) since untagged push-by-digest images are harmless — only the publish step controls which repos get tagged manifests. Co-Authored-By: Claude Opus 4.6 (1M context) --- .../sync_service_dockerhub_image.yml | 20 ++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/.github/workflows/sync_service_dockerhub_image.yml b/.github/workflows/sync_service_dockerhub_image.yml index 15e461bf5c..483ab7d83c 100644 --- a/.github/workflows/sync_service_dockerhub_image.yml +++ b/.github/workflows/sync_service_dockerhub_image.yml @@ -75,7 +75,18 @@ jobs: - uses: actions/checkout@v4 with: + # The checked out commit influences the value of the ELECTRIC_VERSION variable + # that is baked into the Docker image. + # + # For regular pushes to main, we check out the HEAD commit and publish canary images. + # + # For releases we check out the tag corresponding to the release, e.g. + # @core/sync-service@v1.2.10. + # + # For custom builds, we check out the branch, tag, or SHA specified by the caller. ref: ${{ steps.git_ref.outputs.git_ref }} + # Also important to fetch the whole history since otherwise we won't get the tags + # that are required to determine the correct ELECTRIC_VERSION. fetch-depth: 0 fetch-tags: true @@ -149,7 +160,7 @@ jobs: outputs: type=image,push-by-digest=true,name-canonical=true,push=true tags: | ${{ env.DOCKERHUB_REPO }} - ${{ needs.derive_build_vars.outputs.build_mode == 'canary' && env.DOCKERHUB_CANARY_REPO || '' }} + ${{ env.DOCKERHUB_CANARY_REPO }} # Save the digest so the merge job can find both platform images - name: Export digest @@ -187,12 +198,15 @@ jobs: SHORT_SHA: ${{ needs.derive_build_vars.outputs.short_commit_sha }} run: | if [ "$BUILD_MODE" = "release" ]; then + # A release triggers official release image publishing echo "ELECTRIC_TAGS=-t $DOCKERHUB_REPO:latest -t $DOCKERHUB_REPO:$ELECTRIC_VERSION" >> $GITHUB_ENV echo "ELECTRIC_CANARY_TAGS=" >> $GITHUB_ENV elif [ "$BUILD_MODE" = "custom" ]; then + # A custom build publishes with the caller-specified tag only echo "ELECTRIC_TAGS=-t $DOCKERHUB_REPO:$DOCKER_TAG" >> $GITHUB_ENV echo "ELECTRIC_CANARY_TAGS=" >> $GITHUB_ENV else + # A regular push to the main branch triggers canary image publishing echo "ELECTRIC_TAGS=-t $DOCKERHUB_REPO:canary" >> $GITHUB_ENV echo "ELECTRIC_CANARY_TAGS=-t $DOCKERHUB_CANARY_REPO:latest -t $DOCKERHUB_CANARY_REPO:$SHORT_SHA" >> $GITHUB_ENV fi @@ -201,20 +215,24 @@ jobs: run: | set -euo pipefail + # Build a list of $DOCKERHUB_REPO@sha256:... source images ELECTRIC_IMAGES=$( for f in /tmp/digests/*.digest; do echo $DOCKERHUB_REPO@$(cat $f) done ) + # Create a manifest list that includes both platforms docker buildx imagetools create $ELECTRIC_TAGS $ELECTRIC_IMAGES if [ -n "$ELECTRIC_CANARY_TAGS" ]; then + # Build a list of $DOCKERHUB_CANARY_REPO@sha256:... source images ELECTRIC_CANARY_IMAGES=$( for f in /tmp/digests/*.digest; do echo $DOCKERHUB_CANARY_REPO@$(cat $f) done ) + # Create a manifest list for $DOCKERHUB_CANARY_REPO:... that includes both platforms docker buildx imagetools create $ELECTRIC_CANARY_TAGS $ELECTRIC_CANARY_IMAGES fi From 117931da941eedd8ec359c1dc97e8418da171737 Mon Sep 17 00:00:00 2001 From: rob Date: Mon, 30 Mar 2026 18:44:35 +0100 Subject: [PATCH 7/7] Remove sha fallback --- .github/workflows/sync_service_dockerhub_image.yml | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/.github/workflows/sync_service_dockerhub_image.yml b/.github/workflows/sync_service_dockerhub_image.yml index 483ab7d83c..58364cd294 100644 --- a/.github/workflows/sync_service_dockerhub_image.yml +++ b/.github/workflows/sync_service_dockerhub_image.yml @@ -100,14 +100,9 @@ jobs: git rev-parse --short HEAD )" >> $GITHUB_OUTPUT - electric_version=$( + echo "electric_version=$( git describe --abbrev=7 --tags --always --first-parent --match '@core/sync-service@*' | sed -En 's|^@core/sync-service@||p' - ) - # Fall back to short SHA if no matching tag exists (e.g. custom builds from old branches) - if [ -z "$electric_version" ]; then - electric_version=$(git rev-parse --short HEAD) - fi - echo "electric_version=$electric_version" >> $GITHUB_OUTPUT + )" >> $GITHUB_OUTPUT # Determine the docker tag to use if [ -n "$INPUT_DOCKER_TAG" ]; then