From 5eaa08518d40b638c804ea1f32e2f7ecb0df2221 Mon Sep 17 00:00:00 2001 From: kpango Date: Thu, 23 Apr 2026 14:49:59 +0900 Subject: [PATCH] ci: add individual github workflows for building dockerfiles Signed-off-by: kpango --- .github/actions/docker-build/action.yml | 79 +++++++++++++ .github/workflows/docker-base.yml | 55 --------- .github/workflows/docker-matrix.yml | 151 ++++++++++++++++++++++++ Makefile | 97 ++++++++++++--- 4 files changed, 310 insertions(+), 72 deletions(-) create mode 100644 .github/actions/docker-build/action.yml delete mode 100755 .github/workflows/docker-base.yml create mode 100644 .github/workflows/docker-matrix.yml diff --git a/.github/actions/docker-build/action.yml b/.github/actions/docker-build/action.yml new file mode 100644 index 00000000..501ea0d0 --- /dev/null +++ b/.github/actions/docker-build/action.yml @@ -0,0 +1,79 @@ +name: "Build and Push Docker Image" +description: "Sets up QEMU, Buildx, and builds/pushes a Docker image using the repository's Makefile." +inputs: + target: + description: "The Makefile target to run (e.g., build_base)" + required: true + docker_user: + description: "DockerHub username" + required: true + default: "kpango" + docker_pass: + description: "DockerHub password" + required: true + github_token: + description: "GitHub Token for registry/cache" + required: true + docker_push: + description: "Whether this is a push build" + required: false + default: "false" + platform: + description: "The platform to build (e.g., linux/amd64)" + required: false + suffix: + description: "The suffix for the docker tag (e.g., amd64)" + required: false + ghcr_user: + description: "GHCR username" + required: false + default: ${{ github.repository_owner }} + +runs: + using: "composite" + steps: + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Login to DockerHub + if: inputs.docker_push == 'true' + uses: docker/login-action@v4 + with: + username: ${{ inputs.docker_user }} + password: ${{ inputs.docker_pass }} + + - name: Login to GitHub Container Registry + if: inputs.docker_push == 'true' + uses: docker/login-action@v4 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ inputs.github_token }} + + - name: Run Makefile Target + shell: bash + env: + TARGET: ${{ inputs.target }} + USER_DEFAULT: ${{ inputs.docker_user }} + GHCR_USER: ${{ inputs.ghcr_user }} + DOCKER_PUSH: ${{ inputs.docker_push }} + GITHUB_ACCESS_TOKEN: ${{ inputs.github_token }} + DOCKER_BUILDER_PLATFORM: ${{ inputs.platform }} + DOCKER_ARCH_SUFFIX: ${{ inputs.suffix }} + EVENT_NAME: ${{ github.event_name }} + EVENT_PATH: ${{ github.event_path }} + GITHUB_REF_VAR: ${{ github.ref }} + run: | + VERSION="nightly" + + if [ "$EVENT_NAME" == "pull_request" ]; then + PR_NUM=$(jq -r ".number" "$EVENT_PATH") + VERSION="pr-$PR_NUM" + elif [[ "$GITHUB_REF_VAR" == refs/tags/* ]]; then + VERSION="${GITHUB_REF_VAR#refs/tags/}" + fi + + make "$TARGET" DOCKER_PUSH="$DOCKER_PUSH" USER="$USER_DEFAULT" GHCR_USER="$GHCR_USER" VERSION="$VERSION" GITHUB_ACCESS_TOKEN="$GITHUB_ACCESS_TOKEN" DOCKER_BUILDER_PLATFORM="$DOCKER_BUILDER_PLATFORM" DOCKER_ARCH_SUFFIX="$DOCKER_ARCH_SUFFIX" diff --git a/.github/workflows/docker-base.yml b/.github/workflows/docker-base.yml deleted file mode 100755 index f1ae0bfb..00000000 --- a/.github/workflows/docker-base.yml +++ /dev/null @@ -1,55 +0,0 @@ -name: 'Build docker image: base' -on: - push: - branches: - - master - tags: - - '*.*.*' - - 'v*.*.*' - - '*.*.*-*' - - 'v*.*.*-*' - paths: - - 'dockers/base.Dockerfile' - pull_request: - paths: - - 'dockers/base.Dockerfile' - -jobs: - build: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - with: - fetch-depth: 10 - - name: Build the Docker image - run: | - make build_Base - - name: login to DockerHub - run: | - echo ${DOCKERHUB_PASS} | docker login --username ${DOCKERHUB_USER} --password-stdin - env: - DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }} - DOCKERHUB_PASS: ${{ secrets.DOCKERHUB_PASS }} - - name: push to DockerHub (master) - if: github.ref == 'refs/heads/master' - run: | - imagename=`make name_base` - docker tag ${imagename} ${imagename}:nightly - docker push ${imagename}:nightly - - name: push to DockerHub (pull request) - if: github.event_name == 'pull_request' - run: | - imagename=`make name_base` - pr_num=`cat $GITHUB_EVENT_PATH | jq -r ".number"` - docker tag ${imagename} ${imagename}:pr-${pr_num} - docker push ${imagename}:pr-${pr_num} - - name: push to DockerHub (tags) - if: startsWith( github.ref, 'refs/tags/') - run: | - imagename=`make name_base` - docker push ${imagename}:latest - tag_name=`echo $GITHUB_REF | sed -e 's:^refs/tags/::'` - docker tag ${imagename} ${imagename}:${tag_name} - docker push ${imagename}:${tag_name} - docker tag ${imagename} ${imagename}:nightly - docker push ${imagename}:nightly diff --git a/.github/workflows/docker-matrix.yml b/.github/workflows/docker-matrix.yml new file mode 100644 index 00000000..8edaa6c1 --- /dev/null +++ b/.github/workflows/docker-matrix.yml @@ -0,0 +1,151 @@ +name: "Build docker images" +on: + push: + branches: + - master + tags: + - "*.*.*" + - "v*.*.*" + - "*.*.*-*" + - "v*.*.*-*" + paths: + - "dockers/**" + - "Makefile" + pull_request: + paths: + - "dockers/**" + - "Makefile" + workflow_dispatch: + +jobs: + build: + strategy: + fail-fast: false + matrix: + arch: + - platform: linux/amd64 + runner: ubuntu-latest + suffix: amd64 + - platform: linux/arm64/v8 + runner: ubuntu-24.04-arm + suffix: arm64 + image_target: + - image: base + target: build_base + - image: dart + target: build_dart + - image: docker + target: build_docker + - image: env + target: build_env + - image: gcloud + target: build_gcloud + - image: go + target: build_go + - image: kube + target: build_k8s + - image: nim + target: build_nim + - image: rust + target: build_rust + runs-on: ${{ matrix.arch.runner }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 10 + + - name: Build and Push Docker Image + uses: ./.github/actions/docker-build + with: + target: ${{ matrix.image_target.target }} + docker_user: ${{ secrets.DOCKERHUB_USER || 'kpango' }} + docker_pass: ${{ secrets.DOCKERHUB_PASS }} + github_token: ${{ secrets.GITHUB_TOKEN }} + docker_push: true + platform: ${{ matrix.arch.platform }} + suffix: ${{ matrix.arch.suffix }} + + merge: + needs: build + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + image_target: + - image: base + target: merge_base + - image: dart + target: merge_dart + - image: docker + target: merge_docker + - image: env + target: merge_env + - image: gcloud + target: merge_gcloud + - image: go + target: merge_go + - image: kube + target: merge_k8s + - image: nim + target: merge_nim + - image: rust + target: merge_rust + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 10 + + - name: Merge and Push Manifest + uses: ./.github/actions/docker-build + with: + target: ${{ matrix.image_target.target }} + docker_user: ${{ secrets.DOCKERHUB_USER || 'kpango' }} + docker_pass: ${{ secrets.DOCKERHUB_PASS }} + github_token: ${{ secrets.GITHUB_TOKEN }} + docker_push: true + + build_dev: + needs: merge + strategy: + fail-fast: false + matrix: + arch: + - platform: linux/amd64 + runner: ubuntu-latest + suffix: amd64 + - platform: linux/arm64/v8 + runner: ubuntu-24.04-arm + suffix: arm64 + runs-on: ${{ matrix.arch.runner }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 10 + + - name: Build and Push Docker Image + uses: ./.github/actions/docker-build + with: + target: prod_build + docker_user: ${{ secrets.DOCKERHUB_USER || 'kpango' }} + docker_pass: ${{ secrets.DOCKERHUB_PASS }} + github_token: ${{ secrets.GITHUB_TOKEN }} + docker_push: true + platform: ${{ matrix.arch.platform }} + suffix: ${{ matrix.arch.suffix }} + + merge_dev: + needs: build_dev + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 10 + + - name: Merge and Push Manifest + uses: ./.github/actions/docker-build + with: + target: merge_dev + docker_user: ${{ secrets.DOCKERHUB_USER || 'kpango' }} + docker_pass: ${{ secrets.DOCKERHUB_PASS }} + github_token: ${{ secrets.GITHUB_TOKEN }} + docker_push: true diff --git a/Makefile b/Makefile index ae56edf1..198479c9 100755 --- a/Makefile +++ b/Makefile @@ -1,16 +1,20 @@ .PHONY: all link zsh bash build prod_build profile run push pull ROOTDIR = $(eval ROOTDIR := $(shell git rev-parse --show-toplevel))$(ROOTDIR) -USER = $(eval USER := $(shell whoami))$(USER) -USER_ID = $(eval USER_ID := $(shell id -u $(USER)))$(USER_ID) -GROUP_ID = $(eval GROUP_ID := $(shell id -g $(USER)))$(GROUP_ID) -GROUP_IDS = $(eval GROUP_IDS := $(shell id -G $(USER)))$(GROUP_IDS) +SYS_USER = $(shell whoami) +USER ?= $(SYS_USER) +USER_ID = $(shell id -u $(SYS_USER)) +GROUP_ID = $(shell id -g $(SYS_USER)) +GROUP_IDS = $(shell id -G $(SYS_USER)) GITHUB_ACCESS_TOKEN = $(eval GITHUB_ACCESS_TOKEN := $(shell pass github.api.ro.token))$(GITHUB_ACCESS_TOKEN) GITHUB_SHA = $(eval GITHUB_SHA := $(shell git rev-parse HEAD))$(GITHUB_SHA) GITHUB_URL = https://github.com/kpango/dotfiles EMAIL = kpango@vdaas.org DOCKER_EXTRA_OPTS ?= +DOCKER_ARCH_SUFFIX ?= +GHCR_USER ?= $(USER) +DOCKER_PUSH ?= true DOCKER_BUILDER_NAME = "kpango-builder" DOCKER_BUILDER_DRIVER = "docker-container" DOCKER_BUILDER_PLATFORM = "linux/amd64,linux/arm64/v8" @@ -18,7 +22,13 @@ DOCKER_CACHE_REPO := $(USER)/$(NAME):buildcache DOCKER_BUILD_CACHE_DIR:= $(HOME)/.docker/buildcache DOCKER_MEMORY_LIMIT = 32G -VERSION = latest +VERSION ?= latest + +ifneq ($(DOCKER_ARCH_SUFFIX),) + DOCKER_TAG_VERSION = $(VERSION)-$(DOCKER_ARCH_SUFFIX) +else + DOCKER_TAG_VERSION = $(VERSION) +endif echo: @echo $(ROOTDIR) @@ -122,7 +132,7 @@ arch_link: \ ln -sfv $(ROOTDIR)/arch/workstyle.toml $(HOME)/.config/workstyle/config.toml ln -sfv $(ROOTDIR)/arch/Xmodmap $(HOME)/.Xmodmap sudo cp $(ROOTDIR)/arch/chrony.conf /etc/chrony.conf - sudo cp $(ROOTDIR)/arch/suduers /etc/sudoers.d/$(USER) + sudo cp $(ROOTDIR)/arch/suduers /etc/sudoers.d/$(SYS_USER) sudo cp $(ROOTDIR)/arch/environment /etc/environment sudo cp $(ROOTDIR)/network/NetworkManager-dispatcher.service /etc/systemd/system/NetworkManager-dispatcher.service sudo cp $(ROOTDIR)/network/nmcli-wifi-eth-autodetect.sh /etc/NetworkManager/dispatcher.d/nmcli-wifi-eth-autodetect.sh @@ -154,8 +164,8 @@ arch_link: \ sudo chown root:root /etc/NetworkManager/dispatcher.d/nmcli-bond-auto-connect.sh sudo chown -R 0:0 /etc/sudoers.d sudo chmod -R 0440 /etc/sudoers.d - sudo chown -R 0:0 /etc/sudoers.d/$(USER) - sudo chmod -R 0440 /etc/sudoers.d/$(USER) + sudo chown -R 0:0 /etc/sudoers.d/$(SYS_USER) + sudo chmod -R 0440 /etc/sudoers.d/$(SYS_USER) sudo sysctl -e -p /etc/sysctl.d/99-sysctl.conf sudo systemctl daemon-reload @@ -350,7 +360,7 @@ clean: perm /etc/pulse/default.pa \ /etc/resolv.dnsmasq.conf \ /etc/resolv.pre-tailscale-backup.conf \ - /etc/sudoers.d/$(USER) \ + /etc/sudoers.d/$(SYS_USER) \ /etc/sysctl.conf \ /etc/sysctl.d/99-sysctl.conf \ /etc/systemd/system/NetworkManager-dispatcher.service \ @@ -398,6 +408,7 @@ docker_build: $(eval TMP_DIR := $(shell mktemp -d)) @echo $(GITHUB_ACCESS_TOKEN) > $(TMP_DIR)/gat @chmod 600 $(TMP_DIR)/gat + @mkdir -p $(DOCKER_BUILD_CACHE_DIR) DOCKER_BUILDKIT=1 docker buildx build \ $(DOCKER_EXTRA_OPTS) \ --builder "$(DOCKER_BUILDER_NAME)" \ @@ -412,10 +423,15 @@ docker_build: --build-arg HOME="$(HOME)" \ --build-arg USER="$(USER)" \ --build-arg USER_ID="$(USER_ID)" \ - --build-arg WHOAMI="$(USER)" \ + --build-arg WHOAMI="$(SYS_USER)" \ --cache-from type=local,src=$(DOCKER_BUILD_CACHE_DIR) \ + --cache-from "ghcr.io/$(GHCR_USER)/$(NAME):$(DOCKER_TAG_VERSION)" \ + --cache-from type=gha \ --cache-from type=registry,ref=$(DOCKER_CACHE_REPO) \ --cache-to type=local,dest=$(DOCKER_BUILD_CACHE_DIR),mode=max \ + --cache-to type=registry,ref=$(DOCKER_BUILD_CACHE_REPO),mode=max \ + --cache-to type=gha,mode=max \ + --cache-to type=inline, mode=max \ --label org.opencontainers.image.revision="$(GITHUB_SHA)" \ --label org.opencontainers.image.source="$(GITHUB_URL)" \ --label org.opencontainers.image.title="$(USER)/$(NAME)" \ @@ -424,12 +440,13 @@ docker_build: --memory $(DOCKER_MEMORY_LIMIT) \ --memory-swap $(DOCKER_MEMORY_LIMIT) \ --network=host \ - --output type=registry,oci-mediatypes=true,compression=zstd,compression-level=5,force-compression=true,push=true \ + --output type=registry,oci-mediatypes=true,compression=zstd,compression-level=5,force-compression=true,push=$(DOCKER_PUSH) \ --provenance=mode=max \ --secret id=gat,src="$(TMP_DIR)/gat" \ - -t "$(USER)/$(NAME):$(VERSION)" \ - -f $(DOCKERFILE) . - # --cache-to type=registry,ref=$(DOCKER_CACHE_REPO),mode=max,inline=true \ + -t "$(USER)/$(NAME):$(DOCKER_TAG_VERSION)" \ + -t "ghcr.io/$(GHCR_USER)/$(NAME):$(DOCKER_TAG_VERSION)" \ + -f $(DOCKERFILE) . \ + --cache-to type=registry,ref=$(DOCKER_CACHE_REPO),mode=max,inline=true docker buildx rm --force "$(DOCKER_BUILDER_NAME)" @rm -rf $(TMP_DIR) @@ -453,7 +470,7 @@ create_buildx: # make add_nodes docker buildx ls docker buildx inspect --bootstrap $(DOCKER_BUILDER_NAME) - sudo chown -R $(USER):$(GROUP_ID) "$(HOME)/.docker" + sudo chown -R $(SYS_USER):$(GROUP_ID) "$(HOME)/.docker" add_nodes: @echo $(DOCKER_BUILDER_PLATFORM) | tr ',' '\n' | while read platform; do \ @@ -508,6 +525,52 @@ build_gcloud: build_k8s: @make NAME="kube" do_build + + +merge_dev: + @make NAME="dev" do_merge + +merge_mkl: + @make NAME="mkl" do_merge + +merge_go: + @make NAME="go" do_merge + +merge_rust: + @make NAME="rust" do_merge + +merge_nim: + @make NAME="nim" do_merge + +merge_dart: + @make NAME="dart" do_merge + +merge_docker: + @make NAME="docker" do_merge + +merge_base: + @make NAME="base" do_merge + +merge_env: + @make NAME="env" do_merge + +merge_gcloud: + @make NAME="gcloud" do_merge + +merge_k8s: + @make NAME="kube" do_merge + +docker_merge: + docker buildx imagetools create -t "$(USER)/$(NAME):$(VERSION)" \ + "$(USER)/$(NAME):$(VERSION)-amd64" \ + "$(USER)/$(NAME):$(VERSION)-arm64" + docker buildx imagetools create -t "ghcr.io/$(GHCR_USER)/$(NAME):$(VERSION)" \ + "ghcr.io/$(GHCR_USER)/$(NAME):$(VERSION)-amd64" \ + "ghcr.io/$(GHCR_USER)/$(NAME):$(VERSION)-arm64" + +do_merge: + @make NAME="$(NAME)" docker_merge + profile: rm -f analyze.txt type dlayer >/dev/null 2>&1 && docker save kpango/dev:latest | dlayer >> analyze.txt @@ -526,8 +589,8 @@ pull: perm: sudo chmod -R 755 $(ROOTDIR)/* sudo chmod -R 755 $(ROOTDIR)/.* - sudo chown -R $(USER):$(GROUP_ID) $(ROOTDIR)/* - sudo chown -R $(USER):$(GROUP_ID) $(ROOTDIR)/.* + sudo chown -R $(SYS_USER):$(GROUP_ID) $(ROOTDIR)/* + sudo chown -R $(SYS_USER):$(GROUP_ID) $(ROOTDIR)/.* sudo chmod -R 644 $(ROOTDIR)/gpg-agent.conf sudo chmod -R 644 $(ROOTDIR)/arch/waybar.json \find $(ROOTDIR) -type d -name '.git' -prune -o -type f -not -name 'tmux.conf' -exec nkf -Lu -w --overwrite {} \;