ci(feat): add mkosi os image building

.github/workflows/build-os.yaml

@@ -0,0 +1,216 @@
+name: Build OS
+
+on:
+  schedule:
+    - cron: '0 16 * * *'  # Daily at 8am PST / 4pm UTC
+  push:
+    branches:
+      - main
+    paths:
+      - 'deps/mkosi/**'
+  pull_request:
+    branches:
+      - main
+    paths:
+      - 'deps/mkosi/**'
+  workflow_dispatch:
+
+permissions:
+  contents: read
+  id-token: write
+
+jobs:
+  build-os-image:
+    runs-on:
+      - ${{ matrix.os }}
+    container:
+      image: public.ecr.aws/docker/library/fedora:43
+      options: --privileged
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-24.04, ubuntu-24.04-arm]
+    timeout-minutes: 60
+    steps:
+      - name: Install packages
+        run: |
+          dnf install -y git \
+            systemd-boot \
+            systemd-repart \
+            systemd-ukify \
+            qemu-img \
+            docker \
+            jq \
+            e2fsprogs \
+            dosfstools \
+            cpio \
+            zstd \
+            procps-ng \
+            distribution-gpg-keys \
+            openssl \
+            mtools \
+            skopeo \
+            awscli2
+      - name: Checkout repo
+        uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
+        with:
+          submodules: true
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@7474bc4690e29a8392af63c5b98e7449536d5c3a # v4.3.1
+        with:
+          role-to-assume: ${{ secrets.ROLE }}
+          aws-region: ${{ secrets.REGION }}
+          role-session-name: rootfs-ecr-image-upload-session
+      - name: Login to Amazon ECR
+        uses: aws-actions/amazon-ecr-login@062b18b96a7aff071d4dc91bc00c4c1a7945b076 # v2.0.1
+      - name: Install mkosi
+        run: |
+          mkdir -p "$HOME/.local/bin/"
+          export PATH="/root/.local/bin/:$PATH"
+          ./deps/mkosi/install-mkosi.sh
+      - name: mkosi arch
+        id: mkosi-arch
+        shell: bash
+        run: |
+          arch=""
+          if [ "$(uname -m)" == "aarch64" ]; then
+            arch="arm64"
+          else
+            arch="x86-64"
+          fi
+          echo "arch=${arch}" >> $GITHUB_OUTPUT
+      - name: Fix git permissions
+        run: |
+          git config --global --add safe.directory "${GITHUB_WORKSPACE}"
+      - name: Build QEMU
+        run: |
+          ln -s "$HOME/.docker" /root/.docker
+          export ECR_CACHE_REPO="${{ secrets.ECR_CACHE_REPO }}"
+          ./deps/mkosi/scripts/build-qemu.sh
+      - name: Setup binary packages
+        run: |
+          ./deps/mkosi/scripts/bin-packages.sh
+      - name: Build custom AL packages
+        run: |
+          ln -s "$HOME/.docker" /root/.docker
+          ./deps/mkosi/scripts/al2023-package-build.sh
+      - name: Build OS image
+        run: |
+          arch="${{ steps.mkosi-arch.outputs.arch }}"
+          export PATH="$HOME/.local/bin/:$PATH"
+          ./deps/mkosi/mkosi.sh --arch "${arch}" -- --image-id os-image
+      - name: Convert OS image
+        run: |
+          arch="${{ steps.mkosi-arch.outputs.arch }}"
+          sudo qemu-img convert -f raw -c -O qcow2 "./deps/mkosi/out/${arch}/os-image.raw" \
+            "./deps/mkosi/out/${arch}/os-image.qcow2"
+      - name: Calculate SHA512 checksum for OS image
+        run: |
+          arch="${{ steps.mkosi-arch.outputs.arch }}"
+          cd "./deps/mkosi/out/${arch}"
+          sha512sum os-image.qcow2 | cut -d ' ' -f 1 > os-image.qcow2.sha512sum
+      - name: Upload OS image to S3
+        run: |
+          arch="${{ steps.mkosi-arch.outputs.arch }}"
+          filename="finch-al2023-os-image-${arch}-${{ github.run_id }}.qcow2"
+          aws s3 cp "./deps/mkosi/out/${arch}/os-image.qcow2" \
+            "s3://${{ secrets.DEPENDENCY_BUCKET_NAME }}/${filename}" \
+            --region "${{ secrets.DEPENDENCY_BUCKET_REGION }}"
+          aws s3 cp "./deps/mkosi/out/${arch}/os-image.qcow2.sha512sum" \
+            "s3://${{ secrets.DEPENDENCY_BUCKET_NAME }}/${filename}.sha512sum" \
+            --region "${{ secrets.DEPENDENCY_BUCKET_REGION }}"
+      - name: Push OS image
+        run: |
+          arch="${{ steps.mkosi-arch.outputs.arch }}"
+          tar -cvf "./deps/mkosi/out/${arch}/container-with-kernel-image.tar" -C "./deps/mkosi/out/${arch}/container-with-kernel" .
+          skopeo copy \
+            "oci-archive:./deps/mkosi/out/${arch}/container-with-kernel-image.tar" \
+            docker://"${{ secrets.ROOTFS_IMAGE_ECR_REPOSITORY_NAME }}:${{ steps.mkosi-arch.outputs.arch }}-with-kernel-${{github.run_id}}"
+      - name: Run wsl rootfs build
+        if: steps.mkosi-arch.outputs.arch == 'x86-64'
+        run: |
+          export PATH="$HOME/.local/bin/:$PATH"
+          ./deps/mkosi/mkosi.sh --arch x86-64 -- --image-id wsl-rootfs
+      - name: Remove init -> systemd symlink and delete efi directory
+        if: steps.mkosi-arch.outputs.arch == 'x86-64'
+        run: |
+          arch="${{ steps.mkosi-arch.outputs.arch }}"
+          cd "./deps/mkosi/out/${arch}/wsl-rootfs/"
+          rm -f init
+          rm -rf efi
+      - name: Compress rootfs for S3
+        if: steps.mkosi-arch.outputs.arch == 'x86-64'
+        run: |
+          arch="${{ steps.mkosi-arch.outputs.arch }}"
+          cd "./deps/mkosi/out/${arch}"
+          tar -czvf wsl-rootfs.tar.gz -C wsl-rootfs .
+      - name: Calculate SHA512 checksum for rootfs
+        if: steps.mkosi-arch.outputs.arch == 'x86-64'
+        run: |
+          arch="${{ steps.mkosi-arch.outputs.arch }}"
+          cd "./deps/mkosi/out/${arch}"
+          sha512sum wsl-rootfs.tar.gz | cut -d ' ' -f 1 > wsl-rootfs.tar.gz.sha512sum
+      - name: Upload rootfs to S3
+        if: steps.mkosi-arch.outputs.arch == 'x86-64'
+        run: |
+          arch="${{ steps.mkosi-arch.outputs.arch }}"
+          filename="finch-al2023-rootfs-${arch}-${{ github.run_id }}.tar.gz"
+          aws s3 cp "./deps/mkosi/out/${arch}/wsl-rootfs.tar.gz" \
+            "s3://${{ secrets.DEPENDENCY_BUCKET_NAME }}/common/${arch}/${filename}" \
+            --region "${{ secrets.DEPENDENCY_BUCKET_REGION }}"
+          aws s3 cp "./deps/mkosi/out/${arch}/wsl-rootfs.tar.gz.sha512sum" \
+            "s3://${{ secrets.DEPENDENCY_BUCKET_NAME }}/common/${arch}/${filename}.sha512sum" \
+            --region "${{ secrets.DEPENDENCY_BUCKET_REGION }}"
+      - name: Upload container image artifacts
+        if: steps.mkosi-arch.outputs.arch == 'x86-64'
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+        with:
+          name: image-oci-${{ steps.mkosi-arch.outputs.arch }}-${{github.run_id}}.tar.gz
+          path: ./deps/mkosi/out/${{ steps.mkosi-arch.outputs.arch }}/wsl-rootfs.tar.gz
+          if-no-files-found: error
+      - name: Upload OS image artifacts
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+        with:
+          name: image-${{ steps.mkosi-arch.outputs.arch }}-${{github.run_id}}.qcow2
+          path: ./deps/mkosi/out/${{ steps.mkosi-arch.outputs.arch }}/os-image.qcow2
+          if-no-files-found: error
+
+  update-deps-config:
+    needs: build-os-image
+    runs-on: ubuntu-latest
+    if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'
+    permissions:
+      contents: write
+      pull-requests: write
+    steps:
+      - name: Checkout repo
+        uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@7474bc4690e29a8392af63c5b98e7449536d5c3a # v4.3.1
+        with:
+          role-to-assume: ${{ secrets.ROLE }}
+          aws-region: ${{ secrets.REGION }}
+          role-session-name: update-deps-config-session
+      - name: Run update-os-image script
+        run: |
+          bash bin/update-os-image.sh \
+            -d "${{ secrets.DEPENDENCY_BUCKET_NAME }}"
+      - name: Run update-rootfs script
+        run: |
+          bash bin/update-rootfs.sh \
+            -d "${{ secrets.DEPENDENCY_BUCKET_NAME }}"
+      - name: Create Pull Request
+        uses: peter-evans/create-pull-request@5e914681df9dc83aa4e4905692ca88beb2f9e91f # v7.0.5
+        if: github.event_name != 'pull_request'
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+          commit-message: "deps(os): update OS image to run ${{ github.run_id }}"
+          title: "deps(os): update OS image"
+          body: |
+            This PR was automatically created by the [build-os workflow run](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}).
+
+            Updates the OS image and rootfs artifacts in:
+            - `deps/full-os.conf` (OS images for both architectures)
+            - `deps/rootfs.conf` (rootfs for x86-64)
+          branch: deps/update-os-image-${{ github.run_id }}
+          delete-branch: true

.github/workflows/submodulesync.yaml

@@ -28,11 +28,18 @@ jobs:
           TAG=`cd src/lima && git describe --tags $(git rev-list --tags --max-count=1)`
           echo "Pulling changes from release: $TAG"
           (cd src/lima && git checkout $TAG)
+
           # finch-daemon
           (cd src/finch-daemon && git fetch --tags)
           FINCH_DAEMON_TAG=`cd src/finch-daemon && git describe --tags $(git rev-list --tags --max-count=1)`
           echo "Pulling changes from release: $FINCH_DAEMON_TAG"
           (cd src/finch-daemon && git checkout $FINCH_DAEMON_TAG)
+
+          # binfmt
+          (cd deps/mkosi/binfmt && git fetch --tags)
+          BINFMT_TAG=`cd deps/mkosi/binfmt && git describe --tags $(git rev-list --tags --max-count=1)`
+          echo "Pulling changes from release: $BINFMT_TAG"
+          (cd deps/mkosi/binfmt && git checkout $BINFMT_TAG)
 
       - name: Create PR
         uses: peter-evans/create-pull-request@271a8d0340265f705b14b6d32b9829c1cb33d45e # v7.0.8

.github/workflows/update-dependencies.yaml

@@ -26,7 +26,7 @@ permissions:
   contents: read
 
 jobs:
-  update-deps:
+  update-lima-bundle:
     runs-on: ubuntu-latest
 
     permissions:
@@ -50,12 +50,6 @@ jobs:
       - name: Update Lima bundle archive for Finch on macOS
         run: bash bin/update-lima-bundles.sh -d ${{ secrets.DEPENDENCY_BUCKET_NAME }}
 
-      - name: Update base OS for Finch on macOS
-        run: bash bin/update-os-image.sh -d ${{ secrets.DEPENDENCY_BUCKET_NAME }}
-
-      - name: Update rootfs for Finch on Windows
-        run: bash bin/update-rootfs.sh -d ${{ secrets.DEPENDENCY_BUCKET_NAME }}
-
       - name: create PR
         if: github.event_name != 'pull_request'
         uses: peter-evans/create-pull-request@271a8d0340265f705b14b6d32b9829c1cb33d45e # v7.0.8

.gitignore

@@ -8,3 +8,13 @@ wingit-temp/
 **/*.tar.gz
 .vscode/
 **/node_modules
+deps/mkosi/mkosi.cache/*.*
+!deps/mkosi/mkosi.cache/*.gitkeep
+deps/mkosi/mkosi.builddir/
+deps/mkosi/mkosi.images/base/mkosi.extra/
+deps/mkosi/binfmt/bin
+deps/mkosi/qemu-binfmt-conf.sh
+deps/mkosi/out/
+deps/mkosi/al2023-package-build/al2023-build*
+deps/mkosi/al2023-package-build/artifacts
+deps/mkosi/al2023-package-build/_output

.gitmodules

@@ -7,3 +7,6 @@
 [submodule "src/finch-daemon"]
 	path = src/finch-daemon
 	url = https://github.com/runfinch/finch-daemon.git
+[submodule "deps/mkosi/binfmt"]
+	path = deps/mkosi/binfmt
+	url = https://github.com/tonistiigi/binfmt

bin/update-os-image.sh

@@ -3,7 +3,7 @@
 # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
 # SPDX-License-Identifier: Apache-2.0
 
-# A script to update the base os image used for Finch on macOS.
+# A script to update the OS image artifacts in deps/full-os.conf
 #
 # Usage: bash update-os-image.sh -d <S3 bucket>
 
@@ -16,8 +16,8 @@ PROJECT_ROOT="$(cd -- "${CURRENT_DIR}/.." && pwd)"
 source "${PROJECT_ROOT}/bin/utility.sh"
 
 DEPENDENCY_CLOUDFRONT_URL="https://deps.runfinch.com"
-AARCH64_FILENAME_PATTERN="Fedora-Cloud-Base-.*\.aarch64-[0-9]+\.qcow2$"
-AMD64_FILENAME_PATTERN="Fedora-Cloud-Base-.*\.x86_64-[0-9]+\.qcow2$"
+AARCH64_FILENAME_PATTERN="finch-al2023-os-image-arm64-[0-9]+\.qcow2$"
+AMD64_FILENAME_PATTERN="finch-al2023-os-image-x86-64-[0-9]+\.qcow2$"
 
 while getopts d: flag
 do
@@ -48,15 +48,17 @@ pull_artifact_and_verify_shasum "${DEPENDENCY_CLOUDFRONT_URL}/${amd64_deps}" "${
 
 # Update base os file with latest artifacts and digests
 OS_FILE="${PROJECT_ROOT}/deps/full-os.conf"
+
+# Regenerate the file with OS artifacts and cosign config
 truncate -s 0 "${OS_FILE}"
 {
     echo "ARTIFACT_BASE_URL=${DEPENDENCY_CLOUDFRONT_URL}"
     echo ""
-    echo "# From https://dl.fedoraproject.org/pub/fedora/linux/releases/42/Cloud/aarch64/images/"
+    echo "# Built by mkosi from deps/mkosi"
     echo "AARCH64_ARTIFACT=$(basename "${aarch64_deps}")"
     echo "AARCH64_512_DIGEST=${aarch64_deps_shasum}"
     echo ""
-    echo "# From https://dl.fedoraproject.org/pub/fedora/linux/releases/42/Cloud/x86_64/images/"
+    echo "# Built by mkosi from deps/mkosi"
     echo "X86_64_ARTIFACT=$(basename "${amd64_deps}")"
     echo "X86_64_512_DIGEST=${amd64_deps_shasum}"
 } >> "${OS_FILE}"

bin/update-rootfs.sh

@@ -3,7 +3,7 @@
 # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
 # SPDX-License-Identifier: Apache-2.0
 
-# A script to update the rootfs dependency used for Finch on Windows.
+# A script to update the rootfs artifacts in deps/rootfs.conf
 #
 # Usage: bash update-rootfs.sh -d <S3 bucket>
 
@@ -16,7 +16,7 @@ PROJECT_ROOT="$(cd -- "${CURRENT_DIR}/.." && pwd)"
 source "${PROJECT_ROOT}/bin/utility.sh"
 
 DEPENDENCY_CLOUDFRONT_URL="https://deps.runfinch.com"
-AMD64_FILENAME_PATTERN="finch-rootfs-production-amd64-[0-9]+\.tar\.gz$"
+AMD64_FILENAME_PATTERN="finch-al2023-rootfs-x86-64-[0-9]+\.tar\.gz$"
 PLATFORM="common"
 # ARM not currently supported for Finch on Windows
 # AARCH64="aarch64"

deps/cosign.conf

@@ -0,0 +1,5 @@
+# Cosign binary for image signing
+COSIGN_VERSION=2.6.2-1
+COSIGN_RELEASE=2.6.2
+COSIGN_AARCH64_RPM_SHA256_DIGEST=0d7f20ae4edc713e72ad7b1d0b580b41c72c0fa2b1079ecbbf108268f80fc288
+COSIGN_X86_64_RPM_SHA256_DIGEST=9e7a0f324e2603315349a367d7822c59ba66e228f60abb9d887988118bd08632

deps/full-os.conf

@@ -1,9 +1,9 @@
 ARTIFACT_BASE_URL=https://deps.runfinch.com
 
-# From https://dl.fedoraproject.org/pub/fedora/linux/releases/42/Cloud/aarch64/images/
-AARCH64_ARTIFACT=Fedora-Cloud-Base-Generic-42-1.1.aarch64-20260217220711.qcow2
-AARCH64_512_DIGEST=dd1d74b0a97ef77ce07981c2b8f415b98a45a3cd206694100df2daa9261db9487425339f55848eec65a6b1657f72cf6012dfdcd80246e61c94d11573d9a8d1a1
+# built with mkosi
+AARCH64_ARTIFACT=finch-al2023-os-image-arm64-22922664487.qcow2
+AARCH64_512_DIGEST=49429e07332c79df6cf822d16a10047076db9e8e128601e3730ca5f258f96d44a307aae46466d8cc0d2906b7a61edd4aed5ef5fd7c865164e567e14b85a56dac
 
-# From https://dl.fedoraproject.org/pub/fedora/linux/releases/42/Cloud/x86_64/images/
-X86_64_ARTIFACT=Fedora-Cloud-Base-Generic-42-1.1.x86_64-20260217221823.qcow2
-X86_64_512_DIGEST=4e2da044cb7b1a04c5c25ffd1c62fb07ec352ee1edf4dda64a4f1c3ff08661024df57b4187ea819504885afbfe888b61fbd71d28d75c41ba168b7b691e206f1b
+# built with mkosi
+X86_64_ARTIFACT=finch-al2023-os-image-x86-64-22922664487.qcow2
+X86_64_512_DIGEST=850fc4677e0974667be5ae8f5a25b0a507e9babf2aac4c25c57747f2cb555d85ffbe475e3069dac62025aa9201aa4cbba44d848d91d867373c0ebc04deb99316

deps/mkosi/.gitmodules

@@ -0,0 +1,3 @@
+[submodule "binfmt"]
+	path = binfmt
+	url = https://github.com/tonistiigi/binfmt