diff --git a/.github/actions/build/action.yaml b/.github/actions/build/action.yaml index 70a697f7..0f7cca11 100644 --- a/.github/actions/build/action.yaml +++ b/.github/actions/build/action.yaml @@ -43,6 +43,11 @@ inputs: - all - rpms - bootc-image + rpm-builder: + type: string + description: RPM builder image to use + required: false + default: rpm-local-builder runs: using: "composite" @@ -56,9 +61,12 @@ runs: uses: ./.github/actions/debug-info - name: Prepare the build and run environment + # Skip if the rpm-builder is overridden - release.yaml builds RPMs using COPR. + if: inputs.rpm-builder == 'rpm-local-builder' uses: ./.github/actions/prebuild - name: Build MicroShift RPMs + if: inputs.rpm-builder == 'rpm-local-builder' shell: bash run: | # See https://github.com/microshift-io/microshift/blob/main/docs/build.md @@ -89,6 +97,7 @@ runs: make image \ BOOTC_IMAGE_URL=${{ inputs.bootc-image-url }} \ BOOTC_IMAGE_TAG=${{ inputs.bootc-image-tag }} \ + BUILDER_IMAGE=${{ inputs.rpm-builder }} \ ${make_opts[@]} - name: Run a test to verify that MicroShift is functioning properly @@ -143,17 +152,21 @@ runs: set -euo pipefail set -x - # Check if the MicroShift container is running - if ! sudo podman ps --format "{{.Names}}" | grep -q '^microshift-okd$' ; then - echo "WARNING: MicroShift container is not running - cannot collect sos report" - exit 0 - fi + sudo podman ps --all - # Collect sos report from the MicroShift container - sudo podman exec -i microshift-okd microshift-sos-report - for f in $(sudo podman exec -i microshift-okd bash -c 'ls -1 /tmp/sosreport-*'); do - sudo podman cp microshift-okd:${f} /mnt/tmp/ - sudo chmod 644 "/mnt/tmp/$(basename "${f}")" + for i in $(seq 1 ${{ inputs.node-count }}); do + # Check if the MicroShift container is running + if ! sudo podman ps --format "{{.Names}}" | grep -q "^microshift-okd-${i}$" ; then + echo "WARNING: MicroShift container number ${i} is not running - cannot collect sos report" + exit 0 + fi + + # Collect sos report from the MicroShift container + sudo podman exec -i "microshift-okd-${i}" microshift-sos-report + for f in $(sudo podman exec -i "microshift-okd-${i}" bash -c 'ls -1 /tmp/sosreport-*'); do + sudo podman cp "microshift-okd-${i}":${f} /mnt/tmp/ + sudo chmod 644 "/mnt/tmp/$(basename "${f}")" + done done - name: Upload sos report to the GitHub Actions artifact diff --git a/.github/workflows/builders.yaml b/.github/workflows/builders.yaml index ae0660f7..62211ac9 100644 --- a/.github/workflows/builders.yaml +++ b/.github/workflows/builders.yaml @@ -61,7 +61,7 @@ jobs: ushift-branch: main okd-version-tag: ${{ steps.detect-okd-version.outputs.okd-version-tag }} bootc-image-url: registry.fedoraproject.org/fedora-bootc - bootc-image-tag: latest + bootc-image-tag: 42 build: bootc-image node-count: 2 diff --git a/.github/workflows/release.md b/.github/workflows/release.md index 837e4788..50b45f53 100644 --- a/.github/workflows/release.md +++ b/.github/workflows/release.md @@ -2,19 +2,32 @@ #### RPM -Review the instructions in [MicroShift RPMs](https://github.com/microshift-io/microshift/blob/main/docs/run.md#microshift-rpms) to install the packages and run MicroShift. +Enable COPR repository (optionally specify chroot such as `centos-stream-9-{x86_64,aarch64}`, `fedora-42-{x86_64,aarch64}`): +```sh +sudo dnf copr enable $COPR_REPO_NAME [chroot] +``` + +Next, install MicroShift: +```sh +sudo dnf install -y \ + microshift-$VERSION \ + microshift-kindnet-$VERSION \ + microshift-topolvm-$VERSION +``` + +Review the instructions in [MicroShift RPMs](https://github.com/microshift-io/microshift/blob/main/docs/run.md#microshift-rpms) to run MicroShift. #### Bootc Image Load the Bootc container image using the following command: ```bash -sudo podman pull ghcr.io/microshift-io/microshift:$TAG +sudo podman pull ghcr.io/microshift-io/microshift:$VERSION ``` Or use the image with the `quickstart.sh`: ```bash -curl -s https://microshift-io.github.io/microshift/quickstart.sh | sudo TAG=$TAG bash +curl -s https://microshift-io.github.io/microshift/quickstart.sh | sudo TAG=$VERSION bash ``` Review the instructions in [MicroShift Bootc Image](https://github.com/microshift-io/microshift/blob/main/docs/run.md#microshift-bootc-image) to run the image. diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 10cae9d7..2867c31d 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -9,17 +9,13 @@ on: description: MicroShift branch from https://github.com/openshift/microshift/branches type: string okd-version-tag: - default: "4.19.0-okd-scos.19" + default: "4.21.0-okd-scos.ec.5" description: OKD version tag from https://quay.io/repository/okd/scos-release?tab=tags type: string - build: - type: choice - description: Types of artifacts to build - default: all - options: - - all - - rpms - - bootc-image + copr-repo: + default: pmtk0/test123 + description: COPR repository name + type: string jobs: build: @@ -34,47 +30,56 @@ jobs: - name: Check out MicroShift upstream repository uses: actions/checkout@v4 - - name: Run the build action + - name: Prepare the build and run environment + uses: ./.github/actions/prebuild + + - name: Build MicroShift RPMs + shell: bash + env: + COPR_TOKEN: | + ${{ secrets.COPR_TOKEN }} + run: | + cd ${GITHUB_WORKSPACE}/ + echo "${COPR_TOKEN}" > /tmp/copr-token + make copr-rpm \ + USHIFT_BRANCH=${{ inputs.ushift-branch }} \ + OKD_VERSION_TAG=${{ inputs.okd-version-tag }} \ + RPM_OUTDIR=/mnt/rpms \ + COPR_CONFIG=/tmp/copr-token \ + COPR_REPO_NAME="${{ inputs.copr-repo }}" + + - name: Build and test bootc image uses: ./.github/actions/build with: ushift-branch: ${{ inputs.ushift-branch }} okd-version-tag: ${{ inputs.okd-version-tag }} - build: ${{ inputs.build }} - - # Test the local container image with the quick start and clean procedures - # before releasing the artifacts. - - name: Run the quick start script and clean scripts - if: contains(fromJSON('["all", "bootc-image"]'), inputs.build) - uses: ./.github/actions/quick-start-clean - with: - image-ref: localhost/microshift-okd:latest + build: bootc-image + rpm-builder: rpm-copr-builder - # The release process consumes the RPMs and the container image - # prepared by the build action. - - name: Prepare the RPM archives - if: contains(fromJSON('["all", "rpms"]'), inputs.build) + - name: COPR - Delete builds + if: failure() shell: bash + env: + COPR_TOKEN: | + ${{ secrets.COPR_TOKEN }} run : | - # Archive sources separately from the RPMs - sudo mv /mnt/rpms/srpms /mnt/srpms - cd /mnt/srpms - sudo tar zcvf /mnt/release/microshift-src.tgz . - - cd /mnt/rpms - sudo tar zcvf /mnt/release/microshift-rpms-$(uname -m).tgz . + echo "${COPR_TOKEN}" > /tmp/copr-token + make copr-delete-builds \ + COPR_CONFIG=/tmp/copr-token \ + COPR_BUILDS="$(cat /mnt/rpms/builds.txt)" - - name: Release RPMs - if: contains(fromJSON('["all", "rpms"]'), inputs.build) - uses: softprops/action-gh-release@v2 - with: - tag_name: ${{ inputs.ushift-branch }}-${{ inputs.okd-version-tag }} - files: | - /mnt/release/microshift-rpms-*.tgz - /mnt/release/microshift-src.tgz - overwrite_files: true + - name: COPR - Regenerate RPM repo + shell: bash + env: + COPR_TOKEN: | + ${{ secrets.COPR_TOKEN }} + run : | + echo "${COPR_TOKEN}" > /tmp/copr-token + make copr-regenerate-repos \ + COPR_CONFIG=/tmp/copr-token \ + COPR_REPO_NAME="${{ inputs.copr-repo }}" - name: Login to GitHub Container Registry - if: contains(fromJSON('["all", "bootc-image"]'), inputs.build) uses: redhat-actions/podman-login@v1 with: registry: ghcr.io/${{ github.repository_owner }} @@ -83,21 +88,24 @@ jobs: auth_file_path: /tmp/ghcr-auth.json - name: Publish Bootc image - if: contains(fromJSON('["all", "bootc-image"]'), inputs.build) + id: publish-bootc-image shell: bash run: | + VERSION="$(cat /mnt/rpms/version.txt)" sudo podman tag microshift-okd \ - ghcr.io/${{ github.repository }}:${{ inputs.ushift-branch }}-${{ inputs.okd-version-tag }} \ + ghcr.io/${{ github.repository }}:${VERSION} \ ghcr.io/${{ github.repository }}:latest - sudo podman push --authfile /tmp/ghcr-auth.json ghcr.io/${{ github.repository }}:${{ inputs.ushift-branch }}-${{ inputs.okd-version-tag }} + sudo podman push --authfile /tmp/ghcr-auth.json ghcr.io/${{ github.repository }}:${VERSION} sudo podman push --authfile /tmp/ghcr-auth.json ghcr.io/${{ github.repository }}:latest # Prepare the release note for the bootc image usage - TAG=${{ inputs.ushift-branch }}-${{ inputs.okd-version-tag }} envsubst < .github/workflows/release.md > /tmp/release.md + VERSION="${VERSION}" \ + COPR_REPO_NAME="${{ inputs.copr-repo }}" \ + envsubst < .github/workflows/release.md > /tmp/release.md + echo "version=${VERSION}" >> $GITHUB_OUTPUT - name: Add release note for bootc image usage - if: contains(fromJSON('["all", "bootc-image"]'), inputs.build) uses: softprops/action-gh-release@v2 with: - tag_name: ${{ inputs.ushift-branch }}-${{ inputs.okd-version-tag }} + tag_name: ${{ steps.publish-bootc-image.outputs.version }} body_path: /tmp/release.md diff --git a/Makefile b/Makefile index 34d3e197..70cddaef 100644 --- a/Makefile +++ b/Makefile @@ -20,11 +20,14 @@ ISOLATED_NETWORK ?= 0 # Internal variables SHELL := /bin/bash -BUILDER_IMAGE := microshift-okd-builder +BUILDER_IMAGE ?= rpm-local-builder USHIFT_IMAGE := microshift-okd LVM_DISK := /var/lib/microshift-okd/lvmdisk.image VG_NAME := myvg1 +PROJECT_DIR := $(shell dirname $(abspath $(lastword $(MAKEFILE_LIST)))) +include $(PROJECT_DIR)/src/copr/copr.mk + # # Define the main targets # @@ -47,16 +50,23 @@ all: @echo " run-status: show the status of the MicroShift cluster" @echo " clean-all: perform a full cleanup, including the container images" @echo "" + @echo "COPR related targets:" + @echo " copr-rpm: build the MicroShift RPMs using COPR build service" + @echo " copr-delete-builds: delete the COPR builds using the COPR_BUILDS environment variable" + @echo " copr-regenerate-repos: regenerate the COPR repository" + @echo " copr-cfg-ensure-podman-secret: create podman secret from COPR_CONFIG" + @echo " copr-cli: build the COPR CLI container image used by copr-delete-builds and copr-regenerate-repos" + @echo "" .PHONY: rpm rpm: - @echo "Building the MicroShift builder image" + @echo "Building the MicroShift RPMs" sudo podman build \ -t "${BUILDER_IMAGE}" \ --ulimit nofile=524288:524288 \ --build-arg USHIFT_BRANCH="${USHIFT_BRANCH}" \ --build-arg OKD_VERSION_TAG="${OKD_VERSION_TAG}" \ - -f packaging/microshift-builder.Containerfile . + -f packaging/rpm-local-builder.Containerfile . @echo "Extracting the MicroShift RPMs" outdir="$${RPM_OUTDIR:-$$(mktemp -d /tmp/microshift-rpms-XXXXXX)}" && \ @@ -80,8 +90,8 @@ rpm-to-deb: .PHONY: image image: - @if ! sudo podman image exists microshift-okd-builder ; then \ - echo "ERROR: Run 'make rpm' to build the MicroShift RPMs" ; \ + @if ! sudo podman image exists "${BUILDER_IMAGE}" ; then \ + echo "ERROR: Run 'make rpm' or 'make copr-rpm' to build the MicroShift RPMs" ; \ exit 1 ; \ fi @@ -93,11 +103,12 @@ image: --label okd.version="${OKD_VERSION_TAG}" \ --build-arg BOOTC_IMAGE_URL="${BOOTC_IMAGE_URL}" \ --build-arg BOOTC_IMAGE_TAG="${BOOTC_IMAGE_TAG}" \ + --build-arg RPM_BUILDER_IMAGE="${BUILDER_IMAGE}" \ --env WITH_KINDNET="${WITH_KINDNET}" \ --env WITH_TOPOLVM="${WITH_TOPOLVM}" \ --env WITH_OLM="${WITH_OLM}" \ --env EMBED_CONTAINER_IMAGES="${EMBED_CONTAINER_IMAGES}" \ - -f packaging/microshift-runner.Containerfile . + -f packaging/bootc.Containerfile . # Notes: # - An isolated network is created if the ISOLATED_NETWORK environment variable is set @@ -157,6 +168,7 @@ clean-all: $(MAKE) clean sudo podman rmi -f "${USHIFT_IMAGE}" || true sudo podman rmi -f "${BUILDER_IMAGE}" || true + sudo podman rmi -f "${COPR_BUILDER_IMAGE}" || true .PHONY: check check: _hadolint _shellcheck diff --git a/docs/run.md b/docs/run.md index 3a9a3ae3..b8d576e6 100644 --- a/docs/run.md +++ b/docs/run.md @@ -6,25 +6,12 @@ This document describes how to run MicroShift on the host. See [MicroShift Bootc Deployment](./run-bootc.md) on how to run MicroShift inside a Bootc container. -## MicroShift RPM Packages +## MicroShift - optional packages -### Install RPM - -Run the following command to install MicroShift RPM packages from the local -repository copied from the build container image. -See [Create RPM Packages](../docs/build.md#create-rpm-packages) for more information. - -```bash -RPM_REPO_DIR=/tmp/microshift-rpms - -sudo ./src/rpm/create_repos.sh -create "${RPM_REPO_DIR}" -sudo dnf install -y microshift microshift-kindnet -sudo ./src/rpm/create_repos.sh -delete -``` - -The following optional RPM packages are available in the repository. It is +The following optional RPM packages are available. It is mandatory to install either `microshift-kindnet` or `microshift-networking` to enable the Kindnet or OVN-K networking support. +MicroShift with OVN-K CNI is not supported for Ubuntu. | Package | Description | Comments | |-----------------------|----------------------------|----------| @@ -33,28 +20,45 @@ to enable the Kindnet or OVN-K networking support. | microshift-topolvm | TopoLVM CSI | Install to enable storage support | | microshift-olm | Operator Lifecycle Manager | See [Operator Hub Catalogs](https://okd.io/docs/operators/) | -### Start MicroShift Service +## Package based systems (non-bootc) + +### Install MicroShift -Run the following commands to configure the minimum required firewall rules, -disable LVMS, and start the MicroShift service. +#### Local RPMs + +Run the following command to install MicroShift RPM packages from the local +repository copied from the build container image. +See [Create RPM Packages](../docs/build.md#create-rpm-packages) for more information. ```bash -sudo ./src/rpm/postinstall.sh -sudo systemctl start microshift.service +RPM_REPO_DIR=/tmp/microshift-rpms + +sudo ./src/rpm/create_repos.sh -create "${RPM_REPO_DIR}" +sudo dnf install -y microshift microshift-kindnet +sudo ./src/rpm/create_repos.sh -delete ``` -Verify that all the MicroShift pods are up and running successfully. +#### RPMs from COPR -```bash -mkdir -p ~/.kube -sudo cat /var/lib/microshift/resources/kubeadmin/kubeconfig > ~/.kube/config +Run following command to enable COPR repository: +```sh +sudo dnf copr enable $COPR_REPO_NAME +``` -oc get pods -A +Optionally specify chroot like `centos-stream-9-{x86_64,aarch64}`, `fedora-42-{x86_64,aarch64}`, for example: +```sh +sudo dnf copr enable $COPR_REPO_NAME centos-stream-9-x86_64 +sudo dnf copr enable $COPR_REPO_NAME centos-stream-9-aarch64 +sudo dnf copr enable $COPR_REPO_NAME fedora-42-x86_64 +sudo dnf copr enable $COPR_REPO_NAME fedora-42-aarch64 ``` -## MicroShift DEB Packages +Next, install MicroShift: +```sh +sudo dnf install -y microshift microshift-kindnet +``` -### Install DEB +#### Local DEB (Ubuntu) Run the following command to install MicroShift DEB packages from the local repository copied from the build container image. @@ -65,19 +69,17 @@ DEB_REPO_DIR=/tmp/microshift-rpms/deb sudo ./src/deb/install.sh "${DEB_REPO_DIR}" ``` -The following optional DEB packages are available in the repository. - -| Package | Description | Comments | -|--------------------|----------------------------|----------| -| microshift-topolvm | TopoLVM CSI | Install to enable storage support | -| microshift-olm | Operator Lifecycle Manager | See [Operator Hub Catalogs](https://okd.io/docs/operators/) | +### Start MicroShift Service -> Note: All of the optional packages are installed by default. +On RPM-based systems, run the following commands to configure the minimum +required firewall rules, disable LVMS, and enable the MicroShift service. +Skip this command on Ubuntu. -### Start MicroShift Service +```bash +sudo ./src/rpm/postinstall.sh +``` -Run the following command to start the MicroShift service. All the necessary system -configuration was performed during the installation step. +Run the following command to start the MicroShift service. ```bash sudo systemctl start microshift.service diff --git a/packaging/microshift-runner.Containerfile b/packaging/bootc.Containerfile similarity index 96% rename from packaging/microshift-runner.Containerfile rename to packaging/bootc.Containerfile index f5c5b3a3..248b0d06 100644 --- a/packaging/microshift-runner.Containerfile +++ b/packaging/bootc.Containerfile @@ -2,7 +2,9 @@ ARG BOOTC_IMAGE_URL=quay.io/centos-bootc/centos-bootc ARG BOOTC_IMAGE_TAG=stream9 -FROM localhost/microshift-okd-builder:latest AS builder +ARG RPM_BUILDER_IMAGE=microshift-okd-builder + +FROM localhost/${RPM_BUILDER_IMAGE}:latest AS builder FROM ${BOOTC_IMAGE_URL}:${BOOTC_IMAGE_TAG} ARG REPO_CONFIG_SCRIPT=/tmp/create_repos.sh diff --git a/packaging/rpm-copr-builder.Containerfile b/packaging/rpm-copr-builder.Containerfile new file mode 100644 index 00000000..d0e5ac89 --- /dev/null +++ b/packaging/rpm-copr-builder.Containerfile @@ -0,0 +1,73 @@ +FROM quay.io/fedora/fedora:42 + +RUN dnf install -y \ + --setopt=install_weak_deps=False \ + git rpm-build jq python3-pip copr-cli python3-specfile createrepo && \ + dnf clean all + +# Variables controlling the source of MicroShift components to build +ARG USHIFT_BRANCH=main +ARG OKD_VERSION_TAG + +# Internal variables +ARG OKD_REPO=quay.io/okd/scos-release +ARG USHIFT_GIT_URL=https://github.com/openshift/microshift.git +ENV HOME=/home/microshift +ARG BUILDER_RPM_REPO_PATH=${HOME}/microshift/_output/rpmbuild/RPMS +ARG USHIFT_PREBUILD_SCRIPT=/tmp/prebuild.sh +ARG USHIFT_POSTBUILD_SCRIPT=/tmp/postbuild.sh +ARG USHIFT_BUILDRPMS_SCRIPT=/tmp/build-rpms.sh +ARG USHIFT_MODIFY_SPEC_SCRIPT=/tmp/modify-spec.py + +ENV COPR_REPO_NAME=pmtk0/test123 + +# Verify mandatory build arguments +RUN if [ -z "${OKD_VERSION_TAG}" ]; then \ + echo "ERROR: OKD_VERSION_TAG is not set"; \ + echo "See quay.io/okd/scos-release for a list of tags"; \ + exit 1; \ + fi + +RUN ARCH="" ; if [ "$(uname -m)" = "aarch64" ]; then ARCH="-arm64"; fi && \ + OKD_CLIENT_URL=https://github.com/okd-project/okd/releases/download/${OKD_VERSION_TAG}/openshift-client-linux${ARCH}-${OKD_VERSION_TAG}.tar.gz && \ + echo "OKD_CLIENT_URL: ${OKD_CLIENT_URL}" && \ + curl -L -o /tmp/okd-client.tar.gz "${OKD_CLIENT_URL}" && \ + tar -xzf /tmp/okd-client.tar.gz -C /tmp && \ + mv /tmp/oc /usr/local/bin/oc && \ + rm -rf /tmp/okd-client.tar.gz ; + +WORKDIR ${HOME} + +RUN git clone --branch "${USHIFT_BRANCH}" --single-branch "${USHIFT_GIT_URL}" "${HOME}/microshift" + +# Preparing the build scripts +COPY --chmod=755 ./src/image/prebuild.sh ${USHIFT_PREBUILD_SCRIPT} +RUN "${USHIFT_PREBUILD_SCRIPT}" --replace "${OKD_REPO}" "${OKD_VERSION_TAG}" + +COPY --chmod=755 ./src/image/build-rpms.sh ${USHIFT_BUILDRPMS_SCRIPT} +COPY --chmod=755 ./src/image/modify-spec.py ${USHIFT_MODIFY_SPEC_SCRIPT} + +WORKDIR ${HOME}/microshift/ +RUN sed -i -e 's,CHECK_RPMS="y",,g' -e 's,CHECK_SRPMS="y",,g' ./packaging/rpm/make-rpm.sh && \ + python3 ${USHIFT_MODIFY_SPEC_SCRIPT} && \ + "${USHIFT_BUILDRPMS_SCRIPT}" srpm + +# Building Kindnet upstream RPM +COPY ./src/kindnet/kindnet.spec ./packaging/rpm/microshift.spec +COPY ./src/kindnet/assets/ ./assets/optional/ +COPY ./src/kindnet/dropins/ ./packaging/kindnet/ +COPY ./src/kindnet/crio.conf.d/ ./packaging/crio.conf.d/ +# Prepare and build Kindnet upstream RPM +RUN "${USHIFT_PREBUILD_SCRIPT}" --replace-kindnet "${OKD_REPO}" "${OKD_VERSION_TAG}" && \ + "${USHIFT_BUILDRPMS_SCRIPT}" srpm + +# Building TopoLVM upstream RPM +COPY ./src/topolvm/topolvm.spec ./packaging/rpm/microshift.spec +COPY ./src/topolvm/assets/ ./assets/optional/topolvm/ +COPY ./src/topolvm/dropins/ ./packaging/microshift/dropins/ +COPY ./src/topolvm/greenboot/ ./packaging/greenboot/ +COPY ./src/topolvm/release/ ./assets/optional/topolvm/ +RUN "${USHIFT_BUILDRPMS_SCRIPT}" srpm + +COPY ./src/copr/create-builds-and-wait.sh /tmp/create-builds-and-wait.sh +RUN --mount=type=secret,id=copr-cfg bash /tmp/create-builds-and-wait.sh diff --git a/packaging/microshift-builder.Containerfile b/packaging/rpm-local-builder.Containerfile similarity index 60% rename from packaging/microshift-builder.Containerfile rename to packaging/rpm-local-builder.Containerfile index 21a338b6..9d961e86 100644 --- a/packaging/microshift-builder.Containerfile +++ b/packaging/rpm-local-builder.Containerfile @@ -12,6 +12,7 @@ ENV HOME=/home/microshift ARG BUILDER_RPM_REPO_PATH=${HOME}/microshift/_output/rpmbuild/RPMS ARG USHIFT_PREBUILD_SCRIPT=/tmp/prebuild.sh ARG USHIFT_POSTBUILD_SCRIPT=/tmp/postbuild.sh +ARG USHIFT_BUILDRPMS_SCRIPT=/tmp/build-rpms.sh # Verify mandatory build arguments RUN if [ -z "${OKD_VERSION_TAG}" ]; then \ @@ -40,26 +41,31 @@ RUN git clone --branch "${USHIFT_BRANCH}" --single-branch "${USHIFT_GIT_URL}" "$ COPY --chmod=755 ./src/image/prebuild.sh ${USHIFT_PREBUILD_SCRIPT} RUN "${USHIFT_PREBUILD_SCRIPT}" --replace "${OKD_REPO}" "${OKD_VERSION_TAG}" -# Building all MicroShift downstream RPMs and SRPMs -# hadolint ignore=DL3059 -RUN MICROSHIFT_VARIANT="community" make -C "${HOME}/microshift" rpm srpm +COPY --chmod=755 ./src/image/build-rpms.sh ${USHIFT_BUILDRPMS_SCRIPT} +COPY --chmod=755 ./src/image/modify-spec.py ${USHIFT_MODIFY_SPEC_SCRIPT} + +WORKDIR ${HOME}/microshift/ +# Building MicroShift downstream RPMs and SRPMs +RUN sed -i -e 's,CHECK_RPMS="y",,g' -e 's,CHECK_SRPMS="y",,g' ./packaging/rpm/make-rpm.sh && \ + python3 ${USHIFT_MODIFY_SPEC_SCRIPT} && \ + "${USHIFT_BUILDRPMS_SCRIPT}" # Building Kindnet upstream RPM -COPY --chown=${USER}:${USER} ./src/kindnet/kindnet.spec "${HOME}/microshift/packaging/rpm/microshift.spec" -COPY --chown=${USER}:${USER} ./src/kindnet/assets/ "${HOME}/microshift/assets/optional/" -COPY --chown=${USER}:${USER} ./src/kindnet/dropins/ "${HOME}/microshift/packaging/kindnet/" -COPY --chown=${USER}:${USER} ./src/kindnet/crio.conf.d/ "${HOME}/microshift/packaging/crio.conf.d/" +COPY --chown=${USER}:${USER} ./src/kindnet/kindnet.spec "./packaging/rpm/microshift.spec" +COPY --chown=${USER}:${USER} ./src/kindnet/assets/ "./assets/optional/" +COPY --chown=${USER}:${USER} ./src/kindnet/dropins/ "./packaging/kindnet/" +COPY --chown=${USER}:${USER} ./src/kindnet/crio.conf.d/ "./packaging/crio.conf.d/" # Prepare and build Kindnet upstream RPM RUN "${USHIFT_PREBUILD_SCRIPT}" --replace-kindnet "${OKD_REPO}" "${OKD_VERSION_TAG}" && \ - MICROSHIFT_VARIANT="community" make -C "${HOME}/microshift" rpm + "${USHIFT_BUILDRPMS_SCRIPT}" # Building TopoLVM upstream RPM -COPY --chown=${USER}:${USER} ./src/topolvm/topolvm.spec "${HOME}/microshift/packaging/rpm/microshift.spec" -COPY --chown=${USER}:${USER} ./src/topolvm/assets/ "${HOME}/microshift/assets/optional/topolvm/" -COPY --chown=${USER}:${USER} ./src/topolvm/dropins/ "${HOME}/microshift/packaging/microshift/dropins/" -COPY --chown=${USER}:${USER} ./src/topolvm/greenboot/ "${HOME}/microshift/packaging/greenboot/" -COPY --chown=${USER}:${USER} ./src/topolvm/release/ "${HOME}/microshift/assets/optional/topolvm/" -RUN MICROSHIFT_VARIANT="community" make -C "${HOME}/microshift" rpm +COPY --chown=${USER}:${USER} ./src/topolvm/topolvm.spec "./packaging/rpm/microshift.spec" +COPY --chown=${USER}:${USER} ./src/topolvm/assets/ "./assets/optional/topolvm/" +COPY --chown=${USER}:${USER} ./src/topolvm/dropins/ "./packaging/microshift/dropins/" +COPY --chown=${USER}:${USER} ./src/topolvm/greenboot/ "./packaging/greenboot/" +COPY --chown=${USER}:${USER} ./src/topolvm/release/ "./assets/optional/topolvm/" +RUN "${USHIFT_BUILDRPMS_SCRIPT}" # Post-build MicroShift configuration COPY --chmod=755 ./src/image/postbuild.sh ${USHIFT_POSTBUILD_SCRIPT} diff --git a/src/copr/copr-cli.Containerfile b/src/copr/copr-cli.Containerfile new file mode 100644 index 00000000..6a00140e --- /dev/null +++ b/src/copr/copr-cli.Containerfile @@ -0,0 +1,3 @@ +FROM quay.io/fedora/fedora:42 + +RUN dnf install -y copr-cli && dnf clean all diff --git a/src/copr/copr.mk b/src/copr/copr.mk new file mode 100644 index 00000000..70231616 --- /dev/null +++ b/src/copr/copr.mk @@ -0,0 +1,62 @@ + +COPR_CONFIG ?= $(HOME)/.config/copr +COPR_BUILDS ?= +COPR_REPO_NAME ?= pmtk0/test123 + +COPR_SECRET_NAME := copr-cfg +COPR_BUILDER_IMAGE := rpm-copr-builder +COPR_CLI_IMAGE := localhost/copr-cli:latest + +.PHONY: copr-rpm +copr-rpm: + @echo "Building the MicroShift RPMs using the COPR build service" + sudo podman build \ + --tag "${COPR_BUILDER_IMAGE}" \ + --secret id=${COPR_SECRET_NAME},src=${COPR_CONFIG} \ + --ulimit nofile=524288:524288 \ + --build-arg USHIFT_BRANCH="${USHIFT_BRANCH}" \ + --build-arg OKD_VERSION_TAG="${OKD_VERSION_TAG}" \ + --env COPR_REPO_NAME="${COPR_REPO_NAME}" \ + --file packaging/rpm-copr-builder.Containerfile . + + @echo "Extracting the MicroShift RPMs" + outdir="$${RPM_OUTDIR:-$$(mktemp -d /tmp/microshift-rpms-XXXXXX)}" && \ + mntdir="$$(sudo podman image mount "${COPR_BUILDER_IMAGE}")" && \ + sudo cp -r "$${mntdir}/home/microshift/microshift/_output/rpmbuild/RPMS/." "$${outdir}" && \ + sudo podman image umount "${COPR_BUILDER_IMAGE}" && \ + echo "" && \ + echo "Build completed successfully" && \ + echo "RPMs are available in '$${outdir}'" + +.PHONY: copr-cfg-ensure-podman-secret +copr-cfg-ensure-podman-secret: + @echo "Ensuring the COPR secret is available and is up to date" + if sudo podman secret exists "${COPR_SECRET_NAME}"; then \ + sudo podman secret rm "${COPR_SECRET_NAME}" ; \ + fi && \ + sudo podman secret create "${COPR_SECRET_NAME}" "${COPR_CONFIG}" + +.PHONY: copr-cli +copr-cli: + @echo "Building the COPR CLI container" + sudo podman build \ + --tag "${COPR_CLI_IMAGE}" \ + --file src/copr/copr-cli.Containerfile . + +.PHONY: copr-delete-builds +copr-delete-builds: copr-cfg-ensure-podman-secret copr-cli + @echo "Deleting the COPR builds" + sudo podman run \ + --rm \ + --secret ${COPR_SECRET_NAME} \ + "${COPR_CLI_IMAGE}" \ + bash -c "copr-cli --config /run/secrets/copr-cfg delete-build ${COPR_BUILDS}" + +.PHONY: copr-regenerate-repos +copr-regenerate-repos: copr-cfg-ensure-podman-secret copr-cli + @echo "Regenerating the COPR repository" + sudo podman run \ + --rm \ + --secret ${COPR_SECRET_NAME} \ + "${COPR_CLI_IMAGE}" \ + bash -c "copr-cli --config /run/secrets/copr-cfg regenerate-repos ${COPR_REPO_NAME}" diff --git a/src/copr/create-builds-and-wait.sh b/src/copr/create-builds-and-wait.sh new file mode 100644 index 00000000..9ca7cc23 --- /dev/null +++ b/src/copr/create-builds-and-wait.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +set -euxo pipefail + +SRPMS="${HOME}/microshift/_output/rpmbuild/SRPMS" + +out="$(copr-cli --config /run/secrets/copr-cfg build --nowait "${COPR_REPO_NAME}" "${SRPMS}"/microshift*.src.rpm)" +echo "${out}" + +builds=$(echo "${out}" | grep "Created builds" | cut -d: -f2 | xargs) +# shellcheck disable=SC2086 +copr-cli watch-build ${builds} +mkdir -p ./rpms +# shellcheck disable=SC2086 +for b in $builds ; do + copr download-build --rpms --chroot centos-stream-9-x86_64 --dest ./rpms "${b}"; +done + +mkdir -p "${HOME}/microshift/_output/rpmbuild/RPMS/" +echo "${builds}" > "${HOME}/microshift/_output/rpmbuild/RPMS/builds.txt" +cp "${HOME}/microshift/_output/rpmbuild/version.txt" "${HOME}/microshift/_output/rpmbuild/RPMS/version.txt" + +cp -v ./rpms/centos-stream-9-x86_64/*.rpm "${HOME}/microshift/_output/rpmbuild/RPMS/" +createrepo -v "${HOME}/microshift/_output/rpmbuild/RPMS/" diff --git a/src/image/build-rpms.sh b/src/image/build-rpms.sh new file mode 100755 index 00000000..1d99e81e --- /dev/null +++ b/src/image/build-rpms.sh @@ -0,0 +1,60 @@ +#!/usr/bin/env bash +set -xeuo pipefail + +# Primary purpose of this script is to build the MicroShift RPMs and SRPMs +# with adjusted version string. +# Using `make {rpm,srpm}` it would hardcode a meaningless version string based on downstream git variables +# which would not provide good information to identify the build and trace back its contents. +# Following script overrides the version to include information about the downstream version and commit, and OKD tag. + +target=all +if [[ "$#" -eq 1 ]]; then + if [[ "${1}" != "all" && "${1}" != "rpm" && "${1}" != "srpm" ]]; then + echo "Script accepts at most one argument: all, rpm or srpm" + echo "If no argument is provided, the default is 'all'" + exit 1 + fi + target="${1}" +fi + +cd "${HOME}/microshift" + +SOURCE_GIT_COMMIT="$(git rev-parse --short 'HEAD^{commit}')" + +# MICROSHIFT_VERSION must start with X.Y.Z for the internals to correctly parse the version. +# If USHIFT_BRANCH is a tag, use it. Otherwise parse the version from Makefile.version.*.var file. +if [[ $(git tag -l "${USHIFT_BRANCH}") ]]; then + MICROSHIFT_VERSION="${USHIFT_BRANCH}" +else + MICROSHIFT_VERSION="$(awk -F'[=.-]' '{print $2 "." $3 "." $4}' Makefile.version.aarch64.var | sed -e 's/ //g')" +fi +# Example results: +# - 4.21.0_ga9cd00b34_4.21.0_okd_scos.ec.5 for build against HEAD of main which was 4.21 at the time. +# - 4.20.0-202510201126.p0-g1c4675ace_4.20.0-okd-scos.6 for build against a specific tag. +MICROSHIFT_VERSION="${MICROSHIFT_VERSION}-g${SOURCE_GIT_COMMIT}-${OKD_VERSION_TAG}" +# MicroShift's make-rpm.sh makes this substitution. Although we don't use the script, +# let's do it as well for keeping the version consistent with existing downstream RPMs. +# Version is also used for release.md file. +MICROSHIFT_VERSION=${MICROSHIFT_VERSION//-/_} + +RPM_RELEASE="1" +SOURCE_GIT_TAG="$(git describe --long --tags --abbrev=7 --match 'v[0-9]*' || echo "v0.0.0-unknown-${SOURCE_GIT_COMMIT}")" +SOURCE_GIT_TREE_STATE=clean # Because we're updating downstream specfile, but that shouldn't be a reason to have -dirty suffix. +MICROSHIFT_VARIANT=community + +export MICROSHIFT_VERSION +export RPM_RELEASE +export SOURCE_GIT_TAG +export SOURCE_GIT_COMMIT +export SOURCE_GIT_TREE_STATE +export MICROSHIFT_VARIANT + +if [[ "${target}" == "all" || "${target}" == "rpm" ]]; then + ./packaging/rpm/make-rpm.sh rpm local +fi + +if [[ "${target}" == "all" || "${target}" == "srpm" ]]; then + ./packaging/rpm/make-rpm.sh srpm local +fi + +echo "${MICROSHIFT_VERSION}" > "${HOME}/microshift/_output/rpmbuild/version.txt" diff --git a/src/image/modify-spec.py b/src/image/modify-spec.py new file mode 100644 index 00000000..e5213ed2 --- /dev/null +++ b/src/image/modify-spec.py @@ -0,0 +1,69 @@ +#!/usr/bin/env python + +import specfile +from itertools import product + +pkgs_to_remove = [ + 'multus', + 'low-latency', + 'gateway-api', + 'ai-model-serving', + 'cert-manager', + 'observability', +] + +sections_to_remove = [ + 'package', + 'description', + 'files', + 'preun', + 'post', +] + +full_sections_to_remove = [ f"{p[0]} {p[1]}{p[2]}" for p in product(sections_to_remove, pkgs_to_remove, ["", "-release-info"]) ] + +install_keywords_to_remove = [ + 'multus', + 'low-latency', + 'lib/tuned', + '05-high-performance-runtime.conf', + 'microshift-baseline', + 'microshift-tuned', + 'gateway-api', + 'ai-model-serving', + 'cert-manager', + 'observability', +] + +s = specfile.Specfile('./packaging/rpm/microshift.spec', macros=[('release', '1'), ('version', '4.0.0'), ('commit', 'x'), ('embedded_git_tag', 'tag'), ('embedded_git_tree_state', 'clean')]) + +with s.sections() as sections: + for id in full_sections_to_remove: + try: + sec = sections.get(id) + sections.remove(sec) + print(f"Removing section: '%{id}'") + except ValueError as e: + pass + + i = sections.install + new_install = [] + nl_present = False + for line in i: + if any(substring in line for substring in install_keywords_to_remove): + print(f"Removing line: '{line}'") + else: + if line == "": + # Skip extraneous newlines for aesthetic reasons + if nl_present: + continue + else: + nl_present = True + new_install.append(line) + else: + nl_present = False + new_install.append(line) + i.clear() + i.extend(new_install) + +s.save() \ No newline at end of file diff --git a/src/image/postbuild.sh b/src/image/postbuild.sh index d75324cb..64940a7d 100644 --- a/src/image/postbuild.sh +++ b/src/image/postbuild.sh @@ -5,15 +5,6 @@ set -x # Variables BUILDER_RPM_REPO_PATH="$1" -# Delete unsupported RPMs, which are built unconditionally. -# To add support for an RPM, undo the file removal and add a presubmit test for it. -rm -f "${BUILDER_RPM_REPO_PATH}"/*/microshift-ai-model-serving*.rpm -rm -f "${BUILDER_RPM_REPO_PATH}"/*/microshift-cert-manager*.rpm -rm -f "${BUILDER_RPM_REPO_PATH}"/*/microshift-gateway-api*.rpm -rm -f "${BUILDER_RPM_REPO_PATH}"/*/microshift-low-latency*.rpm -rm -f "${BUILDER_RPM_REPO_PATH}"/*/microshift-multus*.rpm -rm -f "${BUILDER_RPM_REPO_PATH}"/*/microshift-observability*.rpm - # Create a local RPM repository and add SRPMs on top of it mkdir -p "${BUILDER_RPM_REPO_PATH}/srpms" createrepo -v "${BUILDER_RPM_REPO_PATH}" diff --git a/src/image/prebuild.sh b/src/image/prebuild.sh index 6aa25a55..b84a1ae1 100755 --- a/src/image/prebuild.sh +++ b/src/image/prebuild.sh @@ -20,11 +20,14 @@ replace_base_assets() { local -r arch=$(uname -m) local -r temp_json=$(mktemp "/tmp/release-${arch}.XXXXX.json") - # replace Microshift images with upstream (from OKD release) + # Replace Microshift images with upstream (from OKD release) + local -r okd_json=$(mktemp "/tmp/okd-release.XXXXX.json") + oc adm release info "${okd_url}:${okd_releaseTag}" -o json > "${okd_json}" + for op in $(jq -e -r '.images | keys []' "${MICROSHIFT_ROOT}/assets/release/release-${arch}.json") do local image - image=$(oc adm release info --image-for="${op}" "${okd_url}:${okd_releaseTag}" || true) + image=$(jq -r --arg NAME "${op}" '.references.spec.tags[] | select(.name == $NAME) | .from.name' "${okd_json}" || true) if [ -n "${image}" ] ; then echo "${op} ${image}" jq --arg a "${op}" --arg b "${image}" '.images[$a] = $b' "${MICROSHIFT_ROOT}/assets/release/release-${arch}.json" >"${temp_json}" @@ -32,8 +35,8 @@ replace_base_assets() { fi done - pod_image=$(oc adm release info --image-for=pod "${okd_url}:${okd_releaseTag}" || true) - # update the infra pods for crio + pod_image=$(jq -r --arg NAME pod '.references.spec.tags[] | select(.name == $NAME) | .from.name' "${okd_json}" || true) + # Update the infra pods for crio sed -i 's,pause_image .*,pause_image = '"\"${pod_image}\""',' "${MICROSHIFT_ROOT}/packaging/crio.conf.d/10-microshift_${UNAME_TO_GOARCH_MAP[${arch}]}.conf" } @@ -61,6 +64,9 @@ replace_olm_assets() { images: EOF + local -r okd_json=$(mktemp "/tmp/okd-release.XXXXX.json") + oc adm release info "${okd_url}:${okd_releaseTag}" -o json > "${okd_json}" + # Read from the image-references file to find the images we need to update local -r containers=$("${MICROSHIFT_ROOT}"/_output/bin/yq -r '.spec.tags[].name' "${olm_image_refs_file}") for container in "${containers[@]}" ; do @@ -70,7 +76,7 @@ EOF # Get the new image from OKD release local new_image - new_image=$(oc adm release info --image-for="${container}" "${okd_url}:${okd_releaseTag}" || true) + new_image=$(jq -r --arg NAME "${op}" '.references.spec.tags[] | select(.name == $NAME) | .from.name' "${okd_json}" || true) if [ -n "${new_image}" ] ; then echo "${container} ${new_image}" diff --git a/src/kindnet/kindnet.spec b/src/kindnet/kindnet.spec index ccda8646..79943352 100644 --- a/src/kindnet/kindnet.spec +++ b/src/kindnet/kindnet.spec @@ -1,48 +1,35 @@ -# -# Beginning of the header copied from microshift/packaging/rpm/microshift.spec -# %global shortcommit %(c=%{commit}; echo ${c:0:7}) # Debug info not supported with Go %global debug_package %{nil} -Name: microshift +Name: microshift-kindnet Version: %{version} Release: %{release}%{dist} -Summary: MicroShift service +Summary: kindnet CNI for MicroShift License: ASL 2.0 URL: https://github.com/openshift/microshift Source0: https://github.com/openshift/microshift/archive/%{commit}/microshift-%{shortcommit}.tar.gz -ExclusiveArch: x86_64 aarch64 - -%description -The microshift package provides an OpenShift Kubernetes distribution optimized for small form factor and edge computing. - -%prep -%setup -n microshift-%{commit} -# -# End of the header copied from microshift/packaging/rpm/microshift.spec -# - -%package kindnet -Summary: kindnet CNI for MicroShift ExclusiveArch: x86_64 aarch64 Requires: microshift = %{version} -%description kindnet +%description The microshift-kindnet package provides the required manifests for the kindnet CNI and the dependent kube-proxy to be installed on MicroShift. -%package kindnet-release-info +%package release-info Summary: Release information for kindnet CNI for MicroShift BuildArch: noarch Requires: microshift-release-info = %{version} -%description kindnet-release-info +%description release-info The microshift-kindnet-release-info package provides release information files for this release. These files contain the list of container image references used by the kindnet CNI with the dependent kube-proxy for MicroShift. +%prep +%setup -n microshift-%{commit} + %install install -d -m755 %{buildroot}/%{_sysconfdir}/microshift/config.d install -d -m755 %{buildroot}/%{_sysconfdir}/microshift/manifests.d @@ -85,7 +72,7 @@ mkdir -p -m755 %{buildroot}%{_datadir}/microshift/release install -p -m644 assets/optional/kindnet/release-kindnet-{x86_64,aarch64}.json %{buildroot}%{_datadir}/microshift/release/ install -p -m644 assets/optional/kube-proxy/release-kube-proxy-{x86_64,aarch64}.json %{buildroot}%{_datadir}/microshift/release/ -%files kindnet +%files %dir %{_prefix}/lib/microshift/manifests.d/000-microshift-kindnet %dir %{_prefix}/lib/microshift/manifests.d/000-microshift-kube-proxy %{_prefix}/lib/microshift/manifests.d/000-microshift-kindnet/* @@ -94,6 +81,6 @@ install -p -m644 assets/optional/kube-proxy/release-kube-proxy-{x86_64,aarch64}. %{_sysconfdir}/systemd/system/microshift.service %{_sysconfdir}/crio/crio.conf.d/13-microshift-kindnet.conf -%files kindnet-release-info +%files release-info %{_datadir}/microshift/release/release-kindnet-{x86_64,aarch64}.json %{_datadir}/microshift/release/release-kube-proxy-{x86_64,aarch64}.json diff --git a/src/topolvm/topolvm.spec b/src/topolvm/topolvm.spec index 8f6cfe97..c79bf538 100644 --- a/src/topolvm/topolvm.spec +++ b/src/topolvm/topolvm.spec @@ -1,56 +1,42 @@ -# -# Beginning of the header copied from microshift/packaging/rpm/microshift.spec -# %global shortcommit %(c=%{commit}; echo ${c:0:7}) # Debug info not supported with Go %global debug_package %{nil} -Name: microshift +Name: microshift-topolvm Version: %{version} Release: %{release}%{dist} -Summary: MicroShift service +Summary: TopoLVM CSI Plugin for MicroShift License: ASL 2.0 URL: https://github.com/openshift/microshift Source0: https://github.com/openshift/microshift/archive/%{commit}/microshift-%{shortcommit}.tar.gz - -ExclusiveArch: x86_64 aarch64 - -%description -The microshift package provides an OpenShift Kubernetes distribution optimized for small form factor and edge computing. - -%prep -%setup -n microshift-%{commit} -# -# End of the header copied from microshift/packaging/rpm/microshift.spec -# - -%package topolvm -Summary: TopoLVM CSI Plugin for MicroShift ExclusiveArch: x86_64 aarch64 Requires: microshift = %{version} -%description topolvm +%description The microshift-topolvm package provides the required manifests for the TopoLVM CSI and the dependent cert-manager to be installed on MicroShift. -%files topolvm +%files %dir %{_prefix}/lib/microshift/manifests.d/001-microshift-topolvm %{_prefix}/lib/microshift/manifests.d/001-microshift-topolvm/* %{_sysconfdir}/greenboot/check/required.d/50_microshift_topolvm_check.sh %config(noreplace) %{_sysconfdir}/microshift/config.d/01-disable-storage-csi.yaml -%package topolvm-release-info +%package release-info Summary: Release information for TopoLVM components for MicroShift BuildArch: noarch Requires: microshift-release-info = %{version} -%description topolvm-release-info +%description release-info The microshift-topolvm-release-info package provides release information files for this release. These files contain the list of container image references used by the TopoLVM CSI. -%files topolvm-release-info +%files release-info %{_datadir}/microshift/release/release-topolvm-{x86_64,aarch64}.json +%prep +%setup -n microshift-%{commit} + %install install -d -m755 %{buildroot}/%{_prefix}/lib/microshift/manifests.d/001-microshift-topolvm install -p -m644 assets/optional/topolvm/*.yaml %{buildroot}/%{_prefix}/lib/microshift/manifests.d/001-microshift-topolvm