diff --git a/Containerfile b/Containerfile index 9f84eeff5..9d85d9f6a 100644 --- a/Containerfile +++ b/Containerfile @@ -1,15 +1,27 @@ # This builds the final OCP/OKD node image on top of the base CoreOS image. For # instructions on how to build this, see `docs/building.md`. -FROM quay.io/openshift-release-dev/ocp-v4.0-art-dev:c9s-coreos as build +ARG IMAGE_FROM=overridden +FROM ${IMAGE_FROM} as build ARG OPENSHIFT_CI=0 RUN --mount=type=bind,target=/run/src --mount=type=secret,id=yumrepos,target=/etc/yum.repos.d/secret.repo /run/src/build-node-image.sh FROM build as metadata +ARG IMAGE_NAME +ARG IMAGE_CPE +ARG TARGETARCH RUN --mount=type=bind,target=/run/src /run/src/scripts/generate-metadata +RUN --mount=type=bind,target=/run/src /run/src/scripts/generate-labels FROM build COPY --from=metadata /usr/share/openshift /usr/share/openshift +COPY --from=metadata /usr/share/buildinfo /usr/share/buildinfo +ARG IMAGE_NAME +ARG IMAGE_CPE +ARG TARGETARCH +LABEL name=${IMAGE_NAME} +LABEL cpe=${IMAGE_CPE} +LABEL architecture=${TARGETARCH} LABEL io.openshift.metalayer=true # Add a hack to get OpenShift tests working again because a # revert of the new architecture happened in diff --git a/build-args-10.2-4.22.conf b/build-args-10.2-4.22.conf new file mode 100644 index 000000000..33e509600 --- /dev/null +++ b/build-args-10.2-4.22.conf @@ -0,0 +1,3 @@ +IMAGE_FROM=registry.ci.openshift.org/coreos/rhel-coreos-base:10.2 +IMAGE_NAME=openshift/ose-rhel-coreos-10 +IMAGE_CPE=cpe:/a:redhat:openshift:4.22::el10 diff --git a/build-args-9.8-4.22.conf b/build-args-9.8-4.22.conf new file mode 100644 index 000000000..879f96823 --- /dev/null +++ b/build-args-9.8-4.22.conf @@ -0,0 +1,3 @@ +IMAGE_FROM=registry.ci.openshift.org/coreos/rhel-coreos-base:9.8 +IMAGE_NAME=openshift/ose-rhel-coreos-9 +IMAGE_CPE=cpe:/a:redhat:openshift:4.22::el9 diff --git a/build-args-c10s-4.22.conf b/build-args-c10s-4.22.conf new file mode 100644 index 000000000..1ae66b2b1 --- /dev/null +++ b/build-args-c10s-4.22.conf @@ -0,0 +1,2 @@ +IMAGE_FROM=registry.ci.openshift.org/coreos/stream-coreos-base:10 +# SCOS/OKD: no labels.json or OCI labels for name/cpe diff --git a/build-node-image.sh b/build-node-image.sh index 00a6a618b..d8249346b 100755 --- a/build-node-image.sh +++ b/build-node-image.sh @@ -23,8 +23,8 @@ source /etc/os-release # should come from CentOS. We should eventually sever this. if [ $ID = centos ]; then # this says: "if the line starts with [.*], turn off printing. if the line starts with [our-repo], turn it on." - awk "/\[.*\]/{p=0} /\[rhel-9.8-server-ose-4.22\]/{p=1} p" /etc/yum.repos.d/*.repo > /etc/yum.repos.d/okd.repo.tmp - sed -i -e 's,\[rhel-9.8-server-ose-4.22\],\[rhel-9.8-server-ose-4.22-okd\],' /etc/yum.repos.d/okd.repo.tmp + awk "/\[.*\]/{p=0} /\[rhel-9-server-ose\]/{p=1} p" /etc/yum.repos.d/*.repo > /etc/yum.repos.d/okd.repo.tmp + sed -i -e 's,\[rhel-9-server-ose\],\[rhel-9-server-ose-okd\],' /etc/yum.repos.d/okd.repo.tmp echo 'includepkgs=openshift-*,ose-aws-ecr-*,ose-azure-acr-*,ose-gcp-gcr-*,ose-crio-* ' >> /etc/yum.repos.d/okd.repo.tmp mv /etc/yum.repos.d/okd.repo{.tmp,} fi diff --git a/c9s-mirror.repo b/c9s-mirror.repo deleted file mode 100644 index 801dada3d..000000000 --- a/c9s-mirror.repo +++ /dev/null @@ -1,36 +0,0 @@ -# These are the official c9s repos. They are slower to update, but contain older -# versions of packages, which is useful when pinning for lack of a "coreos-pool" -# equivalent. When no pinning is needed you may find the compose repo URLs -# defined in c9s.repo are quicker to get new content. - -[c9s-baseos-mirror] -name=CentOS Stream 9 - BaseOS -baseurl=https://mirror.stream.centos.org/9-stream/BaseOS/$basearch/os -gpgcheck=1 -repo_gpgcheck=0 -enabled=1 -gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-centosofficial - -[c9s-appstream-mirror] -name=CentOS Stream 9 - AppStream -baseurl=https://mirror.stream.centos.org/9-stream/AppStream/$basearch/os -gpgcheck=1 -repo_gpgcheck=0 -enabled=1 -gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-centosofficial - -[c9s-nfv-mirror] -name=CentOS Stream 9 - NFV -baseurl=https://mirror.stream.centos.org/9-stream/NFV/$basearch/os -gpgcheck=1 -repo_gpgcheck=0 -enabled=1 -gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-centosofficial - -[c9s-rt-mirror] -name=CentOS Stream 9 - RT -baseurl=https://mirror.stream.centos.org/9-stream/RT/$basearch/os -gpgcheck=1 -repo_gpgcheck=0 -enabled=1 -gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-centosofficial diff --git a/c9s.repo b/c9s.repo deleted file mode 100644 index 9ccc6bd69..000000000 --- a/c9s.repo +++ /dev/null @@ -1,76 +0,0 @@ -# These are compose repo URLs that represent the latest composes in -# CentOS Stream 9. Sometimes these repos get content a little faster -# than the mirror repos defined in c9s-mirror.repo, but they won't -# have multiple versions of packages, which make them not ideal when -# needing to pin on older package versions. - -[c9s-baseos] -name=CentOS Stream 9 - BaseOS -baseurl=https://composes.stream.centos.org/production/latest-CentOS-Stream/compose/BaseOS/$basearch/os -gpgcheck=1 -repo_gpgcheck=0 -enabled=1 -gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-centosofficial - -[c9s-appstream] -name=CentOS Stream 9 - AppStream -baseurl=https://composes.stream.centos.org/production/latest-CentOS-Stream/compose/AppStream/$basearch/os -gpgcheck=1 -repo_gpgcheck=0 -enabled=1 -gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-centosofficial - -[c9s-nfv] -name=CentOS Stream 9 - NFV -baseurl=https://composes.stream.centos.org/production/latest-CentOS-Stream/compose/NFV/$basearch/os -gpgcheck=1 -repo_gpgcheck=0 -enabled=1 -gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-centosofficial - -[c9s-rt] -name=CentOS Stream 9 - RT -baseurl=https://composes.stream.centos.org/production/latest-CentOS-Stream/compose/RT/$basearch/os -gpgcheck=1 -repo_gpgcheck=0 -enabled=1 -gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-centosofficial - -[c9s-extras-common] -name=CentOS Stream 9 - Extras packages -baseurl=https://mirror.stream.centos.org/SIGs/9-stream/extras/$basearch/extras-common -gpgcheck=1 -repo_gpgcheck=0 -enabled=1 -gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-SIG-Extras-SHA512 - -# Note: We can't find a composes.stream.centos.org URL for this repo -# so we use the mirror.stream.centos.org URL here. -[c9s-sig-nfv] -name=CentOS Stream 9 - SIG NFV -baseurl=https://mirror.stream.centos.org/SIGs/9-stream/nfv/$basearch/openvswitch-2/ -gpgcheck=1 -repo_gpgcheck=0 -enabled=1 -gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-SIG-NFV - -# Note: We can't find a composes.stream.centos.org URL for this repo -# so we use the mirror.stream.centos.org URL here. -[c9s-sig-virtualization] -name=CentOS Stream 9 - SIG Virtualization -baseurl=https://mirror.stream.centos.org/SIGs/9-stream/virt/$basearch/kata-containers/ -gpgcheck=1 -repo_gpgcheck=0 -enabled=1 -gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-SIG-Virtualization - -# Note: We can't find a composes.stream.centos.org URL for this repo -# so we use the mirror.stream.centos.org URL here. -# This needs to be updated to okd-4.21 once it becomes available -[c9s-sig-cloud-okd] -name=CentOS Stream 9 - SIG Cloud OKD 4.20 -baseurl=https://mirror.stream.centos.org/SIGs/9-stream/cloud/$basearch/okd-4.20/ -gpgcheck=1 -repo_gpgcheck=0 -enabled=1 -gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-SIG-Cloud diff --git a/ci/get-ocp-repo.sh b/ci/get-ocp-repo.sh index 62a73a2d1..907fa9fb7 100755 --- a/ci/get-ocp-repo.sh +++ b/ci/get-ocp-repo.sh @@ -6,8 +6,9 @@ set -euo pipefail # content. urls=( - # theoretically that's the only one we need - "http://base-4-22-rhel98.ocp.svc.cluster.local" + # NB: we use rhel9 here for pre-release 9.8 content. switch back to 98 once + # it's GA and there are proper repos + "http://base-4-22-rhel9.ocp.svc.cluster.local" "http://base-4-22-rhel102.ocp.svc.cluster.local" ) diff --git a/docs/building.md b/docs/building.md index 953f07723..1508a3360 100644 --- a/docs/building.md +++ b/docs/building.md @@ -27,28 +27,29 @@ SCOS or RHCOS image (see building instructions in ## Building -If the base image is SCOS, then the OKD node image is built (`stream-coreos`). -If the base image is RHCOS, then the OCP node image is built (`rhel-coreos`). -The default base image is SCOS. +Each variant has a `build-args-*.conf` file that specifies the base image +and metadata for that build. Choose the appropriate one for your target: -To build SCOS: +- `build-args-9.8-4.22.conf` — RHCOS on RHEL 9.8 +- `build-args-10.2-4.22.conf` — RHCOS on RHEL 10.2 +- `build-args-c10s-4.22.conf` — SCOS on CentOS Stream 10 -``` -podman build . --secret id=yumrepos,src=/path/to/all.repo \ - -v /etc/pki/ca-trust:/etc/pki/ca-trust:ro \ - --security-opt label=disable -t localhost/stream-coreos:4.21 -``` - -To build RHCOS, the command is identical, but you must pass in the RHCOS base -image using `--from`: +To build: ``` -podman build --from quay.io/openshift-release-dev/ocp-v4.0-art-dev:rhel-9.6-coreos ... +podman build . --build-arg-file build-args-c10s-4.22.conf \ + --secret id=yumrepos,src=/path/to/all.repo \ + -v /etc/pki/ca-trust:/etc/pki/ca-trust:ro \ + --security-opt label=disable -t localhost/stream-coreos:4.22 ``` -To build from a local OCI archive (e.g. from a cosa workdir), you can use the -`oci-archive` transport: +To override the base image (e.g. to use a locally built OCI archive), +pass `--from`: ``` -podman build --from oci-archive:$(ls builds/latest/x86_64/*.ociarchive) ... +podman build . --build-arg-file build-args-c10s-4.22.conf \ + --from oci-archive:$(ls builds/latest/x86_64/*.ociarchive) \ + --secret id=yumrepos,src=/path/to/all.repo \ + -v /etc/pki/ca-trust:/etc/pki/ca-trust:ro \ + --security-opt label=disable -t localhost/stream-coreos:4.22 ``` diff --git a/extensions/build.sh b/extensions/build.sh index d201e69d5..b3ad82d95 100755 --- a/extensions/build.sh +++ b/extensions/build.sh @@ -10,6 +10,15 @@ fi # hack around this for now by deleting the problematic bits; we should tweak rpm-ostree instead jq 'del(.["check-passwd","check-groups"])' /usr/share/rpm-ostree/treefile.json > filtered.json +# The base image treefile references rhel-9.8-* repo names, but we use rhel-9-* +# repos in CI for pre-release content (see ci/get-ocp-repo.sh). Rewrite the repo +# names to match. rhel-9.8-early-kernel is excluded because it exists under the +# same name in both repo sets. +jq '.repos |= map(if startswith("rhel-9.8-") and . != "rhel-9.8-early-kernel" + then "rhel-9-" + ltrimstr("rhel-9.8-") + else . end)' \ + filtered.json > filtered.json.tmp && mv filtered.json{.tmp,} + . /etc/os-release rpm-ostree compose extensions filtered.json "extensions/${ID}-${VERSION_ID}.yaml" \ --rootfs=/ --output-dir=/usr/share/rpm-ostree/extensions/ diff --git a/extensions/rhel-10.2.yaml b/extensions/rhel-10.2.yaml index 864858952..b8160ac4c 100644 --- a/extensions/rhel-10.2.yaml +++ b/extensions/rhel-10.2.yaml @@ -14,7 +14,7 @@ repos: # Repo placed here to respect the rule above. # We do not have kata-container yet in 10.2 # https://github.com/openshift/os/issues/1911 - - rhel-9.8-server-ose-4.22 + - rhel-9-server-ose - rhel-10.2-server-ose-4.22 # For two-node-ha extension. # Repo placed here to respect the rule above. diff --git a/extensions/rhel-9.8.yaml b/extensions/rhel-9.8.yaml index e32a04edd..255a00877 100644 --- a/extensions/rhel-9.8.yaml +++ b/extensions/rhel-9.8.yaml @@ -9,16 +9,16 @@ repos: # Generically used for various extensions. # Repo placed here to respect the rule above. - - rhel-9.8-appstream + - rhel-9-appstream # For kata-containers (sandboxed-containers). # Repo placed here to respect the rule above. - - rhel-9.8-server-ose-4.22 + - rhel-9-server-ose # For two-node-ha extension. # Repo placed here to respect the rule above. - - rhel-9.8-highavailability + - rhel-9-highavailability # For ipsec extension include the fast-datapath repo. # Repo placed here to respect the rule above. - - rhel-9.8-fast-datapath + - rhel-9-fast-datapath extensions: # https://github.com/coreos/fedora-coreos-tracker/issues/1504 @@ -63,7 +63,7 @@ extensions: - x86_64 repos: # this is not available on all arches, so keep here and not in the global repo list - - rhel-9.8-nfv + - rhel-9-nfv packages: - kernel-rt-core - kernel-rt-modules diff --git a/packages-openshift.yaml b/packages-openshift.yaml index 7658205be..62688432f 100644 --- a/packages-openshift.yaml +++ b/packages-openshift.yaml @@ -8,18 +8,17 @@ conditional-include: - if: - osversion != "rhel-9.8" - osversion != "rhel-10.2" - - osversion != "centos-9" - osversion != "centos-10" include: repos: [ENOEXIST] # We want an error in this case - if: osversion == "rhel-9.8" include: repos: - - rhel-9.8-baseos - - rhel-9.8-appstream + - rhel-9-baseos + - rhel-9-appstream - rhel-9.8-early-kernel - - rhel-9.8-fast-datapath - - rhel-9.8-server-ose-4.22 + - rhel-9-fast-datapath + - rhel-9-server-ose - if: osversion == "rhel-10.2" include: repos: @@ -28,15 +27,6 @@ conditional-include: - rhel-10.2-early-kernel - rhel-10.2-fast-datapath - rhel-10.2-server-ose-4.22 - - if: osversion == "centos-9" - include: - repos: - - c9s-baseos - - c9s-appstream - - c9s-sig-nfv - - c9s-sig-cloud-okd - # XXX: this shouldn't be here; see related XXX in build-node-image.sh - - rhel-9.8-server-ose-4.22-okd - if: osversion == "centos-10" include: repos: @@ -47,7 +37,7 @@ conditional-include: # XXX: this shouldn't be here; see related XXX in build-node-image.sh # XXX: using 9.8 repo for now until 10.2 plashets exist # - rhel-10.2-server-ose-4.22-okd - - rhel-9.8-server-ose-4.22-okd + - rhel-9-server-ose-okd packages: # The packages below are required by OpenShift/OKD diff --git a/scripts/generate-labels b/scripts/generate-labels new file mode 100755 index 000000000..8a13ccf7a --- /dev/null +++ b/scripts/generate-labels @@ -0,0 +1,79 @@ +#!/usr/bin/python3 -u + +""" +This script generates /usr/share/buildinfo/labels.json, which provides embedded +metadata for security scanners that only have filesystem access (not OCI image +metadata). + +For RHEL builds, IMAGE_NAME and IMAGE_CPE must be set (via build-args file). +For non-RHEL builds (e.g. SCOS/OKD), no labels.json is written. +""" + +import datetime +import json +import os +import sys + +LABELS_FILE = "/usr/share/buildinfo/labels.json" + + +def parse_os_release(): + """Parse /etc/os-release into a dict.""" + release = {} + with open('/etc/os-release') as f: + for line in f: + line = line.strip() + if not line or line.startswith('#'): + continue + k, v = line.split('=', 1) + release[k] = v.strip('"\'') + return release + + +def main(): + image_name = os.environ.get('IMAGE_NAME', '') + image_cpe = os.environ.get('IMAGE_CPE', '') + target_arch = os.environ.get('TARGETARCH', '') + + os.makedirs(os.path.dirname(LABELS_FILE), exist_ok=True) + + if not all([image_name, image_cpe, target_arch]): + os_release = parse_os_release() + if os_release.get('ID') == 'rhel': + print("error: IMAGE_NAME, IMAGE_CPE, and TARGETARCH must be set for RHEL builds", + file=sys.stderr) + return 1 + return + + # Ideally the creation date we set here is consistent with the creation date + # of the OCI image itself. We'll get that once we're hermetic and we hook + # up SOURCE_DATE_EPOCH. So prepare for that eventuality, but for now just + # use the current time (which will be a few seconds different from the OCI + # timestamp, which is still fine for our purposes). + source_date_epoch = os.environ.get('SOURCE_DATE_EPOCH', '') + if source_date_epoch: + created = datetime.datetime.fromtimestamp( + int(source_date_epoch), tz=datetime.timezone.utc + ).strftime('%Y-%m-%dT%H:%M:%SZ') + else: + created = datetime.datetime.now( + tz=datetime.timezone.utc + ).strftime('%Y-%m-%dT%H:%M:%SZ') + + # this schema is documented at: + # https://github.com/RedHatProductSecurity/security-data-guidelines/blob/main/schema/embedded_metadata.v1.schema.json + labels = { + 'architecture': target_arch, + 'cpe': image_cpe, + 'name': image_name, + 'org.opencontainers.image.created': created, + } + + os.makedirs(os.path.dirname(LABELS_FILE), exist_ok=True) + with open(LABELS_FILE, encoding='utf-8', mode='w') as f: + json.dump(labels, f, sort_keys=True, indent=2) + f.write('\n') + + +if __name__ == '__main__': + sys.exit(main())