build-sys: Various improvements

Pass SOURCE_DATE_EPOCH from git commit timestamp through to rpmbuild,
enabling bit-for-bit reproducible RPM builds. This is useful for
verification and caching.

Then fix the idempotency of the default `just build` to ensure
we're not incorrectly invalidating caches.

Add `just check-buildsys` command that builds packages twice and
verifies checksums match, confirming reproducibility. The CI package
job now uses this to catch regressions.

Assisted-by: OpenCode (Opus 4.5)
Signed-off-by: Colin Walters <walters@verbum.org>

Author: cgwalters

.dockerignore

@@ -21,7 +21,5 @@
 # Workaround for podman bug with secrets + remote
 # https://github.com/containers/podman/issues/25314
 !podman-build-secret*
-# Pre-built packages for builds that use them
-!target/packages/
 # And finally of course all the Rust sources
 !crates/

.github/workflows/ci.yml

@@ -134,8 +134,8 @@ jobs:
           BASE=$(just pullspec-for-os base ${{ matrix.test_os }})
           echo "BOOTC_base=${BASE}" >> $GITHUB_ENV
 
-      - name: Build packages
-        run: just package
+      - name: Build packages (and verify build system)
+        run: just check-buildsys
 
       - name: Upload package artifacts
         uses: actions/upload-artifact@v6

Cargo.lock

@@ -15,11 +15,6 @@ COPY . /src
 FROM scratch as packaging
 COPY contrib/packaging /
 
-# This image captures pre-built packages from the context.
-# By COPYing into a stage, we avoid SELinux issues with context bind mounts.
-FROM scratch as packages
-COPY target/packages/*.rpm /
-
 FROM $base as base
 # Mark this as a test image (moved from --label build flag to fix layer caching)
 LABEL bootc.testimage="1"
@@ -51,6 +46,9 @@ RUN /src/contrib/packaging/configure-systemdboot download
 FROM buildroot as build
 # Version for RPM build (optional, computed from git in Justfile)
 ARG pkgversion
+# For reproducible builds, SOURCE_DATE_EPOCH must be exported as ENV for rpmbuild to see it
+ARG SOURCE_DATE_EPOCH
+ENV SOURCE_DATE_EPOCH=${SOURCE_DATE_EPOCH}
 # Build RPM directly from source, using cached target directory
 RUN --mount=type=cache,target=/src/target --mount=type=cache,target=/var/roothome --network=none RPM_VERSION="${pkgversion}" /src/contrib/packaging/build-rpm
 
@@ -71,27 +69,33 @@ ENV TMPDIR=/var/tmp
 RUN --mount=type=cache,target=/src/target --mount=type=cache,target=/var/roothome --network=none make install-unit-tests
 
 # This just does syntax checking
-FROM build as validate
+FROM buildroot as validate
 RUN --mount=type=cache,target=/src/target --mount=type=cache,target=/var/roothome --network=none make validate
 
-# The final image that derives from the original base and adds the release binaries
-FROM base
-# See the Justfile for possible variants
+# Common base for final images: configures variant, rootfs, and injects extra content
+FROM base as final-common
 ARG variant
 RUN --network=none --mount=type=bind,from=packaging,target=/run/packaging \
     --mount=type=bind,from=sdboot-content,target=/run/sdboot-content \
     --mount=type=bind,from=sdboot-signed,target=/run/sdboot-signed \
     /run/packaging/configure-variant "${variant}"
-# Support overriding the rootfs at build time conveniently
-ARG rootfs
+ARG rootfs=""
 RUN --mount=type=bind,from=packaging,target=/run/packaging /run/packaging/configure-rootfs "${variant}" "${rootfs}"
-# Inject additional content
 COPY --from=packaging /usr-extras/ /usr/
-# Install packages from the packages stage
-# Using bind from a stage avoids SELinux issues with context bind mounts
+
+# Default target for source builds (just build)
+# Installs packages from the internal build stage
+FROM final-common as final
+RUN --mount=type=bind,from=packaging,target=/run/packaging \
+    --mount=type=bind,from=build,target=/build-output \
+    --network=none \
+    /run/packaging/install-rpm-and-setup /build-output/out
+RUN bootc container lint --fatal-warnings
+
+# Alternative target for pre-built packages (CI workflow)
+# Use with: podman build --target=final-from-packages -v path/to/packages:/run/packages:ro
+FROM final-common as final-from-packages
 RUN --mount=type=bind,from=packaging,target=/run/packaging \
-    --mount=type=bind,from=packages,target=/build-packages \
     --network=none \
-    /run/packaging/install-rpm-and-setup /build-packages
-# Finally, testour own linting
+    /run/packaging/install-rpm-and-setup /run/packages
 RUN bootc container lint --fatal-warnings

Dockerfile

@@ -43,11 +43,39 @@ buildargs := base_buildargs + " --secret=id=secureboot_key,src=target/test-secur
 # Args for build-sealed (no base arg, it sets that itself)
 sealed_buildargs := "--build-arg=variant=" + variant + " --secret=id=secureboot_key,src=target/test-secureboot/db.key --secret=id=secureboot_cert,src=target/test-secureboot/db.crt"
 
+# Compute SOURCE_DATE_EPOCH and VERSION from git for reproducible builds.
+# Outputs shell variable assignments that can be eval'd.
+_git-build-vars:
+    #!/bin/bash
+    set -euo pipefail
+    SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct)
+    # Compute version from git (matching xtask.rs gitrev logic)
+    if VERSION=$(git describe --tags --exact-match 2>/dev/null); then
+        VERSION="${VERSION#v}"
+        VERSION="${VERSION//-/.}"
+    else
+        COMMIT=$(git rev-parse HEAD | cut -c1-10)
+        COMMIT_TS=$(git show -s --format=%ct)
+        TIMESTAMP=$(date -u -d @${COMMIT_TS} +%Y%m%d%H%M)
+        VERSION="${TIMESTAMP}.g${COMMIT}"
+    fi
+    echo "SOURCE_DATE_EPOCH=${SOURCE_DATE_EPOCH}"
+    echo "VERSION=${VERSION}"
+
 # The default target: build the container image from current sources.
 # Note commonly you might want to override the base image via e.g.
 # `just build --build-arg=base=quay.io/fedora/fedora-bootc:42`
-build: package _keygen
-    podman build {{base_buildargs}} -t {{base_img}}-bin {{buildargs}} .
+#
+# The Dockerfile builds RPMs internally in its 'build' stage, so we don't need
+# to call 'package' first. This avoids cache invalidation from external files.
+build: _keygen
+    #!/bin/bash
+    set -xeuo pipefail
+    eval $(just _git-build-vars)
+    podman build {{base_buildargs}} --target=final \
+        --build-arg=SOURCE_DATE_EPOCH=${SOURCE_DATE_EPOCH} \
+        --build-arg=pkgversion=${VERSION} \
+        -t {{base_img}}-bin {{buildargs}} .
     ./hack/build-sealed {{variant}} {{base_img}}-bin {{base_img}} {{sealed_buildargs}}
 
 # Generate Secure Boot keys (only for our own CI/testing)
@@ -62,18 +90,9 @@ build-sealed:
 _packagecontainer:
     #!/bin/bash
     set -xeuo pipefail
-    # Compute version from git (matching xtask.rs gitrev logic)
-    if VERSION=$(git describe --tags --exact-match 2>/dev/null); then
-        VERSION="${VERSION#v}"
-        VERSION="${VERSION//-/.}"
-    else
-        COMMIT=$(git rev-parse HEAD | cut -c1-10)
-        COMMIT_TS=$(git show -s --format=%ct)
-        TIMESTAMP=$(date -u -d @${COMMIT_TS} +%Y%m%d%H%M)
-        VERSION="${TIMESTAMP}.g${COMMIT}"
-    fi
+    eval $(just _git-build-vars)
     echo "Building RPM with version: ${VERSION}"
-    podman build {{base_buildargs}} --build-arg=pkgversion=${VERSION} -t localhost/bootc-pkg --target=build .
+    podman build {{base_buildargs}} --build-arg=SOURCE_DATE_EPOCH=${SOURCE_DATE_EPOCH} --build-arg=pkgversion=${VERSION} -t localhost/bootc-pkg --target=build .
 
 # Build packages (e.g. RPM) into target/packages/
 # Any old packages will be removed.
@@ -86,7 +105,8 @@ package: _packagecontainer
     podman rmi localhost/bootc-pkg
 
 # Copy pre-existing packages from PATH into target/packages/
-# Used to prepare for building with pre-built packages
+# Note: This is mainly for CI artifact extraction; build-from-package
+# now uses volume mounts directly instead of copying to target/packages/.
 copy-packages-from PATH:
     #!/bin/bash
     set -xeuo pipefail
@@ -101,11 +121,15 @@ copy-packages-from PATH:
     chmod a+r target/packages/*.rpm
 
 # Build the container image using pre-existing packages from PATH
-# Note: The Dockerfile reads from target/packages/, so copy the packages there first
-# if they're in a different location.
+# Uses the 'final-from-packages' target with a volume mount to inject packages,
+# avoiding Docker context cache invalidation issues.
 build-from-package PATH: _keygen
-    @if [ "{{PATH}}" != "target/packages" ]; then just copy-packages-from {{PATH}}; fi
-    podman build {{base_buildargs}} -t {{base_img}}-bin {{buildargs}} .
+    #!/bin/bash
+    set -xeuo pipefail
+    # Resolve to absolute path for podman volume mount
+    # Use :z for SELinux relabeling
+    pkg_path=$(realpath "{{PATH}}")
+    podman build {{base_buildargs}} --target=final-from-packages -v "${pkg_path}":/run/packages:ro,z -t {{base_img}}-bin {{buildargs}} .
     ./hack/build-sealed {{variant}} {{base_img}}-bin {{base_img}} {{sealed_buildargs}}
 
 # Pull images used by hack/lbi
@@ -137,7 +161,10 @@ run-container-external-tests:
 
 # We build the unit tests into a container image
 build-units:
-    podman build {{base_buildargs}} --target units -t localhost/bootc-units .
+    #!/bin/bash
+    set -xeuo pipefail
+    eval $(just _git-build-vars)
+    podman build {{base_buildargs}} --build-arg=SOURCE_DATE_EPOCH=${SOURCE_DATE_EPOCH} --build-arg=pkgversion=${VERSION} --target units -t localhost/bootc-units .
 
 # Perform validation (build, linting) in a container build environment
 validate:
@@ -209,3 +236,10 @@ mdbook-serve: build-mdbook
 # Use this after adding, removing, or modifying CLI options or schemas.
 update-generated:
     cargo run -p xtask update-generated
+
+# Verify build system properties (reproducible builds)
+#
+# This runs `just package` twice and verifies that the resulting RPMs
+# are bit-for-bit identical, confirming SOURCE_DATE_EPOCH is working.
+check-buildsys:
+    cargo run -p xtask check-buildsys

Justfile

@@ -162,7 +162,7 @@ chmod +x %{?buildroot}/%{system_reinstall_bootc_install_podman_path}
 # generate doc file list excluding directories; workaround for
 # https://github.com/coreos/rpm-ostree/issues/5420
 touch %{?buildroot}/%{_docdir}/bootc/baseimage/base/sysroot/.keepdir
-find %{?buildroot}/%{_docdir} ! -type d -printf '%{_docdir}/%%P\n' > bootcdoclist.txt
+find %{?buildroot}/%{_docdir} ! -type d -printf '%{_docdir}/%%P\n' | sort > bootcdoclist.txt
 
 %if %{with check}
 %check

contrib/packaging/bootc.spec

@@ -29,10 +29,17 @@ else
 fi
 
 # Build RPM
+# For reproducible builds:
+# - use_source_date_epoch_as_buildtime: RPM build timestamp uses SOURCE_DATE_EPOCH
+# - clamp_mtime_to_source_date_epoch: file mtimes clamped to SOURCE_DATE_EPOCH
+# - _buildhost: fixed hostname for consistent RPM metadata
 rpmbuild -bb \
   --define "_topdir /tmp/rpmbuild" \
   --define "_builddir ${SRC_DIR}" \
   --define "container_build 1" \
+  --define "use_source_date_epoch_as_buildtime 1" \
+  --define "clamp_mtime_to_source_date_epoch 1" \
+  --define "_buildhost reproducible" \
   --with tests \
   --nocheck \
   "${SPEC_FILE}"

contrib/packaging/build-rpm

@@ -31,6 +31,7 @@ mandown = "1.1.0"
 rand = "0.9"
 serde_yaml = "0.9"
 tar = "0.4"
+itertools = "0.14.0"
 
 [lints]
 workspace = true

crates/xtask/Cargo.toml

@@ -51,6 +51,8 @@ enum Commands {
     RunTmt(RunTmtArgs),
     /// Provision a VM for manual TMT testing
     TmtProvision(TmtProvisionArgs),
+    /// Check build system properties (e.g., reproducible builds)
+    CheckBuildsys,
 }
 
 /// Arguments for run-tmt command
@@ -135,6 +137,7 @@ fn try_main() -> Result<()> {
         Commands::Spec => spec(&sh),
         Commands::RunTmt(args) => tmt::run_tmt(&sh, &args),
         Commands::TmtProvision(args) => tmt::tmt_provision(&sh, &args),
+        Commands::CheckBuildsys => check_buildsys(&sh),
     }
 }
 
@@ -402,3 +405,48 @@ fn update_generated(sh: &Shell) -> Result<()> {
 
     Ok(())
 }
+
+/// Check build system properties
+///
+/// - Reproducible builds for the RPM
+#[context("Checking build system")]
+fn check_buildsys(sh: &Shell) -> Result<()> {
+    use std::collections::BTreeMap;
+
+    println!("Checking reproducible builds...");
+    // Helper to compute SHA256 of bootc RPMs in target/packages/
+    fn get_rpm_checksums(sh: &Shell) -> Result<BTreeMap<String, String>> {
+        // Find bootc*.rpm files in target/packages/
+        let packages_dir = Utf8Path::new("target/packages");
+        let mut rpm_files: Vec<Utf8PathBuf> = Vec::new();
+        for entry in std::fs::read_dir(packages_dir).context("Reading target/packages")? {
+            let entry = entry?;
+            let path = Utf8PathBuf::try_from(entry.path())?;
+            if path.extension() == Some("rpm") {
+                rpm_files.push(path);
+            }
+        }
+
+        assert!(!rpm_files.is_empty());
+
+        let mut checksums = BTreeMap::new();
+        for rpm_path in &rpm_files {
+            let output = cmd!(sh, "sha256sum {rpm_path}").read()?;
+            let (hash, filename) = output
+                .split_once("  ")
+                .with_context(|| format!("failed to parse sha256sum output: '{}'", output))?;
+            checksums.insert(filename.to_owned(), hash.to_owned());
+        }
+        Ok(checksums)
+    }
+
+    cmd!(sh, "just package").run()?;
+    let first_checksums = get_rpm_checksums(sh)?;
+    cmd!(sh, "just package").run()?;
+    let second_checksums = get_rpm_checksums(sh)?;
+
+    itertools::assert_equal(first_checksums, second_checksums);
+    println!("ok package reproducibility");
+
+    Ok(())
+}