rpm: speed up cross-arch worker builds by installing deps into mounted target rootfs

targets/linux/rpm/almalinux/v8.go

@@ -20,7 +20,7 @@ var ConfigV8 = &distro.Config{
 	ContextRef: v8WorkerContextName,
 
 	CacheName: dnfCacheNameV8,
-	CacheDir:  "/var/cache/dnf",
+	CacheDir:  []string{"/var/cache/dnf"},
 	// Alma's repo configs do not include the $basearch variable in the mirrorlist URL
 	// This means that the cache key that dnf computes for /var/cache/dnf/<repoid>-<hash>
 	// is the same across x86_64 and aarch64, which leads to incorrect repo metadata

targets/linux/rpm/almalinux/v9.go

@@ -20,7 +20,7 @@ var ConfigV9 = &distro.Config{
 	ContextRef: v9WorkerContextName,
 
 	CacheName: dnfCacheNameV9,
-	CacheDir:  "/var/cache/dnf",
+	CacheDir:  []string{"/var/cache/dnf"},
 	// Alma's repo configs do not include the $basearch variable in the mirrorlist URL
 	// This means that the cache key that dnf computes for /var/cache/dnf/<repoid>-<hash>
 	// is the same across x86_64 and aarch64, which leads to incorrect repo metadata

targets/linux/rpm/azlinux/azlinux3.go

@@ -19,11 +19,12 @@ var Azlinux3Config = &distro.Config{
 	ImageRef:   Azlinux3Ref,
 	ContextRef: Azlinux3WorkerContextName,
 
-	CacheName: tdnfCacheNameAzlinux3,
-	CacheDir:  "/var/cache/tdnf",
+	CacheName:        tdnfCacheNameAzlinux3,
+	CacheDir:         []string{"/var/cache/tdnf", "/var/cache/dnf"},
+	CacheAddPlatform: true,
 
 	ReleaseVer:         "3.0",
-	BuilderPackages:    builderPackages,
+	BuilderPackages:    append(builderPackages, "dnf"),
 	BasePackages:       basePackages(AzLinux3TargetKey),
 	RepoPlatformConfig: &defaultAzlinuxRepoPlatform,
 	InstallFunc:        distro.TdnfInstall,

targets/linux/rpm/azlinux/mariner2.go

@@ -17,11 +17,12 @@ var Mariner2Config = &distro.Config{
 	ImageRef:   "mcr.microsoft.com/cbl-mariner/base/core:2.0",
 	ContextRef: Mariner2WorkerContextName,
 
-	CacheName: tdnfCacheNameMariner2,
-	CacheDir:  "/var/cache/tdnf",
+	CacheName:        tdnfCacheNameMariner2,
+	CacheDir:         []string{"/var/cache/tdnf", "/var/cache/dnf"},
+	CacheAddPlatform: true,
 
 	ReleaseVer:         "2.0",
-	BuilderPackages:    builderPackages,
+	BuilderPackages:    append(builderPackages, "dnf"),
 	BasePackages:       basePackages(Mariner2TargetKey),
 	RepoPlatformConfig: &defaultAzlinuxRepoPlatform,
 	InstallFunc:        distro.TdnfInstall,

targets/linux/rpm/distro/distro.go

@@ -37,11 +37,10 @@ type Config struct {
 	CacheName string
 	// Whether to namespace the cache key by platform
 	// Not all distros need this, hence why it is configurable.
-	// The cache key is only added when the build platform and target platform differ.
 	CacheAddPlatform bool
 
-	// e.g. /var/cache/tdnf or /var/cache/dnf
-	CacheDir string
+	// Cache directories to mount (e.g. ["/var/cache/dnf"] or ["/var/cache/tdnf", "/var/cache/dnf"])
+	CacheDir []string
 
 	// erofs-utils 1.7+ is required for tar support.
 	SysextSupported bool
@@ -61,7 +60,27 @@ func (cfg *Config) PackageCacheMount(root string) llb.RunOption {
 			}
 			cacheKey += "-" + platforms.Format(*p)
 		}
-		llb.AddMount(filepath.Join(root, cfg.CacheDir), llb.Scratch(), llb.AsPersistentCacheDir(cacheKey, llb.CacheMountLocked)).SetRunOption(ei)
+
+		if len(cfg.CacheDir) == 0 {
+			return
+		}
+
+		// Mount each cache dir. If there are multiple, suffix key with the dir base.
+		for _, d := range cfg.CacheDir {
+			if d == "" {
+				continue
+			}
+			k := cacheKey
+			if len(cfg.CacheDir) > 1 {
+				k = cacheKey + "-" + filepath.Base(d)
+			}
+			llb.AddMount(
+				joinUnderRoot(root, d),
+				llb.Scratch(),
+				llb.AsPersistentCacheDir(k, llb.CacheMountLocked),
+			).SetRunOption(ei)
+		}
+
 	})
 }
 

targets/linux/rpm/distro/dnf_install.go

@@ -2,10 +2,12 @@ package distro
 
 import (
 	"fmt"
+	"path"
 	"path/filepath"
 	"strings"
 
 	"github.com/moby/buildkit/client/llb"
+	ocispecs "github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/project-dalec/dalec"
 	"github.com/project-dalec/dalec/packaging/linux/rpm"
 )
@@ -40,10 +42,21 @@ type dnfInstallConfig struct {
 
 	// When true, don't omit docs from the installed RPMs.
 	includeDocs bool
+
+	forceArch string
 }
 
 type DnfInstallOpt func(*dnfInstallConfig)
 
+// joinUnderRoot joins a rootfs path with an absolute container path.
+// We must not use filepath.Join(root, "/abs") because that drops `root`.
+func joinUnderRoot(root, abs string) string {
+	if root == "" {
+		return abs
+	}
+	return path.Join(root, strings.TrimPrefix(abs, "/"))
+}
+
 // see comment in tdnfInstall for why this additional option is needed
 func DnfImportKeys(keys []string) DnfInstallOpt {
 	return func(cfg *dnfInstallConfig) {
@@ -63,6 +76,12 @@ func DnfAtRoot(root string) DnfInstallOpt {
 	}
 }
 
+func DnfForceArch(arch string) DnfInstallOpt {
+	return func(cfg *dnfInstallConfig) {
+		cfg.forceArch = arch
+	}
+}
+
 func DnfDownloadAllDeps(dest string) DnfInstallOpt {
 	return func(cfg *dnfInstallConfig) {
 		cfg.downloadOnly = true
@@ -145,23 +164,33 @@ func dnfCommand(cfg *dnfInstallConfig, releaseVer string, exe string, dnfSubCmd
 
 	cacheDir := "/var/cache/" + exe
 	if cfg.root != "" {
-		cacheDir = filepath.Join(cfg.root, cacheDir)
+		cacheDir = joinUnderRoot(cfg.root, cacheDir)
 	}
 	installFlags := dnfInstallFlags(cfg)
 	installFlags += " -y --setopt varsdir=/etc/dnf/vars --releasever=" + releaseVer + " "
+	forceArch := cfg.forceArch
 	installScriptDt := `#!/usr/bin/env bash
 set -eux -o pipefail
 
 import_keys_path="` + importKeysPath + `"
 cmd="` + exe + `"
 install_flags="` + installFlags + `"
+force_arch="` + forceArch + `"
 dnf_sub_cmd="` + strings.Join(dnfSubCmd, " ") + `"
 cache_dir="` + cacheDir + `"
 
 if [ -x "$import_keys_path" ]; then
 	"$import_keys_path"
 fi
 
+if [ -n "$force_arch" ]; then
+        if [ "$cmd" = "tdnf" ]; then
+                echo "tdnf does not support --forcearch; cross-arch installs must use dnf" >&2
+                exit 70
+        fi
+        install_flags="$install_flags --forcearch=$force_arch"
+fi
+
 $cmd $dnf_sub_cmd $install_flags "${@}"
 `
 	var runOpts []llb.RunOption
@@ -196,6 +225,53 @@ $cmd $dnf_sub_cmd $install_flags "${@}"
 	return dalec.WithRunOptions(runOpts...)
 }
 
+func (cfg *Config) InstallIntoRoot(rootfsPath string, pkgs []string, targetArch string, buildPlat ocispecs.Platform) llb.RunOption {
+	return dalec.RunOptFunc(func(ei *llb.ExecInfo) {
+		// Ensure the package manager runs on the build/executor platform (native),
+		// while installing into the mounted target rootfs via --installroot.
+		bp := buildPlat
+		ei.Constraints.Platform = &bp
+
+		installOpts := []DnfInstallOpt{
+			DnfAtRoot(rootfsPath),
+			DnfForceArch(targetArch),
+			DnfInstallWithConstraints([]llb.ConstraintsOpt{dalec.WithConstraint(&ei.Constraints)}),
+		}
+
+		var installCfg dnfInstallConfig
+		dnfInstallOptions(&installCfg, installOpts)
+
+		cacheKey := cfg.CacheName
+		if cfg.CacheAddPlatform {
+			cacheKey += "-" + targetArch
+		}
+		// Cross-arch installs always use dnf --forcearch --installroot
+		runOpts := []llb.RunOption{
+			DnfInstall(&installCfg, cfg.ReleaseVer, pkgs),
+		}
+
+		// Mount package manager caches under the target rootfs (may include multiple dirs).
+		for _, d := range cfg.CacheDir {
+			if d == "" {
+				continue
+			}
+			k := cacheKey
+			if len(cfg.CacheDir) > 1 {
+				k = cacheKey + "-" + filepath.Base(d)
+			}
+			runOpts = append(runOpts,
+				llb.AddMount(
+					joinUnderRoot(rootfsPath, d),
+					llb.Scratch(),
+					llb.AsPersistentCacheDir(k, llb.CacheMountLocked),
+				),
+			)
+		}
+
+		dalec.WithRunOptions(runOpts...).SetRunOption(ei)
+	})
+}
+
 func DnfInstall(cfg *dnfInstallConfig, releaseVer string, pkgs []string) llb.RunOption {
 	return dnfCommand(cfg, releaseVer, "dnf", append([]string{"install"}, pkgs...), nil)
 }