From 205cedb93760bcb6e3a5b402ab27f989dc92b431 Mon Sep 17 00:00:00 2001 From: vtripathi Date: Sun, 16 Nov 2025 09:24:21 +0000 Subject: [PATCH] feat: Add custom CRI-O storage paths with automatic bind mount --- cmd/minikube/cmd/start_flags.go | 46 +++++++++++++++++++ pkg/minikube/bootstrapper/kubeadm/kubeadm.go | 2 + pkg/minikube/config/types.go | 2 + pkg/minikube/cruntime/crio.go | 48 +++++++++++++++++++- pkg/minikube/cruntime/cruntime.go | 6 +++ pkg/minikube/node/node.go | 9 +++- pkg/minikube/node/start.go | 10 +++- 7 files changed, 119 insertions(+), 4 deletions(-) diff --git a/cmd/minikube/cmd/start_flags.go b/cmd/minikube/cmd/start_flags.go index c7627f0468da..6e6777a207e7 100644 --- a/cmd/minikube/cmd/start_flags.go +++ b/cmd/minikube/cmd/start_flags.go @@ -146,6 +146,8 @@ const ( staticIP = "static-ip" gpus = "gpus" autoPauseInterval = "auto-pause-interval" + containerStorageRoot = "container-storage-root" + containerStorageRunRoot = "container-storage-runroot" ) var ( @@ -211,6 +213,8 @@ func initMinikubeFlags() { startCmd.Flags().String(staticIP, "", "Set a static IP for the minikube cluster, the IP must be: private, IPv4, and the last octet must be between 2 and 254, for example 192.168.200.200 (Docker and Podman drivers only)") startCmd.Flags().StringP(gpus, "g", "", "Allow pods to use your GPUs. Options include: [all,nvidia,amd] (Docker driver with Docker container-runtime only)") startCmd.Flags().Duration(autoPauseInterval, time.Minute*1, "Duration of inactivity before the minikube VM is paused (default 1m0s)") + startCmd.Flags().String(containerStorageRoot, "", "Absolute path for container runtime storage root directory (e.g. /host-containers/storage for CRI-O). Works with CRI-O and other compatible runtimes.") + startCmd.Flags().String(containerStorageRunRoot, "", "Absolute path for container runtime storage state directory (e.g. /host-containers/storage for CRI-O). Works with CRI-O and other compatible runtimes.") } // initKubernetesFlags inits the commandline flags for Kubernetes related options @@ -652,6 +656,8 @@ func generateNewConfigFromFlags(cmd *cobra.Command, k8sVersion string, rtime str MultiNodeRequested: viper.GetInt(nodes) > 1 || viper.GetBool(ha), GPUs: viper.GetString(gpus), AutoPauseInterval: viper.GetDuration(autoPauseInterval), + ContainerStorageRoot: viper.GetString(containerStorageRoot), + ContainerStorageRunRoot: viper.GetString(containerStorageRunRoot), } cc.VerifyComponents = interpretWaitFlag(*cmd) @@ -659,6 +665,44 @@ func generateNewConfigFromFlags(cmd *cobra.Command, k8sVersion string, rtime str cc.ContainerVolumeMounts = []string{viper.GetString(mountString)} } + // Validate and warn about custom storage paths + if cc.ContainerStorageRoot != "" || cc.ContainerStorageRunRoot != "" { + // Validate that custom storage paths are only used with supported runtimes + if rtime != constants.CRIO && rtime != "cri-o" { + out.WarningT("Custom container storage paths (--container-storage-root, --container-storage-runroot) are currently only supported with CRI-O runtime. The specified paths may be ignored.") + } + + // Validate paths are absolute + if cc.ContainerStorageRoot != "" && !strings.HasPrefix(cc.ContainerStorageRoot, "/") { + exit.Message(reason.Usage, "--container-storage-root must be an absolute path, got: {{.path}}", out.V{"path": cc.ContainerStorageRoot}) + } + if cc.ContainerStorageRunRoot != "" && !strings.HasPrefix(cc.ContainerStorageRunRoot, "/") { + exit.Message(reason.Usage, "--container-storage-runroot must be an absolute path, got: {{.path}}", out.V{"path": cc.ContainerStorageRunRoot}) + } + + // Automatically add storage path mounts for KIC drivers + if driver.IsKIC(drvName) { + if cc.ContainerVolumeMounts == nil { + cc.ContainerVolumeMounts = []string{} + } + if cc.ContainerStorageRoot != "" { + // Mount host storage directory to the same path in container + cc.ContainerVolumeMounts = append(cc.ContainerVolumeMounts, fmt.Sprintf("%s:%s", cc.ContainerStorageRoot, cc.ContainerStorageRoot)) + } + if cc.ContainerStorageRunRoot != "" { + // Mount host runroot directory to the same path in container + cc.ContainerVolumeMounts = append(cc.ContainerVolumeMounts, fmt.Sprintf("%s:%s", cc.ContainerStorageRunRoot, cc.ContainerStorageRunRoot)) + } + } + + // Informational message about the feature + if rtime == constants.CRIO || rtime == "cri-o" { + if driver.IsKIC(drvName) { + out.Styled(style.Tip, "Using custom storage paths with {{.runtime}}. Storage directories will be automatically mounted.", out.V{"runtime": rtime}) + } + } + } + if driver.IsKIC(drvName) { si, err := oci.CachedDaemonInfo(drvName) if err != nil { @@ -882,6 +926,8 @@ func updateExistingConfigFromFlags(cmd *cobra.Command, existing *config.ClusterC updateStringFromFlag(cmd, &cc.SocketVMnetClientPath, socketVMnetClientPath) updateStringFromFlag(cmd, &cc.SocketVMnetPath, socketVMnetPath) updateDurationFromFlag(cmd, &cc.AutoPauseInterval, autoPauseInterval) + updateStringFromFlag(cmd, &cc.ContainerStorageRoot, containerStorageRoot) + updateStringFromFlag(cmd, &cc.ContainerStorageRunRoot, containerStorageRunRoot) if cmd.Flags().Changed(kubernetesVersion) { kubeVer, err := getKubernetesVersion(existing) diff --git a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go index b4b7f4699dc6..591a71084376 100644 --- a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go +++ b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go @@ -897,6 +897,8 @@ func (k *Bootstrapper) UpdateCluster(cfg config.ClusterConfig) error { Runner: k.c, Socket: cfg.KubernetesConfig.CRISocket, KubernetesVersion: ver, + StorageRoot: cfg.ContainerStorageRoot, + StorageRunRoot: cfg.ContainerStorageRunRoot, }) if err != nil { return errors.Wrap(err, "runtime") diff --git a/pkg/minikube/config/types.go b/pkg/minikube/config/types.go index 1077ac3ca255..969ff227de5a 100644 --- a/pkg/minikube/config/types.go +++ b/pkg/minikube/config/types.go @@ -109,6 +109,8 @@ type ClusterConfig struct { SSHAgentPID int GPUs string AutoPauseInterval time.Duration // Specifies interval of time to wait before checking if cluster should be paused + ContainerStorageRoot string // Custom storage root directory for container runtime (e.g., CRI-O root path) + ContainerStorageRunRoot string // Custom storage state directory for container runtime (e.g., CRI-O runroot path) } // KubernetesConfig contains the parameters used to configure the VM Kubernetes. diff --git a/pkg/minikube/cruntime/crio.go b/pkg/minikube/cruntime/crio.go index 85699fa0d375..ebd4b59b1049 100644 --- a/pkg/minikube/cruntime/crio.go +++ b/pkg/minikube/cruntime/crio.go @@ -51,10 +51,12 @@ type CRIO struct { ImageRepository string KubernetesVersion semver.Version Init sysinit.Manager + StorageRoot string // Custom storage root directory path + StorageRunRoot string // Custom storage state directory path } // generateCRIOConfig sets up pause image and cgroup manager for cri-o in crioConfigFile -func generateCRIOConfig(cr CommandRunner, imageRepository string, kv semver.Version, cgroupDriver string) error { +func generateCRIOConfig(cr CommandRunner, imageRepository string, kv semver.Version, cgroupDriver string, storageRoot string, storageRunRoot string) error { pauseImage := images.Pause(kv, imageRepository) klog.Infof("configure cri-o to use %q pause image...", pauseImage) c := exec.Command("sh", "-c", fmt.Sprintf(`sudo sed -i 's|^.*pause_image = .*$|pause_image = %q|' %s`, pauseImage, crioConfigFile)) @@ -62,6 +64,48 @@ func generateCRIOConfig(cr CommandRunner, imageRepository string, kv semver.Vers return errors.Wrap(err, "update pause_image") } + // configure custom storage paths if provided + if storageRoot != "" || storageRunRoot != "" { + // Check if [crio] section exists, if not add it at the beginning + checkCrioSection := exec.Command("sh", "-c", fmt.Sprintf(`sudo grep -q "^\[crio\]$" %s`, crioConfigFile)) + if _, err := cr.RunCmd(checkCrioSection); err != nil { + // [crio] section doesn't exist, add it at the beginning + if _, err := cr.RunCmd(exec.Command("sh", "-c", fmt.Sprintf(`sudo sed -i '1i[crio]' %s`, crioConfigFile))); err != nil { + return errors.Wrap(err, "adding [crio] section") + } + } + } + + if storageRoot != "" { + // Remove any existing root configuration + if _, err := cr.RunCmd(exec.Command("sh", "-c", fmt.Sprintf(`sudo sed -i '/^root = /d' %s`, crioConfigFile))); err != nil { + return errors.Wrap(err, "removing existing root configuration") + } + // Add new root configuration after [crio] section + if _, err := cr.RunCmd(exec.Command("sh", "-c", fmt.Sprintf(`sudo sed -i '/^\[crio\]$/a root = %q' %s`, storageRoot, crioConfigFile))); err != nil { + return errors.Wrap(err, "configuring storage root") + } + } + + if storageRunRoot != "" { + // Remove any existing runroot configuration + if _, err := cr.RunCmd(exec.Command("sh", "-c", fmt.Sprintf(`sudo sed -i '/^runroot = /d' %s`, crioConfigFile))); err != nil { + return errors.Wrap(err, "removing existing runroot configuration") + } + // Add new runroot configuration after [crio] section (or after root if it exists) + if storageRoot != "" { + // Add after root line + if _, err := cr.RunCmd(exec.Command("sh", "-c", fmt.Sprintf(`sudo sed -i '/^root = /a runroot = %q' %s`, storageRunRoot, crioConfigFile))); err != nil { + return errors.Wrap(err, "configuring storage runroot") + } + } else { + // Add after [crio] section + if _, err := cr.RunCmd(exec.Command("sh", "-c", fmt.Sprintf(`sudo sed -i '/^\[crio\]$/a runroot = %q' %s`, storageRunRoot, crioConfigFile))); err != nil { + return errors.Wrap(err, "configuring storage runroot") + } + } + } + // configure cgroup driver if cgroupDriver == constants.UnknownCgroupDriver { klog.Warningf("unable to configure cri-o to use unknown cgroup driver, will use default %q instead", constants.DefaultCgroupDriver) @@ -223,7 +267,7 @@ func (r *CRIO) Enable(disOthers bool, cgroupDriver string, inUserNamespace bool) if err := populateCRIConfig(r.Runner, r.SocketPath()); err != nil { return err } - if err := generateCRIOConfig(r.Runner, r.ImageRepository, r.KubernetesVersion, cgroupDriver); err != nil { + if err := generateCRIOConfig(r.Runner, r.ImageRepository, r.KubernetesVersion, cgroupDriver, r.StorageRoot, r.StorageRunRoot); err != nil { return err } if err := enableIPForwarding(r.Runner); err != nil { diff --git a/pkg/minikube/cruntime/cruntime.go b/pkg/minikube/cruntime/cruntime.go index 8b06f042d9d0..6bc10fb76b04 100644 --- a/pkg/minikube/cruntime/cruntime.go +++ b/pkg/minikube/cruntime/cruntime.go @@ -157,6 +157,10 @@ type Config struct { InsecureRegistry []string // GPUs add GPU devices to the container GPUs string + // StorageRoot custom storage root directory for container runtime + StorageRoot string + // StorageRunRoot custom storage state directory for container runtime + StorageRunRoot string } // ListContainersOptions are the options to use for listing containers @@ -238,6 +242,8 @@ func New(c Config) (Manager, error) { ImageRepository: c.ImageRepository, KubernetesVersion: c.KubernetesVersion, Init: sm, + StorageRoot: c.StorageRoot, + StorageRunRoot: c.StorageRunRoot, }, nil case "containerd": return &Containerd{ diff --git a/pkg/minikube/node/node.go b/pkg/minikube/node/node.go index 663372597ed4..f26c32a47e65 100644 --- a/pkg/minikube/node/node.go +++ b/pkg/minikube/node/node.go @@ -139,7 +139,14 @@ func teardown(cc config.ClusterConfig, name string, options *run.CommandOptions) kv, kerr = util.ParseKubernetesVersion(cc.KubernetesConfig.KubernetesVersion) if kerr == nil { var crt cruntime.Manager - crt, kerr = cruntime.New(cruntime.Config{Type: cc.KubernetesConfig.ContainerRuntime, Runner: r, Socket: cc.KubernetesConfig.CRISocket, KubernetesVersion: kv}) + crt, kerr = cruntime.New(cruntime.Config{ + Type: cc.KubernetesConfig.ContainerRuntime, + Runner: r, + Socket: cc.KubernetesConfig.CRISocket, + KubernetesVersion: kv, + StorageRoot: cc.ContainerStorageRoot, + StorageRunRoot: cc.ContainerStorageRunRoot, + }) if kerr == nil { sp := crt.SocketPath() // avoid warning/error: diff --git a/pkg/minikube/node/start.go b/pkg/minikube/node/start.go index f62799853e7d..35d1058cb000 100755 --- a/pkg/minikube/node/start.go +++ b/pkg/minikube/node/start.go @@ -263,7 +263,13 @@ func handleNoKubernetes(starter Starter) (bool, error) { if starter.Node.KubernetesVersion == constants.NoKubernetesVersion { // Stop existing Kubernetes node if applicable. if starter.StopK8s { - cr, err := cruntime.New(cruntime.Config{Type: starter.Cfg.KubernetesConfig.ContainerRuntime, Runner: starter.Runner, Socket: starter.Cfg.KubernetesConfig.CRISocket}) + cr, err := cruntime.New(cruntime.Config{ + Type: starter.Cfg.KubernetesConfig.ContainerRuntime, + Runner: starter.Runner, + Socket: starter.Cfg.KubernetesConfig.CRISocket, + StorageRoot: starter.Cfg.ContainerStorageRoot, + StorageRunRoot: starter.Cfg.ContainerStorageRunRoot, + }) if err != nil { return false, err } @@ -418,6 +424,8 @@ func configureRuntimes(runner cruntime.CommandRunner, cc config.ClusterConfig, k ImageRepository: cc.KubernetesConfig.ImageRepository, KubernetesVersion: kv, InsecureRegistry: cc.InsecureRegistry, + StorageRoot: cc.ContainerStorageRoot, + StorageRunRoot: cc.ContainerStorageRunRoot, } if cc.GPUs != "" { co.GPUs = cc.GPUs