Skip to content

Commit d9643eb

Browse files
saschagrunertsaschagrunert
committed
Add additional storage configuration fields to ContainerRuntimeConfig
This enhancement extends ContainerRuntimeConfig API with three features for improved container storage flexibility: - additionalLayerStores: Enable lazy image pulling via storage plugins (BYOS approach with stargz-snapshotter, nydus-storage-plugin) - additionalImageStores: Read-only container image caches on shared or high-performance storage (NFS, SSD) for faster startup and reduced network overhead - additionalArtifactStores: Configurable OCI artifact storage locations (SSD-backed storage, pre-populated caches, air-gapped deployments) All features target AI/ML workload performance improvements and will ship as Tech Preview in 4.22 behind TechPreviewNoUpgrade feature gate. Path-based configuration with graceful fallback to standard behavior on failure. Unified API design pattern across all three features. Signed-off-by: Sascha Grunert <sgrunert@redhat.com>
1 parent c16ec2b commit d9643eb

26 files changed

Lines changed: 3118 additions & 7 deletions

File tree

‎features.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@
2424
| AWSDedicatedHosts| | | <span style="background-color: #519450">Enabled</span> | <span style="background-color: #519450">Enabled</span> | | | <span style="background-color: #519450">Enabled</span> | <span style="background-color: #519450">Enabled</span> |
2525
| AWSDualStackInstall| | | <span style="background-color: #519450">Enabled</span> | <span style="background-color: #519450">Enabled</span> | | | <span style="background-color: #519450">Enabled</span> | <span style="background-color: #519450">Enabled</span> |
2626
| AWSServiceLBNetworkSecurityGroup| | | <span style="background-color: #519450">Enabled</span> | <span style="background-color: #519450">Enabled</span> | | | <span style="background-color: #519450">Enabled</span> | <span style="background-color: #519450">Enabled</span> |
27+
| AdditionalStorageConfig| | | <span style="background-color: #519450">Enabled</span> | <span style="background-color: #519450">Enabled</span> | | | <span style="background-color: #519450">Enabled</span> | <span style="background-color: #519450">Enabled</span> |
2728
| AutomatedEtcdBackup| | | <span style="background-color: #519450">Enabled</span> | <span style="background-color: #519450">Enabled</span> | | | <span style="background-color: #519450">Enabled</span> | <span style="background-color: #519450">Enabled</span> |
2829
| AzureClusterHostedDNSInstall| | | <span style="background-color: #519450">Enabled</span> | <span style="background-color: #519450">Enabled</span> | | | <span style="background-color: #519450">Enabled</span> | <span style="background-color: #519450">Enabled</span> |
2930
| AzureDedicatedHosts| | | <span style="background-color: #519450">Enabled</span> | <span style="background-color: #519450">Enabled</span> | | | <span style="background-color: #519450">Enabled</span> | <span style="background-color: #519450">Enabled</span> |

‎features/features.go‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -401,6 +401,14 @@ var (
401401
enableIn(configv1.Default, configv1.OKD, configv1.DevPreviewNoUpgrade, configv1.TechPreviewNoUpgrade).
402402
mustRegister()
403403

404+
FeatureGateAdditionalStorageConfig = newFeatureGate("AdditionalStorageConfig").
405+
reportProblemsToJiraComponent("MachineConfigOperator").
406+
contactPerson("saschagrunert").
407+
productScope(ocpSpecific).
408+
enhancementPR("https://github.com/openshift/enhancements/pull/1934").
409+
enableIn(configv1.DevPreviewNoUpgrade, configv1.TechPreviewNoUpgrade).
410+
mustRegister()
411+
404412
FeatureGateUpgradeStatus = newFeatureGate("UpgradeStatus").
405413
reportProblemsToJiraComponent("Cluster Version Operator").
406414
contactPerson("pmuller").

‎machineconfiguration/v1/tests/containerruntimeconfigs.machineconfiguration.openshift.io/AdditionalStorageConfig.yaml‎

Lines changed: 213 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,213 @@
1+
apiVersion: apiextensions.k8s.io/v1 # Hack because controller-gen complains if we don't have this
2+
name: "ContainerRuntimeConfig"
3+
crdName: containerruntimeconfigs.machineconfiguration.openshift.io
4+
tests:
5+
onCreate:
6+
# AdditionalLayerStores - comprehensive validation tests
7+
- name: Should be able to create ContainerRuntimeConfig with multiple additionalLayerStores
8+
initial: |
9+
apiVersion: machineconfiguration.openshift.io/v1
10+
kind: ContainerRuntimeConfig
11+
spec:
12+
containerRuntimeConfig:
13+
additionalLayerStores:
14+
- path: /var/lib/stargz-store
15+
- path: /mnt/nydus-store
16+
- path: /opt/layer_store-v1.0
17+
expected: |
18+
apiVersion: machineconfiguration.openshift.io/v1
19+
kind: ContainerRuntimeConfig
20+
spec:
21+
containerRuntimeConfig:
22+
additionalLayerStores:
23+
- path: /var/lib/stargz-store
24+
- path: /mnt/nydus-store
25+
- path: /opt/layer_store-v1.0
26+
27+
- name: Should fail if additionalLayerStores path is empty
28+
initial: |
29+
apiVersion: machineconfiguration.openshift.io/v1
30+
kind: ContainerRuntimeConfig
31+
spec:
32+
containerRuntimeConfig:
33+
additionalLayerStores:
34+
- path: ""
35+
expectedError: "path in body should be at least 1 chars long"
36+
37+
- name: Should fail if additionalLayerStores path is not absolute
38+
initial: |
39+
apiVersion: machineconfiguration.openshift.io/v1
40+
kind: ContainerRuntimeConfig
41+
spec:
42+
containerRuntimeConfig:
43+
additionalLayerStores:
44+
- path: var/lib/stargz-store
45+
expectedError: "path must be absolute and contain only alphanumeric characters, '/', '_', and '-'"
46+
47+
- name: Should fail if additionalLayerStores path contains spaces
48+
initial: |
49+
apiVersion: machineconfiguration.openshift.io/v1
50+
kind: ContainerRuntimeConfig
51+
spec:
52+
containerRuntimeConfig:
53+
additionalLayerStores:
54+
- path: /var/lib/stargz store
55+
expectedError: "path must be absolute and contain only alphanumeric characters, '/', '_', and '-'"
56+
57+
- name: Should fail if additionalLayerStores path contains invalid characters
58+
initial: |
59+
apiVersion: machineconfiguration.openshift.io/v1
60+
kind: ContainerRuntimeConfig
61+
spec:
62+
containerRuntimeConfig:
63+
additionalLayerStores:
64+
- path: /var/lib/stargz@store
65+
expectedError: "path must be absolute and contain only alphanumeric characters, '/', '_', and '-'"
66+
67+
- name: Should fail if additionalLayerStores path is too long
68+
initial: |
69+
apiVersion: machineconfiguration.openshift.io/v1
70+
kind: ContainerRuntimeConfig
71+
spec:
72+
containerRuntimeConfig:
73+
additionalLayerStores:
74+
- path: /aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
75+
expectedError: "path in body should be at most 256 chars long"
76+
77+
- name: Should fail if additionalLayerStores exceeds maximum of 5 items
78+
initial: |
79+
apiVersion: machineconfiguration.openshift.io/v1
80+
kind: ContainerRuntimeConfig
81+
spec:
82+
containerRuntimeConfig:
83+
additionalLayerStores:
84+
- path: /var/lib/store1
85+
- path: /var/lib/store2
86+
- path: /var/lib/store3
87+
- path: /var/lib/store4
88+
- path: /var/lib/store5
89+
- path: /var/lib/store6
90+
expectedError: "additionalLayerStores in body should have at most 5 items"
91+
92+
- name: Should fail if additionalLayerStores item has no path field
93+
initial: |
94+
apiVersion: machineconfiguration.openshift.io/v1
95+
kind: ContainerRuntimeConfig
96+
spec:
97+
containerRuntimeConfig:
98+
additionalLayerStores:
99+
- {}
100+
expectedError: "spec.containerRuntimeConfig.additionalLayerStores[0] in body should have at least 1 properties"
101+
102+
# AdditionalImageStores - test max items validation (different from layer stores)
103+
- name: Should fail if additionalImageStores exceeds maximum of 10 items
104+
initial: |
105+
apiVersion: machineconfiguration.openshift.io/v1
106+
kind: ContainerRuntimeConfig
107+
spec:
108+
containerRuntimeConfig:
109+
additionalImageStores:
110+
- path: /var/lib/store1
111+
- path: /var/lib/store2
112+
- path: /var/lib/store3
113+
- path: /var/lib/store4
114+
- path: /var/lib/store5
115+
- path: /var/lib/store6
116+
- path: /var/lib/store7
117+
- path: /var/lib/store8
118+
- path: /var/lib/store9
119+
- path: /var/lib/store10
120+
- path: /var/lib/store11
121+
expectedError: "additionalImageStores in body should have at most 10 items"
122+
123+
- name: Should fail if additionalImageStores path is not absolute
124+
initial: |
125+
apiVersion: machineconfiguration.openshift.io/v1
126+
kind: ContainerRuntimeConfig
127+
spec:
128+
containerRuntimeConfig:
129+
additionalImageStores:
130+
- path: var/lib/images
131+
expectedError: "path must be absolute and contain only alphanumeric characters, '/', '.', '_', and '-'"
132+
133+
- name: Should fail if additionalImageStores item has no path field
134+
initial: |
135+
apiVersion: machineconfiguration.openshift.io/v1
136+
kind: ContainerRuntimeConfig
137+
spec:
138+
containerRuntimeConfig:
139+
additionalImageStores:
140+
- {}
141+
expectedError: "spec.containerRuntimeConfig.additionalImageStores[0] in body should have at least 1 properties"
142+
143+
# AdditionalArtifactStores - test max items validation (different from layer stores)
144+
- name: Should fail if additionalArtifactStores exceeds maximum of 10 items
145+
initial: |
146+
apiVersion: machineconfiguration.openshift.io/v1
147+
kind: ContainerRuntimeConfig
148+
spec:
149+
containerRuntimeConfig:
150+
additionalArtifactStores:
151+
- path: /var/lib/store1
152+
- path: /var/lib/store2
153+
- path: /var/lib/store3
154+
- path: /var/lib/store4
155+
- path: /var/lib/store5
156+
- path: /var/lib/store6
157+
- path: /var/lib/store7
158+
- path: /var/lib/store8
159+
- path: /var/lib/store9
160+
- path: /var/lib/store10
161+
- path: /var/lib/store11
162+
expectedError: "additionalArtifactStores in body should have at most 10 items"
163+
164+
- name: Should fail if additionalArtifactStores path is not absolute
165+
initial: |
166+
apiVersion: machineconfiguration.openshift.io/v1
167+
kind: ContainerRuntimeConfig
168+
spec:
169+
containerRuntimeConfig:
170+
additionalArtifactStores:
171+
- path: var/lib/artifacts
172+
expectedError: "path must be absolute and contain only alphanumeric characters, '/', '.', '_', and '-'"
173+
174+
- name: Should fail if additionalArtifactStores item has no path field
175+
initial: |
176+
apiVersion: machineconfiguration.openshift.io/v1
177+
kind: ContainerRuntimeConfig
178+
spec:
179+
containerRuntimeConfig:
180+
additionalArtifactStores:
181+
- {}
182+
expectedError: "spec.containerRuntimeConfig.additionalArtifactStores[0] in body should have at least 1 properties"
183+
184+
# Combined test - all storage types together with other fields
185+
- name: Should be able to create ContainerRuntimeConfig with all storage types and existing fields
186+
initial: |
187+
apiVersion: machineconfiguration.openshift.io/v1
188+
kind: ContainerRuntimeConfig
189+
spec:
190+
containerRuntimeConfig:
191+
defaultRuntime: crun
192+
logLevel: info
193+
additionalLayerStores:
194+
- path: /var/lib/stargz-store
195+
additionalImageStores:
196+
- path: /mnt/nfs-images
197+
- path: /mnt/ssd-images
198+
additionalArtifactStores:
199+
- path: /mnt/ssd-artifacts
200+
expected: |
201+
apiVersion: machineconfiguration.openshift.io/v1
202+
kind: ContainerRuntimeConfig
203+
spec:
204+
containerRuntimeConfig:
205+
defaultRuntime: crun
206+
logLevel: info
207+
additionalLayerStores:
208+
- path: /var/lib/stargz-store
209+
additionalImageStores:
210+
- path: /mnt/nfs-images
211+
- path: /mnt/ssd-images
212+
additionalArtifactStores:
213+
- path: /mnt/ssd-artifacts

‎machineconfiguration/v1/types.go‎

Lines changed: 114 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -892,6 +892,54 @@ type ContainerRuntimeConfiguration struct {
892892
// +kubebuilder:validation:Enum=crun;runc
893893
// +optional
894894
DefaultRuntime ContainerRuntimeDefaultRuntime `json:"defaultRuntime,omitempty"`
895+
896+
// additionalLayerStores configures additional layer store locations.
897+
//
898+
// Stores are checked in order until a layer is found.
899+
// Maximum of 5 stores allowed.
900+
//
901+
// When omitted, no additional layer stores are configured.
902+
//
903+
// +openshift:enable:FeatureGate=AdditionalStorageConfig
904+
// +optional
905+
// +listType=atomic
906+
// +kubebuilder:validation:MinItems=1
907+
// +kubebuilder:validation:MaxItems=5
908+
AdditionalLayerStores []AdditionalLayerStore `json:"additionalLayerStores,omitempty"`
909+
910+
// additionalImageStores configures additional read-only container image store
911+
// locations for complete Open Container Initiative (OCI) images.
912+
//
913+
// Images are checked in order: additional stores first, then default location.
914+
// Stores are read-only.
915+
//
916+
// Maximum of 10 stores allowed.
917+
//
918+
// When omitted, only the default image location is used.
919+
//
920+
// +openshift:enable:FeatureGate=AdditionalStorageConfig
921+
// +optional
922+
// +listType=atomic
923+
// +kubebuilder:validation:MinItems=1
924+
// +kubebuilder:validation:MaxItems=10
925+
AdditionalImageStores []AdditionalImageStore `json:"additionalImageStores,omitempty"`
926+
927+
// additionalArtifactStores configures additional read-only artifact storage
928+
// locations for Open Container Initiative (OCI) volumes.
929+
//
930+
// Artifacts are checked in order: additional stores first, then default location.
931+
// Stores are read-only.
932+
//
933+
// Maximum of 10 stores allowed.
934+
//
935+
// When omitted, only the default artifact location (/var/lib/containers/storage/artifacts/) is used.
936+
//
937+
// +openshift:enable:FeatureGate=AdditionalStorageConfig
938+
// +optional
939+
// +listType=atomic
940+
// +kubebuilder:validation:MinItems=1
941+
// +kubebuilder:validation:MaxItems=10
942+
AdditionalArtifactStores []AdditionalArtifactStore `json:"additionalArtifactStores,omitempty"`
895943
}
896944

897945
type ContainerRuntimeDefaultRuntime string
@@ -904,6 +952,72 @@ const (
904952
ContainerRuntimeDefaultRuntimeDefault = ContainerRuntimeDefaultRuntimeCrun
905953
)
906954

955+
// AdditionalLayerStore defines a storage location for container image layers.
956+
// At least one field must be specified.
957+
// +kubebuilder:validation:MinProperties=1
958+
type AdditionalLayerStore struct {
959+
// path is the absolute path to the additional layer store location.
960+
//
961+
// The path must exist on the node before configuration is applied.
962+
// When a container image is requested, layers found at this location will be used instead of
963+
// retrieving from the registry.
964+
//
965+
// This field is required and must:
966+
// - Have length between 1 and 256 characters
967+
// - Start with '/' (absolute path)
968+
// - Contain only: a-z, A-Z, 0-9, '/', '.', '_', '-' (no spaces or special characters)
969+
//
970+
// +required
971+
// +kubebuilder:validation:MinLength=1
972+
// +kubebuilder:validation:MaxLength=256
973+
// +kubebuilder:validation:XValidation:rule="self.matches('^/[a-zA-Z0-9/._-]+$')",message="path must be absolute and contain only alphanumeric characters, '/', '.', '_', and '-'"
974+
Path string `json:"path,omitempty"`
975+
}
976+
977+
// AdditionalImageStore defines an additional read-only storage location for complete container images.
978+
// At least one field must be specified.
979+
// +kubebuilder:validation:MinProperties=1
980+
type AdditionalImageStore struct {
981+
// path is the absolute path to the additional image store location.
982+
//
983+
// The path must exist on the node before configuration is applied.
984+
// When a container image is requested, images found at this location will be used instead of
985+
// retrieving from the registry.
986+
//
987+
// This field is required and must:
988+
// - Have length between 1 and 256 characters
989+
// - Start with '/' (absolute path)
990+
// - Contain only: a-z, A-Z, 0-9, '/', '.', '_', '-' (no spaces or special characters)
991+
//
992+
// +required
993+
// +kubebuilder:validation:MinLength=1
994+
// +kubebuilder:validation:MaxLength=256
995+
// +kubebuilder:validation:XValidation:rule="self.matches('^/[a-zA-Z0-9/._-]+$')",message="path must be absolute and contain only alphanumeric characters, '/', '.', '_', and '-'"
996+
Path string `json:"path,omitempty"`
997+
}
998+
999+
// AdditionalArtifactStore defines an additional storage location for Open Container Initiative (OCI) artifacts.
1000+
// At least one field must be specified.
1001+
// +kubebuilder:validation:MinProperties=1
1002+
type AdditionalArtifactStore struct {
1003+
// path is the absolute path to the additional artifact store location.
1004+
//
1005+
// The path must exist on the node before configuration is applied.
1006+
// When an Open Container Initiative (OCI) artifact is requested, artifacts found at this location will be used instead of
1007+
// retrieving from the registry.
1008+
//
1009+
// This field is required and must:
1010+
// - Have length between 1 and 256 characters
1011+
// - Start with '/' (absolute path)
1012+
// - Contain only: a-z, A-Z, 0-9, '/', '.', '_', '-' (no spaces or special characters)
1013+
//
1014+
// +required
1015+
// +kubebuilder:validation:MinLength=1
1016+
// +kubebuilder:validation:MaxLength=256
1017+
// +kubebuilder:validation:XValidation:rule="self.matches('^/[a-zA-Z0-9/._-]+$')",message="path must be absolute and contain only alphanumeric characters, '/', '.', '_', and '-'"
1018+
Path string `json:"path,omitempty"`
1019+
}
1020+
9071021
// ContainerRuntimeConfigStatus defines the observed state of a ContainerRuntimeConfig
9081022
type ContainerRuntimeConfigStatus struct {
9091023
// observedGeneration represents the generation observed by the controller.

0 commit comments

Comments
 (0)