Skip to content

Commit 0fbbf8e

Browse files
saschagrunertsaschagrunert
committed
Add additional storage configuration fields to ContainerRuntimeConfig
This enhancement extends ContainerRuntimeConfig API with three features for improved container storage flexibility: - additionalLayerStores: Enable lazy image pulling via storage plugins (BYOS approach with stargz-snapshotter, nydus-storage-plugin) - additionalImageStores: Read-only container image caches on shared or high-performance storage (NFS, SSD) for faster startup and reduced network overhead - additionalArtifactStores: Configurable OCI artifact storage locations (SSD-backed storage, pre-populated caches, air-gapped deployments) All features target AI/ML workload performance improvements and will ship as Tech Preview in 4.22 behind TechPreviewNoUpgrade feature gate. Signed-off-by: Sascha Grunert <sgrunert@redhat.com>
1 parent 8ef4bf7 commit 0fbbf8e

26 files changed

Lines changed: 3209 additions & 7 deletions

File tree

‎features.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323
| AWSDedicatedHosts| | | <span style="background-color: #519450">Enabled</span> | <span style="background-color: #519450">Enabled</span> | | | <span style="background-color: #519450">Enabled</span> | <span style="background-color: #519450">Enabled</span> |
2424
| AWSDualStackInstall| | | <span style="background-color: #519450">Enabled</span> | <span style="background-color: #519450">Enabled</span> | | | <span style="background-color: #519450">Enabled</span> | <span style="background-color: #519450">Enabled</span> |
2525
| AWSServiceLBNetworkSecurityGroup| | | <span style="background-color: #519450">Enabled</span> | <span style="background-color: #519450">Enabled</span> | | | <span style="background-color: #519450">Enabled</span> | <span style="background-color: #519450">Enabled</span> |
26+
| AdditionalStorageConfig| | | <span style="background-color: #519450">Enabled</span> | <span style="background-color: #519450">Enabled</span> | | | <span style="background-color: #519450">Enabled</span> | <span style="background-color: #519450">Enabled</span> |
2627
| AutomatedEtcdBackup| | | <span style="background-color: #519450">Enabled</span> | <span style="background-color: #519450">Enabled</span> | | | <span style="background-color: #519450">Enabled</span> | <span style="background-color: #519450">Enabled</span> |
2728
| AzureClusterHostedDNSInstall| | | <span style="background-color: #519450">Enabled</span> | <span style="background-color: #519450">Enabled</span> | | | <span style="background-color: #519450">Enabled</span> | <span style="background-color: #519450">Enabled</span> |
2829
| AzureDedicatedHosts| | | <span style="background-color: #519450">Enabled</span> | <span style="background-color: #519450">Enabled</span> | | | <span style="background-color: #519450">Enabled</span> | <span style="background-color: #519450">Enabled</span> |

‎features/features.go‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -361,6 +361,14 @@ var (
361361
enableIn(configv1.Default, configv1.OKD, configv1.DevPreviewNoUpgrade, configv1.TechPreviewNoUpgrade).
362362
mustRegister()
363363

364+
FeatureGateAdditionalStorageConfig = newFeatureGate("AdditionalStorageConfig").
365+
reportProblemsToJiraComponent("node").
366+
contactPerson("saschagrunert").
367+
productScope(ocpSpecific).
368+
enhancementPR("https://github.com/openshift/enhancements/pull/1934").
369+
enableIn(configv1.DevPreviewNoUpgrade, configv1.TechPreviewNoUpgrade).
370+
mustRegister()
371+
364372
FeatureGateUpgradeStatus = newFeatureGate("UpgradeStatus").
365373
reportProblemsToJiraComponent("Cluster Version Operator").
366374
contactPerson("pmuller").

‎machineconfiguration/v1/tests/containerruntimeconfigs.machineconfiguration.openshift.io/AdditionalStorageConfig.yaml‎

Lines changed: 248 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,248 @@
1+
apiVersion: apiextensions.k8s.io/v1 # Hack because controller-gen complains if we don't have this
2+
name: "ContainerRuntimeConfig"
3+
crdName: containerruntimeconfigs.machineconfiguration.openshift.io
4+
featureGates:
5+
- AdditionalStorageConfig
6+
tests:
7+
onCreate:
8+
# AdditionalLayerStores - comprehensive validation tests
9+
- name: Should be able to create ContainerRuntimeConfig with multiple additionalLayerStores
10+
initial: |
11+
apiVersion: machineconfiguration.openshift.io/v1
12+
kind: ContainerRuntimeConfig
13+
spec:
14+
containerRuntimeConfig:
15+
additionalLayerStores:
16+
- path: /var/lib/stargz-store
17+
- path: /mnt/nydus-store
18+
- path: /opt/layer_store-v1.0
19+
expected: |
20+
apiVersion: machineconfiguration.openshift.io/v1
21+
kind: ContainerRuntimeConfig
22+
spec:
23+
containerRuntimeConfig:
24+
additionalLayerStores:
25+
- path: /var/lib/stargz-store
26+
- path: /mnt/nydus-store
27+
- path: /opt/layer_store-v1.0
28+
29+
- name: Should fail if additionalLayerStores path is empty
30+
initial: |
31+
apiVersion: machineconfiguration.openshift.io/v1
32+
kind: ContainerRuntimeConfig
33+
spec:
34+
containerRuntimeConfig:
35+
additionalLayerStores:
36+
- path: ""
37+
expectedError: "path in body should be at least 1 chars long"
38+
39+
- name: Should fail if additionalLayerStores path is not absolute
40+
initial: |
41+
apiVersion: machineconfiguration.openshift.io/v1
42+
kind: ContainerRuntimeConfig
43+
spec:
44+
containerRuntimeConfig:
45+
additionalLayerStores:
46+
- path: var/lib/stargz-store
47+
expectedError: "path must be absolute and contain only alphanumeric characters, '/', '.', '_', and '-'"
48+
49+
- name: Should fail if additionalLayerStores path contains spaces
50+
initial: |
51+
apiVersion: machineconfiguration.openshift.io/v1
52+
kind: ContainerRuntimeConfig
53+
spec:
54+
containerRuntimeConfig:
55+
additionalLayerStores:
56+
- path: /var/lib/stargz store
57+
expectedError: "path must be absolute and contain only alphanumeric characters, '/', '.', '_', and '-'"
58+
59+
- name: Should fail if additionalLayerStores path contains invalid characters
60+
initial: |
61+
apiVersion: machineconfiguration.openshift.io/v1
62+
kind: ContainerRuntimeConfig
63+
spec:
64+
containerRuntimeConfig:
65+
additionalLayerStores:
66+
- path: /var/lib/stargz@store
67+
expectedError: "path must be absolute and contain only alphanumeric characters, '/', '.', '_', and '-'"
68+
69+
- name: Should fail if additionalLayerStores path is too long
70+
initial: |
71+
apiVersion: machineconfiguration.openshift.io/v1
72+
kind: ContainerRuntimeConfig
73+
spec:
74+
containerRuntimeConfig:
75+
additionalLayerStores:
76+
- path: /aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
77+
expectedError: "Too long: may not be more than 256 bytes"
78+
79+
- name: Should fail if additionalLayerStores exceeds maximum of 5 items
80+
initial: |
81+
apiVersion: machineconfiguration.openshift.io/v1
82+
kind: ContainerRuntimeConfig
83+
spec:
84+
containerRuntimeConfig:
85+
additionalLayerStores:
86+
- path: /var/lib/store1
87+
- path: /var/lib/store2
88+
- path: /var/lib/store3
89+
- path: /var/lib/store4
90+
- path: /var/lib/store5
91+
- path: /var/lib/store6
92+
expectedError: "Too many: 6: must have at most 5 items"
93+
94+
- name: Should fail if additionalLayerStores item has no path field
95+
initial: |
96+
apiVersion: machineconfiguration.openshift.io/v1
97+
kind: ContainerRuntimeConfig
98+
spec:
99+
containerRuntimeConfig:
100+
additionalLayerStores:
101+
- {}
102+
expectedError: "path: Required value"
103+
104+
- name: Should fail if additionalLayerStores contains duplicate paths
105+
initial: |
106+
apiVersion: machineconfiguration.openshift.io/v1
107+
kind: ContainerRuntimeConfig
108+
spec:
109+
containerRuntimeConfig:
110+
additionalLayerStores:
111+
- path: /var/lib/stargz-store
112+
- path: /var/lib/stargz-store
113+
expectedError: "additionalLayerStores must not contain duplicate paths"
114+
115+
# AdditionalImageStores - test max items validation (different from layer stores)
116+
- name: Should fail if additionalImageStores exceeds maximum of 10 items
117+
initial: |
118+
apiVersion: machineconfiguration.openshift.io/v1
119+
kind: ContainerRuntimeConfig
120+
spec:
121+
containerRuntimeConfig:
122+
additionalImageStores:
123+
- path: /var/lib/store1
124+
- path: /var/lib/store2
125+
- path: /var/lib/store3
126+
- path: /var/lib/store4
127+
- path: /var/lib/store5
128+
- path: /var/lib/store6
129+
- path: /var/lib/store7
130+
- path: /var/lib/store8
131+
- path: /var/lib/store9
132+
- path: /var/lib/store10
133+
- path: /var/lib/store11
134+
expectedError: "Too many: 11: must have at most 10 items"
135+
136+
- name: Should fail if additionalImageStores path is not absolute
137+
initial: |
138+
apiVersion: machineconfiguration.openshift.io/v1
139+
kind: ContainerRuntimeConfig
140+
spec:
141+
containerRuntimeConfig:
142+
additionalImageStores:
143+
- path: var/lib/images
144+
expectedError: "path must be absolute and contain only alphanumeric characters, '/', '.', '_', and '-'"
145+
146+
- name: Should fail if additionalImageStores item has no path field
147+
initial: |
148+
apiVersion: machineconfiguration.openshift.io/v1
149+
kind: ContainerRuntimeConfig
150+
spec:
151+
containerRuntimeConfig:
152+
additionalImageStores:
153+
- {}
154+
expectedError: "path: Required value"
155+
156+
- name: Should fail if additionalImageStores contains duplicate paths
157+
initial: |
158+
apiVersion: machineconfiguration.openshift.io/v1
159+
kind: ContainerRuntimeConfig
160+
spec:
161+
containerRuntimeConfig:
162+
additionalImageStores:
163+
- path: /mnt/nfs-images
164+
- path: /mnt/nfs-images
165+
expectedError: "additionalImageStores must not contain duplicate paths"
166+
167+
# AdditionalArtifactStores - test max items validation (different from layer stores)
168+
- name: Should fail if additionalArtifactStores exceeds maximum of 10 items
169+
initial: |
170+
apiVersion: machineconfiguration.openshift.io/v1
171+
kind: ContainerRuntimeConfig
172+
spec:
173+
containerRuntimeConfig:
174+
additionalArtifactStores:
175+
- path: /var/lib/store1
176+
- path: /var/lib/store2
177+
- path: /var/lib/store3
178+
- path: /var/lib/store4
179+
- path: /var/lib/store5
180+
- path: /var/lib/store6
181+
- path: /var/lib/store7
182+
- path: /var/lib/store8
183+
- path: /var/lib/store9
184+
- path: /var/lib/store10
185+
- path: /var/lib/store11
186+
expectedError: "Too many: 11: must have at most 10 items"
187+
188+
- name: Should fail if additionalArtifactStores path is not absolute
189+
initial: |
190+
apiVersion: machineconfiguration.openshift.io/v1
191+
kind: ContainerRuntimeConfig
192+
spec:
193+
containerRuntimeConfig:
194+
additionalArtifactStores:
195+
- path: var/lib/artifacts
196+
expectedError: "path must be absolute and contain only alphanumeric characters, '/', '.', '_', and '-'"
197+
198+
- name: Should fail if additionalArtifactStores item has no path field
199+
initial: |
200+
apiVersion: machineconfiguration.openshift.io/v1
201+
kind: ContainerRuntimeConfig
202+
spec:
203+
containerRuntimeConfig:
204+
additionalArtifactStores:
205+
- {}
206+
expectedError: "path: Required value"
207+
208+
- name: Should fail if additionalArtifactStores contains duplicate paths
209+
initial: |
210+
apiVersion: machineconfiguration.openshift.io/v1
211+
kind: ContainerRuntimeConfig
212+
spec:
213+
containerRuntimeConfig:
214+
additionalArtifactStores:
215+
- path: /mnt/ssd-artifacts
216+
- path: /mnt/ssd-artifacts
217+
expectedError: "additionalArtifactStores must not contain duplicate paths"
218+
219+
# Combined test - all storage types together with other fields
220+
- name: Should be able to create ContainerRuntimeConfig with all storage types and existing fields
221+
initial: |
222+
apiVersion: machineconfiguration.openshift.io/v1
223+
kind: ContainerRuntimeConfig
224+
spec:
225+
containerRuntimeConfig:
226+
defaultRuntime: crun
227+
logLevel: info
228+
additionalLayerStores:
229+
- path: /var/lib/stargz-store
230+
additionalImageStores:
231+
- path: /mnt/nfs-images
232+
- path: /mnt/ssd-images
233+
additionalArtifactStores:
234+
- path: /mnt/ssd-artifacts
235+
expected: |
236+
apiVersion: machineconfiguration.openshift.io/v1
237+
kind: ContainerRuntimeConfig
238+
spec:
239+
containerRuntimeConfig:
240+
defaultRuntime: crun
241+
logLevel: info
242+
additionalLayerStores:
243+
- path: /var/lib/stargz-store
244+
additionalImageStores:
245+
- path: /mnt/nfs-images
246+
- path: /mnt/ssd-images
247+
additionalArtifactStores:
248+
- path: /mnt/ssd-artifacts

‎machineconfiguration/v1/types.go‎

Lines changed: 114 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -892,6 +892,60 @@ type ContainerRuntimeConfiguration struct {
892892
// +kubebuilder:validation:Enum=crun;runc
893893
// +optional
894894
DefaultRuntime ContainerRuntimeDefaultRuntime `json:"defaultRuntime,omitempty"`
895+
896+
// additionalLayerStores configures additional read-only container image layer store locations for Open Container Initiative (OCI) images.
897+
//
898+
// Layers are checked in order: additional stores first, then the default location.
899+
// Stores are read-only.
900+
// Maximum of 5 stores allowed.
901+
// Each path must be unique.
902+
//
903+
// When omitted, only the default layer location is used.
904+
// When specified, at least one store must be provided.
905+
//
906+
// +openshift:enable:FeatureGate=AdditionalStorageConfig
907+
// +optional
908+
// +listType=atomic
909+
// +kubebuilder:validation:MinItems=1
910+
// +kubebuilder:validation:MaxItems=5
911+
// +kubebuilder:validation:XValidation:rule="self.all(x, self.exists_one(y, x.path == y.path))",message="additionalLayerStores must not contain duplicate paths"
912+
AdditionalLayerStores []AdditionalLayerStore `json:"additionalLayerStores,omitempty"`
913+
914+
// additionalImageStores configures additional read-only container image store locations for Open Container Initiative (OCI) images.
915+
//
916+
// Images are checked in order: additional stores first, then the default location.
917+
// Stores are read-only.
918+
// Maximum of 10 stores allowed.
919+
// Each path must be unique.
920+
//
921+
// When omitted, only the default image location is used.
922+
// When specified, at least one store must be provided.
923+
//
924+
// +openshift:enable:FeatureGate=AdditionalStorageConfig
925+
// +optional
926+
// +listType=atomic
927+
// +kubebuilder:validation:MinItems=1
928+
// +kubebuilder:validation:MaxItems=10
929+
// +kubebuilder:validation:XValidation:rule="self.all(x, self.exists_one(y, x.path == y.path))",message="additionalImageStores must not contain duplicate paths"
930+
AdditionalImageStores []AdditionalImageStore `json:"additionalImageStores,omitempty"`
931+
932+
// additionalArtifactStores configures additional read-only artifact storage locations for Open Container Initiative (OCI) artifacts.
933+
//
934+
// Artifacts are checked in order: additional stores first, then the default location (/var/lib/containers/storage/artifacts).
935+
// Stores are read-only.
936+
// Maximum of 10 stores allowed.
937+
// Each path must be unique.
938+
//
939+
// When omitted, only the default artifact location is used.
940+
// When specified, at least one store must be provided.
941+
//
942+
// +openshift:enable:FeatureGate=AdditionalStorageConfig
943+
// +optional
944+
// +listType=atomic
945+
// +kubebuilder:validation:MinItems=1
946+
// +kubebuilder:validation:MaxItems=10
947+
// +kubebuilder:validation:XValidation:rule="self.all(x, self.exists_one(y, x.path == y.path))",message="additionalArtifactStores must not contain duplicate paths"
948+
AdditionalArtifactStores []AdditionalArtifactStore `json:"additionalArtifactStores,omitempty"`
895949
}
896950

897951
type ContainerRuntimeDefaultRuntime string
@@ -904,6 +958,66 @@ const (
904958
ContainerRuntimeDefaultRuntimeDefault = ContainerRuntimeDefaultRuntimeCrun
905959
)
906960

961+
// AdditionalLayerStore defines a read-only storage location for Open Container Initiative (OCI) container image layers.
962+
type AdditionalLayerStore struct {
963+
// path specifies the absolute location of the additional layer store.
964+
//
965+
// The path must exist on the node before configuration is applied.
966+
// When a container image is requested, layers found at this location will be used instead of
967+
// retrieving from the registry.
968+
//
969+
// This field is required and must:
970+
// - Have length between 1 and 256 characters
971+
// - Start with '/' (absolute path)
972+
// - Contain only: a-z, A-Z, 0-9, '/', '.', '_', '-' (no spaces or special characters)
973+
//
974+
// +required
975+
// +kubebuilder:validation:MinLength=1
976+
// +kubebuilder:validation:MaxLength=256
977+
// +kubebuilder:validation:XValidation:rule="self.matches('^/[a-zA-Z0-9/._-]+$')",message="path must be absolute and contain only alphanumeric characters, '/', '.', '_', and '-'"
978+
Path string `json:"path,omitempty"`
979+
}
980+
981+
// AdditionalImageStore defines an additional read-only storage location for Open Container Initiative (OCI) images.
982+
type AdditionalImageStore struct {
983+
// path specifies the absolute location of the additional image store.
984+
//
985+
// The path must exist on the node before configuration is applied.
986+
// When a container image is requested, images found at this location will be used instead of
987+
// retrieving from the registry.
988+
//
989+
// This field is required and must:
990+
// - Have length between 1 and 256 characters
991+
// - Start with '/' (absolute path)
992+
// - Contain only: a-z, A-Z, 0-9, '/', '.', '_', '-' (no spaces or special characters)
993+
//
994+
// +required
995+
// +kubebuilder:validation:MinLength=1
996+
// +kubebuilder:validation:MaxLength=256
997+
// +kubebuilder:validation:XValidation:rule="self.matches('^/[a-zA-Z0-9/._-]+$')",message="path must be absolute and contain only alphanumeric characters, '/', '.', '_', and '-'"
998+
Path string `json:"path,omitempty"`
999+
}
1000+
1001+
// AdditionalArtifactStore defines an additional read-only storage location for Open Container Initiative (OCI) artifacts.
1002+
type AdditionalArtifactStore struct {
1003+
// path specifies the absolute location of the additional artifact store.
1004+
//
1005+
// The path must exist on the node before configuration is applied.
1006+
// When an artifact is requested, artifacts found at this location will be used instead of
1007+
// retrieving from the registry.
1008+
//
1009+
// This field is required and must:
1010+
// - Have length between 1 and 256 characters
1011+
// - Start with '/' (absolute path)
1012+
// - Contain only: a-z, A-Z, 0-9, '/', '.', '_', '-' (no spaces or special characters)
1013+
//
1014+
// +required
1015+
// +kubebuilder:validation:MinLength=1
1016+
// +kubebuilder:validation:MaxLength=256
1017+
// +kubebuilder:validation:XValidation:rule="self.matches('^/[a-zA-Z0-9/._-]+$')",message="path must be absolute and contain only alphanumeric characters, '/', '.', '_', and '-'"
1018+
Path string `json:"path,omitempty"`
1019+
}
1020+
9071021
// ContainerRuntimeConfigStatus defines the observed state of a ContainerRuntimeConfig
9081022
type ContainerRuntimeConfigStatus struct {
9091023
// observedGeneration represents the generation observed by the controller.

0 commit comments

Comments
 (0)