From a1e094a1770dd15cd7addb45d86056c008617012 Mon Sep 17 00:00:00 2001 From: Bart Wensley Date: Wed, 13 May 2026 15:59:01 -0400 Subject: [PATCH] OCPBUGS-83865: Revert stalld backend to sched_debug Updating the MC to revert the stalld backend to sched_debug. This is necessary because the new default queue_task backend causes an unacceptable latency penalty (see RHEL-175242). Co-Authored-By: Claude Opus 4.6 Signed-off-by: Bart Wensley --- .../components/machineconfig/machineconfig.go | 10 ++++++++ .../machineconfig/machineconfig_test.go | 24 +++++++++++++++++++ .../functests/1_performance/performance.go | 16 +++++++++++++ ...nshift-bootstrap-master_machineconfig.yaml | 14 +++++++++++ ...nshift-bootstrap-worker_machineconfig.yaml | 14 +++++++++++ ...nshift-bootstrap-master_machineconfig.yaml | 14 +++++++++++ ...nshift-bootstrap-worker_machineconfig.yaml | 14 +++++++++++ .../default/arm/manual_machineconfig.yaml | 14 +++++++++++ .../cpuFrequency/manual_machineconfig.yaml | 14 +++++++++++ .../default/manual_machineconfig.yaml | 14 +++++++++++ .../pp-norps/manual_machineconfig.yaml | 14 +++++++++++ .../manual_machineconfig.yaml | 14 +++++++++++ .../no-ref/manual_machineconfig.yaml | 14 +++++++++++ 13 files changed, 190 insertions(+) diff --git a/pkg/performanceprofile/controller/performanceprofile/components/machineconfig/machineconfig.go b/pkg/performanceprofile/controller/performanceprofile/components/machineconfig/machineconfig.go index 12c01df1ef..4c9eec0393 100644 --- a/pkg/performanceprofile/controller/performanceprofile/components/machineconfig/machineconfig.go +++ b/pkg/performanceprofile/controller/performanceprofile/components/machineconfig/machineconfig.go @@ -391,6 +391,16 @@ func getIgnitionConfig(profile *performancev2.PerformanceProfile, opts *componen addContent(ignitionConfig, content, ovsDynamicPinningTriggerHostFile, &ovsMode) } + // Configure a systemd dropin and sysconfig file so stalld uses sched_debug as its backend. + // The dropin has no effect if stalld is not running, so it is safe to include unconditionally. + // This can potentially be removed when RHEL-175242 is fixed. + stalldSysconfigContent := []byte("BE=\"-b sched_debug\"\n") + stalldSysconfigMode := 0644 + addContent(ignitionConfig, stalldSysconfigContent, "/etc/sysconfig/stalld-backend", &stalldSysconfigMode) + + stalldDropinContent := []byte("[Service]\nEnvironmentFile=/etc/sysconfig/stalld-backend\n") + addContent(ignitionConfig, stalldDropinContent, "/etc/systemd/system/stalld.service.d/stalld-backend.conf", &stalldSysconfigMode) + if opts.MixedCPUsEnabled { // ContainersLimit should be exposed as user-facing API in future updates content, err := renderMixedCPUsConfig(defaultContainersLimit, filepath.Join("configs", mixedCPUsConfig)) diff --git a/pkg/performanceprofile/controller/performanceprofile/components/machineconfig/machineconfig_test.go b/pkg/performanceprofile/controller/performanceprofile/components/machineconfig/machineconfig_test.go index d02bba6916..e65d67a7a3 100644 --- a/pkg/performanceprofile/controller/performanceprofile/components/machineconfig/machineconfig_test.go +++ b/pkg/performanceprofile/controller/performanceprofile/components/machineconfig/machineconfig_test.go @@ -258,6 +258,30 @@ var _ = Describe("Machine Config", func() { }) }) + Context("stalld backend configuration", func() { + It("should include stalld sysconfig and systemd dropin files", func() { + profile := testutils.NewPerformanceProfile("test") + + mc, err := New(profile, &components.MachineConfigOptions{}) + Expect(err).ToNot(HaveOccurred()) + + result := igntypes.Config{} + Expect(json.Unmarshal(mc.Spec.Config.Raw, &result)).To(Succeed()) + + fileContents := map[string]string{} + for _, f := range result.Storage.Files { + Expect(f.Contents.Source).ToNot(BeNil(), "file %s has nil source", f.Path) + base64Data := strings.TrimPrefix(*f.Contents.Source, "data:text/plain;charset=utf-8;base64,") + decoded, err := base64.StdEncoding.DecodeString(base64Data) + Expect(err).ToNot(HaveOccurred()) + fileContents[f.Path] = string(decoded) + } + + Expect(fileContents).To(HaveKeyWithValue("/etc/sysconfig/stalld-backend", "BE=\"-b sched_debug\"\n")) + Expect(fileContents).To(HaveKeyWithValue("/etc/systemd/system/stalld.service.d/stalld-backend.conf", "[Service]\nEnvironmentFile=/etc/sysconfig/stalld-backend\n")) + }) + }) + Context("check listToString ", func() { It("should create string from CPUSet", func() { res := components.ListToString(CPUs) diff --git a/test/e2e/performanceprofile/functests/1_performance/performance.go b/test/e2e/performanceprofile/functests/1_performance/performance.go index 1a5b2060ba..cb8b886248 100644 --- a/test/e2e/performanceprofile/functests/1_performance/performance.go +++ b/test/e2e/performanceprofile/functests/1_performance/performance.go @@ -342,6 +342,22 @@ var _ = Describe("[rfe_id:27368][performance]", Ordered, func() { Expect(tunedExitTime).To(BeNumerically("<", kubeletStartTime), "kubelet started before ocp-tuned-one-shot.service exits") } }) + + It("Should have the stalld service configured with the stalld-backend drop-in", func() { + for _, node := range workerRTNodes { + By("verify that the stalld systemd unit has the stalld-backend drop-in path") + out, err := systemd.ShowProperty(context.TODO(), "stalld.service", "DropInPaths", &node) + Expect(err).ToNot(HaveOccurred()) + Expect(out).To(ContainSubstring("/etc/systemd/system/stalld.service.d/stalld-backend.conf"), + "stalld.service on node %s does not have the stalld-backend.conf drop-in: %s", node.Name, out) + + By("verify that the stalld systemd unit has the stalld-backend environment file") + out, err = systemd.ShowProperty(context.TODO(), "stalld.service", "EnvironmentFiles", &node) + Expect(err).ToNot(HaveOccurred()) + Expect(out).To(ContainSubstring("/etc/sysconfig/stalld-backend"), + "stalld.service on node %s does not have /etc/sysconfig/stalld-backend in EnvironmentFiles: %s", node.Name, out) + } + }) }) Context("Tuned kernel parameters", Label(string(label.Tier0)), func() { diff --git a/test/e2e/performanceprofile/testdata/render-expected-output/bootstrap/extra-mcp/openshift-bootstrap-master_machineconfig.yaml b/test/e2e/performanceprofile/testdata/render-expected-output/bootstrap/extra-mcp/openshift-bootstrap-master_machineconfig.yaml index 4c115aeb9a..60c3734916 100644 --- a/test/e2e/performanceprofile/testdata/render-expected-output/bootstrap/extra-mcp/openshift-bootstrap-master_machineconfig.yaml +++ b/test/e2e/performanceprofile/testdata/render-expected-output/bootstrap/extra-mcp/openshift-bootstrap-master_machineconfig.yaml @@ -104,6 +104,20 @@ spec: mode: 420 path: /var/lib/ovn-ic/etc/enable_dynamic_cpu_affinity user: {} + - contents: + source: data:text/plain;charset=utf-8;base64,QkU9Ii1iIHNjaGVkX2RlYnVnIgo= + verification: {} + group: {} + mode: 420 + path: /etc/sysconfig/stalld-backend + user: {} + - contents: + source: data:text/plain;charset=utf-8;base64,W1NlcnZpY2VdCkVudmlyb25tZW50RmlsZT0vZXRjL3N5c2NvbmZpZy9zdGFsbGQtYmFja2VuZAo= + verification: {} + group: {} + mode: 420 + path: /etc/systemd/system/stalld.service.d/stalld-backend.conf + user: {} systemd: units: - contents: | diff --git a/test/e2e/performanceprofile/testdata/render-expected-output/bootstrap/extra-mcp/openshift-bootstrap-worker_machineconfig.yaml b/test/e2e/performanceprofile/testdata/render-expected-output/bootstrap/extra-mcp/openshift-bootstrap-worker_machineconfig.yaml index ece677a000..cf8b5aee7b 100644 --- a/test/e2e/performanceprofile/testdata/render-expected-output/bootstrap/extra-mcp/openshift-bootstrap-worker_machineconfig.yaml +++ b/test/e2e/performanceprofile/testdata/render-expected-output/bootstrap/extra-mcp/openshift-bootstrap-worker_machineconfig.yaml @@ -104,6 +104,20 @@ spec: mode: 420 path: /var/lib/ovn-ic/etc/enable_dynamic_cpu_affinity user: {} + - contents: + source: data:text/plain;charset=utf-8;base64,QkU9Ii1iIHNjaGVkX2RlYnVnIgo= + verification: {} + group: {} + mode: 420 + path: /etc/sysconfig/stalld-backend + user: {} + - contents: + source: data:text/plain;charset=utf-8;base64,W1NlcnZpY2VdCkVudmlyb25tZW50RmlsZT0vZXRjL3N5c2NvbmZpZy9zdGFsbGQtYmFja2VuZAo= + verification: {} + group: {} + mode: 420 + path: /etc/systemd/system/stalld.service.d/stalld-backend.conf + user: {} systemd: units: - contents: | diff --git a/test/e2e/performanceprofile/testdata/render-expected-output/bootstrap/no-mcp/openshift-bootstrap-master_machineconfig.yaml b/test/e2e/performanceprofile/testdata/render-expected-output/bootstrap/no-mcp/openshift-bootstrap-master_machineconfig.yaml index 4c115aeb9a..60c3734916 100644 --- a/test/e2e/performanceprofile/testdata/render-expected-output/bootstrap/no-mcp/openshift-bootstrap-master_machineconfig.yaml +++ b/test/e2e/performanceprofile/testdata/render-expected-output/bootstrap/no-mcp/openshift-bootstrap-master_machineconfig.yaml @@ -104,6 +104,20 @@ spec: mode: 420 path: /var/lib/ovn-ic/etc/enable_dynamic_cpu_affinity user: {} + - contents: + source: data:text/plain;charset=utf-8;base64,QkU9Ii1iIHNjaGVkX2RlYnVnIgo= + verification: {} + group: {} + mode: 420 + path: /etc/sysconfig/stalld-backend + user: {} + - contents: + source: data:text/plain;charset=utf-8;base64,W1NlcnZpY2VdCkVudmlyb25tZW50RmlsZT0vZXRjL3N5c2NvbmZpZy9zdGFsbGQtYmFja2VuZAo= + verification: {} + group: {} + mode: 420 + path: /etc/systemd/system/stalld.service.d/stalld-backend.conf + user: {} systemd: units: - contents: | diff --git a/test/e2e/performanceprofile/testdata/render-expected-output/bootstrap/no-mcp/openshift-bootstrap-worker_machineconfig.yaml b/test/e2e/performanceprofile/testdata/render-expected-output/bootstrap/no-mcp/openshift-bootstrap-worker_machineconfig.yaml index ece677a000..cf8b5aee7b 100644 --- a/test/e2e/performanceprofile/testdata/render-expected-output/bootstrap/no-mcp/openshift-bootstrap-worker_machineconfig.yaml +++ b/test/e2e/performanceprofile/testdata/render-expected-output/bootstrap/no-mcp/openshift-bootstrap-worker_machineconfig.yaml @@ -104,6 +104,20 @@ spec: mode: 420 path: /var/lib/ovn-ic/etc/enable_dynamic_cpu_affinity user: {} + - contents: + source: data:text/plain;charset=utf-8;base64,QkU9Ii1iIHNjaGVkX2RlYnVnIgo= + verification: {} + group: {} + mode: 420 + path: /etc/sysconfig/stalld-backend + user: {} + - contents: + source: data:text/plain;charset=utf-8;base64,W1NlcnZpY2VdCkVudmlyb25tZW50RmlsZT0vZXRjL3N5c2NvbmZpZy9zdGFsbGQtYmFja2VuZAo= + verification: {} + group: {} + mode: 420 + path: /etc/systemd/system/stalld.service.d/stalld-backend.conf + user: {} systemd: units: - contents: | diff --git a/test/e2e/performanceprofile/testdata/render-expected-output/default/arm/manual_machineconfig.yaml b/test/e2e/performanceprofile/testdata/render-expected-output/default/arm/manual_machineconfig.yaml index 6a8cfbaf06..fa483d570a 100644 --- a/test/e2e/performanceprofile/testdata/render-expected-output/default/arm/manual_machineconfig.yaml +++ b/test/e2e/performanceprofile/testdata/render-expected-output/default/arm/manual_machineconfig.yaml @@ -89,6 +89,20 @@ spec: mode: 420 path: /var/lib/ovn-ic/etc/enable_dynamic_cpu_affinity user: {} + - contents: + source: data:text/plain;charset=utf-8;base64,QkU9Ii1iIHNjaGVkX2RlYnVnIgo= + verification: {} + group: {} + mode: 420 + path: /etc/sysconfig/stalld-backend + user: {} + - contents: + source: data:text/plain;charset=utf-8;base64,W1NlcnZpY2VdCkVudmlyb25tZW50RmlsZT0vZXRjL3N5c2NvbmZpZy9zdGFsbGQtYmFja2VuZAo= + verification: {} + group: {} + mode: 420 + path: /etc/systemd/system/stalld.service.d/stalld-backend.conf + user: {} systemd: units: - contents: | diff --git a/test/e2e/performanceprofile/testdata/render-expected-output/default/cpuFrequency/manual_machineconfig.yaml b/test/e2e/performanceprofile/testdata/render-expected-output/default/cpuFrequency/manual_machineconfig.yaml index 0ab6a4f06d..1e64b3b6a6 100644 --- a/test/e2e/performanceprofile/testdata/render-expected-output/default/cpuFrequency/manual_machineconfig.yaml +++ b/test/e2e/performanceprofile/testdata/render-expected-output/default/cpuFrequency/manual_machineconfig.yaml @@ -89,6 +89,20 @@ spec: mode: 420 path: /var/lib/ovn-ic/etc/enable_dynamic_cpu_affinity user: {} + - contents: + source: data:text/plain;charset=utf-8;base64,QkU9Ii1iIHNjaGVkX2RlYnVnIgo= + verification: {} + group: {} + mode: 420 + path: /etc/sysconfig/stalld-backend + user: {} + - contents: + source: data:text/plain;charset=utf-8;base64,W1NlcnZpY2VdCkVudmlyb25tZW50RmlsZT0vZXRjL3N5c2NvbmZpZy9zdGFsbGQtYmFja2VuZAo= + verification: {} + group: {} + mode: 420 + path: /etc/systemd/system/stalld.service.d/stalld-backend.conf + user: {} systemd: units: - contents: | diff --git a/test/e2e/performanceprofile/testdata/render-expected-output/default/manual_machineconfig.yaml b/test/e2e/performanceprofile/testdata/render-expected-output/default/manual_machineconfig.yaml index 6e78cd7420..6769ff4114 100644 --- a/test/e2e/performanceprofile/testdata/render-expected-output/default/manual_machineconfig.yaml +++ b/test/e2e/performanceprofile/testdata/render-expected-output/default/manual_machineconfig.yaml @@ -90,6 +90,20 @@ spec: mode: 420 path: /var/lib/ovn-ic/etc/enable_dynamic_cpu_affinity user: {} + - contents: + source: data:text/plain;charset=utf-8;base64,QkU9Ii1iIHNjaGVkX2RlYnVnIgo= + verification: {} + group: {} + mode: 420 + path: /etc/sysconfig/stalld-backend + user: {} + - contents: + source: data:text/plain;charset=utf-8;base64,W1NlcnZpY2VdCkVudmlyb25tZW50RmlsZT0vZXRjL3N5c2NvbmZpZy9zdGFsbGQtYmFja2VuZAo= + verification: {} + group: {} + mode: 420 + path: /etc/systemd/system/stalld.service.d/stalld-backend.conf + user: {} systemd: units: - contents: | diff --git a/test/e2e/performanceprofile/testdata/render-expected-output/default/pp-norps/manual_machineconfig.yaml b/test/e2e/performanceprofile/testdata/render-expected-output/default/pp-norps/manual_machineconfig.yaml index 6e78cd7420..6769ff4114 100644 --- a/test/e2e/performanceprofile/testdata/render-expected-output/default/pp-norps/manual_machineconfig.yaml +++ b/test/e2e/performanceprofile/testdata/render-expected-output/default/pp-norps/manual_machineconfig.yaml @@ -90,6 +90,20 @@ spec: mode: 420 path: /var/lib/ovn-ic/etc/enable_dynamic_cpu_affinity user: {} + - contents: + source: data:text/plain;charset=utf-8;base64,QkU9Ii1iIHNjaGVkX2RlYnVnIgo= + verification: {} + group: {} + mode: 420 + path: /etc/sysconfig/stalld-backend + user: {} + - contents: + source: data:text/plain;charset=utf-8;base64,W1NlcnZpY2VdCkVudmlyb25tZW50RmlsZT0vZXRjL3N5c2NvbmZpZy9zdGFsbGQtYmFja2VuZAo= + verification: {} + group: {} + mode: 420 + path: /etc/systemd/system/stalld.service.d/stalld-backend.conf + user: {} systemd: units: - contents: | diff --git a/test/e2e/performanceprofile/testdata/render-expected-output/manual_machineconfig.yaml b/test/e2e/performanceprofile/testdata/render-expected-output/manual_machineconfig.yaml index 86ac5c9d71..1b8d6c8bb6 100644 --- a/test/e2e/performanceprofile/testdata/render-expected-output/manual_machineconfig.yaml +++ b/test/e2e/performanceprofile/testdata/render-expected-output/manual_machineconfig.yaml @@ -116,6 +116,20 @@ spec: mode: 420 path: /var/lib/ovn-ic/etc/enable_dynamic_cpu_affinity user: {} + - contents: + source: data:text/plain;charset=utf-8;base64,QkU9Ii1iIHNjaGVkX2RlYnVnIgo= + verification: {} + group: {} + mode: 420 + path: /etc/sysconfig/stalld-backend + user: {} + - contents: + source: data:text/plain;charset=utf-8;base64,W1NlcnZpY2VdCkVudmlyb25tZW50RmlsZT0vZXRjL3N5c2NvbmZpZy9zdGFsbGQtYmFja2VuZAo= + verification: {} + group: {} + mode: 420 + path: /etc/systemd/system/stalld.service.d/stalld-backend.conf + user: {} systemd: units: - contents: | diff --git a/test/e2e/performanceprofile/testdata/render-expected-output/no-ref/manual_machineconfig.yaml b/test/e2e/performanceprofile/testdata/render-expected-output/no-ref/manual_machineconfig.yaml index c127be204f..cc8fcab611 100644 --- a/test/e2e/performanceprofile/testdata/render-expected-output/no-ref/manual_machineconfig.yaml +++ b/test/e2e/performanceprofile/testdata/render-expected-output/no-ref/manual_machineconfig.yaml @@ -89,6 +89,20 @@ spec: mode: 420 path: /var/lib/ovn-ic/etc/enable_dynamic_cpu_affinity user: {} + - contents: + source: data:text/plain;charset=utf-8;base64,QkU9Ii1iIHNjaGVkX2RlYnVnIgo= + verification: {} + group: {} + mode: 420 + path: /etc/sysconfig/stalld-backend + user: {} + - contents: + source: data:text/plain;charset=utf-8;base64,W1NlcnZpY2VdCkVudmlyb25tZW50RmlsZT0vZXRjL3N5c2NvbmZpZy9zdGFsbGQtYmFja2VuZAo= + verification: {} + group: {} + mode: 420 + path: /etc/systemd/system/stalld.service.d/stalld-backend.conf + user: {} systemd: units: - contents: |