From ad8e5cdb1beac4c72e65166d820e7f6640145ef7 Mon Sep 17 00:00:00 2001 From: Valeriy Khorunzhin Date: Mon, 23 Mar 2026 11:17:25 +0300 Subject: [PATCH 1/8] unscheduled hotplug pod Signed-off-by: Valeriy Khorunzhin --- api/core/v1alpha2/vmbdacondition/condition.go | 2 ++ .../controller/vmbda/internal/life_cycle.go | 15 ++++++++ .../internal/service/attachment_service.go | 34 +++++++++++++++++++ 3 files changed, 51 insertions(+) diff --git a/api/core/v1alpha2/vmbdacondition/condition.go b/api/core/v1alpha2/vmbdacondition/condition.go index 19d30e18cd..628b0c9fd7 100644 --- a/api/core/v1alpha2/vmbdacondition/condition.go +++ b/api/core/v1alpha2/vmbdacondition/condition.go @@ -65,6 +65,8 @@ const ( Conflict AttachedReason = "Conflict" // DeviceNotAvailableOnNode indicates that the block device's PersistentVolume is not available on the node where the virtual machine is running. DeviceNotAvailableOnNode AttachedReason = "DeviceNotAvailableOnNode" + // HotPlugPodNotScheduled indicates that the hotplug pod cannot be scheduled on any node. + HotPlugPodNotScheduled AttachedReason = "HotPlugPodNotScheduled" // CapacityAvailable signifies that the capacity not reached and attaching available. CapacityAvailable DiskAttachmentCapacityAvailableReason = "CapacityAvailable" diff --git a/images/virtualization-artifact/pkg/controller/vmbda/internal/life_cycle.go b/images/virtualization-artifact/pkg/controller/vmbda/internal/life_cycle.go index d12c94f99d..4aff763e21 100644 --- a/images/virtualization-artifact/pkg/controller/vmbda/internal/life_cycle.go +++ b/images/virtualization-artifact/pkg/controller/vmbda/internal/life_cycle.go @@ -22,6 +22,7 @@ import ( "fmt" "time" + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" virtv1 "kubevirt.io/api/core/v1" "sigs.k8s.io/controller-runtime/pkg/reconcile" @@ -199,6 +200,20 @@ func (h LifeCycleHandler) Handle(ctx context.Context, vmbda *v1alpha2.VirtualMac if err != nil { if errors.Is(err, intsvc.ErrVolumeStatusNotReady) { vmbda.Status.Phase = v1alpha2.BlockDeviceAttachmentPhaseInProgress + + podScheduled, podErr := h.attacher.GetHotPlugPodCondition(ctx, ad, kvvmi, corev1.PodScheduled) + if podErr != nil { + return reconcile.Result{}, podErr + } + if podScheduled != nil && podScheduled.Status == corev1.ConditionFalse && podScheduled.Message != "" { + vmbda.Status.Phase = v1alpha2.BlockDeviceAttachmentPhasePending + cb. + Status(metav1.ConditionFalse). + Reason(vmbdacondition.HotPlugPodNotScheduled). + Message(fmt.Sprintf("%s: %s", podScheduled.Reason, podScheduled.Message)) + return reconcile.Result{}, nil + } + cb. Status(metav1.ConditionFalse). Reason(vmbdacondition.AttachmentRequestSent). diff --git a/images/virtualization-artifact/pkg/controller/vmbda/internal/service/attachment_service.go b/images/virtualization-artifact/pkg/controller/vmbda/internal/service/attachment_service.go index 6032079351..24f1f77420 100644 --- a/images/virtualization-artifact/pkg/controller/vmbda/internal/service/attachment_service.go +++ b/images/virtualization-artifact/pkg/controller/vmbda/internal/service/attachment_service.go @@ -314,6 +314,40 @@ func (s AttachmentService) IsPVAvailableOnVMNode(ctx context.Context, pvc *corev return true, nil } +func (s AttachmentService) GetHotPlugPodCondition(ctx context.Context, ad *AttachmentDisk, kvvmi *virtv1.VirtualMachineInstance, condType corev1.PodConditionType) (*corev1.PodCondition, error) { + if ad == nil || kvvmi == nil { + return nil, nil + } + + for _, vs := range kvvmi.Status.VolumeStatus { + if vs.HotplugVolume == nil || vs.Name != ad.GenerateName { + continue + } + if vs.HotplugVolume.AttachPodName == "" { + return nil, nil + } + + pod, err := object.FetchObject(ctx, types.NamespacedName{ + Namespace: kvvmi.Namespace, + Name: vs.HotplugVolume.AttachPodName, + }, s.client, &corev1.Pod{}) + if err != nil { + return nil, err + } + if pod == nil { + return nil, nil + } + + for i, c := range pod.Status.Conditions { + if c.Type == condType { + return &pod.Status.Conditions[i], nil + } + } + return nil, nil + } + return nil, nil +} + func isSameBlockDeviceRefs(a, b v1alpha2.VMBDAObjectRef) bool { return a.Kind == b.Kind && a.Name == b.Name } From 92d78c286cf3701ea950c7fb537100b96bd92145 Mon Sep 17 00:00:00 2001 From: Valeriy Khorunzhin Date: Mon, 23 Mar 2026 12:13:06 +0300 Subject: [PATCH 2/8] failed attachment Signed-off-by: Valeriy Khorunzhin --- api/core/v1alpha2/vmbdacondition/condition.go | 2 + api/core/v1alpha2/vmcondition/condition.go | 1 + .../pkg/controller/vm/internal/lifecycle.go | 29 ++++ .../controller/vmbda/internal/life_cycle.go | 70 ++++++++-- .../internal/service/attachment_service.go | 57 ++++++-- .../internal/watcher/volumeevent_watcher.go | 127 ++++++++++++++++++ .../pkg/controller/vmbda/vmbda_reconciler.go | 1 + 7 files changed, 263 insertions(+), 24 deletions(-) create mode 100644 images/virtualization-artifact/pkg/controller/vmbda/internal/watcher/volumeevent_watcher.go diff --git a/api/core/v1alpha2/vmbdacondition/condition.go b/api/core/v1alpha2/vmbdacondition/condition.go index 628b0c9fd7..a77a61aff6 100644 --- a/api/core/v1alpha2/vmbdacondition/condition.go +++ b/api/core/v1alpha2/vmbdacondition/condition.go @@ -67,6 +67,8 @@ const ( DeviceNotAvailableOnNode AttachedReason = "DeviceNotAvailableOnNode" // HotPlugPodNotScheduled indicates that the hotplug pod cannot be scheduled on any node. HotPlugPodNotScheduled AttachedReason = "HotPlugPodNotScheduled" + // FailedAttachVolume indicates that the hotplug pod failed to attach a volume. + FailedAttachVolume AttachedReason = "FailedAttachVolume" // CapacityAvailable signifies that the capacity not reached and attaching available. CapacityAvailable DiskAttachmentCapacityAvailableReason = "CapacityAvailable" diff --git a/api/core/v1alpha2/vmcondition/condition.go b/api/core/v1alpha2/vmcondition/condition.go index 6abfecbe0c..f757457b04 100644 --- a/api/core/v1alpha2/vmcondition/condition.go +++ b/api/core/v1alpha2/vmcondition/condition.go @@ -169,6 +169,7 @@ const ( ReasonVirtualMachineRunning RunningReason = "Running" ReasonInternalVirtualMachineError RunningReason = "InternalVirtualMachineError" ReasonPodNotStarted RunningReason = "PodNotStarted" + ReasonPodContainerCreating RunningReason = "PodContainerCreating" ReasonPodTerminating RunningReason = "PodTerminating" ReasonPodNotFound RunningReason = "PodNotFound" ReasonPodConditionMissing RunningReason = "PodConditionMissing" diff --git a/images/virtualization-artifact/pkg/controller/vm/internal/lifecycle.go b/images/virtualization-artifact/pkg/controller/vm/internal/lifecycle.go index 347068a476..87f4804683 100644 --- a/images/virtualization-artifact/pkg/controller/vm/internal/lifecycle.go +++ b/images/virtualization-artifact/pkg/controller/vm/internal/lifecycle.go @@ -131,6 +131,14 @@ func (h *LifeCycleHandler) syncRunning(ctx context.Context, vm *v1alpha2.Virtual return } + if containerCreatingPod := h.findContainerCreatingPod(ctx, vm, log); containerCreatingPod != nil { + cb.Status(metav1.ConditionFalse). + Reason(vmcondition.ReasonPodContainerCreating). + Message(fmt.Sprintf("Pod %q is in ContainerCreating phase. Check the pod for more details.", containerCreatingPod.Name)) + conditions.SetCondition(cb, &vm.Status.Conditions) + return + } + if kvvm != nil { podScheduled := service.GetKVVMCondition(string(corev1.PodScheduled), kvvm.Status.Conditions) if podScheduled != nil && podScheduled.Status == corev1.ConditionFalse { @@ -236,6 +244,27 @@ func (h *LifeCycleHandler) checkPodVolumeErrors(ctx context.Context, vm *v1alpha return nil } +func (h *LifeCycleHandler) findContainerCreatingPod(ctx context.Context, vm *v1alpha2.VirtualMachine, log *slog.Logger) *corev1.Pod { + var podList corev1.PodList + err := h.client.List(ctx, &podList, &client.ListOptions{ + Namespace: vm.Namespace, + LabelSelector: labels.SelectorFromSet(map[string]string{ + virtv1.VirtualMachineNameLabel: vm.Name, + }), + }) + if err != nil { + log.Error("Failed to list pods", "error", err) + return nil + } + + for i := range podList.Items { + if isContainerCreating(&podList.Items[i]) { + return &podList.Items[i] + } + } + return nil +} + func isContainerCreating(pod *corev1.Pod) bool { if pod.Status.Phase != corev1.PodPending { return false diff --git a/images/virtualization-artifact/pkg/controller/vmbda/internal/life_cycle.go b/images/virtualization-artifact/pkg/controller/vmbda/internal/life_cycle.go index 4aff763e21..427e5d87ac 100644 --- a/images/virtualization-artifact/pkg/controller/vmbda/internal/life_cycle.go +++ b/images/virtualization-artifact/pkg/controller/vmbda/internal/life_cycle.go @@ -201,17 +201,8 @@ func (h LifeCycleHandler) Handle(ctx context.Context, vmbda *v1alpha2.VirtualMac if errors.Is(err, intsvc.ErrVolumeStatusNotReady) { vmbda.Status.Phase = v1alpha2.BlockDeviceAttachmentPhaseInProgress - podScheduled, podErr := h.attacher.GetHotPlugPodCondition(ctx, ad, kvvmi, corev1.PodScheduled) - if podErr != nil { - return reconcile.Result{}, podErr - } - if podScheduled != nil && podScheduled.Status == corev1.ConditionFalse && podScheduled.Message != "" { - vmbda.Status.Phase = v1alpha2.BlockDeviceAttachmentPhasePending - cb. - Status(metav1.ConditionFalse). - Reason(vmbdacondition.HotPlugPodNotScheduled). - Message(fmt.Sprintf("%s: %s", podScheduled.Reason, podScheduled.Message)) - return reconcile.Result{}, nil + if result, handled, podErr := h.handleHotPlugPodIssues(ctx, ad, kvvmi, vmbda, cb); podErr != nil || handled { + return result, podErr } cb. @@ -315,3 +306,60 @@ func (h LifeCycleHandler) Handle(ctx context.Context, vmbda *v1alpha2.VirtualMac return reconcile.Result{}, err } } + +const reasonFailedAttachVolume = "FailedAttachVolume" + +func (h LifeCycleHandler) handleHotPlugPodIssues( + ctx context.Context, + ad *intsvc.AttachmentDisk, + kvvmi *virtv1.VirtualMachineInstance, + vmbda *v1alpha2.VirtualMachineBlockDeviceAttachment, + cb *conditions.ConditionBuilder, +) (reconcile.Result, bool, error) { + hotPlugPod, err := h.attacher.GetHotPlugPod(ctx, ad, kvvmi) + if err != nil { + return reconcile.Result{}, false, err + } + if hotPlugPod == nil { + return reconcile.Result{}, false, nil + } + + for _, c := range hotPlugPod.Status.Conditions { + if c.Type == corev1.PodScheduled && c.Status == corev1.ConditionFalse && c.Message != "" { + vmbda.Status.Phase = v1alpha2.BlockDeviceAttachmentPhasePending + cb. + Status(metav1.ConditionFalse). + Reason(vmbdacondition.HotPlugPodNotScheduled). + Message(fmt.Sprintf("%s: %s", c.Reason, c.Message)) + return reconcile.Result{}, true, nil + } + } + + if isContainerCreating(hotPlugPod) { + lastEvent, err := h.attacher.GetLastPodEvent(ctx, hotPlugPod) + if err != nil { + return reconcile.Result{}, false, err + } + if lastEvent != nil && lastEvent.Reason == reasonFailedAttachVolume { + cb. + Status(metav1.ConditionFalse). + Reason(vmbdacondition.FailedAttachVolume). + Message(fmt.Sprintf("%s: %s", lastEvent.Reason, lastEvent.Message)) + return reconcile.Result{}, true, nil + } + } + + return reconcile.Result{}, false, nil +} + +func isContainerCreating(pod *corev1.Pod) bool { + if pod.Status.Phase != corev1.PodPending { + return false + } + for _, cs := range pod.Status.ContainerStatuses { + if cs.State.Waiting != nil && cs.State.Waiting.Reason == "ContainerCreating" { + return true + } + } + return false +} diff --git a/images/virtualization-artifact/pkg/controller/vmbda/internal/service/attachment_service.go b/images/virtualization-artifact/pkg/controller/vmbda/internal/service/attachment_service.go index 24f1f77420..c97fa2f25d 100644 --- a/images/virtualization-artifact/pkg/controller/vmbda/internal/service/attachment_service.go +++ b/images/virtualization-artifact/pkg/controller/vmbda/internal/service/attachment_service.go @@ -20,9 +20,11 @@ import ( "context" "errors" "fmt" + "slices" "strings" corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/fields" "k8s.io/apimachinery/pkg/types" "k8s.io/component-helpers/scheduling/corev1/nodeaffinity" virtv1 "kubevirt.io/api/core/v1" @@ -314,7 +316,7 @@ func (s AttachmentService) IsPVAvailableOnVMNode(ctx context.Context, pvc *corev return true, nil } -func (s AttachmentService) GetHotPlugPodCondition(ctx context.Context, ad *AttachmentDisk, kvvmi *virtv1.VirtualMachineInstance, condType corev1.PodConditionType) (*corev1.PodCondition, error) { +func (s AttachmentService) GetHotPlugPod(ctx context.Context, ad *AttachmentDisk, kvvmi *virtv1.VirtualMachineInstance) (*corev1.Pod, error) { if ad == nil || kvvmi == nil { return nil, nil } @@ -327,27 +329,56 @@ func (s AttachmentService) GetHotPlugPodCondition(ctx context.Context, ad *Attac return nil, nil } - pod, err := object.FetchObject(ctx, types.NamespacedName{ + return object.FetchObject(ctx, types.NamespacedName{ Namespace: kvvmi.Namespace, Name: vs.HotplugVolume.AttachPodName, }, s.client, &corev1.Pod{}) - if err != nil { - return nil, err - } - if pod == nil { - return nil, nil - } + } + return nil, nil +} - for i, c := range pod.Status.Conditions { - if c.Type == condType { - return &pod.Status.Conditions[i], nil - } +func (s AttachmentService) GetHotPlugPodCondition(ctx context.Context, ad *AttachmentDisk, kvvmi *virtv1.VirtualMachineInstance, condType corev1.PodConditionType) (*corev1.PodCondition, error) { + pod, err := s.GetHotPlugPod(ctx, ad, kvvmi) + if err != nil || pod == nil { + return nil, err + } + + for i, c := range pod.Status.Conditions { + if c.Type == condType { + return &pod.Status.Conditions[i], nil } - return nil, nil } return nil, nil } +func (s AttachmentService) GetLastPodEvent(ctx context.Context, pod *corev1.Pod) (*corev1.Event, error) { + if pod == nil { + return nil, nil + } + + eventList := &corev1.EventList{} + err := s.client.List(ctx, eventList, &client.ListOptions{ + Namespace: pod.Namespace, + FieldSelector: fields.SelectorFromSet(fields.Set{ + "involvedObject.name": pod.Name, + "involvedObject.kind": "Pod", + }), + }) + if err != nil { + return nil, err + } + + if len(eventList.Items) == 0 { + return nil, nil + } + + last := slices.MaxFunc(eventList.Items, func(a, b corev1.Event) int { + return a.LastTimestamp.Time.Compare(b.LastTimestamp.Time) + }) + + return &last, nil +} + func isSameBlockDeviceRefs(a, b v1alpha2.VMBDAObjectRef) bool { return a.Kind == b.Kind && a.Name == b.Name } diff --git a/images/virtualization-artifact/pkg/controller/vmbda/internal/watcher/volumeevent_watcher.go b/images/virtualization-artifact/pkg/controller/vmbda/internal/watcher/volumeevent_watcher.go new file mode 100644 index 0000000000..e3c8b6ede7 --- /dev/null +++ b/images/virtualization-artifact/pkg/controller/vmbda/internal/watcher/volumeevent_watcher.go @@ -0,0 +1,127 @@ +/* +Copyright 2026 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package watcher + +import ( + "context" + "fmt" + "log/slog" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/types" + virtv1 "kubevirt.io/api/core/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/predicate" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + "sigs.k8s.io/controller-runtime/pkg/source" + + "github.com/deckhouse/virtualization-controller/pkg/controller/kvbuilder" + "github.com/deckhouse/virtualization/api/core/v1alpha2" +) + +const reasonFailedAttachVolume = "FailedAttachVolume" + +type VolumeEventWatcher struct { + client client.Client +} + +func NewVolumeEventWatcher(client client.Client) *VolumeEventWatcher { + return &VolumeEventWatcher{client: client} +} + +func (w *VolumeEventWatcher) Watch(mgr manager.Manager, ctr controller.Controller) error { + if err := ctr.Watch( + source.Kind( + mgr.GetCache(), + &corev1.Event{}, + handler.TypedEnqueueRequestsFromMapFunc(w.enqueueVMBDAs), + predicate.TypedFuncs[*corev1.Event]{ + CreateFunc: func(e event.TypedCreateEvent[*corev1.Event]) bool { + return isFailedAttachVolumeEvent(e.Object) + }, + UpdateFunc: func(e event.TypedUpdateEvent[*corev1.Event]) bool { + return isFailedAttachVolumeEvent(e.ObjectNew) + }, + DeleteFunc: func(_ event.TypedDeleteEvent[*corev1.Event]) bool { + return false + }, + }, + ), + ); err != nil { + return fmt.Errorf("error setting watch on Event: %w", err) + } + return nil +} + +func isFailedAttachVolumeEvent(e *corev1.Event) bool { + return e.InvolvedObject.Kind == "Pod" && + e.Type == corev1.EventTypeWarning && + e.Reason == reasonFailedAttachVolume +} + +func (w *VolumeEventWatcher) enqueueVMBDAs(ctx context.Context, e *corev1.Event) []reconcile.Request { + if e.InvolvedObject.Kind != "Pod" { + return nil + } + + ns := e.InvolvedObject.Namespace + podName := e.InvolvedObject.Name + + var kvvmiList virtv1.VirtualMachineInstanceList + if err := w.client.List(ctx, &kvvmiList, &client.ListOptions{Namespace: ns}); err != nil { + slog.Default().Error(fmt.Sprintf("failed to list kvvmis: %s", err)) + return nil + } + + for _, kvvmi := range kvvmiList.Items { + for _, vs := range kvvmi.Status.VolumeStatus { + if vs.HotplugVolume == nil || vs.HotplugVolume.AttachPodName != podName { + continue + } + + name, kind := kvbuilder.GetOriginalDiskName(vs.Name) + if kind == "" { + continue + } + + var vmbdas v1alpha2.VirtualMachineBlockDeviceAttachmentList + if err := w.client.List(ctx, &vmbdas, &client.ListOptions{Namespace: ns}); err != nil { + slog.Default().Error(fmt.Sprintf("failed to list vmbdas: %s", err)) + return nil + } + + var requests []reconcile.Request + for _, vmbda := range vmbdas.Items { + if vmbda.Spec.BlockDeviceRef.Name == name { + requests = append(requests, reconcile.Request{ + NamespacedName: types.NamespacedName{ + Namespace: vmbda.Namespace, + Name: vmbda.Name, + }, + }) + } + } + return requests + } + } + + return nil +} diff --git a/images/virtualization-artifact/pkg/controller/vmbda/vmbda_reconciler.go b/images/virtualization-artifact/pkg/controller/vmbda/vmbda_reconciler.go index 45e8a7b147..10b41e41d6 100644 --- a/images/virtualization-artifact/pkg/controller/vmbda/vmbda_reconciler.go +++ b/images/virtualization-artifact/pkg/controller/vmbda/vmbda_reconciler.go @@ -84,6 +84,7 @@ func (r *Reconciler) SetupController(_ context.Context, mgr manager.Manager, ctr watcher.NewClusterVirtualImageWatcher(mgr.GetClient()), watcher.NewVirtualImageWatcherr(mgr.GetClient()), watcher.NewKVVMIWatcher(mgr.GetClient()), + watcher.NewVolumeEventWatcher(mgr.GetClient()), } { err := w.Watch(mgr, ctr) if err != nil { From c0265d2d562e802cf269c3e8bde6a0b46b3046ca Mon Sep 17 00:00:00 2001 From: Valeriy Khorunzhin Date: Tue, 24 Mar 2026 02:08:51 +0300 Subject: [PATCH 3/8] dirty Signed-off-by: Valeriy Khorunzhin --- .../pkg/controller/vm/internal/lifecycle.go | 59 +++++---- .../vm/internal/watcher/pod_watcher.go | 4 +- .../controller/vmbda/internal/life_cycle.go | 5 + .../internal/watcher/hotplug_pod_watcher.go | 117 ++++++++++++++++++ .../internal/watcher/volumeevent_watcher.go | 35 +++--- .../pkg/controller/vmbda/vmbda_reconciler.go | 1 + 6 files changed, 179 insertions(+), 42 deletions(-) create mode 100644 images/virtualization-artifact/pkg/controller/vmbda/internal/watcher/hotplug_pod_watcher.go diff --git a/images/virtualization-artifact/pkg/controller/vm/internal/lifecycle.go b/images/virtualization-artifact/pkg/controller/vm/internal/lifecycle.go index 87f4804683..1269353e4b 100644 --- a/images/virtualization-artifact/pkg/controller/vm/internal/lifecycle.go +++ b/images/virtualization-artifact/pkg/controller/vm/internal/lifecycle.go @@ -104,6 +104,34 @@ func (h *LifeCycleHandler) Handle(ctx context.Context, s state.VirtualMachineSta log := logger.FromContext(ctx).With(logger.SlogHandler(nameLifeCycleHandler)) + if pod == nil { + podList, err := s.Pods(ctx) + if err != nil { + return reconcile.Result{}, err + } + + for _, innerPod := range podList.Items { + if isContainerCreating(&innerPod) { + cb := conditions.NewConditionBuilder(vmcondition.TypeRunning).Generation(changed.GetGeneration()) + + if volumeErr := h.checkPodVolumeErrors(ctx, changed, log); volumeErr != nil { + cb.Status(metav1.ConditionFalse). + Reason(vmcondition.ReasonPodNotStarted). + Message(volumeErr.Error()) + conditions.SetCondition(cb, &changed.Status.Conditions) + return reconcile.Result{}, nil + } + + cb.Status(metav1.ConditionFalse). + Reason(vmcondition.ReasonPodNotStarted). + Message(fmt.Sprintf("Pod %q is in ContainerCreating phase. Check the pod for more details.", innerPod.Name)) + conditions.SetCondition(cb, &changed.Status.Conditions) + + return reconcile.Result{}, nil + } + } + } + h.syncRunning(ctx, changed, kvvm, kvvmi, pod, log) return reconcile.Result{}, nil } @@ -131,10 +159,10 @@ func (h *LifeCycleHandler) syncRunning(ctx context.Context, vm *v1alpha2.Virtual return } - if containerCreatingPod := h.findContainerCreatingPod(ctx, vm, log); containerCreatingPod != nil { + if isContainerCreating(pod) { cb.Status(metav1.ConditionFalse). - Reason(vmcondition.ReasonPodContainerCreating). - Message(fmt.Sprintf("Pod %q is in ContainerCreating phase. Check the pod for more details.", containerCreatingPod.Name)) + Reason(vmcondition.ReasonPodNotStarted). + Message(fmt.Sprintf("Pod %q is in ContainerCreating phase. Check the pod for more details.", pod.Name)) conditions.SetCondition(cb, &vm.Status.Conditions) return } @@ -218,6 +246,7 @@ func (h *LifeCycleHandler) syncRunning(ctx context.Context, vm *v1alpha2.Virtual } else { vm.Status.Node = "" } + cb.Reason(vmcondition.ReasonVirtualMachineNotRunning).Status(metav1.ConditionFalse) conditions.SetCondition(cb, &vm.Status.Conditions) } @@ -244,28 +273,10 @@ func (h *LifeCycleHandler) checkPodVolumeErrors(ctx context.Context, vm *v1alpha return nil } -func (h *LifeCycleHandler) findContainerCreatingPod(ctx context.Context, vm *v1alpha2.VirtualMachine, log *slog.Logger) *corev1.Pod { - var podList corev1.PodList - err := h.client.List(ctx, &podList, &client.ListOptions{ - Namespace: vm.Namespace, - LabelSelector: labels.SelectorFromSet(map[string]string{ - virtv1.VirtualMachineNameLabel: vm.Name, - }), - }) - if err != nil { - log.Error("Failed to list pods", "error", err) - return nil - } - - for i := range podList.Items { - if isContainerCreating(&podList.Items[i]) { - return &podList.Items[i] - } - } - return nil -} - func isContainerCreating(pod *corev1.Pod) bool { + if pod == nil { + return false + } if pod.Status.Phase != corev1.PodPending { return false } diff --git a/images/virtualization-artifact/pkg/controller/vm/internal/watcher/pod_watcher.go b/images/virtualization-artifact/pkg/controller/vm/internal/watcher/pod_watcher.go index dbe8b008ac..050b13c57c 100644 --- a/images/virtualization-artifact/pkg/controller/vm/internal/watcher/pod_watcher.go +++ b/images/virtualization-artifact/pkg/controller/vm/internal/watcher/pod_watcher.go @@ -19,6 +19,7 @@ package watcher import ( "context" "fmt" + "reflect" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" @@ -66,7 +67,8 @@ func (w *PodWatcher) Watch(mgr manager.Manager, ctr controller.Controller) error DeleteFunc: func(e event.TypedDeleteEvent[*corev1.Pod]) bool { return true }, UpdateFunc: func(e event.TypedUpdateEvent[*corev1.Pod]) bool { return e.ObjectOld.Status.Phase != e.ObjectNew.Status.Phase || - e.ObjectOld.Annotations[annotations.AnnNetworksStatus] != e.ObjectNew.Annotations[annotations.AnnNetworksStatus] + e.ObjectOld.Annotations[annotations.AnnNetworksStatus] != e.ObjectNew.Annotations[annotations.AnnNetworksStatus] || + !reflect.DeepEqual(e.ObjectOld.Status.ContainerStatuses, e.ObjectNew.Status.ContainerStatuses) }, }, ), diff --git a/images/virtualization-artifact/pkg/controller/vmbda/internal/life_cycle.go b/images/virtualization-artifact/pkg/controller/vmbda/internal/life_cycle.go index 427e5d87ac..b284548c03 100644 --- a/images/virtualization-artifact/pkg/controller/vmbda/internal/life_cycle.go +++ b/images/virtualization-artifact/pkg/controller/vmbda/internal/life_cycle.go @@ -347,6 +347,11 @@ func (h LifeCycleHandler) handleHotPlugPodIssues( Message(fmt.Sprintf("%s: %s", lastEvent.Reason, lastEvent.Message)) return reconcile.Result{}, true, nil } + + cb.Status(metav1.ConditionFalse). + Reason(vmbdacondition.FailedAttachVolume). + Message(fmt.Sprintf("Pod %q is in ContainerCreating phase. Check the pod for more details.", hotPlugPod.Name)) + return reconcile.Result{}, true, nil } return reconcile.Result{}, false, nil diff --git a/images/virtualization-artifact/pkg/controller/vmbda/internal/watcher/hotplug_pod_watcher.go b/images/virtualization-artifact/pkg/controller/vmbda/internal/watcher/hotplug_pod_watcher.go new file mode 100644 index 0000000000..cc6b9307ef --- /dev/null +++ b/images/virtualization-artifact/pkg/controller/vmbda/internal/watcher/hotplug_pod_watcher.go @@ -0,0 +1,117 @@ +/* +Copyright 2026 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package watcher + +import ( + "context" + "fmt" + "reflect" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/types" + virtv1 "kubevirt.io/api/core/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/predicate" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + "sigs.k8s.io/controller-runtime/pkg/source" + + "github.com/deckhouse/virtualization-controller/pkg/controller/kvbuilder" + "github.com/deckhouse/virtualization/api/core/v1alpha2" +) + +func NewHotPlugPodWatcher(client client.Client) *HotPlugPodWatcher { + return &HotPlugPodWatcher{ + client: client, + } +} + +type HotPlugPodWatcher struct { + client client.Client +} + +func (w *HotPlugPodWatcher) Watch(mgr manager.Manager, ctr controller.Controller) error { + if err := ctr.Watch( + source.Kind( + mgr.GetCache(), + &corev1.Pod{}, + handler.TypedEnqueueRequestsFromMapFunc(w.enqueueVMBDAs), + predicate.TypedFuncs[*corev1.Pod]{ + CreateFunc: func(e event.TypedCreateEvent[*corev1.Pod]) bool { return true }, + DeleteFunc: func(e event.TypedDeleteEvent[*corev1.Pod]) bool { return true }, + UpdateFunc: func(e event.TypedUpdateEvent[*corev1.Pod]) bool { + return e.ObjectOld.Status.Phase != e.ObjectNew.Status.Phase || + !reflect.DeepEqual(e.ObjectOld.Status.Conditions, e.ObjectNew.Status.Conditions) || + !reflect.DeepEqual(e.ObjectOld.Status.ContainerStatuses, e.ObjectNew.Status.ContainerStatuses) + }, + }, + ), + ); err != nil { + return fmt.Errorf("error setting watch on hot-plug Pod: %w", err) + } + return nil +} + +func (w *HotPlugPodWatcher) enqueueVMBDAs(ctx context.Context, pod *corev1.Pod) []reconcile.Request { + if pod == nil { + return nil + } + + ns := pod.Namespace + podName := pod.Name + + var kvvmiList virtv1.VirtualMachineInstanceList + if err := w.client.List(ctx, &kvvmiList, &client.ListOptions{Namespace: ns}); err != nil { + return nil + } + + for _, kvvmi := range kvvmiList.Items { + for _, vs := range kvvmi.Status.VolumeStatus { + if vs.HotplugVolume == nil || vs.HotplugVolume.AttachPodName != podName { + continue + } + + name, kind := kvbuilder.GetOriginalDiskName(vs.Name) + if kind == "" { + continue + } + + var vmbdas v1alpha2.VirtualMachineBlockDeviceAttachmentList + if err := w.client.List(ctx, &vmbdas, &client.ListOptions{Namespace: ns}); err != nil { + return nil + } + + var requests []reconcile.Request + for _, vmbda := range vmbdas.Items { + if vmbda.Spec.BlockDeviceRef.Name == name { + requests = append(requests, reconcile.Request{ + NamespacedName: types.NamespacedName{ + Namespace: vmbda.Namespace, + Name: vmbda.Name, + }, + }) + } + } + return requests + } + } + + return nil +} diff --git a/images/virtualization-artifact/pkg/controller/vmbda/internal/watcher/volumeevent_watcher.go b/images/virtualization-artifact/pkg/controller/vmbda/internal/watcher/volumeevent_watcher.go index e3c8b6ede7..424669b4f6 100644 --- a/images/virtualization-artifact/pkg/controller/vmbda/internal/watcher/volumeevent_watcher.go +++ b/images/virtualization-artifact/pkg/controller/vmbda/internal/watcher/volumeevent_watcher.go @@ -19,7 +19,6 @@ package watcher import ( "context" "fmt" - "log/slog" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" @@ -37,14 +36,19 @@ import ( "github.com/deckhouse/virtualization/api/core/v1alpha2" ) -const reasonFailedAttachVolume = "FailedAttachVolume" +const ( + ReasonFailedAttachVolume = "FailedAttachVolume" + ReasonFailedMount = "FailedMount" +) -type VolumeEventWatcher struct { - client client.Client +func NewVolumeEventWatcher(client client.Client) *VolumeEventWatcher { + return &VolumeEventWatcher{ + client: client, + } } -func NewVolumeEventWatcher(client client.Client) *VolumeEventWatcher { - return &VolumeEventWatcher{client: client} +type VolumeEventWatcher struct { + client client.Client } func (w *VolumeEventWatcher) Watch(mgr manager.Manager, ctr controller.Controller) error { @@ -55,12 +59,13 @@ func (w *VolumeEventWatcher) Watch(mgr manager.Manager, ctr controller.Controlle handler.TypedEnqueueRequestsFromMapFunc(w.enqueueVMBDAs), predicate.TypedFuncs[*corev1.Event]{ CreateFunc: func(e event.TypedCreateEvent[*corev1.Event]) bool { - return isFailedAttachVolumeEvent(e.Object) + return e.Object.Type == corev1.EventTypeWarning && + (e.Object.Reason == ReasonFailedAttachVolume || e.Object.Reason == ReasonFailedMount) }, UpdateFunc: func(e event.TypedUpdateEvent[*corev1.Event]) bool { - return isFailedAttachVolumeEvent(e.ObjectNew) + return false }, - DeleteFunc: func(_ event.TypedDeleteEvent[*corev1.Event]) bool { + DeleteFunc: func(e event.TypedDeleteEvent[*corev1.Event]) bool { return false }, }, @@ -71,23 +76,20 @@ func (w *VolumeEventWatcher) Watch(mgr manager.Manager, ctr controller.Controlle return nil } -func isFailedAttachVolumeEvent(e *corev1.Event) bool { - return e.InvolvedObject.Kind == "Pod" && - e.Type == corev1.EventTypeWarning && - e.Reason == reasonFailedAttachVolume -} - func (w *VolumeEventWatcher) enqueueVMBDAs(ctx context.Context, e *corev1.Event) []reconcile.Request { if e.InvolvedObject.Kind != "Pod" { return nil } + if e.Reason != ReasonFailedAttachVolume && e.Reason != ReasonFailedMount { + return nil + } + ns := e.InvolvedObject.Namespace podName := e.InvolvedObject.Name var kvvmiList virtv1.VirtualMachineInstanceList if err := w.client.List(ctx, &kvvmiList, &client.ListOptions{Namespace: ns}); err != nil { - slog.Default().Error(fmt.Sprintf("failed to list kvvmis: %s", err)) return nil } @@ -104,7 +106,6 @@ func (w *VolumeEventWatcher) enqueueVMBDAs(ctx context.Context, e *corev1.Event) var vmbdas v1alpha2.VirtualMachineBlockDeviceAttachmentList if err := w.client.List(ctx, &vmbdas, &client.ListOptions{Namespace: ns}); err != nil { - slog.Default().Error(fmt.Sprintf("failed to list vmbdas: %s", err)) return nil } diff --git a/images/virtualization-artifact/pkg/controller/vmbda/vmbda_reconciler.go b/images/virtualization-artifact/pkg/controller/vmbda/vmbda_reconciler.go index 10b41e41d6..c06ed1bf15 100644 --- a/images/virtualization-artifact/pkg/controller/vmbda/vmbda_reconciler.go +++ b/images/virtualization-artifact/pkg/controller/vmbda/vmbda_reconciler.go @@ -85,6 +85,7 @@ func (r *Reconciler) SetupController(_ context.Context, mgr manager.Manager, ctr watcher.NewVirtualImageWatcherr(mgr.GetClient()), watcher.NewKVVMIWatcher(mgr.GetClient()), watcher.NewVolumeEventWatcher(mgr.GetClient()), + watcher.NewHotPlugPodWatcher(mgr.GetClient()), } { err := w.Watch(mgr, ctr) if err != nil { From 987084eb8c7e2d2a3e67d7b3eb1af20acbc0114e Mon Sep 17 00:00:00 2001 From: Valeriy Khorunzhin Date: Tue, 24 Mar 2026 11:17:41 +0300 Subject: [PATCH 4/8] resolve self-review Signed-off-by: Valeriy Khorunzhin --- api/core/v1alpha2/vmcondition/condition.go | 1 + .../pkg/controller/vm/internal/lifecycle.go | 45 ++++++------------- .../controller/vmbda/internal/life_cycle.go | 11 +++-- 3 files changed, 20 insertions(+), 37 deletions(-) diff --git a/api/core/v1alpha2/vmcondition/condition.go b/api/core/v1alpha2/vmcondition/condition.go index f757457b04..dd0cd1f9db 100644 --- a/api/core/v1alpha2/vmcondition/condition.go +++ b/api/core/v1alpha2/vmcondition/condition.go @@ -170,6 +170,7 @@ const ( ReasonInternalVirtualMachineError RunningReason = "InternalVirtualMachineError" ReasonPodNotStarted RunningReason = "PodNotStarted" ReasonPodContainerCreating RunningReason = "PodContainerCreating" + ReasonPodVolumeErrors RunningReason = "PodVolumeErrors" ReasonPodTerminating RunningReason = "PodTerminating" ReasonPodNotFound RunningReason = "PodNotFound" ReasonPodConditionMissing RunningReason = "PodConditionMissing" diff --git a/images/virtualization-artifact/pkg/controller/vm/internal/lifecycle.go b/images/virtualization-artifact/pkg/controller/vm/internal/lifecycle.go index 1269353e4b..493b814e88 100644 --- a/images/virtualization-artifact/pkg/controller/vm/internal/lifecycle.go +++ b/images/virtualization-artifact/pkg/controller/vm/internal/lifecycle.go @@ -103,33 +103,24 @@ func (h *LifeCycleHandler) Handle(ctx context.Context, s state.VirtualMachineSta } log := logger.FromContext(ctx).With(logger.SlogHandler(nameLifeCycleHandler)) - + // While the pod is not running, the VMI does not set the node and the method returns nil, so it is necessary to check if there are any issues with the pod if pod == nil { - podList, err := s.Pods(ctx) - if err != nil { - return reconcile.Result{}, err - } - - for _, innerPod := range podList.Items { - if isContainerCreating(&innerPod) { - cb := conditions.NewConditionBuilder(vmcondition.TypeRunning).Generation(changed.GetGeneration()) + cb := conditions.NewConditionBuilder(vmcondition.TypeRunning).Generation(changed.GetGeneration()) - if volumeErr := h.checkPodVolumeErrors(ctx, changed, log); volumeErr != nil { - cb.Status(metav1.ConditionFalse). - Reason(vmcondition.ReasonPodNotStarted). - Message(volumeErr.Error()) - conditions.SetCondition(cb, &changed.Status.Conditions) - return reconcile.Result{}, nil - } + if volumeErr := h.checkPodVolumeErrors(ctx, changed, log); volumeErr != nil { + cb.Status(metav1.ConditionFalse). + Reason(vmcondition.ReasonPodVolumeErrors). + Message(fmt.Sprintf("Volume errors detected on Pod: %s", volumeErr.Error())) + conditions.SetCondition(cb, &changed.Status.Conditions) + return reconcile.Result{}, nil + } - cb.Status(metav1.ConditionFalse). - Reason(vmcondition.ReasonPodNotStarted). - Message(fmt.Sprintf("Pod %q is in ContainerCreating phase. Check the pod for more details.", innerPod.Name)) - conditions.SetCondition(cb, &changed.Status.Conditions) + cb.Status(metav1.ConditionFalse). + Reason(vmcondition.ReasonPodContainerCreating). + Message("Pod is in ContainerCreating phase. Check the pod for more details.") + conditions.SetCondition(cb, &changed.Status.Conditions) - return reconcile.Result{}, nil - } - } + return reconcile.Result{}, nil } h.syncRunning(ctx, changed, kvvm, kvvmi, pod, log) @@ -159,14 +150,6 @@ func (h *LifeCycleHandler) syncRunning(ctx context.Context, vm *v1alpha2.Virtual return } - if isContainerCreating(pod) { - cb.Status(metav1.ConditionFalse). - Reason(vmcondition.ReasonPodNotStarted). - Message(fmt.Sprintf("Pod %q is in ContainerCreating phase. Check the pod for more details.", pod.Name)) - conditions.SetCondition(cb, &vm.Status.Conditions) - return - } - if kvvm != nil { podScheduled := service.GetKVVMCondition(string(corev1.PodScheduled), kvvm.Status.Conditions) if podScheduled != nil && podScheduled.Status == corev1.ConditionFalse { diff --git a/images/virtualization-artifact/pkg/controller/vmbda/internal/life_cycle.go b/images/virtualization-artifact/pkg/controller/vmbda/internal/life_cycle.go index b284548c03..7335aae05a 100644 --- a/images/virtualization-artifact/pkg/controller/vmbda/internal/life_cycle.go +++ b/images/virtualization-artifact/pkg/controller/vmbda/internal/life_cycle.go @@ -30,6 +30,7 @@ import ( "github.com/deckhouse/virtualization-controller/pkg/controller/conditions" "github.com/deckhouse/virtualization-controller/pkg/controller/service" intsvc "github.com/deckhouse/virtualization-controller/pkg/controller/vmbda/internal/service" + "github.com/deckhouse/virtualization-controller/pkg/controller/vmbda/internal/watcher" "github.com/deckhouse/virtualization-controller/pkg/logger" "github.com/deckhouse/virtualization/api/core/v1alpha2" "github.com/deckhouse/virtualization/api/core/v1alpha2/vmbdacondition" @@ -307,8 +308,6 @@ func (h LifeCycleHandler) Handle(ctx context.Context, vmbda *v1alpha2.VirtualMac } } -const reasonFailedAttachVolume = "FailedAttachVolume" - func (h LifeCycleHandler) handleHotPlugPodIssues( ctx context.Context, ad *intsvc.AttachmentDisk, @@ -330,7 +329,7 @@ func (h LifeCycleHandler) handleHotPlugPodIssues( cb. Status(metav1.ConditionFalse). Reason(vmbdacondition.HotPlugPodNotScheduled). - Message(fmt.Sprintf("%s: %s", c.Reason, c.Message)) + Message(fmt.Sprintf("Hot plug pod not scheduled: %s: %s", c.Reason, c.Message)) return reconcile.Result{}, true, nil } } @@ -340,16 +339,16 @@ func (h LifeCycleHandler) handleHotPlugPodIssues( if err != nil { return reconcile.Result{}, false, err } - if lastEvent != nil && lastEvent.Reason == reasonFailedAttachVolume { + if lastEvent != nil && (lastEvent.Reason == watcher.ReasonFailedAttachVolume || lastEvent.Reason == watcher.ReasonFailedMount) { cb. Status(metav1.ConditionFalse). Reason(vmbdacondition.FailedAttachVolume). - Message(fmt.Sprintf("%s: %s", lastEvent.Reason, lastEvent.Message)) + Message(fmt.Sprintf("Hot plug pod failed to attach volume: %s: %s", lastEvent.Reason, lastEvent.Message)) return reconcile.Result{}, true, nil } cb.Status(metav1.ConditionFalse). - Reason(vmbdacondition.FailedAttachVolume). + Reason(vmbdacondition.AttachmentRequestSent). Message(fmt.Sprintf("Pod %q is in ContainerCreating phase. Check the pod for more details.", hotPlugPod.Name)) return reconcile.Result{}, true, nil } From 905c0109ea41c46ee68350457d1fee183bf63976 Mon Sep 17 00:00:00 2001 From: Valeriy Khorunzhin Date: Tue, 24 Mar 2026 11:44:35 +0300 Subject: [PATCH 5/8] fix logic Signed-off-by: Valeriy Khorunzhin --- .../pkg/controller/vm/internal/lifecycle.go | 37 ++++++++++++++++--- 1 file changed, 31 insertions(+), 6 deletions(-) diff --git a/images/virtualization-artifact/pkg/controller/vm/internal/lifecycle.go b/images/virtualization-artifact/pkg/controller/vm/internal/lifecycle.go index 493b814e88..2d632e58df 100644 --- a/images/virtualization-artifact/pkg/controller/vm/internal/lifecycle.go +++ b/images/virtualization-artifact/pkg/controller/vm/internal/lifecycle.go @@ -115,12 +115,17 @@ func (h *LifeCycleHandler) Handle(ctx context.Context, s state.VirtualMachineSta return reconcile.Result{}, nil } - cb.Status(metav1.ConditionFalse). - Reason(vmcondition.ReasonPodContainerCreating). - Message("Pod is in ContainerCreating phase. Check the pod for more details.") - conditions.SetCondition(cb, &changed.Status.Conditions) - - return reconcile.Result{}, nil + isVMInContainerCreating, err := h.isVMInContainerCreatingState(ctx, changed, log) + if err != nil { + return reconcile.Result{}, err + } + if isVMInContainerCreating { + cb.Status(metav1.ConditionFalse). + Reason(vmcondition.ReasonPodContainerCreating). + Message("Pod is in ContainerCreating phase. Check the pod for more details.") + conditions.SetCondition(cb, &changed.Status.Conditions) + return reconcile.Result{}, nil + } } h.syncRunning(ctx, changed, kvvm, kvvmi, pod, log) @@ -234,6 +239,26 @@ func (h *LifeCycleHandler) syncRunning(ctx context.Context, vm *v1alpha2.Virtual conditions.SetCondition(cb, &vm.Status.Conditions) } +func (h *LifeCycleHandler) isVMInContainerCreatingState(ctx context.Context, vm *v1alpha2.VirtualMachine, log *slog.Logger) (bool, error) { + var podList corev1.PodList + err := h.client.List(ctx, &podList, &client.ListOptions{ + Namespace: vm.Namespace, + LabelSelector: labels.SelectorFromSet(map[string]string{ + virtv1.VirtualMachineNameLabel: vm.Name, + }), + }) + if err != nil { + log.Error("Failed to list pods", "error", err) + return false, err + } + + if len(podList.Items) == 1 { + return isContainerCreating(&podList.Items[0]), nil + } + + return false, nil +} + func (h *LifeCycleHandler) checkPodVolumeErrors(ctx context.Context, vm *v1alpha2.VirtualMachine, log *slog.Logger) error { var podList corev1.PodList err := h.client.List(ctx, &podList, &client.ListOptions{ From 597c33a61af45ad8fe5c40895930924bec976c82 Mon Sep 17 00:00:00 2001 From: Valeriy Khorunzhin Date: Tue, 24 Mar 2026 12:02:52 +0300 Subject: [PATCH 6/8] fix linter Signed-off-by: Valeriy Khorunzhin --- .../controller/vmbda/internal/life_cycle.go | 20 +++++++++---------- .../internal/service/attachment_service.go | 2 +- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/images/virtualization-artifact/pkg/controller/vmbda/internal/life_cycle.go b/images/virtualization-artifact/pkg/controller/vmbda/internal/life_cycle.go index 7335aae05a..dc5050fab2 100644 --- a/images/virtualization-artifact/pkg/controller/vmbda/internal/life_cycle.go +++ b/images/virtualization-artifact/pkg/controller/vmbda/internal/life_cycle.go @@ -202,8 +202,8 @@ func (h LifeCycleHandler) Handle(ctx context.Context, vmbda *v1alpha2.VirtualMac if errors.Is(err, intsvc.ErrVolumeStatusNotReady) { vmbda.Status.Phase = v1alpha2.BlockDeviceAttachmentPhaseInProgress - if result, handled, podErr := h.handleHotPlugPodIssues(ctx, ad, kvvmi, vmbda, cb); podErr != nil || handled { - return result, podErr + if handled, podErr := h.handleHotPlugPodIssues(ctx, ad, kvvmi, vmbda, cb); podErr != nil || handled { + return reconcile.Result{}, podErr } cb. @@ -314,13 +314,13 @@ func (h LifeCycleHandler) handleHotPlugPodIssues( kvvmi *virtv1.VirtualMachineInstance, vmbda *v1alpha2.VirtualMachineBlockDeviceAttachment, cb *conditions.ConditionBuilder, -) (reconcile.Result, bool, error) { +) (bool, error) { hotPlugPod, err := h.attacher.GetHotPlugPod(ctx, ad, kvvmi) if err != nil { - return reconcile.Result{}, false, err + return false, err } if hotPlugPod == nil { - return reconcile.Result{}, false, nil + return false, nil } for _, c := range hotPlugPod.Status.Conditions { @@ -330,30 +330,30 @@ func (h LifeCycleHandler) handleHotPlugPodIssues( Status(metav1.ConditionFalse). Reason(vmbdacondition.HotPlugPodNotScheduled). Message(fmt.Sprintf("Hot plug pod not scheduled: %s: %s", c.Reason, c.Message)) - return reconcile.Result{}, true, nil + return true, nil } } if isContainerCreating(hotPlugPod) { lastEvent, err := h.attacher.GetLastPodEvent(ctx, hotPlugPod) if err != nil { - return reconcile.Result{}, false, err + return false, err } if lastEvent != nil && (lastEvent.Reason == watcher.ReasonFailedAttachVolume || lastEvent.Reason == watcher.ReasonFailedMount) { cb. Status(metav1.ConditionFalse). Reason(vmbdacondition.FailedAttachVolume). Message(fmt.Sprintf("Hot plug pod failed to attach volume: %s: %s", lastEvent.Reason, lastEvent.Message)) - return reconcile.Result{}, true, nil + return true, nil } cb.Status(metav1.ConditionFalse). Reason(vmbdacondition.AttachmentRequestSent). Message(fmt.Sprintf("Pod %q is in ContainerCreating phase. Check the pod for more details.", hotPlugPod.Name)) - return reconcile.Result{}, true, nil + return true, nil } - return reconcile.Result{}, false, nil + return false, nil } func isContainerCreating(pod *corev1.Pod) bool { diff --git a/images/virtualization-artifact/pkg/controller/vmbda/internal/service/attachment_service.go b/images/virtualization-artifact/pkg/controller/vmbda/internal/service/attachment_service.go index c97fa2f25d..04159549f1 100644 --- a/images/virtualization-artifact/pkg/controller/vmbda/internal/service/attachment_service.go +++ b/images/virtualization-artifact/pkg/controller/vmbda/internal/service/attachment_service.go @@ -373,7 +373,7 @@ func (s AttachmentService) GetLastPodEvent(ctx context.Context, pod *corev1.Pod) } last := slices.MaxFunc(eventList.Items, func(a, b corev1.Event) int { - return a.LastTimestamp.Time.Compare(b.LastTimestamp.Time) + return a.LastTimestamp.Compare(b.LastTimestamp.Time) }) return &last, nil From 18ad92f1cd2c561d2149efdbdefa14bdec04755c Mon Sep 17 00:00:00 2001 From: Valeriy Khorunzhin Date: Tue, 24 Mar 2026 12:58:58 +0300 Subject: [PATCH 7/8] fix getPodVolumeError Signed-off-by: Valeriy Khorunzhin --- .../pkg/controller/vm/internal/lifecycle.go | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/images/virtualization-artifact/pkg/controller/vm/internal/lifecycle.go b/images/virtualization-artifact/pkg/controller/vm/internal/lifecycle.go index 2d632e58df..fb7d07756a 100644 --- a/images/virtualization-artifact/pkg/controller/vm/internal/lifecycle.go +++ b/images/virtualization-artifact/pkg/controller/vm/internal/lifecycle.go @@ -20,6 +20,7 @@ import ( "context" "fmt" "log/slog" + "slices" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -314,10 +315,15 @@ func (h *LifeCycleHandler) getPodVolumeError(ctx context.Context, pod *corev1.Po return nil } - for _, e := range eventList.Items { - if e.Type == corev1.EventTypeWarning && (e.Reason == watcher.ReasonFailedAttachVolume || e.Reason == watcher.ReasonFailedMount) { - return fmt.Errorf("%s: %s", e.Reason, e.Message) - } + if len(eventList.Items) == 0 { + return nil + } + + last := slices.MaxFunc(eventList.Items, func(a, b corev1.Event) int { + return a.LastTimestamp.Compare(b.LastTimestamp.Time) + }) + if last.Reason == watcher.ReasonFailedAttachVolume || last.Reason == watcher.ReasonFailedMount { + return fmt.Errorf("%s: %s", last.Reason, last.Message) } return nil From ed1f0cf28b70f468fd713db07300f5d3f3eabf37 Mon Sep 17 00:00:00 2001 From: Valeriy Khorunzhin Date: Mon, 30 Mar 2026 13:16:59 +0300 Subject: [PATCH 8/8] final Signed-off-by: Valeriy Khorunzhin --- api/core/v1alpha2/vmcondition/condition.go | 1 - .../pkg/controller/vm/internal/lifecycle.go | 54 ++++++------------- .../vm/internal/watcher/pod_watcher.go | 4 +- .../controller/vmbda/internal/life_cycle.go | 9 +--- .../internal/watcher/hotplug_pod_watcher.go | 5 +- .../pkg/controller/vmbda/vmbda_reconciler.go | 1 - 6 files changed, 19 insertions(+), 55 deletions(-) diff --git a/api/core/v1alpha2/vmcondition/condition.go b/api/core/v1alpha2/vmcondition/condition.go index dd0cd1f9db..786e5e57f1 100644 --- a/api/core/v1alpha2/vmcondition/condition.go +++ b/api/core/v1alpha2/vmcondition/condition.go @@ -169,7 +169,6 @@ const ( ReasonVirtualMachineRunning RunningReason = "Running" ReasonInternalVirtualMachineError RunningReason = "InternalVirtualMachineError" ReasonPodNotStarted RunningReason = "PodNotStarted" - ReasonPodContainerCreating RunningReason = "PodContainerCreating" ReasonPodVolumeErrors RunningReason = "PodVolumeErrors" ReasonPodTerminating RunningReason = "PodTerminating" ReasonPodNotFound RunningReason = "PodNotFound" diff --git a/images/virtualization-artifact/pkg/controller/vm/internal/lifecycle.go b/images/virtualization-artifact/pkg/controller/vm/internal/lifecycle.go index fb7d07756a..b908fde213 100644 --- a/images/virtualization-artifact/pkg/controller/vm/internal/lifecycle.go +++ b/images/virtualization-artifact/pkg/controller/vm/internal/lifecycle.go @@ -55,6 +55,11 @@ type LifeCycleHandler struct { recorder eventrecord.EventRecorderLogger } +type podVolumeErrorEvent struct { + Reason string + Message string +} + func (h *LifeCycleHandler) Handle(ctx context.Context, s state.VirtualMachineState) (reconcile.Result, error) { if s.VirtualMachine().IsEmpty() { return reconcile.Result{}, nil @@ -111,19 +116,7 @@ func (h *LifeCycleHandler) Handle(ctx context.Context, s state.VirtualMachineSta if volumeErr := h.checkPodVolumeErrors(ctx, changed, log); volumeErr != nil { cb.Status(metav1.ConditionFalse). Reason(vmcondition.ReasonPodVolumeErrors). - Message(fmt.Sprintf("Volume errors detected on Pod: %s", volumeErr.Error())) - conditions.SetCondition(cb, &changed.Status.Conditions) - return reconcile.Result{}, nil - } - - isVMInContainerCreating, err := h.isVMInContainerCreatingState(ctx, changed, log) - if err != nil { - return reconcile.Result{}, err - } - if isVMInContainerCreating { - cb.Status(metav1.ConditionFalse). - Reason(vmcondition.ReasonPodContainerCreating). - Message("Pod is in ContainerCreating phase. Check the pod for more details.") + Message(fmt.Sprintf("Error attaching block devices to virtual machine: %s: %s", volumeErr.Reason, volumeErr.Message)) conditions.SetCondition(cb, &changed.Status.Conditions) return reconcile.Result{}, nil } @@ -150,8 +143,8 @@ func (h *LifeCycleHandler) syncRunning(ctx context.Context, vm *v1alpha2.Virtual if volumeError := h.checkPodVolumeErrors(ctx, vm, log); volumeError != nil { cb.Status(metav1.ConditionFalse). - Reason(vmcondition.ReasonPodNotStarted). - Message(volumeError.Error()) + Reason(vmcondition.ReasonPodVolumeErrors). + Message(fmt.Sprintf("Error attaching block devices to virtual machine: %s: %s", volumeError.Reason, volumeError.Message)) conditions.SetCondition(cb, &vm.Status.Conditions) return } @@ -163,7 +156,7 @@ func (h *LifeCycleHandler) syncRunning(ctx context.Context, vm *v1alpha2.Virtual if podScheduled.Message != "" { cb.Status(metav1.ConditionFalse). Reason(vmcondition.ReasonPodNotStarted). - Message(fmt.Sprintf("%s: %s", podScheduled.Reason, podScheduled.Message)) + Message(fmt.Sprintf("Could not schedule the virtual machine: %s: %s", podScheduled.Reason, podScheduled.Message)) conditions.SetCondition(cb, &vm.Status.Conditions) } @@ -240,27 +233,7 @@ func (h *LifeCycleHandler) syncRunning(ctx context.Context, vm *v1alpha2.Virtual conditions.SetCondition(cb, &vm.Status.Conditions) } -func (h *LifeCycleHandler) isVMInContainerCreatingState(ctx context.Context, vm *v1alpha2.VirtualMachine, log *slog.Logger) (bool, error) { - var podList corev1.PodList - err := h.client.List(ctx, &podList, &client.ListOptions{ - Namespace: vm.Namespace, - LabelSelector: labels.SelectorFromSet(map[string]string{ - virtv1.VirtualMachineNameLabel: vm.Name, - }), - }) - if err != nil { - log.Error("Failed to list pods", "error", err) - return false, err - } - - if len(podList.Items) == 1 { - return isContainerCreating(&podList.Items[0]), nil - } - - return false, nil -} - -func (h *LifeCycleHandler) checkPodVolumeErrors(ctx context.Context, vm *v1alpha2.VirtualMachine, log *slog.Logger) error { +func (h *LifeCycleHandler) checkPodVolumeErrors(ctx context.Context, vm *v1alpha2.VirtualMachine, log *slog.Logger) *podVolumeErrorEvent { var podList corev1.PodList err := h.client.List(ctx, &podList, &client.ListOptions{ Namespace: vm.Namespace, @@ -297,7 +270,7 @@ func isContainerCreating(pod *corev1.Pod) bool { return false } -func (h *LifeCycleHandler) getPodVolumeError(ctx context.Context, pod *corev1.Pod, log *slog.Logger) error { +func (h *LifeCycleHandler) getPodVolumeError(ctx context.Context, pod *corev1.Pod, log *slog.Logger) *podVolumeErrorEvent { if !isContainerCreating(pod) { return nil } @@ -323,7 +296,10 @@ func (h *LifeCycleHandler) getPodVolumeError(ctx context.Context, pod *corev1.Po return a.LastTimestamp.Compare(b.LastTimestamp.Time) }) if last.Reason == watcher.ReasonFailedAttachVolume || last.Reason == watcher.ReasonFailedMount { - return fmt.Errorf("%s: %s", last.Reason, last.Message) + return &podVolumeErrorEvent{ + Reason: last.Reason, + Message: last.Message, + } } return nil diff --git a/images/virtualization-artifact/pkg/controller/vm/internal/watcher/pod_watcher.go b/images/virtualization-artifact/pkg/controller/vm/internal/watcher/pod_watcher.go index 050b13c57c..dbe8b008ac 100644 --- a/images/virtualization-artifact/pkg/controller/vm/internal/watcher/pod_watcher.go +++ b/images/virtualization-artifact/pkg/controller/vm/internal/watcher/pod_watcher.go @@ -19,7 +19,6 @@ package watcher import ( "context" "fmt" - "reflect" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" @@ -67,8 +66,7 @@ func (w *PodWatcher) Watch(mgr manager.Manager, ctr controller.Controller) error DeleteFunc: func(e event.TypedDeleteEvent[*corev1.Pod]) bool { return true }, UpdateFunc: func(e event.TypedUpdateEvent[*corev1.Pod]) bool { return e.ObjectOld.Status.Phase != e.ObjectNew.Status.Phase || - e.ObjectOld.Annotations[annotations.AnnNetworksStatus] != e.ObjectNew.Annotations[annotations.AnnNetworksStatus] || - !reflect.DeepEqual(e.ObjectOld.Status.ContainerStatuses, e.ObjectNew.Status.ContainerStatuses) + e.ObjectOld.Annotations[annotations.AnnNetworksStatus] != e.ObjectNew.Annotations[annotations.AnnNetworksStatus] }, }, ), diff --git a/images/virtualization-artifact/pkg/controller/vmbda/internal/life_cycle.go b/images/virtualization-artifact/pkg/controller/vmbda/internal/life_cycle.go index dc5050fab2..c0f5e29fc2 100644 --- a/images/virtualization-artifact/pkg/controller/vmbda/internal/life_cycle.go +++ b/images/virtualization-artifact/pkg/controller/vmbda/internal/life_cycle.go @@ -329,7 +329,7 @@ func (h LifeCycleHandler) handleHotPlugPodIssues( cb. Status(metav1.ConditionFalse). Reason(vmbdacondition.HotPlugPodNotScheduled). - Message(fmt.Sprintf("Hot plug pod not scheduled: %s: %s", c.Reason, c.Message)) + Message(fmt.Sprintf("Error attaching block device to virtual machine: %s: %s", c.Reason, c.Message)) return true, nil } } @@ -343,14 +343,9 @@ func (h LifeCycleHandler) handleHotPlugPodIssues( cb. Status(metav1.ConditionFalse). Reason(vmbdacondition.FailedAttachVolume). - Message(fmt.Sprintf("Hot plug pod failed to attach volume: %s: %s", lastEvent.Reason, lastEvent.Message)) + Message(fmt.Sprintf("Error attaching block device to virtual machine: %s: %s", lastEvent.Reason, lastEvent.Message)) return true, nil } - - cb.Status(metav1.ConditionFalse). - Reason(vmbdacondition.AttachmentRequestSent). - Message(fmt.Sprintf("Pod %q is in ContainerCreating phase. Check the pod for more details.", hotPlugPod.Name)) - return true, nil } return false, nil diff --git a/images/virtualization-artifact/pkg/controller/vmbda/internal/watcher/hotplug_pod_watcher.go b/images/virtualization-artifact/pkg/controller/vmbda/internal/watcher/hotplug_pod_watcher.go index cc6b9307ef..97cf18fe4b 100644 --- a/images/virtualization-artifact/pkg/controller/vmbda/internal/watcher/hotplug_pod_watcher.go +++ b/images/virtualization-artifact/pkg/controller/vmbda/internal/watcher/hotplug_pod_watcher.go @@ -19,7 +19,6 @@ package watcher import ( "context" "fmt" - "reflect" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" @@ -57,9 +56,7 @@ func (w *HotPlugPodWatcher) Watch(mgr manager.Manager, ctr controller.Controller CreateFunc: func(e event.TypedCreateEvent[*corev1.Pod]) bool { return true }, DeleteFunc: func(e event.TypedDeleteEvent[*corev1.Pod]) bool { return true }, UpdateFunc: func(e event.TypedUpdateEvent[*corev1.Pod]) bool { - return e.ObjectOld.Status.Phase != e.ObjectNew.Status.Phase || - !reflect.DeepEqual(e.ObjectOld.Status.Conditions, e.ObjectNew.Status.Conditions) || - !reflect.DeepEqual(e.ObjectOld.Status.ContainerStatuses, e.ObjectNew.Status.ContainerStatuses) + return e.ObjectOld.Status.Phase != e.ObjectNew.Status.Phase }, }, ), diff --git a/images/virtualization-artifact/pkg/controller/vmbda/vmbda_reconciler.go b/images/virtualization-artifact/pkg/controller/vmbda/vmbda_reconciler.go index c06ed1bf15..10b41e41d6 100644 --- a/images/virtualization-artifact/pkg/controller/vmbda/vmbda_reconciler.go +++ b/images/virtualization-artifact/pkg/controller/vmbda/vmbda_reconciler.go @@ -85,7 +85,6 @@ func (r *Reconciler) SetupController(_ context.Context, mgr manager.Manager, ctr watcher.NewVirtualImageWatcherr(mgr.GetClient()), watcher.NewKVVMIWatcher(mgr.GetClient()), watcher.NewVolumeEventWatcher(mgr.GetClient()), - watcher.NewHotPlugPodWatcher(mgr.GetClient()), } { err := w.Watch(mgr, ctr) if err != nil {