Chore/api/provide pod errors to vm and vmbda

api/core/v1alpha2/vmbdacondition/condition.go

@@ -65,6 +65,10 @@ const (
 	Conflict AttachedReason = "Conflict"
 	// DeviceNotAvailableOnNode indicates that the block device's PersistentVolume is not available on the node where the virtual machine is running.
 	DeviceNotAvailableOnNode AttachedReason = "DeviceNotAvailableOnNode"
+	// HotPlugPodNotScheduled indicates that the hotplug pod cannot be scheduled on any node.
+	HotPlugPodNotScheduled AttachedReason = "HotPlugPodNotScheduled"
+	// FailedAttachVolume indicates that the hotplug pod failed to attach a volume.
+	FailedAttachVolume AttachedReason = "FailedAttachVolume"
 
 	// CapacityAvailable signifies that the capacity not reached and attaching available.
 	CapacityAvailable DiskAttachmentCapacityAvailableReason = "CapacityAvailable"

api/core/v1alpha2/vmcondition/condition.go

@@ -169,6 +169,8 @@ const (
 	ReasonVirtualMachineRunning       RunningReason = "Running"
 	ReasonInternalVirtualMachineError RunningReason = "InternalVirtualMachineError"
 	ReasonPodNotStarted               RunningReason = "PodNotStarted"
+	ReasonPodContainerCreating        RunningReason = "PodContainerCreating"
+	ReasonPodVolumeErrors             RunningReason = "PodVolumeErrors"
 	ReasonPodTerminating              RunningReason = "PodTerminating"
 	ReasonPodNotFound                 RunningReason = "PodNotFound"
 	ReasonPodConditionMissing         RunningReason = "PodConditionMissing"

images/virtualization-artifact/pkg/controller/vm/internal/lifecycle.go

@@ -20,6 +20,7 @@ import (
 	"context"
 	"fmt"
 	"log/slog"
+	"slices"
 
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -103,6 +104,30 @@ func (h *LifeCycleHandler) Handle(ctx context.Context, s state.VirtualMachineSta
 	}
 
 	log := logger.FromContext(ctx).With(logger.SlogHandler(nameLifeCycleHandler))
+	// While the pod is not running, the VMI does not set the node and the method returns nil, so it is necessary to check if there are any issues with the pod
+	if pod == nil {
+		cb := conditions.NewConditionBuilder(vmcondition.TypeRunning).Generation(changed.GetGeneration())
+
+		if volumeErr := h.checkPodVolumeErrors(ctx, changed, log); volumeErr != nil {
+			cb.Status(metav1.ConditionFalse).
+				Reason(vmcondition.ReasonPodVolumeErrors).
+				Message(fmt.Sprintf("Volume errors detected on Pod: %s", volumeErr.Error()))
+			conditions.SetCondition(cb, &changed.Status.Conditions)
+			return reconcile.Result{}, nil
+		}
+
+		isVMInContainerCreating, err := h.isVMInContainerCreatingState(ctx, changed, log)
+		if err != nil {
+			return reconcile.Result{}, err
+		}
+		if isVMInContainerCreating {
+			cb.Status(metav1.ConditionFalse).
+				Reason(vmcondition.ReasonPodContainerCreating).
+				Message("Pod is in ContainerCreating phase. Check the pod for more details.")
+			conditions.SetCondition(cb, &changed.Status.Conditions)
+			return reconcile.Result{}, nil
+		}
+	}
 
 	h.syncRunning(ctx, changed, kvvm, kvvmi, pod, log)
 	return reconcile.Result{}, nil
@@ -210,10 +235,31 @@ func (h *LifeCycleHandler) syncRunning(ctx context.Context, vm *v1alpha2.Virtual
 	} else {
 		vm.Status.Node = ""
 	}
+
 	cb.Reason(vmcondition.ReasonVirtualMachineNotRunning).Status(metav1.ConditionFalse)
 	conditions.SetCondition(cb, &vm.Status.Conditions)
 }
 
+func (h *LifeCycleHandler) isVMInContainerCreatingState(ctx context.Context, vm *v1alpha2.VirtualMachine, log *slog.Logger) (bool, error) {
+	var podList corev1.PodList
+	err := h.client.List(ctx, &podList, &client.ListOptions{
+		Namespace: vm.Namespace,
+		LabelSelector: labels.SelectorFromSet(map[string]string{
+			virtv1.VirtualMachineNameLabel: vm.Name,
+		}),
+	})
+	if err != nil {
+		log.Error("Failed to list pods", "error", err)
+		return false, err
+	}
+
+	if len(podList.Items) == 1 {
+		return isContainerCreating(&podList.Items[0]), nil
+	}
+
+	return false, nil
+}
+
 func (h *LifeCycleHandler) checkPodVolumeErrors(ctx context.Context, vm *v1alpha2.VirtualMachine, log *slog.Logger) error {
 	var podList corev1.PodList
 	err := h.client.List(ctx, &podList, &client.ListOptions{
@@ -237,6 +283,9 @@ func (h *LifeCycleHandler) checkPodVolumeErrors(ctx context.Context, vm *v1alpha
 }
 
 func isContainerCreating(pod *corev1.Pod) bool {
+	if pod == nil {
+		return false
+	}
 	if pod.Status.Phase != corev1.PodPending {
 		return false
 	}
@@ -266,10 +315,15 @@ func (h *LifeCycleHandler) getPodVolumeError(ctx context.Context, pod *corev1.Po
 		return nil
 	}
 
-	for _, e := range eventList.Items {
-		if e.Type == corev1.EventTypeWarning && (e.Reason == watcher.ReasonFailedAttachVolume || e.Reason == watcher.ReasonFailedMount) {
-			return fmt.Errorf("%s: %s", e.Reason, e.Message)
-		}
+	if len(eventList.Items) == 0 {
+		return nil
+	}
+
+	last := slices.MaxFunc(eventList.Items, func(a, b corev1.Event) int {
+		return a.LastTimestamp.Compare(b.LastTimestamp.Time)
+	})
+	if last.Reason == watcher.ReasonFailedAttachVolume || last.Reason == watcher.ReasonFailedMount {
+		return fmt.Errorf("%s: %s", last.Reason, last.Message)
 	}
 
 	return nil

images/virtualization-artifact/pkg/controller/vm/internal/watcher/pod_watcher.go

@@ -19,6 +19,7 @@ package watcher
 import (
 	"context"
 	"fmt"
+	"reflect"
 
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/types"
@@ -66,7 +67,8 @@ func (w *PodWatcher) Watch(mgr manager.Manager, ctr controller.Controller) error
 				DeleteFunc: func(e event.TypedDeleteEvent[*corev1.Pod]) bool { return true },
 				UpdateFunc: func(e event.TypedUpdateEvent[*corev1.Pod]) bool {
 					return e.ObjectOld.Status.Phase != e.ObjectNew.Status.Phase ||
-						e.ObjectOld.Annotations[annotations.AnnNetworksStatus] != e.ObjectNew.Annotations[annotations.AnnNetworksStatus]
+						e.ObjectOld.Annotations[annotations.AnnNetworksStatus] != e.ObjectNew.Annotations[annotations.AnnNetworksStatus] ||
+						!reflect.DeepEqual(e.ObjectOld.Status.ContainerStatuses, e.ObjectNew.Status.ContainerStatuses)
 				},
 			},
 		),

images/virtualization-artifact/pkg/controller/vmbda/internal/life_cycle.go

@@ -22,13 +22,15 @@ import (
 	"fmt"
 	"time"
 
+	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	virtv1 "kubevirt.io/api/core/v1"
 	"sigs.k8s.io/controller-runtime/pkg/reconcile"
 
 	"github.com/deckhouse/virtualization-controller/pkg/controller/conditions"
 	"github.com/deckhouse/virtualization-controller/pkg/controller/service"
 	intsvc "github.com/deckhouse/virtualization-controller/pkg/controller/vmbda/internal/service"
+	"github.com/deckhouse/virtualization-controller/pkg/controller/vmbda/internal/watcher"
 	"github.com/deckhouse/virtualization-controller/pkg/logger"
 	"github.com/deckhouse/virtualization/api/core/v1alpha2"
 	"github.com/deckhouse/virtualization/api/core/v1alpha2/vmbdacondition"
@@ -199,6 +201,11 @@ func (h LifeCycleHandler) Handle(ctx context.Context, vmbda *v1alpha2.VirtualMac
 	if err != nil {
 		if errors.Is(err, intsvc.ErrVolumeStatusNotReady) {
 			vmbda.Status.Phase = v1alpha2.BlockDeviceAttachmentPhaseInProgress
+
+			if handled, podErr := h.handleHotPlugPodIssues(ctx, ad, kvvmi, vmbda, cb); podErr != nil || handled {
+				return reconcile.Result{}, podErr
+			}
+
 			cb.
 				Status(metav1.ConditionFalse).
 				Reason(vmbdacondition.AttachmentRequestSent).
@@ -300,3 +307,63 @@ func (h LifeCycleHandler) Handle(ctx context.Context, vmbda *v1alpha2.VirtualMac
 		return reconcile.Result{}, err
 	}
 }
+
+func (h LifeCycleHandler) handleHotPlugPodIssues(
+	ctx context.Context,
+	ad *intsvc.AttachmentDisk,
+	kvvmi *virtv1.VirtualMachineInstance,
+	vmbda *v1alpha2.VirtualMachineBlockDeviceAttachment,
+	cb *conditions.ConditionBuilder,
+) (bool, error) {
+	hotPlugPod, err := h.attacher.GetHotPlugPod(ctx, ad, kvvmi)
+	if err != nil {
+		return false, err
+	}
+	if hotPlugPod == nil {
+		return false, nil
+	}
+
+	for _, c := range hotPlugPod.Status.Conditions {
+		if c.Type == corev1.PodScheduled && c.Status == corev1.ConditionFalse && c.Message != "" {
+			vmbda.Status.Phase = v1alpha2.BlockDeviceAttachmentPhasePending
+			cb.
+				Status(metav1.ConditionFalse).
+				Reason(vmbdacondition.HotPlugPodNotScheduled).
+				Message(fmt.Sprintf("Hot plug pod not scheduled: %s: %s", c.Reason, c.Message))
+			return true, nil
+		}
+	}
+
+	if isContainerCreating(hotPlugPod) {
+		lastEvent, err := h.attacher.GetLastPodEvent(ctx, hotPlugPod)
+		if err != nil {
+			return false, err
+		}
+		if lastEvent != nil && (lastEvent.Reason == watcher.ReasonFailedAttachVolume || lastEvent.Reason == watcher.ReasonFailedMount) {
+			cb.
+				Status(metav1.ConditionFalse).
+				Reason(vmbdacondition.FailedAttachVolume).
+				Message(fmt.Sprintf("Hot plug pod failed to attach volume: %s: %s", lastEvent.Reason, lastEvent.Message))
+			return true, nil
+		}
+
+		cb.Status(metav1.ConditionFalse).
+			Reason(vmbdacondition.AttachmentRequestSent).
+			Message(fmt.Sprintf("Pod %q is in ContainerCreating phase. Check the pod for more details.", hotPlugPod.Name))
+		return true, nil
+	}
+
+	return false, nil
+}
+
+func isContainerCreating(pod *corev1.Pod) bool {
+	if pod.Status.Phase != corev1.PodPending {
+		return false
+	}
+	for _, cs := range pod.Status.ContainerStatuses {
+		if cs.State.Waiting != nil && cs.State.Waiting.Reason == "ContainerCreating" {
+			return true
+		}
+	}
+	return false
+}

images/virtualization-artifact/pkg/controller/vmbda/internal/service/attachment_service.go

@@ -20,9 +20,11 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"slices"
 	"strings"
 
 	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/fields"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/component-helpers/scheduling/corev1/nodeaffinity"
 	virtv1 "kubevirt.io/api/core/v1"
@@ -314,6 +316,69 @@ func (s AttachmentService) IsPVAvailableOnVMNode(ctx context.Context, pvc *corev
 	return true, nil
 }
 
+func (s AttachmentService) GetHotPlugPod(ctx context.Context, ad *AttachmentDisk, kvvmi *virtv1.VirtualMachineInstance) (*corev1.Pod, error) {
+	if ad == nil || kvvmi == nil {
+		return nil, nil
+	}
+
+	for _, vs := range kvvmi.Status.VolumeStatus {
+		if vs.HotplugVolume == nil || vs.Name != ad.GenerateName {
+			continue
+		}
+		if vs.HotplugVolume.AttachPodName == "" {
+			return nil, nil
+		}
+
+		return object.FetchObject(ctx, types.NamespacedName{
+			Namespace: kvvmi.Namespace,
+			Name:      vs.HotplugVolume.AttachPodName,
+		}, s.client, &corev1.Pod{})
+	}
+	return nil, nil
+}
+
+func (s AttachmentService) GetHotPlugPodCondition(ctx context.Context, ad *AttachmentDisk, kvvmi *virtv1.VirtualMachineInstance, condType corev1.PodConditionType) (*corev1.PodCondition, error) {
+	pod, err := s.GetHotPlugPod(ctx, ad, kvvmi)
+	if err != nil || pod == nil {
+		return nil, err
+	}
+
+	for i, c := range pod.Status.Conditions {
+		if c.Type == condType {
+			return &pod.Status.Conditions[i], nil
+		}
+	}
+	return nil, nil
+}
+
+func (s AttachmentService) GetLastPodEvent(ctx context.Context, pod *corev1.Pod) (*corev1.Event, error) {
+	if pod == nil {
+		return nil, nil
+	}
+
+	eventList := &corev1.EventList{}
+	err := s.client.List(ctx, eventList, &client.ListOptions{
+		Namespace: pod.Namespace,
+		FieldSelector: fields.SelectorFromSet(fields.Set{
+			"involvedObject.name": pod.Name,
+			"involvedObject.kind": "Pod",
+		}),
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	if len(eventList.Items) == 0 {
+		return nil, nil
+	}
+
+	last := slices.MaxFunc(eventList.Items, func(a, b corev1.Event) int {
+		return a.LastTimestamp.Compare(b.LastTimestamp.Time)
+	})
+
+	return &last, nil
+}
+
 func isSameBlockDeviceRefs(a, b v1alpha2.VMBDAObjectRef) bool {
 	return a.Kind == b.Kind && a.Name == b.Name
 }