Add support for Preservation of Machines and Backing nodes

docs/documents/apis.md

@@ -513,6 +513,21 @@ not be estimated during the time a MachineDeployment is paused. This is not set
 by default, which is treated as infinite deadline.</p>
 </td>
 </tr>
+<tr>
+<td>
+<code>autoPreserveFailedMachineMax</code>
+</td>
+<td>
+<em>
+int32
+</em>
+</td>
+<td>
+<em>(Optional)</em>
+<p>The maximum number of machines in the machine deployment that will be auto-preserved.
+In the gardener context, this number is derived from the AutoPreserveFailedMachineMax set at the worker level, distributed amongst the worker&rsquo;s machine deployments</p>
+</td>
+</tr>
 </table>
 </td>
 </tr>
@@ -678,6 +693,19 @@ int32
 <em>(Optional)</em>
 </td>
 </tr>
+<tr>
+<td>
+<code>autoPreserveFailedMachineMax</code>
+</td>
+<td>
+<em>
+int32
+</em>
+</td>
+<td>
+<em>(Optional)</em>
+</td>
+</tr>
 </table>
 </td>
 </tr>
@@ -833,6 +861,21 @@ Kubernetes meta/v1.Time
 <p>Last update time of current status</p>
 </td>
 </tr>
+<tr>
+<td>
+<code>preserveExpiryTime</code>
+</td>
+<td>
+<em>
+<a href="https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.29/#time-v1-meta">
+Kubernetes meta/v1.Time
+</a>
+</em>
+</td>
+<td>
+<p>PreserveExpiryTime is the time at which MCM will stop preserving the machine</p>
+</td>
+</tr>
 </tbody>
 </table>
 <br>
@@ -1071,6 +1114,22 @@ Kubernetes meta/v1.Duration
 </tr>
 <tr>
 <td>
+<code>preserveTimeout</code>
+</td>
+<td>
+<em>
+<a href="https://godoc.org/k8s.io/apimachinery/pkg/apis/meta/v1#Duration">
+Kubernetes meta/v1.Duration
+</a>
+</em>
+</td>
+<td>
+<em>(Optional)</em>
+<p>MachinePreserveTimeout is the timeout after which the machine preservation is stopped</p>
+</td>
+</tr>
+<tr>
+<td>
 <code>disableHealthTimeout</code>
 </td>
 <td>
@@ -1398,6 +1457,21 @@ not be estimated during the time a MachineDeployment is paused. This is not set
 by default, which is treated as infinite deadline.</p>
 </td>
 </tr>
+<tr>
+<td>
+<code>autoPreserveFailedMachineMax</code>
+</td>
+<td>
+<em>
+int32
+</em>
+</td>
+<td>
+<em>(Optional)</em>
+<p>The maximum number of machines in the machine deployment that will be auto-preserved.
+In the gardener context, this number is derived from the AutoPreserveFailedMachineMax set at the worker level, distributed amongst the worker&rsquo;s machine deployments</p>
+</td>
+</tr>
 </tbody>
 </table>
 <br>
@@ -1543,8 +1617,8 @@ newest MachineSet.</p>
 </td>
 <td>
 <em>
-<a href="#machine.sapcloud.io/v1alpha1.*../../pkg/apis/machine/v1alpha1.MachineSummary">
-[]*../../pkg/apis/machine/v1alpha1.MachineSummary
+<a href="#machine.sapcloud.io/v1alpha1.*github.com/thiyyakat/machine-controller-manager/pkg/apis/machine/v1alpha1.MachineSummary">
+[]*github.com/thiyyakat/machine-controller-manager/pkg/apis/machine/v1alpha1.MachineSummary
 </a>
 </em>
 </td>
@@ -1860,6 +1934,19 @@ int32
 <em>(Optional)</em>
 </td>
 </tr>
+<tr>
+<td>
+<code>autoPreserveFailedMachineMax</code>
+</td>
+<td>
+<em>
+int32
+</em>
+</td>
+<td>
+<em>(Optional)</em>
+</td>
+</tr>
 </tbody>
 </table>
 <br>
@@ -1988,8 +2075,8 @@ LastOperation
 </td>
 <td>
 <em>
-<a href="#machine.sapcloud.io/v1alpha1.[]../../pkg/apis/machine/v1alpha1.MachineSummary">
-[]../../pkg/apis/machine/v1alpha1.MachineSummary
+<a href="#machine.sapcloud.io/v1alpha1.[]github.com/thiyyakat/machine-controller-manager/pkg/apis/machine/v1alpha1.MachineSummary">
+[]github.com/thiyyakat/machine-controller-manager/pkg/apis/machine/v1alpha1.MachineSummary
 </a>
 </em>
 </td>
@@ -1998,6 +2085,20 @@ LastOperation
 <p>FailedMachines has summary of machines on which lastOperation Failed</p>
 </td>
 </tr>
+<tr>
+<td>
+<code>autoPreserveFailedMachineCount</code>
+</td>
+<td>
+<em>
+int32
+</em>
+</td>
+<td>
+<em>(Optional)</em>
+<p>AutoPreserveFailedMachineCount has a count of the number of failed machines in the machineset that have been auto-preserved</p>
+</td>
+</tr>
 </tbody>
 </table>
 <br>

kubernetes/crds/machine.sapcloud.io_machinedeployments.yaml

@@ -67,6 +67,12 @@ spec:
           spec:
             description: Specification of the desired behavior of the MachineDeployment.
             properties:
+              autoPreserveFailedMachineMax:
+                description: |-
+                  The maximum number of machines in the machine deployment that will be auto-preserved.
+                  In the gardener context, this number is derived from the AutoPreserveFailedMachineMax set at the worker level, distributed amongst the worker's machine deployments
+                format: int32
+                type: integer
               minReadySeconds:
                 description: |-
                   Minimum number of seconds for which a newly created machine should be ready
@@ -296,6 +302,10 @@ spec:
                         description: MachineInPlaceUpdateTimeout is the timeout after
                           which in-place update is declared failed.
                         type: string
+                      machinePreserveTimeout:
+                        description: MachinePreserveTimeout is the timeout after which
+                          the machine preservation is stopped
+                        type: string
                       maxEvictRetries:
                         description: MaxEvictRetries is the number of retries that
                           will be attempted while draining the node.

kubernetes/crds/machine.sapcloud.io_machines.yaml

@@ -95,6 +95,10 @@ spec:
                 description: MachineInPlaceUpdateTimeout is the timeout after which
                   in-place update is declared failed.
                 type: string
+              machinePreserveTimeout:
+                description: MachinePreserveTimeout is the timeout after which the
+                  machine preservation is stopped
+                type: string
               maxEvictRetries:
                 description: MaxEvictRetries is the number of retries that will be
                   attempted while draining the node.
@@ -287,6 +291,11 @@ spec:
                     description: MachinePhase is a label for the condition of a machine
                       at the current time.
                     type: string
+                  preserveExpiryTime:
+                    description: PreserveExpiryTime is the time at which MCM will
+                      stop preserving the machine
+                    format: date-time
+                    type: string
                   timeoutActive:
                     type: boolean
                 type: object

kubernetes/crds/machine.sapcloud.io_machinesets.yaml

@@ -60,6 +60,9 @@ spec:
           spec:
             description: MachineSetSpec is the specification of a MachineSet.
             properties:
+              autoPreserveFailedMachineMax:
+                format: int32
+                type: integer
               machineClass:
                 description: ClassSpec is the class specification of machine
                 properties:
@@ -178,6 +181,10 @@ spec:
                         description: MachineInPlaceUpdateTimeout is the timeout after
                           which in-place update is declared failed.
                         type: string
+                      machinePreserveTimeout:
+                        description: MachinePreserveTimeout is the timeout after which
+                          the machine preservation is stopped
+                        type: string
                       maxEvictRetries:
                         description: MaxEvictRetries is the number of retries that
                           will be attempted while draining the node.
@@ -312,6 +319,11 @@ spec:
             description: MachineSetStatus holds the most recently observed status
               of MachineSet.
             properties:
+              autoPreserveFailedMachineCount:
+                description: AutoPreserveFailedMachineCount has a count of the number
+                  of failed machines in the machineset that have been auto-preserved
+                format: int32
+                type: integer
               availableReplicas:
                 description: The number of available replicas (ready for at least
                   minReadySeconds) for this replica set.

pkg/apis/machine/types.go

@@ -97,6 +97,8 @@ type MachineConfiguration struct {
 	// MachineInPlaceUpdateTimeout is the timeout after which in-place update is declared failed.
 	MachineInPlaceUpdateTimeout *metav1.Duration
 
+	// MachinePreserveTimeout is the timeout after which the machine preservation is stopped
+	MachinePreserveTimeout *metav1.Duration
 	// DisableHealthTimeout if set to true, health timeout will be ignored. Leading to machine never being declared failed.
 	// This is intended to be used only for in-place updates.
 	DisableHealthTimeout *bool
@@ -158,6 +160,9 @@ type CurrentStatus struct {
 
 	// Last update time of current status
 	LastUpdateTime metav1.Time
+
+	// PreserveExpiryTime is the time at which MCM will stop preserving the machine
+	PreserveExpiryTime metav1.Time
 }
 
 // MachineStatus holds the most recently observed status of Machine.
@@ -351,6 +356,8 @@ type MachineSetSpec struct {
 	Template MachineTemplateSpec
 
 	MinReadySeconds int32
+
+	AutoPreserveFailedMachineMax int32
 }
 
 // MachineSetConditionType is the condition on machineset object
@@ -409,6 +416,9 @@ type MachineSetStatus struct {
 
 	// FailedMachines has summary of machines on which lastOperation Failed
 	FailedMachines *[]MachineSummary
+
+	// AutoPreserveFailedMachineCount has a count of the number of failed machines in the machineset that have been auto-preserved
+	AutoPreserveFailedMachineCount int32
 }
 
 // MachineSummary store the summary of machine.
@@ -487,6 +497,10 @@ type MachineDeploymentSpec struct {
 	// not be estimated during the time a MachineDeployment is paused. This is not set
 	// by default.
 	ProgressDeadlineSeconds *int32
+
+	// The maximum number of machines in the machine deployment that will be auto-preserved.
+	// In the gardener context, this number is derived from the AutoPreserveFailedMachineMax set at the worker level, distributed amongst the worker's machine deployments
+	AutoPreserveFailedMachineMax int32
 }
 
 // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object

pkg/apis/machine/v1alpha1/machine_types.go

@@ -244,6 +244,26 @@ const (
 	UpdateFailed string = "UpdateFailed"
 )
 
+const (
+	// NodePreserved is a node condition type for preservation of machines to allow end-user to know that a node is preserved
+	NodePreserved corev1.NodeConditionType = "NodePreserved"
+
+	// NodePreservedByMCM is a node condition reason for preservation of machines to indicate that the node is auto-preserved by MCM
+	NodePreservedByMCM string = "PreservedByMCM"
+
+	// NodePreservedByUser is a node condition reason to indicate that a machine/node has been preserved due to explicit annotation by user
+	NodePreservedByUser string = "PreservedByUser"
+
+	// NodePreservationStopped is a node condition reason to indicate that a machine/node preservation has been stopped due to annotation update or timeout
+	NodePreservationStopped string = "PreservationStopped"
+
+	// PreservedNodeDrainSuccessful is a constant for the message in condition that indicates that the preserved node's drain is successful
+	PreservedNodeDrainSuccessful string = "Preserved Node drained successfully"
+
+	// PreservedNodeDrainUnsuccessful is a constant for the message in condition that indicates that the preserved node's drain was not successful
+	PreservedNodeDrainUnsuccessful string = "Preserved Node could not be drained"
+)
+
 // CurrentStatus contains information about the current status of Machine.
 type CurrentStatus struct {
 	Phase MachinePhase `json:"phase,omitempty"`
@@ -252,6 +272,9 @@ type CurrentStatus struct {
 
 	// Last update time of current status
 	LastUpdateTime metav1.Time `json:"lastUpdateTime,omitempty"`
+
+	// PreserveExpiryTime is the time at which MCM will stop preserving the machine
+	PreserveExpiryTime metav1.Time `json:"preserveExpiryTime,omitempty"`
 }
 
 // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object

pkg/apis/machine/v1alpha1/machinedeployment_types.go

@@ -91,6 +91,11 @@ type MachineDeploymentSpec struct {
 	// by default, which is treated as infinite deadline.
 	// +optional
 	ProgressDeadlineSeconds *int32 `json:"progressDeadlineSeconds,omitempty"`
+
+	// The maximum number of machines in the machine deployment that will be auto-preserved.
+	// In the gardener context, this number is derived from the AutoPreserveFailedMachineMax set at the worker level, distributed amongst the worker's machine deployments
+	// +optional
+	AutoPreserveFailedMachineMax int32 `json:"autoPreserveFailedMachineMax,omitempty"`
 }
 
 const (

pkg/apis/machine/v1alpha1/machineset_types.go

@@ -68,6 +68,9 @@ type MachineSetSpec struct {
 
 	// +optional
 	MinReadySeconds int32 `json:"minReadySeconds,omitempty"`
+
+	// +optional
+	AutoPreserveFailedMachineMax int32 `json:"autoPreserveFailedMachineMax,omitempty"`
 }
 
 // MachineSetConditionType is the condition on machineset object
@@ -135,4 +138,8 @@ type MachineSetStatus struct {
 	// FailedMachines has summary of machines on which lastOperation Failed
 	// +optional
 	FailedMachines *[]MachineSummary `json:"failedMachines,omitempty"`
+
+	// AutoPreserveFailedMachineCount has a count of the number of failed machines in the machineset that have been auto-preserved
+	// +optional
+	AutoPreserveFailedMachineCount int32 `json:"autoPreserveFailedMachineCount,omitempty"`
 }

pkg/apis/machine/v1alpha1/shared_types.go

@@ -44,6 +44,10 @@ type MachineConfiguration struct {
 	// +optional
 	MachineInPlaceUpdateTimeout *metav1.Duration `json:"inPlaceUpdateTimeout,omitempty"`
 
+	// MachinePreserveTimeout is the timeout after which the machine preservation is stopped
+	// +optional
+	MachinePreserveTimeout *metav1.Duration `json:"machinePreserveTimeout,omitempty"`
+
 	// DisableHealthTimeout if set to true, health timeout will be ignored. Leading to machine never being declared failed.
 	// This is intended to be used only for in-place updates.
 	// +optional