[Observability] Use metric filter to generate the clustermgtd heartbeat metric.

Author: gmarciani

cli/src/pcluster/constants.py

@@ -188,6 +188,12 @@
 CW_ALARM_DATAPOINTS_TO_ALARM_DEFAULT = 1
 DETAILED_MONITORING_ENABLED_DEFAULT = False
 
+# CloudWatch Metrics
+CW_METRICS_NAMESPACE = "ParallelCluster"
+CW_METRICS_DIMENSION_CLUSTER_NAME = "ClusterName"
+CW_METRICS_DIMENSION_INSTANCE_ID = "InstanceId"
+CW_METRICS_CLUSTERMGTD_HEARTBEAT = "ClustermgtdHeartbeat"
+
 STACK_EVENTS_LOG_STREAM_NAME_FORMAT = "{}-cfn-events"
 
 PCLUSTER_IMAGE_NAME_REGEX = r"^[-_A-Za-z0-9{][-_A-Za-z0-9\s:{}\.]+[-_A-Za-z0-9}]$"

cli/src/pcluster/templates/cluster_stack.py

@@ -66,6 +66,7 @@
     CW_LOGS_CFN_PARAM_NAME,
     CW_METRICS_CLUSTERMGTD_HEARTBEAT,
     CW_METRICS_DIMENSION_CLUSTER_NAME,
+    CW_METRICS_DIMENSION_INSTANCE_ID,
     CW_METRICS_NAMESPACE,
     DEFAULT_EPHEMERAL_DIR,
     EFS_PORT,
@@ -351,9 +352,18 @@ def _add_internal_efs_shared_storage(self):
         self._add_shared_storage(internal_efs_storage_shared)
 
     def _cw_metric_head_node(
-        self, namespace, metric_name, statistic="Maximum", period_seconds=CW_ALARM_PERIOD_DEFAULT, extra_dimensions=None
+        self,
+        namespace,
+        metric_name,
+        statistic="Maximum",
+        period_seconds=CW_ALARM_PERIOD_DEFAULT,
+        extra_dimensions=None,
+        override_dimensions=None,
     ):
-        dimensions = {"InstanceId": self.head_node_instance.ref}
+        if override_dimensions:
+            dimensions = override_dimensions
+        else:
+            dimensions = {CW_METRICS_DIMENSION_INSTANCE_ID: self.head_node_instance.ref}
         if extra_dimensions:
             dimensions.update(extra_dimensions)
         return cloudwatch.Metric(
@@ -384,12 +394,35 @@ def _add_head_node_alarms(self):
             },
         }
 
-        if self._condition_is_slurm():
-            metrics_for_alarms["ClustermgtdHeartbeat"] = {
+        # These alarms required Cw logging enabled because they are based on CW Metrics Filters.
+        if self._condition_is_slurm() and self.config.is_cw_logging_enabled:
+            # Create metric filter to extract heartbeat metric from clustermgtd event logs
+            clustermgtd_heartbeat_metric_filter = logs.CfnMetricFilter(
+                scope=self.stack,
+                id=f"{CW_METRICS_CLUSTERMGTD_HEARTBEAT}Filter",
+                filter_pattern='{ $.event-type = "clustermgtd-heartbeat" }',
+                log_group_name=self.log_group_name,
+                metric_transformations=[
+                    logs.CfnMetricFilter.MetricTransformationProperty(
+                        metric_namespace=CW_METRICS_NAMESPACE,
+                        metric_name=CW_METRICS_CLUSTERMGTD_HEARTBEAT,
+                        metric_value="1",
+                        unit="Count",
+                        dimensions=[
+                            logs.CfnMetricFilter.DimensionProperty(
+                                key=CW_METRICS_DIMENSION_CLUSTER_NAME, value="$.cluster-name"
+                            )
+                        ],
+                    )
+                ],
+            )
+            clustermgtd_heartbeat_metric_filter.add_depends_on(self.log_group)
+
+            metrics_for_alarms[CW_METRICS_CLUSTERMGTD_HEARTBEAT] = {
                 "metric": self._cw_metric_head_node(
                     CW_METRICS_NAMESPACE,
                     CW_METRICS_CLUSTERMGTD_HEARTBEAT,
-                    extra_dimensions={CW_METRICS_DIMENSION_CLUSTER_NAME: self.config.cluster_name},
+                    override_dimensions={CW_METRICS_DIMENSION_CLUSTER_NAME: self.config.cluster_name},
                 ),
                 "evaluation_periods": 10,
                 "datapoints_to_alarm": 10,

cli/tests/pcluster/example_configs/slurm.logging_disabled.yaml

@@ -0,0 +1,22 @@
+Image:
+  Os: alinux2
+HeadNode:
+  InstanceType: t3.micro
+  Networking:
+    SubnetId: subnet-12345678
+  Ssh:
+    KeyName: ec2-key-name
+Scheduling:
+  Scheduler: slurm
+  SlurmQueues:
+    - Name: queue1
+      Networking:
+        SubnetIds:
+          - subnet-12345678
+      ComputeResources:
+        - Name: compute-resource1
+          InstanceType: c5.2xlarge
+Monitoring:
+  Logs:
+    CloudWatch:
+      Enabled: false

cli/tests/pcluster/templates/test_cluster_stack.py

@@ -250,6 +250,7 @@ def test_add_efs_shared_storage(mocker, test_datadir, config_file_name, expected
     [
         "slurm.required.yaml",
         "slurm.full.yaml",
+        "slurm.logging_disabled.yaml",
         "awsbatch.simple.yaml",
         "awsbatch.full.yaml",
     ],
@@ -315,7 +316,7 @@ def test_add_alarms(mocker, config_file_name):
         },
     }
 
-    if cluster.scheduling.scheduler == "slurm":
+    if cluster.scheduling.scheduler == "slurm" and cluster.is_cw_logging_enabled:
         expected_alarms["Clustermgtd-Heartbeat"] = {
             "name": "clustername-HeadNode-ClustermgtdHeartbeat",
             "metric_name": "ClustermgtdHeartbeat",