From d277b59c43af6dec993f345328b23797e470d07e Mon Sep 17 00:00:00 2001 From: hutiefang76 <137664623+hutiefang76@users.noreply.github.com> Date: Sun, 21 Jun 2026 03:36:50 +0800 Subject: [PATCH] [alerter]bugfix: preserve group alert recovery events --- .../notice/impl/DbAlertStoreHandlerImpl.java | 17 +++++++ .../alert/reduce/AlarmGroupReduce.java | 13 +++++- .../impl/DbAlertStoreHandlerImplTest.java | 46 +++++++++++++++++++ .../alert/reduce/AlarmGroupReduceTest.java | 42 ++++++++++++++++- 4 files changed, 115 insertions(+), 3 deletions(-) diff --git a/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/notice/impl/DbAlertStoreHandlerImpl.java b/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/notice/impl/DbAlertStoreHandlerImpl.java index 24fd7d11412..2ac441508aa 100644 --- a/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/notice/impl/DbAlertStoreHandlerImpl.java +++ b/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/notice/impl/DbAlertStoreHandlerImpl.java @@ -127,9 +127,26 @@ public GroupAlert store(GroupAlert groupAlert) { } // Save alert group groupAlert.setAlertFingerprints(alertFingerprints.stream().toList()); + refreshGroupStatus(groupAlert); GroupAlert savedGroupAlert = groupAlertDao.save(groupAlert); savedGroupAlert.setAlerts(groupAlert.getAlerts()); return savedGroupAlert; } } + + private void refreshGroupStatus(GroupAlert groupAlert) { + List alertFingerprints = groupAlert.getAlertFingerprints(); + if (alertFingerprints == null || alertFingerprints.isEmpty()) { + return; + } + List alerts = singleAlertDao.findSingleAlertsByFingerprintIn(alertFingerprints); + if (alerts == null) { + return; + } + boolean hasFiringAlert = alerts.stream() + .anyMatch(alert -> CommonConstants.ALERT_STATUS_FIRING.equals(alert.getStatus())); + groupAlert.setStatus(hasFiringAlert + ? CommonConstants.ALERT_STATUS_FIRING + : CommonConstants.ALERT_STATUS_RESOLVED); + } } diff --git a/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/reduce/AlarmGroupReduce.java b/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/reduce/AlarmGroupReduce.java index 597ef9a58a6..2d44b7fc6e1 100644 --- a/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/reduce/AlarmGroupReduce.java +++ b/hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/reduce/AlarmGroupReduce.java @@ -294,13 +294,17 @@ private void sendGroupAlert(GroupAlertCache cache) { AlertGroupConverge ruleConfig = groupDefines.get(cache.getGroupDefineName()); long repeatInterval = ruleConfig.getRepeatInterval() != null ? ruleConfig.getRepeatInterval() * MS_PER_SECOND : DEFAULT_REPEAT_INTERVAL; + boolean hasResolvedAlert = hasResolvedAlert(cache.getAlertFingerprints().values()); // Skip if within repeat interval - if (cache.getLastRepeatTime() > 0 + if (!hasResolvedAlert + && cache.getLastRepeatTime() > 0 && now - cache.getLastRepeatTime() < repeatInterval) { return; } - cache.setLastRepeatTime(now); + if (!hasResolvedAlert) { + cache.setLastRepeatTime(now); + } } GroupAlert groupAlert = GroupAlert.builder() @@ -392,6 +396,11 @@ private String determineGroupStatus(Collection alerts) { .anyMatch(alert -> CommonConstants.ALERT_STATUS_FIRING.equals(alert.getStatus())) ? CommonConstants.ALERT_STATUS_FIRING : CommonConstants.ALERT_STATUS_RESOLVED; } + + private boolean hasResolvedAlert(Collection alerts) { + return alerts.stream() + .anyMatch(alert -> CommonConstants.ALERT_STATUS_RESOLVED.equals(alert.getStatus())); + } @Data private static class GroupAlertCache { diff --git a/hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/notice/impl/DbAlertStoreHandlerImplTest.java b/hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/notice/impl/DbAlertStoreHandlerImplTest.java index 560d6501fdd..b97f35bf7f9 100644 --- a/hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/notice/impl/DbAlertStoreHandlerImplTest.java +++ b/hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/notice/impl/DbAlertStoreHandlerImplTest.java @@ -19,11 +19,13 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyList; import static org.mockito.Mockito.never; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; import org.apache.hertzbeat.alert.dao.GroupAlertDao; import org.apache.hertzbeat.alert.dao.SingleAlertDao; +import org.apache.hertzbeat.common.constants.CommonConstants; import org.apache.hertzbeat.common.entity.alerter.GroupAlert; import org.apache.hertzbeat.common.entity.alerter.SingleAlert; import org.junit.jupiter.api.BeforeEach; @@ -122,4 +124,48 @@ public void testStoreExistingAlert() { assertEquals(1L, groupAlert.getId()); } + @Test + public void storeResolvedAlertKeepsGroupFiringWhenOtherGroupAlertsStillFire() { + String groupKey = "instance:host1"; + String resolvedFingerprint = "cpu"; + String firingFingerprint = "memory"; + groupAlert.setGroupKey(groupKey); + groupAlert.setStatus(CommonConstants.ALERT_STATUS_RESOLVED); + singleAlert.setFingerprint(resolvedFingerprint); + singleAlert.setStatus(CommonConstants.ALERT_STATUS_RESOLVED); + + GroupAlert existingGroup = new GroupAlert(); + existingGroup.setId(1L); + existingGroup.setAlertFingerprints(List.of(resolvedFingerprint, firingFingerprint)); + when(groupAlertDao.findByGroupKey(groupKey)).thenReturn(existingGroup); + + SingleAlert previousFiringAlert = new SingleAlert(); + previousFiringAlert.setId(1L); + previousFiringAlert.setFingerprint(resolvedFingerprint); + previousFiringAlert.setStatus(CommonConstants.ALERT_STATUS_FIRING); + previousFiringAlert.setStartAt(1000L); + previousFiringAlert.setActiveAt(2000L); + previousFiringAlert.setTriggerTimes(3); + when(singleAlertDao.findByFingerprint(resolvedFingerprint)).thenReturn(previousFiringAlert); + + SingleAlert savedResolvedAlert = new SingleAlert(); + savedResolvedAlert.setId(1L); + savedResolvedAlert.setFingerprint(resolvedFingerprint); + savedResolvedAlert.setStatus(CommonConstants.ALERT_STATUS_RESOLVED); + when(singleAlertDao.save(any(SingleAlert.class))).thenReturn(savedResolvedAlert); + + SingleAlert existingFiringAlert = new SingleAlert(); + existingFiringAlert.setFingerprint(firingFingerprint); + existingFiringAlert.setStatus(CommonConstants.ALERT_STATUS_FIRING); + when(singleAlertDao.findSingleAlertsByFingerprintIn(anyList())) + .thenReturn(List.of(savedResolvedAlert, existingFiringAlert)); + + when(groupAlertDao.save(any(GroupAlert.class))).thenAnswer(invocation -> invocation.getArgument(0)); + + GroupAlert savedGroupAlert = dbAlertStoreHandler.store(groupAlert); + + assertEquals(CommonConstants.ALERT_STATUS_FIRING, savedGroupAlert.getStatus()); + assertEquals(CommonConstants.ALERT_STATUS_FIRING, groupAlert.getStatus()); + } + } diff --git a/hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/reduce/AlarmGroupReduceTest.java b/hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/reduce/AlarmGroupReduceTest.java index a042032cab4..30339a1a167 100644 --- a/hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/reduce/AlarmGroupReduceTest.java +++ b/hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/reduce/AlarmGroupReduceTest.java @@ -44,6 +44,7 @@ import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.argThat; import static org.mockito.Mockito.never; +import static org.mockito.Mockito.clearInvocations; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; import java.util.Arrays; @@ -55,6 +56,7 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import org.apache.hertzbeat.alert.dao.AlertGroupConvergeDao; +import org.apache.hertzbeat.common.constants.CommonConstants; import org.apache.hertzbeat.common.config.VirtualThreadProperties; import org.apache.hertzbeat.common.entity.alerter.AlertGroupConverge; import org.apache.hertzbeat.common.entity.alerter.SingleAlert; @@ -83,7 +85,7 @@ void setUp() { when(alertGroupConvergeDao.findAlertGroupConvergesByEnableIsTrue()) .thenReturn(Collections.emptyList()); alarmGroupReduce = new AlarmGroupReduce(alarmInhibitReduce, alertGroupConvergeDao, - new VirtualThreadProperties(), false); + new VirtualThreadProperties(false, null, null, null, null, null, null), false); } @AfterEach @@ -129,6 +131,35 @@ void whenMatchingGroupRule_shouldGroup() { verify(alarmInhibitReduce, never()).inhibitAlarm(any()); // Should not send immediately due to group wait } + @Test + void resolvedAlertInFiringGroupShouldBypassRepeatThrottle() { + AlertGroupConverge rule = new AlertGroupConverge(); + rule.setName("test-rule"); + rule.setGroupLabels(Collections.singletonList("instance")); + rule.setGroupWait(0L); + rule.setGroupInterval(0L); + rule.setRepeatInterval(60L); + alarmGroupReduce.refreshGroupDefines(Collections.singletonList(rule)); + + alarmGroupReduce.processGroupAlert(createAlert("cpu", CommonConstants.ALERT_STATUS_FIRING)); + alarmGroupReduce.dispatchCheckAndSendGroups(); + verify(alarmInhibitReduce).inhibitAlarm(argThat(group -> + CommonConstants.ALERT_STATUS_FIRING.equals(group.getStatus()) + && group.getAlerts().size() == 1 + && group.getAlerts().get(0).getFingerprint().equals("cpu"))); + clearInvocations(alarmInhibitReduce); + + alarmGroupReduce.processGroupAlert(createAlert("cpu", CommonConstants.ALERT_STATUS_FIRING)); + alarmGroupReduce.processGroupAlert(createAlert("memory", CommonConstants.ALERT_STATUS_RESOLVED)); + alarmGroupReduce.dispatchCheckAndSendGroups(); + + verify(alarmInhibitReduce).inhibitAlarm(argThat(group -> + CommonConstants.ALERT_STATUS_FIRING.equals(group.getStatus()) + && group.getAlerts().stream() + .anyMatch(alert -> "memory".equals(alert.getFingerprint()) + && CommonConstants.ALERT_STATUS_RESOLVED.equals(alert.getStatus())))); + } + @Test void dispatchCheckAndSendGroupsRunsOnVirtualThread() throws Exception { CountDownLatch latch = new CountDownLatch(1); @@ -173,6 +204,15 @@ private Map createLabels(String... keyValues) { return labels; } + private SingleAlert createAlert(String fingerprint, String status) { + return SingleAlert.builder() + .fingerprint(fingerprint) + .status(status) + .labels(createLabels("instance", "host1")) + .annotations(Collections.emptyMap()) + .build(); + } + private static final class TestAlarmGroupReduce extends AlarmGroupReduce { private final CountDownLatch virtualThreadLatch;