feat:Enhance the alarm kernel with recovered status notification capability for alarm rules

.github/workflows/skywalking.yaml

@@ -780,7 +780,7 @@ jobs:
         if: matrix.test.docker != null
         run: docker build -t ${{ matrix.test.docker.name }} -f ${{ matrix.test.docker.base }}/${{ matrix.test.docker.file }} ${{ matrix.test.docker.base }}
       - name: ${{ matrix.test.name }}
-        uses: apache/skywalking-infra-e2e@cf589b4a0b9f8e6f436f78e9cfd94a1ee5494180
+        uses: apache/skywalking-infra-e2e@01b80d98a38154f4f80d9cdb128b9d81727f2b80
         with:
           e2e-file: $GITHUB_WORKSPACE/${{ matrix.test.config }}
       - if: ${{ failure() }}
@@ -844,7 +844,7 @@ jobs:
           username: ${{ github.repository_owner }}
           password: ${{ secrets.GITHUB_TOKEN }}
       - name: ${{ matrix.test.name }}
-        uses: apache/skywalking-infra-e2e@cf589b4a0b9f8e6f436f78e9cfd94a1ee5494180
+        uses: apache/skywalking-infra-e2e@01b80d98a38154f4f80d9cdb128b9d81727f2b80
         env:
           ISTIO_VERSION: ${{ matrix.versions.istio }}
           KUBERNETES_VERSION: ${{ matrix.versions.kubernetes }}
@@ -905,7 +905,7 @@ jobs:
           username: ${{ github.repository_owner }}
           password: ${{ secrets.GITHUB_TOKEN }}
       - name: ${{ matrix.test.name }}
-        uses: apache/skywalking-infra-e2e@cf589b4a0b9f8e6f436f78e9cfd94a1ee5494180
+        uses: apache/skywalking-infra-e2e@01b80d98a38154f4f80d9cdb128b9d81727f2b80
         env:
           ISTIO_VERSION: ${{ matrix.versions.istio }}
           KUBERNETES_VERSION: ${{ matrix.versions.kubernetes }}
@@ -968,7 +968,7 @@ jobs:
         shell: bash
         run: ./mvnw -B -q -f test/e2e-v2/java-test-service/pom.xml clean package
       - name: Java version ${{ matrix.java-version }}
-        uses: apache/skywalking-infra-e2e@cf589b4a0b9f8e6f436f78e9cfd94a1ee5494180
+        uses: apache/skywalking-infra-e2e@01b80d98a38154f4f80d9cdb128b9d81727f2b80
         env:
           SW_AGENT_JDK_VERSION: ${{ matrix.java-version }}
         with:
@@ -1064,7 +1064,7 @@ jobs:
 #          fi
 #          docker compose -f ${BANYANDB_DATA_GENERATE_ROOT}/docker-compose.yml down -v
 #      - name: ${{ matrix.test.name }}
-#        uses: apache/skywalking-infra-e2e@cf589b4a0b9f8e6f436f78e9cfd94a1ee5494180
+#        uses: apache/skywalking-infra-e2e@01b80d98a38154f4f80d9cdb128b9d81727f2b80
 #        with:
 #          e2e-file: $GITHUB_WORKSPACE/${{ matrix.test.config }}
 #      - if: ${{ failure() }}

dist-material/alarm-settings.yml

@@ -23,6 +23,8 @@ rules:
     expression: sum(service_resp_time > 1000) >= 3
     period: 10
     silence-period: 5
+    # Number of periods to wait before considering the alarm recovered,default as 0.
+    recovery-observation-period: 3
     message: Response time of service {name} is more than 1000ms in 3 minutes of last 10 minutes.
 #  service_resp_time_rule:
 #    expression: avg(service_resp_time) > 1000
@@ -35,16 +37,20 @@ rules:
     period: 10
     # How many times of checks, the alarm keeps silence after alarm triggered, default as same as period.
     silence-period: 3
+    # Number of periods to wait before considering the alarm recovered,default as 0.
+    recovery-observation-period: 2
     message: Successful rate of service {name} is lower than 80% in 2 minutes of last 10 minutes
   service_resp_time_percentile_rule:
     expression: sum(service_percentile{p='50,75,90,95,99'} > 1000) >= 3
     period: 10
     silence-period: 5
+    recovery-observation-period: 3
     message: Percentile response time of service {name} alarm in 3 minutes of last 10 minutes, due to more than one condition of p50 > 1000, p75 > 1000, p90 > 1000, p95 > 1000, p99 > 1000
   service_instance_resp_time_rule:
     expression: sum(service_instance_resp_time > 1000) >= 2
     period: 10
     silence-period: 5
+    recovery-observation-period: 2
     message: Response time of service instance {name} is more than 1000ms in 2 minutes of last 10 minutes
   database_access_resp_time_rule:
     expression: sum(database_access_resp_time > 1000) >= 2
@@ -63,11 +69,36 @@ rules:
 #    silence-period: 5
 #    message: Response time of endpoint {name} is more than 1000ms in 2 minutes of last 10 minutes
 
+
 #hooks:
 #  webhook:
 #    default:
 #      is-default: true
 #      urls:
-#        - http://127.0.0.1/notify/
-#        - http://127.0.0.1/go-wechat/
-
+#        - http://127.0.0.1/default/alarm
+#       recovery-urls:
+#        - http://127.0.0.1/default/alarm-recovery
+#      custom1:
+#        urls:
+#          - http://127.0.0.1/custom1/alarm
+#        recovery-urls:
+#          - http://127.0.0.1/custom1/alarm-recovery
+#  wechat:
+#    default:
+#      is-default: true
+#      text-template: |-
+#        {
+#          "msgtype": "text",
+#          "text": {
+#            "content": "Apache SkyWalking Alarm: \n %s."
+#          }
+#        }
+#      recovery-text-template: |-
+#        {
+#          "msgtype": "text",
+#          "text": {
+#            "content": "Apache SkyWalking Alarm Recovered: \n %s."
+#          }
+#        }
+#      webhooks:
+#        - https://qyapi.weixin.qq.com/cgi-bin/webhook/send?key=dummy_key

dist-material/config-examples/alarm-settings.yml

@@ -23,6 +23,8 @@ rules:
     period: 10
     # How many times of checks, the alarm keeps silence after alarm triggered, default as same as period.
     silence-period: 10
+    # Number of periods to wait before considering the alarm recovered,default as 0.
+    recovery-observation-period: 3
     message: Successful rate of endpoint {name} is lower than 75%
     tags:
       level: WARNING
@@ -43,7 +45,35 @@ rules:
     silence-period: 5
     message: Response time of service instance {name} is more than 1000ms in 2 minutes of last 10 minutes
 
-#webhooks:
-#  - http://127.0.0.1/notify/
-#  - http://127.0.0.1/go-wechat/
-
+#hooks:
+#  webhook:
+#    default:
+#      is-default: true
+#      urls:
+#        - http://127.0.0.1/default/alarm
+#       recovery-urls:
+#        - http://127.0.0.1/default/alarm-recovery
+#      custom1:
+#        urls:
+#          - http://127.0.0.1/custom1/alarm
+#        recovery-urls:
+#          - http://127.0.0.1/custom1/alarm-recovery
+#  wechat:
+#    default:
+#      is-default: true
+#      text-template: |-
+#        {
+#          "msgtype": "text",
+#          "text": {
+#            "content": "Apache SkyWalking Alarm: \n %s."
+#          }
+#        }
+#      recovery-text-template: |-
+#        {
+#          "msgtype": "text",
+#          "text": {
+#            "content": "Apache SkyWalking Alarm Recovered: \n %s."
+#          }
+#        }
+#      webhooks:
+#        - https://qyapi.weixin.qq.com/cgi-bin/webhook/send?key=dummy_key

docs/en/changes/changes.md

@@ -5,8 +5,11 @@
 #### OAP Server
 
 * KubernetesCoordinator: make self instance return real pod IP address instead of `127.0.0.1`.
+* Enhance the alarm kernel with recovered status notification capability
 
 #### UI
+* Fix the missing icon in new native trace view.
+* Enhance the alert page to show the recovery time of resolved alerts.
 
 #### Documentation