added failed halth check metrics

depot/steps/emit_check_failure_metric_step.go

@@ -0,0 +1,57 @@
+package steps
+
+import (
+	"fmt"
+	"os"
+
+	loggingclient "code.cloudfoundry.org/diego-logging-client"
+	"code.cloudfoundry.org/executor"
+	"github.com/tedsuo/ifrit"
+)
+
+type emitCheckFailureMetricStep struct {
+	subStep       ifrit.Runner
+	checkProtocol executor.CheckProtocol
+	checkType     executor.HealthcheckType
+	metronClient  loggingclient.IngressClient
+}
+
+const (
+	CheckFailedCount = "ChecksFailedCount"
+)
+
+func NewEmitCheckFailureMetricStep(
+	subStep ifrit.Runner,
+	checkProtocol executor.CheckProtocol,
+	checkType executor.HealthcheckType,
+	metronClient loggingclient.IngressClient) ifrit.Runner {
+	return &emitCheckFailureMetricStep{
+		subStep:       subStep,
+		checkProtocol: checkProtocol,
+		checkType:     checkType,
+		metronClient:  metronClient,
+	}
+}
+
+func (step *emitCheckFailureMetricStep) Run(signals <-chan os.Signal, ready chan<- struct{}) error {
+	if step.subStep == nil {
+		return nil
+	}
+
+	subStepErr := step.subStep.Run(signals, ready)
+
+	if subStepErr != nil {
+		step.emitFailureMetric()
+	}
+
+	return subStepErr
+}
+
+func (step *emitCheckFailureMetricStep) emitFailureMetric() {
+	metricName := constructMetricName(step.checkProtocol, step.checkType)
+	go step.metronClient.IncrementCounter(metricName)
+}
+
+func constructMetricName(checkProtocol executor.CheckProtocol, checkType executor.HealthcheckType) string {
+	return fmt.Sprintf("%s%s%s", checkProtocol, checkType, CheckFailedCount)
+}

depot/steps/emit_check_failure_metric_step_test.go

@@ -0,0 +1,125 @@
+package steps_test
+
+import (
+	"errors"
+	"fmt"
+	"os"
+
+	mfakes "code.cloudfoundry.org/diego-logging-client/testhelpers"
+	"code.cloudfoundry.org/executor"
+	"code.cloudfoundry.org/executor/depot/steps"
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+	"github.com/tedsuo/ifrit"
+	fake_runner "github.com/tedsuo/ifrit/fake_runner_v2"
+)
+
+var _ = Describe("EmitCheckFailureMetricStep", func() {
+	var (
+		subStep          *fake_runner.FakeRunner
+		step             ifrit.Runner
+		fakeMetronClient *mfakes.FakeIngressClient
+		errorToReturn    error
+		checkProtocol    executor.CheckProtocol
+		checkType        executor.HealthcheckType
+	)
+
+	BeforeEach(func() {
+		checkProtocol = executor.HTTPCheck
+		checkType = executor.IsLivenessCheck
+		fakeMetronClient = new(mfakes.FakeIngressClient)
+		errorToReturn = nil
+		subStep = &fake_runner.FakeRunner{}
+
+		subStep.RunStub = func(signals <-chan os.Signal, ready chan<- struct{}) error {
+			return errorToReturn
+		}
+	})
+
+	JustBeforeEach(func() {
+		step = steps.NewEmitCheckFailureMetricStep(subStep, checkProtocol, checkType, fakeMetronClient)
+	})
+
+	Describe("Ready", func() {
+		var (
+			fakeRunner *fake_runner.TestRunner
+		)
+
+		Context("when substep is not nil", func() {
+			BeforeEach(func() {
+				fakeRunner = fake_runner.NewTestRunner()
+				subStep = fakeRunner.FakeRunner
+			})
+
+			AfterEach(func() {
+				fakeRunner.EnsureExit()
+			})
+
+			It("becomes ready when the substep is ready", func() {
+				p := ifrit.Background(step)
+				Consistently(p.Ready()).ShouldNot(BeClosed())
+				fakeRunner.TriggerReady()
+				Eventually(p.Ready()).Should(BeClosed())
+			})
+		})
+	})
+
+	Describe("Running", func() {
+		var (
+			err error
+		)
+
+		JustBeforeEach(func() {
+			err = <-ifrit.Invoke(step).Wait()
+		})
+
+		Context("when the substep succeeds", func() {
+			It("should not emit any metric", func() {
+				Expect(err).NotTo(HaveOccurred())
+				Eventually(fakeMetronClient.IncrementCounterCallCount).Should(Equal(0))
+			})
+		})
+
+		Context("when the substep fails", func() {
+			BeforeEach(func() {
+				errorToReturn = errors.New("BOOM")
+			})
+
+			It("should pass the error along", func() {
+				Expect(err).To(MatchError(errorToReturn))
+			})
+
+			It("should emit metric for the correct health check protocol and type", func() {
+				Eventually(fakeMetronClient.IncrementCounterCallCount).Should(Equal(1))
+				name := fakeMetronClient.IncrementCounterArgsForCall(0)
+				Expect(name).To(Equal(fmt.Sprintf("%s%s%s", checkProtocol, checkType, steps.CheckFailedCount)))
+			})
+		})
+	})
+
+	Describe("Signal", func() {
+		var (
+			p        ifrit.Process
+			finished chan struct{}
+		)
+
+		BeforeEach(func() {
+			finished = make(chan struct{})
+			subStep.RunStub = func(signals <-chan os.Signal, ready chan<- struct{}) error {
+				<-signals
+				close(finished)
+				return new(steps.CancelledError)
+			}
+		})
+
+		JustBeforeEach(func() {
+			p = ifrit.Background(step)
+		})
+
+		It("should pass the signal", func() {
+			Consistently(finished).ShouldNot(BeClosed())
+			p.Signal(os.Interrupt)
+			Eventually(finished).Should(BeClosed())
+		})
+	})
+})