Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
43 changes: 43 additions & 0 deletions pkg/controller/monitor/monitor_controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ import (

crdv1 "github.com/tigera/operator/pkg/apis/crd.projectcalico.org/v1"

monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
Expand Down Expand Up @@ -430,6 +431,48 @@ func (r *ReconcileMonitor) Reconcile(ctx context.Context, request reconcile.Requ
}
}

p, err := utils.GetPrometheus(ctx, r.client)
if err != nil {
r.status.SetDegraded(operatorv1.ResourceReadError, "An error occurred trying to retrieve the Prometheus status", err, reqLogger)
return reconcile.Result{}, err
}

if p != nil {
available := monitoringv1.ConditionFalse

for _, cond := range p.Status.Conditions {
if cond.Type == monitoringv1.Available {
available = cond.Status
}
}

if available != monitoringv1.ConditionTrue {
r.status.SetDegraded(operatorv1.ResourceNotReady, "Prometheus component is not available", err, reqLogger)
return reconcile.Result{}, err
}
}

am, err := utils.GetAlertmanager(ctx, r.client)
if err != nil {
r.status.SetDegraded(operatorv1.ResourceReadError, "An error occurred trying to retrieve the Alertmanager status", err, reqLogger)
return reconcile.Result{}, err
}

if am != nil {
available := monitoringv1.ConditionFalse

for _, cond := range am.Status.Conditions {
if cond.Type == monitoringv1.Available {
available = cond.Status
}
}

if available != monitoringv1.ConditionTrue {
r.status.SetDegraded(operatorv1.ResourceNotReady, "Alertmanager component is not available", err, reqLogger)
return reconcile.Result{}, err
}
}

// Tell the status manager that we're ready to monitor the resources we've told it about and receive statuses.
r.status.ReadyToMonitor()

Expand Down
136 changes: 134 additions & 2 deletions pkg/controller/monitor/monitor_controller_test.go
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
// Copyright (c) 2021-2024 Tigera, Inc. All rights reserved.
// Copyright (c) 2021-2026 Tigera, Inc. All rights reserved.

// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
Expand Down Expand Up @@ -68,10 +68,14 @@ var _ = Describe("Monitor controller tests", func() {
Expect(apis.AddToScheme(scheme)).NotTo(HaveOccurred())
Expect(appsv1.SchemeBuilder.AddToScheme(scheme)).NotTo(HaveOccurred())
Expect(rbacv1.SchemeBuilder.AddToScheme(scheme)).NotTo(HaveOccurred())
Expect(monitoringv1.AddToScheme(scheme)).NotTo(HaveOccurred())

// Create a client that will have a crud interface of k8s objects.
ctx = context.Background()
cli = ctrlrfake.DefaultFakeClientBuilder(scheme).Build()
cli = ctrlrfake.DefaultFakeClientBuilder(scheme).
WithStatusSubresource(&monitoringv1.Prometheus{}).
WithStatusSubresource(&monitoringv1.Alertmanager{}).
Build()

// Create an object we can use throughout the test to do the monitor reconcile loops.
mockStatus = &status.MockStatus{}
Expand All @@ -85,6 +89,7 @@ var _ = Describe("Monitor controller tests", func() {
mockStatus.On("ReadyToMonitor")
mockStatus.On("RemoveDeployments", mock.Anything)
mockStatus.On("RemoveCertificateSigningRequests", common.TigeraPrometheusNamespace)
mockStatus.On("SetDegraded", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return()
mockStatus.On("SetMetaData", mock.Anything).Return()

// Create an object we can use throughout the test to do the monitor reconcile loops.
Expand Down Expand Up @@ -135,6 +140,133 @@ var _ = Describe("Monitor controller tests", func() {
r.tierWatchReady.MarkAsReady()
})

Context("prometheus resources", func() {
BeforeEach(func() {
// Add the Prometheus and Alertmanager instances
prom := &monitoringv1.Prometheus{
ObjectMeta: metav1.ObjectMeta{
Name: monitor.CalicoNodePrometheus,
Namespace: common.TigeraPrometheusNamespace,
},
}
Expect(cli.Create(ctx, prom)).To(BeNil())

prom.Status = monitoringv1.PrometheusStatus{
Conditions: []monitoringv1.Condition{
{
Type: monitoringv1.Available,
Status: monitoringv1.ConditionTrue,
},
{
Type: monitoringv1.Reconciled,
Status: monitoringv1.ConditionTrue,
},
},
}
Expect(cli.Status().Update(ctx, prom)).To(Succeed())

alertManager := &monitoringv1.Alertmanager{
ObjectMeta: metav1.ObjectMeta{
Name: monitor.CalicoNodeAlertmanager,
Namespace: common.TigeraPrometheusNamespace,
},
}
Expect(cli.Create(ctx, alertManager)).To(BeNil())

alertManager.Status = monitoringv1.AlertmanagerStatus{
Conditions: []monitoringv1.Condition{
{
Type: monitoringv1.Available,
Status: monitoringv1.ConditionTrue,
},
{
Type: monitoringv1.Reconciled,
Status: monitoringv1.ConditionTrue,
},
},
}
Expect(cli.Status().Update(ctx, alertManager)).To(Succeed())
})

It("should be ready if the prometheus statefulset is ready", func() {
_, err := r.Reconcile(ctx, reconcile.Request{})
Expect(err).ShouldNot(HaveOccurred())

mockStatus.AssertNotCalled(GinkgoT(), "SetDegraded",
operatorv1.ResourceNotReady,
mock.Anything,
mock.Anything,
mock.Anything,
)
})

It("should be ready if the alertmanager statefulset is ready", func() {
_, err := r.Reconcile(ctx, reconcile.Request{})
Expect(err).ShouldNot(HaveOccurred())

mockStatus.AssertNotCalled(GinkgoT(), "SetDegraded",
operatorv1.ResourceNotReady,
mock.Anything,
mock.Anything,
mock.Anything,
)
})

It("should degrade if the prometheus statefulset isn't ready", func() {
prom := &monitoringv1.Prometheus{}
Expect(cli.Get(ctx, client.ObjectKey{Name: monitor.CalicoNodePrometheus, Namespace: common.TigeraPrometheusNamespace}, prom)).NotTo(HaveOccurred())

prom.Status.Conditions = []monitoringv1.Condition{
{
Type: monitoringv1.Available,
Status: monitoringv1.ConditionFalse,
},
{
Type: monitoringv1.Reconciled,
Status: monitoringv1.ConditionTrue,
},
}
Expect(cli.Status().Update(ctx, prom)).To(Succeed())

_, err := r.Reconcile(ctx, reconcile.Request{})
Expect(err).ShouldNot(HaveOccurred())

mockStatus.AssertCalled(GinkgoT(), "SetDegraded",
operatorv1.ResourceNotReady,
"Prometheus component is not available",
mock.Anything,
mock.Anything,
)
})

It("should degrade if the alertmanager statefulset isn't ready", func() {
alertManager := &monitoringv1.Alertmanager{}
Expect(cli.Get(ctx, client.ObjectKey{Name: monitor.CalicoNodeAlertmanager, Namespace: common.TigeraPrometheusNamespace}, alertManager)).NotTo(HaveOccurred())

alertManager.Status.Conditions = []monitoringv1.Condition{
{
Type: monitoringv1.Available,
Status: monitoringv1.ConditionFalse,
},
{
Type: monitoringv1.Reconciled,
Status: monitoringv1.ConditionTrue,
},
}
Expect(cli.Status().Update(ctx, alertManager)).To(Succeed())

_, err := r.Reconcile(ctx, reconcile.Request{})
Expect(err).ShouldNot(HaveOccurred())

mockStatus.AssertCalled(GinkgoT(), "SetDegraded",
operatorv1.ResourceNotReady,
"Alertmanager component is not available",
mock.Anything,
mock.Anything,
)
})
})

Context("controller reconciliation", func() {
var (
am = &monitoringv1.Alertmanager{}
Expand Down
26 changes: 26 additions & 0 deletions pkg/controller/utils/utils.go
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ import (

esv1 "github.com/elastic/cloud-on-k8s/v2/pkg/apis/elasticsearch/v1"
"github.com/elastic/cloud-on-k8s/v2/pkg/utils/stringsutil"
monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1"
csiv1 "sigs.k8s.io/secrets-store-csi-driver/apis/v1"

"github.com/go-logr/logr"
Expand Down Expand Up @@ -58,6 +59,7 @@ import (
"github.com/tigera/operator/pkg/ctrlruntime"
"github.com/tigera/operator/pkg/render"
"github.com/tigera/operator/pkg/render/logstorage/eck"
"github.com/tigera/operator/pkg/render/monitor"
)

const (
Expand Down Expand Up @@ -859,6 +861,30 @@ func GetElasticsearch(ctx context.Context, c client.Client) (*esv1.Elasticsearch
return &es, nil
}

func GetAlertmanager(ctx context.Context, c client.Client) (*monitoringv1.Alertmanager, error) {
a := monitoringv1.Alertmanager{}
err := c.Get(ctx, client.ObjectKey{Name: monitor.CalicoNodeAlertmanager, Namespace: common.TigeraPrometheusNamespace}, &a)
if err != nil {
if errors.IsNotFound(err) {
return nil, nil
}
return nil, err
}
return &a, nil
}

func GetPrometheus(ctx context.Context, c client.Client) (*monitoringv1.Prometheus, error) {
p := monitoringv1.Prometheus{}
err := c.Get(ctx, client.ObjectKey{Name: monitor.CalicoNodePrometheus, Namespace: common.TigeraPrometheusNamespace}, &p)
if err != nil {
if errors.IsNotFound(err) {
return nil, nil
}
return nil, err
}
return &p, nil
}

// AddKubeProxyWatch creates a watch on the kube-proxy DaemonSet.
func AddKubeProxyWatch(c ctrlruntime.Controller) error {
ds := &appsv1.DaemonSet{
Expand Down
Loading