Add custom Prometheus metrics

internal/controller/httpbin_controller.go

@@ -23,6 +23,7 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/controller"
 
 	orchestratev1alpha1 "http-operator/api/v1alpha1"
+	"http-operator/internal/metrics"
 
 	"k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/api/meta"
@@ -81,6 +82,7 @@ func (r *HttpBinReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct
 			"error_type", errors.IsNotFound(err),
 			"error_forbidden", errors.IsForbidden(err),
 			"error_invalid", errors.IsInvalid(err))
+		metrics.HttpBinReconciled.WithLabelValues("error").Inc()
 		return ctrl.Result{}, err
 	}
 
@@ -148,6 +150,7 @@ func (r *HttpBinReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct
 			"HttpBinDeployment.Name", httpBinDeployment.Name,
 			"HttpBinDeployment.Spec", httpBinDeployment.Spec)
 
+		metrics.HttpBinReconciled.WithLabelValues("created").Inc()
 		// Deployment created successfully - return and requeue
 		return ctrl.Result{Requeue: true}, nil
 	} else if err != nil {
@@ -191,6 +194,7 @@ func (r *HttpBinReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct
 		}
 	}
 
+	metrics.HttpBinReconciled.WithLabelValues("success").Inc()
 	return ctrl.Result{}, nil
 }
 

internal/controller/httpbin_controller_test.go

@@ -22,12 +22,14 @@ import (
 
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
+	"github.com/prometheus/client_golang/prometheus/testutil"
 	"k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
 	"sigs.k8s.io/controller-runtime/pkg/reconcile"
 
 	orchestratev1alpha1 "http-operator/api/v1alpha1"
+	"http-operator/internal/metrics"
 )
 
 var _ = Describe("HttpBin Controller", func() {
@@ -86,13 +88,19 @@ var _ = Describe("HttpBin Controller", func() {
 				Scheme:       k8sClient.Scheme(),
 			}
 
+			before := testutil.ToFloat64(metrics.HttpBinReconciled.WithLabelValues("created"))
+
 			result, err := controllerReconciler.Reconcile(ctx, reconcile.Request{
 				NamespacedName: typeNamespacedName,
 			})
 			Expect(err).NotTo(HaveOccurred())
 			// Check that reconciliation requests a requeue
 			Expect(result).NotTo(Equal(reconcile.Result{}))
 
+			By("Checking that the created metric was incremented")
+			after := testutil.ToFloat64(metrics.HttpBinReconciled.WithLabelValues("created"))
+			Expect(after - before).To(Equal(float64(1)))
+
 			By("Verifying HttpBinDeployment was created")
 			httpBinDeployment := &orchestratev1alpha1.HttpBinDeployment{}
 			Eventually(func() error {
@@ -190,11 +198,16 @@ var _ = Describe("HttpBin Controller", func() {
 			Expect(k8sClient.Status().Update(ctx, httpBinDeployment)).To(Succeed())
 
 			By("Reconciling again to propagate status")
+			beforeSuccess := testutil.ToFloat64(metrics.HttpBinReconciled.WithLabelValues("success"))
 			_, err = controllerReconciler.Reconcile(ctx, reconcile.Request{
 				NamespacedName: typeNamespacedName,
 			})
 			Expect(err).NotTo(HaveOccurred())
 
+			By("Checking that the success metric was incremented")
+			afterSuccess := testutil.ToFloat64(metrics.HttpBinReconciled.WithLabelValues("success"))
+			Expect(afterSuccess - beforeSuccess).To(Equal(float64(1)))
+
 			By("Verifying HttpBin status was updated")
 			httpBin := &orchestratev1alpha1.HttpBin{}
 			Expect(k8sClient.Get(ctx, typeNamespacedName, httpBin)).To(Succeed())

internal/controller/httpbindeployment_controller.go

@@ -16,6 +16,7 @@ import (
 	gatewayApi "sigs.k8s.io/gateway-api/apis/v1"
 
 	orchestratev1alpha1 "http-operator/api/v1alpha1"
+	"http-operator/internal/metrics"
 
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
@@ -176,6 +177,7 @@ func (r *HttpBinDeploymentReconciler) Reconcile(ctx context.Context, req ctrl.Re
 			_ = r.RemoteClient.Status().Update(ctx, httpBinDeployment)
 			return ctrl.Result{}, err
 		}
+		metrics.DeploymentOperations.WithLabelValues("created").Inc()
 	} else if err != nil {
 		logger.Error(err, "Failed to get Deployment")
 		return ctrl.Result{}, err
@@ -191,6 +193,7 @@ func (r *HttpBinDeploymentReconciler) Reconcile(ctx context.Context, req ctrl.Re
 			_ = r.RemoteClient.Status().Update(ctx, httpBinDeployment)
 			return ctrl.Result{}, err
 		}
+		metrics.DeploymentOperations.WithLabelValues("updated").Inc()
 	}
 
 	// Handle Service in local cluster (Cluster A)
@@ -212,6 +215,7 @@ func (r *HttpBinDeploymentReconciler) Reconcile(ctx context.Context, req ctrl.Re
 			_ = r.RemoteClient.Status().Update(ctx, httpBinDeployment)
 			return ctrl.Result{}, err
 		}
+		metrics.ServiceOperations.WithLabelValues("created").Inc()
 	} else if err != nil {
 		logger.Error(err, "Failed to get Service")
 		return ctrl.Result{}, err
@@ -228,6 +232,7 @@ func (r *HttpBinDeploymentReconciler) Reconcile(ctx context.Context, req ctrl.Re
 			_ = r.RemoteClient.Status().Update(ctx, httpBinDeployment)
 			return ctrl.Result{}, err
 		}
+		metrics.ServiceOperations.WithLabelValues("updated").Inc()
 	}
 
 	ingress := &networkingv1.Ingress{}
@@ -400,9 +405,12 @@ func (r *HttpBinDeploymentReconciler) Reconcile(ctx context.Context, req ctrl.Re
 		statusNeedsUpdate = true
 		if httpBinDeployment.Status.IsDeploymentReady {
 			setHttpBinDeploymentStatusCondition(httpBinDeployment, metav1.ConditionTrue, orchestratev1alpha1.HttpBinDeploymentConditionTypeReady, orchestratev1alpha1.HttpBinDeploymentConditionReasonReady, "HttpBinDeployment instance is ready")
+			metrics.DeploymentReady.WithLabelValues(httpBinDeployment.Name).Set(1)
 		} else {
 			setHttpBinDeploymentStatusCondition(httpBinDeployment, metav1.ConditionFalse, orchestratev1alpha1.HttpBinDeploymentConditionTypeReady, orchestratev1alpha1.HttpBinDeploymentConditionReasonHttpBinInstanceProgressing, "HttpBinDeployment instance is not yet available")
+			metrics.DeploymentReady.WithLabelValues(httpBinDeployment.Name).Set(0)
 		}
+		metrics.ReadyReplicas.WithLabelValues(httpBinDeployment.Name).Set(float64(deployment.Status.ReadyReplicas))
 	}
 
 	// Update status if any changes were made
@@ -490,6 +498,9 @@ func (r *HttpBinDeploymentReconciler) finalizeHttpBinDeployment(ctx context.Cont
 	if err := r.LocalClient.Delete(ctx, deployment); err != nil && !errors.IsNotFound(err) {
 		return err
 	}
+	metrics.DeploymentOperations.WithLabelValues("deleted").Inc()
+	metrics.DeploymentReady.DeleteLabelValues(httpBinDeployment.Name)
+	metrics.ReadyReplicas.DeleteLabelValues(httpBinDeployment.Name)
 
 	// Delete Service
 	serviceName := r.getResourceName(httpBinDeployment)

internal/metrics/metrics.go

@@ -0,0 +1,54 @@
+package metrics
+
+import (
+	"github.com/prometheus/client_golang/prometheus"
+	ctrlmetrics "sigs.k8s.io/controller-runtime/pkg/metrics"
+)
+
+var (
+	HttpBinReconciled = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: "httpbin_operator_httpbin_reconciled_total",
+			Help: "Total number of HttpBin reconcile calls by result.",
+		},
+		[]string{"result"},
+	)
+	DeploymentOperations = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: "httpbin_operator_deployments_total",
+			Help: "Total number of local Deployment operations performed.",
+		},
+		[]string{"operation"},
+	)
+	ServiceOperations = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: "httpbin_operator_services_total",
+			Help: "Total number of local Service operations performed.",
+		},
+		[]string{"operation"},
+	)
+	DeploymentReady = prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Name: "httpbin_operator_deployment_ready",
+			Help: "Whether the HttpBinDeployment is ready (1) or not (0).",
+		},
+		[]string{"name"},
+	)
+	ReadyReplicas = prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Name: "httpbin_operator_ready_replicas",
+			Help: "Number of ready replicas for each HttpBinDeployment.",
+		},
+		[]string{"name"},
+	)
+)
+
+func init() {
+	ctrlmetrics.Registry.MustRegister(
+		HttpBinReconciled,
+		DeploymentOperations,
+		ServiceOperations,
+		DeploymentReady,
+		ReadyReplicas,
+	)
+}

internal/metrics/metrics_test.go

@@ -0,0 +1,54 @@
+package metrics
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/prometheus/client_golang/prometheus/testutil"
+)
+
+func TestMetricsManual(t *testing.T) {
+	// Simulate HttpBin reconcile outcomes
+	HttpBinReconciled.WithLabelValues("success").Inc()
+	HttpBinReconciled.WithLabelValues("success").Inc()
+	HttpBinReconciled.WithLabelValues("created").Inc()
+	HttpBinReconciled.WithLabelValues("error").Inc()
+
+	// Simulate Deployment operations
+	DeploymentOperations.WithLabelValues("created").Inc()
+	DeploymentOperations.WithLabelValues("created").Inc()
+	DeploymentOperations.WithLabelValues("updated").Inc()
+	DeploymentOperations.WithLabelValues("deleted").Inc()
+
+	// Simulate Service operations
+	ServiceOperations.WithLabelValues("created").Inc()
+	ServiceOperations.WithLabelValues("updated").Inc()
+
+	// Simulate ready state for two instances
+	DeploymentReady.WithLabelValues("httpbin-a").Set(1)
+	DeploymentReady.WithLabelValues("httpbin-b").Set(0)
+	ReadyReplicas.WithLabelValues("httpbin-a").Set(2)
+	ReadyReplicas.WithLabelValues("httpbin-b").Set(0)
+
+	fmt.Println("\n--- httpbin_operator_httpbin_reconciled_total ---")
+	fmt.Printf("  success: %.0f\n", testutil.ToFloat64(HttpBinReconciled.WithLabelValues("success")))
+	fmt.Printf("  created: %.0f\n", testutil.ToFloat64(HttpBinReconciled.WithLabelValues("created")))
+	fmt.Printf("  error:   %.0f\n", testutil.ToFloat64(HttpBinReconciled.WithLabelValues("error")))
+
+	fmt.Println("\n--- httpbin_operator_deployments_total ---")
+	fmt.Printf("  created: %.0f\n", testutil.ToFloat64(DeploymentOperations.WithLabelValues("created")))
+	fmt.Printf("  updated: %.0f\n", testutil.ToFloat64(DeploymentOperations.WithLabelValues("updated")))
+	fmt.Printf("  deleted: %.0f\n", testutil.ToFloat64(DeploymentOperations.WithLabelValues("deleted")))
+
+	fmt.Println("\n--- httpbin_operator_services_total ---")
+	fmt.Printf("  created: %.0f\n", testutil.ToFloat64(ServiceOperations.WithLabelValues("created")))
+	fmt.Printf("  updated: %.0f\n", testutil.ToFloat64(ServiceOperations.WithLabelValues("updated")))
+
+	fmt.Println("\n--- httpbin_operator_deployment_ready ---")
+	fmt.Printf("  httpbin-a: %.0f\n", testutil.ToFloat64(DeploymentReady.WithLabelValues("httpbin-a")))
+	fmt.Printf("  httpbin-b: %.0f\n", testutil.ToFloat64(DeploymentReady.WithLabelValues("httpbin-b")))
+
+	fmt.Println("\n--- httpbin_operator_ready_replicas ---")
+	fmt.Printf("  httpbin-a: %.0f\n", testutil.ToFloat64(ReadyReplicas.WithLabelValues("httpbin-a")))
+	fmt.Printf("  httpbin-b: %.0f\n", testutil.ToFloat64(ReadyReplicas.WithLabelValues("httpbin-b")))
+}