fix(core): remove metrics for deleted resources

charts/manager/crds/greenhouse.sap_catalogs.yaml

@@ -6,7 +6,7 @@ apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.20.1
+    controller-gen.kubebuilder.io/version: v0.21.0
   name: catalogs.greenhouse.sap
 spec:
   group: greenhouse.sap

charts/manager/crds/greenhouse.sap_clusterkubeconfigs.yaml

@@ -6,7 +6,7 @@ apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.20.1
+    controller-gen.kubebuilder.io/version: v0.21.0
   name: clusterkubeconfigs.greenhouse.sap
 spec:
   group: greenhouse.sap

charts/manager/crds/greenhouse.sap_clusterplugindefinitions.yaml

@@ -6,7 +6,7 @@ apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.20.1
+    controller-gen.kubebuilder.io/version: v0.21.0
   name: clusterplugindefinitions.greenhouse.sap
 spec:
   group: greenhouse.sap

charts/manager/crds/greenhouse.sap_clusters.yaml

@@ -6,7 +6,7 @@ apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.20.1
+    controller-gen.kubebuilder.io/version: v0.21.0
   name: clusters.greenhouse.sap
 spec:
   group: greenhouse.sap

charts/manager/crds/greenhouse.sap_organizations.yaml

@@ -6,7 +6,7 @@ apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.20.1
+    controller-gen.kubebuilder.io/version: v0.21.0
   name: organizations.greenhouse.sap
 spec:
   group: greenhouse.sap

charts/manager/crds/greenhouse.sap_plugindefinitions.yaml

@@ -6,7 +6,7 @@ apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.20.1
+    controller-gen.kubebuilder.io/version: v0.21.0
   name: plugindefinitions.greenhouse.sap
 spec:
   group: greenhouse.sap

charts/manager/crds/greenhouse.sap_pluginpresets.yaml

@@ -6,7 +6,7 @@ apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.20.1
+    controller-gen.kubebuilder.io/version: v0.21.0
   name: pluginpresets.greenhouse.sap
 spec:
   group: greenhouse.sap

charts/manager/crds/greenhouse.sap_plugins.yaml

@@ -6,7 +6,7 @@ apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.20.1
+    controller-gen.kubebuilder.io/version: v0.21.0
   name: plugins.greenhouse.sap
 spec:
   group: greenhouse.sap

charts/manager/crds/greenhouse.sap_teamrolebindings.yaml

@@ -6,7 +6,7 @@ apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.20.1
+    controller-gen.kubebuilder.io/version: v0.21.0
   name: teamrolebindings.greenhouse.sap
 spec:
   group: greenhouse.sap

charts/manager/crds/greenhouse.sap_teamroles.yaml

@@ -6,7 +6,7 @@ apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.20.1
+    controller-gen.kubebuilder.io/version: v0.21.0
   name: teamroles.greenhouse.sap
 spec:
   group: greenhouse.sap

charts/manager/crds/greenhouse.sap_teams.yaml

@@ -6,7 +6,7 @@ apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.20.1
+    controller-gen.kubebuilder.io/version: v0.21.0
   name: teams.greenhouse.sap
 spec:
   group: greenhouse.sap

internal/controller/cluster/cluster_controller.go

@@ -294,6 +294,7 @@ func (r *RemoteClusterReconciler) EnsureDeleted(ctx context.Context, resource li
 	cluster := resource.(*greenhousev1alpha1.Cluster)
 	c := cluster.Status.GetConditionByType(greenhousev1alpha1.KubeConfigValid)
 	if c != nil && c.IsFalse() {
+		deleteClusterMetrics(cluster)
 		return ctrl.Result{}, lifecycle.Success, nil
 	}
 	// delete all plugins that are bound to this cluster
@@ -312,6 +313,7 @@ func (r *RemoteClusterReconciler) EnsureDeleted(ctx context.Context, resource li
 	// early return if the cluster connectivity is via OIDC
 	if kubeConfigSecret.Type == greenhouseapis.SecretTypeOIDCConfig {
 		log.FromContext(ctx).Info("no resources to clean up", "secretType", kubeConfigSecret.Type, "cluster", cluster.Name)
+		deleteClusterMetrics(cluster)
 		return ctrl.Result{}, lifecycle.Success, nil
 	}
 	restClientGetter, err := clientutil.NewRestClientGetterFromSecret(kubeConfigSecret, cluster.Namespace)
@@ -328,6 +330,7 @@ func (r *RemoteClusterReconciler) EnsureDeleted(ctx context.Context, resource li
 	if err := r.deleteClusterRoleBindingInRemoteCluster(ctx, remoteClient); err != nil {
 		return ctrl.Result{}, lifecycle.Failed, err
 	}
+	deleteClusterMetrics(cluster)
 	return ctrl.Result{}, lifecycle.Success, nil
 }
 

internal/controller/cluster/metrics.go

@@ -41,6 +41,13 @@ func init() {
 }
 
 func UpdateClusterMetrics(cluster *greenhousev1alpha1.Cluster) {
+	// Drop any previous KubernetesVersionsGauge series for this cluster so that
+	// a version change does not leave a stale time series behind.
+	KubernetesVersionsGauge.DeletePartialMatch(prometheus.Labels{
+		"clusterName":  cluster.Name,
+		"organization": cluster.Namespace,
+	})
+
 	kubernetesVersionLabels := prometheus.Labels{
 		"clusterName":  cluster.Name,
 		"organization": cluster.Namespace,
@@ -68,3 +75,13 @@ func UpdateClusterMetrics(cluster *greenhousev1alpha1.Cluster) {
 		ClusterReadyGauge.With(clusterReadyLabels).Set(float64(0))
 	}
 }
+
+func deleteClusterMetrics(cluster *greenhousev1alpha1.Cluster) {
+	matchLabels := prometheus.Labels{
+		"clusterName":  cluster.Name,
+		"organization": cluster.Namespace,
+	}
+	KubernetesVersionsGauge.DeletePartialMatch(matchLabels)
+	SecondsToTokenExpiryGauge.DeletePartialMatch(matchLabels)
+	ClusterReadyGauge.DeletePartialMatch(matchLabels)
+}

internal/controller/cluster/metrics_test.go

@@ -57,4 +57,37 @@ var _ = Describe("Cluster Metrics", Ordered, func() {
 		readyGauge := prometheusTest.ToFloat64(cluster.ClusterReadyGauge.WithLabelValues(c.Name, c.Namespace, "test-owner"))
 		Expect(readyGauge).To(BeEquivalentTo(float64(0)), "clusterReady metric should be present and the cluster should not be ready")
 	})
+
+	It("Should remove the previous kubernetes version time series when the version changes", func() {
+		const clusterName = "test-cluster-version-change"
+		setup := test.NewTestSetup(test.Ctx, test.K8sClient, "clustermetrics")
+		c := &greenhousev1alpha1.Cluster{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      clusterName,
+				Namespace: setup.Namespace(),
+				Labels: map[string]string{
+					greenhouseapis.LabelKeyOwnedBy: "test-owner",
+				},
+			},
+			Status: greenhousev1alpha1.ClusterStatus{
+				KubernetesVersion:              "1.31.1",
+				BearerTokenExpirationTimestamp: metav1.Time{Time: time.Now().Add(600 * time.Second)},
+			},
+		}
+
+		cluster.UpdateClusterMetrics(c)
+		Expect(prometheusTest.ToFloat64(cluster.KubernetesVersionsGauge.WithLabelValues(c.Name, c.Namespace, "1.31.1", "test-owner"))).
+			To(BeEquivalentTo(1), "the initial kubernetes version time series should be present")
+
+		c.Status.KubernetesVersion = "1.32.0"
+		cluster.UpdateClusterMetrics(c)
+
+		Expect(prometheusTest.ToFloat64(cluster.KubernetesVersionsGauge.WithLabelValues(c.Name, c.Namespace, "1.32.0", "test-owner"))).
+			To(BeEquivalentTo(1), "the new kubernetes version time series should be present")
+		// If the previous version's time series had survived, .Set(1) from the first
+		// UpdateClusterMetrics call would still be observable. A value of 0 means the
+		// series was dropped (and only re-created here as a side effect of querying it).
+		Expect(prometheusTest.ToFloat64(cluster.KubernetesVersionsGauge.WithLabelValues(c.Name, c.Namespace, "1.31.1", "test-owner"))).
+			To(BeEquivalentTo(0), "the previous kubernetes version time series should have been removed")
+	})
 })

internal/controller/organization/metrics.go

@@ -33,7 +33,7 @@ func init() {
 	crmetrics.Registry.MustRegister(scimAccessReadyGauge)
 }
 
-func UpdateOrganizationMetrics(organization *greenhousev1alpha1.Organization) {
+func updateOrganizationMetrics(organization *greenhousev1alpha1.Organization) {
 	organizationLabels := prometheus.Labels{
 		"organization": organization.Name,
 	}
@@ -50,3 +50,11 @@ func UpdateOrganizationMetrics(organization *greenhousev1alpha1.Organization) {
 		scimAccessReadyGauge.With(organizationLabels).Set(float64(0))
 	}
 }
+
+func deleteOrganizationMetrics(organization *greenhousev1alpha1.Organization) {
+	organizationLabels := prometheus.Labels{
+		"organization": organization.Name,
+	}
+	organizationReadyGauge.Delete(organizationLabels)
+	scimAccessReadyGauge.Delete(organizationLabels)
+}

internal/controller/organization/organization_controller.go

@@ -139,7 +139,8 @@ func (r *OrganizationReconciler) EnsureDeleted(ctx context.Context, obj lifecycl
 			return ctrl.Result{}, lifecycle.Failed, err
 		}
 	}
-	return ctrl.Result{}, lifecycle.Success, nil // nothing to do in that case
+	deleteOrganizationMetrics(org)
+	return ctrl.Result{}, lifecycle.Success, nil
 }
 
 func (r *OrganizationReconciler) EnsureCreated(ctx context.Context, object lifecycle.RuntimeObject) (ctrl.Result, lifecycle.ReconcileResult, error) {
@@ -373,7 +374,7 @@ func (r *OrganizationReconciler) setStatus() lifecycle.Conditioner {
 		scimAPIAvailableCondition := r.checkSCIMAPIAvailability(ctx, org)
 		readyCondition := calculateReadyCondition(scimAPIAvailableCondition)
 		org.Status.SetConditions(scimAPIAvailableCondition, readyCondition)
-		UpdateOrganizationMetrics(org)
+		updateOrganizationMetrics(org)
 	}
 }
 

internal/controller/plugin/metrics.go

@@ -24,7 +24,7 @@ func init() {
 	crmetrics.Registry.MustRegister(pluginReady)
 }
 
-func UpdatePluginReadyMetric(plugin *greenhousev1alpha1.Plugin, ready bool) {
+func updatePluginReadyMetric(plugin *greenhousev1alpha1.Plugin, ready bool) {
 	pluginReadyLabels := prometheus.Labels{
 		"pluginDefinition": plugin.Spec.PluginDefinitionRef.Name,
 		"clusterName":      plugin.Spec.ClusterName,
@@ -38,3 +38,10 @@ func UpdatePluginReadyMetric(plugin *greenhousev1alpha1.Plugin, ready bool) {
 		pluginReady.With(pluginReadyLabels).Set(0)
 	}
 }
+
+func deletePluginReadyMetric(plugin *greenhousev1alpha1.Plugin) {
+	pluginReady.DeletePartialMatch(prometheus.Labels{
+		"plugin":       plugin.Name,
+		"organization": plugin.Namespace,
+	})
+}

internal/controller/plugin/plugin_controller.go

@@ -126,7 +126,7 @@ func (r *PluginReconciler) setConditions() lifecycle.Conditioner {
 		}
 
 		readyCondition := r.computeReadyConditionFlux(ctx, plugin)
-		UpdatePluginReadyMetric(plugin, readyCondition.Status == metav1.ConditionTrue)
+		updatePluginReadyMetric(plugin, readyCondition.Status == metav1.ConditionTrue)
 
 		ownerLabelCondition := util.ComputeOwnerLabelCondition(ctx, r.Client, plugin)
 		util.UpdateOwnedByLabelMissingMetric(plugin, ownerLabelCondition.IsFalse())
@@ -165,7 +165,11 @@ func (r *PluginReconciler) reconcileTechnicalLabels(ctx context.Context, plugin
 
 func (r *PluginReconciler) EnsureDeleted(ctx context.Context, resource lifecycle.RuntimeObject) (ctrl.Result, lifecycle.ReconcileResult, error) {
 	plugin := resource.(*greenhousev1alpha1.Plugin)
-	return r.EnsureFluxDeleted(ctx, plugin)
+	result, lifecycleResult, err := r.EnsureFluxDeleted(ctx, plugin)
+	if lifecycleResult == lifecycle.Success && err == nil {
+		deletePluginReadyMetric(plugin)
+	}
+	return result, lifecycleResult, err
 }
 
 func (r *PluginReconciler) EnsureCreated(ctx context.Context, resource lifecycle.RuntimeObject) (ctrl.Result, lifecycle.ReconcileResult, error) {

internal/controller/team/metrics.go

@@ -24,14 +24,14 @@ func init() {
 	crmetrics.Registry.MustRegister(membersCountMetric)
 }
 
-func UpdateTeamMembersCountMetric(team *greenhousev1alpha1.Team, membersCount int) {
+func updateTeamMembersCountMetric(team *greenhousev1alpha1.Team, membersCount int) {
 	membersCountMetric.With(prometheus.Labels{
 		"organization": team.Namespace,
 		"team":         team.Name,
 	}).Set(float64(membersCount))
 }
 
-func DeleteTeamMembersCountMetric(team *greenhousev1alpha1.Team) {
+func deleteTeamMembersCountMetric(team *greenhousev1alpha1.Team) {
 	membersCountMetric.Delete(prometheus.Labels{
 		"organization": team.Namespace,
 		"team":         team.Name,

internal/controller/team/team_controller.go

@@ -105,7 +105,7 @@ func (r *TeamController) setConditions() lifecycle.Conditioner {
 
 func (r *TeamController) EnsureDeleted(_ context.Context, obj lifecycle.RuntimeObject) (ctrl.Result, lifecycle.ReconcileResult, error) {
 	team := obj.(*greenhousev1alpha1.Team)
-	DeleteTeamMembersCountMetric(team)
+	deleteTeamMembersCountMetric(team)
 	return ctrl.Result{}, lifecycle.Success, nil
 }
 
@@ -161,7 +161,7 @@ func (r *TeamController) EnsureCreated(ctx context.Context, object lifecycle.Run
 	team.Status.Members = users
 	team.SetCondition(greenhousemetav1alpha1.TrueCondition(greenhousev1alpha1.SCIMAccessReadyCondition, "", ""))
 
-	UpdateTeamMembersCountMetric(team, len(users))
+	updateTeamMembersCountMetric(team, len(users))
 
 	// Reconcile ServiceAccount for support group teams
 	if team.Labels[greenhouseapis.LabelKeySupportGroup] == "true" {

internal/controller/teamrbac/metrics.go

@@ -33,7 +33,7 @@ func init() {
 	crmetrics.Registry.MustRegister(teamRBACReadyGauge, teamRBACClustersGauge)
 }
 
-func UpdateTeamRBACMetrics(teamRoleBinding *greenhousev1alpha2.TeamRoleBinding) {
+func updateTeamRBACMetrics(teamRoleBinding *greenhousev1alpha2.TeamRoleBinding) {
 	teamRBACReadyGauge.DeletePartialMatch(prometheus.Labels{
 		"team_role_binding": teamRoleBinding.Name,
 		"organization":      teamRoleBinding.Namespace,
@@ -55,7 +55,7 @@ func UpdateTeamRBACMetrics(teamRoleBinding *greenhousev1alpha2.TeamRoleBinding)
 	}
 }
 
-func DeleteTeamRBACMetrics(teamRoleBinding *greenhousev1alpha2.TeamRoleBinding) {
+func deleteTeamRBACMetrics(teamRoleBinding *greenhousev1alpha2.TeamRoleBinding) {
 	teamRBACReadyGauge.DeletePartialMatch(prometheus.Labels{
 		"team_role_binding": teamRoleBinding.Name,
 		"organization":      teamRoleBinding.Namespace,

internal/controller/teamrbac/teamrolebinding_controller.go

@@ -108,7 +108,7 @@ func (r *TeamRoleBindingReconciler) setConditions() lifecycle.Conditioner {
 		ownerLabelCondition := util.ComputeOwnerLabelCondition(ctx, r.Client, trb)
 		util.UpdateOwnedByLabelMissingMetric(trb, ownerLabelCondition.IsFalse())
 		trb.Status.SetConditions(readyCondition, ownerLabelCondition)
-		UpdateTeamRBACMetrics(trb)
+		updateTeamRBACMetrics(trb)
 	}
 }
 
@@ -218,7 +218,7 @@ func (r *TeamRoleBindingReconciler) EnsureDeleted(ctx context.Context, resource
 	// all clusters have been processed, finalizer can be removed
 	if len(trb.Status.PropagationStatus) == 0 {
 		r.recorder.Eventf(trb, nil, corev1.EventTypeNormal, greenhousemetav1alpha1.SuccessfulDeletedEvent, "deleting TeamRoleBinding", "Deleted TeamRoleBinding %s from all clusters", trb.GetName())
-		DeleteTeamRBACMetrics(trb)
+		deleteTeamRBACMetrics(trb)
 		return ctrl.Result{}, lifecycle.Success, nil
 	}
 	return ctrl.Result{}, lifecycle.Pending, nil