OCPSTRAT-2485: Introduce feature gate based inclusion/exclusion of manifests

bootstrap/bootstrap-pod.yaml

@@ -5,6 +5,8 @@ metadata:
   namespace: openshift-cluster-version
   labels:
     k8s-app: cluster-version-operator
+  annotations:
+    include.release.openshift.io/{{ .ClusterProfile }}: "true"
 spec:
   containers:
   - name: cluster-version-operator

go.mod

@@ -33,6 +33,8 @@ require (
 	k8s.io/utils v0.0.0-20250604170112-4c0f3b243397
 )
 
+replace github.com/openshift/library-go => github.com/JoelSpeed/library-go v0.0.0-20251217102247-bcd739ac7ffa
+
 require (
 	cel.dev/expr v0.24.0 // indirect
 	github.com/antlr4-go/antlr/v4 v4.13.0 // indirect

go.sum

@@ -1,5 +1,7 @@
 cel.dev/expr v0.24.0 h1:56OvJKSH3hDGL0ml5uSxZmz3/3Pq4tJ+fb1unVLAFcY=
 cel.dev/expr v0.24.0/go.mod h1:hLPLo1W4QUmuYdA72RBX06QTs6MXw941piREPl3Yfiw=
+github.com/JoelSpeed/library-go v0.0.0-20251217102247-bcd739ac7ffa h1:XI0l1W9wZLiVI/VV8nDtuX3RN7vzAKH9HnoEnCgJF/s=
+github.com/JoelSpeed/library-go v0.0.0-20251217102247-bcd739ac7ffa/go.mod h1:ErDfiIrPHH+menTP/B4LKd0nxFDdvCbTamAc6SWMIh8=
 github.com/antlr4-go/antlr/v4 v4.13.0 h1:lxCg3LAv+EUK6t1i0y1V6/SLeUi0eKEKdhQAlS8TVTI=
 github.com/antlr4-go/antlr/v4 v4.13.0/go.mod h1:pfChB/xh/Unjila75QW7+VU4TSnWnnk9UTnmpPaOR2g=
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
@@ -90,8 +92,6 @@ github.com/openshift/api v0.0.0-20251127005036-0e3c378fdedc h1:p83VYAk7mlqYZrMaK
 github.com/openshift/api v0.0.0-20251127005036-0e3c378fdedc/go.mod h1:d5uzF0YN2nQQFA0jIEWzzOZ+edmo6wzlGLvx5Fhz4uY=
 github.com/openshift/client-go v0.0.0-20251201171210-333716c1124a h1:iJYjd+rxyjMa3Sk6Vg55secJ4yMrabr/ulnTiy+vDH0=
 github.com/openshift/client-go v0.0.0-20251201171210-333716c1124a/go.mod h1:WD7m8ADeqiAKTHWx/mBoE/1MFMtnt9MYTyBOnf0L3LI=
-github.com/openshift/library-go v0.0.0-20251120164824-14a789e09884 h1:6512TMT14gnXQ4vyshzAQGjkctU0PO9G+y0tcBjw6Vk=
-github.com/openshift/library-go v0.0.0-20251120164824-14a789e09884/go.mod h1:ErDfiIrPHH+menTP/B4LKd0nxFDdvCbTamAc6SWMIh8=
 github.com/openshift/onsi-ginkgo/v2 v2.6.1-0.20241205171354-8006f302fd12 h1:AKx/w1qpS8We43bsRgf8Nll3CGlDHpr/WAXvuedTNZI=
 github.com/openshift/onsi-ginkgo/v2 v2.6.1-0.20241205171354-8006f302fd12/go.mod h1:7Du3c42kxCUegi0IImZ1wUQzMBVecgIHjR1C+NkhLQo=
 github.com/operator-framework/api v0.17.1 h1:J/6+Xj4IEV8C7hcirqUFwOiZAU3PbnJhWvB0/bB51c4=

hack/cluster-version-util/task_graph.go

@@ -6,6 +6,7 @@ import (
 	"time"
 
 	"github.com/spf13/cobra"
+	"k8s.io/apimachinery/pkg/util/sets"
 
 	"github.com/openshift/cluster-version-operator/pkg/payload"
 )
@@ -30,7 +31,7 @@ func newTaskGraphCmd() *cobra.Command {
 
 func runTaskGraphCmd(cmd *cobra.Command, args []string) error {
 	manifestDir := args[0]
-	release, err := payload.LoadUpdate(manifestDir, "", "", "", payload.DefaultClusterProfile, nil)
+	release, err := payload.LoadUpdate(manifestDir, "", "", "", payload.DefaultClusterProfile, nil, sets.Set[string]{})
 	if err != nil {
 		return err
 	}

install/0000_90_cluster-version-operator_03_configmaps.yaml

@@ -0,0 +1,32 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: dev-preview-created
+  namespace: openshift-cluster-version
+  annotations:
+    include.release.openshift.io/self-managed-high-availability: "true"
+    release.openshift.io/feature-gates: "Example,Example2"
+data:
+  example: "this should be created on dev-preview-only as it's annotated with Example and Example2"
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: tech-preview-created
+  namespace: openshift-cluster-version
+  annotations:
+    include.release.openshift.io/self-managed-high-availability: "true"
+    release.openshift.io/feature-gates: "Example,-Example2"
+data:
+  example: "this should be created on tech-preview-only as it's annotated with Example and not Example2"
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: default-created
+  namespace: openshift-cluster-version
+  annotations:
+    include.release.openshift.io/self-managed-high-availability: "true"
+    release.openshift.io/feature-gates: "-Example,-Example2"
+data:
+  example: "this should be created on default-only as it's annotated with negation for Example and Example2"

lib/manifest/manifest.go

@@ -48,6 +48,7 @@ func GetImplicitlyEnabledCapabilities(
 	currentPayloadManifests []manifest.Manifest,
 	manifestInclusionConfiguration InclusionConfiguration,
 	currentImplicitlyEnabled sets.Set[configv1.ClusterVersionCapability],
+	enabledFeatureGates sets.Set[string],
 ) sets.Set[configv1.ClusterVersionCapability] {
 	ret := currentImplicitlyEnabled.Clone()
 	for _, updateManifest := range updatePayloadManifests {
@@ -57,6 +58,7 @@ func GetImplicitlyEnabledCapabilities(
 			manifestInclusionConfiguration.Profile,
 			manifestInclusionConfiguration.Capabilities,
 			manifestInclusionConfiguration.Overrides,
+			enabledFeatureGates,
 			true,
 		)
 		// update manifest is enabled, no need to check
@@ -74,6 +76,7 @@ func GetImplicitlyEnabledCapabilities(
 				manifestInclusionConfiguration.Profile,
 				manifestInclusionConfiguration.Capabilities,
 				manifestInclusionConfiguration.Overrides,
+				enabledFeatureGates,
 				true,
 			); err != nil {
 				continue

pkg/cvo/cvo.go

@@ -13,6 +13,7 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
+	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/apimachinery/pkg/util/wait"
 	informerscorev1 "k8s.io/client-go/informers/core/v1"
 	"k8s.io/client-go/kubernetes"
@@ -120,6 +121,7 @@ type Operator struct {
 	cmConfigLister        listerscorev1.ConfigMapNamespaceLister
 	cmConfigManagedLister listerscorev1.ConfigMapNamespaceLister
 	proxyLister           configlistersv1.ProxyLister
+	featureGateLister     configlistersv1.FeatureGateLister
 	cacheSynced           []cache.InformerSynced
 
 	// queue tracks applying updates to a cluster.
@@ -189,6 +191,10 @@ type Operator struct {
 	// featurechangestopper controller will detect when cluster feature gate config changes and shutdown the CVO.
 	enabledFeatureGates featuregates.CvoGateChecker
 
+	// featureGatesMutex protects access to enabledManifestFeatureGates
+	featureGatesMutex           sync.RWMutex
+	enabledManifestFeatureGates sets.Set[string]
+
 	clusterProfile string
 	uid            types.UID
 
@@ -213,6 +219,7 @@ func New(
 	cmConfigManagedInformer informerscorev1.ConfigMapInformer,
 	proxyInformer configinformersv1.ProxyInformer,
 	operatorInformerFactory operatorexternalversions.SharedInformerFactory,
+	featureGateInformer configinformersv1.FeatureGateInformer,
 	client clientset.Interface,
 	kubeClient kubernetes.Interface,
 	operatorClient operatorclientset.Interface,
@@ -248,9 +255,9 @@ func New(
 		kubeClient:            kubeClient,
 		operatorClient:        operatorClient,
 		eventRecorder:         eventBroadcaster.NewRecorder(scheme.Scheme, corev1.EventSource{Component: namespace}),
-		queue:                 workqueue.NewTypedRateLimitingQueueWithConfig[any](workqueue.DefaultTypedControllerRateLimiter[any](), workqueue.TypedRateLimitingQueueConfig[any]{Name: "clusterversion"}),
-		availableUpdatesQueue: workqueue.NewTypedRateLimitingQueueWithConfig[any](workqueue.DefaultTypedControllerRateLimiter[any](), workqueue.TypedRateLimitingQueueConfig[any]{Name: "availableupdates"}),
-		upgradeableQueue:      workqueue.NewTypedRateLimitingQueueWithConfig[any](workqueue.DefaultTypedControllerRateLimiter[any](), workqueue.TypedRateLimitingQueueConfig[any]{Name: "upgradeable"}),
+		queue:                 workqueue.NewTypedRateLimitingQueueWithConfig(workqueue.DefaultTypedControllerRateLimiter[any](), workqueue.TypedRateLimitingQueueConfig[any]{Name: "clusterversion"}),
+		availableUpdatesQueue: workqueue.NewTypedRateLimitingQueueWithConfig(workqueue.DefaultTypedControllerRateLimiter[any](), workqueue.TypedRateLimitingQueueConfig[any]{Name: "availableupdates"}),
+		upgradeableQueue:      workqueue.NewTypedRateLimitingQueueWithConfig(workqueue.DefaultTypedControllerRateLimiter[any](), workqueue.TypedRateLimitingQueueConfig[any]{Name: "upgradeable"}),
 
 		hypershift:                hypershift,
 		exclude:                   exclude,
@@ -276,6 +283,9 @@ func New(
 	if _, err := coInformer.Informer().AddEventHandler(optr.clusterOperatorEventHandler()); err != nil {
 		return nil, err
 	}
+	if _, err := featureGateInformer.Informer().AddEventHandler(optr.featureGateEventHandler()); err != nil {
+		return nil, err
+	}
 
 	optr.coLister = coInformer.Lister()
 	optr.cacheSynced = append(optr.cacheSynced, coInformer.Informer().HasSynced)
@@ -287,6 +297,12 @@ func New(
 	optr.cmConfigLister = cmConfigInformer.Lister().ConfigMaps(internal.ConfigNamespace)
 	optr.cmConfigManagedLister = cmConfigManagedInformer.Lister().ConfigMaps(internal.ConfigManagedNamespace)
 
+	optr.featureGateLister = featureGateInformer.Lister()
+	optr.cacheSynced = append(optr.cacheSynced, featureGateInformer.Informer().HasSynced)
+
+	// Initialize cluster feature gates
+	optr.initializeFeatureGates()
+
 	// make sure this is initialized after all the listers are initialized
 	optr.upgradeableChecks = optr.defaultUpgradeableChecks()
 
@@ -318,7 +334,7 @@ func (optr *Operator) LoadInitialPayload(ctx context.Context, restConfig *rest.C
 	}
 
 	update, err := payload.LoadUpdate(optr.defaultPayloadDir(), optr.release.Image, optr.exclude, string(optr.requiredFeatureSet),
-		optr.clusterProfile, configv1.KnownClusterVersionCapabilities)
+		optr.clusterProfile, configv1.KnownClusterVersionCapabilities, optr.getEnabledFeatureGates())
 
 	if err != nil {
 		return nil, fmt.Errorf("the local release contents are invalid - no current version can be determined from disk: %v", err)
@@ -779,7 +795,7 @@ func (optr *Operator) sync(ctx context.Context, key string) error {
 	}
 
 	// inform the config sync loop about our desired state
-	status := optr.configSync.Update(ctx, config.Generation, desired, config, state)
+	status := optr.configSync.Update(ctx, config.Generation, desired, config, state, optr.getEnabledFeatureGates())
 
 	// write cluster version status
 	return optr.syncStatus(ctx, original, config, status, errs)
@@ -1084,6 +1100,97 @@ func (optr *Operator) HTTPClient() (*http.Client, error) {
 	}, nil
 }
 
+// featureGateEventHandler handles changes to FeatureGate objects and updates the cluster feature gates
+func (optr *Operator) featureGateEventHandler() cache.ResourceEventHandler {
+	workQueueKey := optr.queueKey()
+	return cache.ResourceEventHandlerFuncs{
+		AddFunc: func(obj interface{}) {
+			if optr.updateEnabledFeatureGates(obj) {
+				optr.queue.Add(workQueueKey)
+			}
+		},
+		UpdateFunc: func(old, new interface{}) {
+			if optr.updateEnabledFeatureGates(new) {
+				optr.queue.Add(workQueueKey)
+			}
+		},
+	}
+}
+
+// initializeFeatureGates initializes the cluster feature gates from the current FeatureGate object
+func (optr *Operator) initializeFeatureGates() {
+	// Try to load initial state from the cluster FeatureGate object
+	if optr.featureGateLister != nil {
+		if featureGate, err := optr.featureGateLister.Get("cluster"); err == nil {
+			optr.updateEnabledFeatureGates(featureGate)
+		}
+	}
+}
+
+// updateEnabledFeatureGates updates the cluster feature gates based on a FeatureGate object
+func (optr *Operator) updateEnabledFeatureGates(obj interface{}) bool {
+	featureGate, ok := obj.(*configv1.FeatureGate)
+	if !ok {
+		klog.Warningf("Expected FeatureGate object but got %T", obj)
+		return false
+	}
+
+	newGates := optr.extractEnabledGates(featureGate)
+
+	// Check if gates actually changed to avoid unnecessary work
+	if !optr.enabledManifestFeatureGates.Equal(newGates) {
+		optr.featureGatesMutex.Lock()
+		defer optr.featureGatesMutex.Unlock()
+
+		klog.V(2).Infof("Cluster feature gates changed from %v to %v",
+			sets.List(optr.enabledManifestFeatureGates), sets.List(newGates))
+
+		optr.enabledManifestFeatureGates = newGates
+
+		return true
+	}
+
+	return false
+}
+
+// getEnabledFeatureGates returns a copy of the current cluster feature gates for safe consumption
+func (optr *Operator) getEnabledFeatureGates() sets.Set[string] {
+	optr.featureGatesMutex.RLock()
+	defer optr.featureGatesMutex.RUnlock()
+
+	// Return a copy to prevent external modification
+	result := sets.Set[string]{}
+	for gate := range optr.enabledManifestFeatureGates {
+		result.Insert(gate)
+	}
+	return result
+}
+
+// extractEnabledGates extracts the list of enabled feature gates for the current cluster version
+func (optr *Operator) extractEnabledGates(featureGate *configv1.FeatureGate) sets.Set[string] {
+	enabledGates := sets.Set[string]{}
+
+	// Find the feature gate details for the current cluster version
+	currentVersion := optr.enabledFeatureGates.DesiredVersion()
+	for _, details := range featureGate.Status.FeatureGates {
+		if details.Version == currentVersion {
+			for _, enabled := range details.Enabled {
+				enabledGates.Insert(string(enabled.Name))
+			}
+			klog.V(4).Infof("Found %d enabled feature gates for version %s: %v",
+				enabledGates.Len(), currentVersion, sets.List(enabledGates))
+			break
+		}
+	}
+
+	// If no matching version found, log a warning but continue with empty set
+	if enabledGates.Len() == 0 {
+		klog.V(2).Infof("No feature gates found for current version %s, using empty set", currentVersion)
+	}
+
+	return enabledGates
+}
+
 // shouldReconcileCVOConfiguration returns whether the CVO should reconcile its configuration using the API server.
 //
 // enabledFeatureGates must be initialized before the function is called.