OCPBUGS-38676: Add retry for transient API errors to prevent Degraded blips

pkg/console/controllers/clidownloads/controller.go

@@ -228,10 +228,15 @@ func ApplyCLIDownloads(ctx context.Context, consoleClient consoleclientv1.Consol
 	existingCLIDownloads, err := consoleClient.Get(ctx, cliDownloadsName, metav1.GetOptions{})
 	existingCLIDownloadsCopy := existingCLIDownloads.DeepCopy()
 	if apierrors.IsNotFound(err) {
-		actualCLIDownloads, err := consoleClient.Create(ctx, requiredCLIDownloads, metav1.CreateOptions{})
-		if err != nil {
-			klog.V(4).Infof("error creating %s consoleclidownloads custom resource: %s", cliDownloadsName, err)
-			return nil, "FailedCreate", err
+		var actualCLIDownloads *v1.ConsoleCLIDownload
+		createErr := controllersutil.RetryOnTransientError(func() error {
+			var e error
+			actualCLIDownloads, e = consoleClient.Create(ctx, requiredCLIDownloads, metav1.CreateOptions{})
+			return e
+		})
+		if createErr != nil {
+			klog.V(4).Infof("error creating %s consoleclidownloads custom resource: %s", cliDownloadsName, createErr)
+			return nil, "FailedCreate", createErr
 		}
 		klog.V(4).Infof("%s consoleclidownloads custom resource created", cliDownloadsName)
 		return actualCLIDownloads, "", nil
@@ -249,10 +254,15 @@ func ApplyCLIDownloads(ctx context.Context, consoleClient consoleclientv1.Consol
 	}
 
 	existingCLIDownloadsCopy.Spec = requiredCLIDownloads.Spec
-	actualCLIDownloads, err := consoleClient.Update(ctx, existingCLIDownloadsCopy, metav1.UpdateOptions{})
-	if err != nil {
-		klog.V(4).Infof("error updating %s consoleclidownloads custom resource: %v", cliDownloadsName, err)
-		return nil, "FailedUpdate", err
+	var actualCLIDownloads *v1.ConsoleCLIDownload
+	updateErr := controllersutil.RetryOnTransientError(func() error {
+		var e error
+		actualCLIDownloads, e = consoleClient.Update(ctx, existingCLIDownloadsCopy, metav1.UpdateOptions{})
+		return e
+	})
+	if updateErr != nil {
+		klog.V(4).Infof("error updating %s consoleclidownloads custom resource: %v", cliDownloadsName, updateErr)
+		return nil, "FailedUpdate", updateErr
 	}
 	return actualCLIDownloads, "", nil
 }

pkg/console/controllers/downloadsdeployment/controller.go

@@ -118,12 +118,19 @@ func (c *DownloadsDeploymentSyncController) SyncDownloadsDeployment(ctx context.
 
 	requiredDownloadsDeployment := deploymentsub.DefaultDownloadsDeployment(operatorConfigCopy, infrastructureConfig)
 
-	return resourceapply.ApplyDeployment(ctx,
-		c.deploymentClient,
-		controllerContext.Recorder(),
-		requiredDownloadsDeployment,
-		resourcemerge.ExpectedDeploymentGeneration(requiredDownloadsDeployment, operatorConfigCopy.Status.Generations),
-	)
+	var actualDeployment *appsv1.Deployment
+	var deploymentChanged bool
+	deploymentErr := util.RetryOnTransientError(func() error {
+		var e error
+		actualDeployment, deploymentChanged, e = resourceapply.ApplyDeployment(ctx,
+			c.deploymentClient,
+			controllerContext.Recorder(),
+			requiredDownloadsDeployment,
+			resourcemerge.ExpectedDeploymentGeneration(requiredDownloadsDeployment, operatorConfigCopy.Status.Generations),
+		)
+		return e
+	})
+	return actualDeployment, deploymentChanged, deploymentErr
 }
 
 func (c *DownloadsDeploymentSyncController) removeDownloadsDeployment(ctx context.Context) error {

pkg/console/controllers/oauthclients/oauthclients.go

@@ -226,7 +226,10 @@ func (c *oauthClientsController) syncOAuthClient(
 
 	clientCopy := oauthClient.DeepCopy()
 	oauthsub.RegisterConsoleToOAuthClient(clientCopy, consoleURL, secretsub.GetSecretString(sec))
-	_, _, oauthErr := oauthsub.CustomApplyOAuth(c.oauthClient, clientCopy, ctx)
+	oauthErr := util.RetryOnTransientError(func() error {
+		_, _, e := oauthsub.CustomApplyOAuth(c.oauthClient, clientCopy, ctx)
+		return e
+	})
 	if oauthErr != nil {
 		return "FailedRegister", oauthErr
 	}

pkg/console/controllers/oauthclientsecret/oauthclientsecret.go

@@ -24,6 +24,7 @@ import (
 	"github.com/openshift/library-go/pkg/operator/resource/resourceapply"
 	v1helpers "github.com/openshift/library-go/pkg/operator/v1helpers"
 
+	"github.com/openshift/console-operator/pkg/console/controllers/util"
 	authnsub "github.com/openshift/console-operator/pkg/console/subresource/authentication"
 	secretsub "github.com/openshift/console-operator/pkg/console/subresource/secret"
 )
@@ -160,7 +161,10 @@ func (c *oauthClientSecretController) syncSecret(ctx context.Context, clientSecr
 
 	secret, err := c.targetNSSecretsLister.Secrets(api.TargetNamespace).Get("console-oauth-config")
 	if apierrors.IsNotFound(err) || secretsub.GetSecretString(secret) != clientSecret {
-		_, _, err = resourceapply.ApplySecret(ctx, c.secretsClient, recorder, secretsub.DefaultSecret(operatorConfig, clientSecret))
+		err = util.RetryOnTransientError(func() error {
+			_, _, e := resourceapply.ApplySecret(ctx, c.secretsClient, recorder, secretsub.DefaultSecret(operatorConfig, clientSecret))
+			return e
+		})
 	}
 	return err
 }

pkg/console/controllers/poddisruptionbudget/controller.go

@@ -97,11 +97,11 @@ func (c *PodDisruptionBudgetController) Sync(ctx context.Context, controllerCont
 	statusHandler := status.NewStatusHandler(c.operatorClient)
 
 	requiredPDB := c.getDefaultPodDisruptionBudget()
-	_, _, pdbErr := resourceapply.ApplyPodDisruptionBudget(ctx, c.pdbClient, controllerContext.Recorder(), requiredPDB)
+	pdbErr := util.RetryOnTransientError(func() error {
+		_, _, err := resourceapply.ApplyPodDisruptionBudget(ctx, c.pdbClient, controllerContext.Recorder(), requiredPDB)
+		return err
+	})
 	statusHandler.AddConditions(status.HandleProgressingOrDegraded("PDBSync", "FailedApply", pdbErr))
-	if pdbErr != nil {
-		return statusHandler.FlushAndReturn(pdbErr)
-	}
 	return statusHandler.FlushAndReturn(pdbErr)
 }
 

pkg/console/controllers/service/controller.go

@@ -130,7 +130,10 @@ func (c *ServiceSyncController) Sync(ctx context.Context, controllerContext fact
 	routeConfig := routesub.NewRouteConfig(updatedOperatorConfig, ingressConfig, c.serviceName)
 
 	requiredSvc := c.getDefaultService(ingressDisabled)
-	_, _, svcErr := resourceapply.ApplyService(ctx, c.serviceClient, controllerContext.Recorder(), requiredSvc)
+	svcErr := util.RetryOnTransientError(func() error {
+		_, _, err := resourceapply.ApplyService(ctx, c.serviceClient, controllerContext.Recorder(), requiredSvc)
+		return err
+	})
 	statusHandler.AddConditions(status.HandleProgressingOrDegraded("ServiceSync", "FailedApply", svcErr))
 	if svcErr != nil {
 		return statusHandler.FlushAndReturn(svcErr)
@@ -147,17 +150,23 @@ func (c *ServiceSyncController) Sync(ctx context.Context, controllerContext fact
 
 func (c *ServiceSyncController) SyncRedirectService(ctx context.Context, routeConfig *routesub.RouteConfig, controllerContext factory.SyncContext) (string, error) {
 	if !routeConfig.IsCustomHostnameSet() {
-		if err := c.removeService(ctx, c.getRedirectServiceName()); err != nil {
+		err := util.RetryOnTransientError(func() error {
+			return c.removeService(ctx, c.getRedirectServiceName())
+		})
+		if err != nil {
 			return "FailedDelete", err
 		}
 		return "", nil
 	}
 	requiredRedirectService := c.getRedirectService()
-	_, _, redirectSvcErr := resourceapply.ApplyService(ctx, c.serviceClient, controllerContext.Recorder(), requiredRedirectService)
+	redirectSvcErr := util.RetryOnTransientError(func() error {
+		_, _, err := resourceapply.ApplyService(ctx, c.serviceClient, controllerContext.Recorder(), requiredRedirectService)
+		return err
+	})
 	if redirectSvcErr != nil {
 		return "FailedApply", redirectSvcErr
 	}
-	return "", redirectSvcErr
+	return "", nil
 }
 
 func (c *ServiceSyncController) removeService(ctx context.Context, serviceName string) error {

pkg/console/controllers/serviceaccounts/controller.go

@@ -152,14 +152,17 @@ func (c *ServiceAccountSyncController) SyncServiceAccount(ctx context.Context, o
 		return fmt.Errorf("failed to get existing service account %s: %w", c.serviceAccountName, err)
 	}
 
-	_, _, err = resourceapply.ApplyServiceAccount(ctx,
-		c.serviceAccountClient,
-		controllerContext.Recorder(),
-		serviceAccount,
-	)
-
-	if err != nil {
-		return fmt.Errorf("failed to apply service account %s: %w", c.serviceAccountName, err)
+	applyErr := util.RetryOnTransientError(func() error {
+		_, _, e := resourceapply.ApplyServiceAccount(ctx,
+			c.serviceAccountClient,
+			controllerContext.Recorder(),
+			serviceAccount,
+		)
+		return e
+	})
+
+	if applyErr != nil {
+		return fmt.Errorf("failed to apply service account %s: %w", c.serviceAccountName, applyErr)
 	}
 
 	return nil

pkg/console/controllers/storageversionmigration/controller.go

@@ -14,6 +14,7 @@ import (
 	"k8s.io/client-go/dynamic/dynamicinformer"
 	"k8s.io/klog/v2"
 
+	"github.com/openshift/console-operator/pkg/console/controllers/util"
 	"github.com/openshift/console-operator/pkg/console/status"
 	"github.com/openshift/library-go/pkg/controller/factory"
 	"github.com/openshift/library-go/pkg/operator/events"
@@ -103,7 +104,9 @@ func (c *StorageVersionMigrationController) syncStorageVersionMigration(ctx cont
 	if !succeeded {
 		klog.V(4).Infof("StorageVersionMigration has not succeeded yet")
 		// Delete the StorageVersionMigration if it has not succeeded yet
-		if err := c.deleteStorageVersionMigration(ctx); err != nil {
+		if err := util.RetryOnTransientError(func() error {
+			return c.deleteStorageVersionMigration(ctx)
+		}); err != nil {
 			return "FailedDeleteSVM", err
 		}
 		return "", nil

pkg/console/controllers/upgradenotification/controller.go

@@ -131,14 +131,19 @@ func (c *UpgradeNotificationController) syncClusterUpgradeNotification(ctx conte
 				BackgroundColor: "#F0AB00",
 			},
 		}
-		_, err = c.consoleNotificationClient.Create(ctx, notification, metav1.CreateOptions{})
-		if err != nil && !apierrors.IsAlreadyExists(err) {
-			return "FailedCreate", err
+		createErr := util.RetryOnTransientError(func() error {
+			_, e := c.consoleNotificationClient.Create(ctx, notification, metav1.CreateOptions{})
+			return e
+		})
+		if createErr != nil && !apierrors.IsAlreadyExists(createErr) {
+			return "FailedCreate", createErr
 		}
 	} else {
-		err = c.removeUpgradeNotification(ctx)
-		if err != nil {
-			return "FailedDelete", err
+		deleteErr := util.RetryOnTransientError(func() error {
+			return c.removeUpgradeNotification(ctx)
+		})
+		if deleteErr != nil {
+			return "FailedDelete", deleteErr
 		}
 	}
 

pkg/console/controllers/util/retry.go

@@ -0,0 +1,51 @@
+package util
+
+import (
+	"time"
+
+	// kube
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/client-go/util/retry"
+	"k8s.io/klog/v2"
+)
+
+// TransientBackoff defines the retry parameters for transient API errors.
+// 3 steps with 500ms base and 2.0 factor gives: 500ms, 1s, 2s = ~3.5s max per call.
+var TransientBackoff = wait.Backoff{
+	Steps:    3,
+	Duration: 500 * time.Millisecond,
+	Factor:   2.0,
+	Jitter:   0.1,
+}
+
+// IsRetryableError returns true for errors worth retrying — everything
+// except known permanent errors. This naturally handles both API status
+// errors (apierrors.StatusError) and network-level errors (connection
+// refused, EOF, TLS failures) without explicit net.Error detection.
+func IsRetryableError(err error) bool {
+	if apierrors.IsForbidden(err) ||
+		apierrors.IsInvalid(err) ||
+		apierrors.IsMethodNotSupported(err) ||
+		apierrors.IsNotAcceptable(err) ||
+		apierrors.IsAlreadyExists(err) {
+		return false
+	}
+	return true
+}
+
+// RetryOnTransientError wraps a function call with retry logic to absorb
+// transient API server errors (conflicts, timeouts, connection refused)
+// that occur during upgrades. Only API write operations (Apply, Create,
+// Update, Delete) should be wrapped — not lister/cache reads.
+func RetryOnTransientError(fn func() error) error {
+	attempt := 0
+	return retry.OnError(TransientBackoff, IsRetryableError, func() error {
+		err := fn()
+		if err != nil {
+			attempt++
+			klog.V(4).Infof("transient error (attempt %d/%d): %v", attempt, TransientBackoff.Steps, err)
+		}
+		return err
+	})
+}