feat: enhance Gateway API plugin with in-progress label management (#155)

* feat: enhance Gateway API plugin with in-progress label management

- Added support for automatic labeling of Gateway API routes during canary deployments to prevent GitOps drift.
- Updated documentation to reflect new features, including the ability to customize or disable the in-progress label.
- Improved tests to verify the addition and removal of the in-progress label for HTTP, gRPC, TCP, and TLS routes.

This change enhances the integration with GitOps tools like Argo CD, ensuring smoother deployments and better resource management.

Signed-off-by: rick.stokkingreef <rick.stokkingreef@airalo.com>

* ci

Signed-off-by: rick.stokkingreef <rick.stokkingreef@airalo.com>

* fix ignorediff doc section

Signed-off-by: rick.stokkingreef <rick.stokkingreef@airalo.com>

* chore: Add test

Signed-off-by: rick.stokkingreef <rick.stokkingreef@airalo.com>

* update versions of rollouts and traefik

Signed-off-by: rick.stokkingreef <rick.stokkingreef@airalo.com>

* cleanup

Signed-off-by: rick.stokkingreef <rick.stokkingreef@airalo.com>

* test

Signed-off-by: rick.stokkingreef <rick.stokkingreef@airalo.com>

* add wait to sanity check

Signed-off-by: rick.stokkingreef <rick.stokkingreef@airalo.com>

* chore: move label change into it's own test

Signed-off-by: rick.stokkingreef <rick.stokkingreef@airalo.com>

* Fix lint

Signed-off-by: rick.stokkingreef <rick.stokkingreef@airalo.com>

* cleanup

Signed-off-by: rick.stokkingreef <rick.stokkingreef@airalo.com>

* Enhance sanity check script to retry gateway class acceptance with a maximum of 5 attempts, improving reliability of e2e tests.

Signed-off-by: rick.stokkingreef <rick.stokkingreef@airalo.com>

---------

Signed-off-by: rick.stokkingreef <rick.stokkingreef@airalo.com>

Author: stokkie90

Makefile

@@ -4,11 +4,11 @@ E2E_CLUSTER_NAME=gatewayapi-plugin-e2e
 IS_E2E_CLUSTER=$(shell kind get clusters | grep -e "^${E2E_CLUSTER_NAME}$$")
 
 # Versions of components used in e2e tests
-GATEWAY_API_VERSION=v1.1.0
+GATEWAY_API_VERSION=v1.4.0
 # See more versions at https://artifacthub.io/packages/helm/argo/argo-rollouts
-ARGO_ROLLOUTS_HELM_VERSION=2.37.2 # Contains Argo Rollouts 1.7.1
+ARGO_ROLLOUTS_HELM_VERSION=2.40.5 # Contains Argo Rollouts 1.8.3
 # See more versions at https://artifacthub.io/packages/helm/traefik/traefik
-TRAEFIK_HELM_VERSION=31.0.0 # Contains Traefik proxy v3.1.2
+TRAEFIK_HELM_VERSION=37.4.0 # Contains Traefik proxy v3.6.2
 
 
 
@@ -21,9 +21,9 @@ define add_helm_repo
 endef
 
 define setup_cluster
-	kubectl apply -f https://github.com/kubernetes-sigs/gateway-api/releases/download/${GATEWAY_API_VERSION}/experimental-install.yaml
 	helm install argo-rollouts argo/argo-rollouts --values ./test/cluster-setup/argo-rollouts-values.yml --version ${ARGO_ROLLOUTS_HELM_VERSION} --wait
 	helm install traefik traefik/traefik --values ./test/cluster-setup/traefik-values.yml --version ${TRAEFIK_HELM_VERSION} --wait
+	kubectl apply -f https://github.com/kubernetes-sigs/gateway-api/releases/download/${GATEWAY_API_VERSION}/experimental-install.yaml --server-side=true --force-conflicts
 endef
 
 define install_k8s_resources
@@ -60,7 +60,7 @@ unit-tests:
 	go test -v -count=1 ./pkg/...
 
 .PHONY: setup-e2e-cluster
-setup-e2e-cluster:	
+setup-e2e-cluster:
 	make BIN_NAME=gatewayapi-plugin-linux-amd64 GOOS=linux GOARCH=amd64 gatewayapi-plugin-build
 ifeq (${IS_E2E_CLUSTER},)
 	kind create cluster --name ${E2E_CLUSTER_NAME} --config ./test/cluster-setup/cluster-config.yml
@@ -79,20 +79,20 @@ endif
 sanity-check-e2e:
 	./test/cluster-setup/sanity-check.sh
 
-.PHONY: run-e2e-tests 
+.PHONY: run-e2e-tests
 run-e2e-tests: sanity-check-e2e
 	go test -v -timeout 5m -count=1 -run ${RUN} ./test/e2e/...
 
 # Flaky tests usually fail with GitHub actions. You should be able to run them locally though.
 .PHONY: e2e-tests-flaky
-e2e-tests-flaky: setup-e2e-cluster run-e2e-tests-flaky 
+e2e-tests-flaky: setup-e2e-cluster run-e2e-tests-flaky
 ifeq (${CLUSTER_DELETE},true)
 	make clear-e2e-cluster
 endif
 
-.PHONY: run-e2e-tests-flaky 
+.PHONY: run-e2e-tests-flaky
 run-e2e-tests-flaky: sanity-check-e2e
-	go test -tags "flaky" -v -timeout 5m -count=1 -run ${RUN} ./test/e2e/... 
+	go test -tags "flaky" -v -timeout 5m -count=1 -run ${RUN} ./test/e2e/...
 
 .PHONY: clear-e2e-cluster
 clear-e2e-cluster:

RELEASE_NOTES.md

@@ -1,2 +1,3 @@
 * Added support for [TLSRoute](https://rollouts-plugin-trafficrouter-gatewayapi.readthedocs.io/en/latest/features/tls/).
-* You can now use  [filters with Header based routing](https://github.com/argoproj-labs/rollouts-plugin-trafficrouter-gatewayapi/issues/87).
+* You can now use  [filters with Header based routing](https://github.com/argoproj-labs/rollouts-plugin-trafficrouter-gatewayapi/issues/87).
+* Gateway API routes are labeled while a canary is running to avoid GitOps drift and the label is removed once traffic returns to 100% stable.

docs/features/multiple-routes.md

@@ -23,7 +23,7 @@ spec:
   parentRefs:
     - name: eg
   hostnames:
-    - backend.example.com  
+    - backend.example.com
   rules:
   - matches:
     - path:
@@ -46,7 +46,7 @@ spec:
   parentRefs:
     - name: eg
   hostnames:
-    - api.example.com     
+    - api.example.com
   rules:
   - matches:
     - path:
@@ -106,10 +106,36 @@ spec:
             - name: http
               containerPort: 8080
               protocol: TCP
-```              
+```
 
 If you now start a canary deployment both routes will change to 10%, 50% and 100% as the canary progresses to all its steps.
 
+## Working with GitOps controllers
+
+GitOps tools such as Argo CD continuously reconcile Gateway API resources and can revert the temporary weight changes that occur
+while a canary is progressing. The plugin automatically adds the label
+`rollouts.argoproj.io/gatewayapi-canary=in-progress` to every HTTPRoute/GRPCRoute/TCPRoute/TLSRoute it mutates so that you can
+configure your GitOps policy to ignore those resources during a rollout. The label disappears as soon as the stable service
+returns to 100% weight. You can customise the key/value or disable the feature altogether with the
+`inProgressLabelKey`, `inProgressLabelValue` and `disableInProgressLabel` fields under the plugin configuration.
+
+### Argo CD `ignoreDifferences`
+
+When you use Argo CD (either through the Application CRD or its Helm chart), add the following snippet so that Argo CD skips the
+temporary rule edits while the `rollouts.argoproj.io/gatewayapi-canary` label is present:
+
+```yaml
+configs:
+  cm:
+    resource.customizations.ignoreDifferences.gateway.networking.k8s.io_HTTPRoute: |
+      jqPathExpressions:
+        - select(.metadata.labels["rollouts.argoproj.io/gatewayapi-canary"] == "in-progress") | .spec.rules
+```
+
+Duplicate the block for `GRPCRoute`, `TCPRoute` and `TLSRoute` if you manage those kinds as well. If you have customised the
+label key or value on the plugin, update the `jqPathExpressions` condition to match your configuration. The same structure applies
+when you configure `resource.customizations` directly on an Application manifest (outside of Helm).
+
 ## Automatic Route Discovery with Label Selectors
 
 Instead of explicitly listing each route name, you can use label selectors to automatically discover routes. This is particularly useful when managing many routes or when routes are created dynamically.
@@ -200,7 +226,7 @@ trafficRouting:
 The plugin supports selectors for different route types:
 
 - `httpRouteSelector`: Discovers HTTPRoutes
-- `grpcRouteSelector`: Discovers GRPCRoutes  
+- `grpcRouteSelector`: Discovers GRPCRoutes
 - `tcpRouteSelector`: Discovers TCPRoutes
 
 You can use multiple selectors simultaneously:
@@ -247,4 +273,4 @@ To verify which routes will be discovered by your selector, use kubectl:
 kubectl get httproutes -n default -l app=my-app,canary-enabled=true
 ```
 
-The plugin logs discovered routes during reconciliation, which can help with debugging.
+The plugin logs discovered routes during reconciliation, which can help with debugging.

docs/quick-start.md

@@ -5,7 +5,7 @@ to control your Http Routes. In this guide we will see how to use [the Rollouts
 
 You can find more examples at the [provider status page](provider-status.md).
 
-## Prerequisites 
+## Prerequisites
 
 Get access to a Kubernetes cluster. You can use a cluster on the cloud or on your workstation like [k3s](https://k3s.io/), [k3d](https://k3d.io/) or [Docker for Desktop](https://www.docker.com/products/docker-desktop/).
 
@@ -40,7 +40,7 @@ kubectl wait --timeout=5m -n envoy-gateway-system deployment/envoy-gateway --for
 !!! note
     This process needs to happen only once per cluster. The task is normally handled by infrastructure operators.
 
-Create a Gateway 
+Create a Gateway
 
 ```yaml
 ---
@@ -63,7 +63,7 @@ spec:
     - name: http
       protocol: HTTP
       port: 80
-```      
+```
 Apply the file with kubectl and then verify it works correctly with
 
 ```
@@ -109,12 +109,12 @@ subjects:
   - namespace: argo-rollouts
     kind: ServiceAccount
     name: argo-rollouts
-```  
+```
 
 Apply the file with kubectl. Note that this role is **NOT** to be used in production clusters as it is super permissive.
 
 
-## Step 4 - Create an HTTP route 
+## Step 4 - Create an HTTP route
 
 !!! note
     This process needs to happen only once per application. The task is normally handled by cluster operators or application developers.
@@ -135,7 +135,7 @@ spec:
   - matches:
     - path:
         type: PathPrefix
-        value: /  
+        value: /
     backendRefs:
     - name: argo-rollouts-stable-service
       kind: Service
@@ -148,7 +148,7 @@ spec:
 Apply the file with kubectl.
 Verify it with `kubectl get httproutes`
 
-## Step 5 - Create a Rollout 
+## Step 5 - Create a Rollout
 
 !!! note
     This process needs to happen only once per application. The task is normally handled by cluster operators or application developers.
@@ -207,6 +207,10 @@ spec:
           argoproj-labs/gatewayAPI:
             httpRoute: argo-rollouts-http-route # our created httproute
             namespace: default
+            # Optional: customize or disable the temporary label that marks routes as managed during a canary
+            # inProgressLabelKey: rollouts.argoproj.io/gatewayapi-canary
+            # inProgressLabelValue: in-progress
+            # disableInProgressLabel: false
       steps:
       - setWeight: 50
       - pause: {}
@@ -241,7 +245,7 @@ You should see that all requests return with blue color:
 ![First deployment](images/quick-start/canary-start.png)
 
 
-## Daily Task - Perform a Canary 
+## Daily Task - Perform a Canary
 
 !!! note
     This process happens multiple times per day/week. The task is normally handled by application developers.
@@ -260,14 +264,34 @@ At this point each color should get 50% of requests. You can see this visually i
 
 You should also inspect the Http Route and verify that Argo Rollouts has changed the weights of the backend services
 
-Run 
+Run
 
 ```
 kubectl get httproute -o yaml
 ```
 
 In the response you should see the following information about the weights for each backing service.
 
+!!! info
+    While the canary is running, the plugin adds the label `rollouts.argoproj.io/gatewayapi-canary=in-progress` to every managed
+    Gateway API route so that GitOps tools such as Argo CD can be configured to ignore those temporary changes. The label is
+    removed automatically once the stable service goes back to 100% weight. Use `disableInProgressLabel`, `inProgressLabelKey`
+    or `inProgressLabelValue` if you need to adjust this behaviour.
+
+    **Argo CD example (Helm chart values)**
+
+    ```yaml
+    configs:
+      cm:
+        resource.customizations.ignoreDifferences.gateway.networking.k8s.io_HTTPRoute: |
+          jqPathExpressions:
+            - if .metadata.labels["rollouts.argoproj.io/gatewayapi-canary"] == "in-progress" then .spec.rules
+    ```
+
+    Apply the same snippet to `GRPCRoute`, `TCPRoute` and `TLSRoute` kinds if you manage them. If you configure `resource.customizations`
+    directly inside an Application manifest rather than Helm values, reuse the same structure under `spec.source.plugin` or
+    `spec.source.helm.values`.
+
 ```yaml
 [...snip...]
   spec:
@@ -307,4 +331,4 @@ The application should gradually change now to yellow.
 
 The deployment has finished. If you change the Rollout image again, the process will start over.
 
-Feel free to learn more about all Rollout options in the [Specification documentation](https://argo-rollouts.readthedocs.io/en/stable/features/specification/).
+Feel free to learn more about all Rollout options in the [Specification documentation](https://argo-rollouts.readthedocs.io/en/stable/features/specification/).

internal/defaults/defaults.go

@@ -1,3 +1,7 @@
 package defaults
 
-const ConfigMap = "argo-gatewayapi-configmap"
+const (
+	ConfigMap            = "argo-gatewayapi-configmap"
+	InProgressLabelKey   = "rollouts.argoproj.io/gatewayapi-canary"
+	InProgressLabelValue = "in-progress"
+)

pkg/plugin/grpcroute.go

@@ -52,6 +52,7 @@ func (r *RpcPlugin) setGRPCRouteWeight(rollout *v1alpha1.Rollout, desiredWeight
 	for _, ref := range stableBackendRefs {
 		ref.Weight = &restWeight
 	}
+	ensureInProgressLabel(grpcRoute, desiredWeight, gatewayAPIConfig)
 	updatedGRPCRoute, err := grpcRouteClient.Update(ctx, grpcRoute, metav1.UpdateOptions{})
 	if r.IsTest {
 		r.UpdatedGRPCRouteMock = updatedGRPCRoute
@@ -396,6 +397,16 @@ func removeManagedGRPCRouteEntry(managedRouteMap ManagedRouteMap, routeRuleList
 		managedRouteMapKey := managedRouteName + "." + grpcRouteName
 		return nil, fmt.Errorf(ManagedRouteMapEntryDeleteError, managedRouteMapKey, managedRouteMapKey)
 	}
+	if managedRouteIndex < 0 || managedRouteIndex >= len(routeRuleList) {
+		// stale or corrupted managed route index; clean references for this route and continue gracefully
+		for name, managedMap := range managedRouteMap {
+			delete(managedMap, grpcRouteName)
+			if len(managedMap) == 0 {
+				delete(managedRouteMap, name)
+			}
+		}
+		return routeRuleList, nil
+	}
 	delete(routeManagedRouteMap, grpcRouteName)
 	if len(managedRouteMap[managedRouteName]) == 0 {
 		delete(managedRouteMap, managedRouteName)

pkg/plugin/httproute.go

@@ -56,6 +56,7 @@ func (r *RpcPlugin) setHTTPRouteWeight(rollout *v1alpha1.Rollout, desiredWeight
 	if err != nil {
 		r.LogCtx.Error(err, "Failed to handle experiment services")
 	}
+	ensureInProgressLabel(httpRoute, desiredWeight, gatewayAPIConfig)
 	updatedHTTPRoute, err := httpRouteClient.Update(ctx, httpRoute, metav1.UpdateOptions{})
 	if r.IsTest {
 		r.UpdatedHTTPRouteMock = updatedHTTPRoute
@@ -395,6 +396,16 @@ func removeManagedHTTPRouteEntry(managedRouteMap ManagedRouteMap, routeRuleList
 		managedRouteMapKey := managedRouteName + "." + httpRouteName
 		return nil, fmt.Errorf(ManagedRouteMapEntryDeleteError, managedRouteMapKey, managedRouteMapKey)
 	}
+	if managedRouteIndex < 0 || managedRouteIndex >= len(routeRuleList) {
+		// stale or corrupted managed route index; clean references for this route and continue gracefully
+		for name, managedMap := range managedRouteMap {
+			delete(managedMap, httpRouteName)
+			if len(managedMap) == 0 {
+				delete(managedRouteMap, name)
+			}
+		}
+		return routeRuleList, nil
+	}
 	delete(routeManagedRouteMap, httpRouteName)
 	if len(managedRouteMap[managedRouteName]) == 0 {
 		delete(managedRouteMap, managedRouteName)

pkg/plugin/labels.go

@@ -0,0 +1,56 @@
+package plugin
+
+import (
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	"github.com/argoproj-labs/rollouts-plugin-trafficrouter-gatewayapi/internal/defaults"
+)
+
+func ensureInProgressLabel(obj metav1.Object, desiredWeight int32, config *GatewayAPITrafficRouting) bool {
+	if obj == nil || config == nil || config.DisableInProgressLabel {
+		return false
+	}
+
+	key := config.inProgressLabelKey()
+	if key == "" {
+		return false
+	}
+
+	labels := obj.GetLabels()
+	if desiredWeight == 0 {
+		if labels == nil {
+			return false
+		}
+		if _, ok := labels[key]; ok {
+			delete(labels, key)
+			obj.SetLabels(labels)
+			return true
+		}
+		return false
+	}
+
+	value := config.inProgressLabelValue()
+	if labels == nil {
+		labels = make(map[string]string)
+	}
+	if current, ok := labels[key]; ok && current == value {
+		return false
+	}
+	labels[key] = value
+	obj.SetLabels(labels)
+	return true
+}
+
+func (c *GatewayAPITrafficRouting) inProgressLabelKey() string {
+	if c.InProgressLabelKey != "" {
+		return c.InProgressLabelKey
+	}
+	return defaults.InProgressLabelKey
+}
+
+func (c *GatewayAPITrafficRouting) inProgressLabelValue() string {
+	if c.InProgressLabelValue != "" {
+		return c.InProgressLabelValue
+	}
+	return defaults.InProgressLabelValue
+}