feat: enable site clusters to run Nautobot Celery workers locally

Nautobot currently runs entirely on the global cluster, including its
Celery workers. Sites that generate heavy background task load have no
way to offload that processing closer to where the work originates,
and a single global worker pool becomes a bottleneck as sites scale.

This adds a site-scoped ArgoCD Application that deploys only the
Celery worker portion of the Nautobot helm chart. The web server,
Redis, and PostgreSQL are all disabled because they remain on the
global cluster — site workers connect back to those shared services.

This lets operators scale worker capacity per-site independently,
run queue-specific workers closer to the hardware they manage, and
reduce cross-cluster task latency for site-driven automation.

Author: syedhaseebahmed

charts/argocd-understack/templates/application-nautobot-worker.yaml

@@ -0,0 +1,49 @@
+{{- if eq (include "understack.isEnabled" (list $.Values.site "nautobot_worker")) "true" }}
+---
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: {{ printf "%s-%s" $.Release.Name "nautobot-worker" }}
+  finalizers:
+  - resources-finalizer.argocd.argoproj.io
+  annotations:
+    argocd.argoproj.io/compare-options: ServerSideDiff=true,IncludeMutationWebhook=true
+spec:
+  destination:
+    namespace: nautobot
+    server: {{ $.Values.cluster_server }}
+  project: understack
+  sources:
+  - chart: nautobot
+    helm:
+      fileParameters:
+      - name: nautobot.config
+        path: $understack/components/nautobot/nautobot_config.py
+      ignoreMissingValueFiles: true
+      releaseName: nautobot-worker
+      valueFiles:
+      - $understack/components/nautobot-worker/values.yaml
+      - $deploy/{{ include "understack.deploy_path" $ }}/nautobot-worker/values.yaml
+    repoURL: https://nautobot.github.io/helm-charts/
+    targetRevision: 2.5.6
+  - path: components/nautobot-worker
+    ref: understack
+    repoURL: {{ include "understack.understack_url" $ }}
+    targetRevision: {{ include "understack.understack_ref" $ }}
+  - path: {{ include "understack.deploy_path" $ }}/nautobot-worker
+    ref: deploy
+    repoURL: {{ include "understack.deploy_url" $ }}
+    targetRevision: {{ include "understack.deploy_ref" $ }}
+  syncPolicy:
+    automated:
+      prune: true
+      selfHeal: true
+    managedNamespaceMetadata:
+      annotations:
+        argocd.argoproj.io/sync-options: Delete=false
+    syncOptions:
+    - CreateNamespace=true
+    - ServerSideApply=true
+    - RespectIgnoreDifferences=true
+    - ApplyOutOfSyncOnly=true
+{{- end }}

charts/argocd-understack/values.yaml

@@ -514,6 +514,12 @@ site:
     # @default -- false
     enabled: false
 
+  # -- Nautobot Celery workers (site-level, connects to global Nautobot)
+  nautobot_worker:
+    # -- Enable/disable deploying Nautobot workers at the site level
+    # @default -- false
+    enabled: false
+
   # -- Site-specific workflows and event handlers
   site_workflows:
     # -- Enable/disable deploying site workflows

components/envoy-configs/templates/gw-external.yaml.tpl

@@ -52,6 +52,20 @@ spec:
           from: {{ .from | default "All" }}
           {{- end }}
     {{- end }}
+    {{- range .Values.routes.tcp }}
+    - name: {{ .listenerName }}
+      port: {{ .gatewayPort }}
+      protocol: TCP
+      allowedRoutes:
+        namespaces:
+          {{- if .selector }}
+          from: Selector
+          selector:
+            {{- .selector | toYaml | nindent 12 }}
+          {{- else }}
+          from: {{ .from | default "All" }}
+          {{- end }}
+    {{- end }}
   {{- if .Values.gateways.external.serviceAnnotations }}
   infrastructure:
     parametersRef:

components/envoy-configs/templates/tcproute.yaml.tpl

@@ -0,0 +1,26 @@
+{{- range .Values.routes.tcp }}
+---
+apiVersion: gateway.networking.k8s.io/v1alpha2
+kind: TCPRoute
+metadata:
+  {{- if .name }}
+  name: {{ .name }}
+  {{- else }}
+  name: {{ .service.name }}
+  {{- end }}
+  namespace: {{ .namespace | default "envoy-gateway" }}
+  labels:
+    {{- include "envoy-configs.labels" $ | nindent 4 }}
+spec:
+  parentRefs:
+    - name: {{ $.Values.gateways.external.name }}
+      namespace: {{ $.Values.gateways.external.namespace }}
+      sectionName: {{ .listenerName }}
+  rules:
+    - backendRefs:
+        - name: {{ .service.name }}
+          {{- with .namespace }}
+          namespace: {{ . }}
+          {{- end }}
+          port: {{ .service.port }}
+{{- end }}

components/envoy-configs/values.schema.json

@@ -224,6 +224,76 @@
             ],
             "additionalProperties": false
           }
+        },
+        "tcp": {
+          "type": "array",
+          "description": "TCP routes for non-HTTP services (e.g., PostgreSQL, Redis)",
+          "items": {
+            "type": "object",
+            "properties": {
+              "name": {
+                "type": "string",
+                "description": "Name identifier for the TCPRoute resource"
+              },
+              "listenerName": {
+                "type": "string",
+                "description": "Name of the TCP listener on the gateway (must match)"
+              },
+              "gatewayPort": {
+                "type": "integer",
+                "minimum": 1,
+                "maximum": 65535,
+                "description": "Port exposed on the gateway for this TCP route"
+              },
+              "namespace": {
+                "type": "string",
+                "description": "Namespace of the backend service"
+              },
+              "service": {
+                "type": "object",
+                "description": "Kubernetes service backend configuration",
+                "properties": {
+                  "name": {
+                    "type": "string",
+                    "description": "Name of the Kubernetes service"
+                  },
+                  "port": {
+                    "type": "integer",
+                    "minimum": 1,
+                    "maximum": 65535,
+                    "description": "Port of the backend service"
+                  }
+                },
+                "required": [
+                  "name",
+                  "port"
+                ],
+                "additionalProperties": false
+              },
+              "selector": {
+                "type": "object",
+                "description": "Kubernetes-style label selector (key-value pairs)",
+                "additionalProperties": {
+                  "type": "string"
+                }
+              },
+              "from": {
+                "type": "string",
+                "enum": [
+                  "Same",
+                  "All",
+                  "Selector"
+                ],
+                "description": "Specifies where traffic can originate from"
+              }
+            },
+            "required": [
+              "listenerName",
+              "gatewayPort",
+              "service"
+            ],
+            "additionalProperties": false
+          }
         }
       }
     }

components/envoy-configs/values.yaml

@@ -2,3 +2,4 @@ gateways: {}
 routes:
   http: []
   tls: []
+  tcp: []

components/nautobot-worker/kustomization.yaml

@@ -0,0 +1,5 @@
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources: []

components/nautobot-worker/values.yaml

@@ -0,0 +1,67 @@
+# Nautobot Worker (site-level)
+#
+# Deploys only Celery workers that connect back to the global Nautobot
+# database and Redis. The web server is disabled because it lives on
+# the global cluster. Redis and PostgreSQL are disabled because the
+# workers reach the global instances over the network.
+#
+# The deploy repo for each site MUST provide:
+#   - ExternalSecrets for nautobot-django, nautobot-redis, nautobot-db,
+#     nautobot-custom-env, and dockerconfigjson-github-com
+#   - values.yaml overrides for nautobot.db.host and nautobot.redis.host
+#     pointing to the global cluster endpoints
+---
+
+# Disable the Nautobot web server — workers only
+nautobot:
+  enabled: false
+
+  db:
+    engine: "django.db.backends.postgresql"
+    # Override in deploy repo values to point at the global CNPG service
+    host: ""
+    port: "5432"
+    name: "app"
+    user: "app"
+    existingSecret: "nautobot-db"
+    existingSecretPasswordKey: "password"
+
+  django:
+    existingSecret: nautobot-django
+
+  redis:
+    # Override in deploy repo values to point at the global Redis service
+    host: ""
+    port: "6379"
+    ssl: false
+    username: ""
+
+celery:
+  enabled: true
+  concurrency: 2
+  replicaCount: 1
+  extraEnvVarsSecret:
+    - nautobot-django
+    - nautobot-custom-env
+  livenessProbe:
+    initialDelaySeconds: 60
+    periodSeconds: 120
+    timeoutSeconds: 60
+  readinessProbe:
+    initialDelaySeconds: 60
+    periodSeconds: 120
+    timeoutSeconds: 60
+
+# Do not deploy local Redis — use the global instance
+redis:
+  enabled: false
+
+# Do not deploy local PostgreSQL — use the global CNPG instance
+postgresql:
+  enabled: false
+
+ingress:
+  enabled: false
+
+metrics:
+  enabled: false

docs/deploy-guide/components/nautobot-worker.md

@@ -0,0 +1,58 @@
+---
+charts:
+- nautobot
+kustomize_paths:
+- components/nautobot-worker
+deploy_overrides:
+  helm:
+    mode: values
+  kustomize:
+    mode: second_source
+---
+
+# nautobot-worker
+
+Site-level Nautobot Celery workers that connect to the global Nautobot
+database and Redis. This allows sites to run their own worker pods for
+processing background tasks without deploying the full Nautobot web
+application.
+
+## Deployment Scope
+
+- Cluster scope: site
+- Values key: `site.nautobot_worker`
+- ArgoCD Application template: `charts/argocd-understack/templates/application-nautobot-worker.yaml`
+
+## How ArgoCD Builds It
+
+{{ component_argocd_builds() }}
+
+## How to Enable
+
+Enable this component in your site deployment values file:
+
+```yaml title="$CLUSTER_NAME/deploy.yaml"
+site:
+  nautobot_worker:
+    enabled: true
+```
+
+## Deployment Repo Content
+
+{{ secrets_disclaimer }}
+
+Required or commonly required items:
+
+- `values.yaml`: Override celery worker settings such as image, replica
+  count, concurrency, environment variables, and task queue assignments.
+- `nautobot-django` Secret: Provide a `NAUTOBOT_SECRET_KEY` value
+  (must match the global Nautobot instance).
+- `nautobot-cluster-app` Secret: Database credentials for the global
+  CloudNativePG cluster.
+
+Optional additions:
+
+- `nautobot-custom-env` Secret: Extra environment variables to inject
+  into the worker pods.
+- Additional `workers` entries in `values.yaml` to run dedicated
+  workers for specific Celery queues.

mkdocs.yml

@@ -190,6 +190,7 @@ nav:
       - deploy-guide/components/nautobot-site.md
       - deploy-guide/components/nautobot.md
       - deploy-guide/components/nautobotop.md
+      - deploy-guide/components/nautobot-worker.md
       - deploy-guide/components/neutron.md
       - deploy-guide/components/nova.md
       - deploy-guide/components/octavia.md