feat: enable site clusters to run Nautobot Celery workers locally

Nautobot currently runs entirely on the global cluster, including its
Celery workers. Sites that generate heavy background task load have no
way to offload that processing closer to where the work originates,
and a single global worker pool becomes a bottleneck as sites scale.

This adds a site-scoped ArgoCD Application that deploys only the
Celery worker portion of the Nautobot helm chart. The web server,
Redis, and PostgreSQL are all disabled because they remain on the
global cluster — site workers connect back to those shared services.

This lets operators scale worker capacity per-site independently,
run queue-specific workers closer to the hardware they manage, and
reduce cross-cluster task latency for site-driven automation.

Author: syedhaseebahmed

charts/argocd-understack/templates/application-nautobot-worker.yaml

@@ -0,0 +1,49 @@
+{{- if eq (include "understack.isEnabled" (list $.Values.site "nautobot_worker")) "true" }}
+---
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: {{ printf "%s-%s" $.Release.Name "nautobot-worker" }}
+  finalizers:
+  - resources-finalizer.argocd.argoproj.io
+  annotations:
+    argocd.argoproj.io/compare-options: ServerSideDiff=true,IncludeMutationWebhook=true
+spec:
+  destination:
+    namespace: nautobot
+    server: {{ $.Values.cluster_server }}
+  project: understack
+  sources:
+  - chart: nautobot
+    helm:
+      fileParameters:
+      - name: nautobot.config
+        path: $understack/components/nautobot/nautobot_config.py
+      ignoreMissingValueFiles: true
+      releaseName: nautobot-worker
+      valueFiles:
+      - $understack/components/nautobot-worker/values.yaml
+      - $deploy/{{ include "understack.deploy_path" $ }}/nautobot-worker/values.yaml
+    repoURL: https://nautobot.github.io/helm-charts/
+    targetRevision: 2.5.6
+  - path: components/nautobot-worker
+    ref: understack
+    repoURL: {{ include "understack.understack_url" $ }}
+    targetRevision: {{ include "understack.understack_ref" $ }}
+  - path: {{ include "understack.deploy_path" $ }}/nautobot-worker
+    ref: deploy
+    repoURL: {{ include "understack.deploy_url" $ }}
+    targetRevision: {{ include "understack.deploy_ref" $ }}
+  syncPolicy:
+    automated:
+      prune: true
+      selfHeal: true
+    managedNamespaceMetadata:
+      annotations:
+        argocd.argoproj.io/sync-options: Delete=false
+    syncOptions:
+    - CreateNamespace=true
+    - ServerSideApply=true
+    - RespectIgnoreDifferences=true
+    - ApplyOutOfSyncOnly=true
+{{- end }}

charts/argocd-understack/values.yaml

@@ -514,6 +514,12 @@ site:
     # @default -- false
     enabled: false
 
+  # -- Nautobot Celery workers (site-level, connects to global Nautobot)
+  nautobot_worker:
+    # -- Enable/disable deploying Nautobot workers at the site level
+    # @default -- false
+    enabled: false
+
   # -- Site-specific workflows and event handlers
   site_workflows:
     # -- Enable/disable deploying site workflows

components/nautobot-worker/kustomization.yaml

@@ -0,0 +1,5 @@
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources: []

components/nautobot-worker/values.yaml

@@ -0,0 +1,54 @@
+# Nautobot Worker (site-level)
+#
+# This deploys only Celery workers that connect back to the global
+# Nautobot's database and Redis. The web server, Redis, and PostgreSQL
+# are all disabled because they live on the global cluster.
+---
+
+# Disable the Nautobot web server — workers only
+nautobot:
+  enabled: false
+
+  db:
+    engine: "django.db.backends.postgresql"
+    # Points to the global cluster's CloudNativePG instance.
+    # Override host in your deploy repo values if the global DB
+    # endpoint differs per site.
+    host: "nautobot-cluster-rw.nautobot.svc"
+    name: "app"
+    user: "app"
+    existingSecret: "nautobot-cluster-app"
+    existingSecretPasswordKey: "password"
+
+  django:
+    existingSecret: nautobot-django
+
+celery:
+  enabled: true
+  concurrency: 2
+  replicaCount: 1
+  extraEnvVarsSecret:
+    - nautobot-django
+    - nautobot-custom-env
+  livenessProbe:
+    initialDelaySeconds: 60
+    periodSeconds: 120
+    timeoutSeconds: 60
+  readinessProbe:
+    initialDelaySeconds: 60
+    periodSeconds: 120
+    timeoutSeconds: 60
+
+# Workers use the global Redis — do not deploy a local instance
+redis:
+  enabled: false
+
+# Workers use the global PostgreSQL — do not deploy a local instance
+postgresql:
+  enabled: false
+
+ingress:
+  enabled: false
+
+metrics:
+  enabled: false

docs/deploy-guide/components/nautobot-worker.md

@@ -0,0 +1,58 @@
+---
+charts:
+- nautobot
+kustomize_paths:
+- components/nautobot-worker
+deploy_overrides:
+  helm:
+    mode: values
+  kustomize:
+    mode: second_source
+---
+
+# nautobot-worker
+
+Site-level Nautobot Celery workers that connect to the global Nautobot
+database and Redis. This allows sites to run their own worker pods for
+processing background tasks without deploying the full Nautobot web
+application.
+
+## Deployment Scope
+
+- Cluster scope: site
+- Values key: `site.nautobot_worker`
+- ArgoCD Application template: `charts/argocd-understack/templates/application-nautobot-worker.yaml`
+
+## How ArgoCD Builds It
+
+{{ component_argocd_builds() }}
+
+## How to Enable
+
+Enable this component in your site deployment values file:
+
+```yaml title="$CLUSTER_NAME/deploy.yaml"
+site:
+  nautobot_worker:
+    enabled: true
+```
+
+## Deployment Repo Content
+
+{{ secrets_disclaimer }}
+
+Required or commonly required items:
+
+- `values.yaml`: Override celery worker settings such as image, replica
+  count, concurrency, environment variables, and task queue assignments.
+- `nautobot-django` Secret: Provide a `NAUTOBOT_SECRET_KEY` value
+  (must match the global Nautobot instance).
+- `nautobot-cluster-app` Secret: Database credentials for the global
+  CloudNativePG cluster.
+
+Optional additions:
+
+- `nautobot-custom-env` Secret: Extra environment variables to inject
+  into the worker pods.
+- Additional `workers` entries in `values.yaml` to run dedicated
+  workers for specific Celery queues.

mkdocs.yml

@@ -190,6 +190,7 @@ nav:
       - deploy-guide/components/nautobot-site.md
       - deploy-guide/components/nautobot.md
       - deploy-guide/components/nautobotop.md
+      - deploy-guide/components/nautobot-worker.md
       - deploy-guide/components/neutron.md
       - deploy-guide/components/nova.md
       - deploy-guide/components/octavia.md