feat(api): add stable build ID override via spec.workerOptions.customBuildID

api/v1alpha1/worker_types.go

@@ -27,6 +27,24 @@ type WorkerOptions struct {
 	// The Temporal namespace for the worker to connect to.
 	// +kubebuilder:validation:MinLength=1
 	TemporalNamespace string `json:"temporalNamespace"`
+	// CustomBuildID optionally overrides the auto-generated build ID for this worker deployment.
+	// When set, the controller uses this value instead of computing a build ID from the
+	// pod template hash. This enables rolling updates for non-workflow code changes
+	// (bug fixes, config changes) while preserving the same build ID.
+	//
+	// WARNING: Using a custom build ID requires careful management. If workflow code changes
+	// but CustomBuildID stays the same, pinned workflows may execute on workers running incompatible
+	// code. Only use this when you have a reliable way to detect changes in your workflow
+	// definitions (e.g., hashing workflow source files in CI/CD).
+	//
+	// When the CustomBuildID is stable but pod template spec changes, the controller triggers
+	// a rolling update instead of creating a new deployment version. The controller uses
+	// a hash of the user-provided pod template spec to detect ANY changes, including
+	// container images, env vars, commands, volumes, resources, and all other fields.
+	// +optional
+	// +kubebuilder:validation:MaxLength=63
+	// +kubebuilder:validation:Pattern=`^[a-zA-Z0-9]([a-zA-Z0-9._-]*[a-zA-Z0-9])?$`
+	CustomBuildID string `json:"customBuildID,omitempty"`
 }
 
 // TemporalWorkerDeploymentSpec defines the desired state of TemporalWorkerDeployment

helm/temporal-worker-controller/crds/temporal.io_temporalworkerdeployments.yaml

@@ -64,7 +64,6 @@ spec:
                   gate:
                     properties:
                       input:
-                        type: object
                         x-kubernetes-preserve-unknown-fields: true
                       inputFrom:
                         properties:
@@ -73,25 +72,27 @@ spec:
                               key:
                                 type: string
                               name:
+                                default: ""
                                 type: string
                               optional:
                                 type: boolean
                             required:
                             - key
-                            - name
                             type: object
+                            x-kubernetes-map-type: atomic
                           secretKeyRef:
                             properties:
                               key:
                                 type: string
                               name:
+                                default: ""
                                 type: string
                               optional:
                                 type: boolean
                             required:
                             - key
-                            - name
                             type: object
+                            x-kubernetes-map-type: atomic
                         type: object
                       workflowType:
                         type: string
@@ -3949,6 +3950,10 @@ spec:
                     required:
                     - name
                     type: object
+                  customBuildID:
+                    maxLength: 63
+                    pattern: ^[a-zA-Z0-9]([a-zA-Z0-9._-]*[a-zA-Z0-9])?$
+                    type: string
                   temporalNamespace:
                     minLength: 1
                     type: string

internal/k8s/deployments.go

@@ -11,7 +11,9 @@ import (
 	"fmt"
 	"regexp"
 	"sort"
+	"strings"
 
+	"github.com/davecgh/go-spew/spew"
 	"github.com/distribution/reference"
 	temporaliov1alpha1 "github.com/temporalio/temporal-worker-controller/api/v1alpha1"
 	"github.com/temporalio/temporal-worker-controller/internal/controller/k8s.io/utils"
@@ -32,6 +34,7 @@ const (
 	ResourceNameSeparator         = "-"
 	MaxBuildIdLen                 = 63
 	ConnectionSpecHashAnnotation  = "temporal.io/connection-spec-hash"
+	PodTemplateSpecHashAnnotation = "temporal.io/pod-template-spec-hash"
 )
 
 // DeploymentState represents the Kubernetes state of all deployments for a temporal worker deployment
@@ -112,6 +115,15 @@ func NewObjectRef(obj client.Object) *corev1.ObjectReference {
 }
 
 func ComputeBuildID(w *temporaliov1alpha1.TemporalWorkerDeployment) string {
+	// Check for user-provided build ID in spec.workerOptions.customBuildID
+	if override := w.Spec.WorkerOptions.CustomBuildID; override != "" {
+		cleaned := cleanBuildID(override)
+		if cleaned != "" {
+			return TruncateString(cleaned, MaxBuildIdLen)
+		}
+		// Fall through to default hash-based generation if buildID is invalid after cleaning
+	}
+
 	if containers := w.Spec.Template.Spec.Containers; len(containers) > 0 {
 		if img := containers[0].Image; img != "" {
 			shortHashSuffix := ResourceNameSeparator + utils.ComputeHash(&w.Spec.Template, nil, true)
@@ -177,9 +189,12 @@ func CleanStringForDNS(s string) string {
 //
 // Temporal build IDs only need to be ASCII.
 func cleanBuildID(s string) string {
-	// Keep only letters, numbers, dashes, and dots.
+	// Keep only letters, numbers, dashes, underscores, and dots.
 	re := regexp.MustCompile(`[^a-zA-Z0-9-._]+`)
-	return re.ReplaceAllString(s, ResourceNameSeparator)
+	s = re.ReplaceAllString(s, ResourceNameSeparator)
+	// Trim leading/trailing separators to comply with K8s label requirements
+	// (must begin and end with alphanumeric character)
+	return strings.Trim(s, "-._")
 }
 
 // NewDeploymentWithOwnerRef creates a new deployment resource, including owner references
@@ -207,6 +222,108 @@ func NewDeploymentWithOwnerRef(
 
 	podSpec := spec.Template.Spec.DeepCopy()
 
+	// Apply controller-managed environment variables and volume mounts
+	ApplyControllerPodSpecModifications(podSpec, connection, spec.WorkerOptions.TemporalNamespace, workerDeploymentName, buildID)
+
+	// Build pod annotations
+	podAnnotations := make(map[string]string)
+	for k, v := range spec.Template.Annotations {
+		podAnnotations[k] = v
+	}
+	podAnnotations[ConnectionSpecHashAnnotation] = ComputeConnectionSpecHash(connection)
+	// Store hash of user-provided pod template spec BEFORE controller modifications
+	// This enables drift detection when build ID is stable
+	podAnnotations[PodTemplateSpecHashAnnotation] = ComputePodTemplateSpecHash(spec.Template)
+	blockOwnerDeletion := true
+
+	return &appsv1.Deployment{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:                       ComputeVersionedDeploymentName(objectMeta.Name, buildID),
+			Namespace:                  objectMeta.Namespace,
+			DeletionGracePeriodSeconds: nil,
+			Labels:                     selectorLabels,
+			Annotations:                spec.Template.Annotations,
+			OwnerReferences: []metav1.OwnerReference{{
+				APIVersion:         typeMeta.APIVersion,
+				Kind:               typeMeta.Kind,
+				Name:               objectMeta.Name,
+				UID:                objectMeta.UID,
+				BlockOwnerDeletion: &blockOwnerDeletion,
+				Controller:         nil,
+			}},
+			// TODO(jlegrone): Add finalizer managed by the controller in order to prevent
+			//                 deleting deployments that are still reachable.
+		},
+		Spec: appsv1.DeploymentSpec{
+			Replicas: spec.Replicas,
+			Selector: &metav1.LabelSelector{
+				MatchLabels: selectorLabels,
+			},
+			Template: corev1.PodTemplateSpec{
+				ObjectMeta: metav1.ObjectMeta{
+					Labels:      podLabels,
+					Annotations: podAnnotations,
+				},
+				Spec: *podSpec,
+			},
+			MinReadySeconds: spec.MinReadySeconds,
+		},
+	}
+}
+
+// TODO (Shivam): Change hash when secret name is updated as well.
+func ComputeConnectionSpecHash(connection temporaliov1alpha1.TemporalConnectionSpec) string {
+	// HostPort is required, but MutualTLSSecret can be empty for non-mTLS connections
+	if connection.HostPort == "" {
+		return ""
+	}
+
+	hasher := sha256.New()
+
+	// Hash connection spec fields in deterministic order
+	_, _ = hasher.Write([]byte(connection.HostPort))
+	if connection.MutualTLSSecretRef != nil {
+		_, _ = hasher.Write([]byte(connection.MutualTLSSecretRef.Name))
+	} else if connection.APIKeySecretRef != nil {
+		_, _ = hasher.Write([]byte(connection.APIKeySecretRef.Name))
+	}
+
+	return hex.EncodeToString(hasher.Sum(nil))
+}
+
+// ComputePodTemplateSpecHash computes a SHA256 hash of the user-provided pod template spec.
+// This hash is used to detect drift when the build ID is stable but the pod spec has changed.
+// The hash captures ALL user-controllable fields in the pod template spec.
+func ComputePodTemplateSpecHash(template corev1.PodTemplateSpec) string {
+	hasher := sha256.New()
+
+	// Use spew to get a deterministic string representation of the entire struct.
+	// This captures ALL fields including env vars, commands, volumes, etc.
+	// The config MUST NOT be changed because that could change the result of a hash operation.
+	printer := &spew.ConfigState{
+		Indent:                  " ",
+		SortKeys:                true,
+		DisableMethods:          true,
+		SpewKeys:                true,
+		DisablePointerAddresses: true,
+		DisableCapacities:       true,
+	}
+
+	_, _ = hasher.Write([]byte(printer.Sprintf("%#v", template)))
+
+	return hex.EncodeToString(hasher.Sum(nil))
+}
+
+// ApplyControllerPodSpecModifications applies controller-managed environment variables and
+// volume mounts to a pod spec. This is used both when creating new deployments and when
+// updating existing deployments for drift detection.
+func ApplyControllerPodSpecModifications(
+	podSpec *corev1.PodSpec,
+	connection temporaliov1alpha1.TemporalConnectionSpec,
+	temporalNamespace string,
+	workerDeploymentName string,
+	buildID string,
+) {
 	// Add environment variables to containers
 	for i, container := range podSpec.Containers {
 		container.Env = append(container.Env,
@@ -216,7 +333,7 @@ func NewDeploymentWithOwnerRef(
 			},
 			corev1.EnvVar{
 				Name:  "TEMPORAL_NAMESPACE",
-				Value: spec.WorkerOptions.TemporalNamespace,
+				Value: temporalNamespace,
 			},
 			corev1.EnvVar{
 				Name:  "TEMPORAL_DEPLOYMENT_NAME",
@@ -274,68 +391,6 @@ func NewDeploymentWithOwnerRef(
 			podSpec.Containers[i] = container
 		}
 	}
-
-	// Build pod annotations
-	podAnnotations := make(map[string]string)
-	for k, v := range spec.Template.Annotations {
-		podAnnotations[k] = v
-	}
-	podAnnotations[ConnectionSpecHashAnnotation] = ComputeConnectionSpecHash(connection)
-	blockOwnerDeletion := true
-
-	return &appsv1.Deployment{
-		ObjectMeta: metav1.ObjectMeta{
-			Name:                       ComputeVersionedDeploymentName(objectMeta.Name, buildID),
-			Namespace:                  objectMeta.Namespace,
-			DeletionGracePeriodSeconds: nil,
-			Labels:                     selectorLabels,
-			Annotations:                spec.Template.Annotations,
-			OwnerReferences: []metav1.OwnerReference{{
-				APIVersion:         typeMeta.APIVersion,
-				Kind:               typeMeta.Kind,
-				Name:               objectMeta.Name,
-				UID:                objectMeta.UID,
-				BlockOwnerDeletion: &blockOwnerDeletion,
-				Controller:         nil,
-			}},
-			// TODO(jlegrone): Add finalizer managed by the controller in order to prevent
-			//                 deleting deployments that are still reachable.
-		},
-		Spec: appsv1.DeploymentSpec{
-			Replicas: spec.Replicas,
-			Selector: &metav1.LabelSelector{
-				MatchLabels: selectorLabels,
-			},
-			Template: corev1.PodTemplateSpec{
-				ObjectMeta: metav1.ObjectMeta{
-					Labels:      podLabels,
-					Annotations: podAnnotations,
-				},
-				Spec: *podSpec,
-			},
-			MinReadySeconds: spec.MinReadySeconds,
-		},
-	}
-}
-
-// TODO (Shivam): Change hash when secret name is updated as well.
-func ComputeConnectionSpecHash(connection temporaliov1alpha1.TemporalConnectionSpec) string {
-	// HostPort is required, but MutualTLSSecret can be empty for non-mTLS connections
-	if connection.HostPort == "" {
-		return ""
-	}
-
-	hasher := sha256.New()
-
-	// Hash connection spec fields in deterministic order
-	_, _ = hasher.Write([]byte(connection.HostPort))
-	if connection.MutualTLSSecretRef != nil {
-		_, _ = hasher.Write([]byte(connection.MutualTLSSecretRef.Name))
-	} else if connection.APIKeySecretRef != nil {
-		_, _ = hasher.Write([]byte(connection.APIKeySecretRef.Name))
-	}
-
-	return hex.EncodeToString(hasher.Sum(nil))
 }
 
 func NewDeploymentWithControllerRef(