Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 12 additions & 0 deletions api/v2/condition_types.go
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,10 @@ const (
// (uninstall/rollback) due to a failure of the last release attempt against the
// latest desired state.
RemediatedCondition string = "Remediated"

// DriftedCondition represents the status of the Helm release drift detection,
// indicating that the deployed release has drifted from the desired state.
DriftedCondition string = "Drifted"
)

const (
Expand Down Expand Up @@ -79,4 +83,12 @@ const (
// DependencyNotReadyReason represents the fact that
// one of the dependencies is not ready.
DependencyNotReadyReason string = "DependencyNotReady"

// DriftDetectedReason represents the fact that drift has been detected in the
// Helm release compared to the expected state.
DriftDetectedReason string = "DriftDetected"

// NoDriftDetectedReason represents the fact that no drift has been detected in
// the Helm release compared to the expected state.
NoDriftDetectedReason string = "NoDriftDetected"
)
28 changes: 26 additions & 2 deletions docs/spec/v2/helmreleases.md
Original file line number Diff line number Diff line change
Expand Up @@ -1886,8 +1886,9 @@ A HelmRelease enters various states during its lifecycle, reflected as
[Kubernetes Conditions](https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md#typical-status-properties).
It can be [reconciling](#reconciling-helmrelease) when it is being processed by
the controller, it can be [ready](#ready-helmrelease) when the Helm release is
installed and up-to-date, or it can [fail](#failed-helmrelease) during
reconciliation.
installed and up-to-date, it can [fail](#failed-helmrelease) during
reconciliation, or it can be [drifted](#drifted-helmrelease) if the
drift detection mode is set to enabled/warn and there is a drift.

The HelmRelease API is compatible with the [kstatus specification](https://github.com/kubernetes-sigs/cli-utils/tree/master/pkg/kstatus),
and reports `Reconciling` and `Stalled` conditions where applicable to provide
Expand Down Expand Up @@ -1972,6 +1973,29 @@ HelmRelease's `.status.conditions`:
The `TestSuccess` Condition will retain a status value of `"True"` until the
next Helm install or upgrade occurs, or the Helm tests are disabled.

#### Drifted HelmRelease

The helm-controller marks the HelmRelease as _drifted_ when it has the following
characteristics:

- The HelmRelease have drift detection mode set to enabled or warn.
- There is a drift detected against the cluster state.

When the HelmRelease is "drifted", the controller sets a Condition with the
following attributes in the HelmRelease's `.status.conditions`:

- `type: Drifted`
- `status: "True"`
- `reason: DriftDetected`

When the HelmRelease have drift detection mode set to enabled or warn there
and there is no drift, the controller sets a Condition with the following
attributes in the HelmRelease's `.status.conditions`:

- `type: Drifted`
- `status: "False"`
- `reason: NoDriftDetected`

#### Failed HelmRelease

The helm-controller may get stuck trying to determine state or produce a Helm
Expand Down
17 changes: 13 additions & 4 deletions internal/reconcile/atomic_release.go
Original file line number Diff line number Diff line change
Expand Up @@ -188,6 +188,11 @@ func (r *AtomicRelease) Reconcile(ctx context.Context, req *Request) error {
}
return fmt.Errorf("atomic release canceled: %w", ctx.Err())
default:
// Remove the drifted condition if drift detection is disabled.
if conditions.Has(req.Object, v2.DriftedCondition) && !req.Object.GetDriftDetection().MustDetectChanges() {
conditions.Delete(req.Object, v2.DriftedCondition)
}

// Determine the current state of the Helm release.
log.V(logger.DebugLevel).Info("determining current state of Helm release")
state, err := DetermineReleaseState(ctx, r.configFactory, req, r.disallowedFieldManagers)
Expand Down Expand Up @@ -385,6 +390,10 @@ func (r *AtomicRelease) actionForState(ctx context.Context, req *Request, state
conditions.MarkTrue(req.Object, v2.ReleasedCondition, v2.UpgradeSucceededReason, "%s", msg)
}

if req.Object.GetDriftDetection().MustDetectChanges() {
conditions.MarkFalse(req.Object, v2.DriftedCondition, v2.NoDriftDetectedReason, "No drift detected against the cluster state")
}

return nil, nil
case ReleaseStatusLocked:
log.Info(msgWithReason("release locked", state.Reason))
Expand Down Expand Up @@ -440,10 +449,10 @@ func (r *AtomicRelease) actionForState(ctx context.Context, req *Request, state
}
}

r.eventRecorder.Eventf(req.Object, corev1.EventTypeWarning, "DriftDetected",
"Cluster state of release %s has drifted from the desired state:\n%s",
req.Object.Status.History.Latest().FullReleaseName(), diff.SummarizeDiffSet(state.Diff),
)
msg := fmt.Sprintf("Cluster state of release %s has drifted from the desired state:\n%s",
req.Object.Status.History.Latest().FullReleaseName(), diff.SummarizeDiffSet(state.Diff))
r.eventRecorder.Eventf(req.Object, corev1.EventTypeWarning, v2.DriftDetectedReason, msg)
conditions.MarkTrue(req.Object, v2.DriftedCondition, v2.DriftDetectedReason, "%s", msg)

if req.Object.GetDriftDetection().GetMode() == v2.DriftDetectionEnabled {
return NewCorrectClusterDrift(r.configFactory, r.eventRecorder, state.Diff, kube.ManagedFieldsManager), nil
Expand Down
192 changes: 184 additions & 8 deletions internal/reconcile/atomic_release_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -214,14 +214,15 @@ func TestAtomicRelease_Reconcile(t *testing.T) {

func TestAtomicRelease_Reconcile_Scenarios(t *testing.T) {
tests := []struct {
name string
releases func(namespace string) []*helmrelease.Release
spec func(spec *v2.HelmReleaseSpec)
status func(namespace string, releases []*helmrelease.Release) v2.HelmReleaseStatus
chart *helmchart.Chart
values map[string]interface{}
expectHistory func(releases []*helmrelease.Release) v2.Snapshots
wantErr error
name string
releases func(namespace string) []*helmrelease.Release
spec func(spec *v2.HelmReleaseSpec)
status func(namespace string, releases []*helmrelease.Release) v2.HelmReleaseStatus
chart *helmchart.Chart
values map[string]interface{}
expectHistory func(releases []*helmrelease.Release) v2.Snapshots
expectConditions []metav1.Condition
wantErr error
}{
{
name: "release is in-sync",
Expand Down Expand Up @@ -1161,6 +1162,42 @@ func TestAtomicRelease_Reconcile_Scenarios(t *testing.T) {
},
chart: testutil.BuildChart(),
},
{
name: "removes Drifted condition when drift detection is disabled",
releases: func(namespace string) []*helmrelease.Release {
return []*helmrelease.Release{
testutil.BuildRelease(&helmrelease.MockReleaseOptions{
Name: mockReleaseName,
Namespace: namespace,
Version: 1,
Chart: testutil.BuildChart(),
Status: helmrelease.StatusDeployed,
}, testutil.ReleaseWithConfig(nil)),
}
},
spec: func(spec *v2.HelmReleaseSpec) {
spec.DriftDetection = &v2.DriftDetection{
Mode: v2.DriftDetectionDisabled,
}
},
status: func(namespace string, releases []*helmrelease.Release) v2.HelmReleaseStatus {
return v2.HelmReleaseStatus{
History: v2.Snapshots{
release.ObservedToSnapshot(release.ObserveRelease(releases[0])),
},
Conditions: []metav1.Condition{
*conditions.TrueCondition(v2.DriftedCondition, v2.DriftDetectedReason, "drift detected"),
},
}
},
chart: testutil.BuildChart(),
expectHistory: func(releases []*helmrelease.Release) v2.Snapshots {
return v2.Snapshots{
release.ObservedToSnapshot(release.ObserveRelease(releases[0])),
}
},
expectConditions: []metav1.Condition{},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
Expand Down Expand Up @@ -1242,6 +1279,10 @@ func TestAtomicRelease_Reconcile_Scenarios(t *testing.T) {

g.Expect(req.Object.Status.History).To(testutil.Equal(tt.expectHistory(history)))
}

if tt.expectConditions != nil {
g.Expect(req.Object.Status.Conditions).To(conditions.MatchConditions(tt.expectConditions))
}
})
}
}
Expand Down Expand Up @@ -1552,6 +1593,63 @@ func TestAtomicRelease_actionForState(t *testing.T) {
*conditions.FalseCondition(meta.ReadyCondition, v2.UpgradeFailedReason, "upgrade failed"),
},
},
{
name: "in-sync release with drift detection enabled sets Drifted condition to false",
spec: func(spec *v2.HelmReleaseSpec) {
spec.DriftDetection = &v2.DriftDetection{
Mode: v2.DriftDetectionEnabled,
}
},
status: func(releases []*helmrelease.Release) v2.HelmReleaseStatus {
return v2.HelmReleaseStatus{
History: v2.Snapshots{
{Version: 1},
},
}
},
state: ReleaseState{Status: ReleaseStatusInSync},
want: nil,
assertConditions: []metav1.Condition{
*conditions.FalseCondition(v2.DriftedCondition, v2.NoDriftDetectedReason, "No drift detected against the cluster state"),
},
},
{
name: "in-sync release with drift detection warn sets Drifted condition to false",
spec: func(spec *v2.HelmReleaseSpec) {
spec.DriftDetection = &v2.DriftDetection{
Mode: v2.DriftDetectionWarn,
}
},
status: func(releases []*helmrelease.Release) v2.HelmReleaseStatus {
return v2.HelmReleaseStatus{
History: v2.Snapshots{
{Version: 1},
},
}
},
state: ReleaseState{Status: ReleaseStatusInSync},
want: nil,
assertConditions: []metav1.Condition{
*conditions.FalseCondition(v2.DriftedCondition, v2.NoDriftDetectedReason, "No drift detected against the cluster state"),
},
},
{
name: "in-sync release with drift detection disabled does not set Drifted condition",
spec: func(spec *v2.HelmReleaseSpec) {
spec.DriftDetection = &v2.DriftDetection{
Mode: v2.DriftDetectionDisabled,
}
},
status: func(releases []*helmrelease.Release) v2.HelmReleaseStatus {
return v2.HelmReleaseStatus{
History: v2.Snapshots{
{Version: 1},
},
}
},
state: ReleaseState{Status: ReleaseStatusInSync},
want: nil,
},
{
name: "locked release triggers unlock action",
state: ReleaseState{Status: ReleaseStatusLocked},
Expand Down Expand Up @@ -1634,6 +1732,9 @@ func TestAtomicRelease_actionForState(t *testing.T) {
"Deployment/something/mock removed",
),
},
assertConditions: []metav1.Condition{
*conditions.TrueCondition(v2.DriftedCondition, v2.DriftDetectedReason, "Cluster state of release mock-ns/mock-release.v1 has drifted from the desired state:\nDeployment/something/mock removed"),
},
},
{
name: "drifted release only triggers event if mode is warn",
Expand Down Expand Up @@ -1703,6 +1804,81 @@ func TestAtomicRelease_actionForState(t *testing.T) {
"Deployment/something/mock changed (0 additions, 1 changes, 0 removals)",
),
},
assertConditions: []metav1.Condition{
*conditions.TrueCondition(v2.DriftedCondition, v2.DriftDetectedReason, "Cluster state of release mock-ns/mock-release.v1 has drifted from the desired state:\nDeployment/something/mock changed (0 additions, 1 changes, 0 removals)"),
},
},
{
name: "drifted release sets Drifted condition if mode is warn",
spec: func(spec *v2.HelmReleaseSpec) {
spec.DriftDetection = &v2.DriftDetection{
Mode: v2.DriftDetectionWarn,
}
},
status: func(releases []*helmrelease.Release) v2.HelmReleaseStatus {
return v2.HelmReleaseStatus{
History: v2.Snapshots{
{
Name: mockReleaseName,
Namespace: mockReleaseNamespace,
Version: 1,
},
},
}
},
state: ReleaseState{Status: ReleaseStatusDrifted, Diff: jsondiff.DiffSet{
{
Type: jsondiff.DiffTypeUpdate,
DesiredObject: &unstructured.Unstructured{
Object: map[string]interface{}{
"apiVersion": "apps/v1",
"kind": "Deployment",
"metadata": map[string]interface{}{
"name": "mock",
"namespace": "something",
},
"spec": map[string]interface{}{
"replicas": 2,
},
},
},
ClusterObject: &unstructured.Unstructured{
Object: map[string]interface{}{
"apiVersion": "apps/v1",
"kind": "Deployment",
"metadata": map[string]interface{}{
"name": "mock",
"namespace": "something",
},
"spec": map[string]interface{}{
"replicas": 1,
},
},
},
Patch: extjsondiff.Patch{
{
Type: extjsondiff.OperationReplace,
Path: "/spec/replicas",
OldValue: 1,
Value: 2,
},
},
},
}},
want: nil,
wantErr: nil,
wantEvent: &corev1.Event{
Reason: "DriftDetected",
Type: corev1.EventTypeWarning,
Message: fmt.Sprintf(
"Cluster state of release %s has drifted from the desired state:\n%s",
mockReleaseNamespace+"/"+mockReleaseName+".v1",
"Deployment/something/mock changed (0 additions, 1 changes, 0 removals)",
),
},
assertConditions: []metav1.Condition{
*conditions.TrueCondition(v2.DriftedCondition, v2.DriftDetectedReason, "Cluster state of release mock-ns/mock-release.v1 has drifted from the desired state:\nDeployment/something/mock changed (0 additions, 1 changes, 0 removals)"),
},
},
{
name: "out-of-sync release triggers upgrade",
Expand Down