feat(hooks): run post-proposal hooks even when timelock execution fails

.changeset/calm-wombats-sin.md

@@ -0,0 +1,5 @@
+---
+"chainlink-deployments-framework": minor
+---
+
+feat: run post-proposal hooks even when the timelock execution fails

engine/cld/changeset/hooks.go

@@ -85,6 +85,7 @@ type PostProposalHookParams struct {
 	Proposal     *mcms.TimelockProposal
 	Config       any
 	Reports      []MCMSTimelockExecuteReport
+	Err          string
 
 	// Deprecated: use `Config` instead. Will be removed in a future version.
 	Input any

engine/cld/changeset/mcms.go

@@ -83,7 +83,7 @@ func (*EVMForkContext) ChainFamily() string {
 //  2. Global post-proposal-hooks
 func (r *ChangesetsRegistry) RunProposalHooks(
 	key string, e fdeployment.Environment, proposal *mcms.TimelockProposal, input, config any,
-	reports []MCMSTimelockExecuteReport, forkCtx ForkContext,
+	reports []MCMSTimelockExecuteReport, execError string, forkCtx ForkContext,
 ) error {
 	applySnapshot, err := r.getApplySnapshot(key)
 	if err != nil {
@@ -112,6 +112,7 @@ func (r *ChangesetsRegistry) RunProposalHooks(
 		Input:        input,
 		Config:       config,
 		Reports:      reports,
+		Err:          execError,
 	}
 
 	for _, h := range applySnapshot.registryEntry.postProposalHooks {

engine/cld/changeset/mcms_test.go

@@ -148,7 +148,7 @@ func Test_RunProposalHooks(t *testing.T) {
 			execLogs := []string{}
 			registry := tt.setup(&execLogs)
 
-			err := registry.RunProposalHooks(tt.key, hookTestEnv(t), &mcms.TimelockProposal{}, nil, nil, nil, nil)
+			err := registry.RunProposalHooks(tt.key, hookTestEnv(t), &mcms.TimelockProposal{}, nil, nil, nil, "", nil)
 
 			if tt.wantErr == "" {
 				require.NoError(t, err)
@@ -182,7 +182,7 @@ func Test_RunProposalHooks_HookReceivesCorrectParams(t *testing.T) {
 		}},
 	}
 
-	err := r.RunProposalHooks("test-cs", hookTestEnv(t), proposal, input, config, reports, nil)
+	err := r.RunProposalHooks("test-cs", hookTestEnv(t), proposal, input, config, reports, "", nil)
 	require.NoError(t, err)
 
 	expectedParams := PostProposalHookParams{

engine/cld/changeset/registry_test.go

@@ -964,7 +964,7 @@ func Test_FluentAPI_HooksExtractedByAdd(t *testing.T) {
 		require.Len(t, entry.postProposalHooks, 1, "Add should extract post-proposal-hooks via hookCarrier")
 		require.Equal(t, "proposal-hook", entry.postProposalHooks[0].Name)
 
-		err := r.RunProposalHooks("test-cs", hookTestEnv(t), nil, "input", "config", nil, nil)
+		err := r.RunProposalHooks("test-cs", hookTestEnv(t), nil, "input", "config", nil, "", nil)
 		require.NoError(t, err)
 		require.Equal(t, []string{"proposal"}, hookExecutions)
 	})
@@ -985,7 +985,7 @@ func Test_FluentAPI_HooksExtractedByAdd(t *testing.T) {
 		require.Len(t, entry.postProposalHooks, 1)
 		require.Equal(t, "proposal-hook", entry.postProposalHooks[0].Name)
 
-		err := r.RunProposalHooks("test-cs", hookTestEnv(t), nil, "input", "config", nil, nil)
+		err := r.RunProposalHooks("test-cs", hookTestEnv(t), nil, "input", "config", nil, "", nil)
 		require.NoError(t, err)
 		require.Equal(t, []string{"proposal"}, hookExecutions)
 	})

engine/cld/commands/mcms/cmd_execute_fork.go

@@ -177,10 +177,6 @@ func executeFork(
 	if len(chainConfig.HTTPRPCs) == 0 {
 		return fmt.Errorf("no rpcs loaded in forked environment for chain %d (fork tests require public RPCs)", cfg.chainSelector)
 	}
-	forkClient, ok := cfg.forkedEnv.ForkClients[cfg.chainSelector]
-	if !ok {
-		return fmt.Errorf("failed to get fork client for chain %d", cfg.chainSelector)
-	}
 
 	// zkSync VM chains (zkSync Era, Lens, Cronos zkEVM, etc.) require anvil-zksync,
 	// not standard Anvil. Derive this from the loaded chain which is set by the chain
@@ -260,8 +256,14 @@ func executeFork(
 	}
 
 	lggr.Info("Executing timelock chain command")
-	reports, err := timelockExecuteChainCommand(ctx, lggr, cfg)
-	if err != nil {
+	reports, execErr := timelockExecuteChainCommand(ctx, lggr, cfg)
+
+	herr := runPostProposalHooks(lggr, cfg, mcmsCfg, reports, errorString(execErr))
+	if herr != nil {
+		lggr.Warnw("failed to run proposal hooks", "err", herr)
+	}
+
+	if execErr != nil {
 		lggr.Warnw("Timelock.execute() - failure; starting calling individual ops for debugging", "err", err)
 		if derr := diagnoseTimelockRevert(ctx, lggr, anvilClient.URL, cfg.chainSelector, cfg.timelockProposal.Operations,
 			timelockAddress, cfg.env.ExistingAddresses, cfg.proposalCtx); derr != nil { //nolint:staticcheck
@@ -272,16 +274,33 @@ func executeFork(
 
 		return fmt.Errorf("failed to timelock execute chain: %w", err)
 	}
+
 	lggr.Info("Timelock.execute() - success")
 
+	return nil
+}
+
+func runPostProposalHooks(
+	lggr logger.Logger, forkCfg *forkConfig, mcmsCfg Config, reports []cldfchangeset.MCMSTimelockExecuteReport,
+	execError string,
+) error {
 	if mcmsCfg.LoadChangesets == nil {
 		lggr.Debug("LoadChangesets function not set in mcms config; skipping proposal hooks")
 		return nil
 	}
+	forkClient, ok := forkCfg.forkedEnv.ForkClients[forkCfg.chainSelector]
+	if !ok {
+		return fmt.Errorf("failed to get fork client for chain %d", forkCfg.chainSelector)
+	}
+	chainConfig, ok := forkCfg.forkedEnv.ChainConfigs[forkCfg.chainSelector]
+	if !ok {
+		return fmt.Errorf("failed to get forked env's chain config for chain %d", forkCfg.chainSelector)
+	}
 
 	forkContext := &cldfchangeset.EVMForkContext{ChainConfig: chainConfig, Client: forkClient}
-	cfg.env.Name = cfg.envStr // ensure hooks load the correct env config for the fork
-	err = runHooksInternal(mcmsCfg, cfg.env, cfg.timelockProposal, reports, forkContext)
+	forkCfg.env.Name = forkCfg.envStr // ensure hooks load the correct env config for the fork
+
+	err := runHooksInternal(mcmsCfg, forkCfg.env, forkCfg.timelockProposal, reports, execError, forkContext)
 	if err != nil {
 		return fmt.Errorf("failed to run post-proposal hooks: %w", err)
 	}
@@ -486,3 +505,11 @@ func (c *loggingRPCClient) SendTransaction(ctx context.Context, tx *gethtypes.Tr
 
 	return c.OnchainClient.SendTransaction(ctx, tx)
 }
+
+func errorString(err error) string {
+	if err == nil {
+		return ""
+	}
+
+	return err.Error()
+}

engine/cld/commands/mcms/cmd_run_hooks.go

@@ -42,6 +42,7 @@ type runHooksFlags struct {
 	proposalKind  string
 	chainSelector uint64
 	reports       []cldfchangeset.MCMSTimelockExecuteReport
+	error         string
 }
 
 type proposalMetadata struct {
@@ -78,6 +79,7 @@ func newRunProposalHooksCmd(cfg Config) *cobra.Command {
 				proposalKind:  flags.MustString(cmd.Flags().GetString("proposalKind")),
 				chainSelector: flags.MustUint64(cmd.Flags().GetUint64("selector")),
 				reports:       reports,
+				error:         flags.MustString(cmd.Flags().GetString("error")),
 			}
 
 			return runHooks(cmd.Context(), cfg, f)
@@ -88,7 +90,8 @@ func newRunProposalHooksCmd(cfg Config) *cobra.Command {
 	flags.Proposal(cmd)
 	flags.ProposalKind(cmd, string(mcmstypes.KindTimelockProposal))
 	flags.ChainSelector(cmd, true)
-	cmd.Flags().String("report", "", "File with timelock execution report.")
+	cmd.Flags().String("report", "", "File with timelock execution report (required).")
+	cmd.Flags().String("error", "", "The error message, in case the timelock execution failed.")
 	_ = cmd.MarkFlagRequired("report")
 
 	return cmd
@@ -114,14 +117,15 @@ func runHooks(ctx context.Context, cfg Config, hFlags runHooksFlags) error {
 		return errors.New("expected proposal to be a TimelockProposal")
 	}
 
-	return runHooksInternal(cfg, proposalCfg.Env, proposalCfg.TimelockProposal, hFlags.reports, nil)
+	return runHooksInternal(cfg, proposalCfg.Env, proposalCfg.TimelockProposal, hFlags.reports, hFlags.error, nil)
 }
 
 func runHooksInternal(
 	cfg Config,
 	env cldf.Environment,
 	timelockProposal *mcms.TimelockProposal,
 	reports []cldfchangeset.MCMSTimelockExecuteReport,
+	execError string,
 	forkCtx cldfchangeset.ForkContext,
 ) error {
 	if cfg.LoadChangesets == nil {
@@ -145,7 +149,7 @@ func runHooksInternal(
 		})
 
 		herr := changesetRegistry.RunProposalHooks(changeset.Name, env, timelockProposal,
-			changeset.Input, changeset.Config, changesetReports, forkCtx)
+			changeset.Input, changeset.Config, changesetReports, execError, forkCtx)
 		if herr != nil {
 			return fmt.Errorf("proposal hook for changeset %q failed: %w", changeset.Name, herr)
 		}

engine/cld/commands/mcms/operations.go

@@ -196,51 +196,53 @@ func timelockExecuteChainCommand(
 
 	reports := []cldfchangeset.MCMSTimelockExecuteReport{}
 	for i := range cfg.timelockProposal.Operations {
-		if uint64(cfg.timelockProposal.Operations[i].ChainSelector) == cfg.chainSelector {
-			// Check if operation is done, if so, skip it
-			if err := executable.IsOperationDone(ctx, i); err == nil {
-				lggr.Warnf("Operation %d is already done, skipping...\n", i)
-
-				continue
-			}
-
-			if err := executable.IsOperationReady(ctx, i); err != nil {
-				return nil, fmt.Errorf("operation %d is not ready to be executed: %w", i, err)
-			}
+		if uint64(cfg.timelockProposal.Operations[i].ChainSelector) != cfg.chainSelector {
+			continue
+		}
 
-			timestamp := options.clockFn()
+		// Check if operation is done, if so, skip it
+		if err = executable.IsOperationDone(ctx, i); err == nil {
+			lggr.Warnf("Operation %d is already done, skipping...\n", i)
+			continue
+		}
 
-			result, err := executable.Execute(ctx, i, executeOptions...)
-			if err != nil {
-				return nil, fmt.Errorf("failed to execute operation %d: %w", i, err)
-			}
+		if err = executable.IsOperationReady(ctx, i); err != nil {
+			return nil, fmt.Errorf("operation %d is not ready to be executed: %w", i, err)
+		}
 
-			reports = append(reports, cldfchangeset.MCMSTimelockExecuteReport{
-				ID:        options.idFn(),
-				Type:      cldfchangeset.MCMSTimelockExecuteReportType,
-				Status:    "SUCCESS",
-				Timestamp: timestamp,
-				Input: cldfchangeset.MCMSTimelockExecuteReportInput{
-					Index:            i,
-					ChainSelector:    cfg.chainSelector,
-					OperationID:      operationIDs[i],
-					TimelockAddress:  timelockAddress,
-					MCMAddress:       chainMetadata.MCMAddress,
-					AdditionalFields: chainMetadata.AdditionalFields,
-					Changeset:        findChangeset(cfg.timelockProposal, i),
-				},
-				Output: cldfchangeset.MCMSTimelockExecuteReportOutput{
-					TransactionResult: result,
-				},
-			})
-
-			err = confirmTransaction(ctx, lggr, result, cfg)
-			if err != nil {
-				return nil, fmt.Errorf("failed to confirm execute transaction: %w", err)
-			}
+		timestamp := options.clockFn()
+
+		reports = append(reports, cldfchangeset.MCMSTimelockExecuteReport{
+			ID:        options.idFn(),
+			Status:    "SUCCESS",
+			Type:      cldfchangeset.MCMSTimelockExecuteReportType,
+			Timestamp: timestamp,
+			Input: cldfchangeset.MCMSTimelockExecuteReportInput{
+				Index:            i,
+				ChainSelector:    cfg.chainSelector,
+				OperationID:      operationIDs[i],
+				TimelockAddress:  timelockAddress,
+				MCMAddress:       chainMetadata.MCMAddress,
+				AdditionalFields: chainMetadata.AdditionalFields,
+				Changeset:        findChangeset(cfg.timelockProposal, i),
+			},
+			Output: cldfchangeset.MCMSTimelockExecuteReportOutput{},
+		})
+		report := &reports[len(reports)-1]
+
+		report.Output.TransactionResult, err = executable.Execute(ctx, i, executeOptions...)
+		if err != nil {
+			report.Status, report.Error = "FAILURE", err.Error()
+			return reports, fmt.Errorf("failed to execute operation %d: %w", i, err)
+		}
 
-			lggr.Infof("Operation %d executed successfully: %s\n", i, result)
+		err = confirmTransaction(ctx, lggr, report.Output.TransactionResult, cfg)
+		if err != nil {
+			report.Status, report.Error = "FAILURE", err.Error()
+			return reports, fmt.Errorf("failed to confirm execute transaction: %w", err)
 		}
+
+		lggr.Infof("Operation %d executed successfully: %s\n", i, report.Output.TransactionResult)
 	}
 
 	lggr.Infof("All operations executed successfully")