[DX-3449] parallel cre regression tests

.github/workflows/cre-regression-system-tests.yaml

@@ -183,6 +183,11 @@ jobs:
         env:
           TEST_NAME: ${{ matrix.tests.test_name }}
           TEST_TIMEOUT: 30m
+          PARALLEL_COUNT: "10"
+          # parallelisation flags for tests
+          # fanout is necessary, because multiple tests would otherwise start chip test sinks at the same port, causing conflicts
+          CRE_TEST_PARALLEL_ENABLED: "true"
+          CRE_TEST_CHIP_SINK_FANOUT_ENABLED: "true"
         run: |
           echo "Starting test: '${TEST_NAME}'"
           echo "⚠️⚠️⚠️ Add 'skip-e2e-regression' label to skip this step if necessary ⚠️⚠️⚠️"
@@ -192,7 +197,7 @@ jobs:
             --junitfile=/tmp/junit-report-regression.xml \
             --format=github-actions \
             -- \
-            -v -run "^(${TEST_NAME})$" -timeout ${TEST_TIMEOUT} -count=1 -parallel=1 \
+            -v -run "^(${TEST_NAME})$" -timeout ${TEST_TIMEOUT} -count=1 -parallel=${PARALLEL_COUNT} \
             github.com/smartcontractkit/chainlink/system-tests/tests/regression/cre
 
           echo "⚠️⚠️⚠️ Add 'skip-e2e-regression' label to skip this step if necessary ⚠️⚠️⚠️"

system-tests/lib/infra/docker.go

@@ -62,7 +62,7 @@ func PrintFailedContainerLogs(logger zerolog.Logger, logLinesCount uint64) {
 
 		content = strings.TrimSpace(content)
 		if len(content) > 0 {
-			logger.Info().Str("Container", cName).Msgf("Last 100 lines of logs")
+			logger.Info().Str("Container", cName).Msgf("Last %d lines of logs", logLinesCount)
 			fmt.Println(text.RedText("%s\n", content))
 		}
 		_ = ioReader.Close() // can't do much about the error here

system-tests/tests/regression/cre/cre_regression_suite_test.go

@@ -10,6 +10,11 @@ import (
 	"github.com/smartcontractkit/chainlink-testing-framework/framework"
 )
 
+var (
+	parallelEnabled = t_helpers.ParallelEnabled()
+	fanoutEnabled   = t_helpers.ChipSinkFanoutEnabled()
+)
+
 // REGRESSION TESTS target edge cases, negative conditions, etc., all happy path and sanity checks should go to a `smoke` package.
 /*
 To execute tests locally start the local CRE first:
@@ -28,12 +33,16 @@ func Test_CRE_V2_Consensus_Regression(t *testing.T) {
 	for _, tCase := range consensusNegativeTestsGenerateReport {
 		testName := fmt.Sprintf(consensusTestNameTemplate, tCase.caseToTrigger, tCase.name)
 		t.Run(testName, func(t *testing.T) {
-			testEnv := t_helpers.SetupTestEnvironmentWithConfig(t, t_helpers.GetDefaultTestConfig(t))
+			if parallelEnabled && fanoutEnabled {
+				t.Parallel()
+			}
+			testEnv := t_helpers.SetupTestEnvironmentWithPerTestKeys(t, t_helpers.GetDefaultTestConfig(t))
 			ConsensusFailsTest(t, testEnv, tCase)
 		})
 	}
 }
 
+// For now we did not parallelize this suite to avoid complications related to using real ChIP Ingress stack (DX-3543)
 func Test_CRE_V2_Cron_Regression(t *testing.T) {
 	for _, tCase := range cronInvalidSchedulesTests {
 		testName := "[v2] Cron (Beholder) fails when schedule is " + tCase.name
@@ -49,7 +58,10 @@ func Test_CRE_V2_HTTP_Regression(t *testing.T) {
 	for _, tCase := range httpNegativeTests {
 		testName := "[v2] HTTP Trigger fails with " + tCase.name
 		t.Run(testName, func(t *testing.T) {
-			testEnv := t_helpers.SetupTestEnvironmentWithConfig(t, t_helpers.GetDefaultTestConfig(t))
+			if parallelEnabled && fanoutEnabled {
+				t.Parallel()
+			}
+			testEnv := t_helpers.SetupTestEnvironmentWithPerTestKeys(t, t_helpers.GetDefaultTestConfig(t))
 			HTTPTriggerFailsTest(t, testEnv, tCase)
 		})
 	}
@@ -63,7 +75,10 @@ func runEVMNegativeTestSuite(t *testing.T, testCases []evmNegativeTest) {
 	for _, tCase := range testCases {
 		testName := fmt.Sprintf(evmTestNameTemplate, tCase.functionToTest, tCase.name)
 		t.Run(testName, func(t *testing.T) {
-			testEnv := t_helpers.SetupTestEnvironmentWithConfig(t, t_helpers.GetDefaultTestConfig(t))
+			if parallelEnabled && fanoutEnabled {
+				t.Parallel()
+			}
+			testEnv := t_helpers.SetupTestEnvironmentWithPerTestKeys(t, t_helpers.GetDefaultTestConfig(t))
 
 			// Check if test name contains "write" to determine which test function to run
 			switch {
@@ -145,7 +160,10 @@ func Test_CRE_V2_HTTP_Action_CRUD_Regression(t *testing.T) {
 	for _, tCase := range httpActionFailureTests {
 		testName := "[v2] HTTP Action fails with " + tCase.name
 		t.Run(testName, func(t *testing.T) {
-			testEnv := t_helpers.SetupTestEnvironmentWithConfig(t, t_helpers.GetDefaultTestConfig(t))
+			if parallelEnabled && fanoutEnabled {
+				t.Parallel()
+			}
+			testEnv := t_helpers.SetupTestEnvironmentWithPerTestKeys(t, t_helpers.GetDefaultTestConfig(t))
 			HTTPActionFailureTest(t, testEnv, tCase)
 		})
 	}

system-tests/tests/regression/cre/v2_consensus_regression_test.go

@@ -2,7 +2,6 @@ package cre
 
 import (
 	"fmt"
-	"math/rand"
 	"testing"
 	"time"
 
@@ -54,18 +53,30 @@ func ConsensusFailsTest(t *testing.T, testEnv *ttypes.TestEnvironment, consensus
 		chainID := bcOutput.CtfOutput().ChainID
 
 		testLogger.Info().Msg("Creating Consensus Fail workflow configuration...")
-		workflowName := fmt.Sprintf("consensus-fail-workflow-%s-%04d", chainID, rand.Intn(10000))
+		workflowName := t_helpers.UniqueWorkflowName(
+			testEnv,
+			fmt.Sprintf("consensus-fail-%s-%s", chainID, consensusNegativeTest.name),
+		)
 		feedID := "018e16c38e000320000000000000000000000000000000000000000000000000" // 32 hex characters (16 bytes)
 		workflowConfig := consensus_negative_config.Config{
 			CaseToTrigger: consensusNegativeTest.caseToTrigger,
 			FeedID:        feedID,
 			PayloadSizeKB: 101, // only used for oversized payload test
 		}
-		_ = t_helpers.CompileAndDeployWorkflow(t, testEnv, testLogger, workflowName, &workflowConfig, workflowFileLocation)
+		workflowID := t_helpers.CompileAndDeployWorkflow(t, testEnv, testLogger, workflowName, &workflowConfig, workflowFileLocation)
 
 		expectedError := consensusNegativeTest.expectedError
 
-		t_helpers.WatchWorkflowLogs(t, testLogger, userLogsCh, baseMessageCh, t_helpers.WorkflowEngineInitErrorLog, expectedError, 2*time.Minute)
-		testLogger.Info().Msg("Consensus Fail test successfully completed")
+		t_helpers.WatchWorkflowLogs(
+			t,
+			testLogger,
+			userLogsCh,
+			baseMessageCh,
+			t_helpers.WorkflowEngineInitErrorLog,
+			expectedError,
+			2*time.Minute,
+			t_helpers.WithUserLogWorkflowID(workflowID),
+		)
+		testLogger.Info().Str("test case", consensusNegativeTest.name).Msg("Consensus Fail test successfully completed")
 	}
 }

system-tests/tests/regression/cre/v2_evm_regression_test.go

@@ -109,7 +109,7 @@ var evmNegativeTestsEstimateGasInvalidToAddress = []evmNegativeTest{
 	{"empty", "", estimateGasInvalidToAddress, "EVM error StackUnderflow"},
 	{"a letter", "a", estimateGasInvalidToAddress, "EVM error PrecompileError"},
 	{"a symbol", "/", estimateGasInvalidToAddress, "EVM error StackUnderflow"},
-	{"not authored contract", "0xe7f1725E7734CE288F8367e1Bb143E90bb3F0512", estimateGasInvalidToAddress, "execution reverted"},
+	{"invalid call data", "replaced-during-runtime-with-contract-address", estimateGasInvalidToAddress, "execution reverted"},
 	{"cut hex", "0x", estimateGasInvalidToAddress, "EVM error StackUnderflow"}, // equivalent to "0x0"
 }
 
@@ -225,11 +225,32 @@ func EVMReadFailsTest(t *testing.T, testEnv *ttypes.TestEnvironment, evmNegative
 				BalanceReaderAddress: readBalancesAddress,
 			},
 		}
-		workflowName := fmt.Sprintf("evm-read-fail-workflow-%s-%04d", chainID, rand.Intn(10000))
-		_ = t_helpers.CompileAndDeployWorkflow(t, testEnv, testLogger, workflowName, &workflowConfig, workflowFileLocation)
 
-		t_helpers.WatchWorkflowLogs(t, testLogger, userLogsCh, baseMessageCh, t_helpers.WorkflowEngineInitErrorLog, evmNegativeTest.expectedError, 2*time.Minute)
-		testLogger.Info().Msg("EVM Read Fail test successfully completed")
+		if evmNegativeTest.functionToTest == estimateGasInvalidToAddress && evmNegativeTest.name == "invalid call data" {
+			workflowConfig.InvalidInput = readBalancesAddress.String()
+		}
+
+		workflowName := fmt.Sprintf("evm-read-fail-workflow-%s-%04d", chainID, rand.Intn(10000))
+		workflowID := t_helpers.CompileAndDeployWorkflow(
+			t,
+			testEnv,
+			testLogger,
+			workflowName,
+			&workflowConfig,
+			workflowFileLocation,
+		)
+
+		t_helpers.WatchWorkflowLogs(
+			t,
+			testLogger,
+			userLogsCh,
+			baseMessageCh,
+			t_helpers.WorkflowEngineInitErrorLog,
+			evmNegativeTest.expectedError,
+			2*time.Minute,
+			t_helpers.WithUserLogWorkflowID(workflowID),
+		)
+		testLogger.Info().Msgf("EVM Read Fail test successfully completed for test case %s and chain %s", evmNegativeTest.name, chainID)
 	}
 }
 
@@ -264,16 +285,31 @@ func EVMLogTriggerFailsTest(t *testing.T, testEnv *ttypes.TestEnvironment, evmNe
 			ChainSelector:  bcOutput.ChainSelector(),
 			InvalidAddress: evmNegativeTest.invalidInput,
 		}
+
 		workflowName := fmt.Sprintf("evm-logtrigger-fail-workflow-%s-%04d", chainID, rand.Intn(10000))
-		t_helpers.CompileAndDeployWorkflow(t, testEnv, testLogger, workflowName, &workflowConfig, workflowFileLocation)
+		workflowID := t_helpers.CompileAndDeployWorkflow(
+			t,
+			testEnv,
+			testLogger,
+			workflowName,
+			&workflowConfig,
+			workflowFileLocation,
+			// log event trigger is located on the capabilities DON, so it needs workflow artifacts copied to the capabilities DON
+			t_helpers.WithArtifactCopyDONTypes(cre.WorkflowDON, cre.CapabilitiesDON),
+		)
 
 		// For LogTrigger with EOA address, we expect engine initialization failure
 		// This is the correct behavior - the workflow engine should fail to initialize when trying to register a trigger with an invalid address
-		baseMsg := t_helpers.WatchBaseMessages(t, testLogger, baseMessageCh, t_helpers.WorkflowEngineInitErrorLog, 2*time.Minute)
-		require.NotEmpty(t, baseMsg.Labels, "no labels found in base message")
-		require.NotEmpty(t, baseMsg.Labels["err"], "no error label found in base message")
-		require.Contains(t, baseMsg.Labels["err"], evmNegativeTest.expectedError, "expected error message to contain "+evmNegativeTest.expectedError)
-		testLogger.Info().Msg("EVM LogTrigger Fail test successfully completed")
+		_ = t_helpers.WatchBaseMessages(
+			t,
+			testLogger,
+			baseMessageCh,
+			t_helpers.WorkflowEngineInitErrorLog,
+			2*time.Minute,
+			t_helpers.WithBaseMessageWorkflowID(workflowID),
+			t_helpers.WithBaseMessageLabelContains("err", evmNegativeTest.expectedError),
+		)
+		testLogger.Info().Msgf("EVM LogTrigger Fail test successfully completed for test case %s and chain %s", evmNegativeTest.name, chainID)
 	}
 }
 
@@ -313,7 +349,8 @@ var evmNegativeTestsWriteReportInvalidGas = []evmNegativeTest{
 var evmNegativeTestsLogTriggerInvalidAddress = []evmNegativeTest{
 	// using a well-known EOA address that is guaranteed to not be a contract
 	{"EOA address", "0x0000000000000000000000000000000000000001", logTriggerInvalidAddress, expectedLogTriggerInvalidAddress},
-	{"another EOA", "0x742d35Cc6634C0532925a3b844Bc9e7595f0bEb0", logTriggerInvalidAddress, expectedLogTriggerInvalidAddress},
+	// Anvil's & Geth's default dev account
+	{"another EOA", "0xf39Fd6e51aad88F6F4ce6aB8827279cffFb92266", logTriggerInvalidAddress, expectedLogTriggerInvalidAddress},
 }
 
 func EVMWriteFailsTest(t *testing.T, testEnv *ttypes.TestEnvironment, evmNegativeTest evmNegativeTest) {
@@ -357,9 +394,18 @@ func EVMWriteFailsTest(t *testing.T, testEnv *ttypes.TestEnvironment, evmNegativ
 				DataFeedsCacheAddress: dataFeedsCacheAddress,
 			},
 		}
-		_ = t_helpers.CompileAndDeployWorkflow(t, testEnv, testLogger, workflowName, &workflowConfig, workflowFileLocation)
-
-		t_helpers.WatchWorkflowLogs(t, testLogger, userLogsCh, baseMessageCh, t_helpers.WorkflowEngineInitErrorLog, evmNegativeTest.expectedError, 2*time.Minute)
+		workflowID := t_helpers.CompileAndDeployWorkflow(t, testEnv, testLogger, workflowName, &workflowConfig, workflowFileLocation)
+
+		t_helpers.WatchWorkflowLogs(
+			t,
+			testLogger,
+			userLogsCh,
+			baseMessageCh,
+			t_helpers.WorkflowEngineInitErrorLog,
+			evmNegativeTest.expectedError,
+			2*time.Minute,
+			t_helpers.WithUserLogWorkflowID(workflowID),
+		)
 		testLogger.Info().Msg("EVM Write Regression test successfully completed")
 	}
 }

system-tests/tests/regression/cre/v2_http_action_regression_test.go

@@ -5,8 +5,6 @@ import (
 	"testing"
 	"time"
 
-	"github.com/google/uuid"
-
 	commonevents "github.com/smartcontractkit/chainlink-protos/workflows/go/common"
 	workflowevents "github.com/smartcontractkit/chainlink-protos/workflows/go/events"
 
@@ -79,7 +77,7 @@ var httpActionFailureTests = []httpActionFailureTest{
 		testCase:      "crud-failure",
 		method:        "POST",
 		url:           "http://host.docker.internal:8080/test",
-		body:          strings.Repeat("a", 10*1024*1024), // 10MB body
+		body:          strings.Repeat("a", 2*1024*1024), // 2MB body, still above the 1MB threshold
 		expectedError: "HTTP Action failure test completed: oversized-request-body",
 	},
 	{
@@ -115,7 +113,10 @@ func HTTPActionFailureTest(t *testing.T, testEnv *ttypes.TestEnvironment, httpAc
 		TimeoutMs: httpActionTest.timeout,
 	}
 
-	workflowName := "http-action-fail-workflow-" + httpActionTest.method + "-" + uuid.New().String()[0:8]
+	workflowName := t_helpers.UniqueWorkflowName(
+		testEnv,
+		"http-action-fail-"+httpActionTest.method+"-"+httpActionTest.name,
+	)
 
 	// Start Beholder listener BEFORE registering workflow to avoid missing messages
 	userLogsCh := make(chan *workflowevents.UserLogs, 1000)
@@ -130,12 +131,21 @@ func HTTPActionFailureTest(t *testing.T, testEnv *ttypes.TestEnvironment, httpAc
 	})
 
 	// Now register and deploy the workflow
-	_ = t_helpers.CompileAndDeployWorkflow(t, testEnv, testLogger, workflowName, &workflowConfig, workflowFileLocation)
+	workflowID := t_helpers.CompileAndDeployWorkflow(t, testEnv, testLogger, workflowName, &workflowConfig, workflowFileLocation)
 
 	// Wait for specific error message in Beholder based on test case
 	testLogger.Info().Msgf("Waiting for expected HTTP Action failure: '%s' in Beholder...", httpActionTest.expectedError)
 
 	// Expect exact error message for this test case - no fallbacks
-	t_helpers.WatchWorkflowLogs(t, testLogger, userLogsCh, baseMessageCh, t_helpers.WorkflowEngineInitErrorLog, httpActionTest.expectedError, 60*time.Second)
+	t_helpers.WatchWorkflowLogs(
+		t,
+		testLogger,
+		userLogsCh,
+		baseMessageCh,
+		t_helpers.WorkflowEngineInitErrorLog,
+		httpActionTest.expectedError,
+		120*time.Second,
+		t_helpers.WithUserLogWorkflowID(workflowID),
+	)
 	testLogger.Info().Msg("HTTP Action failure test completed successfully")
 }

system-tests/tests/regression/cre/v2_http_trigger_regression_test.go

@@ -123,8 +123,11 @@ func HTTPTriggerFailsTest(t *testing.T, testEnv *ttypes.TestEnvironment, httpNeg
 		TestCase:      httpNegativeTest.testCase,
 	}
 
-	workflowName := "http-trigger-fail-workflow-" + httpNegativeTest.testCase
-	_ = t_helpers.CompileAndDeployWorkflow(t, testEnv, testLogger, workflowName, &workflowConfig, workflowFileLocation)
+	workflowName := t_helpers.UniqueWorkflowName(
+		testEnv,
+		"http-trigger-fail-"+httpNegativeTest.testCase+"-"+httpNegativeTest.name,
+	)
+	workflowID := t_helpers.CompileAndDeployWorkflow(t, testEnv, testLogger, workflowName, &workflowConfig, workflowFileLocation)
 
 	// For invalid key type and invalid public key format, we expect the workflow deployment/trigger setup to fail
 	// For non-existing public key, we expect the trigger execution to fail with unauthorized error at gateway level
@@ -140,11 +143,16 @@ func HTTPTriggerFailsTest(t *testing.T, testEnv *ttypes.TestEnvironment, httpNeg
 	}
 
 	// expect engine initialisation failure due to incorrect trigger configuration
-	baseMsg := t_helpers.WatchBaseMessages(t, testLogger, baseMessageCh, t_helpers.WorkflowEngineInitErrorLog, 2*time.Minute)
-	require.NotEmpty(t, baseMsg.Labels, "no labels found in base message")
-	require.NotEmpty(t, baseMsg.Labels["err"], "no error label found in base message")
-	require.Contains(t, baseMsg.Labels["err"], httpNegativeTest.expectedError, "expected error message to contain "+httpNegativeTest.expectedError)
-	testLogger.Info().Msgf("Found expected error - %s - in base message's labels", httpNegativeTest.expectedError)
+	_ = t_helpers.WatchBaseMessages(
+		t,
+		testLogger,
+		baseMessageCh,
+		t_helpers.WorkflowEngineInitErrorLog,
+		2*time.Minute,
+		t_helpers.WithBaseMessageWorkflowID(workflowID),
+		t_helpers.WithBaseMessageLabelContains("err", httpNegativeTest.expectedError),
+	)
+	testLogger.Info().Msgf("Found expected error - %s - in base message", httpNegativeTest.expectedError)
 	testLogger.Info().Msg("HTTP Trigger Fail test successfully completed")
 }
 

system-tests/tests/smoke/cre/v2_consensus_capability_test.go

@@ -28,9 +28,9 @@ func ExecuteConsensusTest(t *testing.T, testEnv *ttypes.TestEnvironment) {
 	})
 
 	workflowName := t_helpers.UniqueWorkflowName(testEnv, "consensustest")
-	_ = t_helpers.CompileAndDeployWorkflow(t, testEnv, testLogger, workflowName, &t_helpers.None{}, "../../../../core/scripts/cre/environment/examples/workflows/v2/node-mode/main.go")
+	workflowID := t_helpers.CompileAndDeployWorkflow(t, testEnv, testLogger, workflowName, &t_helpers.None{}, "../../../../core/scripts/cre/environment/examples/workflows/v2/node-mode/main.go")
 
 	expectedBeholderLog := "Successfully passed all consensus tests"
-	t_helpers.WatchWorkflowLogs(t, testLogger, userLogsCh, baseMessageCh, t_helpers.WorkflowEngineInitErrorLog, expectedBeholderLog, 4*time.Minute)
+	t_helpers.WatchWorkflowLogs(t, testLogger, userLogsCh, baseMessageCh, t_helpers.WorkflowEngineInitErrorLog, expectedBeholderLog, 4*time.Minute, t_helpers.WithUserLogWorkflowID(workflowID))
 	testLogger.Info().Msg("Consensus capability test completed")
 }

system-tests/tests/smoke/cre/v2_dontime_test.go

@@ -35,9 +35,9 @@ func ExecuteDonTimeTest(t *testing.T, testEnv *ttypes.TestEnvironment) {
 	workflowConfig := crontypes.WorkflowConfig{
 		Schedule: "*/30 * * * * *", // every 30 seconds
 	}
-	_ = t_helpers.CompileAndDeployWorkflow(t, testEnv, testLogger, workflowName, &workflowConfig, workflowFileLocation)
+	workflowID := t_helpers.CompileAndDeployWorkflow(t, testEnv, testLogger, workflowName, &workflowConfig, workflowFileLocation)
 
 	expectedBeholderLog := "Verified consensus on DON Time"
-	t_helpers.WatchWorkflowLogs(t, testLogger, userLogsCh, baseMessageCh, t_helpers.WorkflowEngineInitErrorLog, expectedBeholderLog, 2*time.Minute)
+	t_helpers.WatchWorkflowLogs(t, testLogger, userLogsCh, baseMessageCh, t_helpers.WorkflowEngineInitErrorLog, expectedBeholderLog, 2*time.Minute, t_helpers.WithUserLogWorkflowID(workflowID))
 	testLogger.Info().Msg("DON Time test completed")
 }

system-tests/tests/smoke/cre/v2_evm_capability_test.go

@@ -416,7 +416,7 @@ func ExecuteEVMLogTriggerTest(t *testing.T, testEnv *ttypes.TestEnvironment) {
 
 		workflowName := fmt.Sprintf("evm-logTrigger-workflow-%s-%04d", chainID, rand.Intn(10000))
 		lggr.Info().Msgf("About to deploy Workflow %s on chain %s", workflowName, chainID)
-		t_helpers.CompileAndDeployWorkflow(t, testEnv, lggr, workflowName, &workflowConfig, workflowFileLocation)
+		workflowID := t_helpers.CompileAndDeployWorkflow(t, testEnv, lggr, workflowName, &workflowConfig, workflowFileLocation)
 
 		message := "Data for log trigger chain " + chainID
 		// start background event emission every 10s while WatchWorkflowLogs is running, so that the workflow has events to pick up eventually
@@ -444,7 +444,7 @@ func ExecuteEVMLogTriggerTest(t *testing.T, testEnv *ttypes.TestEnvironment) {
 		}()
 		expectedUserLog := "OnTrigger decoded message: message:" + message
 
-		t_helpers.WatchWorkflowLogs(t, lggr, userLogsCh, baseMessageCh, t_helpers.WorkflowEngineInitErrorLog, expectedUserLog, 4*time.Minute)
+		t_helpers.WatchWorkflowLogs(t, lggr, userLogsCh, baseMessageCh, t_helpers.WorkflowEngineInitErrorLog, expectedUserLog, 4*time.Minute, t_helpers.WithUserLogWorkflowID(workflowID))
 		emitCancelFn()
 		lggr.Info().Msgf("Found expected user log: '%s' on chain %s", expectedUserLog, chainID)