fix workers crash (#5370)

* fix workers crash

* fix failed runner tests

Author: kobenguyent

lib/command/run-workers.js

@@ -40,11 +40,22 @@ export default async function (workerCount, selectedRuns, options) {
 
   output.print(`CodeceptJS v${Codecept.version()} ${output.standWithUkraine()}`)
   output.print(`Running tests in ${output.styles.bold(numberOfWorkers)} workers...`)
-  output.print()
   store.hasWorkers = true
 
   const workers = new Workers(numberOfWorkers, config)
   workers.overrideConfig(overrideConfigs)
+  
+  // Show test distribution after workers are initialized
+  await workers.bootstrapAll()
+  
+  const workerObjects = workers.getWorkers()
+  output.print()
+  output.print('Test distribution:')
+  workerObjects.forEach((worker, index) => {
+    const testCount = worker.tests.length
+    output.print(`  Worker ${index + 1}: ${testCount} test${testCount !== 1 ? 's' : ''}`)
+  })
+  output.print()
 
   workers.on(event.test.failed, test => {
     output.test.failed(test)
@@ -68,7 +79,6 @@ export default async function (workerCount, selectedRuns, options) {
     if (options.verbose) {
       await getMachineInfo()
     }
-    await workers.bootstrapAll()
     await workers.run()
   } catch (err) {
     output.error(err)

lib/command/workers/runTests.js

@@ -19,13 +19,45 @@ const stderr = ''
 
 const { options, tests, testRoot, workerIndex, poolMode } = workerData
 
+// Global error handlers to catch critical errors but not test failures
+process.on('uncaughtException', (err) => {
+  // Log to stderr to bypass stdout suppression
+  process.stderr.write(`[Worker ${workerIndex}] UNCAUGHT EXCEPTION: ${err.message}\n`)
+  process.stderr.write(`${err.stack}\n`)
+  
+  // Don't exit on test assertion errors - those are handled by mocha
+  if (err.name === 'AssertionError' || err.message?.includes('expected')) {
+    return
+  }
+  process.exit(1)
+})
+
+process.on('unhandledRejection', (reason, promise) => {
+  // Log to stderr to bypass stdout suppression
+  const msg = reason?.message || String(reason)
+  process.stderr.write(`[Worker ${workerIndex}] UNHANDLED REJECTION: ${msg}\n`)
+  if (reason?.stack) {
+    process.stderr.write(`${reason.stack}\n`)
+  }
+  
+  // Don't exit on test-related rejections
+  if (msg.includes('expected') || msg.includes('AssertionError')) {
+    return
+  }
+  process.exit(1)
+})
+
 // hide worker output
 // In pool mode, only suppress output if debug is NOT enabled
 // In regular mode, hide result output but allow step output in verbose/debug
 if (poolMode && !options.debug) {
   // In pool mode without debug, allow test names and important output but suppress verbose details
   const originalWrite = process.stdout.write
   process.stdout.write = string => {
+    // Always allow Worker logs
+    if (string.includes('[Worker')) {
+      return originalWrite.call(process.stdout, string)
+    }
     // Allow test names (✔ or ✖), Scenario Steps, failures, and important markers
     if (
       string.includes('✔') ||
@@ -45,7 +77,12 @@ if (poolMode && !options.debug) {
     return originalWrite.call(process.stdout, string)
   }
 } else if (!poolMode && !options.debug && !options.verbose) {
+  const originalWrite = process.stdout.write
   process.stdout.write = string => {
+    // Always allow Worker logs
+    if (string.includes('[Worker')) {
+      return originalWrite.call(process.stdout, string)
+    }
     stdout += string
     return true
   }
@@ -82,6 +119,13 @@ let config
 // Load test and run
 initPromise = (async function () {
   try {
+    // Add staggered delay at the very start to prevent resource conflicts
+    // Longer delay for browser initialization conflicts
+    const delay = (workerIndex - 1) * 2000 // 0ms, 2s, 4s, etc.
+    if (delay > 0) {
+      await new Promise(resolve => setTimeout(resolve, delay))
+    }
+    
     // Import modules dynamically to avoid ES Module loader race conditions in Node 22.x
     const eventModule = await import('../../event.js')
     const containerModule = await import('../../container.js')
@@ -98,16 +142,32 @@ initPromise = (async function () {
 
     const overrideConfigs = tryOrDefault(() => JSON.parse(options.override), {})
 
-    // IMPORTANT: await is required here since getConfig is async
-    const baseConfig = await getConfig(options.config || testRoot)
+    let baseConfig
+    try {
+      // IMPORTANT: await is required here since getConfig is async
+      baseConfig = await getConfig(options.config || testRoot)
+    } catch (configErr) {
+      process.stderr.write(`[Worker ${workerIndex}] FAILED loading config: ${configErr.message}\n`)
+      process.stderr.write(`${configErr.stack}\n`)
+      await new Promise(resolve => setTimeout(resolve, 100))
+      process.exit(1)
+    }
 
     // important deep merge so dynamic things e.g. functions on config are not overridden
     config = deepMerge(baseConfig, overrideConfigs)
 
     // Pass workerIndex as child option for output.process() to display worker prefix
     const optsWithChild = { ...options, child: workerIndex }
     codecept = new Codecept(config, optsWithChild)
-    await codecept.init(testRoot)
+    
+    try {
+      await codecept.init(testRoot)
+    } catch (initErr) {
+      process.stderr.write(`[Worker ${workerIndex}] FAILED during codecept.init(): ${initErr.message}\n`)
+      process.stderr.write(`${initErr.stack}\n`)
+      process.exit(1)
+    }
+    
     codecept.loadTests()
     mocha = container.mocha()
 
@@ -126,10 +186,12 @@ initPromise = (async function () {
       await runTests()
     } else {
       // No tests to run, close the worker
+      console.error(`[Worker ${workerIndex}] ERROR: No tests found after filtering! Assigned ${tests.length} UIDs but none matched.`)
       parentPort?.close()
     }
   } catch (err) {
-    console.error('Error in worker initialization:', err)
+    process.stderr.write(`[Worker ${workerIndex}] FATAL ERROR: ${err.message}\n`)
+    process.stderr.write(`${err.stack}\n`)
     process.exit(1)
   }
 })()
@@ -147,8 +209,14 @@ async function runTests() {
   disablePause()
   try {
     await codecept.run()
+  } catch (err) {
+    throw err
   } finally {
-    await codecept.teardown()
+    try {
+      await codecept.teardown()
+    } catch (err) {
+      // Ignore teardown errors
+    }
   }
 }
 
@@ -336,8 +404,16 @@ function filterTests() {
   mocha.files = files
   mocha.loadFiles()
 
-  for (const suite of mocha.suite.suites) {
+  // Recursively filter tests in all suites (including nested ones)
+  const filterSuiteTests = (suite) => {
     suite.tests = suite.tests.filter(test => tests.indexOf(test.uid) >= 0)
+    for (const childSuite of suite.suites) {
+      filterSuiteTests(childSuite)
+    }
+  }
+
+  for (const suite of mocha.suite.suites) {
+    filterSuiteTests(suite)
   }
 }
 

lib/helper/Playwright.js

@@ -912,7 +912,7 @@ class Playwright extends Helper {
   }
 
   async _finishTest() {
-    if ((restartsSession() || restartsContext() || restartsBrowser()) && this.isRunning) {
+    if (this.isRunning) {
       try {
         await Promise.race([this._stopBrowser(), new Promise((_, reject) => setTimeout(() => reject(new Error('Test finish timeout')), 10000))])
       } catch (e) {

lib/listener/helpers.js

@@ -73,30 +73,18 @@ export default function () {
   })
 
   event.dispatcher.on(event.all.result, () => {
-    // Skip _finishTest for all helpers if any browser helper restarts to avoid double cleanup
-    const hasBrowserRestart = Object.values(helpers).some(helper => 
-      (helper.config && (helper.config.restart === 'browser' || helper.config.restart === 'context' || helper.config.restart === true)) || 
-      (helper.options && (helper.options.restart === 'browser' || helper.options.restart === 'context' || helper.options.restart === true))
-    )
-
     Object.keys(helpers).forEach(key => {
       const helper = helpers[key]
-      if (helper._finishTest && !hasBrowserRestart) {
+      if (helper._finishTest) {
         recorder.add(`hook ${key}._finishTest()`, () => helper._finishTest(), true, false)
       }
     })
   })
 
   event.dispatcher.on(event.all.after, () => {
-    // Skip _cleanup for all helpers if any browser helper restarts to avoid double cleanup
-    const hasBrowserRestart = Object.values(helpers).some(helper => 
-      (helper.config && (helper.config.restart === 'browser' || helper.config.restart === 'context' || helper.config.restart === true)) || 
-      (helper.options && (helper.options.restart === 'browser' || helper.options.restart === 'context' || helper.options.restart === true))
-    )
-
     Object.keys(helpers).forEach(key => {
       const helper = helpers[key]
-      if (helper._cleanup && !hasBrowserRestart) {
+      if (helper._cleanup) {
         recorder.add(`hook ${key}._cleanup()`, () => helper._cleanup(), true, false)
       }
     })

lib/utils/typescript.js

@@ -119,7 +119,7 @@ const __dirname = __dirname_fn(__filename);
     let jsContent = transpileTS(filePath)
 
     // Find all relative TypeScript imports in this file
-    const importRegex = /from\s+['"](\..+?)(?:\.ts)?['"]/g
+    const importRegex = /from\s+['"](\.[^'"]+?)(?:\.ts)?['"]/g
     let match
     const imports = []
 
@@ -170,7 +170,7 @@ const __dirname = __dirname_fn(__filename);
 
     // After all dependencies are transpiled, rewrite imports in this file
     jsContent = jsContent.replace(
-      /from\s+['"](\..+?)(?:\.ts)?['"]/g,
+      /from\s+['"](\.[^'"]+?)(?:\.ts)?['"]/g,
       (match, importPath) => {
         let resolvedPath = path.resolve(fileBaseDir, importPath)
         const originalExt = path.extname(importPath)

lib/workers.js

@@ -370,6 +370,9 @@ class Workers extends EventEmitter {
     // If Codecept isn't initialized yet, return empty groups as a safe fallback
     if (!this.codecept) return populateGroups(numberOfWorkers)
     const files = this.codecept.testFiles
+    
+    // Create a fresh mocha instance to avoid state pollution
+    Container.createMocha(this.codecept.config.mocha || {}, this.options)
     const mocha = Container.mocha()
     mocha.files = files
     mocha.loadFiles()
@@ -384,6 +387,10 @@ class Workers extends EventEmitter {
         groupCounter++
       }
     })
+    
+    // Clean up after collecting test UIDs
+    mocha.unloadFiles()
+    
     return groups
   }
 
@@ -452,9 +459,12 @@ class Workers extends EventEmitter {
     const files = this.codecept.testFiles
     const groups = populateGroups(numberOfWorkers)
 
+    // Create a fresh mocha instance to avoid state pollution
+    Container.createMocha(this.codecept.config.mocha || {}, this.options)
     const mocha = Container.mocha()
     mocha.files = files
     mocha.loadFiles()
+    
     mocha.suite.suites.forEach(suite => {
       const i = indexOfSmallestElement(groups)
       suite.tests.forEach(test => {
@@ -463,6 +473,10 @@ class Workers extends EventEmitter {
         }
       })
     })
+    
+    // Clean up after collecting test UIDs
+    mocha.unloadFiles()
+    
     return groups
   }
 
@@ -504,8 +518,24 @@ class Workers extends EventEmitter {
       // Workers are already running, this is just a placeholder step
     })
 
+    // Add overall timeout to prevent infinite hanging
+    const overallTimeout = setTimeout(() => {
+      console.error('[Main] Overall timeout reached (10 minutes). Force terminating remaining workers...')
+      workerThreads.forEach(w => {
+        try {
+          w.terminate()
+        } catch (e) {
+          // ignore
+        }
+      })
+      this._finishRun()
+    }, 600000) // 10 minutes
+
     return new Promise(resolve => {
-      this.on('end', resolve)
+      this.on('end', () => {
+        clearTimeout(overallTimeout)
+        resolve()
+      })
     })
   }
 
@@ -528,8 +558,32 @@ class Workers extends EventEmitter {
     if (this.isPoolMode) {
       this.activeWorkers.set(worker, { available: true, workerIndex: null })
     }
+    
+    // Track last activity time to detect hanging workers
+    let lastActivity = Date.now()
+    let currentTest = null
+    const workerTimeout = 300000 // 5 minutes
+    
+    const timeoutChecker = setInterval(() => {
+      const elapsed = Date.now() - lastActivity
+      if (elapsed > workerTimeout) {
+        console.error(`[Main] Worker appears to be hanging (no activity for ${Math.floor(elapsed/1000)}s). Terminating...`)
+        if (currentTest) {
+          console.error(`[Main] Last test: ${currentTest}`)
+        }
+        clearInterval(timeoutChecker)
+        worker.terminate()
+      }
+    }, 30000) // Check every 30 seconds
 
     worker.on('message', message => {
+      lastActivity = Date.now() // Update activity timestamp
+      
+      // Track current test
+      if (message.event === event.test.started && message.data) {
+        currentTest = message.data.title || message.data.fullTitle
+      }
+      
       output.process(message.workerIndex)
 
       // Handle test requests for pool mode
@@ -646,11 +700,25 @@ class Workers extends EventEmitter {
     })
 
     worker.on('error', err => {
+      console.error(`[Main] Worker error:`, err.message || err)
+      if (currentTest) {
+        console.error(`[Main] Failed during test: ${currentTest}`)
+      }
       this.errors.push(err)
     })
 
-    worker.on('exit', () => {
+    worker.on('exit', (code) => {
+      clearInterval(timeoutChecker)
       this.closedWorkers += 1
+      
+      if (code !== 0) {
+        console.error(`[Main] Worker exited with code ${code}`)
+        if (currentTest) {
+          console.error(`[Main] Last test running: ${currentTest}`)
+        }
+        // Mark as failed
+        process.exitCode = 1
+      }
 
       if (this.isPoolMode) {
         // Pool mode: finish when all workers have exited and no more tests
@@ -666,7 +734,7 @@ class Workers extends EventEmitter {
 
   _finishRun() {
     event.dispatcher.emit(event.workers.after, { tests: this.workers.map(worker => worker.tests) })
-    if (Container.result().hasFailed) {
+    if (Container.result().hasFailed || this.errors.length > 0) {
       process.exitCode = 1
     } else {
       process.exitCode = 0

package.json

@@ -1,6 +1,6 @@
 {
   "name": "codeceptjs",
-  "version": "4.0.1-beta.9",
+  "version": "4.0.2-beta.17",
   "type": "module",
   "description": "Supercharged End 2 End Testing Framework for NodeJS",
   "keywords": [