feat: add guardrails for headed mode

packages/cli/src/constructs/__tests__/fixtures/playwright-check/playwright.config.headless-false-project.ts

@@ -0,0 +1,13 @@
+import { defineConfig } from '@playwright/test'
+
+export default defineConfig({
+  testDir: './tests',
+  projects: [
+    {
+      name: 'chromium',
+      use: {
+        headless: false,
+      },
+    },
+  ],
+})

packages/cli/src/constructs/__tests__/fixtures/playwright-check/playwright.config.headless-false.ts

@@ -0,0 +1,8 @@
+import { defineConfig } from '@playwright/test'
+
+export default defineConfig({
+  testDir: './tests',
+  use: {
+    headless: false,
+  },
+})

packages/cli/src/constructs/__tests__/fixtures/playwright-check/playwright.config.headless-true.ts

@@ -0,0 +1,8 @@
+import { defineConfig } from '@playwright/test'
+
+export default defineConfig({
+  testDir: './tests',
+  use: {
+    headless: true,
+  },
+})

packages/cli/src/constructs/__tests__/playwright-check.spec.ts

@@ -271,6 +271,101 @@ describe('PlaywrightCheck', () => {
         }),
       ]))
     })
+
+    it('should error if headless: false is set globally', async () => {
+      Session.basePath = path.resolve(__dirname, './fixtures/playwright-check')
+      Session.project = new Project('project-id', {
+        name: 'Test Project',
+        repoUrl: 'https://github.com/checkly/checkly-cli',
+      })
+
+      const check = new PlaywrightCheck('foo', {
+        name: 'Test Check',
+        playwrightConfigPath: path.resolve(__dirname, './fixtures/playwright-check/playwright.config.headless-false.ts'),
+      })
+
+      const diags = new Diagnostics()
+      await check.validate(diags)
+
+      expect(diags.isFatal()).toEqual(true)
+      expect(diags.observations).toEqual(expect.arrayContaining([
+        expect.objectContaining({
+          message: expect.stringContaining('Property "headless" is not supported.'),
+        }),
+      ]))
+    })
+
+    it('should error if headless: false is set in a project', async () => {
+      Session.basePath = path.resolve(__dirname, './fixtures/playwright-check')
+      Session.project = new Project('project-id', {
+        name: 'Test Project',
+        repoUrl: 'https://github.com/checkly/checkly-cli',
+      })
+
+      const check = new PlaywrightCheck('foo', {
+        name: 'Test Check',
+        playwrightConfigPath: path.resolve(__dirname, './fixtures/playwright-check/playwright.config.headless-false-project.ts'),
+      })
+
+      const diags = new Diagnostics()
+      await check.validate(diags)
+
+      expect(diags.isFatal()).toEqual(true)
+      expect(diags.observations).toEqual(expect.arrayContaining([
+        expect.objectContaining({
+          message: expect.stringContaining('Property "headless" is not supported.'),
+        }),
+        expect.objectContaining({
+          message: expect.stringContaining('in project "chromium"'),
+        }),
+      ]))
+    })
+
+    it('should not error if headless: true is set', async () => {
+      Session.basePath = path.resolve(__dirname, './fixtures/playwright-check')
+      Session.project = new Project('project-id', {
+        name: 'Test Project',
+        repoUrl: 'https://github.com/checkly/checkly-cli',
+      })
+
+      const check = new PlaywrightCheck('foo', {
+        name: 'Test Check',
+        playwrightConfigPath: path.resolve(__dirname, './fixtures/playwright-check/playwright.config.headless-true.ts'),
+      })
+
+      const diags = new Diagnostics()
+      await check.validate(diags)
+
+      expect(diags.isFatal()).toEqual(false)
+      expect(diags.observations).not.toEqual(expect.arrayContaining([
+        expect.objectContaining({
+          message: expect.stringContaining('Property "headless" is not supported.'),
+        }),
+      ]))
+    })
+
+    it('should not error if headless is not set', async () => {
+      Session.basePath = path.resolve(__dirname, './fixtures/playwright-check')
+      Session.project = new Project('project-id', {
+        name: 'Test Project',
+        repoUrl: 'https://github.com/checkly/checkly-cli',
+      })
+
+      const check = new PlaywrightCheck('foo', {
+        name: 'Test Check',
+        playwrightConfigPath: path.resolve(__dirname, './fixtures/playwright-check/playwright.config.ts'),
+      })
+
+      const diags = new Diagnostics()
+      await check.validate(diags)
+
+      expect(diags.isFatal()).toEqual(false)
+      expect(diags.observations).not.toEqual(expect.arrayContaining([
+        expect.objectContaining({
+          message: expect.stringContaining('Property "headless" is not supported.'),
+        }),
+      ]))
+    })
   })
 
   describe('defaults', () => {

packages/cli/src/constructs/playwright-check.ts

@@ -238,6 +238,51 @@ export class PlaywrightCheck extends RuntimeCheck {
     }
   }
 
+  protected async validateHeadlessMode (diagnostics: Diagnostics): Promise<void> {
+    try {
+      const playwrightConfig = await Session.loadFile<any>(this.playwrightConfigPath)
+
+      if (playwrightConfig?.use?.headless === false) {
+        diagnostics.add(new UnsupportedPropertyDiagnostic(
+          'headless',
+          new Error(
+            `headless: false is not supported.`
+            + `\n\n`
+            + `Checkly runs all Playwright checks in headless mode. `
+            + `Please remove this setting from your Playwright configuration, `
+            + `or set it to true.`,
+          ),
+        ))
+      }
+
+      if (Array.isArray(playwrightConfig?.projects)) {
+        for (const project of playwrightConfig.projects) {
+          if (project?.use?.headless === false) {
+            const projectName = project.name ? ` in project "${project.name}"` : ''
+            diagnostics.add(new UnsupportedPropertyDiagnostic(
+              'headless',
+              new Error(
+                `headless: false is not supported${projectName}.`
+                + `\n\n`
+                + `Checkly runs all Playwright checks in headless mode. `
+                + `Please remove this setting from your Playwright configuration, `
+                + `or set it to true.`,
+              ),
+            ))
+          }
+        }
+      }
+    } catch (err: any) {
+      diagnostics.add(new InvalidPropertyValueDiagnostic(
+        'playwrightConfigPath',
+        new Error(
+          `Unable to parse Playwright config "${this.playwrightConfigPath}": ${err.message}`,
+          { cause: err },
+        ),
+      ))
+    }
+  }
+
   async validate (diagnostics: Diagnostics): Promise<void> {
     await super.validate(diagnostics)
     await this.validateRetryStrategy(diagnostics)
@@ -251,6 +296,8 @@ export class PlaywrightCheck extends RuntimeCheck {
       ))
     }
 
+    await this.validateHeadlessMode(diagnostics)
+
     this.#validateGroupReferences(diagnostics)
   }