feat(playwright): optimizes playwright tests

.github/workflows/enforce-labels.yml

@@ -9,6 +9,6 @@ jobs:
     steps:
       - uses: yogevbd/enforce-label-action@2.2.2
         with:
-          REQUIRED_LABELS_ANY: "bug,dependencies,documentation,enhancement,feature,skip-changelog,techdebt,tests"
-          REQUIRED_LABELS_ANY_DESCRIPTION: "Select at least one label from the following list: bug, dependencies, documentation, enhancement, feature, skip-changelog, techdebt, tests"
+          REQUIRED_LABELS_ANY: "bug,dependencies,documentation,enhancement,feature,skip-changelog,skip-e2e,techdebt,tests"
+          REQUIRED_LABELS_ANY_DESCRIPTION: "Select at least one label from the following list: bug, dependencies, documentation, enhancement, feature, skip-changelog, skip-e2e, techdebt, tests"
           BANNED_LABELS: "banned"

.github/workflows/playwright.yml

@@ -1,21 +1,80 @@
-name: "playwright"
-on: # rebuild any PRs and main branch changes
+name: Playwright E2E Tests
+on:
   pull_request:
+    types: [opened, synchronize, reopened, ready_for_review]
   push:
     branches:
       - main
 
 env:
   LOG_LEVEL: ERROR
-  STATIC_DIR:
+  STATIC_DIR: ""
   DATABASE_HOSTNAME: localhost
   DATABASE_CREDENTIALS: dispatch:dispatch
   DISPATCH_ENCRYPTION_KEY: NJHDWDJ3PbHT8h
   DISPATCH_JWT_SECRET: foo
 
 jobs:
+  # Job to determine if e2e tests should run
+  should-run-e2e:
+    runs-on: ubuntu-latest
+    outputs:
+      run-tests: ${{ steps.check.outputs.run-tests }}
+    steps:
+      - name: Check out Git repository
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Check if e2e tests should run
+        id: check
+        run: |
+          # Skip if draft PR
+          if [[ "${{ github.event.pull_request.draft }}" == "true" ]]; then
+            echo "Skipping e2e tests: Draft PR"
+            echo "run-tests=false" >> $GITHUB_OUTPUT
+            exit 0
+          fi
+
+          # Skip if only docs changed
+          if git diff --name-only origin/${{ github.base_ref }}..HEAD | grep -v -E '^(docs/|.*\.md$|.*\.mdx$|LICENSE|.*\.txt$)' | wc -l | grep -q '^0$'; then
+            echo "Skipping e2e tests: Documentation-only changes"
+            echo "run-tests=false" >> $GITHUB_OUTPUT
+            exit 0
+          fi
+
+          # Skip if only backend tests changed
+          if git diff --name-only origin/${{ github.base_ref }}..HEAD | grep -v -E '^(tests/(?!static/e2e)|.*test.*\.py$)' | wc -l | grep -q '^0$'; then
+            echo "Skipping e2e tests: Test-only changes"
+            echo "run-tests=false" >> $GITHUB_OUTPUT
+            exit 0
+          fi
+
+          # Skip if only backend-only changes (no frontend impact)
+          if git diff --name-only origin/${{ github.base_ref }}..HEAD | grep -v -E '^(src/dispatch/(?!static)|tests/(?!static)|\.github/workflows/(?!playwright)|requirements.*\.txt|setup\.py|pyproject\.toml|\.python-version|Dockerfile|docker/)' | wc -l | grep -q '^0$'; then
+            echo "Skipping e2e tests: Backend-only changes"
+            echo "run-tests=false" >> $GITHUB_OUTPUT
+            exit 0
+          fi
+
+          # Skip if labeled with skip-e2e
+          if echo '${{ toJson(github.event.pull_request.labels.*.name) }}' | grep -q 'skip-e2e'; then
+            echo "Skipping e2e tests: skip-e2e label found"
+            echo "run-tests=false" >> $GITHUB_OUTPUT
+            exit 0
+          fi
+
+          echo "Running e2e tests: Frontend or critical changes detected"
+          echo "run-tests=true" >> $GITHUB_OUTPUT
+
   end-to-end:
+    needs: should-run-e2e
+    if: needs.should-run-e2e.outputs.run-tests == 'true'
     runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        shard: [1, 2, 3, 4]
     services:
       postgres:
         image: postgres
@@ -58,10 +117,29 @@ jobs:
       - name: Setup sample database
         run: dispatch database restore --dump-file data/dispatch-sample-data.dump --skip-check && dispatch database upgrade
       - name: Run tests
-        run: npx playwright test --project=chromium
+        run: npx playwright test --project=chromium --shard=${{ matrix.shard }}/4
       - uses: actions/upload-artifact@v4
         if: always()
         with:
-          name: playwright-report
+          name: playwright-report-shard-${{ matrix.shard }}
           path: playwright-report/
           retention-days: 30
+
+  # Summary job for required checks
+  e2e-tests-complete:
+    runs-on: ubuntu-latest
+    needs: [should-run-e2e, end-to-end]
+    if: always()
+    steps:
+      - name: Check e2e test results
+        run: |
+          if [[ "${{ needs.should-run-e2e.outputs.run-tests }}" == "false" ]]; then
+            echo "✅ E2E tests skipped (not needed for this change)"
+            exit 0
+          elif [[ "${{ needs.end-to-end.result }}" == "success" ]]; then
+            echo "✅ E2E tests passed"
+            exit 0
+          else
+            echo "❌ E2E tests failed"
+            exit 1
+          fi

playwright.config.ts

@@ -13,52 +13,59 @@ const config: PlaywrightTestConfig = {
     /* Base URL to use in actions like `await page.goto('/')`. */
     baseURL: "http://localhost:8080/",
     /* Collect trace when retrying the failed test. See https://playwright.dev/docs/trace-viewer */
-    trace: "on",
-    video: "on",
-    screenshot: "on",
+    trace: "retain-on-failure",
+    video: "retain-on-failure",
+    screenshot: "only-on-failure",
   },
   /* Maximum time one test can run for. */
-  timeout: 200 * 1000,
+  timeout: process.env.CI ? 200 * 1000 : 60 * 1000,
   expect: {
     /**
      * Maximum time expect() should wait for the condition to be met.
      * For example in `await expect(locator).toHaveText();`
      */
-    timeout: 20000,
+    timeout: process.env.CI ? 20000 : 10000,
   },
   /* Run tests in files in parallel */
   fullyParallel: true,
   /* Fail the build on CI if you accidentally left test.only in the source code. */
   forbidOnly: !!process.env.CI,
   /* Retry on CI only */
   retries: process.env.CI ? 2 : 0,
-  /* Opt out of parallel tests on CI. */
-  workers: process.env.CI ? 1 : undefined,
+  /* Optimize workers for CI - use more workers for faster execution */
+  workers: process.env.CI ? 4 : undefined,
   /* Reporter to use. See https://playwright.dev/docs/test-reporters */
-  reporter: "html",
+  reporter: process.env.CI ? [["html"], ["github"]] : "html",
   /* Configure projects for major browsers */
-  projects: [
-    {
-      name: "chromium",
-      use: {
-        ...devices["Desktop Chrome"],
-      },
-    },
-
-    {
-      name: "firefox",
-      use: {
-        ...devices["Desktop Firefox"],
-      },
-    },
-
-    {
-      name: "webkit",
-      use: {
-        ...devices["Desktop Safari"],
-      },
-    },
-  ],
+  projects: process.env.CI
+    ? [
+        {
+          name: "chromium",
+          use: {
+            ...devices["Desktop Chrome"],
+          },
+        },
+      ]
+    : [
+        {
+          name: "chromium",
+          use: {
+            ...devices["Desktop Chrome"],
+          },
+        },
+        {
+          name: "firefox",
+          use: {
+            ...devices["Desktop Firefox"],
+          },
+        },
+        {
+          name: "webkit",
+          use: {
+            ...devices["Desktop Safari"],
+          },
+        },
+      ],
   /* Folder for test artifacts such as screenshots, videos, traces, etc. */
   // outputDir: 'test-results/',
 
@@ -67,6 +74,8 @@ const config: PlaywrightTestConfig = {
     command: "dispatch server develop",
     url: "http://localhost:8080/",
     reuseExistingServer: !process.env.CI,
+    /* Increase timeout to allow server to fully start and settle - especially important after removing the 2-minute wait from tests */
+    timeout: 240 * 1000, // 4 minutes to ensure server is fully ready
   },
 }
 

tests/static/e2e/pages/auth-page.ts

@@ -51,9 +51,27 @@
     ])
   }
 
+  /**
+   * Wait for the application to be fully ready by checking that all necessary elements are loaded
+   * and the page is stable. This replaces the previous 2-minute static wait.
+   */
+  async waitForAppReady() {
+    // Wait for the page to be loaded and interactive
+    await this.page.waitForLoadState("networkidle")
+
+    // Additional wait to ensure any background initialization is complete
+    // This is much shorter than the previous 2-minute wait but ensures stability
+    await this.page.waitForTimeout(5000)
+
+    // Verify the basic UI elements are working by ensuring we can navigate to login
+    await this.page.goto(this.loginRoute, { waitUntil: "networkidle" })
+    await expect(this.loginHeader).toBeVisible()
+  }
+
   async registerNewUser(email: string, password: string) {
-    // wait for 2 minutes to let server settle
-    await new Promise(resolve => setTimeout(resolve, 120000));
+    // Ensure the application is ready before attempting registration
+    await this.waitForAppReady()
+
     await this.gotoRegisterWithLink()
     await this.emailLabel.first().click()
     await this.emailLabel.fill(email)
@@ -63,7 +81,7 @@
 
     await Promise.all([
       this.registerButton.click(),
       this.page.waitForURL(orgSlug + Routes.Dashboards),
     ])
   }