Migrate cucumber tests to e2e and vitest

package.json

@@ -29,6 +29,7 @@
     "shopify:run": "node packages/cli/bin/dev.js",
     "shopify": "nx build cli && node packages/cli/bin/dev.js",
     "test:e2e": "nx run-many --target=build --projects=cli,create-app --skip-nx-cache && pnpm --filter e2e exec playwright test",
+    "test:regenerate-snapshots": "packages/e2e/scripts/regenerate-snapshots.sh",
     "test:unit": "pnpm vitest run",
     "test": "pnpm vitest run",
     "type-check:affected": "nx affected --target=type-check",
@@ -172,6 +173,7 @@
         ],
         "ignoreBinaries": [
           "bin/*",
+          "packages/e2e/scripts/*",
           "shopify",
           "shopify-nightly",
           "brew",

packages/cli/src/cli/repo-health.test.ts

@@ -0,0 +1,111 @@
+/* eslint-disable no-restricted-imports, no-await-in-loop */
+import {describe, test, expect} from 'vitest'
+import glob from 'fast-glob'
+import * as fs from 'fs/promises'
+import * as path from 'path'
+
+const repoRoot = path.join(__dirname, '../../../..')
+
+describe('GitHub Actions pinning', () => {
+  test('all non-official actions are pinned to SHA', async () => {
+    const workflowDir = path.join(repoRoot, '.github/workflows')
+    const workflowFiles = await glob('*.yml', {cwd: workflowDir, absolute: true})
+    expect(workflowFiles.length).toBeGreaterThan(0)
+
+    const allActions: string[] = []
+    for (const file of workflowFiles) {
+      const content = await fs.readFile(file, 'utf-8')
+      const matches = content.match(/uses:\s+\S+/g) ?? []
+      allActions.push(...matches.map((match) => match.split(/\s+/)[1]!))
+    }
+
+    const thirdParty = allActions.filter(
+      (action) => !action.startsWith('actions/') && !action.startsWith('./') && !action.startsWith('Shopify/'),
+    )
+
+    const unpinned = thirdParty.filter((action) => !action.match(/^[^@]+@[0-9a-f]+/))
+
+    expect(
+      unpinned,
+      [
+        'The following unofficial GitHub actions have not been pinned:\n',
+        ...unpinned.map((el) => `  - ${el}\n`),
+        '\nRun bin/pin-github-actions.js, verify the action is not doing anything malicious, then commit your changes.',
+      ].join(''),
+    ).toHaveLength(0)
+  })
+})
+
+describe('Node dependency version sync', () => {
+  const sharedDependencies = [
+    '@babel/core',
+    '@oclif/core',
+    '@shopify/cli-kit',
+    '@types/node',
+    '@typescript-eslint/parser',
+    'esbuild',
+    'execa',
+    'fast-glob',
+    'graphql',
+    'graphql-request',
+    'graphql-tag',
+    'ink',
+    'liquidjs',
+    'node-fetch',
+    'typescript',
+    'vite',
+    'vitest',
+    'zod',
+  ]
+
+  interface PackageJson {
+    dependencies?: Record<string, string>
+    devDependencies?: Record<string, string>
+    peerDependencies?: Record<string, string>
+    resolutions?: Record<string, string>
+  }
+
+  test('shared dependencies are on the same version across packages', async () => {
+    const packageJsonPaths = await glob('packages/*/package.json', {cwd: repoRoot, absolute: true})
+    const packageJsonMap: Record<string, PackageJson> = {}
+
+    for (const pkgPath of packageJsonPaths) {
+      const name = path.dirname(pkgPath).split('/').pop()!
+      packageJsonMap[name] = JSON.parse(await fs.readFile(pkgPath, 'utf-8')) as PackageJson
+    }
+    packageJsonMap.root = JSON.parse(await fs.readFile(path.join(repoRoot, 'package.json'), 'utf-8')) as PackageJson
+
+    const different: {dep: string; versions: {packageName: string; version: string}[]}[] = []
+
+    for (const dep of sharedDependencies) {
+      const depVersions: {packageName: string; version: string}[] = []
+
+      for (const [packageName, json] of Object.entries(packageJsonMap)) {
+        const version =
+          json.dependencies?.[dep] ??
+          json.devDependencies?.[dep] ??
+          json.peerDependencies?.[dep] ??
+          json.resolutions?.[dep]
+        if (version) {
+          depVersions.push({packageName, version: version.replace(/^\^/, '')})
+        }
+      }
+
+      const uniqueVersions = [...new Set(depVersions.map((ver) => ver.version))]
+      if (uniqueVersions.length > 1) {
+        different.push({dep, versions: depVersions})
+      }
+    }
+
+    const errorMessage = [
+      'The following node dependencies are on different versions across packages:\n\n',
+      ...different.map(
+        ({dep, versions}) =>
+          `  - ${dep}:\n${versions.map(({packageName, version}) => `    - ${packageName}: ${version}`).join('\n')}`,
+      ),
+      '\n\nPlease make sure they are all on the same version.',
+    ].join('')
+
+    expect(different, errorMessage).toHaveLength(0)
+  })
+})

packages/e2e/data/snapshots/commands.txt

@@ -0,0 +1,111 @@
+├─ app
+│  ├─ build
+│  ├─ bulk
+│  │  ├─ cancel
+│  │  ├─ execute
+│  │  └─ status
+│  ├─ config
+│  │  ├─ link
+│  │  ├─ pull
+│  │  └─ use
+│  ├─ deploy
+│  ├─ dev
+│  │  └─ clean
+│  ├─ env
+│  │  ├─ pull
+│  │  └─ show
+│  ├─ execute
+│  ├─ function
+│  │  ├─ build
+│  │  ├─ info
+│  │  ├─ replay
+│  │  ├─ run
+│  │  ├─ schema
+│  │  └─ typegen
+│  ├─ generate
+│  │  └─ extension
+│  ├─ import-custom-data-definitions
+│  ├─ import-extensions
+│  ├─ info
+│  ├─ init
+│  ├─ logs
+│  │  └─ sources
+│  ├─ release
+│  ├─ versions
+│  │  └─ list
+│  └─ webhook
+│     └─ trigger
+├─ auth
+│  ├─ login
+│  └─ logout
+├─ commands
+├─ config
+│  └─ autocorrect
+│     ├─ off
+│     ├─ on
+│     └─ status
+├─ help
+├─ hydrogen
+│  ├─ build
+│  ├─ check
+│  ├─ codegen
+│  ├─ customer-account-push
+│  ├─ debug
+│  │  └─ cpu
+│  ├─ deploy
+│  ├─ dev
+│  ├─ env
+│  │  ├─ list
+│  │  ├─ pull
+│  │  └─ push
+│  ├─ generate
+│  │  ├─ route
+│  │  └─ routes
+│  ├─ init
+│  ├─ link
+│  ├─ list
+│  ├─ login
+│  ├─ logout
+│  ├─ preview
+│  ├─ setup
+│  │  ├─ css
+│  │  ├─ markets
+│  │  └─ vite
+│  ├─ shortcut
+│  ├─ unlink
+│  └─ upgrade
+├─ organization
+│  └─ list
+├─ plugins
+│  ├─ add
+│  ├─ inspect
+│  ├─ install
+│  ├─ link
+│  ├─ remove
+│  ├─ reset
+│  ├─ uninstall
+│  ├─ unlink
+│  └─ update
+├─ search
+├─ theme
+│  ├─ check
+│  ├─ console
+│  ├─ delete
+│  ├─ dev
+│  ├─ duplicate
+│  ├─ info
+│  ├─ init
+│  ├─ language-server
+│  ├─ list
+│  ├─ metafields
+│  │  └─ pull
+│  ├─ open
+│  ├─ package
+│  ├─ profile
+│  ├─ publish
+│  ├─ pull
+│  ├─ push
+│  ├─ rename
+│  └─ share
+├─ upgrade
+└─ version

packages/e2e/scripts/regenerate-snapshots.sh

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+# enter this dir so that we can run this script from the top-level
+cd "$(dirname "$0")"
+
+# regenerate commands snapshot file
+node ../../cli/bin/dev.js commands --tree > ../data/snapshots/commands.txt

packages/e2e/tests/commands.spec.ts

@@ -0,0 +1,38 @@
+/* eslint-disable no-restricted-imports */
+import {cliFixture as test} from '../setup/cli.js'
+import {expect} from '@playwright/test'
+import * as fs from 'fs/promises'
+import * as path from 'path'
+import {fileURLToPath} from 'url'
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url))
+const snapshotPath = path.join(__dirname, '../data/snapshots/commands.txt')
+
+const errorMessage = `
+SNAPSHOT TEST FAILED!
+
+The result of 'shopify commands --tree' has changed! We run this to check that
+all commands can load successfully.
+
+It's normal to see this test fail when you add or remove a command in the CLI.
+In this case you can run this command to regenerate the snapshot file:
+
+$ pnpm test:regenerate-snapshots
+
+Then you can commit this change and this test will pass.
+
+If instead you didn't mean to change a command, UH OH. Check the commands in
+the diff below and figure out what is broken.
+`
+
+const normalize = (value: string) => value.replace(/\r\n/g, '\n').trimEnd()
+
+test.describe('Command snapshot', () => {
+  test('shopify commands --tree matches snapshot', async ({cli}) => {
+    const result = await cli.exec(['commands', '--tree'])
+    expect(result.exitCode).toBe(0)
+
+    const snapshot = await fs.readFile(snapshotPath, {encoding: 'utf8'})
+    expect(normalize(result.stdout), errorMessage).toBe(normalize(snapshot))
+  })
+})