Move integration tester from typespec-azure

.github/workflows/external-integration.yml

@@ -66,3 +66,47 @@ jobs:
 
       - name: E2E Test
         run: pnpm test:e2e
+
+  azure-rest-api-specs:
+    name: Azure REST API Specs
+    runs-on: ubuntu-latest
+    if: contains(github.event.pull_request.labels.*.name, 'int:azure-specs') || github.event_name == 'workflow_dispatch'
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          submodules: true
+          fetch-depth: 0
+          ref: ${{ github.event.pull_request.head.sha || github.sha }}
+
+      - name: Setup Node.js and pnpm
+        uses: ./.github/actions/setup
+
+      - name: Install dependencies
+        run: pnpm install
+
+      - name: Build and pack TypeSpec Azure packages
+        run: |
+          pnpm --filter "!@typespec/playground-website" --filter "!@typespec/website"  -r build
+
+          pnpm chronus pack --pack-destination ./tgz-packages --exclude standalone
+
+          echo "Created tgz packages:"
+          ls -la ./tgz-packages/
+
+          cd packages/tsp-integration
+          npm link
+
+      - name: Checkout
+        run: tsp-integration azure-specs --stage checkout
+
+      - name: Patch package.json
+        run: tsp-integration azure-specs --stage patch --stage install --tgz-dir ./tgz-packages
+
+      - name: Run TypeSpec validation in azure-rest-api-specs
+        run: tsp-integration azure-specs --stage validate
+
+      - name: Check for git changes
+        if: success() || failure() # Still run this step even if validation fails to ensure as much information as possible
+        run: tsp-integration azure-specs --stage validate:clean

.typespec-integration/config.yaml

@@ -0,0 +1,17 @@
+suites:
+  azure-specs:
+    repo: https://github.com/Azure/azure-rest-api-specs
+    branch: typespec-next
+    pattern: "specification/**/tspconfig.yaml"
+    entrypoints:
+      - name: "client.tsp"
+        options: ["--no-emit"]
+      - name: "main.tsp"
+  azure-specs-pr:
+    repo: https://github.com/Azure/azure-rest-api-specs-pr
+    branch: typespec-next
+    pattern: "specification/**/tspconfig.yaml"
+    entrypoints:
+      - name: "client.tsp"
+        options: ["--no-emit"]
+      - name: "main.tsp"

eng/common/config/labels.ts

@@ -218,6 +218,10 @@ export default defineConfig({
           color: "0969da",
           description: "Good candidate for MQ",
         },
+        "int:azure-specs": {
+          color: "0e8a16",
+          description: "Run integration tests against azure-rest-api-specs",
+        },
       },
     },
   },

packages/tsp-integration/cmd/tsp-integration.js

@@ -0,0 +1,2 @@
+#!/usr/bin/env node
+import "../dist/cli.js";

packages/tsp-integration/package.json

@@ -0,0 +1,44 @@
+{
+  "name": "@azure-tools/integration-tester",
+  "private": true,
+  "version": "0.1.0",
+  "type": "module",
+  "description": "CLI tool for testing typespec package changes against external repos",
+  "homepage": "https://github.com/microsoft/typespec",
+  "license": "MIT",
+  "author": "Microsoft",
+  "files": [
+    "dist"
+  ],
+  "bin": {
+    "tsp-integration": "./cmd/tsp-integration.js"
+  },
+  "repository": "https://github.com/microsoft/typespec.git",
+  "engines": {
+    "node": ">=20.0.0"
+  },
+  "scripts": {
+    "watch": "tsc -p ./tsconfig.build.json --watch",
+    "build": "tsc -p ./tsconfig.build.json",
+    "clean": "rimraf dist/ temp/",
+    "test": "vitest run",
+    "test:watch": "vitest -w"
+  },
+  "dependencies": {
+    "@pnpm/workspace.find-packages": "^1000.0.24",
+    "execa": "^9.6.1",
+    "globby": "~16.1.0",
+    "log-symbols": "^7.0.1",
+    "ora": "^9.0.0",
+    "pathe": "^2.0.3",
+    "picocolors": "~1.1.1",
+    "simple-git": "^3.28.0",
+    "tar": "^7.5.2",
+    "yaml": "~2.8.2"
+  },
+  "devDependencies": {
+    "typescript": "~5.9.2",
+    "vitest": "^4.0.15"
+  },
+  "bugs": "https://github.com/microsoft/typespec/issues"
+}

packages/tsp-integration/src/cli.ts

@@ -0,0 +1,72 @@
+import { readFile } from "node:fs/promises";
+import { parseArgs } from "node:util";
+import { join, resolve } from "pathe";
+import { parse } from "yaml";
+import { runIntegrationTestSuite, Stages, type Stage } from "./run.js";
+import { ValidationFailedError } from "./utils.js";
+
+process.on("SIGINT", () => process.exit(0));
+
+const args = parseArgs({
+  args: process.argv.slice(2),
+  allowPositionals: true,
+  options: {
+    clean: {
+      type: "boolean",
+      default: false,
+    },
+    stage: {
+      type: "string",
+      multiple: true,
+    },
+    "tgz-dir": {
+      type: "string",
+    },
+    repo: {
+      type: "string",
+      description: "The path to the repository to test. Defaults temp/{suiteName}.",
+    },
+    interactive: {
+      type: "boolean",
+      default: false,
+      short: "i",
+      description: "Enable interactive mode for validation.",
+    },
+  },
+});
+
+const cwd = process.cwd();
+const integrationDir = join(cwd, ".typespec-integration");
+const suiteName = args.positionals[0];
+const config = parse(await readFile(join(integrationDir, "config.yaml"), "utf8"));
+const suite = config.suites[suiteName];
+if (suite === undefined) {
+  throw new Error(`Integration test suite "${suiteName}" not found in config.`);
+}
+
+let stages: Stage[] | undefined = undefined;
+if (args.values.stage) {
+  stages = args.values.stage as Stage[];
+  for (const stage of stages) {
+    if (!Stages.includes(stage)) {
+      throw new Error(
+        `Invalid stage "${stage}" specified. Valid stages are: ${Stages.join(", ")}.`,
+      );
+    }
+  }
+}
+
+const wd = args.values.repo ?? join(integrationDir, "temp", suiteName);
+try {
+  await runIntegrationTestSuite(wd, suiteName, suite, {
+    clean: args.values.clean,
+    stages,
+    tgzDir: args.values["tgz-dir"] && resolve(process.cwd(), args.values["tgz-dir"]),
+    interactive: args.values.interactive,
+  });
+} catch (error) {
+  if (error instanceof ValidationFailedError) {
+    process.exit(1);
+  }
+  throw error;
+}

packages/tsp-integration/src/config/types.ts

@@ -0,0 +1,15 @@
+export interface IntegrationTestsConfig {
+  suites: Record<string, IntegrationTestSuite>;
+}
+
+export interface IntegrationTestSuite {
+  repo: string;
+  branch: string;
+  pattern?: string;
+  entrypoints?: Entrypoint[];
+}
+
+export interface Entrypoint {
+  name: string;
+  options?: string[];
+}

packages/tsp-integration/src/find-packages.ts

@@ -0,0 +1,157 @@
+import { findWorkspacePackagesNoCheck } from "@pnpm/workspace.find-packages";
+import { readdir } from "node:fs/promises";
+import { relative, resolve } from "pathe";
+import pc from "picocolors";
+import * as tar from "tar";
+import { log } from "./utils.js";
+
+/**
+ * Collection of packages indexed by package name.
+ */
+export interface Packages {
+  [key: string]: {
+    /** The package name (e.g., "@typespec/compiler") */
+    name: string;
+    /** Absolute path to the package directory or .tgz file */
+    path: string;
+  };
+}
+
+/**
+ * Options for {@link findPackages}
+ */
+export interface FindPackageOptions {
+  /** Directory containing PNPM workspace to scan for packages */
+  wsDir?: string;
+  /** Directory containing .tgz artifact files */
+  tgzDir?: string;
+}
+
+/**
+ * Finds packages from either a workspace directory or tgz artifact directory.
+ *
+ * @param options - Configuration specifying the source to find packages from
+ * @returns Promise resolving to a collection of discovered packages
+ * @throws Error if neither wsDir nor tgzDir is provided
+ */
+export function findPackages(options: FindPackageOptions): Promise<Packages> {
+  if (options.tgzDir) {
+    return findPackagesFromTgzArtifactDir(options.tgzDir);
+  }
+  if (options.wsDir) {
+    return findPackagesFromWorkspace(options.wsDir);
+  } else {
+    throw new Error("Either wsDir or tgzDir must be provided to findPackages");
+  }
+}
+
+/**
+ * Prints a formatted list of discovered packages to the console.
+ *
+ * @param packages - Collection of packages to display
+ */
+export function printPackages(packages: Packages): void {
+  log("Found packages:");
+  for (const [name, pkg] of Object.entries(packages)) {
+    log(`  ${pc.green(name)}: ${pc.cyan(relative(process.cwd(), pkg.path))}`);
+  }
+}
+
+/**
+ * Discovers packages from a directory containing .tgz artifact files.
+ *
+ * This function scans a directory for .tgz files and extracts package information
+ * by reading the package.json from within each tar file.
+ *
+ * @param tgzDir - Directory containing .tgz artifact files
+ * @returns Promise resolving to discovered packages with paths pointing to .tgz files
+ */
+export async function findPackagesFromTgzArtifactDir(tgzDir: string): Promise<Packages> {
+  const packages: Packages = {};
+
+  const items = await readdir(tgzDir, { withFileTypes: true });
+  const tgzFiles = items
+    .filter((item) => item.isFile() && item.name.endsWith(".tgz"))
+    .map((item) => item.name);
+
+  // Process tar files in parallel
+  await Promise.all(
+    tgzFiles.map(async (tgzFile) => {
+      const fullPath = resolve(tgzDir, tgzFile);
+      const packageName = await extractPackageNameFromTgzFile(fullPath);
+
+      if (packageName) {
+        packages[packageName] = {
+          name: packageName,
+          path: fullPath,
+        };
+      }
+    }),
+  );
+
+  return packages;
+}
+
+/**
+ * Extracts the package name by reading package.json from a .tgz file.
+ *
+ * This function reads the package.json file from the root of the tar archive
+ * to get the accurate package name, which is more reliable than parsing filenames.
+ *
+ * @param tgzFilePath - Path to the .tgz file
+ * @returns Promise resolving to the package name, or null if not found
+ */
+async function extractPackageNameFromTgzFile(tgzFilePath: string): Promise<string | null> {
+  try {
+    let packageJsonContent: string | null = null;
+
+    await tar.t({
+      file: tgzFilePath,
+      // cspell:ignore onentry
+      onentry: (entry) => {
+        if (entry.path === "package/package.json") {
+          entry.on("data", (chunk) => {
+            if (packageJsonContent === null) {
+              packageJsonContent = "";
+            }
+            packageJsonContent += chunk.toString();
+          });
+        }
+      },
+    });
+
+    if (packageJsonContent) {
+      const packageJson = JSON.parse(packageJsonContent);
+      return packageJson.name || null;
+    }
+
+    return null;
+  } catch (error) {
+    throw new Error(`Failed to read package.json from ${tgzFilePath}: ${error}`);
+  }
+}
+
+/**
+ * Discovers packages from a PNPM workspace configuration.
+ *
+ * This function uses PNPM's workspace discovery to find all packages in a monorepo.
+ * It filters out private packages and packages without names.
+ *
+ * @param root - Root directory of the PNPM workspace
+ * @returns Promise resolving to discovered packages with paths pointing to package directories
+ */
+export async function findPackagesFromWorkspace(root: string): Promise<Packages> {
+  const pnpmPackages = await findWorkspacePackagesNoCheck(root);
+  const packages: Packages = {};
+
+  for (const pkg of pnpmPackages) {
+    if (!pkg.manifest.name || pkg.manifest.private) continue;
+
+    packages[pkg.manifest.name] = {
+      name: pkg.manifest.name,
+      path: pkg.rootDirRealPath,
+    };
+  }
+
+  return packages;
+}