[python] test infra improvements

.chronus/changes/python-testInfraImprovements-2026-2-31-14-30-46.md

@@ -0,0 +1,7 @@
+---
+changeKind: internal
+packages:
+  - "@typespec/http-client-python"
+---
+
+Clean up test infrastructure, run in parallel, and use uv pip when acceptable. CI runs 2-3 x faster

.gitignore

@@ -228,8 +228,8 @@ BenchmarkDotnet.Artifacts/
 !packages/http-client-java/package-lock.json
 
 # python emitter
-packages/http-client-python/generator/test/**/generated/
-packages/http-client-python/generator/test/**/cadl-ranch-coverage.json
+packages/http-client-python/tests/**/generated/
+packages/http-client-python/tests/**/cadl-ranch-coverage.json
 !packages/http-client-python/package-lock.json
 packages/http-client-python/micropip.lock
 packages/http-client-python/venv_build_wheel/

cspell.yaml

@@ -276,6 +276,7 @@ words:
   - unassignable
   - Uncapitalize
   - uncollapsed
+  - unconfigure
   - undifferentiable
   - undoc
   - Ungroup
@@ -307,6 +308,7 @@ words:
   - WHATWG
   - WINDOWSARMVMIMAGE
   - WINDOWSVMIMAGE
+  - workerid
   - xiangyan
   - xiaofei
   - xlarge

packages/http-client-python/emitter/src/code-model.ts

@@ -25,14 +25,7 @@ import {
   emitPagingHttpMethod,
 } from "./http.js";
 import { PythonSdkContext } from "./lib.js";
-import {
-  KnownTypes,
-  disableGenerationMap,
-  emitEndpointType,
-  getType,
-  simpleTypesMap,
-  typesMap,
-} from "./types.js";
+import { KnownTypes, emitEndpointType, getType } from "./types.js";
 import { emitParamBase, getClientNamespace, getImplementation, getRootNamespace } from "./utils.js";
 
 function emitBasicMethod<TServiceOperation extends SdkServiceOperation>(
@@ -366,7 +359,7 @@ export function emitCodeModel(sdkContext: PythonSdkContext) {
       continue;
     }
     // filter out specific models not used in python, e.g., pageable models
-    if (disableGenerationMap.has(model)) {
+    if (sdkContext.__disableGenerationMap.has(model)) {
       continue;
     }
     // filter out core models
@@ -391,7 +384,7 @@ export function emitCodeModel(sdkContext: PythonSdkContext) {
   }
 
   // clear usage when a model is only used by paging
-  for (const type of typesMap.values()) {
+  for (const type of sdkContext.__typesMap.values()) {
     if (
       type["type"] === "model" &&
       type["referredByOperationType"] === ReferredByOperationTypes.PagingOnly &&
@@ -402,9 +395,9 @@ export function emitCodeModel(sdkContext: PythonSdkContext) {
   }
 
   codeModel["types"] = [
-    ...typesMap.values(),
+    ...sdkContext.__typesMap.values(),
     ...Object.values(KnownTypes),
-    ...simpleTypesMap.values(),
+    ...sdkContext.__simpleTypesMap.values(),
   ];
   codeModel["crossLanguagePackageId"] = ignoreDiagnostics(getCrossLanguagePackageId(sdkContext));
   if ((sdkContext.emitContext.options as any).flavor === "azure") {

packages/http-client-python/emitter/src/emitter.ts

@@ -10,7 +10,6 @@ import { emitCodeModel } from "./code-model.js";
 import { saveCodeModelAsYaml } from "./external-process.js";
 import { PythonEmitterOptions, PythonSdkContext, reportDiagnostic } from "./lib.js";
 import { runPython3 } from "./run-python3.js";
-import { disableGenerationMap, simpleTypesMap, typesMap } from "./types.js";
 import { getRootNamespace, md2Rst } from "./utils.js";
 
 function addDefaultOptions(sdkContext: PythonSdkContext) {
@@ -59,6 +58,9 @@ async function createPythonSdkContext(
   return {
     ...sdkContext,
     __endpointPathParameters: [],
+    __typesMap: new Map(),
+    __simpleTypesMap: new Map(),
+    __disableGenerationMap: new Set(),
   };
 }
 
@@ -100,12 +102,6 @@ function walkThroughNodes(yamlMap: Record<string, any>): Record<string, any> {
   return yamlMap;
 }
 
-function cleanAllCache() {
-  typesMap.clear();
-  simpleTypesMap.clear();
-  disableGenerationMap.clear();
-}
-
 export async function $onEmit(context: EmitContext<PythonEmitterOptions>) {
   try {
     await onEmitMain(context);
@@ -124,9 +120,6 @@ export async function $onEmit(context: EmitContext<PythonEmitterOptions>) {
 }
 
 async function onEmitMain(context: EmitContext<PythonEmitterOptions>) {
-  // clean all cache to make sure emitter could work in watch mode
-  cleanAllCache();
-
   const program = context.program;
   const sdkContext = await createPythonSdkContext(context);
   const root = path.join(dirname(fileURLToPath(import.meta.url)), "..", "..");
@@ -168,7 +161,7 @@ async function onEmitMain(context: EmitContext<PythonEmitterOptions>) {
       try {
         await runPython3(path.join(root, "/eng/scripts/setup/install.py"));
         await runPython3(path.join(root, "/eng/scripts/setup/prepare.py"));
-      } catch (error) {
+      } catch {
         // if the python env is not ready, we use pyodide instead
         resolvedOptions["use-pyodide"] = true;
       }
@@ -250,7 +243,7 @@ async function onEmitMain(context: EmitContext<PythonEmitterOptions>) {
       execSync(
         `${venvPath} -m black --line-length=120 --quiet --fast ${outputDir} --exclude "${excludePattern}"`,
       );
-      checkForPylintIssues(outputDir, excludePattern);
+      await checkForPylintIssues(outputDir, excludePattern);
     }
   }
 }
@@ -269,7 +262,7 @@ async function setupPyodideCall(root: string) {
         if (lockAge > 300) {
           fs.unlinkSync(micropipLockPath);
         }
-      } catch (err) {
+      } catch {
         // ignore
       }
     }
@@ -288,14 +281,14 @@ async function setupPyodideCall(root: string) {
       fs.closeSync(fd);
       fs.unlinkSync(micropipLockPath);
       break;
-    } catch (err) {
+    } catch {
       await new Promise((resolve) => setTimeout(resolve, 1000));
     }
   }
   return pyodide;
 }
 
-function checkForPylintIssues(outputDir: string, excludePattern: string) {
+async function checkForPylintIssues(outputDir: string, excludePattern: string) {
   const excludeRegex = new RegExp(excludePattern);
 
   const shouldExcludePath = (filePath: string): boolean => {
@@ -304,9 +297,8 @@ function checkForPylintIssues(outputDir: string, excludePattern: string) {
     return excludeRegex.test(normalizedPath);
   };
 
-  const processFile = (filePath: string) => {
-    let fileContent;
-    fileContent = fs.readFileSync(filePath, "utf-8");
+  const processFile = async (filePath: string) => {
+    let fileContent = await fs.promises.readFile(filePath, "utf-8");
     const pylintDisables: string[] = [];
     const lineEnding = fileContent.includes("\r\n") && os.platform() === "win32" ? "\r\n" : "\n";
     const lines: string[] = fileContent.split(lineEnding);
@@ -321,32 +313,38 @@ function checkForPylintIssues(outputDir: string, excludePattern: string) {
         fileContent = lines[0].includes("pylint: disable=")
           ? [lines[0] + "," + pylintDisables.join(",")].concat(lines.slice(1)).join(lineEnding)
           : `# pylint: disable=${pylintDisables.join(",")}${lineEnding}` + fileContent;
+        await fs.promises.writeFile(filePath, fileContent);
       }
     }
-
-    fs.writeFileSync(filePath, fileContent);
   };
 
-  const walkDir = (dir: string) => {
+  const collectPythonFiles = async (dir: string): Promise<string[]> => {
     if (shouldExcludePath(dir)) {
-      return;
+      return [];
     }
 
-    const files = fs.readdirSync(dir);
-    files.forEach((file) => {
-      const filePath = path.join(dir, file);
+    const entries = await fs.promises.readdir(dir, { withFileTypes: true });
+
+    const promises = entries.map(async (entry) => {
+      const filePath = path.join(dir, entry.name);
 
       if (shouldExcludePath(filePath)) {
-        return;
+        return [];
       }
 
-      if (fs.statSync(filePath).isDirectory()) {
-        walkDir(filePath);
-      } else if (file.endsWith(".py")) {
-        processFile(filePath);
+      if (entry.isDirectory()) {
+        return collectPythonFiles(filePath);
+      } else if (entry.name.endsWith(".py")) {
+        return [filePath];
       }
+      return [];
     });
+
+    const results = await Promise.all(promises);
+    return results.flat();
   };
 
-  walkDir(outputDir);
+  // Collect all Python files first, then process in parallel
+  const pythonFiles = await collectPythonFiles(outputDir);
+  await Promise.all(pythonFiles.map(processFile));
 }

packages/http-client-python/emitter/src/lib.ts

@@ -1,5 +1,6 @@
 import {
   SdkContext,
+  SdkType,
   UnbrandedSdkEmitterOptions,
 } from "@azure-tools/typespec-client-generator-core";
 import { createTypeSpecLibrary, JSONSchemaType, paramMessage } from "@typespec/compiler";
@@ -28,6 +29,9 @@ export interface PythonEmitterOptions {
 
 export interface PythonSdkContext extends SdkContext<PythonEmitterOptions> {
   __endpointPathParameters: Record<string, any>[];
+  __typesMap: Map<SdkType, Record<string, any>>;
+  __simpleTypesMap: Map<string | null, Record<string, any>>;
+  __disableGenerationMap: Set<SdkType>;
 }
 
 export const PythonEmitterOptionsSchema: JSONSchemaType<PythonEmitterOptions> = {

packages/http-client-python/emitter/src/system-requirements.ts

@@ -16,8 +16,8 @@ const execute = (
       options.onCreate(cp);
     }
 
-    options.onStdOutData && cp.stdout.on("data", options.onStdOutData);
-    options.onStdErrData && cp.stderr.on("data", options.onStdErrData);
+    if (options.onStdOutData) cp.stdout.on("data", options.onStdOutData);
+    if (options.onStdErrData) cp.stderr.on("data", options.onStdErrData);
 
     let err = "";
     let out = "";
@@ -34,7 +34,7 @@ const execute = (
     cp.on("error", (err) => {
       reject(err);
     });
-    cp.on("close", (code, signal) =>
+    cp.on("close", (code, _signal) =>
       resolve({
         stdout: out,
         stderr: err,
@@ -110,7 +110,7 @@ const tryPython = async (
       `"${PRINT_PYTHON_VERSION_SCRIPT}"`,
     ]);
     return validateVersionRequirement(resolution, result.stdout.trim(), requirement);
-  } catch (e) {
+  } catch {
     return {
       error: true,
       ...resolution,