Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
32 commits
Select commit Hold shift + click to select a range
5c8a5bd
initial pipeline and upload script
JennyPng Mar 27, 2026
2e7394e
minor
JennyPng Mar 30, 2026
57ea584
1es pipeline, minor clean
JennyPng Mar 30, 2026
fa39d2a
minor comment cleanup
JennyPng Mar 30, 2026
e185330
refactor binary asset uploading
JennyPng Mar 30, 2026
0ce3283
minor, use indexversion
JennyPng Mar 30, 2026
4222592
catch error on wheel finding, read peerdeps from json
JennyPng Mar 30, 2026
1a72243
parallelize azure-tools bundling
JennyPng Mar 30, 2026
4ba8baf
Merge branch 'main' into playground-upload-python
JennyPng Apr 1, 2026
f36a9b8
remove unnecessary things
JennyPng Apr 1, 2026
1fd1a02
copy over changes for generic emitter bundling
JennyPng Apr 1, 2026
671b9d4
initial python-specific updates
JennyPng Apr 1, 2026
9808aa2
minor fix
JennyPng Apr 1, 2026
c609405
remove leading slash
JennyPng Apr 1, 2026
66cc2e9
resolve ci complaints
JennyPng Apr 2, 2026
20979c9
upload pygen whl as zip
JennyPng Apr 2, 2026
31c70f7
update version to preview for testing
JennyPng Apr 2, 2026
6a02c27
copy over changes to make python browser-compatible
JennyPng Apr 2, 2026
a9ef8b2
build pygen wheel url
JennyPng Apr 2, 2026
d3fbecc
lock file
JennyPng Apr 2, 2026
aa4458c
attempt to resolve ci complaints abt unbuilt emitter
JennyPng Apr 3, 2026
04e6fb0
undo smth wrong
JennyPng Apr 3, 2026
3d76d57
matrix
JennyPng Apr 3, 2026
13cde79
more pipeline update
JennyPng Apr 6, 2026
e0bf95b
pipeline fix, prettier on emitter
JennyPng Apr 6, 2026
cacf4f8
Merge branch 'main' into playground-upload-python
JennyPng Apr 6, 2026
09fe269
merge fix
JennyPng Apr 6, 2026
afba0dd
gitignore generated files and use action templat
JennyPng Apr 6, 2026
6176533
external integration fix
JennyPng Apr 6, 2026
bd085aa
use catalog for deps
JennyPng Apr 6, 2026
e6bd83e
undo catalog, doesnt work w azure-tools?
JennyPng Apr 6, 2026
f5048ff
changelog
JennyPng Apr 6, 2026
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
---
changeKind: internal
packages:
- "@typespec/http-client-python"
- "@typespec/bundle-uploader"
- "@typespec/playground-website"
---

Extend publish pipeline to upload emitter bundles to Playground storage account. Update Python emitter to be browser-compatible for use in the TypeSpec playground.
16 changes: 16 additions & 0 deletions .github/actions/build-playground-emitters/action.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
name: Build playground emitters
description: Build emitter packages required by the playground (excluded from pnpm workspace)

runs:
using: composite

steps:
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: "3.x"

- name: Build http-client-python emitter
shell: bash
working-directory: packages/http-client-python
run: npm ci && npm run build
6 changes: 6 additions & 0 deletions .github/workflows/core-ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,8 @@ jobs:
- name: Install dependencies
run: pnpm install

- uses: ./.github/actions/build-playground-emitters

- name: Restore .NET dependencies
run: dotnet restore
working-directory: packages/typespec-vs
Expand Down Expand Up @@ -72,6 +74,8 @@ jobs:
- name: Install dependencies
run: pnpm install

- uses: ./.github/actions/build-playground-emitters

- name: Install Playwright browsers
run: npx playwright install --with-deps

Expand All @@ -98,6 +102,8 @@ jobs:
- name: Install dependencies
run: pnpm install

- uses: ./.github/actions/build-playground-emitters

- name: Install Playwright browsers
run: |
sudo dpkg --configure -a
Expand Down
3 changes: 3 additions & 0 deletions .github/workflows/external-integration.yml
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,9 @@ jobs:
- name: Install Playwright
run: pnpm exec playwright install

- name: Build playground emitters
run: cd core/packages/http-client-python && npm ci && npm run build

- name: Build
run: pnpm build

Expand Down
28 changes: 28 additions & 0 deletions eng/emitters/pipelines/templates/stages/emitter-stages.yml
Original file line number Diff line number Diff line change
Expand Up @@ -82,6 +82,11 @@ parameters:
type: string
default: "3.12"

# Whether to bundle and upload the emitter package to the playground package storage.
- name: UploadPlaygroundBundle
type: boolean
default: false

stages:
# Build stage
# Responsible for building the autorest generator and typespec emitter packages
Expand Down Expand Up @@ -333,6 +338,29 @@ stages:
ArtifactPath: $(buildArtifactsPath)
LanguageShortName: ${{ parameters.LanguageShortName }}

- ${{ if parameters.UploadPlaygroundBundle }}:
- script: npm ci
displayName: Install emitter dependencies for playground bundle
workingDirectory: $(Build.SourcesDirectory)/${{ parameters.PackagePath }}
- script: npm run build
displayName: Build emitter for playground bundle
workingDirectory: $(Build.SourcesDirectory)/${{ parameters.PackagePath }}
- script: npm install -g pnpm
displayName: Install pnpm for playground bundle upload
- script: pnpm install --filter "@typespec/bundle-uploader..."
displayName: Install bundle-uploader dependencies
workingDirectory: $(Build.SourcesDirectory)
- script: pnpm --filter "@typespec/bundle-uploader..." build
displayName: Build bundle-uploader
workingDirectory: $(Build.SourcesDirectory)
- task: AzureCLI@1
displayName: Upload playground bundle
inputs:
azureSubscription: "Azure SDK Engineering System"
scriptLocation: inlineScript
inlineScript: node ./eng/emitters/scripts/upload-bundled-emitter.js ${{ parameters.PackagePath }}
workingDirectory: $(Build.SourcesDirectory)

templateContext:
outputs:
- output: pipelineArtifact
Expand Down
18 changes: 18 additions & 0 deletions eng/emitters/scripts/upload-bundled-emitter.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
// @ts-check
import { resolve } from "path";
import { bundleAndUploadStandalonePackage } from "../../../packages/bundle-uploader/dist/src/index.js";
import { repoRoot } from "../../common/scripts/helpers.js";

const packageRelativePath = process.argv[2];
if (!packageRelativePath) {
// eslint-disable-next-line no-console
console.error("Usage: node upload-bundled-emitter.js <package-path>");
// eslint-disable-next-line no-console
console.error(" e.g. node upload-bundled-emitter.js packages/http-client-csharp");
process.exit(1);
}

// remove leading slash if exists, then resolve to absolute path
const packagePath = resolve(repoRoot, packageRelativePath.replace(/^\//, ""));

await bundleAndUploadStandalonePackage({ packagePath });
2 changes: 2 additions & 0 deletions eng/tsp-core/pipelines/jobs/build-and-test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,8 @@ steps:
nodeVersion: ${{ parameters.nodeVersion }}
useDotnet: true

- template: /eng/tsp-core/pipelines/templates/build-playground-emitters.yml

- template: /eng/tsp-core/pipelines/templates/build.yml

- script: pnpm run test:ci
Expand Down
1 change: 1 addition & 0 deletions eng/tsp-core/pipelines/jobs/e2e.yml
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ jobs:

steps:
- template: /eng/tsp-core/pipelines/templates/install.yml
- template: /eng/tsp-core/pipelines/templates/build-playground-emitters.yml
- template: /eng/tsp-core/pipelines/templates/install-browsers.yml
- template: /eng/tsp-core/pipelines/templates/setup-linux-ui.yml
- template: /eng/tsp-core/pipelines/templates/build.yml
Expand Down
1 change: 1 addition & 0 deletions eng/tsp-core/pipelines/jobs/website.yml
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
steps:
- template: /eng/tsp-core/pipelines/templates/install.yml
- template: /eng/tsp-core/pipelines/templates/build-playground-emitters.yml

- script: pnpm exec playwright install
displayName: Install browsers
Expand Down
1 change: 1 addition & 0 deletions eng/tsp-core/pipelines/pr-tools.yml
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ extends:
- template: /eng/tsp-core/pipelines/templates/install.yml
- template: /eng/tsp-core/pipelines/templates/install-browsers.yml

- template: /eng/tsp-core/pipelines/templates/build-playground-emitters.yml
- template: /eng/tsp-core/pipelines/templates/build.yml

- task: AzureCLI@1
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
# Build emitter packages required by the playground (excluded from pnpm workspace)
steps:
- task: UsePythonVersion@0
displayName: Set up Python
inputs:
versionSpec: "3.x"

- script: cd packages/http-client-python && npm ci && npm run build
displayName: Build http-client-python emitter
1 change: 1 addition & 0 deletions packages/bundle-uploader/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@
"@azure/storage-blob": "catalog:",
"@pnpm/workspace.find-packages": "catalog:",
"@typespec/bundler": "workspace:^",
"globby": "catalog:",
"json5": "catalog:",
"picocolors": "catalog:",
"semver": "catalog:"
Expand Down
142 changes: 139 additions & 3 deletions packages/bundle-uploader/src/index.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,9 @@
import { AzureCliCredential } from "@azure/identity";
import { findWorkspacePackagesNoCheck } from "@pnpm/workspace.find-packages";
import { createTypeSpecBundle } from "@typespec/bundler";
import { resolve } from "path";
import { readFile } from "fs/promises";
import { globby } from "globby";
import { relative, resolve } from "path";
import { join as joinUnix } from "path/posix";
import pc from "picocolors";
import { parse } from "semver";
Expand All @@ -16,6 +18,137 @@ function logSuccess(message: string) {
logInfo(pc.green(`✔ ${message}`));
}

export interface PlaygroundAssetConfig {
/** Glob pattern relative to the package root (e.g. "generator/dist/pygen-*.whl"). */
path: string;
/** MIME content type for the blob upload. */
contentType: string;
}

export interface PlaygroundConfig {
/** Static files to upload as binary blobs. Paths support simple glob patterns. */
assets?: PlaygroundAssetConfig[];
/** Peer dependencies that should be bundled and uploaded. */
bundlePeerDependencies?: string[];
}

export interface BundleAndUploadStandalonePackageOptions {
/**
* Absolute path to the package directory.
*/
packagePath: string;
}

/**
* Bundle and upload a standalone package that is not part of the pnpm workspace.
* Uploads the bundle files and writes a `latest.json` under the package's blob directory
* (e.g. `@typespec/http-client-csharp/latest.json`).
*
* If the package's `package.json` contains a `playgroundConfig` section, this function
* will also upload static assets (resolved via glob patterns) and bundle peer dependencies.
*/
export async function bundleAndUploadStandalonePackage({
packagePath,
}: BundleAndUploadStandalonePackageOptions) {
const pkgJsonPath = resolve(packagePath, "package.json");
const pkgJson = JSON.parse(await readFile(pkgJsonPath, "utf-8"));
const playgroundConfig: PlaygroundConfig | undefined = pkgJson.playgroundConfig;

const bundle = await createTypeSpecBundle(packagePath);
const manifest = bundle.manifest;
logInfo(`Bundling standalone package: ${manifest.name}@${manifest.version}`);

const uploader = new TypeSpecBundledPackageUploader(new AzureCliCredential());
await uploader.createIfNotExists();

const result = await uploader.upload(bundle);
if (result.status === "uploaded") {
logSuccess(`Bundle for package ${manifest.name}@${manifest.version} uploaded.`);
} else {
logInfo(`Bundle for package ${manifest.name} already exists for version ${manifest.version}.`);
}

const importMap: Record<string, string> = {};
for (const [key, value] of Object.entries(result.imports)) {
importMap[joinUnix(manifest.name, key)] = value;
}

await uploadPlaygroundAssets(uploader, packagePath, manifest, importMap, playgroundConfig);

await uploader.updatePackageLatest(manifest.name, {
version: manifest.version,
imports: importMap,
});
logSuccess(`Updated ${manifest.name}/latest.json for version ${manifest.version}.`);
}

/**
* Upload playground assets and bundle peer dependencies based on the provided config.
*/
async function uploadPlaygroundAssets(
uploader: TypeSpecBundledPackageUploader,
packagePath: string,
manifest: { name: string; version: string },
importMap: Record<string, string>,
config: PlaygroundConfig | undefined,
) {
if (!config) {
return;
}

// Upload static assets (e.g. .whl files)
if (config.assets) {
for (const asset of config.assets) {
const matchedFiles = await globby(asset.path, { cwd: packagePath, absolute: true });
if (matchedFiles.length === 0) {
logInfo(pc.yellow(`⚠ No files matched asset pattern: ${asset.path}`));
continue;
}
for (const filePath of matchedFiles) {
const relativePath = relative(packagePath, filePath).replace(/\\/g, "/");
const blobPath = joinUnix(manifest.name, manifest.version, relativePath);
const content = await readFile(filePath);
const assetResult = await uploader.uploadBinaryAsset(blobPath, content, asset.contentType);
const importKey = joinUnix(manifest.name, relativePath);
importMap[importKey] = assetResult.url;
if (assetResult.status === "uploaded") {
logSuccess(`Uploaded asset: ${relativePath}`);
} else {
logInfo(`Asset already exists: ${relativePath}`);
}
}
}
}

// Bundle and upload peer dependencies
if (config.bundlePeerDependencies) {
for (const depName of config.bundlePeerDependencies) {
const depPath = resolve(packagePath, "node_modules", depName);
try {
const depBundle = await createTypeSpecBundle(depPath);
const depResult = await uploader.upload(depBundle);
if (depResult.status === "uploaded") {
logSuccess(
`Bundle for peer dep ${depBundle.manifest.name}@${depBundle.manifest.version} uploaded.`,
);
} else {
logInfo(
`Bundle for peer dep ${depBundle.manifest.name} already exists for version ${depBundle.manifest.version}.`,
);
}
for (const [key, value] of Object.entries(depResult.imports)) {
importMap[joinUnix(depBundle.manifest.name, key)] = value;
}
} catch (e: unknown) {
throw new Error(
`Failed to bundle peer dependency ${depName}: ${e instanceof Error ? e.message : e}`,
{ cause: e },
);
}
}
}
}

export interface BundleAndUploadPackagesOptions {
repoRoot: string;
/**
Expand Down Expand Up @@ -85,9 +218,12 @@ export async function bundleAndUploadPackages({
}
}
logInfo(`Import map for ${indexVersion}:`, importMap);
await uploader.updateIndex(indexName, {
const index = {
version: indexVersion,
imports: importMap,
});
};
await uploader.updateIndex(indexName, index);
logSuccess(`Updated index for version ${indexVersion}.`);
await uploader.updateLatestIndex(indexName, index);
logSuccess(`Updated latest index for version ${indexVersion}.`);
}
Loading
Loading