Skip to content

Commit f086129

Browse files
WebReflectionWebReflection
committed
Add experimental_remote_packages config flag for remote packages
1 parent 64e82c4 commit f086129

6 files changed

Lines changed: 103 additions & 56 deletions

File tree

‎docs/index.js‎

Lines changed: 1 addition & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎docs/index.js.map‎

Lines changed: 1 addition & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎esm/interpreter/_remote_package.js‎

Lines changed: 59 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -1,41 +1,74 @@
1-
// hackity hack hack for PyScript FUN
1+
import fetch from '@webreflection/fetch';
22

33
import { toml } from '../3rd-party.js';
44

5-
const remote = async (config, packages) => {
5+
const { parse } = JSON;
6+
7+
const href = (key, pkg) => new URL(key, pkg).href;
8+
9+
const addPath = (target, key, value) => {
10+
if (key in target)
11+
throw new Error(`Duplicated path: ${key}`);
12+
target[key] = value;
13+
};
14+
15+
const addPaths = (target, source, pkg) => {
16+
for (const key in source)
17+
addPath(target, href(key, pkg), source[key]);
18+
};
19+
20+
const pollute = (t_js_modules, s_js_modules, name, pkg) => {
21+
const source = s_js_modules[name];
22+
if (source) {
23+
t_js_modules[name] ??= {};
24+
addPaths(t_js_modules[name], source, pkg);
25+
}
26+
};
27+
28+
const remote = async (
29+
config,
30+
packages = config.packages,
31+
set = new Set(),
32+
) => {
633
const repackaged = [];
734
for (const pkg of packages) {
8-
if (pkg.endsWith('.toml')) {
9-
const text = await (await fetch(pkg)).text();
10-
const { files, js_modules, packages } = await toml(text);
11-
if (packages)
12-
repackaged.push(...(await remote(config, packages)));
35+
// avoid re-processing already processed packages
36+
if (set.has(pkg)) continue;
37+
set.add(pkg);
38+
const isTOML = pkg.endsWith('.toml');
39+
if (isTOML || pkg.endsWith('.json')) {
40+
const text = await fetch(pkg).text();
41+
const {
42+
name,
43+
files,
44+
js_modules,
45+
packages,
46+
} = isTOML ? await toml(text) : parse(text);
47+
48+
if (set.has(name))
49+
throw new Error(`Unable to process ${name} @ ${pkg}`);
50+
51+
set.add(name);
52+
53+
if (packages) {
54+
// process nested packages from the remote config
55+
repackaged.push(...(await remote(config, packages, set)));
56+
}
57+
1358
if (js_modules) {
14-
if (!config.js_modules) config.js_modules = {};
15-
const { main, worker } = js_modules;
16-
if (main) {
17-
if (!config.js_modules.main) config.js_modules.main = {};
18-
for (const key in main) {
19-
config.js_modules.main[new URL(key, pkg).href] = main[key];
20-
}
21-
}
22-
if (worker) {
23-
if (!config.js_modules.worker) config.js_modules.worker = {};
24-
for (const key in worker) {
25-
config.js_modules.worker[new URL(key, pkg).href] = worker[key];
26-
}
27-
}
59+
config.js_modules ??= {};
60+
pollute(config.js_modules, js_modules, 'main', pkg);
61+
pollute(config.js_modules, js_modules, 'worker', pkg);
2862
}
63+
2964
if (files) {
30-
if (!config.files) config.files = {};
31-
for (const key in files) {
32-
config.files[new URL(key, pkg).href] = files[key];
33-
}
65+
config.files ??= {};
66+
addPaths(config.files, files, pkg);
3467
}
3568
}
3669
else repackaged.push(pkg);
3770
}
38-
return [...new Set(repackaged)];
71+
return repackaged;
3972
};
4073

4174
export default remote;

‎esm/interpreter/micropython.js‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ import mip from '../python/mip.js';
88
import { zip } from '../3rd-party.js';
99

1010
import { initializeNativeFS } from './_nativefs.js';
11+
import _remote_package from './_remote_package.js';
1112

1213
const type = 'micropython';
1314

@@ -44,6 +45,11 @@ export default {
4445
// Install Micropython Package
4546
this.writeFile(interpreter, './mip.py', mip);
4647
if (config.packages) {
48+
if (config.experimental_remote_packages) {
49+
progress('Loading remote packages');
50+
config.packages = await _remote_package(config);
51+
progress('Loaded remote packages');
52+
}
4753
progress('Loading packages');
4854
await py_imports(config.packages.map(fixedRelative, baseURL));
4955
progress('Loaded packages');

‎esm/interpreter/pyodide.js‎

Lines changed: 35 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -85,47 +85,55 @@ export default {
8585
// https://github.com/pyodide/pyodide/issues/5736
8686
const save = config.packages_cache !== 'never' && version !== '0.28.0';
8787
await storage.sync();
88+
progress('Loaded Storage');
8889
// packages_cache = 'never' means: erase the whole DB
8990
if (!save) storage.clear();
9091
// otherwise check if cache is known
91-
else if (packages) {
92-
config.packages = packages = await _remote_package(config, packages);
93-
// packages_cache = 'passthrough' means: do not use micropip.install
94-
if (config.packages_cache === 'passthrough') {
95-
options.packages = packages;
96-
packages = null;
97-
storage.clear();
92+
if (packages) {
93+
if (config.experimental_remote_packages) {
94+
progress('Loading remote packages');
95+
config.packages = (packages = await _remote_package(config, packages));
96+
progress('Loaded remote packages');
9897
}
99-
else {
100-
packages = packages.sort();
101-
// packages are uniquely stored as JSON key
102-
const key = stringify(packages);
103-
if (storage.has(key)) {
104-
const value = storage.get(key);
98+
if (save) {
99+
// packages_cache = 'passthrough' means: do not use micropip.install
100+
if (config.packages_cache === 'passthrough') {
101+
options.packages = packages;
102+
packages = null;
103+
storage.clear();
104+
}
105+
else {
106+
packages = packages.sort();
107+
// packages are uniquely stored as JSON key
108+
const key = stringify(packages);
109+
if (storage.has(key)) {
110+
const value = storage.get(key);
105111

106-
// versions are not currently understood by pyodide when
107-
// a lockFileURL is used instead of micropip.install(packages)
108-
// https://github.com/pyodide/pyodide/issues/5135#issuecomment-2441038644
109-
// https://github.com/pyscript/pyscript/issues/2245
110-
options.packages = packages.map(name => name.split(/[>=<]=/)[0]);
112+
// versions are not currently understood by pyodide when
113+
// a lockFileURL is used instead of micropip.install(packages)
114+
// https://github.com/pyodide/pyodide/issues/5135#issuecomment-2441038644
115+
// https://github.com/pyscript/pyscript/issues/2245
116+
options.packages = packages.map(name => name.split(/[>=<]=/)[0]);
111117

112-
if (version.startsWith('0.27')) {
113-
const blob = new Blob([value], { type: 'application/json' });
114-
options.lockFileURL = URL.createObjectURL(blob);
115-
}
116-
else {
117-
options.lockFileContents = value;
118-
}
118+
if (version.startsWith('0.27')) {
119+
const blob = new Blob([value], { type: 'application/json' });
120+
options.lockFileURL = URL.createObjectURL(blob);
121+
}
122+
else {
123+
options.lockFileContents = value;
124+
}
119125

120-
packages = null;
126+
packages = null;
127+
}
121128
}
122129
}
123130
}
124-
progress('Loaded Storage');
125131
const { stderr, stdout, get } = stdio();
132+
progress('Loading interpreter');
126133
const interpreter = await get(
127134
loadPyodide({ stderr, stdout, ...options }),
128135
);
136+
progress('Loaded interpreter');
129137
globalThis[js_modules].set('-T-', this.transform.bind(this, interpreter));
130138
if (config.debug) interpreter.setDebug(true);
131139
const py_imports = importPackages.bind(interpreter);

‎package.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -95,6 +95,6 @@
9595
"to-json-callback": "^0.1.1"
9696
},
9797
"worker": {
98-
"blob": "sha256-Ld3btStbQlVEeRf5aj1sn88sm3AJewGCIOfEDMQXijc="
98+
"blob": "sha256-u4s/ss3EOnkTX4hrZ/GUovLYI9GxTeDHu+26khZiSnA="
9999
}
100100
}

0 commit comments

Comments
 (0)