-
Notifications
You must be signed in to change notification settings - Fork 100
SeBS Cloudflare Compatibility #274
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: master
Are you sure you want to change the base?
Changes from all commits
9a9d42e
aa24a07
4cc0476
cd24fcf
eaa42a1
57452fa
9e47e0f
822a9d9
f7bb950
1f0a979
b117e75
d42b157
ffd3f78
272a372
556d799
93c8a73
e17982f
214c947
b8f7c5c
dba2992
5390021
24497a2
8812708
5b3d784
cd183b8
9379f39
5f9ad9c
92db5ae
51892b0
3235d3f
416b67b
5284880
b6de39b
812f592
9229f9f
5899d87
6e0cd2b
3cd741f
2615a36
e69243a
f39aad0
e76f846
dc2f6ed
437cc97
1eb375c
6c0768e
0dfcfa8
b2465f9
0eb4d0b
db84f2d
7e2d8ac
35a556d
92c5dea
bcd5ecb
96ac2c1
b028151
03e274e
a11236a
35755d6
b427c5b
734eadf
5ffcb06
4da0c31
4874794
6b8e695
865ca06
20eb8db
ebe0794
9dd0a6e
b4f08a1
fa7b2f5
1311f20
02cb35a
60aa631
cf9d333
cf5a547
91bb9a1
8ea37c5
a60e5d4
18070f0
044b9ef
7ac2b8c
78b2979
6d7e4d0
2cc8f93
3f8e69c
c8ca384
b71e2c8
53a1cd2
32debbf
e22bb62
acf2e33
e4b2abf
28d90d7
7517eaa
6eae47c
6a3a8db
95fdaba
6b9434e
8966909
3665df8
2dafdf9
eb21ce5
0be8706
5908cef
2040294
6d8a1c6
6a5ef6b
9ed8f9c
26ea601
f14bce9
5a1fdcc
bcd3dae
4f7a279
8acec87
812fe72
d8f08db
97bb674
e69c836
42eef0e
4a7f036
cd1def7
9b9894a
6d94293
31624ec
0253844
b84c330
b7ee302
8bc3595
4574d8c
220574c
4a3fc61
33441b2
ac51c3e
2ce61f3
184695c
38cd1c0
360b2a2
51f1c3b
76d5d17
4af9a0f
d6baba3
cec0f9d
89270cf
cfb935c
ff45c7a
fad8b40
fbe8185
bf1b0ac
584a3f8
bd2111d
c476ac8
b9eb11f
e34c67d
cd5992b
6500fd2
16e7454
d3e6568
9681b6e
4d2db1f
eda4d0f
305c4fb
1e402e4
9340696
867f6b7
c076d79
7c8867b
617e7d2
88e232b
606e1ac
c123da6
c9c5372
2053171
50ca4d6
a083ef3
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,6 +1,6 @@ | ||
| { | ||
| "timeout": 120, | ||
| "memory": 128, | ||
| "languages": ["python", "nodejs"], | ||
| "languages": ["python"], | ||
| "modules": [] | ||
| } |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,6 +1,22 @@ | ||
| { | ||
| "timeout": 10, | ||
| "memory": 128, | ||
| "languages": ["python", "nodejs", "java"], | ||
| "languages": [ | ||
| { | ||
| "language": "python", | ||
| "variants": { | ||
| "default": "default", | ||
| "cloudflare": "default" | ||
| } | ||
| }, | ||
| { | ||
| "language": "nodejs", | ||
| "variants": { | ||
| "default": "default", | ||
| "cloudflare": "default" | ||
| } | ||
| }, | ||
| "java" | ||
| ], | ||
| "modules": [] | ||
| } |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,6 +1,21 @@ | ||
| { | ||
| "timeout": 60, | ||
| "memory": 128, | ||
| "languages": ["python", "nodejs"], | ||
| "languages": [ | ||
| { | ||
| "language": "python", | ||
| "variants": { | ||
| "default": "default", | ||
| "cloudflare": {"workers": "cloudflare", "containers": "default"} | ||
| } | ||
| }, | ||
| { | ||
| "language": "nodejs", | ||
| "variants": { | ||
| "default": "default", | ||
| "cloudflare": {"workers": "cloudflare", "containers": "default"} | ||
| } | ||
| } | ||
| ], | ||
| "modules": ["storage"] | ||
| } | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,35 @@ | ||
| // Copyright 2020-2025 ETH Zurich and the SeBS authors. All rights reserved. | ||
| // Cloudflare Workers differ from the default Node.js version: Workers require | ||
| // ES module syntax (no CommonJS `require`) and do not ship the `request` npm | ||
| // package, so we use the platform-native `fetch` API and buffer the response | ||
| // into /tmp instead of piping a stream. | ||
| import * as fs from 'node:fs'; | ||
|
Collaborator
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Can we add a short comment - in both Python and Node.js - why Cloudflare version is different (and how)?
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Added short comments at the top of both files describing the runtime constraint (ES-module-only Workers lacking the |
||
| import * as path from 'node:path'; | ||
| import { storage } from './storage'; | ||
|
|
||
| let storage_handler = new storage(); | ||
|
|
||
| export const handler = async function(event) { | ||
| let bucket = event.bucket.bucket; | ||
| let output_prefix = event.bucket.output; | ||
| let url = event.object.url; | ||
| let upload_key = path.basename(url); | ||
| let download_path = path.join('/tmp', upload_key); | ||
|
|
||
| const response = await fetch(url, { | ||
| headers: { | ||
| 'User-Agent': 'SeBS/1.2 (https://github.com/spcl/serverless-benchmarks) SeBS Benchmark Suite/1.2' | ||
| } | ||
| }); | ||
| const buffer = await response.arrayBuffer(); | ||
| fs.writeFileSync(download_path, Buffer.from(buffer)); | ||
|
|
||
| let [keyName, uploadPromise] = storage_handler.upload( | ||
| bucket, | ||
| path.join(output_prefix, upload_key), | ||
| download_path | ||
| ); | ||
| await uploadPromise; | ||
|
|
||
| return {result: {bucket: bucket, url: url, key: keyName}}; | ||
| }; | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,61 @@ | ||
| # Copyright 2020-2025 ETH Zurich and the SeBS authors. All rights reserved. | ||
| # Cloudflare Workers differ from the default Python version: the Workers | ||
| # Python runtime is Pyodide-based and does not support `urllib.request`, so | ||
| # we download via Pyodide's async `pyfetch` and wrap it with `run_sync` to | ||
| # keep the synchronous handler signature. | ||
|
|
||
| import datetime | ||
| import os | ||
|
|
||
| from pyodide.ffi import run_sync | ||
| from pyodide.http import pyfetch | ||
|
|
||
| from . import storage | ||
| client = storage.storage.get_instance() | ||
|
|
||
| SEBS_USER_AGENT = "SeBS/1.2 (https://github.com/spcl/serverless-benchmarks) SeBS Benchmark Suite/1.2" | ||
|
|
||
| async def do_request(url, download_path): | ||
| headers = {'User-Agent': SEBS_USER_AGENT} | ||
|
|
||
| res = await pyfetch(url, headers=headers) | ||
| bs = await res.bytes() | ||
|
|
||
| with open(download_path, 'wb') as f: | ||
| f.write(bs) | ||
|
Comment on lines
+21
to
+25
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Validate HTTP download success before writing and uploading. Right now any HTTP response body (including 404/500 pages) is written and then uploaded as if it were valid content. Add an explicit status check and fail fast on non-success responses. 🔧 Proposed fix async def do_request(url, download_path):
headers = {'User-Agent': SEBS_USER_AGENT}
res = await pyfetch(url, headers=headers)
+ if not res.ok:
+ raise RuntimeError(f"Download failed with status {res.status} for URL: {url}")
bs = await res.bytes()
with open(download_path, 'wb') as f:
f.write(bs)🤖 Prompt for AI Agents |
||
|
|
||
| def handler(event): | ||
|
|
||
| bucket = event.get('bucket').get('bucket') | ||
| output_prefix = event.get('bucket').get('output') | ||
| url = event.get('object').get('url') | ||
| name = os.path.basename(url) | ||
| download_path = '/tmp/{}'.format(name) | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Use a secure temporary file path instead of a predictable The current path is predictable and can collide across concurrent invocations. Use 🔒 Proposed fix import datetime
import os
+import tempfile
@@
- name = os.path.basename(url)
- download_path = '/tmp/{}'.format(name)
+ name = os.path.basename(url)
+ fd, download_path = tempfile.mkstemp(prefix="sebs-", suffix=f"-{name}", dir="/tmp")
+ os.close(fd)🧰 Tools🪛 Ruff (0.15.6)[error] 28-28: Probable insecure usage of temporary file or directory: "/tmp/{}" (S108) 🤖 Prompt for AI Agents |
||
|
|
||
| process_begin = datetime.datetime.now() | ||
|
|
||
| run_sync(do_request(url, download_path)) | ||
|
|
||
| size = os.path.getsize(download_path) | ||
| process_end = datetime.datetime.now() | ||
|
|
||
| upload_begin = datetime.datetime.now() | ||
| key_name = client.upload(bucket, os.path.join(output_prefix, name), download_path) | ||
| upload_end = datetime.datetime.now() | ||
|
|
||
| process_time = (process_end - process_begin) / datetime.timedelta(microseconds=1) | ||
| upload_time = (upload_end - upload_begin) / datetime.timedelta(microseconds=1) | ||
| return { | ||
| 'result': { | ||
| 'bucket': bucket, | ||
| 'url': url, | ||
| 'key': key_name | ||
| }, | ||
| 'measurement': { | ||
| 'download_time': 0, | ||
| 'download_size': 0, | ||
| 'upload_time': upload_time, | ||
| 'upload_size': size, | ||
| 'compute_time': process_time | ||
| } | ||
| } | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,7 +1,23 @@ | ||
| { | ||
| "timeout": 60, | ||
| "memory": 256, | ||
| "languages": ["python", "nodejs", "cpp"], | ||
| "languages": [ | ||
| { | ||
| "language": "python", | ||
| "variants": { | ||
| "default": "default", | ||
| "cloudflare": "default" | ||
| } | ||
| }, | ||
| { | ||
| "language": "nodejs", | ||
| "variants": { | ||
| "default": "default", | ||
| "cloudflare": "default" | ||
| } | ||
| }, | ||
| "cpp" | ||
| ], | ||
| "modules": ["storage"], | ||
| "cpp_dependencies": ["sdk", "opencv", "libjpeg-turbo", "boost"] | ||
| } |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,6 +1,14 @@ | ||
| { | ||
| "timeout": 60, | ||
| "memory": 512, | ||
| "languages": ["python"], | ||
| "languages": [ | ||
| { | ||
| "language": "python", | ||
| "variants": { | ||
| "default": "default", | ||
| "cloudflare": "default" | ||
| } | ||
| } | ||
| ], | ||
| "modules": ["storage"] | ||
| } |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,6 +1,22 @@ | ||
| { | ||
| "timeout": 60, | ||
| "memory": 256, | ||
| "languages": ["python", "nodejs"], | ||
| "languages": [ | ||
| { | ||
| "language": "python", | ||
| "variants": { | ||
| "default": "default", | ||
| "cloudflare": "default" | ||
| } | ||
| }, | ||
| { | ||
| "language": "nodejs", | ||
| "variants": { | ||
| "default": "default", | ||
| "cloudflare": {"workers": "default", "containers": "default"} | ||
| } | ||
| } | ||
| ], | ||
| "modules": ["storage"] | ||
| } | ||
|
|
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,7 +1,16 @@ | ||
| { | ||
| "timeout": 60, | ||
| "memory": 768, | ||
| "languages": ["python", "cpp"], | ||
| "languages": [ | ||
| { | ||
| "language": "python", | ||
| "variants": { | ||
| "default": "default", | ||
| "cloudflare": {"workers": "default", "containers": "cloudflare"} | ||
| } | ||
| }, | ||
| "cpp" | ||
| ], | ||
| "modules": ["storage"], | ||
| "cpp_dependencies": ["sdk", "torch", "opencv"] | ||
| } |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,6 @@ | ||
| # Copyright 2020-2025 ETH Zurich and the SeBS authors. All rights reserved. | ||
| pillow==10.3.0 | ||
| torch==2.0.0 | ||
| torchvision==0.15.1 | ||
| # prevent installing numpy 2.0 | ||
| numpy==1.24.0 |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,6 @@ | ||
| # Copyright 2020-2025 ETH Zurich and the SeBS authors. All rights reserved. | ||
| pillow==10.3.0 | ||
| torch==2.0.0 | ||
| torchvision==0.15.1 | ||
| # prevent installing numpy 2.0 | ||
| numpy==1.24.0 |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,7 +1,16 @@ | ||
| { | ||
| "timeout": 120, | ||
| "memory": 512, | ||
| "languages": ["python", "cpp"], | ||
| "languages": [ | ||
| { | ||
| "language": "python", | ||
| "variants": { | ||
| "default": "default", | ||
| "cloudflare": "default" | ||
| } | ||
| }, | ||
| "cpp" | ||
| ], | ||
| "modules": [], | ||
| "cpp_dependencies": ["igraph"] | ||
| } |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,6 +1,14 @@ | ||
| { | ||
| "timeout": 120, | ||
| "memory": 512, | ||
| "languages": ["python"], | ||
| "languages": [ | ||
| { | ||
| "language": "python", | ||
| "variants": { | ||
| "default": "default", | ||
| "cloudflare": "default" | ||
| } | ||
| } | ||
| ], | ||
| "modules": [] | ||
| } |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,7 +1,16 @@ | ||
| { | ||
| "timeout": 120, | ||
| "memory": 512, | ||
| "languages": ["python", "cpp"], | ||
| "languages": [ | ||
| { | ||
| "language": "python", | ||
| "variants": { | ||
| "default": "default", | ||
| "cloudflare": "default" | ||
| } | ||
| }, | ||
| "cpp" | ||
| ], | ||
| "modules": [], | ||
| "cpp_dependencies": ["igraph"] | ||
| } |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,6 +1,14 @@ | ||
| { | ||
| "timeout": 60, | ||
| "memory": 2048, | ||
| "languages": ["python"], | ||
| "languages": [ | ||
| { | ||
| "language": "python", | ||
| "variants": { | ||
| "default": "default", | ||
| "cloudflare": "default" | ||
| } | ||
| } | ||
| ], | ||
| "modules": ["storage"] | ||
| } |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Shouldn't this be a mapping to a directory? So, "default" value should be "python", because "python" contains the main implementation (same for "containers" key below). "default" mapping to "default" seems a bit weird :)
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
But the value is resolved as subdirectory of the language dir... so default would map to "no subdirectory", whereas the others map to their respective subdirectories. how about "null" for default so the mapping would be "default": null . this would make sense as per default we'd need "no subdirectory".