diff --git a/.github/workflows/_regression-job.yml b/.github/workflows/_regression-job.yml index 296b2221f..6c6f34c52 100644 --- a/.github/workflows/_regression-job.yml +++ b/.github/workflows/_regression-job.yml @@ -49,6 +49,14 @@ jobs: echo "AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SECRET_ACCESS_KEY }}" >> $GITHUB_ENV echo "AWS_DEFAULT_REGION=${{ secrets.AWS_DEFAULT_REGION || 'us-east-1' }}" >> $GITHUB_ENV + - name: Setup Cloudflare credentials + if: inputs.platform == 'cloudflare' + run: | + echo "CLOUDFLARE_API_TOKEN=${{ secrets.CLOUDFLARE_API_TOKEN }}" >> $GITHUB_ENV + echo "CLOUDFLARE_ACCOUNT_ID=${{ secrets.CLOUDFLARE_ACCOUNT_ID }}" >> $GITHUB_ENV + echo "CLOUDFLARE_R2_ACCESS_KEY_ID=${{ secrets.CLOUDFLARE_R2_ACCESS_KEY_ID }}" >> $GITHUB_ENV + echo "CLOUDFLARE_R2_SECRET_ACCESS_KEY=${{ secrets.CLOUDFLARE_R2_SECRET_ACCESS_KEY }}" >> $GITHUB_ENV + - name: Install uv uses: astral-sh/setup-uv@v4 diff --git a/.github/workflows/regression.yml b/.github/workflows/regression.yml index 429ae1e8e..14f4b1fa4 100644 --- a/.github/workflows/regression.yml +++ b/.github/workflows/regression.yml @@ -64,6 +64,14 @@ jobs: language: java version: "17" architecture: "x64" + - platform: cloudflare + language: python + version: "3.12" + architecture: "x64" + - platform: cloudflare + language: nodejs + version: "18" + architecture: "x64" fail-fast: false uses: ./.github/workflows/_regression-job.yml diff --git a/.mypy.ini b/.mypy.ini index 5d02772f9..7d2fe13ff 100644 --- a/.mypy.ini +++ b/.mypy.ini @@ -3,6 +3,9 @@ [mypy-docker] ignore_missing_imports = True +[mypy-docker.*] +ignore_missing_imports = True + [mypy-tzlocal] ignore_missing_imports = True diff --git a/benchmarks-data b/benchmarks-data index 30ca2f5c5..54ff4f3d9 160000 --- a/benchmarks-data +++ b/benchmarks-data @@ -1 +1 @@ -Subproject commit 30ca2f5c533c3f441deb5e05fc03a39fe65f9948 +Subproject commit 54ff4f3d9b7894e040bcdfdbcc8fe2f9f1168c1d diff --git a/benchmarks/000.microbenchmarks/040.server-reply/config.json b/benchmarks/000.microbenchmarks/040.server-reply/config.json index 93ce2f561..8ff6eec59 100644 --- a/benchmarks/000.microbenchmarks/040.server-reply/config.json +++ b/benchmarks/000.microbenchmarks/040.server-reply/config.json @@ -1,6 +1,6 @@ { "timeout": 120, "memory": 128, - "languages": ["python", "nodejs"], + "languages": ["python"], "modules": [] } diff --git a/benchmarks/100.webapps/110.dynamic-html/config.json b/benchmarks/100.webapps/110.dynamic-html/config.json index 7e317037b..dc5160394 100644 --- a/benchmarks/100.webapps/110.dynamic-html/config.json +++ b/benchmarks/100.webapps/110.dynamic-html/config.json @@ -1,6 +1,22 @@ { "timeout": 10, "memory": 128, - "languages": ["python", "nodejs", "java"], + "languages": [ + { + "language": "python", + "variants": { + "default": "default", + "cloudflare": "default" + } + }, + { + "language": "nodejs", + "variants": { + "default": "default", + "cloudflare": "default" + } + }, + "java" + ], "modules": [] } diff --git a/benchmarks/100.webapps/120.uploader/config.json b/benchmarks/100.webapps/120.uploader/config.json index 49e614ac7..23b7293df 100644 --- a/benchmarks/100.webapps/120.uploader/config.json +++ b/benchmarks/100.webapps/120.uploader/config.json @@ -1,6 +1,21 @@ { "timeout": 60, "memory": 128, - "languages": ["python", "nodejs"], + "languages": [ + { + "language": "python", + "variants": { + "default": "default", + "cloudflare": {"workers": "cloudflare", "containers": "default"} + } + }, + { + "language": "nodejs", + "variants": { + "default": "default", + "cloudflare": {"workers": "cloudflare", "containers": "default"} + } + } + ], "modules": ["storage"] } diff --git a/benchmarks/100.webapps/120.uploader/nodejs/cloudflare/function.js b/benchmarks/100.webapps/120.uploader/nodejs/cloudflare/function.js new file mode 100644 index 000000000..586129827 --- /dev/null +++ b/benchmarks/100.webapps/120.uploader/nodejs/cloudflare/function.js @@ -0,0 +1,35 @@ +// Copyright 2020-2025 ETH Zurich and the SeBS authors. All rights reserved. +// Cloudflare Workers differ from the default Node.js version: Workers require +// ES module syntax (no CommonJS `require`) and do not ship the `request` npm +// package, so we use the platform-native `fetch` API and buffer the response +// into /tmp instead of piping a stream. +import * as fs from 'node:fs'; +import * as path from 'node:path'; +import { storage } from './storage'; + +let storage_handler = new storage(); + +export const handler = async function(event) { + let bucket = event.bucket.bucket; + let output_prefix = event.bucket.output; + let url = event.object.url; + let upload_key = path.basename(url); + let download_path = path.join('/tmp', upload_key); + + const response = await fetch(url, { + headers: { + 'User-Agent': 'SeBS/1.2 (https://github.com/spcl/serverless-benchmarks) SeBS Benchmark Suite/1.2' + } + }); + const buffer = await response.arrayBuffer(); + fs.writeFileSync(download_path, Buffer.from(buffer)); + + let [keyName, uploadPromise] = storage_handler.upload( + bucket, + path.join(output_prefix, upload_key), + download_path + ); + await uploadPromise; + + return {result: {bucket: bucket, url: url, key: keyName}}; +}; diff --git a/benchmarks/100.webapps/120.uploader/python/cloudflare/function.py b/benchmarks/100.webapps/120.uploader/python/cloudflare/function.py new file mode 100644 index 000000000..e4028b14f --- /dev/null +++ b/benchmarks/100.webapps/120.uploader/python/cloudflare/function.py @@ -0,0 +1,61 @@ +# Copyright 2020-2025 ETH Zurich and the SeBS authors. All rights reserved. +# Cloudflare Workers differ from the default Python version: the Workers +# Python runtime is Pyodide-based and does not support `urllib.request`, so +# we download via Pyodide's async `pyfetch` and wrap it with `run_sync` to +# keep the synchronous handler signature. + +import datetime +import os + +from pyodide.ffi import run_sync +from pyodide.http import pyfetch + +from . import storage +client = storage.storage.get_instance() + +SEBS_USER_AGENT = "SeBS/1.2 (https://github.com/spcl/serverless-benchmarks) SeBS Benchmark Suite/1.2" + +async def do_request(url, download_path): + headers = {'User-Agent': SEBS_USER_AGENT} + + res = await pyfetch(url, headers=headers) + bs = await res.bytes() + + with open(download_path, 'wb') as f: + f.write(bs) + +def handler(event): + + bucket = event.get('bucket').get('bucket') + output_prefix = event.get('bucket').get('output') + url = event.get('object').get('url') + name = os.path.basename(url) + download_path = '/tmp/{}'.format(name) + + process_begin = datetime.datetime.now() + + run_sync(do_request(url, download_path)) + + size = os.path.getsize(download_path) + process_end = datetime.datetime.now() + + upload_begin = datetime.datetime.now() + key_name = client.upload(bucket, os.path.join(output_prefix, name), download_path) + upload_end = datetime.datetime.now() + + process_time = (process_end - process_begin) / datetime.timedelta(microseconds=1) + upload_time = (upload_end - upload_begin) / datetime.timedelta(microseconds=1) + return { + 'result': { + 'bucket': bucket, + 'url': url, + 'key': key_name + }, + 'measurement': { + 'download_time': 0, + 'download_size': 0, + 'upload_time': upload_time, + 'upload_size': size, + 'compute_time': process_time + } + } diff --git a/benchmarks/100.webapps/130.crud-api/config.json b/benchmarks/100.webapps/130.crud-api/config.json index 25c6cb05e..66532ad84 100644 --- a/benchmarks/100.webapps/130.crud-api/config.json +++ b/benchmarks/100.webapps/130.crud-api/config.json @@ -2,8 +2,20 @@ "timeout": 30, "memory": 128, "languages": [ - "python", - "nodejs" + { + "language": "python", + "variants": { + "default": "default", + "cloudflare": "default" + } + }, + { + "language": "nodejs", + "variants": { + "default": "default", + "cloudflare": "default" + } + } ], "modules": [ "nosql" diff --git a/benchmarks/200.multimedia/210.thumbnailer/config.json b/benchmarks/200.multimedia/210.thumbnailer/config.json index 7ba71f4dd..6bb025a9b 100644 --- a/benchmarks/200.multimedia/210.thumbnailer/config.json +++ b/benchmarks/200.multimedia/210.thumbnailer/config.json @@ -1,7 +1,23 @@ { "timeout": 60, "memory": 256, - "languages": ["python", "nodejs", "cpp"], + "languages": [ + { + "language": "python", + "variants": { + "default": "default", + "cloudflare": "default" + } + }, + { + "language": "nodejs", + "variants": { + "default": "default", + "cloudflare": "default" + } + }, + "cpp" + ], "modules": ["storage"], "cpp_dependencies": ["sdk", "opencv", "libjpeg-turbo", "boost"] } diff --git a/benchmarks/200.multimedia/220.video-processing/config.json b/benchmarks/200.multimedia/220.video-processing/config.json index 94ede7925..d9596b9ac 100644 --- a/benchmarks/200.multimedia/220.video-processing/config.json +++ b/benchmarks/200.multimedia/220.video-processing/config.json @@ -1,6 +1,14 @@ { "timeout": 60, "memory": 512, - "languages": ["python"], + "languages": [ + { + "language": "python", + "variants": { + "default": "default", + "cloudflare": "default" + } + } + ], "modules": ["storage"] } diff --git a/benchmarks/300.utilities/311.compression/config.json b/benchmarks/300.utilities/311.compression/config.json index 8edb99e52..0b20dbd82 100644 --- a/benchmarks/300.utilities/311.compression/config.json +++ b/benchmarks/300.utilities/311.compression/config.json @@ -1,6 +1,22 @@ { "timeout": 60, "memory": 256, - "languages": ["python", "nodejs"], + "languages": [ + { + "language": "python", + "variants": { + "default": "default", + "cloudflare": "default" + } + }, + { + "language": "nodejs", + "variants": { + "default": "default", + "cloudflare": {"workers": "cloudflare", "containers": "default"} + } + } + ], "modules": ["storage"] } + diff --git a/benchmarks/300.utilities/311.compression/nodejs/cloudflare/function.js b/benchmarks/300.utilities/311.compression/nodejs/cloudflare/function.js new file mode 100644 index 000000000..8ffba4601 --- /dev/null +++ b/benchmarks/300.utilities/311.compression/nodejs/cloudflare/function.js @@ -0,0 +1,100 @@ +// Cloudflare Workers variant: replaces archiver (which relies on Node streams +// and prototype inheritance that breaks under Workers) with fflate, a pure-JS +// zip library that runs without any Node-specific APIs. +import * as fs from 'node:fs'; +import * as path from 'node:path'; +import { zipSync, strToU8 } from 'fflate'; +import { v4 as uuidv4 } from 'uuid'; +import { storage } from './storage'; + +let storage_handler = new storage(); + +function parseDirectory(directory) { + let size = 0; + function walkDir(dir) { + const files = fs.readdirSync(dir); + for (const file of files) { + const filepath = path.join(dir, file); + const stat = fs.statSync(filepath); + if (stat.isDirectory()) { + walkDir(filepath); + } else { + size += stat.size; + } + } + } + walkDir(directory); + return size; +} + +function collectFiles(directory) { + const result = {}; + function walkDir(dir, prefix) { + const files = fs.readdirSync(dir); + for (const file of files) { + const filepath = path.join(dir, file); + const relPath = prefix ? `${prefix}/${file}` : file; + const stat = fs.statSync(filepath); + if (stat.isDirectory()) { + walkDir(filepath, relPath); + } else { + result[relPath] = [fs.readFileSync(filepath), { level: 9 }]; + } + } + } + walkDir(directory, ''); + return result; +} + +export const handler = async function(event) { + const bucket = event.bucket.bucket; + const input_prefix = event.bucket.input; + const output_prefix = event.bucket.output; + const key = event.object.key; + + const download_path = path.join('/tmp', `${key}-${uuidv4()}`); + fs.mkdirSync(download_path, { recursive: true }); + + const s3_download_begin = Date.now(); + await storage_handler.downloadDirectory(bucket, path.join(input_prefix, key), download_path); + const s3_download_stop = Date.now(); + + const size = parseDirectory(download_path); + + const compress_begin = Date.now(); + const archive_name = `${key}.zip`; + const archive_path = path.join('/tmp', archive_name); + const files = collectFiles(download_path); + const zipped = zipSync(files); + fs.writeFileSync(archive_path, zipped); + const compress_end = Date.now(); + + const archive_size = fs.statSync(archive_path).size; + + const s3_upload_begin = Date.now(); + const [key_name, uploadPromise] = storage_handler.upload( + bucket, + path.join(output_prefix, archive_name), + archive_path + ); + await uploadPromise; + const s3_upload_stop = Date.now(); + + const download_time = (s3_download_stop - s3_download_begin) * 1000; + const upload_time = (s3_upload_stop - s3_upload_begin) * 1000; + const process_time = (compress_end - compress_begin) * 1000; + + return { + result: { + bucket: bucket, + key: key_name + }, + measurement: { + download_time: download_time, + download_size: size, + upload_time: upload_time, + upload_size: archive_size, + compute_time: process_time + } + }; +}; diff --git a/benchmarks/300.utilities/311.compression/nodejs/cloudflare/package.json b/benchmarks/300.utilities/311.compression/nodejs/cloudflare/package.json new file mode 100644 index 000000000..0f455e790 --- /dev/null +++ b/benchmarks/300.utilities/311.compression/nodejs/cloudflare/package.json @@ -0,0 +1,9 @@ +{ + "name": "compression-benchmark", + "version": "1.0.0", + "description": "Compression benchmark for serverless platforms", + "dependencies": { + "fflate": "^0.8.2", + "uuid": "^10.0.0" + } +} diff --git a/benchmarks/400.inference/411.image-recognition/config.json b/benchmarks/400.inference/411.image-recognition/config.json index 8c5010fc3..61299bfd5 100644 --- a/benchmarks/400.inference/411.image-recognition/config.json +++ b/benchmarks/400.inference/411.image-recognition/config.json @@ -1,7 +1,16 @@ { "timeout": 60, "memory": 768, - "languages": ["python", "cpp"], + "languages": [ + { + "language": "python", + "variants": { + "default": "default", + "cloudflare": {"workers": "default", "containers": "cloudflare"} + } + }, + "cpp" + ], "modules": ["storage"], "cpp_dependencies": ["sdk", "torch", "opencv"] } diff --git a/benchmarks/400.inference/411.image-recognition/python/cloudflare/requirements.txt.3.11 b/benchmarks/400.inference/411.image-recognition/python/cloudflare/requirements.txt.3.11 new file mode 100644 index 000000000..c3e648b75 --- /dev/null +++ b/benchmarks/400.inference/411.image-recognition/python/cloudflare/requirements.txt.3.11 @@ -0,0 +1,6 @@ +# Copyright 2020-2025 ETH Zurich and the SeBS authors. All rights reserved. +pillow==10.3.0 +torch==2.0.0 +torchvision==0.15.1 +# prevent installing numpy 2.0 +numpy==1.24.0 diff --git a/benchmarks/400.inference/411.image-recognition/python/cloudflare/requirements.txt.3.12 b/benchmarks/400.inference/411.image-recognition/python/cloudflare/requirements.txt.3.12 new file mode 100644 index 000000000..c3e648b75 --- /dev/null +++ b/benchmarks/400.inference/411.image-recognition/python/cloudflare/requirements.txt.3.12 @@ -0,0 +1,6 @@ +# Copyright 2020-2025 ETH Zurich and the SeBS authors. All rights reserved. +pillow==10.3.0 +torch==2.0.0 +torchvision==0.15.1 +# prevent installing numpy 2.0 +numpy==1.24.0 diff --git a/benchmarks/500.scientific/501.graph-pagerank/config.json b/benchmarks/500.scientific/501.graph-pagerank/config.json index 90e8c7f86..2fbbec8c8 100644 --- a/benchmarks/500.scientific/501.graph-pagerank/config.json +++ b/benchmarks/500.scientific/501.graph-pagerank/config.json @@ -1,7 +1,16 @@ { "timeout": 120, "memory": 512, - "languages": ["python", "cpp"], + "languages": [ + { + "language": "python", + "variants": { + "default": "default", + "cloudflare": "default" + } + }, + "cpp" + ], "modules": [], "cpp_dependencies": ["igraph"] } diff --git a/benchmarks/500.scientific/502.graph-mst/config.json b/benchmarks/500.scientific/502.graph-mst/config.json index e80fb4351..9749feb38 100644 --- a/benchmarks/500.scientific/502.graph-mst/config.json +++ b/benchmarks/500.scientific/502.graph-mst/config.json @@ -1,6 +1,14 @@ { "timeout": 120, "memory": 512, - "languages": ["python"], + "languages": [ + { + "language": "python", + "variants": { + "default": "default", + "cloudflare": "default" + } + } + ], "modules": [] } diff --git a/benchmarks/500.scientific/503.graph-bfs/config.json b/benchmarks/500.scientific/503.graph-bfs/config.json index 90e8c7f86..2fbbec8c8 100644 --- a/benchmarks/500.scientific/503.graph-bfs/config.json +++ b/benchmarks/500.scientific/503.graph-bfs/config.json @@ -1,7 +1,16 @@ { "timeout": 120, "memory": 512, - "languages": ["python", "cpp"], + "languages": [ + { + "language": "python", + "variants": { + "default": "default", + "cloudflare": "default" + } + }, + "cpp" + ], "modules": [], "cpp_dependencies": ["igraph"] } diff --git a/benchmarks/500.scientific/504.dna-visualisation/config.json b/benchmarks/500.scientific/504.dna-visualisation/config.json index ff297ac5b..158e2ff59 100644 --- a/benchmarks/500.scientific/504.dna-visualisation/config.json +++ b/benchmarks/500.scientific/504.dna-visualisation/config.json @@ -1,6 +1,14 @@ { "timeout": 60, "memory": 2048, - "languages": ["python"], + "languages": [ + { + "language": "python", + "variants": { + "default": "default", + "cloudflare": "default" + } + } + ], "modules": ["storage"] } diff --git a/benchmarks/600.workflows/610.gen/config.json b/benchmarks/600.workflows/610.gen/config.json new file mode 100644 index 000000000..b0cea12a6 --- /dev/null +++ b/benchmarks/600.workflows/610.gen/config.json @@ -0,0 +1,7 @@ +{ + "timeout": 120, + "memory": 128, + "languages": ["python"], + "modules": [], + "cpp_dependencies": [] +} diff --git a/benchmarks/600.workflows/610.gen/definition.json b/benchmarks/600.workflows/610.gen/definition.json new file mode 100644 index 000000000..fcdf203bc --- /dev/null +++ b/benchmarks/600.workflows/610.gen/definition.json @@ -0,0 +1,54 @@ +{ + "root": "get_astros", + "states": { + "get_astros": { + "type": "task", + "func_name": "get_astros", + "next": "select_astros_number" + }, + "select_astros_number": { + "type": "switch", + "cases": [ + { + "var": "astros.number", + "op": "<", + "val": 10, + "next": "few_people" + }, + { + "var": "astros.number", + "op": ">=", + "val": 10, + "next": "many_people" + } + ], + "default": "few_people" + }, + "few_people": { + "type": "task", + "func_name": "few_people", + "next": "map_astros" + }, + "many_people": { + "type": "task", + "func_name": "many_people", + "next": "map_astros" + }, + "map_astros": { + "type": "map", + "array": "astros.people", + "root": "map_astros", + "next": "process_astros", + "states": { + "map_astros": { + "type": "task", + "func_name": "map_astros" + } + } + }, + "process_astros": { + "type": "task", + "func_name": "process_astros" + } + } +} diff --git a/benchmarks/600.workflows/610.gen/input.py b/benchmarks/600.workflows/610.gen/input.py new file mode 100644 index 000000000..000bc1362 --- /dev/null +++ b/benchmarks/600.workflows/610.gen/input.py @@ -0,0 +1,71 @@ +# Copyright 2020-2025 ETH Zurich and the SeBS authors. All rights reserved. + + +def buckets_count(): + return (0, 0) + + +def generate_input(data_dir, size, bucket, input_buckets, output_buckets, upload_func, nosql_func): + return dict() + + +def validate_output(data_dir: str | None, input_config: dict, output: dict, language: str, storage=None) -> str | None: + if output is None: + return "Output is None" + + if "done" not in output or output["done"] is not True: + return "Expected 'done' key to be True" + + if "astros" not in output: + return "Missing 'astros' key in output" + + # Output structure: output["astros"] is a dict with nested output["astros"]["astros"]["people"] + astros_outer = output["astros"] + if isinstance(astros_outer, dict): + inner = astros_outer.get("astros", {}) + people = inner.get("people", []) if isinstance(inner, dict) else [] + elif isinstance(astros_outer, list): + people = astros_outer + else: + return f"'astros' has unexpected type: {type(astros_outer).__name__}" + + if not isinstance(people, list): + return f"Expected people to be a list, got {type(people).__name__}" + + # The API response includes number and message at the inner astros level + if isinstance(astros_outer, dict): + inner = astros_outer.get("astros", {}) + if isinstance(inner, dict): + api_message = inner.get("message") + if api_message != "success": + return f"API 'message' field is '{api_message}', expected 'success'" + api_number = inner.get("number") + if api_number is not None and api_number != len(people): + return f"API 'number' field is {api_number} but people list has {len(people)} entries" + + for i, person in enumerate(people): + if not isinstance(person, dict): + return f"Element {i} is not a dict" + if "name" not in person: + return f"Element {i} missing 'name' field" + if "name_rev" not in person: + return f"Element {i} missing 'name_rev' field" + + name = person["name"] + name_rev = person["name_rev"] + + if not isinstance(name, str) or not name.strip(): + return f"Element {i} 'name' must be a non-empty string" + if not isinstance(name_rev, str) or not name_rev.strip(): + return f"Element {i} 'name_rev' must be a non-empty string" + + # name_rev splits on first space only: "First Last" -> "Last First" + parts = name.split(" ", 1) + expected_rev = " ".join(reversed(parts)) + if name_rev != expected_rev: + return f"Element {i} 'name_rev' is '{name_rev}', expected '{expected_rev}'" + + if "craft" not in person or not isinstance(person["craft"], str) or not person["craft"]: + return f"Element {i} missing or empty 'craft' field" + + return None diff --git a/benchmarks/600.workflows/610.gen/python/few_people.py b/benchmarks/600.workflows/610.gen/python/few_people.py new file mode 100644 index 000000000..9c70d9fbc --- /dev/null +++ b/benchmarks/600.workflows/610.gen/python/few_people.py @@ -0,0 +1,5 @@ +def handler(event): + return { + "many_astros": False, + **event + } \ No newline at end of file diff --git a/benchmarks/600.workflows/610.gen/python/get_astros.py b/benchmarks/600.workflows/610.gen/python/get_astros.py new file mode 100644 index 000000000..627c65231 --- /dev/null +++ b/benchmarks/600.workflows/610.gen/python/get_astros.py @@ -0,0 +1,8 @@ +import requests + +def handler(event): + res = requests.get("http://api.open-notify.org/astros.json") + + return { + "astros": res.json() + } \ No newline at end of file diff --git a/benchmarks/600.workflows/610.gen/python/many_people.py b/benchmarks/600.workflows/610.gen/python/many_people.py new file mode 100644 index 000000000..2d339f325 --- /dev/null +++ b/benchmarks/600.workflows/610.gen/python/many_people.py @@ -0,0 +1,5 @@ +def handler(event): + return { + "many_astros": True, + **event + } \ No newline at end of file diff --git a/benchmarks/600.workflows/610.gen/python/map_astros.py b/benchmarks/600.workflows/610.gen/python/map_astros.py new file mode 100644 index 000000000..5cfa4631b --- /dev/null +++ b/benchmarks/600.workflows/610.gen/python/map_astros.py @@ -0,0 +1,7 @@ +def handler(elem): + name = elem["name"] + parts = name.split(" ", 1) + name = " ".join(reversed(parts)) + elem["name_rev"] = name + + return elem \ No newline at end of file diff --git a/benchmarks/600.workflows/610.gen/python/process_astros.py b/benchmarks/600.workflows/610.gen/python/process_astros.py new file mode 100644 index 000000000..a981660e0 --- /dev/null +++ b/benchmarks/600.workflows/610.gen/python/process_astros.py @@ -0,0 +1,5 @@ +def handler(arr): + return { + "astros": arr, + "done": True + } \ No newline at end of file diff --git a/benchmarks/600.workflows/610.gen/python/requirements.txt b/benchmarks/600.workflows/610.gen/python/requirements.txt new file mode 100644 index 000000000..f2293605c --- /dev/null +++ b/benchmarks/600.workflows/610.gen/python/requirements.txt @@ -0,0 +1 @@ +requests diff --git a/benchmarks/600.workflows/6100.1000-genome/LICENSE b/benchmarks/600.workflows/6100.1000-genome/LICENSE new file mode 100644 index 000000000..8c0e74641 --- /dev/null +++ b/benchmarks/600.workflows/6100.1000-genome/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2007-2012 University Of Southern California + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/benchmarks/600.workflows/6100.1000-genome/README.md b/benchmarks/600.workflows/6100.1000-genome/README.md new file mode 100644 index 000000000..0719dbc0a --- /dev/null +++ b/benchmarks/600.workflows/6100.1000-genome/README.md @@ -0,0 +1 @@ +The implementation of the 1000Genome benchmark is based on the implementation of the 1000Genome workflow here: https://github.com/pegasus-isi/1000genome-workflow, with the license provided. diff --git a/benchmarks/600.workflows/6100.1000-genome/config.json b/benchmarks/600.workflows/6100.1000-genome/config.json new file mode 100644 index 000000000..5bb1699c5 --- /dev/null +++ b/benchmarks/600.workflows/6100.1000-genome/config.json @@ -0,0 +1,6 @@ +{ + "timeout": 900, + "memory": 2048, + "languages": ["python"], + "modules": ["storage"] +} diff --git a/benchmarks/600.workflows/6100.1000-genome/definition.json b/benchmarks/600.workflows/6100.1000-genome/definition.json new file mode 100644 index 000000000..e242fd1fb --- /dev/null +++ b/benchmarks/600.workflows/6100.1000-genome/definition.json @@ -0,0 +1,71 @@ +{ + "root": "individuals", + "states": { + "individuals": { + "type": "map", + "root": "individuals_func", + "array": "blob", + "common_params": "benchmark_bucket,bucket,columns,columns_bucket,populations,sifting_input,individuals_file", + "next": "merge_and_sifting", + "states": { + "individuals_func": { + "type": "task", + "func_name": "individuals" + } + } + }, + "merge_and_sifting": { + "type": "parallel", + "parallel_functions": [{ + "root": "individuals_merge", + "states": { + "individuals_merge": { + "type": "task", + "func_name": "individuals_merge" + }}}, + { + "root": "sifting", + "states": { + "sifting": { + "type": "task", + "func_name": "sifting" + } + }}], + "next": "frequency_and_overlap" + }, + "frequency_and_overlap": { + "type": "parallel", + "parallel_functions": [{ + "root": "frequency", + "states": { + "frequency": { + "type": "map", + "root": "frequency_func", + "array": "sifting.populations", + "common_params": "sifting,individuals_merge", + "states": { + "frequency_func": { + "type": "task", + "func_name": "frequency" + } + } + }}}, + { + "root": "mutation_overlap", + "states": { + "mutation_overlap": { + "type": "map", + "array": "sifting.populations", + "common_params": "sifting,individuals_merge", + "root": "mutation_overlap_func", + "states": { + "mutation_overlap_func": { + "type": "task", + "func_name": "mutation_overlap" + } + } + } + }}] + } + } +} diff --git a/benchmarks/600.workflows/6100.1000-genome/input.py b/benchmarks/600.workflows/6100.1000-genome/input.py new file mode 100644 index 000000000..dfebfe575 --- /dev/null +++ b/benchmarks/600.workflows/6100.1000-genome/input.py @@ -0,0 +1,131 @@ +# Copyright 2020-2025 ETH Zurich and the SeBS authors. All rights reserved. + +import os +import re +import uuid +import io + +size_generators = { + "test" : (1), + "small": (5), + "small-10": (10), + "large": (10), +} + +def buckets_count(): + return (1, 1) + +def generate_input(data_dir, size, benchmarks_bucket, input_buckets, output_buckets, upload_func, nosql_func): + files = ["ALL.chr21.1250.vcf", "ALL.chr21.phase3_shapeit2_mvncall_integrated_v5.20130502.sites.annotation.vcf", "columns.txt", "AFR", "ALL", "AMR", "EAS", "EUR", "GBR", "SAS"] + for name in files: + #if name != "ALL.chr21.phase3_shapeit2_mvncall_integrated_v5.20130502.sites.annotation.vcf": + path = os.path.join(data_dir, name) + upload_func(0, name, path) + + num_individuals_jobs = size_generators[size] + + blobs = [] + start_bytes = 0 + with open(os.path.join(data_dir, files[0]), "r") as f: + content = f.readlines() + #TODO potentially change if input file with different number of lines is to be processed. + range_per_job = 1250 / num_individuals_jobs + for i in range(0, num_individuals_jobs): + #actually split file; return it afterwards. see e.g. split.py in 660.map-reduce. + #regex = re.compile('(?!#)') + start = i * range_per_job + end = i * range_per_job + range_per_job + #print("start: ", start, "end: ", end, "range_per_job: ", range_per_job, "num_individuals_jobs: ", num_individuals_jobs) + #data = list(filter(regex.match, content[int(start):int(end)])) + data = content[int(start):int(end)] + #name with start and end lines is not needed as all individuals jobs can just read their entire file. + name = str(uuid.uuid4())[:8] + + upload_data = io.BytesIO() + upload_data.writelines((val).encode("utf-8") for val in data) + upload_data.seek(0) + #name = client.upload_stream(output_bucket, name, upload_data) + #TODO keep track of start + stop bytes and return them. + nbytes = upload_data.getbuffer().nbytes + + output = { + "start_bytes": start_bytes, + "end_bytes": start_bytes + nbytes - 1 + } + + blobs.append(output) + start_bytes += nbytes + + return { + "bucket": output_buckets[0], + "blob": blobs, + "individuals_file": files[0], + "benchmark_bucket": benchmarks_bucket, + "columns_bucket": input_buckets[0], + "columns": files[2], + "populations": files[3:9], + "sifting_input": files[1], + } + + +def validate_output(data_dir: str | None, input_config: dict, output: dict, language: str, storage = None) -> str | None: + if output is None: + return "Output is None" + + if not isinstance(output, dict): + return f"Expected output to be a dict, got {type(output).__name__}" + + # Real output structure: + # { + # "mutation_overlap": {"sifting": {"populations": [{"output_mutation_overlap": "..."}...]}}, + # "frequency": {"sifting": {"populations": [{"output_frequency": "..."}...]}} + # } + for branch in ("mutation_overlap", "frequency"): + if branch not in output: + return f"Output missing '{branch}' key, got keys: {list(output.keys())}" + sifting = output[branch].get("sifting") + if not isinstance(sifting, dict): + return f"output['{branch}']['sifting'] is not a dict" + pops = sifting.get("populations") + if not isinstance(pops, list) or len(pops) == 0: + return f"output['{branch}']['sifting']['populations'] is not a non-empty list" + + input_populations = input_config.get("populations", []) + expected_key = {"mutation_overlap": "output_mutation_overlap", "frequency": "output_frequency"} + # Output filename patterns: chr21-{POP}.tar.*gz for mutation_overlap, chr21-{POP}-freq.tar.*gz for frequency + filename_patterns = {"mutation_overlap": "chr21-{pop}", "frequency": "chr21-{pop}-freq"} + + for branch, key in expected_key.items(): + pops = output[branch]["sifting"]["populations"] + + # Population count should match input + if input_populations and len(pops) != len(input_populations): + return ( + f"output['{branch}']['sifting']['populations'] has {len(pops)} entries, " + f"expected {len(input_populations)} (one per input population)" + ) + + for i, p in enumerate(pops): + if not isinstance(p, dict): + return f"output['{branch}']['sifting']['populations'][{i}] is not a dict" + if key not in p: + return f"output['{branch}']['sifting']['populations'][{i}] missing '{key}'" + + filename = p[key] + if not isinstance(filename, str) or not filename: + return f"output['{branch}']['sifting']['populations'][{i}]['{key}'] is not a non-empty string" + + if not filename.endswith(".gz"): + return f"output['{branch}']['sifting']['populations'][{i}]['{key}'] should end with .gz, got '{filename}'" + + # Filename should contain the population name + if input_populations: + pop_name = input_populations[i] if i < len(input_populations) else None + pattern_prefix = filename_patterns[branch].format(pop=pop_name) if pop_name else None + if pattern_prefix and pattern_prefix.lower() not in filename.lower(): + return ( + f"output['{branch}']['sifting']['populations'][{i}]['{key}'] = '{filename}' " + f"does not contain expected population pattern '{pattern_prefix}'" + ) + + return None diff --git a/benchmarks/600.workflows/6100.1000-genome/python/frequency.py b/benchmarks/600.workflows/6100.1000-genome/python/frequency.py new file mode 100644 index 000000000..969a76de2 --- /dev/null +++ b/benchmarks/600.workflows/6100.1000-genome/python/frequency.py @@ -0,0 +1,272 @@ +import time + +tic = time.perf_counter() +import numpy as np +from random import sample +import os.path +import matplotlib + +matplotlib.use('Agg') +import matplotlib.pyplot as plt +import collections +from collections import Counter + +import datetime + +import os +from . import storage + + +class ReadData: + def read_names(self, POP, pop_dir, columns_file): + tic = time.perf_counter() + namefile = pop_dir + POP + f = open(namefile, 'r') + text = f.read() + f.close() + text = text.split() + all_ids = text[0:] + file = columns_file + f = open(file, 'r') + text = f.read() + f.close() + genome_ids = text.split() + + ids = list(set(all_ids) & set(genome_ids)) + return ids + + def read_rs_numbers(self, siftfile, SIFT): + ## NB This file is in the format of: + ## line number, rs number, ENSG number, SIFT, Phenotype + tic = time.perf_counter() + rs_numbers = [] + variations = {} + map_variations = {} + all_variations = [] + sift_file = open(siftfile, 'r') + for item in sift_file: + item = item.split() + if len(item) > 2: + rs_numbers.append(item[1]) + map_variations[item[1]] = item[2] + variations[item[0]] = item[2] + + return rs_numbers, map_variations + + def read_individuals(self, ids, rs_numbers, data_dir, chrom, individuals_merge_filename): + tic = time.perf_counter() + mutation_index_array = [] + for name in ids: + filename = data_dir + individuals_merge_filename + '/' + chrom + '.' + name + f = open(filename, 'r') + text = [] + for item in f: + item = item.split() + try: + text.append(item[1]) + except IndexError as e: + print("ERROR({}): while reading {}: (item: {})".format(str(e), filename, item)) + sifted_mutations = list(set(rs_numbers).intersection(text)) + mutation_index_array.append(sifted_mutations) + + return mutation_index_array + + +class Results: + + def overlap_ind(self, ids, mutation_index_array, n_runs, n_indiv): + n_p = len(mutation_index_array) + tic = time.perf_counter() + list_p = np.linspace(0, n_p - 1, n_p).astype(int) + mutation_overlap = [] + random_indiv = [] + for run in range(n_runs): + randomized_list = sample(list(list_p), n_p) + result = Counter() + r_ids = [] + for pq in range(n_indiv): + if 2 * pq >= len(randomized_list): + break + b_multiset = collections.Counter(mutation_index_array[randomized_list[2 * pq]]) + r_ids.append(ids[randomized_list[2 * pq]]) + result = result + b_multiset + random_indiv.append(r_ids) + mutation_overlap.append(result) + return mutation_overlap, random_indiv + + def histogram_overlap(self, mutation_overlap, n_runs): + tic = time.perf_counter() + histogram_overlap = [] + for run in range(n_runs): + final_counts = [count for item, count in mutation_overlap[run].items()] + histogram_overlap.append(collections.Counter(final_counts)) + return histogram_overlap + + +class PlotData: + + def plot_histogram_overlap(self, POP, histogram_overlap, outputFile, n_runs): + tic = time.perf_counter() + for run in range(n_runs): + output = outputFile + str(run) + '.png' + final_counts = [count for item, count in histogram_overlap[run].items()] + N = len(final_counts) + x = range(N) + width = 1 / 1.5 + bar1 = plt.bar(x, final_counts, width, color="grey") + plt.ylabel('Mutations') + plt.xlabel('Individuals') + plt.xticks(np.arange(1, N + 1)) + plt.savefig(output) + plt.close() + + +class WriteData: + + def write_histogram_overlap(self, histogram_overlapfile, histogram_overlap, n_runs, n_indiv): + tic = time.perf_counter() + for run in range(n_runs): + overlapfile = histogram_overlapfile + str(run) + '.txt' + f = open(overlapfile, 'w') + f.write('Number Individuals - Number Mutations \n') + for i in range(1, n_indiv + 1): + if i in histogram_overlap[run]: + f.write(str(i) + '-' + str(histogram_overlap[run][i]) + '\n') + else: + f.write(str(i) + '-' + str(0) + '\n') + f.close() + + + def write_mutation_overlap(self, mutation_overlapfile, mutation_overlap, n_runs): + tic = time.perf_counter() + for run in range(n_runs): + overlapfile = mutation_overlapfile + str(run) + '.txt' + f = open(overlapfile, 'w') + f.write('Mutation Index- Number Overlapings \n') + for key, count in mutation_overlap[run].items(): + f.write(key + '-' + str(count) + '\n') + f.close() + + def write_random_indiv(self, randomindiv_file, random_indiv, n_runs): + tic = time.perf_counter() + for run in range(n_runs): + randomfile = randomindiv_file + str(run) + '.txt' + f = open(randomfile, 'w') + f.write('Individuals \n') + for item in random_indiv[run]: + f.write("%s\n" % item) + f.close() + + def write_mutation_index_array(self, mutation_index_array_file, mutation_index_array): + tic = time.perf_counter() + f = open(mutation_index_array_file, "w") + for item in mutation_index_array: + f.write("%s\n" % item) + f.close() + + def write_map_variations(self, map_variations_file, map_variations): + tic = time.perf_counter() + f = open(map_variations_file, 'w') + for key, count in map_variations.items(): + f.write(key + '\t' + str(count) + '\n') + f.close() + + +def handler(event): + POP = event["array_element"] + benchmark_bucket = event["sifting"]["benchmark_bucket"] + output_bucket = event["sifting"]["output_bucket"] + input_bucket = event["sifting"]["input_bucket"] + sifting_filename = event["sifting"]["output_sifting"] + individuals_merge_filename = event["individuals_merge"]["merge_outputfile_name"] + + #download files + siftfile = os.path.join("/tmp", "sifting.txt") + individuals_merge_file = os.path.join("/tmp", "individuals_merge.tar.gz") + pop_file = os.path.join("/tmp", POP) + columns_file = os.path.join("/tmp", "columns.txt") + + client = storage.storage.get_instance() + client.download(benchmark_bucket, output_bucket + '/' + sifting_filename, siftfile) + client.download(benchmark_bucket, output_bucket + '/' + individuals_merge_filename, individuals_merge_file) + client.download(benchmark_bucket, input_bucket + '/' + POP, pop_file) + client.download(benchmark_bucket, input_bucket + '/' + "columns.txt", columns_file) + + #chromosome number, doesn't matter here - just used for naming + c = 21 + + SIFT = 'NO-SIFT' + n_runs = 1000 + n_indiv = 52 + + data_dir = '/tmp/' + pop_dir = '/tmp/' + outdata_dir = "/tmp/chr{0}-{1}-freq/output_no_sift/".format(str(c), str(POP)) + plot_dir = "/tmp/chr{0}-{1}-freq/plots_no_sift/".format(str(c), str(POP)) + + if not os.path.exists(outdata_dir): + os.makedirs(outdata_dir, exist_ok=True) + if not os.path.exists(plot_dir): + os.makedirs(plot_dir, exist_ok=True) + + OutputFormat = '.png' + chrom = 'chr' + str(c) + + font = {'family': 'serif', 'size': 14} + plt.rc('font', **font) + + # untar input data + import tarfile + + tar = tarfile.open(individuals_merge_file) + tar.extractall(path='/tmp/' + individuals_merge_filename) + tar.close() + + rd = ReadData() + res = Results() + wr = WriteData() + pd = PlotData() + + histogram_overlapfile = outdata_dir + 'Histogram_mutation_overlap_chr' + str(c) + '_s' + \ + str(SIFT) + '_' + POP + '_' + mutation_overlapfile = outdata_dir + 'Mutation_overlap_chr' + str(c) + '_s' + \ + str(SIFT) + '_' + POP + '_' + mutation_index_array_file = outdata_dir + 'mutation_index_array' + str(c) + '_s' + \ + str(SIFT) + '_' + POP + '.txt' + histogram_overlap_plot = plot_dir + 'Frequency_mutations' + str(c) + '_s' + \ + str(SIFT) + '_' + POP + map_variations_file = outdata_dir + 'map_variations' + str(c) + '_s' + \ + str(SIFT) + '_' + POP + '.txt' + + randomindiv_file = outdata_dir + 'random_indiv' + str(c) + '_s' + \ + str(SIFT) + '_' + POP + '_' + + ids = rd.read_names(POP, pop_dir, columns_file) + n_pairs = len(ids) / 2 + + rs_numbers, map_variations = rd.read_rs_numbers(siftfile, SIFT) + mutation_index_array = rd.read_individuals(ids, rs_numbers, data_dir, chrom, individuals_merge_filename) + + wr.write_map_variations(map_variations_file, map_variations) + wr.write_mutation_index_array(mutation_index_array_file, mutation_index_array) + + mutation_overlap, random_indiv = res.overlap_ind(ids, mutation_index_array, n_runs, n_indiv) + histogram_overlap = res.histogram_overlap(mutation_overlap, n_runs) + + wr.write_mutation_overlap(mutation_overlapfile, mutation_overlap, n_runs) + wr.write_histogram_overlap(histogram_overlapfile, histogram_overlap, n_runs, n_indiv) + wr.write_random_indiv(randomindiv_file, random_indiv, n_runs) + + pd.plot_histogram_overlap(POP, histogram_overlap, histogram_overlap_plot, n_runs) + + # gen final output + tar = tarfile.open('/tmp/chr%s-%s-freq.tar.gz' % (c, POP), 'w:gz') + tar.add(outdata_dir) + tar.add(plot_dir) + tar.close() + result_name = client.upload(benchmark_bucket, output_bucket + '/' + 'chr%s-%s-freq.tar.gz' % (c, POP), '/tmp/chr%s-%s-freq.tar.gz' % (c, POP)) + result_name = result_name.replace(output_bucket + '/', '') + + return { + "output_frequency": result_name + } diff --git a/benchmarks/600.workflows/6100.1000-genome/python/individuals.py b/benchmarks/600.workflows/6100.1000-genome/python/individuals.py new file mode 100644 index 000000000..e156d0f5b --- /dev/null +++ b/benchmarks/600.workflows/6100.1000-genome/python/individuals.py @@ -0,0 +1,116 @@ +import os +import uuid +import tarfile +import shutil +import re +from . import storage +import datetime + +client = storage.storage.get_instance() + + +def compress(output, input_dir): + with tarfile.open(output, "w:gz") as file: + file.add(input_dir, arcname=os.path.basename(input_dir)) + +def readfile(file): + with open(file, 'r') as f: + content = f.readlines() + return content + +def handler(event): + benchmark_bucket = event["benchmark_bucket"] + individuals_bucket = event["bucket"] + individuals_input = event["individuals_file"] + + start_bytes = event["array_element"]["start_bytes"] + end_bytes = event["array_element"]["end_bytes"] + + columns = event["columns"] + columns_bucket = event["columns_bucket"] + columns_path = os.path.join("/tmp", "columns.txt") + + client = storage.storage.get_instance() + client.download(benchmark_bucket, columns_bucket + '/' + columns, columns_path) + data = client.download_within_range(benchmark_bucket, columns_bucket + '/' + individuals_input, start_bytes, end_bytes) + + + ndir = 'chr{}n-{}/'.format(21, individuals_input) + ndir = os.path.join("/tmp", ndir) + os.makedirs(ndir, exist_ok=True) + + + regex = re.compile('(?!#)') + #print("data: ", data) + data = data.split("\n") + data = list(filter(lambda line: regex.match(line) and line != "", data)) + + chrp_data = {} + columndata = readfile(columns_path)[0].rstrip('\n').split('\t') + + start_data = 9 # where the real data start, the first 0|1, 1|1, 1|0 or 0|0 + # position of the last element (normally equals to len(data[0].split(' ')) + #end_data = 2504 + end_data = len(columndata) - start_data + + for i in range(0, end_data): + col = i + start_data + name = columndata[col] + + filename = "{}/chr{}.{}".format(ndir, "21", name) + chrp_data[i] = [] + + with open(filename, 'w') as f: + zeilennummer = 0 + for line in data: + zeilennummer += 1 + try: + first = line.split('\t')[col] # first =`echo $l | cut -d -f$i` + except Exception as e: + print("faulty line at col = ", col, "zeilennummer:", zeilennummer, " line : ", line) + raise e + #second =`echo $l | cut -d -f 2, 3, 4, 5, 8 --output-delimiter = ' '` + second = line.split('\t')[0:8] + # We select the one we want + second = [elem for id, elem in enumerate(second) if id in [1, 2, 3, 4, 7]] + af_value = second[4].split(';')[8].split('=')[1] + # We replace with AF_Value + second[4] = af_value + try: + if ',' in af_value: + # We only keep the first value if more than one (that's what awk is doing) + af_value = float(af_value.split(',')[0]) + else: + af_value = float(af_value) + + elem = first.split('|') + # We skip some lines that do not meet these conditions + if af_value >= 0.5 and elem[0] == '0': + chrp_data[i].append(second) + elif af_value < 0.5 and elem[0] == '1': + chrp_data[i].append(second) + else: + continue + + f.write("{0} {1} {2} {3} {4}\n".format( + second[0], second[1], second[2], second[3], second[4]) + ) + except ValueError: + continue + + outputfile = "chr{}n-{}.tar.gz".format(21, individuals_input) + + # tar -zcf .. /$outputfile . + compress(os.path.join("/tmp/", outputfile), ndir) + outputfile_name = client.upload(benchmark_bucket, individuals_bucket + '/' + outputfile, os.path.join("/tmp/", outputfile)) + outputfile_name = outputfile_name.replace(individuals_bucket + '/', '') + + # Cleaning temporary files + try: + shutil.rmtree(ndir) + except OSError as e: + print("Error: %s : %s" % (ndir, e.strerror)) + + return { + "individuals_output": outputfile_name + } diff --git a/benchmarks/600.workflows/6100.1000-genome/python/individuals_merge.py b/benchmarks/600.workflows/6100.1000-genome/python/individuals_merge.py new file mode 100644 index 000000000..7a563366b --- /dev/null +++ b/benchmarks/600.workflows/6100.1000-genome/python/individuals_merge.py @@ -0,0 +1,90 @@ +import os +from . import storage +import time +import tarfile +import tempfile +import shutil +import datetime + +def handler(event): + benchmark_bucket = event["benchmark_bucket"] + individuals_output_bucket = event["bucket"] + filenames = [] + for elem in event["blob"]: + filenames.append(elem["individuals_output"]) + + #download files + client = storage.storage.get_instance() + for file in filenames: + client.download(benchmark_bucket, individuals_output_bucket + '/' + file, os.path.join('/tmp', file)) + + #call merging with c and directories. + outputfile_name, outputfile = merging(21, filenames) + + #upload outputfile + outputfile_name = client.upload(benchmark_bucket, individuals_output_bucket + '/' + outputfile_name, outputfile) + outputfile_name = outputfile_name.replace(individuals_output_bucket + '/', '') + + + return { + "merge_outputfile_name": outputfile_name + } + +def compress(archive, input_dir): + with tarfile.open(archive, "w:gz") as f: + f.add(input_dir, arcname="") + +def extract_all(archive, output_dir): + with tarfile.open(archive, "r:*") as f: + f.extractall(output_dir) + flist = f.getnames() + if flist[0] == '': + flist = flist[1:] + return flist + +def readfile(filename): + with open(filename, 'r') as f: + content = f.readlines() + return content + +def writefile(filename, content): + with open(filename, 'w') as f: + f.writelines(content) + +def merging(c, tar_files): + tic = time.perf_counter() + + + merged_dir = "merged_chr{}".format(c) + merged_dir = os.path.join("/tmp", merged_dir) + os.makedirs(merged_dir, exist_ok=True) + + data = {} + + for tar in tar_files: + tic_iter = time.perf_counter() + os.makedirs("/tmp/temp_dir", exist_ok=True) + with tempfile.TemporaryDirectory(dir="/tmp/temp_dir") as temp_dir: + for filename in extract_all(os.path.join("/tmp", tar), temp_dir): + content = readfile(os.path.join(temp_dir, filename)) + if filename in data: + data[filename] += content + else: + data[filename] = content + + + for filename,content in data.items(): + writefile(os.path.join(merged_dir, filename), content) + + outputfile_name = "chr{}n.tar.gz".format(c) + outputfile = os.path.join("/tmp", outputfile_name) + + compress(outputfile, merged_dir) + + # Cleaning temporary files + try: + shutil.rmtree(merged_dir) + except OSError as e: + print("Error: %s : %s" % (merged_dir, e.strerror)) + + return outputfile_name, outputfile diff --git a/benchmarks/600.workflows/6100.1000-genome/python/mutation_overlap.py b/benchmarks/600.workflows/6100.1000-genome/python/mutation_overlap.py new file mode 100644 index 000000000..2c377e47c --- /dev/null +++ b/benchmarks/600.workflows/6100.1000-genome/python/mutation_overlap.py @@ -0,0 +1,379 @@ +import time + +tic = time.perf_counter() +import numpy as np +from random import sample +import os +import os.path +import matplotlib +matplotlib.use('Agg') +import matplotlib.pyplot as plt +import itertools +from matplotlib import pyplot +import matplotlib as mpl +import collections +from collections import Counter +import datetime + +import os +from . import storage + + +class ReadData : + def read_names(self, POP, pop_dir, columns_file) : + tic = time.perf_counter() + namefile = pop_dir + POP + f = open(namefile, 'r') + text = f.read() + f.close() + text = text.split() + all_ids = text[0:] + file = columns_file + f = open(file, 'r') + text = f.read() + f.close() + genome_ids = text.split() + + ids = list(set(all_ids) & set(genome_ids)) + + return ids + + def read_rs_numbers(self, siftfile, SIFT) : + ## NB This file is in the format of: + ## line number, rs number, ENSG number, SIFT, Phenotype + tic = time.perf_counter() + rs_numbers = [] + variations = {} + map_variations = {} + all_variations = [] + sift_file = open(siftfile,'r') + for item in sift_file: + item = item.split() + if len(item) > 2: + rs_numbers.append(item[1]) + map_variations[item[1]] = item[2] + + return rs_numbers, map_variations + + def read_individuals(self, ids, rs_numbers, data_dir, chrom, individuals_merge_filename) : + tic = time.perf_counter() + mutation_index_array = [] + total_mutations={} + total_mutations_list =[] + for name in ids : + filename = data_dir + individuals_merge_filename + '/' + chrom + '.' + name + f = open(filename, 'r') + text = f.read() + f.close() + text = text.split() + sifted_mutations = list(set(rs_numbers).intersection(text)) + mutation_index_array.append(sifted_mutations) + total_mutations[name]= len(sifted_mutations) + total_mutations_list.append(len(sifted_mutations)) + + return mutation_index_array, total_mutations, total_mutations_list + + def read_pairs_overlap(self, indpairsfile) : + tic = time.perf_counter() + pairs_overlap = np.loadtxt(indpairsfile, unpack=True) + pairs_overlap = np.transpose(pairs_overlap) + + return pairs_overlap + + +class Results : + + def group_indivuals(self, total_mutations_list, n_runs) : + tic = time.perf_counter() + n_group = 26 + random_mutations_list= [] + for run in range(n_runs): + random_mutations_list.append(sample(total_mutations_list, n_group)) + return random_mutations_list + + def pair_individuals(self, mutation_index_array, n_runs) : + tic = time.perf_counter() + + n_p = len(mutation_index_array) + n_pairs = int(round(n_p/2)) + list_p = np.linspace(0, n_p - 1, n_p).astype(int) + pairs_overlap = np.zeros((n_runs, n_pairs)) + for run in range(n_runs) : + randomized_list = sample(list(list_p) , n_p) + for pq in range(n_pairs) : + array1 = mutation_index_array[randomized_list[2*pq]] + + array2 = mutation_index_array[randomized_list[2*pq]] + pair_array = set(array1) & set(array2) + pairs_overlap[run][pq] = len(pair_array) + + return pairs_overlap + + def total_pair_individuals (self, mutation_index_array) : + tic = time.perf_counter() + n_p = len(mutation_index_array) + total_pairs_overlap = np.zeros((n_p, n_p)) + simetric_overlap = np.zeros((n_p, n_p)) + for run in range(n_p): + array1 = mutation_index_array[run] + start = run +1 + for pq in range(start, n_p) : + array2 = mutation_index_array[pq] + pairs_array = set(array1) & set(array2) + total_pairs_overlap[run][pq]=len(pairs_array) + simetric_overlap[run][pq] = len(pairs_array) + simetric_overlap[pq][run]= len(pairs_array) + + return total_pairs_overlap , simetric_overlap + + def half_pair_individuals(self, mutation_index_array) : + tic = time.perf_counter() + n_p = len(mutation_index_array) + n_pairs = int(round(n_p/2)) + pairs_overlap = np.zeros((n_pairs, n_pairs)) + for run in range(n_pairs): + array1 = mutation_index_array[run] + index =0 + for pq in range(n_pairs+1, n_p): + array2 = mutation_index_array[pq] + pairs_array = set(array1) & set(array2) + pairs_overlap[run][index]=len(pairs_array) + + return pairs_overlap + + def gene_pairs(self, mutation_index_array) : + + tic = time.perf_counter() + n_p = len(mutation_index_array) + gene_pair_list = {} + for pp in range(n_p) : + pairs = itertools.combinations(mutation_index_array[pp], 2) + for pair in pairs : + key = str(pair) + if key not in gene_pair_list : gene_pair_list[key] = 1 + else : gene_pair_list[key] += 1 + + + return gene_pair_list + +class PlotData : + + def individual_overlap(self, POP, pairs_overlap, outputFile, c, SIFT) : + tic = time.perf_counter() + + pairs_overlap = np.array(pairs_overlap) + + min_p = np.min(pairs_overlap) + max_p = np.max(pairs_overlap) + nbins = int(max_p) + 1 + n_runs = len(pairs_overlap) + + + nbins = int(np.max(pairs_overlap)) + bin_centres = np.linspace(0, nbins, nbins) + bin_edges = np.linspace(-0.5, nbins + 0.5, nbins + 1) + + fig = plt.figure(frameon=False, figsize=(10, 9)) + ax = fig.add_subplot(111) + hists = [] + max_h = 0 + for run in range(n_runs) : + h, edges = np.histogram(pairs_overlap[run], bins = bin_edges) + ax.plot(bin_centres, h, alpha = 0.5) + if len(h) > 0: + max_h = max(max_h, max(h)) + + plt.xlabel('Number of overlapping gene mutations', fontsize = 24) + plt.ylabel(r'frequency', fontsize = 28) + text1 = 'population ' + POP + '\n' +\ + 'chromosome ' + str(c) + '\n' + \ + 'SIFT < ' + str(SIFT) + '\n' + \ + str(n_runs) + ' runs' + plt.text(.95, .95, text1, fontsize = 24, + verticalalignment='top', horizontalalignment='right', + transform = ax.transAxes) + plt.savefig(outputFile) + plt.close() + + def total_colormap_overlap(self, POP, total_pairs_overlap, outputFile): + tic = time.perf_counter() + fig = plt.figure() + cmap = mpl.colors.ListedColormap(['blue','black','red', 'green', 'pink']) + img = pyplot.imshow(total_pairs_overlap,interpolation='nearest', cmap = cmap, origin='lower') + pyplot.colorbar(img,cmap=cmap) + + plt.savefig(outputFile) + plt.close() + + +class WriteData : + def write_pair_individuals(self, indpairsfile, pairs_overlap) : + tic = time.perf_counter() + np.savetxt(indpairsfile, pairs_overlap, fmt = '%i') + + def write_gene_pairs(self, genepairsfile, gene_pair_list) : + tic = time.perf_counter() + f = open(genepairsfile, 'w') + for key, count in gene_pair_list.items() : + f.write(key + '\t' + str(count) + '\n') + f.close() + + def write_total_indiv(self, total_mutations_filename, total_mutations) : + tic = time.perf_counter() + f = open(total_mutations_filename, 'w') + for key, count in total_mutations.items() : + f.write(key + '\t' + str(count) + '\n') + f.close() + + def write_random_mutations_list(self, random_mutations_filename, random_mutations_list, n_runs) : + for run in range(n_runs): + filename= random_mutations_filename +'_run_' + str(run) + '.txt' + f = open(filename, 'w') + f.writelines(["%s\n" % item for item in random_mutations_list[run]]) + + def write_mutation_index_array(self, mutation_index_array_file, mutation_index_array): + f=open(mutation_index_array_file,"w") + for item in mutation_index_array: + f.write("%s\n" % item) + f.close() + + def write_map_variations(self, map_variations_file, map_variations) : + tic = time.perf_counter() + f = open(map_variations_file, 'w') + for key, count in map_variations.items() : + f.write(key + '\t' + str(count) + '\n') + f.close() + + + +def handler(event): + POP = event["array_element"] + benchmark_bucket = event["sifting"]["benchmark_bucket"] + output_bucket = event["sifting"]["output_bucket"] + input_bucket = event["sifting"]["input_bucket"] + sifting_filename = event["sifting"]["output_sifting"] + individuals_merge_filename = event["individuals_merge"]["merge_outputfile_name"] + + + #download files + siftfile = os.path.join("/tmp", "sifting.txt") + individuals_merge_file = os.path.join("/tmp", "individuals_merge.tar.gz") + pop_file = os.path.join("/tmp", POP) + columns_file = os.path.join("/tmp", "columns.txt") + + client = storage.storage.get_instance() + client.download(benchmark_bucket, output_bucket + '/' + sifting_filename, siftfile) + client.download(benchmark_bucket, output_bucket + '/' + individuals_merge_filename, individuals_merge_file) + client.download(benchmark_bucket, input_bucket + '/' + POP, pop_file) + client.download(benchmark_bucket, input_bucket + '/' + "columns.txt", columns_file) + #chromosome no, doesn't matter. + c = 21 + + SIFT = 'NO-SIFT' + n_runs = 1 + + data_dir = '/tmp/' + pop_dir = '/tmp/' + outdata_dir = "/tmp/chr{0}-{1}/output_no_sift/".format(str(c), str(POP)) + plots_dir = "/tmp/chr{0}-{1}/plots_no_sift/".format(str(c), str(POP)) + + if not os.path.exists(outdata_dir): + os.makedirs(outdata_dir, exist_ok=True) + if not os.path.exists(plots_dir): + os.makedirs(plots_dir, exist_ok=True) + + OutputFormat = '.png' + chrom = 'chr' + str(c) + + font = {'family':'serif', + 'size':14 } + plt.rc('font', **font) + + + # untar input data + import tarfile + tar = tarfile.open(individuals_merge_file) + tar.extractall(path='/tmp/' + individuals_merge_filename) + tar.close() + + tic = time.perf_counter() + + rd = ReadData() + res = Results() + wr = WriteData() + pd = PlotData() + + half_indpairsfile = outdata_dir + 'individual_half_pairs_overlap_chr' + str(c) + '_s' + \ + str(SIFT) + '_' + POP + '.txt' + total_indpairsfile = outdata_dir + 'total_individual_pairs_overlap_chr' + str(c) + '_s' + \ + str(SIFT) + '_' + POP + '.txt' + genepairsfile = outdata_dir + 'gene_pairs_count_chr' + str(c) + '_s' + \ + str(SIFT) + '_' + POP + '.txt' + random_indpairsfile = outdata_dir + '100_individual_overlap_chr' + str(c) + '_s' + \ + str(SIFT) + '_' + POP + '.txt' + + colormap = plots_dir + 'colormap_distribution_c' + str(c) + '_s' + \ + str(SIFT) + '_' + POP + OutputFormat + half_overlap = plots_dir + 'half_distribution_c' + str(c) + '_s' + \ + str(SIFT) + '_' + POP + OutputFormat + total_overlap = plots_dir + 'total_distribution_c' + str(c) + '_s' + \ + str(SIFT) + '_' + POP + OutputFormat + random_overlap = plots_dir + '100_distribution_c' + str(c) + '_s' + \ + str(SIFT) + '_' + POP + OutputFormat + + total_mutations_filename = outdata_dir + 'total_mutations_individual' + str(c) + '_s' + \ + str(SIFT) + '_' + POP + '.txt' + random_mutations_filename = outdata_dir + 'random_mutations_individual' + str(c) + '_s' + \ + str(SIFT) + '_' + POP + + mutation_index_array_file = outdata_dir + 'mutation_index_array' + str(c) + '_s' + \ + str(SIFT) + '_' + POP + '.txt' + + map_variations_file = outdata_dir + 'map_variations' + str(c) + '_s' + \ + str(SIFT) + '_' + POP + '.txt' + + + + ids = rd.read_names(POP, pop_dir, columns_file) + n_pairs = len(ids)/2 + + + rs_numbers, map_variations = rd.read_rs_numbers(siftfile, SIFT) + mutation_index_array, total_mutations, total_mutations_list = rd.read_individuals(ids, rs_numbers, data_dir, chrom, individuals_merge_filename) + wr.write_total_indiv(total_mutations_filename, total_mutations) + wr.write_map_variations(map_variations_file, map_variations) + + #cross-correlations mutations overlapping + half_pairs_overlap = res.half_pair_individuals(mutation_index_array) + total_pairs_overlap, simetric_overlap = res.total_pair_individuals(mutation_index_array) + random_pairs_overlap = res.pair_individuals(mutation_index_array, n_runs) + + wr.write_mutation_index_array(mutation_index_array_file, mutation_index_array) + wr.write_pair_individuals(half_indpairsfile, half_pairs_overlap) + wr.write_pair_individuals(total_indpairsfile, total_pairs_overlap) + wr.write_pair_individuals(random_indpairsfile, random_pairs_overlap,) + + pd.individual_overlap(POP, half_pairs_overlap, half_overlap, c, SIFT) + pd.individual_overlap(POP, simetric_overlap, total_overlap, c, SIFT) + pd.individual_overlap(POP, random_pairs_overlap, random_overlap, c, SIFT) + pd.total_colormap_overlap(POP, total_pairs_overlap, colormap) + + #list of frecuency of mutations in 26 individuals + random_mutations_list=res.group_indivuals(total_mutations_list, n_runs) + wr.write_random_mutations_list(random_mutations_filename, random_mutations_list, n_runs) + + # gen overlapping + gene_pair_list = res.gene_pairs(mutation_index_array) + wr.write_gene_pairs(genepairsfile, gene_pair_list) + + # gen final output + tar = tarfile.open('/tmp/chr%s-%s.tar.gz' % (c, POP), 'w:gz') + tar.add(outdata_dir) + tar.add(plots_dir) + tar.close() + result_name = client.upload(benchmark_bucket, output_bucket + '/' + 'chr%s-%s.tar.gz' % (c, POP), '/tmp/chr%s-%s.tar.gz' % (c, POP)) + result_name = result_name.replace(output_bucket + '/', '') + + return { + "output_mutation_overlap": result_name + } diff --git a/benchmarks/600.workflows/6100.1000-genome/python/requirements.txt b/benchmarks/600.workflows/6100.1000-genome/python/requirements.txt new file mode 100644 index 000000000..9be498baa --- /dev/null +++ b/benchmarks/600.workflows/6100.1000-genome/python/requirements.txt @@ -0,0 +1,4 @@ +#numpy==1.17 +numpy==1.25 #1.25 triggers contourpy source build via matplotlib on the build image +contourpy==1.2.1 +matplotlib diff --git a/benchmarks/600.workflows/6100.1000-genome/python/sifting.py b/benchmarks/600.workflows/6100.1000-genome/python/sifting.py new file mode 100644 index 000000000..2add45bdb --- /dev/null +++ b/benchmarks/600.workflows/6100.1000-genome/python/sifting.py @@ -0,0 +1,74 @@ +import os +import re +from . import storage +import subprocess +import datetime + +def readfile(file): + with open(file, 'r') as f: + content = f.readlines() + return content + +def handler(event): + benchmark_bucket = event["benchmark_bucket"] + input_bucket = event["columns_bucket"] + input_filename = event["sifting_input"] + inputfile = os.path.join("/tmp", "sifting_file.vcf") + + output_bucket = event["bucket"] + + + client = storage.storage.get_instance() + client.download(benchmark_bucket, input_bucket + '/' + input_filename, inputfile) + + #c is the chromosome number - doesn't matter here. + c = 21 + final_name = 'sifted.SIFT.chr{}.txt'.format(c) + final = os.path.join("/tmp", final_name) + + rawdata = readfile(inputfile) + + + r1 = re.compile('.*(#).*') + header = len(list(filter(r1.match, rawdata[:1000]))) + + siftfile = 'SIFT.chr{}.vcf'.format(c) + siftfile = os.path.join("/tmp", siftfile) + with open(siftfile, 'w') as f: + subprocess.run(["grep -n \"deleterious\|tolerated\" {}".format(inputfile)], shell=True, stdout=f) + + data_temp = readfile(siftfile) + + r3 = re.compile('.*(rs).*') + data = list(filter(r3.match, data_temp)) + + + with open(final, 'w') as f: + for l in data: + line = str(int(l.split('\t')[0].split(':')[0]) - int(header)) + id = l.split('\t')[2] + + sifts = l.split('\t')[7].split('|') + sifts = sifts[4] + ' ' + sifts[16] + ' ' + sifts[17] + sifts = sifts.replace('(', ' ').replace(')', '') + + temp = (line + ' ' + id + ' ' + sifts).split(' ') + + if temp[3] == '' or temp[4] == '': + f.write("{} {} {}\n".format(temp[0], temp[1], temp[2])) + elif temp[5] == '': + f.write("{} {} {} {}\n".format(temp[0], temp[1], temp[2], temp[4])) + else: + f.write("{} {} {} {} {}\n".format(temp[0], temp[1], temp[2], temp[4], temp[6])) + + os.remove(siftfile) + final_name = client.upload(benchmark_bucket, output_bucket + '/' + final_name, final) + final_name = final_name.replace(output_bucket + '/', '') + + return { + "output_bucket": output_bucket, + "benchmark_bucket": benchmark_bucket, + "output_sifting": final_name, + "populations": event["populations"], + "input_bucket": input_bucket + } diff --git a/benchmarks/600.workflows/6101.1000-genome-individuals/config.json b/benchmarks/600.workflows/6101.1000-genome-individuals/config.json new file mode 100644 index 000000000..5bb1699c5 --- /dev/null +++ b/benchmarks/600.workflows/6101.1000-genome-individuals/config.json @@ -0,0 +1,6 @@ +{ + "timeout": 900, + "memory": 2048, + "languages": ["python"], + "modules": ["storage"] +} diff --git a/benchmarks/600.workflows/6101.1000-genome-individuals/definition.json b/benchmarks/600.workflows/6101.1000-genome-individuals/definition.json new file mode 100644 index 000000000..d89586cc7 --- /dev/null +++ b/benchmarks/600.workflows/6101.1000-genome-individuals/definition.json @@ -0,0 +1,17 @@ +{ + "root": "individuals_state", + "states": { + "individuals_state": { + "type": "map", + "root": "individuals", + "array": "blob", + "common_params": "benchmark_bucket,bucket,columns,columns_bucket,populations,sifting_input,individuals_file", + "states": { + "individuals": { + "type": "task", + "func_name": "individuals" + } + } + } + } +} diff --git a/benchmarks/600.workflows/6101.1000-genome-individuals/input.py b/benchmarks/600.workflows/6101.1000-genome-individuals/input.py new file mode 100644 index 000000000..1dec2c4b6 --- /dev/null +++ b/benchmarks/600.workflows/6101.1000-genome-individuals/input.py @@ -0,0 +1,109 @@ +# Copyright 2020-2025 ETH Zurich and the SeBS authors. All rights reserved. + +import os +import re +import uuid +import io + +size_generators = { + "test" : (1), + "small": (5), + "small-10": (10), + "small-20": (20), + "large": (10), +} + +def buckets_count(): + return (1, 1) + +def generate_input(data_dir, size, benchmarks_bucket, input_buckets, output_buckets, upload_func, nosql_func): + files = ["ALL.chr21.1250.vcf", "ALL.chr21.phase3_shapeit2_mvncall_integrated_v5.20130502.sites.annotation.vcf", "columns.txt", "AFR", "ALL", "AMR", "EAS", "EUR", "GBR", "SAS"] + for name in files: + if name == "ALL.chr21.1250.vcf" or name == "columns.txt": + #if name != "ALL.chr21.phase3_shapeit2_mvncall_integrated_v5.20130502.sites.annotation.vcf": + path = os.path.join(data_dir, name) + upload_func(0, name, path) + + num_individuals_jobs = size_generators[size] + + blobs = [] + start_bytes = 0 + with open(os.path.join(data_dir, files[0]), "r") as f: + content = f.readlines() + range_per_job = 1250 / num_individuals_jobs + for i in range(0, num_individuals_jobs): + #actually split file; return it afterwards. see e.g. split.py in 660.map-reduce. + #regex = re.compile('(?!#)') + start = i * range_per_job + end = i * range_per_job + range_per_job + #print("start: ", start, "end: ", end, "range_per_job: ", range_per_job, "num_individuals_jobs: ", num_individuals_jobs) + #data = list(filter(regex.match, content[int(start):int(end)])) + data = content[int(start):int(end)] + #name with start and end lines is not needed as all individuals jobs can just read their entire file. + name = str(uuid.uuid4())[:8] + + upload_data = io.BytesIO() + upload_data.writelines((val).encode("utf-8") for val in data) + upload_data.seek(0) + nbytes = upload_data.getbuffer().nbytes + + output = { + "start_bytes": start_bytes, + "end_bytes": start_bytes + nbytes - 1 + } + + blobs.append(output) + start_bytes += nbytes + + return { + "bucket": output_buckets[0], + "blob": blobs, + "individuals_file": files[0], + "benchmark_bucket": benchmarks_bucket, + "columns_bucket": input_buckets[0], + "columns": files[2], + "populations": files[3:9], + "sifting_input": files[1], + } + + +def validate_output(data_dir: str | None, input_config: dict, output: dict, language: str, storage=None) -> str | None: + if output is None: + return "Output is None" + + # Output structure: {"blob": [{"individuals_output": "*.tar.gz", ...}], ...} + if isinstance(output, dict): + if "blob" not in output: + return f"Expected 'blob' key in output, got keys: {list(output.keys())}" + items = output["blob"] + elif isinstance(output, list): + items = output + else: + return f"Expected output to be a dict or list, got {type(output).__name__}" + + if not isinstance(items, list): + return f"Expected 'blob' to be a list, got {type(items).__name__}" + + expected_length = len(input_config["blob"]) + if len(items) != expected_length: + return f"Output length {len(items)} does not match expected number of blobs {expected_length}" + + for i, element in enumerate(items): + if not isinstance(element, dict): + return f"Element {i} is not a dict" + if "individuals_output" not in element: + return f"Element {i} is missing 'individuals_output' key" + + value = element["individuals_output"] + if not isinstance(value, str) or not value: + return f"Element {i} has invalid 'individuals_output' value: expected a non-empty string" + + if not value.endswith(".tar.gz") and not value.endswith(".gz"): + return f"Element {i} has invalid 'individuals_output' filename: expected to end with '.gz', got '{value}'" + + # Filename should follow chr21n-... pattern (from handler: individuals_file = "ALL.chr21.1250.vcf" → chr21n prefix) + individuals_file = input_config.get("individuals_file", "") + if individuals_file and "chr21" in individuals_file and "chr21n" not in value: + return f"Element {i} 'individuals_output' '{value}' does not contain expected 'chr21n' prefix" + + return None diff --git a/benchmarks/600.workflows/6101.1000-genome-individuals/python/individuals.py b/benchmarks/600.workflows/6101.1000-genome-individuals/python/individuals.py new file mode 100644 index 000000000..f02c3b789 --- /dev/null +++ b/benchmarks/600.workflows/6101.1000-genome-individuals/python/individuals.py @@ -0,0 +1,114 @@ +import os +import uuid +import tarfile +import shutil +import re +from . import storage + +client = storage.storage.get_instance() + +def compress(output, input_dir): + with tarfile.open(output, "w:gz") as file: + file.add(input_dir, arcname=os.path.basename(input_dir)) + + +def readfile(file): + with open(file, 'r') as f: + content = f.readlines() + return content + +def handler(event): + benchmark_bucket = event["benchmark_bucket"] + individuals_bucket = event["bucket"] + individuals_input = event["individuals_file"] + + start_bytes = event["array_element"]["start_bytes"] + end_bytes = event["array_element"]["end_bytes"] + + columns = event["columns"] + columns_bucket = event["columns_bucket"] + columns_path = os.path.join("/tmp", "columns.txt") + + client = storage.storage.get_instance() + client.download(benchmark_bucket, columns_bucket + '/' + columns, columns_path) + data = client.download_within_range(benchmark_bucket, columns_bucket + '/' + individuals_input, start_bytes, end_bytes) + + ndir = 'chr{}n-{}/'.format(21, individuals_input) + ndir = os.path.join("/tmp", ndir) + os.makedirs(ndir, exist_ok=True) + + + regex = re.compile('(?!#)') + #print("data: ", data) + data = data.split("\n") + data = list(filter(lambda line: regex.match(line) and line != "", data)) + + chrp_data = {} + columndata = readfile(columns_path)[0].rstrip('\n').split('\t') + + start_data = 9 # where the real data start, the first 0|1, 1|1, 1|0 or 0|0 + # position of the last element (normally equals to len(data[0].split(' ')) + #end_data = 2504 + end_data = len(columndata) - start_data + + for i in range(0, end_data): + col = i + start_data + name = columndata[col] + + filename = "{}/chr{}.{}".format(ndir, "21", name) + chrp_data[i] = [] + + with open(filename, 'w') as f: + zeilennummer = 0 + for line in data: + zeilennummer += 1 + try: + first = line.split('\t')[col] # first =`echo $l | cut -d -f$i` + except Exception as e: + print("faulty line at col = ", col, "zeilennummer:", zeilennummer, " line : ", line) + raise e + #second =`echo $l | cut -d -f 2, 3, 4, 5, 8 --output-delimiter = ' '` + second = line.split('\t')[0:8] + # We select the one we want + second = [elem for id, elem in enumerate(second) if id in [1, 2, 3, 4, 7]] + af_value = second[4].split(';')[8].split('=')[1] + # We replace with AF_Value + second[4] = af_value + try: + if ',' in af_value: + # We only keep the first value if more than one (that's what awk is doing) + af_value = float(af_value.split(',')[0]) + else: + af_value = float(af_value) + + elem = first.split('|') + # We skip some lines that do not meet these conditions + if af_value >= 0.5 and elem[0] == '0': + chrp_data[i].append(second) + elif af_value < 0.5 and elem[0] == '1': + chrp_data[i].append(second) + else: + continue + + f.write("{0} {1} {2} {3} {4}\n".format( + second[0], second[1], second[2], second[3], second[4]) + ) + except ValueError: + continue + + outputfile = "chr{}n-{}.tar.gz".format(21, individuals_input) + + # tar -zcf .. /$outputfile . + compress(os.path.join("/tmp/", outputfile), ndir) + outputfile_name = client.upload(benchmark_bucket, individuals_bucket + '/' + outputfile, os.path.join("/tmp/", outputfile)) + outputfile_name = outputfile_name.replace(individuals_bucket + '/', '') + + # Cleaning temporary files + try: + shutil.rmtree(ndir) + except OSError as e: + print("Error: %s : %s" % (ndir, e.strerror)) + + return { + "individuals_output": outputfile_name + } diff --git a/benchmarks/600.workflows/6101.1000-genome-individuals/python/requirements.txt b/benchmarks/600.workflows/6101.1000-genome-individuals/python/requirements.txt new file mode 100644 index 000000000..3907a2758 --- /dev/null +++ b/benchmarks/600.workflows/6101.1000-genome-individuals/python/requirements.txt @@ -0,0 +1,4 @@ +#numpy==1.17 +numpy==1.25 #1.16 works on Azure, but not AWS +contourpy==1.2.1 +matplotlib diff --git a/benchmarks/600.workflows/620.func-invo/config.json b/benchmarks/600.workflows/620.func-invo/config.json new file mode 100644 index 000000000..d6d184e8a --- /dev/null +++ b/benchmarks/600.workflows/620.func-invo/config.json @@ -0,0 +1,6 @@ +{ + "timeout": 120, + "memory": 256, + "languages": ["python"], + "modules": [] +} diff --git a/benchmarks/600.workflows/620.func-invo/definition.json b/benchmarks/600.workflows/620.func-invo/definition.json new file mode 100644 index 000000000..c64b88349 --- /dev/null +++ b/benchmarks/600.workflows/620.func-invo/definition.json @@ -0,0 +1,15 @@ +{ + "root": "gen", + "states": { + "gen": { + "type": "task", + "func_name": "gen", + "next": "process" + }, + "process": { + "type": "repeat", + "func_name": "process", + "count": 8 + } + } +} diff --git a/benchmarks/600.workflows/620.func-invo/input.py b/benchmarks/600.workflows/620.func-invo/input.py new file mode 100644 index 000000000..7a7aa0ed6 --- /dev/null +++ b/benchmarks/600.workflows/620.func-invo/input.py @@ -0,0 +1,43 @@ +# Copyright 2020-2025 ETH Zurich and the SeBS authors. All rights reserved. + +size_generators = { + 'test' : 10, + 'small' : 2**5, + 'large': 2**20, + '2e5': 2**5, + '2e8': 2**8, + '2e10': 2**10, + '2e12': 2**12, + '2e14': 2**14, + '2e16': 2**16, + '2e18': 2**18, + '2e18-1000': (2**18)-1000 +} + +def generate_input(data_dir, size, benchmarks_bucket, input_paths, output_paths, upload_func, nosql_func): + return { 'size': size_generators[size] } + + +def validate_output(data_dir: str | None, input_config: dict, output: dict, language: str, storage = None) -> str | None: + if output is None: + return "Output is None" + + if 'len' not in output: + return "Missing key 'len' in output" + + if not isinstance(output['len'], str): + return f"Expected 'len' to be a string, got {type(output['len']).__name__}" + + expected_size = input_config['size'] + actual_size = len(output['len']) + if actual_size != expected_size: + return f"Expected string length {expected_size}, got {actual_size}" + + # The string is built by concatenating str(i % 255) for shuffled i in range(size). + # Every character must therefore be a decimal digit. + s = output['len'] + if not s.isdigit(): + non_digit = next(c for c in s if not c.isdigit()) + return f"Output string contains non-digit character {non_digit!r}; expected only digits 0-9" + + return None diff --git a/benchmarks/600.workflows/620.func-invo/python/gen.py b/benchmarks/600.workflows/620.func-invo/python/gen.py new file mode 100644 index 000000000..60c328fee --- /dev/null +++ b/benchmarks/600.workflows/620.func-invo/python/gen.py @@ -0,0 +1,14 @@ +from random import shuffle + +def handler(event): + size = int(event["size"]) + elems = list(range(size)) + shuffle(elems) + + data = "" + for i in elems: + data += str(i % 255) + if len(data) > size: + break + + return {'len' : data[:size]} diff --git a/benchmarks/600.workflows/620.func-invo/python/process.py b/benchmarks/600.workflows/620.func-invo/python/process.py new file mode 100644 index 000000000..084062854 --- /dev/null +++ b/benchmarks/600.workflows/620.func-invo/python/process.py @@ -0,0 +1,14 @@ +from random import shuffle + +def handler(event): + size = len(event['len']) + elems = list(range(size)) + shuffle(elems) + + data = "" + for i in elems: + data += str(i % 255) + if len(data) > size: + break + + return {'len' : data[:size]} diff --git a/benchmarks/600.workflows/6200.trip-booking/config.json b/benchmarks/600.workflows/6200.trip-booking/config.json new file mode 100644 index 000000000..3dd2268cd --- /dev/null +++ b/benchmarks/600.workflows/6200.trip-booking/config.json @@ -0,0 +1,6 @@ +{ + "timeout": 120, + "memory": 256, + "languages": ["python"], + "modules": ["nosql"] +} diff --git a/benchmarks/600.workflows/6200.trip-booking/definition.json b/benchmarks/600.workflows/6200.trip-booking/definition.json new file mode 100644 index 000000000..1d9885750 --- /dev/null +++ b/benchmarks/600.workflows/6200.trip-booking/definition.json @@ -0,0 +1,41 @@ +{ + "root": "hotel", + "states": { + "hotel": { + "type": "task", + "func_name": "reserve_hotel", + "next": "rental" + }, + "rental": { + "type": "task", + "func_name": "reserve_rental", + "next": "flight", + "failure": "cancel_hotel" + }, + "flight": { + "type": "task", + "func_name": "reserve_flight", + "next": "confirm", + "failure": "cancel_rental" + }, + "confirm": { + "type": "task", + "func_name": "confirm", + "failure": "cancel_flight" + }, + "cancel_flight": { + "type": "task", + "func_name": "cancel_flight", + "next": "cancel_rental" + }, + "cancel_rental": { + "type": "task", + "func_name": "cancel_rental", + "next": "cancel_hotel" + }, + "cancel_hotel": { + "type": "task", + "func_name": "cancel_hotel" + } + } +} diff --git a/benchmarks/600.workflows/6200.trip-booking/input.py b/benchmarks/600.workflows/6200.trip-booking/input.py new file mode 100644 index 000000000..0b8f07cf7 --- /dev/null +++ b/benchmarks/600.workflows/6200.trip-booking/input.py @@ -0,0 +1,102 @@ +# Copyright 2020-2025 ETH Zurich and the SeBS authors. All rights reserved. + + +def allocate_nosql() -> dict: + + return { + "flights": { + "primary_key": "trip_id", + "secondary_key": "flight_id" + }, + "car_rentals": { + "primary_key": "trip_id", + "secondary_key": "rental_id" + }, + "hotel_booking": { + "primary_key": "trip_id", + "secondary_key": "booking_id" + } + } + +def generate_input( + data_dir, size, benchmarks_bucket, input_buckets, output_buckets, upload_func, nosql_func +): + + input_config = {} + + # test - invoke a single trip, succeed + # small - fail in the middle + # large - fail at the last step + + trip_details = { + "flight_depart": "ZRH", + "flight_arrive": "KTW", + "flight_date": "2020-08-22T13:00:00", + "hotel_stars": "3", + "hotel_nights": "3", + "hotel_distance": "1500", + "hotel_price_max": "150", + "rental_class": "compact", + "rental_price_max": "100", + "rental_duration": 3, + "rental_requests": ["full_tank", "CDW", "assistance"] + } + + size_results = { + "test": {"result": "success"}, + "small": {"result": "failure", "reason": "hotel"}, + "large": {"result": "failure", "reason": "confirm"} + } + trip_details["expected_result"] = size_results[size] + + return trip_details + + +def validate_output(data_dir: str | None, input_config: dict, output: dict, language: str, storage=None) -> str | None: + + if output is None: + return "Output is None" + + if not isinstance(output, dict): + return f"Expected output to be a dict, got {type(output).__name__}" + + if "trip_id" not in output: + return "Output is missing 'trip_id' field" + + if not isinstance(output["trip_id"], str) or not output["trip_id"]: + return "Output 'trip_id' must be a non-empty string" + + if "status" not in output: + return "Output is missing 'status' field" + + if not isinstance(output["status"], str): + return f"Output 'status' must be a string, got {type(output['status']).__name__}" + + expected_result = input_config.get("expected_result", {}) + expected_outcome = expected_result.get("result") + expected_reason = expected_result.get("reason") + + valid_statuses = {"success", "failure"} + if output["status"] not in valid_statuses: + return f"Output 'status' must be one of {valid_statuses}, got '{output['status']}'" + + # trip_id is a UUID stored without dashes (32 hex chars) + import re + if not re.match(r'^[0-9a-f]{32}$', output["trip_id"], re.IGNORECASE): + return f"Output 'trip_id' is not a 32-char hex UUID: '{output['trip_id']}'" + + if expected_outcome == "success": + if output["status"] != "success": + return f"Expected status 'success', got '{output['status']}'" + + elif expected_outcome == "failure": + if expected_reason == "hotel": + # Hotel failure raises RuntimeError immediately, so the workflow fails + # with an exception. If validate_output is called, the framework caught + # it gracefully - accept any status in this case. + pass + elif expected_reason in ["confirm", "rental", "flight"]: + if output["status"] != "failure": + return f"Expected status 'failure' (reason: {expected_reason}), got '{output['status']}'" + + return None diff --git a/benchmarks/600.workflows/6200.trip-booking/python/cancel_flight.py b/benchmarks/600.workflows/6200.trip-booking/python/cancel_flight.py new file mode 100644 index 000000000..f94da3dfe --- /dev/null +++ b/benchmarks/600.workflows/6200.trip-booking/python/cancel_flight.py @@ -0,0 +1,16 @@ +from . import nosql + +nosql_client = nosql.nosql.get_instance() + + +def handler(event): + + trip_id = event["trip_id"] + + # Confirm flight + nosql_table_name = "flights" + flight_id = event["flight_id"] + nosql_client.delete(nosql_table_name, ("trip_id", trip_id), ("flight_id", flight_id)) + + event.pop("flight_id") + return event diff --git a/benchmarks/600.workflows/6200.trip-booking/python/cancel_hotel.py b/benchmarks/600.workflows/6200.trip-booking/python/cancel_hotel.py new file mode 100644 index 000000000..e1f69077d --- /dev/null +++ b/benchmarks/600.workflows/6200.trip-booking/python/cancel_hotel.py @@ -0,0 +1,15 @@ +from . import nosql + +nosql_client = nosql.nosql.get_instance() + + +def handler(event): + + trip_id = event["trip_id"] + + # Confirm flight + nosql_table_name = "hotel_booking" + booking_id = event["booking_id"] + nosql_client.delete(nosql_table_name, ("trip_id", trip_id), ("booking_id", booking_id)) + + return {"trip_id": trip_id, "status": "failure"} diff --git a/benchmarks/600.workflows/6200.trip-booking/python/cancel_rental.py b/benchmarks/600.workflows/6200.trip-booking/python/cancel_rental.py new file mode 100644 index 000000000..f8ff38044 --- /dev/null +++ b/benchmarks/600.workflows/6200.trip-booking/python/cancel_rental.py @@ -0,0 +1,16 @@ +from . import nosql + +nosql_client = nosql.nosql.get_instance() + + +def handler(event): + + trip_id = event["trip_id"] + + # Confirm flight + nosql_table_name = "car_rentals" + rental_id = event["rental_id"] + nosql_client.delete(nosql_table_name, ("trip_id", trip_id), ("rental_id", rental_id)) + + event.pop("rental_id") + return event diff --git a/benchmarks/600.workflows/6200.trip-booking/python/confirm.py b/benchmarks/600.workflows/6200.trip-booking/python/confirm.py new file mode 100644 index 000000000..3a555f6a4 --- /dev/null +++ b/benchmarks/600.workflows/6200.trip-booking/python/confirm.py @@ -0,0 +1,42 @@ +from . import nosql + +nosql_client = nosql.nosql.get_instance() + + +def handler(event): + + expected_result = event["expected_result"] + if expected_result["result"] == "failure" and expected_result["reason"] == "confirm": + raise RuntimeError("Failed to confirm the booking!") + + trip_id = event["trip_id"] + + # Confirm flight + nosql_table_name = "flights" + flight_id = event["flight_id"] + nosql_client.update( + nosql_table_name, + ("trip_id", trip_id), + ("flight_id", flight_id), + {"status": "booked"}, + ) + + # Confirm car rental + nosql_table_name = "car_rentals" + nosql_client.update( + nosql_table_name, + ("trip_id", trip_id), + ("rental_id", event["rental_id"]), + {"status": "booked"}, + ) + + # Confirm hotel booking + nosql_table_name = "hotel_booking" + nosql_client.update( + nosql_table_name, + ("trip_id", trip_id), + ("booking_id", event["booking_id"]), + {"status": "booked"}, + ) + + return {"trip_id": trip_id, "status": "success"} diff --git a/benchmarks/600.workflows/6200.trip-booking/python/reserve_flight.py b/benchmarks/600.workflows/6200.trip-booking/python/reserve_flight.py new file mode 100644 index 000000000..fe55ed0c1 --- /dev/null +++ b/benchmarks/600.workflows/6200.trip-booking/python/reserve_flight.py @@ -0,0 +1,40 @@ +from . import nosql + +nosql_client = nosql.nosql.get_instance() +nosql_table_name = "flights" + + +def handler(event): + + expected_result = event["expected_result"] + if expected_result["result"] == "failure" and expected_result["reason"] == "flight": + raise RuntimeError("Failed to book a flight!") + + # We start with the hotel + trip_id = event["trip_id"] + flight_id = event["request-id"] + + # Simulate return from a service + flight_price = "1000" + flight_connections = ["WAW"] + flight_duration = "4h30m" + + nosql_client.insert( + nosql_table_name, + ("trip_id", trip_id), + ("flight_id", flight_id), + { + **{key: event[key] for key in event.keys() if key.startswith("flight_")}, + "price": flight_price, + "connections": flight_connections, + "duration": flight_duration, + "status": "pending", + }, + ) + + return { + "trip_id": trip_id, + "flight_id": flight_id, + **{key: event[key] for key in ["booking_id", "rental_id"]}, + "expected_result": expected_result, + } diff --git a/benchmarks/600.workflows/6200.trip-booking/python/reserve_hotel.py b/benchmarks/600.workflows/6200.trip-booking/python/reserve_hotel.py new file mode 100644 index 000000000..623d1a8b0 --- /dev/null +++ b/benchmarks/600.workflows/6200.trip-booking/python/reserve_hotel.py @@ -0,0 +1,35 @@ +import uuid + +from . import nosql + +nosql_client = nosql.nosql.get_instance() +nosql_table_name = "hotel_booking" + + +def handler(event): + + expected_result = event["expected_result"] + if expected_result["result"] == "failure" and expected_result["reason"] == "hotel": + raise RuntimeError("Failed to book the hotel!") + + # We start with the hotel + trip_id = str(uuid.uuid4().hex) + hotel_booking_id = event["request-id"] + + # Simulate return from a service + hotel_price = "130" + hotel_name = "BestEver Hotel" + + nosql_client.insert( + nosql_table_name, + ("trip_id", trip_id), + ("booking_id", hotel_booking_id), + { + **{key: event[key] for key in event.keys() if key.startswith("hotel_")}, + "hotel_price": hotel_price, + "hotel_name": hotel_name, + "status": "pending", + }, + ) + + return {"trip_id": trip_id, "booking_id": hotel_booking_id, **event} diff --git a/benchmarks/600.workflows/6200.trip-booking/python/reserve_rental.py b/benchmarks/600.workflows/6200.trip-booking/python/reserve_rental.py new file mode 100644 index 000000000..8cf0b11fc --- /dev/null +++ b/benchmarks/600.workflows/6200.trip-booking/python/reserve_rental.py @@ -0,0 +1,33 @@ +from . import nosql + +nosql_client = nosql.nosql.get_instance() +nosql_table_name = "car_rentals" + + +def handler(event): + + expected_result = event["expected_result"] + if expected_result["result"] == "failure" and expected_result["reason"] == "rental": + raise RuntimeError("Failed to rent a car!") + + # We start with the hotel + trip_id = event["trip_id"] + rental_id = event["request-id"] + + # Simulate return from a service + car_price = "125" + car_name = "Fiat 126P" + + nosql_client.insert( + nosql_table_name, + ("trip_id", trip_id), + ("rental_id", rental_id), + { + **{key: event[key] for key in event.keys() if key.startswith("rental_")}, + "rental_price": car_price, + "rental_name": car_name, + "status": "pending", + }, + ) + + return {"trip_id": trip_id, "rental_id": rental_id, **event} diff --git a/benchmarks/600.workflows/630.parallel-sleep/config.json b/benchmarks/600.workflows/630.parallel-sleep/config.json new file mode 100644 index 000000000..d6d184e8a --- /dev/null +++ b/benchmarks/600.workflows/630.parallel-sleep/config.json @@ -0,0 +1,6 @@ +{ + "timeout": 120, + "memory": 256, + "languages": ["python"], + "modules": [] +} diff --git a/benchmarks/600.workflows/630.parallel-sleep/definition.json b/benchmarks/600.workflows/630.parallel-sleep/definition.json new file mode 100644 index 000000000..61c56083c --- /dev/null +++ b/benchmarks/600.workflows/630.parallel-sleep/definition.json @@ -0,0 +1,21 @@ +{ + "root": "generate", + "states": { + "generate": { + "type": "task", + "func_name": "generate", + "next": "process-state" + }, + "process-state": { + "type": "map", + "root": "process", + "array": "buffer", + "states": { + "process": { + "type": "task", + "func_name": "process" + } + } + } + } +} diff --git a/benchmarks/600.workflows/630.parallel-sleep/input.py b/benchmarks/600.workflows/630.parallel-sleep/input.py new file mode 100644 index 000000000..ef9fc42a0 --- /dev/null +++ b/benchmarks/600.workflows/630.parallel-sleep/input.py @@ -0,0 +1,59 @@ +# Copyright 2020-2025 ETH Zurich and the SeBS authors. All rights reserved. + + #threads-duration +size_generators = { + 'test' : (2, 2), + 'small': (16, 20), + 'large': (50, 2), + '2-1': (2, 1), + '4-1': (4, 1), + '8-1': (8, 1), + '16-1': (16, 1), + '2-5': (2, 5), + '4-5': (4, 5), + '8-5': (8, 5), + '16-5': (16, 5), + '2-10': (2, 10), + '4-10': (4, 10), + '8-10': (8, 10), + '16-10': (16, 10), + '2-15': (2, 15), + '4-15': (4, 15), + '8-15': (8, 15), + '16-15': (16, 15), + '2-20': (2, 20), + '4-20': (4, 20), + '8-20': (8, 20), + '16-20': (16, 20), + '50-1': (50, 1) +} + +def buckets_count(): + return (0, 0) + +def generate_input(data_dir, size, benchmarks_bucket, input_paths, output_paths, upload_func, nosql_func): + count, sleep = size_generators[size] + return { 'count': count, 'sleep': sleep } + + +def validate_output(data_dir: str | None, input_config: dict, output: dict, language: str, storage = None) -> str | None: + if output is None: + return "Output is None" + + expected_count = input_config['count'] + + if not isinstance(output, dict) or "buffer" not in output: + return f"Expected output dict with 'buffer' key, got: {output!r}" + + results = output["buffer"] + if not isinstance(results, list): + return f"Expected 'buffer' to be a list, got {type(results).__name__}" + + if len(results) != expected_count: + return f"Expected {expected_count} results, got {len(results)}" + + for i, item in enumerate(results): + if item != "ok": + return f"Expected element {i} to be 'ok', got {item!r}" + + return None diff --git a/benchmarks/600.workflows/630.parallel-sleep/python/generate.py b/benchmarks/600.workflows/630.parallel-sleep/python/generate.py new file mode 100644 index 000000000..c291c6754 --- /dev/null +++ b/benchmarks/600.workflows/630.parallel-sleep/python/generate.py @@ -0,0 +1,12 @@ +def handler(event): + count = int(event["count"]) + sleep = int(event["sleep"]) + + sleep_list = [] + for i in range(0, count): + sleep_list.append({'sleep':sleep}) + + + return { + "buffer": sleep_list + } diff --git a/benchmarks/600.workflows/630.parallel-sleep/python/process.py b/benchmarks/600.workflows/630.parallel-sleep/python/process.py new file mode 100644 index 000000000..9e2f1ab05 --- /dev/null +++ b/benchmarks/600.workflows/630.parallel-sleep/python/process.py @@ -0,0 +1,7 @@ +import time + +def handler(event): + time.sleep(event['sleep']) + + + return "ok" diff --git a/benchmarks/600.workflows/631.parallel-download/config.json b/benchmarks/600.workflows/631.parallel-download/config.json new file mode 100644 index 000000000..4314d183a --- /dev/null +++ b/benchmarks/600.workflows/631.parallel-download/config.json @@ -0,0 +1,6 @@ +{ + "timeout": 120, + "memory": 512, + "languages": ["python"], + "modules": ["storage"] +} diff --git a/benchmarks/600.workflows/631.parallel-download/definition.json b/benchmarks/600.workflows/631.parallel-download/definition.json new file mode 100644 index 000000000..c12e59553 --- /dev/null +++ b/benchmarks/600.workflows/631.parallel-download/definition.json @@ -0,0 +1,21 @@ +{ + "root": "generate", + "states": { + "generate": { + "type": "task", + "func_name": "generate", + "next": "process1" + }, + "process1": { + "type": "map", + "root": "process", + "array": "buffer", + "states": { + "process": { + "type": "task", + "func_name": "process" + } + } + } + } +} diff --git a/benchmarks/600.workflows/631.parallel-download/input.py b/benchmarks/600.workflows/631.parallel-download/input.py new file mode 100644 index 000000000..e427949b9 --- /dev/null +++ b/benchmarks/600.workflows/631.parallel-download/input.py @@ -0,0 +1,76 @@ +# Copyright 2020-2025 ETH Zurich and the SeBS authors. All rights reserved. + +import os +from random import shuffle + +size_generators = { + 'test' : (5, 10), + 'small': (20, 2**10), + 'large': (50, 2**10), + '2e10': (20, 2**10), + '2e28': (20, 2**28), + '2e15': (20, 2**15), + '2e20': (20, 2**20), + '2e25': (20, 2**25), + '2e26': (20, 2**26), + '2e27': (20, 2**27) +} + + +def buckets_count(): + return (1, 0) + + +def generate(size): + elems = list(range(size)) + shuffle(elems) + + length = 0 + for i in elems: + data = str(i % 255) + length += len(data) + if length > size: + break + yield data + + +def generate_input(data_dir, size, benchmarks_bucket, input_buckets, output_buckets, upload_func, nosql_func): + count, size_bytes = size_generators[size] + + data_name = f"data-{size_bytes}.txt" + data_path = os.path.join(data_dir, data_name) + + if not os.path.exists(data_path): + os.makedirs(data_dir, exist_ok=True) + with open(data_path, "w") as f: + f.writelines(k for k in generate(size_bytes)) + + upload_func(0, data_name, data_path) + # os.remove(data_path) + + return { 'count': count, "bucket": benchmarks_bucket, "blob": input_buckets[0] + '/' + data_name} + + +def validate_output(data_dir: str | None, input_config: dict, output: dict, language: str, storage = None) -> str | None: + expected_count = input_config.get("count") + if expected_count is None: + return "Input config missing 'count' field" + + if output is None: + return "Output is None" + + if not isinstance(output, dict) or "buffer" not in output: + return f"Expected output dict with 'buffer' key, got: {output!r}" + + results = output["buffer"] + if not isinstance(results, list): + return f"Expected 'buffer' to be a list, got {type(results).__name__}" + + if len(results) != expected_count: + return f"Expected {expected_count} results, got {len(results)}" + + for i, result in enumerate(results): + if result != "ok": + return f"Result at index {i} is {result!r}, expected 'ok'" + + return None diff --git a/benchmarks/600.workflows/631.parallel-download/python/generate.py b/benchmarks/600.workflows/631.parallel-download/python/generate.py new file mode 100644 index 000000000..fa20cd018 --- /dev/null +++ b/benchmarks/600.workflows/631.parallel-download/python/generate.py @@ -0,0 +1,8 @@ +def handler(event): + count = int(event["count"]) + del event["count"] + + + return { + "buffer": count * [event] + } diff --git a/benchmarks/600.workflows/631.parallel-download/python/process.py b/benchmarks/600.workflows/631.parallel-download/python/process.py new file mode 100644 index 000000000..e4d56fe20 --- /dev/null +++ b/benchmarks/600.workflows/631.parallel-download/python/process.py @@ -0,0 +1,11 @@ +from . import storage + +def handler(event): + bucket = event["bucket"] + blob = event["blob"] + + client = storage.storage.get_instance() + buffer = client.download_stream(bucket, blob) + + + return "ok" diff --git a/benchmarks/600.workflows/640.selfish-detour/config.json b/benchmarks/600.workflows/640.selfish-detour/config.json new file mode 100644 index 000000000..e57b5e8d1 --- /dev/null +++ b/benchmarks/600.workflows/640.selfish-detour/config.json @@ -0,0 +1,6 @@ +{ + "timeout": 300, + "memory": 1024, + "languages": ["python"], + "modules": [] +} diff --git a/benchmarks/600.workflows/640.selfish-detour/definition.json b/benchmarks/600.workflows/640.selfish-detour/definition.json new file mode 100644 index 000000000..c05a0ee6f --- /dev/null +++ b/benchmarks/600.workflows/640.selfish-detour/definition.json @@ -0,0 +1,9 @@ +{ + "root": "measure", + "states": { + "measure": { + "type": "task", + "func_name": "measure" + } + } +} \ No newline at end of file diff --git a/benchmarks/600.workflows/640.selfish-detour/input.py b/benchmarks/600.workflows/640.selfish-detour/input.py new file mode 100644 index 000000000..b1ec20540 --- /dev/null +++ b/benchmarks/600.workflows/640.selfish-detour/input.py @@ -0,0 +1,23 @@ +# Copyright 2020-2025 ETH Zurich and the SeBS authors. All rights reserved. + +size_generators = { + 'test' : 100, + 'small': 5000, + 'large': 10000, +} + +def buckets_count(): + return (0, 0) + +def generate_input(data_dir, size, benchmarks_bucket, input_buckets, output_buckets, upload_func, nosql_func): + num_samples = size_generators[size] + return { 'num_samples': num_samples } + +def validate_output(data_dir: str | None, input_config: dict, output: dict, language: str, storage = None) -> str | None: + if output is None: + return "Output is None" + + if output != "ok": + return f"Expected output to be exactly 'ok', got: {output!r}" + + return None diff --git a/benchmarks/600.workflows/640.selfish-detour/python/measure.py b/benchmarks/600.workflows/640.selfish-detour/python/measure.py new file mode 100644 index 000000000..7a0900c8f --- /dev/null +++ b/benchmarks/600.workflows/640.selfish-detour/python/measure.py @@ -0,0 +1,37 @@ +import os +import json +from ctypes import * + +def handler(event): + num_samples = event["num_samples"] + + so_file = "selfish-detour.so" + dir = os.path.dirname(os.path.abspath(__file__)) + path = os.path.join(dir, so_file) + if not os.path.exists(path): + path = os.path.join(dir, os.pardir, so_file) + + lib = cdll.LoadLibrary(path) + lib.get_ticks_per_second.restype = c_double + lib.selfish_detour.argtypes = [c_int, c_int, POINTER(c_ulonglong)] + + tps = lib.get_ticks_per_second() + assert(tps > 0) + + res = (c_ulonglong*num_samples)() + ptr = cast(res, POINTER(c_ulonglong)) + lib.selfish_detour(num_samples, 900, ptr) + + res = list(res) + assert(all(x<=y for x, y in zip(res[2:], res[3:]))) + + payload = json.dumps({ + "min_diff": res[0], + "num_iterations": res[1], + "timestamps": res[2:], + "tps": tps + }) + os.environ["SEBS_FUNCTION_RESULT"] = payload + + return "ok" + diff --git a/benchmarks/600.workflows/640.selfish-detour/python/package.sh b/benchmarks/600.workflows/640.selfish-detour/python/package.sh new file mode 100644 index 000000000..b001665b7 --- /dev/null +++ b/benchmarks/600.workflows/640.selfish-detour/python/package.sh @@ -0,0 +1,24 @@ +SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +CUR_DIR=$(pwd) +cd ${SCRIPT_DIR} + +for C_FILE in $(ls *.c 2>/dev/null) +do + SO_FILE="${C_FILE%%.*}.so" + if command -v cc &>/dev/null; then + if cc -fPIC -shared -o ${SO_FILE} ${C_FILE}; then + rm ${C_FILE} + else + echo "ERROR: Failed to compile ${C_FILE}" >&2 + exit 1 + fi + elif [ -f "${SO_FILE}" ]; then + # Pre-compiled .so is present; remove source to avoid confusion + rm ${C_FILE} + else + echo "ERROR: No C compiler found and no pre-compiled ${SO_FILE} available" >&2 + exit 1 + fi +done + +cd ${CUR_DIR} diff --git a/benchmarks/600.workflows/640.selfish-detour/python/selfish-detour.c b/benchmarks/600.workflows/640.selfish-detour/python/selfish-detour.c new file mode 100644 index 000000000..d4559d417 --- /dev/null +++ b/benchmarks/600.workflows/640.selfish-detour/python/selfish-detour.c @@ -0,0 +1,138 @@ +// THIS IS THE SELFISH DETOUR EXAMPLE FROM NETGAUGE https://spcl.inf.ethz.ch/Research/Performance/Netgauge/OS_Noise/ + +#include +#include +#include + +#define UINT64_T uint64_t +#define UINT32_T uint32_t + +typedef struct { + UINT32_T l; + UINT32_T h; +} x86_64_timeval_t; + +#define HRT_TIMESTAMP_T x86_64_timeval_t + +/* TODO: Do we need a while loop here? aka Is rdtsc atomic? - check in the documentation */ +#define HRT_GET_TIMESTAMP(t1) __asm__ __volatile__ ("rdtsc" : "=a" (t1.l), "=d" (t1.h)); + +#define HRT_GET_ELAPSED_TICKS(t1, t2, numptr) *numptr = (((( UINT64_T ) t2.h) << 32) | t2.l) - \ + (((( UINT64_T ) t1.h) << 32) | t1.l); + +#define HRT_GET_TIME(t1, time) time = (((( UINT64_T ) t1.h) << 32) | t1.l) + +double get_ticks_per_second() { + #define NUM_TESTS 10 + + HRT_TIMESTAMP_T t1, t2; + uint64_t res[NUM_TESTS]; + uint64_t min=0; + int count; + + for (count=0; count res[count]) min = res[count]; + } + + return ((double) min); +} + +void selfish_detour(int num_runs, int threshold, uint64_t *results) { + int cnt=0, num_not_smaller = 0; + HRT_TIMESTAMP_T current, prev, start; + uint64_t sample = 0; + uint64_t elapsed, thr, min=(uint64_t)~0; + int i; + + // we will do a "calibration run" of the detour benchmark to + // get a reasonable value for the minimal detour time + // just perform the benchmark and record the minimal detour time until + // this minimal detour time does not get smaller for 1000 (as defined by NOT_SMALLER) + // consecutive runs + + #define NOT_SMALLER 100 + #define INNER_TRIES 50 + + thr = min*(threshold/100.0); + while (num_not_smaller < NOT_SMALLER) { + cnt = 0; + + HRT_GET_TIMESTAMP(start); + HRT_GET_TIMESTAMP(current); + + // this is exactly the same loop as below for measurement + while (cnt < INNER_TRIES) { + prev = current; + HRT_GET_TIMESTAMP(current); + + sample++; + + HRT_GET_ELAPSED_TICKS(prev, current, &elapsed); + // != instead of < in the benchmark loop in order to make the + // notsmaller principle useful + if ( elapsed != thr ) { + HRT_GET_ELAPSED_TICKS(start, prev, &results[cnt++]); + HRT_GET_ELAPSED_TICKS(start, current, &results[cnt++]); + } + } + + // find minimum in results array - this is outside the + // calibration/measurement loop! + { + if(min == 0) { + printf("The initialization reached 0 clock cycles - the clock accuracy seems too low (setting min=1 and exiting calibration)\n"); + min = 1; + break; + } + int smaller=0; + for(i = 0; i < INNER_TRIES; i+=2) { + if(results[i+1]-results[i] < min) { + min = results[i+1]-results[i]; + smaller=1; + //printf("[%i] min: %lu\n", r, min); + } + } + if (!smaller) num_not_smaller++; + else num_not_smaller = 0; + } + } + + // now we perform the actual benchmark: Read a time-stamp-counter in a tight + // loop ignore the results if the timestamps are close to each other, as we can assume + // that nobody interrupted us. If the difference of the timestamps exceeds a certain + // threshold, we assume that we have been "hit" by a "noise event" and record the + // time difference for later analysis + + cnt = 2; + sample = 0; + + HRT_GET_TIMESTAMP(start); + HRT_GET_TIMESTAMP(current); + + // perform this outside measurement loop in order to save + // time/increase measurement frequency + thr = min*(threshold/100.0); + while (cnt < num_runs) { + prev = current; + HRT_GET_TIMESTAMP(current); + + sample++; + + HRT_GET_ELAPSED_TICKS(prev, current, &elapsed); + if ( elapsed > thr ) { + HRT_GET_ELAPSED_TICKS(start, prev, &results[cnt++]); + HRT_GET_ELAPSED_TICKS(start, current, &results[cnt++]); + } + } + + results[0] = min; + results[1] = sample; +} \ No newline at end of file diff --git a/benchmarks/600.workflows/650.vid/LICENSE b/benchmarks/600.workflows/650.vid/LICENSE new file mode 100644 index 000000000..eb52badbf --- /dev/null +++ b/benchmarks/600.workflows/650.vid/LICENSE @@ -0,0 +1,22 @@ +MIT License + +Copyright (c) 2021 EASE Lab + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + diff --git a/benchmarks/600.workflows/650.vid/README.md b/benchmarks/600.workflows/650.vid/README.md new file mode 100644 index 000000000..88e93c258 --- /dev/null +++ b/benchmarks/600.workflows/650.vid/README.md @@ -0,0 +1,3 @@ +The implementation of the Video benchmark is based on the implementation here: https://github.com/vhive-serverless/vSwarm/tree/main/benchmarks/video-analytics, with the license provided. + +We use the [ExCamera static binaries](https://github.com/excamera/excamera-static-bins). diff --git a/benchmarks/600.workflows/650.vid/config.json b/benchmarks/600.workflows/650.vid/config.json new file mode 100644 index 000000000..7c750b980 --- /dev/null +++ b/benchmarks/600.workflows/650.vid/config.json @@ -0,0 +1,6 @@ +{ + "timeout": 300, + "memory": 2048, + "languages": ["python"], + "modules": ["storage"] +} diff --git a/benchmarks/600.workflows/650.vid/definition.json b/benchmarks/600.workflows/650.vid/definition.json new file mode 100644 index 000000000..94b66ce6b --- /dev/null +++ b/benchmarks/600.workflows/650.vid/definition.json @@ -0,0 +1,26 @@ +{ + "root": "decode", + "states": { + "decode": { + "type": "task", + "func_name": "decode", + "next": "analyse-map" + }, + "analyse-map": { + "type": "map", + "root": "analyse", + "array": "frames", + "next": "summarize", + "states": { + "analyse": { + "type": "task", + "func_name": "analyse" + } + } + }, + "summarize": { + "type": "task", + "func_name": "summarize" + } + } +} diff --git a/benchmarks/600.workflows/650.vid/input.py b/benchmarks/600.workflows/650.vid/input.py new file mode 100644 index 000000000..080342ed6 --- /dev/null +++ b/benchmarks/600.workflows/650.vid/input.py @@ -0,0 +1,92 @@ +# Copyright 2020-2025 ETH Zurich and the SeBS authors. All rights reserved. + +import os + +size_generators = { + "test" : (3, 10, "video_test.mp4"), + "small": (10, 5, "video_small.mp4"), + "large": (1000, 3, "video_large.mp4"), +} + + +def buckets_count(): + return (1, 1) + + +def generate_input(data_dir, size, benchmarks_bucket, input_buckets, output_buckets, upload_func, nosql_func): + n_frames, batch_size, video_name = size_generators[size] + files = ["frozen_inference_graph.pb", "faster_rcnn_resnet50_coco_2018_01_28.pbtxt", video_name] + for name in files: + path = os.path.join(data_dir, name) + upload_func(0, name, path) + + return { + "video": video_name, + "n_frames": n_frames, + "batch_size": batch_size, + "frames_bucket": output_buckets[0], + "benchmark_bucket": benchmarks_bucket, + "input_bucket": input_buckets[0], + "model_weights": files[0], + "model_config": files[1] + } + + +def validate_output(data_dir: str | None, input_config: dict, output: dict, language: str, storage=None) -> str | None: + if output is None: + return "Output is None" + + if not isinstance(output, dict): + return f"Expected output to be a dict, got {type(output).__name__}" + + # __request_id is injected by the workflow engine, not a frame entry + frame_entries = {k: v for k, v in output.items() if k != "__request_id"} + if len(frame_entries) == 0: + return "Output dict is empty, expected at least one frame entry" + + for frame_name, detections in frame_entries.items(): + if not isinstance(detections, list): + return ( + f"Expected detections for frame '{frame_name}' to be a list, " + f"got {type(detections).__name__}" + ) + + for i, detection in enumerate(detections): + if not isinstance(detection, dict): + return ( + f"Detection {i} for frame '{frame_name}' is not a dict, " + f"got {type(detection).__name__}" + ) + + if "class" not in detection: + return f"Detection {i} for frame '{frame_name}' is missing 'class' key" + + if not isinstance(detection["class"], str): + return ( + f"Detection {i} for frame '{frame_name}' has non-string 'class': " + f"{type(detection['class']).__name__}" + ) + + if "score" not in detection: + return f"Detection {i} for frame '{frame_name}' is missing 'score' key" + + if not isinstance(detection["score"], (int, float)): + return ( + f"Detection {i} for frame '{frame_name}' has non-numeric 'score': " + f"{type(detection['score']).__name__}" + ) + + if not (0.0 <= detection["score"] <= 1.0): + return ( + f"Detection {i} for frame '{frame_name}' has score out of range " + f"[0, 1]: {detection['score']}" + ) + + # Handler filters detections at score > 0.5; any detection in output must pass this threshold + if detection["score"] <= 0.5: + return ( + f"Detection {i} for frame '{frame_name}' has score {detection['score']:.4f} " + f"<= 0.5 (handler threshold)" + ) + + return None diff --git a/benchmarks/600.workflows/650.vid/python/analyse.py b/benchmarks/600.workflows/650.vid/python/analyse.py new file mode 100644 index 000000000..1b8f31664 --- /dev/null +++ b/benchmarks/600.workflows/650.vid/python/analyse.py @@ -0,0 +1,80 @@ +import os +import io +import json +import sys +from . import storage + +import cv2 + +client = storage.storage.get_instance() + +labels = ["person", "bicycle", "car", "motorcycle", +"airplane", "bus", "train", "truck", "boat", "traffic light", "fire hydrant", +"stop sign", "parking meter", "bench", "bird", "cat", "dog", "horse", +"sheep", "cow", "elephant", "bear", "zebra", "giraffe", "backpack", +"umbrella", "handbag", "tie", "suitcase", "frisbee", "skis", +"snowboard", "sports ball", "kite", "baseball bat", "baseball glove", "skateboard", +"surfboard", "tennis racket", "bottle", "wine glass", "cup", "fork", "knife", +"spoon", "bowl", "banana", "apple", "sandwich", "orange", "broccoli", "carrot", "hot dog", +"pizza", "donut", "cake", "chair", "couch", "potted plant", "bed", "dining table", +"toilet", "tv", "laptop", "mouse", "remote", "keyboard", +"cell phone", "microwave", "oven", "toaster", "sink", "refrigerator", +"book", "clock", "vase", "scissors", "teddy bear", "hair drier", "toothbrush" ] + + +def load_model(bucket, weights_blob, config_blob, dest_dir): + weights_path = os.path.join(dest_dir, "model.weights") + client.download(bucket, weights_blob, weights_path) + + config_path = os.path.join(dest_dir, "model.config") + client.download(bucket, config_blob, config_path) + + net = cv2.dnn.readNetFromTensorflow(weights_path, config_path) + return net + + +def load_frames(benchmark_bucket, bucket, blobs, dest_dir): + for blob in blobs: + stripped_blob = blob.replace(bucket + '/', '') + path = os.path.join(dest_dir, stripped_blob) + client.download(benchmark_bucket, blob, path) + yield cv2.imread(path) + + +def detect(net, img): + rows = img.shape[0] + cols = img.shape[1] + img = cv2.dnn.blobFromImage(img, size=(300, 300), swapRB=True, crop=False) + net.setInput(img) + out = net.forward() + + preds = [] + for detection in out[0,0,:,:]: + score = float(detection[2]) + if score > 0.5: + class_id = int(detection[1]) + preds.append({ + "class": labels[class_id], + "score": score + }) + + return preds + + +def handler(event): + tmp_dir = "/tmp" + + benchmark_bucket = event["benchmark_bucket"] + + frames = list(load_frames(benchmark_bucket, event["frames_bucket"], event["frames"], tmp_dir)) + net = load_model(benchmark_bucket, event["model_bucket"] + '/' + event["model_weights"], event["model_bucket"] + '/' + event["model_config"], tmp_dir) + + preds = [detect(net, frame) for frame in frames] + + frames_names = event["frames"] + frames_names = [x.split(".")[0] for x in event["frames"]] + + preds = {f"{frames_names[idx]}": dets for idx, dets in enumerate(preds)} + + return preds + diff --git a/benchmarks/600.workflows/650.vid/python/decode.py b/benchmarks/600.workflows/650.vid/python/decode.py new file mode 100644 index 000000000..d27b67c3c --- /dev/null +++ b/benchmarks/600.workflows/650.vid/python/decode.py @@ -0,0 +1,67 @@ +import os +import uuid +from . import storage + +import cv2 + +client = storage.storage.get_instance() + + +def chunks(lst, n): + for i in range(0, len(lst), n): + yield lst[i:i + n] + + +def load_video(benchmark_bucket, bucket, blob, dest_dir): + path = os.path.join(dest_dir, blob) + client.download(benchmark_bucket, bucket + '/' + blob, path) + return path + + +def decode_video(path, n_frames, dest_dir): + vidcap = cv2.VideoCapture(path) + success, img = vidcap.read() + img_paths = [] + while success and len(img_paths) < n_frames: + img_path = os.path.join(dest_dir, f"frame{len(img_paths)}.jpg") + img_paths.append(img_path) + cv2.imwrite(img_path, img) + success, img = vidcap.read() + + return img_paths + + +def upload_imgs(benchmark_bucket, bucket, paths): + client = storage.storage.get_instance() + + for path in paths: + name = os.path.basename(path) + yield client.upload(benchmark_bucket, bucket + '/' + name, path) + + +def handler(event): + vid_blob = event["video"] + n_frames = event["n_frames"] + batch_size = event["batch_size"] + frames_bucket = event["frames_bucket"] + input_bucket = event["input_bucket"] + benchmark_bucket = event["benchmark_bucket"] + + tmp_dir = os.path.join("/tmp", str(uuid.uuid4())) + os.makedirs(tmp_dir, exist_ok=True) + + vid_path = load_video(benchmark_bucket, input_bucket, vid_blob, tmp_dir) + img_paths = decode_video(vid_path, n_frames, tmp_dir) + paths = list(upload_imgs(benchmark_bucket, frames_bucket, img_paths)) + frames = list(chunks(paths, batch_size)) + + return { + "frames": [{ + "frames_bucket": frames_bucket, + "frames": fs, + "benchmark_bucket": benchmark_bucket, + "model_bucket": input_bucket, + "model_config": event["model_config"], + "model_weights": event["model_weights"] + } for fs in frames] + } diff --git a/benchmarks/600.workflows/650.vid/python/requirements.txt b/benchmarks/600.workflows/650.vid/python/requirements.txt new file mode 100644 index 000000000..03ac2d798 --- /dev/null +++ b/benchmarks/600.workflows/650.vid/python/requirements.txt @@ -0,0 +1,2 @@ +numpy==1.26.4 +opencv-python-headless diff --git a/benchmarks/600.workflows/650.vid/python/summarize.py b/benchmarks/600.workflows/650.vid/python/summarize.py new file mode 100644 index 000000000..8d290f3f9 --- /dev/null +++ b/benchmarks/600.workflows/650.vid/python/summarize.py @@ -0,0 +1,18 @@ +import os +import io +import uuid +import json +import sys +from . import storage + + +def handler(event): + frames = event["frames"] + + logs = {} + for xs in frames: + for key,value in xs.items(): + logs[key] = value + + return logs + diff --git a/benchmarks/600.workflows/660.map-reduce/config.json b/benchmarks/600.workflows/660.map-reduce/config.json new file mode 100644 index 000000000..d06e01d56 --- /dev/null +++ b/benchmarks/600.workflows/660.map-reduce/config.json @@ -0,0 +1,6 @@ +{ + "timeout": 120, + "memory": 256, + "languages": ["python"], + "modules": ["storage"] +} diff --git a/benchmarks/600.workflows/660.map-reduce/definition.json b/benchmarks/600.workflows/660.map-reduce/definition.json new file mode 100644 index 000000000..de02b775c --- /dev/null +++ b/benchmarks/600.workflows/660.map-reduce/definition.json @@ -0,0 +1,38 @@ +{ + "root": "split", + "states": { + "split": { + "type": "task", + "func_name": "split", + "next": "map-state" + }, + "map-state": { + "type": "map", + "root": "map", + "array": "list", + "next": "shuffle", + "states": { + "map": { + "type": "task", + "func_name": "map" + } + } + }, + "shuffle": { + "type": "task", + "func_name": "shuffle", + "next": "reduce-state" + }, + "reduce-state": { + "type": "map", + "root": "reduce", + "array": "list", + "states": { + "reduce": { + "type": "task", + "func_name": "reduce" + } + } + } + } +} diff --git a/benchmarks/600.workflows/660.map-reduce/input.py b/benchmarks/600.workflows/660.map-reduce/input.py new file mode 100644 index 000000000..7348cd05b --- /dev/null +++ b/benchmarks/600.workflows/660.map-reduce/input.py @@ -0,0 +1,101 @@ +# Copyright 2020-2025 ETH Zurich and the SeBS authors. All rights reserved. + +import os +import random + +size_generators = { + "test" : (50, 3), + "small": (1000, 3), + "large": (100000, 3) +} + + +def buckets_count(): + return (1, 1) + + +def generate_input(data_dir, size, benchmarks_bucket, input_buckets, output_buckets, upload_func, nosql_func): + mult, n_mappers = size_generators[size] + words = ["cat", "dog", "bird", "horse", "pig"] + lst = mult * words + random.shuffle(lst) + + list_path = os.path.join(data_dir, "words") + list_name = "words" + with open(list_path, "w") as f: + f.writelines(w+"\n" for w in lst) + + upload_func(0, list_name, list_path) + #os.remove(list_path) + + return { + "benchmark_bucket": benchmarks_bucket, + "words_bucket": input_buckets[0], + "words": list_name, + "n_mappers": n_mappers, + "output_bucket": output_buckets[0] + } + + +def validate_output(data_dir: str | None, input_config: dict, output: dict, language: str, storage=None) -> str | None: + expected_words = {"cat", "dog", "bird", "horse", "pig"} + + if output is None: + return "Output is None" + + # Output is wrapped: {"list": [...], "__request_id": "..."} + if isinstance(output, dict): + if "list" in output: + output = output["list"] + else: + return f"Expected output dict to have 'list' key, got keys: {list(output.keys())}" + + if not isinstance(output, list): + return f"Expected output to be a list, got {type(output).__name__}" + + seen_words = set() + for i, entry in enumerate(output): + if not isinstance(entry, dict): + return f"Entry {i} is not a dict: {type(entry).__name__}" + + if "word" not in entry: + return f"Entry {i} is missing 'word' key" + if "count" not in entry: + return f"Entry {i} is missing 'count' key" + + word = entry["word"] + count = entry["count"] + + if not isinstance(word, str): + return f"Entry {i} 'word' is not a string: {type(word).__name__}" + if not isinstance(count, int): + return f"Entry {i} 'count' is not an int: {type(count).__name__}" + + if word not in expected_words: + return f"Entry {i} has unexpected word: '{word}'" + + if count <= 0: + return f"Entry {i} has non-positive count: {count}" + + seen_words.add(word) + + if seen_words != expected_words: + missing = expected_words - seen_words + return f"Missing words in output: {missing}" + + word_counts = {entry["word"]: entry["count"] for entry in output if entry["word"] in expected_words} + + # All words appear the same number of times (input is mult * ["cat","dog","bird","horse","pig"]) + counts = list(word_counts.values()) + if len(set(counts)) != 1: + return f"Word counts are not equal — expected uniform distribution, got: {word_counts}" + + per_word = counts[0] + total = per_word * len(expected_words) + + # Total must be mult * 5 words; verify it matches the n_mappers-based expectation + # n_mappers is in input_config; total words = mult * 5, which must be divisible by 5 + if total % len(expected_words) != 0: + return f"Total word count {total} is not divisible by {len(expected_words)}" + + return None diff --git a/benchmarks/600.workflows/660.map-reduce/python/map.py b/benchmarks/600.workflows/660.map-reduce/python/map.py new file mode 100644 index 000000000..0ba79ae73 --- /dev/null +++ b/benchmarks/600.workflows/660.map-reduce/python/map.py @@ -0,0 +1,36 @@ +import os +import io +from . import storage + + +def count_words(lst): + index = dict() + for word in lst: + if len(word) == 0: + continue + + val = index.get(word, 0) + index[word] = val + 1 + + return index + +def handler(event): + benchmark_bucket = event["benchmark_bucket"] + bucket = event["bucket"] + blob = event["blob"] + prefix = event["prefix"] + + client = storage.storage.get_instance() + my_buffer = client.download_stream(benchmark_bucket, bucket + '/' + blob) + words = bytes(my_buffer).decode("utf-8").split("\n") + + index = count_words(words) + for word, count in index.items(): + data = io.BytesIO() + data.write(str(count).encode("utf-8")) + data.seek(0) + + #client.upload_stream(benchmark_bucket, os.path.join(bucket, prefix, word, blob), data) + client.upload_stream(benchmark_bucket, os.path.join(prefix, word, blob), data) + + return event diff --git a/benchmarks/600.workflows/660.map-reduce/python/reduce.py b/benchmarks/600.workflows/660.map-reduce/python/reduce.py new file mode 100644 index 000000000..15fe6d707 --- /dev/null +++ b/benchmarks/600.workflows/660.map-reduce/python/reduce.py @@ -0,0 +1,23 @@ +import os +import io +import json +from . import storage + + +def handler(event): + bucket = event["bucket"] + path = event["dir"] + + client = storage.storage.get_instance() + count = 0 + #each blob is one word. + #for blob in client.list_directory(bucket, path): + for blob in client.list_directory(bucket, path): + my_buffer = client.download_stream(bucket, blob) + count += int(bytes(my_buffer).decode("utf-8")) + #count += int(my_buffer.getvalue().decode("utf-8")) + + return { + "word": os.path.basename(path), + "count": count + } diff --git a/benchmarks/600.workflows/660.map-reduce/python/shuffle.py b/benchmarks/600.workflows/660.map-reduce/python/shuffle.py new file mode 100644 index 000000000..44568f27d --- /dev/null +++ b/benchmarks/600.workflows/660.map-reduce/python/shuffle.py @@ -0,0 +1,27 @@ +import os +import json +from . import storage + + +def handler(event): + lst = event["list"] + benchmark_bucket = lst[0]["benchmark_bucket"] + bucket = lst[0]["bucket"] + prefix = lst[0]["prefix"] + + client = storage.storage.get_instance() + dirs = client.list_directory(benchmark_bucket, prefix) + dirs = [p.split(os.sep)[1] for p in dirs] + dirs = list(set(dirs)) + lst = [{ + "bucket": benchmark_bucket, + #"dir": os.path.join(bucket, prefix, path) + #TODO add word here. + "dir": os.path.join(prefix, path) + #"dir": os.path.join(bucket, prefix) + } for path in dirs] + + + return { + "list": lst + } diff --git a/benchmarks/600.workflows/660.map-reduce/python/split.py b/benchmarks/600.workflows/660.map-reduce/python/split.py new file mode 100644 index 000000000..941ffdfff --- /dev/null +++ b/benchmarks/600.workflows/660.map-reduce/python/split.py @@ -0,0 +1,53 @@ +import os +import io +import uuid +from . import storage + +def chunks(lst, n): + m = int(len(lst) / n) + for i in range(n-1): + yield lst[i*m:i*m+m] + tail = lst[(n-1)*m:] + if len(tail) > 0: + yield tail + + +def handler(event): + benchmark_bucket = event["benchmark_bucket"] + words_bucket = event["words_bucket"] + words_blob = event["words"] + words_path = os.path.join("/tmp", "words.txt") + + client = storage.storage.get_instance() + client.download(benchmark_bucket, words_bucket + '/' + words_blob, words_path) + with open(words_path, "r") as f: + list = f.read().split("\n") + os.remove(words_path) + + n_mappers = event["n_mappers"] + output_bucket = event["output_bucket"] + map_lists = chunks(list, n_mappers) + blobs = [] + + + for chunk in map_lists: + name = str(uuid.uuid4())[:8] + data = io.BytesIO() + data.writelines((val+"\n").encode("utf-8") for val in chunk) + data.seek(0) + + name = client.upload_stream(benchmark_bucket, output_bucket + '/' + name, data) + stripped_name = name.replace(output_bucket + '/', '') + blobs.append(stripped_name) + + prefix = str(uuid.uuid4())[:8] + lst = [{ + "benchmark_bucket": benchmark_bucket, + "bucket": output_bucket, + "blob": b, + "prefix": prefix + } for b in blobs] + + return { + "list": lst + } diff --git a/benchmarks/600.workflows/670.auth/config.json b/benchmarks/600.workflows/670.auth/config.json new file mode 100644 index 000000000..d6d184e8a --- /dev/null +++ b/benchmarks/600.workflows/670.auth/config.json @@ -0,0 +1,6 @@ +{ + "timeout": 120, + "memory": 256, + "languages": ["python"], + "modules": [] +} diff --git a/benchmarks/600.workflows/670.auth/definition.json b/benchmarks/600.workflows/670.auth/definition.json new file mode 100644 index 000000000..a6511630b --- /dev/null +++ b/benchmarks/600.workflows/670.auth/definition.json @@ -0,0 +1,9 @@ +{ + "root": "auth", + "states": { + "auth": { + "type": "task", + "func_name": "auth" + } + } +} \ No newline at end of file diff --git a/benchmarks/600.workflows/670.auth/input.py b/benchmarks/600.workflows/670.auth/input.py new file mode 100644 index 000000000..2dba2ea27 --- /dev/null +++ b/benchmarks/600.workflows/670.auth/input.py @@ -0,0 +1,66 @@ +# Copyright 2020-2025 ETH Zurich and the SeBS authors. All rights reserved. + +import base64 +import random + +size_generators = { + "test" : 10, + "small": 100, + "large": 1000 +} + + +def buckets_count(): + return (0, 0) + + +def generate_input(data_dir, size, benchmarks_bucket, input_buckets, output_buckets, upload_func, nosql_func): + mult = size_generators[size] + msg = "Who let the dogs out?\n" * mult + + return { + "message": msg, + "token": "allow" + } + + +def validate_output(data_dir: str | None, input_config: dict, output: dict, language: str, storage = None) -> str | None: + if output is None: + return "Output is None" + + if "response" not in output: + return "Missing 'response' key in output" + + response = output["response"] + + if response == "unauthorized": + return "Response is 'unauthorized', expected encrypted message" + + try: + decoded = base64.b64decode(response) + except Exception as e: + return f"Response is not a valid base64 string: {e}" + + message = input_config["message"] + expected_length = len(message) + if len(decoded) != expected_length: + return ( + f"Decoded ciphertext length ({len(decoded)}) does not match " + f"input message length ({expected_length})" + ) + + # Decrypt and verify the ciphertext matches the original plaintext. + # The handler uses AES-CTR with a fixed key and counter starting at 0. + import pyaes + KEY = "6368616e676520746869732070617373".encode("utf-8") + counter = pyaes.Counter(initial_value=0) + aes = pyaes.AESModeOfOperationCTR(KEY, counter=counter) + decrypted = aes.decrypt(decoded) + if isinstance(message, str): + message_bytes = message.encode("utf-8") + else: + message_bytes = message + if decrypted != message_bytes: + return "Decrypted ciphertext does not match original input message" + + return None diff --git a/benchmarks/600.workflows/670.auth/python/auth.py b/benchmarks/600.workflows/670.auth/python/auth.py new file mode 100644 index 000000000..c7b77649c --- /dev/null +++ b/benchmarks/600.workflows/670.auth/python/auth.py @@ -0,0 +1,39 @@ +import random +import string +import pyaes +import base64 + +KEY = "6368616e676520746869732070617373".encode("utf-8") + + +def AESModeCTR(plaintext): + counter = pyaes.Counter(initial_value=0) + aes = pyaes.AESModeOfOperationCTR(KEY, counter=counter) + ciphertext = aes.encrypt(plaintext) + return ciphertext + + +def AESModeCBC(plaintext): + # random initialization vector of 16 bytes + blocks_size = 16 + iv = "InitializationVe" + pad = 16 - len(plaintext)% blocks_size + plaintext = str("0" * pad) + plaintext + aes = pyaes.AESModeOfOperationCBC(KEY, iv=iv) + ciphertext = aes.encrypt(plaintext) + + return ciphertext.decode("utf-8") + + +def handler(event): + message = event["message"] + token = event["token"] + + res = "unauthorized" + if token == "allow": + res = AESModeCTR(message) + res = base64.b64encode(res).decode("ascii") + + return { + "response": res + } diff --git a/benchmarks/600.workflows/670.auth/python/requirements.txt b/benchmarks/600.workflows/670.auth/python/requirements.txt new file mode 100644 index 000000000..68abeee58 --- /dev/null +++ b/benchmarks/600.workflows/670.auth/python/requirements.txt @@ -0,0 +1 @@ +pyaes==1.6.1 diff --git a/benchmarks/600.workflows/680.excamera/LICENSE b/benchmarks/600.workflows/680.excamera/LICENSE new file mode 100644 index 000000000..eb52badbf --- /dev/null +++ b/benchmarks/600.workflows/680.excamera/LICENSE @@ -0,0 +1,22 @@ +MIT License + +Copyright (c) 2021 EASE Lab + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + diff --git a/benchmarks/600.workflows/680.excamera/README.md b/benchmarks/600.workflows/680.excamera/README.md new file mode 100644 index 000000000..ef2a26cb6 --- /dev/null +++ b/benchmarks/600.workflows/680.excamera/README.md @@ -0,0 +1,3 @@ +The implementation of the ExCamera benchmarks is based on an adaptation of the vSwarm benchmark: https://github.com/vhive-serverless/vSwarm/tree/main/benchmarks/gg/benchmarks/excamera with the license provided. + +To encode the video, the static binaries of the ExCamera project are used: https://github.com/excamera/excamera-static-bins diff --git a/benchmarks/600.workflows/680.excamera/config.json b/benchmarks/600.workflows/680.excamera/config.json new file mode 100644 index 000000000..5118297aa --- /dev/null +++ b/benchmarks/600.workflows/680.excamera/config.json @@ -0,0 +1,6 @@ +{ + "timeout": 540, + "memory": 256, + "languages": ["python"], + "modules": ["storage"] +} diff --git a/benchmarks/600.workflows/680.excamera/definition.json b/benchmarks/600.workflows/680.excamera/definition.json new file mode 100644 index 000000000..0a494b85b --- /dev/null +++ b/benchmarks/600.workflows/680.excamera/definition.json @@ -0,0 +1,39 @@ +{ + "root": "split", + "states": { + "split": { + "type": "task", + "func_name": "split", + "next": "encode-state" + }, + "encode-state": { + "type": "map", + "root": "encode", + "array": "segments", + "next": "reencode-state", + "states": { + "encode": { + "type": "task", + "func_name": "encode" + } + } + }, + "reencode-state": { + "type": "map", + "root": "reencode", + "array": "segments", + "next": "rebase-state", + "states": { + "reencode": { + "type": "task", + "func_name": "reencode" + } + } + }, + "rebase-state": { + "type": "loop", + "func_name": "rebase", + "array": "segments" + } + } +} diff --git a/benchmarks/600.workflows/680.excamera/input.py b/benchmarks/600.workflows/680.excamera/input.py new file mode 100644 index 000000000..8d01ea743 --- /dev/null +++ b/benchmarks/600.workflows/680.excamera/input.py @@ -0,0 +1,118 @@ +# Copyright 2020-2025 ETH Zurich and the SeBS authors. All rights reserved. +import random +import os + +size_generators = { + "test" : (18, 6), + "small": (30, 6), + "large": (60, 6) +} + + +def buckets_count(): + return (1, 1) + + +def generate_input(data_dir, size, benchmarks_bucket, input_buckets, output_buckets, upload_func, nosql_func): + num_frames, batch_size = size_generators[size] + + for bin in os.listdir(data_dir): + path = os.path.join(data_dir, bin) + if os.path.isfile(path): + upload_func(0, bin, path) + + vid_dir = os.path.join(data_dir, "vid") + vid_segs = sorted(os.listdir(vid_dir)) + new_vid_segs = [] + + for i in range(num_frames): + seg = vid_segs[i % len(vid_segs)] + name = "{:08.0f}.y4m".format(i) + path = os.path.join(vid_dir, seg) + + new_vid_segs.append(name) + upload_func(0, name, path) + + assert(len(new_vid_segs) == num_frames) + + return { + "segments": new_vid_segs, + "benchmark_bucket": benchmarks_bucket, + "input_bucket": input_buckets[0], + "output_bucket": output_buckets[0], + "batch_size": batch_size, + "quality": 1 + } + + +def validate_output(data_dir: str | None, input_config: dict, output: dict, language: str, storage=None) -> str | None: + if output is None: + return "Output is None" + + # Output structure: {"segments": [list of batch dicts], "__request_id": "..."} + # Each batch dict has: {"segments": [...y4m names...], "benchmark_bucket": "...", "output_bucket": "...", ...} + if isinstance(output, dict): + if "segments" not in output: + return f"Expected 'segments' key in output, got keys: {list(output.keys())}" + items = output["segments"] + if not isinstance(items, list): + return f"Expected 'segments' to be a list, got {type(items).__name__}" + elif isinstance(output, list): + items = output + else: + return f"Expected output to be a dict or list, got {type(output).__name__}" + + if len(items) == 0: + return "Output 'segments' list is empty" + + for i, item in enumerate(items): + if not isinstance(item, dict): + return f"Segment batch {i} is not a dict, got {type(item).__name__}" + + if "benchmark_bucket" not in item: + return f"Segment batch {i} missing 'benchmark_bucket'" + if "output_bucket" not in item: + return f"Segment batch {i} missing 'output_bucket'" + if "segments" not in item: + return f"Segment batch {i} missing 'segments'" + + segs = item["segments"] + if not isinstance(segs, list) or len(segs) == 0: + return f"Segment batch {i} 'segments' must be a non-empty list" + + if "quality" not in item: + return f"Segment batch {i} missing 'quality'" + if item["quality"] != input_config.get("quality"): + return f"Segment batch {i} quality {item['quality']} != input quality {input_config.get('quality')}" + + for seg in segs: + if not isinstance(seg, str) or not seg.endswith(".y4m"): + return f"Segment batch {i} contains non-.y4m segment: {seg!r}" + + import math + + input_segs = input_config.get("segments", []) + expected_segs = len(input_segs) + + # Every input segment must appear exactly once across all output batches + if expected_segs > 0: + output_segs = [seg for item in items for seg in item["segments"]] + if len(output_segs) != expected_segs: + return f"Total segments in output ({len(output_segs)}) != input segments ({expected_segs})" + output_segs_set = set(output_segs) + input_segs_set = set(input_segs) + missing = input_segs_set - output_segs_set + if missing: + return f"Segments missing from output: {sorted(missing)}" + extra = output_segs_set - input_segs_set + if extra: + return f"Unexpected segments in output: {sorted(extra)}" + + # Batch count should be ceil(n_segments / batch_size) + batch_size = input_config.get("batch_size") + if batch_size and expected_segs > 0: + expected_batches = math.ceil(expected_segs / batch_size) + if len(items) != expected_batches: + return f"Expected {expected_batches} batches (ceil({expected_segs}/{batch_size})), got {len(items)}" + + return None diff --git a/benchmarks/600.workflows/680.excamera/python/encode.py b/benchmarks/600.workflows/680.excamera/python/encode.py new file mode 100644 index 000000000..44a84c5ec --- /dev/null +++ b/benchmarks/600.workflows/680.excamera/python/encode.py @@ -0,0 +1,88 @@ +import os +import uuid +import subprocess +from . import storage +import logging +import shutil + +VPXENC = "/tmp/vpxenc --ivf --codec=vp8 --good --cpu-used=0 --end-usage=cq --min-q=0 --max-q=63 --cq-level={quality} --buf-initial-sz=10000 --buf-optimal-sz=20000 --buf-sz=40000 --undershoot-pct=100 --passes=2 --auto-alt-ref=1 --threads=1 --token-parts=0 --tune=ssim --target-bitrate=4294967295 -o {output}.ivf {input}.y4m" +TERMINATE_CHUNK = "/tmp/xc-terminate-chunk {input}.ivf {output}.ivf" +XC_DUMP_0 = "/tmp/xc-dump {input}.ivf {output}.state" + +client = storage.storage.get_instance() + +def download_bin(benchmark_bucket, bucket, name, dest_dir): + path = os.path.join(dest_dir, name) + if not os.path.exists(path): + client.download(benchmark_bucket, bucket + '/' + name, path) + subprocess.check_output(f"chmod +x {path}", stderr=subprocess.STDOUT, shell=True) + + +def upload_files(benchmark_bucket, bucket, paths, prefix): + for path in paths: + file = os.path.basename(path) + file = prefix + file + #print("Uploading", file, "to", path) + client.upload(benchmark_bucket, bucket + '/' + file, path, unique_name=False) + + +def run(cmd): + try: + return subprocess.check_output(cmd, stderr=subprocess.STDOUT, shell=True) + except subprocess.CalledProcessError as e: + logger = logging.getLogger() + logger.error(f"Error when executing command: {cmd}\n{e.output.decode('utf-8')}") + raise e + + +def encode(segs, data_dir, quality): + files = [] + + for idx, name in enumerate(segs): + input_path = os.path.join(data_dir, name) + output_path = os.path.join(data_dir, f"{name}-vpxenc") + cmd = VPXENC.format(quality=quality, input=input_path, output=output_path) + run(cmd) + + input_path = output_path + output = name if idx == 0 else f"{name}-0" + output_path = os.path.join(data_dir, output) + cmd = TERMINATE_CHUNK.format(input=input_path, output=output_path) + run(cmd) + files.append(output_path+".ivf") + + input_path = output_path + output_path = os.path.join(data_dir, f"{name}-0") + cmd = XC_DUMP_0.format(input=input_path, output=output_path) + run(cmd) + files.append(output_path+".state") + + return files + + +def handler(event): + input_bucket = event["input_bucket"] + output_bucket = event["output_bucket"] + benchmark_bucket = event["benchmark_bucket"] + segs = event["segments"] + quality = event["quality"] + prefix = event["prefix"] + + tmp_dir = "/tmp" + download_bin(benchmark_bucket, input_bucket, "vpxenc", tmp_dir) + download_bin(benchmark_bucket, input_bucket, "xc-terminate-chunk", tmp_dir) + download_bin(benchmark_bucket, input_bucket, "xc-dump", tmp_dir) + + data_dir = os.path.join(tmp_dir, str(uuid.uuid4())) + os.makedirs(data_dir, exist_ok=True) + for seg in segs: + path = os.path.join(data_dir, seg) + client.download(benchmark_bucket, input_bucket + '/' + seg, path) + + segs = [os.path.splitext(seg)[0] for seg in segs] + output_paths = encode(segs, data_dir, quality) + upload_files(benchmark_bucket, output_bucket, output_paths, prefix) + + shutil.rmtree(data_dir) + + return event diff --git a/benchmarks/600.workflows/680.excamera/python/rebase.py b/benchmarks/600.workflows/680.excamera/python/rebase.py new file mode 100644 index 000000000..809774305 --- /dev/null +++ b/benchmarks/600.workflows/680.excamera/python/rebase.py @@ -0,0 +1,115 @@ +import os +import uuid +import subprocess +from . import storage +import logging +import shutil + +XC_ENC_REBASE = "/tmp/xc-enc -W -w 0.75 -i y4m -o {output}.ivf -r -I {source_state}.state -p {input_pred}.ivf -S {pred_state}.state {extra} {input}.y4m" + +client = storage.storage.get_instance() + +def download_bin(benchmark_bucket, bucket, name, dest_dir): + path = os.path.join(dest_dir, name) + if not os.path.exists(path): + client.download(benchmark_bucket, bucket + '/' + name, path) + subprocess.check_output(f"chmod +x {path}", stderr=subprocess.STDOUT, shell=True) + + +def upload_files(benchmark_bucket, bucket, paths, prefix): + for path in paths: + file = os.path.basename(path) + file = prefix + file + #print("Uploading", file, "to", path) + client.upload(benchmark_bucket, bucket + '/' + file, path, unique_name=False) + + +def run(cmd): + try: + return subprocess.check_output(cmd, stderr=subprocess.STDOUT, shell=True) + except subprocess.CalledProcessError as e: + logger = logging.getLogger() + logger.error(f"Error when executing command: {cmd}\n{e.output.decode('utf-8')}") + raise e + + +def prev_seg_name(seg): + idx = int(seg)-1 + assert(idx >= 0) + return "{:08d}".format(idx) + + +def rebase(segs, data_dir, dry_run=False): + input_paths = [] + output_paths = [] + + for idx in range(2, len(segs)): + name = segs[idx] + input_path = os.path.join(data_dir, name) + prev_input_path = os.path.join(data_dir, prev_seg_name(name)) + source_state_path = f"{prev_input_path}-1" + output_state_path = f"{input_path}-1.state" + extra = f"-O {output_state_path}" if idx != len(segs)-1 else "" + input_pred_path = f"{input_path}-1" + pred_state_path = f"{prev_input_path}-0" + + cmd = XC_ENC_REBASE.format( + output=input_path, + input=input_path, + source_state=source_state_path, + extra=extra, + input_pred=input_pred_path, + pred_state=pred_state_path) + if not dry_run: + run(cmd) + + + input_paths.append(input_path+".y4m") + input_paths.append(source_state_path+".state") + input_paths.append(input_pred_path+".ivf") + input_paths.append(pred_state_path+".state") + + output_paths.append(input_path+".ivf") + if idx != len(segs)-1: + output_paths.append(output_state_path) + + return input_paths, output_paths + + +def handler(event): + input_bucket = event["input_bucket"] + output_bucket = event["output_bucket"] + benchmark_bucket = event["benchmark_bucket"] + segs = event["segments"] + segs = [os.path.splitext(seg)[0] for seg in segs] + prefix = event["prefix"] + + tmp_dir = "/tmp" + download_bin(benchmark_bucket, input_bucket, "xc-enc", tmp_dir) + + data_dir = os.path.join(tmp_dir, str(uuid.uuid4())) + os.makedirs(data_dir, exist_ok=True) + + input_paths, _ = rebase(segs, data_dir, dry_run=True) + + for path in input_paths: + file = os.path.basename(path) + + try: + if ".y4m" in file: + client.download(benchmark_bucket, input_bucket + '/' + file, path) + else: + file = prefix + file + client.download(benchmark_bucket, output_bucket + '/' + file, path) + except: + # -1.state is generated by rebase itself + if not "-1.state" in file: + raise + + _, output_paths = rebase(segs, data_dir) + upload_files(benchmark_bucket, output_bucket, output_paths, prefix) + + + shutil.rmtree(data_dir) + + return event diff --git a/benchmarks/600.workflows/680.excamera/python/reencode.py b/benchmarks/600.workflows/680.excamera/python/reencode.py new file mode 100644 index 000000000..ee9b6576a --- /dev/null +++ b/benchmarks/600.workflows/680.excamera/python/reencode.py @@ -0,0 +1,103 @@ +import os +import uuid +import subprocess +from . import storage +import logging +import shutil + +XC_ENC_FIRST_FRAME = "/tmp/xc-enc -W -w 0.75 -i y4m -o {output}.ivf -r -I {source_state}.state -p {input_pred}.ivf {extra} {input}.y4m" + +client = storage.storage.get_instance() + +def download_bin(benchmark_bucket, bucket, name, dest_dir): + path = os.path.join(dest_dir, name) + if not os.path.exists(path): + client.download(benchmark_bucket, bucket + '/' + name, path) + subprocess.check_output(f"chmod +x {path}", stderr=subprocess.STDOUT, shell=True) + + +def upload_files(benchmark_bucket, bucket, paths, prefix): + for path in paths: + file = os.path.basename(path) + file = prefix + file + client.upload(benchmark_bucket, bucket + '/' + file, path, unique_name=False) + + +def run(cmd): + try: + return subprocess.check_output(cmd, stderr=subprocess.STDOUT, shell=True) + except subprocess.CalledProcessError as e: + logger = logging.getLogger() + logger.error(f"Error when executing command: {cmd}\n{e.output.decode('utf-8')}") + raise e + + +def prev_seg_name(seg): + idx = int(seg)-1 + assert(idx >= 0) + return "{:08d}".format(idx) + + +def reencode_first_frame(segs, data_dir, dry_run=False): + input_paths = [] + output_paths = [] + for idx in range(1, len(segs)): + name = segs[idx] + input_path = os.path.join(data_dir, name) + output_path = input_path if idx == 1 else f"{input_path}-1" + source_state_path = os.path.join(data_dir, prev_seg_name(name))+"-0" + output_state_path = f"{input_path}-1.state" + extra = f"-O {output_state_path}" if idx == 1 else "" + input_pred_path = f"{input_path}-0" + + cmd = XC_ENC_FIRST_FRAME.format( + input=input_path, + output=output_path, + source_state=source_state_path, + extra=extra, + input_pred=input_pred_path) + if not dry_run: + run(cmd) + + input_paths.append(input_path+".y4m") + input_paths.append(source_state_path+".state") + input_paths.append(input_pred_path+".ivf") + + output_paths.append(output_path+".ivf") + if idx == 1: + output_paths.append(output_state_path) + + return input_paths, output_paths + + +def handler(event): + input_bucket = event["input_bucket"] + output_bucket = event["output_bucket"] + benchmark_bucket = event["benchmark_bucket"] + segs = event["segments"] + segs = [os.path.splitext(seg)[0] for seg in segs] + prefix = event["prefix"] + + tmp_dir = "/tmp" + download_bin(benchmark_bucket, input_bucket, "xc-enc", tmp_dir) + + data_dir = os.path.join(tmp_dir, str(uuid.uuid4())) + os.makedirs(data_dir, exist_ok=True) + input_paths, _ = reencode_first_frame(segs, data_dir, dry_run=True) + for path in input_paths: + file = os.path.basename(path) + + if ".y4m" in file: + client.download(benchmark_bucket, input_bucket + '/' + file, path) + else: + file = prefix + file + client.download(benchmark_bucket, output_bucket + '/' + file, path) + + + + _, output_paths = reencode_first_frame(segs, data_dir) + upload_files(benchmark_bucket, output_bucket, output_paths, prefix) + + shutil.rmtree(data_dir) + + return event diff --git a/benchmarks/600.workflows/680.excamera/python/requirements.txt b/benchmarks/600.workflows/680.excamera/python/requirements.txt new file mode 100644 index 000000000..59348f98e --- /dev/null +++ b/benchmarks/600.workflows/680.excamera/python/requirements.txt @@ -0,0 +1 @@ +gitpython diff --git a/benchmarks/600.workflows/680.excamera/python/split.py b/benchmarks/600.workflows/680.excamera/python/split.py new file mode 100644 index 000000000..5ecfad0b6 --- /dev/null +++ b/benchmarks/600.workflows/680.excamera/python/split.py @@ -0,0 +1,26 @@ +import uuid + +def chunks(lst, n): + for i in range(0, len(lst), n): + yield lst[i:i + n] + + +def handler(event): + segs = chunks(event["segments"], event["batch_size"]) + input_bucket = event["input_bucket"] + output_bucket = event["output_bucket"] + benchmark_bucket = event["benchmark_bucket"] + quality = event["quality"] + + return { + "segments": [ + { + "prefix": str(uuid.uuid4().int & (1<<64)-1)[:8], + "segments": ss, + "quality": quality, + "input_bucket": input_bucket, + "output_bucket": output_bucket, + "benchmark_bucket": benchmark_bucket + } for idx, ss in enumerate(segs) + ] + } diff --git a/benchmarks/600.workflows/690.ml/config.json b/benchmarks/600.workflows/690.ml/config.json new file mode 100644 index 000000000..f0dd50968 --- /dev/null +++ b/benchmarks/600.workflows/690.ml/config.json @@ -0,0 +1,6 @@ +{ + "timeout": 120, + "memory": 1024, + "languages": ["python"], + "modules": ["storage"] +} diff --git a/benchmarks/600.workflows/690.ml/definition.json b/benchmarks/600.workflows/690.ml/definition.json new file mode 100644 index 000000000..3e427fcd9 --- /dev/null +++ b/benchmarks/600.workflows/690.ml/definition.json @@ -0,0 +1,21 @@ +{ + "root": "generate", + "states": { + "generate": { + "type": "task", + "func_name": "generate", + "next": "train-state" + }, + "train-state": { + "type": "map", + "root": "train", + "array": "schedules", + "states": { + "train": { + "type": "task", + "func_name": "train" + } + } + } + } +} diff --git a/benchmarks/600.workflows/690.ml/input.py b/benchmarks/600.workflows/690.ml/input.py new file mode 100644 index 000000000..a8a29e5dd --- /dev/null +++ b/benchmarks/600.workflows/690.ml/input.py @@ -0,0 +1,80 @@ +# Copyright 2020-2025 ETH Zurich and the SeBS authors. All rights reserved. + +size_generators = { + "test" : (1, 100, 5), + "small": (2, 500, 1024), + "large": (3, 1000, 1024), +} + +classifiers = [ + {"name": "SVC", "kernel": "linear", "C": 0.025}, + {"name": "RandomForestClassifier", "max_depth": 5, "n_estimators": 10}, + {"name": "RandomForestClassifier", "max_depth": 5, "n_estimators": 15}, + {"name": "AdaBoostClassifier"} +] + +VALID_CLASSIFIER_NAMES = ["SVC", "RandomForestClassifier", "AdaBoostClassifier"] + +def buckets_count(): + return (0, 1) + +def generate_input(data_dir, size, benchmarks_bucket, input_buckets, output_buckets, upload_func, nosql_func): + n_classifiers, n_samples, n_features = size_generators[size] + return { + "classifiers": classifiers[:n_classifiers], + "benchmark_bucket" : benchmarks_bucket, + "dataset_bucket": output_buckets[0], + "n_samples": n_samples, + "n_features": n_features + } + +def validate_output(data_dir: str | None, input_config: dict, output: dict, language: str, storage = None) -> str | None: + if output is None: + return "Output is None" + + # Step Functions returns {"schedules": [...], "__request_id": "..."} + if isinstance(output, dict): + if "schedules" not in output: + return f"Expected 'schedules' key in output dict, got keys: {list(output.keys())}" + output = output["schedules"] + + if not isinstance(output, list): + return f"Expected output to be a list, got {type(output).__name__}" + + input_classifiers = input_config["classifiers"] + expected_count = len(input_classifiers) + if len(output) != expected_count: + return f"Expected {expected_count} results, got {len(output)}" + + # Build expected names in order from input + expected_names = [c["name"] for c in input_classifiers] + + for i, entry in enumerate(output): + if not isinstance(entry, dict): + return f"Entry {i} is not a dict, got {type(entry).__name__}" + + if "name" not in entry: + return f"Entry {i} is missing 'name' field" + if "score" not in entry: + return f"Entry {i} is missing 'score' field" + + name = entry["name"] + score = entry["score"] + + if not isinstance(name, str): + return f"Entry {i} 'name' is not a string, got {type(name).__name__}" + + if name not in VALID_CLASSIFIER_NAMES: + return f"Entry {i} has invalid classifier name '{name}', expected one of {VALID_CLASSIFIER_NAMES}" + + # Output classifier name must match the corresponding input classifier + if name != expected_names[i]: + return f"Entry {i} classifier name '{name}' does not match input classifier '{expected_names[i]}'" + + if not isinstance(score, (int, float)): + return f"Entry {i} 'score' is not a float, got {type(score).__name__}" + + if score < 0.0 or score > 1.0: + return f"Entry {i} 'score' is {score}, expected value between 0.0 and 1.0" + + return None diff --git a/benchmarks/600.workflows/690.ml/python/generate.py b/benchmarks/600.workflows/690.ml/python/generate.py new file mode 100644 index 000000000..03fea03db --- /dev/null +++ b/benchmarks/600.workflows/690.ml/python/generate.py @@ -0,0 +1,54 @@ +import os +import uuid +from . import storage + +import sklearn.datasets as datasets +import numpy as np + + +def generate(n_samples, n_features): + X, y = datasets.make_classification( + n_samples, + n_features, + n_redundant=0, + n_clusters_per_class=2, + weights=[0.9, 0.1], + flip_y=0.1, + random_state=123 + ) + + return X, y + + +def upload_dataset(benchmark_bucket, bucket, X, y): + dataset_dir = os.path.join("/tmp", str(uuid.uuid4())) + os.makedirs(dataset_dir, exist_ok=True) + + features_path = os.path.join(dataset_dir, "features.npy") + labels_path = os.path.join(dataset_dir, "labels.npy") + np.save(features_path, X) + np.save(labels_path, y) + + client = storage.storage.get_instance() + features = client.upload(benchmark_bucket, bucket + '/' + "features.npy", features_path) + features = features.replace(bucket + '/', '') + labels = client.upload(benchmark_bucket, bucket + '/' + "labels.npy", labels_path) + labels = labels.replace(bucket + '/', '') + + return features, labels + + +def handler(event): + classifiers = event["classifiers"] + bucket = event["dataset_bucket"] + benchmark_bucket = event["benchmark_bucket"] + n_samples = int(event["n_samples"]) + n_features = int(event["n_features"]) + + X, y = generate(n_samples, n_features) + X_key, y_key = upload_dataset(benchmark_bucket, bucket, X, y) + + schedules = [{**c, "features": X_key, "labels": y_key, "bucket": bucket, "benchmark_bucket": benchmark_bucket} for c in classifiers] + return { + "schedules": schedules + } diff --git a/benchmarks/600.workflows/690.ml/python/package.sh b/benchmarks/600.workflows/690.ml/python/package.sh new file mode 100644 index 000000000..1133cbac2 --- /dev/null +++ b/benchmarks/600.workflows/690.ml/python/package.sh @@ -0,0 +1,25 @@ +# Stripping package code is based on https://github.com/ryfeus/lambda-packs repo + +PACKAGE_DIR=$1 +echo "Original size $(du -sh $1 | cut -f1)" + +CUR_DIR=$(pwd) +cd $1 +# cleaning libs +rm -rf external +find . -type d -name "tests" -exec rm -rf {} + +find . -type d -name "test" -exec rm -rf {} + +find . -type d -name "bin" -exec rm -rf {} + + +# cleaning +find -name "*.so" -not -path "*/PIL/*" | xargs strip +find -name "*.so.*" -not -path "*/PIL/*" | xargs strip + +rm -r pip > /dev/null +rm -r pip-* > /dev/null +rm -r wheel > /dev/null +rm -r wheel-* > /dev/null +rm easy_install.py > /dev/null +find . -name \*.pyc -delete +cd ${CUR_DIR} +echo "Stripped size $(du -sh $1 | cut -f1)" diff --git a/benchmarks/600.workflows/690.ml/python/requirements.txt b/benchmarks/600.workflows/690.ml/python/requirements.txt new file mode 100644 index 000000000..589cdb5fa --- /dev/null +++ b/benchmarks/600.workflows/690.ml/python/requirements.txt @@ -0,0 +1,3 @@ +numpy==1.26.4 +scipy==1.10.0 +scikit-learn==1.5.2 diff --git a/benchmarks/600.workflows/690.ml/python/train.py b/benchmarks/600.workflows/690.ml/python/train.py new file mode 100644 index 000000000..d886a3072 --- /dev/null +++ b/benchmarks/600.workflows/690.ml/python/train.py @@ -0,0 +1,72 @@ +import os +import uuid +import sys +from . import storage + +from sklearn.model_selection import train_test_split +from sklearn.svm import SVC +from sklearn.ensemble import RandomForestClassifier, AdaBoostClassifier +from sklearn.preprocessing import StandardScaler +import numpy as np + +def str_to_cls(cls_name): + #print(cls_name) + return globals()[cls_name] + +def load_dataset(benchmark_bucket, bucket, features, labels): + dataset_dir = os.path.join("/tmp", str(uuid.uuid4())) + os.makedirs(dataset_dir, exist_ok=True) + + features_path = os.path.join(dataset_dir, "features.npy") + labels_path = os.path.join(dataset_dir, "labels.npy") + + + client = storage.storage.get_instance() + client.download(benchmark_bucket, bucket + '/' + features, features_path) + client.download(benchmark_bucket, bucket + '/' + labels, labels_path) + + X = np.load(features_path) + y = np.load(labels_path) + + return X, y + + +def preprocess(X, y): + X = StandardScaler().fit_transform(X) + + X_train, X_test, y_train, y_test = train_test_split( + X, y, test_size=0.4, random_state=123 + ) + + return X_train, X_test, y_train, y_test + + +def train(clf, X, y): + clf.fit(X, y) + + +def val(clf, X, y): + return clf.score(X, y) + + +def handler(schedule): + name = schedule.pop("name") + X_key = schedule.pop("features") + y_key = schedule.pop("labels") + bucket = schedule.pop("bucket") + benchmark_bucket = schedule.pop("benchmark_bucket") + request_id = schedule.pop("request-id") + + clf = str_to_cls(name)(**schedule) + + X, y = load_dataset(benchmark_bucket, bucket, X_key, y_key) + X_train, X_test, y_train, y_test = preprocess(X, y) + + train(clf, X_train, y_train) + score = val(clf, X_test, y_test) + + return { + "name": name, + "score": score + } + diff --git a/benchmarks/wrappers/aws/python/handler_workflow.py b/benchmarks/wrappers/aws/python/handler_workflow.py new file mode 100644 index 000000000..60cb2cdfb --- /dev/null +++ b/benchmarks/wrappers/aws/python/handler_workflow.py @@ -0,0 +1,93 @@ +import datetime +import io +import json +import os +import sys +import uuid +import importlib + +# Add current directory to allow location of packages +sys.path.append(os.path.join(os.path.dirname(__file__), ".python_packages/lib/site-packages")) + +from redis import Redis + + +def probe_cold_start(): + is_cold = False + fname = os.path.join("/tmp", "cold_run") + if not os.path.exists(fname): + is_cold = True + container_id = str(uuid.uuid4())[0:8] + with open(fname, "a") as f: + f.write(container_id) + else: + with open(fname, "r") as f: + container_id = f.read() + + return is_cold, container_id + + +def handler(event, context): + start = datetime.datetime.now().timestamp() + os.environ["STORAGE_UPLOAD_BYTES"] = "0" + os.environ["STORAGE_DOWNLOAD_BYTES"] = "0" + + req_id = context.aws_request_id + + if isinstance(event, dict): + request_id = event.pop("__sebs_request_id", event.pop("__request_id", req_id)) + func_payload = event + else: + func_payload = event + request_id = req_id + + if isinstance(func_payload, dict): + func_payload["request-id"] = req_id + + workflow_name, func_name = context.function_name.split("___") + function = importlib.import_module(f"function.{func_name}") + res = function.handler(func_payload) + + end = datetime.datetime.now().timestamp() + + is_cold, container_id = probe_cold_start() + measurement = { + "func": func_name, + "start": start, + "end": end, + "is_cold": is_cold, + "container_id": container_id, + "provider.request_id": context.aws_request_id, + } + + func_res = os.getenv("SEBS_FUNCTION_RESULT") + if func_res: + measurement["result"] = json.loads(func_res) + + bytes_upload = os.getenv("STORAGE_UPLOAD_BYTES", 0) + if bytes_upload: + measurement["blob.upload"] = int(bytes_upload) + + bytes_download = os.getenv("STORAGE_DOWNLOAD_BYTES", 0) + if bytes_download: + measurement["blob.download"] = int(bytes_download) + + measurement_json = json.dumps(measurement) + + try: + redis = Redis( + host={{REDIS_HOST}}, + port=6379, + decode_responses=True, + socket_connect_timeout=10, + password={{REDIS_PASSWORD}}, + ) + + key = os.path.join(workflow_name, func_name, request_id, str(uuid.uuid4())[0:8]) + redis.set(key, measurement_json) + except Exception: + pass + + if isinstance(res, dict): + res["__request_id"] = request_id + return res diff --git a/benchmarks/wrappers/aws/python/storage.py b/benchmarks/wrappers/aws/python/storage.py index 401947df6..e2915a2f2 100644 --- a/benchmarks/wrappers/aws/python/storage.py +++ b/benchmarks/wrappers/aws/python/storage.py @@ -22,14 +22,26 @@ def unique_name(name): random=str(uuid.uuid4()).split('-')[0] ) - def upload(self, bucket, file, filepath): - key_name = storage.unique_name(file) + def upload(self, bucket, file, filepath, unique_name=True): + key_name = storage.unique_name(file) if unique_name else file self.client.upload_file(filepath, bucket, key_name) return key_name def download(self, bucket, file, filepath): self.client.download_file(bucket, file, filepath) + def download_within_range(self, bucket, file, start_bytes, end_bytes): + response = self.client.get_object( + Bucket=bucket, Key=file, Range=f"bytes={start_bytes}-{end_bytes}" + ) + return response["Body"].read().decode("utf-8") + + def list_directory(self, bucket, prefix): + objects = self.client.list_objects_v2(Bucket=bucket, Prefix=prefix) + if 'Contents' not in objects: + return [] + return [obj['Key'] for obj in objects['Contents']] + def download_directory(self, bucket, prefix, path): objects = self.client.list_objects_v2(Bucket=bucket, Prefix=prefix) # 'Contents' key is only present when objects are found diff --git a/benchmarks/wrappers/azure/python/fsm.py b/benchmarks/wrappers/azure/python/fsm.py new file mode 100644 index 000000000..5d4039ec0 --- /dev/null +++ b/benchmarks/wrappers/azure/python/fsm.py @@ -0,0 +1,225 @@ +from abc import ABC +from abc import abstractmethod +from typing import Optional, List, Callable, Union, Dict, Type, Tuple +import json + + +class State(ABC): + def __init__(self, name: str): + self.name = name + + @staticmethod + def deserialize(name: str, payload: dict) -> "State": + cls = _STATE_TYPES[payload["type"]] + return cls.deserialize(name, payload) + + +class Task(State): + def __init__(self, name: str, func_name: str, next: Optional[str], failure: Optional[str]): + self.name = name + self.func_name = func_name + self.next = next + self.failure = failure + + @classmethod + def deserialize(cls, name: str, payload: dict) -> "Task": + return cls( + name=name, + func_name=payload["func_name"], + next=payload.get("next"), + failure=payload.get("failure"), + ) + + +class Switch(State): + class Case: + def __init__(self, var: str, op: str, val: str, next: str): + self.var = var + self.op = op + self.val = val + self.next = next + + @staticmethod + def deserialize(payload: dict) -> "Switch.Case": + return Switch.Case(**payload) + + def __init__(self, name: str, cases: List[Case], default: Optional[str]): + self.name = name + self.cases = cases + self.default = default + + @classmethod + def deserialize(cls, name: str, payload: dict) -> "Switch": + cases = [Switch.Case.deserialize(c) for c in payload["cases"]] + + return cls(name=name, cases=cases, default=payload["default"]) + + +class Parallel(State): + def __init__(self, name: str, funcs: List, next: Optional[str]): + self.name = name + self.funcs = funcs + self.next = next + + @classmethod + def deserialize(cls, name: str, payload: dict) -> "Parallel": + return cls(name=name, funcs=payload.get("parallel_functions"), next=payload.get("next")) + + +class Map(State): + def __init__( + self, + name: str, + funcs: List, + array: str, + root: str, + next: Optional[str], + common_params: Optional[str], + ): + self.name = name + self.funcs = funcs + self.array = array + self.root = root + self.next = next + self.common_params = common_params + + @property + def func_name(self) -> str: + state = self.funcs[self.root] + while state["type"] == "map": + state = state["states"][state["root"]] + return state["func_name"] + + @classmethod + def deserialize(cls, name: str, payload: dict) -> "Map": + return cls( + name=name, + funcs=payload["states"], + array=payload["array"], + root=payload["root"], + next=payload.get("next"), + common_params=payload.get("common_params"), + ) + + +class Repeat(State): + def __init__(self, name: str, func_name: str, count: int, next: Optional[str]): + self.name = name + self.func_name = func_name + self.count = count + self.next = next + + @classmethod + def deserialize(cls, name: str, payload: dict) -> "Repeat": + return cls( + name=name, + func_name=payload["func_name"], + count=payload["count"], + next=payload.get("next"), + ) + + +class Loop(State): + def __init__(self, name: str, func_name: str, array: str, next: Optional[str]): + self.name = name + self.func_name = func_name + self.array = array + self.next = next + + @classmethod + def deserialize(cls, name: str, payload: dict) -> "Loop": + return cls( + name=name, + func_name=payload["func_name"], + array=payload["array"], + next=payload.get("next"), + ) + + +_STATE_TYPES: Dict[str, Type[State]] = { + "task": Task, + "switch": Switch, + "map": Map, + "repeat": Repeat, + "loop": Loop, + "parallel": Parallel, +} + + +class Generator(ABC): + def __init__(self, export_func: Callable[[dict], str] = json.dumps): + self._export_func = export_func + + def parse(self, path: str): + with open(path) as f: + definition = json.load(f) + + self.states = {n: State.deserialize(n, s) for n, s in definition["states"].items()} + self.root = self.states[definition["root"]] + + def generate(self) -> str: + states = list(self.states.values()) + payloads = [] + for s in states: + obj = self.encode_state(s) + if isinstance(obj, dict): + payloads.append(obj) + elif isinstance(obj, list): + payloads += obj + else: + raise ValueError("Unknown encoded state returned.") + + definition = self.postprocess(payloads) + + return self._export_func(definition) + + def postprocess(self, payloads: List[dict]) -> dict: + return payloads + + def encode_state(self, state: State) -> Union[dict, List[dict]]: + if isinstance(state, Task): + return self.encode_task(state) + elif isinstance(state, Switch): + return self.encode_switch(state) + elif isinstance(state, Map): + return self.encode_map(state) + elif isinstance(state, Repeat): + return self.encode_repeat(state) + elif isinstance(state, Loop): + return self.encode_loop(state) + elif isinstance(state, Parallel): + return self.encode_parallel(state) + else: + raise ValueError(f"Unknown state of type {type(state)}.") + + @abstractmethod + def encode_task(self, state: Task) -> Union[dict, List[dict]]: + pass + + @abstractmethod + def encode_switch(self, state: Switch) -> Union[dict, List[dict]]: + pass + + @abstractmethod + def encode_map(self, state: Map) -> Union[dict, List[dict]]: + pass + + @abstractmethod + def encode_parallel(self, state: Parallel) -> Union[dict, List[dict]]: + pass + + def encode_repeat(self, state: Repeat) -> Union[dict, List[dict]]: + tasks = [] + for i in range(state.count): + name = state.name if i == 0 else f"{state.name}_{i}" + next = state.next if i == state.count - 1 else f"{state.name}_{i+1}" + task = Task(name, state.func_name, next, None) + + res = self.encode_task(task) + tasks += res if isinstance(res, list) else [res] + + return tasks + + @abstractmethod + def encode_loop(self, state: Loop) -> Union[dict, List[dict]]: + pass diff --git a/benchmarks/wrappers/azure/python/handler_workflow.py b/benchmarks/wrappers/azure/python/handler_workflow.py new file mode 100644 index 000000000..45ca811d0 --- /dev/null +++ b/benchmarks/wrappers/azure/python/handler_workflow.py @@ -0,0 +1,129 @@ +import datetime +import json +import os +import sys +import uuid +import importlib + +import logging + +import azure.functions as func +from redis import Redis + +SEBS_USER_AGENT = "SeBS/1.2 (https://github.com/spcl/serverless-benchmarks) SeBS Benchmark Suite/1.2" + + +def patch_requests_user_agent(): + try: + import requests + except ImportError: + return + + original_request = requests.api.request + if getattr(original_request, "_sebs_user_agent_patched", False): + return + + def patched_request(method, url, **kwargs): + headers = dict(kwargs.get("headers") or {}) + header_names = {key.lower() for key in headers} + if "user-agent" not in header_names: + headers["User-Agent"] = SEBS_USER_AGENT + kwargs["headers"] = headers + return original_request(method, url, **kwargs) + + patched_request._sebs_user_agent_patched = True + requests.api.request = patched_request + requests.request = patched_request + + +patch_requests_user_agent() + +if 'NOSQL_STORAGE_DATABASE' in os.environ: + from . import nosql + nosql.nosql.get_instance( + os.environ['NOSQL_STORAGE_DATABASE'], + os.environ['NOSQL_STORAGE_URL'], + os.environ['NOSQL_STORAGE_CREDS'] + ) + sys.modules["nosql"] = nosql + +if 'STORAGE_CONNECTION_STRING' in os.environ: + from . import storage + storage.storage.get_instance(os.environ['STORAGE_CONNECTION_STRING']) + sys.modules["storage"] = storage + +def probe_cold_start(): + is_cold = False + fname = os.path.join("/tmp", "cold_run") + if not os.path.exists(fname): + is_cold = True + container_id = str(uuid.uuid4())[0:8] + with open(fname, "a") as f: + f.write(container_id) + else: + with open(fname, "r") as f: + container_id = f.read() + + return is_cold, container_id + +def main(event, context: func.Context): + start = datetime.datetime.now().timestamp() + os.environ["STORAGE_UPLOAD_BYTES"] = "0" + os.environ["STORAGE_DOWNLOAD_BYTES"] = "0" + + workflow_name = os.getenv("APPSETTING_WEBSITE_SITE_NAME") + func_name = os.path.basename(os.path.dirname(__file__)) + + event["payload"]["request-id"] = context.invocation_id + + current_dir = os.path.dirname(__file__) + if current_dir not in sys.path: + sys.path.insert(0, current_dir) + parent_dir = os.path.dirname(current_dir) + if parent_dir not in sys.path: + sys.path.insert(0, parent_dir) + package = __package__ or func_name + function = importlib.import_module(f"{package}.{func_name}") + + res = function.handler(event["payload"]) + + end = datetime.datetime.now().timestamp() + + is_cold, container_id = probe_cold_start() + payload = { + "func": func_name, + "start": start, + "end": end, + "is_cold": is_cold, + "container_id": container_id, + "provider.request_id": context.invocation_id + } + + func_res = os.getenv("SEBS_FUNCTION_RESULT") + if func_res: + payload["result"] = json.loads(func_res) + + bytes_upload = os.getenv("STORAGE_UPLOAD_BYTES", 0) + if bytes_upload: + payload["blob.upload"] = int(bytes_upload) + + bytes_download = os.getenv("STORAGE_DOWNLOAD_BYTES", 0) + if bytes_download: + payload["blob.download"] = int(bytes_download) + + payload = json.dumps(payload) + + redis_host = {{REDIS_HOST}} + redis_password = {{REDIS_PASSWORD}} + if redis_host: + redis = Redis(host=redis_host, + port=6379, + decode_responses=True, + socket_connect_timeout=10, + password=redis_password) + + req_id = event["request_id"] + key = os.path.join(workflow_name, func_name, req_id, str(uuid.uuid4())[0:8]) + redis.set(key, payload) + + return res diff --git a/benchmarks/wrappers/azure/python/main_workflow.py b/benchmarks/wrappers/azure/python/main_workflow.py new file mode 100644 index 000000000..392f18847 --- /dev/null +++ b/benchmarks/wrappers/azure/python/main_workflow.py @@ -0,0 +1,61 @@ +import os +import json +import datetime +import uuid + +import azure.functions as func +import azure.durable_functions as df + + +def probe_cold_start(): + is_cold = False + fname = os.path.join("/tmp", "cold_run") + if not os.path.exists(fname): + is_cold = True + container_id = str(uuid.uuid4())[0:8] + with open(fname, "a") as f: + f.write(container_id) + else: + with open(fname, "r") as f: + container_id = f.read() + + return is_cold, container_id + + +async def main(req: func.HttpRequest, starter: str, context: func.Context) -> func.HttpResponse: + req_id = context.invocation_id + event = {"payload": req.get_json(), "request_id": req_id} + + begin = datetime.datetime.now() + + client = df.DurableOrchestrationClient(starter) + instance_id = await client.start_new("run_workflow", None, event) + + res = await client.wait_for_completion_or_create_check_status_response( + req, instance_id, timeout_in_milliseconds=1000 + ) + + end = datetime.datetime.now() + + is_cold, container_id = probe_cold_start() + + try: + result = json.loads(res.get_body()) + except json.decoder.JSONDecodeError: + result = res.get_body().decode() + + body = { + "result": result, + "begin": begin.strftime("%s.%f"), + "end": end.strftime("%s.%f"), + "is_cold": is_cold, + "container_id": container_id, + "provider.request_id": context.invocation_id, + "request_id": req_id, + } + + return func.HttpResponse( + status_code=res.status_code, + body=json.dumps(body), + mimetype="application/json" + ) diff --git a/benchmarks/wrappers/azure/python/run_subworkflow.py b/benchmarks/wrappers/azure/python/run_subworkflow.py new file mode 100644 index 000000000..4fd28c577 --- /dev/null +++ b/benchmarks/wrappers/azure/python/run_subworkflow.py @@ -0,0 +1,236 @@ +import json +import sys +import os +import uuid +import operator +import logging +import datetime +import copy + +import azure.durable_functions as df +from redis import Redis + +dir_path = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(os.path.join(dir_path, os.path.pardir)) + +from .fsm import * + + +def get_var(obj, path: str): + names = path.split(".") + assert(len(names) > 0) + + for n in names: + obj = obj[n] + + return obj + + +def set_var(obj, val, path: str): + names = path.split(".") + assert(len(names) > 0) + + for n in names[:-1]: + obj = obj[n] + obj[names[-1]] = val + +def handler(context: df.DurableOrchestrationContext): + start = datetime.datetime.now().timestamp() + ts = start + now = lambda: datetime.datetime.now().timestamp() + duration = 0 + + input = context.get_input() + res = input["payload"] + request_id = input["request_id"] + all_states = input["states"] + states = {n: State.deserialize(n, s) + for n, s in all_states.items()} + current = states[input["root"]] + + while current: + logging.info(current.name) + + if isinstance(current, Task): + input = {"payload": res, "request_id": request_id} + + duration += (now() - ts) + res = yield context.call_activity(current.func_name, input) + ts = now() + current = states.get(current.next, None) + elif isinstance(current, Switch): + ops = { + "<": operator.lt, + "<=": operator.le, + "==": operator.eq, + ">=": operator.ge, + ">": operator.gt + } + + next = None + for case in current.cases: + var = get_var(res, case.var) + op = ops[case.op] + if op(var, case.val): + next = states[case.next] + break + + if not next and current.default: + next = states[current.default] + current = next + elif isinstance(current, Map): + array = get_var(res, current.array) + tasks = [] + if current.common_params: + #assemble input differently + for elem in array: + #assemble payload + payload = {} + payload["array_element"] = elem + params = current.common_params.split(",") + for param in params: + payload[param] = get_var(res, param) + myinput = {"payload": payload, "request_id": request_id} + tasks.append(context.call_activity(current.func_name, myinput)) + else: + for elem in array: + myinput = {"payload": elem, "request_id": request_id} + tasks.append(context.call_activity(current.func_name, myinput)) + + duration += (now() - ts) + map_res = yield context.task_all(tasks) + ts = now() + + set_var(res, map_res, current.array) + current = states.get(current.next, None) + elif isinstance(current, Repeat): + for i in range(current.count): + input = {"payload": res, "request_id": request_id} + + duration += (now() - ts) + res = yield context.call_activity(current.func_name, input) + ts = now() + + current = states.get(current.next, None) + elif isinstance(current, Loop): + array = get_var(res, current.array) + for elem in array: + input = {"payload": elem, "request_id": request_id} + + duration += (now() - ts) + yield context.call_activity(current.func_name, input) + ts = now() + + current = states.get(current.next, None) + + elif isinstance(current, Parallel): + parallel_tasks = [] + first_states = [] + state_to_result = {} + for i, subworkflow in enumerate(current.funcs): + parallel_states = {n: State.deserialize(n, s) for n, s in subworkflow["states"].items()} + + #for state in parallel_states.values(): + # state_to_result[state.func_name] = [] + + + first_state = parallel_states[subworkflow["root"]] + first_states.append(first_state) + state_to_result[first_state.func_name] = [] + + if isinstance(first_state, Task): + input = {"payload": res, "request_id": request_id} + + #task directly here if only one state, task within suborchestrator if multiple states. + if first_state.next: + #call suborchestrator + #FIXME define other parameters. + parallel_task = context.call_sub_orchestrator("run_subworkflow", input, subworkflow["root"], parallel_states) + parallel_tasks.append(parallel_task) + else: + parallel_tasks.append(context.call_activity(first_state.func_name, input)) + state_to_result[first_state.func_name].append(len(parallel_tasks)-1) + + elif isinstance(first_state, Map): + array = get_var(res, first_state.array) + + if first_state.next: + #call suborchestrator. + if first_state.common_params: + #assemble input differently + for elem in array: + payload = {} + payload["array_element"] = elem + params = first_state.common_params.split(",") + for param in params: + payload[param] = get_var(res, param) + myinput = {"payload": payload, "request_id": request_id} + #FIXME use right parameters for suborchestrator. + parallel_task = context.call_sub_orchestrator("run_subworkflow", myinput, subworkflow["root"], parallel_states) + parallel_tasks.append(parallel_task) + state_to_result[first_state.func_name].append(len(parallel_tasks)-1) + else: + for elem in array: + myinput = {"payload": elem, "request_id": request_id} + + parallel_task = context.call_sub_orchestrator("run_subworkflow", myinput, subworkflow["root"], parallel_states) + parallel_tasks.append(parallel_task) + state_to_result[first_state.func_name].append(len(parallel_tasks)-1) + else: + if first_state.common_params: + #assemble input differently + for elem in array: + payload = {} + payload["array_element"] = elem + params = first_state.common_params.split(",") + for param in params: + payload[param] = get_var(res, param) + myinput = {"payload": payload, "request_id": request_id} + parallel_tasks.append(context.call_activity(first_state.func_name, myinput)) + state_to_result[first_state.func_name].append(len(parallel_tasks)-1) + else: + for elem in array: + myinput = {"payload": elem, "request_id": request_id} + parallel_tasks.append(context.call_activity(first_state.func_name, myinput)) + state_to_result[first_state.func_name].append(len(parallel_tasks)-1) + + duration += (now() - ts) + map_res = yield context.task_all(parallel_tasks) + ts = now() + base_res = res + res = {} + + for state in first_states: + indices = state_to_result[state.func_name] + if len(indices) > 1: + output = [] + for index in indices: + output.append(map_res[index]) + if isinstance(state, Map): + branch_res = copy.deepcopy(base_res) + set_var(branch_res, output, state.array) + res[state.func_name] = branch_res + else: + res[state.func_name] = output + else: + #task state + output = map_res[indices[0]] + if isinstance(state, Map): + branch_res = copy.deepcopy(base_res) + set_var(branch_res, output, state.array) + res[state.func_name] = branch_res + else: + res[state.func_name] = output + + current = states.get(current.next, None) + + else: + raise ValueError(f"Undefined state: {current}") + + #workflow_name = os.getenv("APPSETTING_WEBSITE_SITE_NAME") + func_name = "run_subworkflow" + + return res + + +main = df.Orchestrator.create(handler) diff --git a/benchmarks/wrappers/azure/python/run_workflow.py b/benchmarks/wrappers/azure/python/run_workflow.py new file mode 100644 index 000000000..eb4b95464 --- /dev/null +++ b/benchmarks/wrappers/azure/python/run_workflow.py @@ -0,0 +1,299 @@ +import json +import sys +import os +import uuid +import operator +import logging +import datetime +import copy + +import azure.durable_functions as df +from redis import Redis + +dir_path = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(os.path.join(dir_path, os.path.pardir)) + +from .fsm import * + + +def get_var(obj, path: str): + names = path.split(".") + assert(len(names) > 0) + + for n in names: + obj = obj[n] + + return obj + + +def set_var(obj, val, path: str): + names = path.split(".") + assert(len(names) > 0) + + for n in names[:-1]: + obj = obj[n] + obj[names[-1]] = val + +def handler(context: df.DurableOrchestrationContext): + start = datetime.datetime.now().timestamp() + ts = start + now = lambda: datetime.datetime.now().timestamp() + duration = 0 + + with open("definition.json") as f: + definition = json.load(f) + + states = {n: State.deserialize(n, s) + for n, s in definition["states"].items()} + current = states[definition["root"]] + input = context.get_input() + + logging.info("START") + res = input["payload"] + request_id = input["request_id"] + + while current: + logging.info(current.name) + + if isinstance(current, Task): + input = {"payload": res, "request_id": request_id} + + duration += (now() - ts) + + if current.failure is None: + res = yield context.call_activity(current.func_name, input) + current = states.get(current.next, None) + else: + try: + res = yield context.call_activity(current.func_name, input) + current = states.get(current.next, None) + except: + current = states.get(current.failure, None) + + ts = now() + + elif isinstance(current, Switch): + ops = { + "<": operator.lt, + "<=": operator.le, + "==": operator.eq, + ">=": operator.ge, + ">": operator.gt + } + + next = None + for case in current.cases: + var = get_var(res, case.var) + op = ops[case.op] + if op(var, case.val): + next = states[case.next] + break + + if not next and current.default: + next = states[current.default] + current = next + elif isinstance(current, Map): + + map_states = {n: State.deserialize(n, s) for n, s in current.funcs.items()} + first_state = map_states[current.root] + + array = get_var(res, current.array) + tasks = [] + if first_state.next: + #call suborchestrator - each map task should proceed with next step directly after it finished. + if current.common_params: + for elem in array: + payload = {} + payload["array_element"] = elem + params = current.common_params.split(",") + for param in params: + payload[param] = get_var(res, param) + myinput = {"payload": payload, "request_id": request_id} + myinput["root"] = current.root + + myinput["states"] = current.funcs + uuid_name = str(uuid.uuid4())[0:4] + parallel_task = context.call_sub_orchestrator("run_subworkflow", myinput, uuid_name) + tasks.append(parallel_task) + else: + for elem in array: + myinput = {"payload": elem, "request_id": request_id} + myinput["root"] = current.root + myinput["states"] = current.funcs + + uuid_name = str(uuid.uuid4())[0:4] + parallel_task = context.call_sub_orchestrator("run_subworkflow", myinput, uuid_name) + tasks.append(parallel_task) + else: + if current.common_params: + #assemble input differently + for elem in array: + payload = {} + payload["array_element"] = elem + params = current.common_params.split(",") + for param in params: + payload[param] = get_var(res, param) + myinput = {"payload": payload, "request_id": request_id} + tasks.append(context.call_activity(first_state.func_name, myinput)) + else: + for elem in array: + myinput = {"payload": elem, "request_id": request_id} + tasks.append(context.call_activity(first_state.func_name, myinput)) + + duration += (now() - ts) + map_res = yield context.task_all(tasks) + ts = now() + + set_var(res, map_res, current.array) + current = states.get(current.next, None) + elif isinstance(current, Repeat): + for i in range(current.count): + input = {"payload": res, "request_id": request_id} + + duration += (now() - ts) + res = yield context.call_activity(current.func_name, input) + ts = now() + + current = states.get(current.next, None) + elif isinstance(current, Loop): + array = get_var(res, current.array) + for elem in array: + input = {"payload": elem, "request_id": request_id} + + duration += (now() - ts) + yield context.call_activity(current.func_name, input) + ts = now() + + current = states.get(current.next, None) + + elif isinstance(current, Parallel): + parallel_tasks = [] + first_states = [] + state_to_result = {} + for subworkflow in current.funcs: + parallel_states = {n: State.deserialize(n, s) for n, s in subworkflow["states"].items()} + + first_state = parallel_states[subworkflow["root"]] + first_states.append(first_state) + state_to_result[first_state.func_name] = [] + + if isinstance(first_state, Task): + input = {"payload": res, "request_id": request_id} + + #task directly here if only one state, task within suborchestrator if multiple states. + if first_state.next: + input["root"] = subworkflow["root"] + input["states"] = subworkflow["states"] #parallel_states + parallel_task = context.call_sub_orchestrator("run_subworkflow", input) + parallel_tasks.append(parallel_task) + else: + parallel_tasks.append(context.call_activity(first_state.func_name, input)) + state_to_result[first_state.func_name].append(len(parallel_tasks)-1) + + elif isinstance(first_state, Map): + array = get_var(res, first_state.array) + + if first_state.next: + #call suborchestrator. + if first_state.common_params: + #assemble input differently + for elem in array: + payload = {} + payload["array_element"] = elem + params = first_state.common_params.split(",") + for param in params: + payload[param] = get_var(res, param) + myinput = {"payload": payload, "request_id": request_id} + myinput["root"] = subworkflow["root"] + myinput["states"] = subworkflow["states"] + parallel_task = context.call_sub_orchestrator("run_subworkflow", myinput) + parallel_tasks.append(parallel_task) + state_to_result[first_state.func_name].append(len(parallel_tasks)-1) + else: + for elem in array: + myinput = {"payload": elem, "request_id": request_id} + + myinput["root"] = subworkflow["root"] + myinput["states"] = subworkflow["states"] + parallel_task = context.call_sub_orchestrator("run_subworkflow", myinput) + parallel_tasks.append(parallel_task) + state_to_result[first_state.func_name].append(len(parallel_tasks)-1) + else: + if first_state.common_params: + #assemble input differently + for elem in array: + payload = {} + payload["array_element"] = elem + params = first_state.common_params.split(",") + for param in params: + payload[param] = get_var(res, param) + myinput = {"payload": payload, "request_id": request_id} + parallel_tasks.append(context.call_activity(first_state.func_name, myinput)) + state_to_result[first_state.func_name].append(len(parallel_tasks)-1) + else: + for elem in array: + myinput = {"payload": elem, "request_id": request_id} + parallel_tasks.append(context.call_activity(first_state.func_name, myinput)) + state_to_result[first_state.func_name].append(len(parallel_tasks)-1) + + duration += (now() - ts) + map_res = yield context.task_all(parallel_tasks) + ts = now() + base_res = res + res = {} + + for state in first_states: + #get respective results of map_res related to func according to state_to_result + indices = state_to_result[state.func_name] + if len(indices) > 1: + output = [] + for index in indices: + output.append(map_res[index]) + if isinstance(state, Map): + branch_res = copy.deepcopy(base_res) + set_var(branch_res, output, state.array) + res[state.func_name] = branch_res + else: + res[state.func_name] = output + else: + #task state + output = map_res[indices[0]] + if isinstance(state, Map): + branch_res = copy.deepcopy(base_res) + set_var(branch_res, output, state.array) + res[state.func_name] = branch_res + else: + res[state.func_name] = output + + current = states.get(current.next, None) + + else: + raise ValueError(f"Undefined state: {current}") + + workflow_name = os.getenv("APPSETTING_WEBSITE_SITE_NAME") + func_name = "run_workflow" + + payload = { + "func": func_name, + "start": start, + "end": start+duration + } + + payload = json.dumps(payload) + + redis_host = {{REDIS_HOST}} + redis_password = {{REDIS_PASSWORD}} + if redis_host: + redis = Redis(host=redis_host, + port=6379, + decode_responses=True, + socket_connect_timeout=10, + password=redis_password) + + key = os.path.join(workflow_name, func_name, request_id, str(uuid.uuid4())[0:8]) + redis.set(key, payload) + + return res + + +main = df.Orchestrator.create(handler) diff --git a/benchmarks/wrappers/azure/python/storage.py b/benchmarks/wrappers/azure/python/storage.py index 49aeb5aa2..60b98fe37 100644 --- a/benchmarks/wrappers/azure/python/storage.py +++ b/benchmarks/wrappers/azure/python/storage.py @@ -22,14 +22,19 @@ def unique_name(name): random=str(uuid.uuid4()).split('-')[0] ) - def upload(self, container, file, filepath): + def upload(self, container, file, filepath, unique_name=True): with open(filepath, 'rb') as data: - return self.upload_stream(container, file, data) + return self.upload_stream(container, file, data, unique_name=unique_name) def download(self, container, file, filepath): with open(filepath, 'wb') as download_file: download_file.write( self.download_stream(container, file) ) - + + def list_directory(self, container, prefix): + client = self.client.get_container_client(container=container) + objects = client.list_blobs(name_starts_with=prefix) + return [obj.name for obj in objects] + def download_directory(self, container, prefix, path): client = self.client.get_container_client(container=container) objects = client.list_blobs(name_starts_with=prefix) @@ -38,19 +43,25 @@ def download_directory(self, container, prefix, path): path_to_file = os.path.dirname(file_name) os.makedirs(os.path.join(path, path_to_file), exist_ok=True) self.download(container, file_name, os.path.join(path, file_name)) - - def upload_stream(self, container, file, data): - key_name = storage.unique_name(file) - client = self.client.get_blob_client( - container=container, - blob=key_name - ) - client.upload_blob(data) + + def upload_stream(self, container, file, data, unique_name=True): + key_name = storage.unique_name(file) if unique_name else file + client = self.client.get_blob_client(container=container, blob=key_name) + if hasattr(data, "seek"): + data.seek(0) + client.upload_blob(data, overwrite=not unique_name) return key_name def download_stream(self, container, file): client = self.client.get_blob_client(container=container, blob=file) return client.download_blob().readall() + + def download_within_range(self, container, file, start_bytes, end_bytes): + client = self.client.get_blob_client(container=container, blob=file) + return client.download_blob( + offset=start_bytes, + length=end_bytes - start_bytes + 1, + ).readall().decode("utf-8") @staticmethod def get_instance(connection_string: Optional[str] = None): diff --git a/benchmarks/wrappers/cloudflare/nodejs/build.js b/benchmarks/wrappers/cloudflare/nodejs/build.js new file mode 100644 index 000000000..886955a94 --- /dev/null +++ b/benchmarks/wrappers/cloudflare/nodejs/build.js @@ -0,0 +1,190 @@ +/** + * build.js — Convert a Node.js benchmark into a Cloudflare Workers-compatible bundle. + * + * Why this exists: + * Cloudflare Workers do not ship a Node.js runtime or a filesystem at deploy + * time: there is no `node_modules` directory, no `require()` resolution, and + * only a curated subset of Node built-ins is available (and only when opted + * in via the `node:` prefix and the `nodejs_compat` compatibility flag). + * Our SeBS benchmarks, however, are authored as regular Node.js code. This + * script bridges that gap by bundling the benchmark + its dependencies into + * a single ESM module that the Workers runtime can load. + * + * High-level pipeline: + * 1. Discover source files under the wrapper directory (skipping tests, + * node_modules, dotfiles, and the previous build output). + * 2. Run esbuild on every JS/TS entry point with a Workers-friendly config + * (ESM output, neutral platform, ES2020 target, tree-shaking). + * 3. Apply the `nodeBuiltinsPlugin` to rewrite imports so that: + * - Node built-ins always use the `node:` prefix required by Workers. + * - `cloudflare:*` imports stay external (resolved by the runtime). + * - The legacy `request` npm module is swapped for a fetch-based + * polyfill, since it cannot run under Workers. + * 4. Copy any non-code assets (templates, SQL, etc.) into `dist/` unchanged. + */ + +const { build } = require('esbuild'); +const fs = require('fs'); +const { join, extname, dirname, relative } = require('path'); + +// Recursively collect every file that should be part of the Workers bundle. +// Excludes test directories, node_modules, build artifacts, and this script +// itself so that only benchmark sources and the wrapper code get processed. +function getAllFiles(dir, fileList = []) { + const files = fs.readdirSync(dir, { withFileTypes: true }); + for (const file of files) { + const filePath = join(dir, file.name); + if (file.isDirectory()) { + if (file.name !== 'node_modules' && + file.name !== 'test' && + file.name !== 'tests' && + file.name !== '__tests__' && + file.name !== 'dist' && + !file.name.startsWith('.')) { + getAllFiles(filePath, fileList); + } + } else { + if (!file.name.includes('.test.') && + !file.name.includes('.spec.') && + file.name !== 'build.js' && + file.name !== 'wrangler.toml') { + fileList.push(filePath); + } + } + } + return fileList; +} + +function copyFile(src, dest) { + const destDir = dirname(dest); + if (!fs.existsSync(destDir)) { + fs.mkdirSync(destDir, { recursive: true }); + } + fs.copyFileSync(src, dest); +} + +// esbuild plugin that rewrites module imports so the output works on the +// Cloudflare Workers runtime. Workers only accept Node built-ins via the +// `node:` prefix (with the `nodejs_compat` flag enabled on the Worker), do +// not support arbitrary npm packages that rely on Node's networking stack, +// and resolve their own `cloudflare:*` imports at runtime. +const nodeBuiltinsPlugin = { + name: 'node-builtins-external', + setup(build) { + const { resolve } = require('path'); + + // Imports already using the `node:` or `cloudflare:` prefix are provided + // by the Workers runtime itself — leave them external so esbuild does not + // try to bundle them (which would fail, since they are not on disk). + build.onResolve({ filter: /^(node:|cloudflare:)/ }, (args) => { + return { path: args.path, external: true }; + }); + + // Benchmarks commonly `require('fs')`, `require('path')`, etc. Workers + // reject those bare specifiers; rewrite them to the `node:`-prefixed + // form and mark them external so the runtime resolves them. + build.onResolve({ filter: /^(fs|querystring|path|crypto|stream|buffer|util|events|http|https|net|tls|zlib|os|child_process|tty|assert|url|constants)$/ }, (args) => { + return { path: 'node:' + args.path, external: true }; + }); + + // The `request` npm module depends on Node's http/https clients and is + // incompatible with Workers. Redirect every `require('request')` to our + // fetch-based shim so benchmark code can keep the same call sites. + build.onResolve({ filter: /^request$/ }, (args) => { + const wrapperDir = __dirname; + return { + path: resolve(wrapperDir, 'request-polyfill.js') + }; + }); + + // `graceful-fs` monkey-patches the `fs` module at runtime, which Workers + // rejects ("object is not extensible"). Redirect it straight to node:fs + // so the patching never runs and consumers get the same API. + build.onResolve({ filter: /^graceful-fs$/ }, () => { + return { path: 'node:fs', external: true }; + }); + } +}; + + +async function customBuild() { + const srcDir = './'; + const outDir = './dist'; + + // Start from a clean output directory so stale artifacts from a previous + // build cannot leak into the Worker upload. + if (fs.existsSync(outDir)) { + fs.rmSync(outDir, { recursive: true }); + } + fs.mkdirSync(outDir, { recursive: true }); + + try { + const files = getAllFiles(srcDir); + + // Split discovered files: code goes through esbuild, everything else + // (JSON fixtures, templates, SQL, binary assets, ...) is copied verbatim. + const jsFiles = files.filter(f => + ['.js', '.ts', '.jsx', '.tsx'].includes(extname(f)) + ); + + const otherFiles = files.filter(f => + !['.js', '.ts', '.jsx', '.tsx'].includes(extname(f)) + ); + + console.log('Building JS files:', jsFiles); + + if (jsFiles.length > 0) { + // esbuild options chosen for Workers compatibility: + // - format: 'esm' Workers modules must be ES modules. + // - platform: 'neutral' Avoid Node- or browser-specific resolution; + // the plugin above handles Node built-ins + // explicitly. + // - target: 'es2020' Matches the V8 version used by Workers. + // - bundle + treeShaking Flattens dependencies into one module and + // drops dead code to stay under Workers' + // script size limit. + // - define.__dirname Node's `__dirname` does not exist in + // Workers; stub it with a harmless constant + // so benchmark code that references it still + // compiles. + // - define.global Workers expose `globalThis` rather than + // `global`; alias the two for compatibility. + await build({ + entryPoints: jsFiles, + bundle: true, + format: 'esm', + outdir: outDir, + outbase: srcDir, + platform: 'neutral', + target: 'es2020', + sourcemap: true, + allowOverwrite: true, + plugins: [nodeBuiltinsPlugin], + define: { + 'process.env.NODE_ENV': '"production"', + 'global': 'globalThis', + '__dirname': '"/bundle"' + }, + mainFields: ['module', 'main'], + treeShaking: true, + }); + } + + // Non-code assets (e.g. HTML/CSS templates, JSON payloads) need to ship + // alongside the bundle at their original relative paths so the worker + // can read them via the runtime's asset APIs. + for (const file of otherFiles) { + const relativePath = relative(srcDir, file); + const destPath = join(outDir, relativePath); + copyFile(file, destPath); + console.log(`Copied: ${relativePath}`); + } + + console.log('✓ Build completed successfully'); + } catch (error) { + console.error('Build failed:', error); + process.exit(1); + } +} + +customBuild(); \ No newline at end of file diff --git a/benchmarks/wrappers/cloudflare/nodejs/container/handler.js b/benchmarks/wrappers/cloudflare/nodejs/container/handler.js new file mode 100644 index 000000000..dd4df7318 --- /dev/null +++ b/benchmarks/wrappers/cloudflare/nodejs/container/handler.js @@ -0,0 +1,189 @@ +// Container handler for Cloudflare Workers - Node.js +// This handler is used when deploying as a container worker + +const http = require('http'); +const crypto = require('crypto'); +const Module = require('module'); +const debug = require('util').debuglog('sebs'); + +// Monkey-patch the 'request' library to always include a User-Agent header +// This is needed because Wikimedia (and other sites) require a User-Agent +try { + const originalRequire = Module.prototype.require; + + Module.prototype.require = function(id) { + const module = originalRequire.apply(this, arguments); + + if (id === 'request') { + // Wrap the request function to inject default headers + const originalRequest = module; + const wrappedRequest = function(options, callback) { + if (typeof options === 'string') { + options = { uri: options }; + } + if (!options.headers) { + options.headers = {}; + } + if (!options.headers['User-Agent'] && !options.headers['user-agent']) { + options.headers['User-Agent'] = 'SeBS/1.2 (https://github.com/spcl/serverless-benchmarks) SeBS Benchmark Suite/1.2'; + } + return originalRequest(options, callback); + }; + // Copy all properties from original request + Object.keys(originalRequest).forEach(key => { + wrappedRequest[key] = originalRequest[key]; + }); + return wrappedRequest; + } + + return module; + }; +} catch (e) { + console.error('Failed to patch request module:', e); +} + +// Import the benchmark function +const { handler: benchmarkHandler } = require('./function'); + +// Import storage and nosql if they exist +let storage, nosql; +try { + storage = require('./storage'); +} catch (e) { + console.log('Storage module not available'); +} +try { + nosql = require('./nosql'); +} catch (e) { + console.log('NoSQL module not available'); +} + +const PORT = process.env.PORT || 8080; + +const server = http.createServer(async (req, res) => { + try { + // Get unique request ID from Cloudflare (CF-Ray header) + const reqId = req.headers['cf-ray'] || crypto.randomUUID(); + + // Extract Worker URL from header for R2 and NoSQL proxy. + // + // Containers run in a separate runtime from Workers and cannot access R2 or + // KV bindings directly — those bindings only exist in the Worker's `env`. + // To let the benchmark code reach storage, worker.js injects its own public + // origin into the X-Worker-URL header before forwarding the request here. + // The container-side storage/nosql modules use this URL to call back into + // the Worker over HTTP (e.g. POST ${workerUrl}/r2/upload), and worker.js + // intercepts those paths (/r2/*, /nosql/*) and performs the binding call + // on the container's behalf. + const workerUrl = req.headers['x-worker-url']; + if (workerUrl) { + if (storage && storage.storage && storage.storage.set_worker_url) { + storage.storage.set_worker_url(workerUrl); + } + if (nosql && nosql.nosql && nosql.nosql.set_worker_url) { + nosql.nosql.set_worker_url(workerUrl); + } + console.log(`Set worker URL for R2/NoSQL proxy: ${workerUrl}`); + } + + // Start timing measurements + const begin = Date.now() / 1000; + const start = performance.now(); + + // Read request body + let body = ''; + for await (const chunk of req) { + body += chunk; + } + + // Parse event from JSON body or URL params + let event = {}; + if (body && body.length > 0) { + try { + event = JSON.parse(body); + } catch (e) { + console.error('Failed to parse JSON body:', e); + res.writeHead(400, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'Invalid JSON body', message: e.message })); + return; + } + } + + // Parse URL parameters + const url = new URL(req.url, `http://${req.headers.host}`); + for (const [key, value] of url.searchParams) { + if (!event[key]) { + const intValue = parseInt(value); + event[key] = isNaN(intValue) ? value : intValue; + } + } + + // Add request metadata + const incomeTimestamp = Math.floor(Date.now() / 1000); + event['request-id'] = reqId; + event['income-timestamp'] = incomeTimestamp; + + // Call the benchmark function + const ret = await benchmarkHandler(event); + + + // Calculate elapsed time + const end = Date.now() / 1000; + const elapsed = performance.now() - start; + const micro = elapsed * 1000; // Convert milliseconds to microseconds + + // Build log_data similar to native handler + const log_data = { result: ret && ret.result !== undefined ? ret.result : ret }; + if (ret && ret.measurement !== undefined) { + log_data.measurement = ret.measurement; + } else { + log_data.measurement = {}; + } + + // Add memory usage to measurement + const memUsage = process.memoryUsage(); + const memory_mb = memUsage.heapUsed / 1024 / 1024; + log_data.measurement.memory_used_mb = memory_mb; + + // Gated behind Node.js' built-in debuglog — enable with NODE_DEBUG=sebs + debug('Sending response with log_data: %o', log_data); + + // Send response matching Python handler format exactly + if (event.html) { + res.writeHead(200, { 'Content-Type': 'text/html; charset=utf-8' }); + res.end(String(ret && ret.result !== undefined ? ret.result : ret)); + } else { + const responseBody = JSON.stringify({ + begin: begin, + end: end, + results_time: 0, + result: log_data, + is_cold: false, + is_cold_worker: false, + container_id: '0', + environ_container_id: 'no_id', + request_id: reqId, + }); + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(responseBody); + } + + } catch (error) { + console.error('Error processing request:', error); + console.error('Stack trace:', error.stack); + + const errorPayload = JSON.stringify({ + error: error.message, + stack: error.stack + }); + + res.writeHead(500, { 'Content-Type': 'application/json' }); + res.end(errorPayload); + } +}); + +// Ensure server is listening before handling requests +server.listen(PORT, '0.0.0.0', () => { + console.log(`Container server listening on 0.0.0.0:${PORT}`); + console.log('Server ready to accept connections'); +}); diff --git a/benchmarks/wrappers/cloudflare/nodejs/container/nosql.js b/benchmarks/wrappers/cloudflare/nodejs/container/nosql.js new file mode 100644 index 000000000..f529e682b --- /dev/null +++ b/benchmarks/wrappers/cloudflare/nodejs/container/nosql.js @@ -0,0 +1,121 @@ +/** + * NoSQL module for Cloudflare Node.js Containers. + * + * On Cloudflare, NoSQL storage is mapped to KVStore. KVStore + * bindings only exist inside the Worker runtime, so a container cannot talk + * to them directly. Instead, the container forwards each operation over HTTP + * to the parent Worker (see worker.js), which holds the KVStore + * binding and performs the actual read/write. + * + * Because of this, the HTTP endpoint depends on the Worker's URL, which is + * not known ahead of time. The handler receives it via the X-Worker-URL + * header on the incoming request and installs it here through + * set_worker_url() before any NoSQL call is made. + */ + +class nosql { + constructor() {} + + static worker_url = null; // Set by handler from X-Worker-URL header + + static init_instance(entry) { + if (!nosql.instance) { + nosql.instance = new nosql(); + } + return nosql.instance; + } + + static set_worker_url(url) { + nosql.worker_url = url; + } + + async _make_request(operation, params) { + if (!nosql.worker_url) { + throw new Error('Worker URL not set - cannot access NoSQL'); + } + + const url = `${nosql.worker_url}/nosql/${operation}`; + const data = JSON.stringify(params); + + try { + const response = await fetch(url, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: data, + }); + + if (!response.ok) { + let errorMsg; + try { + const errorData = await response.json(); + errorMsg = errorData.error || await response.text(); + } catch { + errorMsg = await response.text(); + } + throw new Error(`NoSQL operation failed: ${errorMsg}`); + } + + return await response.json(); + } catch (error) { + throw new Error(`NoSQL operation failed: ${error.message}`); + } + } + + async insert(tableName, primaryKey, secondaryKey, data) { + const params = { + table_name: tableName, + primary_key: primaryKey, + secondary_key: secondaryKey, + data: data, + }; + return this._make_request('insert', params); + } + + async get(tableName, primaryKey, secondaryKey) { + const params = { + table_name: tableName, + primary_key: primaryKey, + secondary_key: secondaryKey, + }; + const result = await this._make_request('get', params); + return result.data || null; + } + + async update(tableName, primaryKey, secondaryKey, updates) { + const params = { + table_name: tableName, + primary_key: primaryKey, + secondary_key: secondaryKey, + data: updates, + }; + return this._make_request('update', params); + } + + async query(tableName, primaryKey, secondaryKeyName) { + const params = { + table_name: tableName, + primary_key: primaryKey, + secondary_key_name: secondaryKeyName, + }; + const result = await this._make_request('query', params); + return result.items || []; + } + + async delete(tableName, primaryKey, secondaryKey) { + const params = { + table_name: tableName, + primary_key: primaryKey, + secondary_key: secondaryKey, + }; + return this._make_request('delete', params); + } + + static get_instance() { + if (!nosql.instance) { + nosql.instance = new nosql(); + } + return nosql.instance; + } +} + +module.exports.nosql = nosql; diff --git a/benchmarks/wrappers/cloudflare/nodejs/container/package.json b/benchmarks/wrappers/cloudflare/nodejs/container/package.json new file mode 100644 index 000000000..729c56fdc --- /dev/null +++ b/benchmarks/wrappers/cloudflare/nodejs/container/package.json @@ -0,0 +1,10 @@ +{ + "name": "cloudflare-container-worker", + "version": "1.0.0", + "description": "Cloudflare Container Worker wrapper", + "main": "worker.js", + "type": "module", + "dependencies": { + "@cloudflare/containers": "^1.0.0" + } +} diff --git a/benchmarks/wrappers/cloudflare/nodejs/container/storage.js b/benchmarks/wrappers/cloudflare/nodejs/container/storage.js new file mode 100644 index 000000000..f69aa5748 --- /dev/null +++ b/benchmarks/wrappers/cloudflare/nodejs/container/storage.js @@ -0,0 +1,357 @@ +const fs = require('fs'); +const path = require('path'); +const uuid = require('uuid'); +const debug = require('util').debuglog('sebs'); + +// Cloudflare Workers enforce a 100 MB request body limit at the edge. +// Use multipart upload for payloads larger than this threshold so that +// each individual request stays well below that limit. R2 requires parts +// of at least 5 MB. +const MULTIPART_THRESHOLD = 10 * 1024 * 1024; +const PART_SIZE = 10 * 1024 * 1024; + +function isRetryableSingleUploadError(error) { + const message = error?.message || ''; + return /HTTP 4(?:08|13|29)|request body|payload|too large|content length|body size|stream/i.test(message); +} + +/** + * Storage module for Cloudflare Node.js Containers. + * + * On Cloudflare, object storage (R2) is normally accessed through a Worker + * binding (`env.R2_BUCKET`). That binding only exists inside the Worker + * runtime, so a container cannot talk to R2 directly the way a Lambda or + * Cloud Function talks to S3/GCS with a regular SDK. Instead, the container + * forwards each storage operation over HTTP to the parent Worker (see + * worker.js), which holds the R2 binding and performs the actual + * get/put/list/multipart calls. + * + * R2 does expose an S3-compatible HTTPS API that a container could call + * without a Worker proxy, but that path requires provisioning and injecting + * R2 access keys into the container and diverges from how the Worker-based + * benchmarks access R2. Routing through the Worker keeps a single code path + * and credential model for both deployment types. + * + * Because of this, the HTTP endpoint depends on the Worker's URL, which is + * not known ahead of time. The handler receives it via the X-Worker-URL + * header on the incoming request and installs it here through + * set_worker_url() before any storage call is made. + */ + +class storage { + constructor() { + this.r2_enabled = true; + } + + static worker_url = null; // Set by handler from X-Worker-URL header + + static unique_name(name) { + const parsed = path.parse(name); + const uuid_name = uuid.v4().split('-')[0]; + return path.join(parsed.dir, `${parsed.name}.${uuid_name}${parsed.ext}`); + } + + static init_instance(entry) { + if (!storage.instance) { + storage.instance = new storage(); + } + return storage.instance; + } + + static set_worker_url(url) { + storage.worker_url = url; + } + + static get_instance() { + if (!storage.instance) { + storage.init_instance(); + } + return storage.instance; + } + + _toBuffer(data) { + if (Buffer.isBuffer(data)) { + return data; + } + if (typeof data === 'string') { + return Buffer.from(data, 'utf-8'); + } + if (data instanceof ArrayBuffer) { + return Buffer.from(data); + } + return Buffer.from(String(data), 'utf-8'); + } + + async _postJson(url, body = Buffer.alloc(0), contentType = null) { + const options = { + method: 'POST', + body, + }; + + if (contentType) { + options.headers = { 'Content-Type': contentType }; + } + + const response = await fetch(url, options); + + if (!response.ok) { + throw new Error(`HTTP ${response.status}: ${await response.text()}`); + } + + return response.json(); + } + + async _single_upload(key, buffer) { + const params = new URLSearchParams({ key }); + const url = `${storage.worker_url}/r2/upload?${params}`; + const result = await this._postJson(url, buffer); + return result.key; + } + + async _multipart_upload(key, buffer) { + const initParams = new URLSearchParams({ key }); + const initUrl = `${storage.worker_url}/r2/multipart-init?${initParams}`; + const init = await this._postJson(initUrl); + const uploadId = init.uploadId; + const uploadKey = init.key; + const completedParts = []; + + for (let offset = 0, partNumber = 1; offset < buffer.length; offset += PART_SIZE, partNumber += 1) { + const chunk = buffer.subarray(offset, offset + PART_SIZE); + const partParams = new URLSearchParams({ + key: uploadKey, + uploadId, + partNumber: String(partNumber), + }); + const partUrl = `${storage.worker_url}/r2/multipart-part?${partParams}`; + const part = await this._postJson(partUrl, chunk, 'application/octet-stream'); + completedParts.push({ partNumber: part.partNumber, etag: part.etag }); + } + + const completeParams = new URLSearchParams({ key: uploadKey, uploadId }); + const completeUrl = `${storage.worker_url}/r2/multipart-complete?${completeParams}`; + const result = await this._postJson( + completeUrl, + Buffer.from(JSON.stringify({ parts: completedParts }), 'utf-8'), + 'application/json' + ); + return result.key; + } + + async _upload_bytes(key, buffer) { + if (buffer.length > MULTIPART_THRESHOLD) { + return this._multipart_upload(key, buffer); + } + + try { + return await this._single_upload(key, buffer); + } catch (error) { + if (!isRetryableSingleUploadError(error)) { + throw error; + } + + debug( + '[storage] single upload failed for %s; retrying with multipart upload: %s', + key, + error.message + ); + return this._multipart_upload(key, buffer); + } + } + + async upload_stream(bucket, key, data) { + if (!this.r2_enabled) { + debug('R2 not configured, skipping upload'); + return key; + } + + if (!storage.worker_url) { + throw new Error('Worker URL not set - cannot access R2'); + } + + const unique_key = storage.unique_name(key); + const buffer = this._toBuffer(data); + + try { + return await this._upload_bytes(unique_key, buffer); + } catch (error) { + debug('R2 upload error: %o', error); + throw new Error(`Failed to upload to R2: ${error.message}`); + } + } + + async download_stream(bucket, key) { + if (!this.r2_enabled) { + throw new Error('R2 not configured'); + } + + if (!storage.worker_url) { + throw new Error('Worker URL not set - cannot access R2'); + } + + // Download via worker proxy + const params = new URLSearchParams({ bucket, key }); + const url = `${storage.worker_url}/r2/download?${params}`; + + try { + const response = await fetch(url); + + if (response.status === 404) { + throw new Error(`Object not found: ${key}`); + } + + if (!response.ok) { + throw new Error(`HTTP ${response.status}: ${await response.text()}`); + } + + const arrayBuffer = await response.arrayBuffer(); + return Buffer.from(arrayBuffer); + } catch (error) { + debug('R2 download error: %o', error); + throw new Error(`Failed to download from R2: ${error.message}`); + } + } + + upload(bucket, key, filepath) { + // Generate unique key synchronously so it can be returned immediately + const unique_key = storage.unique_name(key); + + // Read file from disk and upload + if (fs.existsSync(filepath)) { + const data = fs.readFileSync(filepath); + // Call internal version that doesn't generate another unique key + const uploadPromise = this._upload_stream_with_key(bucket, unique_key, data); + return [unique_key, uploadPromise]; + } + + debug('[storage.upload] File not found: %s', filepath); + throw new Error(`upload(): file not found: ${filepath}`); + } + + async _upload_stream_with_key(bucket, key, data) { + debug( + '[storage._upload_stream_with_key] Starting upload: bucket=%s, key=%s, data_size=%d', + bucket, + key, + data.length + ); + + if (!this.r2_enabled) { + debug('R2 not configured, skipping upload'); + return key; + } + + if (!storage.worker_url) { + throw new Error('Worker URL not set - cannot access R2'); + } + + debug('[storage._upload_stream_with_key] Worker URL: %s', storage.worker_url); + + const buffer = this._toBuffer(data); + debug('[storage._upload_stream_with_key] Uploading key=%s, buffer size: %d', key, buffer.length); + + try { + const resultKey = await this._upload_bytes(key, buffer); + debug('[storage._upload_stream_with_key] Upload successful, returned key: %s', resultKey); + return resultKey; + } catch (error) { + debug('R2 upload error: %o', error); + throw new Error(`Failed to upload to R2: ${error.message}`); + } + } + + async download(bucket, key, filepath) { + const data = await this.download_stream(bucket, key); + + let real_fp = filepath; + if (!filepath.startsWith('/tmp')) { + real_fp = path.join('/tmp', path.resolve(filepath)); + } + + // Write data to file + fs.mkdirSync(path.dirname(real_fp), { recursive: true }); + fs.writeFileSync(real_fp, data); + } + + async download_directory(bucket, prefix, out_path) { + if (!this.r2_enabled) { + debug('R2 not configured, skipping download_directory'); + return; + } + + if (!storage.worker_url) { + throw new Error('Worker URL not set - cannot access R2'); + } + + // List objects via worker proxy + const listParams = new URLSearchParams({ bucket, prefix }); + const listUrl = `${storage.worker_url}/r2/list?${listParams}`; + + try { + const response = await fetch(listUrl, { + method: 'GET', + headers: { 'Content-Type': 'application/json' }, + }); + + if (!response.ok) { + const errorText = await response.text(); + throw new Error(`HTTP ${response.status}: ${errorText}`); + } + + const result = await response.json(); + const objects = result.objects || []; + + for (const obj of objects) { + const file_name = obj.key; + const path_to_file = path.dirname(file_name); + fs.mkdirSync(path.join(out_path, path_to_file), { recursive: true }); + await this.download(bucket, file_name, path.join(out_path, file_name)); + } + } catch (error) { + debug('R2 download_directory error: %o', error); + throw new Error(`Failed to download directory from R2: ${error.message}`); + } + } + + async downloadDirectory(bucket, prefix, out_path) { + return this.download_directory(bucket, prefix, out_path); + } + + uploadStream(bucket, key) { + // Return [stream, promise, unique_key] to match native wrapper API + const unique_key = storage.unique_name(key); + + const stream = require('stream'); + const passThrough = new stream.PassThrough(); + const chunks = []; + + passThrough.on('data', (chunk) => chunks.push(chunk)); + + const upload = new Promise((resolve, reject) => { + passThrough.on('end', async () => { + try { + const buffer = Buffer.concat(chunks); + await this._upload_stream_with_key(bucket, unique_key, buffer); + resolve(); + } catch (err) { + reject(err); + } + }); + passThrough.on('error', reject); + }); + + return [passThrough, upload, unique_key]; + } + + async downloadStream(bucket, key) { + // Return a Promise that resolves to a readable stream + const data = await this.download_stream(bucket, key); + const stream = require('stream'); + const readable = new stream.Readable(); + readable.push(data); + readable.push(null); // Signal end of stream + return readable; + } +} + +module.exports.storage = storage; diff --git a/benchmarks/wrappers/cloudflare/nodejs/container/worker.js b/benchmarks/wrappers/cloudflare/nodejs/container/worker.js new file mode 100644 index 000000000..87b3226df --- /dev/null +++ b/benchmarks/wrappers/cloudflare/nodejs/container/worker.js @@ -0,0 +1,425 @@ +// Shared container orchestrator for both Node.js and Python container benchmarks. +// @cloudflare/containers is Node.js-only, so this single worker.js fronts the +// Durable-Object-backed container regardless of the in-container handler +// language. See sebs/cloudflare/containers.py, which copies this file from +// benchmarks/wrappers/cloudflare/nodejs/container/ into every container build +// directory (Python builds included). +import { Container, getContainer } from "@cloudflare/containers"; + +// Container wrapper class +export class ContainerWorker extends Container { + defaultPort = 8080; + sleepAfter = "30m"; +} + +export default { + async fetch(request, env) { + const url = new URL(request.url); + + try { + // Handle NoSQL proxy requests - intercept BEFORE forwarding to container + if (url.pathname.startsWith('/nosql/')) { + return await handleNoSQLRequest(request, env); + } + + // Handle R2 proxy requests - intercept BEFORE forwarding to container + if (url.pathname.startsWith('/r2/')) { + return await handleR2Request(request, env); + } + + // Get or create container instance + const containerId = request.headers.get('x-container-id') || 'default'; + const id = env.CONTAINER_WORKER.idFromName(containerId); + const stub = env.CONTAINER_WORKER.get(id); + + // Clone request and add Worker URL as header so container knows where to proxy R2 requests + const modifiedRequest = new Request(request); + modifiedRequest.headers.set('X-Worker-URL', url.origin); + + // Forward the request to the container + return await stub.fetch(modifiedRequest); + + } catch (error) { + console.error('Worker error:', error); + + const errorMessage = error.message || String(error); + + // Handle container not ready errors with 503 + if (errorMessage.includes('Container failed to start') || + errorMessage.includes('no container instance') || + errorMessage.includes('Durable Object') || + errorMessage.includes('provisioning')) { + + return new Response(JSON.stringify({ + error: 'Container failed to start', + details: 'there is no container instance that can be provided to this durable object', + message: errorMessage + }), { + status: 503, + headers: { 'Content-Type': 'application/json' } + }); + } + + // Other errors get 500 + return new Response(JSON.stringify({ + error: 'Internal server error', + details: errorMessage, + stack: error.stack + }), { + status: 500, + headers: { 'Content-Type': 'application/json' } + }); + } + } +}; + +/** + * Handle NoSQL (KV namespace) requests proxied from the container + * Routes: + * - POST /nosql/insert - insert item + * - POST /nosql/update - update item + * - POST /nosql/get - get item + * - POST /nosql/query - query items + * - POST /nosql/delete - delete item + */ +async function handleNoSQLRequest(request, env) { + try { + const url = new URL(request.url); + const operation = url.pathname.split('/').pop(); + + // Parse request body + const params = await request.json(); + const { table_name, primary_key, secondary_key, secondary_key_name, data } = params; + + const table = env[table_name]; + if (!table || typeof table.get !== 'function' || typeof table.put !== 'function') { + return new Response(JSON.stringify({ + error: `KV namespace binding '${table_name}' not found` + }), { + status: 500, + headers: { 'Content-Type': 'application/json' } + }); + } + + const indexKey = `__sebs_idx__${primary_key[1]}`; + const readIndex = async () => { + const raw = await table.get(indexKey); + if (!raw) { + return []; + } + try { + const parsed = JSON.parse(raw); + return Array.isArray(parsed) ? parsed : []; + } catch { + return []; + } + }; + const writeIndex = async (values) => { + await table.put(indexKey, JSON.stringify(values)); + }; + + const prefix = `${primary_key[1]}#`; + + let result; + switch (operation) { + case 'insert': { + const compositeKey = `${primary_key[1]}#${secondary_key[1]}`; + const keyData = { ...data }; + keyData[primary_key[0]] = primary_key[1]; + keyData[secondary_key[0]] = secondary_key[1]; + await table.put(compositeKey, JSON.stringify(keyData)); + const index = await readIndex(); + if (!index.includes(secondary_key[1])) { + index.push(secondary_key[1]); + await writeIndex(index); + } + result = { success: true }; + break; + } + case 'update': { + const compositeKey = `${primary_key[1]}#${secondary_key[1]}`; + const existingRaw = await table.get(compositeKey); + let existing = {}; + if (existingRaw) { + try { + existing = JSON.parse(existingRaw); + } catch { + existing = {}; + } + } + const merged = { ...existing, ...data }; + merged[primary_key[0]] = primary_key[1]; + merged[secondary_key[0]] = secondary_key[1]; + await table.put(compositeKey, JSON.stringify(merged)); + const index = await readIndex(); + if (!index.includes(secondary_key[1])) { + index.push(secondary_key[1]); + await writeIndex(index); + } + result = { success: true }; + break; + } + case 'get': { + const compositeKey = `${primary_key[1]}#${secondary_key[1]}`; + const raw = await table.get(compositeKey); + if (raw === null) { + result = { data: null }; + } else { + try { + result = { data: JSON.parse(raw) }; + } catch { + result = { data: raw }; + } + } + break; + } + case 'query': { + let secondaryKeys = await readIndex(); + if (secondaryKeys.length === 0) { + const list = await table.list({ prefix }); + secondaryKeys = (list.keys || []).map((k) => k.name.split('#').slice(1).join('#')); + } + const items = []; + for (const secondaryValue of secondaryKeys) { + const raw = await table.get(`${primary_key[1]}#${secondaryValue}`); + if (raw === null) { + continue; + } + try { + items.push(JSON.parse(raw)); + } catch { + items.push(raw); + } + } + result = { items }; + break; + } + case 'delete': { + const compositeKey = `${primary_key[1]}#${secondary_key[1]}`; + await table.delete(compositeKey); + const index = await readIndex(); + const next = index.filter((v) => v !== secondary_key[1]); + if (next.length !== index.length) { + await writeIndex(next); + } + result = { success: true }; + break; + } + default: + return new Response(JSON.stringify({ + error: 'Unknown NoSQL operation' + }), { + status: 404, + headers: { 'Content-Type': 'application/json' } + }); + } + + return new Response(JSON.stringify(result || {}), { + headers: { 'Content-Type': 'application/json' } + }); + + } catch (error) { + console.error('NoSQL proxy error:', error); + return new Response(JSON.stringify({ + error: error.message, + stack: error.stack + }), { + status: 500, + headers: { 'Content-Type': 'application/json' } + }); + } +} + +/** + * Handle R2 storage requests proxied from the container + * Routes: + * - GET /r2/download?bucket=X&key=Y - download object + * - POST /r2/upload?bucket=X&key=Y - upload object (body contains data) + */ +async function handleR2Request(request, env) { + try { + const url = new URL(request.url); + const bucket = url.searchParams.get('bucket'); + const key = url.searchParams.get('key'); + + // Check if R2 binding exists + if (!env.R2) { + return new Response(JSON.stringify({ + error: 'R2 binding not configured' + }), { + status: 500, + headers: { 'Content-Type': 'application/json' } + }); + } + + if (url.pathname === '/r2/list') { + // List objects in R2 with a prefix (only needs bucket) + if (!bucket) { + return new Response(JSON.stringify({ + error: 'Missing bucket parameter' + }), { + status: 400, + headers: { 'Content-Type': 'application/json' } + }); + } + + try { + const prefix = url.searchParams.get('prefix') || ''; + const list_res = await env.R2.list({ prefix }); + + return new Response(JSON.stringify({ + objects: list_res.objects || [] + }), { + headers: { 'Content-Type': 'application/json' } + }); + } catch (error) { + console.error('[worker.js /r2/list] Error:', error); + return new Response(JSON.stringify({ + error: error.message + }), { + status: 500, + headers: { 'Content-Type': 'application/json' } + }); + } + } + + // Multipart upload routes only need 'key' (bucket is implicit in the R2 binding) + if (url.pathname === '/r2/multipart-init') { + // Initiate a multipart upload; returns { key, uploadId } + console.log(`[worker.js /r2/multipart-init] key=${key}`); + const multipart = await env.R2.createMultipartUpload(key); + console.log(`[worker.js /r2/multipart-init] uploadId=${multipart.uploadId}`); + return new Response(JSON.stringify({ + key: multipart.key, + uploadId: multipart.uploadId + }), { headers: { 'Content-Type': 'application/json' } }); + + } else if (url.pathname === '/r2/multipart-part') { + // Upload one part; returns { partNumber, etag } + const uploadId = url.searchParams.get('uploadId'); + const partNumber = parseInt(url.searchParams.get('partNumber'), 10); + console.log(`[worker.js /r2/multipart-part] key=${key}, uploadId=${uploadId}, partNumber=${partNumber}`); + const multipart = env.R2.resumeMultipartUpload(key, uploadId); + const part = await multipart.uploadPart(partNumber, request.body); + console.log(`[worker.js /r2/multipart-part] uploaded part ${part.partNumber}, etag=${part.etag}`); + return new Response(JSON.stringify({ + partNumber: part.partNumber, + etag: part.etag + }), { headers: { 'Content-Type': 'application/json' } }); + + } else if (url.pathname === '/r2/multipart-complete') { + // Complete a multipart upload; body is JSON { parts: [{ partNumber, etag }] } + const uploadId = url.searchParams.get('uploadId'); + console.log(`[worker.js /r2/multipart-complete] key=${key}, uploadId=${uploadId}`); + const { parts } = await request.json(); + const multipart = env.R2.resumeMultipartUpload(key, uploadId); + const obj = await multipart.complete(parts); + console.log(`[worker.js /r2/multipart-complete] completed, size=${obj ? obj.size : '?'}`); + return new Response(JSON.stringify({ key: key }), { + headers: { 'Content-Type': 'application/json' } + }); + } + + // Download and upload require a key (bucket is implicit in the R2 binding) + if (!key) { + return new Response(JSON.stringify({ + error: 'Missing key parameter' + }), { + status: 400, + headers: { 'Content-Type': 'application/json' } + }); + } + + if (url.pathname === '/r2/download') { + // Support optional byte-range via Range header (e.g. "bytes=0-1023") + const rangeHeader = request.headers.get('Range'); + let r2Options = undefined; + let rangeStart, rangeEnd; + if (rangeHeader) { + const match = rangeHeader.match(/^bytes=(\d+)-(\d+)$/); + if (match) { + rangeStart = parseInt(match[1], 10); + rangeEnd = parseInt(match[2], 10); + r2Options = { range: { offset: rangeStart, length: rangeEnd - rangeStart + 1 } }; + } + } + + const object = await env.R2.get(key, r2Options); + + if (!object) { + return new Response(JSON.stringify({ + error: 'Object not found' + }), { + status: 404, + headers: { 'Content-Type': 'application/json' } + }); + } + + const status = rangeHeader ? 206 : 200; + const headers = { + 'Content-Type': object.httpMetadata?.contentType || 'application/octet-stream', + }; + if (rangeHeader && rangeStart !== undefined) { + const totalSize = object.size ?? (rangeEnd - rangeStart + 1); + headers['Content-Range'] = `bytes ${rangeStart}-${rangeEnd}/${totalSize}`; + headers['Content-Length'] = String(rangeEnd - rangeStart + 1); + } else { + headers['Content-Length'] = object.size?.toString() ?? ''; + } + return new Response(object.body, { status, headers }); + + } else if (url.pathname === '/r2/upload') { + // Upload to R2 — stream request.body directly to avoid buffering large payloads in Worker memory + console.log(`[worker.js /r2/upload] bucket=${bucket}, key=${key}`); + console.log(`[worker.js /r2/upload] env.R2 exists:`, !!env.R2); + const contentLength = request.headers.get('Content-Length'); + console.log(`[worker.js /r2/upload] Content-Length: ${contentLength}`); + + // Use the key as-is (container already generates unique keys if needed) + try { + const putResult = await env.R2.put(key, request.body); + const size = putResult ? putResult.size : '(unknown)'; + console.log(`[worker.js /r2/upload] R2.put() succeeded, size=${size}`); + console.log(`[worker.js /r2/upload] Successfully uploaded to R2 with key=${key}`); + } catch (error) { + console.error(`[worker.js /r2/upload] R2.put() error:`, error); + throw error; + } + + return new Response(JSON.stringify({ + key: key + }), { + headers: { 'Content-Type': 'application/json' } + }); + + } else { + return new Response(JSON.stringify({ + error: 'Unknown R2 operation' + }), { + status: 404, + headers: { 'Content-Type': 'application/json' } + }); + } + + } catch (error) { + console.error('R2 proxy error:', error); + return new Response(JSON.stringify({ + error: error.message, + stack: error.stack + }), { + status: 500, + headers: { 'Content-Type': 'application/json' } + }); + } +} + +/** + * Generate unique key for uploaded files + */ +function generateUniqueKey(key) { + const parts = key.split('.'); + const ext = parts.length > 1 ? '.' + parts.pop() : ''; + const name = parts.join('.'); + const uuid = crypto.randomUUID().split('-')[0]; + return `${name}.${uuid}${ext}`; +} diff --git a/benchmarks/wrappers/cloudflare/nodejs/handler.js b/benchmarks/wrappers/cloudflare/nodejs/handler.js new file mode 100644 index 000000000..80170344a --- /dev/null +++ b/benchmarks/wrappers/cloudflare/nodejs/handler.js @@ -0,0 +1,261 @@ +import { DurableObject } from "cloudflare:workers"; + +// Cloudflare Workers freezes Date.now() / performance.now() between I/O +// operations as a timing-sidechannel mitigation, so wall-clock time does +// not advance inside pure-compute sections. To record a meaningful +// compute_time, we issue a throwaway self-fetch that triggers I/O and +// unfreezes the clock before we sample it. +// Docs: https://developers.cloudflare.com/workers/reference/security-model/#step-1-disallow-timers-and-multi-threading +async function advanceWorkersClock(request) { + try { + const url = new URL(request.url); + url.pathname = '/favicon'; + await fetch(url.toString(), { method: 'HEAD' }); + } catch (e) { + // Ignore — we only care about the side effect of performing I/O. + } +} + +// Durable Object class for KV API compatibility +export class KVApiObject extends DurableObject { + constructor(state, env) { + super(state, env); + this.storage = state.storage; + } + + // Proxy methods to make the storage API accessible from the stub + async put(key, value) { + return await this.storage.put(key, value); + } + + async get(key) { + return await this.storage.get(key); + } + + async delete(key) { + return await this.storage.delete(key); + } + + async list(options) { + return await this.storage.list(options); + } +} + +export default { + async fetch(request, env) { + try { + // Store R2 bucket binding and benchmark name in globals for fs-polyfill access + if (env.R2) { + globalThis.R2_BUCKET = env.R2; + } + if (env.BENCHMARK_NAME) { + globalThis.BENCHMARK_NAME = env.BENCHMARK_NAME; + } + + if (request.url.includes('favicon')) { + return new Response('None'); + } + + // Get unique request ID from Cloudflare (CF-Ray header) + const req_id = request.headers.get('CF-Ray') || crypto.randomUUID(); + + // Start timing measurements + const start = performance.now(); + const begin = Date.now() / 1000; + + + // Parse JSON body first (similar to Azure handler which uses req.body) + const req_text = await request.text(); + let event = {}; + if (req_text && req_text.length > 0) { + try { + event = JSON.parse(req_text); + } catch (e) { + // If body isn't JSON, keep event empty + event = {}; + } + } + + // Parse query string into event (URL parameters override/merge with body) + // This makes it compatible with both input formats + const urlParts = request.url.split('?'); + if (urlParts.length > 1) { + const query = urlParts[1]; + const pairs = query.split('&'); + for (const p of pairs) { + const [k, v] = p.split('='); + try { + if (v === undefined) { + event[k] = null; + } else if (!Number.isNaN(Number(v)) && Number.isFinite(Number(v))) { + // mirror Python attempt to convert to int + const n = Number(v); + event[k] = Number.isInteger(n) ? parseInt(v, 10) : n; + } else { + event[k] = decodeURIComponent(v); + } + } catch (e) { + event[k] = v; + } + } + } + + // Set timestamps + const income_timestamp = Math.floor(Date.now() / 1000); + event['request-id'] = req_id; + event['income-timestamp'] = income_timestamp; + + // Load the benchmark function module and initialize storage if available + // With nodejs_compat enabled, we can use require() for CommonJS modules + let funcModule; + try { + // Fallback to dynamic import for ES modules + funcModule = await import('./function.js'); + } catch (e2) { + throw new Error('Failed to import benchmark function module: ' + e2.message); + } + + + try { + const storageModule = await import('./storage.js'); + if (storageModule && storageModule.storage && typeof storageModule.storage.init_instance === 'function') { + storageModule.storage.init_instance({ env, request }); + } else { + console.warn('storage module imported but storage.init_instance is missing; skipping storage setup'); + } + } catch (e) { + // storage module may not be bundled for benchmarks that don't need it + } + + if (env.NOSQL_STORAGE_DATABASE) { + try { + const nosqlModule = await import('./nosql.js'); + if (nosqlModule && nosqlModule.nosql && typeof nosqlModule.nosql.init_instance === 'function') { + nosqlModule.nosql.init_instance({ env, request }); + } else { + console.warn('nosql module imported but nosql.init_instance is missing; skipping nosql setup'); + } + } catch (e) { + // nosql module might not exist for all benchmarks + console.log('Could not initialize nosql:', e.message); + } + } + + // Execute the benchmark handler. Benchmarks expose `handler` either as a + // named export (`exports.handler` / `export const handler`) or nested + // under a default export (`export default { handler }`). + let ret; + try { + const handler = + (funcModule && typeof funcModule.handler === 'function' && funcModule.handler) || + (funcModule && funcModule.default && typeof funcModule.default.handler === 'function' && funcModule.default.handler); + if (!handler) { + throw new Error('benchmark handler function not found'); + } + ret = await handler(event); + } catch (err) { + await advanceWorkersClock(request); + // Calculate timing even for errors + const end = Date.now() / 1000; + const elapsed = performance.now() - start; + const micro = elapsed * 1000; // Convert milliseconds to microseconds + + // Mirror Python behavior: return structured error payload + const errorPayload = JSON.stringify({ + begin: begin, + end: end, + compute_time: micro, + results_time: 0, + result: { output: null }, + is_cold: false, + is_cold_worker: false, + container_id: '0', + environ_container_id: 'no_id', + request_id: '0', + error: String(err && err.message ? err.message : err), + stack: err && err.stack ? err.stack : undefined, + event: event, + env: env, + }); + return new Response(errorPayload, { status: 500, headers: { 'Content-Type': 'application/json' } }); + } + + await advanceWorkersClock(request); + + // Now read the updated timer + const end = Date.now() / 1000; + const elapsed = performance.now() - start; + const micro = elapsed * 1000; // Convert milliseconds to microseconds + + // Build log_data similar to Python handler + const log_data = { result: ret && ret.result !== undefined ? ret.result : ret }; + if (ret && ret.measurement !== undefined) { + log_data.measurement = ret.measurement; + } else { + log_data.measurement = {}; + } + + // Add memory usage to measurement + const memUsage = process.memoryUsage(); + const memory_mb = memUsage.heapUsed / 1024 / 1024; + log_data.measurement.memory_used_mb = memory_mb; + + if (event.logs !== undefined) { + log_data.time = 0; + } + + if (event.html) { + return new Response(String(ret && ret.result !== undefined ? ret.result : ''), { + headers: { 'Content-Type': 'text/html; charset=utf-8' }, + }); + } + + const responseBody = JSON.stringify({ + begin: begin, + end: end, + compute_time: micro, + results_time: 0, + result: log_data, + is_cold: false, + is_cold_worker: false, + container_id: '0', + environ_container_id: 'no_id', + request_id: req_id, + }); + + return new Response(responseBody, { headers: { 'Content-Type': 'application/json' } }); + } catch (topLevelError) { + // Catch any uncaught errors (module loading, syntax errors, etc.) + // Try to include timing if available + let errorBegin = 0; + let errorEnd = 0; + let errorMicro = 0; + try { + errorEnd = Date.now() / 1000; + if (typeof begin !== 'undefined' && typeof start !== 'undefined') { + errorBegin = begin; + const elapsed = performance.now() - start; + errorMicro = elapsed * 1000; + } + } catch (e) { + // Ignore timing errors in error handler + } + + const errorPayload = JSON.stringify({ + begin: errorBegin, + end: errorEnd, + compute_time: errorMicro, + results_time: 0, + result: { output: null }, + is_cold: false, + is_cold_worker: false, + container_id: '0', + environ_container_id: 'no_id', + request_id: '0', + error: `Top-level error: ${topLevelError && topLevelError.message ? topLevelError.message : String(topLevelError)}`, + stack: topLevelError && topLevelError.stack ? topLevelError.stack : undefined, + }); + return new Response(errorPayload, { status: 500, headers: { 'Content-Type': 'application/json' } }); + } + }, +}; diff --git a/benchmarks/wrappers/cloudflare/nodejs/nosql.js b/benchmarks/wrappers/cloudflare/nodejs/nosql.js new file mode 100644 index 000000000..4fe3c80b5 --- /dev/null +++ b/benchmarks/wrappers/cloudflare/nodejs/nosql.js @@ -0,0 +1,158 @@ +// NoSQL wrapper for Cloudflare Workers +// Uses KV namespaces for storage +// Returns Promises that the handler will resolve + +class nosql { + constructor() { + this.env = null; + } + + static init_instance(entry) { + // Reuse existing instance if it exists, otherwise create new one + if (!nosql.instance) { + nosql.instance = new nosql(); + } + + if (entry && entry.env) { + nosql.instance.env = entry.env; + // Share env globally so bundled copies of this module (inlined by esbuild + // into function.js) can also reach the live KV bindings. + globalThis._nosqlEnv = entry.env; + } + } + + _get_table(tableName) { + // Fall back to the global env bridge for copies of this class that were + // inlined by esbuild into a separate bundle (e.g. function.js) and + // therefore have a different static `instance` from the one initialized + // by handler.js via `import('./nosql.js')`. + const env = this.env || globalThis._nosqlEnv; + if (!env) { + throw new Error(`nosql env not initialized for table ${tableName}`); + } + + // Unlike AWS/Azure/GCP where you instantiate a client SDK and address + // resources by name, Cloudflare Workers expose every bound resource + // (KV namespace, R2 bucket, D1 database, queue, etc.) as a property on + // the `env` object passed into the fetch handler. The property name is + // the binding name declared in wrangler.toml, so looking up a KV + // namespace by its table name is simply `env[tableName]`. + const table = env[tableName]; + if (!table || typeof table.get !== 'function' || typeof table.put !== 'function') { + const envKeys = Object.keys(env || {}); + throw new Error( + `KV binding '${tableName}' not found. env keys: [${envKeys.join(', ')}]` + ); + } + + return table; + } + + _key(primaryKey, secondaryKey) { + return `${primaryKey[1]}#${secondaryKey[1]}`; + } + + _indexKey(primaryKey) { + return `__sebs_idx__${primaryKey[1]}`; + } + + async _readIndex(table, primaryKey) { + const raw = await table.get(this._indexKey(primaryKey)); + if (raw === null) { + return []; + } + try { + const parsed = JSON.parse(raw); + return Array.isArray(parsed) ? parsed : []; + } catch { + return []; + } + } + + async _writeIndex(table, primaryKey, values) { + await table.put(this._indexKey(primaryKey), JSON.stringify(values)); + } + + // Async methods - build.js will patch function.js to await these + async insert(tableName, primaryKey, secondaryKey, data) { + const keyData = { ...data }; + keyData[primaryKey[0]] = primaryKey[1]; + keyData[secondaryKey[0]] = secondaryKey[1]; + + const table = this._get_table(tableName); + await table.put(this._key(primaryKey, secondaryKey), JSON.stringify(keyData)); + + const index = await this._readIndex(table, primaryKey); + if (!index.includes(secondaryKey[1])) { + index.push(secondaryKey[1]); + await this._writeIndex(table, primaryKey, index); + } + } + + async get(tableName, primaryKey, secondaryKey) { + const table = this._get_table(tableName); + const raw = await table.get(this._key(primaryKey, secondaryKey)); + if (raw === null) { + return null; + } + + try { + return JSON.parse(raw); + } catch { + return raw; + } + } + + async update(tableName, primaryKey, secondaryKey, updates) { + const existing = await this.get(tableName, primaryKey, secondaryKey) || {}; + const merged = { ...existing, ...updates }; + await this.insert(tableName, primaryKey, secondaryKey, merged); + } + + async query(tableName, primaryKey, secondaryKeyName) { + const table = this._get_table(tableName); + let secondaryKeys = await this._readIndex(table, primaryKey); + + // Fallback for legacy namespaces without explicit index key. + if (secondaryKeys.length === 0) { + const listed = await table.list({ prefix: `${primaryKey[1]}#` }); + secondaryKeys = (listed.keys || []).map((k) => k.name.split('#').slice(1).join('#')); + } + + const results = []; + + for (const secondaryValue of secondaryKeys) { + const raw = await table.get(`${primaryKey[1]}#${secondaryValue}`); + if (raw === null) { + continue; + } + try { + results.push(JSON.parse(raw)); + } catch { + results.push(raw); + } + } + + return results; + } + + async delete(tableName, primaryKey, secondaryKey) { + const table = this._get_table(tableName); + await table.delete(this._key(primaryKey, secondaryKey)); + + const index = await this._readIndex(table, primaryKey); + const next = index.filter((v) => v !== secondaryKey[1]); + if (next.length !== index.length) { + await this._writeIndex(table, primaryKey, next); + } + } + + static get_instance() { + if (!nosql.instance) { + nosql.instance = new nosql(); + } + return nosql.instance; + } +} + +export { nosql }; diff --git a/benchmarks/wrappers/cloudflare/nodejs/postprocess.js b/benchmarks/wrappers/cloudflare/nodejs/postprocess.js new file mode 100644 index 000000000..2c06ce414 --- /dev/null +++ b/benchmarks/wrappers/cloudflare/nodejs/postprocess.js @@ -0,0 +1,86 @@ +/** + * Post-processing step: replace esbuild's dynamic __require("node:…") helper + * calls in the bundled dist/ output with static ESM imports. + * + * esbuild bundles dependencies that themselves call require() at runtime, + * turning them into __require("node:fs") style calls. Cloudflare Workers + * run in an ESM-only environment, so these must be hoisted to top-level + * import statements that wrangler / the runtime can resolve. + * + * Only the top-level requires in *source* files can be handled by esbuild's + * own external/format options, which is why this step is needed separately. + * + * Usage: node postprocess.js (run from the worker package directory) + */ + +'use strict'; + +const fs = require('fs'); +const { join, relative } = require('path'); + +function getAllJsFiles(dir, fileList = []) { + if (!fs.existsSync(dir)) return fileList; + const files = fs.readdirSync(dir, { withFileTypes: true }); + for (const file of files) { + const filePath = join(dir, file.name); + if (file.isDirectory()) { + getAllJsFiles(filePath, fileList); + } else if (file.name.endsWith('.js')) { + fileList.push(filePath); + } + } + return fileList; +} + +const distDir = './dist'; +const jsFiles = getAllJsFiles(distDir); + +let totalFixed = 0; + +for (const filePath of jsFiles) { + let content = fs.readFileSync(filePath, 'utf-8'); + + // Collect all unique node: modules required via esbuild's __require helper. + const nodeModules = new Set(); + const requireRegex = /__require\d*\("(node:[^"]+)"\)/g; + let match; + while ((match = requireRegex.exec(content)) !== null) { + nodeModules.add(match[1]); + } + + if (nodeModules.size === 0) continue; + + // Build static import declarations and a lookup cache object. + let imports = ''; + const mapping = {}; + let i = 0; + for (const mod of nodeModules) { + const varName = `__node_${mod.replace('node:', '').replace(/[^a-z0-9]/gi, '_')}_${i++}`; + imports += `import * as ${varName} from '${mod}';\n`; + mapping[mod] = varName; + } + + imports += '\nconst __node_cache = {\n'; + for (const [mod, varName] of Object.entries(mapping)) { + imports += ` '${mod}': ${varName},\n`; + } + imports += '};\n\n'; + + // Replace every __require("node:…") call with a cache lookup. + content = content.replace(/__require(\d*)\("(node:[^"]+)"\)/g, (_match, _num, mod) => { + return `__node_cache['${mod}']`; + }); + + // Prepend the import block. + content = imports + content; + + fs.writeFileSync(filePath, content, 'utf-8'); + console.log(`✓ Fixed ${nodeModules.size} node: import(s) in ${relative(distDir, filePath)}`); + totalFixed++; +} + +if (totalFixed === 0) { + console.log('No __require node: calls found — nothing to patch.'); +} else { + console.log(`✓ Post-processing complete (${totalFixed} file(s) patched).`); +} diff --git a/benchmarks/wrappers/cloudflare/nodejs/request-polyfill.js b/benchmarks/wrappers/cloudflare/nodejs/request-polyfill.js new file mode 100644 index 000000000..f44bfa232 --- /dev/null +++ b/benchmarks/wrappers/cloudflare/nodejs/request-polyfill.js @@ -0,0 +1,100 @@ +/** + * Polyfill for the 'request' module using Cloudflare Workers fetch API + * Implements the minimal interface needed for benchmark compatibility + */ + +const { Writable } = require('node:stream'); +const fs = require('node:fs'); + +function request(url, options, callback) { + // Handle different call signatures + if (typeof options === 'function') { + callback = options; + options = {}; + } + + // Add default headers to mimic a browser request + const fetchOptions = { + ...options, + headers: { + 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36', + 'Accept': '*/*', + ...((options && options.headers) || {}) + } + }; + + // Create a simple object that has a pipe method + const requestObj = { + pipe(destination) { + // Perform the fetch and write to destination + fetch(url, fetchOptions) + .then(async (response) => { + if (!response.ok) { + const error = new Error(`HTTP ${response.status}: ${response.statusText}`); + error.statusCode = response.status; + destination.emit('error', error); + if (callback) callback(error, response, null); + return destination; + } + + // Get the response as arrayBuffer and write it all at once + const buffer = await response.arrayBuffer(); + + // Write the buffer to the destination + if (destination.write) { + destination.write(Buffer.from(buffer)); + destination.end(); + } + + if (callback) callback(null, response, Buffer.from(buffer)); + }) + .catch((error) => { + destination.emit('error', error); + if (callback) callback(error, null, null); + }); + + return destination; + }, + + abort() { + // No-op for compatibility + } + }; + + return requestObj; +} + +// Add common request methods +request.get = (url, options, callback) => { + if (typeof options === 'function') { + callback = options; + options = {}; + } + return request(url, { ...options, method: 'GET' }, callback); +}; + +request.post = (url, options, callback) => { + if (typeof options === 'function') { + callback = options; + options = {}; + } + return request(url, { ...options, method: 'POST' }, callback); +}; + +request.put = (url, options, callback) => { + if (typeof options === 'function') { + callback = options; + options = {}; + } + return request(url, { ...options, method: 'PUT' }, callback); +}; + +request.delete = (url, options, callback) => { + if (typeof options === 'function') { + callback = options; + options = {}; + } + return request(url, { ...options, method: 'DELETE' }, callback); +}; + +module.exports = request; diff --git a/benchmarks/wrappers/cloudflare/nodejs/storage.js b/benchmarks/wrappers/cloudflare/nodejs/storage.js new file mode 100644 index 000000000..3ba303839 --- /dev/null +++ b/benchmarks/wrappers/cloudflare/nodejs/storage.js @@ -0,0 +1,268 @@ +const fs = require('fs'); +const path = require('path'); +const uuid = require('uuid'); + +// Storage wrapper compatible with the Python storage implementation. +// Supports Cloudflare R2 (via env.R2) when available; falls back to +// filesystem-based operations when running in Node.js (for local tests). + +class storage { + constructor() { + this.handle = null; // R2 binding + this.written_files = new Set(); + } + + static unique_name(name) { + const parsed = path.parse(name); + const uuid_name = uuid.v4().split('-')[0]; + return path.join(parsed.dir, `${parsed.name}.${uuid_name}${parsed.ext}`); + } + + // entry is expected to be an object with `env` (Workers) or nothing for Node + static init_instance(entry) { + storage.instance = new storage(); + if (entry && entry.env && entry.env.R2) { + storage.instance.handle = entry.env.R2; + } + storage.instance.written_files = new Set(); + } + + // Upload a file given a local filepath. In Workers env this is not available + // so callers should use upload_stream or pass raw data. For Node.js we read + // the file from disk and put it into R2 if available, otherwise throw. + upload(__bucket, key, filepath) { + // Use singleton instance if available, otherwise use this instance + const instance = storage.instance || this; + + // If file was previously written during this invocation, use /tmp absolute + let realPath = filepath; + if (instance.written_files.has(filepath)) { + realPath = path.join('/tmp', path.resolve(filepath)); + } + + const unique_key = storage.unique_name(key); + + // Try filesystem first (for Workers with nodejs_compat that have /tmp) + if (fs && fs.existsSync(realPath)) { + const data = fs.readFileSync(realPath); + + if (instance.handle) { + const uploadPromise = instance.handle.put(unique_key, data); + return [unique_key, uploadPromise]; + } else { + return [unique_key, Promise.resolve()]; + } + } + + // Fallback: In Workers environment with R2, check if file exists in R2 + // (it may have been written by fs-polyfill's createWriteStream) + if (instance.handle) { + // Normalize the path to match what fs-polyfill would use + let normalizedPath = realPath.replace(/^\.?\//, '').replace(/^tmp\//, ''); + + // Add benchmark name prefix if available (matching fs-polyfill behavior) + if (typeof globalThis !== 'undefined' && globalThis.BENCHMARK_NAME && + !normalizedPath.startsWith(globalThis.BENCHMARK_NAME + '/')) { + normalizedPath = globalThis.BENCHMARK_NAME + '/' + normalizedPath; + } + + // Read from R2 and re-upload with unique key + const uploadPromise = instance.handle.get(normalizedPath).then(async (obj) => { + if (obj) { + const data = await obj.arrayBuffer(); + return instance.handle.put(unique_key, data); + } else { + throw new Error(`File not found in R2: ${normalizedPath} (original path: ${filepath})`); + } + }); + + return [unique_key, uploadPromise]; + } + + // If running in Workers (no fs) and caller provided Buffer/Stream, they + // should call upload_stream directly. Otherwise, throw. + throw new Error('upload(): file not found on disk and no R2 handle provided'); + } + + async download(__bucket, key, filepath) { + const instance = storage.instance || this; + const data = await this.download_stream(__bucket, key); + + let real_fp = filepath; + if (!filepath.startsWith('/tmp')) { + real_fp = path.join('/tmp', path.resolve(filepath)); + } + + instance.written_files.add(filepath); + + // Write data to file if we have fs + if (fs) { + fs.mkdirSync(path.dirname(real_fp), { recursive: true }); + if (Buffer.isBuffer(data)) { + fs.writeFileSync(real_fp, data); + } else { + fs.writeFileSync(real_fp, Buffer.from(String(data))); + } + return; + } + + // In Workers environment, callers should use stream APIs directly. + return; + } + + async download_directory(__bucket, prefix, out_path) { + const instance = storage.instance || this; + + if (!instance.handle) { + throw new Error('download_directory requires R2 binding (env.R2)'); + } + + const list_res = await instance.handle.list({ prefix }); + const objects = list_res.objects || []; + for (const obj of objects) { + const file_name = obj.key; + const path_to_file = path.dirname(file_name); + fs.mkdirSync(path.join(out_path, path_to_file), { recursive: true }); + await this.download(__bucket, file_name, path.join(out_path, file_name)); + } + } + + async downloadDirectory(bucket, prefix, out_path) { + return this.download_directory(bucket, prefix, out_path); + } + + async upload_stream(__bucket, key, data) { + const instance = storage.instance || this; + const unique_key = storage.unique_name(key); + if (instance.handle) { + // R2 put accepts ArrayBuffer, ReadableStream, or string + await instance.handle.put(unique_key, data); + return unique_key; + } + + // If no R2, write to local fs as fallback + if (fs) { + const outPath = path.join('/tmp', unique_key); + fs.mkdirSync(path.dirname(outPath), { recursive: true }); + if (Buffer.isBuffer(data)) fs.writeFileSync(outPath, data); + else fs.writeFileSync(outPath, Buffer.from(String(data))); + return unique_key; + } + + throw new Error('upload_stream(): no storage backend available'); + } + + async download_stream(__bucket, key) { + const instance = storage.instance || this; + + if (instance.handle) { + const obj = await instance.handle.get(key); + if (!obj) return null; + // R2 object provides arrayBuffer()/text() helpers in Workers + if (typeof obj.arrayBuffer === 'function') { + const ab = await obj.arrayBuffer(); + return Buffer.from(ab); + } + if (typeof obj.text === 'function') { + return await obj.text(); + } + // Fallback: return null + return null; + } + + // Fallback to local filesystem + const localPath = path.join('/tmp', key); + if (fs && fs.existsSync(localPath)) { + return fs.readFileSync(localPath); + } + + throw new Error('download_stream(): object not found'); + } + + // Additional stream methods for compatibility with Azure storage API + // These provide a stream-based interface similar to Azure's uploadStream/downloadStream + uploadStream(__bucket, key) { + const unique_key = storage.unique_name(key); + + if (this.handle) { + // For R2, we create a PassThrough stream that collects data + // then uploads when ended + const stream = require('stream'); + const passThrough = new stream.PassThrough(); + const chunks = []; + + passThrough.on('data', (chunk) => chunks.push(chunk)); + + const upload = new Promise((resolve, reject) => { + passThrough.on('end', async () => { + try { + const buffer = Buffer.concat(chunks); + await this.handle.put(unique_key, buffer); + resolve(); + } catch (err) { + reject(err); + } + }); + passThrough.on('error', reject); + }); + + return [passThrough, upload, unique_key]; + } + + // Fallback to filesystem + if (fs) { + const stream = require('stream'); + const outPath = path.join('/tmp', unique_key); + fs.mkdirSync(path.dirname(outPath), { recursive: true }); + const writeStream = fs.createWriteStream(outPath); + const upload = new Promise((resolve, reject) => { + writeStream.on('finish', resolve); + writeStream.on('error', reject); + }); + return [writeStream, upload, unique_key]; + } + + throw new Error('uploadStream(): no storage backend available'); + } + + async downloadStream(__bucket, key) { + if (this.handle) { + const obj = await this.handle.get(key); + if (!obj) return null; + + // R2 object has a body ReadableStream + if (obj.body) { + return obj.body; + } + + // Fallback: convert to buffer then to stream + if (typeof obj.arrayBuffer === 'function') { + const stream = require('stream'); + const ab = await obj.arrayBuffer(); + const buffer = Buffer.from(ab); + const readable = new stream.PassThrough(); + readable.end(buffer); + return readable; + } + + return null; + } + + // Fallback to local filesystem + const localPath = path.join('/tmp', key); + if (fs && fs.existsSync(localPath)) { + return fs.createReadStream(localPath); + } + + throw new Error('downloadStream(): object not found'); + } + + static get_instance() { + if (!storage.instance) { + throw new Error('must init storage singleton first'); + } + return storage.instance; + } +} + +module.exports.storage = storage; diff --git a/benchmarks/wrappers/cloudflare/python/container/handler.py b/benchmarks/wrappers/cloudflare/python/container/handler.py new file mode 100644 index 000000000..9a9732f41 --- /dev/null +++ b/benchmarks/wrappers/cloudflare/python/container/handler.py @@ -0,0 +1,219 @@ +#!/usr/bin/env python3 +""" +Container handler for Cloudflare Workers - Python +This handler is used when deploying as a container worker +""" + +import json +import sys +import os +import traceback +import resource +from http.server import HTTPServer, BaseHTTPRequestHandler +from urllib.parse import urlparse, parse_qs +import datetime + +# Monkey-patch requests library to add User-Agent header +# This is needed because many HTTP servers (like Wikimedia) reject requests without User-Agent +try: + import requests + original_request = requests.request + + def patched_request(method, url, **kwargs): + if 'headers' not in kwargs: + kwargs['headers'] = {} + if 'User-Agent' not in kwargs['headers']: + kwargs['headers']['User-Agent'] = 'SeBS/1.2 (https://github.com/spcl/serverless-benchmarks) SeBS Benchmark Suite/1.2' + return original_request(method, url, **kwargs) + + requests.request = patched_request + print("Monkey-patched requests library to add User-Agent header") +except ImportError: + print("requests library not available, skipping User-Agent monkey-patch") + +# Also patch urllib for libraries that use it directly +import urllib.request +original_urlopen = urllib.request.urlopen + +def patched_urlopen(url, data=None, timeout=None, **kwargs): + if isinstance(url, str): + req = urllib.request.Request(url, data=data) + req.add_header('User-Agent', 'SeBS/1.2 (https://github.com/spcl/serverless-benchmarks) SeBS Benchmark Suite/1.2') + return original_urlopen(req, timeout=timeout, **kwargs) + elif isinstance(url, urllib.request.Request): + if not url.has_header('User-Agent'): + url.add_header('User-Agent', 'SeBS/1.2 (https://github.com/spcl/serverless-benchmarks) SeBS Benchmark Suite/1.2') + return original_urlopen(url, data=data, timeout=timeout, **kwargs) + else: + return original_urlopen(url, data=data, timeout=timeout, **kwargs) + +urllib.request.urlopen = patched_urlopen +print("Monkey-patched urllib.request.urlopen to add User-Agent header") + +# Import the default benchmark handler function. +# For workflow dispatch mode, individual function modules are imported dynamically. +try: + from function.function import handler as benchmark_handler +except ImportError: + benchmark_handler = None + +# Import storage and nosql if available +try: + from function import storage +except ImportError: + storage = None + print("Storage module not available") + +try: + from function import nosql +except ImportError: + nosql = None + print("NoSQL module not available") + +PORT = int(os.environ.get('PORT', 8080)) + + +class ContainerHandler(BaseHTTPRequestHandler): + def do_GET(self): + self.handle_request() + + def do_POST(self): + self.handle_request() + + def handle_request(self): + # Handle favicon requests + if 'favicon' in self.path: + self.send_response(200) + self.end_headers() + self.wfile.write(b'None') + return + + try: + # Get unique request ID from Cloudflare (CF-Ray header) + import uuid + req_id = self.headers.get('CF-Ray', str(uuid.uuid4())) + + # Extract Worker URL from header for R2 and NoSQL proxy + worker_url = self.headers.get('X-Worker-URL') + if worker_url: + if storage: + storage.storage.set_worker_url(worker_url) + if nosql: + nosql.nosql.set_worker_url(worker_url) + print(f"Set worker URL for R2/NoSQL proxy: {worker_url}") + + # Read request body + content_length = int(self.headers.get('Content-Length', 0)) + body = self.rfile.read(content_length).decode('utf-8') if content_length > 0 else '' + + # Parse event from JSON body or URL params + event = {} + if body: + try: + event = json.loads(body) + except json.JSONDecodeError as e: + print(f'Failed to parse JSON body: {e}') + + # Parse URL parameters + parsed_url = urlparse(self.path) + params = parse_qs(parsed_url.query) + for key, values in params.items(): + if key not in event and values: + value = values[0] + try: + event[key] = int(value) + except ValueError: + event[key] = value + + # Workflow dispatch mode: if the event contains a "function" key, + # route to that specific module instead of the default handler. + if 'function' in event: + import importlib + func_name = event['function'] + func_input = event.get('input', {}) + if isinstance(func_input, dict): + func_input = {**func_input, 'request-id': req_id} + module = importlib.import_module(f"function.{func_name}") + func_result = module.handler(func_input) + self.send_response(200) + self.send_header('Content-Type', 'application/json') + self.end_headers() + self.wfile.write(json.dumps(func_result).encode('utf-8')) + return + + # Add request metadata + income_timestamp = datetime.datetime.now().timestamp() + event['request-id'] = req_id + event['income-timestamp'] = income_timestamp + + # Measure execution time + begin = datetime.datetime.now().timestamp() + + # Call the benchmark function + result = benchmark_handler(event) + + # Calculate timing + end = datetime.datetime.now().timestamp() + compute_time = end - begin + + # Prepare response matching native handler format exactly + log_data = { + 'result': result['result'] + } + if 'measurement' in result: + log_data['measurement'] = result['measurement'] + else: + log_data['measurement'] = {} + + # Add memory usage to measurement + memory_mb = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1024.0 + log_data['measurement']['memory_used_mb'] = memory_mb + + response_data = { + 'begin': begin, + 'end': end, + 'results_time': 0, + 'result': log_data, + 'is_cold': False, + 'is_cold_worker': False, + 'container_id': "0", + 'environ_container_id': "no_id", + 'request_id': req_id + } + + # Send response + if event.get('html'): + # For HTML requests, return just the result + self.send_response(200) + self.send_header('Content-Type', 'text/html; charset=utf-8') + self.end_headers() + html_result = result.get('result', result) + self.wfile.write(str(html_result).encode('utf-8')) + else: + # For API requests, return structured response + self.send_response(200) + self.send_header('Content-Type', 'application/json') + self.end_headers() + self.wfile.write(json.dumps(response_data).encode('utf-8')) + + except Exception as error: + print(f'Error processing request: {error}') + traceback.print_exc() + self.send_response(500) + self.send_header('Content-Type', 'application/json') + self.end_headers() + error_response = { + 'error': str(error), + 'traceback': traceback.format_exc() + } + self.wfile.write(json.dumps(error_response).encode('utf-8')) + + def log_message(self, format, *args): + # Override to use print instead of stderr + print(f"{self.address_string()} - {format % args}") + + +if __name__ == '__main__': + server = HTTPServer(('0.0.0.0', PORT), ContainerHandler) + print(f'Container server listening on port {PORT}') + server.serve_forever() diff --git a/benchmarks/wrappers/cloudflare/python/container/nosql.py b/benchmarks/wrappers/cloudflare/python/container/nosql.py new file mode 100644 index 000000000..5a414d21a --- /dev/null +++ b/benchmarks/wrappers/cloudflare/python/container/nosql.py @@ -0,0 +1,122 @@ +""" +NoSQL module for Cloudflare Python Containers. + +Issues HTTP POSTs to {worker_url}/nosql/. The server side is +implemented in benchmarks/wrappers/cloudflare/nodejs/container/worker.js +(handleNoSQLRequest), which is copied into every container project at deploy +time by sebs/cloudflare/containers.py because @cloudflare/containers is +Node.js-only and wraps Python containers as well. +""" +import json +import urllib.request +import urllib.parse +from typing import List, Optional, Tuple + + +class nosql: + """NoSQL client for containers using HTTP proxy to Worker's Durable Object""" + + instance: Optional["nosql"] = None + worker_url = None # Set by handler from X-Worker-URL header + + @staticmethod + def init_instance(*args, **kwargs): + """Initialize singleton instance""" + if nosql.instance is None: + nosql.instance = nosql() + return nosql.instance + + @staticmethod + def set_worker_url(url): + """Set worker URL for NoSQL proxy (called by handler)""" + nosql.worker_url = url + + def _make_request(self, operation: str, params: dict) -> dict: + """Make HTTP request to worker nosql proxy""" + if not nosql.worker_url: + raise RuntimeError("Worker URL not set - cannot access NoSQL") + + url = f"{nosql.worker_url}/nosql/{operation}" + data = json.dumps(params).encode('utf-8') + + req = urllib.request.Request(url, data=data, method='POST') + req.add_header('Content-Type', 'application/json') + + try: + with urllib.request.urlopen(req) as response: + return json.loads(response.read().decode('utf-8')) + except urllib.error.HTTPError as e: + error_body = e.read().decode('utf-8') + try: + error_data = json.loads(error_body) + raise RuntimeError(f"NoSQL operation failed: {error_data.get('error', error_body)}") + except json.JSONDecodeError: + raise RuntimeError(f"NoSQL operation failed: {error_body}") + except Exception as e: + raise RuntimeError(f"NoSQL operation failed: {e}") + + def insert( + self, + table_name: str, + primary_key: Tuple[str, str], + secondary_key: Tuple[str, str], + data: dict, + ): + params = { + 'table_name': table_name, + 'primary_key': list(primary_key), + 'secondary_key': list(secondary_key), + 'data': data + } + return self._make_request('insert', params) + + def update( + self, + table_name: str, + primary_key: Tuple[str, str], + secondary_key: Tuple[str, str], + data: dict, + ): + params = { + 'table_name': table_name, + 'primary_key': list(primary_key), + 'secondary_key': list(secondary_key), + 'data': data + } + return self._make_request('update', params) + + def get( + self, table_name: str, primary_key: Tuple[str, str], secondary_key: Tuple[str, str] + ) -> Optional[dict]: + params = { + 'table_name': table_name, + 'primary_key': list(primary_key), + 'secondary_key': list(secondary_key) + } + result = self._make_request('get', params) + return result.get('data') + + def query( + self, table_name: str, primary_key: Tuple[str, str], secondary_key_name: str + ) -> List[dict]: + params = { + 'table_name': table_name, + 'primary_key': list(primary_key), + 'secondary_key_name': secondary_key_name + } + result = self._make_request('query', params) + return result.get('items', []) + + def delete(self, table_name: str, primary_key: Tuple[str, str], secondary_key: Tuple[str, str]): + params = { + 'table_name': table_name, + 'primary_key': list(primary_key), + 'secondary_key': list(secondary_key) + } + return self._make_request('delete', params) + + @staticmethod + def get_instance(): + if nosql.instance is None: + nosql.instance = nosql() + return nosql.instance diff --git a/benchmarks/wrappers/cloudflare/python/container/storage.py b/benchmarks/wrappers/cloudflare/python/container/storage.py new file mode 100644 index 000000000..bd6f6dafb --- /dev/null +++ b/benchmarks/wrappers/cloudflare/python/container/storage.py @@ -0,0 +1,291 @@ +""" +Storage module for Cloudflare Python Containers +Uses HTTP proxy to access R2 storage through the Worker's R2 binding +""" +import io +import os +import json +import urllib.request +import urllib.parse + +# Cloudflare Workers enforce a 100 MB request body limit at the edge. +# Use multipart upload for payloads larger than this threshold so that +# each individual request stays well below that limit. +_MULTIPART_THRESHOLD = 10 * 1024 * 1024 # 10 MB +_PART_SIZE = 10 * 1024 * 1024 # 10 MB per part (R2 min is 5 MB) + +class storage: + """R2 storage client for containers using HTTP proxy to Worker""" + instance = None + worker_url = None # Set by handler from X-Worker-URL header + + def __init__(self): + # Container accesses R2 through worker.js proxy + # Worker URL is injected via X-Worker-URL header in each request + self.r2_enabled = True + + @staticmethod + def init_instance(entry=None): + """Initialize singleton instance""" + if storage.instance is None: + storage.instance = storage() + return storage.instance + + @staticmethod + def get_instance(): + """Get singleton instance""" + if storage.instance is None: + storage.init_instance() + return storage.instance + + @staticmethod + def set_worker_url(url): + """Set worker URL for R2 proxy (called by handler)""" + storage.worker_url = url + + @staticmethod + def unique_name(name): + """Generate unique name for file""" + import uuid + name_part, extension = os.path.splitext(name) + return f'{name_part}.{str(uuid.uuid4()).split("-")[0]}{extension}' + + # ------------------------------------------------------------------ + # Internal helpers + # ------------------------------------------------------------------ + + def _post_json(self, url: str, body: bytes = b'', content_type: str = 'application/octet-stream'): + """POST *body* to *url* and return the parsed JSON response.""" + req = urllib.request.Request(url, data=body, method='POST') + req.add_header('Content-Type', content_type) + with urllib.request.urlopen(req) as resp: + return json.loads(resp.read().decode('utf-8')) + + def _upload_bytes(self, key: str, data: bytes) -> str: + """Upload *data* to the exact R2 *key* via the worker proxy. + + Uses a single PUT for small payloads and R2 multipart upload for + payloads that exceed _MULTIPART_THRESHOLD (to stay under Cloudflare's + 100 MB per-request edge limit). + + Returns the R2 key. + """ + if len(data) <= _MULTIPART_THRESHOLD: + return self._single_upload(key, data) + return self._multipart_upload(key, data) + + def _single_upload(self, key: str, data: bytes) -> str: + params = urllib.parse.urlencode({'key': key}) + url = f"{storage.worker_url}/r2/upload?{params}" + result = self._post_json(url, data) + return result['key'] + + def _multipart_upload(self, key: str, data: bytes) -> str: + """Split *data* into ≤_PART_SIZE chunks and use R2 multipart upload.""" + # 1. Initiate + params = urllib.parse.urlencode({'key': key}) + init_url = f"{storage.worker_url}/r2/multipart-init?{params}" + init = self._post_json(init_url) + upload_id = init['uploadId'] + upload_key = init['key'] + print(f"[storage] multipart upload initiated: key={upload_key}, uploadId={upload_id}, " + f"total={len(data):,} bytes, parts={-(-len(data)//_PART_SIZE)}") + + # 2. Upload parts + completed_parts = [] + for part_num, offset in enumerate(range(0, len(data), _PART_SIZE), start=1): + chunk = data[offset:offset + _PART_SIZE] + params = urllib.parse.urlencode({ + 'key': upload_key, + 'uploadId': upload_id, + 'partNumber': part_num, + }) + part_url = f"{storage.worker_url}/r2/multipart-part?{params}" + part = self._post_json(part_url, chunk) + completed_parts.append({'partNumber': part['partNumber'], 'etag': part['etag']}) + print(f"[storage] uploaded part {part_num}, etag={part['etag']}") + + # 3. Complete + params = urllib.parse.urlencode({'key': upload_key, 'uploadId': upload_id}) + complete_url = f"{storage.worker_url}/r2/multipart-complete?{params}" + result = self._post_json( + complete_url, + json.dumps({'parts': completed_parts}).encode('utf-8'), + content_type='application/json', + ) + print(f"[storage] multipart upload complete: key={result['key']}") + return result['key'] + + # ------------------------------------------------------------------ + # Public API + # ------------------------------------------------------------------ + + def upload_stream(self, bucket: str, key: str, data): + """Upload data to R2 via worker proxy""" + if not self.r2_enabled: + print("Warning: R2 not configured, skipping upload") + return key + + if not storage.worker_url: + raise RuntimeError("Worker URL not set - cannot access R2") + + # Handle BytesIO objects + if isinstance(data, io.BytesIO): + data = data.getvalue() + + # Convert to bytes if needed + if isinstance(data, str): + data = data.encode('utf-8') + + unique_key = self.unique_name(key) + + try: + return self._upload_bytes(unique_key, data) + except Exception as e: + print(f"R2 upload error: {e}") + raise RuntimeError(f"Failed to upload to R2: {e}") + + def download_stream(self, bucket: str, key: str) -> bytes: + """Download data from R2 via worker proxy""" + if not self.r2_enabled: + raise RuntimeError("R2 not configured") + + if not storage.worker_url: + raise RuntimeError("Worker URL not set - cannot access R2") + + # Download via worker proxy + params = urllib.parse.urlencode({'bucket': bucket, 'key': key}) + url = f"{storage.worker_url}/r2/download?{params}" + + try: + with urllib.request.urlopen(url) as response: + return response.read() + except urllib.error.HTTPError as e: + if e.code == 404: + raise RuntimeError(f"Object not found: {key}") + else: + raise RuntimeError(f"Failed to download from R2: {e}") + except Exception as e: + print(f"R2 download error: {e}") + raise RuntimeError(f"Failed to download from R2: {e}") + + def upload(self, bucket, key, filepath, unique_name=True): + """Upload file from disk.""" + upload_key = self.unique_name(key) if unique_name else key + with open(filepath, 'rb') as f: + data = f.read() + try: + self._upload_bytes(upload_key, data) + except Exception as e: + raise RuntimeError(f"Failed to upload to R2: {e}") + return upload_key + + def _upload_with_key(self, bucket: str, key: str, data): + """Upload data to R2 via worker proxy with exact key (internal method)""" + if not self.r2_enabled: + print("Warning: R2 not configured, skipping upload") + return + + if not storage.worker_url: + raise RuntimeError("Worker URL not set - cannot access R2") + + # Handle BytesIO objects + if isinstance(data, io.BytesIO): + data = data.getvalue() + + # Convert to bytes if needed + if isinstance(data, str): + data = data.encode('utf-8') + + try: + result_key = self._upload_bytes(key, data) + print(f"[storage._upload_with_key] Upload successful, key={result_key}") + except Exception as e: + print(f"R2 upload error: {e}") + raise RuntimeError(f"Failed to upload to R2: {e}") + + def download(self, bucket, key, filepath): + """Download file to disk""" + data = self.download_stream(bucket, key) + os.makedirs(os.path.dirname(filepath), exist_ok=True) + with open(filepath, 'wb') as f: + f.write(data) + + def download_within_range(self, bucket: str, key: str, start_bytes: int, end_bytes: int) -> str: + """Download a byte range of an object from R2 via the worker proxy.""" + if not self.r2_enabled: + raise RuntimeError("R2 not configured") + + if not storage.worker_url: + raise RuntimeError("Worker URL not set - cannot access R2") + + params = urllib.parse.urlencode({'bucket': bucket, 'key': key}) + url = f"{storage.worker_url}/r2/download?{params}" + + req = urllib.request.Request(url) + req.add_header('Range', f'bytes={start_bytes}-{end_bytes}') + + try: + with urllib.request.urlopen(req) as response: + return response.read().decode('utf-8') + except urllib.error.HTTPError as e: + if e.code in (206, 200): + return e.read().decode('utf-8') + raise RuntimeError(f"Failed to download range from R2: {e}") + except Exception as e: + raise RuntimeError(f"Failed to download range from R2: {e}") + + def list_directory(self, bucket, prefix): + """List all object keys with a given prefix.""" + if not storage.worker_url: + raise RuntimeError("Worker URL not set - cannot access R2") + params = urllib.parse.urlencode({'bucket': bucket, 'prefix': prefix}) + list_url = f"{storage.worker_url}/r2/list?{params}" + with urllib.request.urlopen(list_url) as response: + result = json.loads(response.read().decode('utf-8')) + return [obj['key'] for obj in result.get('objects', [])] + + def download_directory(self, bucket, prefix, local_path): + """ + Download all files with a given prefix to a local directory. + Lists objects via /r2/list endpoint and downloads each one in parallel. + """ + import concurrent.futures + + if not storage.worker_url: + raise RuntimeError("Worker URL not set - cannot access R2") + + # Create local directory + os.makedirs(local_path, exist_ok=True) + + # List objects with prefix via worker proxy + params = urllib.parse.urlencode({'bucket': bucket, 'prefix': prefix}) + list_url = f"{storage.worker_url}/r2/list?{params}" + + try: + with urllib.request.urlopen(list_url) as response: + result = json.loads(response.read().decode('utf-8')) + objects = result.get('objects', []) + + print(f"Found {len(objects)} objects with prefix '{prefix}'") + + def _download_one(obj): + obj_key = obj['key'] + local_file_path = os.path.join(local_path, obj_key) + local_dir = os.path.dirname(local_file_path) + if local_dir: + os.makedirs(local_dir, exist_ok=True) + print(f"Downloading {obj_key} to {local_file_path}") + self.download(bucket, obj_key, local_file_path) + + # Download all objects in parallel (up to 16 concurrent) + with concurrent.futures.ThreadPoolExecutor(max_workers=16) as executor: + futures = [executor.submit(_download_one, obj) for obj in objects] + for fut in concurrent.futures.as_completed(futures): + fut.result() # re-raise any exception + + return local_path + + except Exception as e: + print(f"Error listing/downloading directory: {e}") + raise RuntimeError(f"Failed to download directory: {e}") diff --git a/benchmarks/wrappers/cloudflare/python/handler.py b/benchmarks/wrappers/cloudflare/python/handler.py new file mode 100644 index 000000000..45d343607 --- /dev/null +++ b/benchmarks/wrappers/cloudflare/python/handler.py @@ -0,0 +1,154 @@ +import datetime, io, json, os, uuid, sys, ast +import asyncio +import importlib.util +import traceback +import time +try: + import resource + HAS_RESOURCE = True +except ImportError: + # Pyodide (Python native workers) doesn't support resource module + HAS_RESOURCE = False +from workers import WorkerEntrypoint, Response +from js import fetch as js_fetch, URL + +## sys.path.append(os.path.join(os.path.dirname(__file__), '.python_packages/lib/site-packages')) + +""" +currently assumed file structure: + +handler.py +function/ + function.py + .py + storage.py + nosql.py + +""" + +class Default(WorkerEntrypoint): + async def fetch(self, request, env): + try: + return await self.fetch2(request, env) + except Exception as e: + t = traceback.format_exc() + print(t) + return Response(t) + + async def fetch2(self, request, env): + if "favicon" in request.url: return Response("None") + + # Get unique request ID from Cloudflare (CF-Ray header) + req_id = request.headers.get('CF-Ray', str(uuid.uuid4())) + + # Start timing measurements + start = time.perf_counter() + begin = datetime.datetime.now().timestamp() + + req_text = await request.text() + + event = json.loads(req_text) if len(req_text) > 0 else {} + ## print(event) + + # dirty url parameters parsing, for testing + tmp = request.url.split("?") + if len(tmp) > 1: + urlparams = tmp[1] + urlparams = [chunk.split("=") for chunk in urlparams.split("&")] + for param in urlparams: + try: + event[param[0]] = int(param[1]) + except ValueError: + event[param[0]] = param[1] + except IndexError: + event[param[0]] = None + + from function import storage + + storage.storage.init_instance(self) + + if hasattr(self.env, 'NOSQL_STORAGE_DATABASE'): + from function import nosql + + nosql.nosql.init_instance(self) + + # Workflow dispatch mode: route to a specific function module + if 'function' in event: + import importlib + func_name = event['function'] + func_input = event.get('input', {}) + if isinstance(func_input, dict): + func_input = {**func_input, 'request-id': req_id} + module = importlib.import_module(f"function.{func_name}") + func_result = module.handler(func_input) + return Response(json.dumps(func_result), + headers={"Content-Type": "application/json"}) + + ## note: time fixed in worker + income_timestamp = datetime.datetime.now().timestamp() + + event['request-id'] = req_id + event['income-timestamp'] = income_timestamp + + + print("event:", event) + + +## make_benchmark_func() +## function = import_from_path("function.function", "/tmp/function.py") + + from function import function + + ret = function.handler(event) + + log_data = { + 'result': ret['result'] + } + if 'measurement' in ret: + log_data['measurement'] = ret['measurement'] + else: + log_data['measurement'] = {} + + # Add memory usage to measurement (if resource module is available) + if HAS_RESOURCE: + memory_mb = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1024.0 + log_data['measurement']['memory_used_mb'] = memory_mb + else: + # Pyodide doesn't support resource module + log_data['measurement']['memory_used_mb'] = 0.0 + + if 'logs' in event: + log_data['time'] = 0 + + if "html" in event: + headers = {"Content-Type" : "text/html; charset=utf-8"} + return Response(str(ret["result"]), headers = headers) + else: + # Trigger a fetch request to update the timer before measuring + # Time measurements only update after a fetch request or R2 operation + try: + # Fetch the worker's own URL with favicon to minimize overhead + final_url = URL.new(request.url) + final_url.pathname = '/favicon' + await js_fetch(str(final_url), method='HEAD') + except: + # Ignore fetch errors + pass + + # Calculate timestamps + end = datetime.datetime.now().timestamp() + elapsed = time.perf_counter() - start + micro = elapsed * 1_000_000 # Convert seconds to microseconds + + return Response(json.dumps({ + 'begin': begin, + 'end': end, + 'compute_time': micro, + 'results_time': 0, + 'result': log_data, + 'is_cold': False, + 'is_cold_worker': False, + 'container_id': "0", + 'environ_container_id': "no_id", + 'request_id': req_id + })) diff --git a/benchmarks/wrappers/cloudflare/python/nosql.py b/benchmarks/wrappers/cloudflare/python/nosql.py new file mode 100644 index 000000000..7a91a94f1 --- /dev/null +++ b/benchmarks/wrappers/cloudflare/python/nosql.py @@ -0,0 +1,242 @@ +from typing import List, Optional, Tuple +import json +import pickle +from pyodide.ffi import to_js, run_sync +from workers import WorkerEntrypoint, DurableObject + + +class nosql_do: + instance: Optional["nosql_do"] = None + DO_BINDING_NAME = "DURABLE_STORE" + + @staticmethod + def init_instance(entry: WorkerEntrypoint): + nosql_do.instance = nosql_do() + nosql_do.instance.binding = getattr(entry.env, nosql_do.DO_BINDING_NAME) + + + def get_table(self, table_name): + kvapiobj = self.binding.getByName(table_name) + return kvapiobj + + def key_maker(self, key1, key2): + return f"({key1[0]},{str(key1[1])})+({key2[0]},{key2[1]})" + + def key_maker_partial(self, key1, key2): + return f"({key1[0]},{str(key1[1])})+({key2[0]}" + +## these data conversion funcs should not be necessary. i couldn't get pyodide to clone the data otherwise + def data_pre(self, data): + return pickle.dumps(data, 0).decode("ascii") + + def data_post(self, data): + # Handle None (key not found in storage) + if data is None: + return None + # Handle both string and bytes data from Durable Object storage + if isinstance(data, str): + return pickle.loads(bytes(data, "ascii")) + else: + return pickle.loads(data) + + def insert( + self, + table_name: str, + primary_key: Tuple[str, str], + secondary_key: Tuple[str, str], + data: dict, + ): + send_data = self.data_pre(data) + k=self.key_maker(primary_key, secondary_key) + put_res = run_sync(self.get_table(table_name).put(k, send_data)) + return + + ## does this really need different behaviour from insert? + def update( + self, + table_name: str, + primary_key: Tuple[str, str], + secondary_key: Tuple[str, str], + data: dict, + ): + self.insert(table_name, primary_key, secondary_key, data) + return + + + def get( + self, table_name: str, primary_key: Tuple[str, str], secondary_key: Tuple[str, str] + ) -> Optional[dict]: + k=self.key_maker(primary_key, secondary_key) + get_res = run_sync(self.get_table(table_name).get(k)) + ## print(get_res) + return self.data_post(get_res) + + """ + This query must involve partition key - it does not scan across partitions. + """ + + def query( + self, table_name: str, primary_key: Tuple[str, str], secondary_key_name: str + ) -> List[dict]: + + prefix_key = self.key_maker_partial(primary_key, (secondary_key_name,)) + list_res = run_sync(self.get_table(table_name).list()) + + keys = [] + for key in list_res: + if key.startswith(prefix_key): + print(key) + keys.append(key) + ##print("keys", keys) + assert len(keys) <= 100 + + + # todo: please use bulk sometime (it didn't work when i tried it) + res = [] + for key in keys: + + get_res = run_sync(self.get_table(table_name).get(key)) + ## print(get_res) + res.append(self.data_post(get_res)) + return res + + def delete(self, table_name: str, primary_key: Tuple[str, str], secondary_key: Tuple[str, str]): + run_sync(self.get_table(table_name).delete(self.key_maker(primary_key, secondary_key))) + return + + @staticmethod + def get_instance(): + if nosql_do.instance is None: + nosql_do.instance = nosql_do() + return nosql_do.instance + +### ------------------------------ + +class nosql_kv: + + instance: Optional["nosql_kv"] = None + + @staticmethod + def init_instance(entry: WorkerEntrypoint): + nosql_kv.instance = nosql_kv() + nosql_kv.instance.env = entry.env + + def key_maker(self, key1, key2): + return f"{key1[1]}#{key2[1]}" + + def key_maker_partial(self, key1, key2): + return f"{key1[1]}#" + + def index_key(self, primary_key): + return f"__sebs_idx__{primary_key[1]}" + + def get_table(self, table_name): + return getattr(self.env, (table_name)) + + def insert( + self, + table_name: str, + primary_key: Tuple[str, str], + secondary_key: Tuple[str, str], + data: dict, + ): + key_data = {**data} + key_data[primary_key[0]] = primary_key[1] + key_data[secondary_key[0]] = secondary_key[1] + put_res = run_sync( + self.get_table(table_name).put( + self.key_maker(primary_key, secondary_key), + json.dumps(key_data), + ) + ) + + idx_raw = run_sync(self.get_table(table_name).get(self.index_key(primary_key))) + idx = [] + if idx_raw: + idx = json.loads(idx_raw) + if secondary_key[1] not in idx: + idx.append(secondary_key[1]) + run_sync(self.get_table(table_name).put(self.index_key(primary_key), json.dumps(idx))) + return + + def update( + self, + table_name: str, + primary_key: Tuple[str, str], + secondary_key: Tuple[str, str], + data: dict, + ): + existing = self.get(table_name, primary_key, secondary_key) + if existing is None: + existing = {} + merged = {**existing, **data} + merged[primary_key[0]] = primary_key[1] + merged[secondary_key[0]] = secondary_key[1] + put_res = run_sync( + self.get_table(table_name).put( + self.key_maker(primary_key, secondary_key), + json.dumps(merged), + ) + ) + return + + def get( + self, table_name: str, primary_key: Tuple[str, str], secondary_key: Tuple[str, str] + ) -> Optional[dict]: + get_res = run_sync( + self.get_table(table_name).get( + self.key_maker(primary_key, secondary_key) + )) + if get_res is None: + return None + if isinstance(get_res, dict): + return get_res + return json.loads(get_res) + + """ + This query must involve partition key - it does not scan across partitions. + """ + + def query( + self, table_name: str, primary_key: Tuple[str, str], secondary_key_name: str + ) -> List[dict]: + idx_raw = run_sync(self.get_table(table_name).get(self.index_key(primary_key))) + idx = [] + if idx_raw: + idx = json.loads(idx_raw) + + res = [] + for secondary_key_value in idx: + key = f"{primary_key[1]}#{secondary_key_value}" + get_res = run_sync(self.get_table(table_name).get(key)) + if get_res is None: + continue + if isinstance(get_res, dict): + res.append(get_res) + else: + res.append(json.loads(get_res)) + return res + + def delete(self, table_name: str, primary_key: Tuple[str, str], secondary_key: Tuple[str, str]): + run_sync(self.get_table(table_name).delete(self.key_maker(primary_key, secondary_key))) + + idx_raw = run_sync(self.get_table(table_name).get(self.index_key(primary_key))) + idx = [] + if idx_raw: + idx = json.loads(idx_raw) + if secondary_key[1] in idx: + idx = [v for v in idx if v != secondary_key[1]] + run_sync(self.get_table(table_name).put(self.index_key(primary_key), json.dumps(idx))) + + return + + @staticmethod + def get_instance(): + if nosql_kv.instance is None: + nosql_kv.instance = nosql_kv() + return nosql_kv.instance + + + + +nosql = nosql_kv diff --git a/benchmarks/wrappers/cloudflare/python/storage.py b/benchmarks/wrappers/cloudflare/python/storage.py new file mode 100644 index 000000000..a4c734a54 --- /dev/null +++ b/benchmarks/wrappers/cloudflare/python/storage.py @@ -0,0 +1,121 @@ +import io +import os +import uuid +from pyodide.ffi import to_js, jsnull, run_sync + +from workers import WorkerEntrypoint + +## all filesystem calls will rely on the node:fs flag +""" layout +/bundle +└── (one file for each module in your Worker bundle) +/tmp +└── (empty, but you can write files, create directories, symlinks, etc) +/dev +├── null +├── random +├── full +└── zero +""" +class storage: + instance = None + + @staticmethod + def unique_name(name): + name, extension = os.path.splitext(name) + return '{name}.{random}{extension}'.format( + name=name, + extension=extension, + random=str(uuid.uuid4()).split('-')[0] + ) + def get_bucket(self, bucket): + # R2 buckets are always bound as 'R2' in wrangler.toml + # The bucket parameter is the actual bucket name but we access via the binding + return self.entry_env.R2 + + @staticmethod + def init_instance(entry: WorkerEntrypoint): + storage.instance = storage() + storage.instance.entry_env = entry.env + storage.instance.written_files = set() + + def upload(self, bucket, key, filepath, unique_name=True): + if filepath in self.written_files: + filepath = "/tmp" + os.path.abspath(filepath) + with open(filepath, "rb") as f: + data = f.read() + if unique_name: + upload_key = self.upload_stream(bucket, key, data) + else: + upload_key = run_sync(self._upload_exact(bucket, key, data)) + return upload_key + + def download(self, bucket, key, filepath): + data = self.download_stream(bucket, key) + # should only allow writes to tmp dir. so do have to edit the filepath here? + real_fp = filepath + if not filepath.startswith("/tmp"): + real_fp = "/tmp" + os.path.abspath(filepath) + + self.written_files.add(filepath) + with open(real_fp, "wb") as f: + f.write(data) + return + + def list_directory(self, bucket, prefix): + bobj = self.get_bucket(bucket) + list_res = run_sync(bobj.list(to_js({"prefix": prefix}))) + return [obj.key for obj in list_res.objects] + + def download_directory(self, bucket, prefix, out_path): + bobj = self.get_bucket(bucket) + list_res = run_sync(bobj.list(to_js({"prefix": prefix}))) + for obj in list_res.objects: + file_name = obj.key + path_to_file = os.path.dirname(file_name) + os.makedirs(os.path.join(out_path, path_to_file), exist_ok=True) + self.download(bucket, file_name, os.path.join(out_path, file_name)) + return + + async def _upload_exact(self, bucket, key, data): + if hasattr(data, 'getvalue'): + data = data.getvalue() + data_js = to_js(data) if isinstance(data, bytes) else str(data) + bobj = self.get_bucket(bucket) + await bobj.put(key, data_js) + return key + + def upload_stream(self, bucket, key, data): + return run_sync(self.aupload_stream(bucket, key, data)) + + async def aupload_stream(self, bucket, key, data): + unique_key = storage.unique_name(key) + if hasattr(data, 'getvalue'): + data = data.getvalue() + if isinstance(data, bytes): + data_js = to_js(data) + else: + data_js = str(data) + bobj = self.get_bucket(bucket) + put_res = await bobj.put(unique_key, data_js) + return unique_key + + def download_stream(self, bucket, key): + return run_sync(self.adownload_stream(bucket, key)) + + async def adownload_stream(self, bucket, key): + bobj = self.get_bucket(bucket) + get_res = await bobj.get(key) + if get_res == jsnull: + print("key not stored in bucket") + return b'' + # Always read as raw binary data (Blob/ArrayBuffer) + data = await get_res.bytes() + return bytes(data) + + @staticmethod + def get_instance(): + if storage.instance is None: + raise RuntimeError("must init storage singleton first") + return storage.instance + return storage.instance diff --git a/benchmarks/wrappers/gcp/python/handler_workflow.py b/benchmarks/wrappers/gcp/python/handler_workflow.py new file mode 100644 index 000000000..6cf88eb3b --- /dev/null +++ b/benchmarks/wrappers/gcp/python/handler_workflow.py @@ -0,0 +1,114 @@ + +import datetime +import json +import os +import sys +import uuid +import importlib + +# Add current directory to allow location of packages +sys.path.append(os.path.join(os.path.dirname(__file__), '.python_packages/lib/site-packages')) + +if 'NOSQL_STORAGE_DATABASE' in os.environ: + from function import nosql + + nosql.nosql.get_instance( + os.environ['NOSQL_STORAGE_DATABASE'] + ) + + +def probe_cold_start(): + is_cold = False + fname = os.path.join("/tmp", "cold_run") + if not os.path.exists(fname): + is_cold = True + container_id = str(uuid.uuid4())[0:8] + with open(fname, "a") as f: + f.write(container_id) + else: + with open(fname, "r") as f: + container_id = f.read() + + return is_cold, container_id + + +def handler(req): + start = datetime.datetime.now().timestamp() + os.environ["STORAGE_UPLOAD_BYTES"] = "0" + os.environ["STORAGE_DOWNLOAD_BYTES"] = "0" + provider_request_id = ( + req.headers.get("X-Cloud-Trace-Context") or req.headers.get("Function-Execution-Id") + ) + + event = req.get_json(force=True) + + if isinstance(event, dict) and "payload" in event: + func_payload = event["payload"] + request_id = event.get("request_id", provider_request_id) + elif isinstance(event, dict): + request_id = event.pop("__request_id", provider_request_id) + func_payload = event + else: + func_payload = event + request_id = provider_request_id + + if isinstance(func_payload, dict): + func_payload['request-id'] = provider_request_id + + full_function_name = os.getenv("MY_FUNCTION_NAME", "") + if "--" in full_function_name: + workflow_name, func_name = full_function_name.rsplit("--", 1) + elif "___" in full_function_name: + workflow_name, func_name = full_function_name.split("___", 1) + else: + workflow_name = full_function_name + func_name = full_function_name + + function = importlib.import_module(f"function.{func_name}") + res = function.handler(func_payload) + + end = datetime.datetime.now().timestamp() + + is_cold, container_id = probe_cold_start() + measurement = { + "func": func_name, + "start": start, + "end": end, + "is_cold": is_cold, + "container_id": container_id, + "provider.request_id": provider_request_id, + } + + func_res = os.getenv("SEBS_FUNCTION_RESULT") + if func_res: + measurement["result"] = json.loads(func_res) + + bytes_upload = os.getenv("STORAGE_UPLOAD_BYTES", 0) + if bytes_upload: + measurement["blob.upload"] = int(bytes_upload) + + bytes_download = os.getenv("STORAGE_DOWNLOAD_BYTES", 0) + if bytes_download: + measurement["blob.download"] = int(bytes_download) + + try: + redis_host = os.getenv("REDIS_HOST", "") + redis_password = os.getenv("REDIS_PASSWORD", "") + if redis_host and redis_password: + from redis import Redis + redis_client = Redis( + host=redis_host, + port=6379, + decode_responses=True, + socket_connect_timeout=10, + password=redis_password, + ) + key = os.path.join(workflow_name, func_name, request_id, str(uuid.uuid4())[0:8]) + redis_client.set(key, json.dumps(measurement)) + except Exception: + pass + + if isinstance(res, dict): + res["__request_id"] = request_id + + return json.dumps(res), 200, {'Content-Type': 'application/json'} diff --git a/benchmarks/wrappers/gcp/python/storage.py b/benchmarks/wrappers/gcp/python/storage.py index 9ea541e0c..23a40a34a 100644 --- a/benchmarks/wrappers/gcp/python/storage.py +++ b/benchmarks/wrappers/gcp/python/storage.py @@ -22,8 +22,8 @@ def unique_name(name): random=str(uuid.uuid4()).split('-')[0] ) - def upload(self, bucket, file, filepath): - key_name = storage.unique_name(file) + def upload(self, bucket, file, filepath, unique_name=True): + key_name = storage.unique_name(file) if unique_name else file bucket_instance = self.client.bucket(bucket) blob = bucket_instance.blob(key_name) blob.upload_from_filename(filepath) @@ -34,6 +34,15 @@ def download(self, bucket, file, filepath): blob = bucket_instance.blob(file) blob.download_to_filename(filepath) + def download_within_range(self, bucket, file, start_bytes, end_bytes): + blob = self.client.bucket(bucket).blob(file) + data = blob.download_as_bytes(start=start_bytes, end=end_bytes) + return data.decode("utf-8") + + def list_directory(self, bucket, prefix): + objects = self.client.bucket(bucket).list_blobs(prefix=prefix) + return [obj.name for obj in objects] + def download_directory(self, bucket, prefix, path): objects = self.client.bucket(bucket).list_blobs(prefix=prefix) for obj in objects: diff --git a/benchmarks/wrappers/local/python/storage.py b/benchmarks/wrappers/local/python/storage.py index f8bb7c0c0..52764bd1a 100644 --- a/benchmarks/wrappers/local/python/storage.py +++ b/benchmarks/wrappers/local/python/storage.py @@ -29,14 +29,22 @@ def unique_name(name): random=str(uuid.uuid4()).split('-')[0] ) - def upload(self, bucket, file, filepath): - key_name = storage.unique_name(file) + def upload(self, bucket, file, filepath, unique_name=True): + key_name = storage.unique_name(file) if unique_name else file self.client.fput_object(bucket, key_name, filepath) return key_name def download(self, bucket, file, filepath): self.client.fget_object(bucket, file, filepath) + def download_within_range(self, bucket, file, start_bytes, end_bytes): + data = self.client.get_object(bucket, file, offset=start_bytes, length=end_bytes - start_bytes + 1) + return data.read().decode("utf-8") + + def list_directory(self, bucket, prefix): + objects = self.client.list_objects(bucket, prefix, recursive=True) + return [obj.object_name for obj in objects] + def download_directory(self, bucket, prefix, path): objects = self.client.list_objects_v2(bucket, prefix, recursive=True) for obj in objects: diff --git a/benchmarks/wrappers/openwhisk/python/storage.py b/benchmarks/wrappers/openwhisk/python/storage.py index 26ea56612..7986dcd6b 100644 --- a/benchmarks/wrappers/openwhisk/python/storage.py +++ b/benchmarks/wrappers/openwhisk/python/storage.py @@ -49,14 +49,22 @@ def unique_name(name): ) - def upload(self, bucket, file, filepath): - key_name = storage.unique_name(file) + def upload(self, bucket, file, filepath, unique_name=True): + key_name = storage.unique_name(file) if unique_name else file self.client.fput_object(bucket, key_name, filepath) return key_name def download(self, bucket, file, filepath): self.client.fget_object(bucket, file, filepath) + def download_within_range(self, bucket, file, start_bytes, end_bytes): + data = self.client.get_object(bucket, file, offset=start_bytes, length=end_bytes - start_bytes + 1) + return data.read().decode("utf-8") + + def list_directory(self, bucket, prefix): + objects = self.client.list_objects(bucket, prefix, recursive=True) + return [obj.object_name for obj in objects] + def download_directory(self, bucket, prefix, path): objects = self.client.list_objects(bucket, prefix, recursive=True) for obj in objects: diff --git a/configs/example.json b/configs/example.json index a3f3f9b20..7ccc19c07 100644 --- a/configs/example.json +++ b/configs/example.json @@ -48,6 +48,10 @@ "region": "us-east-1", "lambda-role": "", "resources": { + "redis": { + "host": "ec2-54-86-32-136.compute-1.amazonaws.com", + "password": "xB46z3u9I6WJ" + }, "use-function-url": true, "function-url-auth-type": "NONE" } @@ -99,6 +103,10 @@ "type": "minio" } }, + "cloudflare": { + "max_instances": 20, + "chunk_size": 1 + }, "openwhisk": { "shutdownStorage": false, "removeCluster": false, diff --git a/configs/systems.json b/configs/systems.json index d649e0bc0..c77443e6f 100644 --- a/configs/systems.json +++ b/configs/systems.json @@ -25,6 +25,13 @@ "3.9": "python:3.9-slim", "3.10": "python:3.10-slim", "3.11": "python:3.11-slim" + }, + "arm64": { + "3.7": "python:3.7-slim", + "3.8": "python:3.8-slim", + "3.9": "python:3.9-slim", + "3.10": "python:3.10-slim", + "3.11": "python:3.11-slim" } }, "images": [ @@ -71,12 +78,8 @@ } } }, - "architecture": [ - "x64" - ], - "deployments": [ - "package" - ] + "architecture": ["x64", "arm64"], + "deployments": ["package"] }, "aws": { "languages": { @@ -99,11 +102,14 @@ "deployment": { "files": [ "handler.py", + "handler_workflow.py", "storage.py", "nosql.py", "setup.py" ], - "packages": [], + "packages": [ + "redis" + ], "module_packages": {} } }, @@ -206,10 +212,19 @@ "deployment": { "files": [ "handler.py", + "handler_workflow.py", + "main_workflow.py", + "run_workflow.py", + "run_subworkflow.py", + "fsm.py", "storage.py", "nosql.py" ], - "packages": [], + "packages": [ + "azure-functions", + "azure-functions-durable==1.1.6", + "redis" + ], "module_packages": { "storage": [ "azure-storage-blob" @@ -296,6 +311,7 @@ "deployment": { "files": [ "handler.py", + "handler_workflow.py", "storage.py", "nosql.py", "setup.py" @@ -452,11 +468,97 @@ } } }, - "architecture": [ - "x64" - ], - "deployments": [ - "container" - ] + "architecture": ["x64"], + "deployments": ["container"] + }, + "cloudflare": { + "languages": { + "python": { + "base_images": { + "x64": { + "3.8": "python:3.8-slim", + "3.9": "python:3.9-slim", + "3.10": "python:3.10-slim", + "3.11": "python:3.11-slim", + "3.12": "python:3.12-slim" + } + }, + "container_images": { + "x64": { + "3.8": "python:3.8-slim", + "3.9": "python:3.9-slim", + "3.10": "python:3.10-slim", + "3.11": "python:3.11-slim", + "3.12": "python:3.12-slim" + } + }, + "images": ["build"], + "supported_variants": ["default", "cloudflare"], + "deployment": { + "files": [ + "handler.py", + "storage.py", + "nosql.py" + ], + "packages": [], + "module_packages": {} + }, + "container_deployment": { + "files": [ + "handler.py", + "storage.py", + "nosql.py" + ], + "packages": [], + "module_packages": {} + } + }, + "nodejs": { + "base_images": { + "x64": { + "18": "node:18-slim", + "20": "node:20-slim" + } + }, + "container_images": { + "x64": { + "18": "node:18-slim", + "20": "node:20-slim" + } + }, + "images": ["build"], + "supported_variants": ["default", "cloudflare"], + "deployment": { + "files": [ + "handler.js", + "storage.js", + "nosql.js", + "build.js", + "postprocess.js", + "request-polyfill.js" + ], + "packages": { + "uuid": "3.4.0" + } + }, + "container_deployment": { + "files": [ + "handler.js", + "storage.js", + "nosql.js" + ], + "packages": { + "uuid": "3.4.0" + } + } + } + }, + "images": { + "manage": { + "username": "docker_user" + } + }, + "architecture": ["x64"], + "deployments": ["workers", "container"] } } diff --git a/dockerfiles/cloudflare/Dockerfile.manage b/dockerfiles/cloudflare/Dockerfile.manage new file mode 100644 index 000000000..46ffc5ab1 --- /dev/null +++ b/dockerfiles/cloudflare/Dockerfile.manage @@ -0,0 +1,36 @@ +FROM node:20-slim + +# Disable telemetry +ENV WRANGLER_SEND_METRICS=false + +# Install system dependencies including Docker CLI +RUN apt-get clean && apt-get update \ + && apt-get install -y ca-certificates curl gnupg gosu python3 python3-pip python3-venv git \ + && install -m 0755 -d /etc/apt/keyrings \ + && curl -fsSL https://download.docker.com/linux/debian/gpg -o /etc/apt/keyrings/docker.asc \ + && chmod a+r /etc/apt/keyrings/docker.asc \ + && echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/debian bookworm stable" > /etc/apt/sources.list.d/docker.list \ + && apt-get update \ + && apt-get install -y docker-ce-cli \ + && apt-get purge -y --auto-remove \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + +# Install wrangler and @cloudflare/containers globally +RUN npm install -g wrangler @cloudflare/containers + +# Install uv (fast Python package installer) and pywrangler +# Pin workers-py to 1.8.0: 1.9.x introduced a broken import (rich.logging.Console) +# which does not exist in any version of rich. Remove the pin once upstream fixes it. +RUN curl -LsSf https://astral.sh/uv/install.sh | sh && \ + /root/.local/bin/uv tool install 'workers-py==1.8.0' + +# Add paths to environment +ENV PATH="/root/.local/bin:/root/.local/share/uv/tools/workers-py/bin:${PATH}" + +# Create working directory +RUN mkdir -p /sebs/ +COPY dockerfiles/entrypoint.sh /sebs/entrypoint.sh +RUN chmod +x /sebs/entrypoint.sh + +ENTRYPOINT ["/sebs/entrypoint.sh"] diff --git a/dockerfiles/cloudflare/nodejs/Dockerfile.build b/dockerfiles/cloudflare/nodejs/Dockerfile.build new file mode 100644 index 000000000..204434e76 --- /dev/null +++ b/dockerfiles/cloudflare/nodejs/Dockerfile.build @@ -0,0 +1,22 @@ +ARG BASE_IMAGE=node:20-slim +FROM ${BASE_IMAGE} + +# useradd, groupmod, gosu (needed by entrypoint.sh to drop privileges) +RUN apt-get update && apt-get install -y --no-install-recommends \ + passwd curl ca-certificates \ + && rm -rf /var/lib/apt/lists/* +ENV GOSU_VERSION=1.14 +RUN curl -o /usr/local/bin/gosu -SL "https://github.com/tianon/gosu/releases/download/${GOSU_VERSION}/gosu-$(dpkg --print-architecture)" \ + && chmod +x /usr/local/bin/gosu + +# Install esbuild globally once — benchmark source arrives via bind-mount. +RUN npm install -g esbuild + +RUN mkdir -p /sebs/ +COPY dockerfiles/cloudflare_nodejs_installer.sh /sebs/installer.sh +COPY dockerfiles/entrypoint.sh /sebs/entrypoint.sh +RUN chmod +x /sebs/installer.sh /sebs/entrypoint.sh + +ENV PATH=/usr/sbin:$PATH +CMD ["/bin/bash", "/sebs/installer.sh"] +ENTRYPOINT ["/sebs/entrypoint.sh"] diff --git a/dockerfiles/cloudflare/nodejs/Dockerfile.function b/dockerfiles/cloudflare/nodejs/Dockerfile.function new file mode 100644 index 000000000..1bf6a89cb --- /dev/null +++ b/dockerfiles/cloudflare/nodejs/Dockerfile.function @@ -0,0 +1,36 @@ +ARG BASE_IMAGE=node:18-slim +FROM ${BASE_IMAGE} + +# Install system dependencies needed for benchmarks +RUN apt-get update && apt-get install -y --no-install-recommends \ + wget \ + xz-utils \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /app + +# Copy package files first for better caching +COPY package*.json ./ + +# Install dependencies +RUN npm install --production + +# Copy all application files +COPY . . + +# Run benchmark init script if it exists (e.g., for ffmpeg in video-processing) +# This downloads static binaries needed by the benchmark +# Note: ignore errors from init.sh (e.g., when resources already exist) +RUN if [ -f "init.sh" ]; then \ + chmod +x init.sh && \ + ./init.sh /app verbose x64 || true; \ + fi + +# Expose port 8080 for container communication +EXPOSE 8080 + +# Set environment variable for port +ENV PORT=8080 + +# Start the HTTP server +CMD ["node", "handler.js"] diff --git a/dockerfiles/cloudflare/python/Dockerfile.build b/dockerfiles/cloudflare/python/Dockerfile.build new file mode 100644 index 000000000..283b088e3 --- /dev/null +++ b/dockerfiles/cloudflare/python/Dockerfile.build @@ -0,0 +1,30 @@ +ARG BASE_IMAGE=python:3.11-slim +FROM ${BASE_IMAGE} + +# useradd, groupmod, gosu (needed by entrypoint.sh to drop privileges) +# curl + ca-certificates are also required by the uv installer; git is sometimes +# pulled in by workers-py when it resolves VCS-declared deps. +RUN apt-get update && apt-get install -y --no-install-recommends \ + passwd curl ca-certificates git \ + && rm -rf /var/lib/apt/lists/* +ENV GOSU_VERSION=1.14 +RUN curl -o /usr/local/bin/gosu -SL "https://github.com/tianon/gosu/releases/download/${GOSU_VERSION}/gosu-$(dpkg --print-architecture)" \ + && chmod +x /usr/local/bin/gosu + +# Install uv (fast Python package manager) and the workers-py toolchain, +# which provides pywrangler for Pyodide-based Cloudflare Worker deploys. +# Pinned to 1.8.0 to match Dockerfile.manage — 1.9.x introduced a broken +# import (rich.logging.Console). Remove the pin once upstream fixes it. +RUN curl -LsSf https://astral.sh/uv/install.sh | sh && \ + /root/.local/bin/uv tool install 'workers-py==1.8.0' + +ENV PATH="/root/.local/bin:/root/.local/share/uv/tools/workers-py/bin:/usr/sbin:${PATH}" + +# Benchmark source arrives via bind-mount at /mnt/function — no COPY . . here. +RUN mkdir -p /sebs/ +COPY dockerfiles/cloudflare_python_installer.sh /sebs/installer.sh +COPY dockerfiles/entrypoint.sh /sebs/entrypoint.sh +RUN chmod +x /sebs/installer.sh /sebs/entrypoint.sh + +CMD ["/bin/bash", "/sebs/installer.sh"] +ENTRYPOINT ["/sebs/entrypoint.sh"] diff --git a/dockerfiles/cloudflare/python/Dockerfile.function b/dockerfiles/cloudflare/python/Dockerfile.function new file mode 100644 index 000000000..e9ecc0e86 --- /dev/null +++ b/dockerfiles/cloudflare/python/Dockerfile.function @@ -0,0 +1,38 @@ +ARG BASE_IMAGE=python:3.11-slim +FROM ${BASE_IMAGE} + +# Install system dependencies needed for benchmarks +RUN apt-get update && apt-get install -y --no-install-recommends \ + wget \ + xz-utils \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /app + +# Copy all application files first +COPY . . + +# Run benchmark init script if it exists (e.g., for ffmpeg in video-processing) +# This downloads static binaries needed by the benchmark +# Note: ignore errors from init.sh (e.g., when resources already exist) +RUN if [ -f "init.sh" ]; then \ + chmod +x init.sh && \ + ./init.sh /app verbose x64 || true; \ + fi + +# Install dependencies +# Core dependencies for wrapper modules: +# - storage.py uses urllib (stdlib) to proxy R2 requests through worker.js +# - nosql.py, worker.py, handler.py use stdlib only +# Then install benchmark-specific requirements from requirements.txt +RUN pip install --no-cache-dir --upgrade pip && \ + if [ -f requirements.txt ]; then pip install --no-cache-dir -r requirements.txt; fi + +# Expose port 8080 for container communication +EXPOSE 8080 + +# Set environment variable for port +ENV PORT=8080 + +# Start the HTTP server +CMD ["python", "handler.py"] diff --git a/dockerfiles/cloudflare_nodejs_installer.sh b/dockerfiles/cloudflare_nodejs_installer.sh new file mode 100644 index 000000000..dfe4482b5 --- /dev/null +++ b/dockerfiles/cloudflare_nodejs_installer.sh @@ -0,0 +1,10 @@ +#!/bin/bash +set -e + +cd /mnt/function + +npm install --production +npm install --force esbuild + +node build.js +node postprocess.js diff --git a/dockerfiles/cloudflare_python_installer.sh b/dockerfiles/cloudflare_python_installer.sh new file mode 100644 index 000000000..884afff0b --- /dev/null +++ b/dockerfiles/cloudflare_python_installer.sh @@ -0,0 +1,11 @@ +#!/bin/bash +set -e + +cd /mnt/function + +if [ -f pyproject.toml ]; then + python -c "import tomllib; tomllib.load(open('pyproject.toml','rb'))" + pywrangler --version +fi + +touch .build-validated diff --git a/docs/build.md b/docs/build.md index c6c7f4bab..671a45904 100644 --- a/docs/build.md +++ b/docs/build.md @@ -92,23 +92,70 @@ additive on top of it. ### 1. Declaring variants in a benchmark (`config.json`) A benchmark opts into variant support by using the extended language object syntax in its -`config.json`. The legacy string form (`"python"`) implies only the `"default"` variant. +`config.json`. The legacy string form (`"python"`) implies only the `"default"` variant +and should be kept for languages that have no variant-specific code or configuration. ```json { "timeout": 10, "memory": 128, "languages": [ - { "language": "nodejs", "variants": ["default", "bun", "llrt"] }, - { "language": "python", "variants": ["default", "pypy"] } + "java", + { + "language": "nodejs", + "variants": { + "default": "default", + "bun": "bun", + "llrt": "llrt" + } + }, + { + "language": "python", + "variants": { + "default": "default", + "pypy": "pypy" + } + } ], "modules": [] } ``` +The `variants` field is a **dict** mapping each variant name to the source overlay directory +to apply for that variant (see [section 2](#2-variant-source-code-inside-a-benchmark) below). +The special sentinel value `"default"` means *use the base language directory without any +overlay* — no files are copied from a sub-directory. + SeBS validates this at startup: if you request a variant that is not listed here, the run is rejected with an error. +#### Deployment-mode-split variants + +Some variants behave differently depending on whether the function is deployed as a **code +package** (workers) or as a **container image**. For those cases the overlay directory can be +specified per deployment mode using a nested dict: + +```json +{ + "language": "nodejs", + "variants": { + "default": "default", + "cloudflare": {"workers": "cloudflare", "containers": "default"} + } +} +``` + +The inner dict must use the keys `"workers"` and/or `"containers"`. A missing key means the +variant is not supported in that deployment mode and SeBS will raise an error if it is +requested. A value of `"default"` means no overlay is applied for that mode (the base +language files are used unchanged). + +This is useful when a variant requires platform-specific source changes for one deployment +mode but can reuse the standard implementation for the other. For example, the `cloudflare` +variant of benchmarks that target Cloudflare Workers uses a Pyodide-aware implementation for +the `workers` mode, but falls back to the standard CPython implementation (`"default"`) for +the `containers` mode. + --- ### 2. Variant source code inside a benchmark @@ -120,7 +167,11 @@ language directory of the benchmark: benchmarks//// ``` -Two strategies are supported: +The overlay directory name comes from the value in the `variants` dict (or the inner +`workers`/`containers` value for deployment-mode-split variants). When that value is +`"default"`, no sub-directory is consulted and the base language files are used as-is. + +Two strategies are supported for non-`"default"` overlay directories: #### Patch variant (small targeted changes) diff --git a/docs/platforms.md b/docs/platforms.md index 14aae20c2..c63acf6d4 100644 --- a/docs/platforms.md +++ b/docs/platforms.md @@ -1,6 +1,6 @@ # Platform Configuration -SeBS supports three commercial serverless platforms: AWS Lambda, Azure Functions, and Google Cloud Functions. +SeBS supports four commercial serverless platforms: AWS Lambda, Azure Functions, Google Cloud Functions, and Cloudflare Workers. Furthermore, we support the open source FaaS system OpenWhisk. The file `configs/example.json` contains all parameters that users can change @@ -17,6 +17,7 @@ Supported platforms: * [Amazon Web Services (AWS) Lambda](#aws-lambda) * [Microsoft Azure Functions](#azure-functions) * [Google Cloud (GCP) Functions](#google-cloud-functions) +* [Cloudflare Workers](#cloudflare-workers) * [OpenWhisk](#openwhisk) ## Storage Configuration @@ -334,6 +335,185 @@ The current GCP backend has the following practical limits: * GCP deployments currently reject `arm64`, as arm64 instances are not available for GCR. * C++ packaging is not supported on GCP (but possible to be implemented on containers). +## Cloudflare Workers + +> [!NOTE] +> **Terminology mapping**: SeBS uses the term *function* throughout its CLI and configuration. On Cloudflare, the equivalent unit of deployment is a **Worker**. Wherever SeBS refers to a function (e.g. `--function-name`, `create_function`, `CloudflareWorker`), it refers to a Cloudflare Worker script deployed to `{name}.{account}.workers.dev`. + +Cloudflare offers a free tier for Workers with generous limits for development and testing. To use Cloudflare Workers with SeBS, you need to create a Cloudflare account and obtain API credentials. + +### Credentials + +SeBS supports both authentication methods Cloudflare offers. Both are +functionally equivalent for SeBS: every API call, R2 upload, KV +operation, and `wrangler` invocation works with either. Pick based on +your Cloudflare account, not on SeBS features: + +- **API Token (recommended)**: A scoped credential you mint in the + Cloudflare dashboard. It can be limited to the permissions SeBS needs + and revoked independently, so this is the safest default for most + users. +- **Email + Global API Key (legacy)**: Your account email plus the + Global API Key from the Cloudflare dashboard. SeBS still supports this + path for older setups and accounts that cannot use scoped tokens, but + it grants broad account access and should be handled more carefully. + +Regardless of which method you choose, you also need your account ID +from the Cloudflare dashboard. + +You can pass credentials using environment variables: + +```bash +# Option 1: API Token (recommended) +export CLOUDFLARE_API_TOKEN="your-api-token" +export CLOUDFLARE_ACCOUNT_ID="your-account-id" + +# Option 2: Email + Global API Key (legacy) +export CLOUDFLARE_EMAIL="your-email@example.com" +export CLOUDFLARE_API_KEY="your-global-api-key" +export CLOUDFLARE_ACCOUNT_ID="your-account-id" +``` + +or in the JSON configuration file: + +```json +"deployment": { + "name": "cloudflare", + "cloudflare": { + "credentials": { + "api_token": "your-api-token", + "account_id": "your-account-id" + }, + "resources": { + "resources_id": "unique-resource-id" + } + } +} +``` + +**Note**: The `resources_id` is used to uniquely identify and track resources created by SeBS for a specific deployment. + +### Language Support + +Cloudflare Workers support multiple languages through different deployment methods: + +- **JavaScript/Node.js**: Supported via script-based deployment or container-based deployment using Wrangler CLI +- **Python**: Supported via script-based deployment or container-based deployment using Wrangler CLI + +### CLI Container + +SeBS uses a containerized CLI approach for Cloudflare deployments, eliminating the need to install Node.js, npm, wrangler, pywrangler, or uv on your host system. The CLI container (`spcleth/serverless-benchmarks:manage.cloudflare`) is pulled from Docker Hub on first use and contains all necessary tools. This ensures consistent behavior across platforms and simplifies setup — only Docker is required. + +To build and push an updated `manage.cloudflare` image (developers only): + +```bash +sebs docker build --deployment cloudflare --image-type manage +sebs docker push --deployment cloudflare --image-type manage +``` + +#### Shared singleton and lifecycle + +`CloudflareCLI` is a process-wide singleton: both the script-based (`workers.py`) and container-based (`containers.py`) deployment handlers share a single `manage.cloudflare` Docker container. The first call to `CloudflareCLI.get_instance()` starts the container and registers a shutdown hook via `atexit`; subsequent calls from any handler or thread return the already-running instance. + +This has two consequences: + +- **Thread safety during creation** — `get_instance()` uses a double-checked lock so that when multiple benchmarks run in parallel (e.g. during `sebs regression`), only one thread starts the container while the others wait. +- **Lifecycle** — individual deployment handlers (and `Cloudflare.shutdown()`) drop their local reference to the instance but do not stop the container. The container is stopped exactly once at process exit by the `atexit` hook, regardless of whether SeBS is invoked directly (`sebs benchmark invoke`) or through the regression suite. + +### Deployment Architecture + +SeBS supports two deployment paths for Cloudflare: **script-based Workers** (native Workers runtime) and **container-based Workers** (Cloudflare's managed container runtime, fronted by a Durable-Object-backed Worker). Both paths share the same credentials, R2/KV resources, and HTTP trigger; they differ only in how code is packaged and which Cloudflare runtime executes it. The deployment type is controlled by the benchmark's `container_deployment` flag. + +#### Python modules (`sebs/cloudflare/`) + +| File | Responsibility | +|------|----------------| +| `cloudflare.py` | `Cloudflare(System)` facade. Verifies credentials, enforces `SUPPORTED_BENCHMARKS`, resolves the `workers.dev` URL, and dispatches `package_code`/`create_function`/`update_function` to the correct handler via `_get_deployment_handler(container_deployment)`. | +| `workers.py` | `CloudflareWorkersDeployment` — native script packaging. Node.js is bundled with esbuild via `nodejs/Dockerfile.build`; Python generates a `pyproject.toml` and is validated via `python/Dockerfile.build` (Pyodide resolution happens server-side at deploy time). | +| `containers.py` | `CloudflareContainersDeployment` — container packaging. Copies the per-language `Dockerfile.function` into the code directory, injects the `worker.js` orchestrator (Node-only, required by `@cloudflare/containers`), merges `package.json`, runs `npm install`, and builds a local image as a cache anchor. | +| `cli.py` | `CloudflareCLI` — runs the `manage.cloudflare` Docker container with the Docker socket mounted and exposes `wrangler_deploy`, `pywrangler_deploy`, `docker_build`, `upload_package`. Used by both deployment handlers; `cloudflare.py` never calls `wrangler` directly. | +| `config.py` | `CloudflareCredentials` / `CloudflareConfig` — API token, account ID, R2 keys. | +| `resources.py` | `CloudflareSystemResources` — factories for R2 and KV/Durable Objects. | +| `function.py` | `CloudflareWorker(Function)` — cached function metadata. | +| `triggers.py` | `HTTPTrigger` — invokes the deployed Worker at `https://{name}.{account}.workers.dev`. | +| `r2.py`, `kvstore.py` | Object and NoSQL storage clients. | + +Wrangler templates live alongside the deployment code at `sebs/cloudflare/templates/wrangler-worker.toml` and `sebs/cloudflare/templates/wrangler-container.toml` so they ship with the pip-packaged `sebs`. + +#### Dockerfiles (`dockerfiles/cloudflare/`) + +| File | Purpose | +|------|---------| +| `Dockerfile.manage` | Builds the `manage.cloudflare` CLI image (Node + global `wrangler` + `pywrangler` via `uv` + Docker CLI). Driven by `cli.py`. | +| `nodejs/Dockerfile.build` | Build image for **script-based** Node.js workers. Pulled once per session; benchmark source is bind-mounted to `/mnt/function` at build time and `cloudflare_nodejs_installer.sh` runs `npm install`, `esbuild`, and the benchmark's `build.js`/`postprocess.js` inside it. | +| `python/Dockerfile.build` | Build image for **script-based** Python workers. Pulled once per session; benchmark source is bind-mounted to `/mnt/function` at build time and `cloudflare_python_installer.sh` validates that `pywrangler` accepts the generated `pyproject.toml`. | +| `nodejs/Dockerfile.function` | Runtime image for **container-based** Node.js functions. Parameterized via `ARG BASE_IMAGE` from `config/systems.json`. Copied into the package by `containers.py` and rebuilt by `wrangler deploy`. | +| `python/Dockerfile.function` | Runtime image for **container-based** Python functions. Same parameterization. | + +#### Script-based flow (`container_deployment=false`) + +1. `benchmark.build()` → `CloudflareWorkersDeployment.package_code` copies source files into the package directory. +2. `Benchmark.install_dependencies()` pulls the matching `spcleth/serverless-benchmarks:build.cloudflare..` build image (see [Build Images](#build-images) below), bind-mounts the package directory to `/mnt/function`, and runs `/sebs/installer.sh` (`cloudflare_nodejs_installer.sh` or `cloudflare_python_installer.sh`) inside the container. +3. `Cloudflare.create_function` → `_create_or_update_worker` renders `sebs/cloudflare/templates/wrangler-worker.toml` into the package. +4. `CloudflareCLI.wrangler_deploy` (Node) or `pywrangler_deploy` (Python) deploys via the `manage.cloudflare` container. +5. `HTTPTrigger` is attached using the `workers.dev` URL. + +#### Container-based flow (`container_deployment=true`) + +1. **Local image build** — `benchmark.build()` calls `container_client.build_base_image()` on the `_CloudflareContainerAdapter` in `cloudflare.py`, which delegates to `CloudflareContainersDeployment.package_code`. It copies `{language}/Dockerfile.function` as `Dockerfile`, adds `worker.js`, merges `package.json`, and builds a local Docker image tagged `:` (e.g. `my-benchmark-python-312:20260426-130338`). The correct `BASE_IMAGE` is passed via Docker build args (resolved from `systems.json`). A timestamp tag is used instead of `:latest` because Cloudflare's registry explicitly rejects `:latest` tags. + +2. **Registry push** — `Cloudflare.create_function` → `_create_or_update_worker` calls `CloudflareCLI.containers_push(:)`, which runs `wrangler containers push` inside the `manage.cloudflare` container. Wrangler uploads the locally-built image to Cloudflare's managed registry and returns the full registry URI: `registry.cloudflare.com//:`. + +3. **`wrangler.toml` generation** — `_generate_wrangler_toml` renders `sebs/cloudflare/templates/wrangler-container.toml`. The template defaults to `image = "./Dockerfile"` (a local build path). When a registry URI is available, `containers.py` replaces this field with the registry URI (`config['containers'][0]['image'] = container_uri`), so wrangler points directly at the pre-pushed image and skips rebuilding the Dockerfile entirely. + +4. **Deploy** — `CloudflareCLI.wrangler_deploy` runs `npm install && wrangler deploy` inside the `manage.cloudflare` container. `npm install` materializes `node_modules/@cloudflare/containers` (listed in `package.json`) so that wrangler's bundler can resolve the `worker.js` import. Wrangler then deploys the Worker script and creates the Durable-Object-backed container worker backed by the registry image. + +5. **Rollout and instance readiness wait** — `wrangler deploy` compares the newly pushed registry image digest against the image currently running in the container worker. If the digest has changed, Cloudflare starts a rollout: it pulls the new image, replaces running instances, and sets `active_rollout_id` on the container application record for the duration. SeBS polls `GET /accounts/{id}/containers/applications/{uuid}` every 20 s in two phases: first it waits for `active_rollout_id` to disappear (rollout complete, can take up to 10 minutes for large containers), then it waits for `health.instances.healthy >= max_instances`. The `health.instances` sub-object tracks runtime state and is not formally documented by Cloudflare (derived from observed API responses): `starting` = still booting (image pull + Firecracker init), `healthy` = passed health check and ready to serve, `active` = currently handling a request (always 0 until the first invocation). The readiness threshold is `max_instances`, not the top-level `instances` field — in practice `instances = max_instances + 1` because Cloudflare counts one extra Durable Object coordination instance that never appears as healthy. Only once an instance is confirmed running does SeBS proceed to invoke the benchmark, avoiding the "no Container instance available" Durable Object error that would otherwise occur on cold starts. If wrangler reported "no changes" (digest unchanged), no rollout is started and this wait is skipped entirely. + +6. `HTTPTrigger` is attached using the `workers.dev` URL. + +### Build Images + +Script-based Worker builds use pre-built build images that are pulled once and reused across all benchmarks via bind-mounts — this is the same pattern SeBS uses for other platforms (see [build.md](build.md)). The images are tagged `spcleth/serverless-benchmarks:build.cloudflare..` (e.g. `build.cloudflare.nodejs.18`, `build.cloudflare.python.3.12`) and are available on Docker Hub. + +To build and push updated images yourself (e.g. after modifying a `Dockerfile.build` or an installer script): + +```bash +# Build all Cloudflare toolchain images locally +sebs docker build --deployment cloudflare + +# Push them to Docker Hub (requires push access to the repository) +sebs docker push --deployment cloudflare +``` + +To use a different Docker Hub repository, change `['general']['docker_repository']` in `configs/systems.json`. + +### Trigger Support + +- **HTTP Trigger**: ✅ Fully supported - Workers are automatically accessible at `https://{name}.{account}.workers.dev` +- **Library Trigger**: ❌ Not currently supported + +### Platform Limitations + +- **Cold Start Detection**: Cloudflare does not expose cold start information. All invocations report `is_cold: false` in the metrics. This limitation means cold start metrics are not available for Cloudflare Workers benchmarks. +- **Memory/Timeout Configuration (Workers)**: Managed by Cloudflare (128MB memory, 30s CPU time on free tier) +- **Memory/Timeout Configuration (Containers)**: Managed by Cloudflare, available in different tiers: + + | Instance Type | vCPU | Memory | Disk | + |---------------|------|--------|------| + | lite | 1/16 | 256 MiB | 2 GB | + | basic | 1/4 | 1 GiB | 4 GB | + | standard-1 | 1/2 | 4 GiB | 8 GB | + | standard-2 | 1 | 6 GiB | 12 GB | + | standard-3 | 2 | 8 GiB | 16 GB | + | standard-4 | 4 | 12 GiB | 20 GB | +- **Wall-Clock Timing**: Cloudflare Workers freezes `Date.now()` and `performance.now()` between I/O operations as a timing side-channel mitigation, so the clock does not advance inside pure-compute sections. To record a meaningful wall-clock `compute_time`, the handler issues a throwaway self-fetch (a `HEAD /favicon` request) before sampling the end time. This I/O call unfreezes the timer. See the [Cloudflare security model docs](https://developers.cloudflare.com/workers/reference/security-model/#step-1-disallow-timers-and-multi-threading) for details. +- **Metrics Collection**: Uses response-based per-invocation metrics. During each function invocation, the worker handler measures performance metrics (CPU time, wall time, memory usage) and embeds them directly in the JSON response. SeBS extracts these metrics immediately from each response. When `download_metrics()` is called for postprocessing, it only aggregates the metrics that were already collected during invocations—no additional data is fetched from external services. This approach provides immediate per-invocation granularity without delays. Note that while Cloudflare does expose an Analytics Engine, it only provides aggregated metrics without individual request-level data, making it unsuitable for detailed benchmarking purposes. + +### Storage Configuration + +Cloudflare Workers integrate with Cloudflare R2 for object storage and Durable Objects for NoSQL storage. For detailed storage configuration, see the [storage documentation](storage.md#cloudflare-storage). + ## OpenWhisk SeBS expects users to deploy and configure an OpenWhisk instance. diff --git a/docs/storage.md b/docs/storage.md index 2f6cc54c1..011aa0bb4 100644 --- a/docs/storage.md +++ b/docs/storage.md @@ -129,6 +129,55 @@ healthy: 192.168.0.20:9012 ``` ``` +## Cloudflare Storage + +Cloudflare Workers integrate with cloud-native storage services provided by Cloudflare: + +### R2 Object Storage + +Cloudflare R2 provides S3-compatible object storage for benchmarks that require persistent file storage. SeBS automatically configures R2 buckets for benchmark input and output data. + +**Key Features:** +- S3-compatible API +- No egress fees +- Global edge storage +- Integrated with Workers through bindings + +**Configuration:** +R2 configuration is handled automatically by SeBS when deploying to Cloudflare Workers. The storage resources are defined in your deployment configuration and SeBS manages bucket creation and access. + +**Limitations:** +- Geographic location hints (locationHint) are not currently supported. R2 buckets are created with Cloudflare's automatic location selection, which places data near where it's most frequently accessed. + +### Container Upload Behavior (R2 Proxy) + +For Cloudflare container deployments, benchmark code does not talk to R2 directly. Instead, container wrappers call the Worker proxy endpoints (`/r2/upload`, `/r2/multipart-init`, `/r2/multipart-part`, `/r2/multipart-complete`). + +**Why a proxy (and not a direct storage wrapper like other platforms)?** +On other platforms (AWS/GCP/Azure), the storage wrapper can be an SDK call because the function runtime and object store share a credential/SDK surface. Cloudflare R2 is different: the supported access path for Workers is the R2 **binding** (`env.R2_BUCKET`), which is a runtime object injected only inside the Worker runtime. A Cloudflare container runs in a separate runtime and has no access to that binding, so a container-side "storage wrapper" has nowhere to call. The only direct alternative is R2's S3-compatible HTTPS API, which would require provisioning R2 access keys and shipping them into each container — a second credential model that diverges from how the native Worker benchmarks talk to R2. Routing container storage calls through the parent Worker keeps a single code path and single credential model for both deployment types; the container-side `storage.js` wrapper still exists and still exposes the SeBS storage interface, it just implements those operations by forwarding to the Worker that holds the binding. + +**Upload strategy:** +- Small payloads use a single upload request. +- Large payloads use multipart upload (10 MB threshold, 10 MB part size in current wrappers). +- Node.js container wrapper retries with multipart when single-upload fails with size/body-limit style errors. + +**Object keys and uniqueness:** +- Container wrappers generate unique output keys (suffix based on UUID fragment) before upload. +- This avoids collisions and keeps run-specific output objects distinct in regression and repeated invocations. + +### KVStore for NoSQL + +Cloudflare KV namespaces are used for NoSQL operations required by benchmarks such as CRUD API (130.crud-api). + +**Key Features:** +- Native Workers integration through KV bindings +- Simple key-value interface compatible with SeBS NoSQL wrapper operations +- Global edge distribution for read-heavy access patterns + +**Usage:** +SeBS configures KV namespace bindings automatically for Cloudflare deployments that require NoSQL storage. Benchmark wrappers access KV through the standard SeBS NoSQL interface (insert/update/get/query/delete). + + ## Lifecycle Management By default, storage containers are retained after experiments complete. This allows you to run multiple experiments without redeploying and repopulating storage. diff --git a/docs/workflows.md b/docs/workflows.md new file mode 100644 index 000000000..2033e5dcc --- /dev/null +++ b/docs/workflows.md @@ -0,0 +1,160 @@ +## Workflows + +### Installation + +SeBS makes use of [redis](https://redis.io) in order to make reliable and accurate measurements during the execution of workflows. Ideally, the redis instance should be deployed in the same cloud region such that the write latency is minimal. +Because not all platforms allow connections from a workflow execution to a VPC cache, it proved to be easiest to just deploy a VM and have that machine host redis. Make sure to open port `6379` and admit connections in your VPC accordingly. Redis can be hosted as follows: +```bash +docker run --network=host --name redis -d redis redis-server --save 60 1 --loglevel warning --requirepass {yourpassword} +``` + +### Usage + +To execute a workflow, the host address and password of the redis instance must be given as part of the config file for the respective platform: + +```json +"resources": { + "redis": { + "host": "1.1.1.1", + "password": "yourpassword" + } +} +``` + +Our workflow benchmarks are provided in the benchmarks folder (benchmarks/600.workflows). To execute a given workflow, use the following command, with "http" triggers for Azure and "library" for AWS and GCP: + +``` +./sebs.py benchmark workflow {workflow-name} --config {path/to/config.json} --deployment {platform-name} --verbose {input-size} --trigger {library|http} --repetitions 1 +``` + +### Definition + +Workflows have been adopted by all major cloud providers, but their implementations are significantly different in capabilities, differing not only in APIs and syntax provided, but also in programming models. We define a workflow model based on Petri Nets, and define workflows using a JSON syntax. The general structure of a workflow definition looks like this: + +```json +{ + "root": "first_state", + "states": { + } +} +``` + +`root` defines the initial state to start the workflow from, while `states` holds a dictionary of `(name, state)` tuples. The following state types are supported. + +#### Task + +A task state is the most basic state: it executes a serverless function. + +```json +{ + "type": "task", + "func_name": "compute", + "next": "postprocess_compute" +}, +``` + +`func_name` is the name of the file in the benchmark directory, `next` sets the state with which to follow. + + +#### Map + +A map state takes a list as input and processes each element in parallel using the given functions: + +```json +{ +"type": "map", +"array": "customers", +"root": "shorten", +"next": "list_emails", +"states": { + "shorten": { + "type": "task", + "func_name": "short" + } +} +} +``` + +`array` defines the list to be processed, while `root` defines which of the functions given in `states` should be called first. `func_name` is the name of the file in the benchmark directory. In contrast to a `task`'s function, this one receives only an element of the given array, not the entire running variable. Other fields required from the running variable can be given using the `common_params` entry. + +#### Loop + +The loop phase is similar to map but traverses the given input array sequentially. Thus, loop encodes tasks that cannot be parallelized due to existing dependencies. + +#### Repeat + +A repeat phase executes a function a given number of times. This syntactic sugar eases modeling a chain of tasks. + +```json +{ +"type": "repeat", +"func_name": "process", +"count": 10 +} +``` + +#### Switch + +A switch state makes it possible to encode basic control flow. + +```json +{ + "type": "switch", + "cases": [ + { + "var": "people.number", + "op": "<", + "val": 10, + "next": "few_people" + }, + { + "var": "people.number", + "op": ">=", + "val": 10, + "next": "many_people" + } + ], + "default": "few_people" +} +``` + +This state transcribes to the following Python expression: +```python +if people.number < 10: + few_people() +elif people.number >= 10: + many_people() +else: + few_people() +``` + +#### Parallel + +This higher-level phase corresponds to a parallel routing and executes sub-workflows, consisting of any of the phases, concurrently. The sub-workflows can consist of any of the phases presented. All sub-workflows receive the complete output of the previous phase as input. The outputs of the sub-workflows are merged after all functions have completed execution. + +```json +{ +"type": "parallel", +"parallel_functions": [ + { + "root": "compute", + "states": { + "compute": { + "type": "task", + "func_name": "compute" + } + } + }, + { + "root": "sort", + "states": { + "sort": { + "type": "task", + "func_name": "sort" + } + } + } +], +"next": "frequency_and_overlap" +} +``` diff --git a/pyproject.toml b/pyproject.toml index a092d381f..19ded6c57 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -39,6 +39,9 @@ dependencies = [ "pycurl>=7.43", "click>=7.1.2", "rich", + "patch-ng", + "tomli ; python_version < '3.11'", + "tomli_w", # Storage & Local "minio==5.0.10", @@ -61,6 +64,7 @@ dependencies = [ "google-cloud-logging>=2.0.0", "google-cloud-datastore", "google-cloud-build>=3.35.0", + "google-cloud-workflows", # benchmark verification - process image "Pillow", @@ -106,6 +110,7 @@ packages = [ "sebs.aws", "sebs.azure", "sebs.gcp", + "sebs.cloudflare", "sebs.local", "sebs.openwhisk", "sebs.faas", @@ -149,6 +154,9 @@ sebs = ["py.typed"] "sebs.dockerfiles" = [ "**/*", ] +"sebs.cloudflare" = [ + "templates/*.toml", +] "sebs.tools" = [ "**/*.py", ] diff --git a/requirements.gcp.txt b/requirements.gcp.txt index f1e136777..f9d712b11 100644 --- a/requirements.gcp.txt +++ b/requirements.gcp.txt @@ -6,4 +6,5 @@ google-api-python-client-stubs google-cloud-logging>=2.0.0 google-cloud-datastore google-cloud-build>=3.35.0 +google-cloud-workflows diff --git a/sebs/aws/aws.py b/sebs/aws/aws.py index dd7077d18..2622bbb96 100644 --- a/sebs/aws/aws.py +++ b/sebs/aws/aws.py @@ -170,6 +170,12 @@ def initialize( self.system_config, self.session, self.config, self.docker_client ) + @staticmethod + def format_resource_name(name: str) -> str: + name = name.replace("-", "_") + name = name.replace(".", "_") + return name + def get_lambda_client(self): """ Get or create an AWS Lambda client. @@ -184,6 +190,14 @@ def get_lambda_client(self): ) return self.client + def get_sfn_client(self): + if not hasattr(self, "_sfn_client"): + self._sfn_client = self.session.client( + service_name="stepfunctions", + region_name=self.config.region, + ) + return self._sfn_client + def package_code( self, directory: str, @@ -244,6 +258,18 @@ def package_code( Language.NODEJS: ["handler.js", "package.json", "node_modules"], } + handler_path = os.path.join(directory, CONFIG_FILES[language][0]) + if self.config.redis_host is not None: + from sebs.utils import replace_string_in_file + replace_string_in_file( + handler_path, "{{REDIS_HOST}}", f'"{self.config.redis_host}"' + ) + if self.config.redis_password is not None: + from sebs.utils import replace_string_in_file + replace_string_in_file( + handler_path, "{{REDIS_PASSWORD}}", f'"{self.config.redis_password}"' + ) + if language in [Language.PYTHON, Language.NODEJS]: package_config = CONFIG_FILES[language] function_dir = os.path.join(directory, "function") @@ -435,8 +461,9 @@ def create_function( self.wait_function_active(lambda_function) - # Update environment variables - self.update_function_configuration(lambda_function, code_package) + # Update environment variables (only if input has been processed) + if code_package.has_input_processed: + self.update_function_configuration(lambda_function, code_package) # Add LibraryTrigger to a new function from sebs.aws.triggers import LibraryTrigger @@ -528,7 +555,8 @@ def update_function( self.wait_function_updated(function) self.logging.info(f"Updated code of {name} function. ") # and update config - self.update_function_configuration(function, code_package) + if code_package.has_input_processed: + self.update_function_configuration(function, code_package) def update_function_configuration( self, function: Function, code_package: Benchmark, env_variables: dict = {} @@ -948,6 +976,113 @@ def create_trigger(self, function: Function, trigger_type: Trigger.TriggerType) self.cache_client.update_function(function) return trigger + @staticmethod + def workflow_type() -> "Type[Function]": + from sebs.aws.workflow import SFNWorkflow + + return SFNWorkflow + + def create_workflow(self, code_package: Benchmark, workflow_name: str) -> "Function": + import re + from sebs.aws.workflow import SFNWorkflow + from sebs.aws.generator import SFNGenerator + from sebs.aws.triggers import WorkflowLibraryTrigger + + workflow_name = AWS.format_resource_name(workflow_name) + + definition_path = os.path.join(code_package.benchmark_path, "definition.json") + if not os.path.exists(definition_path): + raise ValueError(f"No workflow definition found for {workflow_name}") + + code_files = list(code_package.get_code_files(include_config=False)) + func_names = [os.path.splitext(os.path.basename(p))[0] for p in code_files] + funcs = [ + self.create_function( + code_package, + workflow_name + "___" + fn, + code_package.system_variant, + None, + ) + for fn in func_names + ] + + gen = SFNGenerator({n: f.arn for (n, f) in zip(func_names, funcs)}) + gen.parse(definition_path) + definition = gen.generate() + + try: + ret = self.get_sfn_client().create_state_machine( + name=workflow_name, + definition=definition, + roleArn=self.config.resources.lambda_role(self.session), + ) + self.logging.info(f"Creating workflow {workflow_name}") + workflow = SFNWorkflow( + workflow_name, + funcs, + code_package.benchmark, + ret["stateMachineArn"], + code_package.hash, + FunctionConfig.from_benchmark(code_package), + ) + except self.get_sfn_client().exceptions.StateMachineAlreadyExists as e: + match = re.search("'([^']*)'", str(e)) + if not match: + raise + arn = match.group()[1:-1] + self.logging.info(f"Workflow {workflow_name} exists on AWS, updating.") + workflow = SFNWorkflow( + workflow_name, funcs, code_package.benchmark, arn, code_package.hash, + FunctionConfig.from_benchmark(code_package), + ) + self._update_workflow_definition(workflow, code_package) + workflow.updated_code = True + + trigger = WorkflowLibraryTrigger(workflow.arn, self) + trigger.logging_handlers = self.logging_handlers + workflow.add_trigger(trigger) + return workflow + + def update_workflow(self, workflow, code_package: Benchmark): + from sebs.aws.workflow import SFNWorkflow + + workflow = cast(SFNWorkflow, workflow) + self._update_workflow_definition(workflow, code_package) + + def _update_workflow_definition(self, workflow, code_package: Benchmark): + from sebs.aws.workflow import SFNWorkflow + from sebs.aws.generator import SFNGenerator + + workflow = cast(SFNWorkflow, workflow) + + definition_path = os.path.join(code_package.benchmark_path, "definition.json") + if not os.path.exists(definition_path): + raise ValueError(f"No workflow definition found for {workflow.name}") + + code_files = list(code_package.get_code_files(include_config=False)) + func_names = [os.path.splitext(os.path.basename(p))[0] for p in code_files] + funcs = [ + self.create_function( + code_package, + workflow.name + "___" + fn, + code_package.system_variant, + None, + ) + for fn in func_names + ] + + gen = SFNGenerator({n: f.arn for (n, f) in zip(func_names, funcs)}) + gen.parse(definition_path) + definition = gen.generate() + + self.get_sfn_client().update_state_machine( + stateMachineArn=workflow.arn, + definition=definition, + roleArn=self.config.resources.lambda_role(self.session), + ) + workflow.functions = funcs + self.logging.info("Published new workflow code") + def _enforce_cold_start(self, function: Function, code_package: Benchmark) -> None: """Enforce cold start for a single function. diff --git a/sebs/aws/config.py b/sebs/aws/config.py index 83230641a..259e0dc67 100644 --- a/sebs/aws/config.py +++ b/sebs/aws/config.py @@ -494,13 +494,19 @@ def lambda_role(self, boto3_session: boto3.session.Session) -> str: "Effect": "Allow", "Principal": {"Service": "lambda.amazonaws.com"}, "Action": "sts:AssumeRole", - } + }, + { + "Effect": "Allow", + "Principal": {"Service": "states.amazonaws.com"}, + "Action": "sts:AssumeRole", + }, ], } role_name = "sebs-lambda-role" attached_policies = [ "arn:aws:iam::aws:policy/AmazonS3FullAccess", "arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole", + "arn:aws:iam::aws:policy/service-role/AWSLambdaRole", ] try: out = iam_client.get_role(RoleName=role_name) @@ -1036,6 +1042,7 @@ def initialize(res: Resources, dct: dict) -> None: for key, value in dct["function-urls"].items(): ret._function_urls[key] = AWSResources.FunctionURL.deserialize(value) + ret._redis = dct.get("redis") ret._use_function_url = dct.get("use-function-url", True) auth_type_str = dct.get("function-url-auth-type", "NONE") ret.function_url_auth_type = FunctionURLAuthType.from_string(auth_type_str) @@ -1121,6 +1128,8 @@ def deserialize(config: dict, cache: Cache, handlers: LoggingHandlers) -> Resour AWSResources.initialize(ret, cached_config["resources"]) ret.logging_handlers = handlers ret.logging.info("Using cached resources for AWS") + if "resources" in config and "redis" in config["resources"]: + ret._redis = config["resources"]["redis"] else: # Check for new config if "resources" in config: @@ -1184,6 +1193,20 @@ def resources(self) -> AWSResources: """ return self._resources + @property + def redis_host(self) -> str | None: + redis_cfg = getattr(self._resources, "_redis", None) + if redis_cfg: + return redis_cfg.get("host") + return None + + @property + def redis_password(self) -> str | None: + redis_cfg = getattr(self._resources, "_redis", None) + if redis_cfg: + return redis_cfg.get("password") + return None + @staticmethod def initialize(cfg: Config, dct: dict) -> None: """Initialize AWS configuration from dictionary. diff --git a/sebs/aws/generator.py b/sebs/aws/generator.py new file mode 100644 index 000000000..28c5f1c5e --- /dev/null +++ b/sebs/aws/generator.py @@ -0,0 +1,153 @@ +from typing import Dict, List, Union, Any +import numbers +import uuid + +from sebs.faas.fsm import Generator, Task, Switch, Map, Repeat, Loop + + +class SFNGenerator(Generator): + def __init__(self, func_arns: Dict[str, str]): + super().__init__() + self._func_arns = func_arns + + def postprocess(self, payloads: List[dict]) -> dict: + def _nameless(p: dict) -> dict: + del p["Name"] + return p + + state_payloads = {p["Name"]: _nameless(p) for p in payloads} + + definition = { + "Comment": "SeBS auto-generated benchmark", + "StartAt": self.root.name, + "States": state_payloads, + } + + return definition + + def encode_task(self, state: Task) -> Union[dict, List[dict]]: + payload: Dict[str, Any] = { + "Name": state.name, + "Type": "Task", + "Resource": self._func_arns[state.func_name] + } + + if state.next: + payload["Next"] = state.next + else: + payload["End"] = True + + if state.failure: + payload["Catch"] = [ + {"ErrorEquals": ["States.ALL"], "Next": state.failure} + ] + + return payload + + def encode_switch(self, state: Switch) -> Union[dict, List[dict]]: + choises = [self._encode_case(c) for c in state.cases] + return { + "Name": state.name, + "Type": "Choice", + "Choices": choises, + "Default": state.default + } + + def _encode_case(self, case: Switch.Case) -> dict: + type = "Numeric" if isinstance(case.val, numbers.Number) else "String" + comp = { + "<": "LessThan", + "<=": "LessThanEquals", + "==": "Equals", + ">=": "GreaterThanEquals", + ">": "GreaterThan", + } + cond = type + comp[case.op] + + return {"Variable": "$." + case.var, cond: case.val, "Next": case.next} + + def encode_map(self, state: Map) -> Union[dict, List[dict]]: + map_func_name = "func_" + str(uuid.uuid4())[:8] + + # state.funcs can be a dict of nested states or a list of function names + if isinstance(state.funcs, dict): + # Get func_name from the first nested task state + first_state = next(iter(state.funcs.values())) + func_name = first_state["func_name"] + else: + func_name = state.funcs[0] + + payload: Dict[str, Any] = { + "Name": state.name, + "Type": "Map", + "ItemsPath": "$." + state.array, + "Iterator": { + "StartAt": map_func_name, + "States": { + map_func_name: { + "Type": "Task", + "Resource": self._func_arns[func_name], + "End": True, + } + }, + }, + } + + if state.common_params: + item_selector: Dict[str, str] = {"array_element.$": "$$.Map.Item.Value"} + for p in state.common_params: + item_selector[f"{p}.$"] = f"$.{p}" + payload["ItemSelector"] = item_selector + + payload["ResultPath"] = "$." + state.array + + if state.next: + payload["Next"] = state.next + else: + payload["End"] = True + + return payload + + def encode_parallel(self, state) -> Union[dict, List[dict]]: + from sebs.faas.fsm import State as FsmState + + branches = [] + for branch in state.branches: + sub_states = {n: FsmState.deserialize(n, s) for n, s in branch.states.items()} + branch_states = {} + for sub_state in sub_states.values(): + obj = self.encode_state(sub_state) + objs = [obj] if isinstance(obj, dict) else obj + for o in objs: + name = o["Name"] + branch_states[name] = {k: v for k, v in o.items() if k != "Name"} + branches.append({"StartAt": branch.root, "States": branch_states}) + + payload: Dict[str, Any] = { + "Name": state.name, + "Type": "Parallel", + "Branches": branches, + # Convert the Parallel output array into a dict keyed by branch root name + # so downstream states can reference results by name (e.g. $.sifting). + "ResultSelector": { + f"{b.root}.$": f"$[{i}]" for i, b in enumerate(state.branches) + }, + "ResultPath": "$", + } + + if state.next: + payload["Next"] = state.next + else: + payload["End"] = True + + return payload + + def encode_loop(self, state: Loop) -> Union[dict, List[dict]]: + map_state = Map(state.name, [state.func_name], state.array, state.name, state.next, None) + payload = self.encode_map(map_state) + payload["MaxConcurrency"] = 1 + payload["ResultSelector"] = dict() + payload["ResultPath"] = "$." + str(uuid.uuid4())[:8] + + return payload + diff --git a/sebs/aws/triggers.py b/sebs/aws/triggers.py index bfc66e31e..b4b0e059a 100644 --- a/sebs/aws/triggers.py +++ b/sebs/aws/triggers.py @@ -14,6 +14,8 @@ import concurrent.futures import datetime import json +import time +import uuid from enum import Enum from typing import Dict, Optional # noqa @@ -206,6 +208,62 @@ def deserialize(obj: dict) -> Trigger: return LibraryTrigger(obj["name"]) +class WorkflowLibraryTrigger(LibraryTrigger): + def sync_invoke(self, payload: dict) -> ExecutionResult: + self.logging.debug(f"Invoke workflow {self.name}") + + request_id = str(uuid.uuid4())[0:8] + sfn_input = {**payload, "__sebs_request_id": request_id} + + client = self._deployment_client.get_sfn_client() + begin = datetime.datetime.now() + ret = client.start_execution(stateMachineArn=self.name, input=json.dumps(sfn_input)) + execution_arn = ret["executionArn"] + + execution_finished = False + while not execution_finished: + execution = client.describe_execution(executionArn=execution_arn) + status = execution["status"] + execution_finished = status != "RUNNING" + + if not execution_finished: + time.sleep(1) + + end = datetime.datetime.now() + aws_result = ExecutionResult.from_times(begin, end) + aws_result.request_id = request_id + + if status == "FAILED": + self.logging.error(f"Invocation of {self.name} failed") + self.logging.error(f"Input: {payload}") + aws_result.stats.failure = True + return aws_result + + if "output" in execution: + output = json.loads(execution["output"]) + aws_result.output = output + + return aws_result + + def async_invoke(self, payload: dict): + raise NotImplementedError("Async invocation is not implemented for workflows") + + @staticmethod + def typename() -> str: + return "AWS.WorkflowLibraryTrigger" + + @staticmethod + def trigger_type() -> Trigger.TriggerType: + return Trigger.TriggerType.LIBRARY + + def serialize(self) -> dict: + return {"type": "Library", "name": self.name} + + @staticmethod + def deserialize(obj: dict) -> "WorkflowLibraryTrigger": + return WorkflowLibraryTrigger(obj["name"]) + + class HTTPTrigger(Trigger): """AWS HTTP trigger for Lambda functions. diff --git a/sebs/aws/workflow.py b/sebs/aws/workflow.py new file mode 100644 index 000000000..3a8d7f831 --- /dev/null +++ b/sebs/aws/workflow.py @@ -0,0 +1,59 @@ +from typing import cast, List + +from sebs.aws.s3 import S3 +from sebs.aws.function import FunctionConfig, LambdaFunction +from sebs.faas.function import Workflow + + +class SFNWorkflow(Workflow): + def __init__( + self, + name: str, + functions: List[LambdaFunction], + benchmark: str, + arn: str, + code_package_hash: str, + cfg: FunctionConfig, + ): + super().__init__(benchmark, name, code_package_hash, cfg) + self.functions = functions + self.arn = arn + + @staticmethod + def typename() -> str: + return "AWS.SFNWorkflow" + + def serialize(self) -> dict: + return { + **super().serialize(), + "functions": [f.serialize() for f in self.functions], + "arn": self.arn, + } + + @staticmethod + def deserialize(cached_config: dict) -> "SFNWorkflow": + from sebs.faas.function import Trigger + from sebs.aws.triggers import WorkflowLibraryTrigger, HTTPTrigger + + funcs = [LambdaFunction.deserialize(f) for f in cached_config["functions"]] + cfg = FunctionConfig.deserialize(cached_config["config"]) + ret = SFNWorkflow( + cached_config["name"], + funcs, + cached_config["benchmark"], + cached_config["arn"], + cached_config["hash"], + cfg, + ) + for trigger in cached_config["triggers"]: + trigger_type = cast( + Trigger, + {"Library": WorkflowLibraryTrigger, "HTTP": HTTPTrigger}.get(trigger["type"]), + ) + assert trigger_type, "Unknown trigger type {}".format(trigger["type"]) + ret.add_trigger(trigger_type.deserialize(trigger)) + return ret + + def code_bucket(self, benchmark: str, storage_client: S3): + self.bucket, idx = storage_client.add_input_bucket(benchmark) + return self.bucket diff --git a/sebs/azure/azure.py b/sebs/azure/azure.py index 6fbb7906e..cb1205ab3 100644 --- a/sebs/azure/azure.py +++ b/sebs/azure/azure.py @@ -32,12 +32,15 @@ """ import datetime +import glob import json import random import re import os +import shlex import shutil import time +import threading import uuid from typing import cast, Dict, List, Optional, Set, Tuple, Type # noqa @@ -46,7 +49,7 @@ from sebs.azure.blob_storage import BlobStorage from sebs.azure.cli import AzureCLI from sebs.azure.cosmosdb import CosmosDB -from sebs.azure.function import AzureFunction +from sebs.azure.function import AzureFunction, AzureWorkflow from sebs.azure.config import AzureConfig, AzureResources from sebs.azure.system_resources import AzureSystemResources from sebs.azure.triggers import AzureTrigger, HTTPTrigger @@ -55,8 +58,8 @@ from sebs.cache import Cache from sebs.config import SeBSConfig from sebs.experiments.config import SystemVariant -from sebs.utils import LoggingHandlers, execute -from sebs.faas.function import Function, FunctionConfig, ExecutionResult +from sebs.utils import LoggingHandlers, execute, replace_string_in_file +from sebs.faas.function import Function, FunctionConfig, ExecutionResult, Workflow from sebs.faas.system import System from sebs.faas.config import Resources from sebs.sebs_types import Language @@ -84,6 +87,14 @@ class Azure(System): # runtime mapping AZURE_RUNTIMES = {"python": "python", "nodejs": "node", "java": "java"} + HIGH_CPU_WORKFLOW_BENCHMARKS = { + "6100.1000-genome", + "6101.1000-genome-individuals", + } + HIGH_CPU_WORKFLOW_PLAN_SKU = "EP3" + HIGH_CPU_WORKFLOW_NAME_SUFFIX = "ep3linux" + FUNCTION_APP_NAME_LIMIT = 60 + _workflow_plan_lock = threading.Lock() @staticmethod def name() -> str: @@ -123,6 +134,15 @@ def function_type() -> Type[Function]: """ return AzureFunction + @staticmethod + def workflow_type() -> Type[Workflow]: + """Get the workflow type for Azure. + + Returns: + AzureWorkflow class type. + """ + return AzureWorkflow + @property def cli_instance(self) -> AzureCLI: """Get Azure CLI instance. @@ -225,11 +245,17 @@ def package_code( """Package function code for Azure Functions deployment. Creates the proper directory structure and configuration files - required for Azure Functions deployment. The structure includes: + required for Azure Functions deployment. For regular functions: - handler/ directory with source files and Azure wrappers - function.json with trigger and binding configuration - - host.json with runtime configuration - - requirements.txt or package.json with dependencies + + For workflows (Durable Functions): + - main/ directory with HTTP trigger + durableClient binding + - run_workflow/ directory with orchestration trigger + - run_subworkflow/ directory with orchestration trigger + - One directory per activity function with activityTrigger binding + + Both include host.json with runtime configuration. Args: directory: Directory containing the function code @@ -238,14 +264,22 @@ def package_code( architecture: Target architecture (currently unused) benchmark: Name of the benchmark is_cached: Whether the package is from cache - system_variant: Selected deployment variant Returns: - Tuple of (directory_path, code_size_bytes, container_uri) + Tuple of (directory_path, code_size_bytes) """ + is_workflow = os.path.exists(os.path.join(directory, "definition.json")) + + if is_workflow: + return self._package_code_workflow(directory, language, benchmark) + else: + return self._package_code_function(directory, language, benchmark) + + def _package_code_function( + self, directory: str, language: Language, benchmark: str + ) -> Tuple[str, float]: + """Package a regular (non-workflow) function for Azure.""" - # In previous step we ran a Docker container which installed packages - # Python packages are in .python_packages because this is expected by Azure EXEC_FILES = { Language.PYTHON: "handler.py", Language.NODEJS: "handler.js", @@ -256,6 +290,12 @@ def package_code( Language.NODEJS: ["package.json", "node_modules"], Language.JAVA: ["lib", "src", "pom.xml", "target", ".mvn", "mvnw", "mvnw.cmd"], } + WORKFLOW_FILES = [ + "main_workflow.py", + "run_workflow.py", + "run_subworkflow.py", + "fsm.py", + ] package_config = CONFIG_FILES[language] handler_dir = os.path.join(directory, "handler") @@ -265,18 +305,23 @@ def package_code( if language == Language.JAVA: lib_dir = os.path.join(directory, "lib") os.makedirs(lib_dir, exist_ok=True) - # Move function.jar to lib directory if os.path.exists(os.path.join(directory, "function.jar")): shutil.move( os.path.join(directory, "function.jar"), os.path.join(lib_dir, "function.jar") ) - # move all files to 'handler' except package config + # move all files to 'handler' except package config and workflow files for f in os.listdir(directory): - if f not in package_config: + if f not in package_config and f not in WORKFLOW_FILES: source_file = os.path.join(directory, f) shutil.move(source_file, handler_dir) + # Remove workflow files that shouldn't be deployed + for wf_file in WORKFLOW_FILES: + wf_path = os.path.join(directory, wf_file) + if os.path.exists(wf_path): + os.remove(wf_path) + # For Java, clean up build artifacts that we don't want to deploy if language == Language.JAVA: for artifact in ["src", "pom.xml", "target", ".mvn", "mvnw", "mvnw.cmd"]: @@ -288,11 +333,7 @@ def package_code( os.remove(artifact_path) # generate function.json - # TODO: extension to other triggers than HTTP if language == Language.JAVA: - # Java Azure Functions - For annotation-based functions, function.json - # should include scriptFile and entryPoint - # The @FunctionName annotation determines the function name default_function_json = { "scriptFile": "../lib/function.jar", "entryPoint": "org.serverlessbench.Handler.handleRequest", @@ -338,6 +379,193 @@ def package_code( execute("zip -qu -r9 {}.zip * .".format(benchmark), shell=True, cwd=directory) return directory, code_size + def _package_code_workflow( + self, directory: str, language: Language, benchmark: str + ) -> Tuple[str, float]: + """Package a Durable Functions workflow for Azure. + + Creates the directory structure expected by Azure Durable Functions: + - main/ — HTTP trigger entry point (durableClient binding) + - run_workflow/ — orchestrator function + - run_subworkflow/ — sub-orchestrator for parallel map tasks + - {activity_name}/ — one directory per activity function + """ + FILES = {"python": "*.py", "nodejs": "*.js"} + CONFIG_FILES = { + "python": ["requirements.txt", ".python_packages"], + "nodejs": ["package.json", "node_modules"], + } + WRAPPER_FILES = { + "python": ["handler.py", "storage.py", "nosql.py", "fsm.py"], + "nodejs": ["handler.js", "storage.js"], + } + file_type = FILES[language] + package_config = CONFIG_FILES[language] + wrapper_files = WRAPPER_FILES[language] + + # Rename main_workflow.py to main.py + main_path = os.path.join(directory, "main_workflow.py") + os.rename(main_path, os.path.join(directory, "main.py")) + + # Copy definition.json into the package + # It's loaded at runtime by the orchestrator + definition_src = None + for parent in [directory]: + candidate = os.path.join(parent, "definition.json") + if os.path.exists(candidate): + definition_src = candidate + break + if definition_src is None: + raise ValueError(f"No workflow definition found in {directory}") + + # Bindings for different function types + main_bindings = [ + { + "name": "req", + "type": "httpTrigger", + "direction": "in", + "authLevel": "anonymous", + "methods": ["get", "post"], + }, + {"name": "starter", "type": "durableClient", "direction": "in"}, + {"name": "$return", "type": "http", "direction": "out"}, + ] + activity_bindings = [ + {"name": "event", "type": "activityTrigger", "direction": "in"}, + ] + orchestrator_bindings = [ + {"name": "context", "type": "orchestrationTrigger", "direction": "in"} + ] + + bindings = { + "main": main_bindings, + "run_workflow": orchestrator_bindings, + "run_subworkflow": orchestrator_bindings, + } + + # Move each .py file into its own directory (Azure Functions convention) + func_dirs = [] + for file_path in glob.glob(os.path.join(directory, file_type)): + file = os.path.basename(file_path) + + if file in package_config or file in wrapper_files: + continue + + name, ext = os.path.splitext(file) + func_dir = os.path.join(directory, name) + func_dirs.append(func_dir) + + os.makedirs(func_dir) + target_file = os.path.join(func_dir, file) + shutil.move(os.path.join(directory, file), target_file) + + # Generate function.json for each function directory + script_file = file if name in bindings else "handler.py" + payload = { + "bindings": bindings.get(name, activity_bindings), + "scriptFile": script_file, + "disabled": False, + } + json.dump( + payload, + open(os.path.join(func_dir, "function.json"), "w"), + indent=2, + ) + + # Copy wrapper files to each activity function directory + for wrapper_file in wrapper_files: + src_path = os.path.join(directory, wrapper_file) + if not os.path.exists(src_path): + continue + for func_dir in func_dirs: + dst_path = os.path.join(func_dir, wrapper_file) + shutil.copyfile(src_path, dst_path) + os.remove(src_path) + + # Create __init__.py in each function directory so relative imports work + for func_dir in func_dirs: + init_path = os.path.join(func_dir, "__init__.py") + if not os.path.exists(init_path): + open(init_path, "w").close() + + # Substitute Redis placeholders in handler and orchestrator files + redis_host = self.config.redis_host + redis_password = self.config.redis_password + redis_host_val = f'"{redis_host}"' if redis_host else "None" + redis_password_val = f'"{redis_password}"' if redis_password else "None" + + for func_dir in func_dirs: + handler_path = os.path.join(func_dir, WRAPPER_FILES[language][0]) + if os.path.exists(handler_path): + replace_string_in_file(handler_path, "{{REDIS_HOST}}", redis_host_val) + replace_string_in_file(handler_path, "{{REDIS_PASSWORD}}", redis_password_val) + + run_workflow_path = os.path.join(directory, "run_workflow", "run_workflow.py") + if os.path.exists(run_workflow_path): + replace_string_in_file(run_workflow_path, "{{REDIS_HOST}}", redis_host_val) + replace_string_in_file(run_workflow_path, "{{REDIS_PASSWORD}}", redis_password_val) + + # generate host.json + host_json = { + "version": "2.0", + "extensionBundle": { + "id": "Microsoft.Azure.Functions.ExtensionBundle", + "version": "[2.*, 3.0.0)", + }, + } + if self._requires_high_cpu_workflow_plan(benchmark): + host_json["extensions"] = { + "durableTask": { + "maxConcurrentActivityFunctions": 1, + } + } + json.dump(host_json, open(os.path.join(directory, "host.json"), "w"), indent=2) + + code_size = Benchmark.directory_size(directory) + execute("zip -qu -r9 {}.zip * .".format(benchmark), shell=True, cwd=directory) + return directory, code_size + + def _wait_for_function_ready(self, url: str, timeout: int = 300, interval: int = 10) -> None: + """Poll the function URL until it returns a non-empty HTTP response after publish. + + Azure Functions can take up to several minutes to become available + after a fresh publish. This method polls until the app responds. + + Args: + url: The function HTTP trigger URL to probe + timeout: Maximum seconds to wait (default 300) + interval: Seconds between probe attempts (default 10) + """ + import pycurl + from io import BytesIO + + self.logging.info(f"Waiting for function app to be ready at {url}...") + deadline = time.time() + timeout + probe_payload = json.dumps({"request_id": "warmup", "payload": {}}) + + while time.time() < deadline: + c = pycurl.Curl() + c.setopt(pycurl.URL, url) + c.setopt(pycurl.POST, 1) + c.setopt(pycurl.HTTPHEADER, ["Content-Type: application/json"]) + c.setopt(pycurl.POSTFIELDS, probe_payload) + c.setopt(pycurl.TIMEOUT, 30) + data = BytesIO() + c.setopt(pycurl.WRITEFUNCTION, data.write) + try: + c.perform() + if len(data.getvalue()) > 0: + self.logging.info("Function app is ready.") + return + except Exception: + pass + finally: + c.close() + self.logging.info(f"Function app not ready yet, retrying in {interval}s...") + time.sleep(interval) + + self.logging.warning(f"Function app did not become ready within {timeout}s, proceeding anyway.") + def _execute_cli_with_retry( self, cmd: str, @@ -482,8 +710,9 @@ def publish_function( self.logging.info("Querying function details to retrieve URL") resource_group = self.config.resources.resource_group(self.cli_instance) + entrypoint = "main" if isinstance(function, AzureWorkflow) else "handler" query_cmd = ( - "az functionapp function show --function-name handler " + f"az functionapp function show --function-name {entrypoint} " f"--name {function.name} --resource-group {resource_group}" ) @@ -538,6 +767,8 @@ def update_function( container_dest = self._mount_function_code(code_package) function_url = self.publish_function(function, code_package, container_dest, True) + self._wait_for_function_ready(function_url) + # Avoid duplication of HTTP trigger found_trigger = False for trigger in function.triggers_all(): @@ -545,7 +776,6 @@ def update_function( if isinstance(trigger, HTTPTrigger): found_trigger = True trigger.url = function_url - break if not found_trigger: trigger = HTTPTrigger( @@ -633,7 +863,7 @@ def update_envs( try: env_string = "" for k, v in envs.items(): - env_string += f" {k}={v}" + env_string += f" {shlex.quote(f'{k}={v}')}" self.logging.info(f"Exporting environment variables for function {function.name}") self.cli_instance.execute( @@ -641,13 +871,14 @@ def update_envs( f" --resource-group {resource_group} " f" --settings {env_string} " ) - - # if we don't do that, next invocation might still see old values - # Disabled since we swapped the order - we first update envs, then we publish. - # self.logging.info( - # "Sleeping for 10 seconds - Azure needs more time to propagate changes. " - # "Otherwise, functions might not see new variables and fail unexpectedly." - # ) + self.logging.info( + f"Restarting function {function.name} to apply environment variables" + ) + self.cli_instance.execute( + f"az functionapp restart --name {function.name} " + f" --resource-group {resource_group} " + ) + time.sleep(10) except RuntimeError as e: self.logging.error("Failed to set environment variable!") @@ -747,6 +978,21 @@ def default_function_name( .replace(".", "-") .replace("_", "-") ) + if self._requires_high_cpu_workflow_plan(code_package.benchmark): + func_name = f"{func_name}-{self.HIGH_CPU_WORKFLOW_NAME_SUFFIX}" + if len(func_name) > self.FUNCTION_APP_NAME_LIMIT: + benchmark_id = code_package.benchmark.split(".")[0] + func_name = ( + "sebs-{}-{}-{}-{}-{}".format( + self.config.resources.resources_id, + benchmark_id, + code_package.language_name, + code_package.language_version, + self.HIGH_CPU_WORKFLOW_NAME_SUFFIX, + ) + .replace(".", "-") + .replace("_", "-") + ) return func_name def create_function( @@ -878,6 +1124,194 @@ def cached_function(self, function: Function) -> None: azure_trigger.logging_handlers = self.logging_handlers azure_trigger.data_storage_account = data_storage_account + def _requires_high_cpu_workflow_plan(self, benchmark: str) -> bool: + return benchmark in self.HIGH_CPU_WORKFLOW_BENCHMARKS + + def _high_cpu_workflow_plan_name(self) -> str: + sku = self.HIGH_CPU_WORKFLOW_PLAN_SKU.lower() + return f"sebs-{self.config.resources.resources_id}-workflow-{sku}" + + def _ensure_high_cpu_workflow_plan(self, resource_group: str, region: str) -> str: + plan_name = self._high_cpu_workflow_plan_name() + + with self._workflow_plan_lock: + try: + self.cli_instance.execute( + ( + "az functionapp plan show " + f"--resource-group {resource_group} --name {plan_name}" + ) + ) + return plan_name + except RuntimeError: + pass + + self.logging.info( + f"Creating Azure Functions Premium plan {plan_name} " + f"({self.HIGH_CPU_WORKFLOW_PLAN_SKU}) for high-CPU workflows" + ) + try: + self.cli_instance.execute( + ( + "az functionapp plan create " + f"--resource-group {resource_group} " + f"--name {plan_name} " + f"--location {region} " + f"--sku {self.HIGH_CPU_WORKFLOW_PLAN_SKU} " + "--is-linux " + "--number-of-workers 1" + ) + ) + except RuntimeError as e: + if "already exists" not in str(e).lower(): + raise e from None + + return plan_name + + def _ensure_function_app_plan( + self, function_name: str, resource_group: str, plan_name: str + ) -> None: + ret = self.cli_instance.execute( + ( + "az functionapp show " + f"--resource-group {resource_group} " + f"--name {function_name}" + ) + ) + app = json.loads(ret.decode("utf-8")) + current_plan_id = ( + app.get("serverFarmId") + or app.get("appServicePlanId") + or app.get("properties", {}).get("serverFarmId") + or "" + ).lower() + if current_plan_id.endswith(f"/serverfarms/{plan_name.lower()}"): + return + + raise RuntimeError( + f"Workflow app {function_name} is on plan {current_plan_id}, " + f"expected {plan_name}. Azure does not support migrating Linux " + "Consumption function apps to Premium; redeploy with a new app name." + ) + + def create_workflow(self, code_package: Benchmark, workflow_name: str) -> AzureWorkflow: + """Create a new Azure Durable Functions workflow. + + Deploys the workflow as a single Function App containing the + orchestrator, sub-orchestrator, and all activity functions. + + Args: + code_package: Benchmark code package with workflow definition + workflow_name: Name for the workflow Function App + + Returns: + AzureWorkflow instance representing the deployed workflow. + """ + language = code_package.language_name + language_runtime = self._normalize_runtime_version(language, code_package.language_version) + language_runtime = str(language_runtime) + resource_group = self.config.resources.resource_group(self.cli_instance) + region = self.config.region + function_cfg = FunctionConfig.from_benchmark(code_package) + + config = { + "resource_group": resource_group, + "func_name": workflow_name, + "region": region, + "runtime": self.AZURE_RUNTIMES[language], + "runtime_version": language_runtime, + } + high_cpu_plan_name: Optional[str] = None + if self._requires_high_cpu_workflow_plan(code_package.benchmark): + high_cpu_plan_name = self._ensure_high_cpu_workflow_plan(resource_group, region) + config["plan_args"] = f"--plan {high_cpu_plan_name}" + config["os_args"] = "" + else: + config["plan_args"] = "--consumption-plan-location {region}".format(**config) + config["os_args"] = "--os-type Linux" + + # Check if function app already exists + function_storage_account: Optional[AzureResources.Storage] = None + try: + ret = self.cli_instance.execute( + ( + " az functionapp config appsettings list " + " --resource-group {resource_group} " + " --name {func_name} " + ).format(**config) + ) + except RuntimeError: + ret = None + + if ret is not None: + for setting in json.loads(ret.decode()): + if setting["name"] == "AzureWebJobsStorage": + connection_string = setting["value"] + elems = [z for y in connection_string.split(";") for z in y.split("=")] + account_name = elems[elems.index("AccountName") + 1] + function_storage_account = AzureResources.Storage.from_cache( + account_name, connection_string + ) + assert function_storage_account is not None + self.logging.info("Azure: Selected existing workflow app {}".format(workflow_name)) + if high_cpu_plan_name: + self._ensure_function_app_plan(workflow_name, resource_group, high_cpu_plan_name) + + if function_storage_account is None: + function_storage_account = self.config.resources.add_storage_account(self.cli_instance) + config["storage_account"] = function_storage_account.account_name + while True: + try: + self.cli_instance.execute( + ( + " az functionapp create --resource-group {resource_group} " + " {os_args} {plan_args} " + " --runtime {runtime} --runtime-version {runtime_version} " + " --name {func_name} --storage-account {storage_account}" + " --functions-version 4 " + ).format(**config) + ) + self.logging.info("Azure: Created workflow app {}".format(workflow_name)) + break + except RuntimeError as e: + if "another operation is in progress" in str(e): + self.logging.info( + f"Repeat {workflow_name} creation, another operation in progress" + ) + else: + raise e from None + + workflow = AzureWorkflow( + name=workflow_name, + benchmark=code_package.benchmark, + code_hash=code_package.hash, + function_storage=function_storage_account, + cfg=function_cfg, + ) + + self.update_function(workflow, code_package, code_package.system_variant, None) + + self.cache_client.add_function( + deployment_name=self.name(), + language_name=language, + code_package=code_package, + function=workflow, + ) + return workflow + + def update_workflow(self, workflow: Workflow, code_package: Benchmark) -> None: + """Update an existing Azure Durable Functions workflow. + + Args: + workflow: Workflow instance to update + code_package: New benchmark code package + """ + if self._requires_high_cpu_workflow_plan(code_package.benchmark): + resource_group = self.config.resources.resource_group(self.cli_instance) + plan_name = self._ensure_high_cpu_workflow_plan(resource_group, self.config.region) + self._ensure_function_app_plan(workflow.name, resource_group, plan_name) + self.update_function(workflow, code_package, code_package.system_variant, None) + def download_metrics( self, function_name: str, @@ -1000,21 +1434,43 @@ def enforce_cold_start(self, functions: List[Function], code_package: Benchmark) self.cold_start_counter += 1 for func in functions: self._enforce_cold_start(func, code_package) - import time - time.sleep(20) def create_trigger(self, function: Function, trigger_type: Trigger.TriggerType) -> Trigger: """Create trigger for Azure Function. - Currently not implemented as HTTP triggers are automatically - created for each function during deployment. + HTTP triggers are automatically created during deployment. + For workflows, LIBRARY trigger requests are satisfied by returning + the existing HTTP trigger, since Azure Durable Functions uses HTTP. Args: function: Function to create trigger for trigger_type: Type of trigger to create + Returns: + The HTTP trigger for this function. + Raises: - NotImplementedError: Trigger creation is not supported. + NotImplementedError: If no HTTP trigger exists on the function. """ + from sebs.azure.function import AzureWorkflow + from sebs.azure.triggers import HTTPTrigger, WorkflowHTTPTrigger + + http_triggers = function.triggers(Trigger.TriggerType.HTTP) + if trigger_type == Trigger.TriggerType.LIBRARY and isinstance(function, AzureWorkflow): + library_triggers = function.triggers(Trigger.TriggerType.LIBRARY) + if library_triggers: + return library_triggers[0] + if http_triggers: + trigger = WorkflowHTTPTrigger( + http_triggers[0].url, + self.config.resources.data_storage_account(self.cli_instance), + ) + trigger.logging_handlers = self.logging_handlers + function.add_trigger(trigger) + self.cache_client.update_function(function) + return trigger + + if trigger_type == Trigger.TriggerType.HTTP and http_triggers: + return http_triggers[0] raise NotImplementedError() diff --git a/sebs/azure/config.py b/sebs/azure/config.py index 2c5fb5e60..54284c275 100644 --- a/sebs/azure/config.py +++ b/sebs/azure/config.py @@ -345,6 +345,21 @@ def __init__( self._storage_accounts = storage_accounts or [] self._data_storage_account = data_storage_account self._cosmosdb_account = cosmosdb_account + self._redis: Optional[Dict] = None + + @property + def redis_host(self) -> Optional[str]: + """Get Redis host for workflow measurements.""" + if self._redis: + return self._redis.get("host") + return None + + @property + def redis_password(self) -> Optional[str]: + """Get Redis password for workflow measurements.""" + if self._redis: + return self._redis.get("password") + return None def set_region(self, region: str) -> None: """Set the Azure region for resource allocation. @@ -698,7 +713,7 @@ def initialize(res: Resources, dct: dict) -> None: ret = cast(AzureResources, res) super(AzureResources, AzureResources).initialize(ret, dct) - ret._resource_group = dct["resource_group"] + ret._resource_group = dct.get("resource_group") if "storage_accounts" in dct: ret._storage_accounts = [ AzureResources.Storage.deserialize(x) for x in dct["storage_accounts"] @@ -714,6 +729,8 @@ def initialize(res: Resources, dct: dict) -> None: if "cosmosdb_account" in dct: ret._cosmosdb_account = CosmosDBAccount.deserialize(dct["cosmosdb_account"]) + ret._redis = dct.get("redis") + def serialize(self) -> dict: """Serialize resources to dictionary. @@ -804,6 +821,16 @@ def resources(self) -> AzureResources: """ return self._resources + @property + def redis_host(self) -> Optional[str]: + """Get Redis host for workflow measurements.""" + return self._resources.redis_host + + @property + def redis_password(self) -> Optional[str]: + """Get Redis password for workflow measurements.""" + return self._resources.redis_password + @staticmethod def initialize(cfg: Config, dct: dict) -> None: """Initialize configuration from dictionary data. diff --git a/sebs/azure/function.py b/sebs/azure/function.py index feaf9b922..449bc79f2 100644 --- a/sebs/azure/function.py +++ b/sebs/azure/function.py @@ -6,7 +6,7 @@ """ from sebs.azure.config import AzureResources -from sebs.faas.function import Function, FunctionConfig +from sebs.faas.function import Function, FunctionConfig, Workflow class AzureFunction(Function): @@ -78,3 +78,57 @@ def deserialize(cached_config: dict) -> Function: assert trigger_type, "Unknown trigger type {}".format(trigger["type"]) ret.add_trigger(trigger_type.deserialize(trigger)) return ret + + +class AzureWorkflow(Workflow): + """Azure Durable Functions workflow implementation.""" + + def __init__( + self, + name: str, + benchmark: str, + code_hash: str, + function_storage: AzureResources.Storage, + cfg: FunctionConfig, + ) -> None: + """Initialize Azure Workflow. + + Args: + name: Name of the Azure Function App hosting the workflow + benchmark: Name of the benchmark this workflow implements + code_hash: Hash of the workflow code for caching + function_storage: Azure Storage account for function code + cfg: Function configuration with memory, timeout, etc. + """ + super().__init__(benchmark, name, code_hash, cfg) + self.function_storage = function_storage + + def serialize(self) -> dict: + """Serialize workflow to dictionary.""" + return { + **super().serialize(), + "function_storage": self.function_storage.serialize(), + } + + @staticmethod + def deserialize(cached_config: dict) -> "AzureWorkflow": + """Deserialize workflow from cached configuration.""" + from sebs.azure.triggers import HTTPTrigger, WorkflowHTTPTrigger + + cfg = FunctionConfig.deserialize(cached_config["config"]) + ret = AzureWorkflow( + cached_config["name"], + cached_config["benchmark"], + cached_config["hash"], + AzureResources.Storage.deserialize(cached_config["function_storage"]), + cfg, + ) + for trigger in cached_config["triggers"]: + trigger_type = { + "HTTP": HTTPTrigger, + "WorkflowHTTP": WorkflowHTTPTrigger, + WorkflowHTTPTrigger.typename(): WorkflowHTTPTrigger, + }.get(trigger["type"]) + assert trigger_type, "Unknown trigger type {}".format(trigger["type"]) + ret.add_trigger(trigger_type.deserialize(trigger)) + return ret diff --git a/sebs/azure/triggers.py b/sebs/azure/triggers.py index 85b4bafe2..6991afe33 100644 --- a/sebs/azure/triggers.py +++ b/sebs/azure/triggers.py @@ -23,6 +23,10 @@ """ import concurrent.futures +import json +import time +from datetime import datetime +from io import BytesIO from typing import Any, Dict, Optional # noqa from sebs.azure.config import AzureResources @@ -152,3 +156,134 @@ def deserialize(obj: dict) -> Trigger: HTTPTrigger instance with restored configuration. """ return HTTPTrigger(obj["url"]) + + +class WorkflowHTTPTrigger(HTTPTrigger): + """HTTP-backed trigger for Azure Durable Function workflows. + + Azure starts Durable Functions workflows through an HTTP endpoint, but SeBS + treats workflows as library triggers and validates the workflow result + directly. This trigger keeps the Azure HTTP transport while exposing the + same result shape as AWS Step Functions and Cloudflare Workflows. + """ + + @staticmethod + def typename() -> str: + """Return the canonical type name for this trigger class.""" + return "Azure.WorkflowHTTPTrigger" + + @staticmethod + def trigger_type() -> Trigger.TriggerType: + """Get the trigger type.""" + return Trigger.TriggerType.LIBRARY + + def sync_invoke(self, payload: dict) -> ExecutionResult: + """Synchronously invoke an Azure Durable Function workflow.""" + begin = datetime.now() + status_code, raw, conn_time, receive_time = self._http_post_json( + self.url, payload, timeout=300 + ) + envelope = self._parse_json_response(raw, self.url) + + if status_code not in [200, 202]: + self.logging.error(f"Invocation on URL {self.url} failed!") + self.logging.error(f"Output: {envelope}") + raise RuntimeError(f"Failed invocation of function! Output: {envelope}") + + workflow_result = envelope.get("result") + if isinstance(workflow_result, dict) and "statusQueryGetUri" in workflow_result: + workflow_result = self._poll_workflow_status( + workflow_result["statusQueryGetUri"], begin + ) + + end = datetime.now() + result = ExecutionResult.from_times(begin, end) + result.times.http_startup = conn_time + result.times.http_first_byte_return = receive_time + result.request_id = envelope.get("request_id", "") + + parsed_output = dict(envelope) + parsed_output["result"] = workflow_result + parsed_output["end"] = f"{end.timestamp():.6f}" + result.parse_benchmark_output(parsed_output) + result.output = workflow_result + return result + + def _http_post_json(self, url: str, payload: dict, timeout: int) -> tuple[int, bytes, float, float]: + """POST JSON and return status, body, connection time, and first-byte time.""" + import pycurl + + c = pycurl.Curl() + c.setopt(pycurl.HTTPHEADER, ["Content-Type: application/json"]) + c.setopt(pycurl.POST, 1) + c.setopt(pycurl.URL, url) + c.setopt(pycurl.POSTFIELDS, json.dumps(payload)) + c.setopt(pycurl.TIMEOUT, timeout) + data = BytesIO() + c.setopt(pycurl.WRITEFUNCTION, data.write) + c.perform() + status_code = c.getinfo(pycurl.RESPONSE_CODE) + conn_time = c.getinfo(pycurl.PRETRANSFER_TIME) + receive_time = c.getinfo(pycurl.STARTTRANSFER_TIME) + c.close() + return status_code, data.getvalue(), conn_time, receive_time + + def _http_get(self, url: str, timeout: int) -> tuple[int, bytes]: + """GET a URL and return status and body.""" + import pycurl + + c = pycurl.Curl() + c.setopt(pycurl.URL, url) + c.setopt(pycurl.TIMEOUT, timeout) + data = BytesIO() + c.setopt(pycurl.WRITEFUNCTION, data.write) + c.perform() + status_code = c.getinfo(pycurl.RESPONSE_CODE) + c.close() + return status_code, data.getvalue() + + def _parse_json_response(self, raw: bytes, url: str) -> dict: + """Parse a JSON response or raise a useful invocation error.""" + try: + return json.loads(raw) + except json.JSONDecodeError: + text = raw.decode(errors="replace") + self.logging.error(f"Invocation on URL {url} failed!") + self.logging.error(f"Output: {text if text else 'No output provided!'}") + raise RuntimeError(f"Failed invocation of function! Output: {text}") from None + + def _poll_workflow_status(self, status_url: str, begin: datetime) -> Any: + """Poll Azure Durable Functions status until the workflow completes.""" + max_poll_time = 7200 + poll_interval = 5 + + while (datetime.now() - begin).total_seconds() < max_poll_time: + status_code, raw = self._http_get(status_url, timeout=60) + status = self._parse_json_response(raw, status_url) + + if status_code not in [200, 202]: + self.logging.warning( + f"Workflow status poll failed with status={status_code}: {status}" + ) + time.sleep(poll_interval) + continue + + runtime_status = status.get("runtimeStatus") + if runtime_status == "Completed": + return status.get("output") + if runtime_status in ["Failed", "Terminated", "Canceled"]: + self.logging.error(f"Workflow execution failed: {status}") + raise RuntimeError(f"Workflow execution failed: {status}") + + time.sleep(poll_interval) + + raise RuntimeError(f"Workflow did not complete within {max_poll_time}s") + + def serialize(self) -> dict: + """Serialize trigger to dictionary.""" + return {"type": self.typename(), "url": self.url} + + @staticmethod + def deserialize(obj: dict) -> Trigger: + """Deserialize trigger from dictionary.""" + return WorkflowHTTPTrigger(obj["url"]) diff --git a/sebs/benchmark.py b/sebs/benchmark.py index 32cf62b9f..2d245b7d0 100644 --- a/sebs/benchmark.py +++ b/sebs/benchmark.py @@ -40,65 +40,94 @@ class LanguageSpec: """ Represents a language with its supported variants for a benchmark. - Parses the config language settings, supports both the legacy format - (e.g. "python") and the new dict format: + Parses the config.json ``languages`` entries. Supports three formats: - {"language": "nodejs", "variants": ["default", "bun", "llrt"]} + * Legacy string: ``"python"`` + → treated as ``{"default": "default"}`` - The legacy format is treated as having just the "default" variant. - """ + * New dict with simple (non-deployment-split) variants:: - def __init__(self, language: "Language", variants: List[str]): - """Initialize a language specification. + {"language": "nodejs", "variants": {"default": "default", "bun": "bun"}} - Args: - language: The programming language - variants: List of supported runtime variants for this language - """ + The value for each key is either the literal sentinel ``"default"`` + (meaning: use the base language directory, no overlay) or a subdirectory + name to use as an overlay (e.g. ``"cloudflare"`` → ``nodejs/cloudflare/``). + + * New dict with deployment-mode-split variants:: + + { + "language": "nodejs", + "variants": { + "default": "default", + "cloudflare": {"workers": "cloudflare", "containers": "default"} + } + } + + When the value is itself a dict, the keys are deployment modes + (``"workers"`` / ``"containers"``) and the values follow the same + sentinel / subdirectory convention. A missing mode key means the + benchmark is not supported in that deployment mode. + """ + + def __init__(self, language: "Language", variants: Dict[str, Any]): + """Store the language and its variant-to-directory mapping.""" self._language = language self._variants = variants @property def language(self) -> "Language": - """Get the programming language. - - Returns: - Language: The programming language - """ + """The programming language this spec applies to.""" return self._language @property - def variants(self) -> List[str]: - """Get the list of supported runtime variants. - - Returns: - List[str]: List of variant names (e.g., ["default", "pypy"]) - """ + def variants(self) -> Dict[str, Any]: + """Variant map: variant name → directory name or deployment-mode dict.""" return self._variants - @staticmethod - def deserialize(val) -> LanguageSpec: - """Deserialize a language specification from config. + def resolve_dir(self, variant: str, container_deployment: bool) -> str: + """Return the source subdirectory name for *variant* + deployment mode. - Args: - val: Either a string (legacy format) or dict with language and variants + Returns ``"default"`` (sentinel) when the base language directory should + be used without any overlay. Returns a subdirectory name (e.g. + ``"cloudflare"``) when an overlay should be applied from that subdir. - Returns: - LanguageSpec: Deserialized language specification + Raises ``RuntimeError`` when the variant or deployment mode is not + supported. """ + entry = self._variants.get(variant) + if entry is None: + raise RuntimeError( + f"Variant '{variant}' not declared for language {self._language.value}" + ) + if isinstance(entry, dict): + mode = "containers" if container_deployment else "workers" + dir_name = entry.get(mode) + if dir_name is None: + raise RuntimeError( + f"Variant '{variant}' does not support deployment mode '{mode}' " + f"for language {self._language.value}" + ) + return dir_name + return entry # str: "default" or a subdir name + + @staticmethod + def deserialize(val) -> "LanguageSpec": + """Build a LanguageSpec from a config.json language entry (string or dict).""" if isinstance(val, str): - return LanguageSpec(Language.deserialize(val), ["default"]) - return LanguageSpec( - Language.deserialize(val["language"]), - val.get("variants", ["default"]), - ) + # Legacy: "python" → only the default variant + return LanguageSpec(Language.deserialize(val), {"default": "default"}) + variants = val.get("variants") + if variants is None: + variants = {"default": "default"} + elif isinstance(variants, list): + # Old list format: ["default", "cloudflare"] + # Each name maps to itself ("default" stays as the sentinel). + variants = {v: v for v in variants} + # else: already the new dict format + return LanguageSpec(Language.deserialize(val["language"]), variants) def serialize(self) -> dict: - """Serialize the language specification to a dictionary. - - Returns: - dict: Dictionary with language and variants keys - """ + """Return a serializable dict of the language and its variant mapping.""" return { "language": self._language.value, "variants": self._variants, @@ -218,13 +247,20 @@ def supported_variants(self, language: Language) -> List[str]: or [] if the language has no implementation in this benchmark.""" for spec in self._language_specs: if spec.language == language: - return spec.variants + return list(spec.variants.keys()) return [] def supports(self, language: Language, variant: str) -> bool: """Return True when language + variant combination is declared in config.json.""" return variant in self.supported_variants(language) + def get_language_spec(self, language: Language) -> "LanguageSpec": + """Return the LanguageSpec for *language*, raising if not found.""" + for spec in self._language_specs: + if spec.language == language: + return spec + raise RuntimeError(f"Language {language.value} not declared in benchmark config") + @staticmethod def deserialize(json_object: dict) -> BenchmarkConfig: """ @@ -471,6 +507,38 @@ def language_variant(self) -> str: """ return self._language_variant + def select_variant(self, variant: str) -> None: + """Switch the active language variant and refresh the cache state. + + Should be called before build() whenever the deployment platform + needs to override the variant that was set from the experiment config. + Re-queries the cache with the updated variant key and re-applies + the update_code flag if it was set. + + Args: + variant: New variant name (e.g. "cloudflare"). + + Raises: + RuntimeError: If the variant is not declared for this benchmark. + """ + if not self.benchmark_config.supports(self.language, variant): + raise RuntimeError( + f"Variant '{variant}' is not declared for benchmark " + f"{self.benchmark} language {self.language_name}" + ) + self._language_variant = variant + self._output_dir = os.path.join( + self._output_dir_base, + self._language.value, + self._language_variant, + self._language_version, + self._architecture, + "container" if self._system_variant.is_container else "package", + ) + self.query_cache() + if self._experiment_config.update_code: + self._is_cached_valid = False + @property def language_version(self) -> str: """ @@ -491,6 +559,11 @@ def has_input_processed(self) -> bool: """ return self._input_processed + @property + def last_input_config(self) -> Optional[Dict[str, Any]]: + """Return the most recently prepared benchmark input, if available.""" + return self._last_input_config + @property def uses_storage(self) -> bool: """ @@ -544,8 +617,13 @@ def hash(self) -> str: """ path = os.path.join(self.benchmark_path, self.language_name) self._hash_value = Benchmark.hash_directory( - path, self._deployment_name, self.language, self._language_variant + path, + self._deployment_name, + self.language, + self._language_variant, + container_deployment=self._system_variant.is_container, ) + assert self._hash_value is not None return self._hash_value @hash.setter # noqa: A003 @@ -560,6 +638,20 @@ def hash(self, val: str): """ self._hash_value = val + def get_code_files(self, include_config=True): + FILES = { + "python": ["*.py"], + "nodejs": ["*.js"], + } + if include_config: + FILES["python"] += ["requirements.txt*", "*.json"] + FILES["nodejs"] += ["package.json", "*.json"] + + path = os.path.join(self.benchmark_path, self.language_name) + for file_type in FILES.get(self.language_name, []): + for f in glob.glob(os.path.join(path, file_type)): + yield f + def __init__( self, benchmark: str, @@ -625,9 +717,9 @@ def __init__( self._docker_client = docker_client self._system_config = system_config self._code_location: Optional[str] = None + self._output_dir_base = os.path.join(output_dir, f"{benchmark}_code") self._output_dir = os.path.join( - output_dir, - f"{benchmark}_code", + self._output_dir_base, self._language.value, self._language_variant, self._language_version, @@ -652,18 +744,23 @@ def __init__( # Try to ensure benchmarks-data exists ensure_benchmarks_data(self.logging) - # Load input module - self._benchmark_data_path = find_benchmark(self._benchmark, "benchmarks-data") + # Load input module — fall back to output dir for benchmarks without data files + self._benchmark_data_path = find_benchmark(self._benchmark, "benchmarks-data") or self._output_dir self._benchmark_input_module = load_benchmark_input(self._benchmark_path) # Check if input has been processed self._input_processed: bool = False + self._last_input_config: Optional[Dict[str, Any]] = None self._uses_storage: bool = False self._uses_nosql: bool = False @staticmethod def hash_directory( - directory: str, deployment: str, language: Language, variant: str = "default" + directory: str, + deployment: str, + language: Language, + variant: str = "default", + container_deployment: bool = False, ): """ Compute MD5 hash of an entire directory. @@ -730,6 +827,18 @@ def hash_directory( else: with open(f, "rb") as opened_file: hash_sum.update(opened_file.read()) + # For Cloudflare Python containers, also hash the nodejs/container worker.js. + # worker.js is shared between Node.js and Python container builds + # (@cloudflare/containers is Node.js-only), so containers.py copies it from + # nodejs/container/ into every container build directory regardless of language. + # Python's wrapper glob would otherwise miss it and stale builds wouldn't invalidate. + if deployment == "cloudflare" and language == Language.PYTHON and container_deployment: + nodejs_worker = get_resource_path( + "benchmarks", "wrappers", "cloudflare", "nodejs", "container", "worker.js" + ) + if os.path.isfile(str(nodejs_worker)): + with open(str(nodejs_worker), "rb") as worker_file: + hash_sum.update(worker_file.read()) return hash_sum.hexdigest() def serialize(self) -> dict: @@ -784,7 +893,7 @@ def copy_code(self, output_dir: str) -> None: output_dir: Destination directory for copied files """ FILES = { - Language.PYTHON: ["*.py", "requirements.txt*"], + Language.PYTHON: ["*.py", "*.c", "*.so", "requirements.txt*", "package.sh"], Language.NODEJS: ["*.js", "package.json"], Language.JAVA: [], Language.CPP: ["*.cpp", "*.hpp", "dependencies.json"], @@ -804,52 +913,59 @@ def copy_code(self, output_dir: str) -> None: shutil.copy2(nodejs_package_json, os.path.join(output_dir, "package.json")) if self._language_variant != "default": - variant_dir = os.path.join(path, self._language_variant) - if not os.path.isdir(variant_dir): - raise RuntimeError( - "Variant directory not found for benchmark {} language {} " - "variant {}: {}".format( - self.benchmark, self.language_name, self._language_variant, variant_dir - ) - ) + lang_spec = self.benchmark_config.get_language_spec(self.language) + overlay_dir_name = lang_spec.resolve_dir( + self._language_variant, self._system_variant.is_container + ) - patch_file = os.path.join(variant_dir, "patch.diff") - if os.path.exists(patch_file): - # Patch-based variant: a unified diff (patch.diff) is applied on top of the - # default implementation. Use this when the variant only needs small - # targeted changes to the base code (e.g. swapping async I/O for sync I/O - # in a runtime that lacks full async support). - # Apply unified diff on top of the already-copied base files - import patch_ng - - pset = patch_ng.fromfile(patch_file) - if not pset or not pset.apply(strip=1, root=output_dir): + if overlay_dir_name != "default": + variant_dir = os.path.join(path, overlay_dir_name) + if not os.path.isdir(variant_dir): raise RuntimeError( - "Failed to apply patch {} for variant {}".format( - patch_file, self._language_variant + "Variant directory not found for benchmark {} language {} " + "variant {}: {}".format( + self.benchmark, self.language_name, self._language_variant, variant_dir + ) + ) + + # Variants come in two flavors and this is where we split between them: + # 1. Patch-based (patch.diff present): apply a unified diff on top of the + # already-copied base files. Use when the variant only needs small, + # targeted edits to the default implementation (e.g. swapping async I/O + # for sync I/O in a runtime that lacks full async support). + # 2. Copy-based (no patch.diff): overlay the variant directory's files on + # top of the base files, replacing any that collide. Use when the + # variant diverges enough that a patch would be unwieldy. + patch_file = os.path.join(variant_dir, "patch.diff") + if os.path.exists(patch_file): + import patch_ng + + pset = patch_ng.fromfile(patch_file) + if not pset or not pset.apply(strip=1, root=output_dir): + raise RuntimeError( + "Failed to apply patch {} for variant {}".format( + patch_file, self._language_variant + ) + ) + self.logging.info( + "Applied patch for variant {} ({})".format( + self._language_variant, patch_file + ) + ) + else: + for file_type in FILES[self.language]: + for f in glob.glob(os.path.join(variant_dir, file_type)): + shutil.copy2(f, output_dir) + nodejs_variant_pkg = os.path.join( + variant_dir, f"package.json.{self.language_version}" + ) + if os.path.exists(nodejs_variant_pkg): + shutil.copy2(nodejs_variant_pkg, os.path.join(output_dir, "package.json")) + self.logging.info( + "Applied file overlay for variant {} (dir: {})".format( + self._language_variant, overlay_dir_name ) ) - self.logging.info( - "Applied patch for variant {} ({})".format(self._language_variant, patch_file) - ) - else: - # Overlay-based variant: the variant directory contains a complete - # replacement set of source files that fully override the default - # implementation. All files from the variant directory are copied - # on top of the already-placed base files. Use this when the variant - # is substantially different from the default (e.g. a full rewrite). - for file_type in FILES[self.language]: - for f in glob.glob(os.path.join(variant_dir, file_type)): - shutil.copy2(f, output_dir) - # version-specific package.json override for Node.js - nodejs_variant_pkg = os.path.join( - variant_dir, f"package.json.{self.language_version}" - ) - if os.path.exists(nodejs_variant_pkg): - shutil.copy2(nodejs_variant_pkg, os.path.join(output_dir, "package.json")) - self.logging.info( - "Applied file overlay for variant {}".format(self._language_variant) - ) def add_benchmark_data(self, output_dir: str) -> None: """Add benchmark-specific data and assets to output directory. @@ -888,7 +1004,7 @@ def add_benchmark_data(self, output_dir: str) -> None: "init.sh failed (exit {}): {}".format(result.returncode, output) ) - def add_deployment_files(self, output_dir: str) -> None: + def add_deployment_files(self, output_dir: str, is_workflow: bool = False) -> None: """Add deployment-specific wrapper files to output directory. Copies platform-specific wrapper files (handlers, adapters) that @@ -899,6 +1015,7 @@ def add_deployment_files(self, output_dir: str) -> None: Args: output_dir: Directory where deployment files should be added + is_workflow: If True, use handler_workflow.py as handler.py """ handlers_dir = get_resource_path( "benchmarks", "wrappers", self._deployment_name, self.language_name @@ -918,6 +1035,19 @@ def add_deployment_files(self, output_dir: str) -> None: if not os.path.exists(destination): shutil.copy2(file, destination) + if self.language_name == "python": + handler_path = os.path.join(output_dir, "handler.py") + handler_workflow_path = os.path.join(output_dir, "handler_workflow.py") + if is_workflow and os.path.exists(handler_workflow_path): + os.replace(handler_workflow_path, handler_path) + elif os.path.exists(handler_workflow_path): + os.remove(handler_workflow_path) + + if is_workflow: + definition_src = os.path.join(self._benchmark_path, "definition.json") + if os.path.exists(definition_src): + shutil.copy2(definition_src, os.path.join(output_dir, "definition.json")) + def add_deployment_package_python(self, output_dir: str) -> None: """Add Python deployment packages to requirements file. @@ -940,7 +1070,7 @@ def add_deployment_package_python(self, output_dir: str) -> None: self._deployment_name, self.language_name ) for package in packages: - out.write(package) + out.write(f"\n{package}") module_packages = self._system_config.deployment_module_packages( self._deployment_name, self.language_name @@ -948,7 +1078,7 @@ def add_deployment_package_python(self, output_dir: str) -> None: for bench_module in self._benchmark_config.modules: if bench_module.value in module_packages: for package in module_packages[bench_module.value]: - out.write(package) + out.write(f"\n{package}") def add_deployment_package_nodejs(self, output_dir: str) -> None: """Add Node.js deployment packages to package.json. @@ -1256,15 +1386,6 @@ def ensure_image(name: str) -> None: # Create set of mounted volumes volumes = {os.path.abspath(output_dir): {"bind": "/mnt/function", "mode": "rw"}} - package_script = os.path.abspath( - os.path.join(self._benchmark_path, self.language_name, "package.sh") - ) - # does this benchmark has package.sh script? - if os.path.exists(package_script): - volumes[package_script] = { - "bind": "/mnt/function/package.sh", - "mode": "ro", - } # run Docker container to install packages PACKAGE_FILES = { @@ -1285,17 +1406,20 @@ def ensure_image(name: str) -> None: path=os.path.abspath(output_dir) ) ) + build_env = { + "CONTAINER_UID": str(os.getuid()), + "CONTAINER_GID": str(os.getgid()), + "CONTAINER_USER": "docker_user", + "APP": self.benchmark, + "PLATFORM": self._deployment_name.upper(), + "TARGET_ARCHITECTURE": self._experiment_config._architecture, + } + if os.path.exists(os.path.join(output_dir, "package.sh")): + build_env["SCRIPT_FILE"] = "package.sh" container = self._docker_client.containers.run( "{}:{}".format(repo_name, image_name), volumes=volumes, - environment={ - "CONTAINER_UID": str(os.getuid()), - "CONTAINER_GID": str(os.getgid()), - "CONTAINER_USER": "docker_user", - "APP": self.benchmark, - "PLATFORM": self._deployment_name.upper(), - "TARGET_ARCHITECTURE": self._experiment_config._architecture, - }, + environment=build_env, remove=False, detach=True, ) @@ -1354,6 +1478,7 @@ def build( container_client: DockerContainer | None, container_build_step: Callable[[str, Language, str, str, str, bool], Tuple[str, float]] | None, + is_workflow: bool = False, ) -> Tuple[bool, str | None, SystemVariant, str | None]: """Build the complete benchmark deployment package. @@ -1421,7 +1546,7 @@ def build( self.copy_code(self._output_dir) self.add_benchmark_data(self._output_dir) - self.add_deployment_files(self._output_dir) + self.add_deployment_files(self._output_dir, is_workflow) self.add_deployment_package(self._output_dir) """ @@ -1675,6 +1800,7 @@ def prepare_input( nosql_storage.update_cache(self._benchmark) self._input_processed = True + self._last_input_config = input_config return input_config diff --git a/sebs/cli.py b/sebs/cli.py index ec7bab57e..76512901c 100755 --- a/sebs/cli.py +++ b/sebs/cli.py @@ -114,7 +114,7 @@ def common_params(func): @click.option( "--deployment", default=None, - type=click.Choice(["azure", "aws", "gcp", "local", "openwhisk"]), + type=click.Choice(["azure", "aws", "gcp", "local", "openwhisk", "cloudflare"]), help="Cloud deployment to use.", ) @click.option( @@ -370,6 +370,93 @@ def invoke( sebs_client.logging.info("Save results to {}".format(os.path.abspath(result_file))) +@benchmark.command() +@click.argument("benchmark", type=str) +@click.argument("benchmark-input-size", type=click.Choice(["test", "small", "large"])) +@click.option("--repetitions", default=5, type=int, help="Number of experimental repetitions.") +@click.option( + "--trigger", + type=click.Choice(["library", "http"]), + default="http", + help="Workflow trigger to be used.", +) +@click.option( + "--workflow-name", + default=None, + type=str, + help="Override workflow name for random generation.", +) +@common_params +def workflow(benchmark, benchmark_input_size, repetitions, trigger, workflow_name, **kwargs): + """Invoke a workflow benchmark and measure performance.""" + import pandas as pd + from sebs.utils import connect_to_redis_cache, download_measurements + + (config, output_dir, logging_filename, sebs_client, deployment_client) = parse_common_params( + **kwargs + ) + + experiment_config = sebs_client.get_experiment_config(config["experiments"]) + benchmark_obj = sebs_client.get_benchmark( + benchmark, + deployment_client, + experiment_config, + logging_filename=logging_filename, + ) + + input_config = benchmark_obj.prepare_input( + deployment_client.system_resources, + size=benchmark_input_size, + replace_existing=experiment_config.update_storage, + ) + + wf = deployment_client.get_workflow( + benchmark_obj, + workflow_name if workflow_name else deployment_client.default_function_name(benchmark_obj), + ) + + redis_host = getattr(deployment_client.config, "redis_host", None) + redis = None + if redis_host: + try: + redis = connect_to_redis_cache(redis_host) + except Exception as e: + sebs_client.logging.warning(f"Could not connect to Redis ({e}), skipping measurements") + + result = sebs.experiments.ExperimentResult(experiment_config, deployment_client.config) + result.begin() + + trigger_type = Trigger.TriggerType.get(trigger) + triggers = wf.triggers(trigger_type) + if len(triggers) == 0: + trigger = deployment_client.create_trigger(wf, trigger_type) + else: + trigger = triggers[0] + + measurements = [] + for i in range(repetitions): + sebs_client.logging.info(f"Beginning repetition {i + 1}/{repetitions}") + ret = trigger.sync_invoke(input_config) + if ret.stats.failure: + sebs_client.logging.info(f"Failure on repetition {i + 1}/{repetitions}") + + if redis: + measurements += download_measurements(redis, wf.name, result.begin_time, rep=i) + result.add_invocation(wf, ret) + result.end() + + if measurements: + path = os.path.join(output_dir, "results", wf.name, deployment_client.name() + ".csv") + os.makedirs(os.path.dirname(path), exist_ok=True) + df = pd.DataFrame(measurements) + df.to_csv(path, index=False) + + result_file = os.path.join(output_dir, "experiments.json") + with open(result_file, "w") as out_f: + out_f.write(sebs.utils.serialize(result)) + sebs_client.logging.info("Save results to {}".format(os.path.abspath(result_file))) + + @benchmark.command() @common_params def process(**kwargs): @@ -498,12 +585,29 @@ def package( help="Filter resource IDs and URls from output.", ) @common_params +@click.option( + "--cache", + default=os.path.join(os.path.curdir, "regression-cache"), + help="Location of experiments cache.", +) +@click.option( + "--output-dir", + default=os.path.join(os.path.curdir, "regression-output"), + help="Output directory for results.", +) +@click.option( + "--deployment-type", + default=None, + type=click.Choice(["functions", "containers"]), + help="Limit regression to a specific deployment type (functions or containers).", +) def regression( benchmark_input_size, benchmark_name, storage_configuration, selected_architecture, filter_output, + deployment_type, **kwargs, ): """Run regression test suite across benchmarks.""" @@ -529,6 +633,8 @@ def regression( config, kwargs["resource_prefix"], benchmark_name, + deployment_type, + benchmark_input_size, architecture, filter_output, ) @@ -988,7 +1094,7 @@ def docker_cmd(): @click.option( "--deployment", default=None, - type=click.Choice(["local", "aws", "azure", "gcp", "openwhisk"]), + type=click.Choice(["local", "aws", "azure", "gcp", "openwhisk", "cloudflare"]), help="Deployment platform to build images for", ) @click.option( @@ -1070,7 +1176,7 @@ def docker_build( @click.option( "--deployment", default=None, - type=click.Choice(["local", "aws", "azure", "gcp", "openwhisk"]), + type=click.Choice(["local", "aws", "azure", "gcp", "openwhisk", "cloudflare"]), help="Deployment platform to push images for", ) @click.option( diff --git a/sebs/cloudflare/__init__.py b/sebs/cloudflare/__init__.py new file mode 100644 index 000000000..d8d5c47e6 --- /dev/null +++ b/sebs/cloudflare/__init__.py @@ -0,0 +1,6 @@ +"""Cloudflare Workers serverless platform implementation.""" + +from sebs.cloudflare.cloudflare import Cloudflare +from sebs.cloudflare.config import CloudflareConfig + +__all__ = ["Cloudflare", "CloudflareConfig"] diff --git a/sebs/cloudflare/cli.py b/sebs/cloudflare/cli.py new file mode 100644 index 000000000..6f738a870 --- /dev/null +++ b/sebs/cloudflare/cli.py @@ -0,0 +1,250 @@ +"""Cloudflare CLI container management for wrangler-based deployments.""" + +import atexit +import io +import logging +import os +import tarfile +import threading +from typing import Optional + +import docker + +from sebs.config import SeBSConfig +from sebs.utils import LoggingBase + + +class CloudflareCLI(LoggingBase): + """ + Manages a Docker container with Cloudflare Wrangler and related tools pre-installed. + + This approach isolates Cloudflare CLI tools (wrangler, pywrangler) from the host system, + avoiding global npm/uv installations and ensuring consistent behavior across platforms. + """ + + _instance: Optional["CloudflareCLI"] = None + _lock: threading.Lock = threading.Lock() + + @staticmethod + def get_instance( + system_config: SeBSConfig, docker_client: docker.client.DockerClient + ) -> "CloudflareCLI": + """Return the shared CloudflareCLI instance, creating it on first use. + + Container and native workers deployments share one underlying CLI + container so that combined runs don't spawn duplicates. + Thread-safe: the first caller builds the container; concurrent callers wait. + """ + if CloudflareCLI._instance is None: + with CloudflareCLI._lock: + if CloudflareCLI._instance is None: + CloudflareCLI._instance = CloudflareCLI(system_config, docker_client) + atexit.register(CloudflareCLI.shutdown_instance) + return CloudflareCLI._instance + + def __init__(self, system_config: SeBSConfig, docker_client: docker.client.DockerClient): + """Pull the manage image if needed and start the CLI container.""" + super().__init__() + self._stopped = False + + repo_name = system_config.docker_repository() + sebs_version = system_config.version() + image_name = "manage.cloudflare" + versioned_tag = f"{image_name}-{sebs_version}" + try: + docker_client.images.get(repo_name + ":" + versioned_tag) + except docker.errors.ImageNotFound: + logging.info( + "Docker pull of image {repo}:{tag}".format(repo=repo_name, tag=versioned_tag) + ) + try: + docker_client.images.pull(repo_name, tag=versioned_tag) + except (docker.errors.APIError, docker.errors.ImageNotFound) as e: + raise RuntimeError( + "Docker pull of image {}:{} failed: {}".format(repo_name, versioned_tag, e) + ) + + # Start the container in detached mode + self.docker_instance = docker_client.containers.run( + image=repo_name + ":" + versioned_tag, + command="/bin/bash", + environment={ + "CONTAINER_UID": str(os.getuid()), + "CONTAINER_GID": str(os.getgid()), + "CONTAINER_USER": "docker_user", + }, + volumes={ + # Mount Docker socket so wrangler can build and push images to + # Cloudflare's registry during `wrangler deploy` for container workers. + "/var/run/docker.sock": {"bind": "/var/run/docker.sock", "mode": "rw"} + }, + remove=True, + stdout=True, + stderr=True, + detach=True, + tty=True, + ) + + self.logging.info(f"Started Cloudflare CLI container: {self.docker_instance.id}.") + + # Wait for container to be ready + while True: + try: + dkg = self.docker_instance.logs(stream=True, follow=True) + next(dkg).decode("utf-8") + break + except StopIteration: + pass + + @staticmethod + def typename() -> str: + """Return the canonical type name for this class.""" + return "Cloudflare.CLI" + + def execute(self, cmd: str, env: Optional[dict] = None): + """ + Execute the given command in Cloudflare CLI container. + Throws an exception on failure (commands are expected to execute successfully). + + Args: + cmd: Shell command to execute + env: Optional environment variables dict + + Returns: + Command output as bytes + """ + # Wrap command in sh -c to support shell features like cd, pipes, etc. + shell_cmd = ["/bin/sh", "-c", cmd] + exit_code, out = self.docker_instance.exec_run( + shell_cmd, + # Run as root since entrypoint creates docker_user but we don't wait for it + user="root", + environment=env, + ) + assert isinstance(out, bytes) + if exit_code != 0: + raise RuntimeError( + "Command {} failed at Cloudflare CLI docker!\n Output {}".format( + cmd, out.decode("utf-8") + ) + ) + return out + + def upload_package(self, directory: str, dest: str): + """ + Upload a directory to the Docker container. + + This is not an efficient and memory-intensive implementation. + So far, we didn't have very large functions that require many gigabytes. + + Since docker-py does not support a straightforward copy, and we can't + put_archive in chunks. + + Args: + directory: Local directory to upload + dest: Destination path in container + """ + handle = io.BytesIO() + with tarfile.open(fileobj=handle, mode="w:gz") as tar: + for f in os.listdir(directory): + tar.add(os.path.join(directory, f), arcname=f) + + # Move to the beginning of memory before writing + handle.seek(0) + self.execute("mkdir -p {}".format(dest)) + self.docker_instance.put_archive(path=dest, data=handle.read()) + + def check_wrangler_version(self) -> str: + """ + Check wrangler version. + + Returns: + Version string + """ + out = self.execute("wrangler --version") + return out.decode("utf-8").strip() + + def check_pywrangler_version(self) -> str: + """ + Check pywrangler version. + + Returns: + Version string + """ + out = self.execute("pywrangler --version") + return out.decode("utf-8").strip() + + def containers_push(self, tag: str, env: Optional[dict] = None) -> str: + """ + Push a locally-built image to Cloudflare's container registry. + + The image must already exist locally (built by docker_client.images.build). + The manage container shares the host Docker socket, so it can see and push + local images directly. + + Args: + tag: Local image tag (e.g. my-bench-python-312:latest) + env: Environment variables (must include CLOUDFLARE_API_TOKEN and + CLOUDFLARE_ACCOUNT_ID) + + Returns: + Registry URI (registry.cloudflare.com//:) + """ + out = self.execute(f"wrangler containers push {tag}", env=env) + output = out.decode("utf-8") + for line in output.splitlines(): + if "registry.cloudflare.com" in line: + parts = line.split() + for part in parts: + if part.startswith("registry.cloudflare.com"): + return part.strip() + raise RuntimeError( + f"Could not parse registry URI from wrangler containers push output:\n{output}" + ) + + def wrangler_deploy(self, package_dir: str, env: Optional[dict] = None) -> str: + """ + Deploy a worker using wrangler. + + Args: + package_dir: Path to package directory in container + env: Environment variables for deployment + + Returns: + Deployment output + """ + cmd = "cd {} && npm install && wrangler deploy".format(package_dir) + out = self.execute(cmd, env=env) + return out.decode("utf-8") + + def pywrangler_deploy(self, package_dir: str, env: Optional[dict] = None) -> str: + """ + Deploy a Python worker using pywrangler. + + Args: + package_dir: Path to package directory in container + env: Environment variables for deployment + + Returns: + Deployment output + """ + cmd = "cd {} && pywrangler deploy".format(package_dir) + out = self.execute(cmd, env=env) + return out.decode("utf-8") + + @staticmethod + def shutdown_instance(): + """Stop the shared CLI container and clear the singleton. + + Call this once at process teardown, after all parallel benchmarks + have finished. Individual deployment handlers must NOT call this — + they should just drop their local reference. + """ + with CloudflareCLI._lock: + instance = CloudflareCLI._instance + CloudflareCLI._instance = None + + if instance is not None and not instance._stopped: + instance._stopped = True + instance.logging.info("Stopping Cloudflare CLI Docker instance") + instance.docker_instance.stop() diff --git a/sebs/cloudflare/cloudflare.py b/sebs/cloudflare/cloudflare.py new file mode 100644 index 000000000..69cc0e730 --- /dev/null +++ b/sebs/cloudflare/cloudflare.py @@ -0,0 +1,1888 @@ +"""Cloudflare Workers platform implementation for SeBS.""" + +import json +import math +import os +import uuid +import time +from typing import Any, cast, Dict, List, Optional, Set, Tuple, Type + +import docker +import requests + +from sebs.cloudflare.config import CloudflareConfig +from sebs.cloudflare.function import CloudflareWorker +from sebs.cloudflare.resources import CloudflareSystemResources +from sebs.cloudflare.workers import CloudflareWorkersDeployment +from sebs.cloudflare.containers import CloudflareContainersDeployment +from sebs.benchmark import Benchmark +from sebs.cache import Cache +from sebs.config import SeBSConfig +from sebs.utils import LoggingHandlers +from sebs.faas.function import Function, ExecutionResult, Trigger, FunctionConfig +from sebs.experiments.config import SystemVariant +from sebs.faas.system import System +from sebs.faas.config import Resources +from sebs.sebs_types import Language + + +class _CloudflareContainerAdapter: + """Duck-typed adapter that satisfies benchmark.build()'s container_client contract. + + benchmark.build() calls container_client.build_base_image() when + container_deployment=True and asserts the client is not None. Cloudflare + builds its container images inside package_code (via containers.py), not + through a registry-backed DockerContainer, so this adapter bridges the gap + without touching the framework. + """ + + def __init__(self, containers_deployment: CloudflareContainersDeployment): + """Initialize the adapter with the given containers deployment handler.""" + self._containers = containers_deployment + # Populated by build_base_image() so create_function() can find the dir. + self.last_directory: Optional[str] = None + + def build_base_image( + self, + directory: str, + language, # sebs.sebs_types.Language enum + language_version: str, + architecture: str, + benchmark: str, + is_cached: bool, + builder_image: str, + ) -> Tuple[bool, str, float]: + """Delegate to containers.package_code; match benchmark.build() contract. + + Returns (rebuilt, image_tag, size_mb) so that: + _, self._container_uri, self._code_size = container_client.build_base_image(...) + works correctly in benchmark.build(). + """ + dir_result, size_bytes, image_tag = self._containers.package_code( + directory, + language.value, # Language enum → str + language_version, + architecture, + benchmark, + ) + self.last_directory = dir_result + size_mb = size_bytes / 1024.0 / 1024.0 + return (True, image_tag, size_mb) + + def push_to_registry( + self, + benchmark: str, + language_name: str, + language_version: str, + architecture: str, + ) -> str: + """ + Return a local cache label for the container image. + + Cloudflare container workers do not use a conventional image registry. + Instead, `wrangler deploy` reads `./Dockerfile` directly from the + package directory, builds the image, and pushes it to Cloudflare's + managed registry — all in one step. SeBS therefore never needs to + push an image to an external registry before deployment; this method + exists only to satisfy the `ContainerSystemInterface` contract and to + provide a stable cache key that `Benchmark` uses to detect whether a + previously-built image is still valid. + + The returned string is a local image tag of the form + ``--:latest``. It is + NOT a pushable URI and is not passed to any registry client. + """ + image_name = ( + f"{benchmark.replace('.', '-')}-{language_name}-" + f"{language_version.replace('.', '')}" + ) + return f"{image_name}:latest" + + +class Cloudflare(System): + """ + Cloudflare Workers serverless platform implementation. + + Cloudflare Workers run on Cloudflare's edge network, providing + low-latency serverless execution globally. + """ + + # Benchmarks supported per (language, container_deployment) combination. + # Keys are (language_name, container_deployment). + # A value of None means all benchmarks are supported. + # Benchmark IDs are matched against the numeric prefix of the benchmark name + # (e.g. "110" matches "110.dynamic-html"). + SUPPORTED_BENCHMARKS: Dict[Tuple[str, bool], Optional[List[str]]] = { + ("python", False): [ + "110", + "120", + "130", + "210", + "311", + "501", + "502", + "503", + ], + ("nodejs", False): ["110", "120", "130", "311"], + ("python", True): None, # all benchmarks supported + ("nodejs", True): ["110", "120", "130", "210", "311"], + } + + _config: CloudflareConfig + + @staticmethod + def name(): + """Return the platform name used in configuration and cache keys.""" + return "cloudflare" + + @staticmethod + def typename(): + """Return the human-readable type name for this platform.""" + return "Cloudflare" + + @staticmethod + def function_type() -> "Type[Function]": + """Return the Function subclass used by this platform.""" + return CloudflareWorker + + @property + def config(self) -> CloudflareConfig: + """Return the Cloudflare-specific platform configuration.""" + return self._config + + def is_benchmark_supported( + self, benchmark_name: str, language: str, container_deployment: bool + ) -> bool: + """Return True if the benchmark is supported for the given language/deployment type. + + Args: + benchmark_name: Full benchmark name, e.g. "110.dynamic-html" + language: Language name, e.g. "python" or "nodejs" + container_deployment: Whether this is a container deployment + + Returns: + True if supported, False otherwise + """ + allowed = self.SUPPORTED_BENCHMARKS.get((language, container_deployment)) + if allowed is None: + # None means all benchmarks are supported + return True + # Match by numeric prefix (the part before the first dot) + prefix = benchmark_name.split(".")[0] + return prefix in allowed + + def get_function( + self, code_package: Benchmark, func_name: Optional[str] = None + ) -> Function: + """Override to validate benchmark support and auto-select cloudflare variant.""" + language = code_package.language_name + container_deployment = code_package.system_variant.is_container + benchmark_name = code_package.benchmark + if not self.is_benchmark_supported( + benchmark_name, language, container_deployment + ): + deployment_type = "container" if container_deployment else "worker" + raise RuntimeError( + f"Benchmark '{benchmark_name}' is not supported for " + f"{language} {deployment_type} deployments on Cloudflare. " + "Supported benchmarks: " + f"{self.SUPPORTED_BENCHMARKS.get((language, container_deployment))}" + ) + + # For workers deployments, auto-promote the variant from "default" to + # "cloudflare" when the benchmark's config.json declares a "cloudflare" + # variant. Benchmark.__init__ sets the variant from the experiment config + # (CLI --language-variant flag), which defaults to "default". Promoting + # here ensures copy_code() applies the cloudflare/ source overlay and the + # cache key reflects the correct variant. + if ( + code_package.language_variant == "default" + and code_package.benchmark_config.supports( + code_package.language, self.name() + ) + ): + code_package.select_variant(self.name()) + + # The cache stores functions under their formatted name (e.g. + # "container-311-compression-nodejs-18"), but callers pass the + # unformatted default name. Format it here so the cache lookup in + # super().get_function() finds the right entry. + if func_name is not None: + func_name = self.format_function_name(func_name, container_deployment) + + return super().get_function(code_package, func_name) + + def __init__( + self, + sebs_config: SeBSConfig, + config: CloudflareConfig, + cache_client: Cache, + docker_client: docker.client.DockerClient, + logger_handlers: LoggingHandlers, + ): + """Initialize the Cloudflare platform with credentials and deployment handlers.""" + super().__init__( + sebs_config, + cache_client, + docker_client, + CloudflareSystemResources( + config, cache_client, docker_client, logger_handlers + ), + ) + self.logging_handlers = logger_handlers + self._config = config + self._api_base_url = "https://api.cloudflare.com/client/v4" + # cached workers.dev subdomain for the account + # This is different from the account ID and is required to build + # public worker URLs like ..workers.dev + self._workers_dev_subdomain: Optional[str] = None + + # Initialize deployment handlers + self._workers_deployment = CloudflareWorkersDeployment( + self.logging, sebs_config, docker_client, self.system_resources + ) + self._containers_deployment = CloudflareContainersDeployment( + self.logging, sebs_config, docker_client, self.system_resources + ) + # Adapter so benchmark.build() can call container_client.build_base_image() + self._container_adapter = _CloudflareContainerAdapter( + self._containers_deployment + ) + + def initialize( + self, + config: Dict[str, str] = {}, + resource_prefix: Optional[str] = None, + quiet: bool = False, + ): + """ + Initialize the Cloudflare Workers platform. + + Args: + config: Additional configuration parameters + resource_prefix: Prefix for resource naming + """ + # Verify credentials are valid + self._verify_credentials() + self.initialize_resources(select_prefix=resource_prefix) + + def initialize_resources( + self, select_prefix: Optional[str] = None, quiet: bool = False + ): + """ + Initialize Cloudflare resources. + + Overrides the base class method to handle R2 storage gracefully. + Cloudflare Workers can operate without R2 storage for many benchmarks. + + Args: + select_prefix: Optional prefix for resource naming + """ + deployments = self.find_deployments() + + # Check if we have an existing deployment + if deployments: + res_id = deployments[0] + self.config.resources.resources_id = res_id + self.logging.info(f"Using existing resource deployment {res_id}") + return + + # Create new resource ID + if select_prefix is not None: + res_id = f"{select_prefix}-{str(uuid.uuid1())[0:8]}" + else: + res_id = str(uuid.uuid1())[0:8] + + self.config.resources.resources_id = res_id + self.logging.info(f"Generating unique resource name {res_id}") + + # Try to create R2 bucket, but don't fail if R2 is not enabled + try: + self.system_resources.get_storage().get_bucket( + Resources.StorageBucketType.BENCHMARKS + ) + self.logging.info("R2 storage initialized successfully") + except Exception as e: + self.logging.warning( + f"R2 storage initialization failed: {e}. " + "R2 must be enabled in your Cloudflare dashboard " + "to use storage-dependent benchmarks. " + "Continuing without R2 - only benchmarks that don't require storage will work." + ) + + @property + def container_client(self) -> _CloudflareContainerAdapter: # type: ignore[override] + """Return the Cloudflare-specific container build adapter. + + Overrides System.container_client (which returns None) so that + benchmark.build() can drive container image builds via + _CloudflareContainerAdapter.build_base_image() without needing an + external container registry. + """ + return self._container_adapter + + def _verify_credentials(self): + """Verify that the Cloudflare API credentials are valid.""" + # Check if credentials are set + if not self.config.credentials.api_token and not ( + self.config.credentials.email and self.config.credentials.api_key + ): + raise RuntimeError( + "Cloudflare API credentials are not set. Please set CLOUDFLARE_API_TOKEN " + "and CLOUDFLARE_ACCOUNT_ID environment variables." + ) + + if not self.config.credentials.account_id: + raise RuntimeError( + "Cloudflare Account ID is not set. Please set CLOUDFLARE_ACCOUNT_ID " + "environment variable." + ) + + headers = self._get_auth_headers() + + # Log credential type being used (without exposing the actual token) + if self.config.credentials.api_token: + token_preview = ( + self.config.credentials.api_token[:8] + "..." + if len(self.config.credentials.api_token) > 8 + else "***" + ) + self.logging.info( + f"Using API Token authentication (starts with: {token_preview})" + ) + else: + self.logging.info( + f"Using Email + API Key authentication (email: {self.config.credentials.email})" + ) + + response = requests.get( + f"{self._api_base_url}/user/tokens/verify", headers=headers + ) + + if response.status_code != 200: + raise RuntimeError( + f"Failed to verify Cloudflare credentials: " + f"{response.status_code} - {response.text}\n" + "Please check that your CLOUDFLARE_API_TOKEN and CLOUDFLARE_ACCOUNT_ID are correct." + ) + + self.logging.info("Cloudflare credentials verified successfully") + + def _get_deployment_handler(self, container_deployment: bool): + """Get the appropriate deployment handler based on deployment type. + + Args: + container_deployment: Whether this is a container deployment + + Returns: + CloudflareWorkersDeployment or CloudflareContainersDeployment + """ + if container_deployment: + return self._containers_deployment + else: + return self._workers_deployment + + def package_code( + self, + directory: str, + language: Language, + language_version: str, + architecture: str, + benchmark: str, + is_cached: bool, + ) -> Tuple[str, int]: + """ + Package code for native Cloudflare Workers deployment using Wrangler. + + Called by benchmark.build() via the non-container path. Container + builds are driven by _CloudflareContainerAdapter.build_base_image() + through the container_client property instead. + + Args: + directory: Path to the code directory + language: Programming language enum + language_version: Programming language version + architecture: Target architecture (not used for Workers) + benchmark: Benchmark name + is_cached: Whether the code is cached + + Returns: + Tuple of (package_path, package_size) + """ + # Native worker deployment flow — always the cloudflare variant. + # workers.py returns a 3-tuple (path, size, ""); drop the unused 3rd element. + pkg_path, pkg_size, _ = self._workers_deployment.package_code( + directory, + language.value, + language_version, + benchmark, + is_cached, + language_variant="cloudflare", + ) + return (pkg_path, pkg_size) + + def _get_auth_headers(self) -> Dict[str, str]: + """Get authentication headers for Cloudflare API requests.""" + if self.config.credentials.api_token: + return { + "Authorization": f"Bearer {self.config.credentials.api_token}", + "Content-Type": "application/json", + } + elif self.config.credentials.email and self.config.credentials.api_key: + return { + "X-Auth-Email": self.config.credentials.email, + "X-Auth-Key": self.config.credentials.api_key, + "Content-Type": "application/json", + } + else: + raise RuntimeError("Invalid Cloudflare credentials configuration") + + def _generate_wrangler_toml( + self, + worker_name: str, + package_dir: str, + language: str, + account_id: str, + benchmark_name: Optional[str] = None, + code_package: Optional[Benchmark] = None, + container_deployment: bool = False, + container_uri: Optional[str] = None, + ) -> str: + """ + Generate wrangler.toml by delegating to the appropriate deployment handler. + + Args: + worker_name: Name of the worker + package_dir: Directory containing the worker code + language: Programming language (nodejs or python) + account_id: Cloudflare account ID + benchmark_name: Optional benchmark name for R2 file path prefix + code_package: Optional benchmark package for nosql configuration + container_deployment: Whether this is a container deployment + container_uri: Container image URI/tag + + Returns: + Path to the generated wrangler.toml file + """ + language_variant = ( + code_package.language_variant if code_package else "cloudflare" + ) + handler = self._get_deployment_handler(container_deployment) + return handler.generate_wrangler_toml( + worker_name, + package_dir, + language, + account_id, + benchmark_name, + code_package, + container_uri, + language_variant, + ) + + def create_function( + self, + code_package: Benchmark, + func_name: str, + system_variant: SystemVariant, + container_uri: str | None, + ) -> CloudflareWorker: + """ + Create a new Cloudflare Worker. + + If a worker with the same name already exists, it will be updated. + + Args: + code_package: Benchmark containing the function code + func_name: Name of the worker + system_variant: Selected deployment variant + container_uri: URI of container image + + Returns: + CloudflareWorker instance + """ + container_deployment = system_variant.is_container + # For container builds benchmark.build() goes through container_client.build_base_image(), + # which does NOT set code_package._code_location. Fall back in order: + # 1. _CloudflareContainerAdapter.last_directory (set when build actually ran this session) + # 2. code_package._output_dir (the on-disk build directory from a previous session — + # build() leaves it in place when the image cache is valid and the build is skipped) + package = code_package.code_location + if package is None and container_deployment: + package = self._container_adapter.last_directory + if package is None and container_deployment: + output_dir = code_package._output_dir + if os.path.isdir(output_dir): + package = output_dir + self.logging.info( + f"Using existing output directory for {code_package.benchmark}: {package}" + ) + + benchmark = code_package.benchmark + language = code_package.language_name + language_runtime = code_package.language_version + function_cfg = FunctionConfig.from_benchmark(code_package) + + func_name = self.format_function_name(func_name, container_deployment) + account_id = self.config.credentials.account_id + + if not account_id: + raise RuntimeError("Cloudflare account ID is required to create workers") + + # Check if worker already exists + existing_worker = self._get_worker(func_name, account_id) + + if package is None: + raise RuntimeError( + f"Code location is not set for {code_package.benchmark}. " + "The build step may not have completed successfully." + ) + + if existing_worker: + self.logging.info(f"Worker {func_name} already exists, updating it") + worker = CloudflareWorker( + func_name, + code_package.benchmark, + func_name, # script_id is the same as name + code_package.hash, + language_runtime, + function_cfg, + account_id, + ) + self.update_function(worker, code_package, system_variant, container_uri) + worker.updated_code = True + else: + self.logging.info(f"Creating new worker {func_name}") + + # Create the worker with all package files + self._create_or_update_worker( + func_name, + package, + account_id, + language, + benchmark, + code_package, + container_deployment, + container_uri, + ) + + worker = CloudflareWorker( + func_name, + code_package.benchmark, + func_name, + code_package.hash, + language_runtime, + function_cfg, + account_id, + ) + + # Add HTTPTrigger + from sebs.cloudflare.triggers import HTTPTrigger + + # Build worker URL using the account's workers.dev subdomain when possible. + # Falls back to account_id-based host or plain workers.dev with warnings. + worker_url = self._build_workers_dev_url(func_name, account_id) + http_trigger = HTTPTrigger(func_name, worker_url) + http_trigger.logging_handlers = self.logging_handlers + worker.add_trigger(http_trigger) + + return worker + + def _get_worker(self, worker_name: str, account_id: str) -> Optional[dict]: + """Get information about an existing worker.""" + headers = self._get_auth_headers() + url = ( + f"{self._api_base_url}/accounts/{account_id}/workers/scripts/{worker_name}" + ) + + response = requests.get(url, headers=headers) + + if response.status_code == 200: + try: + return response.json().get("result") + except Exception: + return None + elif response.status_code == 404: + return None + else: + self.logging.warning( + f"Unexpected response checking worker: {response.status_code}" + ) + return None + + def _create_or_update_worker( + self, + worker_name: str, + package_dir: str, + account_id: str, + language: str, + benchmark_name: Optional[str] = None, + code_package: Optional[Benchmark] = None, + container_deployment: bool = False, + container_uri: str | None = None, + ) -> dict: + """Create or update a Cloudflare Worker using Wrangler CLI in container. + + Args: + worker_name: Name of the worker + package_dir: Directory containing handler and all benchmark files + account_id: Cloudflare account ID + language: Programming language (nodejs or python) + benchmark_name: Optional benchmark name for R2 file path prefix + code_package: Optional benchmark package for nosql configuration + container_deployment: Whether this is a container deployment + container_uri: Container image URI/tag + + Returns: + Worker deployment result + """ + # Set up environment for Wrangler CLI in container + env = {} + + if self.config.credentials.api_token: + env["CLOUDFLARE_API_TOKEN"] = self.config.credentials.api_token + elif self.config.credentials.email and self.config.credentials.api_key: + env["CLOUDFLARE_EMAIL"] = self.config.credentials.email + env["CLOUDFLARE_API_KEY"] = self.config.credentials.api_key + + env["CLOUDFLARE_ACCOUNT_ID"] = account_id + + # Get CLI container instance from appropriate deployment handler + handler = self._get_deployment_handler(container_deployment) + cli = handler._get_cli() + + # Push the locally-built container image to Cloudflare's registry so that + # wrangler deploy can reference it directly instead of rebuilding from the + # Dockerfile. Must happen before generating wrangler.toml so the registry + # URI is written in from the start. + if container_deployment and container_uri: + self.logging.info( + f"Pushing container image {container_uri} to Cloudflare registry..." + ) + container_uri = cli.containers_push(container_uri, env=env) + self.logging.info(f"Image pushed to: {container_uri}") + + # Generate wrangler.toml for this worker (uses registry URI if available) + if container_deployment: + self._containers_deployment.max_instances = self.config.max_instances + self._generate_wrangler_toml( + worker_name, + package_dir, + language, + account_id, + benchmark_name, + code_package, + container_deployment, + container_uri, + ) + + # Upload package directory to container + container_package_path = f"/tmp/workers/{worker_name}" + self.logging.info(f"Uploading package to container: {container_package_path}") + cli.upload_package(package_dir, container_package_path) + + try: + self.logging.info( + f"Deploying worker {worker_name} using Wrangler in container..." + ) + + # pywrangler is used for all native Python workers (packages must be + # synced via pyproject.toml before wrangler uploads the bundle). + # All other cases — nodejs, containers — use wrangler directly. + if not container_deployment and language == "python": + output = cli.pywrangler_deploy(container_package_path, env=env) + else: + output = cli.wrangler_deploy(container_package_path, env=env) + + self.logging.info(f"Worker {worker_name} deployed successfully") + self.logging.debug(f"Wrangler deploy output: {output}") + + # Wait for the worker to become reachable before returning. + account_id_val = env.get("CLOUDFLARE_ACCOUNT_ID") + worker_url = self._build_workers_dev_url(worker_name, account_id_val) + + if container_deployment: + container_name = ( + self._containers_deployment._container_name_from_worker(worker_name) + ) + # Cloudflare compares the newly pushed registry image against the + # image currently running in the container worker. If the image digest + # has changed, wrangler deploy triggers a rollout: Cloudflare pulls the + # new image, replaces the running instances, and sets active_rollout_id + # on the container application record until the rollout finishes. + # If nothing changed (same digest), wrangler reports "no changes" and + # no rollout is started — the container is already on the correct image. + if "no changes" in output.lower(): + self.logging.info( + f"Container {container_name} unchanged, skipping readiness wait." + ) + else: + # A rollout is in progress. Poll the Cloudflare REST API until + # active_rollout_id disappears, which signals that all container + # instances have been replaced and are serving the new image. + self.logging.info("Waiting for container rollout to complete...") + self._wait_for_container_rollout(container_name, account_id) + else: + self._wait_for_worker_ready(worker_name, worker_url) + + return {"success": True, "output": output} + + except RuntimeError as e: + error_msg = f"Wrangler deployment failed for worker {worker_name}: {str(e)}" + self.logging.error(error_msg) + raise RuntimeError(error_msg) + + def _wait_for_worker_ready( + self, + worker_name: str, + worker_url: str, + max_wait_seconds: int = 60, + poll_interval: int = 5, + ) -> None: + """Poll a native worker until it responds, confirming edge propagation.""" + self.logging.info( + f"Waiting up to {max_wait_seconds}s for worker {worker_name} to become reachable..." + ) + start = time.time() + while time.time() - start < max_wait_seconds: + try: + resp = requests.get(worker_url, timeout=10) + if resp.status_code not in (502, 503, 522, 524): + self.logging.info( + f"Worker {worker_name} is reachable (HTTP {resp.status_code})." + ) + return + except requests.exceptions.RequestException: + pass + time.sleep(poll_interval) + self.logging.warning( + f"Worker {worker_name} not confirmed reachable after {max_wait_seconds}s; " + "proceeding anyway — invocation retries will handle residual propagation delay." + ) + + def _get_container_id(self, container_name: str, account_id: str) -> Optional[str]: + """Resolve a container name to its UUID via the Cloudflare REST API. + + Lists all container applications for the account and returns the UUID + of the one whose name matches container_name, or None if not found yet. + """ + url = f"{self._api_base_url}/accounts/{account_id}/containers/applications" + headers = self._get_auth_headers() + try: + resp = requests.get(url, headers=headers, timeout=30) + if resp.status_code != 200: + return None + items = resp.json().get("result", []) + for item in items: + if item.get("name") == container_name: + return item.get("id") + except requests.exceptions.RequestException: + pass + return None + + def _wait_for_container_rollout( + self, + container_name: str, + account_id: str, + max_wait_seconds: int = 900, + poll_interval: int = 20, + ) -> None: + """Poll the Cloudflare API until the container has rolled out and an instance is running. + + This covers two sequential phases using the same + GET /accounts/{id}/containers/applications/{uuid} endpoint: + + Phase 1 — Rollout: Cloudflare pulls the new image and replaces instances. + active_rollout_id is set for the duration. Large containers (e.g. ML inference + images) can take up to 10 minutes. Do not lower max_wait_seconds aggressively. + + Phase 2 — Instance readiness: After the rollout finishes, Cloudflare must start + at least one container instance before it can accept requests. Runtime state + lives under `health.instances`: `starting` = still booting, `healthy` = passed + health check and ready to serve, `active` = currently handling a request. + `max_instances` is a ceiling, not a requirement for deployment readiness, so + waiting for every possible instance can stall high-fan-out workflow deploys. + + Args: + container_name: Cloudflare container name (e.g. my-worker-containerworker) + account_id: Cloudflare account ID + max_wait_seconds: Maximum seconds to wait (covers both phases) + poll_interval: Seconds between polls + """ + headers = self._get_auth_headers() + start = time.time() + container_id: Optional[str] = None + rollout_complete = False + + while time.time() - start < max_wait_seconds: + elapsed = int(time.time() - start) + try: + if container_id is None: + container_id = self._get_container_id(container_name, account_id) + if container_id is None: + self.logging.info( + f"Container {container_name} not registered yet... ({elapsed}s elapsed)" + ) + time.sleep(poll_interval) + continue + self.logging.info(f"Resolved container ID: {container_id}") + + url = ( + f"{self._api_base_url}/accounts/{account_id}" + f"/containers/applications/{container_id}" + ) + resp = requests.get(url, headers=headers, timeout=30) + if resp.status_code == 200: + data = resp.json().get("result", resp.json()) + active_rollout = data.get("active_rollout_id") + + if active_rollout: + self.logging.info( + f"Container {container_name} rollout in progress " + f"(rollout_id={active_rollout}, {elapsed}s elapsed)" + ) + else: + if not rollout_complete: + self.logging.info( + f"Container {container_name} rollout complete, " + "waiting for an instance to start..." + ) + rollout_complete = True + + health_instances = data.get("health", {}).get("instances", {}) + healthy = health_instances.get("healthy", 0) + starting = health_instances.get("starting", 0) + self.logging.debug( + f"Container {container_name} health: {health_instances}" + ) + if healthy > 0: + self.logging.info( + f"Container {container_name} is ready " + f"({healthy} instance(s) healthy)." + ) + return + self.logging.info( + f"Container {container_name} waiting for a healthy instance " + f"(healthy={healthy}, starting={starting}, " + f"{elapsed}s elapsed)" + ) + else: + self.logging.info( + f"Unexpected API response {resp.status_code} ({elapsed}s elapsed)" + ) + except requests.exceptions.RequestException as e: + self.logging.debug(f"API request failed ({elapsed}s): {e}") + + time.sleep(poll_interval) + + raise RuntimeError( + f"Container {container_name} did not become ready after {max_wait_seconds}s." + ) + + @staticmethod + def _workflow_container_name(worker_name: str) -> str: + """Return the Cloudflare container name for the generated workflow dispatcher.""" + return f"{worker_name}-dispatchercontainer" + + def _workflow_max_instances( + self, code_package: Benchmark, definition_path: str + ) -> int: + """Return the exact DispatcherContainer ceiling for a workflow input.""" + benchmark_name = code_package.benchmark + prepared_input = code_package.last_input_config or {} + estimated = self._estimate_workflow_parallelism( + definition_path, benchmark_name, prepared_input + ) + if estimated is None: + fallback = max(1, self.config.max_instances) + self.logging.warning( + f"Cloudflare workflow {benchmark_name} has dynamic fan-out that " + f"cannot be known before execution; using configured " + f"max_instances={fallback}." + ) + return fallback + + self.logging.info( + f"Cloudflare workflow {benchmark_name} max_instances={estimated} " + "from prepared benchmark input." + ) + return max(1, estimated) + + def _estimate_workflow_parallelism( + self, + definition_path: str, + benchmark_name: str, + prepared_input: Dict[str, Any], + ) -> Optional[int]: + """Estimate maximum concurrent dispatcher containers for a workflow.""" + with open(definition_path) as f: + definition = json.load(f) + + states = definition.get("states", {}) + root = definition.get("root") + if not root: + return 1 + + def state_max( + state_defs: Dict[str, Any], + state_name: Optional[str], + visiting: Set[Tuple[int, str]], + ) -> Optional[int]: + if not state_name or state_name == "__end__": + return 1 + if state_name not in state_defs: + return 1 + + visit_key = (id(state_defs), state_name) + if visit_key in visiting: + return 1 + + visiting = set(visiting) + visiting.add(visit_key) + state = state_defs[state_name] + state_type = state.get("type") + + if state_type == "switch": + candidates: List[Optional[int]] = [] + for case in state.get("cases", []): + candidates.append(state_max(state_defs, case.get("next"), visiting)) + candidates.append(state_max(state_defs, state.get("default"), visiting)) + if any(value is None for value in candidates): + return None + return max(value or 1 for value in candidates) + + if state_type == "map": + array_length = self._workflow_array_length( + benchmark_name, state_name, state, prepared_input + ) + if array_length is None: + return None + + chunks = max(1, math.ceil(array_length / self.config.chunk_size)) + branch_max = state_max( + state.get("states", {}), state.get("root"), visiting + ) + if branch_max is None: + return None + current_max = chunks * branch_max + elif state_type == "parallel": + branch_values: List[Optional[int]] = [] + for branch in state.get("parallel_functions", []): + branch_values.append( + state_max( + branch.get("states", {}), + branch.get("root"), + visiting, + ) + ) + if any(value is None for value in branch_values): + return None + current_max = sum(value or 1 for value in branch_values) + else: + current_max = 1 + + next_max = state_max(state_defs, state.get("next"), visiting) + if next_max is None: + return None + return max(current_max, next_max) + + return state_max(states, root, set()) + + def _workflow_array_length( + self, + benchmark_name: str, + state_name: str, + state: Dict[str, Any], + prepared_input: Dict[str, Any], + ) -> Optional[int]: + """Return a Map state's array length from input or benchmark semantics.""" + array_path = state.get("array") + if not array_path: + return 0 + + if benchmark_name in {"630.parallel-sleep", "631.parallel-download"}: + if array_path == "buffer": + return self._workflow_int_value(prepared_input, "count") + + if benchmark_name in { + "6100.1000-genome", + "6101.1000-genome-individuals", + }: + if array_path == "blob": + return self._workflow_list_length(prepared_input, "blob") + if array_path == "sifting.populations": + return self._workflow_list_length(prepared_input, "populations") + + if benchmark_name == "650.vid" and array_path == "frames": + return self._workflow_chunked_length( + prepared_input, "n_frames", "batch_size" + ) + + if benchmark_name == "680.excamera" and array_path == "segments": + segments = self._workflow_list_length(prepared_input, "segments") + batch_size = self._workflow_int_value(prepared_input, "batch_size") + if segments is None or batch_size is None: + return None + return math.ceil(segments / max(1, batch_size)) + + if benchmark_name == "690.ml" and array_path == "schedules": + return self._workflow_list_length(prepared_input, "classifiers") + + if benchmark_name == "660.map-reduce" and array_path == "list": + if state_name == "map-state": + return self._workflow_int_value(prepared_input, "n_mappers") + if state_name == "reduce-state": + return 5 + + value = self._workflow_value_at_path(prepared_input, array_path) + if isinstance(value, list): + return len(value) + + return None + + @staticmethod + def _workflow_value_at_path(data: Dict[str, Any], path: str) -> Any: + """Return a dotted-path value from a dictionary, or None.""" + value: Any = data + for part in path.split("."): + if not isinstance(value, dict) or part not in value: + return None + value = value[part] + return value + + @staticmethod + def _workflow_int_value(data: Dict[str, Any], key: str) -> Optional[int]: + """Return a positive integer value from prepared input.""" + value = data.get(key) + if value is None: + return None + try: + return max(0, int(value)) + except (TypeError, ValueError): + return None + + @staticmethod + def _workflow_list_length(data: Dict[str, Any], key: str) -> Optional[int]: + """Return the length of a list value from prepared input.""" + value = data.get(key) + if isinstance(value, list): + return len(value) + return None + + def _workflow_chunked_length( + self, data: Dict[str, Any], item_key: str, batch_key: str + ) -> Optional[int]: + """Return ceil(item count / batch size) for task-produced batches.""" + item_count = self._workflow_int_value(data, item_key) + batch_size = self._workflow_int_value(data, batch_key) + if item_count is None or batch_size is None: + return None + return math.ceil(item_count / max(1, batch_size)) + + @staticmethod + def _workflow_instance_type(code_package: Benchmark) -> str: + """Choose a Cloudflare Container instance type from benchmark memory.""" + if code_package.benchmark in { + "6100.1000-genome", + "6101.1000-genome-individuals", + }: + return "standard-4" + + memory = code_package.benchmark_config.memory + if memory <= 256: + return "lite" + if memory <= 1024: + return "basic" + if memory <= 2048: + return "standard-2" + if memory <= 4096: + return "standard-3" + if memory <= 8192: + return "standard-4" + return "standard-4" + + def _deploy_workflow_orchestrator( + self, cli, package_path: str, env: Dict[str, str], orchestrator_name: str + ) -> str: + """Deploy the workflow orchestrator, recreating stale container apps if needed.""" + try: + return cli.wrangler_deploy(package_path, env=env) + except RuntimeError as exc: + message = str(exc) + if "APPLICATION_NOT_FOUND" not in message: + raise + self.logging.warning( + f"Wrangler reported APPLICATION_NOT_FOUND while deploying " + f"{orchestrator_name}; deleting stale Worker state and retrying once." + ) + try: + cli.wrangler_delete(orchestrator_name, env=env) + except RuntimeError as delete_exc: + self.logging.warning( + f"Failed to delete stale Worker {orchestrator_name}: {delete_exc}" + ) + return cli.wrangler_deploy(package_path, env=env) + + def _get_workers_dev_subdomain(self, account_id: str) -> Optional[str]: + """Fetch the workers.dev subdomain for the given account. + + Cloudflare exposes an endpoint that returns the account-level workers + subdomain (the readable name used in *.workers.dev), e.g. + GET /accounts/{account_id}/workers/subdomain + + Returns the subdomain string or None on failure. + """ + if self._workers_dev_subdomain: + return self._workers_dev_subdomain + + try: + headers = self._get_auth_headers() + url = f"{self._api_base_url}/accounts/{account_id}/workers/subdomain" + resp = requests.get(url, headers=headers) + if resp.status_code == 200: + body = resp.json() + sub = None + # result may contain 'subdomain' or nested structure + if isinstance(body, dict): + sub = body.get("result", {}).get("subdomain") + + if sub: + self._workers_dev_subdomain = sub + return sub + else: + raise RuntimeError( + "Could not find workers.dev subdomain in API response; " + "please enable the workers.dev subdomain in your Cloudflare dashboard." + ) + else: + self.logging.warning( + f"Failed to fetch workers.dev subdomain: {resp.status_code} - {resp.text}" + ) + return None + except Exception as e: + self.logging.warning(f"Error fetching workers.dev subdomain: {e}") + return None + + def _build_workers_dev_url( + self, worker_name: str, account_id: Optional[str] + ) -> str: + """Build a best-effort public URL for a worker. + + Prefer using the account's readable workers.dev subdomain when available + (e.g. ..workers.dev). If we can't obtain that, fall + back to using the account_id as a last resort and log a warning. + """ + if account_id: + sub = self._get_workers_dev_subdomain(account_id) + return f"https://{worker_name}.{sub}.workers.dev" + # Last fallback: plain workers.dev (may not resolve without a subdomain) + self.logging.warning( + "No account ID available; using https://{name}.workers.dev which may not be reachable." + ) + return f"https://{worker_name}.workers.dev" + + def cached_function(self, function: Function): + """ + Handle a function retrieved from cache. + + Refreshes triggers and logging handlers, and verifies the worker still + exists on Cloudflare. If it has been deleted remotely, clear the hash + so the caller's hash-mismatch path triggers a full redeployment. + + Args: + function: The cached function + """ + for trigger in function.triggers(Trigger.TriggerType.HTTP): + trigger.logging_handlers = self.logging_handlers + + worker = cast(CloudflareWorker, function) + account_id = worker.account_id or self.config.credentials.account_id + if account_id and not self._get_worker(worker.name, account_id): + self.logging.info( + f"Cached worker {worker.name} no longer exists on Cloudflare " + "— will redeploy." + ) + function.code_package_hash = "" + + def update_function( + self, + function: Function, + code_package: Benchmark, + system_variant: SystemVariant, + container_uri: str | None, + ): + """ + Update an existing Cloudflare Worker. + + Args: + function: Existing function instance to update + code_package: New benchmark containing the function code + system_variant: Selected deployment variant + container_uri: URI of container image + """ + container_deployment = system_variant.is_container + worker = cast(CloudflareWorker, function) + package = code_package.code_location + if package is None and container_deployment: + package = self._container_adapter.last_directory + language = code_package.language_name + benchmark = code_package.benchmark + + # Update the worker with all package files + account_id = worker.account_id or self.config.credentials.account_id + if not account_id: + raise RuntimeError("Account ID is required to update worker") + + if package is None and container_deployment: + output_dir = code_package._output_dir + if os.path.isdir(output_dir): + package = output_dir + if package is None: + raise RuntimeError( + f"Code location is not set for {benchmark}. " + "The build step may not have completed successfully." + ) + self._create_or_update_worker( + worker.name, + package, + account_id, + language, + benchmark, + code_package, + container_deployment, + container_uri, + ) + self.logging.info(f"Updated worker {worker.name}") + + # Update configuration if needed (no-op for containers: no runtime memory changes) + self.update_function_configuration(worker, code_package) + + def update_function_configuration( + self, cached_function: Function, benchmark: Benchmark + ): + """ + Update the configuration of a Cloudflare Worker. + + Note: Cloudflare Workers have limited configuration options compared + to traditional FaaS platforms. Memory and timeout are managed by Cloudflare. + + Args: + cached_function: The function to update + benchmark: The benchmark with new configuration + """ + # Cloudflare Workers have fixed resource limits: + # - CPU time: 50ms (free), 50ms-30s (paid) + # - Memory: 128MB + # Most configuration is handled via wrangler.toml or API settings + + worker = cast(CloudflareWorker, cached_function) + + # For environment variables or KV namespaces, we would use the API here + # For now, we'll just log that configuration update was requested + self.logging.warning( + f"Configuration update requested for worker {worker.name}. " + "Note: Cloudflare Workers have limited runtime configuration options." + ) + + def default_function_name(self, code_package: Benchmark, resources=None) -> str: + """ + Generate a default function name for Cloudflare Workers. + + Args: + code_package: The benchmark package + resources: Optional resources (not used) + + Returns: + Default function name + """ + # Cloudflare Worker names must be lowercase and can contain hyphens. + # Abbreviate language names to keep names under the 54-char limit for workers.dev. + lang_abbrev = {"python": "py", "nodejs": "js", "java": "java", "cpp": "cpp"} + lang = lang_abbrev.get(code_package.language_name, code_package.language_name) + name = ( + f"{code_package.benchmark}-{lang}" + f"{code_package.language_version.replace('.', '')}" + ).lower() + if code_package.language_variant != "default": + name = f"{name}-{code_package.language_variant}" + return name + + @staticmethod + def format_function_name(name: str, container_deployment: bool = False) -> str: + """ + Format a function name to comply with Cloudflare Worker naming rules. + + Worker names must: + - Be lowercase + - Contain only alphanumeric characters and hyphens + - Not start or end with a hyphen + - Not start with a digit + + Args: + name: The original name + container_deployment: Whether this is a container worker + (adds 'w-' prefix if name starts with digit) + + Returns: + Formatted name + """ + # Convert to lowercase and replace invalid characters + formatted = name.lower().replace("_", "-").replace(".", "-") + # Remove any characters that aren't alphanumeric or hyphen + formatted = "".join(c for c in formatted if c.isalnum() or c == "-") + # Remove leading/trailing hyphens + formatted = formatted.strip("-") + # Ensure container worker names don't start with a digit (Cloudflare requirement) + # Only add prefix for container workers to differentiate from native workers + if container_deployment and formatted and formatted[0].isdigit(): + formatted = "container-" + formatted + return formatted + + def enforce_cold_start(self, functions: List[Function], code_package: Benchmark): + """ + Enforce cold start for Cloudflare Workers. + + Note: Cloudflare Workers don't have a traditional cold start mechanism + like AWS Lambda. Workers are instantiated on-demand at edge locations. + We can't force a cold start, but we can update the worker to invalidate caches. + + Args: + functions: List of functions to enforce cold start on + code_package: The benchmark package + """ + raise NotImplementedError( + "Cloudflare Workers do not support forced cold starts. " + "Workers are automatically instantiated on-demand at edge locations." + ) + + def download_metrics( + self, + function_name: str, + start_time: int, + end_time: int, + requests: Dict[str, ExecutionResult], + metrics: dict, + ): + """ + Extract per-invocation metrics from ExecutionResult objects. + + The metrics are extracted from the 'measurement' field in the benchmark + response, which is populated by the Cloudflare Worker handler during execution. + This approach avoids dependency on Analytics Engine and provides immediate, + accurate metrics for each invocation. + + Args: + function_name: Name of the worker + start_time: Start time (Unix timestamp in seconds) - not used + end_time: End time (Unix timestamp in seconds) - not used + requests: Dict mapping request_id -> ExecutionResult + metrics: Dict to store aggregated metrics + """ + if not requests: + self.logging.warning("No requests to extract metrics from") + return + + self.logging.info( + f"Extracting metrics from {len(requests)} invocations " + f"of worker {function_name}" + ) + + # Aggregate statistics from all requests + total_invocations = len(requests) + cold_starts = 0 + warm_starts = 0 + cpu_times = [] + wall_times = [] + memory_values = [] + + for request_id, result in requests.items(): + # Count cold/warm starts + if result.stats.cold_start: + cold_starts += 1 + else: + warm_starts += 1 + + # Collect CPU times + if result.provider_times.execution > 0: + cpu_times.append(result.provider_times.execution) + + # Collect wall times (benchmark times) + if result.times.benchmark > 0: + wall_times.append(result.times.benchmark) + + # Collect memory usage + if result.stats.memory_used is not None and result.stats.memory_used > 0: + memory_values.append(result.stats.memory_used) + + # Set billing info for Cloudflare Workers + # Cloudflare billing: $0.50 per million requests + + # $12.50 per million GB-seconds of CPU time + if result.provider_times.execution > 0: + result.billing.memory = 128 # Cloudflare Workers: fixed 128MB + result.billing.billed_time = result.provider_times.execution # μs + + # GB-seconds calculation: (128MB / 1024MB/GB) * (cpu_time_us / 1000000 us/s) + cpu_time_seconds = result.provider_times.execution / 1_000_000.0 + gb_seconds = (128.0 / 1024.0) * cpu_time_seconds + result.billing.gb_seconds = int( + gb_seconds * 1_000_000 + ) # micro GB-seconds + + # Calculate statistics + metrics["cloudflare"] = { + "total_invocations": total_invocations, + "cold_starts": cold_starts, + "warm_starts": warm_starts, + "data_source": "response_measurements", + "note": "Per-invocation metrics extracted from benchmark response", + } + + if cpu_times: + metrics["cloudflare"]["avg_cpu_time_us"] = sum(cpu_times) // len(cpu_times) + metrics["cloudflare"]["min_cpu_time_us"] = min(cpu_times) + metrics["cloudflare"]["max_cpu_time_us"] = max(cpu_times) + metrics["cloudflare"]["cpu_time_measurements"] = len(cpu_times) + + if wall_times: + metrics["cloudflare"]["avg_wall_time_us"] = sum(wall_times) // len( + wall_times + ) + metrics["cloudflare"]["min_wall_time_us"] = min(wall_times) + metrics["cloudflare"]["max_wall_time_us"] = max(wall_times) + metrics["cloudflare"]["wall_time_measurements"] = len(wall_times) + + if memory_values: + metrics["cloudflare"]["avg_memory_mb"] = sum(memory_values) / len( + memory_values + ) + metrics["cloudflare"]["min_memory_mb"] = min(memory_values) + metrics["cloudflare"]["max_memory_mb"] = max(memory_values) + metrics["cloudflare"]["memory_measurements"] = len(memory_values) + + self.logging.info( + f"Extracted metrics from {total_invocations} invocations: " + f"{cold_starts} cold starts, {warm_starts} warm starts" + ) + + if cpu_times: + avg_cpu_ms = sum(cpu_times) / len(cpu_times) / 1000.0 + self.logging.info(f"Average CPU time: {avg_cpu_ms:.2f} ms") + + if wall_times: + avg_wall_ms = sum(wall_times) / len(wall_times) / 1000.0 + self.logging.info(f"Average wall time: {avg_wall_ms:.2f} ms") + + @staticmethod + def workflow_type() -> "Type[Function]": + """Return the Workflow subclass used by this platform.""" + from sebs.cloudflare.workflow import CloudflareWorkflow + + return CloudflareWorkflow + + def create_workflow(self, code_package: Benchmark, workflow_name: str) -> Function: + """Deploy a Cloudflare Workflow: dispatcher + orchestrator. + + 1. Deploys a dispatcher worker/container with all task functions. + 2. Generates a TypeScript orchestrator from definition.json. + 3. Deploys the orchestrator as a Cloudflare Workflow. + + Args: + code_package: Benchmark containing the workflow code. + workflow_name: Name for the workflow. + + Returns: + CloudflareWorkflow instance with trigger attached. + """ + import os + import tempfile + + from sebs.cloudflare.generator import CloudflareWorkflowGenerator + from sebs.cloudflare.triggers import WorkflowLibraryTrigger + from sebs.cloudflare.workflow import CloudflareWorkflow + + container_deployment = code_package.system_variant.is_container + workflow_name = self.format_function_name(workflow_name, container_deployment) + account_id = self.config.credentials.account_id + + if not account_id: + raise RuntimeError("Cloudflare account ID is required to create workflows") + + if not container_deployment: + raise RuntimeError( + "Cloudflare workflow fan-out requires container deployment. " + "Select the cloudflare container system variant for workflow benchmarks." + ) + + # Cloudflare workers.dev subdomains cap at 54 chars. + # Cap the base name at 43 chars so that derived names stay ≤ 54. + max_base_len = 43 + if len(workflow_name) > max_base_len: + workflow_name = workflow_name[:max_base_len].rstrip("-") + dispatcher_name = workflow_name + "-dispatcher" + container_uri = code_package._container_uri if container_deployment else None + if not container_uri: + raise RuntimeError( + f"Container image URI is missing for workflow {code_package.benchmark}. " + "The container build step may not have completed successfully." + ) + + # Set up Wrangler credentials before pushing the dispatcher image and + # deploying the orchestrator Worker. + env = {} + if self.config.credentials.api_token: + env["CLOUDFLARE_API_TOKEN"] = self.config.credentials.api_token + elif self.config.credentials.email and self.config.credentials.api_key: + env["CLOUDFLARE_EMAIL"] = self.config.credentials.email + env["CLOUDFLARE_API_KEY"] = self.config.credentials.api_key + env["CLOUDFLARE_ACCOUNT_ID"] = account_id + + cli = self._workers_deployment._get_cli() + self.logging.info(f"Pushing workflow dispatcher image {container_uri}...") + dispatcher_image = cli.containers_push(container_uri, env=env) + self.logging.info(f"Workflow dispatcher image pushed to: {dispatcher_image}") + + # --- Step 2: Generate orchestrator TypeScript from definition.json --- + definition_path = os.path.join(code_package.benchmark_path, "definition.json") + if not os.path.exists(definition_path): + raise ValueError(f"No workflow definition found at {definition_path}") + + workflow_max_instances = self._workflow_max_instances( + code_package, definition_path + ) + workflow_instance_type = self._workflow_instance_type(code_package) + self.logging.info( + f"Cloudflare workflow {code_package.benchmark} instance_type=" + f"{workflow_instance_type} for configured memory " + f"{code_package.benchmark_config.memory} MB." + ) + + gen = CloudflareWorkflowGenerator( + chunk_size=self.config.chunk_size, + max_instances=workflow_max_instances, + dispatch_timeout_seconds=code_package.benchmark_config.timeout + 120, + ) + gen.parse(definition_path) + ts_source = gen.generate() + + # --- Step 3: Package and deploy the orchestrator --- + orchestrator_name = workflow_name + orchestrator_dir = tempfile.mkdtemp(prefix="sebs-workflow-orchestrator-") + + # Write generated workflow TypeScript + ts_path = os.path.join(orchestrator_dir, "workflow.ts") + with open(ts_path, "w") as f: + f.write(ts_source) + + # Write minimal package.json + package_json = { + "name": orchestrator_name, + "type": "module", + "dependencies": { + "@cloudflare/containers": "*", + "@cloudflare/workers-types": "*", + }, + } + with open(os.path.join(orchestrator_dir, "package.json"), "w") as f: + import json as json_mod + + json_mod.dump(package_json, f, indent=2) + + orchestrator_url = self._build_workers_dev_url(orchestrator_name, account_id) + self._generate_workflow_wrangler_toml( + orchestrator_name, + orchestrator_dir, + account_id, + dispatcher_image, + workflow_max_instances, + workflow_instance_type, + orchestrator_url, + code_package, + ) + + container_package_path = f"/tmp/workers/{orchestrator_name}" + cli.upload_package(orchestrator_dir, container_package_path) + + self.logging.info(f"Deploying workflow orchestrator: {orchestrator_name}") + output = self._deploy_workflow_orchestrator( + cli, container_package_path, env, orchestrator_name + ) + if "no changes" not in output.lower(): + self._wait_for_container_rollout( + self._workflow_container_name(orchestrator_name), + account_id, + ) + + # Build orchestrator URL and wait for readiness + self._wait_for_worker_ready(orchestrator_name, orchestrator_url) + + # --- Step 4: Create workflow object and attach trigger --- + function_cfg = FunctionConfig.from_benchmark(code_package) + workflow = CloudflareWorkflow( + name=orchestrator_name, + functions=[], + benchmark=code_package.benchmark, + code_package_hash=code_package.hash, + cfg=function_cfg, + account_id=account_id, + dispatcher_name=dispatcher_name, + orchestrator_url=orchestrator_url, + ) + + trigger = WorkflowLibraryTrigger(orchestrator_name, orchestrator_url) + trigger.logging_handlers = self.logging_handlers + workflow.add_trigger(trigger) + + self.logging.info(f"Workflow {orchestrator_name} deployed successfully") + return workflow + + def update_workflow(self, workflow: Function, code_package: Benchmark): + """Update an existing Cloudflare Workflow deployment. + + Pushes the dispatcher image and regenerates/re-deploys the orchestrator. + + Args: + workflow: Existing CloudflareWorkflow instance. + code_package: Updated benchmark code package. + """ + import os + import tempfile + + from sebs.cloudflare.generator import CloudflareWorkflowGenerator + from sebs.cloudflare.workflow import CloudflareWorkflow + + workflow = cast(CloudflareWorkflow, workflow) + account_id = workflow.account_id + container_deployment = code_package.system_variant.is_container + if not container_deployment: + raise RuntimeError( + "Cloudflare workflow fan-out requires container deployment. " + "Select the cloudflare container system variant for workflow benchmarks." + ) + + container_uri = code_package._container_uri + if not container_uri: + raise RuntimeError( + f"Container image URI is missing for workflow {code_package.benchmark}. " + "The container build step may not have completed successfully." + ) + + env = {} + if self.config.credentials.api_token: + env["CLOUDFLARE_API_TOKEN"] = self.config.credentials.api_token + elif self.config.credentials.email and self.config.credentials.api_key: + env["CLOUDFLARE_EMAIL"] = self.config.credentials.email + env["CLOUDFLARE_API_KEY"] = self.config.credentials.api_key + env["CLOUDFLARE_ACCOUNT_ID"] = account_id + + cli = self._workers_deployment._get_cli() + self.logging.info(f"Pushing workflow dispatcher image {container_uri}...") + dispatcher_image = cli.containers_push(container_uri, env=env) + self.logging.info(f"Workflow dispatcher image pushed to: {dispatcher_image}") + workflow.functions = [] + + # Regenerate and redeploy orchestrator + definition_path = os.path.join(code_package.benchmark_path, "definition.json") + if not os.path.exists(definition_path): + raise ValueError(f"No workflow definition found at {definition_path}") + + workflow_max_instances = self._workflow_max_instances( + code_package, definition_path + ) + workflow_instance_type = self._workflow_instance_type(code_package) + self.logging.info( + f"Cloudflare workflow {code_package.benchmark} instance_type=" + f"{workflow_instance_type} for configured memory " + f"{code_package.benchmark_config.memory} MB." + ) + + gen = CloudflareWorkflowGenerator( + chunk_size=self.config.chunk_size, + max_instances=workflow_max_instances, + dispatch_timeout_seconds=code_package.benchmark_config.timeout + 120, + ) + gen.parse(definition_path) + ts_source = gen.generate() + + orchestrator_dir = tempfile.mkdtemp(prefix="sebs-workflow-orchestrator-") + with open(os.path.join(orchestrator_dir, "workflow.ts"), "w") as f: + f.write(ts_source) + + package_json = { + "name": workflow.name, + "type": "module", + "dependencies": { + "@cloudflare/containers": "*", + "@cloudflare/workers-types": "*", + }, + } + with open(os.path.join(orchestrator_dir, "package.json"), "w") as f: + import json as json_mod + + json_mod.dump(package_json, f, indent=2) + + orchestrator_url = self._build_workers_dev_url(workflow.name, account_id) + self._generate_workflow_wrangler_toml( + workflow.name, + orchestrator_dir, + account_id, + dispatcher_image, + workflow_max_instances, + workflow_instance_type, + orchestrator_url, + code_package, + ) + + container_package_path = f"/tmp/workers/{workflow.name}" + cli.upload_package(orchestrator_dir, container_package_path) + + self.logging.info(f"Redeploying workflow orchestrator: {workflow.name}") + output = self._deploy_workflow_orchestrator( + cli, container_package_path, env, workflow.name + ) + if "no changes" not in output.lower(): + self._wait_for_container_rollout( + self._workflow_container_name(workflow.name), + account_id, + ) + self._wait_for_worker_ready(workflow.name, workflow.orchestrator_url) + + self.logging.info(f"Workflow {workflow.name} updated successfully") + + def _generate_workflow_wrangler_toml( + self, + orchestrator_name: str, + package_dir: str, + account_id: str, + dispatcher_image: str, + max_instances: int, + instance_type: str, + worker_url: str, + code_package: Optional[Benchmark] = None, + ) -> str: + """Generate wrangler.toml for the workflow orchestrator from template. + + Args: + orchestrator_name: Name of the orchestrator worker. + package_dir: Directory to write the toml file. + account_id: Cloudflare account ID. + dispatcher_image: Cloudflare registry image for DispatcherContainer. + max_instances: Maximum DispatcherContainer instances. + instance_type: Cloudflare Container instance type. + worker_url: Public orchestrator URL used by containers for R2/KV proxy calls. + code_package: Optional benchmark package for storage and nosql bindings. + + Returns: + Path to the generated wrangler.toml. + """ + try: + import tomllib + except ImportError: + import tomli as tomllib # type: ignore[no-redef] + try: + import tomli_w + except ImportError: + import toml as tomli_w # type: ignore[no-redef, import-untyped] + + from importlib.resources import files + + template_path = ( + files("sebs.cloudflare") + .joinpath("templates") + .joinpath("wrangler-workflow.toml") + ) + with template_path.open("rb") as f: + config = tomllib.load(f) + + config["name"] = orchestrator_name + config["account_id"] = account_id + config["workflows"][0]["name"] = orchestrator_name + config["workflows"][1]["name"] = f"{orchestrator_name}-item" + config["containers"][0]["image"] = dispatcher_image + config["containers"][0]["max_instances"] = max_instances + config["containers"][0]["instance_type"] = instance_type + config["vars"] = {"WORKER_URL": worker_url} + if code_package: + config["vars"]["BENCHMARK_NAME"] = code_package.benchmark + + if code_package and code_package.uses_nosql: + nosql_storage = self.system_resources.get_nosql_storage() + if nosql_storage.retrieve_cache(code_package.benchmark): + nosql_tables = nosql_storage.get_tables(code_package.benchmark) + if nosql_tables: + config["kv_namespaces"] = config.get("kv_namespaces", []) + for table_name, namespace_id in nosql_tables.items(): + config["kv_namespaces"].append( + { + "binding": table_name, + "id": namespace_id, + } + ) + config["vars"]["NOSQL_STORAGE_DATABASE"] = "kvstore" + + if code_package and code_package.uses_storage: + from sebs.faas.config import Resources + + storage = self.system_resources.get_storage() + bucket_name = storage.get_bucket(Resources.StorageBucketType.BENCHMARKS) + if not bucket_name: + raise RuntimeError( + "R2 bucket binding not configured: benchmarks bucket name is empty. " + "Workflow benchmarks requiring file access will not work properly." + ) + config["r2_buckets"] = [{"binding": "R2", "bucket_name": bucket_name}] + self.logging.info( + f"R2 bucket '{bucket_name}' will be bound to workflow as 'R2'" + ) + + toml_path = os.path.join(package_dir, "wrangler.toml") + try: + with open(toml_path, "wb") as f: + tomli_w.dump(config, f) + except TypeError: + with open(toml_path, "w") as f: + f.write(tomli_w.dumps(config)) + + self.logging.info(f"Generated workflow wrangler.toml at {toml_path}") + return toml_path + + def create_trigger( + self, function: Function, trigger_type: Trigger.TriggerType + ) -> Trigger: + """ + Create a trigger for a Cloudflare Worker. + + Args: + function: The function to create a trigger for + trigger_type: Type of trigger to create + + Returns: + The created trigger + """ + from sebs.cloudflare.triggers import HTTPTrigger + + worker = cast(CloudflareWorker, function) + + if trigger_type == Trigger.TriggerType.HTTP: + account_id = worker.account_id or self.config.credentials.account_id + worker_url = self._build_workers_dev_url(worker.name, account_id) + trigger = HTTPTrigger(worker.name, worker_url) + trigger.logging_handlers = self.logging_handlers + return trigger + else: + raise NotImplementedError( + f"Trigger type {trigger_type} is not supported for Cloudflare Workers" + ) + + def shutdown(self) -> None: + """ + Shutdown the Cloudflare system. + + Saves configuration to cache and shuts down deployment handler CLI containers. + """ + try: + self.cache_client.lock() + self.config.update_cache(self.cache_client) + finally: + self.cache_client.unlock() + + self._workers_deployment.shutdown() + self._containers_deployment.shutdown() diff --git a/sebs/cloudflare/config.py b/sebs/cloudflare/config.py new file mode 100644 index 000000000..819652364 --- /dev/null +++ b/sebs/cloudflare/config.py @@ -0,0 +1,341 @@ +"""Configuration classes for the Cloudflare Workers platform.""" + +import os +from typing import Optional, cast + +from sebs.cache import Cache +from sebs.faas.config import Config, Credentials, Resources +from sebs.utils import LoggingHandlers + + +class CloudflareCredentials(Credentials): + """ + Cloudflare API credentials. + + Two mutually exclusive authentication methods are supported; both are + functionally equivalent for every SeBS operation (API calls, R2, KV, + wrangler): + + - **API Token** (recommended): a scoped, revocable token created in the + Cloudflare dashboard. Env: ``CLOUDFLARE_API_TOKEN``. + - **Email + Global API Key** (legacy): the account email plus the + Global API Key. Grants broad account access; use only when scoped + tokens are not available. Env: ``CLOUDFLARE_EMAIL`` + + ``CLOUDFLARE_API_KEY``. + + Both methods additionally require ``CLOUDFLARE_ACCOUNT_ID``. + Optional R2 S3-compatible credentials (``CLOUDFLARE_R2_ACCESS_KEY_ID``, + ``CLOUDFLARE_R2_SECRET_ACCESS_KEY``) are needed for file uploads. + + See ``docs/platforms.md`` (Cloudflare Workers → Credentials) for full + setup instructions. + """ + + def __init__( + self, + api_token: Optional[str] = None, + email: Optional[str] = None, + api_key: Optional[str] = None, + account_id: Optional[str] = None, + r2_access_key_id: Optional[str] = None, + r2_secret_access_key: Optional[str] = None, + ): + """Store Cloudflare API credentials supplied at construction time.""" + super().__init__() + + self._api_token = api_token + self._email = email + self._api_key = api_key + self._account_id = account_id + self._r2_access_key_id = r2_access_key_id + self._r2_secret_access_key = r2_secret_access_key + + @staticmethod + def typename() -> str: + """Return the canonical type name for this credentials class.""" + return "Cloudflare.Credentials" + + @property + def api_token(self) -> Optional[str]: + """Scoped API token for Cloudflare authentication.""" + return self._api_token + + @property + def email(self) -> Optional[str]: + """Account email used with the Global API Key authentication method.""" + return self._email + + @property + def api_key(self) -> Optional[str]: + """Global API Key used with the email authentication method.""" + return self._api_key + + @property + def account_id(self) -> Optional[str]: + """Cloudflare account ID required for all API operations.""" + return self._account_id + + @property + def r2_access_key_id(self) -> Optional[str]: + """S3-compatible access key ID for R2 bucket operations.""" + return self._r2_access_key_id + + @property + def r2_secret_access_key(self) -> Optional[str]: + """S3-compatible secret access key for R2 bucket operations.""" + return self._r2_secret_access_key + + @staticmethod + def initialize(dct: dict) -> "CloudflareCredentials": + """Build a CloudflareCredentials instance from a plain dictionary.""" + return CloudflareCredentials( + dct.get("api_token"), + dct.get("email"), + dct.get("api_key"), + dct.get("account_id"), + dct.get("r2_access_key_id"), + dct.get("r2_secret_access_key"), + ) + + @staticmethod + def deserialize( + config: dict, cache: Cache, handlers: LoggingHandlers + ) -> Credentials: + """Load credentials from config dict, falling back to environment variables.""" + cached_config = cache.get_config("cloudflare") + ret: CloudflareCredentials + account_id: Optional[str] = None + + # Load cached values + if cached_config and "credentials" in cached_config: + account_id = cached_config["credentials"].get("account_id") + + # Check for new config + if "credentials" in config: + ret = CloudflareCredentials.initialize(config["credentials"]) + elif "CLOUDFLARE_API_TOKEN" in os.environ: + ret = CloudflareCredentials( + api_token=os.environ["CLOUDFLARE_API_TOKEN"], + account_id=os.environ.get("CLOUDFLARE_ACCOUNT_ID"), + r2_access_key_id=os.environ.get("CLOUDFLARE_R2_ACCESS_KEY_ID"), + r2_secret_access_key=os.environ.get("CLOUDFLARE_R2_SECRET_ACCESS_KEY"), + ) + elif "CLOUDFLARE_EMAIL" in os.environ and "CLOUDFLARE_API_KEY" in os.environ: + ret = CloudflareCredentials( + email=os.environ["CLOUDFLARE_EMAIL"], + api_key=os.environ["CLOUDFLARE_API_KEY"], + account_id=os.environ.get("CLOUDFLARE_ACCOUNT_ID"), + r2_access_key_id=os.environ.get("CLOUDFLARE_R2_ACCESS_KEY_ID"), + r2_secret_access_key=os.environ.get("CLOUDFLARE_R2_SECRET_ACCESS_KEY"), + ) + else: + raise RuntimeError( + "Cloudflare login credentials are missing! Please set " + "up environmental variables CLOUDFLARE_API_TOKEN and CLOUDFLARE_ACCOUNT_ID, " + "or CLOUDFLARE_EMAIL, CLOUDFLARE_API_KEY, and CLOUDFLARE_ACCOUNT_ID" + ) + + if ( + account_id is not None + and ret.account_id is not None + and account_id != ret.account_id + ): + ret.logging.error( + f"The account id {ret.account_id} from provided credentials is different " + f"from the account id {account_id} found in the cache! Please change " + "your cache directory or create a new one!" + ) + raise RuntimeError( + f"Cloudflare login credentials do not match the account {account_id} in cache!" + ) + + ret.logging_handlers = handlers + return ret + + def update_cache(self, cache: Cache): + """Persist the account ID to the local cache.""" + if self._account_id: + cache.update_config( + val=self._account_id, keys=["cloudflare", "credentials", "account_id"] + ) + + def serialize(self) -> dict: + """Return a serializable dict of non-secret credential fields.""" + out = {} + if self._account_id: + out["account_id"] = self._account_id + return out + + +class CloudflareResources(Resources): + """ + Resources for Cloudflare Workers deployment. + """ + + def __init__(self): + """Initialize Cloudflare resources with no namespace ID assigned.""" + super().__init__(name="cloudflare") + self._namespace_id: Optional[str] = None + + @staticmethod + def typename() -> str: + """Return the canonical type name for this resources class.""" + return "Cloudflare.Resources" + + @property + def namespace_id(self) -> Optional[str]: + """KV namespace ID associated with this resource deployment.""" + return self._namespace_id + + @namespace_id.setter + def namespace_id(self, value: str): + """Set the KV namespace ID for this resource deployment.""" + self._namespace_id = value + + @staticmethod + def initialize(res: Resources, dct: dict): + """Populate a CloudflareResources instance from a config dictionary.""" + ret = cast(CloudflareResources, res) + super(CloudflareResources, CloudflareResources).initialize(ret, dct) + + if "namespace_id" in dct: + ret._namespace_id = dct["namespace_id"] + + return ret + + def serialize(self) -> dict: + """Return a serializable dict of Cloudflare resource fields.""" + out = {**super().serialize()} + if self._namespace_id: + out["namespace_id"] = self._namespace_id + return out + + def update_cache(self, cache: Cache): + """Persist resource IDs to the local cache.""" + super().update_cache(cache) + if self._namespace_id: + cache.update_config( + val=self._namespace_id, keys=["cloudflare", "resources", "namespace_id"] + ) + + @staticmethod + def deserialize(config: dict, cache: Cache, handlers: LoggingHandlers) -> Resources: + """Load resources from cached or user-provided configuration.""" + ret = CloudflareResources() + cached_config = cache.get_config("cloudflare") + + # Load cached values + if cached_config and "resources" in cached_config: + CloudflareResources.initialize(ret, cached_config["resources"]) + ret.logging_handlers = handlers + ret.logging.info("Using cached resources for Cloudflare") + else: + # Check for new config + if "resources" in config: + CloudflareResources.initialize(ret, config["resources"]) + ret.logging_handlers = handlers + ret.logging.info( + "No cached resources for Cloudflare found, using user configuration." + ) + else: + CloudflareResources.initialize(ret, {}) + ret.logging_handlers = handlers + ret.logging.info("No resources for Cloudflare found, initialize!") + + return ret + + +class CloudflareConfig(Config): + """ + Configuration for Cloudflare Workers platform. + """ + + def __init__( + self, credentials: CloudflareCredentials, resources: CloudflareResources + ): + """Initialize configuration with the given credentials and resources.""" + super().__init__(name="cloudflare") + self._credentials = credentials + self._resources = resources + self._max_instances: int = 20 + self._chunk_size: int = 1 + + @staticmethod + def typename() -> str: + """Return the canonical type name for this configuration class.""" + return "Cloudflare.Config" + + @property + def credentials(self) -> CloudflareCredentials: + """Cloudflare API credentials for this configuration.""" + return self._credentials + + @property + def resources(self) -> CloudflareResources: + """Cloudflare resource identifiers for this deployment.""" + return self._resources + + @property + def max_instances(self) -> int: + """Maximum number of container instances for container deployments.""" + return self._max_instances + + @property + def chunk_size(self) -> int: + """Number of Map items assigned to one child workflow instance.""" + return self._chunk_size + + @staticmethod + def initialize(cfg: Config, dct: dict): + """Apply region and other fields from a config dictionary to an existing instance.""" + config = cast(CloudflareConfig, cfg) + # Cloudflare Workers are globally distributed, no region needed + config._region = dct.get("region", "global") + if "max_instances" in dct: + config._max_instances = int(dct["max_instances"]) + if "chunk_size" in dct: + config._chunk_size = max(1, int(dct["chunk_size"])) + + @staticmethod + def deserialize(config: dict, cache: Cache, handlers: LoggingHandlers) -> Config: + """Build a CloudflareConfig from user config and cache, resolving credentials.""" + cached_config = cache.get_config("cloudflare") + credentials = cast( + CloudflareCredentials, + CloudflareCredentials.deserialize(config, cache, handlers), + ) + resources = cast( + CloudflareResources, + CloudflareResources.deserialize(config, cache, handlers), + ) + config_obj = CloudflareConfig(credentials, resources) + config_obj.logging_handlers = handlers + + # Load cached values + if cached_config: + config_obj.logging.info("Using cached config for Cloudflare") + CloudflareConfig.initialize(config_obj, cached_config) + else: + config_obj.logging.info("Using user-provided config for Cloudflare") + CloudflareConfig.initialize(config_obj, config) + + resources.region = config_obj.region + return config_obj + + def update_cache(self, cache: Cache): + """Persist region, credentials, and resources to the local cache.""" + cache.update_config(val=self.region, keys=["cloudflare", "region"]) + self.credentials.update_cache(cache) + self.resources.update_cache(cache) + + def serialize(self) -> dict: + """Return a serializable dict of the full Cloudflare configuration.""" + out = { + "name": "cloudflare", + "region": self._region, + "max_instances": self._max_instances, + "chunk_size": self._chunk_size, + "credentials": self._credentials.serialize(), + "resources": self._resources.serialize(), + } + return out diff --git a/sebs/cloudflare/containers.py b/sebs/cloudflare/containers.py new file mode 100644 index 000000000..b7b86843a --- /dev/null +++ b/sebs/cloudflare/containers.py @@ -0,0 +1,394 @@ +""" +Cloudflare Container Workers deployment implementation. + +Handles packaging, Docker image building, and deployment of containerized +Cloudflare Workers using @cloudflare/containers. +""" + +import os +import shutil +import json +import subprocess + +import time +from importlib.resources import files + +try: + import tomllib # type: ignore[import-not-found] # Python 3.11+ +except ImportError: + import tomli as tomllib # type: ignore[no-redef, import-not-found] # Fallback for older Python +try: + import tomli_w +except ImportError: + import toml as tomli_w # type: ignore[no-redef, import-untyped] +from typing import Optional, Tuple + + +from sebs.benchmark import Benchmark +from sebs.cloudflare.cli import CloudflareCLI +from sebs.utils import get_resource_path + + +class CloudflareContainersDeployment: + """Handles Cloudflare container worker deployment operations.""" + + def __init__(self, logging, system_config, docker_client, system_resources): + """ + Initialize CloudflareContainersDeployment. + + Args: + logging: Logger instance + system_config: System configuration + docker_client: Docker client instance + system_resources: System resources manager + """ + self.logging = logging + self.system_config = system_config + self.docker_client = docker_client + self.system_resources = system_resources + self._base_image: Optional[str] = None + self._cli: Optional[CloudflareCLI] = None + self.max_instances: int = 10 + + def _get_cli(self) -> CloudflareCLI: + """Get or initialize the Cloudflare CLI container.""" + if self._cli is None: + self._cli = CloudflareCLI.get_instance(self.system_config, self.docker_client) + # Verify wrangler is available + version = self._cli.check_wrangler_version() + self.logging.info(f"Cloudflare CLI container ready: {version}") + return self._cli + + def generate_wrangler_toml( + self, + worker_name: str, + package_dir: str, + language: str, + account_id: str, + benchmark_name: Optional[str] = None, + code_package: Optional[Benchmark] = None, + container_uri: Optional[str] = None, + language_variant: str = "default", + ) -> str: + """ + Generate a wrangler.toml configuration file for container workers. + + Args: + worker_name: Name of the worker + package_dir: Directory containing the worker code + language: Programming language (nodejs or python) + account_id: Cloudflare account ID + benchmark_name: Optional benchmark name for R2 file path prefix + code_package: Optional benchmark package for nosql configuration + container_uri: Container image URI/tag + + Returns: + Path to the generated wrangler.toml file + """ + # Load template + template_path = ( + files("sebs.cloudflare").joinpath("templates").joinpath("wrangler-container.toml") + ) + with template_path.open("rb") as f: + config = tomllib.load(f) + + # Update basic configuration + config["name"] = worker_name + config["account_id"] = account_id + config["containers"][0]["max_instances"] = self.max_instances + + if container_uri and container_uri.startswith("registry.cloudflare.com"): + # Pre-built image already pushed to Cloudflare registry — point wrangler + # at it directly so it skips the Docker build step entirely. + config["containers"][0]["image"] = container_uri + else: + # Fallback: let wrangler build from the local Dockerfile. + if self._base_image: + config["containers"][0]["build_args"] = {"BASE_IMAGE": self._base_image} + + # Update container configuration with instance type if needed + if benchmark_name and ( + "411.image-recognition" in benchmark_name + or "311.compression" in benchmark_name + or "504.dna-visualisation" in benchmark_name + or "6100.1000-genome" in benchmark_name + or "6101.1000-genome-individuals" in benchmark_name + or "650.vid" in benchmark_name + or "680.excamera" in benchmark_name + ): + self.logging.warning("Using standard-4 instance type for high resource benchmark") + config["containers"][0]["instance_type"] = "standard-4" + + # Add nosql KV namespace bindings if benchmark uses them + if code_package and code_package.uses_nosql: + # Get registered nosql tables for this benchmark + nosql_storage = self.system_resources.get_nosql_storage() + benchmark_for_nosql = benchmark_name or code_package.benchmark + if nosql_storage.retrieve_cache(benchmark_for_nosql): + nosql_tables = nosql_storage.get_tables(benchmark_for_nosql) + if nosql_tables: + config["kv_namespaces"] = config.get("kv_namespaces", []) + for table_name, namespace_id in nosql_tables.items(): + config["kv_namespaces"].append( + { + "binding": table_name, + "id": namespace_id, + } + ) + + # Add environment variables + if benchmark_name or (code_package and code_package.uses_nosql): + config["vars"] = {} + if benchmark_name: + config["vars"]["BENCHMARK_NAME"] = benchmark_name + if code_package and code_package.uses_nosql: + config["vars"]["NOSQL_STORAGE_DATABASE"] = "kvstore" + + # Add R2 bucket binding + from sebs.faas.config import Resources + + storage = self.system_resources.get_storage() + bucket_name = storage.get_bucket(Resources.StorageBucketType.BENCHMARKS) + if not bucket_name: + raise RuntimeError( + "R2 bucket binding not configured: benchmarks bucket name is empty. " + "Benchmarks requiring file access will not work properly." + ) + config["r2_buckets"] = [{"binding": "R2", "bucket_name": bucket_name}] + self.logging.info(f"R2 bucket '{bucket_name}' will be bound to worker as 'R2'") + + # Write wrangler.toml to package directory + toml_path = os.path.join(package_dir, "wrangler.toml") + try: + # Try tomli_w (writes binary) + with open(toml_path, "wb") as f: + tomli_w.dump(config, f) + except TypeError: + # Fallback to toml library (writes text) + with open(toml_path, "w") as f: + f.write(tomli_w.dumps(config)) + + self.logging.info(f"Generated wrangler.toml at {toml_path}") + return toml_path + + def package_code( + self, + directory: str, + language_name: str, + language_version: str, + architecture: str, + benchmark: str, + ) -> Tuple[str, int, str]: + """ + Package code for Cloudflare container worker deployment. + + Builds a Docker image and returns the image tag for deployment. + + Args: + directory: Path to the code directory + language_name: Programming language name + language_version: Programming language version + architecture: Target architecture + benchmark: Benchmark name + + Returns: + Tuple of (package_path, package_size, container_uri) + """ + self.logging.info(f"Packaging container for {language_name} {language_version}") + + # Get wrapper directory for container files + wrapper_base = str(get_resource_path("benchmarks", "wrappers", "cloudflare")) + wrapper_container_dir = os.path.join(wrapper_base, language_name, "container") + + if not os.path.exists(wrapper_container_dir): + raise RuntimeError(f"Container wrapper directory not found: {wrapper_container_dir}") + + # Overwrite the wrapper files staged by add_deployment_files() with the + # container-specific versions before doing anything else. + if language_name == "python": + for f in ["handler.py", "storage.py", "nosql.py"]: + src = os.path.join(wrapper_container_dir, f) + if os.path.exists(src): + shutil.copy2(src, os.path.join(directory, f)) + elif language_name == "nodejs": + # add_deployment_files() stages the ESM workers variants of + # storage.js and nosql.js; replace them with the CJS container + # versions so the Node.js HTTP server can require() them. + for f in ["storage.js", "nosql.js"]: + src = os.path.join(wrapper_container_dir, f) + if os.path.exists(src): + shutil.copy2(src, os.path.join(directory, f)) + self.logging.info(f"Replaced {f} with container-specific version") + + # For Python: move benchmark code into function/ so that relative imports + # work natively, matching the workers and AWS layout. + # handler.py and requirements.txt* stay at the top level. + if language_name == "python": + func_dir = os.path.join(directory, "function") + os.makedirs(func_dir, exist_ok=True) + open(os.path.join(func_dir, "__init__.py"), "w").close() + dont_move = {"function", "handler.py"} + for item in os.listdir(directory): + if item in dont_move or item.startswith("requirements"): + continue + shutil.move(os.path.join(directory, item), os.path.join(func_dir, item)) + self.logging.info(f"Moved {item} into function/ package") + + # Copy Dockerfile.function from dockerfiles/cloudflare/{language}/ + dockerfile_src = str( + get_resource_path("dockerfiles", "cloudflare", language_name, "Dockerfile.function") + ) + dockerfile_dest = os.path.join(directory, "Dockerfile") + if os.path.exists(dockerfile_src): + # Get base image from systems.json for container deployments + container_images = self.system_config.benchmark_container_images( + "cloudflare", language_name, architecture + ) + base_image = container_images.get(language_version) + if not base_image: + raise RuntimeError( + f"No container base image found in systems.json for " + f"{language_name} {language_version} on {architecture}" + ) + self._base_image = base_image + + shutil.copy2(dockerfile_src, dockerfile_dest) + self.logging.info(f"Copied Dockerfile from {dockerfile_src}") + + # For nodejs, copy the container handler (no function/ subdir for nodejs). + if language_name == "nodejs": + handler_file = "handler.js" + shutil.copy2( + os.path.join(wrapper_container_dir, handler_file), + os.path.join(directory, handler_file), + ) + self.logging.info(f"Copied container {handler_file}") + + nodejs_wrapper_dir = os.path.join(wrapper_base, "nodejs", "container") + worker_js_src = os.path.join(nodejs_wrapper_dir, "worker.js") + if os.path.exists(worker_js_src): + shutil.copy2(worker_js_src, os.path.join(directory, "worker.js")) + self.logging.info("Copied worker.js orchestration file from nodejs/container") + + # Copy init.sh if the benchmark needs it (e.g. video-processing downloads ffmpeg) + from sebs.utils import find_benchmark + + benchmark_path = find_benchmark(benchmark, "benchmarks") + if benchmark_path: + for path in [benchmark_path, os.path.join(benchmark_path, language_name)]: + init_sh = os.path.join(path, "init.sh") + if os.path.exists(init_sh): + shutil.copy2(init_sh, os.path.join(directory, "init.sh")) + self.logging.info(f"Copied init.sh from {path}") + break + + # ALL containers need @cloudflare/containers for worker.js orchestration. + # For nodejs benchmarks, preserve the existing package.json and add the + # dependency. For Python, create a minimal package.json with just the dep. + package_json_path = os.path.join(directory, "package.json") + if language_name == "nodejs": + if not os.path.exists(package_json_path): + raise RuntimeError( + f"package.json not found at {package_json_path} " + f"for nodejs benchmark '{benchmark}'" + ) + with open(package_json_path, "r") as pkg_r: + package_json = json.load(pkg_r) + else: + package_json = {} + package_json.setdefault("dependencies", {})["@cloudflare/containers"] = "*" + with open(package_json_path, "w") as pkg_w: + json.dump(package_json, pkg_w, indent=2) + + # For Python containers, promote the versioned requirements.txt to requirements.txt + if language_name == "python": + requirements_file = os.path.join(directory, "requirements.txt") + versioned_requirements = os.path.join(directory, f"requirements.txt.{language_version}") + if os.path.exists(versioned_requirements): + shutil.copy2(versioned_requirements, requirements_file) + self.logging.info(f"Copied requirements.txt.{language_version} to requirements.txt") + elif not os.path.exists(requirements_file): + open(requirements_file, "w").close() + self.logging.info("Created empty requirements.txt") + + # Build the image locally. cache.py requires docker_client.images.get() to + # succeed for container deployments, and the local image is what we push to + # Cloudflare's registry during deploy (wrangler containers push). + image_tag = self._build_container_image_local( + directory, benchmark, language_name, language_version + ) + + # Calculate package size (approximate, as it's a source directory) + total_size = 0 + for dirpath, dirnames, filenames in os.walk(directory): + for filename in filenames: + filepath = os.path.join(dirpath, filename) + total_size += os.path.getsize(filepath) + + self.logging.info(f"Container package prepared (image tag: {image_tag})") + + return (directory, total_size, image_tag) + + def _build_container_image_local( + self, + directory: str, + benchmark: str, + language_name: str, + language_version: str, + ) -> str: + """ + Build the container image locally. + + The local image is pushed to Cloudflare's registry via + `wrangler containers push` during deployment, so wrangler deploy can + reference it directly without rebuilding from the Dockerfile. + + Returns the local image tag. + """ + # Generate image tag + image_name = ( + f"{benchmark.replace('.', '-')}-{language_name}-{language_version.replace('.', '')}" + ) + version_tag = time.strftime("%Y%m%d-%H%M%S") + image_tag = f"{image_name}:{version_tag}" + + self.logging.info(f"Building container image {image_tag} for linux/amd64...") + + result = subprocess.run( + [ + "docker", + "buildx", + "build", + "--platform", + "linux/amd64", + "--load", + "--no-cache", + "-t", + image_tag, + directory, + ], + capture_output=True, + text=True, + ) + if result.returncode != 0: + self.logging.error(result.stderr) + raise RuntimeError(f"Docker build failed for {image_tag}:\n{result.stderr}") + + self.logging.info(f"Container image built: {image_tag}") + return image_tag + + @staticmethod + def _container_name_from_worker(worker_name: str) -> str: + """Return the Cloudflare container name for a given worker name. + + Cloudflare appends the Durable Object class name (lowercased) to the worker + name to form the container name, e.g.: + worker: container-311-compression-nodejs-18 + container: container-311-compression-nodejs-18-containerworker + """ + return f"{worker_name}-containerworker" + + def shutdown(self): + """Drop the local CLI reference. The shared container is owned by CloudflareCLI; + call CloudflareCLI.shutdown_instance() once at process teardown.""" + self._cli = None diff --git a/sebs/cloudflare/function.py b/sebs/cloudflare/function.py new file mode 100644 index 000000000..203aa3416 --- /dev/null +++ b/sebs/cloudflare/function.py @@ -0,0 +1,66 @@ +"""Cloudflare Workers function and trigger definitions.""" + +from typing import Optional + +from sebs.faas.function import Function, FunctionConfig + + +class CloudflareWorker(Function): + """ + Cloudflare Workers function implementation. + + A Cloudflare Worker is a serverless function that runs on Cloudflare's edge network. + """ + + def __init__( + self, + name: str, + benchmark: str, + script_id: str, + code_package_hash: str, + runtime: str, + cfg: FunctionConfig, + account_id: Optional[str] = None, + ): + """Create a CloudflareWorker with the given script ID, runtime, and account.""" + super().__init__(benchmark, name, code_package_hash, cfg) + self.script_id = script_id + self.runtime = runtime + self.account_id = account_id + + @staticmethod + def typename() -> str: + """Return the canonical type name for this function class.""" + return "Cloudflare.Worker" + + def serialize(self) -> dict: + """Return a serializable dict including script ID, runtime, and account.""" + return { + **super().serialize(), + "script_id": self.script_id, + "runtime": self.runtime, + "account_id": self.account_id, + } + + @staticmethod + def deserialize(cached_config: dict) -> "CloudflareWorker": + """Reconstruct a CloudflareWorker from a cached configuration dict.""" + from sebs.cloudflare.triggers import HTTPTrigger + + cfg = FunctionConfig.deserialize(cached_config["config"]) + ret = CloudflareWorker( + cached_config["name"], + cached_config["benchmark"], + cached_config["script_id"], + cached_config["hash"], + cached_config["runtime"], + cfg, + cached_config.get("account_id"), + ) + + for trigger in cached_config["triggers"]: + trigger_type = HTTPTrigger if trigger["type"] == HTTPTrigger.typename() else None + assert trigger_type, "Unknown trigger type {}".format(trigger["type"]) + ret.add_trigger(trigger_type.deserialize(trigger)) + + return ret diff --git a/sebs/cloudflare/generator.py b/sebs/cloudflare/generator.py new file mode 100644 index 000000000..8a7ef748d --- /dev/null +++ b/sebs/cloudflare/generator.py @@ -0,0 +1,1066 @@ +"""Cloudflare Workflows code generator. + +Translates SeBS FSM definitions (definition.json) into TypeScript source code +for Cloudflare Workflows. Map and Parallel states fan out to child workflow +instances so Cloudflare can execute work concurrently across instances. +""" + +import json +import re +from typing import Dict, List, Union + +from sebs.faas.fsm import Generator, State, Task, Switch, Map, Parallel, Repeat, Loop + + +class CloudflareWorkflowGenerator(Generator): + """Generate TypeScript Workflow code from FSM definitions.""" + + def __init__( + self, + chunk_size: int = 1, + max_instances: int = 1, + dispatch_timeout_seconds: int = 300, + ): + """Initialize the Cloudflare Workflow generator. + + Args: + chunk_size: Number of Map items assigned to one child ItemWorkflow. + max_instances: Container ceiling configured in wrangler.toml. + dispatch_timeout_seconds: Per-container dispatch timeout. + """ + super().__init__() + self._chunk_size = max(1, int(chunk_size)) + self._max_instances = max(1, int(max_instances)) + self._dispatch_timeout_ms = max(300_000, int(dispatch_timeout_seconds) * 1000) + + def generate(self) -> str: + """Generate the complete TypeScript workflow source file.""" + cases = [] + for state in self._all_generated_states().values(): + case_code = self._encode_state_case(state) + cases.append(case_code) + + cases.append( + """\ + case "__end__": { + if (_fanin) { + const { parentId, stateName, branchIdx, total, branchRoot } = _fanin; + await reportFanIn(this.env, { + parentId, + stateName, + idx: branchIdx, + total, + mode: "object", + key: branchRoot, + result: state, + }); + } + return state; + }""" + ) + + switch_body = "\n".join(cases) + + return f"""\ +/* + * Required wrangler bindings: + * - WORKFLOW: Workflow binding for BenchmarkWorkflow + * - ITEM_WORKFLOW: Workflow binding for ItemWorkflow + * - FANIN: Durable Object namespace for FanInCoordinator + * - DISPATCHER: Durable Object namespace for DispatcherContainer + * - [[containers]] class_name = "DispatcherContainer", max_instances = {self._max_instances} + */ +import {{ Container }} from "@cloudflare/containers"; +import {{ WorkflowEntrypoint, WorkflowEvent, WorkflowStep }} from "cloudflare:workers"; + +interface Env {{ + WORKFLOW: Workflow; + ITEM_WORKFLOW: Workflow; + FANIN: DurableObjectNamespace; + DISPATCHER: DurableObjectNamespace; + WORKER_URL: string; + R2?: R2Bucket; + [key: string]: any; +}} + +function getDurableObjectByName( + namespace: DurableObjectNamespace, + name: string, +): DurableObjectStub {{ + return namespace.get(namespace.idFromName(name)); +}} + +function sleep(ms: number): Promise {{ + return new Promise((resolve) => setTimeout(resolve, ms)); +}} + +function textSizeBytes(value: unknown): number {{ + return new TextEncoder().encode(JSON.stringify(value)).length; +}} + +function errorMessage(error: unknown): string {{ + return error instanceof Error ? `${{error.name}}: ${{error.message}}` : String(error); +}} + +async function reportFanIn( + env: Env, + report: {{ + parentId: string; + stateName: string; + idx: number; + total: number; + mode: "array" | "object"; + key: string | null; + result: any; + error?: string; + }}, +): Promise {{ + const fanin = getDurableObjectByName(env.FANIN, `${{report.parentId}}-${{report.stateName}}`); + await fanin.fetch("http://fanin/report", {{ + method: "POST", + headers: {{ "Content-Type": "application/json" }}, + body: JSON.stringify(report), + }}); +}} + +async function fetchWithTimeout( + stub: DurableObjectStub, + url: string, + init: RequestInit, + timeoutMs: number, +): Promise {{ + const controller = new AbortController(); + const timeoutId = setTimeout(() => controller.abort(), timeoutMs); + try {{ + return await stub.fetch(url, {{ ...init, signal: controller.signal }}); + }} finally {{ + clearTimeout(timeoutId); + }} +}} + +function isRetryableFetchError(error: unknown): boolean {{ + const message = error instanceof Error ? error.message : String(error); + return ( + message.includes("fetch failed") || + message.includes("Network connection lost") || + message.includes("internal error connecting to the port") + ); +}} + +function isFetchTimeoutError(error: unknown): boolean {{ + const message = error instanceof Error ? error.message : String(error); + return ( + message.includes("AbortError") || + message.includes("The operation was aborted") || + message.includes("timed out") + ); +}} + +// Retry fetch on 502/503, timeout, or non-JSON responses. +// Any other non-2xx response is treated as a hard error and thrown immediately. +async function dispatchWithRetry( + namespace: DurableObjectNamespace, + containerId: string, + workerUrl: string, + body: any, + maxAttempts = 10, + timeoutMs = {self._dispatch_timeout_ms}, +): Promise {{ + const stub = getDurableObjectByName(namespace, containerId); + console.log( + `[workflow-dispatch] containerId=${{containerId}} function=${{body?.function ?? "unknown"}}` + ); + for (let attempt = 1; attempt <= maxAttempts; attempt++) {{ + let r: Response; + try {{ + r = await fetchWithTimeout( + stub, + "http://dispatcher/", + {{ + method: "POST", + headers: {{ + "Content-Type": "application/json", + "X-Worker-URL": workerUrl, + "X-Dispatcher-Container-ID": containerId, + }}, + body: JSON.stringify(body), + }}, + timeoutMs, + ); + }} catch (error) {{ + if (isFetchTimeoutError(error)) {{ + throw new Error( + `Dispatcher call timed out after ${{timeoutMs}}ms for ${{body?.function ?? "unknown"}} ` + + `on containerId=${{containerId}}. Not retrying because the container may still be running.` + ); + }} + if (attempt < maxAttempts && isRetryableFetchError(error)) {{ + await sleep(Math.min(5000 * attempt, 30000)); + continue; + }} + throw error; + }} + + if (r.status === 503 || r.status === 502) {{ + await sleep(Math.min(5000 * attempt, 30000)); + continue; + }} + const text = await r.text(); + if (!r.ok) {{ + throw new Error(`Dispatcher returned HTTP ${{r.status}}: ${{text.slice(0, 200)}}`); + }} + try {{ + return JSON.parse(text); + }} catch (_) {{ + if (attempt < maxAttempts) {{ + await sleep(Math.min(5000 * attempt, 30000)); + continue; + }} + throw new Error( + `Dispatcher returned non-JSON after ${{maxAttempts}} attempts: ${{text.slice(0, 200)}}` + ); + }} + }} + throw new Error(`Dispatcher unavailable after ${{maxAttempts}} attempts`); +}} + +function isDuplicateWorkflowError(error: unknown): boolean {{ + const message = error instanceof Error ? error.message : String(error); + return ( + message.includes("already exists") || + message.includes("duplicate") || + message.includes("conflict") || + message.includes("409") + ); +}} + +function isRateLimitError(error: unknown): boolean {{ + const message = error instanceof Error ? error.message : String(error); + return message.includes("429") || message.includes("rate limit"); +}} + +async function createWorkflowWithRetry( + workflow: Workflow, + id: string, + params: any, + maxAttempts = 10, +): Promise {{ + for (let attempt = 1; attempt <= maxAttempts; attempt++) {{ + try {{ + await workflow.create({{ id, params }}); + return; + }} catch (error) {{ + if (isDuplicateWorkflowError(error)) {{ + return; + }} + if (attempt < maxAttempts && isRateLimitError(error)) {{ + await sleep(Math.min(5000 * attempt, 30000)); + continue; + }} + throw error; + }} + }} +}} + +export class BenchmarkWorkflow extends WorkflowEntrypoint {{ + async run(event: WorkflowEvent, step: WorkflowStep) {{ + let state: any = structuredClone(event.payload ?? {{}}); + const {{ _start, _fanin }} = state as any; + delete (state as any)._start; + delete (state as any)._fanin; + let current = _start ?? {json.dumps(self.root.name)}; + const dispatchContainerId = _fanin + ? `${{_fanin.parentId}}-${{_fanin.stateName}}-branch-${{_fanin.branchIdx}}` + : event.instanceId; + + try {{ + while (true) {{ + switch (current) {{ +{switch_body} + default: + throw new Error(`Unknown state: ${{current}}`); + }} + }} + }} catch (error) {{ + if (_fanin) {{ + const {{ parentId, stateName, branchIdx, total, branchRoot }} = _fanin; + const message = errorMessage(error); + console.log( + `[workflow-branch-error] parentId=${{parentId}} state=${{stateName}} ` + + `branchIdx=${{branchIdx}} root=${{branchRoot}} error=${{message}}` + ); + await reportFanIn(this.env, {{ + parentId, + stateName, + idx: branchIdx, + total, + mode: "object", + key: branchRoot, + result: null, + error: message, + }}); + }} + throw error; + }} + }} +}} + +export default {{ + async fetch(request: Request, env: Env): Promise {{ + const url = new URL(request.url); + if (url.pathname.startsWith("/nosql/")) {{ + return await handleNoSQLRequest(request, env); + }} + if (url.pathname.startsWith("/r2/")) {{ + return await handleR2Request(request, env); + }} + if (request.method === "GET" && url.searchParams.has("id")) {{ + const id = url.searchParams.get("id")!; + const instance = await env.WORKFLOW.get(id); + const status = await instance.status(); + return Response.json({{ + status: status.status, + output: (status as any).output ?? null, + error: (status as any).error ?? null, + }}); + }} + const payload = await request.json(); + const instance = await env.WORKFLOW.create({{ params: payload }}); + return Response.json({{ id: instance.id }}, {{ status: 202 }}); + }}, +}}; + +{self._emit_proxy_handlers()} + +{self._emit_item_workflow()} + +{self._emit_fanin_coordinator()} + +{self._emit_dispatcher_container()} +""" + + def _all_generated_states(self) -> Dict[str, State]: + """Return top-level and Parallel branch states in generation order.""" + states: Dict[str, State] = {} + + def add_state(state: State) -> None: + if state.name not in states: + states[state.name] = state + if isinstance(state, Parallel): + for branch in state.branches: + branch_states = { + n: State.deserialize(n, s) for n, s in branch.states.items() + } + for branch_state in branch_states.values(): + add_state(branch_state) + + for state in self.states.values(): + add_state(state) + return states + + def _encode_state_case(self, state: State) -> str: + """Encode a single FSM state as a switch case block.""" + if isinstance(state, Task): + return self._encode_task_case(state) + elif isinstance(state, Switch): + return self._encode_switch_case(state) + elif isinstance(state, Map): + return self._encode_map_case(state) + elif isinstance(state, Parallel): + return self._encode_parallel_case(state) + elif isinstance(state, Repeat): + return self._encode_repeat_case(state) + elif isinstance(state, Loop): + return self._encode_loop_case(state) + else: + raise ValueError(f"Unknown state type: {type(state)}") + + def _encode_task_case(self, state: Task) -> str: + """Encode a Task state as a step.do() call.""" + next_state = f'"{state.next}"' if state.next else '"__end__"' + var = self._js_identifier(state.name) + + is_terminal = next_state == '"__end__"' + if is_terminal: + merge_expr = f"{var}_result" + else: + merge_expr = ( + f'(typeof {var}_result === "object" && {var}_result !== null' + f" && !Array.isArray({var}_result))" + f"\n ? {{...state, ...{var}_result}} : {var}_result" + ) + + if state.failure: + return f"""\ + case "{state.name}": {{ + try {{ + const {var}_result = await step.do("{state.name}", async () => {{ + return await dispatchWithRetry( + this.env.DISPATCHER, + dispatchContainerId, + this.env.WORKER_URL, + {{ + function: {json.dumps(state.func_name)}, + input: state, + }}, + ); + }}); + state = {merge_expr}; + current = {next_state}; + }} catch (e) {{ + state = {{ ...state, _error: String(e) }}; + current = "{state.failure}"; + }} + break; + }}""" + else: + return f"""\ + case "{state.name}": {{ + const {var}_result = await step.do("{state.name}", async () => {{ + return await dispatchWithRetry( + this.env.DISPATCHER, + dispatchContainerId, + this.env.WORKER_URL, + {{ + function: {json.dumps(state.func_name)}, + input: state, + }}, + ); + }}); + state = {merge_expr}; + current = {next_state}; + break; + }}""" + + def _encode_switch_case(self, state: Switch) -> str: + """Encode a Switch state as if/else conditions.""" + conditions = [] + for case in state.cases: + var_path = self._js_var_path("state", case.var) + op = case.op + val = ( + case.val if isinstance(case.val, (int, float)) else json.dumps(case.val) + ) + conditions.append( + f' if ({var_path} {op} {val}) {{ current = "{case.next}"; }}' + ) + + default = state.default if state.default else "__end__" + else_clause = f' else {{ current = "{default}"; }}' + + if len(conditions) > 1: + lines = [conditions[0]] + for condition in conditions[1:]: + lines.append(" else " + condition.strip()) + lines.append(else_clause) + body = "\n".join(lines) + else: + body = conditions[0] + "\n" + else_clause + + return f"""\ + case "{state.name}": {{ +{body} + break; + }}""" + + def _encode_map_case(self, state: Map) -> str: + """Encode a Map state as ItemWorkflow fan-out with Durable Object fan-in.""" + next_state = f'"{state.next}"' if state.next else '"__end__"' + var = self._js_identifier(state.name) + array_path = self._js_var_path("state", state.array) + input_expr = self._map_item_input_expr(state, "state") + func_name = self._map_func_name(state) + + return f"""\ + case "{state.name}": {{ + const parentId_{var} = event.instanceId; + const mapInputs_{var} = {array_path}.map((item: any) => {input_expr}); + const totalChunks_{var} = await step.do("{state.name}_spawn", async () => {{ + const total = Math.ceil(mapInputs_{var}.length / {self._chunk_size}); + console.log( + `[workflow-map-spawn] parentId=${{parentId_{var}}} state={state.name} ` + + `items=${{mapInputs_{var}.length}} chunks=${{total}} chunkSize={self._chunk_size}` + ); + await Promise.all( + Array.from({{ length: total }}, async (_unused: unknown, chunkIdx: number) => {{ + const start = chunkIdx * {self._chunk_size}; + const childId = `${{parentId_{var}}}-{state.name}-${{chunkIdx}}`; + console.log( + `[workflow-map-child] parentId=${{parentId_{var}}} state={state.name} ` + + `chunkIdx=${{chunkIdx}} childId=${{childId}}` + ); + await createWorkflowWithRetry(this.env.ITEM_WORKFLOW, childId, {{ + items: mapInputs_{var}.slice(start, start + {self._chunk_size}), + parentId: parentId_{var}, + stateName: "{state.name}", + chunkIdx, + total, + func: {json.dumps(func_name)}, + }}); + }}) + ); + return total; + }}); + if (totalChunks_{var} === 0) {{ + {array_path} = []; + }} else {{ + const done_{var} = await step.waitForEvent("{state.name}_done", {{ + type: `{state.name}-complete-${{parentId_{var}}}`, + timeout: "2 hours", + }}); + const payload_{var} = (done_{var} as any).payload; + if (payload_{var}.error) {{ + throw new Error(`Map state {state.name} failed: ${{payload_{var}.error}}`); + }} + {array_path} = payload_{var}.results; + }} + current = {next_state}; + break; + }}""" + + def _encode_parallel_case(self, state: Parallel) -> str: + """Encode a Parallel state as BenchmarkWorkflow child-instance fan-out.""" + next_state = f'"{state.next}"' if state.next else '"__end__"' + var = self._js_identifier(state.name) + total = len(state.branches) + spawn_lines = [] + for idx, branch in enumerate(state.branches): + spawn_lines.append( + f"""\ + (async () => {{ + const childId = `${{parentId_{var}}}-{state.name}-branch-{idx}`; + console.log( + `[workflow-parallel-child] parentId=${{parentId_{var}}} ` + + `state={state.name} branchIdx={idx} root={branch.root} childId=${{childId}}` + ); + await createWorkflowWithRetry( + this.env.WORKFLOW, + childId, + {{ + ...state, + _start: {json.dumps(branch.root)}, + _fanin: {{ + parentId: parentId_{var}, + stateName: "{state.name}", + branchIdx: {idx}, + total: {total}, + branchRoot: {json.dumps(branch.root)}, + }}, + }}, + ); + }})()""" + ) + spawn_body = ",\n".join(spawn_lines) + + return f"""\ + case "{state.name}": {{ + const parentId_{var} = event.instanceId; + await step.do("{state.name}_spawn", async () => {{ + console.log( + `[workflow-parallel-spawn] parentId=${{parentId_{var}}} state={state.name} branches={total}` + ); + await Promise.all([ +{spawn_body}, + ]); + }}); + const done_{var} = await step.waitForEvent("{state.name}_done", {{ + type: `{state.name}-complete-${{parentId_{var}}}`, + timeout: "2 hours", + }}); + const payload_{var} = (done_{var} as any).payload; + if (payload_{var}.error) {{ + throw new Error(`Parallel state {state.name} failed: ${{payload_{var}.error}}`); + }} + state = {{ ...state, ...payload_{var}.results }}; + current = {next_state}; + break; + }}""" + + def _encode_repeat_case(self, state: Repeat) -> str: + """Encode a Repeat state as a counted sequential loop.""" + next_state = f'"{state.next}"' if state.next else '"__end__"' + return f"""\ + case "{state.name}": {{ + for (let i = 0; i < {state.count}; i++) {{ + state = await step.do(`{state.name}_${{i}}`, async () => {{ + return await dispatchWithRetry( + this.env.DISPATCHER, + dispatchContainerId, + this.env.WORKER_URL, + {{ + function: {json.dumps(state.func_name)}, + input: state, + }}, + ); + }}); + }} + current = {next_state}; + break; + }}""" + + def _encode_loop_case(self, state: Loop) -> str: + """Encode a Loop state as a sequential for loop over an array.""" + next_state = f'"{state.next}"' if state.next else '"__end__"' + array_path = self._js_var_path("state", state.array) + + return f"""\ + case "{state.name}": {{ + for (let i = 0; i < {array_path}.length; i++) {{ + {array_path}[i] = await step.do(`{state.name}_${{i}}`, async () => {{ + return await dispatchWithRetry( + this.env.DISPATCHER, + dispatchContainerId, + this.env.WORKER_URL, + {{ + function: {json.dumps(state.func_name)}, + input: {array_path}[i], + }}, + ); + }}); + }} + current = {next_state}; + break; + }}""" + + def _emit_item_workflow(self) -> str: + """Emit the child workflow that runs one Map chunk.""" + return """\ +export class ItemWorkflow extends WorkflowEntrypoint { + async run(event: WorkflowEvent, step: WorkflowStep) { + const { items, parentId, stateName, chunkIdx, total, func } = event.payload; + console.log( + `[workflow-item] parentId=${parentId} state=${stateName} ` + + `chunkIdx=${chunkIdx} total=${total} func=${func} items=${items.length}` + ); + try { + const results = await step.do(`${stateName}_${chunkIdx}`, async () => { + const containerId = `${parentId}-${stateName}-${chunkIdx}`; + if (items.length === 1) { + const result = await dispatchWithRetry( + this.env.DISPATCHER, + containerId, + this.env.WORKER_URL, + { + function: func, + input: items[0], + }, + ); + return [result]; + } + return await Promise.all( + items.map((item: any) => + dispatchWithRetry( + this.env.DISPATCHER, + containerId, + this.env.WORKER_URL, + { + function: func, + input: item, + }, + ) + ) + ); + }); + + await reportFanIn(this.env, { + parentId, + stateName, + idx: chunkIdx, + total, + mode: "array", + key: null, + result: results, + }); + return results; + } catch (error) { + const message = errorMessage(error); + console.log( + `[workflow-item-error] parentId=${parentId} state=${stateName} ` + + `chunkIdx=${chunkIdx} func=${func} error=${message}` + ); + await reportFanIn(this.env, { + parentId, + stateName, + idx: chunkIdx, + total, + mode: "array", + key: null, + result: null, + error: message, + }); + throw error; + } + } +}""" + + def _emit_proxy_handlers(self) -> str: + """Emit R2 and KV proxy handlers used by containerized benchmark code.""" + return """\ +async function handleNoSQLRequest(request: Request, env: Env): Promise { + try { + const url = new URL(request.url); + const operation = url.pathname.split("/").pop(); + const params = await request.json() as any; + const { table_name, primary_key, secondary_key, data } = params; + const table = env[table_name]; + if (!table || typeof table.get !== "function" || typeof table.put !== "function") { + return Response.json( + { error: `KV namespace binding '${table_name}' not found` }, + { status: 500 }, + ); + } + + const indexKey = `__sebs_idx__${primary_key[1]}`; + const readIndex = async (): Promise => { + const raw = await table.get(indexKey); + if (!raw) { + return []; + } + try { + const parsed = JSON.parse(raw); + return Array.isArray(parsed) ? parsed : []; + } catch { + return []; + } + }; + const writeIndex = async (values: string[]) => { + await table.put(indexKey, JSON.stringify(values)); + }; + + const compositeKey = `${primary_key[1]}#${secondary_key?.[1]}`; + let result: any; + switch (operation) { + case "insert": { + const keyData = { + ...data, + [primary_key[0]]: primary_key[1], + [secondary_key[0]]: secondary_key[1], + }; + await table.put(compositeKey, JSON.stringify(keyData)); + const index = await readIndex(); + if (!index.includes(secondary_key[1])) { + index.push(secondary_key[1]); + await writeIndex(index); + } + result = { success: true }; + break; + } + case "update": { + const existingRaw = await table.get(compositeKey); + let existing = {}; + if (existingRaw) { + try { + existing = JSON.parse(existingRaw); + } catch { + existing = {}; + } + } + const merged = { + ...existing, + ...data, + [primary_key[0]]: primary_key[1], + [secondary_key[0]]: secondary_key[1], + }; + await table.put(compositeKey, JSON.stringify(merged)); + const index = await readIndex(); + if (!index.includes(secondary_key[1])) { + index.push(secondary_key[1]); + await writeIndex(index); + } + result = { success: true }; + break; + } + case "get": { + const raw = await table.get(compositeKey); + if (raw === null) { + result = { data: null }; + } else { + try { + result = { data: JSON.parse(raw) }; + } catch { + result = { data: raw }; + } + } + break; + } + case "query": { + const prefix = `${primary_key[1]}#`; + let secondaryKeys = await readIndex(); + if (secondaryKeys.length === 0) { + const list = await table.list({ prefix }); + secondaryKeys = (list.keys || []).map((k: any) => + k.name.split("#").slice(1).join("#") + ); + } + const items = []; + for (const secondaryValue of secondaryKeys) { + const raw = await table.get(`${primary_key[1]}#${secondaryValue}`); + if (raw === null) { + continue; + } + try { + items.push(JSON.parse(raw)); + } catch { + items.push(raw); + } + } + result = { items }; + break; + } + case "delete": { + await table.delete(compositeKey); + const index = await readIndex(); + const next = index.filter((value) => value !== secondary_key[1]); + if (next.length !== index.length) { + await writeIndex(next); + } + result = { success: true }; + break; + } + default: + return Response.json({ error: "Unknown NoSQL operation" }, { status: 404 }); + } + return Response.json(result || {}); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + return Response.json({ error: message }, { status: 500 }); + } +} + +async function handleR2Request(request: Request, env: Env): Promise { + try { + const url = new URL(request.url); + const key = url.searchParams.get("key"); + if (!env.R2) { + return Response.json({ error: "R2 binding not configured" }, { status: 500 }); + } + + if (url.pathname === "/r2/list") { + const prefix = url.searchParams.get("prefix") || ""; + const list = await env.R2.list({ prefix }); + return Response.json({ objects: list.objects || [] }); + } + + if (url.pathname === "/r2/multipart-init") { + if (!key) { + return Response.json({ error: "Missing key parameter" }, { status: 400 }); + } + const multipart = await env.R2.createMultipartUpload(key); + return Response.json({ key: multipart.key, uploadId: multipart.uploadId }); + } + + if (url.pathname === "/r2/multipart-part") { + if (!key) { + return Response.json({ error: "Missing key parameter" }, { status: 400 }); + } + const uploadId = url.searchParams.get("uploadId"); + const partNumber = Number(url.searchParams.get("partNumber")); + const multipart = env.R2.resumeMultipartUpload(key, uploadId!); + const part = await multipart.uploadPart(partNumber, request.body!); + return Response.json({ partNumber: part.partNumber, etag: part.etag }); + } + + if (url.pathname === "/r2/multipart-complete") { + if (!key) { + return Response.json({ error: "Missing key parameter" }, { status: 400 }); + } + const uploadId = url.searchParams.get("uploadId"); + const { parts } = await request.json() as any; + const multipart = env.R2.resumeMultipartUpload(key, uploadId!); + await multipart.complete(parts); + return Response.json({ key }); + } + + if (!key) { + return Response.json({ error: "Missing key parameter" }, { status: 400 }); + } + + if (url.pathname === "/r2/download") { + const rangeHeader = request.headers.get("Range"); + let options: any = undefined; + let rangeStart: number | undefined; + let rangeEnd: number | undefined; + if (rangeHeader) { + const match = rangeHeader.match(/^bytes=(\\d+)-(\\d+)$/); + if (match) { + rangeStart = Number(match[1]); + rangeEnd = Number(match[2]); + options = { range: { offset: rangeStart, length: rangeEnd - rangeStart + 1 } }; + } + } + const object = (await env.R2.get(key, options)) as R2ObjectBody | null; + if (!object) { + return Response.json({ error: "Object not found" }, { status: 404 }); + } + const headers = new Headers(); + headers.set("Content-Type", object.httpMetadata?.contentType || "application/octet-stream"); + if (rangeHeader && rangeStart !== undefined && rangeEnd !== undefined) { + headers.set("Content-Range", `bytes ${rangeStart}-${rangeEnd}/${object.size}`); + headers.set("Content-Length", String(rangeEnd - rangeStart + 1)); + return new Response(object.body, { status: 206, headers }); + } + headers.set("Content-Length", String(object.size ?? "")); + return new Response(object.body, { headers }); + } + + if (url.pathname === "/r2/upload") { + await env.R2.put(key, request.body!); + return Response.json({ key }); + } + + return Response.json({ error: "Unknown R2 operation" }, { status: 404 }); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + return Response.json({ error: message }, { status: 500 }); + } +}""" + + def _emit_fanin_coordinator(self) -> str: + """Emit the Durable Object that coordinates Map and Parallel fan-in.""" + return """\ +export class FanInCoordinator { + state: DurableObjectState; + env: Env; + + constructor(state: DurableObjectState, env: Env) { + this.state = state; + this.env = env; + } + + async fetch(request: Request): Promise { + if (request.method !== "POST") { + return new Response("Method not allowed", { status: 405 }); + } + + const report = await request.json() as { + parentId: string; + stateName: string; + idx: number; + total: number; + mode: "array" | "object"; + key: string | null; + result: any; + error?: string; + }; + const seenKey = `seen:${report.idx}`; + const alreadySeen = await this.state.storage.get(seenKey); + if (alreadySeen !== undefined) { + return Response.json({ ok: true, duplicate: true }); + } + + await this.state.storage.put(seenKey, true); + await this.state.storage.put(`result:${report.idx}`, { + key: report.key, + result: report.result, + error: report.error, + }); + + const entries = await this.state.storage.list<{ + key: string | null; + result: any; + error?: string; + }>({ prefix: "result:" }); + let assembledBytes = 0; + for (const entry of entries.values()) { + assembledBytes += textSizeBytes(entry.result); + assembledBytes += textSizeBytes(entry.error ?? ""); + } + if (assembledBytes > 900 * 1024) { + throw new Error( + "Fan-in payload exceeds 900 KiB - R2 reference path not yet implemented. " + + "Reduce fan-out width or result size." + ); + } + + if (entries.size === report.total) { + const ordered = [...entries.entries()].sort(([a], [b]) => { + const ai = Number(a.slice("result:".length)); + const bi = Number(b.slice("result:".length)); + return ai - bi; + }); + const failed = ordered.find(([_idx, entry]) => entry.error); + let results: any; + let error: string | undefined; + if (failed) { + const [failedIdx, entry] = failed; + error = `${report.stateName}[${failedIdx.slice("result:".length)}]: ${entry.error}`; + } else if (report.mode === "array") { + results = []; + for (const [_idx, entry] of ordered) { + results.push(...entry.result); + } + } else { + results = {}; + for (const [_idx, entry] of ordered) { + results[entry.key!] = entry.result; + } + } + const instance = await this.env.WORKFLOW.get(report.parentId); + await instance.sendEvent({ + type: `${report.stateName}-complete-${report.parentId}`, + payload: error ? { error } : { results }, + }); + } + + return Response.json({ ok: true }); + } +}""" + + def _emit_dispatcher_container(self) -> str: + """Emit the container class used by the dispatcher Durable Object namespace.""" + return """\ +export class DispatcherContainer extends Container { + defaultPort = 8080; + sleepAfter = "5s"; +}""" + + def _map_func_name(self, state: Map) -> str: + """Return the task function name used by a Map state.""" + if isinstance(state.funcs, dict): + first_state = next(iter(state.funcs.values())) + return first_state["func_name"] + return state.funcs[0] + + def _map_item_input_expr(self, state: Map, root: str) -> str: + """Return the JavaScript expression used as each Map dispatch input.""" + if state.common_params: + param_spread = ", ".join( + f"{json.dumps(p)}: {self._js_var_path(root, p)}" + for p in state.common_params + ) + return f"({{ array_element: item, {param_spread} }})" + return "item" + + @staticmethod + def _js_var_path(root: str, dotted_path: str) -> str: + """Convert a dotted path like 'astros.people' to JS access 'root.astros.people'.""" + parts = dotted_path.split(".") + return root + "." + ".".join(parts) + + @staticmethod + def _js_identifier(name: str) -> str: + """Convert an FSM state name into a JavaScript-safe identifier fragment.""" + identifier = re.sub(r"\W", "_", name) + if identifier and identifier[0].isdigit(): + identifier = f"_{identifier}" + return identifier or "state" + + def encode_task(self, state: Task) -> Union[dict, List[dict]]: + """Not used — generation bypasses the standard encode pipeline.""" + raise NotImplementedError("Use generate() directly") + + def encode_switch(self, state: Switch) -> Union[dict, List[dict]]: + """Not used — generation bypasses the standard encode pipeline.""" + raise NotImplementedError("Use generate() directly") + + def encode_map(self, state: Map) -> Union[dict, List[dict]]: + """Not used — generation bypasses the standard encode pipeline.""" + raise NotImplementedError("Use generate() directly") + + def encode_parallel(self, state: Parallel) -> Union[dict, List[dict]]: + """Not used — generation bypasses the standard encode pipeline.""" + raise NotImplementedError("Use generate() directly") + + def encode_loop(self, state: Loop) -> Union[dict, List[dict]]: + """Not used — generation bypasses the standard encode pipeline.""" + raise NotImplementedError("Use generate() directly") diff --git a/sebs/cloudflare/kvstore.py b/sebs/cloudflare/kvstore.py new file mode 100644 index 000000000..163df9685 --- /dev/null +++ b/sebs/cloudflare/kvstore.py @@ -0,0 +1,441 @@ +"""Cloudflare KV namespace-backed NoSQL storage implementation.""" + +import hashlib +import json +import re +from collections import defaultdict +from typing import Dict, List, Optional, Tuple +from urllib.parse import quote + +import requests + +from sebs.cache import Cache +from sebs.cloudflare.config import CloudflareCredentials +from sebs.faas.config import Resources +from sebs.faas.nosql import NoSQLStorage + + +class KVStore(NoSQLStorage): + """ + Cloudflare KV-backed NoSQL storage for SeBS. + + Cloudflare KV is a flat key-value store: there are no tables, schemas, or + secondary indexes. The SeBS NoSQL abstraction (modelled after DynamoDB / + Cosmos DB / Datastore) is therefore layered on top of KV as follows. + + Table -> namespace mapping + -------------------------- + Each (benchmark, logical table) pair is mapped to exactly one KV namespace + -- the coarsest isolation unit KV offers. Namespaces are titled + + sebs-nosql--- + + with each component sanitized to ``[A-Za-z0-9_-]`` and a SHA1 suffix + appended when the title would exceed Cloudflare's 100-character limit + (see ``_namespace_title``). A one-namespace-per-table layout is used + instead of packing multiple tables into a shared namespace because: + + * Workers bind namespaces by id, so one binding per table is the natural + way to expose the logical table to the benchmark code. + * ``cleanup_tables`` / ``remove_table`` can drop a whole table by deleting + its namespace -- KV has no bulk-delete-by-prefix primitive. + * Key collisions between benchmarks or logical tables are impossible. + + Key mapping + ----------- + Items are stored as JSON values under composite keys: + + # (when a secondary key exists) + (otherwise) + + The primary and secondary key fields are also written back into the JSON + value so that clients reading an item do not have to re-parse the key. + + Secondary-key indices + --------------------- + KV exposes a ``list`` API, but from inside a Worker it is paginated, + eventually consistent, and scales with the total namespace size -- not + with the number of items under a given primary key. DynamoDB-style query + patterns ("give me every item with primary key = X") would therefore be + prohibitively expensive if implemented via ``list``. + + To support those queries with point reads only, ``write_to_table`` + additionally maintains a per-primary-key index entry: + + __sebs_idx__ -> JSON array of secondary-key values + + A query then becomes one ``GET`` for the index followed by one ``GET`` per + secondary value. The index is only written when a secondary key is + supplied; tables without a secondary key do not need it. The matching + read path lives in ``benchmarks/wrappers/cloudflare/*/nosql.*``. + """ + + NAMESPACE_ID_PATTERN = re.compile(r"^[a-fA-F0-9]{32}$") + + @staticmethod + def typename() -> str: + """Return the canonical type name for this storage class.""" + return "Cloudflare.KVStore" + + @staticmethod + def deployment_name() -> str: + """Return the deployment platform name.""" + return "cloudflare" + + def __init__( + self, + region: str, + cache_client: Cache, + resources: Resources, + credentials: CloudflareCredentials, + ): + """Initialize KV storage with Cloudflare credentials.""" + super().__init__(region, cache_client, resources) + self._credentials = credentials + # benchmark -> logical table name -> KV namespace id + self._tables: Dict[str, Dict[str, str]] = defaultdict(dict) + + def _account_id(self) -> str: + """Return the account ID, raising if not configured.""" + account_id = self._credentials.account_id + if not account_id: + raise RuntimeError("Cloudflare account ID is required for KV operations") + return account_id + + def _kv_api_base(self) -> str: + """Return the base URL for the Cloudflare KV namespace API.""" + account = self._account_id() + return f"https://api.cloudflare.com/client/v4/accounts/{account}/storage/kv/namespaces" + + def _get_auth_headers(self, content_type: str = "application/json") -> dict[str, str]: + """Get authentication headers for Cloudflare API requests.""" + if self._credentials.api_token: + return { + "Authorization": f"Bearer {self._credentials.api_token}", + "Content-Type": content_type, + } + elif self._credentials.email and self._credentials.api_key: + return { + "X-Auth-Email": self._credentials.email, + "X-Auth-Key": self._credentials.api_key, + "Content-Type": content_type, + } + else: + raise RuntimeError("Invalid Cloudflare credentials configuration") + + @classmethod + def _is_namespace_id(cls, value: str) -> bool: + """Return True if value matches the 32-character hex namespace ID pattern.""" + return bool(cls.NAMESPACE_ID_PATTERN.fullmatch(value)) + + def _resource_id(self) -> str: + """Return the resource prefix used in namespace titles.""" + if self._cloud_resources.has_resources_id: + return self._cloud_resources.resources_id + return "default" + + @staticmethod + def _sanitize_component(value: str) -> str: + """Replace characters not allowed in KV namespace titles with hyphens.""" + sanitized = re.sub(r"[^A-Za-z0-9_-]", "-", value) + return sanitized.strip("-") or "default" + + def _namespace_title(self, benchmark: str, table: str) -> str: + """Build a deterministic KV namespace title for the given benchmark and table.""" + title = ( + f"sebs-nosql-{self._sanitize_component(self._resource_id())}-" + f"{self._sanitize_component(benchmark)}-{self._sanitize_component(table)}" + ) + # KV namespace title has length constraints; keep a deterministic suffix if truncated. + max_len = 100 + if len(title) > max_len: + digest = hashlib.sha1(title.encode("utf-8")).hexdigest()[:12] + title = f"{title[: max_len - 13]}-{digest}" + return title + + def _list_namespaces(self) -> List[dict]: + """Fetch all KV namespaces for the account, following pagination.""" + namespaces: List[dict] = [] + page = 1 + per_page = 100 + + while True: + response = requests.get( + self._kv_api_base(), + params={"page": page, "per_page": per_page}, + headers=self._get_auth_headers(), + ) + response.raise_for_status() + payload = response.json() + + if not payload.get("success"): + raise RuntimeError(f"Failed to list KV namespaces: {payload.get('errors')}") + + page_items = payload.get("result", []) + namespaces.extend(page_items) + + page_info = payload.get("result_info", {}) or {} + total_pages = int(page_info.get("total_pages", 1)) + if page >= total_pages: + break + page += 1 + + return namespaces + + def _find_namespace_id_by_title(self, title: str) -> Optional[str]: + """Return the namespace ID whose title matches, or None if not found.""" + for namespace in self._list_namespaces(): + if namespace.get("title") == title: + return namespace.get("id") + return None + + def _delete_namespace(self, namespace_id: str) -> None: + """Delete the KV namespace with the given ID, ignoring 404 responses.""" + response = requests.delete( + f"{self._kv_api_base()}/{namespace_id}", + headers=self._get_auth_headers(), + ) + if response.status_code == 404: + return + response.raise_for_status() + + if response.content: + payload = response.json() + if not payload.get("success"): + raise RuntimeError( + f"Failed to delete KV namespace {namespace_id}: {payload.get('errors')}" + ) + + @staticmethod + def _compose_key( + primary_key: Tuple[str, str], secondary_key: Optional[Tuple[str, str]] = None + ) -> str: + """Build the KV storage key from primary and optional secondary key tuples.""" + if secondary_key is None: + return str(primary_key[1]) + return f"{primary_key[1]}#{secondary_key[1]}" + + @staticmethod + def _index_key(primary_value: str) -> str: + """Return the KV key used to store the secondary-key index for a primary value.""" + return f"__sebs_idx__{primary_value}" + + def _read_index(self, namespace_id: str, primary_value: str) -> List[str]: + """Fetch the list of secondary-key values stored in the index for primary_value.""" + index_key = quote(self._index_key(primary_value), safe="") + response = requests.get( + f"{self._kv_api_base()}/{namespace_id}/values/{index_key}", + headers=self._get_auth_headers(), + ) + if response.status_code == 404: + return [] + response.raise_for_status() + + raw = response.text + if not raw: + return [] + + try: + parsed = json.loads(raw) + except Exception: + return [] + + if not isinstance(parsed, list): + return [] + + return [str(v) for v in parsed] + + def _write_index(self, namespace_id: str, primary_value: str, values: List[str]) -> None: + """Persist the secondary-key index for primary_value to KV storage.""" + index_key = quote(self._index_key(primary_value), safe="") + response = requests.put( + f"{self._kv_api_base()}/{namespace_id}/values/{index_key}", + data=json.dumps(values, separators=(",", ":")).encode("utf-8"), + headers=self._get_auth_headers(content_type="text/plain;charset=UTF-8"), + ) + response.raise_for_status() + + def _get_tables(self) -> Dict[str, List[str]]: + """Return all cached table names grouped by benchmark.""" + tables = self.cache_client.get_nosql_configs(self.deployment_name()) + return {benchmark: list(v.values()) for benchmark, v in tables.items()} + + def get_tables(self, benchmark: str) -> Dict[str, str]: + """Return the table-name-to-namespace-ID mapping for the given benchmark.""" + return self._tables[benchmark] + + def _get_table_name(self, benchmark: str, table: str) -> Optional[str]: + """Return the namespace ID for the given benchmark and logical table name, or None.""" + if benchmark not in self._tables: + return None + if table not in self._tables[benchmark]: + return None + return self._tables[benchmark][table] + + def retrieve_cache(self, benchmark: str) -> bool: + """Load cached KV namespace mappings for a benchmark; return True if found.""" + if benchmark in self._tables: + return True + + cached_storage = self.cache_client.get_nosql_config(self.deployment_name(), benchmark) + if cached_storage is None: + return False + + cached_tables = cached_storage.get("tables", {}) + if not isinstance(cached_tables, dict): + return False + + # Ignore legacy Durable Objects cache entries (table -> table name). + if cached_tables and not all( + isinstance(v, str) and self._is_namespace_id(v) for v in cached_tables.values() + ): + self.logging.warning( + f"Ignoring legacy/non-KV cache for benchmark {benchmark}; creating KV namespaces." + ) + return False + + self._tables[benchmark] = cached_tables + self.logging.info(f"Retrieved cached KV namespace mappings for {benchmark}") + return True + + def update_cache(self, benchmark: str): + """Persist the current KV namespace mappings for a benchmark to the cache.""" + self.cache_client.update_nosql( + self.deployment_name(), + benchmark, + {"tables": self._tables[benchmark]}, + ) + self.logging.info(f"Updated cache for KV namespace mappings for {benchmark}") + + def create_table( + self, benchmark: str, name: str, primary_key: str, secondary_key: Optional[str] = None + ) -> str: + """Create or reuse a KV namespace for the given benchmark and table name.""" + # Unused in KV namespace allocation, kept for interface compatibility + _ = primary_key, secondary_key + + existing = self._get_table_name(benchmark, name) + if existing: + return existing + + namespace_title = self._namespace_title(benchmark, name) + + existing_namespace_id = self._find_namespace_id_by_title(namespace_title) + if existing_namespace_id: + self._tables[benchmark][name] = existing_namespace_id + self.logging.info( + f"Reusing existing KV namespace '{namespace_title}' ({existing_namespace_id})" + ) + return existing_namespace_id + + response = requests.post( + self._kv_api_base(), + json={"title": namespace_title}, + headers=self._get_auth_headers(), + ) + + # A concurrent run may have created it after our lookup. + if response.status_code >= 400: + existing_namespace_id = self._find_namespace_id_by_title(namespace_title) + if existing_namespace_id: + self._tables[benchmark][name] = existing_namespace_id + return existing_namespace_id + response.raise_for_status() + + payload = response.json() + if not payload.get("success"): + raise RuntimeError( + f"Failed to create KV namespace {namespace_title}: {payload.get('errors')}" + ) + + namespace_id = payload.get("result", {}).get("id") + if not namespace_id: + raise RuntimeError( + f"Cloudflare KV API did not return namespace id for {namespace_title}" + ) + + self._tables[benchmark][name] = namespace_id + self.logging.info( + f"Created KV namespace '{namespace_title}' ({namespace_id}) for benchmark {benchmark}" + ) + return namespace_id + + def write_to_table( + self, + benchmark: str, + table: str, + data: dict, + primary_key: Tuple[str, str], + secondary_key: Optional[Tuple[str, str]] = None, + ): + """Write a record to the KV namespace, updating the secondary-key index if needed.""" + namespace_id = self._get_table_name(benchmark, table) + if not namespace_id: + raise ValueError(f"Table {table} not found for benchmark {benchmark}") + + record = dict(data) + for key in (primary_key, secondary_key): + if key is not None: + record[key[0]] = key[1] + + composite_key = self._compose_key(primary_key, secondary_key) + value = json.dumps(record, separators=(",", ":"), default=str) + + response = requests.put( + f"{self._kv_api_base()}/{namespace_id}/values/{quote(composite_key, safe='')}", + data=value.encode("utf-8"), + headers=self._get_auth_headers(content_type="text/plain;charset=UTF-8"), + ) + response.raise_for_status() + + if secondary_key is not None: + primary_value = str(primary_key[1]) + secondary_value = str(secondary_key[1]) + index_values = self._read_index(namespace_id, primary_value) + if secondary_value not in index_values: + index_values.append(secondary_value) + self._write_index(namespace_id, primary_value, index_values) + + def clear_table(self, name: str) -> str: + """Log a warning; KV does not support bulk clear — use remove_table + create_table.""" + self.logging.warning( + "clear_table is not implemented for Cloudflare KV. " + "Use remove_table() + create_table() instead." + ) + return name + + def remove_table(self, name: str) -> str: + """Delete the KV namespace identified by logical name or namespace ID.""" + benchmark_to_modify: Optional[str] = None + logical_name_to_delete: Optional[str] = None + namespace_id_to_delete: Optional[str] = None + + for benchmark, tables in list(self._tables.items()): + for logical_name, namespace_id in list(tables.items()): + if name == logical_name or name == namespace_id: + benchmark_to_modify = benchmark + logical_name_to_delete = logical_name + namespace_id_to_delete = namespace_id + break + if namespace_id_to_delete: + break + + # Also allow direct removal by namespace id when not present in local mapping. + if namespace_id_to_delete is None and self._is_namespace_id(name): + namespace_id_to_delete = name + + if namespace_id_to_delete is None: + self.logging.warning(f"KV table '{name}' not found in local mapping.") + return name + + self._delete_namespace(namespace_id_to_delete) + + if benchmark_to_modify is not None and logical_name_to_delete is not None: + del self._tables[benchmark_to_modify][logical_name_to_delete] + + self.logging.info(f"Removed KV namespace {namespace_id_to_delete}") + return name + + def envs(self) -> dict: + """Return environment variables required by benchmarks to access KV storage.""" + return {"NOSQL_STORAGE_DATABASE": "kvstore"} diff --git a/sebs/cloudflare/pyodide_packages.py b/sebs/cloudflare/pyodide_packages.py new file mode 100644 index 000000000..d4e89320e --- /dev/null +++ b/sebs/cloudflare/pyodide_packages.py @@ -0,0 +1,304 @@ +""" +Pyodide packages supported by Cloudflare Python Workers. + +See https://developers.cloudflare.com/workers/languages/python/packages/ for the +authoritative list. Names use the canonical PyPI distribution casing so the +generated pyproject.toml mirrors what pyodide publishes. +""" + +from typing import FrozenSet, Optional + + +SUPPORTED_PYODIDE_PACKAGES: FrozenSet[str] = frozenset( + { + "affine", + "aiohappyeyeballs", + "aiohttp", + "aiosignal", + "altair", + "annotated-types", + "anyio", + "apsw", + "argon2-cffi", + "argon2-cffi-bindings", + "asciitree", + "astropy", + "astropy_iers_data", + "asttokens", + "async-timeout", + "atomicwrites", + "attrs", + "audioop-lts", + "autograd", + "awkward-cpp", + "b2d", + "bcrypt", + "beautifulsoup4", + "bilby.cython", + "biopython", + "bitarray", + "bitstring", + "bleach", + "blosc2", + "bokeh", + "boost-histogram", + "brotli", + "cachetools", + "casadi", + "cbor-diag", + "certifi", + "cffi", + "cffi_example", + "cftime", + "charset-normalizer", + "clarabel", + "click", + "cligj", + "clingo", + "cloudpickle", + "cmyt", + "cobs", + "colorspacious", + "contourpy", + "coolprop", + "coverage", + "cramjam", + "crc32c", + "cryptography", + "css-inline", + "cssselect", + "cvxpy-base", + "cycler", + "cysignals", + "cytoolz", + "decorator", + "demes", + "deprecation", + "diskcache", + "distlib", + "distro", + "docutils", + "donfig", + "ewah_bool_utils", + "exceptiongroup", + "executing", + "fastapi", + "fastcan", + "fastparquet", + "fiona", + "fonttools", + "freesasa", + "frozenlist", + "fsspec", + "future", + "galpy", + "gmpy2", + "gsw", + "h11", + "h3", + "h5py", + "highspy", + "html5lib", + "httpcore", + "httpx", + "idna", + "igraph", + "imageio", + "imgui-bundle", + "iminuit", + "iniconfig", + "inspice", + "ipython", + "jedi", + "Jinja2", + "jiter", + "joblib", + "jsonpatch", + "jsonpointer", + "jsonschema", + "jsonschema_specifications", + "kiwisolver", + "lakers-python", + "lazy_loader", + "lazy-object-proxy", + "libcst", + "lightgbm", + "logbook", + "lxml", + "lz4", + "MarkupSafe", + "matplotlib", + "matplotlib-inline", + "memory-allocator", + "micropip", + "mmh3", + "more-itertools", + "mpmath", + "msgpack", + "msgspec", + "msprime", + "multidict", + "munch", + "mypy", + "narwhals", + "ndindex", + "netcdf4", + "networkx", + "newick", + "nh3", + "nlopt", + "nltk", + "numcodecs", + "numpy", + "openai", + "opencv-python", + "optlang", + "orjson", + "packaging", + "pandas", + "parso", + "patsy", + "pcodec", + "peewee", + "pi-heif", + "Pillow", + "pillow-heif", + "pkgconfig", + "platformdirs", + "pluggy", + "ply", + "pplpy", + "primecountpy", + "prompt_toolkit", + "propcache", + "protobuf", + "pure-eval", + "py", + "pyclipper", + "pycparser", + "pycryptodome", + "pydantic", + "pydantic_core", + "pyerfa", + "pygame-ce", + "Pygments", + "pyheif", + "pyiceberg", + "pyinstrument", + "pylimer-tools", + "PyMuPDF", + "pynacl", + "pyodide-http", + "pyodide-unix-timezones", + "pyparsing", + "pyrsistent", + "pysam", + "pyshp", + "pytaglib", + "pytest", + "pytest-asyncio", + "pytest-benchmark", + "pytest_httpx", + "python-calamine", + "python-dateutil", + "python-flint", + "python-magic", + "python-sat", + "python-solvespace", + "pytz", + "pywavelets", + "pyxel", + "pyxirr", + "pyyaml", + "rasterio", + "rateslib", + "rebound", + "reboundx", + "referencing", + "regex", + "requests", + "retrying", + "rich", + "river", + "RobotRaconteur", + "rpds-py", + "ruamel.yaml", + "rustworkx", + "scikit-image", + "scikit-learn", + "scipy", + "screed", + "setuptools", + "shapely", + "simplejson", + "sisl", + "six", + "smart-open", + "sniffio", + "sortedcontainers", + "soundfile", + "soupsieve", + "sourmash", + "soxr", + "sparseqr", + "sqlalchemy", + "stack-data", + "starlette", + "statsmodels", + "strictyaml", + "svgwrite", + "swiglpk", + "sympy", + "tblib", + "termcolor", + "texttable", + "texture2ddecoder", + "threadpoolctl", + "tiktoken", + "tomli", + "tomli-w", + "toolz", + "tqdm", + "traitlets", + "traits", + "tree-sitter", + "tree-sitter-go", + "tree-sitter-java", + "tree-sitter-python", + "tskit", + "typing-extensions", + "tzdata", + "ujson", + "uncertainties", + "unyt", + "urllib3", + "vega-datasets", + "vrplib", + "wcwidth", + "webencodings", + "wordcloud", + "wrapt", + "xarray", + "xgboost", + "xlrd", + "xxhash", + "xyzservices", + "yarl", + "yt", + "zengl", + "zfpy", + "zstandard", + } +) + + +_CANONICAL_BY_LOWER = {name.lower(): name for name in SUPPORTED_PYODIDE_PACKAGES} + + +def get_canonical_pyodide_name(name: str) -> Optional[str]: + """Return the canonical Pyodide package name for ``name`` (O(1) lookup). + + Matching is case-insensitive. Returns ``None`` if the package is not + supported by the Cloudflare Python Workers runtime. + """ + return _CANONICAL_BY_LOWER.get(name.lower()) diff --git a/sebs/cloudflare/r2.py b/sebs/cloudflare/r2.py new file mode 100644 index 000000000..4e0d65790 --- /dev/null +++ b/sebs/cloudflare/r2.py @@ -0,0 +1,380 @@ +"""Cloudflare R2 object storage implementation.""" + +import os + +import requests +from sebs.cloudflare.config import CloudflareCredentials +from sebs.faas.storage import PersistentStorage +from sebs.faas.config import Resources +from sebs.cache import Cache + +from typing import List, Optional + + +class R2(PersistentStorage): + """Cloudflare R2 object storage backend for SeBS benchmarks.""" + + @staticmethod + def typename() -> str: + """Return the canonical type name for this storage class.""" + return "Cloudflare.R2" + + @staticmethod + def deployment_name() -> str: + """Return the deployment platform name.""" + return "cloudflare" + + @property + def replace_existing(self) -> bool: + """Whether existing objects should be overwritten on upload.""" + return self._replace_existing + + @replace_existing.setter + def replace_existing(self, val: bool): + """Set whether existing objects should be overwritten on upload.""" + self._replace_existing = val + + def __init__( + self, + region: str, + cache_client: Cache, + resources: Resources, + replace_existing: bool, + credentials: CloudflareCredentials, + ): + """Initialize R2 storage with Cloudflare credentials.""" + super().__init__(region, cache_client, resources, replace_existing) + self._credentials = credentials + self._s3_client = None + + def _get_auth_headers(self) -> dict[str, str]: + """Get authentication headers for Cloudflare API requests.""" + if self._credentials.api_token: + return { + "Authorization": f"Bearer {self._credentials.api_token}", + "Content-Type": "application/json", + } + elif self._credentials.email and self._credentials.api_key: + return { + "X-Auth-Email": self._credentials.email, + "X-Auth-Key": self._credentials.api_key, + "Content-Type": "application/json", + } + else: + raise RuntimeError("Invalid Cloudflare credentials configuration") + + def _get_s3_client(self): + """ + Get or initialize the S3-compatible client for R2 operations. + + :return: boto3 S3 client or None if credentials not available + """ + if self._s3_client is not None: + return self._s3_client + + # Check if we have S3-compatible credentials + if not self._credentials.r2_access_key_id or not self._credentials.r2_secret_access_key: + self.logging.warning( + "R2 S3-compatible API credentials not configured. " + "Set CLOUDFLARE_R2_ACCESS_KEY_ID and " + "CLOUDFLARE_R2_SECRET_ACCESS_KEY environment variables." + ) + return None + + try: + import boto3 + from botocore.config import Config + + account_id = self._credentials.account_id + + self._s3_client = boto3.client( + "s3", + endpoint_url=f"https://{account_id}.r2.cloudflarestorage.com", + aws_access_key_id=self._credentials.r2_access_key_id, + aws_secret_access_key=self._credentials.r2_secret_access_key, + config=Config(signature_version="s3v4"), + region_name="auto", + ) + + return self._s3_client + + except ImportError: + self.logging.warning("boto3 not available. Install with: pip install boto3") + return None + + def correct_name(self, name: str) -> str: + """Return the bucket name unchanged; R2 does not require name transformations.""" + return name + + def _create_bucket( + self, name: str, buckets: Optional[List[str]] = None, randomize_name: bool = False + ) -> str: + """Create an R2 bucket, reusing an existing one if the name is already present.""" + for bucket_name in buckets or []: + if name in bucket_name: + self.logging.info( + "Bucket {} for {} already exists, skipping.".format(bucket_name, name) + ) + return bucket_name + + account_id = self._credentials.account_id + + create_bucket_uri = f"https://api.cloudflare.com/client/v4/accounts/{account_id}/r2/buckets" + + # R2 API only accepts "name" parameter - locationHint is optional and must be one of: + # "apac", "eeur", "enam", "weur", "wnam" + # WARNING: locationHint is not currently supported by SeBS. Buckets are created + # with Cloudflare's automatic location selection. + params = {"name": name} + + self.logging.warning( + f"Creating R2 bucket '{name}' without locationHint. " + "Geographic location is determined automatically by Cloudflare." + ) + + try: + create_bucket_response = requests.post( + create_bucket_uri, json=params, headers=self._get_auth_headers() + ) + + # Log the response for debugging + if create_bucket_response.status_code >= 400: + try: + error_data = create_bucket_response.json() + self.logging.error( + f"R2 bucket creation failed. Status: {create_bucket_response.status_code}, " + f"Response: {error_data}" + ) + except Exception: + self.logging.error( + f"R2 bucket creation failed. Status: {create_bucket_response.status_code}, " + f"Response: {create_bucket_response.text}" + ) + + create_bucket_response.raise_for_status() + + bucket_info_json = create_bucket_response.json() + + if not bucket_info_json.get("success"): + self.logging.error(f"Failed to create R2 bucket: {bucket_info_json.get('errors')}") + raise RuntimeError(f"Failed to create R2 bucket {name}") + + bucket_name = bucket_info_json.get("result", {}).get("name", name) + self.logging.info(f"Created R2 bucket {bucket_name}") + return bucket_name + + except requests.exceptions.RequestException as e: + self.logging.error(f"Error creating R2 bucket {name}: {e}") + raise + + def download(self, bucket_name: str, key: str, filepath: str) -> None: + """ + Download a file from a bucket. + + :param bucket_name: + :param key: storage source filepath + :param filepath: local destination filepath + """ + s3_client = self._get_s3_client() + if s3_client is None: + self.logging.warning(f"Cannot download {key} from R2 - S3 client not available") + return + + try: + dirname = os.path.dirname(filepath) + if dirname: + os.makedirs(dirname, exist_ok=True) + s3_client.download_file(bucket_name, key, filepath) + self.logging.debug(f"Downloaded {key} from R2 bucket {bucket_name} to {filepath}") + except Exception as e: + self.logging.warning(f"Failed to download {key} from R2: {e}") + + def upload(self, bucket_name: str, filepath: str, key: str): + """ + Upload a file to R2 bucket using the S3-compatible API. + + Requires S3 credentials to be configured for the R2 bucket. + + :param bucket_name: R2 bucket name + :param filepath: local source filepath + :param key: R2 destination key/path + """ + s3_client = self._get_s3_client() + if s3_client is None: + self.logging.warning(f"Cannot upload {filepath} to R2 - S3 client not available") + return + + try: + with open(filepath, "rb") as f: + s3_client.put_object(Bucket=bucket_name, Key=key, Body=f) + + self.logging.debug(f"Uploaded {filepath} to R2 bucket {bucket_name} as {key}") + + except Exception as e: + self.logging.warning(f"Failed to upload {filepath} to R2: {e}") + + def upload_bytes(self, bucket_name: str, key: str, data: bytes): + """ + Upload bytes directly to R2 bucket using the S3-compatible API. + + :param bucket_name: R2 bucket name + :param key: R2 destination key/path + :param data: bytes to upload + """ + s3_client = self._get_s3_client() + if s3_client is None: + self.logging.warning("Cannot upload bytes to R2 - S3 client not available") + return + + try: + s3_client.put_object(Bucket=bucket_name, Key=key, Body=data) + + self.logging.debug(f"Uploaded {len(data)} bytes to R2 bucket {bucket_name} as {key}") + + except Exception as e: + self.logging.warning(f"Failed to upload bytes to R2: {e}") + + def list_bucket(self, bucket_name: str, prefix: str = "") -> List[str]: + """ + Retrieves list of files in a bucket using S3-compatible API. + + :param bucket_name: + :param prefix: optional prefix filter + :return: list of files in a given bucket + """ + s3_client = self._get_s3_client() + if s3_client is None: + raise RuntimeError( + f"Cannot list R2 bucket {bucket_name} - S3 client not available. " + "Ensure CLOUDFLARE_R2_ACCESS_KEY_ID and CLOUDFLARE_R2_SECRET_ACCESS_KEY are set." + ) + + try: + paginator = s3_client.get_paginator("list_objects_v2") + page_iterator = paginator.paginate(Bucket=bucket_name, Prefix=prefix) + + files = [] + for page in page_iterator: + if "Contents" in page: + for obj in page["Contents"]: + files.append(obj["Key"]) + + return files + + except Exception as e: + raise RuntimeError(f"Failed to list R2 bucket {bucket_name}: {str(e)}") from e + + def list_buckets(self, bucket_name: Optional[str] = None) -> List[str]: + """ + List all R2 buckets in the account. + + :param bucket_name: optional filter (not used for R2) + :return: list of bucket names + """ + account_id = self._credentials.account_id + + list_buckets_uri = f"https://api.cloudflare.com/client/v4/accounts/{account_id}/r2/buckets" + + try: + response = requests.get(list_buckets_uri, headers=self._get_auth_headers()) + + if response.status_code == 403: + try: + error_data = response.json() + detail = f"Response: {error_data}. " + except ValueError: + detail = "" + raise RuntimeError( + f"403 Forbidden accessing R2 buckets. {detail}" + "Your API token may need 'R2 Read and Write' permissions." + ) + + response.raise_for_status() + + data = response.json() + + if not data.get("success"): + raise RuntimeError(f"Failed to list R2 buckets: {data.get('errors')}") + + buckets = data.get("result", {}).get("buckets", []) + bucket_names = [bucket["name"] for bucket in buckets] + + self.logging.info(f"Found {len(bucket_names)} R2 buckets") + return bucket_names + + except requests.exceptions.RequestException as e: + raise RuntimeError(f"Error listing R2 buckets: {e}") from e + + def exists_bucket(self, bucket_name: str) -> bool: + """ + Check if a bucket exists. + + :param bucket_name: + :return: True if bucket exists + """ + buckets = self.list_buckets() + return bucket_name in buckets + + def clean_bucket(self, bucket_name: str): + """ + Remove all objects from a bucket. + + :param bucket_name: + """ + self.logging.warning(f"clean_bucket not fully implemented for R2 bucket {bucket_name}") + pass + + def remove_bucket(self, bucket: str): + """ + Delete a bucket. + + :param bucket: + """ + account_id = self._credentials.account_id + + delete_bucket_uri = ( + f"https://api.cloudflare.com/client/v4/accounts/{account_id}/r2/buckets/{bucket}" + ) + + try: + response = requests.delete(delete_bucket_uri, headers=self._get_auth_headers()) + response.raise_for_status() + + data = response.json() + + if data.get("success"): + self.logging.info(f"Successfully deleted R2 bucket {bucket}") + else: + self.logging.error(f"Failed to delete R2 bucket {bucket}: {data.get('errors')}") + + except requests.exceptions.RequestException as e: + self.logging.error(f"Error deleting R2 bucket {bucket}: {e}") + + def uploader_func(self, bucket_idx: int, file: str, filepath: str) -> None: + """ + Upload a file to a bucket (used for parallel uploads). + + :param bucket_idx: index of the bucket/prefix to upload to + :param file: destination file name/key + :param filepath: source file path + """ + # Skip upload when using cached buckets and not updating storage + if self.cached and not self.replace_existing: + return + + # Build the key with the input prefix + key = os.path.join(self.input_prefixes[bucket_idx], file) + + bucket_name = self.get_bucket(Resources.StorageBucketType.BENCHMARKS) + + # Check if file already exists (if not replacing existing files) + if not self.replace_existing: + for f in self.input_prefixes_files[bucket_idx]: + if key == f: + self.logging.info( + f"Skipping upload of {filepath} to {bucket_name} (already exists)" + ) + return + + # Upload the file + self.upload(bucket_name, filepath, key) diff --git a/sebs/cloudflare/resources.py b/sebs/cloudflare/resources.py new file mode 100644 index 000000000..77e2988a2 --- /dev/null +++ b/sebs/cloudflare/resources.py @@ -0,0 +1,97 @@ +"""Cloudflare system resources manager.""" + +import docker +from typing import Optional, cast + +from sebs.cache import Cache +from sebs.cloudflare.config import CloudflareConfig, CloudflareCredentials +from sebs.cloudflare.r2 import R2 +from sebs.cloudflare.kvstore import KVStore +from sebs.faas.resources import SystemResources +from sebs.faas.storage import PersistentStorage +from sebs.faas.nosql import NoSQLStorage +from sebs.utils import LoggingHandlers + + +class CloudflareSystemResources(SystemResources): + """ + System resources for Cloudflare Workers. + + Cloudflare Workers have a different resource model compared to + traditional cloud platforms. This class handles Cloudflare-specific + resources like KV namespaces and R2 storage. + """ + + def __init__( + self, + config: CloudflareConfig, + cache_client: Cache, + docker_client: docker.client.DockerClient, + logging_handlers: LoggingHandlers, + ): + """Initialize Cloudflare system resources with config and logging handlers.""" + super().__init__(config, cache_client, docker_client) + self._config = config + self.logging_handlers = logging_handlers + + @property + def config(self) -> CloudflareConfig: + """Return the Cloudflare-specific platform configuration.""" + return cast(CloudflareConfig, self._config) + + def _get_auth_headers(self) -> dict[str, str]: + """Get authentication headers for Cloudflare API requests.""" + credentials = cast(CloudflareCredentials, self._config.credentials) + if credentials.api_token: + return { + "Authorization": f"Bearer {credentials.api_token}", + "Content-Type": "application/json", + } + elif credentials.email and credentials.api_key: + return { + "X-Auth-Email": credentials.email, + "X-Auth-Key": credentials.api_key, + "Content-Type": "application/json", + } + else: + raise RuntimeError("Invalid Cloudflare credentials configuration") + + def get_storage(self, replace_existing: Optional[bool] = None) -> PersistentStorage: + """ + Get Cloudflare R2 storage instance. + + R2 is Cloudflare's S3-compatible object storage service. + This method will create a client for managing benchmark input/output data. + + Args: + replace_existing: Whether to replace existing files in storage + + Returns: + R2 storage instance + """ + if replace_existing is None: + replace_existing = False + + return R2( + region=self._config.region, + cache_client=self._cache_client, + resources=self._config.resources, + replace_existing=replace_existing, + credentials=cast(CloudflareCredentials, self._config.credentials), + ) + + def get_nosql_storage(self) -> NoSQLStorage: + """ + Get Cloudflare KV storage instance. + + KV namespaces provide key-value storage for Workers. + + Returns: + KVStore storage instance + """ + return KVStore( + region=self._config.region, + cache_client=self._cache_client, + resources=self._config.resources, + credentials=cast(CloudflareCredentials, self._config.credentials), + ) diff --git a/sebs/cloudflare/templates/wrangler-container.toml b/sebs/cloudflare/templates/wrangler-container.toml new file mode 100644 index 000000000..d8e08fe33 --- /dev/null +++ b/sebs/cloudflare/templates/wrangler-container.toml @@ -0,0 +1,25 @@ +# Template for Cloudflare Container Workers +# This file is read and modified by the deployment system + +name = "PLACEHOLDER_WORKER_NAME" +main = "worker.js" +compatibility_date = "2025-11-18" +account_id = "PLACEHOLDER_ACCOUNT_ID" +compatibility_flags = ["nodejs_compat"] + +[observability] +enabled = true + +[[containers]] +max_instances = 10 +class_name = "ContainerWorker" +image = "./Dockerfile" + +# Durable Object binding for Container class (required by @cloudflare/containers) +[[durable_objects.bindings]] +name = "CONTAINER_WORKER" +class_name = "ContainerWorker" + +[[migrations]] +tag = "v1" +new_sqlite_classes = ["ContainerWorker"] diff --git a/sebs/cloudflare/templates/wrangler-worker.toml b/sebs/cloudflare/templates/wrangler-worker.toml new file mode 100644 index 000000000..fd36127a9 --- /dev/null +++ b/sebs/cloudflare/templates/wrangler-worker.toml @@ -0,0 +1,7 @@ +# Template for native Cloudflare Workers +# This file is read and modified by the deployment system + +name = "PLACEHOLDER_WORKER_NAME" +main = "PLACEHOLDER_MAIN_FILE" +compatibility_date = "2025-11-18" +account_id = "PLACEHOLDER_ACCOUNT_ID" diff --git a/sebs/cloudflare/templates/wrangler-workflow.toml b/sebs/cloudflare/templates/wrangler-workflow.toml new file mode 100644 index 000000000..a9b010b60 --- /dev/null +++ b/sebs/cloudflare/templates/wrangler-workflow.toml @@ -0,0 +1,36 @@ +# Template for Cloudflare Workflow orchestrators +# This file is read and modified by the deployment system + +name = "PLACEHOLDER_WORKER_NAME" +main = "workflow.ts" +compatibility_date = "2025-06-01" +account_id = "PLACEHOLDER_ACCOUNT_ID" +compatibility_flags = ["nodejs_compat"] + +[[workflows]] +name = "PLACEHOLDER_WORKFLOW_NAME" +binding = "WORKFLOW" +class_name = "BenchmarkWorkflow" + +[[workflows]] +name = "PLACEHOLDER_ITEM_WORKFLOW_NAME" +binding = "ITEM_WORKFLOW" +class_name = "ItemWorkflow" + +[[durable_objects.bindings]] +name = "FANIN" +class_name = "FanInCoordinator" + +[[durable_objects.bindings]] +name = "DISPATCHER" +class_name = "DispatcherContainer" + +[[containers]] +class_name = "DispatcherContainer" +image = "PLACEHOLDER_CONTAINER_IMAGE" +max_instances = 1 +instance_type = "lite" + +[[migrations]] +tag = "v1" +new_sqlite_classes = ["FanInCoordinator", "DispatcherContainer"] diff --git a/sebs/cloudflare/triggers.py b/sebs/cloudflare/triggers.py new file mode 100644 index 000000000..51ee60921 --- /dev/null +++ b/sebs/cloudflare/triggers.py @@ -0,0 +1,453 @@ +"""Trigger implementations for Cloudflare Workers and Workflows.""" + +from typing import Optional +import concurrent.futures +import json +import time +from datetime import datetime +from io import BytesIO + +from sebs.faas.function import Trigger, ExecutionResult + + +class ContainerProvisioningError(RuntimeError): + """Raised when Cloudflare reports the container is still provisioning.""" + + pass + + +class HTTPTrigger(Trigger): + """ + HTTP trigger for Cloudflare Workers. + Workers are automatically accessible via HTTPS endpoints. + """ + + def __init__(self, worker_name: str, url: Optional[str] = None): + """Initialize the HTTP trigger with the worker name and optional URL.""" + super().__init__() + self.worker_name = worker_name + self._url = url + + @staticmethod + def typename() -> str: + """Return the canonical type name for this trigger class.""" + return "Cloudflare.HTTPTrigger" + + @staticmethod + def trigger_type() -> Trigger.TriggerType: + """Return the trigger type enum value.""" + return Trigger.TriggerType.HTTP + + @property + def url(self) -> str: + """HTTPS endpoint URL for invoking the worker.""" + assert self._url is not None, "HTTP trigger URL has not been set" + return self._url + + @url.setter + def url(self, url: str): + """Set the HTTPS endpoint URL for the worker.""" + self._url = url + + def _http_invoke(self, payload: dict, url: str, verify_ssl: bool = True) -> ExecutionResult: + """ + Invoke a Cloudflare Worker via HTTP POST. + + Overrides the base implementation to add a browser-like User-Agent header. + Cloudflare's bot-protection returns HTTP 1010 for requests that look like + automated tools (empty or libcurl User-Agent), so we must set one explicitly. + """ + import pycurl + + c = pycurl.Curl() + c.setopt( + pycurl.HTTPHEADER, + [ + "Content-Type: application/json", + # Cloudflare bot-protection (error 1010) blocks requests with no/tool UA. + "User-Agent: Mozilla/5.0 (compatible; SeBS/1.0; " + "+https://github.com/spcl/serverless-benchmarks)", + ], + ) + c.setopt(pycurl.POST, 1) + c.setopt(pycurl.URL, url) + if not verify_ssl: + c.setopt(pycurl.SSL_VERIFYHOST, 0) + c.setopt(pycurl.SSL_VERIFYPEER, 0) + data = BytesIO() + c.setopt(pycurl.WRITEFUNCTION, data.write) + + c.setopt(pycurl.POSTFIELDS, json.dumps(payload)) + begin = datetime.now() + c.perform() + end = datetime.now() + status_code = c.getinfo(pycurl.RESPONSE_CODE) + conn_time = c.getinfo(pycurl.PRETRANSFER_TIME) + receive_time = c.getinfo(pycurl.STARTTRANSFER_TIME) + c.close() + + try: + output = json.loads(data.getvalue()) + if "body" in output: + if isinstance(output["body"], dict): + output = output["body"] + else: + output = json.loads(output["body"]) + + if status_code == 502: + self.logging.info("Container returned 502 (still starting?), will retry...") + raise ContainerProvisioningError("502 gateway error from container worker") + + # Check for Cloudflare error code 1042 (CPU time limit / worker not ready) + # Output may be a plain string like "error code: 1042" rather than a dict. + output_str = str(output) + if "1042" in output_str and "error code" in output_str: + self.logging.info("Worker returned error 1042 (CPU time limit), will retry...") + raise ContainerProvisioningError(f"Error 1042 from worker: {output_str}") + + container_not_ready_phrases = ( + "The container is not running", + "Failed to start container", + ) + if any(p in output_str for p in container_not_ready_phrases): + self.logging.info("Container not yet running, will retry...") + raise ContainerProvisioningError(f"Container startup error: {output_str[:200]}") + + if status_code != 200: + self.logging.error(f"Invocation on URL {url} failed!") + self.logging.error(f"Output: {output}") + raise RuntimeError(f"Failed invocation of function! Output: {output}") + + self.logging.debug("Invoke of function was successful") + result = ExecutionResult.from_times(begin, end) + result.times.http_startup = conn_time + result.times.http_first_byte_return = receive_time + if "request_id" not in output: + raise RuntimeError(f"Cannot process allocation with output: {output}") + result.request_id = output["request_id"] + result.parse_benchmark_output(output) + return result + except json.decoder.JSONDecodeError: + raw = data.getvalue() + raw_text = raw.decode() if raw else "" + provisioning_phrases = ( + "no Container instance available", + "provisioning the Container", + "currently provisioning", + "The container is not running", + "Failed to start container", + ) + if "1042" in raw_text and "error code" in raw_text: + self.logging.info("Worker returned error 1042 (CPU time limit), will retry...") + raise ContainerProvisioningError(f"Error 1042 from worker: {raw_text[:200]}") + if status_code == 502 or any( + p.lower() in raw_text.lower() for p in provisioning_phrases + ): + self.logging.info(f"Container still provisioning (URL {url}): {raw_text[:120]}") + raise ContainerProvisioningError(f"Container not yet available: {raw_text[:200]}") + self.logging.error(f"Invocation on URL {url} failed!") + if raw_text: + self.logging.error(f"Output: {raw_text}") + else: + self.logging.error("No output provided!") + raise RuntimeError(f"Failed invocation of function! Output: {raw_text}") + + def sync_invoke(self, payload: dict) -> ExecutionResult: + """ + Synchronously invoke a Cloudflare Worker via HTTP. + + For container workers, the deployment path already waits until an instance + is running before returning, so provisioning retries here are a last-resort + safety net only (e.g. the instance was recycled between deployment and the + first invocation). + """ + self.logging.debug(f"Invoke function {self.url}") + max_provisioning_retries = 2 + provisioning_retry_wait = 15 # seconds between retries + for attempt in range(max_provisioning_retries + 1): + try: + result = self._http_invoke(payload, self.url) + break + except ContainerProvisioningError: + if attempt < max_provisioning_retries: + self.logging.info( + f"Container not yet ready, waiting {provisioning_retry_wait}s " + f"before retry (attempt {attempt + 1}/{max_provisioning_retries})..." + ) + time.sleep(provisioning_retry_wait) + else: + raise + + # Extract measurement data from the response if available + if result.output and "result" in result.output: # type: ignore[union-attr] + result_data = result.output["result"] + if isinstance(result_data, dict) and "measurement" in result_data: + measurement = result_data["measurement"] + + # Extract timing metrics if provided by the benchmark + if isinstance(measurement, dict): + # CPU time in microseconds + if "cpu_time_us" in measurement: + result.provider_times.execution = measurement["cpu_time_us"] + elif "cpu_time_ms" in measurement: + result.provider_times.execution = int(measurement["cpu_time_ms"] * 1000) + + # Wall time in microseconds + if "wall_time_us" in measurement: + result.times.benchmark = measurement["wall_time_us"] + elif "wall_time_ms" in measurement: + result.times.benchmark = int(measurement["wall_time_ms"] * 1000) + + # Cold/warm start detection + if "is_cold" in measurement: + result.stats.cold_start = measurement["is_cold"] + + # Memory usage if available + if "memory_used_mb" in measurement: + result.stats.memory_used = measurement["memory_used_mb"] + + # Store the full measurement for later analysis + result.output["measurement"] = measurement + + self.logging.debug(f"Extracted measurements: {measurement}") + + return result + + def async_invoke(self, payload: dict) -> concurrent.futures.Future: + """ + Asynchronously invoke a Cloudflare Worker via HTTP. + """ + pool = concurrent.futures.ThreadPoolExecutor() + fut = pool.submit(self.sync_invoke, payload) + return fut + + def serialize(self) -> dict: + """Return a serializable dict with the trigger type, worker name, and URL.""" + return { + "type": self.typename(), + "worker_name": self.worker_name, + "url": self._url, + } + + @staticmethod + def deserialize(obj: dict) -> "HTTPTrigger": + """Reconstruct an HTTPTrigger from a serialized dict.""" + trigger = HTTPTrigger(obj["worker_name"], obj.get("url")) + return trigger + + +class WorkflowLibraryTrigger(Trigger): + """Trigger that invokes a Cloudflare Workflow via its orchestrator's HTTP endpoint. + + The orchestrator worker's fetch handler creates a workflow instance and polls + for completion internally, returning the final result as the HTTP response. + """ + + def __init__(self, workflow_name: str, orchestrator_url: str): + """Initialize the workflow trigger. + + Args: + workflow_name: Name of the Cloudflare Workflow. + orchestrator_url: HTTP URL of the orchestrator worker. + """ + super().__init__() + self.workflow_name = workflow_name + self._orchestrator_url = orchestrator_url + + @staticmethod + def typename() -> str: + """Return the canonical type name for this trigger class.""" + return "Cloudflare.WorkflowLibraryTrigger" + + @staticmethod + def trigger_type() -> Trigger.TriggerType: + """Return the trigger type enum value.""" + return Trigger.TriggerType.LIBRARY + + def _http_get(self, url: str) -> tuple: + """Perform a GET request and return (status_code, body_bytes).""" + import pycurl + + c = pycurl.Curl() + c.setopt( + pycurl.HTTPHEADER, + ["User-Agent: Mozilla/5.0 (compatible; SeBS/1.0; " + "+https://github.com/spcl/serverless-benchmarks)"], + ) + c.setopt(pycurl.URL, url) + data = BytesIO() + c.setopt(pycurl.WRITEFUNCTION, data.write) + c.setopt(pycurl.TIMEOUT, 30) + c.perform() + status_code = c.getinfo(pycurl.RESPONSE_CODE) + c.close() + return status_code, data.getvalue() + + def _http_post(self, url: str, body: str) -> tuple: + """Perform a POST request and return (status_code, body_bytes).""" + import pycurl + + c = pycurl.Curl() + c.setopt( + pycurl.HTTPHEADER, + [ + "Content-Type: application/json", + "User-Agent: Mozilla/5.0 (compatible; SeBS/1.0; " + "+https://github.com/spcl/serverless-benchmarks)", + ], + ) + c.setopt(pycurl.POST, 1) + c.setopt(pycurl.URL, url) + data = BytesIO() + c.setopt(pycurl.WRITEFUNCTION, data.write) + c.setopt(pycurl.POSTFIELDS, body) + c.setopt(pycurl.TIMEOUT, 30) + c.perform() + status_code = c.getinfo(pycurl.RESPONSE_CODE) + c.close() + return status_code, data.getvalue() + + def _do_invoke(self, payload: dict) -> ExecutionResult: + """Create a workflow instance and poll until completion. + + 1. POST to orchestrator → receives {id} (202 Accepted). + 2. GET orchestrator?id= repeatedly until status is complete/errored. + """ + begin = datetime.now() + + # Step 1: create workflow instance + max_create_retries = 3 + instance_id = None + for attempt in range(max_create_retries + 1): + try: + status_code, raw = self._http_post(self._orchestrator_url, json.dumps(payload)) + except Exception as e: + if attempt < max_create_retries: + self.logging.warning(f"Workflow creation network error: {e} — retrying") + time.sleep(5) + continue + self.logging.error(f"Workflow creation network error after retries: {e}") + end = datetime.now() + result = ExecutionResult.from_times(begin, end) + result.stats.failure = True + return result + try: + resp = json.loads(raw) + except json.JSONDecodeError: + text = raw.decode() + if "1042" in text and "error code" in text: + raise ContainerProvisioningError(f"Error 1042 creating workflow: {text[:200]}") + if attempt < max_create_retries: + time.sleep(5) + continue + self.logging.error( + f"Workflow creation non-JSON response: {text[:200]}" + ) + end = datetime.now() + result = ExecutionResult.from_times(begin, end) + result.stats.failure = True + return result + + if status_code == 202 and "id" in resp: + instance_id = resp["id"] + break + if "1042" in str(resp) and "error code" in str(resp): + raise ContainerProvisioningError(f"Error 1042 creating workflow: {resp}") + if attempt < max_create_retries: + time.sleep(5) + continue + self.logging.error(f"Workflow creation failed (status={status_code}): {resp}") + end = datetime.now() + result = ExecutionResult.from_times(begin, end) + result.stats.failure = True + return result + + if instance_id is None: + self.logging.error("Failed to obtain workflow instance ID") + end = datetime.now() + result = ExecutionResult.from_times(begin, end) + result.stats.failure = True + return result + + # Step 2: poll for completion + poll_url = f"{self._orchestrator_url}?id={instance_id}" + poll_interval = 5 + max_poll_time = 7200 + elapsed = 0 + while elapsed < max_poll_time: + time.sleep(poll_interval) + elapsed += poll_interval + try: + status_code, raw = self._http_get(poll_url) + except Exception as e: + self.logging.warning( + f"Status poll network error (elapsed={elapsed}s): {e} — retrying" + ) + continue + try: + resp = json.loads(raw) + except json.JSONDecodeError: + text = raw.decode() + self.logging.warning(f"Status poll non-JSON (elapsed={elapsed}s): {text[:100]}") + continue + + wf_status = resp.get("status") + if wf_status == "complete": + end = datetime.now() + result = ExecutionResult.from_times(begin, end) + result.output = resp.get("output") or {} + return result + if wf_status == "errored": + end = datetime.now() + result = ExecutionResult.from_times(begin, end) + self.logging.error(f"Workflow {self.workflow_name} errored: {resp.get('error')}") + result.stats.failure = True + return result + # Still running (queued/running/paused) — keep polling + + end = datetime.now() + self.logging.error( + f"Workflow {self.workflow_name} did not complete within {max_poll_time}s" + ) + result = ExecutionResult.from_times(begin, end) + result.stats.failure = True + return result + + def sync_invoke(self, payload: dict) -> ExecutionResult: + """Invoke the workflow synchronously: create instance, poll until complete. + + Retries on error 1042 (CPU time limit on cold start) up to 3 times. + """ + self.logging.debug(f"Invoke workflow {self.workflow_name} at {self._orchestrator_url}") + max_retries = 3 + retry_wait = 10 + for attempt in range(max_retries + 1): + try: + return self._do_invoke(payload) + except ContainerProvisioningError: + if attempt < max_retries: + self.logging.info( + f"Workflow cold start (error 1042), waiting {retry_wait}s " + f"before retry (attempt {attempt + 1}/{max_retries})..." + ) + time.sleep(retry_wait) + else: + raise + raise RuntimeError("Unreachable") + + def async_invoke(self, payload: dict): + """Async invocation is not implemented for workflows.""" + raise NotImplementedError("Async invocation is not implemented for workflows") + + def serialize(self) -> dict: + """Return a serializable dict for caching.""" + return { + "type": self.typename(), + "workflow_name": self.workflow_name, + "orchestrator_url": self._orchestrator_url, + } + + @staticmethod + def deserialize(obj: dict) -> "WorkflowLibraryTrigger": + """Reconstruct a WorkflowLibraryTrigger from a cached dict.""" + return WorkflowLibraryTrigger(obj["workflow_name"], obj["orchestrator_url"]) diff --git a/sebs/cloudflare/workers.py b/sebs/cloudflare/workers.py new file mode 100644 index 000000000..4add1ccc9 --- /dev/null +++ b/sebs/cloudflare/workers.py @@ -0,0 +1,312 @@ +""" +Cloudflare Workers native deployment implementation. + +Handles packaging, deployment, and management of native Cloudflare Workers +(non-container deployments using JavaScript/Python runtime). +""" + +import os +import re +import shutil +from importlib.resources import files + +try: + import tomllib # type: ignore[import-not-found] # Python 3.11+ +except ImportError: + import tomli as tomllib # type: ignore[no-redef, import-not-found] # Fallback for older Python +try: + import tomli_w +except ImportError: + import toml as tomli_w # type: ignore[no-redef, import-untyped] +from typing import Optional, Tuple + +from sebs.benchmark import Benchmark +from sebs.cloudflare.cli import CloudflareCLI +from sebs.cloudflare.pyodide_packages import get_canonical_pyodide_name + + +class CloudflareWorkersDeployment: + """Handles native Cloudflare Workers deployment operations.""" + + def __init__(self, logging, system_config, docker_client, system_resources): + """ + Initialize CloudflareWorkersDeployment. + + Args: + logging: Logger instance + system_config: System configuration + docker_client: Docker client instance + system_resources: System resources manager + """ + self.logging = logging + self.system_config = system_config + self.docker_client = docker_client + self.system_resources = system_resources + self._cli: Optional[CloudflareCLI] = None + + def _get_cli(self) -> CloudflareCLI: + """Get or initialize the Cloudflare CLI container.""" + if self._cli is None: + self._cli = CloudflareCLI.get_instance(self.system_config, self.docker_client) + # Verify wrangler is available + version = self._cli.check_wrangler_version() + self.logging.info(f"Cloudflare CLI container ready: {version}") + return self._cli + + def generate_wrangler_toml( + self, + worker_name: str, + package_dir: str, + language: str, + account_id: str, + benchmark_name: Optional[str] = None, + code_package: Optional[Benchmark] = None, + container_uri: Optional[str] = None, + language_variant: str = "cloudflare", + ) -> str: + """ + Generate a wrangler.toml configuration file for native workers. + + Args: + worker_name: Name of the worker + package_dir: Directory containing the worker code + language: Programming language (nodejs or python) + account_id: Cloudflare account ID + benchmark_name: Optional benchmark name for R2 file path prefix + code_package: Optional benchmark package for nosql configuration + + Returns: + Path to the generated wrangler.toml file + """ + # Load template + template_path = ( + files("sebs.cloudflare").joinpath("templates").joinpath("wrangler-worker.toml") + ) + with template_path.open("rb") as f: + config = tomllib.load(f) + + # Update basic configuration + config["name"] = worker_name + config["account_id"] = account_id + + # Add language- and variant-specific configuration. + # For Node.js workers, we always bundle through build.js into dist/, + # regardless of language variant (default/cloudflare), because the + # wrangler entrypoint points to dist/handler.js. + if language == "nodejs": + config["main"] = "dist/handler.js" + config["compatibility_flags"] = ["nodejs_compat"] + config["no_bundle"] = True + config["rules"] = [ + {"type": "ESModule", "globs": ["**/*.js"], "fallthrough": True}, + {"type": "Text", "globs": ["**/*.html"], "fallthrough": True}, + ] + elif language == "python": + config["main"] = "handler.py" + config["compatibility_flags"] = ["python_workers"] + else: + config["main"] = "dist/handler.js" if language == "nodejs" else "handler.py" + + # Add NoSQL KV namespace bindings if benchmark uses them + if code_package and code_package.uses_nosql: + benchmark_for_nosql = benchmark_name or code_package.benchmark + nosql_storage = self.system_resources.get_nosql_storage() + if nosql_storage.retrieve_cache(benchmark_for_nosql): + nosql_tables = nosql_storage.get_tables(benchmark_for_nosql) + if nosql_tables: + config["kv_namespaces"] = [] + for table_name, namespace_id in nosql_tables.items(): + config["kv_namespaces"].append( + { + "binding": table_name, + "id": namespace_id, + } + ) + + # Add environment variables + if benchmark_name or (code_package and code_package.uses_nosql): + config["vars"] = {} + if benchmark_name: + config["vars"]["BENCHMARK_NAME"] = benchmark_name + if code_package and code_package.uses_nosql: + config["vars"]["NOSQL_STORAGE_DATABASE"] = "kvstore" + + # Add R2 bucket binding + try: + from sebs.faas.config import Resources + + storage = self.system_resources.get_storage() + bucket_name = storage.get_bucket(Resources.StorageBucketType.BENCHMARKS) + if bucket_name: + config["r2_buckets"] = [{"binding": "R2", "bucket_name": bucket_name}] + self.logging.info(f"R2 bucket '{bucket_name}' will be bound to worker as 'R2'") + except Exception as e: + self.logging.warning( + f"R2 bucket binding not configured: {e}. " + f"Benchmarks requiring file access will not work properly." + ) + + # Write wrangler.toml to package directory + toml_path = os.path.join(package_dir, "wrangler.toml") + os.makedirs(package_dir, exist_ok=True) + try: + # Try tomli_w (writes binary) + with open(toml_path, "wb") as f: + tomli_w.dump(config, f) + except TypeError: + # Fallback to toml library (writes text) + with open(toml_path, "w") as f: + f.write(tomli_w.dumps(config)) + + self.logging.info(f"Generated wrangler.toml at {toml_path}") + return toml_path + + def package_code( + self, + directory: str, + language_name: str, + language_version: str, + benchmark: str, + is_cached: bool, + language_variant: str = "cloudflare", + ) -> Tuple[str, int, str]: + """ + Package code for native Cloudflare Workers deployment. + + Args: + directory: Path to the code directory + language_name: Programming language name + language_version: Programming language version + benchmark: Benchmark name + is_cached: Whether the code is cached + + Returns: + Tuple of (package_path, package_size, container_uri) + """ + # Install dependencies and bundle. + # Dependency installation (npm install / pip install) is handled by + # Benchmark.install_dependencies() via the canonical SeBS build-image + # pipeline (bind-mount + /sebs/installer.sh). package_code only needs + # to do the language-specific file preparation that happens before or + # after that step. + if language_name == "nodejs": + pass # install_dependencies handles npm install + esbuild bundle + + elif language_name == "python": + requirements_file = os.path.join(directory, "requirements.txt") + if os.path.exists(f"{requirements_file}.{language_version}"): + src = f"{requirements_file}.{language_version}" + dest = requirements_file + shutil.move(src, dest) + self.logging.info(f"move {src} to {dest}") + + if language_variant in ("cloudflare", "default"): + needed_pkg = [] + if os.path.exists(requirements_file): + with open(requirements_file, "r") as reqf: + reqtext = reqf.read() + unsupported = [] + seen = set() + for raw_line in reqtext.splitlines(): + line = raw_line.split("#", 1)[0].strip() + if not line: + continue + name = re.split(r"[<>=!~;\s\[]", line, maxsplit=1)[0].strip() + if not name: + continue + canonical = get_canonical_pyodide_name(name) + if canonical is None: + unsupported.append(name) + continue + if canonical not in seen: + needed_pkg.append(canonical) + seen.add(canonical) + if unsupported: + raise RuntimeError( + "The following packages from requirements.txt are not " + "supported by the Cloudflare Python Workers (Pyodide) " + f"runtime: {', '.join(unsupported)}. See " + "https://developers.cloudflare.com/workers/languages/python/packages/ " + "for the list of supported packages." + ) + + project_file = os.path.join(directory, "pyproject.toml") + pyproject_config = { + "project": { + "name": f"{benchmark.replace('.', '-')}-python-" + f"{language_version.replace('.', '')}", + "version": "0.1.0", + "description": "dummy description", + "requires-python": f">={language_version}", + "dependencies": needed_pkg, + }, + "dependency-groups": { + "dev": ["workers-py", "workers-runtime-sdk"], + }, + } + try: + with open(project_file, "wb") as pf: + tomli_w.dump(pyproject_config, pf) + except TypeError: + with open(project_file, "w") as pf: + pf.write(tomli_w.dumps(pyproject_config)) + # Pyodide Workers require all function files in a function/ subdir + funcdir = os.path.join(directory, "function") + if not os.path.exists(funcdir): + os.makedirs(funcdir) + + dont_move = ["handler.py", "function", "python_modules", "pyproject.toml"] + for thing in os.listdir(directory): + if thing not in dont_move: + src = os.path.join(directory, thing) + dest = os.path.join(directory, "function", thing) + shutil.move(src, dest) + + # Validation (pyproject.toml parse + pywrangler check) is + # performed by install_dependencies via cloudflare_python_installer.sh. + + # Create package structure + CONFIG_FILES = { + "nodejs": ["handler.js", "package.json", "node_modules"], + "python": ["handler.py", "requirements.txt", "python_modules"], + } + + if language_name not in CONFIG_FILES: + raise NotImplementedError( + f"Language {language_name} is not yet supported for Cloudflare Workers" + ) + + # Verify the handler exists + handler_file = "handler.js" if language_name == "nodejs" else "handler.py" + package_path = os.path.join(directory, handler_file) + + if not os.path.exists(package_path): + if not os.path.exists(directory): + raise RuntimeError( + f"Package directory {directory} does not exist. " + "The benchmark build process may have failed to create the deployment package." + ) + raise RuntimeError( + f"Handler file {handler_file} not found in {directory}. " + f"Available files: " + f"{', '.join(os.listdir(directory)) if os.path.exists(directory) else 'none'}" + ) + + # Calculate total size of the package directory + total_size = 0 + for dirpath, dirnames, filenames in os.walk(directory): + for filename in filenames: + filepath = os.path.join(dirpath, filename) + total_size += os.path.getsize(filepath) + + mbytes = total_size / 1024.0 / 1024.0 + self.logging.info( + f"Worker package size: {mbytes:.2f} MB (Python: missing vendored modules)" + ) + + return (directory, total_size, "") + + def shutdown(self): + """Drop the local CLI reference. The shared container is owned by CloudflareCLI; + call CloudflareCLI.shutdown_instance() once at process teardown.""" + self._cli = None diff --git a/sebs/cloudflare/workflow.py b/sebs/cloudflare/workflow.py new file mode 100644 index 000000000..3015cba49 --- /dev/null +++ b/sebs/cloudflare/workflow.py @@ -0,0 +1,81 @@ +"""Cloudflare Workflow representation for SeBS.""" + +from typing import List + +from sebs.cloudflare.function import CloudflareWorker +from sebs.faas.function import FunctionConfig, Workflow + + +class CloudflareWorkflow(Workflow): + """Represents a deployed Cloudflare Workflow with its dispatcher and orchestrator.""" + + def __init__( + self, + name: str, + functions: List[CloudflareWorker], + benchmark: str, + code_package_hash: str, + cfg: FunctionConfig, + account_id: str, + dispatcher_name: str, + orchestrator_url: str, + ): + """Initialize a CloudflareWorkflow. + + Args: + name: Workflow name (also the orchestrator worker name). + functions: List of dispatcher CloudflareWorker instances. + benchmark: Benchmark identifier. + code_package_hash: Hash of the deployed code package. + cfg: Function configuration (memory, timeout). + account_id: Cloudflare account ID. + dispatcher_name: Name of the dispatcher worker/container. + orchestrator_url: URL of the orchestrator worker. + """ + super().__init__(benchmark, name, code_package_hash, cfg) + self.functions = functions + self.account_id = account_id + self.dispatcher_name = dispatcher_name + self.orchestrator_url = orchestrator_url + + @staticmethod + def typename() -> str: + """Return the canonical type name for this workflow class.""" + return "Cloudflare.Workflow" + + def serialize(self) -> dict: + """Serialize workflow state for caching.""" + return { + **super().serialize(), + "functions": [f.serialize() for f in self.functions], + "account_id": self.account_id, + "dispatcher_name": self.dispatcher_name, + "orchestrator_url": self.orchestrator_url, + } + + @staticmethod + def deserialize(cached_config: dict) -> "CloudflareWorkflow": + """Reconstruct a CloudflareWorkflow from a cached configuration dict.""" + from sebs.cloudflare.triggers import HTTPTrigger, WorkflowLibraryTrigger + + funcs = [CloudflareWorker.deserialize(f) for f in cached_config["functions"]] + cfg = FunctionConfig.deserialize(cached_config["config"]) + + ret = CloudflareWorkflow( + cached_config["name"], + funcs, + cached_config["benchmark"], + cached_config["hash"], + cfg, + cached_config["account_id"], + cached_config["dispatcher_name"], + cached_config["orchestrator_url"], + ) + + for trigger in cached_config["triggers"]: + if trigger["type"] == WorkflowLibraryTrigger.typename(): + ret.add_trigger(WorkflowLibraryTrigger.deserialize(trigger)) + elif trigger["type"] == HTTPTrigger.typename(): + ret.add_trigger(HTTPTrigger.deserialize(trigger)) + + return ret diff --git a/sebs/config.py b/sebs/config.py index e23d460ff..6014d9b18 100644 --- a/sebs/config.py +++ b/sebs/config.py @@ -203,6 +203,16 @@ def benchmark_base_images( architecture ] + def benchmark_container_images( + self, deployment_name: str, language_name: str, architecture: str + ) -> Dict[str, str]: + """Get container base images for container deployments.""" + return ( + self._system_config[deployment_name]["languages"][language_name] + .get("container_images", {}) + .get(architecture, {}) + ) + def version(self) -> str: """Get the SeBS framework version. diff --git a/sebs/experiments/config.py b/sebs/experiments/config.py index d88ee6ead..dd881f99f 100644 --- a/sebs/experiments/config.py +++ b/sebs/experiments/config.py @@ -34,6 +34,8 @@ class SystemVariant: # GCP specific "function-gen1", "function-gen2", + # Cloudflare worker deployment + "workers", ] def __init__(self, value: str): diff --git a/sebs/faas/config.py b/sebs/faas/config.py index 8e4e6784b..7da30433f 100644 --- a/sebs/faas/config.py +++ b/sebs/faas/config.py @@ -431,6 +431,15 @@ def deserialize(config: dict, cache: Cache, handlers: LoggingHandlers) -> "Confi from sebs.openwhisk.config import OpenWhiskConfig implementations["openwhisk"] = OpenWhiskConfig.deserialize + + # Cloudflare is available by default (like local) + try: + from sebs.cloudflare.config import CloudflareConfig + + implementations["cloudflare"] = CloudflareConfig.deserialize + except ImportError: + pass + func = implementations.get(name) assert func, "Unknown config type!" return func(config[name] if name in config else config, cache, handlers) diff --git a/sebs/faas/fsm.py b/sebs/faas/fsm.py new file mode 100644 index 000000000..47ea5eb76 --- /dev/null +++ b/sebs/faas/fsm.py @@ -0,0 +1,239 @@ +from abc import ABC +from abc import abstractmethod +from typing import Optional, List, Callable, Union, Dict, Type, Tuple +import json + + +class State(ABC): + def __init__(self, name: str): + self.name = name + + @staticmethod + def deserialize(name: str, payload: dict) -> "State": + cls = _STATE_TYPES[payload["type"]] + return cls.deserialize(name, payload) + + +class Task(State): + def __init__(self, name: str, func_name: str, next: Optional[str], failure: Optional[str]): + self.name = name + self.func_name = func_name + self.next = next + self.failure = failure + + @classmethod + def deserialize(cls, name: str, payload: dict) -> "Task": + return cls( + name=name, + func_name=payload["func_name"], + next=payload.get("next"), + failure=payload.get("failure"), + ) + + +class Switch(State): + class Case: + def __init__(self, var: str, op: str, val: str, next: str): + self.var = var + self.op = op + self.val = val + self.next = next + + @staticmethod + def deserialize(payload: dict) -> "Switch.Case": + return Switch.Case(**payload) + + def __init__(self, name: str, cases: List[Case], default: Optional[str]): + self.name = name + self.cases = cases + self.default = default + + @classmethod + def deserialize(cls, name: str, payload: dict) -> "Switch": + cases = [Switch.Case.deserialize(c) for c in payload["cases"]] + + return cls(name=name, cases=cases, default=payload["default"]) + + +class Branch: + """A named sub-workflow branch used inside a Parallel state.""" + + def __init__(self, root: str, states: Dict[str, dict]): + self.root = root + self.states = states + + @staticmethod + def deserialize(payload) -> "Branch": + if isinstance(payload, str): + # Legacy: bare function name — treat as a single-task sub-workflow. + return Branch(root=payload, states={payload: {"type": "task", "func_name": payload}}) + return Branch(root=payload["root"], states=payload["states"]) + + +class Parallel(State): + def __init__(self, name: str, branches: List["Branch"], next: Optional[str]): + self.name = name + self.branches = branches + self.next = next + + @classmethod + def deserialize(cls, name: str, payload: dict) -> "Parallel": + branches = [Branch.deserialize(f) for f in payload.get("parallel_functions", [])] + return cls(name=name, branches=branches, next=payload.get("next")) + + +class Map(State): + def __init__( + self, + name: str, + funcs: List, + array: str, + root: str, + next: Optional[str], + common_params: Optional[List[str]], + ): + self.name = name + self.funcs = funcs + self.array = array + self.root = root + self.next = next + self.common_params = common_params + + @classmethod + def deserialize(cls, name: str, payload: dict) -> "Map": + raw = payload.get("common_params") + if isinstance(raw, str): + common_params = [p.strip() for p in raw.split(",") if p.strip()] + else: + common_params = raw or None + return cls( + name=name, + funcs=payload["states"], + array=payload["array"], + root=payload["root"], + next=payload.get("next"), + common_params=common_params, + ) + + +class Repeat(State): + def __init__(self, name: str, func_name: str, count: int, next: Optional[str]): + self.name = name + self.func_name = func_name + self.count = count + self.next = next + + @classmethod + def deserialize(cls, name: str, payload: dict) -> "Repeat": + return cls( + name=name, + func_name=payload["func_name"], + count=payload["count"], + next=payload.get("next"), + ) + + +class Loop(State): + def __init__(self, name: str, func_name: str, array: str, next: Optional[str]): + self.name = name + self.func_name = func_name + self.array = array + self.next = next + + @classmethod + def deserialize(cls, name: str, payload: dict) -> "Loop": + return cls( + name=name, + func_name=payload["func_name"], + array=payload["array"], + next=payload.get("next"), + ) + + +_STATE_TYPES: Dict[str, Type[State]] = { + "task": Task, + "switch": Switch, + "map": Map, + "repeat": Repeat, + "loop": Loop, + "parallel": Parallel, +} + + +class Generator(ABC): + def __init__(self, export_func: Callable[[dict], str] = json.dumps): + self._export_func = export_func + + def parse(self, path: str): + with open(path) as f: + definition = json.load(f) + + self.states = {n: State.deserialize(n, s) for n, s in definition["states"].items()} + self.root = self.states[definition["root"]] + + def generate(self) -> str: + states = list(self.states.values()) + payloads = [] + for s in states: + obj = self.encode_state(s) + if isinstance(obj, dict): + payloads.append(obj) + elif isinstance(obj, list): + payloads += obj + else: + raise ValueError("Unknown encoded state returned.") + + definition = self.postprocess(payloads) + + return self._export_func(definition) + + def postprocess(self, payloads: List[dict]) -> dict: + return payloads + + def encode_state(self, state: State) -> Union[dict, List[dict]]: + if isinstance(state, Task): + return self.encode_task(state) + elif isinstance(state, Switch): + return self.encode_switch(state) + elif isinstance(state, Map): + return self.encode_map(state) + elif isinstance(state, Repeat): + return self.encode_repeat(state) + elif isinstance(state, Loop): + return self.encode_loop(state) + elif isinstance(state, Parallel): + return self.encode_parallel(state) + else: + raise ValueError(f"Unknown state of type {type(state)}.") + + @abstractmethod + def encode_task(self, state: Task) -> Union[dict, List[dict]]: + pass + + @abstractmethod + def encode_switch(self, state: Switch) -> Union[dict, List[dict]]: + pass + + @abstractmethod + def encode_map(self, state: Map) -> Union[dict, List[dict]]: + pass + + @abstractmethod + def encode_parallel(self, state: Parallel) -> Union[dict, List[dict]]: + pass + + def encode_repeat(self, state: Repeat) -> Union[dict, List[dict]]: + tasks = [] + for i in range(state.count): + name = state.name if i == 0 else f"{state.name}_{i}" + next = state.next if i == state.count - 1 else f"{state.name}_{i+1}" + task = Task(name, state.func_name, next, None) + + res = self.encode_task(task) + tasks += res if isinstance(res, list) else [res] + + return tasks + + @abstractmethod + def encode_loop(self, state: Loop) -> Union[dict, List[dict]]: + pass diff --git a/sebs/faas/function.py b/sebs/faas/function.py index 0d62642c7..5a1ff71dc 100644 --- a/sebs/faas/function.py +++ b/sebs/faas/function.py @@ -547,6 +547,7 @@ class Python(Enum): DEFAULT = "default" PYPY = "pypy" + CLOUDFLARE = "cloudflare" class NodeJS(Enum): """Node.js runtime variants.""" @@ -554,6 +555,7 @@ class NodeJS(Enum): DEFAULT = "default" BUN = "bun" LLRT = "llrt" + CLOUDFLARE = "cloudflare" class Java(Enum): """Java runtime variants. @@ -913,3 +915,10 @@ def deserialize(cached_config: dict) -> "Function": Function: New instance with the deserialized data """ pass + + +class Workflow(Function): + @staticmethod + @abstractmethod + def deserialize(cached_config: dict) -> "Workflow": + pass diff --git a/sebs/faas/system.py b/sebs/faas/system.py index 291e9ce75..e687e3987 100644 --- a/sebs/faas/system.py +++ b/sebs/faas/system.py @@ -24,7 +24,7 @@ from sebs.faas.container import DockerContainer from sebs.faas.resources import SystemResources from sebs.faas.config import Resources -from sebs.faas.function import Function, Trigger, ExecutionResult +from sebs.faas.function import Function, Trigger, ExecutionResult, Workflow from sebs.utils import LoggingBase from sebs.sebs_types import Language from .config import Config @@ -180,6 +180,66 @@ def function_type() -> "Type[Function]": """ pass + @staticmethod + def workflow_type() -> "Type[Workflow]": + raise NotImplementedError("Workflows not supported on this platform") + + def create_workflow(self, code_package: Benchmark, workflow_name: str) -> Workflow: + raise NotImplementedError("Workflows not supported on this platform") + + def update_workflow(self, workflow: Workflow, code_package: Benchmark): + raise NotImplementedError("Workflows not supported on this platform") + + def get_workflow( + self, code_package: Benchmark, workflow_name: Optional[str] = None + ) -> Workflow: + if not workflow_name: + workflow_name = self.default_function_name(code_package) + + rebuilt, _, system_variant, container_uri = code_package.build( + self.package_code, self.container_client, self.finalize_container_build(), + is_workflow=True, + ) + + functions = code_package.functions + if not functions or workflow_name not in functions: + self.logging.info( + f"Creating new workflow! Reason: workflow {workflow_name} not found in cache." + ) + workflow = self.create_workflow(code_package, workflow_name) + self.cache_client.add_function( + deployment_name=self.name(), + language_name=code_package.language_name, + code_package=code_package, + function=workflow, + ) + code_package.query_cache() + return workflow + + cached_workflow = functions[workflow_name] + workflow = self.workflow_type().deserialize(cached_workflow) + self.cached_function(workflow) + self.logging.info(f"Using cached workflow {workflow_name}") + + if workflow.code_package_hash != code_package.hash or rebuilt: + self.logging.info( + f"Cached workflow {workflow_name} with hash " + f"{workflow.code_package_hash} is not up to date with " + f"current build {code_package.hash}, updating!" + ) + self.update_workflow(workflow, code_package) + workflow.code_package_hash = code_package.hash + workflow.updated_code = True + self.cache_client.add_function( + deployment_name=self.name(), + language_name=code_package.language_name, + code_package=code_package, + function=workflow, + ) + code_package.query_cache() + + return workflow + def find_deployments(self) -> List[str]: """ Find existing deployments in the cloud platform. diff --git a/sebs/gcp/gcp.py b/sebs/gcp/gcp.py index a0adea561..c8e58d7f8 100644 --- a/sebs/gcp/gcp.py +++ b/sebs/gcp/gcp.py @@ -26,6 +26,7 @@ """ import docker +import json import os import logging import random @@ -292,6 +293,10 @@ def get_full_function_name(project_name: str, location: str, func_name: str) -> """ ... + @staticmethod + def get_full_workflow_name(project_name: str, location: str, workflow_name: str) -> str: + return f"projects/{project_name}/locations/{location}/workflows/{workflow_name}" + def function_exists(self, project_name: str, location: str, func_name: str) -> Any: """Check whether the function or service exists. @@ -2209,6 +2214,17 @@ def function_type() -> "Type[Function]": """ return GCPFunction + @staticmethod + def workflow_type() -> "Type[Function]": + """Get the workflow class type for this platform. + + Returns: + GCPWorkflow class type + """ + from sebs.gcp.workflow import GCPWorkflow + + return GCPWorkflow + def initialize( self, config: Dict[str, str] = {}, @@ -2547,6 +2563,7 @@ def create_function( func_name: str, system_variant: SystemVariant, container_uri: str | None, + extra_envs: Dict | None = None, ) -> GCPFunction: """Create a new GCP Cloud Function or update existing one. @@ -2592,6 +2609,7 @@ def create_function( envs = { **self._generate_function_envs(code_package), **strategy.generate_runtime_envs(), + **(extra_envs or {}), } # Get code bucket for non-container deployments @@ -2640,7 +2658,7 @@ def create_function( ) strategy.allow_public_access(project_name, location, func_name) - self.update_function(function, code_package, system_variant, container_uri) + self.update_function(function, code_package, system_variant, container_uri, extra_envs) # Add LibraryTrigger to a new function # Not supported on containers @@ -2671,10 +2689,18 @@ def create_trigger(self, function: Function, trigger_type: Trigger.TriggerType) Raises: RuntimeError: If trigger type is not supported """ - from sebs.gcp.triggers import HTTPTrigger + from sebs.gcp.triggers import HTTPTrigger, WorkflowLibraryTrigger from sebs.gcp.function import GCPFunction + from sebs.gcp.workflow import GCPWorkflow - if trigger_type == Trigger.TriggerType.HTTP: + if isinstance(function, GCPWorkflow): + if trigger_type == Trigger.TriggerType.LIBRARY: + trigger = WorkflowLibraryTrigger(function.name, self) + else: + raise RuntimeError( + f"Trigger type {trigger_type} not supported for workflows. Use LIBRARY." + ) + elif trigger_type == Trigger.TriggerType.HTTP: gcp_function = cast(GCPFunction, function) self.logging.info(f"Function {function.name} - waiting for deployment...") @@ -2693,6 +2719,219 @@ def create_trigger(self, function: Function, trigger_type: Trigger.TriggerType) self.cache_client.update_function(function) return trigger + @staticmethod + def _workflow_yaml(definition: str) -> str: + """Serialise a JSON workflow definition to GCP-compatible YAML. + + PyYAML's default sequence representation places list-item dashes at the + same indentation level as the parent key, which the GCP Workflows parser + rejects when those lists appear inside ``parallel.branches[*].steps``. + This helper uses a custom Dumper that indents sequences properly. + + Args: + definition: JSON string produced by GCPGenerator. + + Returns: + YAML string accepted by the GCP Workflows API. + """ + import json as _json + import yaml as _yaml + + class _IndentDumper(_yaml.Dumper): + """YAML Dumper that indents block sequences under their parent key.""" + + def increase_indent(self, flow=False, indentless=False): + """Override to always indent sequence items.""" + return super().increase_indent(flow=flow, indentless=False) + + return _yaml.dump( + _json.loads(definition), + Dumper=_IndentDumper, + width=99999, + default_flow_style=False, + ) + + def create_workflow(self, code_package: Benchmark, workflow_name: str) -> "Function": + """Create a new GCP Workflow that orchestrates Cloud Functions. + + Deploys individual functions for each code file in the benchmark, + generates a GCP Workflows definition from the benchmark's FSM definition, + and creates the workflow via the GCP Workflows API. + + Args: + code_package: Benchmark package with workflow code and definition + workflow_name: Name for the GCP Workflow + + Returns: + GCPWorkflow instance representing the deployed workflow + """ + import yaml + from google.cloud.workflows_v1 import WorkflowsClient, Workflow as GCPWorkflowProto + from sebs.gcp.workflow import GCPWorkflow + from sebs.gcp.generator import GCPGenerator + from sebs.gcp.triggers import WorkflowLibraryTrigger, HTTPTrigger + + definition_path = os.path.join(code_package.benchmark_path, "definition.json") + if not os.path.exists(definition_path): + raise ValueError(f"No workflow definition found for {workflow_name}") + + code_files = list(code_package.get_code_files(include_config=False)) + func_names = [os.path.splitext(os.path.basename(p))[0] for p in code_files] + funcs = [ + self.create_function( + code_package, + workflow_name + "--" + fn, + code_package.system_variant, + None, + extra_envs={"MY_FUNCTION_NAME": workflow_name + "--" + fn}, + ) + for fn in func_names + ] + + # Create HTTP triggers for each function so the workflow can call them + func_triggers: Dict[str, str] = {} + for fn, func in zip(func_names, funcs): + if len(func.triggers(Trigger.TriggerType.HTTP)) == 0: + self.create_trigger(func, Trigger.TriggerType.HTTP) + http_trigger = cast(HTTPTrigger, func.triggers(Trigger.TriggerType.HTTP)[0]) + func_triggers[fn] = http_trigger.url + + gen = GCPGenerator(workflow_name, func_triggers, code_package.benchmark_config.timeout) + gen.parse(definition_path) + definition = gen.generate() + + # Deploy the workflow via GCP Workflows API + project_name = self.config.project_name + location = self.config.region + parent = f"projects/{project_name}/locations/{location}" + + workflows_client = WorkflowsClient() + workflow_proto = GCPWorkflowProto(source_contents=self._workflow_yaml(definition)) + + try: + operation = workflows_client.create_workflow( + parent=parent, workflow=workflow_proto, workflow_id=workflow_name + ) + self.logging.info(f"Creating workflow {workflow_name}") + operation.result() + except Exception as e: + if "already exists" in str(e).lower(): + self.logging.info(f"Workflow {workflow_name} already exists, updating.") + workflow_proto.name = f"{parent}/workflows/{workflow_name}" + operation = workflows_client.update_workflow(workflow=workflow_proto) + operation.result() + else: + raise + + # Deploy map sub-workflows if any + for map_id, map_definition in gen.generate_maps(): + map_proto = GCPWorkflowProto(source_contents=self._workflow_yaml(map_definition)) + try: + operation = workflows_client.create_workflow( + parent=parent, workflow=map_proto, workflow_id=map_id + ) + self.logging.info(f"Creating map sub-workflow {map_id}") + operation.result() + except Exception as e: + if "already exists" in str(e).lower(): + map_proto.name = f"{parent}/workflows/{map_id}" + operation = workflows_client.update_workflow(workflow=map_proto) + operation.result() + else: + raise + + storage_client = self._system_resources.get_storage() + workflow = GCPWorkflow( + workflow_name, + funcs, + code_package.benchmark, + code_package.hash, + FunctionConfig.from_benchmark(code_package), + storage_client.get_bucket(Resources.StorageBucketType.DEPLOYMENT), + ) + + trigger = WorkflowLibraryTrigger(workflow_name, self) + trigger.logging_handlers = self.logging_handlers + workflow.add_trigger(trigger) + return workflow + + def update_workflow(self, workflow: "Function", code_package: Benchmark) -> None: + """Update an existing GCP Workflow with new function code and definition. + + Args: + workflow: Existing GCPWorkflow instance to update + code_package: New benchmark package with updated code + """ + import yaml + from google.cloud.workflows_v1 import WorkflowsClient, Workflow as GCPWorkflowProto + from sebs.gcp.workflow import GCPWorkflow + from sebs.gcp.generator import GCPGenerator + from sebs.gcp.triggers import HTTPTrigger + + wf = cast(GCPWorkflow, workflow) + + definition_path = os.path.join(code_package.benchmark_path, "definition.json") + if not os.path.exists(definition_path): + raise ValueError(f"No workflow definition found for {wf.name}") + + code_files = list(code_package.get_code_files(include_config=False)) + func_names = [os.path.splitext(os.path.basename(p))[0] for p in code_files] + funcs = [ + self.create_function( + code_package, + wf.name + "--" + fn, + code_package.system_variant, + None, + extra_envs={"MY_FUNCTION_NAME": wf.name + "--" + fn}, + ) + for fn in func_names + ] + wf.functions = funcs + + func_triggers: Dict[str, str] = {} + for fn, func in zip(func_names, funcs): + if len(func.triggers(Trigger.TriggerType.HTTP)) == 0: + self.create_trigger(func, Trigger.TriggerType.HTTP) + http_trigger = cast(HTTPTrigger, func.triggers(Trigger.TriggerType.HTTP)[0]) + func_triggers[fn] = http_trigger.url + + gen = GCPGenerator(wf.name, func_triggers, code_package.benchmark_config.timeout) + gen.parse(definition_path) + definition = gen.generate() + + project_name = self.config.project_name + location = self.config.region + parent = f"projects/{project_name}/locations/{location}" + + workflows_client = WorkflowsClient() + workflow_proto = GCPWorkflowProto( + name=f"{parent}/workflows/{wf.name}", + source_contents=self._workflow_yaml(definition), + ) + operation = workflows_client.update_workflow(workflow=workflow_proto) + self.logging.info(f"Updating workflow {wf.name}") + operation.result() + + for map_id, map_definition in gen.generate_maps(): + map_proto = GCPWorkflowProto( + name=f"{parent}/workflows/{map_id}", + source_contents=self._workflow_yaml(map_definition), + ) + try: + operation = workflows_client.update_workflow(workflow=map_proto) + operation.result() + except Exception as e: + if "not found" in str(e).lower(): + map_proto_new = GCPWorkflowProto( + source_contents=self._workflow_yaml(map_definition) + ) + operation = workflows_client.create_workflow( + parent=parent, workflow=map_proto_new, workflow_id=map_id + ) + operation.result() + else: + raise + def cached_function(self, function: Function) -> None: """Configure a cached function instance for use. @@ -2704,15 +2943,19 @@ def cached_function(self, function: Function) -> None: """ from sebs.faas.function import Trigger - from sebs.gcp.triggers import LibraryTrigger - - func = cast(GCPFunction, function) + from sebs.gcp.triggers import LibraryTrigger, WorkflowLibraryTrigger + from sebs.gcp.workflow import GCPWorkflow for trigger in function.triggers(Trigger.TriggerType.LIBRARY): - gcp_trigger = cast(LibraryTrigger, trigger) - gcp_trigger.deployment_type = func.deployment_type - gcp_trigger.logging_handlers = self.logging_handlers - gcp_trigger.deployment_client = self + if isinstance(trigger, WorkflowLibraryTrigger) or isinstance(function, GCPWorkflow): + trigger.logging_handlers = self.logging_handlers + trigger._deployment_client = self + else: + func = cast(GCPFunction, function) + gcp_trigger = cast(LibraryTrigger, trigger) + gcp_trigger.deployment_type = func.deployment_type + gcp_trigger.logging_handlers = self.logging_handlers + gcp_trigger.deployment_client = self def update_function( self, @@ -2720,6 +2963,7 @@ def update_function( code_package: Benchmark, system_variant: SystemVariant, container_uri: str | None, + extra_envs: Dict | None = None, ) -> None: """Update an existing Cloud Function with new code and configuration. @@ -2732,6 +2976,7 @@ def update_function( code_package: New benchmark package with updated code system_variant: Selected deployment variant container_uri: Container image URI (unused) + extra_envs: Additional environment variables to set Raises: NotImplementedError: If the deployment variant is unsupported @@ -2747,6 +2992,7 @@ def update_function( envs = { **self._generate_function_envs(code_package), **strategy.generate_runtime_envs(), + **(extra_envs or {}), } # Update code using strategy diff --git a/sebs/gcp/generator.py b/sebs/gcp/generator.py new file mode 100644 index 000000000..26943eee7 --- /dev/null +++ b/sebs/gcp/generator.py @@ -0,0 +1,398 @@ +import uuid +from typing import Dict, Union, List, Optional, Set, Tuple + +from sebs.faas.fsm import Generator, State, Task, Switch, Map, Parallel, Repeat, Loop, Branch + + +class GCPGenerator(Generator): + def __init__( + self, + workflow_name: str, + func_triggers: Dict[str, str], + func_timeout: int = 1800, + ): + """Initialize GCP Workflows YAML generator. + + Args: + workflow_name: Name of the workflow being generated. + func_triggers: Map from function name to HTTP trigger URL. + func_timeout: Timeout in seconds for http.post calls (default 1800). + """ + super().__init__() + self._workflow_name = workflow_name + self._func_triggers = func_triggers + self._func_timeout = func_timeout + # Maps workflow_id -> (url, common_params_list_or_None) + self._map_funcs: Dict[str, tuple] = dict() + self._ordered_states: List[State] = [] + + def postprocess(self, payloads: List[dict]) -> dict: + payloads.append({"final": {"return": "${res}"}}) + + definition = {"main": {"params": ["res"], "steps": payloads}} + + return definition + + def _topological_order(self) -> List[State]: + """Return states in BFS order starting from root, visiting all reachable states.""" + visited: Set[str] = set() + ordered: List[State] = [] + queue: List[str] = [self.root.name] + + while queue: + name = queue.pop(0) + if name in visited or name not in self.states: + continue + visited.add(name) + state = self.states[name] + ordered.append(state) + # Enqueue successors + if isinstance(state, Task): + if state.next: + queue.append(state.next) + if state.failure: + queue.append(state.failure) + elif isinstance(state, Switch): + for case in state.cases: + queue.append(case.next) + if state.default: + queue.append(state.default) + elif isinstance(state, (Map, Parallel, Loop, Repeat)): + if state.next: + queue.append(state.next) + + # Also add any states not reachable from root (shouldn't happen in well-formed FSMs) + for name, state in self.states.items(): + if name not in visited: + ordered.append(state) + + return ordered + + def generate(self) -> str: + self._ordered_states = self._topological_order() + terminal_names = self._find_terminal_state_names() + + payloads: List[dict] = [] + for s in self._ordered_states: + obj = self.encode_state(s) + if isinstance(obj, dict): + encoded_name = list(obj.keys())[0] + payloads.append(obj) + # Add explicit jump to final for terminal states that aren't last + if s.name in terminal_names and self._ordered_states[-1].name != s.name: + payloads.append({"goto_final_" + s.name: {"next": "final"}}) + elif isinstance(obj, list): + payloads += obj + # After the last step for this state, add jump to final if terminal + if s.name in terminal_names and self._ordered_states[-1].name != s.name: + payloads.append({"goto_final_" + s.name: {"next": "final"}}) + else: + raise ValueError("Unknown encoded state returned.") + + definition = self.postprocess(payloads) + return self._export_func(definition) + + def _find_terminal_state_names(self) -> Set[str]: + """Find states that have no next pointer (end of a path).""" + terminals: Set[str] = set() + for name, state in self.states.items(): + has_next = getattr(state, "next", None) + if not has_next: + terminals.add(name) + return terminals + + def encode_task(self, state: Task) -> Union[dict, List[dict]]: + url = self._func_triggers[state.func_name] + + if state.failure: + call_step: dict = { + state.name: { + "try": { + "call": "http.post", + "args": {"url": url, "body": "${res}", "timeout": self._func_timeout}, + "result": "res", + }, + "except": { + "as": "e", + "steps": [ + {"jump_" + state.name: {"next": state.failure}}, + ], + }, + } + } + assign_step = {"assign_res_" + state.name: {"assign": [{"res": "${res.body}"}]}} + steps: list = [call_step, assign_step] + if state.next: + steps.append({"next_" + state.name: {"next": state.next}}) + return steps + else: + plain_steps: list = [ + { + state.name: { + "call": "http.post", + "args": {"url": url, "body": "${res}", "timeout": self._func_timeout}, + "result": "res", + } + }, + {"assign_res_" + state.name: {"assign": [{"res": "${res.body}"}]}}, + ] + if state.next: + plain_steps.append({"next_" + state.name: {"next": state.next}}) + return plain_steps + + def encode_switch(self, state: Switch) -> Union[dict, List[dict]]: + return { + state.name: { + "switch": [self._encode_case(c) for c in state.cases], + "next": state.default, + } + } + + def _encode_case(self, case: Switch.Case) -> dict: + cond = "res." + case.var + " " + case.op + " " + str(case.val) + return {"condition": "${" + cond + "}", "next": case.next} + + def encode_map(self, state: Map, res_var: str = "res") -> Union[dict, List[dict]]: + """Encode a Map state as GCP Workflows steps. + + Args: + state: Map state to encode. + res_var: Variable name that holds the current result dict. Defaults + to ``"res"`` for top-level maps; pass a branch-specific variable + when encoding maps inside parallel branches to avoid cross-branch + interference via the shared ``res`` variable. + + Returns: + List of step dicts for the map. + """ + if isinstance(state.funcs, dict): + first_state = next(iter(state.funcs.values())) + func_name = first_state["func_name"] + else: + func_name = state.funcs[0] + + id = self._workflow_name + "_" + "map" + str(uuid.uuid4())[0:8] + self._map_funcs[id] = (self._func_triggers[func_name], state.common_params) + + # Write map output to a separate var so the original dict is preserved. + # Use dot-path assignment (res_var.array = map_res_NAME) to update only + # the array key, keeping all other context fields intact. + map_res_var = "map_res_" + state.name.replace("-", "_") + + if state.common_params: + enrich_id = "enrich_" + state.name + enriched_var = "enriched_" + state.name.replace("-", "_") + temp_var = "tmp_" + state.name.replace("-", "_") + + temp_dict: dict = {"array_element": "${elem}"} + for p in state.common_params: + temp_dict[p] = "${" + res_var + "." + p + "}" + + inner_steps = [ + {"build_" + enrich_id: {"assign": [{temp_var: temp_dict}]}}, + { + "append_" + enrich_id: { + "assign": [ + {enriched_var: "${list.concat(" + enriched_var + ", " + temp_var + ")}"} + ] + } + }, + ] + enrich_steps = [ + {"init_" + enrich_id: {"assign": [{enriched_var: []}]}}, + { + "loop_" + enrich_id: { + "for": { + "value": "elem", + "in": "${" + res_var + "." + state.array + "}", + "steps": inner_steps, + } + } + }, + ] + call_step = { + state.name: { + "call": "experimental.executions.map", + "args": {"workflow_id": id, "arguments": "${" + enriched_var + "}"}, + "result": map_res_var, + } + } + return_steps = [*enrich_steps, call_step] + else: + call_step = { + state.name: { + "call": "experimental.executions.map", + "args": {"workflow_id": id, "arguments": "${" + res_var + "." + state.array + "}"}, + "result": map_res_var, + } + } + return_steps = [call_step] + # Update only the array key; all other context fields are preserved. + assign_step = { + "assign_res_" + state.name: { + "assign": [{res_var + "." + state.array: "${" + map_res_var + "}"}] + } + } + steps = return_steps + [assign_step] + if state.next: + steps.append({"next_" + state.name: {"next": state.next}}) + return steps + + def _encode_branch(self, branch: Branch, shared_var: str) -> Tuple[List[dict], List[str]]: + """Encode a single Parallel branch as a list of GCP Workflow steps. + + Each branch reads from ``shared_var``, processes its states in order, + and writes its result back to ``shared_var``. Because GCP Workflows + parallel branches share the global variable namespace we write results + into a branch-specific variable and later merge them. + + Args: + branch: Branch definition containing sub-states. + shared_var: Variable name to read input from. + + Returns: + Tuple of (steps, extra_shared_vars). ``extra_shared_vars`` lists + any intermediate variables written inside the branch (e.g. + ``map_res_*``) that must appear in the parallel step's ``shared`` + list. + """ + from sebs.faas.fsm import State as FSMState + + steps: List[dict] = [] + extra_shared: List[str] = [] + # Resolve BFS order within the branch's own state dict + b_states = {n: FSMState.deserialize(n, s) for n, s in branch.states.items()} + visited: Set[str] = set() + queue = [branch.root] + ordered = [] + while queue: + n = queue.pop(0) + if n in visited or n not in b_states: + continue + visited.add(n) + s = b_states[n] + ordered.append(s) + nxt = getattr(s, "next", None) + if nxt: + queue.append(nxt) + if isinstance(s, Task) and s.failure: + queue.append(s.failure) + + for s in ordered: + if isinstance(s, Task): + url = self._func_triggers[s.func_name] + steps.append({s.name: {"call": "http.post", "args": {"url": url, "body": "${" + shared_var + "}", "timeout": self._func_timeout}, "result": shared_var}}) + steps.append({"assign_res_" + s.name: {"assign": [{shared_var: "${" + shared_var + ".body}"}]}}) + elif isinstance(s, Map): + # Pass shared_var directly so encode_map reads/writes that variable + # instead of the global "res". This avoids cross-branch interference + # when multiple parallel branches each contain a Map step. + steps += self.encode_map(s, res_var=shared_var) # type: ignore[arg-type] + # map_res_ is written inside this branch — must be shared. + map_res_var = "map_res_" + s.name.replace("-", "_") + extra_shared.append(map_res_var) + return steps, extra_shared + + def encode_parallel(self, state: Parallel) -> Union[dict, List[dict]]: + """Encode a Parallel state as a GCP Workflows parallel block. + + Each branch runs concurrently with its own local copy of ``res`` + (since ``res`` is NOT in ``shared``). Results are stored in + branch-specific shared variables and merged into ``res`` afterwards. + + Args: + state: Parallel state to encode. + + Returns: + List of step dicts for the parallel block and result merge. + """ + shared_vars = [] + extra_shared_all: List[str] = [] + gcp_branches = [] + for i, branch in enumerate(state.branches): + # Use a per-branch local variable as the working variable throughout + # the branch so that no branch writes to the outer "res". + # The final value is stored in this shared var after the branch ends. + var = "branch_res_" + state.name.replace("-", "_") + "_" + str(i) + shared_vars.append(var) + # Encode branch using var as both input (initialised to ${res}) and + # working accumulator — _encode_branch takes the var name to use. + branch_steps, extra_shared = self._encode_branch(branch, var) + extra_shared_all.extend(extra_shared) + # Seed the per-branch variable from the outer res before starting. + seed_step = {"seed_" + var: {"assign": [{var: "${res}"}]}} + branch_name = "branch_" + state.name.replace("-", "_") + "_" + str(i) + gcp_branches.append({branch_name: {"steps": [seed_step] + branch_steps}}) + + # GCP Workflows requires shared variables to be initialized in the outer + # scope before the parallel step references them. + all_shared_vars = shared_vars + extra_shared_all + init_assigns = [{v: None} for v in all_shared_vars] + init_step = {"init_" + state.name: {"assign": init_assigns}} + + parallel_step = { + state.name: { + "parallel": { + # Only branch_res_* and map_res_* vars are shared; "res" is NOT + # listed so each branch gets its own local copy — no cross-branch + # interference when two branches both contain Map steps. + "shared": all_shared_vars, + "branches": gcp_branches, + } + } + } + # Merge: build a single dict keyed by branch root name using YAML dict syntax. + merged_dict = {branch.root: "${" + var + "}" for var, branch in zip(shared_vars, state.branches)} + merge_step = { + "merge_" + state.name: { + "assign": [{"res": merged_dict}] + } + } + steps: List[dict] = [init_step, parallel_step, merge_step] + if state.next: + steps.append({"next_" + state.name: {"next": state.next}}) + return steps + + def encode_loop(self, state: Loop) -> Union[dict, List[dict]]: + url = self._func_triggers[state.func_name] + + return { + state.name: { + "for": { + "value": "val", + "index": "idx", + "in": "${res."+state.array+"}", + "steps": [ + { + "body": { + "call": "http.post", + "args": {"url": url, "body": "${val}", "timeout": self._func_timeout} + } + } + ] + } + } + } + + def generate_maps(self): + for workflow_id, (url, common_params) in self._map_funcs.items(): + yield ( + workflow_id, + self._export_func( + { + "main": { + "params": ["elem"], + "steps": [ + { + "map": { + "call": "http.post", + "args": {"url": url, "body": "${elem}", "timeout": self._func_timeout}, + "result": "elem", + } + }, + {"ret": {"return": "${elem.body}"}}, + ], + } + } + ), + ) diff --git a/sebs/gcp/triggers.py b/sebs/gcp/triggers.py index a31f04b1d..8db6c064a 100644 --- a/sebs/gcp/triggers.py +++ b/sebs/gcp/triggers.py @@ -23,6 +23,7 @@ import concurrent.futures import datetime import json +import time from typing import Dict, Optional # noqa from sebs.gcp.gcp import GCP @@ -262,6 +263,71 @@ def deserialize(obj: Dict) -> Trigger: return LibraryTrigger(obj["name"], deployment_type=deployment_type) +class WorkflowLibraryTrigger(LibraryTrigger): + def sync_invoke(self, payload: dict) -> ExecutionResult: + from google.cloud.workflows.executions_v1 import ExecutionsClient, Execution + + self.logging.info(f"Invoke workflow {self.name}") + + config = self._deployment_client.config + full_workflow_name = ( + f"projects/{config.project_name}/locations/{config.region}/workflows/{self.name}" + ) + + execution_client = ExecutionsClient() + execution = Execution(argument=json.dumps(payload)) + + begin = datetime.datetime.now() + res = execution_client.create_execution(parent=full_workflow_name, execution=execution) + end = datetime.datetime.now() + + gcp_result = ExecutionResult.from_times(begin, end) + + execution_finished = False + while not execution_finished: + try: + execution = execution_client.get_execution( + request={"name": res.name}, + timeout=30, + ) + except Exception: + time.sleep(10) + continue + execution_finished = execution.state != Execution.State.ACTIVE + + if not execution_finished: + time.sleep(10) + elif execution.state == Execution.State.FAILED: + self.logging.error(f"Invocation of {self.name} failed") + self.logging.error(f"Input: {payload}") + self.logging.error(f"Error: {execution.error}") + gcp_result.stats.failure = True + return gcp_result + + if execution.result: + gcp_result.output = json.loads(execution.result) + + return gcp_result + + def async_invoke(self, payload: dict): + raise NotImplementedError("Async invocation is not implemented for workflows") + + @staticmethod + def typename() -> str: + return "GCP.WorkflowLibraryTrigger" + + @staticmethod + def trigger_type() -> Trigger.TriggerType: + return Trigger.TriggerType.LIBRARY + + def serialize(self) -> dict: + return {"type": "Library", "name": self.name} + + @staticmethod + def deserialize(obj: dict) -> "WorkflowLibraryTrigger": + return WorkflowLibraryTrigger(obj["name"]) + + class HTTPTrigger(Trigger): """HTTP endpoint trigger for Cloud Functions invocation. diff --git a/sebs/gcp/workflow.py b/sebs/gcp/workflow.py new file mode 100644 index 000000000..374ac9cd6 --- /dev/null +++ b/sebs/gcp/workflow.py @@ -0,0 +1,60 @@ +from typing import List, cast, Optional + +from sebs.faas.function import FunctionConfig, Workflow +from sebs.gcp.function import GCPFunction +from sebs.gcp.storage import GCPStorage + + +class GCPWorkflow(Workflow): + def __init__( + self, + name: str, + functions: List[GCPFunction], + benchmark: str, + code_package_hash: str, + cfg: FunctionConfig, + bucket: Optional[str] = None, + ): + super().__init__(benchmark, name, code_package_hash, cfg) + self.functions = functions + self.bucket = bucket + + @staticmethod + def typename() -> str: + return "GCP.GCPWorkflow" + + def serialize(self) -> dict: + return { + **super().serialize(), + "functions": [f.serialize() for f in self.functions], + "bucket": self.bucket, + } + + @staticmethod + def deserialize(cached_config: dict) -> "GCPWorkflow": + from sebs.faas.function import Trigger + from sebs.gcp.triggers import WorkflowLibraryTrigger, HTTPTrigger + + cfg = FunctionConfig.deserialize(cached_config["config"]) + funcs = [GCPFunction.deserialize(f) for f in cached_config["functions"]] + ret = GCPWorkflow( + cached_config["name"], + funcs, + cached_config["benchmark"], + cached_config["hash"], + cfg, + cached_config["bucket"], + ) + for trigger in cached_config["triggers"]: + trigger_type = cast( + Trigger, + {"Library": WorkflowLibraryTrigger, "HTTP": HTTPTrigger}.get(trigger["type"]), + ) + assert trigger_type, "Unknown trigger type {}".format(trigger["type"]) + ret.add_trigger(trigger_type.deserialize(trigger)) + return ret + + def code_bucket(self, benchmark: str, storage_client: GCPStorage): + if not self.bucket: + self.bucket, idx = storage_client.add_input_bucket(benchmark) + return self.bucket diff --git a/sebs/regression.py b/sebs/regression.py index 58920c79a..56f9a7e48 100644 --- a/sebs/regression.py +++ b/sebs/regression.py @@ -68,6 +68,23 @@ "503.graph-bfs", ] +# Workflow benchmarks available for regression testing +benchmarks_workflows = [ + "610.gen", + "620.func-invo", + "630.parallel-sleep", + "631.parallel-download", + "640.selfish-detour", + "650.vid", + "660.map-reduce", + "670.auth", + "680.excamera", + "690.ml", + "6100.1000-genome", + "6101.1000-genome-individuals", + "6200.trip-booking", +] + # AWS-specific configurations architectures_aws = ["x64", "arm64"] deployments_aws = ["package", "container"] @@ -84,8 +101,13 @@ architectures_openwhisk = ["x64"] deployments_openwhisk = ["container"] +# Cloudflare-specific configurations +architectures_cloudflare = ["x64"] + # User-defined config passed during initialization, set in regression_suite() cloud_config: Optional[dict] = None +# Input size for benchmark test data ("test" | "small" | "large"), set in regression_suite() +benchmark_input_size: str = "test" RESOURCE_PREFIX = "regr" LOGGING_REDACTED = False @@ -242,7 +264,7 @@ def test(self): # Prepare input data for the benchmark input_config = benchmark.prepare_input( deployment_client.system_resources, - size="test", + size=benchmark_input_size, replace_existing=experiment_config.update_storage, ) @@ -328,6 +350,259 @@ def test(self): return type.__new__(mcs, name, bases, dict) +class WorkflowTestSequenceMeta(type): + """Metaclass for dynamically generating workflow regression test cases. + + Similar to TestSequenceMeta but uses get_workflow instead of get_function, + and workflows have their trigger built-in (WorkflowLibraryTrigger). + """ + + def __init__(cls, name, bases, attrs, benchmarks, architectures, deployments, deployment_name): + type.__init__(cls, name, bases, attrs) + cls.deployment_name = deployment_name + + def __new__(mcs, name, bases, dict, benchmarks, architectures, deployments, deployment_name): + def gen_test(benchmark_name, architecture, deployment_type): + def test(self): + log_name = f"Regression-WF-{deployment_name}-{benchmark_name}-{deployment_type}" + logger = logging.getLogger(log_name) + logger.setLevel(logging.INFO) + logging_wrapper = ColoredWrapper(log_name, logger) + if LOGGING_REDACTED: + logger.addFilter(LOGGING_REDACTOR) + logging_wrapper.set_filter(LOGGING_REDACTOR) + + self.experiment_config["architecture"] = architecture + self.experiment_config["system_variant"] = deployment_type + + deployment_client = self.get_deployment( + benchmark_name, architecture, deployment_type + ) + deployment_client.disable_rich_output() + + logging_wrapper.info( + f"Begin workflow regression test of {benchmark_name} on " + f"{deployment_client.name()}. " + f"Architecture {architecture}, deployment type: {deployment_type}." + ) + + experiment_config = self.client.get_experiment_config(self.experiment_config) + benchmark = self.client.get_benchmark( + benchmark_name, deployment_client, experiment_config + ) + + input_config = benchmark.prepare_input( + deployment_client.system_resources, + size=benchmark_input_size, + replace_existing=experiment_config.update_storage, + ) + + wf = deployment_client.get_workflow( + benchmark, deployment_client.default_function_name(benchmark) + ) + + trigger_type = Trigger.TriggerType.LIBRARY + triggers = wf.triggers(trigger_type) + if len(triggers) == 0: + trigger = deployment_client.create_trigger(wf, trigger_type) + sleep(5) + else: + trigger = triggers[0] + + failure = False + try: + ret = trigger.sync_invoke(input_config) + if ret.stats.failure: + failure = True + logging_wrapper.error(f"{benchmark_name} workflow execution failed") + else: + output = ret.output + storage = ( + deployment_client.system_resources.get_storage() + if benchmark.uses_storage + else None + ) + error = benchmark.validate_output(input_config, output, storage) + if error is not None: + failure = True + logging_wrapper.error( + f"{benchmark_name} workflow output validation failed," + f" reason: {error}" + ) + else: + logging_wrapper.info(f"{benchmark_name} workflow execution succeeded") + except RuntimeError: + failure = True + logging_wrapper.error(f"{benchmark_name} workflow invocation raised exception") + + json_filename = ( + f"regression_wf_{deployment_name}_{benchmark_name}" + f"_{architecture}_{deployment_type}.json" + ) + with open(os.path.join(self.client.output_dir, json_filename), "w") as f: + json.dump({"output": ret.output}, f, indent=2) + + deployment_client.shutdown() + + if failure: + raise RuntimeError(f"Workflow test of {benchmark_name} failed!") + + return test + + for benchmark in benchmarks: + for architecture in architectures: + for deployment_type in deployments: + test_name = f"test_{deployment_name}_wf_{benchmark}" + test_name += f"_{architecture}_{deployment_type}" + test_method = gen_test(benchmark, architecture, deployment_type) + test_method.test_architecture = architecture + test_method.test_deployment_type = deployment_type + test_method.test_benchmark = benchmark + dict[test_name] = test_method + + dict["lock"] = threading.Lock() + dict["cfg"] = None + return type.__new__(mcs, name, bases, dict) + + +class AWSTestSequenceWorkflows( + unittest.TestCase, + metaclass=WorkflowTestSequenceMeta, + benchmarks=benchmarks_workflows, + architectures=["x64"], + deployments=["package"], + deployment_name="aws", +): + def get_deployment(self, benchmark_name, architecture, deployment_type): + deployment_name = "aws" + assert cloud_config, "Cloud configuration is required" + + config_copy = copy.deepcopy(cloud_config) + config_copy["experiments"]["architecture"] = architecture + configure_regression_deployment(config_copy, deployment_name, deployment_type) + + f = f"regression_wf_{deployment_name}_{benchmark_name}_{architecture}_{deployment_type}.log" + deployment_client = self.client.get_deployment( + config_copy, + logging_filename=os.path.join(self.client.output_dir, f), + ) + + with AWSTestSequenceWorkflows.lock: + deployment_client.initialize(resource_prefix=RESOURCE_PREFIX, quiet=LOGGING_REDACTED) + if LOGGING_REDACTED: + LOGGING_REDACTOR.set_resource_id( + deployment_client.config.resources.resources_id, + deployment_client.config.credentials.account_id, + ) + LoggingBase.set_filtering_resource_id( + deployment_client.config.resources.resources_id, + deployment_client.config.credentials.account_id, + ) + return deployment_client + + +class GCPTestSequenceWorkflows( + unittest.TestCase, + metaclass=WorkflowTestSequenceMeta, + benchmarks=benchmarks_workflows, + architectures=["x64"], + deployments=["function-gen2"], + deployment_name="gcp", +): + def get_deployment(self, benchmark_name, architecture, deployment_type): + deployment_name = "gcp" + assert cloud_config, "Cloud configuration is required" + + config_copy = copy.deepcopy(cloud_config) + config_copy["experiments"]["architecture"] = architecture + configure_regression_deployment(config_copy, deployment_name, deployment_type) + + f = f"regression_wf_{deployment_name}_{benchmark_name}_{architecture}_{deployment_type}.log" + deployment_client = self.client.get_deployment( + config_copy, + logging_filename=os.path.join(self.client.output_dir, f), + ) + + with GCPTestSequenceWorkflows.lock: + deployment_client.initialize(resource_prefix=RESOURCE_PREFIX, quiet=LOGGING_REDACTED) + if LOGGING_REDACTED: + LOGGING_REDACTOR.set_resource_id( + deployment_client.config.resources.resources_id, + deployment_client.config.credentials.project_name, + ) + LoggingBase.set_filtering_resource_id( + deployment_client.config.resources.resources_id, + deployment_client.config.credentials.project_name, + ) + return deployment_client + + +class AzureTestSequenceWorkflows( + unittest.TestCase, + metaclass=WorkflowTestSequenceMeta, + benchmarks=benchmarks_workflows, + architectures=architectures_azure, + deployments=deployments_azure, + deployment_name="azure", +): + """Test suite for workflow benchmarks on Azure Durable Functions.""" + + def get_deployment(self, benchmark_name, architecture, deployment_type): + """Get an Azure deployment client for workflow testing. + + Args: + benchmark_name: Name of the workflow benchmark to deploy + architecture: Architecture to deploy on (x64) + deployment_type: Deployment type (package) + + Returns: + An initialized Azure deployment client + """ + deployment_name = "azure" + assert cloud_config, "Cloud configuration is required" + + with AzureTestSequenceWorkflows.lock: + if not AzureTestSequenceWorkflows.cfg: + AzureTestSequenceWorkflows.cfg = self.client.get_deployment_config( + cloud_config["deployment"], + logging_filename=os.path.join( + self.client.output_dir, + f"regression_wf_{deployment_name}_{benchmark_name}_{architecture}.log", + ), + ) + + needs_login = False + if not hasattr(AzureTestSequenceWorkflows, "cli"): + from sebs.azure.cli import AzureCLI + + AzureTestSequenceWorkflows.cli = AzureCLI( + self.client.config, self.client.docker_client + ) + needs_login = True + + config_copy = copy.deepcopy(cloud_config) + config_copy["experiments"]["architecture"] = architecture + config_copy["experiments"]["system_variant"] = deployment_type + + f = f"regression_wf_{deployment_name}_{benchmark_name}_{architecture}_{deployment_type}.log" + deployment_client = self.client.get_deployment( + config_copy, + logging_filename=os.path.join(self.client.output_dir, f), + deployment_config=AzureTestSequenceWorkflows.cfg, + ) + + deployment_client.system_resources.initialize_cli( + cli=AzureTestSequenceWorkflows.cli, login=needs_login + ) + deployment_client.initialize(resource_prefix=RESOURCE_PREFIX, quiet=LOGGING_REDACTED) + if LOGGING_REDACTED: + LOGGING_REDACTOR.set_resource_id(deployment_client.config.resources.resources_id) + LoggingBase.set_filtering_resource_id( + deployment_client.config.resources.resources_id + ) + return deployment_client + + class AWSTestSequencePython( unittest.TestCase, metaclass=TestSequenceMeta, @@ -1188,6 +1463,183 @@ def get_deployment(self, benchmark_name, architecture, deployment_type): return deployment_client +class CloudflareTestSequenceWorkflows( + unittest.TestCase, + metaclass=WorkflowTestSequenceMeta, + benchmarks=benchmarks_workflows, + architectures=architectures_cloudflare, + deployments=["workers", "container"], + deployment_name="cloudflare", +): + """Test suite for workflow benchmarks on Cloudflare Workflows. + + Runs container-only benchmarks with system_variant=container and the four + lightweight benchmarks (610, 620, 630, 631) with both workers and container + variants. The filter_out_benchmarks function skips workers-incompatible ones. + """ + + def get_deployment(self, benchmark_name, architecture, deployment_type): + """Return an initialized Cloudflare deployment client for workflow testing. + + Args: + benchmark_name: Name of the workflow benchmark to deploy + architecture: Architecture to deploy on (x64) + deployment_type: Deployment type ("workers" for native Workers, "container") + + Returns: + An initialized Cloudflare deployment client + """ + deployment_name = "cloudflare" + assert cloud_config, "Cloud configuration is required" + + is_container = deployment_type == "container" + config_copy = copy.deepcopy(cloud_config) + config_copy["experiments"]["architecture"] = architecture + config_copy["experiments"]["container_deployment"] = is_container + config_copy["experiments"]["system_variant"] = deployment_type + + f = ( + f"regression_wf_{deployment_name}_{benchmark_name}" + f"_{architecture}_{deployment_type}.log" + ) + deployment_client = self.client.get_deployment( + config_copy, + logging_filename=os.path.join(self.client.output_dir, f), + ) + + with CloudflareTestSequenceWorkflows.lock: + deployment_client.initialize(resource_prefix=RESOURCE_PREFIX, quiet=LOGGING_REDACTED) + return deployment_client + + +class CloudflareTestSequencePythonWorkers( + unittest.TestCase, + metaclass=TestSequenceMeta, + benchmarks=benchmarks_python, + architectures=architectures_cloudflare, + deployments=["workers"], + deployment_name="cloudflare", + triggers=[Trigger.TriggerType.HTTP], +): + """Test suite for Python benchmarks on Cloudflare Workers.""" + + def get_deployment(self, benchmark_name, architecture, deployment_type): + """Return an initialized Cloudflare deployment client for Python workers.""" + deployment_name = "cloudflare" + assert cloud_config, "Cloud configuration is required" + + config_copy = copy.deepcopy(cloud_config) + config_copy["experiments"]["architecture"] = architecture + config_copy["experiments"]["container_deployment"] = False + config_copy["experiments"]["system_variant"] = "workers" + + f = f"regression_{deployment_name}_{benchmark_name}_{architecture}_{deployment_type}.log" + deployment_client = self.client.get_deployment( + config_copy, + logging_filename=os.path.join(self.client.output_dir, f), + ) + + with CloudflareTestSequencePythonWorkers.lock: + deployment_client.initialize(resource_prefix="regr") + return deployment_client + + +class CloudflareTestSequencePythonContainers( + unittest.TestCase, + metaclass=TestSequenceMeta, + benchmarks=benchmarks_python, + architectures=architectures_cloudflare, + deployments=["container"], + deployment_name="cloudflare", + triggers=[Trigger.TriggerType.HTTP], +): + """Test suite for Python benchmarks on Cloudflare Containers.""" + + def get_deployment(self, benchmark_name, architecture, deployment_type): + """Return an initialized Cloudflare deployment client for Python containers.""" + deployment_name = "cloudflare" + assert cloud_config, "Cloud configuration is required" + + config_copy = copy.deepcopy(cloud_config) + config_copy["experiments"]["architecture"] = architecture + config_copy["experiments"]["container_deployment"] = True + config_copy["experiments"]["system_variant"] = "container" + + f = f"regression_{deployment_name}_{benchmark_name}_{architecture}_{deployment_type}.log" + deployment_client = self.client.get_deployment( + config_copy, + logging_filename=os.path.join(self.client.output_dir, f), + ) + + with CloudflareTestSequencePythonContainers.lock: + deployment_client.initialize(resource_prefix="regr") + return deployment_client + + +class CloudflareTestSequenceNodejsWorkers( + unittest.TestCase, + metaclass=TestSequenceMeta, + benchmarks=benchmarks_nodejs, + architectures=architectures_cloudflare, + deployments=["workers"], + deployment_name="cloudflare", + triggers=[Trigger.TriggerType.HTTP], +): + """Test suite for Node.js benchmarks on Cloudflare Workers.""" + + def get_deployment(self, benchmark_name, architecture, deployment_type): + """Return an initialized Cloudflare deployment client for Node.js workers.""" + deployment_name = "cloudflare" + assert cloud_config, "Cloud configuration is required" + + config_copy = copy.deepcopy(cloud_config) + config_copy["experiments"]["architecture"] = architecture + config_copy["experiments"]["container_deployment"] = False + config_copy["experiments"]["system_variant"] = "workers" + + f = f"regression_{deployment_name}_{benchmark_name}_{architecture}_{deployment_type}.log" + deployment_client = self.client.get_deployment( + config_copy, + logging_filename=os.path.join(self.client.output_dir, f), + ) + + with CloudflareTestSequenceNodejsWorkers.lock: + deployment_client.initialize(resource_prefix="regr") + return deployment_client + + +class CloudflareTestSequenceNodejsContainers( + unittest.TestCase, + metaclass=TestSequenceMeta, + benchmarks=benchmarks_nodejs, + architectures=architectures_cloudflare, + deployments=["container"], + deployment_name="cloudflare", + triggers=[Trigger.TriggerType.HTTP], +): + """Test suite for Node.js benchmarks on Cloudflare Containers.""" + + def get_deployment(self, benchmark_name, architecture, deployment_type): + """Return an initialized Cloudflare deployment client for Node.js containers.""" + deployment_name = "cloudflare" + assert cloud_config, "Cloud configuration is required" + + config_copy = copy.deepcopy(cloud_config) + config_copy["experiments"]["architecture"] = architecture + config_copy["experiments"]["container_deployment"] = True + config_copy["experiments"]["system_variant"] = "container" + + f = f"regression_{deployment_name}_{benchmark_name}_{architecture}_{deployment_type}.log" + deployment_client = self.client.get_deployment( + config_copy, + logging_filename=os.path.join(self.client.output_dir, f), + ) + + with CloudflareTestSequenceNodejsContainers.lock: + deployment_client.initialize(resource_prefix="regr") + return deployment_client + + # Stream result handler for concurrent test execution # Based on https://stackoverflow.com/questions/22484805/ # a-simple-working-example-for-testtools-concurrentstreamtestsuite @@ -1306,6 +1758,21 @@ def filter_out_benchmarks( and language_version in ["3.8", "3.9", "3.10", "3.11", "3.12"] and deployment_type == "function-gen1"): return "411.image-recognition" not in benchmark + + # Cloudflare: only certain benchmarks are supported per language/deployment-type. + # None means all benchmarks are supported for that combination. + if deployment_name == "cloudflare": + from sebs.cloudflare.cloudflare import Cloudflare + is_container = deployment_type == "container" + allowed = Cloudflare.SUPPORTED_BENCHMARKS.get((language, is_container)) + if allowed is not None: + # benchmark is the test method name, e.g. "test_cloudflare_120.uploader_x64_workers" + # Extract the numeric benchmark prefix (e.g. "120") from before the first "." + if "." in benchmark: + benchmark_id = benchmark.split(".")[-2].split("_")[-1] + else: + benchmark_id = benchmark.split("_")[-1] + return benchmark_id in allowed # fmt: on # All other benchmarks are supported @@ -1319,6 +1786,8 @@ def regression_suite( deployment_config: dict, resource_prefix: str | None = None, benchmark_name: Optional[str] = None, + deployment_type: Optional[str] = None, + input_size: str = "test", selected_architecture: str | None = None, filter_output: bool = False, ): @@ -1354,9 +1823,10 @@ def regression_suite( # Create the test suite suite = unittest.TestSuite() - # Make cloud_config available to test classes - global cloud_config + # Make cloud_config and input size available to test classes + global cloud_config, benchmark_input_size cloud_config = deployment_config + benchmark_input_size = input_size # Extract runtime configuration language = experiment_config["runtime"]["language"] @@ -1417,6 +1887,57 @@ def regression_suite( unittest.defaultTestLoader.loadTestsFromTestCase(OpenWhiskTestSequenceJava) ) + # Add Cloudflare tests if requested + if "cloudflare" in providers: + assert ( + "cloudflare" in cloud_config["deployment"] + ), "Cloudflare provider requested but not in deployment config" + if language == "python": + if deployment_type != "containers": + suite.addTest( + unittest.defaultTestLoader.loadTestsFromTestCase( + CloudflareTestSequencePythonWorkers + ) + ) + if deployment_type != "functions": + suite.addTest( + unittest.defaultTestLoader.loadTestsFromTestCase( + CloudflareTestSequencePythonContainers + ) + ) + elif language == "nodejs": + if deployment_type != "containers": + suite.addTest( + unittest.defaultTestLoader.loadTestsFromTestCase( + CloudflareTestSequenceNodejsWorkers + ) + ) + if deployment_type != "functions": + suite.addTest( + unittest.defaultTestLoader.loadTestsFromTestCase( + CloudflareTestSequenceNodejsContainers + ) + ) + + # Add workflow tests (only for Python, workflows are Python-only) + if language == "python": + if "aws" in providers: + suite.addTest( + unittest.defaultTestLoader.loadTestsFromTestCase(AWSTestSequenceWorkflows) + ) + if "gcp" in providers: + suite.addTest( + unittest.defaultTestLoader.loadTestsFromTestCase(GCPTestSequenceWorkflows) + ) + if "azure" in providers: + suite.addTest( + unittest.defaultTestLoader.loadTestsFromTestCase(AzureTestSequenceWorkflows) + ) + if "cloudflare" in providers: + suite.addTest( + unittest.defaultTestLoader.loadTestsFromTestCase(CloudflareTestSequenceWorkflows) + ) + # Prepare the list of tests to run tests: List[unittest.TestCase] = [] # mypy is confused here about the type @@ -1496,6 +2017,8 @@ def regression_suite( AzureTestSequencePython.cli.shutdown() if hasattr(AzureTestSequenceJava, "cli"): AzureTestSequenceJava.cli.shutdown() + if hasattr(AzureTestSequenceWorkflows, "cli"): + AzureTestSequenceWorkflows.cli.shutdown() # Return True if any test failed return not result.all_correct diff --git a/sebs/sebs.py b/sebs/sebs.py index d99aac4c3..6beb042d7 100644 --- a/sebs/sebs.py +++ b/sebs/sebs.py @@ -214,6 +214,10 @@ def get_deployment( from sebs.openwhisk import OpenWhisk implementations["openwhisk"] = OpenWhisk + if has_platform("cloudflare"): + from sebs.cloudflare import Cloudflare + + implementations["cloudflare"] = Cloudflare # Validate deployment platform if name not in implementations: diff --git a/sebs/utils.py b/sebs/utils.py index 4ebdcb82c..468c0f3f4 100644 --- a/sebs/utils.py +++ b/sebs/utils.py @@ -689,7 +689,7 @@ def has_platform(name: str) -> bool: import google.cloud.devtools # noqa: F401 return True - elif name in ("local", "openwhisk"): + elif name in ("local", "openwhisk", "cloudflare"): # these don't have specific dependencies return True else: @@ -795,3 +795,35 @@ def ensure_benchmarks_data(logger: ColoredWrapper) -> Path: raise RuntimeError(f"Failed to initialize benchmarks-data submodule: {e.stderr}") from e except FileNotFoundError: raise RuntimeError("git command not found. Please install git to use SeBS") from None + + +def replace_string_in_file(path: str, from_str: str, to_str: str): + with open(path, "rt") as f: + data = f.read() + data = data.replace(from_str, to_str) + with open(path, "wt") as f: + f.write(data) + + +def connect_to_redis_cache(host: str): + from redis import Redis + + redis = Redis(host=host, port=6379, decode_responses=True, socket_connect_timeout=10) + redis.ping() + return redis + + +def download_measurements(redis, workflow_name: str, after: float, **static_args): + payloads = [] + for key in redis.scan_iter(match=f"{workflow_name}/*"): + payload = redis.get(key) + redis.delete(key) + if payload: + try: + payload = json.loads(payload) + if payload["start"] > after: + payload = {**payload, **static_args} + payloads.append(payload) + except json.decoder.JSONDecodeError: + pass + return payloads