From 9b30024988014786052f15dbcd192c099c8ebccf Mon Sep 17 00:00:00 2001 From: Lorenz Gruber Date: Tue, 4 Nov 2025 16:49:58 +0100 Subject: [PATCH 01/45] updating ignore --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 0712f6d7b..51a36f6c4 100644 --- a/.gitignore +++ b/.gitignore @@ -188,3 +188,6 @@ cache # IntelliJ IDEA files .idea *.iml + +# Local config +config/config.json From 93c1fe53c32daa688a578a95cae6cfa645fedd89 Mon Sep 17 00:00:00 2001 From: Lorenz Gruber Date: Tue, 4 Nov 2025 16:50:06 +0100 Subject: [PATCH 02/45] fixing dummy workflow --- benchmarks/600.workflows/610.gen/config.json | 3 ++- benchmarks/600.workflows/610.gen/input.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/benchmarks/600.workflows/610.gen/config.json b/benchmarks/600.workflows/610.gen/config.json index 8eae08240..8ff6eec59 100644 --- a/benchmarks/600.workflows/610.gen/config.json +++ b/benchmarks/600.workflows/610.gen/config.json @@ -1,5 +1,6 @@ { "timeout": 120, "memory": 128, - "languages": ["python"] + "languages": ["python"], + "modules": [] } diff --git a/benchmarks/600.workflows/610.gen/input.py b/benchmarks/600.workflows/610.gen/input.py index 68f82e81f..f4073b711 100644 --- a/benchmarks/600.workflows/610.gen/input.py +++ b/benchmarks/600.workflows/610.gen/input.py @@ -1,5 +1,5 @@ def buckets_count(): return (0, 0) -def generate_input(data_dir, size, input_buckets, output_buckets, upload_func): +def generate_input(data_dir, size, benchmarks_bucket,input_buckets, output_buckets, upload_func, nosql_func): return dict() \ No newline at end of file From c4ef1076a4e5849a4e795e5878a949c3c5ef35ec Mon Sep 17 00:00:00 2001 From: Lorenz Gruber Date: Tue, 11 Nov 2025 17:05:55 +0100 Subject: [PATCH 03/45] vscode run config. to be removed later --- .vscode/launch.json | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 .vscode/launch.json diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 000000000..40ea1f6ed --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,16 @@ +{ + // Use IntelliSense to learn about possible attributes. + // Hover to view descriptions of existing attributes. + // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "configurations": [ + { + "name": "Python Debugger: Current File with Arguments", + "type": "debugpy", + "request": "launch", + "program": "sebs.py", + "console": "integratedTerminal", + "args": "benchmark workflow 610.gen test --config config/example.json --deployment aws --trigger library --repetitions 1" + } + ] +} \ No newline at end of file From c9fa1964ef43ea208fade94d65c6c28e45aef554 Mon Sep 17 00:00:00 2001 From: Lorenz Gruber Date: Tue, 11 Nov 2025 17:06:18 +0100 Subject: [PATCH 04/45] fix call site to create_function --- sebs/aws/aws.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sebs/aws/aws.py b/sebs/aws/aws.py index 18706d20e..94e9f8349 100644 --- a/sebs/aws/aws.py +++ b/sebs/aws/aws.py @@ -453,7 +453,7 @@ def create_workflow(self, code_package: Benchmark, workflow_name: str) -> "SFNWo code_files = list(code_package.get_code_files(include_config=False)) func_names = [os.path.splitext(os.path.basename(p))[0] for p in code_files] funcs = [ - self.create_function(code_package, workflow_name + "___" + fn) for fn in func_names + self.create_function(code_package, workflow_name + "___" + fn, code_package.container_deployment,code_package.container_uri if code_package.container_deployment else None) for fn in func_names ] # Generate workflow definition.json From 5de04ed8a34c0995f58bd6bcecb514e7a46de980 Mon Sep 17 00:00:00 2001 From: Lorenz Gruber Date: Tue, 11 Nov 2025 17:06:44 +0100 Subject: [PATCH 05/45] adapt to changed signature --- sebs/faas/system.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sebs/faas/system.py b/sebs/faas/system.py index 0adcfc1d7..0b2b35b20 100644 --- a/sebs/faas/system.py +++ b/sebs/faas/system.py @@ -363,7 +363,7 @@ def update_workflow(self, workflow: Workflow, code_package: Benchmark): def get_workflow(self, code_package: Benchmark, workflow_name: Optional[str] = None): if code_package.language_version not in self.system_config.supported_language_versions( - self.name(), code_package.language_name + self.name(), code_package.language_name, code_package.architecture ): raise Exception( "Unsupported {language} version {version} in {system}!".format( @@ -375,7 +375,7 @@ def get_workflow(self, code_package: Benchmark, workflow_name: Optional[str] = N if not workflow_name: workflow_name = self.default_function_name(code_package) - rebuilt, _ = code_package.build(self.package_code, True) + rebuilt, _ , _ ,_= code_package.build(self.package_code, True) """ There's no function with that name? From 60143bfd31b9a8fee12b4e2a2864fc16e264b18a Mon Sep 17 00:00:00 2001 From: Lorenz Gruber Date: Mon, 17 Nov 2025 17:15:54 +0100 Subject: [PATCH 06/45] fixing state machine --- benchmarks/600.workflows/610.gen/definition.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/benchmarks/600.workflows/610.gen/definition.json b/benchmarks/600.workflows/610.gen/definition.json index fcdf203bc..4ccb52abe 100644 --- a/benchmarks/600.workflows/610.gen/definition.json +++ b/benchmarks/600.workflows/610.gen/definition.json @@ -37,10 +37,10 @@ "map_astros": { "type": "map", "array": "astros.people", - "root": "map_astros", + "root": "map", "next": "process_astros", "states": { - "map_astros": { + "map": { "type": "task", "func_name": "map_astros" } From fab742a7e55d01eff6738a2abad8521a1ffa25ad Mon Sep 17 00:00:00 2001 From: Lorenz Gruber Date: Mon, 17 Nov 2025 17:58:26 +0100 Subject: [PATCH 07/45] adjusting temporary vscode debug config --- .vscode/launch.json | 25 +++++++++++++++++++++---- .vscode/settings.json | 3 +++ 2 files changed, 24 insertions(+), 4 deletions(-) create mode 100644 .vscode/settings.json diff --git a/.vscode/launch.json b/.vscode/launch.json index 40ea1f6ed..a48c57f0b 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -5,12 +5,29 @@ "version": "0.2.0", "configurations": [ { - "name": "Python Debugger: Current File with Arguments", - "type": "debugpy", + "name": "SeBS Benchmark Workflow", + "type": "python", "request": "launch", - "program": "sebs.py", + "program": "${workspaceFolder}/sebs.py", + "cwd": "${workspaceFolder}", "console": "integratedTerminal", - "args": "benchmark workflow 610.gen test --config config/example.json --deployment aws --trigger library --repetitions 1" + "args": [ + "benchmark", + "workflow", + "610.gen", + "test", + "--config", + "config/config.json", + "--deployment", + "aws", + "--trigger", + "library", + "--repetitions", + "1" + ], + "justMyCode": true, + "pythonPath": "${workspaceFolder}/python-venv/bin/python", + "envFile": "${workspaceFolder}/.env" } ] } \ No newline at end of file diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 000000000..7e68766ae --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,3 @@ +{ + "python-envs.pythonProjects": [] +} \ No newline at end of file From 3c846d058589f9b1a07d91fc1ac944b542115fe5 Mon Sep 17 00:00:00 2001 From: Lorenz Gruber Date: Mon, 17 Nov 2025 17:59:02 +0100 Subject: [PATCH 08/45] adding redis configs to gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 51a36f6c4..53f5d8311 100644 --- a/.gitignore +++ b/.gitignore @@ -191,3 +191,4 @@ cache # Local config config/config.json +config/redis From 19027fee233f25c53d9075a0ca2f86c67b44c538 Mon Sep 17 00:00:00 2001 From: Lorenz Gruber Date: Wed, 19 Nov 2025 15:06:10 +0100 Subject: [PATCH 09/45] fixing demo workflow --- benchmarks/600.workflows/610.gen/python/map_astros.py | 9 ++++----- benchmarks/600.workflows/610.gen/python/requirements.txt | 1 + 2 files changed, 5 insertions(+), 5 deletions(-) create mode 100644 benchmarks/600.workflows/610.gen/python/requirements.txt diff --git a/benchmarks/600.workflows/610.gen/python/map_astros.py b/benchmarks/600.workflows/610.gen/python/map_astros.py index b98b5e9d7..54e530ff2 100644 --- a/benchmarks/600.workflows/610.gen/python/map_astros.py +++ b/benchmarks/600.workflows/610.gen/python/map_astros.py @@ -1,7 +1,6 @@ def handler(elem): - name = elem["name"] - fn, ln = name.split(" ") - name = " ".join([ln, fn]) - elem["name_rev"] = name - + full_name:str = elem["name"] + names = full_name.split(" ") + names.reverse() + elem["name_rev"] = " ".join(names) return elem \ No newline at end of file diff --git a/benchmarks/600.workflows/610.gen/python/requirements.txt b/benchmarks/600.workflows/610.gen/python/requirements.txt new file mode 100644 index 000000000..663bd1f6a --- /dev/null +++ b/benchmarks/600.workflows/610.gen/python/requirements.txt @@ -0,0 +1 @@ +requests \ No newline at end of file From 790ad27be4613bd24b25e5936b24637664df1d95 Mon Sep 17 00:00:00 2001 From: Lorenz Gruber Date: Wed, 19 Nov 2025 15:06:31 +0100 Subject: [PATCH 10/45] fix: function update from cache --- sebs/aws/aws.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sebs/aws/aws.py b/sebs/aws/aws.py index 94e9f8349..2fb60f7ed 100644 --- a/sebs/aws/aws.py +++ b/sebs/aws/aws.py @@ -527,7 +527,7 @@ def update_workflow(self, workflow: Workflow, code_package: Benchmark): code_files = list(code_package.get_code_files(include_config=False)) func_names = [os.path.splitext(os.path.basename(p))[0] for p in code_files] funcs = [ - self.create_function(code_package, workflow.name + "___" + fn) for fn in func_names + self.create_function(code_package, workflow.name + "___" + fn,code_package.container_deployment,code_package.container_uri if code_package.container_deployment else None) for fn in func_names ] # Generate workflow definition.json From c954fd5d1a85f05a2c6f740df88833487f82830c Mon Sep 17 00:00:00 2001 From: Lorenz Gruber Date: Wed, 19 Nov 2025 15:06:56 +0100 Subject: [PATCH 11/45] fix: state machine encoding of switch case variable --- sebs/aws/generator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sebs/aws/generator.py b/sebs/aws/generator.py index 602bc3998..6b941deb7 100644 --- a/sebs/aws/generator.py +++ b/sebs/aws/generator.py @@ -117,7 +117,7 @@ def _encode_case(self, case: Switch.Case) -> dict: } cond = type + comp[case.op] - return {"Variable": "$.payload" + case.var, cond: case.val, "Next": case.next} + return {"Variable": "$.payload." + case.var, cond: case.val, "Next": case.next} def encode_map(self, state: Map) -> Union[dict, List[dict]]: states = {n: State.deserialize(n, s) for n, s in state.funcs.items()} From e10af40c491a250a9c49c3bb7dd129a66eccb73b Mon Sep 17 00:00:00 2001 From: Lorenz Gruber Date: Wed, 19 Nov 2025 15:08:45 +0100 Subject: [PATCH 12/45] updating ignore --- .gitignore | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.gitignore b/.gitignore index 53f5d8311..0efc0e4c0 100644 --- a/.gitignore +++ b/.gitignore @@ -192,3 +192,7 @@ cache # Local config config/config.json config/redis + +# Benchmark Results +results/ +experiments.json \ No newline at end of file From 39af6e57dfdc4d5980e6de54279eb9788c975057 Mon Sep 17 00:00:00 2001 From: Lorenz Gruber Date: Wed, 19 Nov 2025 17:15:22 +0100 Subject: [PATCH 13/45] adding skeleton for sda workflow --- benchmarks/600.workflows/6300.sda-workflow/config.json | 6 ++++++ .../600.workflows/6300.sda-workflow/definition.json | 9 +++++++++ benchmarks/600.workflows/6300.sda-workflow/input.py | 5 +++++ .../600.workflows/6300.sda-workflow/python/filter.py | 2 ++ 4 files changed, 22 insertions(+) create mode 100644 benchmarks/600.workflows/6300.sda-workflow/config.json create mode 100644 benchmarks/600.workflows/6300.sda-workflow/definition.json create mode 100644 benchmarks/600.workflows/6300.sda-workflow/input.py create mode 100644 benchmarks/600.workflows/6300.sda-workflow/python/filter.py diff --git a/benchmarks/600.workflows/6300.sda-workflow/config.json b/benchmarks/600.workflows/6300.sda-workflow/config.json new file mode 100644 index 000000000..8ff6eec59 --- /dev/null +++ b/benchmarks/600.workflows/6300.sda-workflow/config.json @@ -0,0 +1,6 @@ +{ + "timeout": 120, + "memory": 128, + "languages": ["python"], + "modules": [] +} diff --git a/benchmarks/600.workflows/6300.sda-workflow/definition.json b/benchmarks/600.workflows/6300.sda-workflow/definition.json new file mode 100644 index 000000000..a82881b1d --- /dev/null +++ b/benchmarks/600.workflows/6300.sda-workflow/definition.json @@ -0,0 +1,9 @@ +{ + "root": "filter", + "states": { + "filter": { + "type": "task", + "func_name": "filter" + } + } +} diff --git a/benchmarks/600.workflows/6300.sda-workflow/input.py b/benchmarks/600.workflows/6300.sda-workflow/input.py new file mode 100644 index 000000000..f4073b711 --- /dev/null +++ b/benchmarks/600.workflows/6300.sda-workflow/input.py @@ -0,0 +1,5 @@ +def buckets_count(): + return (0, 0) + +def generate_input(data_dir, size, benchmarks_bucket,input_buckets, output_buckets, upload_func, nosql_func): + return dict() \ No newline at end of file diff --git a/benchmarks/600.workflows/6300.sda-workflow/python/filter.py b/benchmarks/600.workflows/6300.sda-workflow/python/filter.py new file mode 100644 index 000000000..4d737b3c9 --- /dev/null +++ b/benchmarks/600.workflows/6300.sda-workflow/python/filter.py @@ -0,0 +1,2 @@ +def handler(event): + return {} \ No newline at end of file From 840a76f40de20f809b5b49f5d69ccb4b6dd59861 Mon Sep 17 00:00:00 2001 From: Lorenz Gruber Date: Thu, 20 Nov 2025 15:01:14 +0100 Subject: [PATCH 14/45] adding another launch config for debugging --- .vscode/launch.json | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index a48c57f0b..e0debce64 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -5,7 +5,7 @@ "version": "0.2.0", "configurations": [ { - "name": "SeBS Benchmark Workflow", + "name": "SeBS Benchmark Workflow 610", "type": "python", "request": "launch", "program": "${workspaceFolder}/sebs.py", @@ -28,6 +28,32 @@ "justMyCode": true, "pythonPath": "${workspaceFolder}/python-venv/bin/python", "envFile": "${workspaceFolder}/.env" + }, + { + "name": "SeBS SDA Workflow", + "type": "python", + "request": "launch", + "program": "${workspaceFolder}/sebs.py", + "cwd": "${workspaceFolder}", + "console": "integratedTerminal", + "args": [ + "benchmark", + "workflow", + "6300.sda-workflow", + "test", + "--config", + "config/config.json", + "--deployment", + "aws", + "--container-deployment", + "--trigger", + "library", + "--repetitions", + "1" + ], + "justMyCode": true, + "pythonPath": "${workspaceFolder}/python-venv/bin/python", + "envFile": "${workspaceFolder}/.env" } ] } \ No newline at end of file From a9f882a271c2af8202253a5d30dc6522aa0352b3 Mon Sep 17 00:00:00 2001 From: Lorenz Gruber Date: Thu, 20 Nov 2025 15:01:35 +0100 Subject: [PATCH 15/45] working on lambda base image for custom containers --- dockerfiles/aws/python/Dockerfile.base | 20 ++++++++++++++++++++ sebs/aws/container.py | 4 ++++ sebs/faas/container.py | 4 ++++ 3 files changed, 28 insertions(+) create mode 100644 dockerfiles/aws/python/Dockerfile.base diff --git a/dockerfiles/aws/python/Dockerfile.base b/dockerfiles/aws/python/Dockerfile.base new file mode 100644 index 000000000..fb6fbe4a6 --- /dev/null +++ b/dockerfiles/aws/python/Dockerfile.base @@ -0,0 +1,20 @@ +# Define custom base image +ARG BASE_IMAGE="python:3.12" + +FROM ${BASE_IMAGE} AS build-image + +# Include global arg in this stage of the build +ARG FUNCTION_DIR="./function" + +# Copy function code +RUN mkdir -p ${FUNCTION_DIR} +COPY . ${FUNCTION_DIR} + +# Install the function's dependencies +RUN pip install \ + --target ${FUNCTION_DIR} \ + awslambdaric +# Set runtime interface client as default command for the container runtime +ENTRYPOINT [ "/usr/local/bin/python", "-m", "awslambdaric" ] +# Pass the name of the function handler as an argument to the runtime +CMD [ "lambda_function.handler" ] \ No newline at end of file diff --git a/sebs/aws/container.py b/sebs/aws/container.py index 6a2f20e0c..05735ab26 100644 --- a/sebs/aws/container.py +++ b/sebs/aws/container.py @@ -81,3 +81,7 @@ def push_image(self, repository_uri, image_tag): self.logging.error(f"Failed to push the image to registry {repository_uri}.") self.logging.error(f"Error: {str(e)}") raise RuntimeError("Couldn't push to Docker registry") + + + def build_custom_image(self, image_tag):str + return "" \ No newline at end of file diff --git a/sebs/faas/container.py b/sebs/faas/container.py index af24d8bd5..0c812fd19 100644 --- a/sebs/faas/container.py +++ b/sebs/faas/container.py @@ -217,3 +217,7 @@ def build_base_image( self.push_image(image_uri, image_tag) return True, image_uri + + @abstractmethod + def build_custom_image(self, image_tag: str): + pass \ No newline at end of file From f64a6611337ca512a12bec637ed6553a25a144d7 Mon Sep 17 00:00:00 2001 From: Lorenz Gruber Date: Thu, 20 Nov 2025 16:39:10 +0100 Subject: [PATCH 16/45] working on custom aws containers --- .../600.workflows/6300.sda-workflow/README.md | 26 +++++++++++++++++++ dockerfiles/aws/python/Dockerfile.base | 5 ++-- 2 files changed, 29 insertions(+), 2 deletions(-) create mode 100644 benchmarks/600.workflows/6300.sda-workflow/README.md diff --git a/benchmarks/600.workflows/6300.sda-workflow/README.md b/benchmarks/600.workflows/6300.sda-workflow/README.md new file mode 100644 index 000000000..2a121a85a --- /dev/null +++ b/benchmarks/600.workflows/6300.sda-workflow/README.md @@ -0,0 +1,26 @@ +# Docker Commands: +### Build Image +``` +docker build -t sebs/sda-aws -f dockerfiles/aws/python/Dockerfile.base 6300.sda-workflow_code/python/3.8/x64/container/function/build/ +``` +### Test Run +[Custom Image Tutorial](https://docs.aws.amazon.com/lambda/latest/dg/python-image.html#python-image-clients) +#### Start Container +``` +docker run \ +--platform linux/amd64 -d \ +-v ~/.aws-lambda-rie:/aws-lambda \ +-p 9000:8080 \ +--entrypoint /aws-lambda/aws-lambda-rie \ +--name sda-aws-container \ +logru/sda-aws:latest \ +/usr/bin/python -m awslambdaric function/handler.handler +``` +#### Debug Container +``` +docker exec -it sda-aws-container /bin/bash +``` +#### Trigger Requests +``` + curl "http://localhost:9000/2015-03-31/functions/function/invocations" -d '{}' +``` diff --git a/dockerfiles/aws/python/Dockerfile.base b/dockerfiles/aws/python/Dockerfile.base index fb6fbe4a6..ec5828c3b 100644 --- a/dockerfiles/aws/python/Dockerfile.base +++ b/dockerfiles/aws/python/Dockerfile.base @@ -2,6 +2,7 @@ ARG BASE_IMAGE="python:3.12" FROM ${BASE_IMAGE} AS build-image +RUN apt-get update && apt-get install -y python3-pip # Include global arg in this stage of the build ARG FUNCTION_DIR="./function" @@ -15,6 +16,6 @@ RUN pip install \ --target ${FUNCTION_DIR} \ awslambdaric # Set runtime interface client as default command for the container runtime -ENTRYPOINT [ "/usr/local/bin/python", "-m", "awslambdaric" ] +ENTRYPOINT [ "/usr/bin/python", "-m", "awslambdaric" ] # Pass the name of the function handler as an argument to the runtime -CMD [ "lambda_function.handler" ] \ No newline at end of file +CMD [ "function/handler.handler" ] \ No newline at end of file From 88ca99f6568ddd4607e2ba7db6f248860baa20a3 Mon Sep 17 00:00:00 2001 From: Lorenz Gruber Date: Mon, 24 Nov 2025 17:51:34 +0100 Subject: [PATCH 17/45] flag not necessary when deploying user containers --- .vscode/launch.json | 1 - 1 file changed, 1 deletion(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index e0debce64..8c71a624c 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -45,7 +45,6 @@ "config/config.json", "--deployment", "aws", - "--container-deployment", "--trigger", "library", "--repetitions", From c3cf46cf1e462d8c43135dd035c4d83d8d8398e5 Mon Sep 17 00:00:00 2001 From: Lorenz Gruber Date: Mon, 24 Nov 2025 17:52:03 +0100 Subject: [PATCH 18/45] moving dev stuff --- .../600.workflows/6300.sda-workflow/dev/Dockerfile | 10 ++++++---- .../6300.sda-workflow/{ => dev}/README.md | 11 ++++++++--- sebs/aws/container.py | 4 ++-- 3 files changed, 16 insertions(+), 9 deletions(-) rename dockerfiles/aws/python/Dockerfile.base => benchmarks/600.workflows/6300.sda-workflow/dev/Dockerfile (75%) rename benchmarks/600.workflows/6300.sda-workflow/{ => dev}/README.md (58%) diff --git a/dockerfiles/aws/python/Dockerfile.base b/benchmarks/600.workflows/6300.sda-workflow/dev/Dockerfile similarity index 75% rename from dockerfiles/aws/python/Dockerfile.base rename to benchmarks/600.workflows/6300.sda-workflow/dev/Dockerfile index ec5828c3b..ab04c6f80 100644 --- a/dockerfiles/aws/python/Dockerfile.base +++ b/benchmarks/600.workflows/6300.sda-workflow/dev/Dockerfile @@ -5,16 +5,18 @@ FROM ${BASE_IMAGE} AS build-image RUN apt-get update && apt-get install -y python3-pip # Include global arg in this stage of the build -ARG FUNCTION_DIR="./function" +ARG LAMBDA_DIR="/lambda" # Copy function code -RUN mkdir -p ${FUNCTION_DIR} -COPY . ${FUNCTION_DIR} +RUN mkdir -p ${LAMBDA_DIR} +RUN mkdir -p ${LAMBDA_DIR}/function +COPY . ${LAMBDA_DIR}/function # Install the function's dependencies RUN pip install \ - --target ${FUNCTION_DIR} \ + --target ${LAMBDA_DIR} \ awslambdaric +WORKDIR ${LAMBDA_DIR} # Set runtime interface client as default command for the container runtime ENTRYPOINT [ "/usr/bin/python", "-m", "awslambdaric" ] # Pass the name of the function handler as an argument to the runtime diff --git a/benchmarks/600.workflows/6300.sda-workflow/README.md b/benchmarks/600.workflows/6300.sda-workflow/dev/README.md similarity index 58% rename from benchmarks/600.workflows/6300.sda-workflow/README.md rename to benchmarks/600.workflows/6300.sda-workflow/dev/README.md index 2a121a85a..b2b005e9b 100644 --- a/benchmarks/600.workflows/6300.sda-workflow/README.md +++ b/benchmarks/600.workflows/6300.sda-workflow/dev/README.md @@ -1,7 +1,11 @@ # Docker Commands: ### Build Image ``` -docker build -t sebs/sda-aws -f dockerfiles/aws/python/Dockerfile.base 6300.sda-workflow_code/python/3.8/x64/container/function/build/ +docker build \ +--build-arg BASE_IMAGE=logru/sda:latest \ +-t logru/sda-sebs-aws \ +-f benchmarks/600.workflows/6300.sda-workflow/dev/Dockerfile \ +6300.sda-workflow_code/python/3.8/x64/package/ ``` ### Test Run [Custom Image Tutorial](https://docs.aws.amazon.com/lambda/latest/dg/python-image.html#python-image-clients) @@ -11,9 +15,10 @@ docker run \ --platform linux/amd64 -d \ -v ~/.aws-lambda-rie:/aws-lambda \ -p 9000:8080 \ +-e AWS_LAMBDA_FUNCTION_NAME=SDA___filter \ --entrypoint /aws-lambda/aws-lambda-rie \ --name sda-aws-container \ -logru/sda-aws:latest \ +logru/sda-sebs-aws:latest \ /usr/bin/python -m awslambdaric function/handler.handler ``` #### Debug Container @@ -22,5 +27,5 @@ docker exec -it sda-aws-container /bin/bash ``` #### Trigger Requests ``` - curl "http://localhost:9000/2015-03-31/functions/function/invocations" -d '{}' +curl "http://localhost:9000/2015-03-31/functions/function/invocations" -d '{"payload":{},"request_id":"1"}' ``` diff --git a/sebs/aws/container.py b/sebs/aws/container.py index 05735ab26..6ef8216c3 100644 --- a/sebs/aws/container.py +++ b/sebs/aws/container.py @@ -83,5 +83,5 @@ def push_image(self, repository_uri, image_tag): raise RuntimeError("Couldn't push to Docker registry") - def build_custom_image(self, image_tag):str - return "" \ No newline at end of file + # def build_custom_image(self, image_tag):str + # return "" \ No newline at end of file From bc197ad07d5275ca4ba4127621baa92797a6d434 Mon Sep 17 00:00:00 2001 From: Lorenz Gruber Date: Mon, 24 Nov 2025 18:03:52 +0100 Subject: [PATCH 19/45] adding debug script for container --- .../600.workflows/6300.sda-workflow/dev/debug.sh | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100755 benchmarks/600.workflows/6300.sda-workflow/dev/debug.sh diff --git a/benchmarks/600.workflows/6300.sda-workflow/dev/debug.sh b/benchmarks/600.workflows/6300.sda-workflow/dev/debug.sh new file mode 100755 index 000000000..2402b1a16 --- /dev/null +++ b/benchmarks/600.workflows/6300.sda-workflow/dev/debug.sh @@ -0,0 +1,10 @@ +#/bin/bash +if [ -n "$(docker ps -aq -f name=^sda-aws-container$)" ]; then + echo "Removing existing container sda-aws-container..." + docker rm -f sda-aws-container >/dev/null 2>&1 || true +fi +WORKFLOW_NAME="SDA" +FUNCTION="filter" +docker run --platform linux/amd64 -d -v ~/.aws-lambda-rie:/aws-lambda -p 9000:8080 -e AWS_LAMBDA_FUNCTION_NAME=${WORKFLOW_NAME}___${FUNCTION} --entrypoint /aws-lambda/aws-lambda-rie --name sda-aws-container logru/sda-sebs-aws:latest /usr/bin/python -m awslambdaric function/handler.handler +docker exec -it sda-aws-container /bin/bash +docker rm -f sda-aws-container >/dev/null 2>&1 || true \ No newline at end of file From 96605d2b2f747199378737a1ce90c20499d00db5 Mon Sep 17 00:00:00 2001 From: Lorenz Gruber Date: Mon, 24 Nov 2025 18:04:04 +0100 Subject: [PATCH 20/45] testing native binaries inside aws container --- benchmarks/600.workflows/6300.sda-workflow/python/filter.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/benchmarks/600.workflows/6300.sda-workflow/python/filter.py b/benchmarks/600.workflows/6300.sda-workflow/python/filter.py index 4d737b3c9..9b27fd1f7 100644 --- a/benchmarks/600.workflows/6300.sda-workflow/python/filter.py +++ b/benchmarks/600.workflows/6300.sda-workflow/python/filter.py @@ -1,2 +1,4 @@ +import subprocess def handler(event): + subprocess.run(["SettlementDelineationFilter"]) return {} \ No newline at end of file From 00ca219777ca552cf38f0e76669d06eeab654d87 Mon Sep 17 00:00:00 2001 From: Lorenz Gruber Date: Mon, 24 Nov 2025 18:23:44 +0100 Subject: [PATCH 21/45] rearrange order for better caching --- benchmarks/600.workflows/6300.sda-workflow/dev/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/benchmarks/600.workflows/6300.sda-workflow/dev/Dockerfile b/benchmarks/600.workflows/6300.sda-workflow/dev/Dockerfile index ab04c6f80..5f6289594 100644 --- a/benchmarks/600.workflows/6300.sda-workflow/dev/Dockerfile +++ b/benchmarks/600.workflows/6300.sda-workflow/dev/Dockerfile @@ -10,12 +10,12 @@ ARG LAMBDA_DIR="/lambda" # Copy function code RUN mkdir -p ${LAMBDA_DIR} RUN mkdir -p ${LAMBDA_DIR}/function -COPY . ${LAMBDA_DIR}/function - # Install the function's dependencies RUN pip install \ --target ${LAMBDA_DIR} \ awslambdaric + +COPY . ${LAMBDA_DIR}/function WORKDIR ${LAMBDA_DIR} # Set runtime interface client as default command for the container runtime ENTRYPOINT [ "/usr/bin/python", "-m", "awslambdaric" ] From e0b24f7b67f30ebbd917867e5e0cc8d51e5da915 Mon Sep 17 00:00:00 2001 From: Lorenz Gruber Date: Mon, 24 Nov 2025 18:23:51 +0100 Subject: [PATCH 22/45] testing with actual data --- .../dev/data/Corvara_IT.tiff | Bin 0 -> 1805008 bytes .../dev/data/sda-workflow-local.json | 41 ++++++++++++++++++ .../6300.sda-workflow/dev/debug.sh | 2 +- .../6300.sda-workflow/python/filter.py | 6 ++- 4 files changed, 46 insertions(+), 3 deletions(-) create mode 100644 benchmarks/600.workflows/6300.sda-workflow/dev/data/Corvara_IT.tiff create mode 100644 benchmarks/600.workflows/6300.sda-workflow/dev/data/sda-workflow-local.json diff --git a/benchmarks/600.workflows/6300.sda-workflow/dev/data/Corvara_IT.tiff b/benchmarks/600.workflows/6300.sda-workflow/dev/data/Corvara_IT.tiff new file mode 100644 index 0000000000000000000000000000000000000000..efe7364bfd0935ee1842bae62d2b823dbb996c94 GIT binary patch literal 1805008 zcmeI*eaL;=K_B>iUy|G;H_5%p&CN~XOoNloXb~&uANeD;vCa5`PA5#2af-Bt#1g@@ z62nwz1~M~>BGL*IW-_2d5g7|3j6yBNDd->h!-6fP6_g==C^#amP+>BlAZFIfUi;;| zJ$s+C_c{CJbM&6G_u6Z%z1C-+XMdk{p6B`H+rI5DR4-Ij^>S4`^IY{z*!s(F2%Gix zZ#-93&(_~fvij54SJnUUBQI9rrFtcd=l}e%=c@OG?GK0TU;MEbt8kj?N5b|m{n)c< z^q&jcpZ&RKtG^q{e=;@-k3ani|F?aCpAJ|4^Y5>!&wW!>{nXzKSO4!<)sKHZygPn5 zyd(ZiRek0^RMii^_)PWbuYRWb!Eb)1`hkxKp##+3M?FeXjbN zuYa!ks*gNZz5kQp9r+W_RbTP*&sDE|;koLS-+iun>Cc|4UVQKK)e9eZzIy(i0hQy6SiSo7Yue{BK`Z{r3O# zy6QK7@paX|{+F+-e&gT0uKM*iyuSLv`(I!E+Bdzv`saW1_0>Q7`>(J5$>(2R{o*gb zzWRlK^ZM!^|A*ICKlkDrs(WCyo8M4<{$p>bKKI!-R6q4IZ>WCqSKd(l_;0IZ-Hh3W@>_J!&bzxqP;@h`qmeea*VP<_v< zZ>+xi>)%*?=SSXHedLpGtiIzX-dKI>&%d$y@E6`#edu@JSbgx%-dKI$y>F_%;RA1~ zzV5r;RDI2--c)_nKYUa5{$F@g^}c`crs^wx|4r3v&%L>N<$Z6iUi!v2S1a!ntTlJYwzODM7RdF^@Cq{TlE9K`?l(}|Nh<8C;seh z)yLoaQuVzbc&YlH?|P~F?oYi`edj-Xsrtw-yi|S1zj&$o*57}r`tWlvS08%c%hdE-IH{?*IX`~UsR)%#v}d-WB6{_WLkf937fD}U?l z)k~jwd-dY~`u6IDfBN?7`G5QNs`{h1SAY8AJE}ka>UUIM`sR03fAq0;RDbx{cT|7y zGw-N=?^oVY{myT_qx#~P-ckMb%db?w`L(Z9|N7fssea=VuT;POqpwt7_}N#gU;EWp zs(=2)SE_&ZC$Cihx-s~3O!-PPxR_np-X|KGc-=imOG>T`eg&Z_#0@2URu+uu|D)O%m8e)0paRzLn- zuU3EjiT6}L@~Ky=&;G+#tIzzxtJM$xi&v{p|Ng7h4?g#<>IdHUuIdxt_^#^Xf8$-% z_x`o{^)1lQ+?;ZdRO(4fB&xPJ6?FL`omv+PxY;T{Vuzot@?v6zNh-VKY35}f&cZj>UUm!Z}r8me{b~-|Ma!$>;CO))z|#d zYt?UmhH#lkAL3}ee6@;|AU|Y{=fTozVa`9><2#jLqGKWfBXAC_R+Zi!M_-L`Acv5 z;9u-_UseC3u7ks&glseb>ZXI}WF9}7pT*Qfn|@SnW#gK-jHQcX0 z`RV`pKl-^(#r{6@zkla+|Q`S0vFQd*{}v3985v`gt%caxeAkIx*qscq@y z>;PsskWW8D$GRjyfB=Dw3&bU4N|BO`XXYDgwN*;irM79Obd#F570>imieu9{h*K|V zw;UYsI<|hl%fZ9!H?v28009EK7wF{A<)v7sVnmC(`MWzQ=C0t5&Um`vd5byzp)!fE4fc!Z4-pO?mRru*DO=qvP+x=T-&I_e#Z?auVv ztyiC=TV$I60RjXFOd;@e3e_RhR!mWsb}wdFu6AmS=}0&b`l$OVFQ;msE(_hQ(ymUt{E2M=$w$gw_x1dd{E2Tk9s@b`8n^5J@=Ld) zGuyGrnL{IspPc^4{!}MGfB=D=2*k{~^a#iEBUcLBDfUvMbl^neZiB;I0RjXFEJh#)TPNhgW*A&7J|1JPd{CYWOPlbT_jCKE_+-n2 zj^b&Li&wAb19r5W5%xy!xe7N2XFR^L+63e@t*(N}M009Ck z7s$~TpTncc(zMZ=SvuA$2|IPGxY6vDpv=eW=fj5f2@oJafWWE+dZAeYF^6GyUL_2> zWu~*1PBmVy+8`SGK*#>W?%d0Za1Y(m8Ma1C~f59XAbtO+9N=K z009DP6NmxVjJa&*ABC3Ul!4_?DGaZ9otlnzO6$!N^^YqlAwYlt0Rn3h$l;cfFQ;c6 zq4_7EVJ7`d4STWfsTp?q=pF7SaPQEios;Ld)W9(U1PBlya9@Fxh&ldZI+l~Cjb^t# ze)exm^6^gd)J@!PIxHP~B|v}x0RkHkh)c>ajTCh;1@n~RYki~?TnxhFgRw91xq&x` zDG(q)fWVPJ&bOR>Iqy>Dr8G_ZxkbBq9Fs?V)&;q{b4QgEAV7csfz=As5f@&<6q@I< z7muc?<(Ie^ZHfj=c~s>A{q?HUBkPsfAwYlt0Rndt2pJd7l#(uQ=JSTaTo!lZPTcPl z1PBn=p+HQ|_!G|fTt8}sZaT%>Y>syi)OEcOAV7csfgK5?z>K*V z=A5=W(HJ*#Spq3LsqT)5OqBou0t5&g3UuNz=VyGbD}~p5I2YtFP0Q57UhESfK!5;& zT?zE!GbU#$NCai46Q}Xuu6RtA009C72%Hp%AsG%dso0IrwB1H#uinYd6cQjnfB=Dg z2(;;#KDKSsvwUzHN@|H_V;?eRNq_(W0tC(rH1U{|Gd`EfsieDG4tOO%fB*pkI~U0D zSSMxNY0|S^fA)OZgT3PW?X+orzjHbh0RjXF5agQxYIRfWQ(3x{(<2@q9+6 z{Cs?ZcE`I!Ph%oLfB=CF3mk+{O2s-KFD2ypaJ`#V`FQzmCv+kL1PBlyPy|X2rpU~{ zfEzdC;`35`URa!#p6~E_ko|W!qf-zdK!5;&Akd~{ipvz4@hL{<#rUL0fB*pk1okY@ zgk(z69Ge3_7foTiXFGEzK!5;&y$W<9Fy&|dJC*uV(ayn>K2IG#*+?M)0t5&U*pfh4 zd8Pzy)}tXx)Bl~HWT)!+N{$d9K!5;&4GZMeXIfEOR&Mw@n;HQE1PDwk(EiA?tU|@; zw8DBLK!5;&?FrPG8Gq+CeTtiAc>d#Ae>Jkbw}2TDAV7e?l>%kmS^wEez5HyVvfZ#S zy;4{+0t5&U*q}g6#mS{)S009C!5;$Crrm6RHc1IefN`L?X0tChibk?6^o2pNM009C72oNAZfB*pk1PBly zK!5-N0uL4#@q^LM4^|&MXs0JYfB*pkGYRBejDO5>=@EWHdg*ZQSo~?~i0Zwzb_ft4 zK!Cs+1nQuBI`M|bXPCD5T*I|DMgjx~5V&3-uM_3n^@3UwAV7e?ZUvrReeRaV#0d}} zK!Ctm0Y9ufOXC0m0t5&UI4uyuQ+}SNQA&US0RjZh3EUo^F@GiSIbY=j2oNB!NrCjE z&s)>)=AWP5I)F`H8j~VGfB=Ez3dDuu&EfU<4wegSWCREhAh0EYl6W~j%ic9J41X3H zJGu68eamals2u?U1PBngR-go4`rI`=UEkA9KmAPcc}xCYJ8ta=5FkK+!1V$#xng`? z)kADmI=HTfeA=rAaMh?aB0zuu0Rp!Ql=Qo*pS*MoP37NJei{)VK!Ctn1?u>`Vfd+s z8`^71fB*pk1Xd)FKd?>f#`tvghp#s@O6)Ze6)>inFv z&&fOM2u|L&4haw-KwzN)?ajZl6_b(Z;#(=D!tR+>!f4PlcmPCQl9_;0tD78 zkR$SV{4ustSeDo0D)%T!!?=HFkDOXfHL{iJ1PBlyuu_4LOF1Kt$DeX596mmB-;#P9 z_dThz5gQEi)&T{2oNA}gTR0var*f52477H5SUNE&1pVcJzpho z^p?I<{2oNAZV7&ruP?q=%bEy}i zx6CT`TX!QwZ-La7009C7mMDD@qp009C7#t6jEZpYMDmjD3*1PBm_ z|5`)(4+Q}N1PBlyFjzo*4yIC(009C72y_J=@&_~B0oW!$fB*pkD--bH?aD5g;SwM~ zfWQ(2a(w#rp(R`kV<13)0D(0L`0#d3SIbxl5FkKc5dwbmb`h7sAP5j3K;V=>_>6S1 zk0p<7v1kTHfB*pk_Yhd_2c%(SONsWy4CEf~2Q_kWtZ4+2@oJafB*pk1PBlyuu1`|&sAPFgC;gB2oNAZ z;8AV7e?wguL{e%|)gF*^bT2oPAN zfYs+RuZvL;AV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&wP9XfHir-aT z+*L6k0t5&UAV7cs0RjXF5FkK+009C72;57+N5uD%b{+x*2oTtgfS<5#=USQx0RjXz zB@llJXsJuQp#xj$_&4=NF&P2`2oUHBr1;!G{tS)U?%Zt?AV7csf$;)y*_qRG_3;^} zF&;n*0t5&UATVAaN6sdKXA_ece+9K5K!5;&bqXwF&1d1c&UcSd6Cglf-2ykHPiope z-PNHTcDr@g0XP`}0t5&UxKZH3r>7SV-Eh)ntb1>mfTjcp5FoI7fo?(`F9o|tC!-3T z9-nEl-gQKP009C7<`xJ!)iqD8**}3(%hWQa7K!Cs#0#`<1^9k+~-8)Qx z009C7HYSi(pI0tH9}u5cj&x%e&2$J5AVA>J0uRa~KQevvg?D}e1PBngL132jd@>uu zSs$=+y@5$n0t5&UATYatY@D6g2?!7%Kwt*~@^l9m*%S#7AV6Sh0iV@QEvi2P1PBly zK!5-N0t5&UI4ux=SIF-RowiX*fB*pkdlxv9km+}AeTKXDcZc~CAV7e?8G-Zhd8UD4 z0t5&UAh2tJ@cXrKk$E;ica3B61PBlyK;Wc649=XMrvmhBte)&kAprse2oTtdKog&P znXq{hAV7csf!hW0$F`@Ip11dXV-M#(QNFQT%?S`7K!Cu_0x>w3`5EmpM|Cr}4hRq+ zKwuLB7<=*xI@M> z-P!a+fB*pk1jY%BiO+F8t4)9a0RjXzD^SPhxcQ~E;fz0(%xh08&~aBqZ2|-c5V)g2 z{o(C^teP<|?>JwlB0zuu0Rj^U435vM)~OQ=P!|LU5FkKcnF5k=nb*ar2oNAZVA%pb zmtFStF**VS2oPALK>l5zMGnoN2oNAZfWXoPI=?{F*_>h2mb){c>XHBf0t6l>aHn9q z(`g>(=*$EN5FoHnfos->Qfr|X)368-AV7e?5P|#$Gec^sLx2DQ0t5&&0%i4FUYi3p z2oNAZfB=Cr0%iG`K2P?c@);$?1PBlyK%g(s{l^+gmiBAdBS3%v0Rr<0T)T98_@|~q zbG|o)o(T{jK;W#vm#e`6)TtOY^WL?wdv-iy59DEtZKXZ|0t5&Um_%Sm4mMF$XXhj* z)(HUu1PBmVmq3opCKQJczSrd8pWhB2x0(b95FkKc5`mDQy(}EqXhvQK~j0RlS~=tg8;g8%^n z1PJU;;9yaif;0Y$x$#jnVt97b!uP~-VL2oPAdfUnbKUmv4;fIvUHQdp)@ zbjnj|b)=b3_Jp2tFq4#>aVu}+*R->ZX_^HA0t5&Um{g#bpCvxq5S{c~rL(ZGEt_5m z5FkK+z#0W|e1_)`onibrIP2&6=98AZ^L4Lh0t5&UATW@rHF=1DejY&rY2oNAZ;3Qv)T-^i|JX%X7if!H9jwE`pX;9vV4q&009C7W)!HyF`uE1$W#{N zGyZ>g)OD5fj=A~Hy%k~%DVFm2T)YkSAzy2hDESxZLGkj>b{c16jgw?lNa5 zAwYltfu#uK6iU$;J_^n2&M>x=O{uiWrcRx+gTH&QHlsT`BnJo(AV7e?^a42|W9)^k zoSu2R9Owm9U1$1XjP5ZGFQxqGmKqQsK!5;&n*~xzhF;QZehQ1w?rzG@c%)6(o6mY% z2W2d2WVdytH30$y2oShNASY)1+{9)tK4VJ7m1!)@HLf|)WNqSD(}<+!WTrYIK!5;& zu>uFlnS-;*(R4KJr2}DWZ2!~O&qK%|I(_eYBtU=w0Rm45#0YBQvuwqJ7?`nbeXzXb zf+sp1dpMnLY@=C5GJX(R5FkK+z%B)v_>8kJA(`_mK6k=39&D;iYL!nny30JEI_5{W z)_?#30t5(LFOU{009EK5Xj*erkX=D=TM!EIU&n|^xA}K z+?W*TrrxF=%%n5Z836(W2y9Ov$7jf{oRYl>=1|H9>gQA~Rmul@^>5iJ-CY;Tadh$Q zH)ZfG!`GGo0RjZ}B~Zs_`N%frPt2mSUv^VjDUU*FJ8n$%m?HV&xIG9EAV7e?`~pp0 zHUXJ8I;+vTpoz>BQK>KuFKo>}OlKfKfB*pkixJ4dn3kF`9P7x8rFB-eNg21&f!rqV zrHM$-#avGVAwYltfjbH``InQie(nsQ1Zzy#9HO}-R|)k)VW+xH0t5&UAg}|06pC@4 zhw)hsl=2*(omFU!(+$%dc$p#r0t5&U=n9mW%&#dX<5M?Q>x`{mdimPz$u>ZH1hy@3!$Po=h%qFa_zYm7 z*Z6<=kwP?A-nOOL5g+&itEo}}Iha5)4vd!_m{r_zQ10RjXFEJdJR7T!3Vn3{EP z#-NP7)q$Bi&O0}rL~{ZJ2oNB!6oH&|VKhztT{p7aJOpP-$|gPAB)zU%tq2exK!Cua z1#-|`J;XXSWBZiI_2RRifoNV`;YqWtW{YjDt>P_48GeD4mXK+{S0#I?VJ@ovBQK z009C7<`f7?m;UUej?eU`o3SiBQV3pg?$qvz@GF1+l7^B7b;ZfgH)-Sfe4QJP)Mz*K zpeX?Y1PH80Al~(J-Ze3r_RIf>%E+07)GWP?YnfW5#wo$;t#N%^u6>vIocSb|XXq#a z0t5)$Paxj#bK=GSk+F39kGmy4N9RHAYh<&!QJ8feK-w6xfE{z5A^qLzK4v_(o=bZI z1PBm#oIpvt_OP6cv0t5&wS|F!g%9av*^=rK*eJZ6=D0Px4rC(?NaQ5d8rL)G$Po1#oa7<1O ziqANNruT_9Chu^fap{Ht0RjYW6DYBjvLo%}r<7xP1$yxWB_G;Wm#d#ksMP*M)8qQ( zj3+8zDmw1ysCF2`QJDe)1PBlya6%v?MhuOdesLq#X-djkFb2zoXsKK0eomaEDH4*I^2kJdg?Kx&QpCjPJjRb0y_|>SUy9EFm3k;Kd=4)4bMD~j z@_Q!QQ*%5$LZt)<5FoHafw(41;h3T=?PZ0Ohle?O1}L|^T)94ariVO=0Sm;uG(O+U zkRx%v2agj)2Q@D5CV^b#sPD@y9VI}3zzzhCZ@X)sbc&7ml<)s3KI0-R?uJKMP99GD zARYczJV7-EL3q zFuSF)&AXZj&PfB=ED2*lN5{x3yX9i(ye z7bC50Qa7txguQfV*z}K5EOj=l?bWr?2(NB^)NtbNj`~+XfB*pk%M(a(6`x`V)M0k` zdS=wAMUKk0Md>AU)%16!eJC!q+Zv^(wq%B_&PnpbXE=v02@oJaV7me-0dt6zBrBm; zUQbWH3EG@Q>A8;J)MsklNvYGloXXF0HRmbMgPo^uy%HcmfWSrtVtC|bXT1o^X_8lt zPlR8rl;baUk{@;Rrk6Fa#lb#3&LMT4?nMm(1PBn=gg{D;I_%g8hN)g4T;TivJ)m;eC+1ZENFM_yiFg|b*2|GWH1)zS*Ij?GX#mX*il*6TYO z=C!^-8!-U_1PDwoPy!*o#?b4eUpU!QVUb(MTB&UuUfjr!xEV{f^w{2vwsdvOg#ZBp z_Y}w>7Zb1hbIj4fmxHAfzi|#*NSi3r(dca6^C~$P0RjXFEK8uyh9)Fpigx2->}X@- z)Y(RIrk+bplJ>@us84_Z0Ro#4=%-_jxtI$ROfH8_TE|Y%+nu@?H1SE9?ldu{AwYlt zfhh(0aWP43*2AmQbILBKocQU!hJNEPwBG5nC?!CE0D-j&9H!fJi^WyO=itlP|F&ar zGZhICAV6T70x3=>$cEfcjL=Q|&d@e5wpkG%K!Cuc0*6b|lby$4JaM3ITOTS*&=VCE z5g9JLNnDbo!3jxvb(0&Frl=HPETtZxUmu-K!Cs^1fELI zdXObXyK$5Ar(3d!84Vdk{4O92JU~x9VBL!ObK3>YV%$)=QyJG%Z2|-c5Fii*a*~ASCpcogwo=*ZmOasd z!vqKrAV6R{0yzjf$#^ieIw0dqdgMP=*-U(fISu>UodXuT1PBlyuxWuoi^?EyzFtbo zIy+<7v}qFy*N(UT8La&Z7(W351Xe0=Dg#fIrH^@gf9107_*{MRuk`(9*aQd=*n&V< zd>uz+ipAsNl%HFebe_YB32tX{WsUv34~U3!371`tk{9d7XJkcj~^L z_npq>Gz16`Ag~>Q6q50&_p{HKlBuwsXguygfB*pk1PEk-n2~i<9tLD*UwC@I5FkK+ z0D%(%N6X8Ut0h6j=Ls@J1PBlyuwQ|5!I;yt#ODrt&OuBu^&K%z!KDuZ1PCl&;AqiV z4>^X(u5S7f;vB~EFQX9>AV7csfrkj>=$vy2`jBZkDFFfm2uv%`4GOESWE&W|D${!F zjQ{}x1XeH5d`>AaI@c0L2g4=-0t5&g3&cfM`oU&SjQD(9X@?~kc6iZEnE(L-BLsSB znKN`mYgLB`^o)lzP=^2k0t8kfkXN17o^xC-s&lC7lK=q%3lWG*Bp+FhzZe6|22>e; zUM&a^AV6T50v1gJu1i3AIvfX78E`692oNAZU=0H9`9rSKkn|kVk2(Yh5FkKcPXbS` zJ@=$xt^^1WAV6S{fJOQsCY1;fAV7dXSHP#YT@Bj=2oNAZV32_L9K@s&0RjXF5a#!Ky%z`&AV7e?V1X2!srI-}dk3FDMFIo}5Lk(T ze}8W9g$m)BAA?({NPqwV0^1Nc49gG)+nCrrX7Nmk=j9^}qJfB=C-2*lg}A_lV7 zLA3d~toU?Q*ZPh!ZUO{WBH%WA%Z19}8G7(d$SsW85+FceLjp}QY-moCPbIJ2A5G<3 zpvf<+4haw-aIL_jBjMU{-?v?k&-hm>_0}Cv-!6FT)3*h89EDR6AV7dXM2Q{9o zQhol(`qL2s0t5(LBM`&D$AZJ;Vshn2Ua*f`29B(LX-uxzB^}2I5FjwCK)#!YIGFV; zo9jH~XP8x;qvPfWdz_cr1PBmVqJZ4I@~Wh7IdXiCGfWMyJgH^`2oNAZVC@276(96V9dVXXl}GfeDggon2oP9{K*++etIxGe%{U1VAh2J7cxO*PpxZCO z%rnox7L+qH>z)7s0{0b23nTaTIniT&-eCdiBF=dc^-X{PfkT18YtJ!1ygNL}uJ_}2 zC@DT?R*;`F1M8ju0RlaNv^eUO*qK(~UimrgsP#sG009CQ1=2^07i&0pqd=3NR`54U zYEFOvfjbFUTHXoz@@bmU+CP09|0|opnbWZ%{1M)df zqHh8O2oSihK#b1xD=7Cp?XJ$2iFY++lO;faz*Yt76=(C}1IAgev?b`QygDa9fWRaI zL+;g`I%)Cg1KdgIbwYpu0Rrn4=tSe$Np$OUx6d|kU>X6~(>Usd009C7_9GBKBA;!Y zKU>H9xjtq}fWWQ=hTTQejeOTmkD2_8KVu$t;U6gMPced5L%K0_>} zN7a`-$Muh{z^{KMW+k9vVV>Si?1PBlyurPs? zo=0QKnQC!#1fzfe0RjXF)B|fRq4Ohfrs00WQSe8J4iIZ2K%?G)0cUi;T(`bh3hilZmO!*qRZhLyG zm@5GS_ZR5pV{_w88(}c<_5NdW##&&ge(y5Ho$!E2fB*pka|-nCv*pIy-hF$A=NyN= zPYJXa;?&W$EMB;>0*%`)TWe#B009EC3d9?3zVYUbbThVunbrJa=P3h+=qqI)NOB>9Rc0L9CPhtLw#<;qgQ(hY1iMKwx44 zci)LYuj?)c;IJ5spBR)6Z^PN7=XK;-5ge8%&J%QpS#^+92oNAJ zx4`8a_S|RJe^=mgEVfI}%k{cF+9p7N009Eq7wG0<*zQj_?8cH*=kQp+rab}#2oNB! zCxIM|X~uE$VCrG{8A|H->>us#9kjDcfB*pk1hy?uXJeC*z1cPynQEm&Ibn0FUR^r` z2oNAZU`GNaKSO}#4DHOl#AnFNb~Ej?dD^LNlK=q%1PJU#pjk=Q3(GR`Ql5&#PA>2N zI#H?Xl>h+(1PJU-pj}F~TL~fVhK>3qFF{k4=4p3UCQN_;0RjXXfgFcnx5>r05%)q- zTO8|mYQ$X+0t5&UAh2J75{glvY{&S_`%Pi@U_9hOfB*pk1a>XZW@i4Mm+S6_?K)H2 zXidS|iXH5>2W$}_K!5;&jSIxcEIHX(XEsS%e~jD7(7ZfNbvJHfdISg%AVA=ZKqo%a zW?YBHfUNT~Y&6?Vq}E-9{WF~^CP07y0RnpvXp*r>&N%C4ryNdYWhcGv0mmE(5FkK+ zz_CDDLv}K;EIi}VvMugzHhqM{&9)5!1PBlyuuFlGl_e?r2<2$|+COfO009C72<%7T zVEvfFG394>>DfkR*@{5Twk7C(-Y;fKfB*pkwLmXGb0mf%bv8B|B`f1=Ib16Bdf-Zc z009C7_9u{I@?>h(30j9{sM5x#0Nvj^#f%9MAV8oMh-=Fjlc(~te;~A}^EAJ-eb^#E zfB*pkI}?b>nc^_b`yd~CyG_l~!cRbV_Fgev0t5&U=m>Ndp0PNmW7;^1&$_f;hQ`_+ z1PBlyKwy^w#~<9rbgWaf7oc@&T7B;FePYrC2oNC96NvG7@MFz3Gh+(1PJV2pr4g#FPXIEVK?ujm)(N_AV7cs0Rkl% z>#*$n{4?6*Fb&UPtL$2Zde;yjK!5;&PAtZ-?Br+Mi;t9@^-io39@q#FAV7e?&IQ^m z%*mLZo2R_&OzY726s&TsouipP0RjXF5GcQ0TY|C4&UPas8w^p5oIxh5!Kq1PHVOF*Q3M(8k1U_q!k6T7I@@wg?a)K!89Ls6UyF zS(?(ctUA-0G#<|fU=bidfB*pkrv&QwOl!^j32rQjkMeq|l`;YZ2oNB!Z-MyHZG6-U zd8h2{o5t)35Fk)5CEOIf6;24`#7qZDmbP(vqP`*m1PJU%Ag(UTe>d`0D&C|q&s80HP$}{-O~n(_7so zyWfCIB-WqcHYZNGSwEG+Qn&1TB|v}x0RsCFIPtAi&dB=rhq6Mfbt*siVPuvB2oUH9 zoVaavj@sOd!0GovjLIO;Jy;i|6s>V`mHsEZdvP;Q0tEIc(7i40Q^2gB5Qv}MhAKHm z>(`v1B}Ze0Qu0J+4ig|iU>gGEp16(Kn8mFEF(;pR#J5b$%J4`v>n6FN6h6_1!vqKr zSg}C5)4AQP_zF%x^!#?{WoRhMrMc}yVCOU}B# zW|diQ9G|8R)8hk52oShk;MAAp?Y-)O0D)-*LKOBoY*v+_xPPRnR;TBgqfNET8;9q; zT-#9s1PBlyuxNptjrli*E(BzgvZE$*p~F%As7!zW0RjXTC6E?{DMI70nw9Bs?lp&o z3_n*+edp@2ptC!q4|NC-AV7e?QU!8wrstGyZAj)L*N^)kfZL9*?@~(w1PBlyun>Wg zocXo;hVwL|(F*|r1PBngQlOKc>9gB9;3iltmZl@fCb zOMMU^K!Cu)1j4E?tTkiEmH)y3<5IEYUS36Bu^!DGlx{9hJD=oo9Y+ZeAV7e?as`?s zJ4no?e1@SPoN9(M=#l^d0t5)$N8ljShAktHhIY`)(eYexW#?ylQ%ryWfiVK%4m9jm zHKtQ_S1WMq+u`Wlbi=#rsiwD%b+s4CzzGmofIz$h&H4G-0;XsP;{>LCKg{~(^4=YH zY1Jk`U`m1TyB||_r?1Tjq;IT-yg4qtg{IBo&Dfd<0Rr0)i1*1Mx5;hH)+~kzq)>eN zjgj6wQ@?$@A%|TfwFnS+q(FW96Q7T~2x~s~(+N9zb~AK2dJGy6AVAc?P@a&Y+++t=oK_90t5&Uc!WS3ShJjc+zPS{%sM+~9^EWc z)+qr31PBngO}8G2oShY;9#Md67`_$#xrP6 zfB*pkhXSWR`a7&;|M3Dt6VcN1@z=@-2oTu3fR)O|F0l)`u?uQC1PCltAl+>jIyS?) zLLhvYFm#oAMemvrAV7dXEl~2)J$DOliEuTyFb#7cK!CvV1Rh_ z;R6{{cX+RA5+Fc;0D-j$jLFY2b=T%#yaWgkm|tMTa%KL(IKyoMF%kz}(4qBi8<5rn z2oTt+z`&dBR;RzUc?rd>T~YHPuuK8bvCQjYR0IeRAV7csfrkqW{z&)XBQ*d51PBly zup9v&v@GXZ7zqIa1PBlyKwwP*emUU!tL1y}>+Q58K!Cv71?1=2ub}Z0AV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FoG;0l&Vzk_%;+1PBly zupj~dh}(iL#xjQD8>(eo2csacHi2;O*wsI&;9j}5cj#o}Z7%)68@?%@>`Xc$Kwt*~ zdrQw9TvAg!EfD_)K~r!Z#HG(8PJ1ccf`E7ToL4ceJrBMcpYweBUYfvezP`dl+sD$b zg7FZTL?FDyd;O%he6PgLgaQG(?>%APx>|w2;n&*=hJ5W1oAkW41MLVbTHr)jo+w)M z6bz2QiUf8PpYf{fY`QCYBO2=M0x2u|%OUZ3JF6ZD5FkKc>jE(?`*S?l?H{y9fB*pk z1PBCy!}NqgfB*pk1PBlyK!5-N0t5&UAV7cs0RmeU@QLtNudI0yAV7e?d;-#QK3hE# zSgU~FN?hxeGj0L|2oQLnK!{HX`aplDUWdT(pTn$U+(x-N0r%teT(a27w{HZ9{NK!CtP1crVC7Vo-#rnZoac6u27OWFSfx3d?oWO$tJLn#3Q1PBZg za3dYWw2n&MOuTSmYEh-0t5(LCy=7EJjDbRl-EIOMSuVS0{at4xmiE$Z}w(PfB*pk z1g;VgpI5PIM1TMR0=pMz5_I^||?3nqVU^<~y>ck&U|wEM3G1 zA2*1-&;N1q&U{v2_g|G~Wj24n9eG)oIOju5&oHj=U)&0hv=Cj^NR5WT?gb?0uHORb z*1T(h1x`K(W=hVSpYeHt<1{1!`xbEf-1+t2_w<)I`%ZX<&1MO@#Gx7!fi(-<{ynMu zT=Vre_6-Ug%fy)9ZKI5`8#b*4ur?2+QVvZ6ZK;TXS zZpU|mzVb9FK-0&zF+;EHL^A>e?j_*s^j^}=L*OZaoS7*`#phF4juRLuF#Q+R^gWK8 zlIjE+0gJBqnXT_>n>U0F0t5&U7$6`$V|4nCcEDRj6#@hZ5NHJ?NBY&F7Q_|-0t5&U zcv8T!>q#ON2oNAZfB*pk1PEL&aKks;eoAsZyOsn95FoHWf&3Gq@Eij)gA7qp zXOjZn{F}V2CPmoZQI!Az0t5&=DUbu{$!g1}kV3wFx;&^h zYg3G-W$Wb@juIe1U}k}I1Dp9Ay1$n|9WM7e-6ZFU(Kz6drq(_UXrT%L0tDt27dBZLkz@UnjM{4>Ff}_fz_VQ*G(TIoQ6bzz|I6p zvUI0?E}il(zIJPd?aA}=y?9XT-wd;PIi2$nPX20C)5EE1Wl7l+uck9-ZC;j;|?&L_g z9QU5-{m*#s89E;U0_zYck<`B{)wvj^(78Rv!dQ|Xd3IsHE}d+)8b5n~*vcj_F`K&A8j{O|n&=zV0ja zAD`;t_|n~uyWvs4oE$)GGW8%&o=7190t5)`MBsF)q`YaLdKULiqI|wM&SpTJ&0N+$ z*c)!#NnOO6M@JM8AV7csfhh%2EH#1E?fIc8l!kS*`m9UK!R~0wc0N}2yb>TlfB=Cx z1P&8wj&rnqHfL*#+O6)Av8*5ad3m%~H!ij7#cJE7Edm4x5Fjv*K!~K?N2K$dCG{H9 zuU_Y#I7=ExDNj|)nm89WZR4@}QI5398xE8{yb>TlfB=E31)j>2csz$u{iDOH&orcA zKGRcU3OzNWDlJm)Wv2`|N2S0VLaz>iRSWnsU-ji%>ENGOj`ebC_9@o+H{d+si=_$6 zW+SvKJv3!;V?ckZ5FoH-0bk{7zIy8(dk&e?NfVEiN1ElxH|mJ^3a2@AJSxaTC<96j z+MQTC?v)s=`->$W1PBlyKwxTt6hNmtO^2GN({0A=2*fwWPNr&_1PIJ0kTd3F|Htuj=Dt%Z_70cAlU-k5n7b{X)rNk` zdD3gzJt(~XTv~2ZpuC5+IXkl=K;TXSC4K7Cb-yb=I8-OqiE)*R@iiV!FKH(}b-O&# zctla#W$O3TQ+@h!Xhi3#?nWTJJ9abMO-=Nso5_T?qD`Ti%s_$uyDz<+>ZC~>9Nt&* zvD|lEFT5O{t3A>=mg}WMhxKpS4`;}y%NgBUe5QWm5b6Qlavp7$FVK0njT`Lb@5*tf zSNmFlaN(|Pdj6aL>wDC4KLRCJIy0_gCcnh@S92`>)G`*Qg3_|7-c-vQopyD>L4|2{ z)9LGR#i7Ohl9XkjsVtoCXh_oxK`+}7i0>Sa^$FZ^+2XCiRbSsVU&M&hJLl*flnUyn z&M-seSXe?Rz3EO@J@JN2Ejyh}v)Pd;|T`+FBS@&4I6$r|Q=+ZXq3 zt=HgVj57&@x6Z*zEuAP9$4A&{HsW?Xc$`qN^tkw_UhXedp6V$NEa;pc%cgj4 zl60an_IA`IGsEF}W^)fj{{#rEQXt+#RylBkeuO~tW@*CZa1>4GG*#;whi|j^&dYVm zkz7(gANE`J54$bp<6m<+#{4Yb0F+aw*U(|!@AOT?Ydgq+dNxo7BLIje6n?{mQpE4Qf`&D+ky~R zgN86sr8J~`_D*`nG1Q}*=sdb1K!5;&4GZMdcxr+vSV|Da%DLb|;M9qjvZ_3V>YcW+ zm)KGMFCMthqn=AXalI4IoHp+>1ZN>|t3Y_W-r9xs1PI(JFyQMjM@#?f^@*?FZW_g^ zX+J-u?WQ2_-8zX;?c*>~UdD6R@tw8?_R@zK*hsYp3vBDE4fv{ku(#6_*onZY@aTMj z)*rx*!3aI(N2gIKimy$4rkzfudu+x#X&Fp8-}K4H z&)&`&scV)ToZ-2ALlDc;?!zYa*t0)u+D=S>0D)NquDPf7*HEE{{J462rGxO=giS2F zy2E=n?2WWDn>sR6Nf^^Zr<8|vZ<+g^@1el=ovO1DxL#mI_txttxzv{ZyHvg}$LDLu zd)Pz0e{Fa7Z`Yl9JDDib?!{4EPc*@t;!q~&X31S_e}+reH@5W%Z2eBK=qu61PZ4q3 zXq${FKS}NOe_Pl0Z(UC&oH%YZ`B^Wk??0aQjGIr-cB-}r5FqeqftUd0(QLOxIU-7h zD_>*Ggh$)D-szOI$Ckg9CKGbAjnDjDL0!K52^b*(0;?1le#;zQbCnQ_7q6hLR(uj+XqAieldd*b=lh))A@bsCX41&^<z-B3 zodAKM0?l1MroqrgqwA%piQVUo_Ed3ybZ=uDoI6MF&)?P$epP&vbc`Vw&OO7NA{#T0RjXFtXd$YXZ&b& ziL*#CHS54a9**k&C_k+}NBOBtfB*pkcMzC3KI1nDsmu73hM)GvOr+C6?n!)(;Zv6Y z0RjZpC6EF$hh^HVN1sc^P9?Q!x+%-oTspQd^$8Fla8e-LQcjjDBrub}sdrpnGoJeG z-zEfyk0!sTQ&{FkX|p^H??+7n1PELZICU?(P~9N{HwwfUJL;nMk?XyOj%T9OkYwq7S=f_>9J1&?q#+iYou6iH9o~d|Td^#QL`1D3U22(7_I|97M@3wR;{QtqZK+D{||XU={OgZ#gH2`^4dsZH5=l+xzfw+`ndhYfm@u<=w?EjVN8S z`(L>8XKV+1P$wLSkN(B&6dfPzSG7lg0D&lQa2LraoqNeirK6kA$yy3W3B)Ur;1_-S zKKXu$^E}L>QFFh#@}X5{cxMMxBlpkQkw5ISU(p@`0t7AyJaq#J$1k*Y=qiCkx_H^e zUHB^o6yM(A(Ys2yD3=}{>s7QvfB=D$0tfe$2dybi4ox9}X$0!1tUJp&TH-Sn#_eg& zq!$7N_9u{TAL*8|zqy<74g@*@Sx03Xn(fvOAWV?}foTQO&Ba}0T2#F~P~gesWO%nX z!|QA|Wi|*9ATV5@y`Kzkqb7j`2poOu*G&N`L?X0tB`#a3S>SOpKq>9*1L! z$~rw`h2t9Ab~8HyyAil>lh}=ci4qtokWw+UIf~Et^;FWe&d?I2BTu0^0RjXF5ZH}C z`F1pAW{%Rd{EW-cQ;##nr>ZI=K!Cu81>6lcd~pX%Ev-g#y*fb4QuIvYbFq4+x?%zZ z2y9fKEIKwiJ=1zZAjW6C+U&j+;`Zs5=gLplR!V>XfrSa2xwnlSWqrq-sJF3aSO0DT z-OxLT#!h_Z-x`YVhJ!lYah?s-@Z?sWxj8=)7U_HE#wV z<1j~J=gm-Wc3L&{Z4e+pfB=CV2{a4PPD-Zfcb1>APKwg>v?Cl-B|v}x0Rm?Q>I6N= z&w4i|W_u)W<$sRD8uYBF0|W>VAh30ToQ`RhB{1`p>)Z@Gr97`eQ`p9X2altIgX4Ay z5FkK+z;*<3Hir3>a4f-?6S9fUHZbd=`X{5cMEOB!AExxxFSSR2009C7)-Mp_Fl1qz zNKVFjO1U@}wK-V!>({1kKTX@>ddmv|0t5&U*qA`h$uOz&Z~9UO(39nUB}cp{y)SJDYXa@lZZo_s~6M)2>@+n*ad<1PH84ph>tk z*Xs03`4~#$Ypj)vQ$cb6tcCIJEj9wyM-0UtI3C*FWS%)~T} zIwyOd($;~QUz+E7)~6D&?abTr);CA=zO|1F?dR@ONNHwit+;xZW|0RoQ^ zC}|mA(+F}w*ldnm7+7g^SjCO^P-Lb1PksNi(rFnsfxQd3M^E$)C_x#!E9qIvF0VdQ z^*oARlJ<5c8l`Rs5FkK+zybwgP^Pe~SD}N3cqS*G2;D(lt3-eR0RjY;A#f(_^5AlO zb^>(Jm`{HMAU+4-sYHMP0Rr0)2>BWx111wH#cF3N59(sTSvFl|2y+i$bc0@NI|K+2 zATXD}nAEGo^JJ40swa=kHS#s~du1A4*=&tM#!P?!fz=5N%)Ya?^3eXw!gO^M226ke z0RjYSfs^TYHVe!&nuW{+=L4W`O0t5&g3S7>|9G{o#9QJ6R009C7 z2oNZL6KWyqy+D8f0RjX@2;`4&#pehzRS6IvK!8B{#UVci^+q5-fB*pk!v$h^#+9el z=WsJM2@oJaV21)nzbTaRGiB%wHB6ZR0RjXF3=ufaN%1+vO&tOR2oNC96G-vt-@fgc z*daiGz=6QgCwB+O?QU8iCgssvAQzrEuxSdDBS3(_D1oDU=cv{yuUsG}Wtv3$P7t3f zzp92$fB*pk_Y>%?Lc@;L=l$%Ri2wlt1Xe82u0P`>;u>_tlQDDx1PBlya2J7tv=pCr zQFjso1PBn=mVm|Qwyvky5FkK+!2AN?V_We#KeICsAV7e?wgj4A9Xj|#cU!YG8v+Cf z5FjwAKw5oH+M~`05FkK+z~%+~-OA=Ku?Z3&K!Cu+0_k^dRK&ivR%v1U4;D$LFT!XmSJy5FkKcb^$*Got@YT z2oNAZV1ojFhiHQr)szSjAV7cs0RjXF5Fl_);6cC2W9@nFf+#0IfB*pkX9fJ|^DK=6 z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBCy2mNL+djbRq5FoH- z0e|5##ntmo$P|kDAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oRWAz#kLM46J(s1PBn=qJa3^;#D;#0t5&UAV7cs0Rr0-@GszO z@0yws0RjXF%r4-gzuAeMfB*pk1PBlyK!5-N0tD_Z;KSm(lRF^+0t5&UAV7cs0RjXF z5FkK+009C79xCwV_;~0Dsz;yyv9ot}0t5&USfjuK;&Y8`#!P?!0RjZpEMV!n=BsDy z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB=Dw3izd#jb2#OB0zuuff)tT zUxYmPFDhr8o~{WHAV7e?90EB$EkWn7)h7W01PE+c!1{B;7uVDX5FkKcK7kOR^L4Ig z0t5&UAV7cs0RjXF5FkK+009Ey1g810YMNfgja+R41PBlyK!5-N0t5&UAV7e?x&?fq zvhM3=^aKbHAn*tQf5q|$b>}2NfB=D&3y9B^UqHhrK!5-N0t5&UAV7cs0RjXF>`uV1 zQ|#`Nn=kv|Xs0RpQN2>H3XF&i)e0t5&Uc&vc*e5}8-6Cgl(5FkKc zw1D^=O{M_>0t5&Us49P-I{00HYA}_G1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1g;eb|D)sAHr0**0RjXF5FkK+009C72n-VNPazFrQi%Wo0t5(j1;l4p z!!`i|1PBn=zJMRvZvPsaAprse9xotm9)AstfB*pk1nw?yQ~Yi6-`w@l^WEQsPDp?N zft?AgHa>6qreS9n*>njIAV6Spfsmh*_j!*F*E#BY$T$Z90t5(5A+W`qS!?D`aXs}x zfB*pk1V##M@mqnB1gaAtK!Cs<1?0{hv7L$l0RjXFEJjML0**rj2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+0D*-F_#KahTnNJ; zK!5;&^$2`9Jk~SX2aJ@@1JIq4009C7RxjY&i`8F36Cgl<009C72oNAZfWYo0RjXF>|7xHZKeh2&ff#3Pk;ac0#^w5#|^GP(}Vy40t5&UAV7cs0RjXF z5FkK+009C72oNAZfWQC&zsWKHY2{U1/dev/null 2>&1 || true \ No newline at end of file diff --git a/benchmarks/600.workflows/6300.sda-workflow/python/filter.py b/benchmarks/600.workflows/6300.sda-workflow/python/filter.py index 9b27fd1f7..e87da8b11 100644 --- a/benchmarks/600.workflows/6300.sda-workflow/python/filter.py +++ b/benchmarks/600.workflows/6300.sda-workflow/python/filter.py @@ -1,4 +1,6 @@ import subprocess def handler(event): - subprocess.run(["SettlementDelineationFilter"]) - return {} \ No newline at end of file + input = "/data/Corvara_IT.tiff" + config = "/data/sda-workflow-local.json" + result = subprocess.run([f"SettlementDelineationFilter", "-i", input, "-c", config],capture_output=True,text=True) + return {"stdout": result.stdout, "stderr": result.stderr} \ No newline at end of file From 7ecc0be52b301c9a58a4446cc9a1b2a3d6270ccf Mon Sep 17 00:00:00 2001 From: Lorenz Gruber Date: Wed, 26 Nov 2025 16:57:34 +0100 Subject: [PATCH 23/45] testing container deployment of sebs vs custom --- .vscode/launch.json | 1 + 1 file changed, 1 insertion(+) diff --git a/.vscode/launch.json b/.vscode/launch.json index 8c71a624c..028f769e3 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -20,6 +20,7 @@ "config/config.json", "--deployment", "aws", + "--container-deployment", "--trigger", "library", "--repetitions", From 9a3dacbe3e104e7d28ce64104f81e0af43674471 Mon Sep 17 00:00:00 2001 From: Lorenz Gruber Date: Wed, 26 Nov 2025 16:58:23 +0100 Subject: [PATCH 24/45] adding option to specify custom container for benchmark --- .../600.workflows/6300.sda-workflow/config.json | 3 ++- sebs/benchmark.py | 15 +++++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/benchmarks/600.workflows/6300.sda-workflow/config.json b/benchmarks/600.workflows/6300.sda-workflow/config.json index 8ff6eec59..23680cd7e 100644 --- a/benchmarks/600.workflows/6300.sda-workflow/config.json +++ b/benchmarks/600.workflows/6300.sda-workflow/config.json @@ -2,5 +2,6 @@ "timeout": 120, "memory": 128, "languages": ["python"], - "modules": [] + "modules": [], + "container-image": "logru/sda:latest" } diff --git a/sebs/benchmark.py b/sebs/benchmark.py index dbcae6b43..ec5d843ca 100644 --- a/sebs/benchmark.py +++ b/sebs/benchmark.py @@ -29,11 +29,13 @@ def __init__( memory: int, languages: List["Language"], modules: List[BenchmarkModule], + container_image: Optional[str] = None ): self._timeout = timeout self._memory = memory self._languages = languages self._modules = modules + self._container_image = container_image @property def timeout(self) -> int: @@ -59,6 +61,10 @@ def languages(self) -> List["Language"]: def modules(self) -> List[BenchmarkModule]: return self._modules + @property + def container_image(self) -> Optional[str]: + return self._container_image + # FIXME: 3.7+ python with future annotations @staticmethod def deserialize(json_object: dict) -> "BenchmarkConfig": @@ -69,6 +75,7 @@ def deserialize(json_object: dict) -> "BenchmarkConfig": json_object["memory"], [Language.deserialize(x) for x in json_object["languages"]], [BenchmarkModule(x) for x in json_object["modules"]], + json_object.get("container-image",None) ) @@ -633,6 +640,13 @@ def build( ], is_workflow: bool, ) -> Tuple[bool, str, bool, str]: + # Check if custom container image is specified which would collide with container deployment option + if self.container_deployment and self.benchmark_config.container_image is not None: + raise RuntimeError( + f"Benchmark {self.benchmark} specifies custom container image " + f"'{self.benchmark_config.container_image}' which collides with " + f"container deployment option enabled in the experiment." + ) # Skip build if files are up to date and user didn't enforce rebuild if self.is_cached and self.is_cached_valid: @@ -681,6 +695,7 @@ def build( self.is_cached_valid, ) ) + self._container_deployment = self._container_uri != "" self.logging.info( ( "Created code package (source hash: {hash}), for run on {deployment}" From 0d086180eb403704e09329c2e102226f0c674712 Mon Sep 17 00:00:00 2001 From: Lorenz Gruber Date: Wed, 26 Nov 2025 16:59:05 +0100 Subject: [PATCH 25/45] aws base image to include lamdba bootstrap code [still dependent on apt to install pip] --- dockerfiles/aws/python/Dockerfile.base | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 dockerfiles/aws/python/Dockerfile.base diff --git a/dockerfiles/aws/python/Dockerfile.base b/dockerfiles/aws/python/Dockerfile.base new file mode 100644 index 000000000..5f6289594 --- /dev/null +++ b/dockerfiles/aws/python/Dockerfile.base @@ -0,0 +1,23 @@ +# Define custom base image +ARG BASE_IMAGE="python:3.12" + +FROM ${BASE_IMAGE} AS build-image +RUN apt-get update && apt-get install -y python3-pip + +# Include global arg in this stage of the build +ARG LAMBDA_DIR="/lambda" + +# Copy function code +RUN mkdir -p ${LAMBDA_DIR} +RUN mkdir -p ${LAMBDA_DIR}/function +# Install the function's dependencies +RUN pip install \ + --target ${LAMBDA_DIR} \ + awslambdaric + +COPY . ${LAMBDA_DIR}/function +WORKDIR ${LAMBDA_DIR} +# Set runtime interface client as default command for the container runtime +ENTRYPOINT [ "/usr/bin/python", "-m", "awslambdaric" ] +# Pass the name of the function handler as an argument to the runtime +CMD [ "function/handler.handler" ] \ No newline at end of file From 360f3ef6769904610808f66aa09cc580e0f6cba7 Mon Sep 17 00:00:00 2001 From: Lorenz Gruber Date: Wed, 26 Nov 2025 16:59:32 +0100 Subject: [PATCH 26/45] splitting registry info function to get information for custom images too --- sebs/aws/container.py | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/sebs/aws/container.py b/sebs/aws/container.py index 6ef8216c3..4bf8ad91d 100644 --- a/sebs/aws/container.py +++ b/sebs/aws/container.py @@ -38,17 +38,19 @@ def client(self) -> ECRClient: def registry_name( self, benchmark: str, language_name: str, language_version: str, architecture: str ) -> Tuple[str, str, str, str]: + image_tag = self.system_config.benchmark_image_tag( + self.name(), benchmark, language_name, language_version, architecture + ) + return self.registry(image_tag) + def registry( + self, image_tag: str + ) -> Tuple[str, str, str, str]: account_id = self.config.credentials.account_id region = self.config.region registry_name = f"{account_id}.dkr.ecr.{region}.amazonaws.com" - repository_name = self.config.resources.get_ecr_repository(self.client) - image_tag = self.system_config.benchmark_image_tag( - self.name(), benchmark, language_name, language_version, architecture - ) image_uri = f"{registry_name}/{repository_name}:{image_tag}" - return registry_name, repository_name, image_tag, image_uri def find_image(self, repository_name, image_tag) -> bool: @@ -80,8 +82,4 @@ def push_image(self, repository_uri, image_tag): except docker.errors.APIError as e: self.logging.error(f"Failed to push the image to registry {repository_uri}.") self.logging.error(f"Error: {str(e)}") - raise RuntimeError("Couldn't push to Docker registry") - - - # def build_custom_image(self, image_tag):str - # return "" \ No newline at end of file + raise RuntimeError("Couldn't push to Docker registry") \ No newline at end of file From 4850ef3a6394beec07d1cb35daebd3d142717a70 Mon Sep 17 00:00:00 2001 From: Lorenz Gruber Date: Wed, 26 Nov 2025 17:00:08 +0100 Subject: [PATCH 27/45] building custom container --- sebs/aws/aws.py | 12 +++++++++- sebs/faas/container.py | 54 ++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 63 insertions(+), 3 deletions(-) diff --git a/sebs/aws/aws.py b/sebs/aws/aws.py index 2fb60f7ed..aec1a7ab6 100644 --- a/sebs/aws/aws.py +++ b/sebs/aws/aws.py @@ -165,7 +165,8 @@ def package_code( "{{REDIS_PASSWORD}}", f'"{self.config.resources.redis_password}"', ) - + assert not (code_package.container_deployment and code_package.benchmark_config.container_image is not None), \ + "Custom container image specified in benchmark config collides with container deployment option." # if the containerized deployment is set to True if code_package.container_deployment: # build base image and upload to ECR @@ -177,6 +178,15 @@ def package_code( code_package.benchmark, is_cached, ) + if code_package.benchmark_config.container_image is not None: + _, container_uri =self.ecr_client.build_custom_image( + code_package.benchmark_config.container_image, + directory, + code_package.language_name, + code_package.language_version, + code_package.architecture, + code_package.benchmark, + ) package_config = CONFIG_FILES[code_package.language_name] function_dir = os.path.join(directory, "function") diff --git a/sebs/faas/container.py b/sebs/faas/container.py index 0c812fd19..97e1bc449 100644 --- a/sebs/faas/container.py +++ b/sebs/faas/container.py @@ -130,6 +130,10 @@ def registry_name( ) -> Tuple[str, str, str, str]: pass + @abstractmethod + def registry(image_tag:str) -> Tuple[str,str,str,str]: + pass + def build_base_image( self, directory: str, @@ -219,5 +223,51 @@ def build_base_image( return True, image_uri @abstractmethod - def build_custom_image(self, image_tag: str): - pass \ No newline at end of file + def build_custom_image(self, base_image_tag: str, package_directory: str, language_name: str, language_version: str, architecture: str,benchmark:str) -> Tuple[bool, str]: + image_tag = f"sebs.{base_image_tag.replace(':','_').replace('/','.')}.{benchmark}.{language_name}.{language_version}.{architecture}" + registry_name, repository_name, image_tag, image_uri = self.registry(image_tag) + if self.find_image(registry_name,image_tag): + self.logging.info( + f"Skipping building custom Docker image for {benchmark}, using " + f"Docker image {image_uri} from registry: {registry_name}." + ) + return False, image_uri + build_dir = os.path.join(package_directory, "build") + os.makedirs(build_dir, exist_ok=True) + + shutil.copy( + os.path.join(DOCKER_DIR, self.name(), language_name, "Dockerfile.base"), + os.path.join(build_dir, "Dockerfile"), + ) + for fn in os.listdir(package_directory): + if fn not in ("index.js", "__main__.py"): + file = os.path.join(package_directory, fn) + shutil.move(file, build_dir) + + with open(os.path.join(build_dir, ".dockerignore"), "w") as f: + f.write("Dockerfile") + + self.logging.info(f"Building custom image from {base_image_tag}.") + + isa = platform.processor() + if (isa == "x86_64" and architecture != "x64") or ( + isa == "arm64" and architecture != "arm64" + ): + self.logging.warning( + f"Building image for architecture: {architecture} on CPU architecture: {isa}. " + "This step requires configured emulation. If the build fails, please consult " + "our documentation. We recommend QEMU as it can be configured to run automatically." + ) + + buildargs = { + "BASE_IMAGE": base_image_tag, + } + image, _ = self.docker_client.images.build( + tag=image_uri, path=build_dir, buildargs=buildargs + ) + self.logging.info( + f"Push the custom benchmark image {repository_name}:{image_tag} " + f"to registry: {registry_name}." + ) + self.push_image(image_uri, image_tag) + return True, image_uri \ No newline at end of file From 856bd9ab48c14ff1afca0d320f421dd687298142 Mon Sep 17 00:00:00 2001 From: Lorenz Gruber Date: Wed, 26 Nov 2025 17:43:57 +0100 Subject: [PATCH 28/45] adding boto3 to storage module packages for aws --- config/systems.json | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/config/systems.json b/config/systems.json index 9acc1dd2d..aacf58f11 100644 --- a/config/systems.json +++ b/config/systems.json @@ -100,7 +100,11 @@ "packages": [ "redis" ], - "module_packages": {} + "module_packages": { + "storage": [ + "boto3" + ] + } } }, "nodejs": { From 0bdd2da60787cf26dd9eaaed7b2ee18df3689efa Mon Sep 17 00:00:00 2001 From: Lorenz Gruber Date: Wed, 26 Nov 2025 17:44:21 +0100 Subject: [PATCH 29/45] working on file transfer --- benchmarks-data | 2 +- .../6300.sda-workflow/config.json | 2 +- .../600.workflows/6300.sda-workflow/input.py | 25 +++++++++++++++++-- .../6300.sda-workflow/python/filter.py | 24 +++++++++++++++--- .../6300.sda-workflow/python/split.py | 4 +++ 5 files changed, 50 insertions(+), 7 deletions(-) create mode 100644 benchmarks/600.workflows/6300.sda-workflow/python/split.py diff --git a/benchmarks-data b/benchmarks-data index 7c7f67be6..1eb8679a7 160000 --- a/benchmarks-data +++ b/benchmarks-data @@ -1 +1 @@ -Subproject commit 7c7f67be6d6efd94a5de10607136ce237a673ef7 +Subproject commit 1eb8679a726867bfd3ff40874f9e740b3f7a6389 diff --git a/benchmarks/600.workflows/6300.sda-workflow/config.json b/benchmarks/600.workflows/6300.sda-workflow/config.json index 23680cd7e..ba8f5b2a4 100644 --- a/benchmarks/600.workflows/6300.sda-workflow/config.json +++ b/benchmarks/600.workflows/6300.sda-workflow/config.json @@ -2,6 +2,6 @@ "timeout": 120, "memory": 128, "languages": ["python"], - "modules": [], + "modules": ["storage"], "container-image": "logru/sda:latest" } diff --git a/benchmarks/600.workflows/6300.sda-workflow/input.py b/benchmarks/600.workflows/6300.sda-workflow/input.py index f4073b711..504e43b58 100644 --- a/benchmarks/600.workflows/6300.sda-workflow/input.py +++ b/benchmarks/600.workflows/6300.sda-workflow/input.py @@ -1,5 +1,26 @@ +import os + +def input_file(size): + size_generators = { + "test" : "Corvara_IT.tiff", + "small": "Corvara_IT.tiff", + "large": "Corvara_IT.tiff", + } + return size_generators[size] + def buckets_count(): - return (0, 0) + return (1, 1) def generate_input(data_dir, size, benchmarks_bucket,input_buckets, output_buckets, upload_func, nosql_func): - return dict() \ No newline at end of file + SDA_CFG_FILE="sda-config.json" + INPUT_FILE = input_file(size) + files = [SDA_CFG_FILE,INPUT_FILE] + for file in files: + upload_func(0, file, os.path.join(data_dir, file)) + return { + "config_file": SDA_CFG_FILE, + "input_file": INPUT_FILE, + "input_bucket": input_buckets[0], + "output_bucket": output_buckets[0], + "benchmark_bucket": benchmarks_bucket + } \ No newline at end of file diff --git a/benchmarks/600.workflows/6300.sda-workflow/python/filter.py b/benchmarks/600.workflows/6300.sda-workflow/python/filter.py index e87da8b11..5061c2931 100644 --- a/benchmarks/600.workflows/6300.sda-workflow/python/filter.py +++ b/benchmarks/600.workflows/6300.sda-workflow/python/filter.py @@ -1,6 +1,24 @@ +import os import subprocess +import uuid + +from . import storage + +storage_client = storage.storage.get_instance() + +def download_file(benchmark_bucket, bucket, basename, dest_dir): + path = os.path.join(dest_dir, basename) + storage_client.download(benchmark_bucket, bucket + '/' + basename, path) + return path + def handler(event): - input = "/data/Corvara_IT.tiff" - config = "/data/sda-workflow-local.json" - result = subprocess.run([f"SettlementDelineationFilter", "-i", input, "-c", config],capture_output=True,text=True) + benchmark_bucket = event["benchmark_bucket"] + input_bucket = event["input_bucket"] + input = event["input_file"] + config = event["config_file"] + TMP_DIR = os.path.join("/tmp",str(uuid.uuid4())) + os.makedirs(TMP_DIR, exist_ok=True) + input_path = download_file(benchmark_bucket, input_bucket, input, TMP_DIR) + config_path = download_file(benchmark_bucket, input_bucket, config, TMP_DIR) + result = subprocess.run([f"SettlementDelineationFilter", "-i", input_path, "-c", config_path],capture_output=True,text=True) return {"stdout": result.stdout, "stderr": result.stderr} \ No newline at end of file diff --git a/benchmarks/600.workflows/6300.sda-workflow/python/split.py b/benchmarks/600.workflows/6300.sda-workflow/python/split.py new file mode 100644 index 000000000..f7d96399a --- /dev/null +++ b/benchmarks/600.workflows/6300.sda-workflow/python/split.py @@ -0,0 +1,4 @@ +import subprocess + +def handler(event): + pass \ No newline at end of file From e429c8af76be9f7914a218bd3e68f831e973fffb Mon Sep 17 00:00:00 2001 From: Lorenz Gruber Date: Thu, 27 Nov 2025 14:35:18 +0100 Subject: [PATCH 30/45] load custom image container from cache if avail --- sebs/benchmark.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/sebs/benchmark.py b/sebs/benchmark.py index ec5d843ca..b03e85cb6 100644 --- a/sebs/benchmark.py +++ b/sebs/benchmark.py @@ -301,8 +301,7 @@ def serialize(self) -> dict: return {"size": self.code_size, "hash": self.hash} def query_cache(self): - - if self.container_deployment: + if self.container_deployment or self.benchmark_config.container_image is not None: self._code_package = self._cache_client.get_container( deployment=self._deployment_name, benchmark=self._benchmark, From bcac8e989008779f2b679d54f8da0c3976c97748 Mon Sep 17 00:00:00 2001 From: Lorenz Gruber Date: Thu, 27 Nov 2025 14:56:17 +0100 Subject: [PATCH 31/45] store outputs in output bucket --- .../6300.sda-workflow/python/filter.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/benchmarks/600.workflows/6300.sda-workflow/python/filter.py b/benchmarks/600.workflows/6300.sda-workflow/python/filter.py index 5061c2931..ea07ba8c7 100644 --- a/benchmarks/600.workflows/6300.sda-workflow/python/filter.py +++ b/benchmarks/600.workflows/6300.sda-workflow/python/filter.py @@ -11,14 +11,25 @@ def download_file(benchmark_bucket, bucket, basename, dest_dir): storage_client.download(benchmark_bucket, bucket + '/' + basename, path) return path +SHP_SUFFIX = [".shp", ".shx", ".dbf", ".prj"] + +def upload_shp_file(benchmark_bucket, bucket, shp_dir): + for f in os.listdir(shp_dir): + if any(f.endswith(suffix) for suffix in SHP_SUFFIX): + full_path = os.path.join(shp_dir, f) + storage_client.upload(benchmark_bucket, bucket + '/' + f, full_path) + def handler(event): benchmark_bucket = event["benchmark_bucket"] input_bucket = event["input_bucket"] + output_bucket = event["output_bucket"] input = event["input_file"] config = event["config_file"] TMP_DIR = os.path.join("/tmp",str(uuid.uuid4())) os.makedirs(TMP_DIR, exist_ok=True) input_path = download_file(benchmark_bucket, input_bucket, input, TMP_DIR) config_path = download_file(benchmark_bucket, input_bucket, config, TMP_DIR) - result = subprocess.run([f"SettlementDelineationFilter", "-i", input_path, "-c", config_path],capture_output=True,text=True) + # Store workflow data in /tmp due to read only filesystem restriction + result = subprocess.run([f"SettlementDelineationFilter", "-i", input_path, "-c", config_path, "-o", TMP_DIR],capture_output=True,text=True) + upload_shp_file(benchmark_bucket, output_bucket, TMP_DIR) return {"stdout": result.stdout, "stderr": result.stderr} \ No newline at end of file From 1f15d2279ff51f6476f5762c2bc02cea3c533630 Mon Sep 17 00:00:00 2001 From: Lorenz Gruber Date: Thu, 27 Nov 2025 15:17:05 +0100 Subject: [PATCH 32/45] (wip) file management --- .../600.workflows/6300.sda-workflow/input.py | 4 +- .../6300.sda-workflow/python/filter.py | 12 +++--- .../6300.sda-workflow/python/neighbors.py | 38 +++++++++++++++++++ 3 files changed, 46 insertions(+), 8 deletions(-) create mode 100644 benchmarks/600.workflows/6300.sda-workflow/python/neighbors.py diff --git a/benchmarks/600.workflows/6300.sda-workflow/input.py b/benchmarks/600.workflows/6300.sda-workflow/input.py index 504e43b58..abe32be4f 100644 --- a/benchmarks/600.workflows/6300.sda-workflow/input.py +++ b/benchmarks/600.workflows/6300.sda-workflow/input.py @@ -9,7 +9,7 @@ def input_file(size): return size_generators[size] def buckets_count(): - return (1, 1) + return (3, 3) def generate_input(data_dir, size, benchmarks_bucket,input_buckets, output_buckets, upload_func, nosql_func): SDA_CFG_FILE="sda-config.json" @@ -21,6 +21,6 @@ def generate_input(data_dir, size, benchmarks_bucket,input_buckets, output_bucke "config_file": SDA_CFG_FILE, "input_file": INPUT_FILE, "input_bucket": input_buckets[0], - "output_bucket": output_buckets[0], + "filter_output_bucket": output_buckets[0], "benchmark_bucket": benchmarks_bucket } \ No newline at end of file diff --git a/benchmarks/600.workflows/6300.sda-workflow/python/filter.py b/benchmarks/600.workflows/6300.sda-workflow/python/filter.py index ea07ba8c7..b7cba6d42 100644 --- a/benchmarks/600.workflows/6300.sda-workflow/python/filter.py +++ b/benchmarks/600.workflows/6300.sda-workflow/python/filter.py @@ -9,20 +9,20 @@ def download_file(benchmark_bucket, bucket, basename, dest_dir): path = os.path.join(dest_dir, basename) storage_client.download(benchmark_bucket, bucket + '/' + basename, path) - return path + yield path SHP_SUFFIX = [".shp", ".shx", ".dbf", ".prj"] def upload_shp_file(benchmark_bucket, bucket, shp_dir): for f in os.listdir(shp_dir): - if any(f.endswith(suffix) for suffix in SHP_SUFFIX): + if "_filtered" in f and any(f.endswith(suffix) for suffix in SHP_SUFFIX): full_path = os.path.join(shp_dir, f) - storage_client.upload(benchmark_bucket, bucket + '/' + f, full_path) + yield storage_client.upload(benchmark_bucket, bucket + '/' + f, full_path) def handler(event): benchmark_bucket = event["benchmark_bucket"] input_bucket = event["input_bucket"] - output_bucket = event["output_bucket"] + output_bucket = event["filter_output_bucket"] input = event["input_file"] config = event["config_file"] TMP_DIR = os.path.join("/tmp",str(uuid.uuid4())) @@ -31,5 +31,5 @@ def handler(event): config_path = download_file(benchmark_bucket, input_bucket, config, TMP_DIR) # Store workflow data in /tmp due to read only filesystem restriction result = subprocess.run([f"SettlementDelineationFilter", "-i", input_path, "-c", config_path, "-o", TMP_DIR],capture_output=True,text=True) - upload_shp_file(benchmark_bucket, output_bucket, TMP_DIR) - return {"stdout": result.stdout, "stderr": result.stderr} \ No newline at end of file + output_files = list(upload_shp_file(benchmark_bucket, output_bucket, TMP_DIR)) + return {"stdout": result.stdout, "stderr": result.stderr,"filtered_files_bucket":output_bucket,"filtered_files":output_files} \ No newline at end of file diff --git a/benchmarks/600.workflows/6300.sda-workflow/python/neighbors.py b/benchmarks/600.workflows/6300.sda-workflow/python/neighbors.py new file mode 100644 index 000000000..9667d2cde --- /dev/null +++ b/benchmarks/600.workflows/6300.sda-workflow/python/neighbors.py @@ -0,0 +1,38 @@ +import os +import subprocess +import uuid + +from . import storage + +storage_client = storage.storage.get_instance() + +SHP_SUFFIX = [".shp", ".shx", ".dbf", ".prj"] + +def download_shp_file(benchmark_bucket, bucket, basename, dest_dir): + path = os.path.join(dest_dir, basename) + storage_client.download(benchmark_bucket, bucket + '/' + basename, path) + return path + + + + +def upload_shp_file(benchmark_bucket, bucket, shp_dir): + for f in os.listdir(shp_dir): + if any(f.endswith(suffix) for suffix in SHP_SUFFIX): + full_path = os.path.join(shp_dir, f) + storage_client.upload(benchmark_bucket, bucket + '/' + f, full_path) + +def handler(event): + benchmark_bucket = event["benchmark_bucket"] + input_bucket = event["input_bucket"] + output_bucket = event["output_bucket"] + input = event["input_file"] + config = event["config_file"] + TMP_DIR = os.path.join("/tmp",str(uuid.uuid4())) + os.makedirs(TMP_DIR, exist_ok=True) + input_path = download_file(benchmark_bucket, input_bucket, input, TMP_DIR) + config_path = download_file(benchmark_bucket, input_bucket, config, TMP_DIR) + # Store workflow data in /tmp due to read only filesystem restriction + result = subprocess.run([f"SettlementDelineationFilter", "-i", input_path, "-c", config_path, "-o", TMP_DIR],capture_output=True,text=True) + upload_shp_file(benchmark_bucket, output_bucket, TMP_DIR) + return {"stdout": result.stdout, "stderr": result.stderr} \ No newline at end of file From a35cce5b575c96b5f2695128481db35267dde729 Mon Sep 17 00:00:00 2001 From: Lorenz Gruber Date: Mon, 8 Dec 2025 18:12:23 +0100 Subject: [PATCH 33/45] runnable chaining of first two sda steps --- .../6300.sda-workflow/.gitignore | 1 + .../6300.sda-workflow/config.json | 2 +- .../6300.sda-workflow/definition.json | 7 +++- .../600.workflows/6300.sda-workflow/input.py | 14 +++++-- .../6300.sda-workflow/python/SDAHelper.py | 40 +++++++++++++++++++ .../6300.sda-workflow/python/filter.py | 34 ++++------------ .../6300.sda-workflow/python/neighbors.py | 34 +++------------- 7 files changed, 72 insertions(+), 60 deletions(-) create mode 100644 benchmarks/600.workflows/6300.sda-workflow/.gitignore create mode 100644 benchmarks/600.workflows/6300.sda-workflow/python/SDAHelper.py diff --git a/benchmarks/600.workflows/6300.sda-workflow/.gitignore b/benchmarks/600.workflows/6300.sda-workflow/.gitignore new file mode 100644 index 000000000..535a7ffea --- /dev/null +++ b/benchmarks/600.workflows/6300.sda-workflow/.gitignore @@ -0,0 +1 @@ +sda_config.json \ No newline at end of file diff --git a/benchmarks/600.workflows/6300.sda-workflow/config.json b/benchmarks/600.workflows/6300.sda-workflow/config.json index ba8f5b2a4..12332999d 100644 --- a/benchmarks/600.workflows/6300.sda-workflow/config.json +++ b/benchmarks/600.workflows/6300.sda-workflow/config.json @@ -3,5 +3,5 @@ "memory": 128, "languages": ["python"], "modules": ["storage"], - "container-image": "logru/sda:latest" + "container-image": "logru/sda-no-db:latest" } diff --git a/benchmarks/600.workflows/6300.sda-workflow/definition.json b/benchmarks/600.workflows/6300.sda-workflow/definition.json index a82881b1d..43ebba772 100644 --- a/benchmarks/600.workflows/6300.sda-workflow/definition.json +++ b/benchmarks/600.workflows/6300.sda-workflow/definition.json @@ -3,7 +3,12 @@ "states": { "filter": { "type": "task", - "func_name": "filter" + "func_name": "filter", + "next": "neighbors" + }, + "neighbors": { + "type": "task", + "func_name": "neighbors" } } } diff --git a/benchmarks/600.workflows/6300.sda-workflow/input.py b/benchmarks/600.workflows/6300.sda-workflow/input.py index abe32be4f..44e428372 100644 --- a/benchmarks/600.workflows/6300.sda-workflow/input.py +++ b/benchmarks/600.workflows/6300.sda-workflow/input.py @@ -1,3 +1,4 @@ +import json import os def input_file(size): @@ -11,14 +12,21 @@ def input_file(size): def buckets_count(): return (3, 3) +def sda_config(): + SDA_CONFIG_PATH = os.path.join(os.path.dirname(__file__), "sda_config.json") + with open(SDA_CONFIG_PATH, "r") as f: + config = json.load(f) + return config + + + def generate_input(data_dir, size, benchmarks_bucket,input_buckets, output_buckets, upload_func, nosql_func): - SDA_CFG_FILE="sda-config.json" INPUT_FILE = input_file(size) - files = [SDA_CFG_FILE,INPUT_FILE] + files = [INPUT_FILE] for file in files: upload_func(0, file, os.path.join(data_dir, file)) return { - "config_file": SDA_CFG_FILE, + "config_file": json.dumps(sda_config()), "input_file": INPUT_FILE, "input_bucket": input_buckets[0], "filter_output_bucket": output_buckets[0], diff --git a/benchmarks/600.workflows/6300.sda-workflow/python/SDAHelper.py b/benchmarks/600.workflows/6300.sda-workflow/python/SDAHelper.py new file mode 100644 index 000000000..425758035 --- /dev/null +++ b/benchmarks/600.workflows/6300.sda-workflow/python/SDAHelper.py @@ -0,0 +1,40 @@ +import os +import uuid +from pathlib import Path +from . import storage + +storage_client = storage.storage.get_instance() + +SHP_SUFFIX = [".shp", ".shx", ".dbf", ".prj"] + +def download_file(benchmark_bucket, path_in_bucket, dest_dir): + path = Path(dest_dir) / Path(path_in_bucket).name + storage_client.download(benchmark_bucket, path_in_bucket, path) + return path + +def download_file_bucket(benchmark_bucket, bucket, basename, dest_dir): + return download_file(benchmark_bucket, bucket + '/' + basename, dest_dir) + +def download_shp_file(benchmark_bucket, files, dest_dir): + path = None + for filename in files: + path = download_file(benchmark_bucket, filename, dest_dir) + return path.with_suffix(".shp") + +def store_config(json_string,directory): + config_path = Path(directory) / "config.json" + with open(config_path,'w') as f: + f.write(json_string) + return config_path + +def upload_shp_file(benchmark_bucket, bucket, shp_basename): + shp_dir = Path(shp_basename).parent + for f in shp_dir.iterdir(): + if Path(shp_basename).stem == Path(f).stem and any(f.name.endswith(suffix) for suffix in SHP_SUFFIX): + full_path = shp_dir / f.name + yield storage_client.upload(benchmark_bucket, bucket + '/' + f.name, full_path,False) + +def create_tmp_dir(): + tmp_dir = os.path.join("/tmp",str(uuid.uuid4())) + os.makedirs(tmp_dir, exist_ok=True) + return tmp_dir \ No newline at end of file diff --git a/benchmarks/600.workflows/6300.sda-workflow/python/filter.py b/benchmarks/600.workflows/6300.sda-workflow/python/filter.py index b7cba6d42..21b2a0eaa 100644 --- a/benchmarks/600.workflows/6300.sda-workflow/python/filter.py +++ b/benchmarks/600.workflows/6300.sda-workflow/python/filter.py @@ -1,23 +1,5 @@ -import os import subprocess -import uuid - -from . import storage - -storage_client = storage.storage.get_instance() - -def download_file(benchmark_bucket, bucket, basename, dest_dir): - path = os.path.join(dest_dir, basename) - storage_client.download(benchmark_bucket, bucket + '/' + basename, path) - yield path - -SHP_SUFFIX = [".shp", ".shx", ".dbf", ".prj"] - -def upload_shp_file(benchmark_bucket, bucket, shp_dir): - for f in os.listdir(shp_dir): - if "_filtered" in f and any(f.endswith(suffix) for suffix in SHP_SUFFIX): - full_path = os.path.join(shp_dir, f) - yield storage_client.upload(benchmark_bucket, bucket + '/' + f, full_path) +from .SDAHelper import * def handler(event): benchmark_bucket = event["benchmark_bucket"] @@ -25,11 +7,11 @@ def handler(event): output_bucket = event["filter_output_bucket"] input = event["input_file"] config = event["config_file"] - TMP_DIR = os.path.join("/tmp",str(uuid.uuid4())) - os.makedirs(TMP_DIR, exist_ok=True) - input_path = download_file(benchmark_bucket, input_bucket, input, TMP_DIR) - config_path = download_file(benchmark_bucket, input_bucket, config, TMP_DIR) + TMP_DIR = create_tmp_dir() + input_path = download_file_bucket(benchmark_bucket, input_bucket, input, TMP_DIR) + config_path = store_config(config, TMP_DIR) # Store workflow data in /tmp due to read only filesystem restriction - result = subprocess.run([f"SettlementDelineationFilter", "-i", input_path, "-c", config_path, "-o", TMP_DIR],capture_output=True,text=True) - output_files = list(upload_shp_file(benchmark_bucket, output_bucket, TMP_DIR)) - return {"stdout": result.stdout, "stderr": result.stderr,"filtered_files_bucket":output_bucket,"filtered_files":output_files} \ No newline at end of file + result = subprocess.run([f"SettlementDelineationFilter", "-i", str(input_path), "-c", str(config_path), "-o", TMP_DIR],capture_output=True,text=True) + output_file = Path(TMP_DIR).glob("*_filtered.shp").__next__() + output_files = list(upload_shp_file(benchmark_bucket, output_bucket, output_file)) + return {"stdout": result.stdout, "stderr": result.stderr,"filtered_files":output_files,"benchmark_bucket":benchmark_bucket,"config_file":config} \ No newline at end of file diff --git a/benchmarks/600.workflows/6300.sda-workflow/python/neighbors.py b/benchmarks/600.workflows/6300.sda-workflow/python/neighbors.py index 9667d2cde..c5625bbfc 100644 --- a/benchmarks/600.workflows/6300.sda-workflow/python/neighbors.py +++ b/benchmarks/600.workflows/6300.sda-workflow/python/neighbors.py @@ -1,38 +1,14 @@ -import os import subprocess -import uuid - -from . import storage - -storage_client = storage.storage.get_instance() - -SHP_SUFFIX = [".shp", ".shx", ".dbf", ".prj"] - -def download_shp_file(benchmark_bucket, bucket, basename, dest_dir): - path = os.path.join(dest_dir, basename) - storage_client.download(benchmark_bucket, bucket + '/' + basename, path) - return path - - - - -def upload_shp_file(benchmark_bucket, bucket, shp_dir): - for f in os.listdir(shp_dir): - if any(f.endswith(suffix) for suffix in SHP_SUFFIX): - full_path = os.path.join(shp_dir, f) - storage_client.upload(benchmark_bucket, bucket + '/' + f, full_path) +from .SDAHelper import * def handler(event): benchmark_bucket = event["benchmark_bucket"] - input_bucket = event["input_bucket"] - output_bucket = event["output_bucket"] - input = event["input_file"] + input = event["filtered_files"] config = event["config_file"] TMP_DIR = os.path.join("/tmp",str(uuid.uuid4())) os.makedirs(TMP_DIR, exist_ok=True) - input_path = download_file(benchmark_bucket, input_bucket, input, TMP_DIR) - config_path = download_file(benchmark_bucket, input_bucket, config, TMP_DIR) + input_path = download_shp_file(benchmark_bucket, input, TMP_DIR) + config_path = store_config(config, TMP_DIR) # Store workflow data in /tmp due to read only filesystem restriction - result = subprocess.run([f"SettlementDelineationFilter", "-i", input_path, "-c", config_path, "-o", TMP_DIR],capture_output=True,text=True) - upload_shp_file(benchmark_bucket, output_bucket, TMP_DIR) + result = subprocess.run([f"SettlementDelineationNeighbours", "-i", str(input_path), "-c", str(config_path)],capture_output=True,text=True) return {"stdout": result.stdout, "stderr": result.stderr} \ No newline at end of file From 634a96048e976ab7225bed666402a63cf604d874 Mon Sep 17 00:00:00 2001 From: Lorenz Gruber Date: Tue, 9 Dec 2025 23:30:49 +0100 Subject: [PATCH 34/45] call update instead of create modifying sda benchmark data --- benchmarks-data | 2 +- sebs/aws/aws.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/benchmarks-data b/benchmarks-data index 1eb8679a7..8676c32e4 160000 --- a/benchmarks-data +++ b/benchmarks-data @@ -1 +1 @@ -Subproject commit 1eb8679a726867bfd3ff40874f9e740b3f7a6389 +Subproject commit 8676c32e4e0aa339202ee4bcf90d9e0d21dcd9f9 diff --git a/sebs/aws/aws.py b/sebs/aws/aws.py index aec1a7ab6..eeda2d7d0 100644 --- a/sebs/aws/aws.py +++ b/sebs/aws/aws.py @@ -463,7 +463,7 @@ def create_workflow(self, code_package: Benchmark, workflow_name: str) -> "SFNWo code_files = list(code_package.get_code_files(include_config=False)) func_names = [os.path.splitext(os.path.basename(p))[0] for p in code_files] funcs = [ - self.create_function(code_package, workflow_name + "___" + fn, code_package.container_deployment,code_package.container_uri if code_package.container_deployment else None) for fn in func_names + self.update_function(code_package, workflow_name + "___" + fn, code_package.container_deployment,code_package.container_uri if code_package.container_deployment else None) for fn in func_names ] # Generate workflow definition.json From 8696869679305384e27bc7e6609e276874ed5ddc Mon Sep 17 00:00:00 2001 From: Lorenz Gruber Date: Wed, 10 Dec 2025 01:14:06 +0100 Subject: [PATCH 35/45] sda workflow implementation --- .../6300.sda-workflow/.gitignore | 2 +- .../6300.sda-workflow/definition.json | 36 ++++++++++++++- .../600.workflows/6300.sda-workflow/input.py | 16 ++++--- .../6300.sda-workflow/python/SDAHelper.py | 23 +++++----- .../6300.sda-workflow/python/analysis.py | 22 +++++++++ .../6300.sda-workflow/python/clearDB.py | 11 +++++ .../6300.sda-workflow/python/clustering.py | 28 +++++++++++ .../6300.sda-workflow/python/components.py | 46 +++++++++++++++++++ .../6300.sda-workflow/python/filter.py | 11 +++-- .../6300.sda-workflow/python/merge.py | 38 +++++++++++++++ .../6300.sda-workflow/python/neighbors.py | 17 ++++--- 11 files changed, 218 insertions(+), 32 deletions(-) create mode 100644 benchmarks/600.workflows/6300.sda-workflow/python/analysis.py create mode 100644 benchmarks/600.workflows/6300.sda-workflow/python/clearDB.py create mode 100644 benchmarks/600.workflows/6300.sda-workflow/python/clustering.py create mode 100644 benchmarks/600.workflows/6300.sda-workflow/python/components.py create mode 100644 benchmarks/600.workflows/6300.sda-workflow/python/merge.py diff --git a/benchmarks/600.workflows/6300.sda-workflow/.gitignore b/benchmarks/600.workflows/6300.sda-workflow/.gitignore index 535a7ffea..253b6868b 100644 --- a/benchmarks/600.workflows/6300.sda-workflow/.gitignore +++ b/benchmarks/600.workflows/6300.sda-workflow/.gitignore @@ -1 +1 @@ -sda_config.json \ No newline at end of file +sda-config.json \ No newline at end of file diff --git a/benchmarks/600.workflows/6300.sda-workflow/definition.json b/benchmarks/600.workflows/6300.sda-workflow/definition.json index 43ebba772..8523f4db4 100644 --- a/benchmarks/600.workflows/6300.sda-workflow/definition.json +++ b/benchmarks/600.workflows/6300.sda-workflow/definition.json @@ -1,6 +1,11 @@ { - "root": "filter", + "root": "clearDB", "states": { + "clearDB":{ + "type": "task", + "func_name": "clearDB", + "next": "filter" + }, "filter": { "type": "task", "func_name": "filter", @@ -8,7 +13,34 @@ }, "neighbors": { "type": "task", - "func_name": "neighbors" + "func_name": "neighbors", + "next": "components" + }, + "components": { + "type": "task", + "func_name": "components", + "next": "cluster-analyze-map" + }, + "cluster-analyze-map":{ + "type": "map", + "root": "clustering", + "array": "workloads", + "next": "merge-results", + "states": { + "clustering": { + "type": "task", + "func_name": "clustering", + "next": "analysis" + }, + "analysis": { + "type": "task", + "func_name": "analysis" + } + } + }, + "merge-results": { + "type": "task", + "func_name": "merge" } } } diff --git a/benchmarks/600.workflows/6300.sda-workflow/input.py b/benchmarks/600.workflows/6300.sda-workflow/input.py index 44e428372..c1b6ab221 100644 --- a/benchmarks/600.workflows/6300.sda-workflow/input.py +++ b/benchmarks/600.workflows/6300.sda-workflow/input.py @@ -10,25 +10,27 @@ def input_file(size): return size_generators[size] def buckets_count(): - return (3, 3) + return (1, 4) def sda_config(): - SDA_CONFIG_PATH = os.path.join(os.path.dirname(__file__), "sda_config.json") + SDA_CONFIG_PATH = os.path.join(os.path.dirname(__file__), "sda-config.json") with open(SDA_CONFIG_PATH, "r") as f: config = json.load(f) return config - def generate_input(data_dir, size, benchmarks_bucket,input_buckets, output_buckets, upload_func, nosql_func): INPUT_FILE = input_file(size) - files = [INPUT_FILE] - for file in files: - upload_func(0, file, os.path.join(data_dir, file)) + upload_func(0, INPUT_FILE, os.path.join(data_dir, INPUT_FILE)) + CONFIG_FILE = "sda-config.json" + upload_func(0, CONFIG_FILE, os.path.join(os.path.dirname(__file__), CONFIG_FILE)) return { - "config_file": json.dumps(sda_config()), + "config_file": CONFIG_FILE, "input_file": INPUT_FILE, "input_bucket": input_buckets[0], "filter_output_bucket": output_buckets[0], + "cluster_output_bucket": output_buckets[1], + "analysis_output_bucket": output_buckets[2], + "final_output_bucket": output_buckets[3], "benchmark_bucket": benchmarks_bucket } \ No newline at end of file diff --git a/benchmarks/600.workflows/6300.sda-workflow/python/SDAHelper.py b/benchmarks/600.workflows/6300.sda-workflow/python/SDAHelper.py index 425758035..5c6ec27a9 100644 --- a/benchmarks/600.workflows/6300.sda-workflow/python/SDAHelper.py +++ b/benchmarks/600.workflows/6300.sda-workflow/python/SDAHelper.py @@ -6,6 +6,7 @@ storage_client = storage.storage.get_instance() SHP_SUFFIX = [".shp", ".shx", ".dbf", ".prj"] +CONFIG_FILE_NAME = "sda-config.json" def download_file(benchmark_bucket, path_in_bucket, dest_dir): path = Path(dest_dir) / Path(path_in_bucket).name @@ -15,24 +16,24 @@ def download_file(benchmark_bucket, path_in_bucket, dest_dir): def download_file_bucket(benchmark_bucket, bucket, basename, dest_dir): return download_file(benchmark_bucket, bucket + '/' + basename, dest_dir) -def download_shp_file(benchmark_bucket, files, dest_dir): - path = None - for filename in files: - path = download_file(benchmark_bucket, filename, dest_dir) - return path.with_suffix(".shp") +def download_shp_file(benchmark_bucket, bucket ,shp_file, dest_dir): + files = [Path(shp_file).with_suffix(suffix) for suffix in filter(lambda x: x is not ".shp",SHP_SUFFIX)] + for f in files: + download_file_bucket(benchmark_bucket, bucket, f.name, dest_dir) + return download_file_bucket(benchmark_bucket, bucket, shp_file, dest_dir) -def store_config(json_string,directory): - config_path = Path(directory) / "config.json" - with open(config_path,'w') as f: - f.write(json_string) - return config_path +def load_config(benchmark_bucket, input_bucket,directory): + return download_file_bucket(benchmark_bucket, input_bucket, CONFIG_FILE_NAME, directory) def upload_shp_file(benchmark_bucket, bucket, shp_basename): shp_dir = Path(shp_basename).parent for f in shp_dir.iterdir(): if Path(shp_basename).stem == Path(f).stem and any(f.name.endswith(suffix) for suffix in SHP_SUFFIX): full_path = shp_dir / f.name - yield storage_client.upload(benchmark_bucket, bucket + '/' + f.name, full_path,False) + storage_client.upload(benchmark_bucket, bucket + '/' + f.name, full_path,False) + +def download_directory(benchmark_bucket, bucket, dest_dir): + storage_client.download_directory(benchmark_bucket, bucket, dest_dir) def create_tmp_dir(): tmp_dir = os.path.join("/tmp",str(uuid.uuid4())) diff --git a/benchmarks/600.workflows/6300.sda-workflow/python/analysis.py b/benchmarks/600.workflows/6300.sda-workflow/python/analysis.py new file mode 100644 index 000000000..382edc9b9 --- /dev/null +++ b/benchmarks/600.workflows/6300.sda-workflow/python/analysis.py @@ -0,0 +1,22 @@ +import subprocess +from .SDAHelper import * + +def handler(event): + benchmark_bucket = event["benchmark_bucket"] + cluster_output_bucket = event["cluster_output_bucket"] + TMP_DIR = create_tmp_dir() + analysis_input_file = download_shp_file(benchmark_bucket,cluster_output_bucket,event["cluster_output_file"],TMP_DIR) + config_file = load_config(benchmark_bucket,event["input_bucket"], TMP_DIR) + OUTPUT_STEM = "Analysis_"+Path(analysis_input_file).stem + command = ["SettlementDelineationAnalysis", "-i", str(analysis_input_file), "-c", str(config_file), "--outputStem", OUTPUT_STEM] + result = subprocess.run(command,capture_output=True,text=True, cwd=TMP_DIR) + if result.returncode != 0: + event["stdout"] = result.stdout + event["stderr"] = result.stderr + return event + event.pop("cluster_output_file", None) + event["analysis_output_files"]= [] + for file in Path(TMP_DIR).glob(f"{OUTPUT_STEM}*.shp"): + upload_shp_file(benchmark_bucket, event["analysis_output_bucket"],file) + event["analysis_output_files"].append(file.name) + return event diff --git a/benchmarks/600.workflows/6300.sda-workflow/python/clearDB.py b/benchmarks/600.workflows/6300.sda-workflow/python/clearDB.py new file mode 100644 index 000000000..4096f3272 --- /dev/null +++ b/benchmarks/600.workflows/6300.sda-workflow/python/clearDB.py @@ -0,0 +1,11 @@ +from .SDAHelper import * +import subprocess + +def handler(event): + TMP_DIR = create_tmp_dir() + config = load_config(event["benchmark_bucket"],event["input_bucket"], TMP_DIR) + result = subprocess.run([f"AfricapolisClearDatabase", "-c", str(config)],capture_output=True,text=True) + if result.returncode != 0: + event["stdout"] = result.stdout + event["stderr"] = result.stderr + return event \ No newline at end of file diff --git a/benchmarks/600.workflows/6300.sda-workflow/python/clustering.py b/benchmarks/600.workflows/6300.sda-workflow/python/clustering.py new file mode 100644 index 000000000..06b114636 --- /dev/null +++ b/benchmarks/600.workflows/6300.sda-workflow/python/clustering.py @@ -0,0 +1,28 @@ +import subprocess +from .SDAHelper import * + +def handler(event): + + benchmark_bucket = event["benchmark_bucket"] + filter_output_bucket = event["filter_output_bucket"] + TMP_DIR = create_tmp_dir() + input_files = [download_shp_file(benchmark_bucket, filter_output_bucket ,shp_file, TMP_DIR ) for shp_file in event["cluster_input_files"]] + config = load_config(benchmark_bucket,event["input_bucket"], TMP_DIR) + components = event["cluster_components"] + OUTPUT_STEM = "Cluster"+str(components[0]) + # Store workflow data in /tmp due to read only filesystem restriction + command = [f"SettlementDelineationContraction", "-i"] + command.extend([str(file) for file in input_files]) + command.extend(["-c", str(config),"--outputStem",OUTPUT_STEM,"--components"]) + command.extend([str(comp) for comp in components]) + result = subprocess.run(command,capture_output=True,text=True, cwd=TMP_DIR) + if result.returncode != 0: + event["stdout"] = result.stdout + event["stderr"] = result.stderr + return event + output_file = Path(TMP_DIR).glob(f"{OUTPUT_STEM}*.shp").__next__() + upload_shp_file(benchmark_bucket, event["cluster_output_bucket"],output_file) + event.pop("cluster_input_files", None) + event.pop("cluster_components", None) + event["cluster_output_file"] = output_file.name + return {"payload":event,"request_id":event.get("request-id","0")} \ No newline at end of file diff --git a/benchmarks/600.workflows/6300.sda-workflow/python/components.py b/benchmarks/600.workflows/6300.sda-workflow/python/components.py new file mode 100644 index 000000000..e77324723 --- /dev/null +++ b/benchmarks/600.workflows/6300.sda-workflow/python/components.py @@ -0,0 +1,46 @@ +import subprocess +from .SDAHelper import * +import json + +def build_cluster_workload(components_files): + workloadID = 0 + workloads = [] + for comp_file in components_files: + with open(comp_file, 'r') as f: + components_data = json.load(f) + workload = { + "cluster_input_files": [Path(f).name for f in components_data["files"]], + "cluster_components": components_data["components"] + } + workloads.append(workload) + workloadID += 1 + return workloads + +def handler(event): + TMP_DIR = create_tmp_dir() + config_file = load_config(event["benchmark_bucket"],event["input_bucket"], TMP_DIR) + COMPONENT_FILE_PREFIX = "components" + result = subprocess.run( + ["AfricapolisGraphComponents", "-c", str(config_file), "-o", COMPONENT_FILE_PREFIX], + cwd=TMP_DIR, + capture_output=True, + text=True + ) + if result.returncode != 0: + event["stdout"] = result.stdout + event["stderr"] = result.stderr + return event + event.pop("stdout", None) + event.pop("stderr", None) + components_files = sorted(str(p) for p in Path(TMP_DIR).glob(f"{COMPONENT_FILE_PREFIX}*.json")) + workloads = build_cluster_workload(components_files) + return { + "workloads":[ + { + "cluster_input_files": workload["cluster_input_files"], + "cluster_components": workload["cluster_components"], + **event + } for workload in workloads + ], + **event + } \ No newline at end of file diff --git a/benchmarks/600.workflows/6300.sda-workflow/python/filter.py b/benchmarks/600.workflows/6300.sda-workflow/python/filter.py index 21b2a0eaa..508ed0066 100644 --- a/benchmarks/600.workflows/6300.sda-workflow/python/filter.py +++ b/benchmarks/600.workflows/6300.sda-workflow/python/filter.py @@ -6,12 +6,15 @@ def handler(event): input_bucket = event["input_bucket"] output_bucket = event["filter_output_bucket"] input = event["input_file"] - config = event["config_file"] TMP_DIR = create_tmp_dir() input_path = download_file_bucket(benchmark_bucket, input_bucket, input, TMP_DIR) - config_path = store_config(config, TMP_DIR) + config_path = load_config(benchmark_bucket,input_bucket, TMP_DIR) # Store workflow data in /tmp due to read only filesystem restriction result = subprocess.run([f"SettlementDelineationFilter", "-i", str(input_path), "-c", str(config_path), "-o", TMP_DIR],capture_output=True,text=True) + if result.returncode != 0: + event["stdout"] = result.stdout + event["stderr"] = result.stderr + return event output_file = Path(TMP_DIR).glob("*_filtered.shp").__next__() - output_files = list(upload_shp_file(benchmark_bucket, output_bucket, output_file)) - return {"stdout": result.stdout, "stderr": result.stderr,"filtered_files":output_files,"benchmark_bucket":benchmark_bucket,"config_file":config} \ No newline at end of file + upload_shp_file(benchmark_bucket, output_bucket, output_file) + return {"filtered_shp_file":Path(output_file).name,**event} \ No newline at end of file diff --git a/benchmarks/600.workflows/6300.sda-workflow/python/merge.py b/benchmarks/600.workflows/6300.sda-workflow/python/merge.py new file mode 100644 index 000000000..16726db12 --- /dev/null +++ b/benchmarks/600.workflows/6300.sda-workflow/python/merge.py @@ -0,0 +1,38 @@ +import subprocess +from .SDAHelper import * + +def run_merge(input_files, output_file, directory,event): + output_location = Path(directory)/output_file + command = [f"SettlementDelineationMerge", "-i"] + command.extend([str(file) for file in input_files]) + command.extend(["--output",str(output_location)]) + res = subprocess.run(command,capture_output=True,text=True, cwd=directory) + upload_shp_file(event["benchmark_bucket"], event["final_output_bucket"],output_location) + return res + +def load_input(event,directory): + benchmark_bucket = event["benchmark_bucket"] + analysis_output_bucket = event["analysis_output_bucket"] + merge_input_files = [] + merge_edges_input_files = [] + for workload in event["workloads"]: + for file in workload["analysis_output_files"]: + shp_file = download_shp_file(benchmark_bucket,analysis_output_bucket,file,directory) + if "_edges" in shp_file.stem: + merge_edges_input_files.append(shp_file) + else: + merge_input_files.append(shp_file) + return merge_input_files, merge_edges_input_files + +def handler(event): + TMP_DIR = create_tmp_dir() + merge_input_files, merge_edges_input_files = load_input(event, TMP_DIR) + OUTPUT_FILE_STEM = Path(event["input_file"]).stem + "_SDA" + OUTPUT_FILE_NAME = OUTPUT_FILE_STEM + ".shp" + EDGE_OUTPUT_FILE_NAME = OUTPUT_FILE_STEM + "_edges.shp" + merge_output_result = run_merge(merge_input_files, OUTPUT_FILE_NAME, TMP_DIR,event) + if merge_output_result.returncode != 0: + return {"stdout": merge_output_result.stdout, "stderr": merge_output_result.stderr,"command": merge_output_result.args} + if len(merge_edges_input_files) > 0: + merge__edge_output_result = run_merge(merge_edges_input_files, EDGE_OUTPUT_FILE_NAME, TMP_DIR,event) + return {"OutputFile":OUTPUT_FILE_NAME,"EdgeOutputFile":EDGE_OUTPUT_FILE_NAME} \ No newline at end of file diff --git a/benchmarks/600.workflows/6300.sda-workflow/python/neighbors.py b/benchmarks/600.workflows/6300.sda-workflow/python/neighbors.py index c5625bbfc..c51bc10c9 100644 --- a/benchmarks/600.workflows/6300.sda-workflow/python/neighbors.py +++ b/benchmarks/600.workflows/6300.sda-workflow/python/neighbors.py @@ -3,12 +3,15 @@ def handler(event): benchmark_bucket = event["benchmark_bucket"] - input = event["filtered_files"] - config = event["config_file"] - TMP_DIR = os.path.join("/tmp",str(uuid.uuid4())) - os.makedirs(TMP_DIR, exist_ok=True) - input_path = download_shp_file(benchmark_bucket, input, TMP_DIR) - config_path = store_config(config, TMP_DIR) + filter_output_bucket = event["filter_output_bucket"] + shp_file = event["filtered_shp_file"] + TMP_DIR = create_tmp_dir() + input_path = download_shp_file(benchmark_bucket, filter_output_bucket ,shp_file, TMP_DIR) + config_path = load_config(benchmark_bucket,event["input_bucket"], TMP_DIR) # Store workflow data in /tmp due to read only filesystem restriction result = subprocess.run([f"SettlementDelineationNeighbours", "-i", str(input_path), "-c", str(config_path)],capture_output=True,text=True) - return {"stdout": result.stdout, "stderr": result.stderr} \ No newline at end of file + if result.returncode != 0: + event["stdout"] = result.stdout + event["stderr"] = result.stderr + event.pop("filtered_shp_file", None) + return event \ No newline at end of file From b5735eccd59de2a45b12aaecba19711eb3fbf464 Mon Sep 17 00:00:00 2001 From: Lorenz Gruber Date: Thu, 11 Dec 2025 15:42:47 +0100 Subject: [PATCH 36/45] restore old version --- sebs/aws/aws.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sebs/aws/aws.py b/sebs/aws/aws.py index eeda2d7d0..aec1a7ab6 100644 --- a/sebs/aws/aws.py +++ b/sebs/aws/aws.py @@ -463,7 +463,7 @@ def create_workflow(self, code_package: Benchmark, workflow_name: str) -> "SFNWo code_files = list(code_package.get_code_files(include_config=False)) func_names = [os.path.splitext(os.path.basename(p))[0] for p in code_files] funcs = [ - self.update_function(code_package, workflow_name + "___" + fn, code_package.container_deployment,code_package.container_uri if code_package.container_deployment else None) for fn in func_names + self.create_function(code_package, workflow_name + "___" + fn, code_package.container_deployment,code_package.container_uri if code_package.container_deployment else None) for fn in func_names ] # Generate workflow definition.json From 90e65288c41c8813ca2d820610580a3cd98e72b5 Mon Sep 17 00:00:00 2001 From: Lorenz Gruber Date: Thu, 11 Dec 2025 15:43:15 +0100 Subject: [PATCH 37/45] adding initial split step to workflow. enable parallel filter/neighbors --- benchmarks-data | 2 +- .../6300.sda-workflow/definition.json | 40 +++++++++++--- .../6300.sda-workflow/dev/debug.sh | 2 +- .../600.workflows/6300.sda-workflow/input.py | 18 +++---- .../6300.sda-workflow/python/components.py | 3 +- .../6300.sda-workflow/python/filter.py | 10 ++-- .../6300.sda-workflow/python/merge.py | 2 +- .../6300.sda-workflow/python/neighbors.py | 15 ++++-- .../6300.sda-workflow/python/pre_neighbors.py | 54 +++++++++++++++++++ .../6300.sda-workflow/python/split.py | 33 +++++++++++- 10 files changed, 143 insertions(+), 36 deletions(-) create mode 100644 benchmarks/600.workflows/6300.sda-workflow/python/pre_neighbors.py diff --git a/benchmarks-data b/benchmarks-data index 8676c32e4..1eb8679a7 160000 --- a/benchmarks-data +++ b/benchmarks-data @@ -1 +1 @@ -Subproject commit 8676c32e4e0aa339202ee4bcf90d9e0d21dcd9f9 +Subproject commit 1eb8679a726867bfd3ff40874f9e740b3f7a6389 diff --git a/benchmarks/600.workflows/6300.sda-workflow/definition.json b/benchmarks/600.workflows/6300.sda-workflow/definition.json index 8523f4db4..76915b068 100644 --- a/benchmarks/600.workflows/6300.sda-workflow/definition.json +++ b/benchmarks/600.workflows/6300.sda-workflow/definition.json @@ -4,17 +4,41 @@ "clearDB":{ "type": "task", "func_name": "clearDB", - "next": "filter" + "next": "split" }, - "filter": { + "split": { "type": "task", - "func_name": "filter", - "next": "neighbors" + "func_name": "split", + "next": "filter-map" }, - "neighbors": { + "filter-map":{ + "type": "map", + "root": "filter", + "array": "filter_workloads", + "next": "prepare-neighbors", + "states": { + "filter": { + "type": "task", + "func_name": "filter" + } + } + }, + "prepare-neighbors": { "type": "task", - "func_name": "neighbors", - "next": "components" + "func_name": "pre_neighbors", + "next": "neighbors-map" + }, + "neighbors-map":{ + "type": "map", + "root": "neighbors", + "array": "neighbors_workloads", + "next": "components", + "states": { + "neighbors": { + "type": "task", + "func_name": "neighbors" + } + } }, "components": { "type": "task", @@ -24,7 +48,7 @@ "cluster-analyze-map":{ "type": "map", "root": "clustering", - "array": "workloads", + "array": "cluster_workloads", "next": "merge-results", "states": { "clustering": { diff --git a/benchmarks/600.workflows/6300.sda-workflow/dev/debug.sh b/benchmarks/600.workflows/6300.sda-workflow/dev/debug.sh index cd7021e85..a5d14cf63 100755 --- a/benchmarks/600.workflows/6300.sda-workflow/dev/debug.sh +++ b/benchmarks/600.workflows/6300.sda-workflow/dev/debug.sh @@ -1,4 +1,4 @@ -#/bin/bash +#!/bin/bash if [ -n "$(docker ps -aq -f name=^sda-aws-container$)" ]; then echo "Removing existing container sda-aws-container..." docker rm -f sda-aws-container >/dev/null 2>&1 || true diff --git a/benchmarks/600.workflows/6300.sda-workflow/input.py b/benchmarks/600.workflows/6300.sda-workflow/input.py index c1b6ab221..7d246283d 100644 --- a/benchmarks/600.workflows/6300.sda-workflow/input.py +++ b/benchmarks/600.workflows/6300.sda-workflow/input.py @@ -10,14 +10,7 @@ def input_file(size): return size_generators[size] def buckets_count(): - return (1, 4) - -def sda_config(): - SDA_CONFIG_PATH = os.path.join(os.path.dirname(__file__), "sda-config.json") - with open(SDA_CONFIG_PATH, "r") as f: - config = json.load(f) - return config - + return (1, 5) def generate_input(data_dir, size, benchmarks_bucket,input_buckets, output_buckets, upload_func, nosql_func): INPUT_FILE = input_file(size) @@ -28,9 +21,10 @@ def generate_input(data_dir, size, benchmarks_bucket,input_buckets, output_bucke "config_file": CONFIG_FILE, "input_file": INPUT_FILE, "input_bucket": input_buckets[0], - "filter_output_bucket": output_buckets[0], - "cluster_output_bucket": output_buckets[1], - "analysis_output_bucket": output_buckets[2], - "final_output_bucket": output_buckets[3], + "split_output_bucket": output_buckets[0], + "filter_output_bucket": output_buckets[1], + "cluster_output_bucket": output_buckets[2], + "analysis_output_bucket": output_buckets[3], + "final_output_bucket": output_buckets[4], "benchmark_bucket": benchmarks_bucket } \ No newline at end of file diff --git a/benchmarks/600.workflows/6300.sda-workflow/python/components.py b/benchmarks/600.workflows/6300.sda-workflow/python/components.py index e77324723..bfea13997 100644 --- a/benchmarks/600.workflows/6300.sda-workflow/python/components.py +++ b/benchmarks/600.workflows/6300.sda-workflow/python/components.py @@ -32,10 +32,11 @@ def handler(event): return event event.pop("stdout", None) event.pop("stderr", None) + event.pop("neighbors_workloads", None) components_files = sorted(str(p) for p in Path(TMP_DIR).glob(f"{COMPONENT_FILE_PREFIX}*.json")) workloads = build_cluster_workload(components_files) return { - "workloads":[ + "cluster_workloads":[ { "cluster_input_files": workload["cluster_input_files"], "cluster_components": workload["cluster_components"], diff --git a/benchmarks/600.workflows/6300.sda-workflow/python/filter.py b/benchmarks/600.workflows/6300.sda-workflow/python/filter.py index 508ed0066..aff6fa6ac 100644 --- a/benchmarks/600.workflows/6300.sda-workflow/python/filter.py +++ b/benchmarks/600.workflows/6300.sda-workflow/python/filter.py @@ -3,12 +3,10 @@ def handler(event): benchmark_bucket = event["benchmark_bucket"] - input_bucket = event["input_bucket"] - output_bucket = event["filter_output_bucket"] - input = event["input_file"] + input = event["filter_input_file"] TMP_DIR = create_tmp_dir() - input_path = download_file_bucket(benchmark_bucket, input_bucket, input, TMP_DIR) - config_path = load_config(benchmark_bucket,input_bucket, TMP_DIR) + input_path = download_shp_file(benchmark_bucket, event["split_output_bucket"], input, TMP_DIR) + config_path = load_config(benchmark_bucket,event["input_bucket"], TMP_DIR) # Store workflow data in /tmp due to read only filesystem restriction result = subprocess.run([f"SettlementDelineationFilter", "-i", str(input_path), "-c", str(config_path), "-o", TMP_DIR],capture_output=True,text=True) if result.returncode != 0: @@ -16,5 +14,5 @@ def handler(event): event["stderr"] = result.stderr return event output_file = Path(TMP_DIR).glob("*_filtered.shp").__next__() - upload_shp_file(benchmark_bucket, output_bucket, output_file) + upload_shp_file(benchmark_bucket, event["filter_output_bucket"], output_file) return {"filtered_shp_file":Path(output_file).name,**event} \ No newline at end of file diff --git a/benchmarks/600.workflows/6300.sda-workflow/python/merge.py b/benchmarks/600.workflows/6300.sda-workflow/python/merge.py index 16726db12..eb9583a1c 100644 --- a/benchmarks/600.workflows/6300.sda-workflow/python/merge.py +++ b/benchmarks/600.workflows/6300.sda-workflow/python/merge.py @@ -15,7 +15,7 @@ def load_input(event,directory): analysis_output_bucket = event["analysis_output_bucket"] merge_input_files = [] merge_edges_input_files = [] - for workload in event["workloads"]: + for workload in event["cluster_workloads"]: for file in workload["analysis_output_files"]: shp_file = download_shp_file(benchmark_bucket,analysis_output_bucket,file,directory) if "_edges" in shp_file.stem: diff --git a/benchmarks/600.workflows/6300.sda-workflow/python/neighbors.py b/benchmarks/600.workflows/6300.sda-workflow/python/neighbors.py index c51bc10c9..a204a4e31 100644 --- a/benchmarks/600.workflows/6300.sda-workflow/python/neighbors.py +++ b/benchmarks/600.workflows/6300.sda-workflow/python/neighbors.py @@ -4,14 +4,19 @@ def handler(event): benchmark_bucket = event["benchmark_bucket"] filter_output_bucket = event["filter_output_bucket"] - shp_file = event["filtered_shp_file"] TMP_DIR = create_tmp_dir() - input_path = download_shp_file(benchmark_bucket, filter_output_bucket ,shp_file, TMP_DIR) + input_path = download_shp_file(benchmark_bucket, filter_output_bucket ,event["filtered_shp_file"], TMP_DIR) + adjacent_input_paths = [download_shp_file(benchmark_bucket, filter_output_bucket ,f, TMP_DIR) for f in event.get("adjacent_files",[])] config_path = load_config(benchmark_bucket,event["input_bucket"], TMP_DIR) # Store workflow data in /tmp due to read only filesystem restriction - result = subprocess.run([f"SettlementDelineationNeighbours", "-i", str(input_path), "-c", str(config_path)],capture_output=True,text=True) + command = ["SettlementDelineationNeighbours", "-i", str(input_path), "-c", str(config_path)] + if len(adjacent_input_paths) > 0: + command.append("-a") + command.extend([str(p) for p in adjacent_input_paths]) + result = subprocess.run(command,capture_output=True,text=True) if result.returncode != 0: event["stdout"] = result.stdout event["stderr"] = result.stderr - event.pop("filtered_shp_file", None) - return event \ No newline at end of file + event["command"] = " ".join(command) + return event + return {} \ No newline at end of file diff --git a/benchmarks/600.workflows/6300.sda-workflow/python/pre_neighbors.py b/benchmarks/600.workflows/6300.sda-workflow/python/pre_neighbors.py new file mode 100644 index 000000000..4c74a20d3 --- /dev/null +++ b/benchmarks/600.workflows/6300.sda-workflow/python/pre_neighbors.py @@ -0,0 +1,54 @@ +import re + +class Coordinate: + def __init__(self,x:int,y:int): + self.x = x + self.y = y + + def chebyshev_distance(self, other) -> int: + return max(abs(self.x - other.x), abs(self.y - other.y)) + + @staticmethod + def from_filename(filename: str): + match = re.search(r'.+_(\d+)_(\d+)_filtered.shp', filename) + if match: + filename = match.group(1) + coord = Coordinate(int(match.group(1)), int(match.group(2))) + return coord + else: + raise ValueError("Filename does not match expected pattern") + +class SpatialFile: + def __init__(self, filename: str): + self.filename = filename + self.coordinate = Coordinate.from_filename(filename) + self.neighbors = [] + + def add_neighbor(self, neighbor_filename: str): + self.neighbors.append(neighbor_filename) + + def __eq__(self, other) -> bool: + if not isinstance(other, SpatialFile): + return False + return self.filename == other.filename + + def is_adjacent(self, other) -> bool: + return self.coordinate.chebyshev_distance(other.coordinate) <= 1 + +def handler(event): + files = [SpatialFile(file_workload["filtered_shp_file"]) for file_workload in event["filter_workloads"]] + for file in files: + for other_file in files: + if file != other_file and file.is_adjacent(other_file): + file.add_neighbor(other_file.filename) + event.pop("filter_workloads", None) + return { + "neighbors_workloads":[ + { + "filtered_shp_file": file.filename, + "adjacent_files": file.neighbors, + **event + } for file in files + ], + **event + } \ No newline at end of file diff --git a/benchmarks/600.workflows/6300.sda-workflow/python/split.py b/benchmarks/600.workflows/6300.sda-workflow/python/split.py index f7d96399a..7b1c166fa 100644 --- a/benchmarks/600.workflows/6300.sda-workflow/python/split.py +++ b/benchmarks/600.workflows/6300.sda-workflow/python/split.py @@ -1,4 +1,35 @@ import subprocess +import json +from .SDAHelper import * + +def get_splits(config_path:Path)->int: + with open(config_path,"r") as f: + config = json.load(f) + return int(config.get("splits",0)) def handler(event): - pass \ No newline at end of file + benchmark_bucket = event["benchmark_bucket"] + INPUT_DIR = create_tmp_dir() + OUTPUT_DIR = create_tmp_dir() + input_path = download_file_bucket(benchmark_bucket, event["input_bucket"], event["input_file"], INPUT_DIR) + splits = get_splits(load_config(benchmark_bucket,event["input_bucket"], INPUT_DIR)) + command = ["FishnetShapefileSplitter","-i", str(input_path), "-o", str(OUTPUT_DIR),"-s", str(splits)] + result = subprocess.run(command,capture_output=True,text=True,cwd=INPUT_DIR) + if result.returncode != 0: + event["stdout"] = result.stdout + event["stderr"] = result.stderr + event["command"] = " ".join(command) + return event + split_output_files = [] + for file in Path(OUTPUT_DIR).glob("*.shp"): + upload_shp_file(benchmark_bucket, event["split_output_bucket"], file) + split_output_files.append(Path(file).name) + return { + "filter_workloads":[ + { + "filter_input_file": split_file, + **event + }for split_file in split_output_files + ], + **event + } \ No newline at end of file From 5ca3c4686059b2ed92cfe7952b0ecf1bf846506a Mon Sep 17 00:00:00 2001 From: Lorenz Gruber Date: Thu, 11 Dec 2025 17:31:51 +0100 Subject: [PATCH 38/45] adding different experiment sizes to sda workflow --- .../6300.sda-workflow/.gitignore | 3 +- .../600.workflows/6300.sda-workflow/input.py | 125 ++++++++++++++++-- .../6300.sda-workflow/python/SDAHelper.py | 5 +- .../6300.sda-workflow/python/analysis.py | 2 +- .../6300.sda-workflow/python/clearDB.py | 2 +- .../6300.sda-workflow/python/clustering.py | 2 +- .../6300.sda-workflow/python/components.py | 4 +- .../6300.sda-workflow/python/filter.py | 2 +- .../6300.sda-workflow/python/neighbors.py | 2 +- .../6300.sda-workflow/python/split.py | 2 +- 10 files changed, 126 insertions(+), 23 deletions(-) diff --git a/benchmarks/600.workflows/6300.sda-workflow/.gitignore b/benchmarks/600.workflows/6300.sda-workflow/.gitignore index 253b6868b..98a3081b2 100644 --- a/benchmarks/600.workflows/6300.sda-workflow/.gitignore +++ b/benchmarks/600.workflows/6300.sda-workflow/.gitignore @@ -1 +1,2 @@ -sda-config.json \ No newline at end of file +cfg/ +dev/ \ No newline at end of file diff --git a/benchmarks/600.workflows/6300.sda-workflow/input.py b/benchmarks/600.workflows/6300.sda-workflow/input.py index 7d246283d..25e8b8064 100644 --- a/benchmarks/600.workflows/6300.sda-workflow/input.py +++ b/benchmarks/600.workflows/6300.sda-workflow/input.py @@ -1,25 +1,130 @@ import json import os -def input_file(size): - size_generators = { +class MemgraphConfig: + def __init__(self, host:str, port:int, username:str, password:str): + self.host = host + self.port = port + self.username = username + self.password = password + if self.host == "localhost": + raise ValueError("Memgraph database for SDA workflow has to be reachable from the internet. Set the following environment variables via the command line or the SeBS .env file: \n\tMEMGRAPH_HOST\n\tMEMGRAPH_PORT\n\tMEMGRAPH_USER\n\tMEMGRAPH_PASSWORD") + + @staticmethod + def from_env(): + with open(".env", "r") as f: + for line in f: + key, value = line.strip().split("=", 1) + os.environ[key] = value + return MemgraphConfig( + host=os.getenv("MEMGRAPH_HOST", "localhost"), + port=int(os.getenv("MEMGRAPH_PORT", 7687)), + username=os.getenv("MEMGRAPH_USER", ""), + password=os.getenv("MEMGRAPH_PASSWORD", "") + ) + +class SDAConfig: + def __init__(self, memgraph_config: MemgraphConfig, splits:int=0, required_area:float=0.0, + max_edge_distance:float=0.0, max_neighbours: int = 5, clustering_distance: float = 500.0,merge_workers:int=2, visualize_edges:bool= True): + self.memgraph_config = memgraph_config + self.splits = splits + self.required_area = required_area + self.max_edge_distance = max_edge_distance + self.max_neighbours = max_neighbours + self.clustering_distance = clustering_distance + self.merge_workers = merge_workers + self.visualize_edges = visualize_edges + + def get(self): + return { + "binary-filters": [ + { + "name": "InsidePolygonFilter" + } + ], + "centrality-measures": [ + { + "name": "DegreeCentrality" + }, + { + "name": "MeanLocalSignificance" + }, + { + "name": "SmallerNeighboursRatio" + } + ], + "contraction-predicates": [ + { + "distance": self.clustering_distance, + "name": "DistanceBiPredicate" + } + ], + "maxDistanceMeters": self.max_edge_distance, + "maxNeighbours": self.max_neighbours, + "memgraph-host": self.memgraph_config.host, + "memgraph-port": self.memgraph_config.port, + "memgraph-user": self.memgraph_config.username, + "memgraph-password": self.memgraph_config.password, + "merge-workers": self.merge_workers, + "neighbouring-predicates": [], + "splits": self.splits, + "unary-filters": [ + { + "name": "ApproxAreaFilter", + "requiredArea": self.required_area + } + ], + "visualize-edges": self.visualize_edges + } + + @staticmethod + def from_benchmark_size(size:str): + memgraph_config:MemgraphConfig = MemgraphConfig.from_env() + configs = { + "test": SDAConfig(memgraph_config, splits=0, required_area=5000.0, max_edge_distance=3000.0, max_neighbours=5, clustering_distance=500.0, merge_workers=1, visualize_edges=False), + "small": SDAConfig(memgraph_config, splits=1, required_area=500.0, max_edge_distance=1000.0, max_neighbours=5, clustering_distance=500.0, merge_workers=2, visualize_edges=True), + "large": SDAConfig(memgraph_config, splits=2, required_area=500.0, max_edge_distance=2000.0, max_neighbours=5, clustering_distance=200.0, merge_workers=2, visualize_edges=True), + } + return configs[size] + +def get_config_file_name(size): + return f"sda-config-{size}.json" + +def create_config_file(size): + config = SDAConfig.from_benchmark_size(size) + cfg_dir = os.path.join(os.path.dirname(__file__), "cfg") + os.makedirs(cfg_dir, exist_ok=True) + config_file_path = os.path.join(cfg_dir, get_config_file_name(size)) + with open(config_file_path, "w") as f: + json.dump(config.get(), f, indent=4) + return config_file_path + +def get_input_file(size): + input_files = { "test" : "Corvara_IT.tiff", "small": "Corvara_IT.tiff", - "large": "Corvara_IT.tiff", + "large": "Wuerzburg_DE.tiff", } - return size_generators[size] + return input_files[size] def buckets_count(): return (1, 5) +def upload_all_data(upload_func,data_dir): + sizes=["test", "small", "large"] + for size in sizes: + input_file = get_input_file(size) + config_path = create_config_file(size) + upload_func(0, input_file, os.path.join(data_dir, input_file)) + upload_func(0, get_config_file_name(size), config_path) + + + def generate_input(data_dir, size, benchmarks_bucket,input_buckets, output_buckets, upload_func, nosql_func): - INPUT_FILE = input_file(size) - upload_func(0, INPUT_FILE, os.path.join(data_dir, INPUT_FILE)) - CONFIG_FILE = "sda-config.json" - upload_func(0, CONFIG_FILE, os.path.join(os.path.dirname(__file__), CONFIG_FILE)) + upload_all_data(upload_func,data_dir) return { - "config_file": CONFIG_FILE, - "input_file": INPUT_FILE, + "config_file": get_config_file_name(size), + "input_file": get_input_file(size), "input_bucket": input_buckets[0], "split_output_bucket": output_buckets[0], "filter_output_bucket": output_buckets[1], diff --git a/benchmarks/600.workflows/6300.sda-workflow/python/SDAHelper.py b/benchmarks/600.workflows/6300.sda-workflow/python/SDAHelper.py index 5c6ec27a9..ab7a71e59 100644 --- a/benchmarks/600.workflows/6300.sda-workflow/python/SDAHelper.py +++ b/benchmarks/600.workflows/6300.sda-workflow/python/SDAHelper.py @@ -6,7 +6,6 @@ storage_client = storage.storage.get_instance() SHP_SUFFIX = [".shp", ".shx", ".dbf", ".prj"] -CONFIG_FILE_NAME = "sda-config.json" def download_file(benchmark_bucket, path_in_bucket, dest_dir): path = Path(dest_dir) / Path(path_in_bucket).name @@ -22,8 +21,8 @@ def download_shp_file(benchmark_bucket, bucket ,shp_file, dest_dir): download_file_bucket(benchmark_bucket, bucket, f.name, dest_dir) return download_file_bucket(benchmark_bucket, bucket, shp_file, dest_dir) -def load_config(benchmark_bucket, input_bucket,directory): - return download_file_bucket(benchmark_bucket, input_bucket, CONFIG_FILE_NAME, directory) +def load_config(event,directory): + return download_file_bucket(event["benchmark_bucket"], event["input_bucket"], event["config_file"], directory) def upload_shp_file(benchmark_bucket, bucket, shp_basename): shp_dir = Path(shp_basename).parent diff --git a/benchmarks/600.workflows/6300.sda-workflow/python/analysis.py b/benchmarks/600.workflows/6300.sda-workflow/python/analysis.py index 382edc9b9..11128472b 100644 --- a/benchmarks/600.workflows/6300.sda-workflow/python/analysis.py +++ b/benchmarks/600.workflows/6300.sda-workflow/python/analysis.py @@ -6,7 +6,7 @@ def handler(event): cluster_output_bucket = event["cluster_output_bucket"] TMP_DIR = create_tmp_dir() analysis_input_file = download_shp_file(benchmark_bucket,cluster_output_bucket,event["cluster_output_file"],TMP_DIR) - config_file = load_config(benchmark_bucket,event["input_bucket"], TMP_DIR) + config_file = load_config(event, TMP_DIR) OUTPUT_STEM = "Analysis_"+Path(analysis_input_file).stem command = ["SettlementDelineationAnalysis", "-i", str(analysis_input_file), "-c", str(config_file), "--outputStem", OUTPUT_STEM] result = subprocess.run(command,capture_output=True,text=True, cwd=TMP_DIR) diff --git a/benchmarks/600.workflows/6300.sda-workflow/python/clearDB.py b/benchmarks/600.workflows/6300.sda-workflow/python/clearDB.py index 4096f3272..e0b57702a 100644 --- a/benchmarks/600.workflows/6300.sda-workflow/python/clearDB.py +++ b/benchmarks/600.workflows/6300.sda-workflow/python/clearDB.py @@ -3,7 +3,7 @@ def handler(event): TMP_DIR = create_tmp_dir() - config = load_config(event["benchmark_bucket"],event["input_bucket"], TMP_DIR) + config = load_config(event, TMP_DIR) result = subprocess.run([f"AfricapolisClearDatabase", "-c", str(config)],capture_output=True,text=True) if result.returncode != 0: event["stdout"] = result.stdout diff --git a/benchmarks/600.workflows/6300.sda-workflow/python/clustering.py b/benchmarks/600.workflows/6300.sda-workflow/python/clustering.py index 06b114636..c7907ce9a 100644 --- a/benchmarks/600.workflows/6300.sda-workflow/python/clustering.py +++ b/benchmarks/600.workflows/6300.sda-workflow/python/clustering.py @@ -7,7 +7,7 @@ def handler(event): filter_output_bucket = event["filter_output_bucket"] TMP_DIR = create_tmp_dir() input_files = [download_shp_file(benchmark_bucket, filter_output_bucket ,shp_file, TMP_DIR ) for shp_file in event["cluster_input_files"]] - config = load_config(benchmark_bucket,event["input_bucket"], TMP_DIR) + config = load_config(event, TMP_DIR) components = event["cluster_components"] OUTPUT_STEM = "Cluster"+str(components[0]) # Store workflow data in /tmp due to read only filesystem restriction diff --git a/benchmarks/600.workflows/6300.sda-workflow/python/components.py b/benchmarks/600.workflows/6300.sda-workflow/python/components.py index bfea13997..48f87e3f2 100644 --- a/benchmarks/600.workflows/6300.sda-workflow/python/components.py +++ b/benchmarks/600.workflows/6300.sda-workflow/python/components.py @@ -3,7 +3,6 @@ import json def build_cluster_workload(components_files): - workloadID = 0 workloads = [] for comp_file in components_files: with open(comp_file, 'r') as f: @@ -13,12 +12,11 @@ def build_cluster_workload(components_files): "cluster_components": components_data["components"] } workloads.append(workload) - workloadID += 1 return workloads def handler(event): TMP_DIR = create_tmp_dir() - config_file = load_config(event["benchmark_bucket"],event["input_bucket"], TMP_DIR) + config_file = load_config(event, TMP_DIR) COMPONENT_FILE_PREFIX = "components" result = subprocess.run( ["AfricapolisGraphComponents", "-c", str(config_file), "-o", COMPONENT_FILE_PREFIX], diff --git a/benchmarks/600.workflows/6300.sda-workflow/python/filter.py b/benchmarks/600.workflows/6300.sda-workflow/python/filter.py index aff6fa6ac..0f547d443 100644 --- a/benchmarks/600.workflows/6300.sda-workflow/python/filter.py +++ b/benchmarks/600.workflows/6300.sda-workflow/python/filter.py @@ -6,7 +6,7 @@ def handler(event): input = event["filter_input_file"] TMP_DIR = create_tmp_dir() input_path = download_shp_file(benchmark_bucket, event["split_output_bucket"], input, TMP_DIR) - config_path = load_config(benchmark_bucket,event["input_bucket"], TMP_DIR) + config_path = load_config(event, TMP_DIR) # Store workflow data in /tmp due to read only filesystem restriction result = subprocess.run([f"SettlementDelineationFilter", "-i", str(input_path), "-c", str(config_path), "-o", TMP_DIR],capture_output=True,text=True) if result.returncode != 0: diff --git a/benchmarks/600.workflows/6300.sda-workflow/python/neighbors.py b/benchmarks/600.workflows/6300.sda-workflow/python/neighbors.py index a204a4e31..96a375560 100644 --- a/benchmarks/600.workflows/6300.sda-workflow/python/neighbors.py +++ b/benchmarks/600.workflows/6300.sda-workflow/python/neighbors.py @@ -7,7 +7,7 @@ def handler(event): TMP_DIR = create_tmp_dir() input_path = download_shp_file(benchmark_bucket, filter_output_bucket ,event["filtered_shp_file"], TMP_DIR) adjacent_input_paths = [download_shp_file(benchmark_bucket, filter_output_bucket ,f, TMP_DIR) for f in event.get("adjacent_files",[])] - config_path = load_config(benchmark_bucket,event["input_bucket"], TMP_DIR) + config_path = load_config(event, TMP_DIR) # Store workflow data in /tmp due to read only filesystem restriction command = ["SettlementDelineationNeighbours", "-i", str(input_path), "-c", str(config_path)] if len(adjacent_input_paths) > 0: diff --git a/benchmarks/600.workflows/6300.sda-workflow/python/split.py b/benchmarks/600.workflows/6300.sda-workflow/python/split.py index 7b1c166fa..dbb85bd57 100644 --- a/benchmarks/600.workflows/6300.sda-workflow/python/split.py +++ b/benchmarks/600.workflows/6300.sda-workflow/python/split.py @@ -12,7 +12,7 @@ def handler(event): INPUT_DIR = create_tmp_dir() OUTPUT_DIR = create_tmp_dir() input_path = download_file_bucket(benchmark_bucket, event["input_bucket"], event["input_file"], INPUT_DIR) - splits = get_splits(load_config(benchmark_bucket,event["input_bucket"], INPUT_DIR)) + splits = get_splits(load_config(event, INPUT_DIR)) command = ["FishnetShapefileSplitter","-i", str(input_path), "-o", str(OUTPUT_DIR),"-s", str(splits)] result = subprocess.run(command,capture_output=True,text=True,cwd=INPUT_DIR) if result.returncode != 0: From 4a3ec64c1548fba8c8654d3274cb6dec300bef7a Mon Sep 17 00:00:00 2001 From: Lorenz Gruber Date: Thu, 11 Dec 2025 17:56:30 +0100 Subject: [PATCH 39/45] changing timeout --- benchmarks/600.workflows/6300.sda-workflow/config.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmarks/600.workflows/6300.sda-workflow/config.json b/benchmarks/600.workflows/6300.sda-workflow/config.json index 12332999d..f7e9577d9 100644 --- a/benchmarks/600.workflows/6300.sda-workflow/config.json +++ b/benchmarks/600.workflows/6300.sda-workflow/config.json @@ -1,5 +1,5 @@ { - "timeout": 120, + "timeout": 500, "memory": 128, "languages": ["python"], "modules": ["storage"], From 9f6979b2ad0c0a19b302f0120c2e7ea1f59089e4 Mon Sep 17 00:00:00 2001 From: Lorenz Gruber Date: Thu, 11 Dec 2025 17:57:12 +0100 Subject: [PATCH 40/45] adding additional launch option for large benchmark --- .vscode/launch.json | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index 028f769e3..fdfd07614 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -31,7 +31,7 @@ "envFile": "${workspaceFolder}/.env" }, { - "name": "SeBS SDA Workflow", + "name": "SeBS SDA Workflow Test", "type": "python", "request": "launch", "program": "${workspaceFolder}/sebs.py", @@ -54,6 +54,31 @@ "justMyCode": true, "pythonPath": "${workspaceFolder}/python-venv/bin/python", "envFile": "${workspaceFolder}/.env" + }, + { + "name": "SeBS SDA Workflow", + "type": "python", + "request": "launch", + "program": "${workspaceFolder}/sebs.py", + "cwd": "${workspaceFolder}", + "console": "integratedTerminal", + "args": [ + "benchmark", + "workflow", + "6300.sda-workflow", + "large", + "--config", + "config/config.json", + "--deployment", + "aws", + "--trigger", + "library", + "--repetitions", + "1" + ], + "justMyCode": true, + "pythonPath": "${workspaceFolder}/python-venv/bin/python", + "envFile": "${workspaceFolder}/.env" } ] } \ No newline at end of file From af79eb435ce8a5c900bce722e3066565fbb4c968 Mon Sep 17 00:00:00 2001 From: Lorenz Gruber Date: Thu, 11 Dec 2025 17:57:20 +0100 Subject: [PATCH 41/45] updating data --- benchmarks-data | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmarks-data b/benchmarks-data index 1eb8679a7..0c2cf44f2 160000 --- a/benchmarks-data +++ b/benchmarks-data @@ -1 +1 @@ -Subproject commit 1eb8679a726867bfd3ff40874f9e740b3f7a6389 +Subproject commit 0c2cf44f27d214bb38a5bf69357f9dc3d18c4bb3 From ea636c60ae1a78d178c6f9f2ffce12c81bb3931c Mon Sep 17 00:00:00 2001 From: Lorenz Gruber Date: Sun, 18 Jan 2026 17:45:25 +0100 Subject: [PATCH 42/45] updating config --- benchmarks/600.workflows/6300.sda-workflow/input.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmarks/600.workflows/6300.sda-workflow/input.py b/benchmarks/600.workflows/6300.sda-workflow/input.py index 25e8b8064..9057a34dd 100644 --- a/benchmarks/600.workflows/6300.sda-workflow/input.py +++ b/benchmarks/600.workflows/6300.sda-workflow/input.py @@ -83,7 +83,7 @@ def from_benchmark_size(size:str): configs = { "test": SDAConfig(memgraph_config, splits=0, required_area=5000.0, max_edge_distance=3000.0, max_neighbours=5, clustering_distance=500.0, merge_workers=1, visualize_edges=False), "small": SDAConfig(memgraph_config, splits=1, required_area=500.0, max_edge_distance=1000.0, max_neighbours=5, clustering_distance=500.0, merge_workers=2, visualize_edges=True), - "large": SDAConfig(memgraph_config, splits=2, required_area=500.0, max_edge_distance=2000.0, max_neighbours=5, clustering_distance=200.0, merge_workers=2, visualize_edges=True), + "large": SDAConfig(memgraph_config, splits=2, required_area=500.0, max_edge_distance=500.0, max_neighbours=5, clustering_distance=200.0, merge_workers=2, visualize_edges=True), } return configs[size] From 3a17d79c1d699d117760c77a085f052736def2c9 Mon Sep 17 00:00:00 2001 From: Lorenz Gruber Date: Thu, 12 Mar 2026 11:00:13 +0100 Subject: [PATCH 43/45] updating ignore to exclude vs code files --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 0efc0e4c0..73655cf7e 100644 --- a/.gitignore +++ b/.gitignore @@ -189,6 +189,9 @@ cache .idea *.iml +# VS-Code files +.vscode/ + # Local config config/config.json config/redis From eb805d042de4c6b0697ba3fc053aff2b8aeb3343 Mon Sep 17 00:00:00 2001 From: Lorenz Gruber Date: Thu, 12 Mar 2026 11:01:06 +0100 Subject: [PATCH 44/45] removing development artifacts --- .vscode/launch.json | 84 ------------------ .vscode/settings.json | 3 - .../6300.sda-workflow/dev/Dockerfile | 23 ----- .../6300.sda-workflow/dev/README.md | 31 ------- .../dev/data/Corvara_IT.tiff | Bin 1805008 -> 0 bytes .../dev/data/sda-workflow-local.json | 41 --------- .../6300.sda-workflow/dev/debug.sh | 10 --- 7 files changed, 192 deletions(-) delete mode 100644 .vscode/launch.json delete mode 100644 .vscode/settings.json delete mode 100644 benchmarks/600.workflows/6300.sda-workflow/dev/Dockerfile delete mode 100644 benchmarks/600.workflows/6300.sda-workflow/dev/README.md delete mode 100644 benchmarks/600.workflows/6300.sda-workflow/dev/data/Corvara_IT.tiff delete mode 100644 benchmarks/600.workflows/6300.sda-workflow/dev/data/sda-workflow-local.json delete mode 100755 benchmarks/600.workflows/6300.sda-workflow/dev/debug.sh diff --git a/.vscode/launch.json b/.vscode/launch.json deleted file mode 100644 index fdfd07614..000000000 --- a/.vscode/launch.json +++ /dev/null @@ -1,84 +0,0 @@ -{ - // Use IntelliSense to learn about possible attributes. - // Hover to view descriptions of existing attributes. - // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 - "version": "0.2.0", - "configurations": [ - { - "name": "SeBS Benchmark Workflow 610", - "type": "python", - "request": "launch", - "program": "${workspaceFolder}/sebs.py", - "cwd": "${workspaceFolder}", - "console": "integratedTerminal", - "args": [ - "benchmark", - "workflow", - "610.gen", - "test", - "--config", - "config/config.json", - "--deployment", - "aws", - "--container-deployment", - "--trigger", - "library", - "--repetitions", - "1" - ], - "justMyCode": true, - "pythonPath": "${workspaceFolder}/python-venv/bin/python", - "envFile": "${workspaceFolder}/.env" - }, - { - "name": "SeBS SDA Workflow Test", - "type": "python", - "request": "launch", - "program": "${workspaceFolder}/sebs.py", - "cwd": "${workspaceFolder}", - "console": "integratedTerminal", - "args": [ - "benchmark", - "workflow", - "6300.sda-workflow", - "test", - "--config", - "config/config.json", - "--deployment", - "aws", - "--trigger", - "library", - "--repetitions", - "1" - ], - "justMyCode": true, - "pythonPath": "${workspaceFolder}/python-venv/bin/python", - "envFile": "${workspaceFolder}/.env" - }, - { - "name": "SeBS SDA Workflow", - "type": "python", - "request": "launch", - "program": "${workspaceFolder}/sebs.py", - "cwd": "${workspaceFolder}", - "console": "integratedTerminal", - "args": [ - "benchmark", - "workflow", - "6300.sda-workflow", - "large", - "--config", - "config/config.json", - "--deployment", - "aws", - "--trigger", - "library", - "--repetitions", - "1" - ], - "justMyCode": true, - "pythonPath": "${workspaceFolder}/python-venv/bin/python", - "envFile": "${workspaceFolder}/.env" - } - ] -} \ No newline at end of file diff --git a/.vscode/settings.json b/.vscode/settings.json deleted file mode 100644 index 7e68766ae..000000000 --- a/.vscode/settings.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "python-envs.pythonProjects": [] -} \ No newline at end of file diff --git a/benchmarks/600.workflows/6300.sda-workflow/dev/Dockerfile b/benchmarks/600.workflows/6300.sda-workflow/dev/Dockerfile deleted file mode 100644 index 5f6289594..000000000 --- a/benchmarks/600.workflows/6300.sda-workflow/dev/Dockerfile +++ /dev/null @@ -1,23 +0,0 @@ -# Define custom base image -ARG BASE_IMAGE="python:3.12" - -FROM ${BASE_IMAGE} AS build-image -RUN apt-get update && apt-get install -y python3-pip - -# Include global arg in this stage of the build -ARG LAMBDA_DIR="/lambda" - -# Copy function code -RUN mkdir -p ${LAMBDA_DIR} -RUN mkdir -p ${LAMBDA_DIR}/function -# Install the function's dependencies -RUN pip install \ - --target ${LAMBDA_DIR} \ - awslambdaric - -COPY . ${LAMBDA_DIR}/function -WORKDIR ${LAMBDA_DIR} -# Set runtime interface client as default command for the container runtime -ENTRYPOINT [ "/usr/bin/python", "-m", "awslambdaric" ] -# Pass the name of the function handler as an argument to the runtime -CMD [ "function/handler.handler" ] \ No newline at end of file diff --git a/benchmarks/600.workflows/6300.sda-workflow/dev/README.md b/benchmarks/600.workflows/6300.sda-workflow/dev/README.md deleted file mode 100644 index b2b005e9b..000000000 --- a/benchmarks/600.workflows/6300.sda-workflow/dev/README.md +++ /dev/null @@ -1,31 +0,0 @@ -# Docker Commands: -### Build Image -``` -docker build \ ---build-arg BASE_IMAGE=logru/sda:latest \ --t logru/sda-sebs-aws \ --f benchmarks/600.workflows/6300.sda-workflow/dev/Dockerfile \ -6300.sda-workflow_code/python/3.8/x64/package/ -``` -### Test Run -[Custom Image Tutorial](https://docs.aws.amazon.com/lambda/latest/dg/python-image.html#python-image-clients) -#### Start Container -``` -docker run \ ---platform linux/amd64 -d \ --v ~/.aws-lambda-rie:/aws-lambda \ --p 9000:8080 \ --e AWS_LAMBDA_FUNCTION_NAME=SDA___filter \ ---entrypoint /aws-lambda/aws-lambda-rie \ ---name sda-aws-container \ -logru/sda-sebs-aws:latest \ -/usr/bin/python -m awslambdaric function/handler.handler -``` -#### Debug Container -``` -docker exec -it sda-aws-container /bin/bash -``` -#### Trigger Requests -``` -curl "http://localhost:9000/2015-03-31/functions/function/invocations" -d '{"payload":{},"request_id":"1"}' -``` diff --git a/benchmarks/600.workflows/6300.sda-workflow/dev/data/Corvara_IT.tiff b/benchmarks/600.workflows/6300.sda-workflow/dev/data/Corvara_IT.tiff deleted file mode 100644 index efe7364bfd0935ee1842bae62d2b823dbb996c94..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1805008 zcmeI*eaL;=K_B>iUy|G;H_5%p&CN~XOoNloXb~&uANeD;vCa5`PA5#2af-Bt#1g@@ z62nwz1~M~>BGL*IW-_2d5g7|3j6yBNDd->h!-6fP6_g==C^#amP+>BlAZFIfUi;;| zJ$s+C_c{CJbM&6G_u6Z%z1C-+XMdk{p6B`H+rI5DR4-Ij^>S4`^IY{z*!s(F2%Gix zZ#-93&(_~fvij54SJnUUBQI9rrFtcd=l}e%=c@OG?GK0TU;MEbt8kj?N5b|m{n)c< z^q&jcpZ&RKtG^q{e=;@-k3ani|F?aCpAJ|4^Y5>!&wW!>{nXzKSO4!<)sKHZygPn5 zyd(ZiRek0^RMii^_)PWbuYRWb!Eb)1`hkxKp##+3M?FeXjbN zuYa!ks*gNZz5kQp9r+W_RbTP*&sDE|;koLS-+iun>Cc|4UVQKK)e9eZzIy(i0hQy6SiSo7Yue{BK`Z{r3O# zy6QK7@paX|{+F+-e&gT0uKM*iyuSLv`(I!E+Bdzv`saW1_0>Q7`>(J5$>(2R{o*gb zzWRlK^ZM!^|A*ICKlkDrs(WCyo8M4<{$p>bKKI!-R6q4IZ>WCqSKd(l_;0IZ-Hh3W@>_J!&bzxqP;@h`qmeea*VP<_v< zZ>+xi>)%*?=SSXHedLpGtiIzX-dKI>&%d$y@E6`#edu@JSbgx%-dKI$y>F_%;RA1~ zzV5r;RDI2--c)_nKYUa5{$F@g^}c`crs^wx|4r3v&%L>N<$Z6iUi!v2S1a!ntTlJYwzODM7RdF^@Cq{TlE9K`?l(}|Nh<8C;seh z)yLoaQuVzbc&YlH?|P~F?oYi`edj-Xsrtw-yi|S1zj&$o*57}r`tWlvS08%c%hdE-IH{?*IX`~UsR)%#v}d-WB6{_WLkf937fD}U?l z)k~jwd-dY~`u6IDfBN?7`G5QNs`{h1SAY8AJE}ka>UUIM`sR03fAq0;RDbx{cT|7y zGw-N=?^oVY{myT_qx#~P-ckMb%db?w`L(Z9|N7fssea=VuT;POqpwt7_}N#gU;EWp zs(=2)SE_&ZC$Cihx-s~3O!-PPxR_np-X|KGc-=imOG>T`eg&Z_#0@2URu+uu|D)O%m8e)0paRzLn- zuU3EjiT6}L@~Ky=&;G+#tIzzxtJM$xi&v{p|Ng7h4?g#<>IdHUuIdxt_^#^Xf8$-% z_x`o{^)1lQ+?;ZdRO(4fB&xPJ6?FL`omv+PxY;T{Vuzot@?v6zNh-VKY35}f&cZj>UUm!Z}r8me{b~-|Ma!$>;CO))z|#d zYt?UmhH#lkAL3}ee6@;|AU|Y{=fTozVa`9><2#jLqGKWfBXAC_R+Zi!M_-L`Acv5 z;9u-_UseC3u7ks&glseb>ZXI}WF9}7pT*Qfn|@SnW#gK-jHQcX0 z`RV`pKl-^(#r{6@zkla+|Q`S0vFQd*{}v3985v`gt%caxeAkIx*qscq@y z>;PsskWW8D$GRjyfB=Dw3&bU4N|BO`XXYDgwN*;irM79Obd#F570>imieu9{h*K|V zw;UYsI<|hl%fZ9!H?v28009EK7wF{A<)v7sVnmC(`MWzQ=C0t5&Um`vd5byzp)!fE4fc!Z4-pO?mRru*DO=qvP+x=T-&I_e#Z?auVv ztyiC=TV$I60RjXFOd;@e3e_RhR!mWsb}wdFu6AmS=}0&b`l$OVFQ;msE(_hQ(ymUt{E2M=$w$gw_x1dd{E2Tk9s@b`8n^5J@=Ld) zGuyGrnL{IspPc^4{!}MGfB=D=2*k{~^a#iEBUcLBDfUvMbl^neZiB;I0RjXFEJh#)TPNhgW*A&7J|1JPd{CYWOPlbT_jCKE_+-n2 zj^b&Li&wAb19r5W5%xy!xe7N2XFR^L+63e@t*(N}M009Ck z7s$~TpTncc(zMZ=SvuA$2|IPGxY6vDpv=eW=fj5f2@oJafWWE+dZAeYF^6GyUL_2> zWu~*1PBmVy+8`SGK*#>W?%d0Za1Y(m8Ma1C~f59XAbtO+9N=K z009DP6NmxVjJa&*ABC3Ul!4_?DGaZ9otlnzO6$!N^^YqlAwYlt0Rn3h$l;cfFQ;c6 zq4_7EVJ7`d4STWfsTp?q=pF7SaPQEios;Ld)W9(U1PBlya9@Fxh&ldZI+l~Cjb^t# ze)exm^6^gd)J@!PIxHP~B|v}x0RkHkh)c>ajTCh;1@n~RYki~?TnxhFgRw91xq&x` zDG(q)fWVPJ&bOR>Iqy>Dr8G_ZxkbBq9Fs?V)&;q{b4QgEAV7csfz=As5f@&<6q@I< z7muc?<(Ie^ZHfj=c~s>A{q?HUBkPsfAwYlt0Rndt2pJd7l#(uQ=JSTaTo!lZPTcPl z1PBn=p+HQ|_!G|fTt8}sZaT%>Y>syi)OEcOAV7csfgK5?z>K*V z=A5=W(HJ*#Spq3LsqT)5OqBou0t5&g3UuNz=VyGbD}~p5I2YtFP0Q57UhESfK!5;& zT?zE!GbU#$NCai46Q}Xuu6RtA009C72%Hp%AsG%dso0IrwB1H#uinYd6cQjnfB=Dg z2(;;#KDKSsvwUzHN@|H_V;?eRNq_(W0tC(rH1U{|Gd`EfsieDG4tOO%fB*pkI~U0D zSSMxNY0|S^fA)OZgT3PW?X+orzjHbh0RjXF5agQxYIRfWQ(3x{(<2@q9+6 z{Cs?ZcE`I!Ph%oLfB=CF3mk+{O2s-KFD2ypaJ`#V`FQzmCv+kL1PBlyPy|X2rpU~{ zfEzdC;`35`URa!#p6~E_ko|W!qf-zdK!5;&Akd~{ipvz4@hL{<#rUL0fB*pk1okY@ zgk(z69Ge3_7foTiXFGEzK!5;&y$W<9Fy&|dJC*uV(ayn>K2IG#*+?M)0t5&U*pfh4 zd8Pzy)}tXx)Bl~HWT)!+N{$d9K!5;&4GZMeXIfEOR&Mw@n;HQE1PDwk(EiA?tU|@; zw8DBLK!5;&?FrPG8Gq+CeTtiAc>d#Ae>Jkbw}2TDAV7e?l>%kmS^wEez5HyVvfZ#S zy;4{+0t5&U*q}g6#mS{)S009C!5;$Crrm6RHc1IefN`L?X0tChibk?6^o2pNM009C72oNAZfB*pk1PBly zK!5-N0uL4#@q^LM4^|&MXs0JYfB*pkGYRBejDO5>=@EWHdg*ZQSo~?~i0Zwzb_ft4 zK!Cs+1nQuBI`M|bXPCD5T*I|DMgjx~5V&3-uM_3n^@3UwAV7e?ZUvrReeRaV#0d}} zK!Ctm0Y9ufOXC0m0t5&UI4uyuQ+}SNQA&US0RjZh3EUo^F@GiSIbY=j2oNB!NrCjE z&s)>)=AWP5I)F`H8j~VGfB=Ez3dDuu&EfU<4wegSWCREhAh0EYl6W~j%ic9J41X3H zJGu68eamals2u?U1PBngR-go4`rI`=UEkA9KmAPcc}xCYJ8ta=5FkK+!1V$#xng`? z)kADmI=HTfeA=rAaMh?aB0zuu0Rp!Ql=Qo*pS*MoP37NJei{)VK!Ctn1?u>`Vfd+s z8`^71fB*pk1Xd)FKd?>f#`tvghp#s@O6)Ze6)>inFv z&&fOM2u|L&4haw-KwzN)?ajZl6_b(Z;#(=D!tR+>!f4PlcmPCQl9_;0tD78 zkR$SV{4ustSeDo0D)%T!!?=HFkDOXfHL{iJ1PBlyuu_4LOF1Kt$DeX596mmB-;#P9 z_dThz5gQEi)&T{2oNA}gTR0var*f52477H5SUNE&1pVcJzpho z^p?I<{2oNAZV7&ruP?q=%bEy}i zx6CT`TX!QwZ-La7009C7mMDD@qp009C7#t6jEZpYMDmjD3*1PBm_ z|5`)(4+Q}N1PBlyFjzo*4yIC(009C72y_J=@&_~B0oW!$fB*pkD--bH?aD5g;SwM~ zfWQ(2a(w#rp(R`kV<13)0D(0L`0#d3SIbxl5FkKc5dwbmb`h7sAP5j3K;V=>_>6S1 zk0p<7v1kTHfB*pk_Yhd_2c%(SONsWy4CEf~2Q_kWtZ4+2@oJafB*pk1PBlyuu1`|&sAPFgC;gB2oNAZ z;8AV7e?wguL{e%|)gF*^bT2oPAN zfYs+RuZvL;AV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&wP9XfHir-aT z+*L6k0t5&UAV7cs0RjXF5FkK+009C72;57+N5uD%b{+x*2oTtgfS<5#=USQx0RjXz zB@llJXsJuQp#xj$_&4=NF&P2`2oUHBr1;!G{tS)U?%Zt?AV7csf$;)y*_qRG_3;^} zF&;n*0t5&UATVAaN6sdKXA_ece+9K5K!5;&bqXwF&1d1c&UcSd6Cglf-2ykHPiope z-PNHTcDr@g0XP`}0t5&UxKZH3r>7SV-Eh)ntb1>mfTjcp5FoI7fo?(`F9o|tC!-3T z9-nEl-gQKP009C7<`xJ!)iqD8**}3(%hWQa7K!Cs#0#`<1^9k+~-8)Qx z009C7HYSi(pI0tH9}u5cj&x%e&2$J5AVA>J0uRa~KQevvg?D}e1PBngL132jd@>uu zSs$=+y@5$n0t5&UATYatY@D6g2?!7%Kwt*~@^l9m*%S#7AV6Sh0iV@QEvi2P1PBly zK!5-N0t5&UI4ux=SIF-RowiX*fB*pkdlxv9km+}AeTKXDcZc~CAV7e?8G-Zhd8UD4 z0t5&UAh2tJ@cXrKk$E;ica3B61PBlyK;Wc649=XMrvmhBte)&kAprse2oTtdKog&P znXq{hAV7csf!hW0$F`@Ip11dXV-M#(QNFQT%?S`7K!Cu_0x>w3`5EmpM|Cr}4hRq+ zKwuLB7<=*xI@M> z-P!a+fB*pk1jY%BiO+F8t4)9a0RjXzD^SPhxcQ~E;fz0(%xh08&~aBqZ2|-c5V)g2 z{o(C^teP<|?>JwlB0zuu0Rj^U435vM)~OQ=P!|LU5FkKcnF5k=nb*ar2oNAZVA%pb zmtFStF**VS2oPALK>l5zMGnoN2oNAZfWXoPI=?{F*_>h2mb){c>XHBf0t6l>aHn9q z(`g>(=*$EN5FoHnfos->Qfr|X)368-AV7e?5P|#$Gec^sLx2DQ0t5&&0%i4FUYi3p z2oNAZfB=Cr0%iG`K2P?c@);$?1PBlyK%g(s{l^+gmiBAdBS3%v0Rr<0T)T98_@|~q zbG|o)o(T{jK;W#vm#e`6)TtOY^WL?wdv-iy59DEtZKXZ|0t5&Um_%Sm4mMF$XXhj* z)(HUu1PBmVmq3opCKQJczSrd8pWhB2x0(b95FkKc5`mDQy(}EqXhvQK~j0RlS~=tg8;g8%^n z1PJU;;9yaif;0Y$x$#jnVt97b!uP~-VL2oPAdfUnbKUmv4;fIvUHQdp)@ zbjnj|b)=b3_Jp2tFq4#>aVu}+*R->ZX_^HA0t5&Um{g#bpCvxq5S{c~rL(ZGEt_5m z5FkK+z#0W|e1_)`onibrIP2&6=98AZ^L4Lh0t5&UATW@rHF=1DejY&rY2oNAZ;3Qv)T-^i|JX%X7if!H9jwE`pX;9vV4q&009C7W)!HyF`uE1$W#{N zGyZ>g)OD5fj=A~Hy%k~%DVFm2T)YkSAzy2hDESxZLGkj>b{c16jgw?lNa5 zAwYltfu#uK6iU$;J_^n2&M>x=O{uiWrcRx+gTH&QHlsT`BnJo(AV7e?^a42|W9)^k zoSu2R9Owm9U1$1XjP5ZGFQxqGmKqQsK!5;&n*~xzhF;QZehQ1w?rzG@c%)6(o6mY% z2W2d2WVdytH30$y2oShNASY)1+{9)tK4VJ7m1!)@HLf|)WNqSD(}<+!WTrYIK!5;& zu>uFlnS-;*(R4KJr2}DWZ2!~O&qK%|I(_eYBtU=w0Rm45#0YBQvuwqJ7?`nbeXzXb zf+sp1dpMnLY@=C5GJX(R5FkK+z%B)v_>8kJA(`_mK6k=39&D;iYL!nny30JEI_5{W z)_?#30t5(LFOU{009EK5Xj*erkX=D=TM!EIU&n|^xA}K z+?W*TrrxF=%%n5Z836(W2y9Ov$7jf{oRYl>=1|H9>gQA~Rmul@^>5iJ-CY;Tadh$Q zH)ZfG!`GGo0RjZ}B~Zs_`N%frPt2mSUv^VjDUU*FJ8n$%m?HV&xIG9EAV7e?`~pp0 zHUXJ8I;+vTpoz>BQK>KuFKo>}OlKfKfB*pkixJ4dn3kF`9P7x8rFB-eNg21&f!rqV zrHM$-#avGVAwYltfjbH``InQie(nsQ1Zzy#9HO}-R|)k)VW+xH0t5&UAg}|06pC@4 zhw)hsl=2*(omFU!(+$%dc$p#r0t5&U=n9mW%&#dX<5M?Q>x`{mdimPz$u>ZH1hy@3!$Po=h%qFa_zYm7 z*Z6<=kwP?A-nOOL5g+&itEo}}Iha5)4vd!_m{r_zQ10RjXFEJdJR7T!3Vn3{EP z#-NP7)q$Bi&O0}rL~{ZJ2oNB!6oH&|VKhztT{p7aJOpP-$|gPAB)zU%tq2exK!Cua z1#-|`J;XXSWBZiI_2RRifoNV`;YqWtW{YjDt>P_48GeD4mXK+{S0#I?VJ@ovBQK z009C7<`f7?m;UUej?eU`o3SiBQV3pg?$qvz@GF1+l7^B7b;ZfgH)-Sfe4QJP)Mz*K zpeX?Y1PH80Al~(J-Ze3r_RIf>%E+07)GWP?YnfW5#wo$;t#N%^u6>vIocSb|XXq#a z0t5)$Paxj#bK=GSk+F39kGmy4N9RHAYh<&!QJ8feK-w6xfE{z5A^qLzK4v_(o=bZI z1PBm#oIpvt_OP6cv0t5&wS|F!g%9av*^=rK*eJZ6=D0Px4rC(?NaQ5d8rL)G$Po1#oa7<1O ziqANNruT_9Chu^fap{Ht0RjYW6DYBjvLo%}r<7xP1$yxWB_G;Wm#d#ksMP*M)8qQ( zj3+8zDmw1ysCF2`QJDe)1PBlya6%v?MhuOdesLq#X-djkFb2zoXsKK0eomaEDH4*I^2kJdg?Kx&QpCjPJjRb0y_|>SUy9EFm3k;Kd=4)4bMD~j z@_Q!QQ*%5$LZt)<5FoHafw(41;h3T=?PZ0Ohle?O1}L|^T)94ariVO=0Sm;uG(O+U zkRx%v2agj)2Q@D5CV^b#sPD@y9VI}3zzzhCZ@X)sbc&7ml<)s3KI0-R?uJKMP99GD zARYczJV7-EL3q zFuSF)&AXZj&PfB=ED2*lN5{x3yX9i(ye z7bC50Qa7txguQfV*z}K5EOj=l?bWr?2(NB^)NtbNj`~+XfB*pk%M(a(6`x`V)M0k` zdS=wAMUKk0Md>AU)%16!eJC!q+Zv^(wq%B_&PnpbXE=v02@oJaV7me-0dt6zBrBm; zUQbWH3EG@Q>A8;J)MsklNvYGloXXF0HRmbMgPo^uy%HcmfWSrtVtC|bXT1o^X_8lt zPlR8rl;baUk{@;Rrk6Fa#lb#3&LMT4?nMm(1PBn=gg{D;I_%g8hN)g4T;TivJ)m;eC+1ZENFM_yiFg|b*2|GWH1)zS*Ij?GX#mX*il*6TYO z=C!^-8!-U_1PDwoPy!*o#?b4eUpU!QVUb(MTB&UuUfjr!xEV{f^w{2vwsdvOg#ZBp z_Y}w>7Zb1hbIj4fmxHAfzi|#*NSi3r(dca6^C~$P0RjXFEK8uyh9)Fpigx2->}X@- z)Y(RIrk+bplJ>@us84_Z0Ro#4=%-_jxtI$ROfH8_TE|Y%+nu@?H1SE9?ldu{AwYlt zfhh(0aWP43*2AmQbILBKocQU!hJNEPwBG5nC?!CE0D-j&9H!fJi^WyO=itlP|F&ar zGZhICAV6T70x3=>$cEfcjL=Q|&d@e5wpkG%K!Cuc0*6b|lby$4JaM3ITOTS*&=VCE z5g9JLNnDbo!3jxvb(0&Frl=HPETtZxUmu-K!Cs^1fELI zdXObXyK$5Ar(3d!84Vdk{4O92JU~x9VBL!ObK3>YV%$)=QyJG%Z2|-c5Fii*a*~ASCpcogwo=*ZmOasd z!vqKrAV6R{0yzjf$#^ieIw0dqdgMP=*-U(fISu>UodXuT1PBlyuxWuoi^?EyzFtbo zIy+<7v}qFy*N(UT8La&Z7(W351Xe0=Dg#fIrH^@gf9107_*{MRuk`(9*aQd=*n&V< zd>uz+ipAsNl%HFebe_YB32tX{WsUv34~U3!371`tk{9d7XJkcj~^L z_npq>Gz16`Ag~>Q6q50&_p{HKlBuwsXguygfB*pk1PEk-n2~i<9tLD*UwC@I5FkK+ z0D%(%N6X8Ut0h6j=Ls@J1PBlyuwQ|5!I;yt#ODrt&OuBu^&K%z!KDuZ1PCl&;AqiV z4>^X(u5S7f;vB~EFQX9>AV7csfrkj>=$vy2`jBZkDFFfm2uv%`4GOESWE&W|D${!F zjQ{}x1XeH5d`>AaI@c0L2g4=-0t5&g3&cfM`oU&SjQD(9X@?~kc6iZEnE(L-BLsSB znKN`mYgLB`^o)lzP=^2k0t8kfkXN17o^xC-s&lC7lK=q%3lWG*Bp+FhzZe6|22>e; zUM&a^AV6T50v1gJu1i3AIvfX78E`692oNAZU=0H9`9rSKkn|kVk2(Yh5FkKcPXbS` zJ@=$xt^^1WAV6S{fJOQsCY1;fAV7dXSHP#YT@Bj=2oNAZV32_L9K@s&0RjXF5a#!Ky%z`&AV7e?V1X2!srI-}dk3FDMFIo}5Lk(T ze}8W9g$m)BAA?({NPqwV0^1Nc49gG)+nCrrX7Nmk=j9^}qJfB=C-2*lg}A_lV7 zLA3d~toU?Q*ZPh!ZUO{WBH%WA%Z19}8G7(d$SsW85+FceLjp}QY-moCPbIJ2A5G<3 zpvf<+4haw-aIL_jBjMU{-?v?k&-hm>_0}Cv-!6FT)3*h89EDR6AV7dXM2Q{9o zQhol(`qL2s0t5(LBM`&D$AZJ;Vshn2Ua*f`29B(LX-uxzB^}2I5FjwCK)#!YIGFV; zo9jH~XP8x;qvPfWdz_cr1PBmVqJZ4I@~Wh7IdXiCGfWMyJgH^`2oNAZVC@276(96V9dVXXl}GfeDggon2oP9{K*++etIxGe%{U1VAh2J7cxO*PpxZCO z%rnox7L+qH>z)7s0{0b23nTaTIniT&-eCdiBF=dc^-X{PfkT18YtJ!1ygNL}uJ_}2 zC@DT?R*;`F1M8ju0RlaNv^eUO*qK(~UimrgsP#sG009CQ1=2^07i&0pqd=3NR`54U zYEFOvfjbFUTHXoz@@bmU+CP09|0|opnbWZ%{1M)df zqHh8O2oSihK#b1xD=7Cp?XJ$2iFY++lO;faz*Yt76=(C}1IAgev?b`QygDa9fWRaI zL+;g`I%)Cg1KdgIbwYpu0Rrn4=tSe$Np$OUx6d|kU>X6~(>Usd009C7_9GBKBA;!Y zKU>H9xjtq}fWWQ=hTTQejeOTmkD2_8KVu$t;U6gMPced5L%K0_>} zN7a`-$Muh{z^{KMW+k9vVV>Si?1PBlyurPs? zo=0QKnQC!#1fzfe0RjXF)B|fRq4Ohfrs00WQSe8J4iIZ2K%?G)0cUi;T(`bh3hilZmO!*qRZhLyG zm@5GS_ZR5pV{_w88(}c<_5NdW##&&ge(y5Ho$!E2fB*pka|-nCv*pIy-hF$A=NyN= zPYJXa;?&W$EMB;>0*%`)TWe#B009EC3d9?3zVYUbbThVunbrJa=P3h+=qqI)NOB>9Rc0L9CPhtLw#<;qgQ(hY1iMKwx44 zci)LYuj?)c;IJ5spBR)6Z^PN7=XK;-5ge8%&J%QpS#^+92oNAJ zx4`8a_S|RJe^=mgEVfI}%k{cF+9p7N009Eq7wG0<*zQj_?8cH*=kQp+rab}#2oNB! zCxIM|X~uE$VCrG{8A|H->>us#9kjDcfB*pk1hy?uXJeC*z1cPynQEm&Ibn0FUR^r` z2oNAZU`GNaKSO}#4DHOl#AnFNb~Ej?dD^LNlK=q%1PJU#pjk=Q3(GR`Ql5&#PA>2N zI#H?Xl>h+(1PJU-pj}F~TL~fVhK>3qFF{k4=4p3UCQN_;0RjXXfgFcnx5>r05%)q- zTO8|mYQ$X+0t5&UAh2J75{glvY{&S_`%Pi@U_9hOfB*pk1a>XZW@i4Mm+S6_?K)H2 zXidS|iXH5>2W$}_K!5;&jSIxcEIHX(XEsS%e~jD7(7ZfNbvJHfdISg%AVA=ZKqo%a zW?YBHfUNT~Y&6?Vq}E-9{WF~^CP07y0RnpvXp*r>&N%C4ryNdYWhcGv0mmE(5FkK+ zz_CDDLv}K;EIi}VvMugzHhqM{&9)5!1PBlyuuFlGl_e?r2<2$|+COfO009C72<%7T zVEvfFG394>>DfkR*@{5Twk7C(-Y;fKfB*pkwLmXGb0mf%bv8B|B`f1=Ib16Bdf-Zc z009C7_9u{I@?>h(30j9{sM5x#0Nvj^#f%9MAV8oMh-=Fjlc(~te;~A}^EAJ-eb^#E zfB*pkI}?b>nc^_b`yd~CyG_l~!cRbV_Fgev0t5&U=m>Ndp0PNmW7;^1&$_f;hQ`_+ z1PBlyKwy^w#~<9rbgWaf7oc@&T7B;FePYrC2oNC96NvG7@MFz3Gh+(1PJV2pr4g#FPXIEVK?ujm)(N_AV7cs0Rkl% z>#*$n{4?6*Fb&UPtL$2Zde;yjK!5;&PAtZ-?Br+Mi;t9@^-io39@q#FAV7e?&IQ^m z%*mLZo2R_&OzY726s&TsouipP0RjXF5GcQ0TY|C4&UPas8w^p5oIxh5!Kq1PHVOF*Q3M(8k1U_q!k6T7I@@wg?a)K!89Ls6UyF zS(?(ctUA-0G#<|fU=bidfB*pkrv&QwOl!^j32rQjkMeq|l`;YZ2oNB!Z-MyHZG6-U zd8h2{o5t)35Fk)5CEOIf6;24`#7qZDmbP(vqP`*m1PJU%Ag(UTe>d`0D&C|q&s80HP$}{-O~n(_7so zyWfCIB-WqcHYZNGSwEG+Qn&1TB|v}x0RsCFIPtAi&dB=rhq6Mfbt*siVPuvB2oUH9 zoVaavj@sOd!0GovjLIO;Jy;i|6s>V`mHsEZdvP;Q0tEIc(7i40Q^2gB5Qv}MhAKHm z>(`v1B}Ze0Qu0J+4ig|iU>gGEp16(Kn8mFEF(;pR#J5b$%J4`v>n6FN6h6_1!vqKr zSg}C5)4AQP_zF%x^!#?{WoRhMrMc}yVCOU}B# zW|diQ9G|8R)8hk52oShk;MAAp?Y-)O0D)-*LKOBoY*v+_xPPRnR;TBgqfNET8;9q; zT-#9s1PBlyuxNptjrli*E(BzgvZE$*p~F%As7!zW0RjXTC6E?{DMI70nw9Bs?lp&o z3_n*+edp@2ptC!q4|NC-AV7e?QU!8wrstGyZAj)L*N^)kfZL9*?@~(w1PBlyun>Wg zocXo;hVwL|(F*|r1PBngQlOKc>9gB9;3iltmZl@fCb zOMMU^K!Cu)1j4E?tTkiEmH)y3<5IEYUS36Bu^!DGlx{9hJD=oo9Y+ZeAV7e?as`?s zJ4no?e1@SPoN9(M=#l^d0t5)$N8ljShAktHhIY`)(eYexW#?ylQ%ryWfiVK%4m9jm zHKtQ_S1WMq+u`Wlbi=#rsiwD%b+s4CzzGmofIz$h&H4G-0;XsP;{>LCKg{~(^4=YH zY1Jk`U`m1TyB||_r?1Tjq;IT-yg4qtg{IBo&Dfd<0Rr0)i1*1Mx5;hH)+~kzq)>eN zjgj6wQ@?$@A%|TfwFnS+q(FW96Q7T~2x~s~(+N9zb~AK2dJGy6AVAc?P@a&Y+++t=oK_90t5&Uc!WS3ShJjc+zPS{%sM+~9^EWc z)+qr31PBngO}8G2oShY;9#Md67`_$#xrP6 zfB*pkhXSWR`a7&;|M3Dt6VcN1@z=@-2oTu3fR)O|F0l)`u?uQC1PCltAl+>jIyS?) zLLhvYFm#oAMemvrAV7dXEl~2)J$DOliEuTyFb#7cK!CvV1Rh_ z;R6{{cX+RA5+Fc;0D-j$jLFY2b=T%#yaWgkm|tMTa%KL(IKyoMF%kz}(4qBi8<5rn z2oTt+z`&dBR;RzUc?rd>T~YHPuuK8bvCQjYR0IeRAV7csfrkqW{z&)XBQ*d51PBly zup9v&v@GXZ7zqIa1PBlyKwwP*emUU!tL1y}>+Q58K!Cv71?1=2ub}Z0AV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FoG;0l&Vzk_%;+1PBly zupj~dh}(iL#xjQD8>(eo2csacHi2;O*wsI&;9j}5cj#o}Z7%)68@?%@>`Xc$Kwt*~ zdrQw9TvAg!EfD_)K~r!Z#HG(8PJ1ccf`E7ToL4ceJrBMcpYweBUYfvezP`dl+sD$b zg7FZTL?FDyd;O%he6PgLgaQG(?>%APx>|w2;n&*=hJ5W1oAkW41MLVbTHr)jo+w)M z6bz2QiUf8PpYf{fY`QCYBO2=M0x2u|%OUZ3JF6ZD5FkKc>jE(?`*S?l?H{y9fB*pk z1PBCy!}NqgfB*pk1PBlyK!5-N0t5&UAV7cs0RmeU@QLtNudI0yAV7e?d;-#QK3hE# zSgU~FN?hxeGj0L|2oQLnK!{HX`aplDUWdT(pTn$U+(x-N0r%teT(a27w{HZ9{NK!CtP1crVC7Vo-#rnZoac6u27OWFSfx3d?oWO$tJLn#3Q1PBZg za3dYWw2n&MOuTSmYEh-0t5(LCy=7EJjDbRl-EIOMSuVS0{at4xmiE$Z}w(PfB*pk z1g;VgpI5PIM1TMR0=pMz5_I^||?3nqVU^<~y>ck&U|wEM3G1 zA2*1-&;N1q&U{v2_g|G~Wj24n9eG)oIOju5&oHj=U)&0hv=Cj^NR5WT?gb?0uHORb z*1T(h1x`K(W=hVSpYeHt<1{1!`xbEf-1+t2_w<)I`%ZX<&1MO@#Gx7!fi(-<{ynMu zT=Vre_6-Ug%fy)9ZKI5`8#b*4ur?2+QVvZ6ZK;TXS zZpU|mzVb9FK-0&zF+;EHL^A>e?j_*s^j^}=L*OZaoS7*`#phF4juRLuF#Q+R^gWK8 zlIjE+0gJBqnXT_>n>U0F0t5&U7$6`$V|4nCcEDRj6#@hZ5NHJ?NBY&F7Q_|-0t5&U zcv8T!>q#ON2oNAZfB*pk1PEL&aKks;eoAsZyOsn95FoHWf&3Gq@Eij)gA7qp zXOjZn{F}V2CPmoZQI!Az0t5&=DUbu{$!g1}kV3wFx;&^h zYg3G-W$Wb@juIe1U}k}I1Dp9Ay1$n|9WM7e-6ZFU(Kz6drq(_UXrT%L0tDt27dBZLkz@UnjM{4>Ff}_fz_VQ*G(TIoQ6bzz|I6p zvUI0?E}il(zIJPd?aA}=y?9XT-wd;PIi2$nPX20C)5EE1Wl7l+uck9-ZC;j;|?&L_g z9QU5-{m*#s89E;U0_zYck<`B{)wvj^(78Rv!dQ|Xd3IsHE}d+)8b5n~*vcj_F`K&A8j{O|n&=zV0ja zAD`;t_|n~uyWvs4oE$)GGW8%&o=7190t5)`MBsF)q`YaLdKULiqI|wM&SpTJ&0N+$ z*c)!#NnOO6M@JM8AV7csfhh%2EH#1E?fIc8l!kS*`m9UK!R~0wc0N}2yb>TlfB=Cx z1P&8wj&rnqHfL*#+O6)Av8*5ad3m%~H!ij7#cJE7Edm4x5Fjv*K!~K?N2K$dCG{H9 zuU_Y#I7=ExDNj|)nm89WZR4@}QI5398xE8{yb>TlfB=E31)j>2csz$u{iDOH&orcA zKGRcU3OzNWDlJm)Wv2`|N2S0VLaz>iRSWnsU-ji%>ENGOj`ebC_9@o+H{d+si=_$6 zW+SvKJv3!;V?ckZ5FoH-0bk{7zIy8(dk&e?NfVEiN1ElxH|mJ^3a2@AJSxaTC<96j z+MQTC?v)s=`->$W1PBlyKwxTt6hNmtO^2GN({0A=2*fwWPNr&_1PIJ0kTd3F|Htuj=Dt%Z_70cAlU-k5n7b{X)rNk` zdD3gzJt(~XTv~2ZpuC5+IXkl=K;TXSC4K7Cb-yb=I8-OqiE)*R@iiV!FKH(}b-O&# zctla#W$O3TQ+@h!Xhi3#?nWTJJ9abMO-=Nso5_T?qD`Ti%s_$uyDz<+>ZC~>9Nt&* zvD|lEFT5O{t3A>=mg}WMhxKpS4`;}y%NgBUe5QWm5b6Qlavp7$FVK0njT`Lb@5*tf zSNmFlaN(|Pdj6aL>wDC4KLRCJIy0_gCcnh@S92`>)G`*Qg3_|7-c-vQopyD>L4|2{ z)9LGR#i7Ohl9XkjsVtoCXh_oxK`+}7i0>Sa^$FZ^+2XCiRbSsVU&M&hJLl*flnUyn z&M-seSXe?Rz3EO@J@JN2Ejyh}v)Pd;|T`+FBS@&4I6$r|Q=+ZXq3 zt=HgVj57&@x6Z*zEuAP9$4A&{HsW?Xc$`qN^tkw_UhXedp6V$NEa;pc%cgj4 zl60an_IA`IGsEF}W^)fj{{#rEQXt+#RylBkeuO~tW@*CZa1>4GG*#;whi|j^&dYVm zkz7(gANE`J54$bp<6m<+#{4Yb0F+aw*U(|!@AOT?Ydgq+dNxo7BLIje6n?{mQpE4Qf`&D+ky~R zgN86sr8J~`_D*`nG1Q}*=sdb1K!5;&4GZMdcxr+vSV|Da%DLb|;M9qjvZ_3V>YcW+ zm)KGMFCMthqn=AXalI4IoHp+>1ZN>|t3Y_W-r9xs1PI(JFyQMjM@#?f^@*?FZW_g^ zX+J-u?WQ2_-8zX;?c*>~UdD6R@tw8?_R@zK*hsYp3vBDE4fv{ku(#6_*onZY@aTMj z)*rx*!3aI(N2gIKimy$4rkzfudu+x#X&Fp8-}K4H z&)&`&scV)ToZ-2ALlDc;?!zYa*t0)u+D=S>0D)NquDPf7*HEE{{J462rGxO=giS2F zy2E=n?2WWDn>sR6Nf^^Zr<8|vZ<+g^@1el=ovO1DxL#mI_txttxzv{ZyHvg}$LDLu zd)Pz0e{Fa7Z`Yl9JDDib?!{4EPc*@t;!q~&X31S_e}+reH@5W%Z2eBK=qu61PZ4q3 zXq${FKS}NOe_Pl0Z(UC&oH%YZ`B^Wk??0aQjGIr-cB-}r5FqeqftUd0(QLOxIU-7h zD_>*Ggh$)D-szOI$Ckg9CKGbAjnDjDL0!K52^b*(0;?1le#;zQbCnQ_7q6hLR(uj+XqAieldd*b=lh))A@bsCX41&^<z-B3 zodAKM0?l1MroqrgqwA%piQVUo_Ed3ybZ=uDoI6MF&)?P$epP&vbc`Vw&OO7NA{#T0RjXFtXd$YXZ&b& ziL*#CHS54a9**k&C_k+}NBOBtfB*pkcMzC3KI1nDsmu73hM)GvOr+C6?n!)(;Zv6Y z0RjZpC6EF$hh^HVN1sc^P9?Q!x+%-oTspQd^$8Fla8e-LQcjjDBrub}sdrpnGoJeG z-zEfyk0!sTQ&{FkX|p^H??+7n1PELZICU?(P~9N{HwwfUJL;nMk?XyOj%T9OkYwq7S=f_>9J1&?q#+iYou6iH9o~d|Td^#QL`1D3U22(7_I|97M@3wR;{QtqZK+D{||XU={OgZ#gH2`^4dsZH5=l+xzfw+`ndhYfm@u<=w?EjVN8S z`(L>8XKV+1P$wLSkN(B&6dfPzSG7lg0D&lQa2LraoqNeirK6kA$yy3W3B)Ur;1_-S zKKXu$^E}L>QFFh#@}X5{cxMMxBlpkQkw5ISU(p@`0t7AyJaq#J$1k*Y=qiCkx_H^e zUHB^o6yM(A(Ys2yD3=}{>s7QvfB=D$0tfe$2dybi4ox9}X$0!1tUJp&TH-Sn#_eg& zq!$7N_9u{TAL*8|zqy<74g@*@Sx03Xn(fvOAWV?}foTQO&Ba}0T2#F~P~gesWO%nX z!|QA|Wi|*9ATV5@y`Kzkqb7j`2poOu*G&N`L?X0tB`#a3S>SOpKq>9*1L! z$~rw`h2t9Ab~8HyyAil>lh}=ci4qtokWw+UIf~Et^;FWe&d?I2BTu0^0RjXF5ZH}C z`F1pAW{%Rd{EW-cQ;##nr>ZI=K!Cu81>6lcd~pX%Ev-g#y*fb4QuIvYbFq4+x?%zZ z2y9fKEIKwiJ=1zZAjW6C+U&j+;`Zs5=gLplR!V>XfrSa2xwnlSWqrq-sJF3aSO0DT z-OxLT#!h_Z-x`YVhJ!lYah?s-@Z?sWxj8=)7U_HE#wV z<1j~J=gm-Wc3L&{Z4e+pfB=CV2{a4PPD-Zfcb1>APKwg>v?Cl-B|v}x0Rm?Q>I6N= z&w4i|W_u)W<$sRD8uYBF0|W>VAh30ToQ`RhB{1`p>)Z@Gr97`eQ`p9X2altIgX4Ay z5FkK+z;*<3Hir3>a4f-?6S9fUHZbd=`X{5cMEOB!AExxxFSSR2009C7)-Mp_Fl1qz zNKVFjO1U@}wK-V!>({1kKTX@>ddmv|0t5&U*qA`h$uOz&Z~9UO(39nUB}cp{y)SJDYXa@lZZo_s~6M)2>@+n*ad<1PH84ph>tk z*Xs03`4~#$Ypj)vQ$cb6tcCIJEj9wyM-0UtI3C*FWS%)~T} zIwyOd($;~QUz+E7)~6D&?abTr);CA=zO|1F?dR@ONNHwit+;xZW|0RoQ^ zC}|mA(+F}w*ldnm7+7g^SjCO^P-Lb1PksNi(rFnsfxQd3M^E$)C_x#!E9qIvF0VdQ z^*oARlJ<5c8l`Rs5FkK+zybwgP^Pe~SD}N3cqS*G2;D(lt3-eR0RjY;A#f(_^5AlO zb^>(Jm`{HMAU+4-sYHMP0Rr0)2>BWx111wH#cF3N59(sTSvFl|2y+i$bc0@NI|K+2 zATXD}nAEGo^JJ40swa=kHS#s~du1A4*=&tM#!P?!fz=5N%)Ya?^3eXw!gO^M226ke z0RjYSfs^TYHVe!&nuW{+=L4W`O0t5&g3S7>|9G{o#9QJ6R009C7 z2oNZL6KWyqy+D8f0RjX@2;`4&#pehzRS6IvK!8B{#UVci^+q5-fB*pk!v$h^#+9el z=WsJM2@oJaV21)nzbTaRGiB%wHB6ZR0RjXF3=ufaN%1+vO&tOR2oNC96G-vt-@fgc z*daiGz=6QgCwB+O?QU8iCgssvAQzrEuxSdDBS3(_D1oDU=cv{yuUsG}Wtv3$P7t3f zzp92$fB*pk_Y>%?Lc@;L=l$%Ri2wlt1Xe82u0P`>;u>_tlQDDx1PBlya2J7tv=pCr zQFjso1PBn=mVm|Qwyvky5FkK+!2AN?V_We#KeICsAV7e?wgj4A9Xj|#cU!YG8v+Cf z5FjwAKw5oH+M~`05FkK+z~%+~-OA=Ku?Z3&K!Cu+0_k^dRK&ivR%v1U4;D$LFT!XmSJy5FkKcb^$*Got@YT z2oNAZV1ojFhiHQr)szSjAV7cs0RjXF5Fl_);6cC2W9@nFf+#0IfB*pkX9fJ|^DK=6 z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBCy2mNL+djbRq5FoH- z0e|5##ntmo$P|kDAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oRWAz#kLM46J(s1PBn=qJa3^;#D;#0t5&UAV7cs0Rr0-@GszO z@0yws0RjXF%r4-gzuAeMfB*pk1PBlyK!5-N0tD_Z;KSm(lRF^+0t5&UAV7cs0RjXF z5FkK+009C79xCwV_;~0Dsz;yyv9ot}0t5&USfjuK;&Y8`#!P?!0RjZpEMV!n=BsDy z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB=Dw3izd#jb2#OB0zuuff)tT zUxYmPFDhr8o~{WHAV7e?90EB$EkWn7)h7W01PE+c!1{B;7uVDX5FkKcK7kOR^L4Ig z0t5&UAV7cs0RjXF5FkK+009Ey1g810YMNfgja+R41PBlyK!5-N0t5&UAV7e?x&?fq zvhM3=^aKbHAn*tQf5q|$b>}2NfB=D&3y9B^UqHhrK!5-N0t5&UAV7cs0RjXF>`uV1 zQ|#`Nn=kv|Xs0RpQN2>H3XF&i)e0t5&Uc&vc*e5}8-6Cgl(5FkKc zw1D^=O{M_>0t5&Us49P-I{00HYA}_G1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1g;eb|D)sAHr0**0RjXF5FkK+009C72n-VNPazFrQi%Wo0t5(j1;l4p z!!`i|1PBn=zJMRvZvPsaAprse9xotm9)AstfB*pk1nw?yQ~Yi6-`w@l^WEQsPDp?N zft?AgHa>6qreS9n*>njIAV6Spfsmh*_j!*F*E#BY$T$Z90t5(5A+W`qS!?D`aXs}x zfB*pk1V##M@mqnB1gaAtK!Cs<1?0{hv7L$l0RjXFEJjML0**rj2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+0D*-F_#KahTnNJ; zK!5;&^$2`9Jk~SX2aJ@@1JIq4009C7RxjY&i`8F36Cgl<009C72oNAZfWYo0RjXF>|7xHZKeh2&ff#3Pk;ac0#^w5#|^GP(}Vy40t5&UAV7cs0RjXF z5FkK+009C72oNAZfWQC&zsWKHY2{U1/dev/null 2>&1 || true -fi -WORKFLOW_NAME="SDA" -FUNCTION="filter" -docker run --platform linux/amd64 -d -v ~/.aws-lambda-rie:/aws-lambda -v /home/lolo/Projects/serverless-benchmarks/benchmarks/600.workflows/6300.sda-workflow/dev/data:/data -p 9000:8080 -e AWS_LAMBDA_FUNCTION_NAME=${WORKFLOW_NAME}___${FUNCTION} --entrypoint /aws-lambda/aws-lambda-rie --name sda-aws-container logru/sda-sebs-aws:latest /usr/bin/python -m awslambdaric function/handler.handler -docker exec -it sda-aws-container /bin/bash -docker rm -f sda-aws-container >/dev/null 2>&1 || true \ No newline at end of file From af35601b04c64ab455a5b1b3288a7f24395afda9 Mon Sep 17 00:00:00 2001 From: Lorenz Gruber Date: Thu, 12 Mar 2026 11:16:36 +0100 Subject: [PATCH 45/45] removed checks for simultaneous containerized deployment and custom deployment, forcing a custom image deployment --- sebs/aws/aws.py | 5 +---- sebs/benchmark.py | 8 -------- 2 files changed, 1 insertion(+), 12 deletions(-) diff --git a/sebs/aws/aws.py b/sebs/aws/aws.py index aec1a7ab6..e1dda2cbd 100644 --- a/sebs/aws/aws.py +++ b/sebs/aws/aws.py @@ -165,10 +165,7 @@ def package_code( "{{REDIS_PASSWORD}}", f'"{self.config.resources.redis_password}"', ) - assert not (code_package.container_deployment and code_package.benchmark_config.container_image is not None), \ - "Custom container image specified in benchmark config collides with container deployment option." - # if the containerized deployment is set to True - if code_package.container_deployment: + if code_package.container_deployment and code_package.benchmark_config.container_image is None: # build base image and upload to ECR _, container_uri = self.ecr_client.build_base_image( directory, diff --git a/sebs/benchmark.py b/sebs/benchmark.py index b03e85cb6..2bc1bcad8 100644 --- a/sebs/benchmark.py +++ b/sebs/benchmark.py @@ -639,14 +639,6 @@ def build( ], is_workflow: bool, ) -> Tuple[bool, str, bool, str]: - # Check if custom container image is specified which would collide with container deployment option - if self.container_deployment and self.benchmark_config.container_image is not None: - raise RuntimeError( - f"Benchmark {self.benchmark} specifies custom container image " - f"'{self.benchmark_config.container_image}' which collides with " - f"container deployment option enabled in the experiment." - ) - # Skip build if files are up to date and user didn't enforce rebuild if self.is_cached and self.is_cached_valid: self.logging.info(