diff --git a/config/example.json b/config/example.json index c19023e59..bb6cfc896 100644 --- a/config/example.json +++ b/config/example.json @@ -46,7 +46,11 @@ "name": "aws", "aws": { "region": "us-east-1", - "lambda-role": "" + "lambda-role": "", + "resources": { + "use-function-url": false, + "function-url-auth-type": "NONE" + } }, "azure": { "region": "westeurope" diff --git a/sebs/aws/aws.py b/sebs/aws/aws.py index 243a6f0f9..4bb039290 100644 --- a/sebs/aws/aws.py +++ b/sebs/aws/aws.py @@ -436,6 +436,13 @@ def delete_function(self, func_name: Optional[str]): except Exception: self.logging.debug("Function {} does not exist!".format(func_name)) + def delete_function_url(self, func_name: str) -> bool: + """ + Delete the Function URL associated with a Lambda function. + Returns True if deleted successfully, False if it didn't exist. + """ + return self.config.resources.delete_function_url(func_name, self.session) + """ Prepare AWS resources to store experiment results. Allocate one bucket. @@ -576,29 +583,41 @@ def download_metrics( ) def create_trigger(self, func: Function, trigger_type: Trigger.TriggerType) -> Trigger: - from sebs.aws.triggers import HTTPTrigger + from sebs.aws.triggers import HTTPTrigger, FunctionURLTrigger function = cast(LambdaFunction, func) if trigger_type == Trigger.TriggerType.HTTP: - api_name = "{}-http-api".format(function.name) - http_api = self.config.resources.http_api(api_name, function, self.session) - # https://aws.amazon.com/blogs/compute/announcing-http-apis-for-amazon-api-gateway/ - # but this is wrong - source arn must be {api-arn}/*/* - self.get_lambda_client().add_permission( - FunctionName=function.name, - StatementId=str(uuid.uuid1()), - Action="lambda:InvokeFunction", - Principal="apigateway.amazonaws.com", - SourceArn=f"{http_api.arn}/*/*", - ) - trigger = HTTPTrigger(http_api.endpoint, api_name) - self.logging.info( - f"Created HTTP trigger for {func.name} function. " - "Sleep 5 seconds to avoid cloud errors." - ) - time.sleep(5) + if self.config.resources.use_function_url: + # Use Lambda Function URL (no 29-second timeout limit) + func_url = self.config.resources.function_url(function, self.session) + trigger = FunctionURLTrigger( + func_url.url, func_url.function_name, func_url.auth_type + ) + self.logging.info( + f"Created Function URL trigger for {func.name} function." + ) + else: + # Use API Gateway (default, for backward compatibility) + api_name = "{}-http-api".format(function.name) + http_api = self.config.resources.http_api(api_name, function, self.session) + # https://aws.amazon.com/blogs/compute/announcing-http-apis-for-amazon-api-gateway/ + # but this is wrong - source arn must be {api-arn}/*/* + self.get_lambda_client().add_permission( + FunctionName=function.name, + StatementId=str(uuid.uuid1()), + Action="lambda:InvokeFunction", + Principal="apigateway.amazonaws.com", + SourceArn=f"{http_api.arn}/*/*", + ) + trigger = HTTPTrigger(http_api.endpoint, api_name) + self.logging.info( + f"Created HTTP API Gateway trigger for {func.name} function. " + "Sleep 5 seconds to avoid cloud errors." + ) + time.sleep(5) + trigger.logging_handlers = self.logging_handlers elif trigger_type == Trigger.TriggerType.LIBRARY: # should already exist diff --git a/sebs/aws/config.py b/sebs/aws/config.py index 2d05e842e..3714adde5 100644 --- a/sebs/aws/config.py +++ b/sebs/aws/config.py @@ -2,9 +2,11 @@ import json import os import time +from enum import Enum from typing import cast, Dict, Optional, Tuple import boto3 +import botocore.exceptions from mypy_boto3_ecr import ECRClient from sebs.cache import Cache @@ -13,6 +15,28 @@ from sebs.utils import LoggingHandlers +class FunctionURLAuthType(Enum): + """ + Authentication types for AWS Lambda Function URLs. + - NONE: Public access, no authentication required + - AWS_IAM: Requires IAM authentication with SigV4 signing + """ + + NONE = "NONE" + AWS_IAM = "AWS_IAM" + + @staticmethod + def from_string(value: str) -> "FunctionURLAuthType": + """Convert string to FunctionURLAuthType enum.""" + try: + return FunctionURLAuthType(value) + except ValueError: + raise ValueError( + f"Invalid auth type '{value}'. Must be one of: " + f"{[e.value for e in FunctionURLAuthType]}" + ) + + class AWSCredentials(Credentials): def __init__(self, access_key: str, secret_key: str): super().__init__() @@ -115,6 +139,45 @@ def serialize(self) -> dict: out = {"arn": self.arn, "endpoint": self.endpoint} return out + class FunctionURL: + def __init__( + self, + url: str, + function_name: str, + auth_type: FunctionURLAuthType = FunctionURLAuthType.NONE, + ): + self._url = url + self._function_name = function_name + self._auth_type = auth_type + + @property + def url(self) -> str: + return self._url + + @property + def function_name(self) -> str: + return self._function_name + + @property + def auth_type(self) -> FunctionURLAuthType: + return self._auth_type + + @staticmethod + def deserialize(dct: dict) -> "AWSResources.FunctionURL": + auth_type_str = dct.get("auth_type", "NONE") + return AWSResources.FunctionURL( + dct["url"], + dct["function_name"], + FunctionURLAuthType.from_string(auth_type_str), + ) + + def serialize(self) -> dict: + return { + "url": self.url, + "function_name": self.function_name, + "auth_type": self.auth_type.value, + } + def __init__( self, registry: Optional[str] = None, @@ -128,6 +191,9 @@ def __init__( self._container_repository: Optional[str] = None self._lambda_role = "" self._http_apis: Dict[str, AWSResources.HTTPApi] = {} + self._function_urls: Dict[str, AWSResources.FunctionURL] = {} + self._use_function_url: bool = False + self._function_url_auth_type: FunctionURLAuthType = FunctionURLAuthType.NONE @staticmethod def typename() -> str: @@ -149,6 +215,26 @@ def docker_password(self) -> Optional[str]: def container_repository(self) -> Optional[str]: return self._container_repository + @property + def use_function_url(self) -> bool: + return self._use_function_url + + @use_function_url.setter + def use_function_url(self, value: bool): + self._use_function_url = value + + @property + def function_url_auth_type(self) -> FunctionURLAuthType: + return self._function_url_auth_type + + @function_url_auth_type.setter + def function_url_auth_type(self, value: FunctionURLAuthType): + if not isinstance(value, FunctionURLAuthType): + raise TypeError( + f"function_url_auth_type must be a FunctionURLAuthType enum, got {type(value)}" + ) + self._function_url_auth_type = value + def lambda_role(self, boto3_session: boto3.session.Session) -> str: if not self._lambda_role: iam_client = boto3_session.client(service_name="iam") @@ -242,6 +328,139 @@ def http_api( self.logging.info(f"Using cached HTTP API {api_name}") return http_api + def function_url( + self, func: LambdaFunction, boto3_session: boto3.session.Session + ) -> "AWSResources.FunctionURL": + """ + Create or retrieve a Lambda Function URL for the given function. + Function URLs provide a simpler alternative to API Gateway without the + 29-second timeout limit. + """ + cached_url = self._function_urls.get(func.name) + if cached_url: + self.logging.info(f"Using cached Function URL for {func.name}") + return cached_url + + # Check for unsupported auth type before attempting to create + if self._function_url_auth_type == FunctionURLAuthType.AWS_IAM: + raise NotImplementedError( + "AWS_IAM authentication for Function URLs is not yet supported. " + "SigV4 request signing is required for AWS_IAM auth type. " + "Please use auth_type='NONE' or implement SigV4 signing." + ) + + lambda_client = boto3_session.client( + service_name="lambda", region_name=cast(str, self._region) + ) + + try: + response = lambda_client.get_function_url_config(FunctionName=func.name) + self.logging.info(f"Using existing Function URL for {func.name}") + url = response["FunctionUrl"] + auth_type = FunctionURLAuthType.from_string(response["AuthType"]) + except lambda_client.exceptions.ResourceNotFoundException: + self.logging.info(f"Creating Function URL for {func.name}") + + auth_type = self._function_url_auth_type + + if auth_type == FunctionURLAuthType.NONE: + self.logging.warning( + f"Creating Function URL with auth_type=NONE for {func.name}. " + "WARNING: This function will have unrestricted public access. " + "Anyone with the URL can invoke this function." + ) + try: + lambda_client.add_permission( + FunctionName=func.name, + StatementId="FunctionURLAllowPublicAccess", + Action="lambda:InvokeFunctionUrl", + Principal="*", + FunctionUrlAuthType="NONE", + ) + except lambda_client.exceptions.ResourceConflictException: + # Permission with this StatementId already exists on the function. + # This can happen if the function was previously configured with + # a Function URL that was deleted but the permission remained, + # or if there's a concurrent creation attempt. Safe to ignore. + pass + + retries = 0 + while retries < 5: + try: + response = lambda_client.create_function_url_config( + FunctionName=func.name, + AuthType=auth_type.value, + ) + break + except lambda_client.exceptions.ResourceConflictException: + # Function URL already exists - can happen if a concurrent process + # created it between our check and create, or if there was a race + # condition. Retrieve the existing configuration instead. + response = lambda_client.get_function_url_config( + FunctionName=func.name + ) + break + except lambda_client.exceptions.TooManyRequestsException as e: + # AWS is throttling requests - apply exponential backoff + retries += 1 + if retries == 5: + self.logging.error("Failed to create Function URL after 5 retries!") + self.logging.exception(e) + raise RuntimeError("Failed to create Function URL!") from e + else: + backoff_seconds = retries + self.logging.info( + f"Function URL creation rate limited, " + f"retrying in {backoff_seconds}s (attempt {retries}/5)..." + ) + time.sleep(backoff_seconds) + + url = response["FunctionUrl"] + + function_url_obj = AWSResources.FunctionURL(url, func.name, auth_type) + self._function_urls[func.name] = function_url_obj + return function_url_obj + + def delete_function_url( + self, function_name: str, boto3_session: boto3.session.Session + ) -> bool: + """ + Delete a Lambda Function URL for the given function. + Returns True if deleted successfully, False if it didn't exist. + """ + lambda_client = boto3_session.client( + service_name="lambda", region_name=cast(str, self._region) + ) + + # Check if we have cached info about the auth type + cached_url = self._function_urls.get(function_name) + cached_auth_type = cached_url.auth_type if cached_url else None + + try: + lambda_client.delete_function_url_config(FunctionName=function_name) + self.logging.info(f"Deleted Function URL for {function_name}") + + # Only remove the public access permission if auth_type was NONE + # (AWS_IAM auth type doesn't create this permission) + if cached_auth_type is None or cached_auth_type == FunctionURLAuthType.NONE: + try: + lambda_client.remove_permission( + FunctionName=function_name, + StatementId="FunctionURLAllowPublicAccess", + ) + except lambda_client.exceptions.ResourceNotFoundException: + # Permission doesn't exist - either it was already removed, + # or the function was using AWS_IAM auth type + pass + except lambda_client.exceptions.ResourceNotFoundException: + self.logging.info(f"No Function URL found for {function_name}") + return False + else: + # Only runs if no exception was raised - cleanup cache + if function_name in self._function_urls: + del self._function_urls[function_name] + return True + def check_ecr_repository_exists( self, ecr_client: ECRClient, repository_name: str ) -> Optional[str]: @@ -310,6 +529,14 @@ def initialize(res: Resources, dct: dict): for key, value in dct["http-apis"].items(): ret._http_apis[key] = AWSResources.HTTPApi.deserialize(value) + if "function-urls" in dct: + for key, value in dct["function-urls"].items(): + ret._function_urls[key] = AWSResources.FunctionURL.deserialize(value) + + ret._use_function_url = dct.get("use-function-url", False) + auth_type_str = dct.get("function-url-auth-type", "NONE") + ret.function_url_auth_type = FunctionURLAuthType.from_string(auth_type_str) + return ret def serialize(self) -> dict: @@ -317,6 +544,11 @@ def serialize(self) -> dict: **super().serialize(), "lambda-role": self._lambda_role, "http-apis": {key: value.serialize() for (key, value) in self._http_apis.items()}, + "function-urls": { + key: value.serialize() for (key, value) in self._function_urls.items() + }, + "use-function-url": self._use_function_url, + "function-url-auth-type": self._function_url_auth_type.value, "docker": { "registry": self.docker_registry, "username": self.docker_username, @@ -339,6 +571,17 @@ def update_cache(self, cache: Cache): cache.update_config(val=self._lambda_role, keys=["aws", "resources", "lambda-role"]) for name, api in self._http_apis.items(): cache.update_config(val=api.serialize(), keys=["aws", "resources", "http-apis", name]) + for name, func_url in self._function_urls.items(): + cache.update_config( + val=func_url.serialize(), keys=["aws", "resources", "function-urls", name] + ) + cache.update_config( + val=self._use_function_url, keys=["aws", "resources", "use-function-url"] + ) + cache.update_config( + val=self._function_url_auth_type.value, + keys=["aws", "resources", "function-url-auth-type"], + ) @staticmethod def deserialize(config: dict, cache: Cache, handlers: LoggingHandlers) -> Resources: diff --git a/sebs/aws/function.py b/sebs/aws/function.py index 27aeb240b..c65f2884c 100644 --- a/sebs/aws/function.py +++ b/sebs/aws/function.py @@ -39,7 +39,7 @@ def serialize(self) -> dict: @staticmethod def deserialize(cached_config: dict) -> "LambdaFunction": from sebs.faas.function import Trigger - from sebs.aws.triggers import LibraryTrigger, HTTPTrigger + from sebs.aws.triggers import LibraryTrigger, HTTPTrigger, FunctionURLTrigger cfg = FunctionConfig.deserialize(cached_config["config"]) ret = LambdaFunction( @@ -55,7 +55,11 @@ def deserialize(cached_config: dict) -> "LambdaFunction": for trigger in cached_config["triggers"]: trigger_type = cast( Trigger, - {"Library": LibraryTrigger, "HTTP": HTTPTrigger}.get(trigger["type"]), + { + "Library": LibraryTrigger, + "HTTP": HTTPTrigger, + "FunctionURL": FunctionURLTrigger, + }.get(trigger["type"]), ) assert trigger_type, "Unknown trigger type {}".format(trigger["type"]) ret.add_trigger(trigger_type.deserialize(trigger)) diff --git a/sebs/aws/triggers.py b/sebs/aws/triggers.py index f18314593..38842b226 100644 --- a/sebs/aws/triggers.py +++ b/sebs/aws/triggers.py @@ -5,6 +5,7 @@ from typing import Dict, Optional # noqa from sebs.aws.aws import AWS +from sebs.aws.config import FunctionURLAuthType from sebs.faas.function import ExecutionResult, Trigger @@ -123,3 +124,62 @@ def serialize(self) -> dict: @staticmethod def deserialize(obj: dict) -> Trigger: return HTTPTrigger(obj["url"], obj["api-id"]) + + +class FunctionURLTrigger(Trigger): + """ + Trigger using AWS Lambda Function URLs instead of API Gateway. + Function URLs provide a simpler alternative without the 29-second timeout limit. + """ + + def __init__( + self, + url: str, + function_name: str, + auth_type: FunctionURLAuthType = FunctionURLAuthType.NONE, + ): + super().__init__() + self.url = url + self.function_name = function_name + self.auth_type = auth_type + + @staticmethod + def typename() -> str: + return "AWS.FunctionURLTrigger" + + @staticmethod + def trigger_type() -> Trigger.TriggerType: + return Trigger.TriggerType.HTTP + + def sync_invoke(self, payload: dict) -> ExecutionResult: + self.logging.debug(f"Invoke function via Function URL {self.url}") + if self.auth_type == FunctionURLAuthType.AWS_IAM: + raise NotImplementedError( + "AWS_IAM auth type requires SigV4 signing, which is not yet " + "implemented in FunctionURLTrigger. Use auth_type=NONE or " + "implement SigV4 signing via botocore.auth.SigV4Auth." + ) + return self._http_invoke(payload, self.url) + + def async_invoke(self, payload: dict) -> concurrent.futures.Future: + pool = concurrent.futures.ThreadPoolExecutor() + fut = pool.submit(self.sync_invoke, payload) + pool.shutdown(wait=False) + return fut + + def serialize(self) -> dict: + return { + "type": "FunctionURL", + "url": self.url, + "function_name": self.function_name, + "auth_type": self.auth_type.value, + } + + @staticmethod + def deserialize(obj: dict) -> Trigger: + auth_type_str = obj.get("auth_type", "NONE") + return FunctionURLTrigger( + obj["url"], + obj["function_name"], + FunctionURLAuthType.from_string(auth_type_str), + )