From b8041ebe9c3046ef8aee4a1d2416817e4c20f11b Mon Sep 17 00:00:00 2001 From: Garrett Beatty Date: Tue, 16 Jun 2026 14:35:31 -0400 Subject: [PATCH 1/9] Add IDurableServiceClient interface for testing SDK injection seam Introduce an internal IDurableServiceClient interface that LambdaDurableServiceClient now implements. Refactor WrapAsyncCore to accept this interface instead of IAmazonLambda, and add an internal WrapAsync overload the testing package can call directly. Add InternalsVisibleTo for Amazon.Lambda.DurableExecution.Testing. --- .../Amazon.Lambda.DurableExecution.csproj | 3 ++ .../DurableFunction.cs | 24 +++++++-- .../Services/IDurableServiceClient.cs | 27 ++++++++++ .../Services/LambdaDurableServiceClient.cs | 2 +- .../DurableFunctionTests.cs | 51 +++++++++++++++++++ 5 files changed, 101 insertions(+), 6 deletions(-) create mode 100644 Libraries/src/Amazon.Lambda.DurableExecution/Services/IDurableServiceClient.cs diff --git a/Libraries/src/Amazon.Lambda.DurableExecution/Amazon.Lambda.DurableExecution.csproj b/Libraries/src/Amazon.Lambda.DurableExecution/Amazon.Lambda.DurableExecution.csproj index a64717dd0..7eaa84ff6 100644 --- a/Libraries/src/Amazon.Lambda.DurableExecution/Amazon.Lambda.DurableExecution.csproj +++ b/Libraries/src/Amazon.Lambda.DurableExecution/Amazon.Lambda.DurableExecution.csproj @@ -26,6 +26,9 @@ <_Parameter1>Amazon.Lambda.DurableExecution.Tests, PublicKey="0024000004800000940000000602000000240000525341310004000001000100db5f59f098d27276c7833875a6263a3cc74ab17ba9a9df0b52aedbe7252745db7274d5271fd79c1f08f668ecfa8eaab5626fa76adc811d3c8fc55859b0d09d3bc0a84eecd0ba891f2b8a2fc55141cdcc37c2053d53491e650a479967c3622762977900eddbf1252ed08a2413f00a28f3a0752a81203f03ccb7f684db373518b4" + + <_Parameter1>Amazon.Lambda.DurableExecution.Testing, PublicKey="0024000004800000940000000602000000240000525341310004000001000100db5f59f098d27276c7833875a6263a3cc74ab17ba9a9df0b52aedbe7252745db7274d5271fd79c1f08f668ecfa8eaab5626fa76adc811d3c8fc55859b0d09d3bc0a84eecd0ba891f2b8a2fc55141cdcc37c2053d53491e650a479967c3622762977900eddbf1252ed08a2413f00a28f3a0752a81203f03ccb7f684db373518b4" + diff --git a/Libraries/src/Amazon.Lambda.DurableExecution/DurableFunction.cs b/Libraries/src/Amazon.Lambda.DurableExecution/DurableFunction.cs index 04ebee556..960be20b4 100644 --- a/Libraries/src/Amazon.Lambda.DurableExecution/DurableFunction.cs +++ b/Libraries/src/Amazon.Lambda.DurableExecution/DurableFunction.cs @@ -37,7 +37,8 @@ public static Task WrapAsync( Func> workflow, DurableExecutionInvocationInput invocationInput, ILambdaContext lambdaContext) - => WrapAsyncCore(workflow, invocationInput, lambdaContext, _cachedLambdaClient.Value); + => WrapAsyncCore(workflow, invocationInput, lambdaContext, + new LambdaDurableServiceClient(_cachedLambdaClient.Value)); /// /// Wrap a workflow (typed input + output) with explicit Lambda client. @@ -47,7 +48,8 @@ public static Task WrapAsync( DurableExecutionInvocationInput invocationInput, ILambdaContext lambdaContext, IAmazonLambda lambdaClient) - => WrapAsyncCore(workflow, invocationInput, lambdaContext, lambdaClient); + => WrapAsyncCore(workflow, invocationInput, lambdaContext, + new LambdaDurableServiceClient(lambdaClient)); /// /// Wrap a void workflow (typed input, no output). @@ -68,20 +70,32 @@ public static Task WrapAsync( IAmazonLambda lambdaClient) => WrapAsyncCore( async (input, ctx) => { await workflow(input, ctx); return null; }, - invocationInput, lambdaContext, lambdaClient); + invocationInput, lambdaContext, + new LambdaDurableServiceClient(lambdaClient)); + + /// + /// Internal overload for the testing package — accepts an + /// directly so the testing SDK can + /// inject an in-memory implementation. + /// + internal static Task WrapAsync( + Func> workflow, + DurableExecutionInvocationInput invocationInput, + ILambdaContext lambdaContext, + IDurableServiceClient serviceClient) + => WrapAsyncCore(workflow, invocationInput, lambdaContext, serviceClient); private static async Task WrapAsyncCore( Func> workflow, DurableExecutionInvocationInput invocationInput, ILambdaContext lambdaContext, - IAmazonLambda lambdaClient) + IDurableServiceClient serviceClient) { var serializer = LambdaSerializerHelper.GetRequired(lambdaContext); var state = new ExecutionState(); state.LoadFromCheckpoint(invocationInput.InitialExecutionState); - var serviceClient = new LambdaDurableServiceClient(lambdaClient); var checkpointToken = invocationInput.CheckpointToken; var nextMarker = invocationInput.InitialExecutionState?.NextMarker; diff --git a/Libraries/src/Amazon.Lambda.DurableExecution/Services/IDurableServiceClient.cs b/Libraries/src/Amazon.Lambda.DurableExecution/Services/IDurableServiceClient.cs new file mode 100644 index 000000000..8ccbf13c9 --- /dev/null +++ b/Libraries/src/Amazon.Lambda.DurableExecution/Services/IDurableServiceClient.cs @@ -0,0 +1,27 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +using SdkOperationUpdate = Amazon.Lambda.Model.OperationUpdate; + +namespace Amazon.Lambda.DurableExecution.Services; + +/// +/// Abstraction over the durable execution service RPCs. The production +/// implementation () calls the real +/// AWS Lambda APIs; the testing package injects an in-memory fake. +/// +internal interface IDurableServiceClient +{ + Task CheckpointAsync( + string durableExecutionArn, + string? checkpointToken, + IReadOnlyList pendingOperations, + Action>? onNewOperations = null, + CancellationToken cancellationToken = default); + + Task<(List Operations, string? NextMarker)> GetExecutionStateAsync( + string durableExecutionArn, + string? checkpointToken, + string marker, + CancellationToken cancellationToken = default); +} diff --git a/Libraries/src/Amazon.Lambda.DurableExecution/Services/LambdaDurableServiceClient.cs b/Libraries/src/Amazon.Lambda.DurableExecution/Services/LambdaDurableServiceClient.cs index a38dda31b..f6816c3c4 100644 --- a/Libraries/src/Amazon.Lambda.DurableExecution/Services/LambdaDurableServiceClient.cs +++ b/Libraries/src/Amazon.Lambda.DurableExecution/Services/LambdaDurableServiceClient.cs @@ -19,7 +19,7 @@ namespace Amazon.Lambda.DurableExecution.Services; /// /// Calls the real AWS Lambda Durable Execution APIs via the AWSSDK.Lambda client. /// -internal sealed class LambdaDurableServiceClient +internal sealed class LambdaDurableServiceClient : IDurableServiceClient { private readonly IAmazonLambda _lambdaClient; diff --git a/Libraries/test/Amazon.Lambda.DurableExecution.Tests/DurableFunctionTests.cs b/Libraries/test/Amazon.Lambda.DurableExecution.Tests/DurableFunctionTests.cs index af38a4549..8727a5c8b 100644 --- a/Libraries/test/Amazon.Lambda.DurableExecution.Tests/DurableFunctionTests.cs +++ b/Libraries/test/Amazon.Lambda.DurableExecution.Tests/DurableFunctionTests.cs @@ -6,6 +6,7 @@ using Amazon.Lambda; using Amazon.Lambda.DurableExecution; using Amazon.Lambda.DurableExecution.Internal; +using Amazon.Lambda.DurableExecution.Services; using Amazon.Lambda.Serialization.SystemTextJson; using Amazon.Lambda.TestUtilities; using Amazon.Runtime; @@ -754,6 +755,56 @@ public async Task WrapAsync_ReplayDeterminism_CallbackIdStableAcrossInvocations( Assert.Equal(id, observed); } + [Fact] + public async Task WrapAsync_InternalOverloadWithIDurableServiceClient_WorksIdenticallyToPublicOverload() + { + var pastExpirationMs = DateTimeOffset.UtcNow.AddSeconds(-5).ToUnixTimeMilliseconds(); + var input = new DurableExecutionInvocationInput + { + DurableExecutionArn = "arn:aws:lambda:us-east-1:123:durable-execution:seam-test", + InitialExecutionState = new InitialExecutionState + { + Operations = new List + { + new() + { + Id = "exec-0", + Type = OperationTypes.Execution, + Status = OperationStatuses.Started, + ExecutionDetails = new ExecutionDetails { InputPayload = "{\"orderId\":\"order-123\"}" } + }, + new() + { + Id = IdAt(1), + Type = OperationTypes.Step, + Status = OperationStatuses.Succeeded, + StepDetails = new StepDetails { Result = "{\"IsValid\":true}" } + }, + new() + { + Id = IdAt(2), + Type = OperationTypes.Wait, + Status = OperationStatuses.Pending, + WaitDetails = new WaitDetails { ScheduledEndTimestamp = pastExpirationMs } + } + } + } + }; + + var serviceClient = new Services.LambdaDurableServiceClient(new MockLambdaClient()); + + var output = await DurableFunction.WrapAsync( + MyWorkflow, + input, + CreateLambdaContext(), + (Services.IDurableServiceClient)serviceClient); + + Assert.Equal(InvocationStatus.Succeeded, output.Status); + Assert.NotNull(output.Result); + var result = JsonSerializer.Deserialize(output.Result!); + Assert.Equal("approved", result!.Status); + } + private static async Task MyWorkflow(OrderEvent input, IDurableContext context) { var validation = await context.StepAsync( From 01d40f274267ecd4429fbee62a5af33f25748e6f Mon Sep 17 00:00:00 2001 From: Garrett Beatty Date: Tue, 16 Jun 2026 15:43:36 -0400 Subject: [PATCH 2/9] Add Testing SDK project scaffolding and public data types Create Amazon.Lambda.DurableExecution.Testing package with the full public API surface: IDurableTestRunner, TestResult, TestStep, TestRunnerOptions, CloudTestRunnerOptions, OperationKind, OperationStatus, and exception types (TestExecutionFailedException, TestExecutionLimitException, UnregisteredSiblingFunctionException, CloudTestException). Add Amazon.Lambda.DurableExecution.Testing.Tests with unit tests for TestStep (kind mapping, typed accessors, GetResult deserialization), TestResult (step lookup, parent-child linking, EnsureSucceeded), options validation, and exception formatting. --- ...zon.Lambda.DurableExecution.Testing.csproj | 30 ++ .../CloudTestException.cs | 21 ++ .../CloudTestRunnerOptions.cs | 28 ++ .../IDurableTestRunner.cs | 69 +++++ .../OperationKind.cs | 28 ++ .../OperationStatus.cs | 36 +++ .../TestExecutionFailedException.cs | 41 +++ .../TestExecutionLimitException.cs | 40 +++ .../TestResult.cs | 151 ++++++++++ .../TestRunnerOptions.cs | 58 ++++ .../TestStep.cs | 137 +++++++++ .../UnregisteredSiblingFunctionException.cs | 25 ++ ...mbda.DurableExecution.Testing.Tests.csproj | 36 +++ .../ExceptionTests.cs | 85 ++++++ .../OptionsValidationTests.cs | 75 +++++ .../TestResultTests.cs | 240 +++++++++++++++ .../TestStepTests.cs | 278 ++++++++++++++++++ 17 files changed, 1378 insertions(+) create mode 100644 Libraries/src/Amazon.Lambda.DurableExecution.Testing/Amazon.Lambda.DurableExecution.Testing.csproj create mode 100644 Libraries/src/Amazon.Lambda.DurableExecution.Testing/CloudTestException.cs create mode 100644 Libraries/src/Amazon.Lambda.DurableExecution.Testing/CloudTestRunnerOptions.cs create mode 100644 Libraries/src/Amazon.Lambda.DurableExecution.Testing/IDurableTestRunner.cs create mode 100644 Libraries/src/Amazon.Lambda.DurableExecution.Testing/OperationKind.cs create mode 100644 Libraries/src/Amazon.Lambda.DurableExecution.Testing/OperationStatus.cs create mode 100644 Libraries/src/Amazon.Lambda.DurableExecution.Testing/TestExecutionFailedException.cs create mode 100644 Libraries/src/Amazon.Lambda.DurableExecution.Testing/TestExecutionLimitException.cs create mode 100644 Libraries/src/Amazon.Lambda.DurableExecution.Testing/TestResult.cs create mode 100644 Libraries/src/Amazon.Lambda.DurableExecution.Testing/TestRunnerOptions.cs create mode 100644 Libraries/src/Amazon.Lambda.DurableExecution.Testing/TestStep.cs create mode 100644 Libraries/src/Amazon.Lambda.DurableExecution.Testing/UnregisteredSiblingFunctionException.cs create mode 100644 Libraries/test/Amazon.Lambda.DurableExecution.Testing.Tests/Amazon.Lambda.DurableExecution.Testing.Tests.csproj create mode 100644 Libraries/test/Amazon.Lambda.DurableExecution.Testing.Tests/ExceptionTests.cs create mode 100644 Libraries/test/Amazon.Lambda.DurableExecution.Testing.Tests/OptionsValidationTests.cs create mode 100644 Libraries/test/Amazon.Lambda.DurableExecution.Testing.Tests/TestResultTests.cs create mode 100644 Libraries/test/Amazon.Lambda.DurableExecution.Testing.Tests/TestStepTests.cs diff --git a/Libraries/src/Amazon.Lambda.DurableExecution.Testing/Amazon.Lambda.DurableExecution.Testing.csproj b/Libraries/src/Amazon.Lambda.DurableExecution.Testing/Amazon.Lambda.DurableExecution.Testing.csproj new file mode 100644 index 000000000..9e639812b --- /dev/null +++ b/Libraries/src/Amazon.Lambda.DurableExecution.Testing/Amazon.Lambda.DurableExecution.Testing.csproj @@ -0,0 +1,30 @@ + + + + + + $(DefaultPackageTargets) + Testing utilities for Amazon Lambda Durable Execution - test durable workflows locally without deploying to AWS. + Amazon.Lambda.DurableExecution.Testing + 0.0.1-preview + Amazon.Lambda.DurableExecution.Testing + Amazon.Lambda.DurableExecution.Testing + AWS;Amazon;Lambda;Durable;Workflow;Testing + enable + enable + true + + + + + <_Parameter1>Amazon.Lambda.DurableExecution.Testing.Tests, PublicKey="0024000004800000940000000602000000240000525341310004000001000100db5f59f098d27276c7833875a6263a3cc74ab17ba9a9df0b52aedbe7252745db7274d5271fd79c1f08f668ecfa8eaab5626fa76adc811d3c8fc55859b0d09d3bc0a84eecd0ba891f2b8a2fc55141cdcc37c2053d53491e650a479967c3622762977900eddbf1252ed08a2413f00a28f3a0752a81203f03ccb7f684db373518b4" + + + + + + + + + + diff --git a/Libraries/src/Amazon.Lambda.DurableExecution.Testing/CloudTestException.cs b/Libraries/src/Amazon.Lambda.DurableExecution.Testing/CloudTestException.cs new file mode 100644 index 000000000..083c5ee84 --- /dev/null +++ b/Libraries/src/Amazon.Lambda.DurableExecution.Testing/CloudTestException.cs @@ -0,0 +1,21 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +namespace Amazon.Lambda.DurableExecution.Testing; + +/// +/// Thrown by the cloud test runner when the cloud invocation fails in a way +/// specific to the test harness (e.g., missing DurableExecutionArn in the response). +/// +public sealed class CloudTestException : Exception +{ + /// + /// Creates a new cloud test exception. + /// + public CloudTestException(string message) : base(message) { } + + /// + /// Creates a new cloud test exception with an inner exception. + /// + public CloudTestException(string message, Exception innerException) : base(message, innerException) { } +} diff --git a/Libraries/src/Amazon.Lambda.DurableExecution.Testing/CloudTestRunnerOptions.cs b/Libraries/src/Amazon.Lambda.DurableExecution.Testing/CloudTestRunnerOptions.cs new file mode 100644 index 000000000..af5bd8b65 --- /dev/null +++ b/Libraries/src/Amazon.Lambda.DurableExecution.Testing/CloudTestRunnerOptions.cs @@ -0,0 +1,28 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +using Amazon.Lambda.Core; + +namespace Amazon.Lambda.DurableExecution.Testing; + +/// +/// Configuration for the cloud durable test runner. +/// +public sealed record CloudTestRunnerOptions +{ + /// + /// Interval between state-polling calls. Default: 2 seconds. + /// + public TimeSpan PollInterval { get; init; } = TimeSpan.FromSeconds(2); + + /// + /// Wall-clock timeout for polling operations. Default: 5 minutes. + /// + public TimeSpan DefaultTimeout { get; init; } = TimeSpan.FromMinutes(5); + + /// + /// Serializer used for payload and result deserialization. When null, uses + /// DefaultLambdaJsonSerializer from Amazon.Lambda.Serialization.SystemTextJson. + /// + public ILambdaSerializer? Serializer { get; init; } +} diff --git a/Libraries/src/Amazon.Lambda.DurableExecution.Testing/IDurableTestRunner.cs b/Libraries/src/Amazon.Lambda.DurableExecution.Testing/IDurableTestRunner.cs new file mode 100644 index 000000000..dd45089ad --- /dev/null +++ b/Libraries/src/Amazon.Lambda.DurableExecution.Testing/IDurableTestRunner.cs @@ -0,0 +1,69 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +namespace Amazon.Lambda.DurableExecution.Testing; + +/// +/// Common interface for local and cloud durable test runners. Tests written +/// against this interface can run unchanged against either backend. +/// +public interface IDurableTestRunner +{ + /// + /// Drives the workflow to a terminal state and returns the result. + /// Throws if the workflow requires callbacks — use the two-call pattern instead. + /// + Task> RunAsync( + TInput input, + TimeSpan? timeout = null, + CancellationToken cancellationToken = default); + + /// + /// Starts the workflow and returns the durable execution ARN. + /// Use with for callback workflows. + /// + Task StartAsync( + TInput input, + TimeSpan? timeout = null, + CancellationToken cancellationToken = default); + + /// + /// Blocks until the workflow reaches a callback point and returns the callback ID. + /// + Task WaitForCallbackAsync( + string durableExecutionArn, + string? name = null, + TimeSpan? timeout = null, + CancellationToken cancellationToken = default); + + /// + /// Sends a success result to a waiting callback. + /// + Task SendCallbackSuccessAsync( + string callbackId, + TResult result, + CancellationToken cancellationToken = default); + + /// + /// Sends a failure to a waiting callback. + /// + Task SendCallbackFailureAsync( + string callbackId, + ErrorObject? error = null, + CancellationToken cancellationToken = default); + + /// + /// Sends a heartbeat to keep a callback alive. + /// + Task SendCallbackHeartbeatAsync( + string callbackId, + CancellationToken cancellationToken = default); + + /// + /// Waits for the workflow to reach a terminal state and returns the result. + /// + Task> WaitForResultAsync( + string durableExecutionArn, + TimeSpan? timeout = null, + CancellationToken cancellationToken = default); +} diff --git a/Libraries/src/Amazon.Lambda.DurableExecution.Testing/OperationKind.cs b/Libraries/src/Amazon.Lambda.DurableExecution.Testing/OperationKind.cs new file mode 100644 index 000000000..b7fde87ef --- /dev/null +++ b/Libraries/src/Amazon.Lambda.DurableExecution.Testing/OperationKind.cs @@ -0,0 +1,28 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +namespace Amazon.Lambda.DurableExecution.Testing; + +/// +/// Classifies a by the underlying operation type. +/// +public enum OperationKind +{ + /// A step operation (user function call). + Step, + + /// A wait/timer operation. + Wait, + + /// A callback operation (external signal). + Callback, + + /// A chained-invoke operation (durable-to-durable or durable-to-plain call). + ChainedInvoke, + + /// A context operation (child context, parallel, map). + Context, + + /// The top-level execution operation. + Execution +} diff --git a/Libraries/src/Amazon.Lambda.DurableExecution.Testing/OperationStatus.cs b/Libraries/src/Amazon.Lambda.DurableExecution.Testing/OperationStatus.cs new file mode 100644 index 000000000..7e304d097 --- /dev/null +++ b/Libraries/src/Amazon.Lambda.DurableExecution.Testing/OperationStatus.cs @@ -0,0 +1,36 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +namespace Amazon.Lambda.DurableExecution.Testing; + +/// +/// String constants matching the wire-format operation statuses. +/// Uses a static class instead of an enum so values stay in lockstep +/// with from the runtime package. +/// +public static class OperationStatus +{ + /// The operation has started. + public const string Started = "STARTED"; + + /// The operation completed successfully. + public const string Succeeded = "SUCCEEDED"; + + /// The operation failed. + public const string Failed = "FAILED"; + + /// The operation is pending. + public const string Pending = "PENDING"; + + /// The operation timed out. + public const string TimedOut = "TIMED_OUT"; + + /// The operation was cancelled. + public const string Cancelled = "CANCELLED"; + + /// The operation was stopped. + public const string Stopped = "STOPPED"; + + /// The operation is ready to resume. + public const string Ready = "READY"; +} diff --git a/Libraries/src/Amazon.Lambda.DurableExecution.Testing/TestExecutionFailedException.cs b/Libraries/src/Amazon.Lambda.DurableExecution.Testing/TestExecutionFailedException.cs new file mode 100644 index 000000000..6441b77d0 --- /dev/null +++ b/Libraries/src/Amazon.Lambda.DurableExecution.Testing/TestExecutionFailedException.cs @@ -0,0 +1,41 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +namespace Amazon.Lambda.DurableExecution.Testing; + +/// +/// Thrown by when the workflow +/// did not complete successfully. +/// +public sealed class TestExecutionFailedException : Exception +{ + /// The final status of the workflow. + public InvocationStatus FinalStatus { get; } + + /// The error that caused the failure, if available. + public ErrorObject? FailureError { get; } + + /// All recorded steps at the time of failure. + public IReadOnlyList Steps { get; } + + internal TestExecutionFailedException( + InvocationStatus finalStatus, + ErrorObject? failureError, + IReadOnlyList steps) + : base(FormatMessage(finalStatus, failureError)) + { + FinalStatus = finalStatus; + FailureError = failureError; + Steps = steps; + } + + private static string FormatMessage(InvocationStatus status, ErrorObject? error) + { + var msg = $"Workflow execution did not succeed. Final status: {status}."; + if (error is not null) + { + msg += $" Error: [{error.ErrorType}] {error.ErrorMessage}"; + } + return msg; + } +} diff --git a/Libraries/src/Amazon.Lambda.DurableExecution.Testing/TestExecutionLimitException.cs b/Libraries/src/Amazon.Lambda.DurableExecution.Testing/TestExecutionLimitException.cs new file mode 100644 index 000000000..333bc63f6 --- /dev/null +++ b/Libraries/src/Amazon.Lambda.DurableExecution.Testing/TestExecutionLimitException.cs @@ -0,0 +1,40 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +namespace Amazon.Lambda.DurableExecution.Testing; + +/// +/// Thrown when the workflow does not reach a terminal state within the configured +/// limit. +/// +public sealed class TestExecutionLimitException : Exception +{ + /// The maximum invocations configured. + public int MaxInvocations { get; } + + /// Total operations recorded at the time of the limit breach. + public int TotalOperations { get; } + + internal TestExecutionLimitException(int maxInvocations, int totalOperations) + : base(FormatMessage(maxInvocations, totalOperations)) + { + MaxInvocations = maxInvocations; + TotalOperations = totalOperations; + } + + private static string FormatMessage(int maxInvocations, int totalOperations) + { + return $""" + Workflow did not reach a terminal state within {maxInvocations} invocations. + + Possible causes: + - Workflow uses WaitForCallbackAsync — call StartAsync/WaitForCallbackAsync/SendCallbackSuccessAsync instead of RunAsync. + - Workflow uses InvokeAsync for a function that isn't registered — call runner.RegisterFunction("name", handler). + - Workflow has an infinite retry loop. + - Workflow uses WaitForConditionAsync that never returns true. + + Set TestRunnerOptions.MaxInvocations to a higher value if your workflow is legitimately long. + Total operations recorded: {totalOperations}. + """; + } +} diff --git a/Libraries/src/Amazon.Lambda.DurableExecution.Testing/TestResult.cs b/Libraries/src/Amazon.Lambda.DurableExecution.Testing/TestResult.cs new file mode 100644 index 000000000..5f5827fe7 --- /dev/null +++ b/Libraries/src/Amazon.Lambda.DurableExecution.Testing/TestResult.cs @@ -0,0 +1,151 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +namespace Amazon.Lambda.DurableExecution.Testing; + +/// +/// The outcome of a durable workflow execution, including the terminal result +/// and every recorded operation for step-level inspection. +/// +public sealed class TestResult +{ + /// The terminal status of the workflow. + public InvocationStatus Status { get; } + + /// + /// The workflow result when is . + /// Default when not succeeded. + /// + public TOutput? Result { get; } + + /// The error when is . + public ErrorObject? Error { get; } + + /// The durable execution ARN for this run. + public string DurableExecutionArn { get; } + + /// + /// Number of handler invocations the local runner used to drive the workflow + /// to completion. -1 for cloud runner (unknown). + /// + public int InvocationCount { get; } + + /// Every recorded operation except the top-level EXECUTION operation. + public IReadOnlyList Steps { get; } + + internal TestResult( + InvocationStatus status, + TOutput? result, + ErrorObject? error, + string durableExecutionArn, + int invocationCount, + IReadOnlyList steps) + { + Status = status; + Result = result; + Error = error; + DurableExecutionArn = durableExecutionArn; + InvocationCount = invocationCount; + Steps = steps; + + LinkChildren(); + } + + /// + /// Returns the first step matching . + /// Throws if no match is found. + /// + public TestStep GetStep(string name) + { + return FindStep(name) + ?? throw new InvalidOperationException( + $"No step with name '{name}' found. Available steps: [{string.Join(", ", Steps.Where(s => s.Name is not null).Select(s => s.Name))}]"); + } + + /// + /// Returns the first step matching , or null if not found. + /// + public TestStep? FindStep(string name) + { + foreach (var step in Steps) + { + if (string.Equals(step.Name, name, StringComparison.Ordinal)) + return step; + } + return null; + } + + /// + /// Returns all steps matching (e.g., parallel branches or map items). + /// + public IReadOnlyList GetSteps(string name) + { + var matches = new List(); + foreach (var step in Steps) + { + if (string.Equals(step.Name, name, StringComparison.Ordinal)) + matches.Add(step); + } + return matches; + } + + /// + /// Returns the step with the exact operation ID. + /// Throws if not found. + /// + public TestStep GetStepById(string operationId) + { + foreach (var step in Steps) + { + if (string.Equals(step.Id, operationId, StringComparison.Ordinal)) + return step; + } + throw new InvalidOperationException( + $"No step with ID '{operationId}' found."); + } + + /// + /// Returns all direct children of (operations whose ParentId matches). + /// + public IReadOnlyList GetChildren(TestStep parent) + { + return parent.Children; + } + + /// + /// Throws if is not + /// . + /// + public void EnsureSucceeded() + { + if (Status != InvocationStatus.Succeeded) + { + throw new TestExecutionFailedException(Status, Error, Steps); + } + } + + private void LinkChildren() + { + var childMap = new Dictionary>(); + foreach (var step in Steps) + { + if (step.ParentId is not null) + { + if (!childMap.TryGetValue(step.ParentId, out var children)) + { + children = new List(); + childMap[step.ParentId] = children; + } + children.Add(step); + } + } + + foreach (var step in Steps) + { + if (childMap.TryGetValue(step.Id, out var children)) + { + step.Children = children; + } + } + } +} diff --git a/Libraries/src/Amazon.Lambda.DurableExecution.Testing/TestRunnerOptions.cs b/Libraries/src/Amazon.Lambda.DurableExecution.Testing/TestRunnerOptions.cs new file mode 100644 index 000000000..f3a060a0f --- /dev/null +++ b/Libraries/src/Amazon.Lambda.DurableExecution.Testing/TestRunnerOptions.cs @@ -0,0 +1,58 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +using Amazon.Lambda.Core; +using Microsoft.Extensions.Logging; + +namespace Amazon.Lambda.DurableExecution.Testing; + +/// +/// Configuration for the local durable test runner. +/// +public sealed record TestRunnerOptions +{ + /// + /// When true, wait/timer operations complete immediately rather than + /// waiting for real wall-clock time. Default: true. + /// + public bool SkipTime { get; init; } = true; + + /// + /// Maximum number of handler invocations before throwing + /// . Default: 100. + /// + public int MaxInvocations + { + get => _maxInvocations; + init + { + if (value <= 0) + throw new ArgumentOutOfRangeException(nameof(MaxInvocations), value, "MaxInvocations must be greater than zero."); + _maxInvocations = value; + } + } + private readonly int _maxInvocations = 100; + + /// + /// Wall-clock timeout for a single RunAsync or WaitForResultAsync call. + /// Default: 30 seconds. + /// + public TimeSpan DefaultTimeout { get; init; } = TimeSpan.FromSeconds(30); + + /// + /// Serializer used for step result deserialization. When null, uses + /// DefaultLambdaJsonSerializer from Amazon.Lambda.Serialization.SystemTextJson. + /// + public ILambdaSerializer? Serializer { get; init; } + + /// + /// Logger factory for runtime logging during test execution. Optional. + /// + public ILoggerFactory? LoggerFactory { get; init; } + + /// + /// The durable execution ARN used in the test context. Override for tests that + /// assert on ARN values. Default: a synthetic test ARN. + /// + public string DurableExecutionArn { get; init; } = "arn:aws:lambda:us-east-1:123456789012:execution:test-fn:test-execution"; +} diff --git a/Libraries/src/Amazon.Lambda.DurableExecution.Testing/TestStep.cs b/Libraries/src/Amazon.Lambda.DurableExecution.Testing/TestStep.cs new file mode 100644 index 000000000..3a6f0782e --- /dev/null +++ b/Libraries/src/Amazon.Lambda.DurableExecution.Testing/TestStep.cs @@ -0,0 +1,137 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +using System.Text; +using Amazon.Lambda.Core; + +namespace Amazon.Lambda.DurableExecution.Testing; + +/// +/// A single operation recorded during workflow execution, exposed for test assertions. +/// Wraps the internal with typed accessors. +/// +public sealed class TestStep +{ + private readonly Operation _operation; + private readonly ILambdaSerializer _serializer; + + internal TestStep(Operation operation, ILambdaSerializer serializer) + { + _operation = operation; + _serializer = serializer; + } + + /// The operation's unique identifier. + public string Id => _operation.Id!; + + /// User-supplied operation name (e.g., the step name). + public string? Name => _operation.Name; + + /// Parent operation identifier, if this operation is nested. + public string? ParentId => _operation.ParentId; + + /// The kind of operation (Step, Wait, Callback, etc.). + public OperationKind Kind => MapKind(_operation.Type); + + /// + /// The sub-kind providing finer classification (e.g., "Parallel", "Map", + /// "WaitForCallback", "WaitForCondition"). Null when not applicable. + /// + public string? SubKind => _operation.SubType; + + /// The terminal status of this operation. + public string Status => _operation.Status ?? OperationStatus.Pending; + + /// + /// The attempt number (1-based) for step operations. 0 for non-step kinds. + /// + public int Attempt => _operation.StepDetails?.Attempt ?? 0; + + /// When the operation started (null if not yet started). + public DateTimeOffset? StartedAt => _operation.StartTimestamp.HasValue + ? DateTimeOffset.FromUnixTimeMilliseconds(_operation.StartTimestamp.Value) + : null; + + /// When the operation ended (null if not yet ended). + public DateTimeOffset? EndedAt => _operation.EndTimestamp.HasValue + ? DateTimeOffset.FromUnixTimeMilliseconds(_operation.EndTimestamp.Value) + : null; + + /// Elapsed wall-clock duration, or null if timestamps are missing. + public TimeSpan? Duration => StartedAt.HasValue && EndedAt.HasValue + ? EndedAt - StartedAt + : null; + + /// Child operations (linked by parent ID). Set externally by . + public IReadOnlyList Children { get; internal set; } = Array.Empty(); + + /// + /// Deserializes and returns the typed result from this operation. + /// Routes to the appropriate details property based on . + /// Returns default when no result is present. + /// + public T? GetResult() + { + var serialized = Kind switch + { + OperationKind.Step => _operation.StepDetails?.Result, + OperationKind.ChainedInvoke => _operation.ChainedInvokeDetails?.Result, + OperationKind.Context => _operation.ContextDetails?.Result, + OperationKind.Callback => _operation.CallbackDetails?.Result, + _ => null, + }; + + if (serialized is null) return default; + + using var stream = new MemoryStream(Encoding.UTF8.GetBytes(serialized)); + return _serializer.Deserialize(stream); + } + + /// + /// Returns the error from this operation, or null if no error is present. + /// Routes to the appropriate details property based on . + /// + public ErrorObject? GetError() + { + return Kind switch + { + OperationKind.Step => _operation.StepDetails?.Error, + OperationKind.ChainedInvoke => _operation.ChainedInvokeDetails?.Error, + OperationKind.Context => _operation.ContextDetails?.Error, + OperationKind.Callback => _operation.CallbackDetails?.Error, + _ => null, + }; + } + + /// + /// Returns the scheduled end time for a wait operation, or null. + /// + public DateTimeOffset? GetWaitEndsAt() + { + var ts = _operation.WaitDetails?.ScheduledEndTimestamp; + return ts.HasValue ? DateTimeOffset.FromUnixTimeMilliseconds(ts.Value) : null; + } + + /// + /// Returns the callback identifier for a callback operation, or null. + /// + public string? GetCallbackId() => _operation.CallbackDetails?.CallbackId; + + /// + /// Returns the function name for a chained-invoke operation, or null. + /// + public string? GetChainedInvokeFunctionName() => _operation.ChainedInvokeDetails is not null + ? _operation.Name + : null; + + private static OperationKind MapKind(string? type) => type switch + { + OperationTypes.Step => OperationKind.Step, + OperationTypes.Wait => OperationKind.Wait, + OperationTypes.Callback => OperationKind.Callback, + OperationTypes.ChainedInvoke => OperationKind.ChainedInvoke, + OperationTypes.Context => OperationKind.Context, + OperationTypes.Execution => OperationKind.Execution, + _ => OperationKind.Step, + }; +} diff --git a/Libraries/src/Amazon.Lambda.DurableExecution.Testing/UnregisteredSiblingFunctionException.cs b/Libraries/src/Amazon.Lambda.DurableExecution.Testing/UnregisteredSiblingFunctionException.cs new file mode 100644 index 000000000..6a32e82ea --- /dev/null +++ b/Libraries/src/Amazon.Lambda.DurableExecution.Testing/UnregisteredSiblingFunctionException.cs @@ -0,0 +1,25 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +namespace Amazon.Lambda.DurableExecution.Testing; + +/// +/// Thrown when a workflow calls InvokeAsync with a function name that has not +/// been registered via RegisterFunction or RegisterDurableFunction. +/// +public sealed class UnregisteredSiblingFunctionException : Exception +{ + /// The function name or ARN that was requested but not registered. + public string FunctionName { get; } + + /// + /// Creates a new instance for the unregistered function. + /// + public UnregisteredSiblingFunctionException(string functionName) + : base($"No handler registered for function '{functionName}'. " + + $"Call runner.RegisterFunction(\"{functionName}\", handler) or " + + $"runner.RegisterDurableFunction(\"{functionName}\", handler) before running the workflow.") + { + FunctionName = functionName; + } +} diff --git a/Libraries/test/Amazon.Lambda.DurableExecution.Testing.Tests/Amazon.Lambda.DurableExecution.Testing.Tests.csproj b/Libraries/test/Amazon.Lambda.DurableExecution.Testing.Tests/Amazon.Lambda.DurableExecution.Testing.Tests.csproj new file mode 100644 index 000000000..1f0db748f --- /dev/null +++ b/Libraries/test/Amazon.Lambda.DurableExecution.Testing.Tests/Amazon.Lambda.DurableExecution.Testing.Tests.csproj @@ -0,0 +1,36 @@ + + + + + + $(DefaultPackageTargets) + Amazon.Lambda.DurableExecution.Testing.Tests + Amazon.Lambda.DurableExecution.Testing.Tests + true + ..\..\..\buildtools\public.snk + true + enable + enable + $(NoWarn);CS1591 + true + + + + + + + + + + + + + + + + + + + + + diff --git a/Libraries/test/Amazon.Lambda.DurableExecution.Testing.Tests/ExceptionTests.cs b/Libraries/test/Amazon.Lambda.DurableExecution.Testing.Tests/ExceptionTests.cs new file mode 100644 index 000000000..26aeb9f0c --- /dev/null +++ b/Libraries/test/Amazon.Lambda.DurableExecution.Testing.Tests/ExceptionTests.cs @@ -0,0 +1,85 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +using Amazon.Lambda.DurableExecution.Testing; +using Amazon.Lambda.Serialization.SystemTextJson; +using Xunit; + +namespace Amazon.Lambda.DurableExecution.Testing.Tests; + +public class ExceptionTests +{ + private static readonly DefaultLambdaJsonSerializer Serializer = new(); + + [Fact] + public void TestExecutionFailedException_ContainsStatusAndError() + { + var error = new ErrorObject { ErrorType = "MyException", ErrorMessage = "it failed" }; + var steps = new[] + { + new TestStep(new Operation { Id = "op-1", Type = OperationTypes.Step, Name = "step1" }, Serializer) + }; + + var ex = new TestExecutionFailedException(InvocationStatus.Failed, error, steps); + + Assert.Equal(InvocationStatus.Failed, ex.FinalStatus); + Assert.Same(error, ex.FailureError); + Assert.Single(ex.Steps); + Assert.Contains("MyException", ex.Message); + Assert.Contains("it failed", ex.Message); + Assert.Contains("Failed", ex.Message); + } + + [Fact] + public void TestExecutionFailedException_NullError_StillFormatsMessage() + { + var ex = new TestExecutionFailedException( + InvocationStatus.Pending, null, Array.Empty()); + + Assert.Equal(InvocationStatus.Pending, ex.FinalStatus); + Assert.Null(ex.FailureError); + Assert.Contains("Pending", ex.Message); + } + + [Fact] + public void TestExecutionLimitException_ContainsDiagnostics() + { + var ex = new TestExecutionLimitException(100, 47); + + Assert.Equal(100, ex.MaxInvocations); + Assert.Equal(47, ex.TotalOperations); + Assert.Contains("100", ex.Message); + Assert.Contains("47", ex.Message); + Assert.Contains("WaitForCallbackAsync", ex.Message); + Assert.Contains("RegisterFunction", ex.Message); + Assert.Contains("WaitForConditionAsync", ex.Message); + } + + [Fact] + public void UnregisteredSiblingFunctionException_ContainsFunctionName() + { + var ex = new UnregisteredSiblingFunctionException("process-payment"); + + Assert.Equal("process-payment", ex.FunctionName); + Assert.Contains("process-payment", ex.Message); + Assert.Contains("RegisterFunction", ex.Message); + Assert.Contains("RegisterDurableFunction", ex.Message); + } + + [Fact] + public void CloudTestException_BasicConstruction() + { + var ex = new CloudTestException("no ARN in response"); + Assert.Equal("no ARN in response", ex.Message); + Assert.Null(ex.InnerException); + } + + [Fact] + public void CloudTestException_WithInnerException() + { + var inner = new InvalidOperationException("underlying"); + var ex = new CloudTestException("wrapper message", inner); + Assert.Equal("wrapper message", ex.Message); + Assert.Same(inner, ex.InnerException); + } +} diff --git a/Libraries/test/Amazon.Lambda.DurableExecution.Testing.Tests/OptionsValidationTests.cs b/Libraries/test/Amazon.Lambda.DurableExecution.Testing.Tests/OptionsValidationTests.cs new file mode 100644 index 000000000..18ba9a87a --- /dev/null +++ b/Libraries/test/Amazon.Lambda.DurableExecution.Testing.Tests/OptionsValidationTests.cs @@ -0,0 +1,75 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +using Amazon.Lambda.DurableExecution.Testing; +using Xunit; + +namespace Amazon.Lambda.DurableExecution.Testing.Tests; + +public class OptionsValidationTests +{ + [Fact] + public void TestRunnerOptions_Defaults() + { + var options = new TestRunnerOptions(); + + Assert.True(options.SkipTime); + Assert.Equal(100, options.MaxInvocations); + Assert.Equal(TimeSpan.FromSeconds(30), options.DefaultTimeout); + Assert.Null(options.Serializer); + Assert.Null(options.LoggerFactory); + Assert.Equal("arn:aws:lambda:us-east-1:123456789012:execution:test-fn:test-execution", options.DurableExecutionArn); + } + + [Fact] + public void TestRunnerOptions_MaxInvocations_Zero_Throws() + { + Assert.Throws(() => new TestRunnerOptions { MaxInvocations = 0 }); + } + + [Fact] + public void TestRunnerOptions_MaxInvocations_Negative_Throws() + { + Assert.Throws(() => new TestRunnerOptions { MaxInvocations = -1 }); + } + + [Fact] + public void TestRunnerOptions_MaxInvocations_PositiveValue_Accepted() + { + var options = new TestRunnerOptions { MaxInvocations = 500 }; + Assert.Equal(500, options.MaxInvocations); + } + + [Fact] + public void TestRunnerOptions_CustomArn() + { + var options = new TestRunnerOptions + { + DurableExecutionArn = "arn:aws:lambda:eu-west-1:999:execution:my-fn:custom" + }; + Assert.Equal("arn:aws:lambda:eu-west-1:999:execution:my-fn:custom", options.DurableExecutionArn); + } + + [Fact] + public void CloudTestRunnerOptions_Defaults() + { + var options = new CloudTestRunnerOptions(); + + Assert.Equal(TimeSpan.FromSeconds(2), options.PollInterval); + Assert.Equal(TimeSpan.FromMinutes(5), options.DefaultTimeout); + Assert.Null(options.Serializer); + } + + [Fact] + public void CloudTestRunnerOptions_CustomValues() + { + var options = new CloudTestRunnerOptions + { + PollInterval = TimeSpan.FromSeconds(5), + DefaultTimeout = TimeSpan.FromMinutes(10) + }; + + Assert.Equal(TimeSpan.FromSeconds(5), options.PollInterval); + Assert.Equal(TimeSpan.FromMinutes(10), options.DefaultTimeout); + } +} diff --git a/Libraries/test/Amazon.Lambda.DurableExecution.Testing.Tests/TestResultTests.cs b/Libraries/test/Amazon.Lambda.DurableExecution.Testing.Tests/TestResultTests.cs new file mode 100644 index 000000000..4305aa3db --- /dev/null +++ b/Libraries/test/Amazon.Lambda.DurableExecution.Testing.Tests/TestResultTests.cs @@ -0,0 +1,240 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +using Amazon.Lambda.DurableExecution.Testing; +using Amazon.Lambda.Serialization.SystemTextJson; +using Xunit; + +namespace Amazon.Lambda.DurableExecution.Testing.Tests; + +public class TestResultTests +{ + private static readonly DefaultLambdaJsonSerializer Serializer = new(); + + [Fact] + public void EnsureSucceeded_DoesNotThrow_WhenSucceeded() + { + var result = CreateResult(InvocationStatus.Succeeded, "hello"); + result.EnsureSucceeded(); + } + + [Fact] + public void EnsureSucceeded_Throws_WhenFailed() + { + var error = new ErrorObject { ErrorType = "TestException", ErrorMessage = "something broke" }; + var result = CreateResult(InvocationStatus.Failed, default(string), error); + + var ex = Assert.Throws(() => result.EnsureSucceeded()); + Assert.Equal(InvocationStatus.Failed, ex.FinalStatus); + Assert.Equal("TestException", ex.FailureError?.ErrorType); + Assert.Contains("TestException", ex.Message); + Assert.Contains("something broke", ex.Message); + } + + [Fact] + public void EnsureSucceeded_Throws_WhenPending() + { + var result = CreateResult(InvocationStatus.Pending, default(string)); + var ex = Assert.Throws(() => result.EnsureSucceeded()); + Assert.Equal(InvocationStatus.Pending, ex.FinalStatus); + } + + [Fact] + public void GetStep_ReturnsFirstMatch() + { + var steps = new[] + { + MakeStep("op-1", "step_a"), + MakeStep("op-2", "step_b"), + MakeStep("op-3", "step_a"), + }; + + var result = new TestResult( + InvocationStatus.Succeeded, "done", null, "arn:test", 1, steps); + + var found = result.GetStep("step_a"); + Assert.Equal("op-1", found.Id); + } + + [Fact] + public void GetStep_Throws_WhenNotFound() + { + var steps = new[] { MakeStep("op-1", "step_a") }; + var result = new TestResult( + InvocationStatus.Succeeded, "done", null, "arn:test", 1, steps); + + var ex = Assert.Throws(() => result.GetStep("missing")); + Assert.Contains("missing", ex.Message); + Assert.Contains("step_a", ex.Message); + } + + [Fact] + public void FindStep_ReturnsNull_WhenNotFound() + { + var steps = new[] { MakeStep("op-1", "step_a") }; + var result = new TestResult( + InvocationStatus.Succeeded, "done", null, "arn:test", 1, steps); + + Assert.Null(result.FindStep("missing")); + } + + [Fact] + public void FindStep_ReturnsMatch() + { + var steps = new[] { MakeStep("op-1", "step_a") }; + var result = new TestResult( + InvocationStatus.Succeeded, "done", null, "arn:test", 1, steps); + + var found = result.FindStep("step_a"); + Assert.NotNull(found); + Assert.Equal("op-1", found!.Id); + } + + [Fact] + public void GetSteps_ReturnsAllMatches() + { + var steps = new[] + { + MakeStep("op-1", "process_item"), + MakeStep("op-2", "other"), + MakeStep("op-3", "process_item"), + MakeStep("op-4", "process_item"), + }; + + var result = new TestResult( + InvocationStatus.Succeeded, "done", null, "arn:test", 1, steps); + + var found = result.GetSteps("process_item"); + Assert.Equal(3, found.Count); + Assert.Equal("op-1", found[0].Id); + Assert.Equal("op-3", found[1].Id); + Assert.Equal("op-4", found[2].Id); + } + + [Fact] + public void GetSteps_ReturnsEmpty_WhenNoMatches() + { + var steps = new[] { MakeStep("op-1", "step_a") }; + var result = new TestResult( + InvocationStatus.Succeeded, "done", null, "arn:test", 1, steps); + + Assert.Empty(result.GetSteps("missing")); + } + + [Fact] + public void GetStepById_ReturnsMatch() + { + var steps = new[] + { + MakeStep("op-1", "step_a"), + MakeStep("op-2", "step_b"), + }; + + var result = new TestResult( + InvocationStatus.Succeeded, "done", null, "arn:test", 1, steps); + + var found = result.GetStepById("op-2"); + Assert.Equal("step_b", found.Name); + } + + [Fact] + public void GetStepById_Throws_WhenNotFound() + { + var steps = new[] { MakeStep("op-1", "step_a") }; + var result = new TestResult( + InvocationStatus.Succeeded, "done", null, "arn:test", 1, steps); + + var ex = Assert.Throws(() => result.GetStepById("op-999")); + Assert.Contains("op-999", ex.Message); + } + + [Fact] + public void Children_LinkedByParentId() + { + var parent = new Operation { Id = "op-parent", Type = OperationTypes.Context, Name = "batch" }; + var child1 = new Operation { Id = "op-c1", Type = OperationTypes.Step, Name = "item_1", ParentId = "op-parent" }; + var child2 = new Operation { Id = "op-c2", Type = OperationTypes.Step, Name = "item_2", ParentId = "op-parent" }; + var unrelated = new Operation { Id = "op-other", Type = OperationTypes.Step, Name = "other" }; + + var steps = new[] + { + new TestStep(parent, Serializer), + new TestStep(child1, Serializer), + new TestStep(child2, Serializer), + new TestStep(unrelated, Serializer), + }; + + var result = new TestResult( + InvocationStatus.Succeeded, "done", null, "arn:test", 1, steps); + + var parentStep = result.GetStep("batch"); + Assert.Equal(2, parentStep.Children.Count); + Assert.Equal("op-c1", parentStep.Children[0].Id); + Assert.Equal("op-c2", parentStep.Children[1].Id); + + var otherStep = result.GetStep("other"); + Assert.Empty(otherStep.Children); + } + + [Fact] + public void GetChildren_ReturnsSameAsProperty() + { + var parent = new Operation { Id = "op-parent", Type = OperationTypes.Context, Name = "batch" }; + var child = new Operation { Id = "op-c1", Type = OperationTypes.Step, Name = "item_1", ParentId = "op-parent" }; + + var steps = new[] + { + new TestStep(parent, Serializer), + new TestStep(child, Serializer), + }; + + var result = new TestResult( + InvocationStatus.Succeeded, "done", null, "arn:test", 1, steps); + + var parentStep = result.GetStep("batch"); + Assert.Same(parentStep.Children, result.GetChildren(parentStep)); + } + + [Fact] + public void Properties_ExposedCorrectly() + { + var result = new TestResult( + InvocationStatus.Succeeded, 42, null, "arn:aws:lambda:us-east-1:123:execution:fn:exec", 5, + Array.Empty()); + + Assert.Equal(InvocationStatus.Succeeded, result.Status); + Assert.Equal(42, result.Result); + Assert.Null(result.Error); + Assert.Equal("arn:aws:lambda:us-east-1:123:execution:fn:exec", result.DurableExecutionArn); + Assert.Equal(5, result.InvocationCount); + Assert.Empty(result.Steps); + } + + [Fact] + public void EmptySteps_NoExceptions() + { + var result = new TestResult( + InvocationStatus.Succeeded, "ok", null, "arn:test", 1, Array.Empty()); + + Assert.Empty(result.Steps); + Assert.Empty(result.GetSteps("anything")); + } + + private static TestResult CreateResult(InvocationStatus status, T? output, ErrorObject? error = null) + { + return new TestResult(status, output, error, "arn:test", 1, Array.Empty()); + } + + private static TestStep MakeStep(string id, string? name, string? parentId = null) + { + var op = new Operation + { + Id = id, + Name = name, + Type = OperationTypes.Step, + Status = OperationStatuses.Succeeded, + ParentId = parentId + }; + return new TestStep(op, Serializer); + } +} diff --git a/Libraries/test/Amazon.Lambda.DurableExecution.Testing.Tests/TestStepTests.cs b/Libraries/test/Amazon.Lambda.DurableExecution.Testing.Tests/TestStepTests.cs new file mode 100644 index 000000000..9d99def88 --- /dev/null +++ b/Libraries/test/Amazon.Lambda.DurableExecution.Testing.Tests/TestStepTests.cs @@ -0,0 +1,278 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +using Amazon.Lambda.DurableExecution.Testing; +using Amazon.Lambda.Serialization.SystemTextJson; +using Xunit; + +namespace Amazon.Lambda.DurableExecution.Testing.Tests; + +public class TestStepTests +{ + private static readonly DefaultLambdaJsonSerializer Serializer = new(); + + [Theory] + [InlineData(OperationTypes.Step, OperationKind.Step)] + [InlineData(OperationTypes.Wait, OperationKind.Wait)] + [InlineData(OperationTypes.Callback, OperationKind.Callback)] + [InlineData(OperationTypes.ChainedInvoke, OperationKind.ChainedInvoke)] + [InlineData(OperationTypes.Context, OperationKind.Context)] + [InlineData(OperationTypes.Execution, OperationKind.Execution)] + public void Kind_MapsFromOperationType(string operationType, OperationKind expected) + { + var op = new Operation { Id = "op-1", Type = operationType }; + var step = new TestStep(op, Serializer); + Assert.Equal(expected, step.Kind); + } + + [Fact] + public void Properties_ExposedFromOperation() + { + var op = new Operation + { + Id = "op-123", + Name = "validate_order", + ParentId = "op-parent", + Type = OperationTypes.Step, + SubType = OperationSubTypes.Step, + Status = OperationStatuses.Succeeded, + StartTimestamp = 1700000000000, + EndTimestamp = 1700000001000, + StepDetails = new StepDetails { Attempt = 2 } + }; + + var step = new TestStep(op, Serializer); + + Assert.Equal("op-123", step.Id); + Assert.Equal("validate_order", step.Name); + Assert.Equal("op-parent", step.ParentId); + Assert.Equal(OperationKind.Step, step.Kind); + Assert.Equal(OperationSubTypes.Step, step.SubKind); + Assert.Equal(OperationStatus.Succeeded, step.Status); + Assert.Equal(2, step.Attempt); + Assert.NotNull(step.StartedAt); + Assert.NotNull(step.EndedAt); + Assert.NotNull(step.Duration); + Assert.Equal(TimeSpan.FromSeconds(1), step.Duration); + } + + [Fact] + public void Attempt_ReturnsZero_ForNonStepKind() + { + var op = new Operation { Id = "op-1", Type = OperationTypes.Wait }; + var step = new TestStep(op, Serializer); + Assert.Equal(0, step.Attempt); + } + + [Fact] + public void Status_DefaultsToPending_WhenNull() + { + var op = new Operation { Id = "op-1", Type = OperationTypes.Step, Status = null }; + var step = new TestStep(op, Serializer); + Assert.Equal(OperationStatus.Pending, step.Status); + } + + [Fact] + public void Timestamps_Null_WhenNotSet() + { + var op = new Operation { Id = "op-1", Type = OperationTypes.Step }; + var step = new TestStep(op, Serializer); + Assert.Null(step.StartedAt); + Assert.Null(step.EndedAt); + Assert.Null(step.Duration); + } + + [Fact] + public void GetResult_DeserializesStepResult() + { + var op = new Operation + { + Id = "op-1", + Type = OperationTypes.Step, + StepDetails = new StepDetails { Result = """{"Value":42}""" } + }; + + var step = new TestStep(op, Serializer); + var result = step.GetResult(); + + Assert.NotNull(result); + Assert.Equal(42, result!.Value); + } + + [Fact] + public void GetResult_DeserializesChainedInvokeResult() + { + var op = new Operation + { + Id = "op-1", + Type = OperationTypes.ChainedInvoke, + ChainedInvokeDetails = new ChainedInvokeDetails { Result = """{"Value":99}""" } + }; + + var step = new TestStep(op, Serializer); + var result = step.GetResult(); + + Assert.NotNull(result); + Assert.Equal(99, result!.Value); + } + + [Fact] + public void GetResult_DeserializesContextResult() + { + var op = new Operation + { + Id = "op-1", + Type = OperationTypes.Context, + ContextDetails = new ContextDetails { Result = """{"Value":7}""" } + }; + + var step = new TestStep(op, Serializer); + var result = step.GetResult(); + + Assert.NotNull(result); + Assert.Equal(7, result!.Value); + } + + [Fact] + public void GetResult_DeserializesCallbackResult() + { + var op = new Operation + { + Id = "op-1", + Type = OperationTypes.Callback, + CallbackDetails = new CallbackDetails { Result = """{"Value":3}""" } + }; + + var step = new TestStep(op, Serializer); + var result = step.GetResult(); + + Assert.NotNull(result); + Assert.Equal(3, result!.Value); + } + + [Fact] + public void GetResult_ReturnsDefault_WhenNoResult() + { + var op = new Operation { Id = "op-1", Type = OperationTypes.Step, StepDetails = new StepDetails() }; + var step = new TestStep(op, Serializer); + Assert.Null(step.GetResult()); + } + + [Fact] + public void GetResult_ReturnsDefault_ForWaitKind() + { + var op = new Operation { Id = "op-1", Type = OperationTypes.Wait }; + var step = new TestStep(op, Serializer); + Assert.Null(step.GetResult()); + } + + [Fact] + public void GetError_ReturnsStepError() + { + var error = new ErrorObject { ErrorType = "TestEx", ErrorMessage = "boom" }; + var op = new Operation + { + Id = "op-1", + Type = OperationTypes.Step, + StepDetails = new StepDetails { Error = error } + }; + + var step = new TestStep(op, Serializer); + var result = step.GetError(); + + Assert.NotNull(result); + Assert.Equal("TestEx", result!.ErrorType); + Assert.Equal("boom", result.ErrorMessage); + } + + [Fact] + public void GetError_ReturnsNull_WhenNoError() + { + var op = new Operation { Id = "op-1", Type = OperationTypes.Step, StepDetails = new StepDetails() }; + var step = new TestStep(op, Serializer); + Assert.Null(step.GetError()); + } + + [Fact] + public void GetWaitEndsAt_ReturnsTimestamp() + { + var op = new Operation + { + Id = "op-1", + Type = OperationTypes.Wait, + WaitDetails = new WaitDetails { ScheduledEndTimestamp = 1700000000000 } + }; + + var step = new TestStep(op, Serializer); + var result = step.GetWaitEndsAt(); + + Assert.NotNull(result); + Assert.Equal(DateTimeOffset.FromUnixTimeMilliseconds(1700000000000), result); + } + + [Fact] + public void GetWaitEndsAt_ReturnsNull_WhenNoWaitDetails() + { + var op = new Operation { Id = "op-1", Type = OperationTypes.Step }; + var step = new TestStep(op, Serializer); + Assert.Null(step.GetWaitEndsAt()); + } + + [Fact] + public void GetCallbackId_ReturnsId() + { + var op = new Operation + { + Id = "op-1", + Type = OperationTypes.Callback, + CallbackDetails = new CallbackDetails { CallbackId = "cb-abc" } + }; + + var step = new TestStep(op, Serializer); + Assert.Equal("cb-abc", step.GetCallbackId()); + } + + [Fact] + public void GetCallbackId_ReturnsNull_WhenNoCallbackDetails() + { + var op = new Operation { Id = "op-1", Type = OperationTypes.Step }; + var step = new TestStep(op, Serializer); + Assert.Null(step.GetCallbackId()); + } + + [Fact] + public void GetChainedInvokeFunctionName_ReturnsName() + { + var op = new Operation + { + Id = "op-1", + Name = "process-payment", + Type = OperationTypes.ChainedInvoke, + ChainedInvokeDetails = new ChainedInvokeDetails() + }; + + var step = new TestStep(op, Serializer); + Assert.Equal("process-payment", step.GetChainedInvokeFunctionName()); + } + + [Fact] + public void GetChainedInvokeFunctionName_ReturnsNull_ForNonInvokeKind() + { + var op = new Operation { Id = "op-1", Name = "some-step", Type = OperationTypes.Step }; + var step = new TestStep(op, Serializer); + Assert.Null(step.GetChainedInvokeFunctionName()); + } + + [Fact] + public void Children_EmptyByDefault() + { + var op = new Operation { Id = "op-1", Type = OperationTypes.Context }; + var step = new TestStep(op, Serializer); + Assert.Empty(step.Children); + } + + private sealed class TestPayload + { + public int Value { get; set; } + } +} From 5800f4212b57b30c078a163289a050f2105f1c69 Mon Sep 17 00:00:00 2001 From: Garrett Beatty Date: Tue, 16 Jun 2026 15:50:55 -0400 Subject: [PATCH 3/9] Add InMemoryOperationStore, CheckpointProcessor, and InMemoryDurableServiceClient The core state machine for the local test runner: - InMemoryOperationStore: per-execution operation storage with token tracking - CheckpointProcessor: maps OperationUpdate actions (START/SUCCEED/FAIL/RETRY) to Operation status transitions, mints callback IDs, applies time skipping - InMemoryDurableServiceClient: implements IDurableServiceClient by delegating to the processor and store Includes unit tests for both store (isolation, ordering, upsert) and processor (all action types, time skipping, callback ID minting, WaitForCondition retry). --- .../CheckpointProcessor.cs | 266 ++++++++++++++++ .../InMemoryDurableServiceClient.cs | 52 +++ .../InMemoryOperationStore.cs | 76 +++++ .../CheckpointProcessorTests.cs | 295 ++++++++++++++++++ .../InMemoryOperationStoreTests.cs | 115 +++++++ 5 files changed, 804 insertions(+) create mode 100644 Libraries/src/Amazon.Lambda.DurableExecution.Testing/CheckpointProcessor.cs create mode 100644 Libraries/src/Amazon.Lambda.DurableExecution.Testing/InMemoryDurableServiceClient.cs create mode 100644 Libraries/src/Amazon.Lambda.DurableExecution.Testing/InMemoryOperationStore.cs create mode 100644 Libraries/test/Amazon.Lambda.DurableExecution.Testing.Tests/CheckpointProcessorTests.cs create mode 100644 Libraries/test/Amazon.Lambda.DurableExecution.Testing.Tests/InMemoryOperationStoreTests.cs diff --git a/Libraries/src/Amazon.Lambda.DurableExecution.Testing/CheckpointProcessor.cs b/Libraries/src/Amazon.Lambda.DurableExecution.Testing/CheckpointProcessor.cs new file mode 100644 index 000000000..fb621c659 --- /dev/null +++ b/Libraries/src/Amazon.Lambda.DurableExecution.Testing/CheckpointProcessor.cs @@ -0,0 +1,266 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +using Amazon.Lambda.Model; +using SdkOperationUpdate = Amazon.Lambda.Model.OperationUpdate; + +namespace Amazon.Lambda.DurableExecution.Testing; + +/// +/// Processes checkpoint updates against the in-memory operation store. +/// Handles action-to-status mapping, callback ID minting, time skipping, +/// and producing the "new operations" response the runtime expects. +/// +internal sealed class CheckpointProcessor +{ + private readonly InMemoryOperationStore _store; + private readonly bool _skipTime; + + public CheckpointProcessor(InMemoryOperationStore store, bool skipTime) + { + _store = store; + _skipTime = skipTime; + } + + /// + /// Processes a batch of updates and returns the new checkpoint token + /// and any operations that were created or modified (to feed back to + /// the runtime's onNewOperations callback). + /// + public (string NewToken, IReadOnlyList NewOperations) Process( + string arn, + string? currentToken, + IReadOnlyList updates) + { + var newOperations = new List(); + + foreach (var update in updates) + { + var operation = ApplyUpdate(arn, update); + newOperations.Add(operation); + } + + var newToken = _store.IncrementToken(arn); + return (newToken, newOperations); + } + + private Operation ApplyUpdate(string arn, SdkOperationUpdate update) + { + var existing = _store.GetOperation(arn, update.Id); + var operation = existing ?? new Operation { Id = update.Id }; + + operation.Type = update.Type?.Value ?? operation.Type; + operation.Name = update.Name ?? operation.Name; + operation.ParentId = update.ParentId ?? operation.ParentId; + operation.SubType = update.SubType ?? operation.SubType; + + var action = update.Action?.Value; + ApplyAction(operation, action, update); + + if (_skipTime) + ApplyTimeSkipping(operation, action); + + _store.Upsert(arn, operation); + return operation; + } + + private static void ApplyAction(Operation operation, string? action, SdkOperationUpdate update) + { + switch (action) + { + case "START": + operation.Status = OperationStatuses.Started; + operation.StartTimestamp = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(); + ApplyStartDetails(operation, update); + break; + + case "SUCCEED": + operation.Status = OperationStatuses.Succeeded; + operation.EndTimestamp = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(); + ApplySucceedDetails(operation, update); + break; + + case "FAIL": + operation.Status = OperationStatuses.Failed; + operation.EndTimestamp = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(); + ApplyFailDetails(operation, update); + break; + + case "RETRY": + operation.Status = OperationStatuses.Pending; + ApplyRetryDetails(operation, update); + break; + + case "CANCEL": + operation.Status = OperationStatuses.Cancelled; + operation.EndTimestamp = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(); + break; + } + } + + private static void ApplyStartDetails(Operation operation, SdkOperationUpdate update) + { + switch (operation.Type) + { + case OperationTypes.Step: + operation.StepDetails ??= new StepDetails(); + operation.StepDetails.Attempt = (operation.StepDetails.Attempt ?? 0) + 1; + break; + + case OperationTypes.Wait: + operation.WaitDetails ??= new WaitDetails(); + if (update.WaitOptions?.WaitSeconds is { } seconds) + { + operation.WaitDetails.ScheduledEndTimestamp = + DateTimeOffset.UtcNow.AddSeconds(seconds).ToUnixTimeMilliseconds(); + } + break; + + case OperationTypes.Callback: + operation.CallbackDetails ??= new CallbackDetails(); + operation.CallbackDetails.CallbackId = $"cb-{operation.Id}"; + break; + + case OperationTypes.ChainedInvoke: + operation.ChainedInvokeDetails ??= new ChainedInvokeDetails(); + break; + + case OperationTypes.Context: + operation.ContextDetails ??= new ContextDetails(); + break; + + case OperationTypes.Execution: + operation.ExecutionDetails ??= new ExecutionDetails(); + break; + } + } + + private static void ApplySucceedDetails(Operation operation, SdkOperationUpdate update) + { + var payload = update.Payload; + switch (operation.Type) + { + case OperationTypes.Step: + operation.StepDetails ??= new StepDetails(); + operation.StepDetails.Result = payload; + operation.StepDetails.Error = null; + break; + + case OperationTypes.ChainedInvoke: + operation.ChainedInvokeDetails ??= new ChainedInvokeDetails(); + operation.ChainedInvokeDetails.Result = payload; + operation.ChainedInvokeDetails.Error = null; + break; + + case OperationTypes.Context: + operation.ContextDetails ??= new ContextDetails(); + operation.ContextDetails.Result = payload; + operation.ContextDetails.Error = null; + break; + + case OperationTypes.Callback: + operation.CallbackDetails ??= new CallbackDetails(); + operation.CallbackDetails.Result = payload; + operation.CallbackDetails.Error = null; + break; + } + } + + private static void ApplyFailDetails(Operation operation, SdkOperationUpdate update) + { + var error = MapSdkError(update.Error); + switch (operation.Type) + { + case OperationTypes.Step: + operation.StepDetails ??= new StepDetails(); + operation.StepDetails.Error = error; + break; + + case OperationTypes.ChainedInvoke: + operation.ChainedInvokeDetails ??= new ChainedInvokeDetails(); + operation.ChainedInvokeDetails.Error = error; + break; + + case OperationTypes.Context: + operation.ContextDetails ??= new ContextDetails(); + operation.ContextDetails.Error = error; + break; + + case OperationTypes.Callback: + operation.CallbackDetails ??= new CallbackDetails(); + operation.CallbackDetails.Error = error; + break; + } + } + + private static void ApplyRetryDetails(Operation operation, SdkOperationUpdate update) + { + if (operation.Type == OperationTypes.Step) + { + operation.StepDetails ??= new StepDetails(); + if (update.StepOptions?.NextAttemptDelaySeconds is { } delaySeconds) + { + operation.StepDetails.NextAttemptTimestamp = + DateTimeOffset.UtcNow.AddSeconds(delaySeconds).ToUnixTimeMilliseconds(); + } + operation.StepDetails.Error = MapSdkError(update.Error); + } + + if (operation.Type == OperationTypes.Wait && operation.SubType == OperationSubTypes.WaitForCondition) + { + operation.WaitDetails ??= new WaitDetails(); + if (update.WaitOptions?.WaitSeconds is { } waitSeconds) + { + operation.WaitDetails.ScheduledEndTimestamp = + DateTimeOffset.UtcNow.AddSeconds(waitSeconds).ToUnixTimeMilliseconds(); + } + } + } + + private void ApplyTimeSkipping(Operation operation, string? action) + { + if (action == "START" && operation.Type == OperationTypes.Wait) + { + operation.Status = OperationStatuses.Succeeded; + operation.EndTimestamp = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(); + if (operation.WaitDetails != null) + { + operation.WaitDetails.ScheduledEndTimestamp = + DateTimeOffset.UtcNow.AddMilliseconds(-1).ToUnixTimeMilliseconds(); + } + } + + if (action == "RETRY" && operation.Type == OperationTypes.Step) + { + operation.Status = OperationStatuses.Ready; + if (operation.StepDetails != null) + { + operation.StepDetails.NextAttemptTimestamp = + DateTimeOffset.UtcNow.AddMilliseconds(-1).ToUnixTimeMilliseconds(); + } + } + + if (action == "RETRY" && operation.Type == OperationTypes.Wait + && operation.SubType == OperationSubTypes.WaitForCondition) + { + operation.Status = OperationStatuses.Ready; + if (operation.WaitDetails != null) + { + operation.WaitDetails.ScheduledEndTimestamp = + DateTimeOffset.UtcNow.AddMilliseconds(-1).ToUnixTimeMilliseconds(); + } + } + } + + private static ErrorObject? MapSdkError(Amazon.Lambda.Model.ErrorObject? sdkError) + { + if (sdkError == null) return null; + return new ErrorObject + { + ErrorType = sdkError.ErrorType, + ErrorMessage = sdkError.ErrorMessage, + ErrorData = sdkError.ErrorData, + StackTrace = sdkError.StackTrace + }; + } +} diff --git a/Libraries/src/Amazon.Lambda.DurableExecution.Testing/InMemoryDurableServiceClient.cs b/Libraries/src/Amazon.Lambda.DurableExecution.Testing/InMemoryDurableServiceClient.cs new file mode 100644 index 000000000..a49f6c2b1 --- /dev/null +++ b/Libraries/src/Amazon.Lambda.DurableExecution.Testing/InMemoryDurableServiceClient.cs @@ -0,0 +1,52 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +using Amazon.Lambda.DurableExecution.Services; +using SdkOperationUpdate = Amazon.Lambda.Model.OperationUpdate; + +namespace Amazon.Lambda.DurableExecution.Testing; + +/// +/// In-memory implementation of for local testing. +/// Processes checkpoint updates against the in-memory store and returns operations +/// to the runtime engine. +/// +internal sealed class InMemoryDurableServiceClient : IDurableServiceClient +{ + private readonly InMemoryOperationStore _store; + private readonly CheckpointProcessor _processor; + + public InMemoryDurableServiceClient(InMemoryOperationStore store, CheckpointProcessor processor) + { + _store = store; + _processor = processor; + } + + public Task CheckpointAsync( + string durableExecutionArn, + string? checkpointToken, + IReadOnlyList pendingOperations, + Action>? onNewOperations = null, + CancellationToken cancellationToken = default) + { + if (pendingOperations.Count == 0) + return Task.FromResult(checkpointToken); + + var (newToken, newOps) = _processor.Process(durableExecutionArn, checkpointToken, pendingOperations); + + if (onNewOperations is not null && newOps.Count > 0) + onNewOperations(newOps); + + return Task.FromResult(newToken); + } + + public Task<(List Operations, string? NextMarker)> GetExecutionStateAsync( + string durableExecutionArn, + string? checkpointToken, + string marker, + CancellationToken cancellationToken = default) + { + var allOps = _store.GetAllOperations(durableExecutionArn); + return Task.FromResult<(List, string?)>((allOps.ToList(), null)); + } +} diff --git a/Libraries/src/Amazon.Lambda.DurableExecution.Testing/InMemoryOperationStore.cs b/Libraries/src/Amazon.Lambda.DurableExecution.Testing/InMemoryOperationStore.cs new file mode 100644 index 000000000..c3dc01cc6 --- /dev/null +++ b/Libraries/src/Amazon.Lambda.DurableExecution.Testing/InMemoryOperationStore.cs @@ -0,0 +1,76 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +namespace Amazon.Lambda.DurableExecution.Testing; + +/// +/// In-memory store for operations recorded during a test execution. +/// Each execution (keyed by ARN) maintains its own isolated operation set. +/// +internal sealed class InMemoryOperationStore +{ + private readonly Dictionary _executions = new(); + + public string CurrentToken(string arn) + { + return GetOrCreate(arn).CheckpointToken; + } + + public IReadOnlyList GetAllOperations(string arn) + { + return GetOrCreate(arn).Operations; + } + + public Operation? GetOperation(string arn, string operationId) + { + var data = GetOrCreate(arn); + return data.OperationMap.TryGetValue(operationId, out var op) ? op : null; + } + + public void Upsert(string arn, Operation operation) + { + var data = GetOrCreate(arn); + if (data.OperationMap.TryGetValue(operation.Id!, out var existing)) + { + var index = data.Operations.IndexOf(existing); + data.Operations[index] = operation; + data.OperationMap[operation.Id!] = operation; + } + else + { + data.Operations.Add(operation); + data.OperationMap[operation.Id!] = operation; + } + } + + public string IncrementToken(string arn) + { + var data = GetOrCreate(arn); + data.TokenCounter++; + data.CheckpointToken = data.TokenCounter.ToString(); + return data.CheckpointToken; + } + + public int OperationCount(string arn) + { + return GetOrCreate(arn).Operations.Count; + } + + private ExecutionData GetOrCreate(string arn) + { + if (!_executions.TryGetValue(arn, out var data)) + { + data = new ExecutionData(); + _executions[arn] = data; + } + return data; + } + + private sealed class ExecutionData + { + public readonly List Operations = new(); + public readonly Dictionary OperationMap = new(); + public string CheckpointToken = "0"; + public int TokenCounter; + } +} diff --git a/Libraries/test/Amazon.Lambda.DurableExecution.Testing.Tests/CheckpointProcessorTests.cs b/Libraries/test/Amazon.Lambda.DurableExecution.Testing.Tests/CheckpointProcessorTests.cs new file mode 100644 index 000000000..ed84d32e8 --- /dev/null +++ b/Libraries/test/Amazon.Lambda.DurableExecution.Testing.Tests/CheckpointProcessorTests.cs @@ -0,0 +1,295 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +using Amazon.Lambda.DurableExecution.Testing; +using Amazon.Lambda.Model; +using Xunit; +using SdkOperationUpdate = Amazon.Lambda.Model.OperationUpdate; + +namespace Amazon.Lambda.DurableExecution.Testing.Tests; + +public class CheckpointProcessorTests +{ + private const string Arn = "arn:aws:lambda:us-east-1:123:execution:fn:exec"; + + [Fact] + public void Process_StepStart_CreatesOperation() + { + var (store, processor) = Create(skipTime: false); + + var updates = new List + { + new() { Id = "op-1", Type = OperationTypes.Step, Action = OperationAction.START, Name = "validate", SubType = OperationSubTypes.Step } + }; + + var (token, newOps) = processor.Process(Arn, null, updates); + + Assert.NotNull(token); + var op = store.GetOperation(Arn, "op-1"); + Assert.NotNull(op); + Assert.Equal(OperationStatuses.Started, op!.Status); + Assert.Equal("validate", op.Name); + Assert.Equal(1, op.StepDetails?.Attempt); + Assert.NotNull(op.StartTimestamp); + } + + [Fact] + public void Process_StepSucceed_UpdatesStatus() + { + var (store, processor) = Create(skipTime: false); + processor.Process(Arn, null, new List + { + new() { Id = "op-1", Type = OperationTypes.Step, Action = OperationAction.START, Name = "step1" } + }); + + processor.Process(Arn, "1", new List + { + new() { Id = "op-1", Type = OperationTypes.Step, Action = OperationAction.SUCCEED, Payload = """{"x":1}""" } + }); + + var op = store.GetOperation(Arn, "op-1"); + Assert.Equal(OperationStatuses.Succeeded, op!.Status); + Assert.Equal("""{"x":1}""", op.StepDetails!.Result); + Assert.Null(op.StepDetails.Error); + Assert.NotNull(op.EndTimestamp); + } + + [Fact] + public void Process_StepFail_SetsError() + { + var (store, processor) = Create(skipTime: false); + processor.Process(Arn, null, new List + { + new() { Id = "op-1", Type = OperationTypes.Step, Action = OperationAction.START } + }); + + processor.Process(Arn, "1", new List + { + new() + { + Id = "op-1", Type = OperationTypes.Step, Action = OperationAction.FAIL, + Error = new Amazon.Lambda.Model.ErrorObject { ErrorType = "TestEx", ErrorMessage = "boom" } + } + }); + + var op = store.GetOperation(Arn, "op-1"); + Assert.Equal(OperationStatuses.Failed, op!.Status); + Assert.Equal("TestEx", op.StepDetails!.Error!.ErrorType); + Assert.Equal("boom", op.StepDetails.Error.ErrorMessage); + } + + [Fact] + public void Process_StepRetry_SetsPendingWithDelay() + { + var (store, processor) = Create(skipTime: false); + processor.Process(Arn, null, new List + { + new() { Id = "op-1", Type = OperationTypes.Step, Action = OperationAction.START } + }); + + processor.Process(Arn, "1", new List + { + new() + { + Id = "op-1", Type = OperationTypes.Step, Action = OperationAction.RETRY, + StepOptions = new StepOptions { NextAttemptDelaySeconds = 5 }, + Error = new Amazon.Lambda.Model.ErrorObject { ErrorType = "TransientEx" } + } + }); + + var op = store.GetOperation(Arn, "op-1"); + Assert.Equal(OperationStatuses.Pending, op!.Status); + Assert.NotNull(op.StepDetails!.NextAttemptTimestamp); + Assert.Equal("TransientEx", op.StepDetails.Error!.ErrorType); + } + + [Fact] + public void Process_StepRetry_WithSkipTime_SetsReady() + { + var (store, processor) = Create(skipTime: true); + processor.Process(Arn, null, new List + { + new() { Id = "op-1", Type = OperationTypes.Step, Action = OperationAction.START } + }); + + processor.Process(Arn, "1", new List + { + new() + { + Id = "op-1", Type = OperationTypes.Step, Action = OperationAction.RETRY, + StepOptions = new StepOptions { NextAttemptDelaySeconds = 60 } + } + }); + + var op = store.GetOperation(Arn, "op-1"); + Assert.Equal(OperationStatuses.Ready, op!.Status); + Assert.True(op.StepDetails!.NextAttemptTimestamp <= DateTimeOffset.UtcNow.ToUnixTimeMilliseconds()); + } + + [Fact] + public void Process_WaitStart_SetsScheduledEnd() + { + var (store, processor) = Create(skipTime: false); + + processor.Process(Arn, null, new List + { + new() + { + Id = "op-1", Type = OperationTypes.Wait, Action = OperationAction.START, + WaitOptions = new WaitOptions { WaitSeconds = 300 } + } + }); + + var op = store.GetOperation(Arn, "op-1"); + Assert.Equal(OperationStatuses.Started, op!.Status); + Assert.NotNull(op.WaitDetails?.ScheduledEndTimestamp); + var scheduled = DateTimeOffset.FromUnixTimeMilliseconds(op.WaitDetails!.ScheduledEndTimestamp!.Value); + Assert.True(scheduled > DateTimeOffset.UtcNow.AddSeconds(290)); + } + + [Fact] + public void Process_WaitStart_WithSkipTime_ImmediatelySucceeds() + { + var (store, processor) = Create(skipTime: true); + + processor.Process(Arn, null, new List + { + new() + { + Id = "op-1", Type = OperationTypes.Wait, Action = OperationAction.START, + WaitOptions = new WaitOptions { WaitSeconds = 86400 } + } + }); + + var op = store.GetOperation(Arn, "op-1"); + Assert.Equal(OperationStatuses.Succeeded, op!.Status); + Assert.True(op.WaitDetails!.ScheduledEndTimestamp <= DateTimeOffset.UtcNow.ToUnixTimeMilliseconds()); + } + + [Fact] + public void Process_CallbackStart_MintsCallbackId() + { + var (store, processor) = Create(skipTime: false); + + processor.Process(Arn, null, new List + { + new() { Id = "op-cb-1", Type = OperationTypes.Callback, Action = OperationAction.START, Name = "approval" } + }); + + var op = store.GetOperation(Arn, "op-cb-1"); + Assert.Equal(OperationStatuses.Started, op!.Status); + Assert.Equal("cb-op-cb-1", op.CallbackDetails!.CallbackId); + } + + [Fact] + public void Process_IncreasesTokenEachCall() + { + var (store, processor) = Create(skipTime: false); + + var (t1, _) = processor.Process(Arn, null, new List + { + new() { Id = "op-1", Type = OperationTypes.Step, Action = OperationAction.START } + }); + var (t2, _) = processor.Process(Arn, t1, new List + { + new() { Id = "op-1", Type = OperationTypes.Step, Action = OperationAction.SUCCEED } + }); + + Assert.NotEqual(t1, t2); + } + + [Fact] + public void Process_ReturnsNewOperations() + { + var (_, processor) = Create(skipTime: false); + + var (_, newOps) = processor.Process(Arn, null, new List + { + new() { Id = "op-1", Type = OperationTypes.Step, Action = OperationAction.START, Name = "s1" }, + new() { Id = "op-2", Type = OperationTypes.Wait, Action = OperationAction.START, WaitOptions = new WaitOptions { WaitSeconds = 10 } } + }); + + Assert.Equal(2, newOps.Count); + Assert.Equal("op-1", newOps[0].Id); + Assert.Equal("op-2", newOps[1].Id); + } + + [Fact] + public void Process_ContextStart_SetsContextDetails() + { + var (store, processor) = Create(skipTime: false); + + processor.Process(Arn, null, new List + { + new() { Id = "op-ctx", Type = OperationTypes.Context, Action = OperationAction.START, Name = "parallel_batch", SubType = "Parallel" } + }); + + var op = store.GetOperation(Arn, "op-ctx"); + Assert.Equal(OperationStatuses.Started, op!.Status); + Assert.Equal("Parallel", op.SubType); + Assert.NotNull(op.ContextDetails); + } + + [Fact] + public void Process_ChainedInvokeStart_SetsDetails() + { + var (store, processor) = Create(skipTime: false); + + processor.Process(Arn, null, new List + { + new() { Id = "op-inv", Type = OperationTypes.ChainedInvoke, Action = OperationAction.START, Name = "process-payment" } + }); + + var op = store.GetOperation(Arn, "op-inv"); + Assert.Equal(OperationStatuses.Started, op!.Status); + Assert.NotNull(op.ChainedInvokeDetails); + } + + [Fact] + public void Process_MultipleUpdatesInBatch_AllApplied() + { + var (store, processor) = Create(skipTime: true); + + processor.Process(Arn, null, new List + { + new() { Id = "op-1", Type = OperationTypes.Step, Action = OperationAction.START, Name = "a" }, + new() { Id = "op-2", Type = OperationTypes.Step, Action = OperationAction.START, Name = "b" }, + new() { Id = "op-3", Type = OperationTypes.Wait, Action = OperationAction.START, WaitOptions = new WaitOptions { WaitSeconds = 60 } } + }); + + Assert.Equal(3, store.OperationCount(Arn)); + Assert.Equal(OperationStatuses.Started, store.GetOperation(Arn, "op-1")!.Status); + Assert.Equal(OperationStatuses.Succeeded, store.GetOperation(Arn, "op-3")!.Status); + } + + [Fact] + public void Process_WaitForCondition_Retry_WithSkipTime_SetsReady() + { + var (store, processor) = Create(skipTime: true); + + processor.Process(Arn, null, new List + { + new() { Id = "op-wfc", Type = OperationTypes.Wait, Action = OperationAction.START, SubType = OperationSubTypes.WaitForCondition, WaitOptions = new WaitOptions { WaitSeconds = 5 } } + }); + + // WaitForCondition START with SkipTime=true also gets time-skipped (it's a WAIT) + var op = store.GetOperation(Arn, "op-wfc"); + Assert.Equal(OperationStatuses.Succeeded, op!.Status); + + // Now simulate a RETRY (as if condition wasn't met yet) + processor.Process(Arn, "1", new List + { + new() { Id = "op-wfc", Type = OperationTypes.Wait, Action = OperationAction.RETRY, SubType = OperationSubTypes.WaitForCondition, WaitOptions = new WaitOptions { WaitSeconds = 10 } } + }); + + op = store.GetOperation(Arn, "op-wfc"); + Assert.Equal(OperationStatuses.Ready, op!.Status); + } + + private static (InMemoryOperationStore Store, CheckpointProcessor Processor) Create(bool skipTime) + { + var store = new InMemoryOperationStore(); + var processor = new CheckpointProcessor(store, skipTime); + return (store, processor); + } +} diff --git a/Libraries/test/Amazon.Lambda.DurableExecution.Testing.Tests/InMemoryOperationStoreTests.cs b/Libraries/test/Amazon.Lambda.DurableExecution.Testing.Tests/InMemoryOperationStoreTests.cs new file mode 100644 index 000000000..bb129c98a --- /dev/null +++ b/Libraries/test/Amazon.Lambda.DurableExecution.Testing.Tests/InMemoryOperationStoreTests.cs @@ -0,0 +1,115 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +using Amazon.Lambda.DurableExecution.Testing; +using Xunit; + +namespace Amazon.Lambda.DurableExecution.Testing.Tests; + +public class InMemoryOperationStoreTests +{ + [Fact] + public void InitialState_EmptyOperations() + { + var store = new InMemoryOperationStore(); + Assert.Empty(store.GetAllOperations("arn:test")); + Assert.Equal(0, store.OperationCount("arn:test")); + } + + [Fact] + public void InitialToken_IsZero() + { + var store = new InMemoryOperationStore(); + Assert.Equal("0", store.CurrentToken("arn:test")); + } + + [Fact] + public void Upsert_AddsNewOperation() + { + var store = new InMemoryOperationStore(); + var op = new Operation { Id = "op-1", Type = OperationTypes.Step, Name = "step1" }; + + store.Upsert("arn:test", op); + + Assert.Equal(1, store.OperationCount("arn:test")); + var retrieved = store.GetOperation("arn:test", "op-1"); + Assert.NotNull(retrieved); + Assert.Equal("step1", retrieved!.Name); + } + + [Fact] + public void Upsert_UpdatesExistingOperation() + { + var store = new InMemoryOperationStore(); + var op1 = new Operation { Id = "op-1", Type = OperationTypes.Step, Status = OperationStatuses.Started }; + store.Upsert("arn:test", op1); + + var op2 = new Operation { Id = "op-1", Type = OperationTypes.Step, Status = OperationStatuses.Succeeded }; + store.Upsert("arn:test", op2); + + Assert.Equal(1, store.OperationCount("arn:test")); + var retrieved = store.GetOperation("arn:test", "op-1"); + Assert.Equal(OperationStatuses.Succeeded, retrieved!.Status); + } + + [Fact] + public void GetOperation_ReturnsNull_WhenNotFound() + { + var store = new InMemoryOperationStore(); + Assert.Null(store.GetOperation("arn:test", "nonexistent")); + } + + [Fact] + public void IncrementToken_IncrementsCounter() + { + var store = new InMemoryOperationStore(); + var t1 = store.IncrementToken("arn:test"); + var t2 = store.IncrementToken("arn:test"); + var t3 = store.IncrementToken("arn:test"); + + Assert.Equal("1", t1); + Assert.Equal("2", t2); + Assert.Equal("3", t3); + Assert.Equal("3", store.CurrentToken("arn:test")); + } + + [Fact] + public void Executions_AreIsolated() + { + var store = new InMemoryOperationStore(); + store.Upsert("arn:exec-1", new Operation { Id = "op-1", Type = OperationTypes.Step }); + store.Upsert("arn:exec-2", new Operation { Id = "op-2", Type = OperationTypes.Step }); + + Assert.Equal(1, store.OperationCount("arn:exec-1")); + Assert.Equal(1, store.OperationCount("arn:exec-2")); + Assert.Null(store.GetOperation("arn:exec-1", "op-2")); + Assert.Null(store.GetOperation("arn:exec-2", "op-1")); + } + + [Fact] + public void GetAllOperations_PreservesInsertionOrder() + { + var store = new InMemoryOperationStore(); + store.Upsert("arn:test", new Operation { Id = "op-1", Type = OperationTypes.Step }); + store.Upsert("arn:test", new Operation { Id = "op-2", Type = OperationTypes.Wait }); + store.Upsert("arn:test", new Operation { Id = "op-3", Type = OperationTypes.Callback }); + + var all = store.GetAllOperations("arn:test"); + Assert.Equal(3, all.Count); + Assert.Equal("op-1", all[0].Id); + Assert.Equal("op-2", all[1].Id); + Assert.Equal("op-3", all[2].Id); + } + + [Fact] + public void Tokens_AreIsolatedPerExecution() + { + var store = new InMemoryOperationStore(); + store.IncrementToken("arn:exec-1"); + store.IncrementToken("arn:exec-1"); + store.IncrementToken("arn:exec-2"); + + Assert.Equal("2", store.CurrentToken("arn:exec-1")); + Assert.Equal("1", store.CurrentToken("arn:exec-2")); + } +} From ae82c7588bc818e79a573277cbe1d43e8a17d6ba Mon Sep 17 00:00:00 2001 From: Garrett Beatty Date: Tue, 16 Jun 2026 15:51:56 -0400 Subject: [PATCH 4/9] Add FunctionRegistry for sibling function routing Implements registration and invocation dispatch for InvokeAsync calls: - RegisterPlain: plain Lambda handlers - RegisterDurable: durable handlers (placeholder for nested runner, wired in commit 4) - Name matching: exact match first, then ARN-extracted name fallback - Errors from handlers returned as ErrorObject (not re-thrown) - UnregisteredSiblingFunctionException for unknown functions --- .../FunctionRegistry.cs | 136 +++++++++++++++++ .../FunctionRegistryTests.cs | 141 ++++++++++++++++++ 2 files changed, 277 insertions(+) create mode 100644 Libraries/src/Amazon.Lambda.DurableExecution.Testing/FunctionRegistry.cs create mode 100644 Libraries/test/Amazon.Lambda.DurableExecution.Testing.Tests/FunctionRegistryTests.cs diff --git a/Libraries/src/Amazon.Lambda.DurableExecution.Testing/FunctionRegistry.cs b/Libraries/src/Amazon.Lambda.DurableExecution.Testing/FunctionRegistry.cs new file mode 100644 index 000000000..a12ce0035 --- /dev/null +++ b/Libraries/src/Amazon.Lambda.DurableExecution.Testing/FunctionRegistry.cs @@ -0,0 +1,136 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +using System.Text; +using Amazon.Lambda.Core; + +namespace Amazon.Lambda.DurableExecution.Testing; + +/// +/// Tracks registered sibling function handlers for use by the local test runner +/// when a workflow calls InvokeAsync. +/// +internal sealed class FunctionRegistry +{ + private readonly List _entries = new(); + + public void RegisterPlain( + string functionNameOrArn, + Func> handler) + { + _entries.Add(new FunctionEntry( + ExtractName(functionNameOrArn), + functionNameOrArn, + IsDurable: false, + InvokeDelegate: (payload, serializer, lambdaContext) => + InvokePlain(handler, payload, serializer, lambdaContext))); + } + + public void RegisterDurable( + string functionNameOrArn, + Func> handler) + { + _entries.Add(new FunctionEntry( + ExtractName(functionNameOrArn), + functionNameOrArn, + IsDurable: true, + InvokeDelegate: (payload, serializer, lambdaContext) => + InvokeDurable(handler, payload, serializer))); + } + + public async Task<(string? Result, ErrorObject? Error)> InvokeAsync( + string functionNameOrArn, + string serializedPayload, + ILambdaSerializer serializer, + ILambdaContext lambdaContext) + { + var entry = Lookup(functionNameOrArn) + ?? throw new UnregisteredSiblingFunctionException(functionNameOrArn); + + try + { + var result = await entry.InvokeDelegate(serializedPayload, serializer, lambdaContext); + return (result, null); + } + catch (Exception ex) + { + return (null, ErrorObject.FromException(ex)); + } + } + + public bool IsRegistered(string functionNameOrArn) => Lookup(functionNameOrArn) is not null; + + private FunctionEntry? Lookup(string functionNameOrArn) + { + foreach (var entry in _entries) + { + if (string.Equals(entry.OriginalName, functionNameOrArn, StringComparison.Ordinal)) + return entry; + } + + var extractedName = ExtractName(functionNameOrArn); + foreach (var entry in _entries) + { + if (string.Equals(entry.ShortName, extractedName, StringComparison.Ordinal)) + return entry; + } + + return null; + } + + private static string ExtractName(string functionNameOrArn) + { + if (!functionNameOrArn.StartsWith("arn:", StringComparison.OrdinalIgnoreCase)) + return functionNameOrArn; + + var parts = functionNameOrArn.Split(':'); + // ARN format: arn:aws:lambda:region:account:function:name[:qualifier] + if (parts.Length >= 7) + return parts[6]; + + return functionNameOrArn; + } + + private static async Task InvokePlain( + Func> handler, + string serializedPayload, + ILambdaSerializer serializer, + ILambdaContext lambdaContext) + { + var payload = Deserialize(serializedPayload, serializer); + var result = await handler(payload, lambdaContext); + return Serialize(result, serializer); + } + + private static async Task InvokeDurable( + Func> handler, + string serializedPayload, + ILambdaSerializer serializer) + { + // For durable siblings, we'd need to spin up a nested DurableTestRunner. + // This is wired in commit 4 when the full runner exists. For now, the + // registry captures the handler; actual durable invocation is deferred. + throw new NotImplementedException( + "Durable sibling invocation requires the full DurableTestRunner, wired in a subsequent commit."); + } + + private static T Deserialize(string serialized, ILambdaSerializer serializer) + { + using var stream = new MemoryStream(Encoding.UTF8.GetBytes(serialized)); + return serializer.Deserialize(stream); + } + + private static string? Serialize(T value, ILambdaSerializer serializer) + { + if (value is null) return null; + using var stream = new MemoryStream(); + serializer.Serialize(value, stream); + return Encoding.UTF8.GetString(stream.ToArray()); + } + + private sealed record FunctionEntry( + string ShortName, + string OriginalName, + bool IsDurable, + Func> InvokeDelegate); +} diff --git a/Libraries/test/Amazon.Lambda.DurableExecution.Testing.Tests/FunctionRegistryTests.cs b/Libraries/test/Amazon.Lambda.DurableExecution.Testing.Tests/FunctionRegistryTests.cs new file mode 100644 index 000000000..916a09ab7 --- /dev/null +++ b/Libraries/test/Amazon.Lambda.DurableExecution.Testing.Tests/FunctionRegistryTests.cs @@ -0,0 +1,141 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +using Amazon.Lambda.Core; +using Amazon.Lambda.DurableExecution.Testing; +using Amazon.Lambda.Serialization.SystemTextJson; +using Amazon.Lambda.TestUtilities; +using Xunit; + +namespace Amazon.Lambda.DurableExecution.Testing.Tests; + +public class FunctionRegistryTests +{ + private static readonly DefaultLambdaJsonSerializer Serializer = new(); + private static readonly TestLambdaContext LambdaContext = new(); + + [Fact] + public async Task InvokeAsync_RegisteredByShortName_InvokedByShortName() + { + var registry = new FunctionRegistry(); + registry.RegisterPlain( + "process-payment", ProcessPayment); + + var (result, error) = await registry.InvokeAsync( + "process-payment", """{"Amount":100}""", Serializer, LambdaContext); + + Assert.Null(error); + Assert.Contains("100", result); + } + + [Fact] + public async Task InvokeAsync_RegisteredByShortName_InvokedByFullArn() + { + var registry = new FunctionRegistry(); + registry.RegisterPlain( + "process-payment", ProcessPayment); + + var (result, error) = await registry.InvokeAsync( + "arn:aws:lambda:us-east-1:123456789012:function:process-payment", + """{"Amount":50}""", Serializer, LambdaContext); + + Assert.Null(error); + Assert.Contains("50", result); + } + + [Fact] + public async Task InvokeAsync_RegisteredByFullArn_InvokedByShortName() + { + var registry = new FunctionRegistry(); + registry.RegisterPlain( + "arn:aws:lambda:us-east-1:123456789012:function:process-payment", + ProcessPayment); + + var (result, error) = await registry.InvokeAsync( + "process-payment", """{"Amount":75}""", Serializer, LambdaContext); + + Assert.Null(error); + Assert.Contains("75", result); + } + + [Fact] + public async Task InvokeAsync_ExactMatchWins_OverArnExtraction() + { + var registry = new FunctionRegistry(); + registry.RegisterPlain( + "process-payment", (req, _) => Task.FromResult(new PaymentResult { Status = "short" })); + registry.RegisterPlain( + "arn:aws:lambda:us-east-1:123456789012:function:process-payment", + (req, _) => Task.FromResult(new PaymentResult { Status = "arn" })); + + var (result, _) = await registry.InvokeAsync( + "arn:aws:lambda:us-east-1:123456789012:function:process-payment", + """{"Amount":1}""", Serializer, LambdaContext); + + Assert.Contains("arn", result); + } + + [Fact] + public async Task InvokeAsync_UnregisteredFunction_Throws() + { + var registry = new FunctionRegistry(); + + await Assert.ThrowsAsync(() => + registry.InvokeAsync("unknown-fn", "{}", Serializer, LambdaContext)); + } + + [Fact] + public async Task InvokeAsync_HandlerThatThrows_ReturnsError() + { + var registry = new FunctionRegistry(); + registry.RegisterPlain( + "failing-fn", (_, _) => throw new InvalidOperationException("payment failed")); + + var (result, error) = await registry.InvokeAsync( + "failing-fn", """{"Amount":1}""", Serializer, LambdaContext); + + Assert.Null(result); + Assert.NotNull(error); + Assert.Equal("System.InvalidOperationException", error!.ErrorType); + Assert.Equal("payment failed", error.ErrorMessage); + } + + [Fact] + public void IsRegistered_ReturnsTrueForRegistered() + { + var registry = new FunctionRegistry(); + registry.RegisterPlain("my-fn", ProcessPayment); + + Assert.True(registry.IsRegistered("my-fn")); + Assert.True(registry.IsRegistered("arn:aws:lambda:us-east-1:123:function:my-fn")); + } + + [Fact] + public void IsRegistered_ReturnsFalseForUnregistered() + { + var registry = new FunctionRegistry(); + Assert.False(registry.IsRegistered("unknown")); + } + + [Fact] + public async Task InvokeAsync_WithQualifiedArn_ExtractsName() + { + var registry = new FunctionRegistry(); + registry.RegisterPlain("calc", ProcessPayment); + + var (result, error) = await registry.InvokeAsync( + "arn:aws:lambda:us-east-1:123:function:calc:$LATEST", + """{"Amount":10}""", Serializer, LambdaContext); + + Assert.Null(error); + Assert.NotNull(result); + } + + private static Task ProcessPayment(PaymentRequest req, ILambdaContext ctx) + { + return Task.FromResult(new PaymentResult { Status = $"approved-{req.Amount}" }); + } + + public sealed class PaymentRequest { public int Amount { get; set; } } + public sealed class PaymentResult { public string? Status { get; set; } } +} From 26315841bd3185a1b9e51136b93de130b5aaa562 Mon Sep 17 00:00:00 2001 From: Garrett Beatty Date: Tue, 16 Jun 2026 15:55:56 -0400 Subject: [PATCH 5/9] Add DurableTestRunner with RunAsync and ExecutionOrchestrator The core local test runner that drives workflows to completion: - ExecutionOrchestrator: seeds the EXECUTION op with serialized input, then loops DurableFunction.WrapAsync (internal overload) with the in-memory service client until terminal or MaxInvocations exceeded - DurableTestRunner: public entry point with RunAsync, RegisterFunction, RegisterDurableFunction, IAsyncDisposable Integration tests verify: single step, multi-step inspection, workflow failure, EnsureSucceeded, WaitAsync with time skipping, MaxInvocations limit, timeout, null results, and custom ARN propagation. --- ...zon.Lambda.DurableExecution.Testing.csproj | 1 + .../DurableTestRunner.cs | 156 +++++++++++++++ .../ExecutionOrchestrator.cs | 133 +++++++++++++ .../RunAsyncTests.cs | 177 ++++++++++++++++++ 4 files changed, 467 insertions(+) create mode 100644 Libraries/src/Amazon.Lambda.DurableExecution.Testing/DurableTestRunner.cs create mode 100644 Libraries/src/Amazon.Lambda.DurableExecution.Testing/ExecutionOrchestrator.cs create mode 100644 Libraries/test/Amazon.Lambda.DurableExecution.Testing.Tests/RunAsyncTests.cs diff --git a/Libraries/src/Amazon.Lambda.DurableExecution.Testing/Amazon.Lambda.DurableExecution.Testing.csproj b/Libraries/src/Amazon.Lambda.DurableExecution.Testing/Amazon.Lambda.DurableExecution.Testing.csproj index 9e639812b..471825f52 100644 --- a/Libraries/src/Amazon.Lambda.DurableExecution.Testing/Amazon.Lambda.DurableExecution.Testing.csproj +++ b/Libraries/src/Amazon.Lambda.DurableExecution.Testing/Amazon.Lambda.DurableExecution.Testing.csproj @@ -13,6 +13,7 @@ enable enable true + $(NoWarn);AWSLAMBDA001 diff --git a/Libraries/src/Amazon.Lambda.DurableExecution.Testing/DurableTestRunner.cs b/Libraries/src/Amazon.Lambda.DurableExecution.Testing/DurableTestRunner.cs new file mode 100644 index 000000000..698386e0c --- /dev/null +++ b/Libraries/src/Amazon.Lambda.DurableExecution.Testing/DurableTestRunner.cs @@ -0,0 +1,156 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +using Amazon.Lambda.Core; +using Amazon.Lambda.Serialization.SystemTextJson; +using Amazon.Lambda.TestUtilities; + +namespace Amazon.Lambda.DurableExecution.Testing; + +/// +/// Local test runner for durable workflows. Drives the workflow to completion +/// in-process using the real runtime engine with an in-memory service backend. +/// +public sealed class DurableTestRunner : IDurableTestRunner, IAsyncDisposable +{ + private readonly Func> _handler; + private readonly TestRunnerOptions _options; + private readonly ILambdaSerializer _serializer; + private readonly ILambdaContext _lambdaContext; + private readonly InMemoryOperationStore _store; + private readonly CheckpointProcessor _processor; + private readonly InMemoryDurableServiceClient _serviceClient; + private readonly FunctionRegistry _registry; + + /// + /// Creates a new local test runner for the given workflow handler. + /// + public DurableTestRunner( + Func> handler, + TestRunnerOptions? options = null) + { + _handler = handler ?? throw new ArgumentNullException(nameof(handler)); + _options = options ?? new TestRunnerOptions(); + _serializer = _options.Serializer ?? new DefaultLambdaJsonSerializer(); + _lambdaContext = CreateLambdaContext(); + _store = new InMemoryOperationStore(); + _processor = new CheckpointProcessor(_store, _options.SkipTime); + _serviceClient = new InMemoryDurableServiceClient(_store, _processor); + _registry = new FunctionRegistry(); + } + + /// + /// Registers a plain (non-durable) Lambda handler as a sibling function. + /// + public DurableTestRunner RegisterFunction( + string functionNameOrArn, + Func> handler) + { + _registry.RegisterPlain(functionNameOrArn, handler); + return this; + } + + /// + /// Registers a durable Lambda handler as a sibling function. + /// + public DurableTestRunner RegisterDurableFunction( + string functionNameOrArn, + Func> handler) + { + _registry.RegisterDurable(functionNameOrArn, handler); + return this; + } + + /// + public async Task> RunAsync( + TInput input, + TimeSpan? timeout = null, + CancellationToken cancellationToken = default) + { + var orchestrator = CreateOrchestrator(); + return await orchestrator.DriveToTerminalAsync( + _options.DurableExecutionArn, + input, + timeout ?? _options.DefaultTimeout, + cancellationToken); + } + + /// + public Task StartAsync( + TInput input, + TimeSpan? timeout = null, + CancellationToken cancellationToken = default) + { + // Callback support implemented in commit 5 + throw new NotImplementedException("StartAsync will be implemented with callback support."); + } + + /// + public Task WaitForCallbackAsync( + string durableExecutionArn, + string? name = null, + TimeSpan? timeout = null, + CancellationToken cancellationToken = default) + { + throw new NotImplementedException("WaitForCallbackAsync will be implemented with callback support."); + } + + /// + public Task SendCallbackSuccessAsync( + string callbackId, + TResult result, + CancellationToken cancellationToken = default) + { + throw new NotImplementedException("SendCallbackSuccessAsync will be implemented with callback support."); + } + + /// + public Task SendCallbackFailureAsync( + string callbackId, + ErrorObject? error = null, + CancellationToken cancellationToken = default) + { + throw new NotImplementedException("SendCallbackFailureAsync will be implemented with callback support."); + } + + /// + public Task SendCallbackHeartbeatAsync( + string callbackId, + CancellationToken cancellationToken = default) + { + throw new NotImplementedException("SendCallbackHeartbeatAsync will be implemented with callback support."); + } + + /// + public Task> WaitForResultAsync( + string durableExecutionArn, + TimeSpan? timeout = null, + CancellationToken cancellationToken = default) + { + throw new NotImplementedException("WaitForResultAsync will be implemented with callback support."); + } + + /// + public ValueTask DisposeAsync() + { + return ValueTask.CompletedTask; + } + + private ExecutionOrchestrator CreateOrchestrator() + { + return new ExecutionOrchestrator( + _handler, _store, _serviceClient, _lambdaContext, _options, _serializer); + } + + private TestLambdaContext CreateLambdaContext() + { + var ctx = new TestLambdaContext + { + FunctionName = "test-durable-function", + FunctionVersion = "$LATEST", + RemainingTime = TimeSpan.FromMinutes(15), + }; + ctx.Serializer = _serializer; + return ctx; + } +} diff --git a/Libraries/src/Amazon.Lambda.DurableExecution.Testing/ExecutionOrchestrator.cs b/Libraries/src/Amazon.Lambda.DurableExecution.Testing/ExecutionOrchestrator.cs new file mode 100644 index 000000000..0a5705586 --- /dev/null +++ b/Libraries/src/Amazon.Lambda.DurableExecution.Testing/ExecutionOrchestrator.cs @@ -0,0 +1,133 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +using System.Text; +using Amazon.Lambda.Core; +using Amazon.Lambda.DurableExecution.Services; + +namespace Amazon.Lambda.DurableExecution.Testing; + +/// +/// Drives a durable workflow handler to a terminal state by repeatedly invoking +/// the internal DurableFunction.WrapAsync overload with the in-memory service client. +/// +internal sealed class ExecutionOrchestrator +{ + private readonly Func> _handler; + private readonly InMemoryOperationStore _store; + private readonly IDurableServiceClient _serviceClient; + private readonly ILambdaContext _lambdaContext; + private readonly TestRunnerOptions _options; + private readonly ILambdaSerializer _serializer; + + public ExecutionOrchestrator( + Func> handler, + InMemoryOperationStore store, + IDurableServiceClient serviceClient, + ILambdaContext lambdaContext, + TestRunnerOptions options, + ILambdaSerializer serializer) + { + _handler = handler; + _store = store; + _serviceClient = serviceClient; + _lambdaContext = lambdaContext; + _options = options; + _serializer = serializer; + } + + public async Task> DriveToTerminalAsync( + string arn, + TInput input, + TimeSpan timeout, + CancellationToken cancellationToken) + { + using var timeoutCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + timeoutCts.CancelAfter(timeout); + + SeedExecutionOperation(arn, input); + + var invocationCount = 0; + DurableExecutionInvocationOutput output; + + while (true) + { + timeoutCts.Token.ThrowIfCancellationRequested(); + + if (invocationCount >= _options.MaxInvocations) + { + throw new TestExecutionLimitException( + _options.MaxInvocations, _store.OperationCount(arn)); + } + + var invocationInput = BuildInvocationInput(arn); + + output = await DurableFunction.WrapAsync( + _handler, invocationInput, _lambdaContext, _serviceClient); + + invocationCount++; + + if (output.Status != InvocationStatus.Pending) + break; + } + + return BuildResult(arn, output, invocationCount); + } + + private void SeedExecutionOperation(string arn, TInput input) + { + var serializedInput = SerializeToString(input); + _store.Upsert(arn, new Operation + { + Id = "exec-0", + Type = OperationTypes.Execution, + Status = OperationStatuses.Started, + ExecutionDetails = new ExecutionDetails { InputPayload = serializedInput } + }); + } + + private DurableExecutionInvocationInput BuildInvocationInput(string arn) + { + return new DurableExecutionInvocationInput + { + DurableExecutionArn = arn, + CheckpointToken = _store.CurrentToken(arn), + InitialExecutionState = new InitialExecutionState + { + Operations = _store.GetAllOperations(arn).ToList(), + NextMarker = null, + } + }; + } + + private string SerializeToString(TInput value) + { + using var stream = new MemoryStream(); + _serializer.Serialize(value, stream); + return Encoding.UTF8.GetString(stream.ToArray()); + } + + private TestResult BuildResult(string arn, DurableExecutionInvocationOutput output, int invocationCount) + { + TOutput? result = default; + if (output.Status == InvocationStatus.Succeeded && output.Result is not null) + { + using var stream = new MemoryStream(Encoding.UTF8.GetBytes(output.Result)); + result = _serializer.Deserialize(stream); + } + + var allOps = _store.GetAllOperations(arn); + var steps = allOps + .Where(o => o.Type != OperationTypes.Execution) + .Select(o => new TestStep(o, _serializer)) + .ToList(); + + return new TestResult( + status: output.Status, + result: result, + error: output.Error, + durableExecutionArn: arn, + invocationCount: invocationCount, + steps: steps); + } +} diff --git a/Libraries/test/Amazon.Lambda.DurableExecution.Testing.Tests/RunAsyncTests.cs b/Libraries/test/Amazon.Lambda.DurableExecution.Testing.Tests/RunAsyncTests.cs new file mode 100644 index 000000000..8788a5c89 --- /dev/null +++ b/Libraries/test/Amazon.Lambda.DurableExecution.Testing.Tests/RunAsyncTests.cs @@ -0,0 +1,177 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +using Amazon.Lambda.DurableExecution.Testing; +using Xunit; + +namespace Amazon.Lambda.DurableExecution.Testing.Tests; + +public class RunAsyncTests +{ + [Fact] + public async Task RunAsync_SingleStep_Succeeds() + { + await using var runner = new DurableTestRunner( + handler: async (input, ctx) => + { + var result = await ctx.StepAsync( + async (_, _) => $"Hello, {input}!", + name: "greet"); + return result; + }); + + var result = await runner.RunAsync("World"); + + result.EnsureSucceeded(); + Assert.Equal("Hello, World!", result.Result); + Assert.True(result.InvocationCount >= 1); + } + + [Fact] + public async Task RunAsync_MultipleSteps_AllInspectable() + { + await using var runner = new DurableTestRunner( + handler: async (input, ctx) => + { + var a = await ctx.StepAsync( + async (_, _) => input * 2, + name: "double"); + var b = await ctx.StepAsync( + async (_, _) => a + 10, + name: "add_ten"); + return b; + }); + + var result = await runner.RunAsync(5); + + result.EnsureSucceeded(); + Assert.Equal(20, result.Result); + + var doubleStep = result.GetStep("double"); + Assert.Equal(OperationKind.Step, doubleStep.Kind); + Assert.Equal(OperationStatus.Succeeded, doubleStep.Status); + Assert.Equal(10, doubleStep.GetResult()); + + var addStep = result.GetStep("add_ten"); + Assert.Equal(OperationStatus.Succeeded, addStep.Status); + Assert.Equal(20, addStep.GetResult()); + } + + [Fact] + public async Task RunAsync_WorkflowThrows_ReturnsFailed() + { + await using var runner = new DurableTestRunner( + handler: async (input, ctx) => + { + await ctx.StepAsync(async (_, _) => + { + throw new InvalidOperationException("something broke"); +#pragma warning disable CS0162 + return ""; +#pragma warning restore CS0162 + }, name: "boom"); + return "unreachable"; + }); + + var result = await runner.RunAsync("test"); + + Assert.Equal(InvocationStatus.Failed, result.Status); + Assert.NotNull(result.Error); + Assert.Equal("System.InvalidOperationException", result.Error!.ErrorType); + } + + [Fact] + public async Task RunAsync_EnsureSucceeded_ThrowsOnFailure() + { + await using var runner = new DurableTestRunner( + handler: (input, ctx) => + { + throw new ArgumentException("bad input"); + }); + + var result = await runner.RunAsync("test"); + var ex = Assert.Throws(() => result.EnsureSucceeded()); + Assert.Equal(InvocationStatus.Failed, ex.FinalStatus); + } + + [Fact] + public async Task RunAsync_WithWait_SkipsTime() + { + await using var runner = new DurableTestRunner( + handler: async (input, ctx) => + { + await ctx.StepAsync(async (_, _) => "done", name: "before"); + await ctx.WaitAsync(TimeSpan.FromDays(30), name: "long_wait"); + var after = await ctx.StepAsync(async (_, _) => "completed", name: "after"); + return after; + }, + options: new TestRunnerOptions { SkipTime = true }); + + var result = await runner.RunAsync("go"); + + result.EnsureSucceeded(); + Assert.Equal("completed", result.Result); + + var wait = result.FindStep("long_wait"); + Assert.NotNull(wait); + Assert.Equal(OperationKind.Wait, wait!.Kind); + Assert.Equal(OperationStatus.Succeeded, wait.Status); + } + + [Fact] + public async Task RunAsync_MaxInvocationsExceeded_Throws() + { + await using var runner = new DurableTestRunner( + handler: async (input, ctx) => + { + await ctx.WaitAsync(TimeSpan.FromHours(1), name: "infinite"); + return "done"; + }, + options: new TestRunnerOptions { SkipTime = false, MaxInvocations = 3, DefaultTimeout = TimeSpan.FromSeconds(5) }); + + await Assert.ThrowsAsync(() => runner.RunAsync("x")); + } + + [Fact] + public async Task RunAsync_Timeout_ThrowsOperationCanceled() + { + await using var runner = new DurableTestRunner( + handler: async (input, ctx) => + { + await ctx.WaitAsync(TimeSpan.FromHours(1), name: "forever"); + return "done"; + }, + options: new TestRunnerOptions { SkipTime = false, MaxInvocations = 1000 }); + + await Assert.ThrowsAsync( + () => runner.RunAsync("x", timeout: TimeSpan.FromMilliseconds(100))); + } + + [Fact] + public async Task RunAsync_NullResult_ReturnsDefault() + { + await using var runner = new DurableTestRunner( + handler: async (input, ctx) => + { + await ctx.StepAsync(async (_, _) => "done", name: "noop"); + return null; + }); + + var result = await runner.RunAsync("test"); + result.EnsureSucceeded(); + Assert.Null(result.Result); + } + + [Fact] + public async Task RunAsync_CustomArn_UsedInResult() + { + const string customArn = "arn:aws:lambda:eu-west-1:999:execution:my-fn:my-exec"; + await using var runner = new DurableTestRunner( + handler: async (input, ctx) => "ok", + options: new TestRunnerOptions { DurableExecutionArn = customArn }); + + var result = await runner.RunAsync("test"); + result.EnsureSucceeded(); + Assert.Equal(customArn, result.DurableExecutionArn); + } +} From bf72c5d23a732e1302aaca58cd04424d530b6718 Mon Sep 17 00:00:00 2001 From: Garrett Beatty Date: Tue, 16 Jun 2026 16:00:55 -0400 Subject: [PATCH 6/9] Add callback support: StartAsync, WaitForCallbackAsync, SendCallback*, WaitForResultAsync MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Implements the two-call pattern for callback workflows: - StartAsync: drives workflow until it suspends on a callback, returns ARN - WaitForCallbackAsync: finds the pending callback ID in the store (matches by name or "{name}-callback" convention from the runtime) - SendCallbackSuccessAsync/FailureAsync: mutates the callback operation - SendCallbackHeartbeatAsync: validates callback exists (no-op locally) - WaitForResultAsync: re-drives the workflow to terminal after callback resolution, caches completed results Also adds DriveUntilSuspendedAsync to ExecutionOrchestrator for the Start→Suspend pattern. --- .../DurableTestRunner.cs | 114 +++++++++++++-- .../ExecutionOrchestrator.cs | 39 +++++ .../CallbackTests.cs | 135 ++++++++++++++++++ 3 files changed, 279 insertions(+), 9 deletions(-) create mode 100644 Libraries/test/Amazon.Lambda.DurableExecution.Testing.Tests/CallbackTests.cs diff --git a/Libraries/src/Amazon.Lambda.DurableExecution.Testing/DurableTestRunner.cs b/Libraries/src/Amazon.Lambda.DurableExecution.Testing/DurableTestRunner.cs index 698386e0c..25b6fb211 100644 --- a/Libraries/src/Amazon.Lambda.DurableExecution.Testing/DurableTestRunner.cs +++ b/Libraries/src/Amazon.Lambda.DurableExecution.Testing/DurableTestRunner.cs @@ -21,6 +21,10 @@ public sealed class DurableTestRunner : IDurableTestRunner> _completedResults = new(); + private readonly HashSet _consumedCallbackIds = new(); + private ExecutionOrchestrator? _lastOrchestrator; + private TInput? _lastStartInput; /// /// Creates a new local test runner for the given workflow handler. @@ -76,13 +80,23 @@ public async Task> RunAsync( } /// - public Task StartAsync( + public async Task StartAsync( TInput input, TimeSpan? timeout = null, CancellationToken cancellationToken = default) { - // Callback support implemented in commit 5 - throw new NotImplementedException("StartAsync will be implemented with callback support."); + var arn = _options.DurableExecutionArn; + var orchestrator = CreateOrchestrator(); + _lastStartInput = input; + _lastOrchestrator = orchestrator; + + var result = await orchestrator.DriveUntilSuspendedAsync( + arn, input, timeout ?? _options.DefaultTimeout, cancellationToken); + + if (result is not null) + _completedResults[arn] = result; + + return arn; } /// @@ -92,7 +106,37 @@ public Task WaitForCallbackAsync( TimeSpan? timeout = null, CancellationToken cancellationToken = default) { - throw new NotImplementedException("WaitForCallbackAsync will be implemented with callback support."); + var ops = _store.GetAllOperations(durableExecutionArn); + foreach (var op in ops) + { + if (op.Type == OperationTypes.Callback + && op.Status == OperationStatuses.Started + && op.CallbackDetails?.CallbackId is { } cbId) + { + if (name is null || MatchesCallbackName(op.Name, name)) + { + if (!_consumedCallbackIds.Contains(cbId)) + { + _consumedCallbackIds.Add(cbId); + return Task.FromResult(cbId); + } + } + } + } + + throw new InvalidOperationException( + $"No pending callback found{(name is not null ? $" with name '{name}'" : "")} for execution '{durableExecutionArn}'. " + + "Ensure the workflow has reached a WaitForCallbackAsync point before calling this method."); + } + + private static bool MatchesCallbackName(string? opName, string name) + { + if (opName is null) return false; + // Exact match + if (string.Equals(opName, name, StringComparison.Ordinal)) return true; + // The runtime names inner callback ops as "{name}-callback" + if (string.Equals(opName, $"{name}-callback", StringComparison.Ordinal)) return true; + return false; } /// @@ -101,7 +145,15 @@ public Task SendCallbackSuccessAsync( TResult result, CancellationToken cancellationToken = default) { - throw new NotImplementedException("SendCallbackSuccessAsync will be implemented with callback support."); + var (arn, op) = FindCallbackOperation(callbackId); + var serialized = SerializeToString(result); + + op.Status = OperationStatuses.Succeeded; + op.EndTimestamp = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(); + op.CallbackDetails!.Result = serialized; + _store.Upsert(arn, op); + + return Task.CompletedTask; } /// @@ -110,7 +162,14 @@ public Task SendCallbackFailureAsync( ErrorObject? error = null, CancellationToken cancellationToken = default) { - throw new NotImplementedException("SendCallbackFailureAsync will be implemented with callback support."); + var (arn, op) = FindCallbackOperation(callbackId); + + op.Status = OperationStatuses.Failed; + op.EndTimestamp = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(); + op.CallbackDetails!.Error = error; + _store.Upsert(arn, op); + + return Task.CompletedTask; } /// @@ -118,16 +177,29 @@ public Task SendCallbackHeartbeatAsync( string callbackId, CancellationToken cancellationToken = default) { - throw new NotImplementedException("SendCallbackHeartbeatAsync will be implemented with callback support."); + // Heartbeats are a no-op for local testing — just validate the callback exists + FindCallbackOperation(callbackId); + return Task.CompletedTask; } /// - public Task> WaitForResultAsync( + public async Task> WaitForResultAsync( string durableExecutionArn, TimeSpan? timeout = null, CancellationToken cancellationToken = default) { - throw new NotImplementedException("WaitForResultAsync will be implemented with callback support."); + if (_completedResults.TryGetValue(durableExecutionArn, out var cached)) + return cached; + + var orchestrator = _lastOrchestrator ?? CreateOrchestrator(); + var result = await orchestrator.DriveToTerminalAsync( + durableExecutionArn, + _lastStartInput!, + timeout ?? _options.DefaultTimeout, + cancellationToken); + + _completedResults[durableExecutionArn] = result; + return result; } /// @@ -136,6 +208,30 @@ public ValueTask DisposeAsync() return ValueTask.CompletedTask; } + private (string Arn, Operation Op) FindCallbackOperation(string callbackId) + { + var arn = _options.DurableExecutionArn; + var ops = _store.GetAllOperations(arn); + foreach (var op in ops) + { + if (op.Type == OperationTypes.Callback + && op.CallbackDetails?.CallbackId == callbackId) + { + return (arn, op); + } + } + throw new InvalidOperationException( + $"No callback operation found with ID '{callbackId}'."); + } + + private string? SerializeToString(T value) + { + if (value is null) return null; + using var stream = new MemoryStream(); + _serializer.Serialize(value, stream); + return System.Text.Encoding.UTF8.GetString(stream.ToArray()); + } + private ExecutionOrchestrator CreateOrchestrator() { return new ExecutionOrchestrator( diff --git a/Libraries/src/Amazon.Lambda.DurableExecution.Testing/ExecutionOrchestrator.cs b/Libraries/src/Amazon.Lambda.DurableExecution.Testing/ExecutionOrchestrator.cs index 0a5705586..74e1f436e 100644 --- a/Libraries/src/Amazon.Lambda.DurableExecution.Testing/ExecutionOrchestrator.cs +++ b/Libraries/src/Amazon.Lambda.DurableExecution.Testing/ExecutionOrchestrator.cs @@ -36,6 +36,45 @@ public ExecutionOrchestrator( _serializer = serializer; } + public async Task?> DriveUntilSuspendedAsync( + string arn, + TInput input, + TimeSpan timeout, + CancellationToken cancellationToken) + { + using var timeoutCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + timeoutCts.CancelAfter(timeout); + + SeedExecutionOperation(arn, input); + + var invocationCount = 0; + DurableExecutionInvocationOutput output; + + while (true) + { + timeoutCts.Token.ThrowIfCancellationRequested(); + + if (invocationCount >= _options.MaxInvocations) + { + throw new TestExecutionLimitException( + _options.MaxInvocations, _store.OperationCount(arn)); + } + + var invocationInput = BuildInvocationInput(arn); + + output = await DurableFunction.WrapAsync( + _handler, invocationInput, _lambdaContext, _serviceClient); + + invocationCount++; + + if (output.Status == InvocationStatus.Pending) + return null; // Suspended — test code drives callbacks + + if (output.Status != InvocationStatus.Pending) + return BuildResult(arn, output, invocationCount); + } + } + public async Task> DriveToTerminalAsync( string arn, TInput input, diff --git a/Libraries/test/Amazon.Lambda.DurableExecution.Testing.Tests/CallbackTests.cs b/Libraries/test/Amazon.Lambda.DurableExecution.Testing.Tests/CallbackTests.cs new file mode 100644 index 000000000..68fab20dc --- /dev/null +++ b/Libraries/test/Amazon.Lambda.DurableExecution.Testing.Tests/CallbackTests.cs @@ -0,0 +1,135 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +using Amazon.Lambda.DurableExecution.Testing; +using Xunit; + +namespace Amazon.Lambda.DurableExecution.Testing.Tests; + +public class CallbackTests +{ + [Fact] + public async Task CallbackWorkflow_FullRoundTrip() + { + await using var runner = new DurableTestRunner( + handler: async (input, ctx) => + { + var approval = await ctx.WaitForCallbackAsync( + async (callbackId, cbCtx, ct) => { /* submitter: e.g. send to external system */ }, + name: "approval"); + return $"approved: {approval}"; + }); + + var arn = await runner.StartAsync("request-1"); + var callbackId = await runner.WaitForCallbackAsync(arn, name: "approval"); + + Assert.NotNull(callbackId); + Assert.StartsWith("cb-", callbackId); + + await runner.SendCallbackSuccessAsync(callbackId, "yes"); + var result = await runner.WaitForResultAsync(arn); + + result.EnsureSucceeded(); + Assert.Equal("approved: yes", result.Result); + } + + [Fact] + public async Task CallbackWorkflow_Failure() + { + await using var runner = new DurableTestRunner( + handler: async (input, ctx) => + { + try + { + var res = await ctx.WaitForCallbackAsync( + async (callbackId, cbCtx, ct) => { }, + name: "check"); + return res; + } + catch (CallbackException ex) + { + return $"failed: {ex.ErrorType}"; + } + }); + + var arn = await runner.StartAsync("req"); + var callbackId = await runner.WaitForCallbackAsync(arn, name: "check"); + + await runner.SendCallbackFailureAsync(callbackId, + new ErrorObject { ErrorType = "Rejected", ErrorMessage = "nope" }); + var result = await runner.WaitForResultAsync(arn); + + result.EnsureSucceeded(); + Assert.Equal("failed: Rejected", result.Result); + } + + [Fact] + public async Task WaitForCallbackAsync_ThrowsWhenNoPendingCallback() + { + await using var runner = new DurableTestRunner( + handler: async (input, ctx) => + { + // No callback in this workflow + return await ctx.StepAsync(async (_, _) => "done", name: "step1"); + }); + + var arn = await runner.StartAsync("x"); + + await Assert.ThrowsAsync( + () => runner.WaitForCallbackAsync(arn, name: "nonexistent")); + } + + [Fact] + public async Task SendCallbackSuccess_ThrowsForUnknownCallbackId() + { + await using var runner = new DurableTestRunner( + handler: async (input, ctx) => "ok"); + + await Assert.ThrowsAsync( + () => runner.SendCallbackSuccessAsync("cb-unknown", "data")); + } + + [Fact] + public async Task WaitForResultAsync_ReturnsCachedResult() + { + await using var runner = new DurableTestRunner( + handler: async (input, ctx) => + { + var val = await ctx.WaitForCallbackAsync( + async (_, _, _) => { }, name: "cb"); + return val; + }); + + var arn = await runner.StartAsync("input"); + var cbId = await runner.WaitForCallbackAsync(arn, name: "cb"); + await runner.SendCallbackSuccessAsync(cbId, "result1"); + var result1 = await runner.WaitForResultAsync(arn); + var result2 = await runner.WaitForResultAsync(arn); + + Assert.Same(result1, result2); + } + + [Fact] + public async Task SendCallbackHeartbeat_DoesNotThrow() + { + await using var runner = new DurableTestRunner( + handler: async (input, ctx) => + { + var val = await ctx.WaitForCallbackAsync( + async (_, _, _) => { }, name: "hb"); + return val; + }); + + var arn = await runner.StartAsync("input"); + var cbId = await runner.WaitForCallbackAsync(arn, name: "hb"); + + // Heartbeat should not throw + await runner.SendCallbackHeartbeatAsync(cbId); + + // Workflow still completes normally + await runner.SendCallbackSuccessAsync(cbId, "alive"); + var result = await runner.WaitForResultAsync(arn); + result.EnsureSucceeded(); + } + +} From f41efb97a9c85a8d8492b85995348240aa024edd Mon Sep 17 00:00:00 2001 From: Garrett Beatty Date: Tue, 16 Jun 2026 16:04:03 -0400 Subject: [PATCH 7/9] Add CloudDurableTestRunner for integration testing against deployed functions Implements the cloud test runner that invokes real Lambda functions: - StartAsync: invokes the function, extracts DurableExecutionArn from response - WaitForResultAsync: polls GetDurableExecutionState until terminal, builds TestResult - WaitForCallbackAsync: polls until a CALLBACK STARTED operation appears - SendCallbackSuccessAsync/FailureAsync/HeartbeatAsync: calls real Lambda APIs - BuildTestResult: reconstructs TestResult from polled operations Tests use a mock AmazonLambdaClient to verify ARN extraction, polling behavior, timeout handling, callback discovery, and API delegation. --- .../CloudDurableTestRunner.cs | 278 ++++++++++++++++++ ...mbda.DurableExecution.Testing.Tests.csproj | 1 + .../CloudDurableTestRunnerTests.cs | 220 ++++++++++++++ 3 files changed, 499 insertions(+) create mode 100644 Libraries/src/Amazon.Lambda.DurableExecution.Testing/CloudDurableTestRunner.cs create mode 100644 Libraries/test/Amazon.Lambda.DurableExecution.Testing.Tests/CloudDurableTestRunnerTests.cs diff --git a/Libraries/src/Amazon.Lambda.DurableExecution.Testing/CloudDurableTestRunner.cs b/Libraries/src/Amazon.Lambda.DurableExecution.Testing/CloudDurableTestRunner.cs new file mode 100644 index 000000000..925999628 --- /dev/null +++ b/Libraries/src/Amazon.Lambda.DurableExecution.Testing/CloudDurableTestRunner.cs @@ -0,0 +1,278 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +using System.Text; +using Amazon.Lambda; +using Amazon.Lambda.Core; +using Amazon.Lambda.DurableExecution.Services; +using Amazon.Lambda.Model; +using Amazon.Lambda.Serialization.SystemTextJson; + +namespace Amazon.Lambda.DurableExecution.Testing; + +/// +/// Cloud test runner that invokes a real deployed durable Lambda function +/// and polls for results. Provides the same +/// interface as the local runner for portable test code. +/// +public sealed class CloudDurableTestRunner : IDurableTestRunner, IAsyncDisposable +{ + private readonly string _functionArn; + private readonly IAmazonLambda _lambdaClient; + private readonly ILambdaSerializer _serializer; + private readonly CloudTestRunnerOptions _options; + + /// + /// Creates a cloud test runner targeting a deployed durable function. + /// + /// Qualified function ARN (with alias, version, or $LATEST). + /// AWS Lambda client. If null, creates a default client. + /// Cloud runner options. If null, uses defaults. + public CloudDurableTestRunner( + string functionArn, + IAmazonLambda? lambdaClient = null, + CloudTestRunnerOptions? options = null) + { + _functionArn = functionArn ?? throw new ArgumentNullException(nameof(functionArn)); + _lambdaClient = lambdaClient ?? new AmazonLambdaClient(); + _options = options ?? new CloudTestRunnerOptions(); + _serializer = _options.Serializer ?? new DefaultLambdaJsonSerializer(); + } + + /// + public async Task> RunAsync( + TInput input, + TimeSpan? timeout = null, + CancellationToken cancellationToken = default) + { + var arn = await StartAsync(input, timeout, cancellationToken); + return await WaitForResultAsync(arn, timeout, cancellationToken); + } + + /// + public async Task StartAsync( + TInput input, + TimeSpan? timeout = null, + CancellationToken cancellationToken = default) + { + var payload = SerializeToString(input); + + var response = await _lambdaClient.InvokeAsync(new InvokeRequest + { + FunctionName = _functionArn, + Payload = payload, + }, cancellationToken); + + var arn = ExtractDurableExecutionArn(response); + if (arn is null) + { + throw new CloudTestException( + "Lambda response did not include a DurableExecutionArn. " + + "Verify the function is configured with [DurableExecution]."); + } + + return arn; + } + + /// + public async Task> WaitForResultAsync( + string durableExecutionArn, + TimeSpan? timeout = null, + CancellationToken cancellationToken = default) + { + using var timeoutCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + timeoutCts.CancelAfter(timeout ?? _options.DefaultTimeout); + + var serviceClient = new LambdaDurableServiceClient(_lambdaClient); + string? marker = ""; + var operations = new Dictionary(); + + while (true) + { + timeoutCts.Token.ThrowIfCancellationRequested(); + + var (page, nextMarker) = await serviceClient.GetExecutionStateAsync( + durableExecutionArn, null, marker ?? "", timeoutCts.Token); + + foreach (var op in page) + operations[op.Id!] = op; + + marker = nextMarker; + + var execOp = operations.Values.FirstOrDefault(o => o.Type == OperationTypes.Execution); + if (execOp?.Status is OperationStatuses.Succeeded or OperationStatuses.Failed) + { + return BuildTestResult(durableExecutionArn, execOp, operations.Values); + } + + if (string.IsNullOrEmpty(marker)) + { + await Task.Delay(_options.PollInterval, timeoutCts.Token); + } + } + } + + /// + public async Task WaitForCallbackAsync( + string durableExecutionArn, + string? name = null, + TimeSpan? timeout = null, + CancellationToken cancellationToken = default) + { + using var timeoutCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + timeoutCts.CancelAfter(timeout ?? _options.DefaultTimeout); + + var serviceClient = new LambdaDurableServiceClient(_lambdaClient); + + while (true) + { + timeoutCts.Token.ThrowIfCancellationRequested(); + + var (page, _) = await serviceClient.GetExecutionStateAsync( + durableExecutionArn, null, "", timeoutCts.Token); + + foreach (var op in page) + { + if (op.Type == OperationTypes.Callback + && op.Status == OperationStatuses.Started + && op.CallbackDetails?.CallbackId is { } cbId) + { + if (name is null || MatchesCallbackName(op.Name, name)) + return cbId; + } + } + + await Task.Delay(_options.PollInterval, timeoutCts.Token); + } + } + + /// + public async Task SendCallbackSuccessAsync( + string callbackId, + TResult result, + CancellationToken cancellationToken = default) + { + var serialized = SerializeToString(result); + await _lambdaClient.SendDurableExecutionCallbackSuccessAsync( + new SendDurableExecutionCallbackSuccessRequest + { + CallbackId = callbackId, + Result = new MemoryStream(Encoding.UTF8.GetBytes(serialized)), + }, cancellationToken); + } + + /// + public async Task SendCallbackFailureAsync( + string callbackId, + ErrorObject? error = null, + CancellationToken cancellationToken = default) + { + await _lambdaClient.SendDurableExecutionCallbackFailureAsync( + new SendDurableExecutionCallbackFailureRequest + { + CallbackId = callbackId, + Error = error is not null ? new Amazon.Lambda.Model.ErrorObject + { + ErrorType = error.ErrorType, + ErrorMessage = error.ErrorMessage, + } : null, + }, cancellationToken); + } + + /// + public async Task SendCallbackHeartbeatAsync( + string callbackId, + CancellationToken cancellationToken = default) + { + await _lambdaClient.SendDurableExecutionCallbackHeartbeatAsync( + new SendDurableExecutionCallbackHeartbeatRequest + { + CallbackId = callbackId, + }, cancellationToken); + } + + /// + public ValueTask DisposeAsync() + { + return ValueTask.CompletedTask; + } + + private TestResult BuildTestResult(string arn, Operation execOp, IEnumerable allOps) + { + var status = execOp.Status switch + { + OperationStatuses.Succeeded => InvocationStatus.Succeeded, + OperationStatuses.Failed => InvocationStatus.Failed, + _ => InvocationStatus.Pending, + }; + + TOutput? result = default; + if (status == InvocationStatus.Succeeded && execOp.ExecutionDetails?.InputPayload is { } payload) + { + // The execution result is stored differently — check ContextDetails or direct result + // For cloud runner, the result comes from the execution operation + } + + var steps = allOps + .Where(o => o.Type != OperationTypes.Execution) + .Select(o => new TestStep(o, _serializer)) + .ToList(); + + return new TestResult( + status: status, + result: result, + error: execOp.ContextDetails?.Error, + durableExecutionArn: arn, + invocationCount: -1, + steps: steps); + } + + private static string? ExtractDurableExecutionArn(InvokeResponse response) + { + // The durable execution ARN is returned in the response headers/payload. + // Exact extraction depends on the SDK version; try known locations. + if (response.ResponseMetadata?.Metadata is { } metadata + && metadata.TryGetValue("x-amz-durable-execution-arn", out var arnFromHeader)) + { + return arnFromHeader; + } + + // Fallback: parse from the payload if the service embeds it there + if (response.Payload?.Length > 0) + { + try + { + response.Payload.Position = 0; + using var reader = new System.IO.StreamReader(response.Payload, Encoding.UTF8, leaveOpen: true); + var body = reader.ReadToEnd(); + if (body.Contains("DurableExecutionArn")) + { + var doc = System.Text.Json.JsonDocument.Parse(body); + if (doc.RootElement.TryGetProperty("DurableExecutionArn", out var arnProp)) + return arnProp.GetString(); + } + } + catch + { + // Parsing failed — fall through to null + } + } + + return null; + } + + private static bool MatchesCallbackName(string? opName, string name) + { + if (opName is null) return false; + if (string.Equals(opName, name, StringComparison.Ordinal)) return true; + if (string.Equals(opName, $"{name}-callback", StringComparison.Ordinal)) return true; + return false; + } + + private string SerializeToString(T value) + { + using var stream = new MemoryStream(); + _serializer.Serialize(value, stream); + return Encoding.UTF8.GetString(stream.ToArray()); + } +} diff --git a/Libraries/test/Amazon.Lambda.DurableExecution.Testing.Tests/Amazon.Lambda.DurableExecution.Testing.Tests.csproj b/Libraries/test/Amazon.Lambda.DurableExecution.Testing.Tests/Amazon.Lambda.DurableExecution.Testing.Tests.csproj index 1f0db748f..a6818b60a 100644 --- a/Libraries/test/Amazon.Lambda.DurableExecution.Testing.Tests/Amazon.Lambda.DurableExecution.Testing.Tests.csproj +++ b/Libraries/test/Amazon.Lambda.DurableExecution.Testing.Tests/Amazon.Lambda.DurableExecution.Testing.Tests.csproj @@ -23,6 +23,7 @@ + diff --git a/Libraries/test/Amazon.Lambda.DurableExecution.Testing.Tests/CloudDurableTestRunnerTests.cs b/Libraries/test/Amazon.Lambda.DurableExecution.Testing.Tests/CloudDurableTestRunnerTests.cs new file mode 100644 index 000000000..efec45db8 --- /dev/null +++ b/Libraries/test/Amazon.Lambda.DurableExecution.Testing.Tests/CloudDurableTestRunnerTests.cs @@ -0,0 +1,220 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +using System.Text; +using System.Text.Json; +using Amazon.Lambda; +using Amazon.Lambda.DurableExecution.Testing; +using Amazon.Lambda.Model; +using Xunit; + +namespace Amazon.Lambda.DurableExecution.Testing.Tests; + +public class CloudDurableTestRunnerTests +{ + private const string FunctionArn = "arn:aws:lambda:us-east-1:123456789012:function:my-durable-fn:$LATEST"; + private const string ExecutionArn = "arn:aws:lambda:us-east-1:123456789012:execution:my-durable-fn:exec-123"; + + [Fact] + public async Task StartAsync_ExtractsArn_FromResponsePayload() + { + var mockClient = new MockCloudLambdaClient(); + mockClient.InvokeHandler = _ => new InvokeResponse + { + StatusCode = 200, + Payload = new MemoryStream(Encoding.UTF8.GetBytes( + JsonSerializer.Serialize(new { DurableExecutionArn = ExecutionArn }))) + }; + + await using var runner = new CloudDurableTestRunner( + FunctionArn, mockClient); + + var arn = await runner.StartAsync("test-input"); + Assert.Equal(ExecutionArn, arn); + } + + [Fact] + public async Task StartAsync_ThrowsCloudTestException_WhenNoArn() + { + var mockClient = new MockCloudLambdaClient(); + mockClient.InvokeHandler = _ => new InvokeResponse + { + StatusCode = 200, + Payload = new MemoryStream(Encoding.UTF8.GetBytes("{}")) + }; + + await using var runner = new CloudDurableTestRunner( + FunctionArn, mockClient); + + await Assert.ThrowsAsync(() => runner.StartAsync("test")); + } + + [Fact] + public async Task WaitForResultAsync_PollsUntilTerminal() + { + var mockClient = new MockCloudLambdaClient(); + var pollCount = 0; + mockClient.GetExecutionStateHandler = _ => + { + pollCount++; + if (pollCount < 3) + { + return new GetDurableExecutionStateResponse + { + Operations = new List + { + new() { Id = "exec-0", Type = "EXECUTION", Status = "STARTED" } + } + }; + } + + return new GetDurableExecutionStateResponse + { + Operations = new List + { + new() + { + Id = "exec-0", + Type = "EXECUTION", + Status = "SUCCEEDED", + ExecutionDetails = new Amazon.Lambda.Model.ExecutionDetails { InputPayload = """{"x":1}""" } + }, + new() + { + Id = "op-1", + Type = "STEP", + Status = "SUCCEEDED", + Name = "step1", + StepDetails = new Amazon.Lambda.Model.StepDetails { Result = """{"Value":"hello"}""" } + } + } + }; + }; + + await using var runner = new CloudDurableTestRunner( + FunctionArn, mockClient, + new CloudTestRunnerOptions { PollInterval = TimeSpan.FromMilliseconds(10) }); + + var result = await runner.WaitForResultAsync(ExecutionArn, timeout: TimeSpan.FromSeconds(5)); + + Assert.Equal(InvocationStatus.Succeeded, result.Status); + Assert.Equal(-1, result.InvocationCount); + Assert.True(pollCount >= 3); + Assert.Single(result.Steps); + Assert.Equal("step1", result.Steps[0].Name); + } + + [Fact] + public async Task WaitForResultAsync_Timeout_Throws() + { + var mockClient = new MockCloudLambdaClient(); + mockClient.GetExecutionStateHandler = _ => new GetDurableExecutionStateResponse + { + Operations = new List + { + new() { Id = "exec-0", Type = "EXECUTION", Status = "STARTED" } + } + }; + + await using var runner = new CloudDurableTestRunner( + FunctionArn, mockClient, + new CloudTestRunnerOptions { PollInterval = TimeSpan.FromMilliseconds(10) }); + + var ex = await Assert.ThrowsAnyAsync( + () => runner.WaitForResultAsync(ExecutionArn, timeout: TimeSpan.FromMilliseconds(50))); + + // Cancellation surfaces as OperationCanceledException or wrapped in DurableExecutionException + Assert.True( + ex is OperationCanceledException || + (ex is DurableExecutionException dee && dee.InnerException is OperationCanceledException), + $"Expected cancellation exception but got: {ex.GetType().Name}: {ex.Message}"); + } + + [Fact] + public async Task WaitForCallbackAsync_FindsCallbackFromPolledState() + { + var mockClient = new MockCloudLambdaClient(); + mockClient.GetExecutionStateHandler = _ => new GetDurableExecutionStateResponse + { + Operations = new List + { + new() { Id = "exec-0", Type = "EXECUTION", Status = "STARTED" }, + new() + { + Id = "op-cb", + Type = "CALLBACK", + Status = "STARTED", + Name = "approval-callback", + CallbackDetails = new Amazon.Lambda.Model.CallbackDetails { CallbackId = "Y2IxMjM0NQ==" } + } + } + }; + + await using var runner = new CloudDurableTestRunner( + FunctionArn, mockClient, + new CloudTestRunnerOptions { PollInterval = TimeSpan.FromMilliseconds(10) }); + + var cbId = await runner.WaitForCallbackAsync(ExecutionArn, name: "approval", timeout: TimeSpan.FromSeconds(2)); + Assert.Equal("Y2IxMjM0NQ==", cbId); + } + + [Fact] + public async Task SendCallbackSuccessAsync_CallsLambdaApi() + { + var mockClient = new MockCloudLambdaClient(); + await using var runner = new CloudDurableTestRunner( + FunctionArn, mockClient); + + await runner.SendCallbackSuccessAsync("Y2IxMjM=", "approved"); + + Assert.Single(mockClient.CallbackSuccessCalls); + Assert.Equal("Y2IxMjM=", mockClient.CallbackSuccessCalls[0].CallbackId); + } + + /// + /// Minimal mock of IAmazonLambda for cloud runner tests. + /// Subclasses AmazonLambdaClient to override relevant methods. + /// + private sealed class MockCloudLambdaClient : AmazonLambdaClient + { + public Func? InvokeHandler { get; set; } + public Func? GetExecutionStateHandler { get; set; } + public List CallbackSuccessCalls { get; } = new(); + + public MockCloudLambdaClient() : base("fake-key", "fake-secret", Amazon.RegionEndpoint.USEast1) { } + + public override Task InvokeAsync(InvokeRequest request, CancellationToken ct = default) + { + if (InvokeHandler is null) + throw new InvalidOperationException("InvokeHandler not configured"); + return Task.FromResult(InvokeHandler(request)); + } + + public override Task GetDurableExecutionStateAsync( + GetDurableExecutionStateRequest request, CancellationToken ct = default) + { + if (GetExecutionStateHandler is null) + return Task.FromResult(new GetDurableExecutionStateResponse()); + return Task.FromResult(GetExecutionStateHandler(request)); + } + + public override Task SendDurableExecutionCallbackSuccessAsync( + SendDurableExecutionCallbackSuccessRequest request, CancellationToken ct = default) + { + CallbackSuccessCalls.Add(request); + return Task.FromResult(new SendDurableExecutionCallbackSuccessResponse()); + } + + public override Task SendDurableExecutionCallbackFailureAsync( + SendDurableExecutionCallbackFailureRequest request, CancellationToken ct = default) + { + return Task.FromResult(new SendDurableExecutionCallbackFailureResponse()); + } + + public override Task SendDurableExecutionCallbackHeartbeatAsync( + SendDurableExecutionCallbackHeartbeatRequest request, CancellationToken ct = default) + { + return Task.FromResult(new SendDurableExecutionCallbackHeartbeatResponse()); + } + } +} From bc21b2dc595e717289e81d38ca27d3e65c203bef Mon Sep 17 00:00:00 2001 From: Garrett Beatty Date: Wed, 17 Jun 2026 11:16:23 -0400 Subject: [PATCH 8/9] Configure Amazon.Lambda.DurableExecution.Testing with autover Add the testing package to .autover/autover.json and set version to 0.0.1. Preview label will be added manually on first release. --- .autover/autover.json | 4 ++++ .../Amazon.Lambda.DurableExecution.Testing.csproj | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/.autover/autover.json b/.autover/autover.json index 02f2ad0db..2a1cb3f73 100644 --- a/.autover/autover.json +++ b/.autover/autover.json @@ -52,6 +52,10 @@ "Path": "Libraries/src/Amazon.Lambda.DurableExecution/Amazon.Lambda.DurableExecution.csproj", "PrereleaseLabel": "preview" }, + { + "Name": "Amazon.Lambda.DurableExecution.Testing", + "Path": "Libraries/src/Amazon.Lambda.DurableExecution.Testing/Amazon.Lambda.DurableExecution.Testing.csproj" + }, { "Name": "Amazon.Lambda.DynamoDBEvents", "Path": "Libraries/src/Amazon.Lambda.DynamoDBEvents/Amazon.Lambda.DynamoDBEvents.csproj" diff --git a/Libraries/src/Amazon.Lambda.DurableExecution.Testing/Amazon.Lambda.DurableExecution.Testing.csproj b/Libraries/src/Amazon.Lambda.DurableExecution.Testing/Amazon.Lambda.DurableExecution.Testing.csproj index 471825f52..0f9178380 100644 --- a/Libraries/src/Amazon.Lambda.DurableExecution.Testing/Amazon.Lambda.DurableExecution.Testing.csproj +++ b/Libraries/src/Amazon.Lambda.DurableExecution.Testing/Amazon.Lambda.DurableExecution.Testing.csproj @@ -6,7 +6,7 @@ $(DefaultPackageTargets) Testing utilities for Amazon Lambda Durable Execution - test durable workflows locally without deploying to AWS. Amazon.Lambda.DurableExecution.Testing - 0.0.1-preview + 0.0.1 Amazon.Lambda.DurableExecution.Testing Amazon.Lambda.DurableExecution.Testing AWS;Amazon;Lambda;Durable;Workflow;Testing From 4b9d0594ed2895919acb6d9d41853d8572991dca Mon Sep 17 00:00:00 2001 From: Garrett Beatty Date: Wed, 17 Jun 2026 14:02:15 -0400 Subject: [PATCH 9/9] Fix testing-package review findings: sibling invoke, cloud runner, WaitForCondition Wire up and correct the Amazon.Lambda.DurableExecution.Testing package so its headline features work end-to-end, with integration coverage for each. Local runner: - Wire RegisterFunction/RegisterDurableFunction into execution: CheckpointProcessor captures CHAINED_INVOKE STARTs and the orchestrator resolves them through the FunctionRegistry between invocations, stamping ChainedInvokeDetails.Result/Error. Unregistered siblings throw UnregisteredSiblingFunctionException instead of looping to TestExecutionLimitException. - Implement durable siblings via a nested DurableTestRunner (was NotImplementedException). - Persist WaitForCondition poll state (RETRY Payload -> StepDetails.Result) and advance its attempt counter on RETRY (it emits START only once); remove the dead Type==Wait WaitForCondition branches (runtime emits Type=STEP). - Accumulate InvocationCount across StartAsync + WaitForResultAsync; make SeedExecutionOperation idempotent so re-drives don't reset the EXECUTION op. - RunAsync throws an actionable error when a workflow suspends on a callback. Cloud runner: - Determine terminal state and the typed result/error from GetDurableExecution (Status/Result/Error) instead of scanning the EXECUTION op in the state stream, which never reaches a terminal status. - Use the typed InvokeResponse.DurableExecutionArn; treat TIMED_OUT/STOPPED as terminal; map all four error fields on SendCallbackFailureAsync. Hardening: - InMemoryOperationStore is lock-guarded and returns snapshot copies. - OperationStatus constants alias the runtime OperationStatuses (compile-time linked). - TestResult gains IsSucceeded/IsFailed and GetStepsByStatus. - Mark the package preview (0.0.1-preview + autover PrereleaseLabel) to match the runtime. Tests: +15 (sibling invoke incl. durable + unregistered + failure, WaitForCondition end-to-end, callback-via-RunAsync error, accumulated invocation count, step retry attempt 1->2->3, cloud typed result/error/terminal-status, store snapshot/concurrency). 106 -> 121 passing on net8.0 and net10.0. --- .autover/autover.json | 3 +- ...zon.Lambda.DurableExecution.Testing.csproj | 13 +- .../CheckpointProcessor.cs | 83 +++++++-- .../CloudDurableTestRunner.cs | 129 +++++++------- .../DurableTestRunner.cs | 18 +- .../ExecutionOrchestrator.cs | 142 +++++++++++---- .../FunctionRegistry.cs | 42 +++-- .../IDurableTestRunner.cs | 8 +- .../InMemoryOperationStore.cs | 61 +++++-- .../OperationStatus.cs | 21 ++- .../TestResult.cs | 34 +++- .../TestRunnerOptions.cs | 11 +- .../CheckpointProcessorTests.cs | 14 +- .../CloudDurableTestRunnerTests.cs | 163 +++++++++++++----- .../InMemoryOperationStoreTests.cs | 35 ++++ .../RunAsyncBehaviorTests.cs | 138 +++++++++++++++ .../SiblingInvokeIntegrationTests.cs | 143 +++++++++++++++ 17 files changed, 849 insertions(+), 209 deletions(-) create mode 100644 Libraries/test/Amazon.Lambda.DurableExecution.Testing.Tests/RunAsyncBehaviorTests.cs create mode 100644 Libraries/test/Amazon.Lambda.DurableExecution.Testing.Tests/SiblingInvokeIntegrationTests.cs diff --git a/.autover/autover.json b/.autover/autover.json index 2a1cb3f73..b0a7dbc47 100644 --- a/.autover/autover.json +++ b/.autover/autover.json @@ -54,7 +54,8 @@ }, { "Name": "Amazon.Lambda.DurableExecution.Testing", - "Path": "Libraries/src/Amazon.Lambda.DurableExecution.Testing/Amazon.Lambda.DurableExecution.Testing.csproj" + "Path": "Libraries/src/Amazon.Lambda.DurableExecution.Testing/Amazon.Lambda.DurableExecution.Testing.csproj", + "PrereleaseLabel": "preview" }, { "Name": "Amazon.Lambda.DynamoDBEvents", diff --git a/Libraries/src/Amazon.Lambda.DurableExecution.Testing/Amazon.Lambda.DurableExecution.Testing.csproj b/Libraries/src/Amazon.Lambda.DurableExecution.Testing/Amazon.Lambda.DurableExecution.Testing.csproj index 0f9178380..90aff9aec 100644 --- a/Libraries/src/Amazon.Lambda.DurableExecution.Testing/Amazon.Lambda.DurableExecution.Testing.csproj +++ b/Libraries/src/Amazon.Lambda.DurableExecution.Testing/Amazon.Lambda.DurableExecution.Testing.csproj @@ -6,7 +6,8 @@ $(DefaultPackageTargets) Testing utilities for Amazon Lambda Durable Execution - test durable workflows locally without deploying to AWS. Amazon.Lambda.DurableExecution.Testing - 0.0.1 + + 0.0.1-preview Amazon.Lambda.DurableExecution.Testing Amazon.Lambda.DurableExecution.Testing AWS;Amazon;Lambda;Durable;Workflow;Testing @@ -22,6 +23,16 @@ + diff --git a/Libraries/src/Amazon.Lambda.DurableExecution.Testing/CheckpointProcessor.cs b/Libraries/src/Amazon.Lambda.DurableExecution.Testing/CheckpointProcessor.cs index fb621c659..8d709d776 100644 --- a/Libraries/src/Amazon.Lambda.DurableExecution.Testing/CheckpointProcessor.cs +++ b/Libraries/src/Amazon.Lambda.DurableExecution.Testing/CheckpointProcessor.cs @@ -15,6 +15,8 @@ internal sealed class CheckpointProcessor { private readonly InMemoryOperationStore _store; private readonly bool _skipTime; + private readonly object _pendingGate = new(); + private readonly List _pendingInvokes = new(); public CheckpointProcessor(InMemoryOperationStore store, bool skipTime) { @@ -22,6 +24,33 @@ public CheckpointProcessor(InMemoryOperationStore store, bool skipTime) _skipTime = skipTime; } + /// + /// A chained-invoke (ctx.InvokeAsync) that has been started by the + /// workflow but not yet resolved. The runtime suspends after emitting the + /// START and expects an external system to run the target function; in the + /// local harness the + /// drains these between invocations and resolves them via the + /// . The target function name lives only on the + /// wire-format OperationUpdate.ChainedInvokeOptions, so it is captured + /// here rather than on the persisted . + /// + internal readonly record struct PendingInvoke(string OperationId, string FunctionName, string? Payload); + + /// + /// Returns and clears the chained-invokes started since the last drain. + /// + public IReadOnlyList DrainPendingInvokes() + { + lock (_pendingGate) + { + if (_pendingInvokes.Count == 0) + return Array.Empty(); + var drained = _pendingInvokes.ToArray(); + _pendingInvokes.Clear(); + return drained; + } + } + /// /// Processes a batch of updates and returns the new checkpoint token /// and any operations that were created or modified (to feed back to @@ -60,6 +89,20 @@ private Operation ApplyUpdate(string arn, SdkOperationUpdate update) if (_skipTime) ApplyTimeSkipping(operation, action); + // A chained-invoke START suspends the workflow until an external system + // resolves it. Record it so the orchestrator can run the registered + // sibling and stamp the result/error before the next replay. The function + // name is only carried on the wire-format update, not on the Operation. + if (action == "START" + && operation.Type == OperationTypes.ChainedInvoke + && update.ChainedInvokeOptions?.FunctionName is { } functionName) + { + lock (_pendingGate) + { + _pendingInvokes.Add(new PendingInvoke(operation.Id!, functionName, update.Payload)); + } + } + _store.Upsert(arn, operation); return operation; } @@ -104,7 +147,12 @@ private static void ApplyStartDetails(Operation operation, SdkOperationUpdate up { case OperationTypes.Step: operation.StepDetails ??= new StepDetails(); - operation.StepDetails.Attempt = (operation.StepDetails.Attempt ?? 0) + 1; + // A plain step re-emits START before every attempt, so START owns + // the attempt count. WaitForCondition (Type=STEP, SubType=WaitForCondition) + // emits START only once and advances the count on each RETRY instead, + // so it must NOT increment here. + if (operation.SubType != OperationSubTypes.WaitForCondition) + operation.StepDetails.Attempt = (operation.StepDetails.Attempt ?? 0) + 1; break; case OperationTypes.Wait: @@ -195,6 +243,9 @@ private static void ApplyFailDetails(Operation operation, SdkOperationUpdate upd private static void ApplyRetryDetails(Operation operation, SdkOperationUpdate update) { + // Both retried steps and WaitForCondition polls are wire-encoded as + // Type=STEP (WaitForCondition uses SubType=WaitForCondition); the runtime + // never emits a WAIT-typed RETRY, so this single STEP branch covers both. if (operation.Type == OperationTypes.Step) { operation.StepDetails ??= new StepDetails(); @@ -204,15 +255,17 @@ private static void ApplyRetryDetails(Operation operation, SdkOperationUpdate up DateTimeOffset.UtcNow.AddSeconds(delaySeconds).ToUnixTimeMilliseconds(); } operation.StepDetails.Error = MapSdkError(update.Error); - } - if (operation.Type == OperationTypes.Wait && operation.SubType == OperationSubTypes.WaitForCondition) - { - operation.WaitDetails ??= new WaitDetails(); - if (update.WaitOptions?.WaitSeconds is { } waitSeconds) + // WaitForCondition emits START once and advances per RETRY: it carries + // the next poll state in Payload and relies on the persistence layer to + // own the attempt count. Persist both so the next replay resumes from + // the latest state with an advanced attempt number (a plain step RETRY + // carries no Payload and owns its count via START, so leave it alone). + if (operation.SubType == OperationSubTypes.WaitForCondition) { - operation.WaitDetails.ScheduledEndTimestamp = - DateTimeOffset.UtcNow.AddSeconds(waitSeconds).ToUnixTimeMilliseconds(); + if (update.Payload is not null) + operation.StepDetails.Result = update.Payload; + operation.StepDetails.Attempt = (operation.StepDetails.Attempt ?? 0) + 1; } } } @@ -230,6 +283,9 @@ private void ApplyTimeSkipping(Operation operation, string? action) } } + // A retried step (or WaitForCondition poll, also Type=STEP) becomes + // immediately READY under time-skipping so the next replay runs the next + // attempt without waiting for the backoff/poll delay. if (action == "RETRY" && operation.Type == OperationTypes.Step) { operation.Status = OperationStatuses.Ready; @@ -239,17 +295,6 @@ private void ApplyTimeSkipping(Operation operation, string? action) DateTimeOffset.UtcNow.AddMilliseconds(-1).ToUnixTimeMilliseconds(); } } - - if (action == "RETRY" && operation.Type == OperationTypes.Wait - && operation.SubType == OperationSubTypes.WaitForCondition) - { - operation.Status = OperationStatuses.Ready; - if (operation.WaitDetails != null) - { - operation.WaitDetails.ScheduledEndTimestamp = - DateTimeOffset.UtcNow.AddMilliseconds(-1).ToUnixTimeMilliseconds(); - } - } } private static ErrorObject? MapSdkError(Amazon.Lambda.Model.ErrorObject? sdkError) diff --git a/Libraries/src/Amazon.Lambda.DurableExecution.Testing/CloudDurableTestRunner.cs b/Libraries/src/Amazon.Lambda.DurableExecution.Testing/CloudDurableTestRunner.cs index 925999628..87ea95db3 100644 --- a/Libraries/src/Amazon.Lambda.DurableExecution.Testing/CloudDurableTestRunner.cs +++ b/Libraries/src/Amazon.Lambda.DurableExecution.Testing/CloudDurableTestRunner.cs @@ -63,8 +63,10 @@ public async Task StartAsync( Payload = payload, }, cancellationToken); - var arn = ExtractDurableExecutionArn(response); - if (arn is null) + // The durable execution ARN is returned as the X-Amz-Durable-Execution-Arn + // response header, which the SDK unmarshals into this typed property. + var arn = response.DurableExecutionArn; + if (string.IsNullOrEmpty(arn)) { throw new CloudTestException( "Lambda response did not include a DurableExecutionArn. " + @@ -83,33 +85,57 @@ public async Task> WaitForResultAsync( using var timeoutCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); timeoutCts.CancelAfter(timeout ?? _options.DefaultTimeout); - var serviceClient = new LambdaDurableServiceClient(_lambdaClient); - string? marker = ""; - var operations = new Dictionary(); - + // Terminal status and the workflow result/error are execution-level concepts + // surfaced by GetDurableExecution — they are NOT on the EXECUTION operation in + // the GetDurableExecutionState stream (that op only carries the input payload). while (true) { timeoutCts.Token.ThrowIfCancellationRequested(); - var (page, nextMarker) = await serviceClient.GetExecutionStateAsync( - durableExecutionArn, null, marker ?? "", timeoutCts.Token); + var execution = await _lambdaClient.GetDurableExecutionAsync( + new GetDurableExecutionRequest { DurableExecutionArn = durableExecutionArn }, + timeoutCts.Token); - foreach (var op in page) - operations[op.Id!] = op; - - marker = nextMarker; - - var execOp = operations.Values.FirstOrDefault(o => o.Type == OperationTypes.Execution); - if (execOp?.Status is OperationStatuses.Succeeded or OperationStatuses.Failed) + if (IsTerminal(execution.Status)) { - return BuildTestResult(durableExecutionArn, execOp, operations.Values); + var operations = await FetchAllOperationsAsync(durableExecutionArn, timeoutCts.Token); + return BuildTestResult(durableExecutionArn, execution, operations); } - if (string.IsNullOrEmpty(marker)) + await Task.Delay(_options.PollInterval, timeoutCts.Token); + } + } + + private static bool IsTerminal(ExecutionStatus? status) => + status == ExecutionStatus.SUCCEEDED + || status == ExecutionStatus.FAILED + || status == ExecutionStatus.TIMED_OUT + || status == ExecutionStatus.STOPPED; + + private async Task> FetchAllOperationsAsync( + string durableExecutionArn, CancellationToken cancellationToken) + { + var serviceClient = new LambdaDurableServiceClient(_lambdaClient); + var operations = new Dictionary(); + string? marker = ""; + + do + { + cancellationToken.ThrowIfCancellationRequested(); + var (page, nextMarker) = await serviceClient.GetExecutionStateAsync( + durableExecutionArn, null, marker ?? "", cancellationToken); + + foreach (var op in page) { - await Task.Delay(_options.PollInterval, timeoutCts.Token); + if (op.Id is { } id) + operations[id] = op; } + + marker = nextMarker; } + while (!string.IsNullOrEmpty(marker)); + + return operations.Values.ToList(); } /// @@ -175,6 +201,8 @@ await _lambdaClient.SendDurableExecutionCallbackFailureAsync( { ErrorType = error.ErrorType, ErrorMessage = error.ErrorMessage, + ErrorData = error.ErrorData, + StackTrace = error.StackTrace?.ToList(), } : null, }, cancellationToken); } @@ -197,20 +225,16 @@ public ValueTask DisposeAsync() return ValueTask.CompletedTask; } - private TestResult BuildTestResult(string arn, Operation execOp, IEnumerable allOps) + private TestResult BuildTestResult( + string arn, GetDurableExecutionResponse execution, IReadOnlyList allOps) { - var status = execOp.Status switch - { - OperationStatuses.Succeeded => InvocationStatus.Succeeded, - OperationStatuses.Failed => InvocationStatus.Failed, - _ => InvocationStatus.Pending, - }; + var status = MapExecutionStatus(execution.Status); TOutput? result = default; - if (status == InvocationStatus.Succeeded && execOp.ExecutionDetails?.InputPayload is { } payload) + if (status == InvocationStatus.Succeeded && !string.IsNullOrEmpty(execution.Result)) { - // The execution result is stored differently — check ContextDetails or direct result - // For cloud runner, the result comes from the execution operation + using var stream = new MemoryStream(Encoding.UTF8.GetBytes(execution.Result)); + result = _serializer.Deserialize(stream); } var steps = allOps @@ -221,44 +245,33 @@ private TestResult BuildTestResult(string arn, Operation execOp, IEnume return new TestResult( status: status, result: result, - error: execOp.ContextDetails?.Error, + error: MapSdkError(execution.Error), durableExecutionArn: arn, invocationCount: -1, steps: steps); } - private static string? ExtractDurableExecutionArn(InvokeResponse response) + // The runtime's terminal states beyond Succeeded (FAILED/TIMED_OUT/STOPPED) + // all map to Failed since InvocationStatus has no finer terminal distinction. + private static InvocationStatus MapExecutionStatus(ExecutionStatus? status) { - // The durable execution ARN is returned in the response headers/payload. - // Exact extraction depends on the SDK version; try known locations. - if (response.ResponseMetadata?.Metadata is { } metadata - && metadata.TryGetValue("x-amz-durable-execution-arn", out var arnFromHeader)) - { - return arnFromHeader; - } + if (status == ExecutionStatus.SUCCEEDED) return InvocationStatus.Succeeded; + if (status == ExecutionStatus.FAILED + || status == ExecutionStatus.TIMED_OUT + || status == ExecutionStatus.STOPPED) return InvocationStatus.Failed; + return InvocationStatus.Pending; + } - // Fallback: parse from the payload if the service embeds it there - if (response.Payload?.Length > 0) + private static ErrorObject? MapSdkError(Amazon.Lambda.Model.ErrorObject? sdkError) + { + if (sdkError is null) return null; + return new ErrorObject { - try - { - response.Payload.Position = 0; - using var reader = new System.IO.StreamReader(response.Payload, Encoding.UTF8, leaveOpen: true); - var body = reader.ReadToEnd(); - if (body.Contains("DurableExecutionArn")) - { - var doc = System.Text.Json.JsonDocument.Parse(body); - if (doc.RootElement.TryGetProperty("DurableExecutionArn", out var arnProp)) - return arnProp.GetString(); - } - } - catch - { - // Parsing failed — fall through to null - } - } - - return null; + ErrorType = sdkError.ErrorType, + ErrorMessage = sdkError.ErrorMessage, + ErrorData = sdkError.ErrorData, + StackTrace = sdkError.StackTrace, + }; } private static bool MatchesCallbackName(string? opName, string name) diff --git a/Libraries/src/Amazon.Lambda.DurableExecution.Testing/DurableTestRunner.cs b/Libraries/src/Amazon.Lambda.DurableExecution.Testing/DurableTestRunner.cs index 25b6fb211..745007404 100644 --- a/Libraries/src/Amazon.Lambda.DurableExecution.Testing/DurableTestRunner.cs +++ b/Libraries/src/Amazon.Lambda.DurableExecution.Testing/DurableTestRunner.cs @@ -32,6 +32,19 @@ public sealed class DurableTestRunner : IDurableTestRunner> handler, TestRunnerOptions? options = null) + : this(handler, options, registry: null) + { + } + + /// + /// Creates a local test runner that shares an existing . + /// Used when a durable sibling is invoked: the nested runner inherits the parent's + /// registered functions so chains of durable-to-durable invokes resolve. + /// + internal DurableTestRunner( + Func> handler, + TestRunnerOptions? options, + FunctionRegistry? registry) { _handler = handler ?? throw new ArgumentNullException(nameof(handler)); _options = options ?? new TestRunnerOptions(); @@ -40,7 +53,7 @@ public DurableTestRunner( _store = new InMemoryOperationStore(); _processor = new CheckpointProcessor(_store, _options.SkipTime); _serviceClient = new InMemoryDurableServiceClient(_store, _processor); - _registry = new FunctionRegistry(); + _registry = registry ?? new FunctionRegistry(_options); } /// @@ -235,7 +248,8 @@ public ValueTask DisposeAsync() private ExecutionOrchestrator CreateOrchestrator() { return new ExecutionOrchestrator( - _handler, _store, _serviceClient, _lambdaContext, _options, _serializer); + _handler, _store, _serviceClient, _lambdaContext, _options, _serializer, + _processor, _registry); } private TestLambdaContext CreateLambdaContext() diff --git a/Libraries/src/Amazon.Lambda.DurableExecution.Testing/ExecutionOrchestrator.cs b/Libraries/src/Amazon.Lambda.DurableExecution.Testing/ExecutionOrchestrator.cs index 74e1f436e..bac7d9399 100644 --- a/Libraries/src/Amazon.Lambda.DurableExecution.Testing/ExecutionOrchestrator.cs +++ b/Libraries/src/Amazon.Lambda.DurableExecution.Testing/ExecutionOrchestrator.cs @@ -19,6 +19,13 @@ internal sealed class ExecutionOrchestrator private readonly ILambdaContext _lambdaContext; private readonly TestRunnerOptions _options; private readonly ILambdaSerializer _serializer; + private readonly CheckpointProcessor? _processor; + private readonly FunctionRegistry? _registry; + + // Accumulates across DriveUntilSuspendedAsync + DriveToTerminalAsync so the + // reported InvocationCount reflects the whole run (including the invocations + // consumed before a callback suspension), not just the final drive. + private int _invocationCount; public ExecutionOrchestrator( Func> handler, @@ -26,7 +33,9 @@ public ExecutionOrchestrator( IDurableServiceClient serviceClient, ILambdaContext lambdaContext, TestRunnerOptions options, - ILambdaSerializer serializer) + ILambdaSerializer serializer, + CheckpointProcessor? processor = null, + FunctionRegistry? registry = null) { _handler = handler; _store = store; @@ -34,12 +43,36 @@ public ExecutionOrchestrator( _lambdaContext = lambdaContext; _options = options; _serializer = serializer; + _processor = processor; + _registry = registry; + } + + public Task?> DriveUntilSuspendedAsync( + string arn, + TInput input, + TimeSpan timeout, + CancellationToken cancellationToken) + => DriveAsync(arn, input, timeout, stopAtSuspend: true, cancellationToken); + + public async Task> DriveToTerminalAsync( + string arn, + TInput input, + TimeSpan timeout, + CancellationToken cancellationToken) + { + var result = await DriveAsync(arn, input, timeout, stopAtSuspend: false, cancellationToken); + // stopAtSuspend:false only returns null when the workflow suspended on a + // callback it cannot drive itself — surface that as a clear, actionable error. + return result ?? throw new InvalidOperationException( + "Workflow suspended waiting on a callback and cannot be driven to completion with RunAsync. " + + "Use the two-call pattern instead: StartAsync, then WaitForCallbackAsync + SendCallbackSuccessAsync/SendCallbackFailureAsync, then WaitForResultAsync."); } - public async Task?> DriveUntilSuspendedAsync( + private async Task?> DriveAsync( string arn, TInput input, TimeSpan timeout, + bool stopAtSuspend, CancellationToken cancellationToken) { using var timeoutCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); @@ -47,14 +80,11 @@ public ExecutionOrchestrator( SeedExecutionOperation(arn, input); - var invocationCount = 0; - DurableExecutionInvocationOutput output; - while (true) { timeoutCts.Token.ThrowIfCancellationRequested(); - if (invocationCount >= _options.MaxInvocations) + if (_invocationCount >= _options.MaxInvocations) { throw new TestExecutionLimitException( _options.MaxInvocations, _store.OperationCount(arn)); @@ -62,59 +92,95 @@ public ExecutionOrchestrator( var invocationInput = BuildInvocationInput(arn); - output = await DurableFunction.WrapAsync( + var output = await DurableFunction.WrapAsync( _handler, invocationInput, _lambdaContext, _serviceClient); - invocationCount++; - - if (output.Status == InvocationStatus.Pending) - return null; // Suspended — test code drives callbacks + _invocationCount++; if (output.Status != InvocationStatus.Pending) - return BuildResult(arn, output, invocationCount); + return BuildResult(arn, output, _invocationCount); + + // Pending. Resolve any sibling invokes the workflow started this pass + // (the runtime suspends on a CHAINED_INVOKE START expecting an external + // system to run the target); then re-drive so replay sees them resolved. + if (await ResolvePendingInvokesAsync(arn, timeoutCts.Token)) + continue; + + // Genuinely suspended with nothing to resolve. A pending callback means + // the workflow is waiting on external input; the caller decides whether + // that is a suspend point (StartAsync) or an error (RunAsync). For other + // pending states (e.g. a real wait with SkipTime disabled) keep looping + // until the workflow progresses, hits MaxInvocations, or times out. + if (stopAtSuspend || HasPendingCallback(arn)) + return null; } } - public async Task> DriveToTerminalAsync( - string arn, - TInput input, - TimeSpan timeout, - CancellationToken cancellationToken) + /// + /// Runs any chained-invoke siblings started during the last invocation through + /// the and stamps the result/error onto the + /// stored operation so the next replay resolves it. Returns true if at least + /// one invoke was resolved (i.e. the workflow should be re-driven). + /// + private async Task ResolvePendingInvokesAsync(string arn, CancellationToken cancellationToken) { - using var timeoutCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); - timeoutCts.CancelAfter(timeout); - - SeedExecutionOperation(arn, input); + if (_processor is null || _registry is null) + return false; - var invocationCount = 0; - DurableExecutionInvocationOutput output; + var pending = _processor.DrainPendingInvokes(); + if (pending.Count == 0) + return false; - while (true) + foreach (var invoke in pending) { - timeoutCts.Token.ThrowIfCancellationRequested(); + cancellationToken.ThrowIfCancellationRequested(); - if (invocationCount >= _options.MaxInvocations) - { - throw new TestExecutionLimitException( - _options.MaxInvocations, _store.OperationCount(arn)); - } + // An unregistered sibling throws UnregisteredSiblingFunctionException + // (out of InvokeAsync) so it surfaces with actionable guidance rather + // than as an opaque MaxInvocations timeout. + var (result, error) = await _registry.InvokeAsync( + invoke.FunctionName, invoke.Payload ?? "null", _serializer, _lambdaContext); - var invocationInput = BuildInvocationInput(arn); + var op = _store.GetOperation(arn, invoke.OperationId); + if (op is null) + continue; - output = await DurableFunction.WrapAsync( - _handler, invocationInput, _lambdaContext, _serviceClient); + op.ChainedInvokeDetails ??= new ChainedInvokeDetails(); + op.EndTimestamp = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(); + if (error is null) + { + op.Status = OperationStatuses.Succeeded; + op.ChainedInvokeDetails.Result = result; + op.ChainedInvokeDetails.Error = null; + } + else + { + op.Status = OperationStatuses.Failed; + op.ChainedInvokeDetails.Error = error; + } + _store.Upsert(arn, op); + } - invocationCount++; + return true; + } - if (output.Status != InvocationStatus.Pending) - break; + private bool HasPendingCallback(string arn) + { + foreach (var op in _store.GetAllOperations(arn)) + { + if (op.Type == OperationTypes.Callback && op.Status == OperationStatuses.Started) + return true; } - - return BuildResult(arn, output, invocationCount); + return false; } private void SeedExecutionOperation(string arn, TInput input) { + // Idempotent: re-driving (e.g. WaitForResultAsync after a callback) must + // not reset the EXECUTION op back to Started or clobber recorded state. + if (_store.GetOperation(arn, "exec-0") is not null) + return; + var serializedInput = SerializeToString(input); _store.Upsert(arn, new Operation { diff --git a/Libraries/src/Amazon.Lambda.DurableExecution.Testing/FunctionRegistry.cs b/Libraries/src/Amazon.Lambda.DurableExecution.Testing/FunctionRegistry.cs index a12ce0035..26afd6314 100644 --- a/Libraries/src/Amazon.Lambda.DurableExecution.Testing/FunctionRegistry.cs +++ b/Libraries/src/Amazon.Lambda.DurableExecution.Testing/FunctionRegistry.cs @@ -13,6 +13,12 @@ namespace Amazon.Lambda.DurableExecution.Testing; internal sealed class FunctionRegistry { private readonly List _entries = new(); + private readonly TestRunnerOptions _options; + + public FunctionRegistry(TestRunnerOptions? options = null) + { + _options = options ?? new TestRunnerOptions(); + } public void RegisterPlain( string functionNameOrArn, @@ -49,8 +55,7 @@ public void RegisterDurable( try { - var result = await entry.InvokeDelegate(serializedPayload, serializer, lambdaContext); - return (result, null); + return await entry.InvokeDelegate(serializedPayload, serializer, lambdaContext); } catch (Exception ex) { @@ -91,7 +96,7 @@ private static string ExtractName(string functionNameOrArn) return functionNameOrArn; } - private static async Task InvokePlain( + private static async Task<(string? Result, ErrorObject? Error)> InvokePlain( Func> handler, string serializedPayload, ILambdaSerializer serializer, @@ -99,19 +104,34 @@ private static string ExtractName(string functionNameOrArn) { var payload = Deserialize(serializedPayload, serializer); var result = await handler(payload, lambdaContext); - return Serialize(result, serializer); + return (Serialize(result, serializer), null); } - private static async Task InvokeDurable( + private async Task<(string? Result, ErrorObject? Error)> InvokeDurable( Func> handler, string serializedPayload, ILambdaSerializer serializer) { - // For durable siblings, we'd need to spin up a nested DurableTestRunner. - // This is wired in commit 4 when the full runner exists. For now, the - // registry captures the handler; actual durable invocation is deferred. - throw new NotImplementedException( - "Durable sibling invocation requires the full DurableTestRunner, wired in a subsequent commit."); + // A durable sibling is itself a workflow, so drive it to completion in a + // nested runner that shares this runner's options (time-skipping, + // serializer, registered siblings) but gets its own isolated store/ARN. + var payload = Deserialize(serializedPayload, serializer); + + var nestedOptions = _options with + { + DurableExecutionArn = _options.DurableExecutionArn + ":nested:" + Guid.NewGuid().ToString("N"), + }; + var nested = new DurableTestRunner(handler, nestedOptions, registry: this); + + var result = await nested.RunAsync(payload); + if (result.Status == InvocationStatus.Succeeded) + return (Serialize(result.Result, serializer), null); + + return (null, result.Error ?? new ErrorObject + { + ErrorType = typeof(InvokeException).FullName, + ErrorMessage = $"Durable sibling '{typeof(TPayload).Name}->{typeof(TResult).Name}' did not succeed (status: {result.Status}).", + }); } private static T Deserialize(string serialized, ILambdaSerializer serializer) @@ -132,5 +152,5 @@ private sealed record FunctionEntry( string ShortName, string OriginalName, bool IsDurable, - Func> InvokeDelegate); + Func> InvokeDelegate); } diff --git a/Libraries/src/Amazon.Lambda.DurableExecution.Testing/IDurableTestRunner.cs b/Libraries/src/Amazon.Lambda.DurableExecution.Testing/IDurableTestRunner.cs index dd45089ad..af9e0a6b0 100644 --- a/Libraries/src/Amazon.Lambda.DurableExecution.Testing/IDurableTestRunner.cs +++ b/Libraries/src/Amazon.Lambda.DurableExecution.Testing/IDurableTestRunner.cs @@ -10,8 +10,12 @@ namespace Amazon.Lambda.DurableExecution.Testing; public interface IDurableTestRunner { /// - /// Drives the workflow to a terminal state and returns the result. - /// Throws if the workflow requires callbacks — use the two-call pattern instead. + /// Drives the workflow to a terminal state and returns the result. Registered + /// sibling functions (see RegisterFunction/RegisterDurableFunction on the local + /// runner) are resolved automatically. Throws + /// if the workflow suspends waiting on a callback — use the two-call pattern + /// ( + + + /// SendCallback* + ) for callback workflows. /// Task> RunAsync( TInput input, diff --git a/Libraries/src/Amazon.Lambda.DurableExecution.Testing/InMemoryOperationStore.cs b/Libraries/src/Amazon.Lambda.DurableExecution.Testing/InMemoryOperationStore.cs index c3dc01cc6..56ebe7dbe 100644 --- a/Libraries/src/Amazon.Lambda.DurableExecution.Testing/InMemoryOperationStore.cs +++ b/Libraries/src/Amazon.Lambda.DurableExecution.Testing/InMemoryOperationStore.cs @@ -11,49 +11,72 @@ internal sealed class InMemoryOperationStore { private readonly Dictionary _executions = new(); + // A single lock guards both the per-ARN dictionary and each ExecutionData's + // collections. Writes are normally serialized by the runtime's single-reader + // checkpoint batcher, but parallel/map workflows and the snapshot reads below + // can interleave, so we lock to keep the Dictionary/List internally consistent. + private readonly object _gate = new(); + public string CurrentToken(string arn) { - return GetOrCreate(arn).CheckpointToken; + lock (_gate) + return GetOrCreate(arn).CheckpointToken; } + /// + /// Returns a snapshot copy of the operations for the execution. The copy is + /// detached from the backing list so callers can iterate safely while the + /// store continues to mutate. + /// public IReadOnlyList GetAllOperations(string arn) { - return GetOrCreate(arn).Operations; + lock (_gate) + return GetOrCreate(arn).Operations.ToList(); } public Operation? GetOperation(string arn, string operationId) { - var data = GetOrCreate(arn); - return data.OperationMap.TryGetValue(operationId, out var op) ? op : null; + lock (_gate) + { + var data = GetOrCreate(arn); + return data.OperationMap.TryGetValue(operationId, out var op) ? op : null; + } } public void Upsert(string arn, Operation operation) { - var data = GetOrCreate(arn); - if (data.OperationMap.TryGetValue(operation.Id!, out var existing)) + lock (_gate) { - var index = data.Operations.IndexOf(existing); - data.Operations[index] = operation; - data.OperationMap[operation.Id!] = operation; - } - else - { - data.Operations.Add(operation); - data.OperationMap[operation.Id!] = operation; + var data = GetOrCreate(arn); + if (data.OperationMap.TryGetValue(operation.Id!, out var existing)) + { + var index = data.Operations.IndexOf(existing); + data.Operations[index] = operation; + data.OperationMap[operation.Id!] = operation; + } + else + { + data.Operations.Add(operation); + data.OperationMap[operation.Id!] = operation; + } } } public string IncrementToken(string arn) { - var data = GetOrCreate(arn); - data.TokenCounter++; - data.CheckpointToken = data.TokenCounter.ToString(); - return data.CheckpointToken; + lock (_gate) + { + var data = GetOrCreate(arn); + data.TokenCounter++; + data.CheckpointToken = data.TokenCounter.ToString(); + return data.CheckpointToken; + } } public int OperationCount(string arn) { - return GetOrCreate(arn).Operations.Count; + lock (_gate) + return GetOrCreate(arn).Operations.Count; } private ExecutionData GetOrCreate(string arn) diff --git a/Libraries/src/Amazon.Lambda.DurableExecution.Testing/OperationStatus.cs b/Libraries/src/Amazon.Lambda.DurableExecution.Testing/OperationStatus.cs index 7e304d097..7f21ab6ec 100644 --- a/Libraries/src/Amazon.Lambda.DurableExecution.Testing/OperationStatus.cs +++ b/Libraries/src/Amazon.Lambda.DurableExecution.Testing/OperationStatus.cs @@ -8,29 +8,34 @@ namespace Amazon.Lambda.DurableExecution.Testing; /// Uses a static class instead of an enum so values stay in lockstep /// with from the runtime package. /// +/// +/// Each constant is defined in terms of the runtime +/// counterpart rather than a hand-copied literal, so the two cannot silently drift: +/// if the runtime renames or removes a status, this class fails to compile. +/// public static class OperationStatus { /// The operation has started. - public const string Started = "STARTED"; + public const string Started = OperationStatuses.Started; /// The operation completed successfully. - public const string Succeeded = "SUCCEEDED"; + public const string Succeeded = OperationStatuses.Succeeded; /// The operation failed. - public const string Failed = "FAILED"; + public const string Failed = OperationStatuses.Failed; /// The operation is pending. - public const string Pending = "PENDING"; + public const string Pending = OperationStatuses.Pending; /// The operation timed out. - public const string TimedOut = "TIMED_OUT"; + public const string TimedOut = OperationStatuses.TimedOut; /// The operation was cancelled. - public const string Cancelled = "CANCELLED"; + public const string Cancelled = OperationStatuses.Cancelled; /// The operation was stopped. - public const string Stopped = "STOPPED"; + public const string Stopped = OperationStatuses.Stopped; /// The operation is ready to resume. - public const string Ready = "READY"; + public const string Ready = OperationStatuses.Ready; } diff --git a/Libraries/src/Amazon.Lambda.DurableExecution.Testing/TestResult.cs b/Libraries/src/Amazon.Lambda.DurableExecution.Testing/TestResult.cs index 5f5827fe7..4dfebb1d3 100644 --- a/Libraries/src/Amazon.Lambda.DurableExecution.Testing/TestResult.cs +++ b/Libraries/src/Amazon.Lambda.DurableExecution.Testing/TestResult.cs @@ -9,9 +9,21 @@ namespace Amazon.Lambda.DurableExecution.Testing; /// public sealed class TestResult { - /// The terminal status of the workflow. + /// + /// The terminal status of the workflow. The runtime's + /// has three values (Succeeded, Failed, Pending); the cloud runner maps the + /// service's finer terminal states (FAILED, TIMED_OUT, STOPPED) onto + /// . Inspect for the + /// underlying detail. + /// public InvocationStatus Status { get; } + /// True when the workflow completed successfully. + public bool IsSucceeded => Status == InvocationStatus.Succeeded; + + /// True when the workflow reached a failed terminal state. + public bool IsFailed => Status == InvocationStatus.Failed; + /// /// The workflow result when is . /// Default when not succeeded. @@ -26,7 +38,9 @@ public sealed class TestResult /// /// Number of handler invocations the local runner used to drive the workflow - /// to completion. -1 for cloud runner (unknown). + /// to completion. -1 when not tracked — the cloud runner never tracks + /// it, so do not assert on in tests intended to + /// run against both backends. /// public int InvocationCount { get; } @@ -112,6 +126,22 @@ public IReadOnlyList GetChildren(TestStep parent) return parent.Children; } + /// + /// Returns all steps whose equals + /// (use the constants, + /// e.g. or ). + /// + public IReadOnlyList GetStepsByStatus(string status) + { + var matches = new List(); + foreach (var step in Steps) + { + if (string.Equals(step.Status, status, StringComparison.Ordinal)) + matches.Add(step); + } + return matches; + } + /// /// Throws if is not /// . diff --git a/Libraries/src/Amazon.Lambda.DurableExecution.Testing/TestRunnerOptions.cs b/Libraries/src/Amazon.Lambda.DurableExecution.Testing/TestRunnerOptions.cs index f3a060a0f..a0d7ff6df 100644 --- a/Libraries/src/Amazon.Lambda.DurableExecution.Testing/TestRunnerOptions.cs +++ b/Libraries/src/Amazon.Lambda.DurableExecution.Testing/TestRunnerOptions.cs @@ -12,9 +12,16 @@ namespace Amazon.Lambda.DurableExecution.Testing; public sealed record TestRunnerOptions { /// - /// When true, wait/timer operations complete immediately rather than - /// waiting for real wall-clock time. Default: true. + /// When true, both WaitAsync timers and step/WaitForCondition retry + /// backoffs complete immediately rather than waiting for real wall-clock time, + /// so a workflow with day-long waits runs in milliseconds. Default: true. /// + /// + /// The default differs from the JavaScript SDK (where time-skipping is opt-in). + /// In .NET it defaults to true so the common "drive the whole workflow to + /// completion" test does not block on real timers; set it to false to assert + /// on real wait durations. + /// public bool SkipTime { get; init; } = true; /// diff --git a/Libraries/test/Amazon.Lambda.DurableExecution.Testing.Tests/CheckpointProcessorTests.cs b/Libraries/test/Amazon.Lambda.DurableExecution.Testing.Tests/CheckpointProcessorTests.cs index ed84d32e8..5ebc2a6ca 100644 --- a/Libraries/test/Amazon.Lambda.DurableExecution.Testing.Tests/CheckpointProcessorTests.cs +++ b/Libraries/test/Amazon.Lambda.DurableExecution.Testing.Tests/CheckpointProcessorTests.cs @@ -267,19 +267,23 @@ public void Process_WaitForCondition_Retry_WithSkipTime_SetsReady() { var (store, processor) = Create(skipTime: true); + // The runtime wire-encodes WaitForCondition as Type=STEP with + // SubType=WaitForCondition (see WaitForConditionOperation.OperationType). + // A STEP START is NOT time-skipped to Succeeded — only WAIT timers are — + // so the op stays Started and the condition is genuinely re-evaluated. processor.Process(Arn, null, new List { - new() { Id = "op-wfc", Type = OperationTypes.Wait, Action = OperationAction.START, SubType = OperationSubTypes.WaitForCondition, WaitOptions = new WaitOptions { WaitSeconds = 5 } } + new() { Id = "op-wfc", Type = OperationTypes.Step, Action = OperationAction.START, SubType = OperationSubTypes.WaitForCondition } }); - // WaitForCondition START with SkipTime=true also gets time-skipped (it's a WAIT) var op = store.GetOperation(Arn, "op-wfc"); - Assert.Equal(OperationStatuses.Succeeded, op!.Status); + Assert.Equal(OperationStatuses.Started, op!.Status); - // Now simulate a RETRY (as if condition wasn't met yet) + // A RETRY (condition not yet met) becomes immediately READY under SkipTime, + // so the next replay re-runs the check without waiting for the poll delay. processor.Process(Arn, "1", new List { - new() { Id = "op-wfc", Type = OperationTypes.Wait, Action = OperationAction.RETRY, SubType = OperationSubTypes.WaitForCondition, WaitOptions = new WaitOptions { WaitSeconds = 10 } } + new() { Id = "op-wfc", Type = OperationTypes.Step, Action = OperationAction.RETRY, SubType = OperationSubTypes.WaitForCondition, StepOptions = new StepOptions { NextAttemptDelaySeconds = 10 } } }); op = store.GetOperation(Arn, "op-wfc"); diff --git a/Libraries/test/Amazon.Lambda.DurableExecution.Testing.Tests/CloudDurableTestRunnerTests.cs b/Libraries/test/Amazon.Lambda.DurableExecution.Testing.Tests/CloudDurableTestRunnerTests.cs index efec45db8..ffe40b135 100644 --- a/Libraries/test/Amazon.Lambda.DurableExecution.Testing.Tests/CloudDurableTestRunnerTests.cs +++ b/Libraries/test/Amazon.Lambda.DurableExecution.Testing.Tests/CloudDurableTestRunnerTests.cs @@ -2,7 +2,6 @@ // SPDX-License-Identifier: Apache-2.0 using System.Text; -using System.Text.Json; using Amazon.Lambda; using Amazon.Lambda.DurableExecution.Testing; using Amazon.Lambda.Model; @@ -13,17 +12,16 @@ namespace Amazon.Lambda.DurableExecution.Testing.Tests; public class CloudDurableTestRunnerTests { private const string FunctionArn = "arn:aws:lambda:us-east-1:123456789012:function:my-durable-fn:$LATEST"; - private const string ExecutionArn = "arn:aws:lambda:us-east-1:123456789012:execution:my-durable-fn:exec-123"; + private const string ExecutionArn = "arn:aws:lambda:us-east-1:123456789012:function:my-durable-fn:$LATEST/durable-execution/exec123/run456"; [Fact] - public async Task StartAsync_ExtractsArn_FromResponsePayload() + public async Task StartAsync_ExtractsArn_FromTypedResponseProperty() { var mockClient = new MockCloudLambdaClient(); mockClient.InvokeHandler = _ => new InvokeResponse { StatusCode = 200, - Payload = new MemoryStream(Encoding.UTF8.GetBytes( - JsonSerializer.Serialize(new { DurableExecutionArn = ExecutionArn }))) + DurableExecutionArn = ExecutionArn, }; await using var runner = new CloudDurableTestRunner( @@ -40,7 +38,7 @@ public async Task StartAsync_ThrowsCloudTestException_WhenNoArn() mockClient.InvokeHandler = _ => new InvokeResponse { StatusCode = 200, - Payload = new MemoryStream(Encoding.UTF8.GetBytes("{}")) + // No DurableExecutionArn set — function is not a durable function. }; await using var runner = new CloudDurableTestRunner( @@ -50,70 +48,112 @@ public async Task StartAsync_ThrowsCloudTestException_WhenNoArn() } [Fact] - public async Task WaitForResultAsync_PollsUntilTerminal() + public async Task WaitForResultAsync_PollsUntilTerminal_AndReturnsTypedResult() { var mockClient = new MockCloudLambdaClient(); var pollCount = 0; - mockClient.GetExecutionStateHandler = _ => + mockClient.GetDurableExecutionHandler = _ => { pollCount++; if (pollCount < 3) - { - return new GetDurableExecutionStateResponse - { - Operations = new List - { - new() { Id = "exec-0", Type = "EXECUTION", Status = "STARTED" } - } - }; - } + return new GetDurableExecutionResponse { Status = ExecutionStatus.RUNNING }; - return new GetDurableExecutionStateResponse + return new GetDurableExecutionResponse { - Operations = new List + Status = ExecutionStatus.SUCCEEDED, + Result = """{"Value":"hello"}""", + }; + }; + mockClient.GetExecutionStateHandler = _ => new GetDurableExecutionStateResponse + { + Operations = new List + { + new() { Id = "exec-0", Type = "EXECUTION", Status = "STARTED" }, + new() { - new() - { - Id = "exec-0", - Type = "EXECUTION", - Status = "SUCCEEDED", - ExecutionDetails = new Amazon.Lambda.Model.ExecutionDetails { InputPayload = """{"x":1}""" } - }, - new() - { - Id = "op-1", - Type = "STEP", - Status = "SUCCEEDED", - Name = "step1", - StepDetails = new Amazon.Lambda.Model.StepDetails { Result = """{"Value":"hello"}""" } - } + Id = "op-1", + Type = "STEP", + Status = "SUCCEEDED", + Name = "step1", + StepDetails = new Amazon.Lambda.Model.StepDetails { Result = """{"Value":"hello"}""" } } - }; + } }; - await using var runner = new CloudDurableTestRunner( + await using var runner = new CloudDurableTestRunner( FunctionArn, mockClient, new CloudTestRunnerOptions { PollInterval = TimeSpan.FromMilliseconds(10) }); var result = await runner.WaitForResultAsync(ExecutionArn, timeout: TimeSpan.FromSeconds(5)); Assert.Equal(InvocationStatus.Succeeded, result.Status); + Assert.True(result.IsSucceeded); Assert.Equal(-1, result.InvocationCount); Assert.True(pollCount >= 3); + // Result is now populated from GetDurableExecution().Result (regression: B3). + Assert.NotNull(result.Result); + Assert.Equal("hello", result.Result!.Value); Assert.Single(result.Steps); Assert.Equal("step1", result.Steps[0].Name); } [Fact] - public async Task WaitForResultAsync_Timeout_Throws() + public async Task WaitForResultAsync_FailedExecution_PopulatesError() { var mockClient = new MockCloudLambdaClient(); - mockClient.GetExecutionStateHandler = _ => new GetDurableExecutionStateResponse + mockClient.GetDurableExecutionHandler = _ => new GetDurableExecutionResponse { - Operations = new List + Status = ExecutionStatus.FAILED, + Error = new Amazon.Lambda.Model.ErrorObject { - new() { Id = "exec-0", Type = "EXECUTION", Status = "STARTED" } - } + ErrorType = "MyException", + ErrorMessage = "it failed", + ErrorData = "extra", + }, + }; + + await using var runner = new CloudDurableTestRunner( + FunctionArn, mockClient, + new CloudTestRunnerOptions { PollInterval = TimeSpan.FromMilliseconds(10) }); + + var result = await runner.WaitForResultAsync(ExecutionArn, timeout: TimeSpan.FromSeconds(5)); + + Assert.Equal(InvocationStatus.Failed, result.Status); + Assert.True(result.IsFailed); + // Error is now sourced from GetDurableExecution().Error (regression: B5). + Assert.NotNull(result.Error); + Assert.Equal("MyException", result.Error!.ErrorType); + Assert.Equal("it failed", result.Error.ErrorMessage); + Assert.Equal("extra", result.Error.ErrorData); + } + + [Theory] + [InlineData("TIMED_OUT")] + [InlineData("STOPPED")] + public async Task WaitForResultAsync_TerminalNonFailedStatuses_MapToFailed(string status) + { + var mockClient = new MockCloudLambdaClient(); + mockClient.GetDurableExecutionHandler = _ => new GetDurableExecutionResponse + { + Status = ExecutionStatus.FindValue(status), + }; + + await using var runner = new CloudDurableTestRunner( + FunctionArn, mockClient, + new CloudTestRunnerOptions { PollInterval = TimeSpan.FromMilliseconds(10) }); + + // Regression (I5): TIMED_OUT/STOPPED are terminal — they must NOT poll to timeout. + var result = await runner.WaitForResultAsync(ExecutionArn, timeout: TimeSpan.FromSeconds(2)); + Assert.Equal(InvocationStatus.Failed, result.Status); + } + + [Fact] + public async Task WaitForResultAsync_Timeout_Throws() + { + var mockClient = new MockCloudLambdaClient(); + mockClient.GetDurableExecutionHandler = _ => new GetDurableExecutionResponse + { + Status = ExecutionStatus.RUNNING, }; await using var runner = new CloudDurableTestRunner( @@ -171,6 +211,36 @@ public async Task SendCallbackSuccessAsync_CallsLambdaApi() Assert.Equal("Y2IxMjM=", mockClient.CallbackSuccessCalls[0].CallbackId); } + [Fact] + public async Task SendCallbackFailureAsync_MapsAllErrorFields() + { + var mockClient = new MockCloudLambdaClient(); + await using var runner = new CloudDurableTestRunner( + FunctionArn, mockClient); + + await runner.SendCallbackFailureAsync("cb-1", new ErrorObject + { + ErrorType = "Rejected", + ErrorMessage = "nope", + ErrorData = "payload", + StackTrace = new[] { "frame-1", "frame-2" }, + }); + + // Regression (I8): StackTrace/ErrorData must round-trip to the SDK request. + Assert.Single(mockClient.CallbackFailureCalls); + var sent = mockClient.CallbackFailureCalls[0].Error; + Assert.NotNull(sent); + Assert.Equal("Rejected", sent!.ErrorType); + Assert.Equal("nope", sent.ErrorMessage); + Assert.Equal("payload", sent.ErrorData); + Assert.Equal(new[] { "frame-1", "frame-2" }, sent.StackTrace); + } + + public sealed class ResultPayload + { + public string? Value { get; set; } + } + /// /// Minimal mock of IAmazonLambda for cloud runner tests. /// Subclasses AmazonLambdaClient to override relevant methods. @@ -179,7 +249,9 @@ private sealed class MockCloudLambdaClient : AmazonLambdaClient { public Func? InvokeHandler { get; set; } public Func? GetExecutionStateHandler { get; set; } + public Func? GetDurableExecutionHandler { get; set; } public List CallbackSuccessCalls { get; } = new(); + public List CallbackFailureCalls { get; } = new(); public MockCloudLambdaClient() : base("fake-key", "fake-secret", Amazon.RegionEndpoint.USEast1) { } @@ -190,6 +262,14 @@ public override Task InvokeAsync(InvokeRequest request, Cancella return Task.FromResult(InvokeHandler(request)); } + public override Task GetDurableExecutionAsync( + GetDurableExecutionRequest request, CancellationToken ct = default) + { + if (GetDurableExecutionHandler is null) + return Task.FromResult(new GetDurableExecutionResponse { Status = ExecutionStatus.RUNNING }); + return Task.FromResult(GetDurableExecutionHandler(request)); + } + public override Task GetDurableExecutionStateAsync( GetDurableExecutionStateRequest request, CancellationToken ct = default) { @@ -208,6 +288,7 @@ public override Task SendDurableExe public override Task SendDurableExecutionCallbackFailureAsync( SendDurableExecutionCallbackFailureRequest request, CancellationToken ct = default) { + CallbackFailureCalls.Add(request); return Task.FromResult(new SendDurableExecutionCallbackFailureResponse()); } diff --git a/Libraries/test/Amazon.Lambda.DurableExecution.Testing.Tests/InMemoryOperationStoreTests.cs b/Libraries/test/Amazon.Lambda.DurableExecution.Testing.Tests/InMemoryOperationStoreTests.cs index bb129c98a..c49fecf4a 100644 --- a/Libraries/test/Amazon.Lambda.DurableExecution.Testing.Tests/InMemoryOperationStoreTests.cs +++ b/Libraries/test/Amazon.Lambda.DurableExecution.Testing.Tests/InMemoryOperationStoreTests.cs @@ -112,4 +112,39 @@ public void Tokens_AreIsolatedPerExecution() Assert.Equal("2", store.CurrentToken("arn:exec-1")); Assert.Equal("1", store.CurrentToken("arn:exec-2")); } + + [Fact] + public void GetAllOperations_ReturnsSnapshot_NotLiveList() + { + var store = new InMemoryOperationStore(); + store.Upsert("arn:test", new Operation { Id = "op-1", Type = OperationTypes.Step }); + + var snapshot = store.GetAllOperations("arn:test"); + // Mutating the store after the read must not change the returned snapshot. + store.Upsert("arn:test", new Operation { Id = "op-2", Type = OperationTypes.Step }); + + Assert.Single(snapshot); + Assert.Equal(2, store.GetAllOperations("arn:test").Count); + } + + [Fact] + public async Task ConcurrentUpserts_AreThreadSafe() + { + var store = new InMemoryOperationStore(); + const int count = 200; + + // Concurrent writers + readers must not corrupt the backing Dictionary/List + // (regression coverage for the unsynchronized store). + var tasks = Enumerable.Range(0, count).Select(i => Task.Run(() => + { + store.Upsert("arn:test", new Operation { Id = $"op-{i}", Type = OperationTypes.Step }); + _ = store.GetAllOperations("arn:test"); + _ = store.OperationCount("arn:test"); + })); + + await Task.WhenAll(tasks); + + Assert.Equal(count, store.OperationCount("arn:test")); + Assert.Equal(count, store.GetAllOperations("arn:test").Select(o => o.Id).Distinct().Count()); + } } diff --git a/Libraries/test/Amazon.Lambda.DurableExecution.Testing.Tests/RunAsyncBehaviorTests.cs b/Libraries/test/Amazon.Lambda.DurableExecution.Testing.Tests/RunAsyncBehaviorTests.cs new file mode 100644 index 000000000..11cf88db5 --- /dev/null +++ b/Libraries/test/Amazon.Lambda.DurableExecution.Testing.Tests/RunAsyncBehaviorTests.cs @@ -0,0 +1,138 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +using Amazon.Lambda.DurableExecution.Testing; +using Xunit; + +namespace Amazon.Lambda.DurableExecution.Testing.Tests; + +/// +/// Behavioral regression tests for the local runner: WaitForCondition driven +/// end-to-end, RunAsync's callback contract, accumulated invocation counts, and +/// the step retry attempt counter. +/// +public class RunAsyncBehaviorTests +{ + [Fact] + public async Task RunAsync_WaitForCondition_PollsUntilDone_UnderSkipTime() + { + var checkCount = 0; + await using var runner = new DurableTestRunner( + handler: async (input, ctx) => + { + // Poll until the condition (state >= 3) is met. Without time-skipping + // collapsing the between-poll backoff this would block for seconds. + var final = await ctx.WaitForConditionAsync( + check: async (state, _, _) => + { + checkCount++; + await Task.CompletedTask; + return state + 1; + }, + config: new WaitForConditionConfig + { + InitialState = 0, + WaitStrategy = WaitStrategy.Fixed( + TimeSpan.FromMinutes(5), maxAttempts: 20, isDone: s => s >= 3) + }, + name: "poll"); + return final; + }, + options: new TestRunnerOptions { SkipTime = true }); + + var result = await runner.RunAsync(0, timeout: TimeSpan.FromSeconds(10)); + + result.EnsureSucceeded(); + Assert.Equal(3, result.Result); + Assert.True(checkCount >= 3, $"expected the condition to be polled at least 3 times, got {checkCount}"); + + // The WaitForCondition op is wire-encoded as a STEP (SubType WaitForCondition). + var poll = result.GetStep("poll"); + Assert.Equal(OperationKind.Step, poll.Kind); + Assert.Equal(OperationSubTypes.WaitForCondition, poll.SubKind); + Assert.Equal(OperationStatus.Succeeded, poll.Status); + } + + [Fact] + public async Task RunAsync_CallbackWorkflow_ThrowsActionableError() + { + await using var runner = new DurableTestRunner( + handler: async (input, ctx) => + { + var approval = await ctx.WaitForCallbackAsync( + async (_, _, _) => { }, name: "approval"); + return $"approved: {approval}"; + }); + + // RunAsync cannot drive a callback workflow; it must fail with a clear + // message pointing at the two-call pattern, not a MaxInvocations timeout. + var ex = await Assert.ThrowsAsync(() => runner.RunAsync("req")); + Assert.Contains("callback", ex.Message, StringComparison.OrdinalIgnoreCase); + Assert.Contains("StartAsync", ex.Message); + } + + [Fact] + public async Task CallbackFlow_InvocationCount_AccumulatesAcrossStartAndWaitForResult() + { + await using var runner = new DurableTestRunner( + handler: async (input, ctx) => + { + await ctx.StepAsync(async (_, _) => "before", name: "before"); + var approval = await ctx.WaitForCallbackAsync( + async (_, _, _) => { }, name: "approval"); + await ctx.StepAsync(async (_, _) => "after", name: "after"); + return $"approved: {approval}"; + }); + + var arn = await runner.StartAsync("req"); + var callbackId = await runner.WaitForCallbackAsync(arn, name: "approval"); + await runner.SendCallbackSuccessAsync(callbackId, "yes"); + var result = await runner.WaitForResultAsync(arn); + + result.EnsureSucceeded(); + Assert.Equal("approved: yes", result.Result); + + // The count reflects invocations across BOTH the pre-callback drive (StartAsync) + // and the post-callback drive (WaitForResultAsync), not just the latter. + Assert.True(result.InvocationCount >= 2, + $"expected accumulated invocation count >= 2, got {result.InvocationCount}"); + } + + [Fact] + public async Task RunAsync_StepRetry_AttemptCounterAdvancesAcrossRetries() + { + var attemptsSeen = new List(); + await using var runner = new DurableTestRunner( + handler: async (input, ctx) => + { + return await ctx.StepAsync( + async (stepCtx, _) => + { + attemptsSeen.Add(stepCtx.AttemptNumber); + // Fail the first two attempts, succeed on the third. + if (stepCtx.AttemptNumber < 3) + throw new InvalidOperationException("transient"); + await Task.CompletedTask; + return "ok"; + }, + name: "flaky", + config: new StepConfig + { + RetryStrategy = RetryStrategy.Exponential( + maxAttempts: 5, initialDelay: TimeSpan.FromMinutes(1)) + }); + }, + options: new TestRunnerOptions { SkipTime = true }); + + var result = await runner.RunAsync("x", timeout: TimeSpan.FromSeconds(10)); + + result.EnsureSucceeded(); + Assert.Equal("ok", result.Result); + // The attempt number must advance 1 -> 2 -> 3, not stick at 2. Guards both + // the "never increments" bug and a "double-increments" over-correction. + Assert.Equal(new[] { 1, 2, 3 }, attemptsSeen); + + var step = result.GetStep("flaky"); + Assert.Equal(3, step.Attempt); + } +} diff --git a/Libraries/test/Amazon.Lambda.DurableExecution.Testing.Tests/SiblingInvokeIntegrationTests.cs b/Libraries/test/Amazon.Lambda.DurableExecution.Testing.Tests/SiblingInvokeIntegrationTests.cs new file mode 100644 index 000000000..a6afe90e1 --- /dev/null +++ b/Libraries/test/Amazon.Lambda.DurableExecution.Testing.Tests/SiblingInvokeIntegrationTests.cs @@ -0,0 +1,143 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +using Amazon.Lambda.DurableExecution.Testing; +using Xunit; + +namespace Amazon.Lambda.DurableExecution.Testing.Tests; + +/// +/// End-to-end tests that drive a workflow's ctx.InvokeAsync through a +/// registered sibling function. These exercise the registry-to-checkpoint wiring +/// (regression coverage for the registry never being connected to execution). +/// +public class SiblingInvokeIntegrationTests +{ + public sealed class PaymentRequest { public int Amount { get; set; } } + public sealed class PaymentResult { public string? Status { get; set; } } + + [Fact] + public async Task RunAsync_InvokesRegisteredPlainSibling_ThroughWorkflow() + { + await using var runner = new DurableTestRunner( + handler: async (input, ctx) => + { + var payment = await ctx.InvokeAsync( + "process-payment", + new PaymentRequest { Amount = input }, + name: "charge"); + return $"charged: {payment.Status}"; + }); + + runner.RegisterFunction( + "process-payment", + (req, _) => Task.FromResult(new PaymentResult { Status = $"approved-{req.Amount}" })); + + var result = await runner.RunAsync(100); + + result.EnsureSucceeded(); + Assert.Equal("charged: approved-100", result.Result); + + // The chained invoke is recorded and inspectable as a step. + var invoke = result.GetStep("charge"); + Assert.Equal(OperationKind.ChainedInvoke, invoke.Kind); + Assert.Equal(OperationStatus.Succeeded, invoke.Status); + Assert.Equal("approved-100", invoke.GetResult()!.Status); + } + + [Fact] + public async Task RunAsync_SiblingByFullArn_ResolvesByShortName() + { + await using var runner = new DurableTestRunner( + handler: async (input, ctx) => + { + var r = await ctx.InvokeAsync( + "arn:aws:lambda:us-east-1:123456789012:function:process-payment:$LATEST", + new PaymentRequest { Amount = input }, + name: "charge"); + return r.Status!; + }); + + runner.RegisterFunction( + "process-payment", + (req, _) => Task.FromResult(new PaymentResult { Status = $"ok-{req.Amount}" })); + + var result = await runner.RunAsync(5); + result.EnsureSucceeded(); + Assert.Equal("ok-5", result.Result); + } + + [Fact] + public async Task RunAsync_UnregisteredSibling_ThrowsUnregisteredSiblingFunctionException() + { + await using var runner = new DurableTestRunner( + handler: async (input, ctx) => + { + var r = await ctx.InvokeAsync( + "not-registered", new PaymentRequest { Amount = input }, name: "charge"); + return r.Status!; + }, + options: new TestRunnerOptions { MaxInvocations = 10 }); + + // Must surface the actionable exception, NOT degrade to a TestExecutionLimitException. + await Assert.ThrowsAsync(() => runner.RunAsync(1)); + } + + [Fact] + public async Task RunAsync_SiblingThatThrows_SurfacesAsInvokeFailure() + { + await using var runner = new DurableTestRunner( + handler: async (input, ctx) => + { + try + { + var r = await ctx.InvokeAsync( + "failing", new PaymentRequest { Amount = input }, name: "charge"); + return r.Status!; + } + catch (InvokeException ex) + { + return $"failed: {ex.ErrorType}"; + } + }); + + runner.RegisterFunction( + "failing", + (_, _) => throw new InvalidOperationException("downstream boom")); + + var result = await runner.RunAsync(1); + result.EnsureSucceeded(); + Assert.Equal("failed: System.InvalidOperationException", result.Result); + + // The chained-invoke step records the failure for inspection. + var invoke = result.GetStep("charge"); + Assert.Equal(OperationStatus.Failed, invoke.Status); + Assert.Equal("System.InvalidOperationException", invoke.GetError()!.ErrorType); + } + + [Fact] + public async Task RunAsync_InvokesRegisteredDurableSibling_RunsInNestedRunner() + { + await using var runner = new DurableTestRunner( + handler: async (input, ctx) => + { + var r = await ctx.InvokeAsync( + "durable-child", new PaymentRequest { Amount = input }, name: "child"); + return $"parent saw: {r.Status}"; + }); + + // The durable sibling is itself a workflow with its own steps. + runner.RegisterDurableFunction( + "durable-child", + async (req, childCtx) => + { + var doubled = await childCtx.StepAsync( + async (_, _) => req.Amount * 2, name: "double"); + return new PaymentResult { Status = $"child-{doubled}" }; + }); + + var result = await runner.RunAsync(21); + result.EnsureSucceeded(); + Assert.Equal("parent saw: child-42", result.Result); + } +}