From d93fc92af551485dfebe5b26543dc33db354bb25 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Wed, 10 Jun 2026 16:14:24 +1200 Subject: [PATCH 1/3] fix(SES): create missing template instead of failing the send MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The bulk path registers a content-hashed template and, on the first send of any new content, must detect the "template missing" error, create the template, and retry. That recovery never fired: SES API v2 (REST-JSON) returns the exception type in the x-amzn-ErrorType response header, but the base HTTP layer discarded response headers, so errorType() found no type, isTemplateMissing() returned false, and the raw "Template utopia- does not exist." surfaced to every recipient. Capture response headers in Adapter::request() and read the SES error type from x-amzn-ErrorType, keeping the body __type/code as a fallback. Tighten missing-template detection so a generic BadRequestException only matches when the message signals non-existence. The existing routing test mocked a missing template as a 200 with a per-entry TEMPLATE_NOT_FOUND status — a shape SES never returns for a missing default template (it is a top-level 400), which is why CI stayed green while production failed. Add regression tests for the real shape, covering create-and-retry and the concurrent AlreadyExists race. Co-Authored-By: Claude Opus 4.8 --- src/Utopia/Messaging/Adapter.php | 12 +++ src/Utopia/Messaging/Adapter/Email/SES.php | 37 ++++++--- .../Adapter/Email/ResendRoutingTest.php | 3 +- .../Adapter/Email/SESRoutingTest.php | 82 ++++++++++++++++++- 4 files changed, 120 insertions(+), 14 deletions(-) diff --git a/src/Utopia/Messaging/Adapter.php b/src/Utopia/Messaging/Adapter.php index 1c5b19b..bade103 100644 --- a/src/Utopia/Messaging/Adapter.php +++ b/src/Utopia/Messaging/Adapter.php @@ -69,6 +69,7 @@ public function send(Message $message): array * url: string, * statusCode: int, * response: array|string|null, + * headers: array, * error: string|null * } * @@ -103,6 +104,8 @@ protected function request( } } + $responseHeaders = []; + \curl_setopt_array($ch, [ CURLOPT_CUSTOMREQUEST => $method, CURLOPT_URL => $url, @@ -111,6 +114,14 @@ protected function request( CURLOPT_USERAGENT => "Appwrite {$this->getName()} Message Sender", CURLOPT_TIMEOUT => $timeout, CURLOPT_CONNECTTIMEOUT => $connectTimeout, + CURLOPT_HEADERFUNCTION => function ($ch, string $header) use (&$responseHeaders): int { + $parts = \explode(':', $header, 2); + if (\count($parts) === 2) { + $responseHeaders[\strtolower(\trim($parts[0]))] = \trim($parts[1]); + } + + return \strlen($header); + }, ]); $response = \curl_exec($ch); @@ -125,6 +136,7 @@ protected function request( 'url' => $url, 'statusCode' => \curl_getinfo($ch, CURLINFO_RESPONSE_CODE), 'response' => $response, + 'headers' => $responseHeaders, 'error' => \curl_error($ch), ]; } diff --git a/src/Utopia/Messaging/Adapter/Email/SES.php b/src/Utopia/Messaging/Adapter/Email/SES.php index bc3db40..17ce027 100644 --- a/src/Utopia/Messaging/Adapter/Email/SES.php +++ b/src/Utopia/Messaging/Adapter/Email/SES.php @@ -266,7 +266,7 @@ private function sendRaw(EmailMessage $message, Response $response): array * marked failed with the SES error. On success each recipient is mapped * from its corresponding BulkEmailEntryResults entry. * - * @param array{url: string, statusCode: int, response: array|string|null, error: string|null} $result + * @param array{url: string, statusCode: int, response: array|string|null, headers: array, error: string|null} $result * @return array{deliveredTo: int, type: string, results: array>} */ private function parseBulkResult(EmailMessage $message, array $result, Response $response): array @@ -383,14 +383,20 @@ private function templateName(EmailMessage $message): string * Whether a SendBulkEmail result indicates the referenced template is * missing, via either the top-level error or per-entry statuses. * - * @param array{url: string, statusCode: int, response: array|string|null, error: string|null} $result + * @param array{url: string, statusCode: int, response: array|string|null, headers: array, error: string|null} $result */ private function isTemplateMissing(array $result): bool { $errorType = $this->errorType($result); if ($errorType === 'NotFoundException' || $errorType === 'BadRequestException') { - $message = $this->errorMessage($result); - if (\stripos($message, 'template') !== false) { + // BadRequestException is generic, so confirm the message is about a + // missing template rather than another template error (e.g. invalid + // template content). + $message = \strtolower($this->errorMessage($result)); + if ( + \str_contains($message, 'template') + && (\str_contains($message, 'does not exist') || \str_contains($message, 'not found')) + ) { return true; } } @@ -521,7 +527,7 @@ private function formatAddress(string $email, ?string $name): string * configured region. * * @param array $body - * @return array{url: string, statusCode: int, response: array|string|null, error: string|null} + * @return array{url: string, statusCode: int, response: array|string|null, headers: array, error: string|null} * * @throws \Exception */ @@ -652,7 +658,7 @@ private function signingKey(string $dateStamp): string /** * Extract a human-readable error message from a SES error response. * - * @param array{url: string, statusCode: int, response: array|string|null, error: string|null} $result + * @param array{url: string, statusCode: int, response: array|string|null, headers: array, error: string|null} $result */ private function errorMessage(array $result): string { @@ -679,16 +685,25 @@ private function errorMessage(array $result): string } /** - * Extract the SES error type. SES signals the type either via the - * x-amzn-ErrorType header (not available here) or a `__type` body field, - * e.g. "AlreadyExistsException" or "NotFoundException". + * Extract the SES error type, e.g. "AlreadyExistsException" or + * "NotFoundException". * - * @param array{url: string, statusCode: int, response: array|string|null, error: string|null} $result + * SES API v2 uses the AWS REST-JSON protocol, which reports the exception + * type in the x-amzn-ErrorType response header, not the body. The header + * value can carry a trailing ":" which is stripped. Older AWS + * JSON-protocol responses instead carry it in a `__type` (optionally + * "prefix#Type") or `code` body field, which is used as a fallback. + * + * @param array{url: string, statusCode: int, response: array|string|null, headers: array, error: string|null} $result */ private function errorType(array $result): ?string { - $body = $result['response']; + $header = $result['headers']['x-amzn-errortype'] ?? null; + if (\is_string($header) && $header !== '') { + return \trim(\explode(':', $header)[0]); + } + $body = $result['response']; if (\is_array($body)) { $type = $body['__type'] ?? $body['code'] ?? null; if (\is_string($type)) { diff --git a/tests/Messaging/Adapter/Email/ResendRoutingTest.php b/tests/Messaging/Adapter/Email/ResendRoutingTest.php index f674e1a..2b68fa0 100644 --- a/tests/Messaging/Adapter/Email/ResendRoutingTest.php +++ b/tests/Messaging/Adapter/Email/ResendRoutingTest.php @@ -135,7 +135,7 @@ class ResendStub extends Resend /** * @param array $headers * @param array|null $body - * @return array{url: string, statusCode: int, response: array|string|null, error: string|null} + * @return array{url: string, statusCode: int, response: array|string|null, headers: array, error: string|null} */ protected function request( string $method, @@ -158,6 +158,7 @@ protected function request( 'url' => $url, 'statusCode' => $stub['statusCode'], 'response' => $stub['response'], + 'headers' => [], 'error' => null, ]; } diff --git a/tests/Messaging/Adapter/Email/SESRoutingTest.php b/tests/Messaging/Adapter/Email/SESRoutingTest.php index f0530e8..67a2430 100644 --- a/tests/Messaging/Adapter/Email/SESRoutingTest.php +++ b/tests/Messaging/Adapter/Email/SESRoutingTest.php @@ -158,6 +158,83 @@ public function testTemplateNotFoundTriggersCreateAndRetry(): void $this->assertSame('success', $response['results'][0]['status']); } + public function testTopLevelMissingTemplateTriggersCreateAndRetry(): void + { + $stub = new SESStub('key', 'secret', 'us-east-1'); + + // The real SES v2 response for a missing DefaultContent template is a + // top-level 400 whose exception type lives only in the x-amzn-ErrorType + // response header, with a plain {"message": ...} body. The auto-create + // must trigger off that, not off a per-entry BulkEmailEntryResults + // status (which SES does not return for a missing default template). + $stub->stubResponses[] = [ + 'statusCode' => 400, + 'headers' => ['x-amzn-errortype' => 'BadRequestException'], + 'response' => ['message' => 'Template utopia-abc123 does not exist.'], + ]; + $stub->stubResponses[] = ['statusCode' => 200, 'response' => []]; + $stub->stubResponses[] = [ + 'statusCode' => 200, + 'response' => ['BulkEmailEntryResults' => [['Status' => 'SUCCESS', 'MessageId' => 'x']]], + ]; + + $message = new Email( + to: [['email' => 'a@example.com']], + subject: 'Subject', + content: 'Body', + fromName: 'Sender', + fromEmail: 'from@example.com', + ); + + $response = $stub->send($message); + + $this->assertCount(3, $stub->capturedRequests); + $this->assertStringEndsWith('/v2/email/outbound-bulk-emails', $stub->capturedRequests[0]['url']); + $this->assertStringEndsWith('/v2/email/templates', $stub->capturedRequests[1]['url']); + $this->assertStringEndsWith('/v2/email/outbound-bulk-emails', $stub->capturedRequests[2]['url']); + + $this->assertSame(1, $response['deliveredTo']); + $this->assertSame('success', $response['results'][0]['status']); + } + + public function testCreateTemplateToleratesAlreadyExists(): void + { + $stub = new SESStub('key', 'secret', 'us-east-1'); + + // Missing on the first send (top-level 400) ... + $stub->stubResponses[] = [ + 'statusCode' => 400, + 'headers' => ['x-amzn-errortype' => 'BadRequestException'], + 'response' => ['message' => 'Template utopia-abc123 does not exist.'], + ]; + // ... but a concurrent sender created it first, so CreateEmailTemplate + // returns AlreadyExistsException (again, type in the header). That must + // be tolerated rather than surfaced as a failure. + $stub->stubResponses[] = [ + 'statusCode' => 400, + 'headers' => ['x-amzn-errortype' => 'AlreadyExistsException'], + 'response' => ['message' => 'Template utopia-abc123 already exists.'], + ]; + $stub->stubResponses[] = [ + 'statusCode' => 200, + 'response' => ['BulkEmailEntryResults' => [['Status' => 'SUCCESS']]], + ]; + + $message = new Email( + to: [['email' => 'a@example.com']], + subject: 'Subject', + content: 'Body', + fromName: 'Sender', + fromEmail: 'from@example.com', + ); + + $response = $stub->send($message); + + $this->assertCount(3, $stub->capturedRequests); + $this->assertSame(1, $response['deliveredTo']); + $this->assertSame('success', $response['results'][0]['status']); + } + public function testTextTemplateUsesTextContent(): void { $stub = new SESStub('key', 'secret', 'us-east-1'); @@ -573,14 +650,14 @@ class SESStub extends SES public array $capturedRequests = []; /** - * @var array|string|null}> + * @var array|string|null, headers?: array}> */ public array $stubResponses = []; /** * @param array $headers * @param array|null $body - * @return array{url: string, statusCode: int, response: array|string|null, error: string|null} + * @return array{url: string, statusCode: int, response: array|string|null, headers: array, error: string|null} */ protected function request( string $method, @@ -603,6 +680,7 @@ protected function request( 'url' => $url, 'statusCode' => $stub['statusCode'], 'response' => $stub['response'], + 'headers' => $stub['headers'] ?? [], 'error' => null, ]; } From d4ace8c1451528e4e60d1d853cb6cfed39d67c47 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Wed, 10 Jun 2026 16:25:21 +1200 Subject: [PATCH 2/3] refactor(Adapter): keep requestMulti response shape in sync with request requestMulti() now returns a headers key like request(), so callers and static analysis see one consistent response shape across both transport paths. Headers are left empty on the multi path because it builds requests with curl_copy_handle(), which segfaults when the copied handle carries a CURLOPT_HEADERFUNCTION closure; a comment records why, so per-handle capture is not naively reintroduced. Co-Authored-By: Claude Opus 4.8 --- src/Utopia/Messaging/Adapter.php | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/Utopia/Messaging/Adapter.php b/src/Utopia/Messaging/Adapter.php index bade103..a631a14 100644 --- a/src/Utopia/Messaging/Adapter.php +++ b/src/Utopia/Messaging/Adapter.php @@ -152,6 +152,7 @@ protected function request( * url: string, * statusCode: int, * response: array|null, + * headers: array, * error: string|null * }> * @@ -256,6 +257,12 @@ protected function requestMulti( 'url' => \curl_getinfo($ch, CURLINFO_EFFECTIVE_URL), 'statusCode' => \curl_getinfo($ch, CURLINFO_RESPONSE_CODE), 'response' => $response, + // Kept in sync with request()'s shape. Response headers are not + // captured here: this path copies a configured handle with + // curl_copy_handle(), and copying a handle that carries a + // CURLOPT_HEADERFUNCTION closure segfaults. Wire up per-handle + // capture (without copy_handle) if a multi-path adapter needs it. + 'headers' => [], 'error' => \curl_error($ch), ]; From 8601e6e852dfba9cd7ec56b049982ea9ec053582 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Wed, 10 Jun 2026 17:07:10 +1200 Subject: [PATCH 3/3] test(SES): wire SES_* credentials into CI and docker test env SESTest is gated on SES_ACCESS_KEY/SES_SECRET_KEY/SES_REGION/ SES_TEST_EMAIL, but those were never passed into docker-compose or the Tests workflow, so the live SES test silently skipped everywhere. Pass them through both paths: credentials and the optional session token as repo secrets, region and the verified test identity as repo variables (mirroring RESEND_TEST_EMAIL). The test stays skipped until a maintainer adds the secrets/vars in repo settings; nothing breaks meanwhile, since unset values resolve to empty strings and SESTest::setUp marks the test skipped. Co-Authored-By: Claude Opus 4.8 --- .github/workflows/test.yml | 5 +++++ docker-compose.yml | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 8794f4b..ce9e3e9 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -18,6 +18,9 @@ jobs: MAILGUN_DOMAIN: ${{ secrets.MAILGUN_DOMAIN }} RESEND_API_KEY: ${{ secrets.RESEND_API_KEY }} SENDGRID_API_KEY: ${{ secrets.SENDGRID_API_KEY }} + SES_ACCESS_KEY: ${{ secrets.SES_ACCESS_KEY }} + SES_SECRET_KEY: ${{ secrets.SES_SECRET_KEY }} + SES_SESSION_TOKEN: ${{ secrets.SES_SESSION_TOKEN }} FCM_SERVICE_ACCOUNT_JSON: ${{ secrets.FCM_SERVICE_ACCOUNT_JSON }} FCM_TO: ${{ secrets.FCM_TO }} TWILIO_ACCOUNT_SID: ${{ secrets.TWILIO_ACCOUNT_SID }} @@ -53,6 +56,8 @@ jobs: INFORU_SENDER_ID: ${{ secrets.INFORU_SENDER_ID }} RESEND_TEST_EMAIL: ${{ vars.RESEND_TEST_EMAIL }} + SES_REGION: ${{ vars.SES_REGION }} + SES_TEST_EMAIL: ${{ vars.SES_TEST_EMAIL }} run: | docker compose up -d --build sleep 5 diff --git a/docker-compose.yml b/docker-compose.yml index 6e036f6..086ea03 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -12,6 +12,11 @@ services: - RESEND_API_KEY - RESEND_TEST_EMAIL - SENDGRID_API_KEY + - SES_ACCESS_KEY + - SES_SECRET_KEY + - SES_REGION + - SES_TEST_EMAIL + - SES_SESSION_TOKEN - FCM_SERVICE_ACCOUNT_JSON - FCM_TO - TWILIO_ACCOUNT_SID