[Feat(tests)] multi-worker freestyle mock

apps/e2e/tests/backend/endpoints/api/v1/auth/password/update.test.ts

@@ -2,7 +2,7 @@ import { it } from "../../../../../../helpers";
 import { Auth, backendContext, niceBackendFetch } from "../../../../../backend-helpers";
 
 it("should update existing passwords", async ({ expect }) => {
-  const signUpRes = await Auth.Password.signUpWithEmail();
+  const signUpRes = await Auth.Password.signUpWithEmail({ noWaitForEmail: true });
   const oldPassword = signUpRes.password;
   const newPassword = "new-password";
   const response = await niceBackendFetch("/api/v1/auth/password/update", {
@@ -26,7 +26,7 @@ it("should update existing passwords", async ({ expect }) => {
 });
 
 it("should sign out other sessions but not own session when updating password", async ({ expect }) => {
-  const signUpRes = await Auth.Password.signUpWithEmail();
+  const signUpRes = await Auth.Password.signUpWithEmail({ noWaitForEmail: true });
   const oldPassword = signUpRes.password;
   const newPassword = "new-password";
 
@@ -64,7 +64,7 @@ it("should sign out other sessions but not own session when updating password",
 });
 
 it("should not update passwords to weak passwords", async ({ expect }) => {
-  const signUpRes = await Auth.Password.signUpWithEmail();
+  const signUpRes = await Auth.Password.signUpWithEmail({ noWaitForEmail: true });
   const oldPassword = signUpRes.password;
   const newPassword = "short";
   const response = await niceBackendFetch("/api/v1/auth/password/update", {
@@ -92,7 +92,7 @@ it("should not update passwords to weak passwords", async ({ expect }) => {
 });
 
 it("should not update passwords without old password", async ({ expect }) => {
-  await Auth.Password.signUpWithEmail();
+  await Auth.Password.signUpWithEmail({ noWaitForEmail: true });
   const newPassword = "new-password";
   const response = await niceBackendFetch("/api/v1/auth/password/update", {
     method: "POST",
@@ -126,7 +126,7 @@ it("should not update passwords without old password", async ({ expect }) => {
 });
 
 it("should not update passwords if the provided old password is wrong", async ({ expect }) => {
-  await Auth.Password.signUpWithEmail();
+  await Auth.Password.signUpWithEmail({ noWaitForEmail: true });
   const newPassword = "new-password";
   const response = await niceBackendFetch("/api/v1/auth/password/update", {
     method: "POST",

apps/e2e/tests/backend/endpoints/api/v1/internal/failed-emails-digest.test.ts

@@ -4,6 +4,69 @@ import { describe, expect } from "vitest";
 import { it } from "../../../../../helpers";
 import { Auth, backendContext, bumpEmailAddress, InternalProjectKeys, niceBackendFetch, Project } from "../../../../backend-helpers";
 
+type FailedEmailsBatch = {
+  emails: Array<{ subject: string, to: string[] }>,
+  tenant_owner_emails: string[],
+  project_id: string,
+  tenancy_id: string,
+};
+
+type DigestResponse = Awaited<ReturnType<typeof niceBackendFetch>>;
+
+// Always uses dry_run=true: the only callers are the polling helper below
+// (which must be side-effect-free since it fires repeatedly) and the snapshot
+// assertion (which uses the same SELECT result regardless of dry_run).
+async function fetchFailedEmailsDigest(): Promise<DigestResponse> {
+  return await niceBackendFetch("/api/v1/internal/failed-emails-digest", {
+    method: "POST",
+    headers: { "Authorization": "Bearer mock_cron_secret" },
+    query: { dry_run: "true" },
+  });
+}
+
+function selectBatchesForCurrentMailbox(response: DigestResponse): FailedEmailsBatch[] {
+  const batches: FailedEmailsBatch[] = response.body.failed_emails_by_tenancy ?? [];
+  const ownerEmail = backendContext.value.mailbox.emailAddress;
+  return batches
+    .filter((batch) => batch.tenant_owner_emails.includes(ownerEmail))
+    .map((batch) => ({
+      ...batch,
+      emails: [...batch.emails].sort((a, b) => stringCompare(a.subject, b.subject)),
+    }));
+}
+
+// Polls the failed-emails-digest until the current mailbox's batch contains
+// at least `expectedFailedEmailCount` emails. Throws on timeout with a clear
+// message — silently returning partial state would make the subsequent
+// snapshot fail with an unhelpful "got 1, expected 2" diff that hides the
+// fact that the test actually timed out.
+async function waitForFailedEmailsDigest(
+  expectedFailedEmailCount: number,
+  options: { timeoutMs?: number, intervalMs?: number } = {},
+): Promise<{ response: DigestResponse, batches: FailedEmailsBatch[] }> {
+  const timeoutMs = options.timeoutMs ?? 30_000;
+  const intervalMs = options.intervalMs ?? 500;
+
+  let response = await fetchFailedEmailsDigest();
+  let batches = selectBatchesForCurrentMailbox(response);
+  const startedAt = performance.now();
+  while ((batches[0]?.emails.length ?? 0) < expectedFailedEmailCount) {
+    if (performance.now() - startedAt >= timeoutMs) {
+      const observed = batches[0]?.emails.length ?? 0;
+      throw new Error(
+        `Timed out after ${timeoutMs}ms waiting for ${expectedFailedEmailCount} ` +
+        `failed email(s) for the current mailbox in the digest response. ` +
+        `Last observed: ${observed} email(s). ` +
+        `Last batches: ${JSON.stringify(batches, null, 2)}`,
+      );
+    }
+    await wait(intervalMs);
+    response = await fetchFailedEmailsDigest();
+    batches = selectBatchesForCurrentMailbox(response);
+  }
+  return { response, batches };
+}
+
 describe("unauthorized requests", () => {
   it("should return 401 when invalid authorization is provided", async ({ expect }) => {
     const response = await niceBackendFetch(
@@ -58,7 +121,7 @@ describe("unauthorized requests", () => {
 });
 
 describe("with valid credentials", () => {
-  async function testFailedEmails(isDryRun: boolean) {
+  async function testFailedEmails() {
     backendContext.set({
       projectKeys: InternalProjectKeys,
       userAuth: null,
@@ -102,25 +165,9 @@ describe("with valid credentials", () => {
       }
     `);
 
-    await wait(10_000);
-
-    const response = await niceBackendFetch("/api/v1/internal/failed-emails-digest", {
-      method: "POST",
-      headers: { "Authorization": "Bearer mock_cron_secret" },
-      query: {
-        dry_run: `${isDryRun}`,
-      },
-    });
+    const { response, batches: mockProjectFailedEmails } = await waitForFailedEmailsDigest(2);
       expect(response.status).toBe(200);
 
-      const failedEmailsByTenancy = response.body.failed_emails_by_tenancy;
-      const mockProjectFailedEmails = failedEmailsByTenancy.filter(
-        (batch: any) => batch.tenant_owner_emails.includes(backendContext.value.mailbox.emailAddress)
-      ).map((batch: any) => ({
-        ...batch,
-        emails: [...batch.emails].sort((a, b) => stringCompare(a.subject, b.subject)),
-      }));
-
       if (process.env.STACK_TEST_SOURCE_OF_TRUTH === "true") {
       expect(mockProjectFailedEmails).toMatchInlineSnapshot(`[]`);
       } else {
@@ -155,15 +202,17 @@ describe("with valid credentials", () => {
   }
 
   it("should return 200 and process dry run request", async ({ expect }) => {
-    const { projectOwnerMailbox } = await testFailedEmails(true);
+    const { projectOwnerMailbox } = await testFailedEmails();
 
     const messages = await projectOwnerMailbox.fetchMessages();
     expect(messages.filter(msg => !msg.subject.includes("Sign in"))).toMatchInlineSnapshot(`[]`);
   }, { repeats: 10 });
 
-  // TODO: failed emails digest is currently disabled, fix that and then re-enable this test
+  // TODO: failed emails digest is currently disabled. When re-enabling, this
+  // test will need to call the digest endpoint with dry_run=false separately
+  // (testFailedEmails / waitForFailedEmailsDigest are dry-run only).
   it.todo("should return 200 and process failed emails digest", async ({ expect }) => {
-    const { projectOwnerMailbox } = await testFailedEmails(false);
+    const { projectOwnerMailbox } = await testFailedEmails();
     const messages = await projectOwnerMailbox.fetchMessages();
     const digestEmail = messages.find(msg => msg.subject === "Failed emails digest");
     expect(digestEmail).toBeDefined();