From 660d87a7fea84df49810d78c5c1b19d73148c96f Mon Sep 17 00:00:00 2001 From: Aidan Daly Date: Wed, 1 Jul 2026 15:43:56 +0000 Subject: [PATCH 1/2] fix(e2e): reap stale OAuth2 credential providers in globalSetup MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The e2e globalSetup already reaps stale API key credential providers, but OAuth2 providers are a separate resource type with their own List/Delete APIs — they do NOT appear in ListApiKeyCredentialProviders, so the existing cleanup can never see them. The CUSTOM_JWT harness test registers a managed OAuth2 provider (`-oauth`) created outside the CloudFormation stack, so teardown leaves it behind. These accumulate against the account's 50-provider OAuth2 quota (L-431051DC) until every CUSTOM_JWT deploy fails with "The number of agent identity Oauth2 credential providers in this account has reached its limit", failing the E2E suite. Add cleanupStaleOAuth2CredentialProviders (mirrors the API key reaper, using ListOauth2CredentialProviders / DeleteOauth2CredentialProvider) and wire it into globalSetup alongside the existing stack / API key / recommendation cleanups. --- e2e-tests/global-setup.ts | 20 +++++++- .../utils/credential-provider-cleanup.ts | 49 +++++++++++++++++++ 2 files changed, 68 insertions(+), 1 deletion(-) diff --git a/e2e-tests/global-setup.ts b/e2e-tests/global-setup.ts index 3e02ba745..45a328b24 100644 --- a/e2e-tests/global-setup.ts +++ b/e2e-tests/global-setup.ts @@ -1,4 +1,7 @@ -import { cleanupStaleCredentialProviders } from './utils/credential-provider-cleanup'; +import { + cleanupStaleCredentialProviders, + cleanupStaleOAuth2CredentialProviders, +} from './utils/credential-provider-cleanup'; import { getLogger } from './utils/logger'; import { cleanupStaleRecommendations } from './utils/recommendation-cleanup'; import { cleanUpOldStacks } from './utils/stack-cleanup'; @@ -45,6 +48,21 @@ export default async function setup(_project: TestProject): Promise<() => void> } catch (e) { logger.error(String(e)); logger.warn(`failed to clean up all credential providers`); + } + + // OAuth2 providers are a separate resource type with their own List/Delete APIs — they do + // NOT show up in ListApiKeyCredentialProviders, so the reaper above can't touch them. The + // CUSTOM_JWT harness test leaks `-oauth` providers (created outside the CFN stack), so + // reap them here too or the account's 50-provider OAuth2 quota fills up and CUSTOM_JWT deploys fail. + logger.info(`cleaning up stale OAuth2 credential providers...`); + try { + await cleanupStaleOAuth2CredentialProviders(bedrockCPClient, logger.child('oauth2-credential-provider-cleanup'), { + minAgeMs: 30 * 60 * 1000, + prefix: 'E2e', + }); + } catch (e) { + logger.error(String(e)); + logger.warn(`failed to clean up all OAuth2 credential providers`); } finally { bedrockCPClient.destroy(); } diff --git a/e2e-tests/utils/credential-provider-cleanup.ts b/e2e-tests/utils/credential-provider-cleanup.ts index a2f1ef42d..5af7a7895 100644 --- a/e2e-tests/utils/credential-provider-cleanup.ts +++ b/e2e-tests/utils/credential-provider-cleanup.ts @@ -2,7 +2,9 @@ import type { Logger } from './logger'; import { BedrockAgentCoreControlClient, DeleteApiKeyCredentialProviderCommand, + DeleteOauth2CredentialProviderCommand, ListApiKeyCredentialProvidersCommand, + ListOauth2CredentialProvidersCommand, } from '@aws-sdk/client-bedrock-agentcore-control'; export async function deleteCredentialProvider( @@ -19,6 +21,20 @@ export async function deleteCredentialProvider( } } +export async function deleteOAuth2CredentialProvider( + client: BedrockAgentCoreControlClient, + logger: Logger, + name: string +): Promise { + try { + await client.send(new DeleteOauth2CredentialProviderCommand({ name })); + logger.info(`Deleted OAuth2 credential provider: ${name}`); + } catch (error) { + const err = error as Error; + logger.warn(`Failed to delete OAuth2 credential provider ${name}: ${err.name}:${err.message}`); + } +} + export async function cleanupStaleCredentialProviders( client: BedrockAgentCoreControlClient, logger: Logger, @@ -40,3 +56,36 @@ export async function cleanupStaleCredentialProviders( nextToken = response.nextToken; } while (nextToken); } + +/** + * Delete stale OAuth2 credential providers matching `prefix` and older than `minAgeMs`. + * + * OAuth2 providers are a distinct resource type from API key providers — they have their + * own List/Delete APIs and do NOT appear in ListApiKeyCredentialProviders. So + * `cleanupStaleCredentialProviders` (which lists only API key providers) can never reap + * them. The CUSTOM_JWT harness e2e test registers a managed OAuth2 provider (`-oauth`) + * that is created outside the CloudFormation stack, so teardown leaves it behind. Without + * this reaper they accumulate against the account's 50-provider OAuth2 quota (L-431051DC) + * until every CUSTOM_JWT deploy fails with a limit-reached error. + */ +export async function cleanupStaleOAuth2CredentialProviders( + client: BedrockAgentCoreControlClient, + logger: Logger, + options: { + minAgeMs: number; + prefix: string; + } +): Promise { + const cutoff = new Date(Date.now() - options.minAgeMs); + + let nextToken: string | undefined; + do { + const response = await client.send(new ListOauth2CredentialProvidersCommand({ nextToken })); + const providers = response.credentialProviders ?? []; + const stale = providers.filter(p => p.name?.startsWith(options.prefix) && p.createdTime && p.createdTime < cutoff); + + await Promise.all(stale.map(p => deleteOAuth2CredentialProvider(client, logger, p.name!))); + + nextToken = response.nextToken; + } while (nextToken); +} From 738ce44929cbce4f4a3358ccc25bd62ab575cae6 Mon Sep 17 00:00:00 2001 From: Aidan Daly Date: Wed, 1 Jul 2026 15:50:40 +0000 Subject: [PATCH 2/2] refactor(e2e): reap both provider types in one cleanupStaleCredentialProviders Fold the OAuth2 reaping into cleanupStaleCredentialProviders rather than a separate function + second globalSetup call. From the caller's view "reap stale credential providers" is one job; splitting it by resource type is what let OAuth2 providers go unreaped in the first place. The function now walks both the API key and OAuth2 list APIs. globalSetup reverts to a single call (no longer changed vs main). --- e2e-tests/global-setup.ts | 20 +----- .../utils/credential-provider-cleanup.ts | 61 +++++++------------ 2 files changed, 24 insertions(+), 57 deletions(-) diff --git a/e2e-tests/global-setup.ts b/e2e-tests/global-setup.ts index 45a328b24..3e02ba745 100644 --- a/e2e-tests/global-setup.ts +++ b/e2e-tests/global-setup.ts @@ -1,7 +1,4 @@ -import { - cleanupStaleCredentialProviders, - cleanupStaleOAuth2CredentialProviders, -} from './utils/credential-provider-cleanup'; +import { cleanupStaleCredentialProviders } from './utils/credential-provider-cleanup'; import { getLogger } from './utils/logger'; import { cleanupStaleRecommendations } from './utils/recommendation-cleanup'; import { cleanUpOldStacks } from './utils/stack-cleanup'; @@ -48,21 +45,6 @@ export default async function setup(_project: TestProject): Promise<() => void> } catch (e) { logger.error(String(e)); logger.warn(`failed to clean up all credential providers`); - } - - // OAuth2 providers are a separate resource type with their own List/Delete APIs — they do - // NOT show up in ListApiKeyCredentialProviders, so the reaper above can't touch them. The - // CUSTOM_JWT harness test leaks `-oauth` providers (created outside the CFN stack), so - // reap them here too or the account's 50-provider OAuth2 quota fills up and CUSTOM_JWT deploys fail. - logger.info(`cleaning up stale OAuth2 credential providers...`); - try { - await cleanupStaleOAuth2CredentialProviders(bedrockCPClient, logger.child('oauth2-credential-provider-cleanup'), { - minAgeMs: 30 * 60 * 1000, - prefix: 'E2e', - }); - } catch (e) { - logger.error(String(e)); - logger.warn(`failed to clean up all OAuth2 credential providers`); } finally { bedrockCPClient.destroy(); } diff --git a/e2e-tests/utils/credential-provider-cleanup.ts b/e2e-tests/utils/credential-provider-cleanup.ts index 5af7a7895..7910d318d 100644 --- a/e2e-tests/utils/credential-provider-cleanup.ts +++ b/e2e-tests/utils/credential-provider-cleanup.ts @@ -35,40 +35,18 @@ export async function deleteOAuth2CredentialProvider( } } -export async function cleanupStaleCredentialProviders( - client: BedrockAgentCoreControlClient, - logger: Logger, - options: { - minAgeMs: number; - prefix: string; - } -): Promise { - const cutoff = new Date(Date.now() - options.minAgeMs); - - let nextToken: string | undefined; - do { - const response = await client.send(new ListApiKeyCredentialProvidersCommand({ nextToken })); - const providers = response.credentialProviders ?? []; - const stale = providers.filter(p => p.name?.startsWith(options.prefix) && p.createdTime && p.createdTime < cutoff); - - await Promise.all(stale.map(p => deleteCredentialProvider(client, logger, p.name!))); - - nextToken = response.nextToken; - } while (nextToken); -} - /** - * Delete stale OAuth2 credential providers matching `prefix` and older than `minAgeMs`. + * Delete stale credential providers matching `prefix` and older than `minAgeMs`. * - * OAuth2 providers are a distinct resource type from API key providers — they have their - * own List/Delete APIs and do NOT appear in ListApiKeyCredentialProviders. So - * `cleanupStaleCredentialProviders` (which lists only API key providers) can never reap - * them. The CUSTOM_JWT harness e2e test registers a managed OAuth2 provider (`-oauth`) - * that is created outside the CloudFormation stack, so teardown leaves it behind. Without - * this reaper they accumulate against the account's 50-provider OAuth2 quota (L-431051DC) - * until every CUSTOM_JWT deploy fails with a limit-reached error. + * Reaps BOTH provider types the e2e suite creates: API key providers and OAuth2 providers. + * These are distinct resource types with separate List/Delete APIs — an OAuth2 provider does + * NOT appear in ListApiKeyCredentialProviders — so both lists must be walked. The CUSTOM_JWT + * harness test registers a managed OAuth2 provider (`-oauth`) created outside the + * CloudFormation stack, so teardown leaves it behind; without reaping it here, they accumulate + * against the account's 50-provider OAuth2 quota (L-431051DC) until every CUSTOM_JWT deploy + * fails with a limit-reached error. */ -export async function cleanupStaleOAuth2CredentialProviders( +export async function cleanupStaleCredentialProviders( client: BedrockAgentCoreControlClient, logger: Logger, options: { @@ -77,15 +55,22 @@ export async function cleanupStaleOAuth2CredentialProviders( } ): Promise { const cutoff = new Date(Date.now() - options.minAgeMs); + const isStale = (p: { name?: string; createdTime?: Date }): boolean => + !!p.name?.startsWith(options.prefix) && !!p.createdTime && p.createdTime < cutoff; - let nextToken: string | undefined; + let apiKeyNextToken: string | undefined; do { - const response = await client.send(new ListOauth2CredentialProvidersCommand({ nextToken })); - const providers = response.credentialProviders ?? []; - const stale = providers.filter(p => p.name?.startsWith(options.prefix) && p.createdTime && p.createdTime < cutoff); + const response = await client.send(new ListApiKeyCredentialProvidersCommand({ nextToken: apiKeyNextToken })); + const stale = (response.credentialProviders ?? []).filter(isStale); + await Promise.all(stale.map(p => deleteCredentialProvider(client, logger, p.name!))); + apiKeyNextToken = response.nextToken; + } while (apiKeyNextToken); + let oauth2NextToken: string | undefined; + do { + const response = await client.send(new ListOauth2CredentialProvidersCommand({ nextToken: oauth2NextToken })); + const stale = (response.credentialProviders ?? []).filter(isStale); await Promise.all(stale.map(p => deleteOAuth2CredentialProvider(client, logger, p.name!))); - - nextToken = response.nextToken; - } while (nextToken); + oauth2NextToken = response.nextToken; + } while (oauth2NextToken); }