From fd422bff1e7bcf196625a7098de09102c0a85f4a Mon Sep 17 00:00:00 2001 From: Blake Johnson Date: Mon, 16 Mar 2026 17:05:02 -0700 Subject: [PATCH] feat: add batch fetching to AWS Secrets Manager provider Override get_batch on AwssmProvider to use the AWS BatchGetSecretValue API, reducing N sequential GetSecretValue calls to ceil(N/20) batched calls. For a project with 30 secrets this means 2 API calls instead of 30, plus only 1 client construction instead of 30. Co-Authored-By: Claude Opus 4.6 (1M context) --- CHANGELOG.md | 8 ++ docs/src/content/docs/providers/awssm.md | 5 + secretspec/src/provider/awssm.rs | 156 +++++++++++++++++++++++ secretspec/src/provider/tests.rs | 45 +++++++ 4 files changed, 214 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0970381..0e8d910 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,14 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [Unreleased] + +### Changed +- AWS Secrets Manager (`awssm`) provider: batch fetching via `BatchGetSecretValue` API, + reducing N sequential API calls to ceil(N/20) batched calls. For 30 secrets this means + 2 API calls instead of 30. **Note:** requires the `secretsmanager:BatchGetSecretValue` + IAM permission in addition to existing permissions. + ## [0.8.1] - 2026-03-15 ### Added diff --git a/docs/src/content/docs/providers/awssm.md b/docs/src/content/docs/providers/awssm.md index 9ee33d5..fa9031d 100644 --- a/docs/src/content/docs/providers/awssm.md +++ b/docs/src/content/docs/providers/awssm.md @@ -80,6 +80,7 @@ AWS Secrets Manager uses the standard AWS SDK credential chain: "Effect": "Allow", "Action": [ "secretsmanager:GetSecretValue", + "secretsmanager:BatchGetSecretValue", "secretsmanager:CreateSecret", "secretsmanager:PutSecretValue" ], @@ -89,6 +90,10 @@ AWS Secrets Manager uses the standard AWS SDK credential chain: } ``` +:::note +The `BatchGetSecretValue` permission is required for batch fetching, which is used automatically during `check` and `run` commands to reduce API calls. If your IAM policy was created before this feature, you may need to add this permission. +::: + ### CI/CD ```bash diff --git a/secretspec/src/provider/awssm.rs b/secretspec/src/provider/awssm.rs index 78066a1..9502789 100644 --- a/secretspec/src/provider/awssm.rs +++ b/secretspec/src/provider/awssm.rs @@ -37,8 +37,12 @@ use crate::{Result, SecretSpecError}; use aws_sdk_secretsmanager::Client; use secrecy::{ExposeSecret, SecretString}; use serde::{Deserialize, Serialize}; +use std::collections::HashMap; use url::Url; +/// Maximum number of secrets per BatchGetSecretValue API call. +const AWS_BATCH_GET_MAX_SECRETS: usize = 20; + /// Configuration for the AWS Secrets Manager provider. #[derive(Debug, Clone, Serialize, Deserialize)] pub struct AwssmConfig { @@ -198,6 +202,77 @@ impl AwssmProvider { } } + /// Builds the full AWS secret names and a reverse map back to the original keys. + fn build_batch_request_names( + project: &str, + keys: &[&str], + profile: &str, + ) -> Result<(Vec, HashMap)> { + let mut secret_names = Vec::with_capacity(keys.len()); + let mut name_to_key = HashMap::with_capacity(keys.len()); + for key in keys { + let name = Self::format_secret_name(project, profile, key)?; + name_to_key.insert(name.clone(), key.to_string()); + secret_names.push(name); + } + Ok((secret_names, name_to_key)) + } + + /// Fetches multiple secrets in batches of 20 using the BatchGetSecretValue API. + async fn get_batch_async( + &self, + project: &str, + keys: &[&str], + profile: &str, + ) -> Result> { + if keys.is_empty() { + return Ok(HashMap::new()); + } + + let client = self.create_client().await?; + let (secret_names, name_to_key) = Self::build_batch_request_names(project, keys, profile)?; + let mut results = HashMap::new(); + + for chunk in secret_names.chunks(AWS_BATCH_GET_MAX_SECRETS) { + let mut request = client.batch_get_secret_value(); + for name in chunk { + request = request.secret_id_list(name.clone()); + } + + let response = request.send().await.map_err(|e| { + SecretSpecError::ProviderOperationFailed(format!( + "BatchGetSecretValue failed: {}", + e.into_service_error() + )) + })?; + + // Process successful values + for secret in response.secret_values() { + if let (Some(name), Some(value)) = (secret.name(), secret.secret_string()) + && let Some(key) = name_to_key.get(name) + { + results.insert(key.clone(), SecretString::new(value.to_string().into())); + } + } + + // Handle per-secret errors + for error in response.errors() { + let error_code = error.error_code().unwrap_or("Unknown"); + if error_code != "ResourceNotFoundException" { + let secret_id = error.secret_id().unwrap_or("unknown"); + let message = error.message().unwrap_or("no message"); + return Err(SecretSpecError::ProviderOperationFailed(format!( + "Failed to get secret '{}': {} - {}", + secret_id, error_code, message + ))); + } + // ResourceNotFoundException: secret not present, omit from results + } + } + + Ok(results) + } + /// Creates or updates a secret in AWS Secrets Manager. async fn set_secret_async( &self, @@ -272,4 +347,85 @@ impl Provider for AwssmProvider { fn allows_set(&self) -> bool { true } + + fn get_batch( + &self, + project: &str, + keys: &[&str], + profile: &str, + ) -> Result> { + super::block_on(self.get_batch_async(project, keys, profile)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_format_secret_name() { + let name = AwssmProvider::format_secret_name("myapp", "prod", "DB_URL").unwrap(); + assert_eq!(name, "secretspec/myapp/prod/DB_URL"); + } + + #[test] + fn test_format_secret_name_too_long() { + let long_key = "A".repeat(500); + let result = AwssmProvider::format_secret_name("myapp", "prod", &long_key); + assert!(result.is_err()); + } + + #[test] + fn test_format_secret_name_empty_inputs() { + assert!(AwssmProvider::format_secret_name("", "prod", "KEY").is_err()); + assert!(AwssmProvider::format_secret_name("proj", "", "KEY").is_err()); + assert!(AwssmProvider::format_secret_name("proj", "prod", "").is_err()); + } + + #[test] + fn test_build_batch_request_names() { + let keys: Vec<&str> = vec!["A", "B", "C"]; + let (secret_names, name_to_key) = + AwssmProvider::build_batch_request_names("proj", &keys, "default").unwrap(); + + assert_eq!(secret_names.len(), 3); + assert_eq!(name_to_key.len(), 3); + assert_eq!(secret_names[0], "secretspec/proj/default/A"); + assert_eq!(name_to_key["secretspec/proj/default/A"], "A"); + assert_eq!(name_to_key["secretspec/proj/default/B"], "B"); + assert_eq!(name_to_key["secretspec/proj/default/C"], "C"); + } + + #[test] + fn test_build_batch_request_names_empty() { + let keys: Vec<&str> = vec![]; + let (secret_names, name_to_key) = + AwssmProvider::build_batch_request_names("proj", &keys, "default").unwrap(); + assert!(secret_names.is_empty()); + assert!(name_to_key.is_empty()); + } + + #[test] + fn test_build_batch_request_names_chunking() { + let keys: Vec = (0..45).map(|i| format!("SECRET_{}", i)).collect(); + let key_refs: Vec<&str> = keys.iter().map(|s| s.as_str()).collect(); + + let (secret_names, name_to_key) = + AwssmProvider::build_batch_request_names("proj", &key_refs, "default").unwrap(); + + assert_eq!(secret_names.len(), 45); + assert_eq!(name_to_key.len(), 45); + + let chunks: Vec<&[String]> = secret_names.chunks(AWS_BATCH_GET_MAX_SECRETS).collect(); + assert_eq!(chunks.len(), 3); // 20 + 20 + 5 + assert_eq!(chunks[0].len(), 20); + assert_eq!(chunks[1].len(), 20); + assert_eq!(chunks[2].len(), 5); + + // Verify reverse mapping is correct for all keys + for key in &key_refs { + let name = AwssmProvider::format_secret_name("proj", "default", key).unwrap(); + assert_eq!(name_to_key[&name], *key); + } + } } diff --git a/secretspec/src/provider/tests.rs b/secretspec/src/provider/tests.rs index 5b67550..3c6e729 100644 --- a/secretspec/src/provider/tests.rs +++ b/secretspec/src/provider/tests.rs @@ -504,6 +504,51 @@ mod integration_tests { ); } + #[cfg(feature = "awssm")] + #[test] + fn test_awssm_batch_get() { + let providers = get_test_providers(); + if !providers.contains(&"awssm".to_string()) { + return; + } + + let (provider, _temp_dir) = create_provider_with_temp_path("awssm"); + let project_name = generate_test_project_name(); + let profile = "default"; + + // Set up test secrets + let test_secrets = vec![ + ("BATCH_TEST_1", "value1"), + ("BATCH_TEST_2", "value2"), + ("BATCH_TEST_3", "value3"), + ]; + for (key, value) in &test_secrets { + provider + .set( + &project_name, + key, + &SecretString::new(value.to_string().into()), + profile, + ) + .unwrap(); + } + + // Batch get including a key that doesn't exist + let keys = vec![ + "BATCH_TEST_1", + "BATCH_TEST_2", + "BATCH_TEST_3", + "NONEXISTENT", + ]; + let result = provider.get_batch(&project_name, &keys, profile).unwrap(); + + assert_eq!(result.len(), 3); + assert_eq!(result["BATCH_TEST_1"].expose_secret(), "value1"); + assert_eq!(result["BATCH_TEST_2"].expose_secret(), "value2"); + assert_eq!(result["BATCH_TEST_3"].expose_secret(), "value3"); + assert!(!result.contains_key("NONEXISTENT")); + } + #[cfg(feature = "awssm")] #[test] fn test_awssm_provider_creation() {