From 107679b5aa20e2eb71c9cdf46b3d525fcb264306 Mon Sep 17 00:00:00 2001 From: Veetaha Date: Thu, 27 Nov 2025 17:11:10 +0000 Subject: [PATCH 1/6] Add `CrossAccount` boolean input to S3 changelog stack to prevent dependencies on Connector lambdas and roles --- ...dformation-multiple-buckets-with-macros.yaml | 4 ++++ .../cloudformation-multiple-buckets.yaml | 15 +++++++++++++++ .../cloudformation-single-bucket.yaml | 17 +++++++++++++++++ 3 files changed, 36 insertions(+) diff --git a/elastio-s3-changelog/cloudformation-multiple-buckets-with-macros.yaml b/elastio-s3-changelog/cloudformation-multiple-buckets-with-macros.yaml index 0e0a62a..cd70972 100644 --- a/elastio-s3-changelog/cloudformation-multiple-buckets-with-macros.yaml +++ b/elastio-s3-changelog/cloudformation-multiple-buckets-with-macros.yaml @@ -10,6 +10,9 @@ Transform: Parameters: BucketNames: Type: String + CrossAccount: + Type: String + Default: 'false' ScanExistingObjects: Type: String Default: 'false' @@ -39,6 +42,7 @@ Resources: Properties: Parameters: BucketName: !Ref BucketName + CrossAccount: !Ref CrossAccount ScanExistingObjects: !Ref ScanExistingObjects KeyPrefixes: !Ref KeyPrefixes DisableEventCollection: !Ref DisableEventCollection diff --git a/elastio-s3-changelog/cloudformation-multiple-buckets.yaml b/elastio-s3-changelog/cloudformation-multiple-buckets.yaml index 62bd438..5bf06dd 100644 --- a/elastio-s3-changelog/cloudformation-multiple-buckets.yaml +++ b/elastio-s3-changelog/cloudformation-multiple-buckets.yaml @@ -12,6 +12,16 @@ Parameters: Comma-delimited list of S3 bucket names. Example: examplebucket1, examplebucket2 + CrossAccount: + Type: String + AllowedValues: ['true', 'false'] + Default: 'false' + Description: > + Set this to true if the scan happens in a different AWS account than the + S3 bucket (i.e., you are using a centralized scanning account). If it is not set + to true in the cross-account case, an error about missing AWS Lambda functions + will occur during stack creation. + ScanExistingObjects: Type: String AllowedValues: ['true', 'false'] @@ -19,6 +29,9 @@ Parameters: Description: > If set to true, the full initial scan of the bucket will be performed. If set to false, only new objects in the bucket will be scanned. + This option can only be enabled when CrossAccount is false. If it is not set + to false in the cross-account case, an error about missing AWS Lambda functions + will occur during stack creation. KeyPrefixes: Type: CommaDelimitedList @@ -75,6 +88,7 @@ Metadata: - Label: { default: Main Parameters } Parameters: - BucketNames + - CrossAccount - ScanExistingObjects - KeyPrefixes @@ -95,6 +109,7 @@ Resources: Properties: Parameters: BucketNames: !Join [',', !Ref BucketNames] + CrossAccount: !Ref CrossAccount ScanExistingObjects: !Ref ScanExistingObjects KeyPrefixes: !Join [',', !Ref KeyPrefixes] DisableEventCollection: !Ref DisableEventCollection diff --git a/elastio-s3-changelog/cloudformation-single-bucket.yaml b/elastio-s3-changelog/cloudformation-single-bucket.yaml index 4db4875..ef407a5 100644 --- a/elastio-s3-changelog/cloudformation-single-bucket.yaml +++ b/elastio-s3-changelog/cloudformation-single-bucket.yaml @@ -9,6 +9,17 @@ Parameters: ConstraintDescription: Bucket name must be at least 3 characters long. Description: S3 bucket name. + + CrossAccount: + Type: String + AllowedValues: ['true', 'false'] + Default: 'false' + Description: > + Set this to true if the scan happens in a different AWS account than the + S3 bucket (i.e., you are using a centralized scanning account). If it is not set + to true in the cross-account case, an error about missing AWS Lambda functions + will occur during stack creation. + ScanExistingObjects: Type: String AllowedValues: ['true', 'false'] @@ -16,6 +27,9 @@ Parameters: Description: > If set to true, the full initial scan of the bucket will be performed. If set to false, only new objects in the bucket will be scanned. + This option can only be enabled when CrossAccount is false. If it is not set + to false in the cross-account case, an error about missing AWS Lambda functions + will occur during stack creation. KeyPrefixes: # Unfortunately, it's impossible to pass CommaDelimitedList to a nested stack, @@ -74,6 +88,7 @@ Metadata: - Label: { default: Main Parameters } Parameters: - BucketName + - CrossAccount - ScanExistingObjects - KeyPrefixes @@ -89,6 +104,7 @@ Metadata: - QrtsMaxDelay Conditions: + NotCrossAccount: !Equals [!Ref CrossAccount, 'false'] ScanExistingObjects: !Equals [!Ref ScanExistingObjects, 'true'] DisableEventCollection: !Equals [!Ref DisableEventCollection, 'true'] EnableDlq: !Equals [!Ref EnableDlq, 'true'] @@ -216,6 +232,7 @@ Resources: queueRefreshSchedule: Type: AWS::Scheduler::Schedule + Condition: NotCrossAccount Properties: Description: A schedule to invoke the periodic changelog queue refresh lambda ScheduleExpression: cron(0/15 * * * ? *) From d05ee6259cbf558cda5f03fcb6956217d08c5827 Mon Sep 17 00:00:00 2001 From: Veetaha Date: Thu, 27 Nov 2025 17:20:54 +0000 Subject: [PATCH 2/6] Update readme --- elastio-s3-changelog/README.md | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/elastio-s3-changelog/README.md b/elastio-s3-changelog/README.md index 311731f..5219236 100644 --- a/elastio-s3-changelog/README.md +++ b/elastio-s3-changelog/README.md @@ -1,7 +1,7 @@ # S3 Changelog For large buckets, scanning all objects might take a lot of time. This CloudFormation template enables -the Changelog feature for S3 buckets, which significantly improves the scan performance after the initial +the Changelog feature for S3 buckets, which improves the scan performance after the initial scan of the entire bucket is done. This template deploys an SQS queue and an EventBridge rule which sends S3 update events to the queue. @@ -32,7 +32,10 @@ Then, the Elastio `iscan` job reads those events to perform the scanning of new 3. Fill in the main parameters: * *BucketNames* - comma-separated list of S3 bucket names; - * *ScanExistingObjects* - set to `true` if you want to perform the initial scan of all objects in the bucket(s); + * *CrossAccount* - set this to `true` if you are using a centralized Elastio Cloud Connector + that lives in a different AWS account than your S3 buckets. + + * *ScanExistingObjects* - set to `true` if you want to perform the initial scan of all objects in the bucket(s). This option must be set to `false` if `CrossAccount` is enabled! * *KeyPrefixes* - (optional) comma-separated list of prefixes of objects to scan. This will be applied to all buckets. If you want to use different prefixes for different buckets, you need to deploy multiple S3 Changelog stacks. From ebe5dbb2acb76c0fe7412bdfcfa32d0bdc8218e8 Mon Sep 17 00:00:00 2001 From: Veetaha Date: Thu, 27 Nov 2025 17:23:18 +0000 Subject: [PATCH 3/6] Clarify --- elastio-s3-changelog/README.md | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/elastio-s3-changelog/README.md b/elastio-s3-changelog/README.md index 5219236..9557378 100644 --- a/elastio-s3-changelog/README.md +++ b/elastio-s3-changelog/README.md @@ -32,10 +32,16 @@ Then, the Elastio `iscan` job reads those events to perform the scanning of new 3. Fill in the main parameters: * *BucketNames* - comma-separated list of S3 bucket names; - * *CrossAccount* - set this to `true` if you are using a centralized Elastio Cloud Connector - that lives in a different AWS account than your S3 buckets. + * *CrossAccount* - set this to true if the scan happens in a different AWS account than the + S3 bucket (i.e., you are using a centralized scanning account). If it is not set + to true in the cross-account case, an error about missing AWS Lambda functions + will occur during stack creation. - * *ScanExistingObjects* - set to `true` if you want to perform the initial scan of all objects in the bucket(s). This option must be set to `false` if `CrossAccount` is enabled! + * *ScanExistingObjects* - if set to true, the full initial scan of the bucket will be performed. + If set to false, only new objects in the bucket will be scanned. + This option can only be enabled when CrossAccount is false. If it is not set + to false in the cross-account case, an error about missing AWS Lambda functions + will occur during stack creation. * *KeyPrefixes* - (optional) comma-separated list of prefixes of objects to scan. This will be applied to all buckets. If you want to use different prefixes for different buckets, you need to deploy multiple S3 Changelog stacks. From 66099d3cde2998ffaee000e7ecbbdc33c7821812 Mon Sep 17 00:00:00 2001 From: Veetaha Date: Thu, 27 Nov 2025 17:24:16 +0000 Subject: [PATCH 4/6] Extra space --- elastio-s3-changelog/cloudformation-single-bucket.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/elastio-s3-changelog/cloudformation-single-bucket.yaml b/elastio-s3-changelog/cloudformation-single-bucket.yaml index ef407a5..c2b043d 100644 --- a/elastio-s3-changelog/cloudformation-single-bucket.yaml +++ b/elastio-s3-changelog/cloudformation-single-bucket.yaml @@ -9,7 +9,6 @@ Parameters: ConstraintDescription: Bucket name must be at least 3 characters long. Description: S3 bucket name. - CrossAccount: Type: String AllowedValues: ['true', 'false'] From 9baab50755cc5d84ea54079341b49623d71bc73d Mon Sep 17 00:00:00 2001 From: Veetaha Date: Thu, 27 Nov 2025 17:25:06 +0000 Subject: [PATCH 5/6] Grammar --- elastio-s3-changelog/README.md | 2 +- elastio-s3-changelog/cloudformation-multiple-buckets.yaml | 2 +- elastio-s3-changelog/cloudformation-single-bucket.yaml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/elastio-s3-changelog/README.md b/elastio-s3-changelog/README.md index 9557378..6178ed7 100644 --- a/elastio-s3-changelog/README.md +++ b/elastio-s3-changelog/README.md @@ -33,7 +33,7 @@ Then, the Elastio `iscan` job reads those events to perform the scanning of new * *BucketNames* - comma-separated list of S3 bucket names; * *CrossAccount* - set this to true if the scan happens in a different AWS account than the - S3 bucket (i.e., you are using a centralized scanning account). If it is not set + S3 bucket (i.e. you are using a centralized scanning account). If it is not set to true in the cross-account case, an error about missing AWS Lambda functions will occur during stack creation. diff --git a/elastio-s3-changelog/cloudformation-multiple-buckets.yaml b/elastio-s3-changelog/cloudformation-multiple-buckets.yaml index 5bf06dd..c56cf89 100644 --- a/elastio-s3-changelog/cloudformation-multiple-buckets.yaml +++ b/elastio-s3-changelog/cloudformation-multiple-buckets.yaml @@ -18,7 +18,7 @@ Parameters: Default: 'false' Description: > Set this to true if the scan happens in a different AWS account than the - S3 bucket (i.e., you are using a centralized scanning account). If it is not set + S3 bucket (i.e. you are using a centralized scanning account). If it is not set to true in the cross-account case, an error about missing AWS Lambda functions will occur during stack creation. diff --git a/elastio-s3-changelog/cloudformation-single-bucket.yaml b/elastio-s3-changelog/cloudformation-single-bucket.yaml index c2b043d..1a61da6 100644 --- a/elastio-s3-changelog/cloudformation-single-bucket.yaml +++ b/elastio-s3-changelog/cloudformation-single-bucket.yaml @@ -15,7 +15,7 @@ Parameters: Default: 'false' Description: > Set this to true if the scan happens in a different AWS account than the - S3 bucket (i.e., you are using a centralized scanning account). If it is not set + S3 bucket (i.e. you are using a centralized scanning account). If it is not set to true in the cross-account case, an error about missing AWS Lambda functions will occur during stack creation. From e1e7b82113f5d2e7a14ac4d679458fe13ca968e9 Mon Sep 17 00:00:00 2001 From: Veetaha Date: Thu, 27 Nov 2025 17:26:23 +0000 Subject: [PATCH 6/6] Bump version --- elastio-s3-changelog/version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/elastio-s3-changelog/version b/elastio-s3-changelog/version index 626799f..8c1384d 100644 --- a/elastio-s3-changelog/version +++ b/elastio-s3-changelog/version @@ -1 +1 @@ -v1 +v2