diff --git a/tests/functional/crrExistingObjects.js b/tests/functional/crrExistingObjects.js new file mode 100644 index 00000000..874f0ab5 --- /dev/null +++ b/tests/functional/crrExistingObjects.js @@ -0,0 +1,152 @@ +const { promisify } = require('util'); +const vaultclient = require('vaultclient'); +const { Logger } = require('werelogs'); +const { PutObjectCommand, HeadObjectCommand } = require('@aws-sdk/client-s3'); +const ReplicationStatusUpdater = require('../../CRR/ReplicationStatusUpdater'); + +const { + iamHost, + iamPort, + s3Host, + s3Port, + adminAccessKeyId, + adminSecretAccessKey, + createTestAccount, + deleteTestAccount, + configureCrr, + removeCrrConfiguration, +} = require('../utils/S3Setup'); + +const log = new Logger('crrExistingObjects:test'); +const SITE_NAME = 'test-site'; + +function makeCaptureLogger() { + const captured = []; + const inner = new Logger('crrExistingObjects:test:capture'); + const wrap = level => ( + (msg, data) => { + captured.push({ level, message: msg, ...(data || {}) }); + return inner[level](msg, data); + } + ); + return { + captured, + logger: { + info: wrap('info'), + warn: wrap('warn'), + error: wrap('error'), + fatal: wrap('fatal'), + debug: wrap('debug'), + trace: wrap('trace'), + }, + }; +} + +describe('crrExistingObjects', () => { + let vaultClient; + let accountSource; + let accountDest; + let endpoint; + + beforeEach(async () => { + endpoint = `http://${s3Host}:${s3Port}`; + vaultClient = new vaultclient.Client( + iamHost, + iamPort, + false, + undefined, + undefined, + undefined, + undefined, + adminAccessKeyId, + adminSecretAccessKey, + ); + log.info('Setting up source and destination test accounts'); + accountSource = await createTestAccount(vaultClient); + accountDest = await createTestAccount(vaultClient); + log.info(`Source: ${accountSource.accountName}, Dest: ${accountDest.accountName}`); + await configureCrr(accountSource, accountDest); + }); + + afterEach(async () => { + log.info('Cleaning up test accounts'); + await removeCrrConfiguration(accountSource); + await removeCrrConfiguration(accountDest); + await deleteTestAccount(vaultClient, accountSource); + await deleteTestAccount(vaultClient, accountDest); + log.info('Test accounts deleted'); + }); + + async function runTest(testObjects) { + for (const obj of testObjects) { + await accountSource.s3Client.send(new PutObjectCommand({ + Bucket: accountSource.bucketName, + Key: obj.Key, + Body: obj.Body, + })); + log.info('Uploaded object', { key: obj.Key }); + } + + const { captured, logger } = makeCaptureLogger(); + const updater = new ReplicationStatusUpdater({ + buckets: [accountSource.bucketName], + replicationStatusToProcess: ['NEW'], + workers: 10, + accessKey: accountSource.accountAccessKey, + secretKey: accountSource.accountSecretKey, + endpoint, + siteName: SITE_NAME, + storageType: '', + listingLimit: 1000, + currentVersionOnly: false, + forceUsingConfiguration: true, + }, logger); + await promisify(updater.run.bind(updater))(); + + const objectUpdateErrors = captured.filter( + entry => entry.level === 'error' && entry.message === 'error updating object', + ); + expect(objectUpdateErrors).toEqual([]); + expect(updater._nErrors).toBe(0); + expect(updater._nUpdated).toBe(testObjects.length); + + for (const obj of testObjects) { + const head = await accountSource.s3Client.send(new HeadObjectCommand({ + Bucket: accountSource.bucketName, + Key: obj.Key, + })); + expect(head.ReplicationStatus).toBe('PENDING'); + } + } + + it('should replicate existing objects whose key contains Polish diacritics', async () => { + await runTest([ + { Key: 'BŚ-test.txt', Body: 'data with Ś' }, + { Key: 'ąęóćśźżł-all-diacritics.txt', Body: 'data with all polish diacritics' }, + { Key: 'mixed/żółć/Łódź.dat', Body: 'mixed path segments' }, + ]); + }, 60000); + + it('should replicate existing objects whose key contains 3-byte UTF-8 characters', async () => { + await runTest([ + { Key: '日本語-test.txt', Body: 'data with Japanese characters' }, + { Key: '中文/文件.dat', Body: 'data with Chinese path segments' }, + { Key: 'مرحبا-arabic.txt', Body: 'data with Arabic characters' }, + ]); + }, 60000); + + it('should replicate existing objects whose key contains 4-byte UTF-8 characters', async () => { + await runTest([ + { Key: '🌍-planet-key.txt', Body: 'data with emoji key' }, + { Key: '📁/📄-nested.txt', Body: 'data with emoji path segments' }, + { Key: '𝔹𝕆𝕃𝔻-math.txt', Body: 'data with mathematical alphanumeric key' }, + ]); + }, 60000); + + it('should replicate existing objects with ASCII-only keys', async () => { + await runTest([ + { Key: 'ascii-test-1.txt', Body: 'ascii data 1' }, + { Key: 'ascii-test-2.txt', Body: 'ascii data 2' }, + ]); + }, 60000); +}); diff --git a/tests/functional/listObjectsByReplicationStatus.js b/tests/functional/listObjectsByReplicationStatus.js index 1900c0b1..d9c7c028 100644 --- a/tests/functional/listObjectsByReplicationStatus.js +++ b/tests/functional/listObjectsByReplicationStatus.js @@ -1,7 +1,7 @@ const vaultclient = require('vaultclient'); const { Logger } = require('werelogs'); -const { PutBucketVersioningCommand, PutBucketReplicationCommand, DeleteBucketReplicationCommand, PutObjectCommand, HeadObjectCommand, DeleteObjectCommand, CreateBucketCommand, DeleteBucketCommand, DeleteObjectsCommand, ListObjectVersionsCommand } = require('@aws-sdk/client-s3'); -const { CreatePolicyCommand, CreateRoleCommand, AttachRolePolicyCommand, DetachRolePolicyCommand, DeleteRoleCommand, DeletePolicyCommand, DeleteUserCommand, CreateUserCommand, CreateAccessKeyCommand, AttachUserPolicyCommand } = require('@aws-sdk/client-iam'); +const { PutObjectCommand, HeadObjectCommand, CreateBucketCommand } = require('@aws-sdk/client-s3'); +const { CreatePolicyCommand, CreateUserCommand, CreateAccessKeyCommand, AttachUserPolicyCommand } = require('@aws-sdk/client-iam'); const { listObjectsByReplicationStatus } = require('../../listObjectsByReplicationStatus'); const { @@ -13,178 +13,12 @@ const { adminSecretAccessKey, createTestAccount, deleteTestAccount, + configureCrr, + removeCrrConfiguration, } = require('../utils/S3Setup'); const log = new Logger('listObjectsByReplicationStatus:test'); -async function configureCrr(accountSource, accountDest) { - // activate bucket versioning on source and destination buckets - log.info('Enabling bucket versioning on source and destination buckets'); - await accountSource.s3Client.send(new PutBucketVersioningCommand({ - Bucket: accountSource.bucketName, - VersioningConfiguration: { - Status: 'Enabled', - }, - })); - - await accountDest.s3Client.send(new PutBucketVersioningCommand({ - Bucket: accountDest.bucketName, - VersioningConfiguration: { - Status: 'Enabled', - }, - })); - - log.info('Creating IAM policies and roles for CRR'); - // create policy - const policy = { - Version:'2012-10-17', - Statement:[ - { - Effect:'Allow', - Action:[ - 's3:GetObjectVersion', - 's3:GetObjectVersionAcl', - 's3:ReplicateObject' - ], - Resource:[ - `arn:aws:s3:::${accountSource.bucketName}/*` - ] - }, - { - Effect:'Allow', - Action:[ - 's3:ListBucket', - 's3:GetReplicationConfiguration' - ], - Resource:[ - 'arn:aws:s3:::source' - ] - }, - { - Effect:'Allow', - Action:[ - 's3:ReplicateObject', - 's3:ReplicateDelete' - ], - Resource:`arn:aws:s3:::${accountDest.bucketName}/*` - } - ] - }; - await accountSource.iamClient.send(new CreatePolicyCommand({ - PolicyName: 'crr-policy', - PolicyDocument: JSON.stringify(policy), - })); - - await accountDest.iamClient.send(new CreatePolicyCommand({ - PolicyName: 'crr-policy', - PolicyDocument: JSON.stringify(policy), - })); - - log.info('Creating IAM roles'); - // create trust - const trust = { - Version:'2012-10-17', - Statement:[ - { - Effect:'Allow', - Principal:{ - Service:'backbeat' - }, - Action:'sts:AssumeRole' - } - ] - }; - await accountSource.iamClient.send(new CreateRoleCommand({ - RoleName: 'crr-trust-role', - AssumeRolePolicyDocument: JSON.stringify(trust), - })); - await accountDest.iamClient.send(new CreateRoleCommand({ - RoleName: 'crr-trust-role', - AssumeRolePolicyDocument: JSON.stringify(trust), - })); - - log.info('Attaching policies to roles'); - // attach role to policy - await accountSource.iamClient.send(new AttachRolePolicyCommand({ - RoleName: 'crr-trust-role', - PolicyArn: `arn:aws:iam::${accountSource.accountId}:policy/crr-policy`, - })); - await accountDest.iamClient.send(new AttachRolePolicyCommand({ - RoleName: 'crr-trust-role', - PolicyArn: `arn:aws:iam::${accountDest.accountId}:policy/crr-policy`, - })); - - log.info('Setting bucket replication configuration on source bucket'); - const replication = { - Role: `arn:aws:iam::${accountSource.accountId}:role/crr-trust-role,arn:aws:iam::${accountDest.accountId}:role/crr-trust-role`, - Rules: [ - { - Prefix: '', - Status: 'Enabled', - Destination: { - Bucket: `arn:aws:s3:::${accountDest.bucketName}` - } - } - ] - }; - await accountSource.s3Client.send(new PutBucketReplicationCommand({ - Bucket: accountSource.bucketName, - ReplicationConfiguration: replication - })); - -} - -async function removeCrrConfiguration(account) { - log.info('Removing bucket crr configuration', { bucket: account.bucketName }); - try { - await account.s3Client.send(new DeleteBucketReplicationCommand({ Bucket: account.bucketName })); - } catch (err) { - log.error('Error removing bucket replication configuration', { - bucket: account.bucketName, - error: err.message, - }); - } - - // Clean up IAM resources first (roles and policies must be deleted before account) - log.info('Cleaning up IAM resources', { account: account.accountName }); - try { - // Detach policy from role - await account.iamClient.send(new DetachRolePolicyCommand({ - RoleName: 'crr-trust-role', - PolicyArn: `arn:aws:iam::${account.accountId}:policy/crr-policy`, - })); - log.info('Detached policy from role'); - } catch (err) { - log.error('Error detaching policy from role', { error: err.message }); - } - - try { - // Delete role - await account.iamClient.send(new DeleteRoleCommand({ RoleName: 'crr-trust-role' })); - log.info('Deleted IAM role'); - } catch (err) { - log.error('Error deleting role', { error: err.message }); - } - - try { - // Delete policy - await account.iamClient.send(new DeletePolicyCommand({ - PolicyArn: `arn:aws:iam::${account.accountId}:policy/crr-policy`, - })); - log.info('Deleted IAM policy'); - } catch (err) { - log.error('Error deleting policy', { error: err.message }); - } - - try { - // Delete IAM user - await account.iamClient.send(new DeleteUserCommand({ UserName: account.iamUser })); - log.info('Deleted IAM user'); - } catch (err) { - log.error('Error deleting IAM user', { error: err.message }); - } -} - describe('listObjectsByReplicationStatus', () => { let vaultClient; diff --git a/tests/utils/S3Setup.js b/tests/utils/S3Setup.js index f7ddd703..5f939cbb 100644 --- a/tests/utils/S3Setup.js +++ b/tests/utils/S3Setup.js @@ -1,4 +1,4 @@ -const { S3Client, CreateBucketCommand, ListBucketsCommand, ListObjectVersionsCommand, DeleteObjectsCommand, DeleteBucketCommand, PutBucketVersioningCommand, PutBucketReplicationCommand, DeleteBucketReplicationCommand, PutObjectCommand, HeadObjectCommand } = require('@aws-sdk/client-s3'); +const { S3Client, CreateBucketCommand, ListBucketsCommand, ListObjectVersionsCommand, DeleteObjectsCommand, DeleteBucketCommand, PutBucketVersioningCommand, PutBucketReplicationCommand, DeleteBucketReplicationCommand } = require('@aws-sdk/client-s3'); const { IAMClient, CreateUserCommand, CreatePolicyCommand, CreateRoleCommand, AttachRolePolicyCommand, DetachRolePolicyCommand, DeleteRoleCommand, DeletePolicyCommand, DeleteUserCommand, ListRolesCommand, ListAttachedRolePoliciesCommand, ListUsersCommand, ListAttachedUserPoliciesCommand, DetachUserPolicyCommand } = require('@aws-sdk/client-iam'); const { promisify } = require('util'); const { Logger } = require('werelogs'); @@ -202,9 +202,167 @@ async function deleteTestAccount(vaultClient, account) { } +/** + * Configure CRR between a source and a destination test account. + * Enables versioning on both buckets, creates the IAM policies/roles + * required by backbeat, and sets a single-rule replication config on + * the source bucket. + * + * @param {Object} accountSource - object returned by createTestAccount + * @param {Object} accountDest - object returned by createTestAccount + * @param {Object} [opts] + * @param {string} [opts.storageClass] - optional StorageClass on the rule's + * Destination (some flows match the destination by StorageClass / siteName). + * If omitted, the rule has no StorageClass. + * @returns {Promise} + */ +async function configureCrr(accountSource, accountDest, opts = {}) { + log.info('Enabling bucket versioning on source and destination buckets'); + await accountSource.s3Client.send(new PutBucketVersioningCommand({ + Bucket: accountSource.bucketName, + VersioningConfiguration: { Status: 'Enabled' }, + })); + + await accountDest.s3Client.send(new PutBucketVersioningCommand({ + Bucket: accountDest.bucketName, + VersioningConfiguration: { Status: 'Enabled' }, + })); + + log.info('Creating IAM policies and roles for CRR'); + const policy = { + Version: '2012-10-17', + Statement: [ + { + Effect: 'Allow', + Action: [ + 's3:GetObjectVersion', + 's3:GetObjectVersionAcl', + 's3:ReplicateObject', + ], + Resource: [`arn:aws:s3:::${accountSource.bucketName}/*`], + }, + { + Effect: 'Allow', + Action: ['s3:ListBucket', 's3:GetReplicationConfiguration'], + Resource: ['arn:aws:s3:::source'], + }, + { + Effect: 'Allow', + Action: ['s3:ReplicateObject', 's3:ReplicateDelete'], + Resource: `arn:aws:s3:::${accountDest.bucketName}/*`, + }, + ], + }; + await accountSource.iamClient.send(new CreatePolicyCommand({ + PolicyName: 'crr-policy', + PolicyDocument: JSON.stringify(policy), + })); + await accountDest.iamClient.send(new CreatePolicyCommand({ + PolicyName: 'crr-policy', + PolicyDocument: JSON.stringify(policy), + })); + + log.info('Creating IAM roles'); + const trust = { + Version: '2012-10-17', + Statement: [{ + Effect: 'Allow', + Principal: { Service: 'backbeat' }, + Action: 'sts:AssumeRole', + }], + }; + await accountSource.iamClient.send(new CreateRoleCommand({ + RoleName: 'crr-trust-role', + AssumeRolePolicyDocument: JSON.stringify(trust), + })); + await accountDest.iamClient.send(new CreateRoleCommand({ + RoleName: 'crr-trust-role', + AssumeRolePolicyDocument: JSON.stringify(trust), + })); + + log.info('Attaching policies to roles'); + await accountSource.iamClient.send(new AttachRolePolicyCommand({ + RoleName: 'crr-trust-role', + PolicyArn: `arn:aws:iam::${accountSource.accountId}:policy/crr-policy`, + })); + await accountDest.iamClient.send(new AttachRolePolicyCommand({ + RoleName: 'crr-trust-role', + PolicyArn: `arn:aws:iam::${accountDest.accountId}:policy/crr-policy`, + })); + + log.info('Setting bucket replication configuration on source bucket'); + const destination = { Bucket: `arn:aws:s3:::${accountDest.bucketName}` }; + if (opts.storageClass) { + destination.StorageClass = opts.storageClass; + } + const replication = { + Role: `arn:aws:iam::${accountSource.accountId}:role/crr-trust-role,` + + `arn:aws:iam::${accountDest.accountId}:role/crr-trust-role`, + Rules: [{ Prefix: '', Status: 'Enabled', Destination: destination }], + }; + await accountSource.s3Client.send(new PutBucketReplicationCommand({ + Bucket: accountSource.bucketName, + ReplicationConfiguration: replication, + })); +} + +/** + * Tear down the CRR configuration set up by configureCrr() for a single + * account: removes the bucket replication config (if any) and deletes the + * IAM role/policy/user. Best-effort; logs and swallows individual errors + * so a partial setup can still be cleaned up. + */ +async function removeCrrConfiguration(account) { + log.info('Removing bucket crr configuration', { bucket: account.bucketName }); + try { + await account.s3Client.send(new DeleteBucketReplicationCommand({ Bucket: account.bucketName })); + } catch (err) { + log.error('Error removing bucket replication configuration', { + bucket: account.bucketName, + error: err.message, + }); + } + + log.info('Cleaning up IAM resources', { account: account.accountName }); + try { + await account.iamClient.send(new DetachRolePolicyCommand({ + RoleName: 'crr-trust-role', + PolicyArn: `arn:aws:iam::${account.accountId}:policy/crr-policy`, + })); + log.info('Detached policy from role'); + } catch (err) { + log.error('Error detaching policy from role', { error: err.message }); + } + + try { + await account.iamClient.send(new DeleteRoleCommand({ RoleName: 'crr-trust-role' })); + log.info('Deleted IAM role'); + } catch (err) { + log.error('Error deleting role', { error: err.message }); + } + + try { + await account.iamClient.send(new DeletePolicyCommand({ + PolicyArn: `arn:aws:iam::${account.accountId}:policy/crr-policy`, + })); + log.info('Deleted IAM policy'); + } catch (err) { + log.error('Error deleting policy', { error: err.message }); + } + + try { + await account.iamClient.send(new DeleteUserCommand({ UserName: account.iamUser })); + log.info('Deleted IAM user'); + } catch (err) { + log.error('Error deleting IAM user', { error: err.message }); + } +} + module.exports = { createTestAccount, deleteTestAccount, + configureCrr, + removeCrrConfiguration, iamHost, iamPort, s3Host,