S3UTILS-238: regression test for crrExistingObjects with non-ASCII keys

tests/functional/crrExistingObjects.js

@@ -0,0 +1,152 @@
+const { promisify } = require('util');
+const vaultclient = require('vaultclient');
+const { Logger } = require('werelogs');
+const { PutObjectCommand, HeadObjectCommand } = require('@aws-sdk/client-s3');
+const ReplicationStatusUpdater = require('../../CRR/ReplicationStatusUpdater');
+
+const {
+    iamHost,
+    iamPort,
+    s3Host,
+    s3Port,
+    adminAccessKeyId,
+    adminSecretAccessKey,
+    createTestAccount,
+    deleteTestAccount,
+    configureCrr,
+    removeCrrConfiguration,
+} = require('../utils/S3Setup');
+
+const log = new Logger('crrExistingObjects:test');
+const SITE_NAME = 'test-site';
+
+function makeCaptureLogger() {
+    const captured = [];
+    const inner = new Logger('crrExistingObjects:test:capture');
+    const wrap = level => (
+        (msg, data) => {
+            captured.push({ level, message: msg, ...(data || {}) });
+            return inner[level](msg, data);
+        }
+    );
+    return {
+        captured,
+        logger: {
+            info: wrap('info'),
+            warn: wrap('warn'),
+            error: wrap('error'),
+            fatal: wrap('fatal'),
+            debug: wrap('debug'),
+            trace: wrap('trace'),
+        },
+    };
+}
+
+describe('crrExistingObjects', () => {
+    let vaultClient;
+    let accountSource;
+    let accountDest;
+    let endpoint;
+
+    beforeEach(async () => {
+        endpoint = `http://${s3Host}:${s3Port}`;
+        vaultClient = new vaultclient.Client(
+            iamHost,
+            iamPort,
+            false,
+            undefined,
+            undefined,
+            undefined,
+            undefined,
+            adminAccessKeyId,
+            adminSecretAccessKey,
+        );
+        log.info('Setting up source and destination test accounts');
+        accountSource = await createTestAccount(vaultClient);
+        accountDest = await createTestAccount(vaultClient);
+        log.info(`Source: ${accountSource.accountName}, Dest: ${accountDest.accountName}`);
+        await configureCrr(accountSource, accountDest);
+    });
+
+    afterEach(async () => {
+        log.info('Cleaning up test accounts');
+        await removeCrrConfiguration(accountSource);
+        await removeCrrConfiguration(accountDest);
+        await deleteTestAccount(vaultClient, accountSource);
+        await deleteTestAccount(vaultClient, accountDest);
+        log.info('Test accounts deleted');
+    });
+
+    async function runTest(testObjects) {
+        for (const obj of testObjects) {
+            await accountSource.s3Client.send(new PutObjectCommand({
+                Bucket: accountSource.bucketName,
+                Key: obj.Key,
+                Body: obj.Body,
+            }));
+            log.info('Uploaded object', { key: obj.Key });
+        }
+
+        const { captured, logger } = makeCaptureLogger();
+        const updater = new ReplicationStatusUpdater({
+            buckets: [accountSource.bucketName],
+            replicationStatusToProcess: ['NEW'],
+            workers: 10,
+            accessKey: accountSource.accountAccessKey,
+            secretKey: accountSource.accountSecretKey,
+            endpoint,
+            siteName: SITE_NAME,
+            storageType: '',
+            listingLimit: 1000,
+            currentVersionOnly: false,
+            forceUsingConfiguration: true,
+        }, logger);
+        await promisify(updater.run.bind(updater))();
+
+        const objectUpdateErrors = captured.filter(
+            entry => entry.level === 'error' && entry.message === 'error updating object',
+        );
+        expect(objectUpdateErrors).toEqual([]);
+        expect(updater._nErrors).toBe(0);
+        expect(updater._nUpdated).toBe(testObjects.length);
+
+        for (const obj of testObjects) {
+            const head = await accountSource.s3Client.send(new HeadObjectCommand({
+                Bucket: accountSource.bucketName,
+                Key: obj.Key,
+            }));
+            expect(head.ReplicationStatus).toBe('PENDING');
+        }
+    }
+
+    it('should replicate existing objects whose key contains Polish diacritics', async () => {
+        await runTest([
+            { Key: 'BŚ-test.txt', Body: 'data with Ś' },
+            { Key: 'ąęóćśźżł-all-diacritics.txt', Body: 'data with all polish diacritics' },
+            { Key: 'mixed/żółć/Łódź.dat', Body: 'mixed path segments' },
+        ]);
+    }, 60000);
+
+    it('should replicate existing objects whose key contains 3-byte UTF-8 characters', async () => {
+        await runTest([
+            { Key: '日本語-test.txt', Body: 'data with Japanese characters' },
+            { Key: '中文/文件.dat', Body: 'data with Chinese path segments' },
+            { Key: 'مرحبا-arabic.txt', Body: 'data with Arabic characters' },
+        ]);
+    }, 60000);
+
+    it('should replicate existing objects whose key contains 4-byte UTF-8 characters', async () => {
+        await runTest([
+            { Key: '🌍-planet-key.txt', Body: 'data with emoji key' },
+            { Key: '📁/📄-nested.txt', Body: 'data with emoji path segments' },
+            { Key: '𝔹𝕆𝕃𝔻-math.txt', Body: 'data with mathematical alphanumeric key' },
+        ]);
+    }, 60000);
+
+    it('should replicate existing objects with ASCII-only keys', async () => {
+        await runTest([
+            { Key: 'ascii-test-1.txt', Body: 'ascii data 1' },
+            { Key: 'ascii-test-2.txt', Body: 'ascii data 2' },
+        ]);
+    }, 60000);
+});

tests/functional/listObjectsByReplicationStatus.js

@@ -1,7 +1,7 @@
 const vaultclient = require('vaultclient');
 const { Logger } = require('werelogs');
-const { PutBucketVersioningCommand, PutBucketReplicationCommand, DeleteBucketReplicationCommand, PutObjectCommand, HeadObjectCommand, DeleteObjectCommand, CreateBucketCommand, DeleteBucketCommand, DeleteObjectsCommand, ListObjectVersionsCommand } = require('@aws-sdk/client-s3');
-const { CreatePolicyCommand, CreateRoleCommand, AttachRolePolicyCommand, DetachRolePolicyCommand, DeleteRoleCommand, DeletePolicyCommand, DeleteUserCommand, CreateUserCommand, CreateAccessKeyCommand, AttachUserPolicyCommand } = require('@aws-sdk/client-iam');
+const { PutObjectCommand, HeadObjectCommand, CreateBucketCommand } = require('@aws-sdk/client-s3');
+const { CreatePolicyCommand, CreateUserCommand, CreateAccessKeyCommand, AttachUserPolicyCommand } = require('@aws-sdk/client-iam');
 const { listObjectsByReplicationStatus } = require('../../listObjectsByReplicationStatus');
 
 const {
@@ -13,178 +13,12 @@ const {
     adminSecretAccessKey,
     createTestAccount,
     deleteTestAccount,
+    configureCrr,
+    removeCrrConfiguration,
 } = require('../utils/S3Setup');
 
 const log = new Logger('listObjectsByReplicationStatus:test');
 
-async function configureCrr(accountSource, accountDest) {
-    // activate bucket versioning on source and destination buckets
-    log.info('Enabling bucket versioning on source and destination buckets');
-    await accountSource.s3Client.send(new PutBucketVersioningCommand({
-        Bucket: accountSource.bucketName,
-        VersioningConfiguration: {
-            Status: 'Enabled',
-        },
-    }));
-
-    await accountDest.s3Client.send(new PutBucketVersioningCommand({
-        Bucket: accountDest.bucketName,
-        VersioningConfiguration: {
-            Status: 'Enabled',
-        },
-    }));
-
-    log.info('Creating IAM policies and roles for CRR');
-    // create policy
-    const policy = {
-        Version:'2012-10-17',
-        Statement:[
-            {
-                Effect:'Allow',
-                Action:[
-                    's3:GetObjectVersion',
-                    's3:GetObjectVersionAcl',
-                    's3:ReplicateObject'
-                ],
-                Resource:[
-                    `arn:aws:s3:::${accountSource.bucketName}/*`
-                ]
-            },
-            {
-                Effect:'Allow',
-                Action:[
-                    's3:ListBucket',
-                    's3:GetReplicationConfiguration'
-                ],
-                Resource:[
-                    'arn:aws:s3:::source'
-                ]
-            },
-            {
-                Effect:'Allow',
-                Action:[
-                    's3:ReplicateObject',
-                    's3:ReplicateDelete'
-                ],
-                Resource:`arn:aws:s3:::${accountDest.bucketName}/*`
-            }
-        ]
-    };
-    await accountSource.iamClient.send(new CreatePolicyCommand({
-        PolicyName: 'crr-policy',
-        PolicyDocument: JSON.stringify(policy),
-    }));
-
-    await accountDest.iamClient.send(new CreatePolicyCommand({
-        PolicyName: 'crr-policy',
-        PolicyDocument: JSON.stringify(policy),
-    }));
-
-    log.info('Creating IAM roles');
-    // create trust
-    const trust = {
-        Version:'2012-10-17',
-        Statement:[
-            {
-                Effect:'Allow',
-                Principal:{
-                    Service:'backbeat'
-                },
-                Action:'sts:AssumeRole'
-            }
-        ]
-    };
-    await accountSource.iamClient.send(new CreateRoleCommand({
-        RoleName: 'crr-trust-role',
-        AssumeRolePolicyDocument: JSON.stringify(trust),
-    }));
-    await accountDest.iamClient.send(new CreateRoleCommand({
-        RoleName: 'crr-trust-role',
-        AssumeRolePolicyDocument: JSON.stringify(trust),
-    }));
-
-    log.info('Attaching policies to roles');
-    // attach role to policy
-    await accountSource.iamClient.send(new AttachRolePolicyCommand({
-        RoleName: 'crr-trust-role',
-        PolicyArn: `arn:aws:iam::${accountSource.accountId}:policy/crr-policy`,
-    }));
-    await accountDest.iamClient.send(new AttachRolePolicyCommand({
-        RoleName: 'crr-trust-role',
-        PolicyArn: `arn:aws:iam::${accountDest.accountId}:policy/crr-policy`,
-    }));
-
-    log.info('Setting bucket replication configuration on source bucket');
-    const replication = {
-        Role: `arn:aws:iam::${accountSource.accountId}:role/crr-trust-role,arn:aws:iam::${accountDest.accountId}:role/crr-trust-role`,
-        Rules: [
-            {
-                Prefix: '',
-                Status: 'Enabled',
-                Destination: {
-                    Bucket: `arn:aws:s3:::${accountDest.bucketName}`
-                }
-            }
-        ]
-    };
-    await accountSource.s3Client.send(new PutBucketReplicationCommand({
-        Bucket: accountSource.bucketName,
-        ReplicationConfiguration: replication
-    }));
-
-}
-
-async function removeCrrConfiguration(account) {
-    log.info('Removing bucket crr configuration', { bucket: account.bucketName });
-    try {
-        await account.s3Client.send(new DeleteBucketReplicationCommand({ Bucket: account.bucketName }));
-    } catch (err) {
-        log.error('Error removing bucket replication configuration', {
-            bucket: account.bucketName,
-            error: err.message,
-        });
-    }
-
-    // Clean up IAM resources first (roles and policies must be deleted before account)
-    log.info('Cleaning up IAM resources', { account: account.accountName });
-    try {
-        // Detach policy from role
-        await account.iamClient.send(new DetachRolePolicyCommand({
-            RoleName: 'crr-trust-role',
-            PolicyArn: `arn:aws:iam::${account.accountId}:policy/crr-policy`,
-        }));
-        log.info('Detached policy from role');
-    } catch (err) {
-        log.error('Error detaching policy from role', { error: err.message });
-    }
-
-    try {
-        // Delete role
-        await account.iamClient.send(new DeleteRoleCommand({ RoleName: 'crr-trust-role' }));
-        log.info('Deleted IAM role');
-    } catch (err) {
-        log.error('Error deleting role', { error: err.message });
-    }
-
-    try {
-        // Delete policy
-        await account.iamClient.send(new DeletePolicyCommand({
-            PolicyArn: `arn:aws:iam::${account.accountId}:policy/crr-policy`,
-        }));
-        log.info('Deleted IAM policy');
-    } catch (err) {
-        log.error('Error deleting policy', { error: err.message });
-    }
-
-    try {
-        // Delete IAM user
-        await account.iamClient.send(new DeleteUserCommand({ UserName: account.iamUser }));
-        log.info('Deleted IAM user');
-    } catch (err) {
-        log.error('Error deleting IAM user', { error: err.message });
-    }
-}
-
 
 describe('listObjectsByReplicationStatus', () => {
     let vaultClient;