diff --git a/extensions/lifecycle/bucketProcessor/policy.json b/extensions/lifecycle/bucketProcessor/policy.json index 0de6c175b..3f6016797 100644 --- a/extensions/lifecycle/bucketProcessor/policy.json +++ b/extensions/lifecycle/bucketProcessor/policy.json @@ -14,7 +14,8 @@ "s3:GetObjectVersionTagging", "s3:GetObject", "s3:GetObjectVersion", - "s3:ReplicateObject" + "s3:ReplicateObject", + "scality:GetObjectArchiveInfo" ], "Resource": [ "*" diff --git a/extensions/lifecycle/tasks/LifecycleDeleteObjectTask.js b/extensions/lifecycle/tasks/LifecycleDeleteObjectTask.js index c6d93dec3..7891673c0 100644 --- a/extensions/lifecycle/tasks/LifecycleDeleteObjectTask.js +++ b/extensions/lifecycle/tasks/LifecycleDeleteObjectTask.js @@ -195,7 +195,12 @@ class LifecycleDeleteObjectTask extends BackbeatTask { const actionType = entry.getActionType(); const transitionTime = entry.getAttribute('transitionTime'); - const location = this.objectMD?.dataStoreName || entry.getAttribute('details.dataStoreName'); + // The entry attribute carries the location resolved at queue time + // (the cold class for cold/restored objects). Fall back to the object + // metadata for entries missing it, e.g. expired object delete markers + // queued by LifecycleTaskV2, which do not set it. objectMD is only + // fetched for versioned deleteObject actions, hence the optional call. + const location = entry.getAttribute('details.dataStoreName') || this.objectMD?.getDataStoreName(); let reqMethod = 'deleteObject'; let actionMethod = this._deleteObject.bind(this); diff --git a/extensions/lifecycle/tasks/LifecycleTask.js b/extensions/lifecycle/tasks/LifecycleTask.js index 69321a944..8a8311840 100644 --- a/extensions/lifecycle/tasks/LifecycleTask.js +++ b/extensions/lifecycle/tasks/LifecycleTask.js @@ -59,6 +59,19 @@ const MAX_RETRIES = 4; // parallel tasks, so the total delay of retries should about 1m30s. const MAX_RETRIES_TOTAL = CONCURRENCY_DEFAULT * MAX_RETRIES * 10; +function attachArchiveInfoHeader(command) { + command.middlewareStack.add(next => async args => { + if (args.request && args.request.headers) { + // eslint-disable-next-line no-param-reassign + args.request.headers['x-amz-scal-archive-info'] = 'true'; + } + return next(args); + }, { + step: 'build', + name: 'attachArchiveInfoHeader', + }); +} + /** * compare 2 version by their stale dates returning: * - LT (-1) if v1 is less than v2 @@ -1588,12 +1601,20 @@ class LifecycleTask extends BackbeatTask { params.VersionId = obj.VersionId; } const command = new HeadObjectCommand(params); + attachArchiveInfoHeader(command); attachReqUids(command, log.getSerializedUids()); return this.s3target.send(command) .then(data => { LifecycleMetrics.onS3Request(log, 'headObject', 'bucket', null); const object = Object.assign({}, obj, - { LastModified: data.LastModified }); + { + LastModified: data.LastModified, + // With the archive-info header, CloudServer always + // returns the real storage class (never STANDARD): + // the preserved cold class for cold/restored objects, + // the data-store name otherwise. + StorageClass: data.StorageClass, + }); // There is an order of importance in cases of conflicts // Expiration and NoncurrentVersionExpiration should be priority diff --git a/tests/unit/lifecycle/LifecycleBucketProcessorPolicy.spec.js b/tests/unit/lifecycle/LifecycleBucketProcessorPolicy.spec.js new file mode 100644 index 000000000..6b4283b54 --- /dev/null +++ b/tests/unit/lifecycle/LifecycleBucketProcessorPolicy.spec.js @@ -0,0 +1,14 @@ +const assert = require('assert'); + +const bucketProcessorPolicy = require('../../../extensions/lifecycle/bucketProcessor/policy.json'); + +describe('LifecycleBucketProcessor policy', () => { + it('should allow archive info reads for lifecycle metric location resolution', () => { + const actions = bucketProcessorPolicy.Statement + .find(statement => statement.Sid === 'LifecycleExpirationBucketProcessor') + .Action; + + assert(actions.includes('scality:GetObjectArchiveInfo')); + assert(!actions.includes('s3:GetBucketLocation')); + }); +}); diff --git a/tests/unit/lifecycle/LifecycleDeleteObjectTask.spec.js b/tests/unit/lifecycle/LifecycleDeleteObjectTask.spec.js index 0c1b0b59d..aca44bbf9 100644 --- a/tests/unit/lifecycle/LifecycleDeleteObjectTask.spec.js +++ b/tests/unit/lifecycle/LifecycleDeleteObjectTask.spec.js @@ -7,6 +7,7 @@ const { ObjectMD } = require('arsenal').models; const ActionQueueEntry = require('../../../lib/models/ActionQueueEntry'); const LifecycleDeleteObjectTask = require( '../../../extensions/lifecycle/tasks/LifecycleDeleteObjectTask'); +const { LifecycleMetrics } = require('../../../extensions/lifecycle/LifecycleMetrics'); const day = 1000 * 60 * 60 * 24; @@ -51,6 +52,7 @@ describe('LifecycleDeleteObjectTask', () => { }); afterEach(() => { + sinon.restore(); backbeatMdProxyClient.setError(null); }); @@ -279,6 +281,55 @@ describe('LifecycleDeleteObjectTask', () => { }); }); + + it('should emit expiration metrics with the cold location carried by the entry', done => { + const startedMetric = sinon.stub(LifecycleMetrics, 'onLifecycleStarted'); + const completedMetric = sinon.stub(LifecycleMetrics, 'onLifecycleCompleted'); + // Simulate a restored cold object: the object metadata carries the + // warm location the restored copy was written to, while the entry + // carries the cold location resolved at queue time. The metric must + // use the entry location, not the metadata one. + objMd.setDataStoreName('warm-restored-location'); + const entry = ActionQueueEntry.create('deleteObject') + .setAttribute('target.owner', 'testowner') + .setAttribute('target.bucket', 'testbucket') + .setAttribute('target.accountId', 'testid') + .setAttribute('target.key', 'testkey') + .setAttribute('target.version', 'testversion') + .setAttribute('details.dataStoreName', 'cold-location') + .setAttribute('transitionTime', Date.now() - day); + backbeatClient.setResponse(null, {}); + task.processActionEntry(entry, err => { + assert.ifError(err); + assert.strictEqual(startedMetric.firstCall.args[2], 'cold-location'); + assert.strictEqual(completedMetric.firstCall.args[2], 'cold-location'); + done(); + }); + }); + + it('should fall back to the metadata location when the entry does not carry one', done => { + const startedMetric = sinon.stub(LifecycleMetrics, 'onLifecycleStarted'); + const completedMetric = sinon.stub(LifecycleMetrics, 'onLifecycleCompleted'); + // Entries queued without details.dataStoreName (e.g. expired object + // delete markers from LifecycleTaskV2): the metric falls back to the + // location from the object metadata fetched during processing. + objMd.setDataStoreName('md-location'); + const entry = ActionQueueEntry.create('deleteObject') + .setAttribute('target.owner', 'testowner') + .setAttribute('target.bucket', 'testbucket') + .setAttribute('target.accountId', 'testid') + .setAttribute('target.key', 'testkey') + .setAttribute('target.version', 'testversion') + .setAttribute('transitionTime', Date.now() - day); + backbeatClient.setResponse(null, {}); + task.processActionEntry(entry, err => { + assert.ifError(err); + assert.strictEqual(startedMetric.firstCall.args[2], 'md-location'); + assert.strictEqual(completedMetric.firstCall.args[2], 'md-location'); + done(); + }); + }); + it('should fallback to deleteObject method if deleteObjectFromExpiration is not supported', done => { const entry = ActionQueueEntry.create('deleteObject') .setAttribute('target.owner', 'testowner') diff --git a/tests/unit/lifecycle/LifecycleTask.spec.js b/tests/unit/lifecycle/LifecycleTask.spec.js index deedfca8a..7150bbe14 100644 --- a/tests/unit/lifecycle/LifecycleTask.spec.js +++ b/tests/unit/lifecycle/LifecycleTask.spec.js @@ -10,6 +10,8 @@ const LifecycleTask = require( '../../../extensions/lifecycle/tasks/LifecycleTask'); const LifecycleTaskV2 = require( '../../../extensions/lifecycle/tasks/LifecycleTaskV2'); +const ActionQueueEntry = require('../../../lib/models/ActionQueueEntry'); +const { LifecycleMetrics } = require('../../../extensions/lifecycle/LifecycleMetrics'); const fakeLogger = require('../../utils/fakeLogger'); const { withActiveSpan } = require('../../utils/withActiveSpan'); const { timeOptions } = require('../../functional/lifecycle/configObjects'); @@ -2470,6 +2472,121 @@ describe('lifecycle task helper methods', () => { }); }); }); + + describe('_sendObjectAction', () => { + it('should emit trigger metrics with the entry location', done => { + const lifecycleTask = new LifecycleTask(lp); + const sentEntries = []; + lifecycleTask.objectTasksTopic = 'object-topic'; + lifecycleTask.circuitBreakers = { + tripped: sinon.stub().returns(false), + }; + lifecycleTask.producer = { + sendToTopic: (topic, entries, cb) => { + sentEntries.push({ topic, entries }); + cb(); + }, + }; + const triggeredMetric = sinon.stub(LifecycleMetrics, 'onLifecycleTriggered'); + + const entry = ActionQueueEntry.create('deleteObject') + .setAttribute('target.owner', 'test-owner') + .setAttribute('target.bucket', 'test-bucket') + .setAttribute('target.accountId', 'test-account') + .setAttribute('target.key', 'test-key') + .setAttribute('details.dataStoreName', 'us-east-1') + .setAttribute('transitionTime', Date.now() - HOUR); + lifecycleTask._sendObjectAction(entry, err => { + assert.ifError(err); + assert.strictEqual(entry.getAttribute('details.dataStoreName'), 'us-east-1'); + assert.strictEqual(lifecycleTask.circuitBreakers.tripped.firstCall.args[1], 'us-east-1'); + assert.strictEqual(triggeredMetric.firstCall.args[3], 'us-east-1'); + assert.strictEqual(sentEntries.length, 1); + done(); + }); + }); + }); + + describe('_compareObject location metrics', () => { + // With the x-amz-scal-archive-info request header, CloudServer's + // HeadObject always returns the real storage class: the preserved + // cold class for cold and restored objects, the data-store name + // otherwise (never the STANDARD placeholder the listing may carry). + [ + { + desc: 'hot object on the default location', + listedStorageClass: 'STANDARD', + headStorageClass: 'us-east-1', + }, + { + desc: 'hot object on a non-default location', + listedStorageClass: 'site-azure', + headStorageClass: 'site-azure', + }, + { + desc: 'cold object', + listedStorageClass: 'location-dmf-v1', + headStorageClass: 'location-dmf-v1', + }, + { + // restored objects keep the cold class in + // x-amz-storage-class even though the restored copy lives + // on a warm location + desc: 'restored cold object', + listedStorageClass: 'location-dmf-v1', + headStorageClass: 'location-dmf-v1', + }, + ].forEach(({ desc, listedStorageClass, headStorageClass }) => { + it(`should queue the expiration of a ${desc} with the ` + + 'archive-info storage class', done => { + class LifecycleTaskMock extends LifecycleTask { + _sendObjectAction(entry, cb) { + this.latestEntry = entry; + return cb(); + } + } + + const lifecycleTask = new LifecycleTaskMock(lp); + lifecycleTask.s3target = { + send: sinon.stub().resolves({ + LastModified: new Date().toISOString(), + StorageClass: headStorageClass, + }), + }; + + const bucketData = { + target: { + owner: 'test-owner', + bucket: 'test-bucket', + accountId: 'test-account', + }, + details: {}, + }; + const rules = { + Expiration: { + Date: new Date(Date.now() - DAY), + }, + }; + const listedObject = Object.assign({}, OBJECT, { + StorageClass: listedStorageClass, + }); + + lifecycleTask._compareObject(bucketData, listedObject, rules, fakeLogger, err => { + assert.ifError(err); + assert.strictEqual(lifecycleTask.s3target.send.calledOnce, true); + const command = lifecycleTask.s3target.send.firstCall.args[0]; + assert(command.middlewareStack.identify() + .includes('attachArchiveInfoHeader - build')); + assert.strictEqual( + lifecycleTask.latestEntry.getAttribute('details.dataStoreName'), + headStorageClass + ); + done(); + }); + }); + }); + }); + }); describe('LifecycleTask trace-context propagation', () => { diff --git a/tests/unit/lifecycle/LifecycleUpdateExpirationTask.spec.js b/tests/unit/lifecycle/LifecycleUpdateExpirationTask.spec.js index f7ed09c10..305956cf3 100644 --- a/tests/unit/lifecycle/LifecycleUpdateExpirationTask.spec.js +++ b/tests/unit/lifecycle/LifecycleUpdateExpirationTask.spec.js @@ -1,10 +1,12 @@ const assert = require('assert'); +const sinon = require('sinon'); const werelogs = require('werelogs'); const { ObjectMD } = require('arsenal').models; const ActionQueueEntry = require('../../../lib/models/ActionQueueEntry'); const LifecycleUpdateExpirationTask = require( '../../../extensions/lifecycle/tasks/LifecycleUpdateExpirationTask'); +const { LifecycleMetrics } = require('../../../extensions/lifecycle/LifecycleMetrics'); const { GarbageCollectorProducerMock, @@ -60,7 +62,13 @@ describe('LifecycleUpdateExpirationTask', () => { task = new LifecycleUpdateExpirationTask(objectProcessor); }); + afterEach(() => { + sinon.restore(); + }); + it('should expire restored object', done => { + const startedMetric = sinon.stub(LifecycleMetrics, 'onLifecycleStarted'); + const completedMetric = sinon.stub(LifecycleMetrics, 'onLifecycleCompleted'); const requestedAt = new Date(); const restoreCompletedAt = new Date(); const expireDate = new Date(); @@ -94,6 +102,8 @@ describe('LifecycleUpdateExpirationTask', () => { const receivedGcEntry = gcProducer.getReceivedEntry(); assert.strictEqual(receivedGcEntry.getActionType(), 'deleteData'); assert.deepStrictEqual(receivedGcEntry.getAttribute('target.locations'), oldLocation); + assert.strictEqual(startedMetric.firstCall.args[2], 'location-dmf-v1'); + assert.strictEqual(completedMetric.firstCall.args[2], 'location-dmf-v1'); done(); }); });