Properly get location in lifecycle metrics

extensions/lifecycle/bucketProcessor/policy.json

@@ -14,7 +14,8 @@
                 "s3:GetObjectVersionTagging",
                 "s3:GetObject",
                 "s3:GetObjectVersion",
-                "s3:ReplicateObject"
+                "s3:ReplicateObject",
+                "scality:GetObjectArchiveInfo"
             ],
             "Resource": [
                 "*"

extensions/lifecycle/tasks/LifecycleDeleteObjectTask.js

@@ -195,7 +195,7 @@ class LifecycleDeleteObjectTask extends BackbeatTask {
 
         const actionType = entry.getActionType();
         const transitionTime = entry.getAttribute('transitionTime');
-        const location = this.objectMD?.dataStoreName || entry.getAttribute('details.dataStoreName');
+        const location = entry.getAttribute('details.dataStoreName');
 
         let reqMethod = 'deleteObject';
         let actionMethod = this._deleteObject.bind(this);

extensions/lifecycle/tasks/LifecycleTask.js

@@ -59,6 +59,19 @@ const MAX_RETRIES = 4;
 // parallel tasks, so the total delay of retries should about 1m30s.
 const MAX_RETRIES_TOTAL = CONCURRENCY_DEFAULT * MAX_RETRIES * 10;
 
+function attachArchiveInfoHeader(command) {
+    command.middlewareStack.add(next => async args => {
+        if (args.request && args.request.headers) {
+            // eslint-disable-next-line no-param-reassign
+            args.request.headers['x-amz-scal-archive-info'] = 'true';
+        }
+        return next(args);
+    }, {
+        step: 'build',
+        name: 'attachArchiveInfoHeader',
+    });
+}
+
 /**
  * compare 2 version by their stale dates returning:
  * - LT (-1) if v1 is less than v2
@@ -1588,12 +1601,20 @@ class LifecycleTask extends BackbeatTask {
             params.VersionId = obj.VersionId;
         }
         const command = new HeadObjectCommand(params);
+        attachArchiveInfoHeader(command);
         attachReqUids(command, log.getSerializedUids());
         return this.s3target.send(command)
             .then(data => {
                 LifecycleMetrics.onS3Request(log, 'headObject', 'bucket', null);
                 const object = Object.assign({}, obj,
-                    { LastModified: data.LastModified });
+                    {
+                        LastModified: data.LastModified,
+                        // With the archive-info header, CloudServer always
+                        // returns the real storage class (never STANDARD):
+                        // the preserved cold class for cold/restored objects,
+                        // the data-store name otherwise.
+                        StorageClass: data.StorageClass,
+                    });
 
                 // There is an order of importance in cases of conflicts
                 // Expiration and NoncurrentVersionExpiration should be priority

tests/unit/lifecycle/LifecycleBucketProcessorPolicy.spec.js

@@ -0,0 +1,14 @@
+const assert = require('assert');
+
+const bucketProcessorPolicy = require('../../../extensions/lifecycle/bucketProcessor/policy.json');
+
+describe('LifecycleBucketProcessor policy', () => {
+    it('should allow archive info reads for lifecycle metric location resolution', () => {
+        const actions = bucketProcessorPolicy.Statement
+            .find(statement => statement.Sid === 'LifecycleExpirationBucketProcessor')
+            .Action;
+
+        assert(actions.includes('scality:GetObjectArchiveInfo'));
+        assert(!actions.includes('s3:GetBucketLocation'));
+    });
+});

tests/unit/lifecycle/LifecycleDeleteObjectTask.spec.js

@@ -7,6 +7,7 @@ const { ObjectMD } = require('arsenal').models;
 const ActionQueueEntry = require('../../../lib/models/ActionQueueEntry');
 const LifecycleDeleteObjectTask = require(
     '../../../extensions/lifecycle/tasks/LifecycleDeleteObjectTask');
+const { LifecycleMetrics } = require('../../../extensions/lifecycle/LifecycleMetrics');
 
 const day = 1000 * 60 * 60 * 24;
 
@@ -51,6 +52,7 @@ describe('LifecycleDeleteObjectTask', () => {
     });
 
     afterEach(() => {
+        sinon.restore();
         backbeatMdProxyClient.setError(null);
     });
 
@@ -279,6 +281,32 @@ describe('LifecycleDeleteObjectTask', () => {
         });
     });
 
+
+    it('should emit expiration metrics with the cold location carried by the entry', done => {
+        const startedMetric = sinon.stub(LifecycleMetrics, 'onLifecycleStarted');
+        const completedMetric = sinon.stub(LifecycleMetrics, 'onLifecycleCompleted');
+        // Simulate a restored cold object: the object metadata carries the
+        // warm location the restored copy was written to, while the entry
+        // carries the cold location resolved at queue time. The metric must
+        // use the entry location, not the metadata one.
+        objMd.setDataStoreName('warm-restored-location');
+        const entry = ActionQueueEntry.create('deleteObject')
+            .setAttribute('target.owner', 'testowner')
+            .setAttribute('target.bucket', 'testbucket')
+            .setAttribute('target.accountId', 'testid')
+            .setAttribute('target.key', 'testkey')
+            .setAttribute('target.version', 'testversion')
+            .setAttribute('details.dataStoreName', 'cold-location')
+            .setAttribute('transitionTime', Date.now() - day);
+        backbeatClient.setResponse(null, {});
+        task.processActionEntry(entry, err => {
+            assert.ifError(err);
+            assert.strictEqual(startedMetric.firstCall.args[2], 'cold-location');
+            assert.strictEqual(completedMetric.firstCall.args[2], 'cold-location');
+            done();
+        });
+    });
+
     it('should fallback to deleteObject method if deleteObjectFromExpiration is not supported', done => {
         const entry = ActionQueueEntry.create('deleteObject')
             .setAttribute('target.owner', 'testowner')

tests/unit/lifecycle/LifecycleTask.spec.js

@@ -10,6 +10,8 @@ const LifecycleTask = require(
     '../../../extensions/lifecycle/tasks/LifecycleTask');
 const LifecycleTaskV2 = require(
     '../../../extensions/lifecycle/tasks/LifecycleTaskV2');
+const ActionQueueEntry = require('../../../lib/models/ActionQueueEntry');
+const { LifecycleMetrics } = require('../../../extensions/lifecycle/LifecycleMetrics');
 const fakeLogger = require('../../utils/fakeLogger');
 const { withActiveSpan } = require('../../utils/withActiveSpan');
 const { timeOptions } = require('../../functional/lifecycle/configObjects');
@@ -2470,6 +2472,121 @@ describe('lifecycle task helper methods', () => {
             });
         });
     });
+
+    describe('_sendObjectAction', () => {
+        it('should emit trigger metrics with the entry location', done => {
+            const lifecycleTask = new LifecycleTask(lp);
+            const sentEntries = [];
+            lifecycleTask.objectTasksTopic = 'object-topic';
+            lifecycleTask.circuitBreakers = {
+                tripped: sinon.stub().returns(false),
+            };
+            lifecycleTask.producer = {
+                sendToTopic: (topic, entries, cb) => {
+                    sentEntries.push({ topic, entries });
+                    cb();
+                },
+            };
+            const triggeredMetric = sinon.stub(LifecycleMetrics, 'onLifecycleTriggered');
+
+            const entry = ActionQueueEntry.create('deleteObject')
+                .setAttribute('target.owner', 'test-owner')
+                .setAttribute('target.bucket', 'test-bucket')
+                .setAttribute('target.accountId', 'test-account')
+                .setAttribute('target.key', 'test-key')
+                .setAttribute('details.dataStoreName', 'us-east-1')
+                .setAttribute('transitionTime', Date.now() - HOUR);
+            lifecycleTask._sendObjectAction(entry, err => {
+                assert.ifError(err);
+                assert.strictEqual(entry.getAttribute('details.dataStoreName'), 'us-east-1');
+                assert.strictEqual(lifecycleTask.circuitBreakers.tripped.firstCall.args[1], 'us-east-1');
+                assert.strictEqual(triggeredMetric.firstCall.args[3], 'us-east-1');
+                assert.strictEqual(sentEntries.length, 1);
+                done();
+            });
+        });
+    });
+
+    describe('_compareObject location metrics', () => {
+        // With the x-amz-scal-archive-info request header, CloudServer's
+        // HeadObject always returns the real storage class: the preserved
+        // cold class for cold and restored objects, the data-store name
+        // otherwise (never the STANDARD placeholder the listing may carry).
+        [
+            {
+                desc: 'hot object on the default location',
+                listedStorageClass: 'STANDARD',
+                headStorageClass: 'us-east-1',
+            },
+            {
+                desc: 'hot object on a non-default location',
+                listedStorageClass: 'site-azure',
+                headStorageClass: 'site-azure',
+            },
+            {
+                desc: 'cold object',
+                listedStorageClass: 'location-dmf-v1',
+                headStorageClass: 'location-dmf-v1',
+            },
+            {
+                // restored objects keep the cold class in
+                // x-amz-storage-class even though the restored copy lives
+                // on a warm location
+                desc: 'restored cold object',
+                listedStorageClass: 'location-dmf-v1',
+                headStorageClass: 'location-dmf-v1',
+            },
+        ].forEach(({ desc, listedStorageClass, headStorageClass }) => {
+            it(`should queue the expiration of a ${desc} with the ` +
+            'archive-info storage class', done => {
+                class LifecycleTaskMock extends LifecycleTask {
+                    _sendObjectAction(entry, cb) {
+                        this.latestEntry = entry;
+                        return cb();
+                    }
+                }
+
+                const lifecycleTask = new LifecycleTaskMock(lp);
+                lifecycleTask.s3target = {
+                    send: sinon.stub().resolves({
+                        LastModified: new Date().toISOString(),
+                        StorageClass: headStorageClass,
+                    }),
+                };
+
+                const bucketData = {
+                    target: {
+                        owner: 'test-owner',
+                        bucket: 'test-bucket',
+                        accountId: 'test-account',
+                    },
+                    details: {},
+                };
+                const rules = {
+                    Expiration: {
+                        Date: new Date(Date.now() - DAY),
+                    },
+                };
+                const listedObject = Object.assign({}, OBJECT, {
+                    StorageClass: listedStorageClass,
+                });
+
+                lifecycleTask._compareObject(bucketData, listedObject, rules, fakeLogger, err => {
+                    assert.ifError(err);
+                    assert.strictEqual(lifecycleTask.s3target.send.calledOnce, true);
+                    const command = lifecycleTask.s3target.send.firstCall.args[0];
+                    assert(command.middlewareStack.identify()
+                        .includes('attachArchiveInfoHeader - build'));
+                    assert.strictEqual(
+                        lifecycleTask.latestEntry.getAttribute('details.dataStoreName'),
+                        headStorageClass
+                    );
+                    done();
+                });
+            });
+        });
+    });
+
 });
 
 describe('LifecycleTask trace-context propagation', () => {

tests/unit/lifecycle/LifecycleUpdateExpirationTask.spec.js

@@ -1,10 +1,12 @@
 const assert = require('assert');
+const sinon = require('sinon');
 const werelogs = require('werelogs');
 
 const { ObjectMD } = require('arsenal').models;
 const ActionQueueEntry = require('../../../lib/models/ActionQueueEntry');
 const LifecycleUpdateExpirationTask = require(
     '../../../extensions/lifecycle/tasks/LifecycleUpdateExpirationTask');
+const { LifecycleMetrics } = require('../../../extensions/lifecycle/LifecycleMetrics');
 
 const {
     GarbageCollectorProducerMock,
@@ -60,7 +62,13 @@ describe('LifecycleUpdateExpirationTask', () => {
         task = new LifecycleUpdateExpirationTask(objectProcessor);
     });
 
+    afterEach(() => {
+        sinon.restore();
+    });
+
     it('should expire restored object', done => {
+        const startedMetric = sinon.stub(LifecycleMetrics, 'onLifecycleStarted');
+        const completedMetric = sinon.stub(LifecycleMetrics, 'onLifecycleCompleted');
         const requestedAt = new Date();
         const restoreCompletedAt = new Date();
         const expireDate = new Date();
@@ -94,6 +102,8 @@ describe('LifecycleUpdateExpirationTask', () => {
             const receivedGcEntry = gcProducer.getReceivedEntry();
             assert.strictEqual(receivedGcEntry.getActionType(), 'deleteData');
             assert.deepStrictEqual(receivedGcEntry.getAttribute('target.locations'), oldLocation);
+            assert.strictEqual(startedMetric.firstCall.args[2], 'location-dmf-v1');
+            assert.strictEqual(completedMetric.firstCall.args[2], 'location-dmf-v1');
             done();
         });
     });