From 45ff195ece61b4e8ceb32bcddbf771f3180312b5 Mon Sep 17 00:00:00 2001 From: Maha Benzekri Date: Thu, 11 Jun 2026 12:46:53 +0200 Subject: [PATCH 1/3] Add lifecycle archive info permission. Allow the lifecycle bucket processor to request archive info during HeadObject and cover the policy so real metric location resolution keeps working. Issue: BB-721 --- extensions/lifecycle/bucketProcessor/policy.json | 3 ++- .../LifecycleBucketProcessorPolicy.spec.js | 14 ++++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) create mode 100644 tests/unit/lifecycle/LifecycleBucketProcessorPolicy.spec.js diff --git a/extensions/lifecycle/bucketProcessor/policy.json b/extensions/lifecycle/bucketProcessor/policy.json index 0de6c175b2..3f6016797d 100644 --- a/extensions/lifecycle/bucketProcessor/policy.json +++ b/extensions/lifecycle/bucketProcessor/policy.json @@ -14,7 +14,8 @@ "s3:GetObjectVersionTagging", "s3:GetObject", "s3:GetObjectVersion", - "s3:ReplicateObject" + "s3:ReplicateObject", + "scality:GetObjectArchiveInfo" ], "Resource": [ "*" diff --git a/tests/unit/lifecycle/LifecycleBucketProcessorPolicy.spec.js b/tests/unit/lifecycle/LifecycleBucketProcessorPolicy.spec.js new file mode 100644 index 0000000000..6b4283b543 --- /dev/null +++ b/tests/unit/lifecycle/LifecycleBucketProcessorPolicy.spec.js @@ -0,0 +1,14 @@ +const assert = require('assert'); + +const bucketProcessorPolicy = require('../../../extensions/lifecycle/bucketProcessor/policy.json'); + +describe('LifecycleBucketProcessor policy', () => { + it('should allow archive info reads for lifecycle metric location resolution', () => { + const actions = bucketProcessorPolicy.Statement + .find(statement => statement.Sid === 'LifecycleExpirationBucketProcessor') + .Action; + + assert(actions.includes('scality:GetObjectArchiveInfo')); + assert(!actions.includes('s3:GetBucketLocation')); + }); +}); From 23dcc78ce653fa0f48fa6d522dfde06e3e50be19 Mon Sep 17 00:00:00 2001 From: Maha Benzekri Date: Thu, 11 Jun 2026 12:46:59 +0200 Subject: [PATCH 2/3] Use archive info for lifecycle expiration metrics. Request archive info on the existing HeadObject call and reuse the returned storage class when queuing lifecycle expiration actions, so trigger metrics keep the real object location without extra metadata or bucket-location lookups. Issue: BB-721 --- extensions/lifecycle/tasks/LifecycleTask.js | 23 +++- tests/unit/lifecycle/LifecycleTask.spec.js | 117 ++++++++++++++++++++ 2 files changed, 139 insertions(+), 1 deletion(-) diff --git a/extensions/lifecycle/tasks/LifecycleTask.js b/extensions/lifecycle/tasks/LifecycleTask.js index 69321a9444..8a83118404 100644 --- a/extensions/lifecycle/tasks/LifecycleTask.js +++ b/extensions/lifecycle/tasks/LifecycleTask.js @@ -59,6 +59,19 @@ const MAX_RETRIES = 4; // parallel tasks, so the total delay of retries should about 1m30s. const MAX_RETRIES_TOTAL = CONCURRENCY_DEFAULT * MAX_RETRIES * 10; +function attachArchiveInfoHeader(command) { + command.middlewareStack.add(next => async args => { + if (args.request && args.request.headers) { + // eslint-disable-next-line no-param-reassign + args.request.headers['x-amz-scal-archive-info'] = 'true'; + } + return next(args); + }, { + step: 'build', + name: 'attachArchiveInfoHeader', + }); +} + /** * compare 2 version by their stale dates returning: * - LT (-1) if v1 is less than v2 @@ -1588,12 +1601,20 @@ class LifecycleTask extends BackbeatTask { params.VersionId = obj.VersionId; } const command = new HeadObjectCommand(params); + attachArchiveInfoHeader(command); attachReqUids(command, log.getSerializedUids()); return this.s3target.send(command) .then(data => { LifecycleMetrics.onS3Request(log, 'headObject', 'bucket', null); const object = Object.assign({}, obj, - { LastModified: data.LastModified }); + { + LastModified: data.LastModified, + // With the archive-info header, CloudServer always + // returns the real storage class (never STANDARD): + // the preserved cold class for cold/restored objects, + // the data-store name otherwise. + StorageClass: data.StorageClass, + }); // There is an order of importance in cases of conflicts // Expiration and NoncurrentVersionExpiration should be priority diff --git a/tests/unit/lifecycle/LifecycleTask.spec.js b/tests/unit/lifecycle/LifecycleTask.spec.js index deedfca8a9..7150bbe147 100644 --- a/tests/unit/lifecycle/LifecycleTask.spec.js +++ b/tests/unit/lifecycle/LifecycleTask.spec.js @@ -10,6 +10,8 @@ const LifecycleTask = require( '../../../extensions/lifecycle/tasks/LifecycleTask'); const LifecycleTaskV2 = require( '../../../extensions/lifecycle/tasks/LifecycleTaskV2'); +const ActionQueueEntry = require('../../../lib/models/ActionQueueEntry'); +const { LifecycleMetrics } = require('../../../extensions/lifecycle/LifecycleMetrics'); const fakeLogger = require('../../utils/fakeLogger'); const { withActiveSpan } = require('../../utils/withActiveSpan'); const { timeOptions } = require('../../functional/lifecycle/configObjects'); @@ -2470,6 +2472,121 @@ describe('lifecycle task helper methods', () => { }); }); }); + + describe('_sendObjectAction', () => { + it('should emit trigger metrics with the entry location', done => { + const lifecycleTask = new LifecycleTask(lp); + const sentEntries = []; + lifecycleTask.objectTasksTopic = 'object-topic'; + lifecycleTask.circuitBreakers = { + tripped: sinon.stub().returns(false), + }; + lifecycleTask.producer = { + sendToTopic: (topic, entries, cb) => { + sentEntries.push({ topic, entries }); + cb(); + }, + }; + const triggeredMetric = sinon.stub(LifecycleMetrics, 'onLifecycleTriggered'); + + const entry = ActionQueueEntry.create('deleteObject') + .setAttribute('target.owner', 'test-owner') + .setAttribute('target.bucket', 'test-bucket') + .setAttribute('target.accountId', 'test-account') + .setAttribute('target.key', 'test-key') + .setAttribute('details.dataStoreName', 'us-east-1') + .setAttribute('transitionTime', Date.now() - HOUR); + lifecycleTask._sendObjectAction(entry, err => { + assert.ifError(err); + assert.strictEqual(entry.getAttribute('details.dataStoreName'), 'us-east-1'); + assert.strictEqual(lifecycleTask.circuitBreakers.tripped.firstCall.args[1], 'us-east-1'); + assert.strictEqual(triggeredMetric.firstCall.args[3], 'us-east-1'); + assert.strictEqual(sentEntries.length, 1); + done(); + }); + }); + }); + + describe('_compareObject location metrics', () => { + // With the x-amz-scal-archive-info request header, CloudServer's + // HeadObject always returns the real storage class: the preserved + // cold class for cold and restored objects, the data-store name + // otherwise (never the STANDARD placeholder the listing may carry). + [ + { + desc: 'hot object on the default location', + listedStorageClass: 'STANDARD', + headStorageClass: 'us-east-1', + }, + { + desc: 'hot object on a non-default location', + listedStorageClass: 'site-azure', + headStorageClass: 'site-azure', + }, + { + desc: 'cold object', + listedStorageClass: 'location-dmf-v1', + headStorageClass: 'location-dmf-v1', + }, + { + // restored objects keep the cold class in + // x-amz-storage-class even though the restored copy lives + // on a warm location + desc: 'restored cold object', + listedStorageClass: 'location-dmf-v1', + headStorageClass: 'location-dmf-v1', + }, + ].forEach(({ desc, listedStorageClass, headStorageClass }) => { + it(`should queue the expiration of a ${desc} with the ` + + 'archive-info storage class', done => { + class LifecycleTaskMock extends LifecycleTask { + _sendObjectAction(entry, cb) { + this.latestEntry = entry; + return cb(); + } + } + + const lifecycleTask = new LifecycleTaskMock(lp); + lifecycleTask.s3target = { + send: sinon.stub().resolves({ + LastModified: new Date().toISOString(), + StorageClass: headStorageClass, + }), + }; + + const bucketData = { + target: { + owner: 'test-owner', + bucket: 'test-bucket', + accountId: 'test-account', + }, + details: {}, + }; + const rules = { + Expiration: { + Date: new Date(Date.now() - DAY), + }, + }; + const listedObject = Object.assign({}, OBJECT, { + StorageClass: listedStorageClass, + }); + + lifecycleTask._compareObject(bucketData, listedObject, rules, fakeLogger, err => { + assert.ifError(err); + assert.strictEqual(lifecycleTask.s3target.send.calledOnce, true); + const command = lifecycleTask.s3target.send.firstCall.args[0]; + assert(command.middlewareStack.identify() + .includes('attachArchiveInfoHeader - build')); + assert.strictEqual( + lifecycleTask.latestEntry.getAttribute('details.dataStoreName'), + headStorageClass + ); + done(); + }); + }); + }); + }); + }); describe('LifecycleTask trace-context propagation', () => { From dbbd0cb11e21c1f7bf6b7ba87c7d088edf7e6105 Mon Sep 17 00:00:00 2001 From: Maha Benzekri Date: Mon, 6 Jul 2026 11:23:46 +0200 Subject: [PATCH 3/3] Take the expiration metric location from the action entry. _compareObject issues a HeadObject with archive info and stores the resolved storage class (the preserved cold class for cold and restored objects) in the action entry's details.dataStoreName. The expiration metric therefore reports the location the action was queued against: prefer the entry attribute over objectMD.dataStoreName, which reported the warm location for restored cold objects. Keep the metadata location as a fallback for entries that do not carry the attribute: expired object delete markers queued by LifecycleTaskV2, or messages produced by a previous backbeat version during a rolling upgrade. Cover the restored-object and fallback cases in the delete and update-expiration task specs. Issue: BB-721 --- .../tasks/LifecycleDeleteObjectTask.js | 7 ++- .../LifecycleDeleteObjectTask.spec.js | 51 +++++++++++++++++++ .../LifecycleUpdateExpirationTask.spec.js | 10 ++++ 3 files changed, 67 insertions(+), 1 deletion(-) diff --git a/extensions/lifecycle/tasks/LifecycleDeleteObjectTask.js b/extensions/lifecycle/tasks/LifecycleDeleteObjectTask.js index c6d93dec30..7891673c01 100644 --- a/extensions/lifecycle/tasks/LifecycleDeleteObjectTask.js +++ b/extensions/lifecycle/tasks/LifecycleDeleteObjectTask.js @@ -195,7 +195,12 @@ class LifecycleDeleteObjectTask extends BackbeatTask { const actionType = entry.getActionType(); const transitionTime = entry.getAttribute('transitionTime'); - const location = this.objectMD?.dataStoreName || entry.getAttribute('details.dataStoreName'); + // The entry attribute carries the location resolved at queue time + // (the cold class for cold/restored objects). Fall back to the object + // metadata for entries missing it, e.g. expired object delete markers + // queued by LifecycleTaskV2, which do not set it. objectMD is only + // fetched for versioned deleteObject actions, hence the optional call. + const location = entry.getAttribute('details.dataStoreName') || this.objectMD?.getDataStoreName(); let reqMethod = 'deleteObject'; let actionMethod = this._deleteObject.bind(this); diff --git a/tests/unit/lifecycle/LifecycleDeleteObjectTask.spec.js b/tests/unit/lifecycle/LifecycleDeleteObjectTask.spec.js index 0c1b0b59dd..aca44bbf9d 100644 --- a/tests/unit/lifecycle/LifecycleDeleteObjectTask.spec.js +++ b/tests/unit/lifecycle/LifecycleDeleteObjectTask.spec.js @@ -7,6 +7,7 @@ const { ObjectMD } = require('arsenal').models; const ActionQueueEntry = require('../../../lib/models/ActionQueueEntry'); const LifecycleDeleteObjectTask = require( '../../../extensions/lifecycle/tasks/LifecycleDeleteObjectTask'); +const { LifecycleMetrics } = require('../../../extensions/lifecycle/LifecycleMetrics'); const day = 1000 * 60 * 60 * 24; @@ -51,6 +52,7 @@ describe('LifecycleDeleteObjectTask', () => { }); afterEach(() => { + sinon.restore(); backbeatMdProxyClient.setError(null); }); @@ -279,6 +281,55 @@ describe('LifecycleDeleteObjectTask', () => { }); }); + + it('should emit expiration metrics with the cold location carried by the entry', done => { + const startedMetric = sinon.stub(LifecycleMetrics, 'onLifecycleStarted'); + const completedMetric = sinon.stub(LifecycleMetrics, 'onLifecycleCompleted'); + // Simulate a restored cold object: the object metadata carries the + // warm location the restored copy was written to, while the entry + // carries the cold location resolved at queue time. The metric must + // use the entry location, not the metadata one. + objMd.setDataStoreName('warm-restored-location'); + const entry = ActionQueueEntry.create('deleteObject') + .setAttribute('target.owner', 'testowner') + .setAttribute('target.bucket', 'testbucket') + .setAttribute('target.accountId', 'testid') + .setAttribute('target.key', 'testkey') + .setAttribute('target.version', 'testversion') + .setAttribute('details.dataStoreName', 'cold-location') + .setAttribute('transitionTime', Date.now() - day); + backbeatClient.setResponse(null, {}); + task.processActionEntry(entry, err => { + assert.ifError(err); + assert.strictEqual(startedMetric.firstCall.args[2], 'cold-location'); + assert.strictEqual(completedMetric.firstCall.args[2], 'cold-location'); + done(); + }); + }); + + it('should fall back to the metadata location when the entry does not carry one', done => { + const startedMetric = sinon.stub(LifecycleMetrics, 'onLifecycleStarted'); + const completedMetric = sinon.stub(LifecycleMetrics, 'onLifecycleCompleted'); + // Entries queued without details.dataStoreName (e.g. expired object + // delete markers from LifecycleTaskV2): the metric falls back to the + // location from the object metadata fetched during processing. + objMd.setDataStoreName('md-location'); + const entry = ActionQueueEntry.create('deleteObject') + .setAttribute('target.owner', 'testowner') + .setAttribute('target.bucket', 'testbucket') + .setAttribute('target.accountId', 'testid') + .setAttribute('target.key', 'testkey') + .setAttribute('target.version', 'testversion') + .setAttribute('transitionTime', Date.now() - day); + backbeatClient.setResponse(null, {}); + task.processActionEntry(entry, err => { + assert.ifError(err); + assert.strictEqual(startedMetric.firstCall.args[2], 'md-location'); + assert.strictEqual(completedMetric.firstCall.args[2], 'md-location'); + done(); + }); + }); + it('should fallback to deleteObject method if deleteObjectFromExpiration is not supported', done => { const entry = ActionQueueEntry.create('deleteObject') .setAttribute('target.owner', 'testowner') diff --git a/tests/unit/lifecycle/LifecycleUpdateExpirationTask.spec.js b/tests/unit/lifecycle/LifecycleUpdateExpirationTask.spec.js index f7ed09c10c..305956cf36 100644 --- a/tests/unit/lifecycle/LifecycleUpdateExpirationTask.spec.js +++ b/tests/unit/lifecycle/LifecycleUpdateExpirationTask.spec.js @@ -1,10 +1,12 @@ const assert = require('assert'); +const sinon = require('sinon'); const werelogs = require('werelogs'); const { ObjectMD } = require('arsenal').models; const ActionQueueEntry = require('../../../lib/models/ActionQueueEntry'); const LifecycleUpdateExpirationTask = require( '../../../extensions/lifecycle/tasks/LifecycleUpdateExpirationTask'); +const { LifecycleMetrics } = require('../../../extensions/lifecycle/LifecycleMetrics'); const { GarbageCollectorProducerMock, @@ -60,7 +62,13 @@ describe('LifecycleUpdateExpirationTask', () => { task = new LifecycleUpdateExpirationTask(objectProcessor); }); + afterEach(() => { + sinon.restore(); + }); + it('should expire restored object', done => { + const startedMetric = sinon.stub(LifecycleMetrics, 'onLifecycleStarted'); + const completedMetric = sinon.stub(LifecycleMetrics, 'onLifecycleCompleted'); const requestedAt = new Date(); const restoreCompletedAt = new Date(); const expireDate = new Date(); @@ -94,6 +102,8 @@ describe('LifecycleUpdateExpirationTask', () => { const receivedGcEntry = gcProducer.getReceivedEntry(); assert.strictEqual(receivedGcEntry.getActionType(), 'deleteData'); assert.deepStrictEqual(receivedGcEntry.getAttribute('target.locations'), oldLocation); + assert.strictEqual(startedMetric.firstCall.args[2], 'location-dmf-v1'); + assert.strictEqual(completedMetric.firstCall.args[2], 'location-dmf-v1'); done(); }); });