From 80cb85262d0e65852917003d6b30f6c3d9ab95fb Mon Sep 17 00:00:00 2001 From: Francois Ferrand Date: Sat, 5 Sep 2026 00:56:24 +0200 Subject: [PATCH 1/2] Do not garbage-collect the from-location when it is an isCRR location In a clean room, objects exist locally in metadata but their data still lives on the production (isCRR) location. Localizing such an object reuses the lifecycle transition pipeline, which normally garbage-collects the from-location once the copy is merged into metadata: against an isCRR source that would delete production data on the remote site. Data on an isCRR location is remote production data and must never be garbage-collected. Make it a general rule keyed on the location type rather than a clean-room special case, so it also holds for a plain lifecycle transition out of such a location. The rollback paths, which collect the freshly written local copy, are unaffected. This also underpins the replay-safety of duplicate copy actions: the first merge must not collect the remote source. Issue: BB-813 --- .../tasks/LifecycleUpdateTransitionTask.js | 14 +++++++ .../LifecycleUpdateTransitionTask.spec.js | 42 +++++++++++++++++++ 2 files changed, 56 insertions(+) diff --git a/extensions/lifecycle/tasks/LifecycleUpdateTransitionTask.js b/extensions/lifecycle/tasks/LifecycleUpdateTransitionTask.js index 924049a3e..2eda378dd 100644 --- a/extensions/lifecycle/tasks/LifecycleUpdateTransitionTask.js +++ b/extensions/lifecycle/tasks/LifecycleUpdateTransitionTask.js @@ -10,6 +10,7 @@ const { TRANSITION_ATTEMPT_MD, getTransitionAttempt, } = require('../../../lib/util/transitionAttempt'); +const locationsConfig = require('../../../conf/locationConfig.json') || {}; /** @typedef { import('../objectProcessor/LifecycleObjectProcessor.js') } LifecycleObjectProcessor */ class LifecycleUpdateTransitionTask extends BackbeatTask { @@ -114,6 +115,19 @@ class LifecycleUpdateTransitionTask extends BackbeatTask { _garbageCollectLocation(entry, locations, log, done) { const { bucket, key, version, eTag, accountId, owner } = this.getTargetAttribute(entry); + // Data on a CRR location means this was pull replication, not a + // transition: the source is the remote site, and must not be removed. + const { dataStoreName } = locations[0] || {}; + if (locationsConfig[dataStoreName]?.isCRR) { + log.info('skipping garbage collection of data on a CRR location', { + method: 'LifecycleUpdateTransitionTask._garbageCollectLocation', + bucket, + objectKey: key, + versionId: version, + dataStoreName, + }); + return process.nextTick(done); + } const gcEntry = ActionQueueEntry.create('deleteData') .addContext({ origin: 'lifecycle', diff --git a/tests/unit/lifecycle/LifecycleUpdateTransitionTask.spec.js b/tests/unit/lifecycle/LifecycleUpdateTransitionTask.spec.js index d2d0473e5..5754ce5de 100644 --- a/tests/unit/lifecycle/LifecycleUpdateTransitionTask.spec.js +++ b/tests/unit/lifecycle/LifecycleUpdateTransitionTask.spec.js @@ -148,6 +148,48 @@ describe('LifecycleUpdateTransitionTask', () => { }); }); + it('should update metadata but not GC the from-location when it is a CRR ' + + 'location', done => { + const crrLocation = [Object.assign({}, oldLocation[0], + { dataStoreName: 'location-crr-source' })]; + mdObj.setLocation(crrLocation); + task.processActionEntry(actionEntry, err => { + assert.ifError(err); + const receivedMd = backbeatMetadataProxyClient.getReceivedMd(); + assert.deepStrictEqual(receivedMd.location, newLocation); + assert.strictEqual(gcProducer.getReceivedEntry(), null); + done(); + }); + }); + + it('should not GC anything for a multipart object on a CRR location', done => { + const crrPart = Object.assign({}, oldLocation[0], + { key: 'crrKey', dataStoreName: 'location-crr-source' }); + const secondCrrPart = Object.assign({}, crrPart, { key: 'crrKey2', start: 10 }); + mdObj.setLocation([crrPart, secondCrrPart]); + task.processActionEntry(actionEntry, err => { + assert.ifError(err); + assert.strictEqual(gcProducer.getReceivedEntry(), null); + done(); + }); + }); + + it('should still GC the new location on rollback even if the ' + + 'from-location is a CRR location', done => { + mdObj.setLocation([Object.assign({}, oldLocation[0], + { dataStoreName: 'location-crr-source' })]); + actionEntry.setAttribute('target.eTag', + '"6713e7cf89b6b16d5abf11d1fabac587"'); + task.processActionEntry(actionEntry, err => { + assert.ifError(err); + assert.strictEqual(backbeatMetadataProxyClient.getReceivedMd(), null); + const receivedGcEntry = gcProducer.getReceivedEntry(); + assert.deepStrictEqual( + receivedGcEntry.getAttribute('target.locations'), newLocation); + done(); + }); + }); + it('should reset transition-in-progress flag when transition fails', done => { actionEntry.setError(errors.InternalError); task.processActionEntry(actionEntry, err => { From f9f4df44278019d69f65cd833f3d368ed5564846 Mon Sep 17 00:00:00 2001 From: Francois Ferrand Date: Sat, 5 Sep 2026 00:56:25 +0200 Subject: [PATCH 2/2] Skip isCRR locations in the GC service Not triggering the collection from the copy engine only covers one publisher. The GC service is the single point where data actually gets deleted, so guard there too: any deleteData action targeting an isCRR location is dropped, whoever published it. The service only has data to delete, and it has no way to delete data living on a remote location. Getting such an action is a bug upstream, hence the warning, but there is nothing that can be done about it beyond skipping it. Issue: BB-818 --- extensions/gc/tasks/GarbageCollectorTask.js | 15 ++- tests/unit/gc/GarbageCollectorTask.spec.js | 107 ++++++++++++++++++++ 2 files changed, 121 insertions(+), 1 deletion(-) diff --git a/extensions/gc/tasks/GarbageCollectorTask.js b/extensions/gc/tasks/GarbageCollectorTask.js index 78dcdf7ca..0c1060f9a 100644 --- a/extensions/gc/tasks/GarbageCollectorTask.js +++ b/extensions/gc/tasks/GarbageCollectorTask.js @@ -6,6 +6,7 @@ const BackbeatTask = require('../../../lib/tasks/BackbeatTask'); const { BatchDeleteCommand } = require('@scality/cloudserverclient'); const { GarbageCollectorMetrics } = require('../GarbageCollectorMetrics'); const { TRANSITION_ATTEMPT_MD } = require('../../../lib/util/transitionAttempt'); +const locationsConfig = require('../../../conf/locationConfig.json') || {}; /** @typedef { import('../GarbageCollector.js') } GarbageCollector */ class GarbageCollectorTask extends BackbeatTask { @@ -143,6 +144,18 @@ class GarbageCollectorTask extends BackbeatTask { _executeDeleteDataOnce(entry, log, done) { const { locations } = entry.getAttribute('target'); const ruleType = entry.getContextAttribute('ruleType'); + // The service can only delete local data: data on a CRR location lives on + // the remote site, out of reach. Getting one here means a bug upstream. + const { dataStoreName } = locations[0] || {}; + if (locationsConfig[dataStoreName]?.isCRR) { + log.warn('refusing to delete data on a CRR location', { + method: 'GarbageCollectorTask._executeDeleteDataOnce', + dataStoreName, + ruleType, + ...entry.getLogInfo(), + }); + return process.nextTick(done); + } const params = { Locations: locations.map(location => ({ key: location.key, @@ -160,7 +173,7 @@ class GarbageCollectorTask extends BackbeatTask { }), }; - this._batchDeleteData(params, entry, log, err => { + return this._batchDeleteData(params, entry, log, err => { // ruleType can be either `transition` or `restore` (for restore-expiration) GarbageCollectorMetrics.onS3Request(log, 'batchdelete', ruleType, err); entry.setEnd(err); diff --git a/tests/unit/gc/GarbageCollectorTask.spec.js b/tests/unit/gc/GarbageCollectorTask.spec.js index b39b2de9d..d979d9580 100644 --- a/tests/unit/gc/GarbageCollectorTask.spec.js +++ b/tests/unit/gc/GarbageCollectorTask.spec.js @@ -424,4 +424,111 @@ describe('GarbageCollectorTask', () => { }); }); + describe('with CRR locations', () => { + let log; + + function createDeleteDataEntry(locations) { + return ActionQueueEntry.create('deleteData') + .addContext({ + origin: 'lifecycle', + ruleType: 'transition', + bucketName: bucket, + objectKey: key, + versionId: version, + }) + .setAttribute('serviceName', 'lifecycle-transition') + .setAttribute('source', { + bucket, + objectKey: key, + storageClass: 'sourceStorageClass', + }) + .setAttribute('target', { + bucket, + key: version, + version: key, + accountId, + owner, + locations, + }); + } + + const crrLocation = { + key: 'crrKey', + dataStoreName: 'location-crr-source', + size: 10, + dataStoreVersionId: 'crrVersionId', + }; + const regularLocation = { + key: 'locationKey', + dataStoreName: 'us-east-1', + size: 20, + dataStoreVersionId: 'dataStoreVersionId', + }; + + beforeEach(() => { + log = { + info: sinon.spy(), + warn: sinon.spy(), + debug: sinon.spy(), + error: sinon.spy(), + getSerializedUids: () => 'uids', + }; + log.end = () => log; + gcTask.logger = { newRequestLogger: () => log }; + backbeatClient.batchDeleteResponse = { error: null, res: null }; + }); + + it('should not delete anything and warn when all locations are on a ' + + 'CRR location', done => { + const entry = createDeleteDataEntry([crrLocation]); + const batchDeleteDataSpy = sinon.spy(gcTask, '_batchDeleteData'); + const onGcCompletedSpy = sinon.spy(GarbageCollectorMetrics, 'onGcCompleted'); + + gcTask.processActionEntry(entry, err => { + assert.ifError(err); + assert.strictEqual(batchDeleteDataSpy.callCount, 0); + assert.strictEqual(backbeatClient.times.batchDeleteResponse, 0); + assert.strictEqual(onGcCompletedSpy.callCount, 0); + assert.strictEqual(log.warn.callCount, 1); + assert.strictEqual( + log.warn.firstCall.args[1].dataStoreName, + 'location-crr-source'); + batchDeleteDataSpy.restore(); + onGcCompletedSpy.restore(); + done(); + }); + }); + + it('should not delete anything for a multipart object on a CRR ' + + 'location', done => { + const secondCrrLocation = Object.assign({}, crrLocation, { key: 'crrKey2' }); + const entry = createDeleteDataEntry([crrLocation, secondCrrLocation]); + const batchDeleteDataSpy = sinon.spy(gcTask, '_batchDeleteData'); + + gcTask.processActionEntry(entry, err => { + assert.ifError(err); + assert.strictEqual(batchDeleteDataSpy.callCount, 0); + assert.strictEqual(log.warn.callCount, 1); + batchDeleteDataSpy.restore(); + done(); + }); + }); + + it('should delete all locations and not warn when none is on a CRR ' + + 'location', done => { + const entry = createDeleteDataEntry([regularLocation]); + const batchDeleteDataSpy = sinon.spy(gcTask, '_batchDeleteData'); + + gcTask.processActionEntry(entry, err => { + assert.ifError(err); + assert.strictEqual(batchDeleteDataSpy.callCount, 1); + assert.deepStrictEqual( + batchDeleteDataSpy.firstCall.args[0].Locations, + [regularLocation]); + assert.strictEqual(log.warn.callCount, 0); + batchDeleteDataSpy.restore(); + done(); + }); + }); + }); });