Never garbage-collect data on an isCRR location

extensions/gc/tasks/GarbageCollectorTask.js

@@ -6,6 +6,7 @@ const BackbeatTask = require('../../../lib/tasks/BackbeatTask');
 const { BatchDeleteCommand } = require('@scality/cloudserverclient');
 const { GarbageCollectorMetrics } = require('../GarbageCollectorMetrics');
 const { TRANSITION_ATTEMPT_MD } = require('../../../lib/util/transitionAttempt');
+const locationsConfig = require('../../../conf/locationConfig.json') || {};
 /** @typedef { import('../GarbageCollector.js') } GarbageCollector */
 
 class GarbageCollectorTask extends BackbeatTask {
@@ -143,6 +144,18 @@ class GarbageCollectorTask extends BackbeatTask {
     _executeDeleteDataOnce(entry, log, done) {
         const { locations } = entry.getAttribute('target');
         const ruleType = entry.getContextAttribute('ruleType');
+        // The service can only delete local data: data on a CRR location lives on
+        // the remote site, out of reach. Getting one here means a bug upstream.
+        const { dataStoreName } = locations[0] || {};
+        if (locationsConfig[dataStoreName]?.isCRR) {
+            log.warn('refusing to delete data on a CRR location', {
+                method: 'GarbageCollectorTask._executeDeleteDataOnce',
+                dataStoreName,
+                ruleType,
+                ...entry.getLogInfo(),
+            });
+            return process.nextTick(done);
+        }
         const params = {
             Locations: locations.map(location => ({
                 key: location.key,
@@ -160,7 +173,7 @@ class GarbageCollectorTask extends BackbeatTask {
             }),
         };
 
-        this._batchDeleteData(params, entry, log, err => {
+        return this._batchDeleteData(params, entry, log, err => {
             // ruleType can be either `transition` or `restore` (for restore-expiration)
             GarbageCollectorMetrics.onS3Request(log, 'batchdelete', ruleType, err);
             entry.setEnd(err);

extensions/lifecycle/tasks/LifecycleUpdateTransitionTask.js

@@ -10,6 +10,7 @@ const {
     TRANSITION_ATTEMPT_MD,
     getTransitionAttempt,
 } = require('../../../lib/util/transitionAttempt');
+const locationsConfig = require('../../../conf/locationConfig.json') || {};
 /** @typedef { import('../objectProcessor/LifecycleObjectProcessor.js') } LifecycleObjectProcessor */
 
 class LifecycleUpdateTransitionTask extends BackbeatTask {
@@ -114,6 +115,19 @@ class LifecycleUpdateTransitionTask extends BackbeatTask {
 
     _garbageCollectLocation(entry, locations, log, done) {
         const { bucket, key, version, eTag, accountId, owner } = this.getTargetAttribute(entry);
+        // Data on a CRR location means this was pull replication, not a
+        // transition: the source is the remote site, and must not be removed.
+        const { dataStoreName } = locations[0] || {};
+        if (locationsConfig[dataStoreName]?.isCRR) {
+            log.info('skipping garbage collection of data on a CRR location', {
+                method: 'LifecycleUpdateTransitionTask._garbageCollectLocation',
+                bucket,
+                objectKey: key,
+                versionId: version,
+                dataStoreName,
+            });
+            return process.nextTick(done);
+        }
         const gcEntry = ActionQueueEntry.create('deleteData')
               .addContext({
                   origin: 'lifecycle',

tests/unit/gc/GarbageCollectorTask.spec.js

@@ -424,4 +424,111 @@ describe('GarbageCollectorTask', () => {
         });
     });
 
+    describe('with CRR locations', () => {
+        let log;
+
+        function createDeleteDataEntry(locations) {
+            return ActionQueueEntry.create('deleteData')
+                .addContext({
+                    origin: 'lifecycle',
+                    ruleType: 'transition',
+                    bucketName: bucket,
+                    objectKey: key,
+                    versionId: version,
+                })
+                .setAttribute('serviceName', 'lifecycle-transition')
+                .setAttribute('source', {
+                    bucket,
+                    objectKey: key,
+                    storageClass: 'sourceStorageClass',
+                })
+                .setAttribute('target', {
+                    bucket,
+                    key: version,
+                    version: key,
+                    accountId,
+                    owner,
+                    locations,
+                });
+        }
+
+        const crrLocation = {
+            key: 'crrKey',
+            dataStoreName: 'location-crr-source',
+            size: 10,
+            dataStoreVersionId: 'crrVersionId',
+        };
+        const regularLocation = {
+            key: 'locationKey',
+            dataStoreName: 'us-east-1',
+            size: 20,
+            dataStoreVersionId: 'dataStoreVersionId',
+        };
+
+        beforeEach(() => {
+            log = {
+                info: sinon.spy(),
+                warn: sinon.spy(),
+                debug: sinon.spy(),
+                error: sinon.spy(),
+                getSerializedUids: () => 'uids',
+            };
+            log.end = () => log;
+            gcTask.logger = { newRequestLogger: () => log };
+            backbeatClient.batchDeleteResponse = { error: null, res: null };
+        });
+
+        it('should not delete anything and warn when all locations are on a ' +
+        'CRR location', done => {
+            const entry = createDeleteDataEntry([crrLocation]);
+            const batchDeleteDataSpy = sinon.spy(gcTask, '_batchDeleteData');
+            const onGcCompletedSpy = sinon.spy(GarbageCollectorMetrics, 'onGcCompleted');
+
+            gcTask.processActionEntry(entry, err => {
+                assert.ifError(err);
+                assert.strictEqual(batchDeleteDataSpy.callCount, 0);
+                assert.strictEqual(backbeatClient.times.batchDeleteResponse, 0);
+                assert.strictEqual(onGcCompletedSpy.callCount, 0);
+                assert.strictEqual(log.warn.callCount, 1);
+                assert.strictEqual(
+                    log.warn.firstCall.args[1].dataStoreName,
+                    'location-crr-source');
+                batchDeleteDataSpy.restore();
+                onGcCompletedSpy.restore();
+                done();
+            });
+        });
+
+        it('should not delete anything for a multipart object on a CRR ' +
+        'location', done => {
+            const secondCrrLocation = Object.assign({}, crrLocation, { key: 'crrKey2' });
+            const entry = createDeleteDataEntry([crrLocation, secondCrrLocation]);
+            const batchDeleteDataSpy = sinon.spy(gcTask, '_batchDeleteData');
+
+            gcTask.processActionEntry(entry, err => {
+                assert.ifError(err);
+                assert.strictEqual(batchDeleteDataSpy.callCount, 0);
+                assert.strictEqual(log.warn.callCount, 1);
+                batchDeleteDataSpy.restore();
+                done();
+            });
+        });
+
+        it('should delete all locations and not warn when none is on a CRR ' +
+        'location', done => {
+            const entry = createDeleteDataEntry([regularLocation]);
+            const batchDeleteDataSpy = sinon.spy(gcTask, '_batchDeleteData');
+
+            gcTask.processActionEntry(entry, err => {
+                assert.ifError(err);
+                assert.strictEqual(batchDeleteDataSpy.callCount, 1);
+                assert.deepStrictEqual(
+                    batchDeleteDataSpy.firstCall.args[0].Locations,
+                    [regularLocation]);
+                assert.strictEqual(log.warn.callCount, 0);
+                batchDeleteDataSpy.restore();
+                done();
+            });
+        });
+    });
 });

tests/unit/lifecycle/LifecycleUpdateTransitionTask.spec.js

@@ -148,6 +148,48 @@ describe('LifecycleUpdateTransitionTask', () => {
         });
     });
 
+    it('should update metadata but not GC the from-location when it is a CRR ' +
+    'location', done => {
+        const crrLocation = [Object.assign({}, oldLocation[0],
+            { dataStoreName: 'location-crr-source' })];
+        mdObj.setLocation(crrLocation);
+        task.processActionEntry(actionEntry, err => {
+            assert.ifError(err);
+            const receivedMd = backbeatMetadataProxyClient.getReceivedMd();
+            assert.deepStrictEqual(receivedMd.location, newLocation);
+            assert.strictEqual(gcProducer.getReceivedEntry(), null);
+            done();
+        });
+    });
+
+    it('should not GC anything for a multipart object on a CRR location', done => {
+        const crrPart = Object.assign({}, oldLocation[0],
+            { key: 'crrKey', dataStoreName: 'location-crr-source' });
+        const secondCrrPart = Object.assign({}, crrPart, { key: 'crrKey2', start: 10 });
+        mdObj.setLocation([crrPart, secondCrrPart]);
+        task.processActionEntry(actionEntry, err => {
+            assert.ifError(err);
+            assert.strictEqual(gcProducer.getReceivedEntry(), null);
+            done();
+        });
+    });
+
+    it('should still GC the new location on rollback even if the ' +
+    'from-location is a CRR location', done => {
+        mdObj.setLocation([Object.assign({}, oldLocation[0],
+            { dataStoreName: 'location-crr-source' })]);
+        actionEntry.setAttribute('target.eTag',
+                                 '"6713e7cf89b6b16d5abf11d1fabac587"');
+        task.processActionEntry(actionEntry, err => {
+            assert.ifError(err);
+            assert.strictEqual(backbeatMetadataProxyClient.getReceivedMd(), null);
+            const receivedGcEntry = gcProducer.getReceivedEntry();
+            assert.deepStrictEqual(
+                receivedGcEntry.getAttribute('target.locations'), newLocation);
+            done();
+        });
+    });
+
     it('should reset transition-in-progress flag when transition fails', done => {
         actionEntry.setError(errors.InternalError);
         task.processActionEntry(actionEntry, err => {