Reclaim local data when a localized version is deleted

conf/locationConfig.json

@@ -42,5 +42,12 @@
     "legacyAwsBehavior": false,
     "isCold": true,
     "details": {}
+  },
+  "location-crr-source": {
+    "type": "scality",
+    "objectId": "location-crr-source",
+    "legacyAwsBehavior": false,
+    "isCRR": true,
+    "details": {}
   }
 }

extensions/gc/tasks/GarbageCollectorTask.js

@@ -5,6 +5,7 @@ const { ObjectMD } = require('arsenal').models;
 const BackbeatTask = require('../../../lib/tasks/BackbeatTask');
 const { BatchDeleteCommand } = require('@scality/cloudserverclient');
 const { GarbageCollectorMetrics } = require('../GarbageCollectorMetrics');
+const { filterOutCRRLocations, getCRRLocationNames } = require('../../../lib/util/locations');
 /** @typedef { import('../GarbageCollector.js') } GarbageCollector */
 
 class GarbageCollectorTask extends BackbeatTask {
@@ -142,8 +143,26 @@ class GarbageCollectorTask extends BackbeatTask {
     _executeDeleteDataOnce(entry, log, done) {
         const { locations } = entry.getAttribute('target');
         const ruleType = entry.getContextAttribute('ruleType');
+        // Last line of defense: whoever published this entry, data on a CRR
+        // location belongs to the remote site and must never be deleted.
+        const locationsToDelete = filterOutCRRLocations(locations);
+        if (locationsToDelete.length !== (locations || []).length) {
+            log.warn('refusing to delete data on CRR location', {
+                method: 'GarbageCollectorTask._executeDeleteDataOnce',
+                bucket: entry.getAttribute('source.bucket'),
+                objectKey: entry.getAttribute('source.objectKey'),
+                dataStoreNames: getCRRLocationNames(locations),
+                ruleType,
+                ...entry.getLogInfo(),
+            });
+        }
+        if (locationsToDelete.length === 0) {
+            entry.setEnd(null);
+            log.info('action execution ended, nothing to delete', entry.getLogInfo());
+            return process.nextTick(done);
+        }
         const params = {
-            Locations: locations.map(location => ({
+            Locations: locationsToDelete.map(location => ({
                 key: location.key,
                 dataStoreName: location.dataStoreName,
                 size: location.size,
@@ -159,7 +178,7 @@ class GarbageCollectorTask extends BackbeatTask {
             }),
         };
 
-        this._batchDeleteData(params, entry, log, err => {
+        return this._batchDeleteData(params, entry, log, err => {
             // ruleType can be either `transition` or `restore` (for restore-expiration)
             GarbageCollectorMetrics.onS3Request(log, 'batchdelete', ruleType, err);
             entry.setEnd(err);
@@ -184,7 +203,7 @@ class GarbageCollectorTask extends BackbeatTask {
             }
 
             GarbageCollectorMetrics.onGcCompleted(log, ruleType,
-                locations[0]?.dataStoreName, Date.now() - entry.getAttribute('timestamp'));
+                locationsToDelete[0]?.dataStoreName, Date.now() - entry.getAttribute('timestamp'));
             return done();
         });
     }

extensions/lifecycle/tasks/LifecycleUpdateTransitionTask.js

@@ -6,6 +6,7 @@ const BackbeatTask = require('../../../lib/tasks/BackbeatTask');
 const ActionQueueEntry = require('../../../lib/models/ActionQueueEntry');
 const ObjectMD = require('arsenal').models.ObjectMD;
 const { LifecycleMetrics } = require('../LifecycleMetrics');
+const { filterOutCRRLocations, getCRRLocationNames } = require('../../../lib/util/locations');
 /** @typedef { import('../objectProcessor/LifecycleObjectProcessor.js') } LifecycleObjectProcessor */
 
 class LifecycleUpdateTransitionTask extends BackbeatTask {
@@ -112,6 +113,21 @@ class LifecycleUpdateTransitionTask extends BackbeatTask {
 
     _garbageCollectLocation(entry, locations, log, done) {
         const { bucket, key, version, eTag, accountId, owner } = this.getTargetAttribute(entry);
+        // Data stored on a CRR location belongs to the remote site: the copy we
+        // just made is an extra local copy, the source must be left untouched.
+        const locationsToGC = filterOutCRRLocations(locations);
+        if (locationsToGC.length !== locations.length) {
+            log.info('skipping garbage collection of data on CRR location', {
+                method: 'LifecycleUpdateTransitionTask._garbageCollectLocation',
+                bucket,
+                objectKey: key,
+                versionId: version,
+                dataStoreNames: getCRRLocationNames(locations),
+            });
+        }
+        if (locationsToGC.length === 0) {
+            return process.nextTick(done);
+        }
         const gcEntry = ActionQueueEntry.create('deleteData')
               .addContext({
                   origin: 'lifecycle',
@@ -126,7 +142,7 @@ class LifecycleUpdateTransitionTask extends BackbeatTask {
               .setAttribute('serviceName', 'lifecycle-transition')
               .setAttribute('target.accountId', accountId)
               .setAttribute('target.owner', owner)
-              .setAttribute('target.locations', locations);
+              .setAttribute('target.locations', locationsToGC);
         this.gcProducer.publishActionEntry(gcEntry);
         return process.nextTick(done);
     }

extensions/mongoProcessor/MongoQueueProcessor.js

@@ -13,6 +13,7 @@ const { extractVersionId } = require('../../lib/util/versioning');
 
 const Config = require('../../lib/Config');
 const BackbeatConsumer = require('../../lib/BackbeatConsumer');
+const ActionQueueEntry = require('../../lib/models/ActionQueueEntry');
 const QueueEntry = require('../../lib/models/QueueEntry');
 const DeleteOpQueueEntry = require('../../lib/models/DeleteOpQueueEntry');
 const ObjectQueueEntry = require('../../lib/models/ObjectQueueEntry');
@@ -22,6 +23,8 @@ const { metricsExtension, metricsTypeCompleted, metricsTypePendingOnly } =
 const getContentType = require('./utils/contentTypeHelper');
 const BucketMemState = require('./utils/BucketMemState');
 const MongoProcessorMetrics = require('./MongoProcessorMetrics');
+const GarbageCollectorProducer = require('../gc/GarbageCollectorProducer');
+const { isCRRLocation, filterOutCRRLocations } = require('../../lib/util/locations');
 
 // batch metrics by location and send to kafka metrics topic every 5 seconds
 const METRIC_REPORT_INTERVAL_MS = process.env.CI === 'true' ? 1000 : 5000;
@@ -64,13 +67,17 @@ class MongoQueueProcessor {
      * @param {number} [mongoProcessorConfig.concurrency] - consumer concurrency
      * @param {Object} mongoClientConfig - config for connecting to mongo
      * @param {Object} mConfig - metrics config
+     * @param {Object} [gcConfig] - garbage collector config, required to reclaim
+     *   the data of localized objects
      */
-    constructor(kafkaConfig, mongoProcessorConfig, mongoClientConfig, mConfig) {
+    constructor(kafkaConfig, mongoProcessorConfig, mongoClientConfig, mConfig, gcConfig) {
         this.kafkaConfig = kafkaConfig;
         this.mongoProcessorConfig = mongoProcessorConfig;
         this.mongoClientConfig = mongoClientConfig;
         this._mConfig = mConfig;
+        this._gcConfig = gcConfig;
 
+        this._gcProducer = null;
         this._consumer = null;
         this._bootstrapList = null;
         this.logger = new Logger('Backbeat:Ingestion:MongoProcessor');
@@ -119,6 +126,16 @@ class MongoQueueProcessor {
                 }
                 return next(err);
             }),
+            next => {
+                if (!this._gcConfig) {
+                    this.logger.info('no garbage collector configured', {
+                        method: 'MongoQueueProcessor.start',
+                    });
+                    return next();
+                }
+                this._gcProducer = new GarbageCollectorProducer();
+                return this._gcProducer.setupProducer(next);
+            },
         ], error => {
             if (error) {
                 this.logger.fatal('error starting mongo queue processor');
@@ -377,16 +394,24 @@ class MongoQueueProcessor {
         async.waterfall([
             cb => this._getZenkoObjectMetadata(log, sourceEntry, versionId, cb),
             (zenkoObjMd, cb) => {
+                // In a clean room the bucket location constraint is the (isCRR) source
+                // location: once an object has been localized its data does not live there
+                // anymore, so it looks exactly like a transitioned object. It must still be
+                // deleted, and the local copy of the data reclaimed.
+                const localizedLocations = isCRRLocation(location) ?
+                    filterOutCRRLocations(zenkoObjMd.location) : [];
+
                 // Skip if the object is in a different location, i.e. when the delete was caused
                 // by restored-object expiration or transition. It works because the dataStoreName
                 // is updated before actually sending the object to GC to effectively delete the
                 // data.
                 const encode = versionId => (versionId ? VersionID.encode(versionId) : 'null');
-                if (zenkoObjMd.dataStoreName !== location ||
+                if (localizedLocations.length === 0 && (
+                    zenkoObjMd.dataStoreName !== location ||
                     zenkoObjMd.location?.length !== 1 ||
                     zenkoObjMd.location[0].dataStoreName !== location ||
                     zenkoObjMd.location[0].key !== key ||
-                    (zenkoObjMd.location[0].dataStoreVersionId || 'null') !== encode(entryVersionId)
+                    (zenkoObjMd.location[0].dataStoreVersionId || 'null') !== encode(entryVersionId))
                 ) {
                     log.end().info('ignore delete entry, transitioned to another location', {
                         entry: sourceEntry.getLogInfo(),
@@ -395,9 +420,9 @@ class MongoQueueProcessor {
                     return done();
                 }
 
-                return cb(null, zenkoObjMd);
+                return cb(null, zenkoObjMd, localizedLocations);
             },
-            (zenkoObjMd, cb) => {
+            (zenkoObjMd, localizedLocations, cb) => {
                 const options = {};
 
                 // Calling deleteObject with empty options to use deleteObjectNoVer which is used
@@ -418,7 +443,19 @@ class MongoQueueProcessor {
                     options.doesNotNeedOpogUpdate = true;
                 }
 
-                return this._mongoClient.deleteObject(bucket, key, options, log, cb);
+                return this._mongoClient.deleteObject(bucket, key, options, log, err => {
+                    if (err) {
+                        return cb(err);
+                    }
+                    // The data copied locally is not referenced by anything anymore: reclaim
+                    // it. Doing it after the metadata delete means a failure here leaks the
+                    // data, instead of losing it.
+                    if (localizedLocations.length > 0) {
+                        this._garbageCollectData(log, sourceEntry, zenkoObjMd, versionId,
+                            localizedLocations);
+                    }
+                    return cb();
+                });
             },
         ], err => {
             if (err?.is.NoSuchKey) {
@@ -448,6 +485,53 @@ class MongoQueueProcessor {
         });
     }
 
+    /**
+     * Publish a garbage collection entry to reclaim the data of an object which has just
+     * been deleted from mongo.
+     * @param {Logger.newRequestLogger} log - request logger object
+     * @param {DeleteOpQueueEntry} sourceEntry - delete object entry
+     * @param {Object} zenkoObjMd - metadata of the object which was deleted
+     * @param {string} versionId - decoded version id of the object which was deleted
+     * @param {Object[]} locations - location parts to delete
+     * @return {undefined}
+     */
+    _garbageCollectData(log, sourceEntry, zenkoObjMd, versionId, locations) {
+        const bucket = sourceEntry.getBucket();
+        const key = sourceEntry.getObjectKey();
+
+        if (!this._gcProducer) {
+            log.error('no garbage collector configured, cannot reclaim the data', {
+                bucket,
+                objectKey: key,
+                versionId,
+            });
+            return;
+        }
+
+        // The version is already gone, so no lastModified is passed along: the GC must not
+        // send any conditional header, only the locations matter here.
+        const gcEntry = ActionQueueEntry.create('deleteData')
+            .addContext({
+                origin: 'localization',
+                ruleType: 'transition',
+                reqId: log.getSerializedUids(),
+                bucketName: bucket,
+                objectKey: key,
+                versionId: versionId ? VersionID.encode(versionId) : undefined,
+                eTag: zenkoObjMd['content-md5'],
+            })
+            .setAttribute('source', {
+                bucket,
+                objectKey: key,
+                storageClass: zenkoObjMd.dataStoreName,
+            })
+            .setAttribute('serviceName', 'md-ingestion')
+            .setAttribute('target.owner', zenkoObjMd['owner-id'])
+            .setAttribute('target.locations', locations);
+
+        this._gcProducer.publishActionEntry(gcEntry);
+    }
+
     /**
      * Process an object entry
      * @param {Logger.newRequestLogger} log - request logger object

extensions/mongoProcessor/mongoProcessorTask.js

@@ -21,14 +21,15 @@ const mongoProcessorConfig = config.extensions.mongoProcessor;
 // TODO: consider whether we would want a separate mongo config
 // for the consumer side
 const mongoClientConfig = config.queuePopulator.mongo;
+const gcConfig = config.extensions.gc;
 
 const log = new werelogs.Logger('Backbeat:MongoProcessor:task');
 const mongoProcessorLogConfig = mongoProcessorConfig.log ?? config.log;
 werelogs.configure({ level: mongoProcessorLogConfig.logLevel,
     dump: mongoProcessorLogConfig.dumpLevel });
 
 const mqp = new MongoQueueProcessor(kafkaConfig, mongoProcessorConfig,
-    mongoClientConfig, mConfig);
+    mongoClientConfig, mConfig, gcConfig);
 
 /**
  * Handle ProbeServer liveness check

lib/util/locations.js

@@ -0,0 +1,52 @@
+const locationsConfig = require('../../conf/locationConfig.json') || {};
+
+/**
+ * Tell whether a location holds data owned by a remote site.
+ *
+ * Data stored on such a location is remote production data: we may read it
+ * (e.g. to copy it locally), but we must never delete it.
+ *
+ * @param {String} dataStoreName - location name
+ * @return {Boolean} true if the location is a CRR (remote) location
+ */
+function isCRRLocation(dataStoreName) {
+    return Boolean(dataStoreName && locationsConfig[dataStoreName] &&
+                   locationsConfig[dataStoreName].isCRR);
+}
+
+/**
+ * Remove from a list of location parts those living on a CRR location,
+ * i.e. those which must never be garbage-collected.
+ *
+ * @param {Object[]} locations - array of location parts
+ * @return {Object[]} the location parts which are safe to delete
+ */
+function filterOutCRRLocations(locations) {
+    if (!Array.isArray(locations)) {
+        return [];
+    }
+    return locations.filter(location => !isCRRLocation(location && location.dataStoreName));
+}
+
+/**
+ * List the distinct CRR location names found in a list of location parts,
+ * for logging purposes.
+ *
+ * @param {Object[]} locations - array of location parts
+ * @return {String[]} distinct CRR location names
+ */
+function getCRRLocationNames(locations) {
+    if (!Array.isArray(locations)) {
+        return [];
+    }
+    const names = locations
+        .map(location => location && location.dataStoreName)
+        .filter(dataStoreName => isCRRLocation(dataStoreName));
+    return [...new Set(names)];
+}
+
+module.exports = {
+    isCRRLocation,
+    filterOutCRRLocations,
+    getCRRLocationNames,
+};