From d58a747436c41eea9c9c86e0ab9d7d17301eeeda Mon Sep 17 00:00:00 2001 From: FlorianGille Date: Mon, 28 Oct 2024 15:57:17 +0100 Subject: [PATCH] Rollback migration s3 --- packages/server/utils/storage-s3.js | 198 +++++++--------------------- 1 file changed, 48 insertions(+), 150 deletions(-) diff --git a/packages/server/utils/storage-s3.js b/packages/server/utils/storage-s3.js index 3ce5689f..3a2c49d2 100644 --- a/packages/server/utils/storage-s3.js +++ b/packages/server/utils/storage-s3.js @@ -12,21 +12,9 @@ const formatName = require('../helpers/format-filename-for-jquery-fileupload.js' if (!config.isAws) { module.exports = {}; } else { - // AWS.config.update(config.storage.aws); + AWS.config.update(config.storage.aws); const endpoint = new AWS.Endpoint(config.storage.aws.endpoint); - const s3 = new AWS.S3({ - endpoint, - accessKeyId: config.storage.aws.accessKeyId, - secretAccessKey: config.storage.aws.secretAccessKey, - region: config.storage.aws.region, - }); - const newEndpoint = new AWS.Endpoint(config.storage.newAws.endpoint); - const newS3 = new AWS.S3({ - endpoint: newEndpoint, - accessKeyId: config.storage.newAws.accessKeyId, - secretAccessKey: config.storage.newAws.secretAccessKey, - region: config.storage.newAws.region, - }); + const s3 = new AWS.S3({ endpoint }); // http://docs.aws.amazon.com/AWSJavaScriptSDK/guide/node-examples.html#Amazon_Simple_Storage_Service__Amazon_S3_ // http://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html#getObject-property @@ -47,162 +35,78 @@ if (!config.isAws) { const deferred = defer(); const { name, path } = file; const source = fs.createReadStream(path); - const source2 = fs.createReadStream(path); - - const upload1 = s3 - .upload( - { - Bucket: config.storage.aws.bucketName, - Key: name, - Body: source, - }, - function (err, data) { - logger.error(err, data); - } - ) - .on('httpUploadProgress', (progress) => { - logger.info( - `writeStreamFromPath 1 – ${name}`, - (progress.loaded / progress.total) * 100 - ); - }) - .on('error', deferred.reject) - .promise(); - - const upload2 = newS3 - .upload( - { - Bucket: config.storage.newAws.bucketName, - Key: name, - Body: source2, - }, - function (err, data) { - logger.error(err, data); - } - ) + + s3.upload( + { + Bucket: config.storage.aws.bucketName, + Key: name, + Body: source, + }, + function (err, data) { + logger.error(err, data); + } + ) .on('httpUploadProgress', (progress) => { - logger.info({ progress }); logger.info( - `writeStreamFromPath 2 – ${name}`, + `writeStreamFromPath – ${name}`, (progress.loaded / progress.total) * 100 ); + if (progress.loaded >= progress.total) deferred.resolve(); }) - .on('error', deferred.reject) - .promise(); - - Promise.all([upload1, upload2]).then(() => { - deferred.resolve(); - }); + .on('error', deferred.reject); return deferred; }; const writeStreamFromStream = (source, name) => { const deferred = defer(); - const source1 = source.clone(); - const source2 = source.clone(); - - const upload1 = s3 - .upload( - { - Bucket: config.storage.aws.bucketName, - Key: name, - Body: source1, - }, - (err, data) => { - logger.error(err, data); - // if (err) return reject( err ) - // resolve( data ) - } - ) - .on('httpUploadProgress', (progress) => { - logger.info( - `writeStreamFromStream 1 – ${name}`, - (progress.loaded / progress.total) * 100 - ); - }) - .on('error', deferred.reject) - .promise(); - - const upload2 = newS3 - .upload( - { - Bucket: config.storage.newAws.bucketName, - Key: name, - Body: source2, - }, - (err, data) => { - logger.error(err, data); - // if (err) return reject( err ) - // resolve( data ) - } - ) + + s3.upload( + { + Bucket: config.storage.aws.bucketName, + Key: name, + Body: source, + }, + (err, data) => { + logger.error(err, data); + // if (err) return reject( err ) + // resolve( data ) + } + ) .on('httpUploadProgress', (progress) => { logger.info( - `writeStreamFromStream 2 – ${name}`, + `writeStreamFromStream – ${name}`, (progress.loaded / progress.total) * 100 ); + if (progress.loaded >= progress.total) deferred.resolve(); }) - .on('error', deferred.reject) - .promise(); - - Promise.all([upload1, upload2]).then(() => { - deferred.resolve(); - }); + .on('error', deferred.reject); return deferred; }; const writeStreamFromStreamWithPrefix = (source, name, prefix) => { const deferred = defer(); - const source1 = source.clone(); - const source2 = source.clone(); - - const upload1 = s3 - .upload( - { - Bucket: config.storage.aws.bucketName, - Prefix: prefix, - Key: name, - Body: source1, - }, - (err, data) => { - logger.error(err, data); - } - ) - .on('httpUploadProgress', (progress) => { - logger.info( - `writeStreamFromStreamWithPrefix 1 – ${name}`, - (progress.loaded / progress.total) * 100 - ); - }) - .on('error', deferred.reject) - .promise(); - - const upload2 = newS3 - .upload( - { - Bucket: config.storage.newAws.bucketName, - Prefix: prefix, - Key: name, - Body: source2, - }, - (err, data) => { - logger.error(err, data); - } - ) + + s3.upload( + { + Bucket: config.storage.aws.bucketName, + Prefix: prefix, + Key: name, + Body: source, + }, + (err, data) => { + logger.error(err, data); + } + ) .on('httpUploadProgress', (progress) => { logger.info( - `writeStreamFromStreamWithPrefix 2 – ${name}`, + `writeStreamFromStream – ${name}`, (progress.loaded / progress.total) * 100 ); + if (progress.loaded >= progress.total) deferred.resolve(); }) - .on('error', deferred.reject) - .promise(); - - Promise.all([upload1, upload2]).then(() => { - deferred.resolve(); - }); + .on('error', deferred.reject); return deferred; }; @@ -225,17 +129,11 @@ if (!config.isAws) { // http://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html#copyObject-property const copyObject = denodeify(s3.copyObject.bind(s3)); - const newCopyObject = denodeify(newS3.copyObject.bind(newS3)); const copyImages = (oldPrefix, newPrefix) => { logger.info('copying images with S3 storage'); return listImages(oldPrefix).then((files) => Promise.all(files.map(copy))); - async function copy(file) { - await newCopyObject({ - Bucket: config.storage.newAws.bucketName, - CopySource: config.storage.newAws.bucketName + '/' + file.name, - Key: file.name.replace(oldPrefix, newPrefix), - }); + function copy(file) { return copyObject({ Bucket: config.storage.aws.bucketName, CopySource: config.storage.aws.bucketName + '/' + file.name,