From 41ae4a06b6fee941c35ea0e1f881c8a71ce55e65 Mon Sep 17 00:00:00 2001 From: FlorianGille Date: Tue, 22 Oct 2024 10:01:39 +0200 Subject: [PATCH 1/2] feat: add a second targeted bucket to upload files --- packages/server/utils/storage-s3.js | 190 +++++++++++++++++++++------- 1 file changed, 144 insertions(+), 46 deletions(-) diff --git a/packages/server/utils/storage-s3.js b/packages/server/utils/storage-s3.js index 3a2c49d2..534a16e0 100644 --- a/packages/server/utils/storage-s3.js +++ b/packages/server/utils/storage-s3.js @@ -12,9 +12,21 @@ const formatName = require('../helpers/format-filename-for-jquery-fileupload.js' if (!config.isAws) { module.exports = {}; } else { - AWS.config.update(config.storage.aws); + // AWS.config.update(config.storage.aws); const endpoint = new AWS.Endpoint(config.storage.aws.endpoint); - const s3 = new AWS.S3({ endpoint }); + const s3 = new AWS.S3({ + endpoint, + accessKeyId: config.storage.aws.accessKeyId, + secretAccessKey: config.storage.aws.secretAccessKey, + region: config.storage.aws.region, + }); + const newEndpoint = new AWS.Endpoint(config.storage.newAws.endpoint); + const newS3 = new AWS.S3({ + endpoint: newEndpoint, + accessKeyId: config.storage.newAws.accessKeyId, + secretAccessKey: config.storage.newAws.secretAccessKey, + region: config.storage.newAws.region, + }); // http://docs.aws.amazon.com/AWSJavaScriptSDK/guide/node-examples.html#Amazon_Simple_Storage_Service__Amazon_S3_ // http://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html#getObject-property @@ -35,25 +47,52 @@ if (!config.isAws) { const deferred = defer(); const { name, path } = file; const source = fs.createReadStream(path); - - s3.upload( - { - Bucket: config.storage.aws.bucketName, - Key: name, - Body: source, - }, - function (err, data) { - logger.error(err, data); - } - ) + const source2 = fs.createReadStream(path); + + const upload1 = s3 + .upload( + { + Bucket: config.storage.aws.bucketName, + Key: name, + Body: source, + }, + function (err, data) { + logger.error(err, data); + } + ) .on('httpUploadProgress', (progress) => { logger.info( - `writeStreamFromPath – ${name}`, + `writeStreamFromPath 1 – ${name}`, (progress.loaded / progress.total) * 100 ); - if (progress.loaded >= progress.total) deferred.resolve(); }) - .on('error', deferred.reject); + .on('error', deferred.reject) + .promise(); + + const upload2 = newS3 + .upload( + { + Bucket: config.storage.newAws.bucketName, + Key: name, + Body: source2, + }, + function (err, data) { + logger.error(err, data); + } + ) + .on('httpUploadProgress', (progress) => { + logger.info({ progress }); + logger.info( + `writeStreamFromPath 2 – ${name}`, + (progress.loaded / progress.total) * 100 + ); + }) + .on('error', deferred.reject) + .promise(); + + Promise.all([upload1, upload2]).then(() => { + deferred.resolve(); + }); return deferred; }; @@ -61,26 +100,53 @@ if (!config.isAws) { const writeStreamFromStream = (source, name) => { const deferred = defer(); - s3.upload( - { - Bucket: config.storage.aws.bucketName, - Key: name, - Body: source, - }, - (err, data) => { - logger.error(err, data); - // if (err) return reject( err ) - // resolve( data ) - } - ) + const upload1 = s3 + .upload( + { + Bucket: config.storage.aws.bucketName, + Key: name, + Body: source, + }, + (err, data) => { + logger.error(err, data); + // if (err) return reject( err ) + // resolve( data ) + } + ) .on('httpUploadProgress', (progress) => { logger.info( - `writeStreamFromStream – ${name}`, + `writeStreamFromStream 1 – ${name}`, (progress.loaded / progress.total) * 100 ); - if (progress.loaded >= progress.total) deferred.resolve(); }) - .on('error', deferred.reject); + .on('error', deferred.reject) + .promise(); + + const upload2 = newS3 + .upload( + { + Bucket: config.storage.newAws.bucketName, + Key: name, + Body: source, + }, + (err, data) => { + logger.error(err, data); + // if (err) return reject( err ) + // resolve( data ) + } + ) + .on('httpUploadProgress', (progress) => { + logger.info( + `writeStreamFromStream 2 – ${name}`, + (progress.loaded / progress.total) * 100 + ); + }) + .on('error', deferred.reject) + .promise(); + + Promise.all([upload1, upload2]).then(() => { + deferred.resolve(); + }); return deferred; }; @@ -88,25 +154,51 @@ if (!config.isAws) { const writeStreamFromStreamWithPrefix = (source, name, prefix) => { const deferred = defer(); - s3.upload( - { - Bucket: config.storage.aws.bucketName, - Prefix: prefix, - Key: name, - Body: source, - }, - (err, data) => { - logger.error(err, data); - } - ) + const upload1 = s3 + .upload( + { + Bucket: config.storage.aws.bucketName, + Prefix: prefix, + Key: name, + Body: source, + }, + (err, data) => { + logger.error(err, data); + } + ) + .on('httpUploadProgress', (progress) => { + logger.info( + `writeStreamFromStreamWithPrefix 1 – ${name}`, + (progress.loaded / progress.total) * 100 + ); + }) + .on('error', deferred.reject) + .promise(); + + const upload2 = newS3 + .upload( + { + Bucket: config.storage.newAws.bucketName, + Prefix: prefix, + Key: name, + Body: source, + }, + (err, data) => { + logger.error(err, data); + } + ) .on('httpUploadProgress', (progress) => { logger.info( - `writeStreamFromStream – ${name}`, + `writeStreamFromStreamWithPrefix 2 – ${name}`, (progress.loaded / progress.total) * 100 ); - if (progress.loaded >= progress.total) deferred.resolve(); }) - .on('error', deferred.reject); + .on('error', deferred.reject) + .promise(); + + Promise.all([upload1, upload2]).then(() => { + deferred.resolve(); + }); return deferred; }; @@ -129,11 +221,17 @@ if (!config.isAws) { // http://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html#copyObject-property const copyObject = denodeify(s3.copyObject.bind(s3)); + const newCopyObject = denodeify(newS3.copyObject.bind(newS3)); const copyImages = (oldPrefix, newPrefix) => { logger.info('copying images with S3 storage'); return listImages(oldPrefix).then((files) => Promise.all(files.map(copy))); - function copy(file) { + async function copy(file) { + await newCopyObject({ + Bucket: config.storage.newAws.bucketName, + CopySource: config.storage.newAws.bucketName + '/' + file.name, + Key: file.name.replace(oldPrefix, newPrefix), + }); return copyObject({ Bucket: config.storage.aws.bucketName, CopySource: config.storage.aws.bucketName + '/' + file.name, From 430edd9bcc4931d8dbce6e10f58b0d6f79bebc01 Mon Sep 17 00:00:00 2001 From: FlorianGille Date: Wed, 23 Oct 2024 14:45:09 +0200 Subject: [PATCH 2/2] fix: clone source for write stream from stream and stream prefix --- packages/server/utils/storage-s3.js | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/packages/server/utils/storage-s3.js b/packages/server/utils/storage-s3.js index 534a16e0..3ce5689f 100644 --- a/packages/server/utils/storage-s3.js +++ b/packages/server/utils/storage-s3.js @@ -99,13 +99,15 @@ if (!config.isAws) { const writeStreamFromStream = (source, name) => { const deferred = defer(); + const source1 = source.clone(); + const source2 = source.clone(); const upload1 = s3 .upload( { Bucket: config.storage.aws.bucketName, Key: name, - Body: source, + Body: source1, }, (err, data) => { logger.error(err, data); @@ -127,7 +129,7 @@ if (!config.isAws) { { Bucket: config.storage.newAws.bucketName, Key: name, - Body: source, + Body: source2, }, (err, data) => { logger.error(err, data); @@ -153,6 +155,8 @@ if (!config.isAws) { const writeStreamFromStreamWithPrefix = (source, name, prefix) => { const deferred = defer(); + const source1 = source.clone(); + const source2 = source.clone(); const upload1 = s3 .upload( @@ -160,7 +164,7 @@ if (!config.isAws) { Bucket: config.storage.aws.bucketName, Prefix: prefix, Key: name, - Body: source, + Body: source1, }, (err, data) => { logger.error(err, data); @@ -181,7 +185,7 @@ if (!config.isAws) { Bucket: config.storage.newAws.bucketName, Prefix: prefix, Key: name, - Body: source, + Body: source2, }, (err, data) => { logger.error(err, data);