Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Release production (double upload) #835

Merged
merged 5 commits into from
Oct 25, 2024
Merged

Release production (double upload) #835

Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
41ae4a0
feat: add a second targeted bucket to upload files
FlorianGille Oct 22, 2024
1a63464
Merge pull request #832 from Badsender-com/chore-migration-s3
FlorianGille Oct 23, 2024
430edd9
fix: clone source for write stream from stream and stream prefix
FlorianGille Oct 23, 2024
371158b
Merge pull request #833 from Badsender-com/chore-migration-s3
FlorianGille Oct 23, 2024
f43795e
Merge pull request #834 from Badsender-com/develop-clever
FlorianGille Oct 24, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
198 changes: 150 additions & 48 deletions packages/server/utils/storage-s3.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,9 +12,21 @@ const formatName = require('../helpers/format-filename-for-jquery-fileupload.js'
if (!config.isAws) {
module.exports = {};
} else {
AWS.config.update(config.storage.aws);
// AWS.config.update(config.storage.aws);
const endpoint = new AWS.Endpoint(config.storage.aws.endpoint);
const s3 = new AWS.S3({ endpoint });
const s3 = new AWS.S3({
endpoint,
accessKeyId: config.storage.aws.accessKeyId,
secretAccessKey: config.storage.aws.secretAccessKey,
region: config.storage.aws.region,
});
const newEndpoint = new AWS.Endpoint(config.storage.newAws.endpoint);
const newS3 = new AWS.S3({
endpoint: newEndpoint,
accessKeyId: config.storage.newAws.accessKeyId,
secretAccessKey: config.storage.newAws.secretAccessKey,
region: config.storage.newAws.region,
});

// http://docs.aws.amazon.com/AWSJavaScriptSDK/guide/node-examples.html#Amazon_Simple_Storage_Service__Amazon_S3_
// http://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html#getObject-property
Expand All @@ -35,78 +47,162 @@ if (!config.isAws) {
const deferred = defer();
const { name, path } = file;
const source = fs.createReadStream(path);

s3.upload(
{
Bucket: config.storage.aws.bucketName,
Key: name,
Body: source,
},
function (err, data) {
logger.error(err, data);
}
)
const source2 = fs.createReadStream(path);

const upload1 = s3
.upload(
{
Bucket: config.storage.aws.bucketName,
Key: name,
Body: source,
},
function (err, data) {
logger.error(err, data);
}
)
.on('httpUploadProgress', (progress) => {
logger.info(
`writeStreamFromPath 1 – ${name}`,
(progress.loaded / progress.total) * 100
);
})
.on('error', deferred.reject)
.promise();

const upload2 = newS3
.upload(
{
Bucket: config.storage.newAws.bucketName,
Key: name,
Body: source2,
},
function (err, data) {
logger.error(err, data);
}
)
.on('httpUploadProgress', (progress) => {
logger.info({ progress });
logger.info(
`writeStreamFromPath – ${name}`,
`writeStreamFromPath 2 – ${name}`,
(progress.loaded / progress.total) * 100
);
if (progress.loaded >= progress.total) deferred.resolve();
})
.on('error', deferred.reject);
.on('error', deferred.reject)
.promise();

Promise.all([upload1, upload2]).then(() => {
deferred.resolve();
});

return deferred;
};

const writeStreamFromStream = (source, name) => {
const deferred = defer();

s3.upload(
{
Bucket: config.storage.aws.bucketName,
Key: name,
Body: source,
},
(err, data) => {
logger.error(err, data);
// if (err) return reject( err )
// resolve( data )
}
)
const source1 = source.clone();
const source2 = source.clone();

const upload1 = s3
.upload(
{
Bucket: config.storage.aws.bucketName,
Key: name,
Body: source1,
},
(err, data) => {
logger.error(err, data);
// if (err) return reject( err )
// resolve( data )
}
)
.on('httpUploadProgress', (progress) => {
logger.info(
`writeStreamFromStream – ${name}`,
`writeStreamFromStream 1 – ${name}`,
(progress.loaded / progress.total) * 100
);
if (progress.loaded >= progress.total) deferred.resolve();
})
.on('error', deferred.reject);
.on('error', deferred.reject)
.promise();

const upload2 = newS3
.upload(
{
Bucket: config.storage.newAws.bucketName,
Key: name,
Body: source2,
},
(err, data) => {
logger.error(err, data);
// if (err) return reject( err )
// resolve( data )
}
)
.on('httpUploadProgress', (progress) => {
logger.info(
`writeStreamFromStream 2 – ${name}`,
(progress.loaded / progress.total) * 100
);
})
.on('error', deferred.reject)
.promise();

Promise.all([upload1, upload2]).then(() => {
deferred.resolve();
});

return deferred;
};

const writeStreamFromStreamWithPrefix = (source, name, prefix) => {
const deferred = defer();

s3.upload(
{
Bucket: config.storage.aws.bucketName,
Prefix: prefix,
Key: name,
Body: source,
},
(err, data) => {
logger.error(err, data);
}
)
const source1 = source.clone();
const source2 = source.clone();

const upload1 = s3
.upload(
{
Bucket: config.storage.aws.bucketName,
Prefix: prefix,
Key: name,
Body: source1,
},
(err, data) => {
logger.error(err, data);
}
)
.on('httpUploadProgress', (progress) => {
logger.info(
`writeStreamFromStream – ${name}`,
`writeStreamFromStreamWithPrefix 1 – ${name}`,
(progress.loaded / progress.total) * 100
);
if (progress.loaded >= progress.total) deferred.resolve();
})
.on('error', deferred.reject);
.on('error', deferred.reject)
.promise();

const upload2 = newS3
.upload(
{
Bucket: config.storage.newAws.bucketName,
Prefix: prefix,
Key: name,
Body: source2,
},
(err, data) => {
logger.error(err, data);
}
)
.on('httpUploadProgress', (progress) => {
logger.info(
`writeStreamFromStreamWithPrefix 2 – ${name}`,
(progress.loaded / progress.total) * 100
);
})
.on('error', deferred.reject)
.promise();

Promise.all([upload1, upload2]).then(() => {
deferred.resolve();
});

return deferred;
};
Expand All @@ -129,11 +225,17 @@ if (!config.isAws) {

// http://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html#copyObject-property
const copyObject = denodeify(s3.copyObject.bind(s3));
const newCopyObject = denodeify(newS3.copyObject.bind(newS3));
const copyImages = (oldPrefix, newPrefix) => {
logger.info('copying images with S3 storage');
return listImages(oldPrefix).then((files) => Promise.all(files.map(copy)));

function copy(file) {
async function copy(file) {
await newCopyObject({
Bucket: config.storage.newAws.bucketName,
CopySource: config.storage.newAws.bucketName + '/' + file.name,
Key: file.name.replace(oldPrefix, newPrefix),
});
return copyObject({
Bucket: config.storage.aws.bucketName,
CopySource: config.storage.aws.bucketName + '/' + file.name,
Expand Down
Loading