Cleanup Staging files and Copy in Parallel (#164)

* Cleanup Staging files and Copy in Parallel - Fixes #148 - Fixes #102 * Allow Deployer to Delete from Staging Bucket
pwrdrvr · Dec 3, 2021 · c6fadc2 · c6fadc2
1 parent c328480
commit c6fadc2
Show file tree

Hide file tree

Showing 5 changed files with 70 additions and 16 deletions.
diff --git a/package-lock.json b/package-lock.json
diff --git a/packages/microapps-cdk/src/MicroAppsSvcs.ts b/packages/microapps-cdk/src/MicroAppsSvcs.ts
@@ -422,7 +422,7 @@ export class MicroAppsSvcs extends cdk.Construct implements IMicroAppsSvcsExport
     const policyReadListStaging = new iam.PolicyStatement({
       effect: iam.Effect.ALLOW,
       // FIXME: Allow Deployer to delete from Staging bucket
-      actions: ['s3:GetObject', 's3:ListBucket'],
+      actions: ['s3:DeleteObject', 's3:GetObject', 's3:ListBucket'],
       resources: [`${bucketAppsStaging.bucketArn}/*`, bucketAppsStaging.bucketArn],
     });
     deployerFunc.addToRolePolicy(policyReadListStaging);

diff --git a/packages/microapps-deployer/package.json b/packages/microapps-deployer/package.json
@@ -33,6 +33,7 @@
     "fs-extra": "^9.1.0",
     "js-yaml": "^4.1.0",
     "lambda-log": "^2.4.0",
+    "p-map": "^4.0.0",
     "reflect-metadata": "^0.1.13",
     "source-map-support": "^0.5.19",
     "ts-convict": "^1.1.0",

diff --git a/packages/microapps-deployer/src/controllers/VersionController.spec.ts b/packages/microapps-deployer/src/controllers/VersionController.spec.ts
@@ -182,6 +182,11 @@ describe('VersionController', () => {
           CopySource: `${config.filestore.stagingBucket}/${appName}/${semVer}/index.html`,
           Key: `${appName}/${semVer}/index.html`,
         })
+        .resolves({})
+        .on(s3.DeleteObjectCommand, {
+          Bucket: config.filestore.stagingBucket,
+          Key: `${appName}/${semVer}/index.html`,
+        })
         .resolves({});
 
       lambdaClient
@@ -287,6 +292,11 @@ describe('VersionController', () => {
           CopySource: `${config.filestore.stagingBucket}/${appName}/${semVer}/index.html`,
           Key: `${appName}/${semVer}/index.html`,
         })
+        .resolves({})
+        .on(s3.DeleteObjectCommand, {
+          Bucket: config.filestore.stagingBucket,
+          Key: `${appName}/${semVer}/index.html`,
+        })
         .resolves({});
       lambdaClient
         .onAnyCommand()
@@ -403,6 +413,11 @@ describe('VersionController', () => {
           CopySource: `${config.filestore.stagingBucket}/${appName}/${semVer}/index.html`,
           Key: `${appName}/${semVer}/index.html`,
         })
+        .resolves({})
+        .on(s3.DeleteObjectCommand, {
+          Bucket: config.filestore.stagingBucket,
+          Key: `${appName}/${semVer}/index.html`,
+        })
         .resolves({});
 
       lambdaClient
@@ -525,6 +540,11 @@ describe('VersionController', () => {
           CopySource: `${config.filestore.stagingBucket}/${appName}/${semVer}/index.html`,
           Key: `${appName}/${semVer}/index.html`,
         })
+        .resolves({})
+        .on(s3.DeleteObjectCommand, {
+          Bucket: config.filestore.stagingBucket,
+          Key: `${appName}/${semVer}/index.html`,
+        })
         .resolves({});
 
       lambdaClient

diff --git a/packages/microapps-deployer/src/controllers/VersionController.ts b/packages/microapps-deployer/src/controllers/VersionController.ts
@@ -5,6 +5,7 @@ import * as lambda from '@aws-sdk/client-lambda';
 import * as s3 from '@aws-sdk/client-s3';
 import * as sts from '@aws-sdk/client-sts';
 import { DBManager, Rules, Version } from '@pwrdrvr/microapps-datalib';
+import pMap from 'p-map';
 import { IConfig } from '../config/Config';
 import {
   IDeployVersionRequest,
@@ -390,6 +391,15 @@ export default class VersionController {
     return crypto.createHash('sha1').update(input).digest('hex');
   }
 
+  /**
+   * Copy a list of files from the Staging bucket to the Destination bucket
+   * @param list
+   * @param stagingBucket
+   * @param sourcePrefix
+   * @param destinationPrefix
+   * @param config
+   * @returns
+   */
   private static async CopyFilesInList(
     list: s3.ListObjectsV2CommandOutput,
     stagingBucket: string,
@@ -400,28 +410,49 @@ export default class VersionController {
     if (list === undefined || list.Contents === undefined) {
       return;
     }
-    for (const obj of list.Contents) {
-      const sourceKeyRootless = obj.Key?.slice(sourcePrefix.length);
-
-      // FIXME: Use p-map to parallelize with limit
-      await s3Client.send(
-        new s3.CopyObjectCommand({
-          // Source
-          CopySource: `${stagingBucket}/${obj.Key}`,
-          // Destination
-          Bucket: config.filestore.destinationBucket,
-          Key: `${destinationPrefix}/${sourceKeyRootless}`,
-        }),
-      );
-    }
+
+    await pMap(
+      list.Contents,
+      async (obj) => {
+        const sourceKeyRootless = obj.Key?.slice(sourcePrefix.length);
+
+        // Copy from the Staging bucket to the Destination bucket
+        await s3Client.send(
+          new s3.CopyObjectCommand({
+            // Source
+            CopySource: `${stagingBucket}/${obj.Key}`,
+            // Destination
+            Bucket: config.filestore.destinationBucket,
+            Key: `${destinationPrefix}/${sourceKeyRootless}`,
+          }),
+        );
+
+        // Remove the file from the Staging bucket
+        await s3Client.send(
+          new s3.DeleteObjectCommand({
+            Bucket: stagingBucket,
+            Key: obj.Key,
+          }),
+        );
+      },
+      { concurrency: 4 },
+    );
   }
 
+  /**
+   * List files in the Staging bucket and each chunk of the list
+   * to the Destination bucket
+   * @param stagingBucket
+   * @param sourcePrefix
+   * @param destinationPrefix
+   * @param config
+   */
   private static async CopyToProdBucket(
     stagingBucket: string,
     sourcePrefix: string,
     destinationPrefix: string,
     config: IConfig,
-  ) {
+  ): Promise<void> {
     let list: s3.ListObjectsV2CommandOutput | undefined;
     do {
       const optionals =