untested canonical improvements, still need to parse out images

aws-community-projects · Feb 17, 2023 · 6e105a9 · 6e105a9
1 parent f3b63e6
commit 6e105a9
Show file tree

Hide file tree

Showing 7 changed files with 80 additions and 49 deletions.
diff --git a/functions/identify-new-content.ts b/functions/identify-new-content.ts
@@ -1,12 +1,8 @@
 import { Octokit } from "octokit";
-import {
-  EventBridgeClient,
-  PutEventsCommand,
-} from "@aws-sdk/client-eventbridge";
+import { SFNClient, StartExecutionCommand } from '@aws-sdk/client-sfn';
 import { getSecret } from "./utils/secrets";
 
-const eb = new EventBridgeClient({});
-
+const sfn = new SFNClient({});
 let octokit: Octokit;
 
 export const handler = async (event: any) => {
@@ -135,14 +131,17 @@ const processNewContent = async (
     sendStatusEmail: boolean;
   }[]
 ) => {
-  const Entries = newContent.map((content) => ({
-    Source: `cross-post`,
-    DetailType: "process-new-content",
-    Detail: JSON.stringify(content),
+  const executions = await Promise.allSettled(newContent.map(async (content) => {
+    const command = new StartExecutionCommand({
+      stateMachineArn: process.env.STATE_MACHINE_ARN,
+      input: JSON.stringify(content)
+    });
+    await sfn.send(command);
   }));
 
-  const putEventsCommand = new PutEventsCommand({
-    Entries,
-  });
-  await eb.send(putEventsCommand);
+  for (const execution of executions) {
+    if (execution.status == 'rejected') {
+      console.error(execution.reason);
+    }
+  }
 };
diff --git a/functions/parse-dev-post.ts b/functions/parse-dev-post.ts
@@ -8,6 +8,7 @@ export const handler = async (state: {
   post: any;
   format: string;
   articleCatalog: any;
+  canonical?: string;
 }) => {
   const details = frontmatter(state.post);
   const links = getLinks(details.content);
@@ -45,7 +46,7 @@ const formatDevData = (
       } else {
         devContent = devContent.replace(
           link[1],
-          `${process.env.BLOG_BASE_URL}${replacement.links.M.url.S}`
+          `${process.env.AMPLIFY_BASE_URL}${replacement.links.M.url.S}`
         );
       }
     }
@@ -60,10 +61,12 @@ const formatDevData = (
     title: postDetail.data.title,
     published: true,
     main_image: postDetail.data.image,
-    canonical_url: `${process.env.BLOG_BASE_URL}/${postDetail.data.slug.replace(
-      /^\/|\/$/g,
-      ""
-    )}`,
+    ...(process.env.CANONICAL === "dev" ? {} : {
+      canonical_url: process.env.AMPLIFY_BASE_URL ? `${process.env.AMPLIFY_BASE_URL}/${postDetail.data.slug.replace(
+        /^\/|\/$/g,
+        ""
+      )}` : ``,
+    }),
     description: postDetail.data.description,
     tags: [
       ...postDetail.data.categories.map((c) => c.replace(/ /g, "")),

diff --git a/functions/parse-hashnode-post.ts b/functions/parse-hashnode-post.ts
@@ -8,6 +8,7 @@ export const handler = async (state: {
   post: any;
   format: string;
   articleCatalog: any;
+  canonical?: string;
 }) => {
   const details = frontmatter(state.post);
   const links = getLinks(details.content);
@@ -50,7 +51,7 @@ const formatHashnodeData = (
       } else {
         hashnodeContent = hashnodeContent.replace(
           link[1],
-          `${process.env.BLOG_BASE_URL}${replacement.links.M.url.S}`
+          `${process.env.AMPLIFY_BASE_URL}${replacement.links.M.url.S}`
         );
       }
     }
@@ -71,9 +72,9 @@ const formatHashnodeData = (
         contentMarkdown: hashnodeContent,
         coverImageURL: postDetail.data.image,
         isRepublished: {
-          originalArticleURL: `${
-            process.env.BLOG_BASE_URL
-          }/${postDetail.data.slug.replace(/^\/|\/$/g, "")}`,
+          ...(process.env.CANONICAL === "hashnode" ? {} : {
+            originalArticleURL: process.env.AMPLIFY_BASE_URL ? `${process.env.AMPLIFY_BASE_URL}/${postDetail.data.slug.replace(/^\/|\/$/g, "")}` : ``,
+          }),
         },
         tags: [],
         subtitle: postDetail.data.description,

diff --git a/functions/parse-medium-post.ts b/functions/parse-medium-post.ts
@@ -4,7 +4,7 @@ import { getTweetUrl } from "./utils/getTweetUrl";
 
 const frontmatter = require('@github-docs/frontmatter');
 
-export const handler = async (state: { post: any; format: string; articleCatalog: any; }) => {
+export const handler = async (state: { post: any; format: string; articleCatalog: any; canonical?: string; }) => {
   const details = frontmatter(state.post);
   const links = getLinks(details.content);
   const tweets = getTweets(details.content);
@@ -29,7 +29,7 @@ const formatMediumData = (postDetail: { data: { title: any; description: any; im
       if (replacement.links.M.mediumUrl && replacement.links.M.mediumUrl.S) {
         mediumContent = mediumContent.replace(link[1], replacement.links.M.mediumUrl.S);
       } else {
-        mediumContent = mediumContent.replace(link[1], `${process.env.BLOG_BASE_URL}${replacement.links.M.url.S}`);
+        mediumContent = mediumContent.replace(link[1], `${process.env.AMPLIFY_BASE_URL}${replacement.links.M.url.S}`);
       }
     }
   }
@@ -43,7 +43,12 @@ const formatMediumData = (postDetail: { data: { title: any; description: any; im
     title: postDetail.data.title,
     contentFormat: 'markdown',
     tags: [...postDetail.data.categories, ...postDetail.data.tags],
-    canonicalUrl: `${process.env.BLOG_BASE_URL}/${postDetail.data.slug.replace(/^\/|\/$/g, '')}`,
+    ...(process.env.CANONICAL === "medium" ? {} : {
+      canonical_url: process.env.AMPLIFY_BASE_URL ? `${process.env.AMPLIFY_BASE_URL}/${postDetail.data.slug.replace(
+        /^\/|\/$/g,
+        ""
+      )}` : ``,
+    }),
     publishStatus: 'draft',
     notifyFollowers: true,
     content: mediumContent

diff --git a/lib/blog-crossposting-automation-stack.ts b/lib/blog-crossposting-automation-stack.ts
@@ -2,10 +2,9 @@ import { StackProps, Stack, CfnOutput, Duration } from "aws-cdk-lib";
 import { EventBus, Rule } from "aws-cdk-lib/aws-events";
 import {
   LambdaFunction,
-  SfnStateMachine,
 } from "aws-cdk-lib/aws-events-targets";
 import { Architecture, FunctionUrlAuthType, Runtime } from "aws-cdk-lib/aws-lambda";
-import { NodejsFunction } from "aws-cdk-lib/aws-lambda-nodejs";
+import { NodejsFunction, NodejsFunctionProps } from "aws-cdk-lib/aws-lambda-nodejs";
 import { Secret } from "aws-cdk-lib/aws-secretsmanager";
 import { Construct } from "constructs";
 import { join } from "path";
@@ -78,9 +77,9 @@ export class BlogCrosspostingAutomationStack extends Stack {
       secretName
     );
 
-    const lambdaProps = {
+    const lambdaProps: NodejsFunctionProps = {
       architecture: Architecture.ARM_64,
-      memory: 1024,
+      memorySize: 1024,
       timeout: Duration.minutes(5),
       runtime: Runtime.NODEJS_18_X,
       environment: {
@@ -105,6 +104,7 @@ export class BlogCrosspostingAutomationStack extends Stack {
       eventBus,
       sendApiRequestFn,
       table,
+      canonical,
     };
     if (devTo?.devOrganizationId) {
       const parseDevFn = new NodejsFunction(this, `ParseDevToFn`, {
@@ -114,7 +114,7 @@ export class BlogCrosspostingAutomationStack extends Stack {
       parseDevFn.addEnvironment("CANONICAL", canonical);
       parseDevFn.addEnvironment("DEV_ORG_ID", devTo.devOrganizationId);
       if (amplify) {
-        parseDevFn.addEnvironment("BLOG_BASE_URL", amplify.blogBaseUrl);
+        parseDevFn.addEnvironment("AMPLIFY_BASE_URL", amplify.blogBaseUrl);
       }
       crossPostStepFunctionProps.devTo = {
         fn: parseDevFn!,
@@ -127,7 +127,7 @@ export class BlogCrosspostingAutomationStack extends Stack {
       });
       parseHashnodeFn.addEnvironment("CANONICAL", canonical);
       if (amplify) {
-        parseHashnodeFn.addEnvironment("BLOG_BASE_URL", amplify.blogBaseUrl);
+        parseHashnodeFn.addEnvironment("AMPLIFY_BASE_URL", amplify.blogBaseUrl);
       }
       if (hashnode.hashnodePublicationId) {
         parseHashnodeFn.addEnvironment(
@@ -147,7 +147,7 @@ export class BlogCrosspostingAutomationStack extends Stack {
       });
       parseMediumFn.addEnvironment("CANONICAL", canonical);
       if (amplify) {
-        parseMediumFn.addEnvironment("BLOG_BASE_URL", amplify.blogBaseUrl);
+        parseMediumFn.addEnvironment("AMPLIFY_BASE_URL", amplify.blogBaseUrl);
       }
       crossPostStepFunctionProps.medium = {
         fn: parseMediumFn!,
@@ -187,7 +187,6 @@ export class BlogCrosspostingAutomationStack extends Stack {
       );
     }
     secret.grantRead(identifyNewContentFn);
-    eventBus.grantPutEventsTo(identifyNewContentFn);
 
     if (amplify?.amplifyProjectId) {
       new Rule(this, `NewArticlesRule`, {
@@ -228,16 +227,8 @@ export class BlogCrosspostingAutomationStack extends Stack {
     }
 
     const { stateMachine } = new CrossPostStepFunction(this, `CrossPostStepFn`, crossPostStepFunctionProps);
+    stateMachine.grantStartExecution(identifyNewContentFn);
     table.grantReadWriteData(stateMachine);
     eventBus.grantPutEventsTo(stateMachine);
-
-    new Rule(this, "CrossPostMachineRule", {
-      eventBus,
-      eventPattern: {
-        source: [`cross-post`],
-        detailType: ["process-new-content"],
-      },
-      targets: [new SfnStateMachine(stateMachine, {})],
-    });
   }
 }
diff --git a/lib/step-function-branch.ts b/lib/step-function-branch.ts
@@ -19,6 +19,7 @@ import { Construct } from "constructs";
 
 export interface StepFunctionBranchProps {
   hashnodeBlogUrl?: string;
+  includeCanonical: boolean;
   parsePostFn: NodejsFunction;
   publishPayload: TaskInput;
   sendApiRequestFn: NodejsFunction;
@@ -34,6 +35,7 @@ export class StepFunctionBranch extends StateMachineFragment {
       super(scope, id);
       const {
         hashnodeBlogUrl,
+        includeCanonical,
         parsePostFn,
         publishPayload,
         sendApiRequestFn,
@@ -61,6 +63,9 @@ export class StepFunctionBranch extends StateMachineFragment {
       const transform = new LambdaInvoke(this, `Transform`, {
         lambdaFunction: parsePostFn,
         payload: TaskInput.fromObject({
+          ...(includeCanonical ? {
+            "canonical.$": `$.canonical.${format}Url`,
+          } : {}),
           "post.$": "$.content",
           "articleCatalog.$": "$.catalog.Items",
           format,

diff --git a/lib/step-function.ts b/lib/step-function.ts
@@ -26,6 +26,7 @@ import { Duration } from "aws-cdk-lib";
 
 export interface CrossPostStepFunctionProps {
   adminEmail?: string;
+  canonical: "dev" | "medium" | "hashnode" | "amplify";
   devTo?: {
     fn: NodejsFunction;
   };
@@ -48,6 +49,7 @@ export class CrossPostStepFunction extends Construct {
 
     const {
       adminEmail,
+      canonical,
       devTo,
       eventBus,
       hashnode,
@@ -156,10 +158,21 @@ export class CrossPostStepFunction extends Construct {
     loadArticleCatalog.addCatch(updateArticleRecordFailure);
 
     // PARALLEL
-    const parallel = new Parallel(this, "TransformAndPublish");
-    loadArticleCatalog.next(parallel);
+    const transformAndPublish = new Parallel(this, "TransformAndPublish");
+    let transformAndPublishCanonical = transformAndPublish;
+    if (canonical !== "amplify") {
+      transformAndPublishCanonical = new Parallel(this, "TransformAndPublishCanonical", {
+        resultPath: "$.canonical"
+      });
+      loadArticleCatalog.next(transformAndPublishCanonical);
+      transformAndPublishCanonical.next(transformAndPublish);
+    } else {
+      loadArticleCatalog.next(transformAndPublish);
+    }
+
     if (devTo) {
       const devToBranch = new StepFunctionBranch(this, `Dev`, {
+        includeCanonical: !(canonical === "dev" || canonical === "amplify"),
         parsePostFn: devTo.fn,
         publishPayload: TaskInput.fromObject({
           secretKey: "dev",
@@ -179,10 +192,15 @@ export class CrossPostStepFunction extends Construct {
         sendApiRequestFn,
         table,
       });
-      parallel.branch(devToBranch.prefixStates());
+      if (canonical === "dev") {
+        transformAndPublishCanonical.branch(devToBranch.prefixStates());
+      } else {
+        transformAndPublish.branch(devToBranch.prefixStates());
+      }
     }
     if (medium) {
       const mediumBranch = new StepFunctionBranch(this, `Medium`, {
+        includeCanonical: !(canonical === "medium" || canonical === "amplify"),
         parsePostFn: medium.fn,
         publishPayload: TaskInput.fromObject({
           secretKey: "medium",
@@ -199,11 +217,16 @@ export class CrossPostStepFunction extends Construct {
         sendApiRequestFn,
         table,
       });
-      parallel.branch(mediumBranch.prefixStates());
+      if (canonical === "medium") {
+        transformAndPublishCanonical.branch(mediumBranch.prefixStates());
+      } else {
+        transformAndPublish.branch(mediumBranch.prefixStates());
+      }
     }
 
     if (hashnode) {
       const hashnodeBranch = new StepFunctionBranch(this, `Hashnode`, {
+        includeCanonical: !(canonical === "hashnode" || canonical === "amplify"),
         hashnodeBlogUrl: hashnode.url,
         parsePostFn: hashnode.fn,
         publishPayload: TaskInput.fromObject({
@@ -224,7 +247,11 @@ export class CrossPostStepFunction extends Construct {
         sendApiRequestFn,
         table,
       });
-      parallel.branch(hashnodeBranch.prefixStates());
+      if (canonical === "hashnode") {
+        transformAndPublishCanonical.branch(hashnodeBranch.prefixStates());
+      } else {
+        transformAndPublish.branch(hashnodeBranch.prefixStates());
+      }
     }
 
     const formatFailureCheck = new Pass(this, "FormatFailureCheck", {
@@ -235,7 +262,7 @@ export class CrossPostStepFunction extends Construct {
         },
       },
     });
-    parallel.next(formatFailureCheck);
+    transformAndPublish.next(formatFailureCheck);
     const checkForFailures = new Pass(this, `CheckForFailures`, {
       parameters: {
         "results.$": "$.results",