step function code working with private github + hashnode (primary) +…

… devTo

functions/identify-new-content.ts

@@ -1,22 +1,63 @@
 import { Octokit } from "octokit";
-import { SFNClient, StartExecutionCommand } from '@aws-sdk/client-sfn';
+import { SFNClient, StartExecutionCommand } from "@aws-sdk/client-sfn";
 import { getSecret } from "./utils/secrets";
+import { join } from "path";
+import { S3Client, PutObjectCommand } from "@aws-sdk/client-s3";
 
 const sfn = new SFNClient({});
+const s3 = new S3Client({});
+const blogPathDefined = !!(
+  process.env.BLOG_PATH && process.env.BLOG_PATH !== "/"
+);
 let octokit: Octokit;
 
 export const handler = async (event: any) => {
   try {
     await initializeOctokit();
 
-    const recentCommits = await getRecentCommits();
-    if (recentCommits.length) {
-      const newContent = await getNewContent(recentCommits);
-      if (newContent.length) {
-        const data = await getContentData(newContent);
-        await processNewContent(data);
+    let newContent: { fileName: string; commit: string }[] = [];
+    if (event.body) {
+      const body = JSON.parse(event.body);
+      console.log(JSON.stringify({ body }, null, 2));
+      if (body.commits) {
+        newContent = body.commits.reduce(
+          (
+            p: { fileName: string; commit: string }[],
+            commit: {
+              id: string;
+              added: string[];
+              modified: string[];
+              // ... there's more stuff here, but this is all we need
+            }
+          ) => {
+            const addedFiles = commit.added.filter(
+              (addedFile: string) =>
+                (!blogPathDefined ||
+                  addedFile.startsWith(`${process.env.BLOG_PATH}/`)) &&
+                addedFile.endsWith(".md")
+            );
+            return [
+              ...p,
+              ...addedFiles.map((addedFile) => ({
+                fileName: addedFile,
+                commit: commit.id,
+              })),
+            ];
+          },
+          [] as { fileName: string; commit: string }[]
+        );
+      } else {
+        const recentCommits = await getRecentCommits();
+        if (recentCommits.length) {
+          newContent = await getNewContent(recentCommits);
+        }
       }
     }
+    if (newContent.length) {
+      const data = await getContentData(newContent);
+      const imagesProcessed = await saveImagesToS3(data);
+      await processNewContent(imagesProcessed);
+    }
   } catch (err) {
     console.error(err);
   }
@@ -60,10 +101,12 @@ const getNewContent = async (commits: string[]) => {
       ref: commits[j],
     });
 
-    const blogPath = process.env.BLOG_PATH && process.env.BLOG_PATH !== "/";
     const newFiles = commitDetail.data.files?.filter(
       (f) =>
-        f.status == "added" && (!blogPath || f.filename.startsWith(`${process.env.BLOG_PATH}/`))
+        f.status == "added" &&
+        (!blogPathDefined ||
+          f.filename.startsWith(`${process.env.BLOG_PATH}/`)) &&
+        f.filename.endsWith(".md")
     );
     newContent.push(
       ...(newFiles?.map((p) => {
@@ -120,8 +163,72 @@ const saveImagesToS3 = async (
     sendStatusEmail: boolean;
   }[]
 ) => {
-  // TODO: regex for images stored in github and fetch them / store them in a public s3 bucket
-}
+  const contentData: {
+    fileName: string;
+    commit: string;
+    content: string;
+    sendStatusEmail: boolean;
+  }[] = [];
+  const imgRegex = /!\[(.*?)\]\((.*?)\)/g;
+  for (let j = 0; j < newContent.length; j++) {
+    const workingContent = { ...newContent[j] };
+    const imageSet = new Set<string>([]);
+    let match;
+    while ((match = imgRegex.exec(newContent[j].content)) !== null) {
+      imageSet.add(match[2]);
+    }
+    const images = [...imageSet];
+    if (images.length === 0) {
+      // no images in the post... passthrough
+      contentData.push(newContent[j]);
+      continue;
+    }
+    const blogFile = newContent[j].fileName;
+    const blogSplit = `${blogFile}`.split("/");
+    blogSplit.pop();
+    const blogBase = blogSplit.join("/");
+    const s3Mapping: Record<string, string> = {};
+    for (let k = 0; k < images.length; k++) {
+      const image = images[k];
+      const githubPath = join(blogBase, image);
+      const imageSplit = image.split(".");
+      const imageExtension = imageSplit[imageSplit.length - 1];
+      const s3Path = `${blogFile}/${k}.${imageExtension}`.replace(/\ /g, "-");
+      const s3Url = `https://s3.amazonaws.com/${process.env.MEDIA_BUCKET}/${s3Path}`;
+      console.log(
+        JSON.stringify({ image, githubPath, s3Path, s3Url }, null, 2)
+      );
+      const postContent = await octokit.request(
+        "GET /repos/{owner}/{repo}/contents/{path}",
+        {
+          owner: `${process.env.OWNER}`,
+          repo: `${process.env.REPO}`,
+          path: githubPath,
+        }
+      );
+
+      const buffer = Buffer.from((postContent.data as any).content, "base64");
+
+      // upload images to s3
+      const putImage = new PutObjectCommand({
+        Bucket: `${process.env.MEDIA_BUCKET}`,
+        Key: s3Path,
+        Body: buffer,
+      });
+      await s3.send(putImage);
+
+      s3Mapping[image] = s3Url;
+    }
+    const rewriteLink = (match: string, text: string, url: string) => {
+      console.log(JSON.stringify({ match, text, url }));
+      return `![${text}](${s3Mapping[url]})`;
+    }
+    workingContent.content = workingContent.content.replace(imgRegex, rewriteLink);
+    contentData.push(workingContent);
+  }
+  console.log(JSON.stringify({ contentData }));
+  return contentData;
+};
 
 const processNewContent = async (
   newContent: {
@@ -131,16 +238,18 @@ const processNewContent = async (
     sendStatusEmail: boolean;
   }[]
 ) => {
-  const executions = await Promise.allSettled(newContent.map(async (content) => {
-    const command = new StartExecutionCommand({
-      stateMachineArn: process.env.STATE_MACHINE_ARN,
-      input: JSON.stringify(content)
-    });
-    await sfn.send(command);
-  }));
+  const executions = await Promise.allSettled(
+    newContent.map(async (content) => {
+      const command = new StartExecutionCommand({
+        stateMachineArn: process.env.STATE_MACHINE_ARN,
+        input: JSON.stringify(content),
+      });
+      await sfn.send(command);
+    })
+  );
 
   for (const execution of executions) {
-    if (execution.status == 'rejected') {
+    if (execution.status == "rejected") {
       console.error(execution.reason);
     }
   }

functions/parse-dev-post.ts

@@ -65,7 +65,7 @@ const formatDevData = (
       canonical_url: process.env.AMPLIFY_BASE_URL ? `${process.env.AMPLIFY_BASE_URL}/${postDetail.data.slug.replace(
         /^\/|\/$/g,
         ""
-      )}` : ``,
+      )}` : `${process.env.CANONICAL}`,
     }),
     description: postDetail.data.description,
     tags: [

functions/parse-hashnode-post.ts

@@ -71,11 +71,17 @@ const formatHashnodeData = (
         title: postDetail.data.title,
         contentMarkdown: hashnodeContent,
         coverImageURL: postDetail.data.image,
-        isRepublished: {
-          ...(process.env.CANONICAL === "hashnode" ? {} : {
-            originalArticleURL: process.env.AMPLIFY_BASE_URL ? `${process.env.AMPLIFY_BASE_URL}/${postDetail.data.slug.replace(/^\/|\/$/g, "")}` : ``,
-          }),
-        },
+        ...(process.env.CANONICAL === "hashnode"
+          ? {}
+          : {
+              isRepublished: {
+                originalArticleURL: process.env.AMPLIFY_BASE_URL
+                  ? `${
+                      process.env.AMPLIFY_BASE_URL
+                    }/${postDetail.data.slug.replace(/^\/|\/$/g, "")}`
+                  : `${process.env.CANONICAL}`,
+              },
+            }),
         tags: [],
         subtitle: postDetail.data.description,
       },

functions/parse-medium-post.ts

@@ -2,34 +2,69 @@ import { getLinks } from "./utils/getLinks";
 import { getTweets } from "./utils/getTweets";
 import { getTweetUrl } from "./utils/getTweetUrl";
 
-const frontmatter = require('@github-docs/frontmatter');
+const frontmatter = require("@github-docs/frontmatter");
 
-export const handler = async (state: { post: any; format: string; articleCatalog: any; canonical?: string; }) => {
+export const handler = async (state: {
+  post: any;
+  format: string;
+  articleCatalog: any;
+  canonical?: string;
+}) => {
   const details = frontmatter(state.post);
   const links = getLinks(details.content);
   const tweets = getTweets(details.content);
 
-  const payload = formatMediumData(details, state.articleCatalog, links, tweets);
-
+  const payload = formatMediumData(
+    details,
+    state.articleCatalog,
+    links,
+    tweets
+  );
+
   return {
     payload,
-    url: `/${details.data.slug.replace(/^\/|\/$/g, '')}`
+    url: `/${details.data.slug.replace(/^\/|\/$/g, "")}`,
   };
 };
 
-const formatMediumData = (postDetail: { data: { title: any; description: any; image_attribution: any; image: any; categories: any; tags: any; slug: string; }; content: string | any[]; }, articleCatalog: any[], links: any, tweets: any) => {
-  let mediumContent = `\n# ${postDetail.data.title}\n`
-    + `#### ${postDetail.data.description}\n`
-    + `![${postDetail.data.image_attribution ?? ''}](${postDetail.data.image})\n`
-    + `${postDetail.content.slice(0)}`;
+const formatMediumData = (
+  postDetail: {
+    data: {
+      title: any;
+      description: any;
+      image_attribution: any;
+      image: any;
+      categories: any;
+      tags: any;
+      slug: string;
+    };
+    content: string | any[];
+  },
+  articleCatalog: any[],
+  links: any,
+  tweets: any
+) => {
+  let mediumContent =
+    `\n# ${postDetail.data.title}\n` +
+    `#### ${postDetail.data.description}\n` +
+    `![${postDetail.data.image_attribution ?? ""}](${
+      postDetail.data.image
+    })\n` +
+    `${postDetail.content.slice(0)}`;
 
   for (const link of links) {
-    const replacement = articleCatalog.find(c => c.links.M.url.S == link[1]);
+    const replacement = articleCatalog.find((c) => c.links.M.url.S == link[1]);
     if (replacement) {
       if (replacement.links.M.mediumUrl && replacement.links.M.mediumUrl.S) {
-        mediumContent = mediumContent.replace(link[1], replacement.links.M.mediumUrl.S);
+        mediumContent = mediumContent.replace(
+          link[1],
+          replacement.links.M.mediumUrl.S
+        );
       } else {
-        mediumContent = mediumContent.replace(link[1], `${process.env.AMPLIFY_BASE_URL}${replacement.links.M.url.S}`);
+        mediumContent = mediumContent.replace(
+          link[1],
+          `${process.env.AMPLIFY_BASE_URL}${replacement.links.M.url.S}`
+        );
       }
     }
   }
@@ -41,18 +76,22 @@ const formatMediumData = (postDetail: { data: { title: any; description: any; im
 
   const mediumData = {
     title: postDetail.data.title,
-    contentFormat: 'markdown',
+    contentFormat: "markdown",
     tags: [...postDetail.data.categories, ...postDetail.data.tags],
-    ...(process.env.CANONICAL === "medium" ? {} : {
-      canonical_url: process.env.AMPLIFY_BASE_URL ? `${process.env.AMPLIFY_BASE_URL}/${postDetail.data.slug.replace(
-        /^\/|\/$/g,
-        ""
-      )}` : ``,
-    }),
-    publishStatus: 'draft',
+    ...(process.env.CANONICAL === "medium"
+      ? {}
+      : {
+          canonical_url: process.env.AMPLIFY_BASE_URL
+            ? `${process.env.AMPLIFY_BASE_URL}/${postDetail.data.slug.replace(
+                /^\/|\/$/g,
+                ""
+              )}`
+            : `${process.env.CANONICAL}`,
+        }),
+    publishStatus: "draft",
     notifyFollowers: true,
-    content: mediumContent
+    content: mediumContent,
   };
 
   return mediumData;
-};
+};

functions/send-api-request.ts

@@ -20,6 +20,7 @@ export const handler = async (state: { secretKey: any; request: { method: any; b
       }
     };
   } else {
+    console.log(JSON.stringify({ config, state }, null, 2));
     const response = await axios.request(config);
     return response.data;
   }

lib/blog-crossposting-automation-stack.ts

@@ -1,10 +1,11 @@
-import { StackProps, Stack, CfnOutput, Duration } from "aws-cdk-lib";
+import { StackProps, Stack, CfnOutput, Duration, RemovalPolicy } from "aws-cdk-lib";
 import { EventBus, Rule } from "aws-cdk-lib/aws-events";
 import {
   LambdaFunction,
 } from "aws-cdk-lib/aws-events-targets";
 import { Architecture, FunctionUrlAuthType, Runtime } from "aws-cdk-lib/aws-lambda";
 import { NodejsFunction, NodejsFunctionProps } from "aws-cdk-lib/aws-lambda-nodejs";
+import { Bucket } from "aws-cdk-lib/aws-s3";
 import { Secret } from "aws-cdk-lib/aws-secretsmanager";
 import { Construct } from "constructs";
 import { join } from "path";
@@ -19,7 +20,6 @@ export interface BlogCrosspostingAutomationStackProps extends StackProps {
     amplifyProjectId: string;
     blogBaseUrl: string;
   };
-  // TODO: properly handle canonical urls for non-amplify blogs
   canonical: "dev" | "medium" | "hashnode" | "amplify";
   commitTimeToleranceMinutes?: number;
   devTo?: {
@@ -163,6 +163,12 @@ export class BlogCrosspostingAutomationStack extends Stack {
     });
     table.grantWriteData(loadCrossPostsFn);
 
+    const mediaBucket = new Bucket(this, `BlogPostMediaBucket`, {
+      autoDeleteObjects: true,
+      publicReadAccess: true,
+      removalPolicy: RemovalPolicy.DESTROY,
+    });
+
     const identifyNewContentFn = new NodejsFunction(
       this,
       `IdentifyNewContentFn`,
@@ -174,6 +180,8 @@ export class BlogCrosspostingAutomationStack extends Stack {
     identifyNewContentFn.addEnvironment("OWNER", github.owner);
     identifyNewContentFn.addEnvironment("REPO", github.repo);
     identifyNewContentFn.addEnvironment("BLOG_PATH", github.path);
+    identifyNewContentFn.addEnvironment("MEDIA_BUCKET", mediaBucket.bucketName);
+    mediaBucket.grantReadWrite(identifyNewContentFn);
     if (commitTimeToleranceMinutes) {
       identifyNewContentFn.addEnvironment(
         "COMMIT_TIME_TOLERANCE_MINUTES",
@@ -228,6 +236,7 @@ export class BlogCrosspostingAutomationStack extends Stack {
 
     const { stateMachine } = new CrossPostStepFunction(this, `CrossPostStepFn`, crossPostStepFunctionProps);
     stateMachine.grantStartExecution(identifyNewContentFn);
+    identifyNewContentFn.addEnvironment("STATE_MACHINE_ARN", stateMachine.stateMachineArn);
     table.grantReadWriteData(stateMachine);
     eventBus.grantPutEventsTo(stateMachine);
   }