Add retry logic to syllabus scraping and Algolia sync functions

Add exponential backoff and error handling to all major fetch, database,
and Algolia operations. Improves reliability of scraping, syllabus
parsing, PDF download/upload, and Algolia sync/export. Adds utility for
exporting Algolia data to Supabase storage.

Author: ImJustChew

src/config/algolia.ts

@@ -10,10 +10,12 @@ const algolia = (c: Context) => {
   }>(c);
 
   return algoliaWithEnv(ALGOLIA_APP_ID, ALGOLIA_API_KEY);
-}
+};
 
 export const algoliaWithEnv = (appId: string, apiKey: string) => {
-  const client = algoliasearch(appId, apiKey, { requester: createFetchRequester() });
+  const client = algoliasearch(appId, apiKey, {
+    requester: createFetchRequester(),
+  });
   const index = client.initIndex("nthu_courses");
   return index;
 };

src/index.ts

@@ -16,6 +16,7 @@ import {
   scrapeArchivedCourses,
   scrapeSyllabus,
   syncCoursesToAlgolia,
+  exportCoursesToAlgoliaFile,
 } from "./scheduled/syllabus";
 
 export type Bindings = {
@@ -59,7 +60,9 @@ const APIHandler = {
         try {
           const cache = await scrapeArchivedCourses(env, semester);
           await scrapeSyllabus(env, semester, cache);
+          // await scrapeSyllabus(env, semester);
           // await syncCoursesToAlgolia(env, semester);
+          // await exportCoursesToAlgoliaFile(env, semester);
           console.log("Scheduled tasks completed successfully.");
           resolve(void 0);
         } catch (error) {
@@ -70,4 +73,4 @@ const APIHandler = {
   },
 };
 
-export default APIHandler;
+export default APIHandler;

src/issue.ts

@@ -5,7 +5,7 @@ import { z } from "zod";
 import { env } from "hono/adapter";
 
 type GithubEnv = {
-  CLIENT_ID: string;
+  GITHUB_CLIENT_ID: string;
   GITHUB_APP_PRIVATE_KEY: string;
   GITHUB_INSTALLATION_ID: string;
 };
@@ -126,14 +126,17 @@ const app = new Hono()
     async (c) => {
       const { title, body, labels } = c.req.valid("form");
 
-      const { CLIENT_ID, GITHUB_APP_PRIVATE_KEY, GITHUB_INSTALLATION_ID } =
-        env<GithubEnv>(c);
+      const {
+        GITHUB_CLIENT_ID,
+        GITHUB_APP_PRIVATE_KEY,
+        GITHUB_INSTALLATION_ID,
+      } = env<GithubEnv>(c);
 
       // base64 encoded private key to utf8, not using buffer
       const privateKey = atob(GITHUB_APP_PRIVATE_KEY);
 
       const accessToken = await getInstallationAccessToken(
-        CLIENT_ID,
+        GITHUB_CLIENT_ID,
         privateKey,
         GITHUB_INSTALLATION_ID,
       );
@@ -165,12 +168,15 @@ const app = new Hono()
     ),
     async (c) => {
       const { tag } = c.req.valid("query");
-      const { CLIENT_ID, GITHUB_APP_PRIVATE_KEY, GITHUB_INSTALLATION_ID } =
-        env<GithubEnv>(c);
+      const {
+        GITHUB_CLIENT_ID,
+        GITHUB_APP_PRIVATE_KEY,
+        GITHUB_INSTALLATION_ID,
+      } = env<GithubEnv>(c);
       const privateKey = atob(GITHUB_APP_PRIVATE_KEY);
 
       const accessToken = await getInstallationAccessToken(
-        CLIENT_ID,
+        GITHUB_CLIENT_ID,
         privateKey,
         GITHUB_INSTALLATION_ID,
       );