feat: Terms data puller

apps/datapuller/src/lib/terms.ts

@@ -91,10 +91,9 @@ export const getTerms = async (
 
     logger.error(`Unexpected error fetching term: "${parsedError}"`);
     logger.error(`Error details: ${JSON.stringify(parsedError, null, 2)}`);
-
-    if (!parsedError.cause) return;
-
     logger.error(`Error cause: ${parsedError.cause}`);
+
+    return terms;
   }
 
   logger.info(`Fetched ${terms.length} initial terms...`);
@@ -122,18 +121,19 @@ export const getTerms = async (
       );
 
       const currentTerms = currentResponse.data.response.terms;
-      if (!currentTerms) continue;
+      if (!currentTerms) throw new Error("No terms found");
 
       terms.push(...currentTerms);
+
+      currentTerm = currentTerms[0];
     } catch (error: unknown) {
       const parsedError = error as Error;
 
       logger.error(`Unexpected error fetching term: "${parsedError}"`);
       logger.error(`Error details: ${JSON.stringify(parsedError, null, 2)}`);
-
-      if (!parsedError.cause) continue;
-
       logger.error(`Error cause: ${parsedError.cause}`);
+
+      break;
     }
   }
 
@@ -171,17 +171,12 @@ export const getPreviousTerms = async (
     const previousTerms = response.data.response.terms;
     if (!previousTerms) throw new Error("No previous terms found");
 
-    logger.info(`Fetched current term...`);
-
     return previousTerms;
   } catch (error: unknown) {
     const parsedError = error as Error;
 
     logger.error(`Unexpected error fetching term: "${parsedError}"`);
     logger.error(`Error details: ${JSON.stringify(parsedError, null, 2)}`);
-
-    if (!parsedError.cause) return;
-
     logger.error(`Error cause: ${parsedError.cause}`);
   }
 };

apps/datapuller/src/main.ts

@@ -4,6 +4,7 @@ import updateEnrollmentHistories from "./pullers/enrollment";
 import updateGradeDistributions from "./pullers/grade-distributions";
 import main from "./pullers/main";
 import updateSections from "./pullers/sections";
+import updateTerms from "./pullers/terms";
 import setup from "./shared";
 import { Config } from "./shared/config";
 
@@ -18,6 +19,7 @@ const pullerMap: { [key: string]: (config: Config) => Promise<void> } = {
   classes: updateClasses,
   grades: updateGradeDistributions,
   enrollments: updateEnrollmentHistories,
+  terms: updateTerms,
   main: main,
 };
 

apps/datapuller/src/pullers/terms.ts

@@ -0,0 +1,56 @@
+import { TermModel } from "@repo/common";
+
+import { getTerms } from "../lib/terms";
+import { Config } from "../shared/config";
+
+const updateTerms = async ({
+  sis: { TERM_APP_ID, TERM_APP_KEY },
+  log,
+}: Config) => {
+  log.info(`Fetching terms.`);
+
+  // Get all courses
+  const terms = await getTerms(log, TERM_APP_ID, TERM_APP_KEY);
+
+  if (terms.length === 0) {
+    log.info("No terms found.");
+
+    return;
+  }
+
+  log.info(`Fetched ${terms.length.toLocaleString()} terms.`);
+
+  log.info("Deleting terms...");
+
+  // Delete existing terms not in SIS
+  const { deletedCount } = await TermModel.deleteMany({
+    id: { $nin: terms.map((term) => term.id) },
+  });
+
+  log.info(`Deleted ${deletedCount.toLocaleString()} existing terms.`);
+
+  // Insert terms in batches of 5000
+  const insertBatchSize = 5000;
+
+  for (let i = 0; i < terms.length; i += insertBatchSize) {
+    const batch = terms.slice(i, i + insertBatchSize);
+
+    log.info(`Inserting batch ${i / insertBatchSize + 1}...`);
+
+    await TermModel.bulkWrite(
+      batch.map((term) => ({
+        updateOne: {
+          filter: { id: term.id },
+          update: { $set: term },
+          upsert: true,
+        },
+      }))
+    );
+  }
+
+  log.info(
+    `Completed updating database with ${terms.length.toLocaleString()} terms.`
+  );
+};
+
+export default updateTerms;