enrollment datapuller (#770)

* enrollment datapuller

* fix EnrollmentSingular duplicate checking logic

* renmae file, add assumption comment

* add infra

* fix enrollment cronjob schedule

* fix enrollment cronjob schedule attmpt 2

* fix typo

.github/workflows/cd-dev.yaml

@@ -72,6 +72,10 @@ jobs:
             suspend: true
             image:
               tag: '${{ needs.compute-sha.outputs.sha_short }}'
+          enrollments:
+            suspend: true
+            image:
+              tag: '${{ needs.compute-sha.outputs.sha_short }}'
         host: ${{ needs.compute-sha.outputs.sha_short }}.dev.stanfurdtime.com
         mongoUri: mongodb://bt-dev-mongo-mongodb-0.bt-dev-mongo-mongodb-headless.bt.svc.cluster.local:27017/bt
         redisUri: redis://bt-dev-redis-master.bt.svc.cluster.local:6379

.github/workflows/cd-stage.yaml

@@ -45,6 +45,9 @@ jobs:
           grades:
             image:
               tag: latest
+          enrollments:
+            image:
+              tag: latest
         host: staging.stanfurdtime.com
         mongoUri: mongodb://bt-stage-mongo-mongodb-0.bt-stage-mongo-mongodb-headless.bt.svc.cluster.local:27017/bt
         redisUri: redis://bt-stage-redis-master.bt.svc.cluster.local:6379

apps/datapuller/src/lib/enrollment.ts

@@ -0,0 +1,91 @@
+import { Logger } from "tslog";
+
+import { IEnrollmentSingularItem } from "@repo/common";
+import { ClassSection, ClassesAPI } from "@repo/sis-api/classes";
+
+import { fetchPaginatedData } from "./api/sis-api";
+import { filterSection } from "./sections";
+
+export const formatEnrollmentSingular = (input: ClassSection, time: Date) => {
+  const termId = input.class?.session?.term?.id;
+  const sessionId = input.class?.session?.id;
+  const sectionId = input.id?.toString();
+
+  const essentialFields = {
+    termId,
+    sessionId,
+    sectionId,
+  };
+
+  const missingField = Object.keys(essentialFields).find(
+    (key) => !essentialFields[key as keyof typeof essentialFields]
+  );
+
+  if (missingField)
+    throw new Error(`Missing essential section field: ${missingField[0]}`);
+
+  const output: IEnrollmentSingularItem = {
+    termId: termId!,
+    sessionId: sessionId!,
+    sectionId: sectionId!,
+    data: {
+      time: time.toISOString(),
+      status: input.enrollmentStatus?.status?.code,
+      enrolledCount: input.enrollmentStatus?.enrolledCount,
+      reservedCount: input.enrollmentStatus?.reservedCount,
+      waitlistedCount: input.enrollmentStatus?.waitlistedCount,
+      minEnroll: input.enrollmentStatus?.minEnroll,
+      maxEnroll: input.enrollmentStatus?.maxEnroll,
+      maxWaitlist: input.enrollmentStatus?.maxWaitlist,
+      openReserved: input.enrollmentStatus?.openReserved,
+      instructorAddConsentRequired:
+        input.enrollmentStatus?.instructorAddConsentRequired,
+      instructorDropConsentRequired:
+        input.enrollmentStatus?.instructorDropConsentRequired,
+      seatReservations: input.enrollmentStatus?.seatReservations?.map(
+        (reservation) => ({
+          number: reservation.number,
+          maxEnroll: reservation.maxEnroll,
+          enrolledCount: reservation.enrolledCount,
+        })
+      ),
+    },
+    seatReservations: input.enrollmentStatus?.seatReservations?.map(
+      (reservation) => ({
+        number: reservation.number,
+        requirementGroup: reservation.requirementGroup?.description,
+        fromDate: reservation.fromDate,
+      })
+    ),
+  };
+
+  return output;
+};
+
+export const getEnrollmentSingulars = async (
+  logger: Logger<unknown>,
+  id: string,
+  key: string,
+  termIds?: string[]
+) => {
+  const classesAPI = new ClassesAPI();
+
+  const sections = await fetchPaginatedData<
+    IEnrollmentSingularItem,
+    ClassSection
+  >(
+    logger,
+    classesAPI.v1,
+    termIds || null,
+    "getClassSectionsUsingGet",
+    {
+      app_id: id,
+      app_key: key,
+    },
+    (data) => data.apiResponse.response.classSections || [],
+    filterSection,
+    (input) => formatEnrollmentSingular(input, new Date())
+  );
+
+  return sections;
+};

apps/datapuller/src/lib/sections.ts

@@ -5,7 +5,7 @@ import { ClassSection, ClassesAPI } from "@repo/sis-api/classes";
 
 import { fetchPaginatedData } from "./api/sis-api";
 
-const filterSection = (input: ClassSection): boolean => {
+export const filterSection = (input: ClassSection): boolean => {
   return input.status?.code === "A";
 };
 

apps/datapuller/src/main.ts

@@ -1,5 +1,6 @@
 import updateClasses from "./pullers/classes";
 import updateCourses from "./pullers/courses";
+import updateEnrollmentHistories from "./pullers/enrollment";
 import updateGradeDistributions from "./pullers/grade-distributions";
 import main from "./pullers/main";
 import updateSections from "./pullers/sections";
@@ -15,7 +16,8 @@ const pullerMap: { [key: string]: (config: Config) => Promise<void> } = {
   courses: updateCourses,
   sections: updateSections,
   classes: updateClasses,
-  "grade-distributions": updateGradeDistributions,
+  grades: updateGradeDistributions,
+  enrollments: updateEnrollmentHistories,
   main: main,
 };
 

apps/datapuller/src/pullers/classes.ts

@@ -16,7 +16,7 @@ const updateClasses = async ({
   );
 
   log.info(
-    `Fetched ${activeTerms.length.toLocaleString()} active terms: ${activeTerms.map((term) => term.name).toLocaleString()}.`
+    `Fetched ${activeTerms.length.toLocaleString()} undergraduate active terms: ${activeTerms.map((term) => term.name).toLocaleString()}.`
   );
 
   log.info(`Fetching classes for active terms`);

apps/datapuller/src/pullers/enrollment.ts

@@ -0,0 +1,135 @@
+import {
+  IEnrollmentSingularItem,
+  NewEnrollmentHistoryModel,
+} from "@repo/common";
+
+import { getEnrollmentSingulars } from "../lib/enrollment";
+import { getActiveTerms } from "../lib/terms";
+import { Config } from "../shared/config";
+
+// enrollmentSingulars are equivalent if their data points are all equal
+const enrollmentSingularsEqual = (
+  a: IEnrollmentSingularItem["data"],
+  b: IEnrollmentSingularItem["data"]
+) => {
+  const conditions = [
+    a.status === b.status,
+    a.enrolledCount === b.enrolledCount,
+    a.reservedCount === b.reservedCount,
+    a.waitlistedCount === b.waitlistedCount,
+    a.minEnroll === b.minEnroll,
+    a.maxEnroll === b.maxEnroll,
+    a.maxWaitlist === b.maxWaitlist,
+    a.openReserved === b.openReserved,
+    a.instructorAddConsentRequired === b.instructorAddConsentRequired,
+    a.instructorDropConsentRequired === b.instructorDropConsentRequired,
+  ] as const;
+  if (!conditions.every((condition) => condition)) {
+    return false;
+  }
+
+  const aSeatReservationsEmpty =
+    a.seatReservations == undefined || a.seatReservations.length == 0;
+  const bSeatReservationsEmpty =
+    b.seatReservations == undefined || b.seatReservations.length == 0;
+  if (aSeatReservationsEmpty != bSeatReservationsEmpty) {
+    return false;
+  }
+
+  if (a.seatReservations && b.seatReservations) {
+    if (a.seatReservations.length !== b.seatReservations.length) return false;
+    for (const aSeats of a.seatReservations) {
+      const bSeats = b.seatReservations.find(
+        (bSeats) => bSeats.number === aSeats.number
+      );
+      if (
+        !bSeats ||
+        aSeats.enrolledCount !== bSeats.enrolledCount ||
+        aSeats.maxEnroll !== bSeats.maxEnroll
+      ) {
+        return false;
+      }
+    }
+  }
+
+  return true;
+};
+
+const updateEnrollmentHistories = async ({
+  log,
+  sis: { TERM_APP_ID, TERM_APP_KEY, CLASS_APP_ID, CLASS_APP_KEY },
+}: Config) => {
+  log.info(`Fetching active terms.`);
+
+  const allActiveTerms = await getActiveTerms(log, TERM_APP_ID, TERM_APP_KEY); // includes LAW, Graduate, etc. which are duplicates of Undergraduate
+  const activeTerms = allActiveTerms.filter(
+    (term) => term.academicCareer?.description === "Undergraduate"
+  );
+
+  log.info(
+    `Fetched ${activeTerms.length.toLocaleString()} undergraduate active terms: ${activeTerms.map((term) => term.name).toLocaleString()}.`
+  );
+
+  log.info(`Fetching enrollment for active terms.`);
+
+  const enrollmentSingulars = await getEnrollmentSingulars(
+    log,
+    CLASS_APP_ID,
+    CLASS_APP_KEY,
+    activeTerms.map((term) => term.id as string)
+  );
+
+  log.info(
+    `Fetched ${enrollmentSingulars.length.toLocaleString()} enrollments for active terms.`
+  );
+
+  let updateCount = 0;
+  for (const enrollmentSingular of enrollmentSingulars) {
+    const session = await NewEnrollmentHistoryModel.startSession();
+
+    await session.withTransaction(async () => {
+      // find existing history
+      const doc = await NewEnrollmentHistoryModel.findOne(
+        {
+          termId: enrollmentSingular.termId,
+          sessionId: enrollmentSingular.sessionId,
+          sectionId: enrollmentSingular.sectionId,
+        },
+        null,
+        { session }
+      ).lean();
+
+      // skip if no change
+      if (doc && doc.history.length > 0) {
+        const lastHistory = doc.history[doc.history.length - 1];
+        if (enrollmentSingularsEqual(lastHistory, enrollmentSingular.data)) {
+          return;
+        }
+      }
+
+      // append to history array, upsert if needed
+      const op = await NewEnrollmentHistoryModel.updateOne(
+        {
+          termId: enrollmentSingular.termId,
+          sessionId: enrollmentSingular.sessionId,
+          sectionId: enrollmentSingular.sectionId,
+        },
+        {
+          $push: {
+            history: enrollmentSingular.data,
+          },
+        },
+        { upsert: true, session }
+      );
+      updateCount += op.modifiedCount + op.upsertedCount;
+    });
+
+    session.endSession();
+  }
+
+  log.info(
+    `Completed updating database with ${enrollmentSingulars.length.toLocaleString()} enrollments, modified ${updateCount.toLocaleString()} documents for ${activeTerms.length.toLocaleString()} active terms.`
+  );
+};
+
+export default updateEnrollmentHistories;

apps/datapuller/src/pullers/sections.ts

@@ -16,7 +16,7 @@ const updateSections = async ({
   );
 
   log.info(
-    `Fetched ${activeTerms.length.toLocaleString()} active terms: ${activeTerms.map((term) => term.name).toLocaleString()}.`
+    `Fetched ${activeTerms.length.toLocaleString()} undergraduate active terms: ${activeTerms.map((term) => term.name).toLocaleString()}.`
   );
 
   log.info(`Fetching sections for active terms.`);

infra/app/values.yaml

@@ -60,7 +60,15 @@ datapuller:
   grades:
     schedule: "25 4 * * *"
     suspend: false
-    puller: "grade-distributions"
+    puller: "grades"
+    image:
+      registry: docker.io
+      repository: octoberkeleytime/bt-datapuller
+      tag: prod
+  enrollments:
+    schedule: "0/15 * * * *"
+    suspend: false
+    puller: "enrollments"
     image:
       registry: docker.io
       repository: octoberkeleytime/bt-datapuller

packages/common/src/models/enrollment-history.ts

@@ -0,0 +1,87 @@
+import { Document, Model, Schema, model } from "mongoose";
+
+export interface IEnrollmentHistoryItem {
+  termId: string;
+  sessionId: string;
+  sectionId: string;
+
+  // maps number to requirementGroup.
+  // this assumes that these fields are constant over time.
+  seatReservations?: {
+    number: number;
+    requirementGroup?: string;
+    fromDate: string;
+  }[];
+  history: {
+    time: string;
+    status?: string;
+    enrolledCount?: number;
+    reservedCount?: number;
+    waitlistedCount?: number;
+    minEnroll?: number;
+    maxEnroll?: number;
+    maxWaitlist?: number;
+    openReserved?: number;
+    instructorAddConsentRequired?: boolean;
+    instructorDropConsentRequired?: boolean;
+    seatReservations?: {
+      number: number; // maps to seatReservations.number to get requirementGroup
+      maxEnroll: number;
+      enrolledCount?: number;
+    }[];
+  }[];
+}
+
+export interface IEnrollmentSingularItem
+  extends Omit<IEnrollmentHistoryItem, "history"> {
+  data: IEnrollmentHistoryItem["history"][0];
+}
+
+export interface IEnrollmentHistoryItemDocument
+  extends IEnrollmentHistoryItem,
+    Document {}
+
+const enrollmentHistorySchema = new Schema<IEnrollmentHistoryItem>({
+  termId: { type: String, required: true },
+  sessionId: { type: String, required: true },
+  sectionId: { type: String, required: true },
+  history: [
+    {
+      _id: false,
+      time: { type: String, required: true },
+      status: { type: String },
+      enrolledCount: { type: Number },
+      reservedCount: { type: Number },
+      waitlistedCount: { type: Number },
+      minEnroll: { type: Number },
+      maxEnroll: { type: Number },
+      maxWaitlist: { type: Number },
+      openReserved: { type: Number },
+      instructorAddConsentRequired: { type: Boolean },
+      instructorDropConsentRequired: { type: Boolean },
+      seatReservations: [
+        {
+          _id: false,
+          number: { type: Number },
+          maxEnroll: { type: Number },
+          enrolledCount: { type: Number },
+        },
+      ],
+    },
+  ],
+  seatReservations: [
+    {
+      _id: false,
+      number: { type: Number },
+      requirementGroup: { type: String },
+      fromDate: { type: String },
+    },
+  ],
+});
+enrollmentHistorySchema.index(
+  { termId: 1, sessionId: 1, sectionId: 1 },
+  { unique: true }
+);
+
+export const NewEnrollmentHistoryModel: Model<IEnrollmentHistoryItem> =
+  model<IEnrollmentHistoryItem>("EnrollmentHistory", enrollmentHistorySchema);