Sort-of port update-remote-settings cron to TS

The request was to port it to TypeScript even though this code
couldn't run yet, because we might be enabling this again soonish.

Thus, this ports it to TypeScript, recovers the module that it
referenced but no longer existed, replaces that module's use of the
`got` package with `fetch`, and ensures it builds. I don't have the
right access to be able to verify that it actually runs as expected
though.

Author: Vinnl

package.json

@@ -13,6 +13,7 @@
     "dev:cron:monthly-activity": "tsx --tsconfig tsconfig.cronjobs.json src/scripts/cronjobs/monthlyActivity.tsx",
     "dev:cron:db-delete-unverified-subscribers": "tsx --tsconfig tsconfig.cronjobs.json src/scripts/cronjobs/deleteUnverifiedSubscribers.ts",
     "dev:cron:db-pull-breaches": "tsx --tsconfig tsconfig.cronjobs.json src/scripts/cronjobs/syncBreaches.ts",
+    "dev:cron:remote-settings-pull-breaches": "tsx --tsconfig tsconfig.cronjobs.json src/scripts/cronjobs/updateBreachesInRemoteSettings.ts",
     "dev:cron:onerep-limits-alert": "tsx --tsconfig tsconfig.cronjobs.json src/scripts/cronjobs/onerepStatsAlert.ts",
     "dev:nimbus": "node --watch-path config/nimbus.yaml src/scripts/build/nimbusTypes.js",
     "build": "npm run get-location-data && npm run build-glean && npm run build-nimbus && next build && npm run build-cronjobs",
@@ -29,7 +30,7 @@
     "cron:breach-alerts": "node src/scripts/emailBreachAlerts.js",
     "cron:db-delete-unverified-subscribers": "node dist/scripts/cronjobs/deleteUnverifiedSubscribers.js",
     "cron:db-pull-breaches": "node dist/scripts/cronjobs/syncBreaches.js",
-    "cron:remote-settings-pull-breaches": "node scripts/updatebreaches.js",
+    "cron:remote-settings-pull-breaches": "node dist/scripts/cronjobs/updateBreachesInRemoteSettings.js",
     "cron:onerep-limits-alert": "node dist/scripts/cronjobs/onerepStatsAlert.js",
     "db:migrate": "node -r dotenv-flow/config node_modules/knex/bin/cli.js migrate:latest --knexfile src/db/knexfile.js",
     "db:rollback": "node -r dotenv-flow/config node_modules/knex/bin/cli.js migrate:rollback --knexfile src/db/knexfile.js",

scripts/updatebreaches.js

@@ -0,0 +1,149 @@
+"use strict";
+
+/**
+ * Cron: Daily
+ * From all the HIBP breaches, we parse out the new breaches that are not already present
+ * in firefox remote settings, and update the data source accordingly
+ */
+
+/*
+ *
+ *
+ *
+ *
+ *********************************** Warning ***********************************
+ *
+ * This script was in the repository unused, and referenced a module
+ * `remote-settings.js` that no longer existed at the time of writing (it was
+ * deleted in commit c727bfe968937e51b0cd42efefd010c7c401aeae).
+ * I dug it up from the Git history, inlined it in this file, and made sure
+ * it compiled, but I didn't get to actually run it.
+ *
+ * Thus, it could be used as a starting point when re-enabling the remote
+ * settings upload, but shouldn't be expected to work without modifications.
+ *
+ *********************************** Warning ***********************************
+ *
+ *
+ *
+ *
+ *
+ *
+ *
+ *
+ */
+
+import AppConstants from "../../appConstants";
+import * as HIBP from "../../utils/hibp";
+import type { Breach } from "../../app/functions/universal/breach";
+
+type RemoteSettingsBreach = Pick<
+  Breach,
+  "Name" | "Domain" | "BreachDate" | "PwnCount" | "AddedDate" | "DataClasses"
+>;
+
+const BREACHES_COLLECTION = "fxmonitor-breaches";
+const FX_RS_COLLECTION = `${AppConstants.FX_REMOTE_SETTINGS_WRITER_SERVER}/buckets/main-workspace/collections/${BREACHES_COLLECTION}`;
+const FX_RS_RECORDS = `${FX_RS_COLLECTION}/records`;
+const FX_RS_WRITER_USER = AppConstants.FX_REMOTE_SETTINGS_WRITER_USER;
+const FX_RS_WRITER_PASS = AppConstants.FX_REMOTE_SETTINGS_WRITER_PASS;
+
+async function whichBreachesAreNotInRemoteSettingsYet(breaches: Breach[]) {
+  const response = await fetch(FX_RS_RECORDS, {
+    headers: {
+      Authorization: `Basic ${Buffer.from(FX_RS_WRITER_USER + ":" + FX_RS_WRITER_PASS).toString("base64")}`,
+    },
+  });
+  const fxRSRecords = await response.json();
+  const remoteSettingsBreachesSet = new Set(
+    fxRSRecords.body.data.map((b: Breach) => b.Name),
+  );
+
+  return breaches.filter(({ Name }) => !remoteSettingsBreachesSet.has(Name));
+}
+
+async function postNewBreachToBreachesCollection(data: RemoteSettingsBreach) {
+  // Create the record
+  const response = await fetch(FX_RS_RECORDS, {
+    method: "POST",
+    body: JSON.stringify(data),
+    headers: {
+      "Content-Type": "application/json",
+      Authorization: `Basic ${Buffer.from(FX_RS_WRITER_USER + ":" + FX_RS_WRITER_PASS).toString("base64")}`,
+    },
+  });
+  return response.json();
+}
+
+async function requestReviewOnBreachesCollection() {
+  const response = await fetch(FX_RS_COLLECTION, {
+    method: "PATCH",
+    body: JSON.stringify({ data: { status: "to-review" } }),
+    headers: {
+      "Content-Type": "application/json",
+      Authorization: `Basic ${Buffer.from(FX_RS_WRITER_USER + ":" + FX_RS_WRITER_PASS).toString("base64")}`,
+    },
+  });
+  return response.json();
+}
+
+if (
+  !AppConstants.FX_REMOTE_SETTINGS_WRITER_USER ||
+  !AppConstants.FX_REMOTE_SETTINGS_WRITER_PASS ||
+  !AppConstants.FX_REMOTE_SETTINGS_WRITER_SERVER
+) {
+  console.error(
+    "updatebreaches requires FX_REMOTE_SETTINGS_WRITER_SERVER, FX_REMOTE_SETTINGS_WRITER_USER, FX_REMOTE_SETTINGS_WRITER_PASS.",
+  );
+  process.exit(1);
+}
+
+(async () => {
+  const allHibpBreaches = (await HIBP.req("/breaches")) as { body: Breach[] };
+  const verifiedSiteBreaches = allHibpBreaches.body.filter((breach) => {
+    return (
+      !breach.IsRetired &&
+      !breach.IsSpamList &&
+      !breach.IsFabricated &&
+      breach.IsVerified &&
+      breach.Domain !== ""
+    );
+  });
+  const verifiedSiteBreachesWithPWs = verifiedSiteBreaches.filter((breach) =>
+    breach.DataClasses.includes("Passwords"),
+  );
+
+  const newBreaches = await whichBreachesAreNotInRemoteSettingsYet(
+    verifiedSiteBreachesWithPWs,
+  );
+
+  if (newBreaches.length <= 0) {
+    console.log("No new breaches detected.");
+    process.exit(0);
+  }
+
+  console.log(`${newBreaches.length} new breach(es) found.`);
+
+  for (const breach of newBreaches) {
+    const data: RemoteSettingsBreach = {
+      Name: breach.Name,
+      Domain: breach.Domain,
+      BreachDate: breach.BreachDate,
+      PwnCount: breach.PwnCount,
+      AddedDate: breach.AddedDate,
+      DataClasses: breach.DataClasses,
+    };
+
+    console.log("New breach detected: \n", data);
+
+    try {
+      await postNewBreachToBreachesCollection(data);
+    } catch (e) {
+      console.error(e);
+      process.exit(1);
+    }
+  }
+
+  console.log("Requesting review on breaches collection");
+  await requestReviewOnBreachesCollection();
+})();