A shared folder structure

dataops/README.md

@@ -0,0 +1,3 @@
+```sh
+pnpm install
+```

dataops/fetchEntities.ts

@@ -0,0 +1,56 @@
+// TODO: share types
+type Entity = any;
+/** Query sent to the translation API => load all entitites */
+const entitiesQuery = `query EntitiesQuery {
+  entities {
+    id
+    name
+    tags
+    type
+    category
+    description
+    patterns
+    apiOnly
+    mdn {
+      locale
+      url
+      title
+      summary
+    }
+    twitterName
+    twitter {
+      userName
+      avatarUrl
+    }
+    companyName
+    company {
+      name
+      homepage {
+        url
+      }
+    }
+  }
+}
+`;
+/**
+ * Fetch raw entities from the translation API
+ * @returns
+ */
+export const fetchEntities = async () => {
+  const response = await fetch(process.env.TRANSLATION_API!, {
+    method: "POST",
+    headers: {
+      "Content-Type": "application/json",
+      Accept: "application/json",
+    },
+    body: JSON.stringify({ query: entitiesQuery, variables: {} }),
+  });
+  const json = await response.json();
+  if (json.errors) {
+    console.log("// entities API query error");
+    console.log(JSON.stringify(json.errors, null, 2));
+    throw new Error();
+  }
+  const entities = json?.data?.entities as Array<Entity>;
+  return entities;
+};

dataops/mongo/connection.ts

@@ -0,0 +1,43 @@
+/**
+ * Mongo db connection with cache,
+ * works correctly with Next as well
+ * @see https://github.com/vercel/next.js/blob/canary/examples/with-mongodb/lib/mongodb.ts
+ *
+ * @example
+ * import mongoConnection from "connection"
+ * const mongoClient = await mongoConnection
+ * // now you are connected
+ */
+import { MongoClient } from "mongodb";
+
+if (!process.env.MONGODB_URI) {
+  throw new Error('Invalid environment variable: "MONGODB_URI"');
+}
+
+const uri = process.env.MONGODB_URI;
+const options = {};
+
+let client: MongoClient;
+let clientPromise: Promise<MongoClient>;
+
+if (!process.env.MONGODB_URI) {
+  throw new Error("Please add your Mongo URI to .env.local");
+}
+
+if (process.env.NODE_ENV === "development") {
+  // In development mode, use a global variable so that the value
+  // is preserved across module reloads caused by HMR (Hot Module Replacement).
+  if (!(global as any)._mongoClientPromise) {
+    client = new MongoClient(uri, options);
+    (global as any)._mongoClientPromise = client.connect();
+  }
+  clientPromise = (global as any)._mongoClientPromise;
+} else {
+  // In production mode, it's best to not use a global variable.
+  client = new MongoClient(uri, options);
+  clientPromise = client.connect();
+}
+
+// Export a module-scoped MongoClient promise. By doing this in a
+// separate module, the client can be shared across functions.
+export default clientPromise;

dataops/mongo/models.ts

@@ -0,0 +1,9 @@
+import { getCollection } from "./utils";
+
+export const getResponseCollection = () => getCollection("responses");
+export const getUserCollection = () => getCollection("users");
+export const getNormalizedResponseCollection = () =>
+  getCollection("normalizedresponses");
+
+export const getPrivateResponseCollection = () =>
+  getCollection("privateresponses");

dataops/mongo/utils.ts

@@ -0,0 +1,17 @@
+import mongoConnection from "./connection";
+
+export const connectToAppDb = async () => {
+  const mongoClient = await mongoConnection;
+  return mongoClient.db();
+};
+
+/**
+ * Get a collection
+ * Guarantees that the database is connected before returning
+ * @param collectionName
+ * @returns
+ */
+export const getCollection = async (collectionName: string) => {
+  const db = await connectToAppDb();
+  return db.collection(collectionName);
+};