feat: add new imgur-based image upload functionality

nhscc · Aug 7, 2021 · b50e614 · b50e614
1 parent 8e78046
commit b50e614
Show file tree

Hide file tree

Showing 7 changed files with 220 additions and 53 deletions.
diff --git a/dist.env b/dist.env
@@ -16,6 +16,14 @@ ANALYZE=false
 # test-ib`. Note that this is ignored with `npm run build-externals`.
 NODE_ENV=development
 
+# The delete/hash of the imgur album used to store images uploaded via the API.
+# See: https://apidocs.imgur.com/#c85c9dfc-7487-4de2-9ecd-66f727cf3139
+IMGUR_ALBUM_HASH=
+
+# The client-id authorization token used to interact with the imgur API.
+# See: https://api.imgur.com/#registerapp
+IMGUR_CLIENT_ID=
+
 # MongoDB connect URI
 # Specify auth credentials if necessary
 # MUST SPECIFY A DATABASE AT THE END! e.g. mongodb://.../your-database-here
@@ -117,5 +125,9 @@ PRUNE_DATA_MAX_USERS=3400
 # amount. Oldest entries are deleted first.
 PRUNE_DATA_MAX_MEMES=35540
 
+# The size of the memes collection will not be allowed to exceed this
+# amount. Oldest entries are deleted first.
+PRUNE_DATA_MAX_UPLOADS=10000
+
 ### TOOLS FRONTEND VARIABLES ###
 # (optional unless using tools)
diff --git a/external-scripts/prune-data.ts b/external-scripts/prune-data.ts
@@ -18,60 +18,86 @@ if (!getEnv().DEBUG && getEnv().NODE_ENV != 'test') {
   debug.enabled = false;
 }
 
-export default async function main() {
-  try {
-    log('initializing');
-
-    const env = getEnv();
-    const limits = {
-      'request-log':
-        env.PRUNE_DATA_MAX_LOGS ||
-        toss(
-          new IllegalExternalEnvironmentError(
-            'PRUNE_DATA_MAX_LOGS must be greater than zero'
-          )
-        ),
-      users:
-        env.PRUNE_DATA_MAX_USERS ||
-        toss(
-          new IllegalExternalEnvironmentError(
-            'PRUNE_DATA_MAX_USERS must be greater than zero'
-          )
-        ),
-      memes:
-        env.PRUNE_DATA_MAX_MEMES ||
+const COLLECTION_LIMITS = (env: ReturnType<typeof getEnv>) => {
+  const limits = {
+    'request-log':
+      env.PRUNE_DATA_MAX_LOGS ||
+      toss(
+        new IllegalExternalEnvironmentError(
+          'PRUNE_DATA_MAX_LOGS must be greater than zero'
+        )
+      ),
+    users:
+      env.PRUNE_DATA_MAX_USERS ||
+      toss(
+        new IllegalExternalEnvironmentError(
+          'PRUNE_DATA_MAX_USERS must be greater than zero'
+        )
+      ),
+    memes:
+      env.PRUNE_DATA_MAX_MEMES ||
+      toss(
+        new IllegalExternalEnvironmentError(
+          'PRUNE_DATA_MAX_MEMES must be greater than zero'
+        )
+      ),
+    'limited-log-mview':
+      env.PRUNE_DATA_MAX_BANNED ||
+      toss(
+        new IllegalExternalEnvironmentError(
+          'PRUNE_DATA_MAX_BANNED must be greater than zero'
+        )
+      ),
+    uploads: {
+      limit:
+        env.PRUNE_DATA_MAX_UPLOADS ||
         toss(
           new IllegalExternalEnvironmentError(
-            'PRUNE_DATA_MAX_MEMES must be greater than zero'
+            'PRUNE_DATA_MAX_UPLOADS must be greater than zero'
           )
         ),
-      'limited-log-mview':
-        env.PRUNE_DATA_MAX_BANNED ||
-        toss(
-          new IllegalExternalEnvironmentError(
-            'PRUNE_DATA_MAX_BANNED must be greater than zero'
-          )
-        )
-    };
+      orderBy: 'lastUsedAt'
+    }
+  };
+
+  debug('limits: %O', limits);
+  return limits;
+};
+
+export default async function main() {
+  try {
+    log('initializing');
+
+    const limits = COLLECTION_LIMITS(getEnv());
 
-    debug(`final limits: %O`, limits);
     log('connecting to external database');
 
     const db = await getDb({ external: true });
 
     await Promise.all(
-      Object.entries(limits).map(async ([collectionName, limitThreshold]) => {
+      Object.entries(limits).map(async ([collectionName, limitObj]) => {
+        const { limit: limitThreshold, orderBy } =
+          typeof limitObj == 'number' ? { limit: limitObj, orderBy: '_id' } : limitObj;
+
         const subLog = log.extend(collectionName);
         const collection = db.collection(collectionName);
         const total = await collection.countDocuments();
-        const cursor = collection.find().sort({ _id: -1 }).skip(limitThreshold).limit(1);
+
+        const cursor = collection
+          .find()
+          .sort({ [orderBy]: -1 })
+          .skip(limitThreshold)
+          .limit(1);
+
         const thresholdEntry = await cursor.next();
 
         if (thresholdEntry) {
           const result = await collection.deleteMany({
-            _id: { $lte: thresholdEntry._id }
+            [orderBy]: { $lte: thresholdEntry[orderBy] }
           });
-          subLog(`pruned ${result.deletedCount}/${total} "${collectionName}" entries`);
+          subLog(
+            `pruned ${result.deletedCount}/${total} "${collectionName}" entries (ordered by "${orderBy}")`
+          );
         } else {
           subLog(
             `no prunable "${collectionName}" entries (${total} <= ${limitThreshold})`

diff --git a/src/backend/db.ts b/src/backend/db.ts
@@ -72,7 +72,8 @@ export async function destroyDb(db: Db) {
     db.dropCollection('limited-log-mview'),
     db.dropCollection('memes'),
     db.dropCollection('users'),
-    db.dropCollection('info')
+    db.dropCollection('info'),
+    db.dropCollection('uploads')
   ]);
 }
 
@@ -89,7 +90,8 @@ export async function initializeDb(db: Db) {
     // ? Collation allows for case-insensitive searching. See:
     // ? https://stackoverflow.com/a/40914924/1367414
     db.createCollection('users', { collation: { locale: 'en', strength: 2 } }),
-    db.createCollection('info')
+    db.createCollection('info'),
+    db.createCollection('uploads')
   ]);
 
   // TODO:

diff --git a/src/backend/env.ts b/src/backend/env.ts
@@ -31,7 +31,7 @@ export function getEnv() {
   const env = {
     NODE_ENV:
       process.env.APP_ENV || process.env.NODE_ENV || process.env.BABEL_ENV || 'unknown',
-    MONGODB_URI: (process.env.MONGODB_URI || '').toString(),
+    MONGODB_URI: process.env.MONGODB_URI || '',
     MONGODB_MS_PORT: !!process.env.MONGODB_MS_PORT
       ? Number(process.env.MONGODB_MS_PORT)
       : null,
@@ -85,10 +85,15 @@ export function getEnv() {
     PRUNE_DATA_MAX_MEMES: !!process.env.PRUNE_DATA_MAX_MEMES
       ? Number(process.env.PRUNE_DATA_MAX_MEMES)
       : null,
+    PRUNE_DATA_MAX_UPLOADS: !!process.env.PRUNE_DATA_MAX_UPLOADS
+      ? Number(process.env.PRUNE_DATA_MAX_UPLOADS)
+      : null,
     HYDRATE_DB_ON_STARTUP:
       !!process.env.HYDRATE_DB_ON_STARTUP &&
       process.env.HYDRATE_DB_ON_STARTUP !== 'false',
-    API_ROOT_URI: (process.env.API_ROOT_URI || '').toString(),
+    API_ROOT_URI: process.env.API_ROOT_URI || '',
+    IMGUR_ALBUM_HASH: process.env.IMGUR_ALBUM_HASH || '',
+    IMGUR_CLIENT_ID: process.env.IMGUR_CLIENT_ID || '',
     DEBUG: process.env.DEBUG ?? null,
     DEBUG_INSPECTING: !!process.env.VSCODE_INSPECTOR_OPTIONS,
     VERCEL_REGION: (process.env.VERCEL_REGION || 'unknown').toString(),
@@ -104,6 +109,7 @@ export function getEnv() {
   const NODE_X: string = env.NODE_ENV;
   const errors = [];
 
+  // TODO: retire all this logic when expect-env is created
   const envIsGtZero = (name: keyof typeof env) => {
     if (
       typeof env[name] != 'number' ||
@@ -116,6 +122,7 @@ export function getEnv() {
 
   if (NODE_X == 'unknown') errors.push(`bad NODE_ENV, saw "${NODE_X}"`);
 
+  // TODO: expect-env should cover this use-case (server-only) as well
   if (isServer()) {
     if (env.MONGODB_URI === '') errors.push(`bad MONGODB_URI, saw "${env.MONGODB_URI}"`);
 
@@ -131,13 +138,17 @@ export function getEnv() {
       (method) =>
         !HTTP2_METHODS.includes(method) &&
         errors.push(
-          `unknown method "${method}", must be one of: ${HTTP2_METHODS.join(',')}`
+          `unknown method "${method}", must be one of: ${HTTP2_METHODS.join(', ')}`
         )
     );
 
     if (env.MONGODB_MS_PORT && env.MONGODB_MS_PORT <= 1024) {
       errors.push(`optional environment variable MONGODB_MS_PORT must be > 1024`);
     }
+
+    if (!env.IMGUR_ALBUM_HASH || !env.IMGUR_CLIENT_ID) {
+      errors.push('IMGUR_ALBUM_HASH and IMGUR_CLIENT_ID must be defined');
+    }
   }
 
   if (errors.length) {