chore: create tests to check migration

src/db/MediaObjectSchema.ts

@@ -1,6 +1,6 @@
 import mongoose from 'mongoose'
 
-import { MediaObject, EntityTag } from './MediaObjectType.js'
+import { MediaObject, EntityTag } from './MediaObjectTypes.js'
 import { PointSchema } from './ClimbSchema.js'
 
 const { Schema } = mongoose

src/db/MediaObjectType.ts → src/db/MediaObjectTypes.ts

@@ -3,6 +3,7 @@ import { MUUID } from 'uuid-mongodb'
 import { Point } from '@turf/helpers'
 
 export type ImageFormatType = 'jpeg' | 'png' | 'webp' | 'avif'
+
 export interface MediaObject {
   _id: ObjectId
   userUuid: MUUID
@@ -36,3 +37,17 @@ export interface MediaByUsersInput {
   maxFiles?: number
   includesNoEntityTags?: boolean
 }
+
+export interface TagByUser {
+  username?: string
+  userUuid: MUUID
+  total: number
+}
+
+export interface AllTimeTagStats {
+  totalMediaWithTags: number
+  byUsers: TagByUser[]
+}
+export interface TagsLeaderboardType {
+  allTime: AllTimeTagStats
+}

src/db/MediaTypes.ts

@@ -3,7 +3,7 @@ import { MUUID } from 'uuid-mongodb'
 
 import { AreaType } from './AreaTypes.js'
 import { ClimbType } from './ClimbTypes.js'
-import { MediaObject } from './MediaObjectType.js'
+import { MediaObject } from './MediaObjectTypes.js'
 
 /**
  * @deprecated to be removed in favor of MediaObject type
@@ -74,17 +74,3 @@ export interface DeleteTagResult {
   destType: number
   destinationId: string
 }
-
-export interface TagByUser {
-  username?: string
-  userUuid: MUUID
-  total: number
-}
-
-export interface AllTimeTagStats {
-  totalMediaWithTags: number
-  byUsers: TagByUser[]
-}
-export interface TagsLeaderboardType {
-  allTime: AllTimeTagStats
-}

src/db/utils/jobs/migration/CreateMediaMetaCollection.ts

@@ -5,7 +5,7 @@ import muuid from 'uuid-mongodb'
 
 import { connectDB, gracefulExit } from '../../../index.js'
 import { logger } from '../../../../logger.js'
-import { MediaObject } from '../../../MediaObjectType.js'
+import { MediaObject } from '../../../MediaObjectTypes.js'
 import { getMediaObjectModel } from '../../../MediaObjectSchema.js'
 import { getFileInfo } from './SirvClient.js'
 
@@ -33,11 +33,14 @@ const onConnected = async (): Promise<void> => {
   for (const image of images) {
     const { width, height, format } = await sharp(image.fullpath()).metadata()
     if (width == null || height == null || image.size == null) continue
-    if ((format !== 'avif' && format !== 'jpeg' && format !== 'png' && format !== 'webp')) continue
+    if ((format !== 'avif' && format !== 'jpeg' && format !== 'png' && format !== 'webp')) {
+      logger.warn({ format, file: image.name }, 'Unexpected media format')
+      continue
+    }
 
     const folderUuidStr = image.parent?.name ?? ''
     if (!uuidValidate(folderUuidStr)) {
-      console.error('Error: expect folder name to have uuid format.  Found ', folderUuidStr)
+      logger.error({ file: image.name, parent: folderUuidStr }, 'Error: expect folder name to have uuid format.  Found ')
       continue
     }
     const userUuid = muuid.from(folderUuidStr)
@@ -66,7 +69,7 @@ const onConnected = async (): Promise<void> => {
     await model.insertMany(list)
   }
 
-  console.log('#count', count)
+  logger.info({ count }, 'Finish')
 
   await gracefulExit()
 }

src/db/utils/jobs/migration/MigrateTagsToMediaCollection.ts

@@ -1,7 +1,7 @@
 import { connectDB, getMediaModel, gracefulExit } from '../../../index.js'
 import { logger } from '../../../../logger.js'
 import { getMediaObjectModel } from '../../../MediaObjectSchema.js'
-import { EntityTag } from '../../../MediaObjectType.js'
+import { EntityTag } from '../../../MediaObjectTypes.js'
 
 /**
  * Move tags in Media collection to embedded tags in the new Media Objects collection.
@@ -11,6 +11,15 @@ const onConnected = async (): Promise<void> => {
   const mediaObjectModel = getMediaObjectModel()
   const oldTagModel = getMediaModel()
 
+  /**
+   * Initialize entityTags to []
+   */
+  await mediaObjectModel.updateMany({}, {
+    $set: {
+      entityTags: []
+    }
+  })
+
   let count = 0
 
   const taggedClimbsPipeline = [
@@ -95,8 +104,6 @@ const onConnected = async (): Promise<void> => {
   ]).cursor().eachAsync(async doc => {
     const mediaUrl: string = doc._id.mediaUrl
 
-    const rs = await mediaObjectModel.findOne({ mediaUrl })
-
     let d: EntityTag[] = []
     switch (doc._id.destType) {
       case 0: {
@@ -115,7 +122,7 @@ const onConnected = async (): Promise<void> => {
       case 1: {
         console.log('#Add area tags')
 
-        doc.taggedAreas.map(tag => ({
+        d = doc.taggedAreas.map(tag => ({
           targetId: tag.metadata.area_id,
           areaName: tag.area_name,
           ancestors: tag.ancestors,
@@ -127,9 +134,11 @@ const onConnected = async (): Promise<void> => {
       }
     }
 
-    if (d != null) {
-      rs?.set('entityTags', rs?.entityTags.concat(d))
-      await rs?.save()
+    if (d.length > 0) {
+      const rs = await mediaObjectModel.updateOne({ mediaUrl }, {
+        $addToSet: { entityTags: d }
+      }).lean()
+
       count = count + d.length
     }
   })

src/db/utils/jobs/migration/Tests.ts

@@ -0,0 +1,92 @@
+import { connectDB, getMediaModel, gracefulExit } from '../../../index.js'
+import { logger } from '../../../../logger.js'
+import { getMediaObjectModel } from '../../../MediaObjectSchema.js'
+
+const knownIssuesFilter = {
+  $match: {
+    $and: [
+      /**
+       * We don't support .heic files
+       */
+      { mediaUrl: { $not: /heic$/i } },
+      /**
+       * User folder was deleted from Sirv.com but this tag is still left behind
+       */
+      { mediaUrl: { $ne: '/u/515b2003-9b53-46f8-ac6e-667718315c10/rCh6Fnbb6G.jpeg' } }
+    ]
+  }
+}
+
+/**
+ * Move tags in Media collection to embedded tags in the new Media Objects collection.
+ */
+const onConnected = async (): Promise<void> => {
+  logger.info('Verifying...')
+  const mediaObjectModel = getMediaObjectModel()
+  const oldTagModel = getMediaModel()
+
+  const checkTagCounts = async (): Promise<void> => {
+    const rs0 = await oldTagModel.aggregate<{ count: number }>([
+      knownIssuesFilter,
+      {
+        $group: {
+          _id: '$onModel',
+          count: { $count: {} }
+        }
+      },
+      {
+        $project: {
+          _id: 0,
+          tagType: '$_id',
+          count: 1
+        }
+      }
+    ])
+
+    const rs1 = await mediaObjectModel.aggregate([
+      {
+        $unwind: {
+          path: '$entityTags'
+        }
+      }
+    ])
+
+    const oldTagCount = rs0[0].count + rs0[1].count
+    const newTagCount = rs1.length
+
+    logger.info({ oldTagCount, newTagCount, result: oldTagCount === newTagCount ? 'PASS' : 'FAIL' }, 'Old vs new tag count')
+  }
+
+  /**
+   * Compare all tagged climbs and areas in the older collection
+   *  to see if they're added to the new entityTags.
+   */
+  const checkOrphaneIDs = async (): Promise<void> => {
+    const rs = await oldTagModel.aggregate([
+      knownIssuesFilter,
+      {
+        $lookup: {
+          from: mediaObjectModel.modelName,
+          foreignField: 'entityTags.targetId',
+          localField: 'destinationId',
+          as: 'matchingTags'
+        }
+      },
+      {
+        $match: {
+          matchingTags: {
+            $size: 0
+          }
+        }
+      }
+    ])
+
+    logger.info({ found: rs, result: rs.length === 0 ? 'PASS' : 'FAIL' }, 'Orphane IDs check')
+  }
+  await checkTagCounts()
+  await checkOrphaneIDs()
+
+  await gracefulExit()
+}
+
+void connectDB(onConnected)

src/graphql/media/MediaResolvers.ts

@@ -1,5 +1,4 @@
-import { EntityTag, MediaByUsers, MediaObject } from '../../db/MediaObjectType.js'
-import { TagByUser } from '../../db/MediaTypes.js'
+import { EntityTag, MediaByUsers, MediaObject, TagByUser } from '../../db/MediaObjectTypes.js'
 import { getUserNickFromMediaDir, geojsonPointToLatitude, geojsonPointToLongitude } from '../../utils/helpers.js'
 
 const MediaResolvers = {

src/graphql/media/queries.ts

@@ -1,5 +1,4 @@
-import { TagsLeaderboardType } from '../../db/MediaTypes.js'
-import { MediaObject, MediaByUsers } from '../../db/MediaObjectType.js'
+import { TagsLeaderboardType, MediaObject, MediaByUsers } from '../../db/MediaObjectTypes.js'
 import { DataSourcesType } from '../../types.js'
 
 const MediaQueries = {

src/model/MediaDataSource.ts

@@ -2,18 +2,19 @@ import { MongoDataSource } from 'apollo-datasource-mongodb'
 import muid, { MUUID } from 'uuid-mongodb'
 import { logger } from '../logger.js'
 import { getMediaObjectModel } from '../db/index.js'
-import { TagsLeaderboardType, AllTimeTagStats } from '../db/MediaTypes.js'
+import { TagsLeaderboardType, AllTimeTagStats, MediaByUsers, MediaByUsersInput, MediaObject } from '../db/MediaObjectTypes.js'
 
-import { MediaByUsers, MediaByUsersInput, MediaObject } from '../db/MediaObjectType.js'
+const HARD_MAX_FILES = 1000
+const HARD_MAX_USERS = 100
 
 export default class MediaDataSourcmnee extends MongoDataSource<MediaObject> {
   mediaObjectModel = getMediaObjectModel()
 
+  /**
+   * A reusable filter to exclude documents with empty entityTags
+   */
   entityTagsNotEmptyFilter = [{
     $match: {
-      /**
-       * exclude documents with empty entityTags
-       */
       entityTags: { $exists: true, $type: 4, $ne: [] }
     }
   }]
@@ -27,6 +28,9 @@ export default class MediaDataSourcmnee extends MongoDataSource<MediaObject> {
    * @returns MediaByUsers array
    */
   async getMediaByUsers ({ uuidStr, maxUsers = 10, maxFiles = 10, includesNoEntityTags = false }: MediaByUsersInput): Promise<MediaByUsers[]> {
+    const safeMaxFiles = maxFiles > HARD_MAX_FILES ? HARD_MAX_FILES : maxFiles
+    const safeMaxUsers = maxUsers > HARD_MAX_USERS ? HARD_MAX_USERS : maxUsers
+
     let userFilter: any[] = []
     if (uuidStr != null) {
       userFilter = [{
@@ -58,14 +62,14 @@ export default class MediaDataSourcmnee extends MongoDataSource<MediaObject> {
         }
       },
       {
-        $limit: maxUsers
+        $limit: safeMaxUsers
       },
       {
         $project: {
           _id: 0,
           userUuid: '$_id.userUuid',
           mediaWithTags: {
-            $slice: ['$mediaWithTags', maxFiles]
+            $slice: ['$mediaWithTags', safeMaxFiles]
           }
         }
       }
@@ -80,7 +84,7 @@ export default class MediaDataSourcmnee extends MongoDataSource<MediaObject> {
   async getOneUserMedia (uuidStr: string, limit: number): Promise<MediaObject[]> {
     const rs = await this.getMediaByUsers({ uuidStr, maxUsers: 1, maxFiles: limit, includesNoEntityTags: true })
     if (rs.length !== 1) {
-      logger.error('Expecting 1 user in result set but got ', rs.length)
+      logger.error(`Expecting 1 user in result set but got ${rs.length}`)
       return []
     }
     return rs[0].mediaWithTags
@@ -126,6 +130,7 @@ export default class MediaDataSourcmnee extends MongoDataSource<MediaObject> {
     ], {
       /**
        * Read from secondary node since data freshness is not too important
+       * for this query.
        * See https://www.mongodb.com/docs/manual/core/read-preference/
        */
       readPreference: 'secondaryPreferred'