Migrate photo storage to GCloud storage (#273)

* chore: extract & save image width/height during migration
* chore: resolve username in the backend instead of frontend
* chore: return image metadata to tag queries
* chore: climb/area/user profile media share the same type
* refactor: embed tags in media object collection
* refactor: update find-tags-by-area-id to work with embedded tags
* refactor: move leaderboard api to new embedded tags model

.env

@@ -9,6 +9,7 @@ MONGO_READ_PREFERENCE=primary
 MONGO_REPLICA_SET_NAME=rs0
 CONTENT_BASEDIR=./tmp
 DEPLOYMENT_ENV=development
+CDN_URL=https://storage.googleapis.com/openbeta-staging
 
 # Typesense
 TYPESENSE_NODE=4wknoyspjq6l7c9fp-1.a1.typesense.net

.vscode/launch.json

@@ -54,7 +54,8 @@
             "preLaunchTask": "tsc: build - tsconfig.json",
             "outFiles": [
                 "${workspaceFolder}/build/**/*.js"
-            ]
+            ],
+            "console": "integratedTerminal",
         },
         {
             "name": "Debug Jest Tests",
@@ -71,6 +72,32 @@
             ],
             "console": "integratedTerminal",
             "internalConsoleOptions": "neverOpen",
-        }
+        },
+        {
+            "type": "node",
+            "request": "launch",
+            "name": "Create MediaObjects collection",
+            "program": "${workspaceFolder}/src/db/utils/jobs/migration/CreateMediaMetaCollection.ts",
+            "preLaunchTask": "tsc: build - tsconfig.json",
+            "outFiles": [
+                "${workspaceFolder}/build/**/*.js"
+            ],
+            "skipFiles": [
+                "<node_internals>/**"
+            ],
+        },
+        {
+            "type": "node",
+            "request": "launch",
+            "name": "Migrate old tags to MediaObjects collection",
+            "program": "${workspaceFolder}/src/db/utils/jobs/migration/MigrateTagsToMediaCollection.ts",
+            "preLaunchTask": "tsc: build - tsconfig.json",
+            "outFiles": [
+                "${workspaceFolder}/build/**/*.js"
+            ],
+            "skipFiles": [
+                "<node_internals>/**"
+            ],
+        },
     ]
 }

package.json

@@ -24,6 +24,7 @@
   },
   "dependencies": {
     "@babel/runtime": "^7.17.2",
+    "@google-cloud/storage": "^6.9.5",
     "@graphql-tools/schema": "^8.3.1",
     "@openbeta/sandbag": "^0.0.37",
     "@turf/area": "^6.5.0",
@@ -34,9 +35,12 @@
     "@types/uuid": "^8.3.3",
     "apollo-datasource-mongodb": "^0.5.4",
     "apollo-server": "^3.9.0",
+    "axios": "^1.3.6",
     "cors": "^2.8.5",
     "dot-object": "^2.1.4",
     "dotenv": "^10.0.0",
+    "expiry-map": "^2.0.0",
+    "glob": "^10.2.2",
     "graphql": "^16.5.0",
     "graphql-middleware": "^6.1.31",
     "graphql-shield": "^7.5.0",
@@ -48,8 +52,10 @@
     "mongoose-lean-virtuals": "0.9.1",
     "node-fetch": "2",
     "p-limit": "^4.0.0",
+    "p-memoize": "^7.1.1",
     "pino": "^8.2.0",
     "sanitize-html": "^2.7.2",
+    "sharp": "^0.32.0",
     "typesense": "^1.2.1",
     "underscore": "^1.13.2",
     "uuid": "^8.3.2",

src/db/AreaTypes.ts

@@ -6,6 +6,7 @@ import { ClimbType } from './ClimbTypes.js'
 import { ChangeRecordMetadataType } from './ChangeLogType.js'
 import { GradeContexts } from '../GradeUtils.js'
 import { ExperimentalAuthorType } from './UserTypes.js'
+import { AuthorMetadata } from '../types.js'
 
 /**
  * Areas are a grouping mechanism in the OpenBeta data model that allow
@@ -32,7 +33,7 @@ export type AreaType = IAreaProps & {
  * See AreaType for the reified version of this object, and always use it
  * if you are working with data that exists inside the database.
 */
-export interface IAreaProps {
+export interface IAreaProps extends AuthorMetadata {
   _id: mongoose.Types.ObjectId
   /**
    * ShortCodes are short, globally uniqe codes that identify significant climbing areas
@@ -99,10 +100,6 @@ export interface IAreaProps {
   _change?: ChangeRecordMetadataType
   /** Used to delete an area.  See https://www.mongodb.com/docs/manual/core/index-ttl/ */
   _deleting?: Date
-  createdAt?: Date
-  updatedAt?: Date
-  updatedBy?: MUUID
-  createdBy?: MUUID
 }
 
 export interface IAreaMetadata {

src/db/MediaObjectSchema.ts

@@ -0,0 +1,54 @@
+import mongoose from 'mongoose'
+
+import { MediaObject, EntityTag } from './MediaObjectTypes.js'
+import { PointSchema } from './ClimbSchema.js'
+
+const { Schema } = mongoose
+
+const UUID_TYPE = {
+  type: 'object', value: { type: 'Buffer' }
+}
+
+const EntitySchema = new Schema<EntityTag>({
+  targetId: { ...UUID_TYPE, index: true },
+  climbName: { type: Schema.Types.String },
+  areaName: { type: Schema.Types.String, required: true },
+  type: { type: Schema.Types.Number, required: true },
+  ancestors: { type: Schema.Types.String, required: true, index: true },
+  lnglat: {
+    type: PointSchema,
+    index: '2dsphere'
+  }
+}, { _id: true })
+
+const schema = new Schema<MediaObject>({
+  userUuid: { ...UUID_TYPE, index: true },
+  mediaUrl: { type: Schema.Types.String, unique: true, index: true },
+  width: { type: Schema.Types.Number, required: true },
+  height: { type: Schema.Types.Number, required: true },
+  size: { type: Schema.Types.Number, required: true },
+  format: { type: Schema.Types.String, required: true },
+  entityTags: [EntitySchema]
+}, { _id: true, timestamps: true })
+
+/**
+ * Additional indices
+ */
+schema.index({
+  /**
+   * For filtering media objects with/without tags
+   */
+  entityTags: 1,
+  /**
+   * For sorting media objects by insertion order
+   */
+  createdAt: -1 // ascending, more recent first
+})
+
+/**
+ * Get media object model
+ * @returns MediaObjectType
+ */
+export const getMediaObjectModel = (): mongoose.Model<MediaObject> => {
+  return mongoose.model('media_objects', schema)
+}

src/db/MediaObjectTypes.ts

@@ -0,0 +1,58 @@
+import { ObjectId } from 'mongoose'
+import { MUUID } from 'uuid-mongodb'
+import { Point } from '@turf/helpers'
+
+export type ImageFormatType = 'jpeg' | 'png' | 'webp' | 'avif'
+
+export interface MediaObject {
+  _id: ObjectId
+  userUuid: MUUID
+  mediaUrl: string
+  width: number
+  height: number
+  format: ImageFormatType
+  createdAt: Date
+  size: number
+  entityTags: EntityTag[]
+}
+
+export interface EntityTag {
+  _id: ObjectId
+  targetId: MUUID
+  type: number
+  ancestors: string
+  climbName?: string
+  areaName: string
+  lnglat: Point
+}
+
+export interface MediaByUsers {
+  username: string
+  userUuid: MUUID
+  mediaWithTags: MediaObject[]
+}
+export interface MediaForFeedInput {
+  uuidStr?: string
+  maxUsers?: number
+  maxFiles?: number
+  includesNoEntityTags?: boolean
+}
+
+export interface TagByUser {
+  username?: string
+  userUuid: MUUID
+  total: number
+}
+
+export interface AllTimeTagStats {
+  totalMediaWithTags: number
+  byUsers: TagByUser[]
+}
+export interface TagsLeaderboardType {
+  allTime: AllTimeTagStats
+}
+
+export interface UserMediaQueryInput {
+  userUuid: string
+  maxFiles?: number
+}

src/db/MediaSchema.ts

@@ -55,6 +55,10 @@ MediaSchema.virtual('area', {
 MediaSchema.plugin(mongooseLeanVirtuals)
 MediaSchema.index({ mediaUuid: 1, destinationId: 1 }, { unique: true })
 
+/**
+ * @deprecated Superseded by MediaObjectSchema
+ * @param name
+ */
 export const getMediaModel = (name: string = 'media'): mongoose.Model<MediaType> => {
   return mongoose.model(name, MediaSchema)
 }

src/db/MediaTypes.ts

@@ -3,8 +3,11 @@ import { MUUID } from 'uuid-mongodb'
 
 import { AreaType } from './AreaTypes.js'
 import { ClimbType } from './ClimbTypes.js'
+import { MediaObject } from './MediaObjectTypes.js'
 
-// Type for 'Media' collection schema
+/**
+ * @deprecated to be removed in favor of MediaObject type
+ */
 export interface MediaType {
   _id?: mongoose.Types.ObjectId
   mediaUuid: MUUID
@@ -20,11 +23,21 @@ export enum RefModelType {
   areas = 'areas'
 }
 
-export interface MediaListByAuthorType {
-  _id: string
-  tagList: MediaType[]
+/**
+ * A tag with media metadata
+ */
+export type BaseTagType = MediaType & MediaObject
+
+export interface CompleteAreaTag extends BaseTagType {
+  area: AreaType
+}
+
+export interface CompleteClimbTag extends BaseTagType {
+  climb: ClimbType
 }
 
+export type TagType = CompleteAreaTag | CompleteClimbTag
+
 export interface MediaInputType {
   mediaUuid: MUUID
   mediaUrl: string
@@ -33,7 +46,10 @@ export interface MediaInputType {
   destType: number
 }
 
-interface BaseTagType {
+/**
+ * TODO: consolidate this type with BaseTagType
+ */
+interface LegacyBaseTagType {
   _id: mongoose.Types.ObjectId
   mediaUuid: MUUID
   mediaUrl: string
@@ -42,11 +58,11 @@ interface BaseTagType {
   onModel: RefModelType
 }
 
-export interface AreaTagType extends BaseTagType {
+export interface AreaTagType extends LegacyBaseTagType {
   area: AreaType
 }
 
-export interface ClimbTagType extends BaseTagType {
+export interface ClimbTagType extends LegacyBaseTagType {
   climb: ClimbType
 }
 
@@ -58,8 +74,3 @@ export interface DeleteTagResult {
   destType: number
   destinationId: string
 }
-
-export interface TagsLeaderboardType {
-  userUuid: string
-  total: number
-}

src/db/XMediaSchema.ts

@@ -38,6 +38,11 @@ export const XMediaSchema = new Schema<XMediaType>({
 
 XMediaSchema.plugin(mongooseLeanVirtuals)
 
+/**
+ * @deprecated Superceded by MediaObjects
+ * @param name
+ * @returns
+ */
 export const getXMediaModel = (name: string = 'xmedia'): mongoose.Model<XMediaType> => {
   return mongoose.model(name, XMediaSchema)
 }

src/db/index.ts

@@ -6,6 +6,7 @@ import { enableAllPlugins } from 'immer'
 import { getAreaModel } from './AreaSchema.js'
 import { getClimbModel } from './ClimbSchema.js'
 import { getMediaModel } from './MediaSchema.js'
+import { getMediaObjectModel } from './MediaObjectSchema.js'
 import { getOrganizationModel } from './OrganizationSchema.js'
 import { getTickModel } from './TickSchema.js'
 import { getXMediaModel } from './XMediaSchema.js'
@@ -71,6 +72,7 @@ export const createIndexes = async (): Promise<void> => {
   await getTickModel().createIndexes()
   await getXMediaModel().createIndexes()
   await getPostModel().createIndexes()
+  await getMediaObjectModel().createIndexes()
 }
 
 export const gracefulExit = async (exitCode: number = 0): Promise<void> => {
@@ -98,5 +100,6 @@ export {
   getChangeLogModel,
   getXMediaModel,
   getPostModel,
-  getExperimentalUserModel
+  getExperimentalUserModel,
+  getMediaObjectModel
 }

src/db/utils/index.ts

@@ -0,0 +1,8 @@
+import { AuthorMetadata } from '../../types'
+
+export const getAuthorMetadataFromBaseNode = ({ updatedAt, updatedBy, createdAt, createdBy }: AuthorMetadata): AuthorMetadata => ({
+  updatedAt,
+  updatedBy,
+  createdAt,
+  createdBy
+})