feat: convert crag data to mbtiles and upload directly to Mapbox (#391)

convert crag data to mbtiles and upload directly to Mapbox

.env

@@ -17,3 +17,5 @@ TYPESENSE_API_KEY_RW=define_me
 # Auth0
 AUTH0_DOMAIN=https://dev-fmjy7n5n.us.auth0.com
 AUTH0_KID=uciP2tJdJ4BKWoz73Fmln
+
+MAPTILES_WORKING_DIR=./maptiles

package.json

@@ -9,6 +9,7 @@
   "devDependencies": {
     "@types/auth0": "^3.3.2",
     "@types/jest": "^29.4.0",
+    "@types/mapbox__mapbox-sdk": "^0.14.0",
     "@types/node": "^18.13.0",
     "@types/supertest": "^2.0.12",
     "@types/underscore": "^1.11.4",
@@ -25,9 +26,11 @@
     "wait-for-expect": "^3.0.2"
   },
   "dependencies": {
+    "@aws-sdk/client-s3": "^3.507.0",
     "@babel/runtime": "^7.17.2",
     "@google-cloud/storage": "^6.9.5",
     "@graphql-tools/schema": "^8.3.1",
+    "@mapbox/mapbox-sdk": "^0.15.3",
     "@openbeta/sandbag": "^0.0.51",
     "@turf/area": "^6.5.0",
     "@turf/bbox": "^6.5.0",
@@ -45,7 +48,7 @@
     "cors": "^2.8.5",
     "date-fns": "^2.30.0",
     "dot-object": "^2.1.4",
-    "dotenv": "^10.0.0",
+    "dotenv": "^16.4.4",
     "express": "^4.18.2",
     "glob": "^10.2.2",
     "graphql": "^16.8.1",
@@ -87,8 +90,11 @@
     "export:json:full": "yarn build && node build/db/export/json/index.js",
     "export-prod": "./export.sh",
     "prepare": "husky install",
-    "import-users": "tsc ; node build/db/utils/jobs/migration/CreateUsersCollection.js",
-    "export-crags": "tsc ; node build/db/utils/jobs/CragGeojson/index.js"
+    "import-users": "yarn build && node build/db/utils/jobs/migration/CreateUsersCollection.js",
+    "maptiles:export-db": "node build/db/utils/jobs/MapTiles/exportCmd.js",
+    "maptiles:generate": "./scripts/gen-tiles.sh",
+    "maptiles:upload": "node build/db/utils/jobs/MapTiles/uploadCmd.js",
+    "maptiles:full": "yarn build && yarn maptiles:export-db && yarn maptiles:generate && yarn maptiles:upload"
   },
   "standard": {
     "plugins": [
@@ -108,4 +114,4 @@
   "engines": {
     "node": ">=16.14.0"
   }
-}
+}

src/db/MediaObjectSchema.ts

@@ -10,7 +10,7 @@ const UUID_TYPE = {
 }
 
 const EntitySchema = new Schema<EntityTag>({
-  targetId: { ...UUID_TYPE, index: true },
+  targetId: { ...UUID_TYPE, index: true, transform: (v: any) => v.toUUID().toString() },
   climbName: { type: Schema.Types.String },
   areaName: { type: Schema.Types.String, required: true },
   type: { type: Schema.Types.Number, required: true },
@@ -20,17 +20,17 @@ const EntitySchema = new Schema<EntityTag>({
     index: '2dsphere',
     required: false
   }
-}, { _id: true })
+}, { _id: true, toObject: { versionKey: false } })
 
 const schema = new Schema<MediaObject>({
-  userUuid: { ...UUID_TYPE, index: true },
+  userUuid: { ...UUID_TYPE, index: true, transform: (v: any) => v.toUUID().toString() },
   mediaUrl: { type: Schema.Types.String, unique: true, index: true },
   width: { type: Schema.Types.Number, required: true },
   height: { type: Schema.Types.Number, required: true },
   size: { type: Schema.Types.Number, required: true },
   format: { type: Schema.Types.String, required: true },
   entityTags: [EntitySchema]
-}, { _id: true, timestamps: true })
+}, { _id: true, timestamps: true, toJSON: { versionKey: false }, toObject: { versionKey: false } })
 
 /**
  * Additional indices

src/db/import/usa/USADay0Seed.ts

@@ -10,7 +10,10 @@ import { logger } from '../../../logger.js'
 const contentDir: string = process.env.CONTENT_BASEDIR ?? ''
 
 const DEFAULT_CONCURRENT_JOBS = 4
-const concurrentJobs: number = process.env.OB_SEED_JOBS !== undefined ? parseInt(process.env.OB_SEED_JOBS) : DEFAULT_CONCURRENT_JOBS
+const concurrentJobs: number =
+  process.env.OB_SEED_JOBS !== undefined
+    ? parseInt(process.env.OB_SEED_JOBS)
+    : DEFAULT_CONCURRENT_JOBS
 
 logger.info('Data dir', contentDir)
 logger.info('Max concurrent jobs: ', concurrentJobs)
@@ -21,7 +24,9 @@ if (contentDir === '') {
 }
 
 const main = async (): Promise<void> => {
-  const limiter = pLimit(concurrentJobs > 0 ? concurrentJobs : DEFAULT_CONCURRENT_JOBS)
+  const limiter = pLimit(
+    concurrentJobs > 0 ? concurrentJobs : DEFAULT_CONCURRENT_JOBS
+  )
 
   // TODO: Allow update.  Right now we drop the entire collection on each run.
   await dropCollection('areas')
@@ -33,17 +38,19 @@ const main = async (): Promise<void> => {
 
   const rootNode = await createRoot('US', 'USA')
 
-  const stats: Array<JobStats | any> = await Promise.all<Array<JobStats | any>>(US_STATES.map(async state => {
-    const code = state.code.toLowerCase()
-    const fRoutes = `${contentDir}/${code}-routes.jsonlines`
-    const fAreas = `${contentDir}/${code}-areas.jsonlines`
+  const stats: Array<JobStats | any> = await Promise.all<Array<JobStats | any>>(
+    US_STATES.map(async state => {
+      const code = state.code.toLowerCase()
+      const fRoutes = `${contentDir}/${code}-routes.jsonlines`
+      const fAreas = `${contentDir}/${code}-areas.jsonlines`
 
-    if (fs.existsSync(fRoutes) && fs.existsSync(fAreas)) {
-      /* eslint-disable-next-line */
-      return limiter(seedState, rootNode, code, fRoutes, fAreas)
-    }
-    return await Promise.resolve()
-  }))
+      if (fs.existsSync(fRoutes) && fs.existsSync(fAreas)) {
+        /* eslint-disable-next-line */
+        return limiter(seedState, rootNode, code, fRoutes, fAreas)
+      }
+      return await Promise.resolve()
+    })
+  )
 
   printStats(stats)