fix: store climb data as json and direct upload to mbox

package.json

@@ -48,7 +48,7 @@
     "cors": "^2.8.5",
     "date-fns": "^2.30.0",
     "dot-object": "^2.1.4",
-    "dotenv": "^10.0.0",
+    "dotenv": "^16.4.4",
     "express": "^4.18.2",
     "glob": "^10.2.2",
     "graphql": "^16.8.1",
@@ -90,10 +90,11 @@
     "export:json:full": "yarn build && node build/db/export/json/index.js",
     "export-prod": "./export.sh",
     "prepare": "husky install",
-    "import-users": "tsc ; node build/db/utils/jobs/migration/CreateUsersCollection.js",
-    "maptiles:export-db": "tsc ; node build/db/utils/jobs/MapTiles/index.js",
-    "maptiles:generate": "tsc ; node build/db/utils/jobs/MapTiles/generate.js",
-    "maptiles:upload": "tsc ; node build/db/utils/jobs/MapTiles/uploadCmd.js"
+    "import-users": "yarn build && node build/db/utils/jobs/migration/CreateUsersCollection.js",
+    "maptiles:export-db": "node build/db/utils/jobs/MapTiles/exportCmd.js",
+    "maptiles:generate": "./scripts/gen-tiles.sh",
+    "maptiles:upload": "node build/db/utils/jobs/MapTiles/uploadCmd.js",
+    "maptiles:full": "yarn build && yarn maptiles:export-db && yarn maptiles:generate && yarn maptiles:upload"
   },
   "standard": {
     "plugins": [

src/db/MediaObjectSchema.ts

@@ -23,7 +23,7 @@ const EntitySchema = new Schema<EntityTag>({
 }, { _id: true, toObject: { versionKey: false } })
 
 const schema = new Schema<MediaObject>({
-  userUuid: { ...UUID_TYPE, index: true },
+  userUuid: { ...UUID_TYPE, index: true, transform: (v: any) => v.toUUID().toString() },
   mediaUrl: { type: Schema.Types.String, unique: true, index: true },
   width: { type: Schema.Types.Number, required: true },
   height: { type: Schema.Types.Number, required: true },

src/db/MediaObjectTypes.ts

@@ -18,7 +18,7 @@ export interface MediaObject {
 
 export interface EntityTag {
   _id: mongoose.Types.ObjectId
-  targetId: MUUID | string
+  targetId: MUUID
   type: number
   ancestors: string
   climbName?: string

src/db/utils/jobs/MapTiles/index.ts → src/db/utils/jobs/MapTiles/exportCmd.ts

@@ -19,8 +19,7 @@ import {
 import { logger } from '../../../../logger.js'
 import { ClimbType } from '../../../ClimbTypes.js'
 import MutableMediaDataSource from '../../../../model/MutableMediaDataSource.js'
-
-export const WORKING_DIR = './maptiles'
+import { workingDir } from './init.js'
 
 /**
  * Export leaf areas as Geojson.  Leaf areas are crags/boulders that have climbs.
@@ -38,7 +37,7 @@ async function exportLeafCrags (): Promise<void> {
   > = []
 
   let fileIndex = 0
-  let stream: WriteStream = createWriteStream(`crags.${fileIndex}.geojson`, {
+  let stream: WriteStream = createWriteStream(`${workingDir}/crags.${fileIndex}.geojson`, {
     encoding: 'utf-8'
   })
   const cursor = model
@@ -70,7 +69,7 @@ async function exportLeafCrags (): Promise<void> {
     const pointFeature = point(
       doc.metadata.lnglat.coordinates,
       {
-        id: metadata.area_id,
+        id: metadata.area_id.toUUID().toString(),
         name: areaName,
         type: 'crag',
         content,
@@ -101,7 +100,7 @@ async function exportLeafCrags (): Promise<void> {
       features = []
 
       fileIndex++
-      stream = createWriteStream(`${WORKING_DIR}/crags.${fileIndex}.geojson`, {
+      stream = createWriteStream(`${workingDir}/crags.${fileIndex}.geojson`, {
         encoding: 'utf-8'
       })
     }
@@ -119,8 +118,10 @@ async function exportLeafCrags (): Promise<void> {
  * Export crag groups as Geojson.  Crag groups are immediate parent of leaf areas (crags/boulders).
  */
 async function exportCragGroups (): Promise<void> {
+  logger.info('Exporting crag groups')
+  const stream = createWriteStream(`${workingDir}/crag-groups.geojson`, { encoding: 'utf-8' })
+
   const model = getAreaModel()
-  const stream = createWriteStream('crag-groups.geojson', { encoding: 'utf-8' })
 
   interface CragGroup {
     uuid: MUUID
@@ -221,7 +222,7 @@ async function exportCragGroups (): Promise<void> {
 async function onDBConnected (): Promise<void> {
   logger.info('Start exporting crag data as Geojson')
   // await exportLeafCrags()
-  // await exportCragGroups()
+  await exportCragGroups()
   await gracefulExit()
 }
 

src/db/utils/jobs/MapTiles/init.ts

@@ -0,0 +1,21 @@
+import { config } from 'dotenv'
+
+config({ path: ['.env.local', '.env'] })
+
+const mapboxUsername = process.env.MAPBOX_USERNAME ?? ''
+const mapboxToken = process.env.MAPBOX_TOKEN ?? ''
+const workingDir = process.env.MAPTILES_WORKING_DIR ?? ''
+
+if (mapboxUsername.trim() === '') {
+  throw new Error('MAPBOX_USERNAME not set')
+}
+
+if (mapboxToken.trim() === '') {
+  throw new Error('MAPBOX_TOKEN not set')
+}
+
+if (workingDir.trim() === '') {
+  throw new Error('MAPTILES_WORKING_DIR not set')
+}
+
+export { mapboxUsername, mapboxToken, workingDir }

src/db/utils/jobs/MapTiles/uploadCmd.ts

@@ -1,5 +1,4 @@
 import fs from 'fs'
-import { config } from 'dotenv'
 import {
   PutObjectCommand,
   PutObjectCommandOutput,
@@ -10,24 +9,7 @@ import UploadService, {
   MapiResponse
 } from '@mapbox/mapbox-sdk/services/uploads.js'
 import { logger } from '../../../../logger.js'
-
-config({ path: ['.env.local', '.env'] })
-
-const mapboxUsername = process.env.MAPBOX_USERNAME
-const mapboxToken = process.env.MAPBOX_TOKEN
-const workingDir = process.env.MAPTILES_WORKING_DIR
-
-if (mapboxUsername == null) {
-  throw new Error('MAPBOX_USERNAME not set')
-}
-
-if (mapboxToken == null) {
-  throw new Error('MAPBOX_TOKEN not set')
-}
-
-if (workingDir == null) {
-  throw new Error('MAPTILES_WORKING_DIR not set')
-}
+import { mapboxUsername, mapboxToken, workingDir } from './init.js'
 
 const uploadsClient = UploadService({ accessToken: mapboxToken })
 
@@ -97,8 +79,8 @@ const notifyMapbox = async (
 export const upload = async (options: UploadOptions): Promise<void> => {
   try {
     const credentials = await getCredentials()
-    const ret = await stageFileOnS3(credentials, options.filePath)
-    logger.info(ret.$metadata, 'File staged on S3')
+    await stageFileOnS3(credentials, options.filePath)
+    logger.info('File staged on S3')
     const res = await notifyMapbox(credentials, options)
     if (res.statusCode >= 200 && res.statusCode < 300) {
       logger.info('File uploaded to Mapbox')
@@ -115,7 +97,22 @@ export const upload = async (options: UploadOptions): Promise<void> => {
 export const uploadCragsTiles = async (): Promise<void> => {
   logger.info('Uploading crags tiles')
   const filePath = `${workingDir}/crags.mbtiles`
-  await upload({ tilesetId: 'crags', name: 'all crags and boulders', filePath })
+  try {
+    await upload({ tilesetId: 'crags', name: 'all crags and boulders', filePath })
+  } catch (err) {
+    logger.error(err, 'Failed to upload crags tiles')
+  }
+}
+
+export const uploadCragGroupsTiles = async (): Promise<void> => {
+  logger.info('Uploading crag-groups tiles')
+  const filePath = `${workingDir}/crag-groups.mbtiles`
+  try {
+    await upload({ tilesetId: 'crag-groups', name: 'crag groups', filePath })
+  } catch (err) {
+    logger.error(err, 'Failed to upload crag-groups tiles')
+  }
 }
 
 await uploadCragsTiles()
+await uploadCragGroupsTiles()

src/graphql/media/MediaResolvers.ts

@@ -26,7 +26,7 @@ const MediaResolvers = {
 
   EntityTag: {
     id: (node: EntityTag) => node._id,
-    targetId: (node: EntityTag) => node.targetId,
+    targetId: (node: EntityTag) => node.targetId.toUUID().toString(),
     lat: (node: EntityTag) => geojsonPointToLatitude(node.lnglat),
     lng: (node: EntityTag) => geojsonPointToLongitude(node.lnglat)
   },

yarn.lock

@@ -4491,10 +4491,10 @@ dot-object@^2.1.4:
     commander "^4.0.0"
     glob "^7.1.5"
 
-dotenv@^16.4.2:
-  version "16.4.2"
-  resolved "https://registry.yarnpkg.com/dotenv/-/dotenv-16.4.2.tgz#7ca798f89ae2011846bbdbf6470785307754120d"
-  integrity sha512-rZSSFxke7d9nYQ5NeMIwp5PP+f8wXgKNljpOb7KtH6SKW1cEqcXAz9VSJYVLKe7Jhup/gUYOkaeSVyK8GJ+nBg==
+dotenv@^16.4.4:
+  version "16.4.4"
+  resolved "https://registry.yarnpkg.com/dotenv/-/dotenv-16.4.4.tgz#a26e7bb95ebd36272ebb56edb80b826aecf224c1"
+  integrity sha512-XvPXc8XAQThSjAbY6cQ/9PcBXmFoWuw1sQ3b8HqUCR6ziGXjkTi//kB9SWa2UwqlgdAIuRqAa/9hVljzPehbYg==
 
 duplexify@^4.0.0, duplexify@^4.1.1:
   version "4.1.2"