feat: crag exporter job

.vscode/launch.json

@@ -4,6 +4,19 @@
     // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
     "version": "0.2.0",
     "configurations": [
+        {
+            "type": "node",
+            "request": "launch",
+            "name": "Crag Geojson",
+            "program": "${workspaceFolder}/src/db/export/CragGeojson/index.ts",
+            "preLaunchTask": "tsc: build - tsconfig.json",
+            "outFiles": [
+                "${workspaceFolder}/build/**/*.js"
+            ],
+            "skipFiles": [
+                "<node_internals>/**"
+            ],
+        },
         {
             "type": "node",
             "request": "launch",
@@ -48,14 +61,21 @@
             "type": "node",
             "request": "launch",
             "name": "Launch API Server (serve-dev)",
-            "skipFiles": ["<node_internals>/**"],
+            "skipFiles": [
+                "<node_internals>/**"
+            ],
             "program": "${workspaceFolder}/src/main.ts",
             "preLaunchTask": "tsc: build - tsconfig.json",
-            "outFiles": ["${workspaceFolder}/build/**/*.js"],
+            "outFiles": [
+                "${workspaceFolder}/build/**/*.js"
+            ],
             "runtimeExecutable": "yarn",
-            "runtimeArgs": ["run", "serve-dev"],
+            "runtimeArgs": [
+                "run",
+                "serve-dev"
+            ],
             "console": "integratedTerminal"
-          },
+        },
         {
             "name": "Debug Jest Tests",
             "type": "node",

export-crag-data.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+
+if [ -z ${GITHUB_ACCESS_TOKEN} ]
+then
+  echo "GITHUB_ACCESS_TOKEN not defined."
+  exit 1
+fi
+
+echo "cloning openbeta-export repository"
+git clone --depth 1 --branch production https://ob-bot-user:${GITHUB_ACCESS_TOKEN}@github.com/OpenBeta/openbeta-export || exit 1
+git config user.name "db-export-bot"
+git config user.email "db-export-bot@noreply"
+cd ..
+
+echo "start exporting CRAG data..."
+yarn export-crags
+
+echo "... finished export. Committing data..."
+
+git add -A
+git commit -am "export crag data"
+git push origin production

package.json

@@ -31,8 +31,9 @@
     "@turf/area": "^6.5.0",
     "@turf/bbox": "^6.5.0",
     "@turf/bbox-polygon": "^6.5.0",
-    "@turf/centroid": "^6.5.0",
     "@turf/circle": "^6.5.0",
+    "@turf/convex": "^6.5.0",
+    "@turf/helpers": "^6.5.0",
     "@types/uuid": "^8.3.3",
     "apollo-datasource-mongodb": "^0.5.4",
     "apollo-server": "^3.9.0",
@@ -83,7 +84,8 @@
     "export:json:full": "yarn build && node build/db/export/json/index.js",
     "export-prod": "./export.sh",
     "prepare": "husky install",
-    "import-users": "tsc ; node build/db/utils/jobs/migration/CreateUsersCollection.js"
+    "import-users": "tsc ; node build/db/utils/jobs/migration/CreateUsersCollection.js",
+    "export-crags": "tsc ; node build/db/utils/jobs/CragGeojson/index.js"
   },
   "standard": {
     "plugins": [
@@ -103,4 +105,4 @@
   "engines": {
     "node": ">=16.14.0"
   }
-}
+}

src/db/AreaSchema.ts

@@ -8,6 +8,20 @@ import { GradeContexts } from '../GradeUtils.js'
 
 const { Schema, connection } = mongoose
 
+const polygonSchema = new mongoose.Schema({
+  type: {
+    type: String,
+    enum: ['Polygon'],
+    required: true
+  },
+  coordinates: {
+    type: [[[Number]]], // Array of arrays of arrays of numbers
+    required: true
+  }
+}, {
+  _id: false
+})
+
 const ChangeRecordMetadata = new Schema<ChangeRecordMetadataType>({
   user: {
     type: 'object',
@@ -32,6 +46,7 @@ const MetadataSchema = new Schema<IAreaMetadata>({
     type: PointSchema,
     index: '2dsphere'
   },
+  polygon: polygonSchema,
   bbox: [{ type: Number, required: true }],
   leftRightIndex: { type: Number, required: false },
   ext_id: { type: String, required: false, index: true },
@@ -121,13 +136,18 @@ AreaSchema.index({ _deleting: 1 }, { expireAfterSeconds: 0 })
 AreaSchema.index({
   'metadata.leftRightIndex': 1
 }, {
+  name: 'leftRightIndex',
   partialFilterExpression: {
     'metadata.leftRightIndex': {
       $gt: -1
     }
   }
 })
 
+AreaSchema.index({
+  children: 1
+})
+
 export const createAreaModel = (name: string = 'areas'): mongoose.Model<AreaType> => {
   return connection.model(name, AreaSchema)
 }

src/db/AreaTypes.ts

@@ -1,7 +1,7 @@
 import mongoose from 'mongoose'
 import { MUUID } from 'uuid-mongodb'
 
-import { BBox, Point } from '@turf/helpers'
+import { BBox, Point, Polygon } from '@turf/helpers'
 import { ClimbType } from './ClimbTypes.js'
 import { ChangeRecordMetadataType } from './ChangeLogType.js'
 import { GradeContexts } from '../GradeUtils.js'
@@ -118,8 +118,7 @@ export interface IAreaMetadata {
    */
   isBoulder?: boolean
   /**
-   * Areas may be very large, and this point may represent the centroid of the area's bounds
-   * or a spec point chosen by users.
+   * Location of a wall or a boulder aka leaf node.  Use `bbox` or `polygon` non-leaf areas.
    * */
   lnglat: Point
   /**
@@ -143,6 +142,11 @@ export interface IAreaMetadata {
    * GQL layer use these values for querying and identification of areas.
    */
   area_id: MUUID
+
+  /**
+   * A polygon (created by convex hull) containing all child areas.
+   */
+  polygon?: Polygon
 }
 export interface IAreaContent {
   /** longform to mediumform description of this area.

src/db/index.ts

@@ -55,7 +55,7 @@ export const connectDB = async (onConnected: () => any = defaultFn): Promise<voi
 
     await mongoose.connect(
       `${scheme}://${user}:${pass}@${server}/${dbName}?authSource=${authDb}&tls=${tlsFlag}&replicaSet=${rsName}`,
-      { autoIndex: false }
+      { autoIndex: true }
     )
   } catch (e) {
     logger.error("Can't connect to db")

src/db/utils/jobs/CragGeojson/index.ts

@@ -0,0 +1,140 @@
+import { createWriteStream } from 'node:fs'
+import { point, feature, featureCollection, Feature, Point, Polygon } from '@turf/helpers'
+import os from 'node:os'
+import { MUUID } from 'uuid-mongodb'
+
+import { connectDB, gracefulExit, getAreaModel } from '../../../index.js'
+import { logger } from '../../../../logger.js'
+
+/**
+ * Export leaf areas as Geojson.  Leaf areas are crags/boulders that have climbs.
+ */
+async function exportLeafCrags (): Promise<void> {
+  const model = getAreaModel()
+
+  const stream = createWriteStream('crags.geojson', { encoding: 'utf-8' })
+
+  const features: Array<Feature<Point, {
+    name: string
+    id: string
+  }>> = []
+
+  for await (const doc of model.find({ 'metadata.leaf': true }).lean()) {
+    const { metadata, area_name: areaName, pathTokens, ancestors } = doc
+
+    const ancestorArray = ancestors.split(',')
+    const pointFeature = point(doc.metadata.lnglat.coordinates, {
+      id: metadata.area_id.toUUID().toString(),
+      name: areaName,
+      type: 'crag',
+      parent: {
+        id: ancestorArray[ancestorArray.length - 2],
+        name: pathTokens[doc.pathTokens.length - 2]
+      }
+    })
+    features.push(pointFeature)
+  }
+  stream.write(JSON.stringify(featureCollection(features)) + os.EOL)
+  stream.close()
+}
+
+/**
+ * Export crag groups as Geojson.  Crag groups are immediate parent of leaf areas (crags/boulders).
+ */
+async function exportCragGroups (): Promise<void> {
+  const model = getAreaModel()
+  const stream = createWriteStream('crag-groups.geojson', { encoding: 'utf-8' })
+
+  interface CragGroup {
+    uuid: MUUID
+    name: string
+    polygon: Polygon
+    childAreaList: Array<{
+      name: string
+      uuid: MUUID
+      leftRightIndex: number
+    }>
+  }
+
+  const rs: CragGroup[] = await model.aggregate([
+    { $match: { 'metadata.leaf': true } },
+    {
+      $lookup: {
+        from: 'areas',
+        localField: '_id',
+        foreignField: 'children',
+        as: 'parentCrags'
+      }
+    },
+    {
+      $match: {
+        $and: [
+          { parentCrags: { $type: 'array', $ne: [] } }
+        ]
+      }
+    },
+    {
+      $unwind: '$parentCrags'
+    },
+    {
+      $addFields: {
+        parentCrags: {
+          childId: '$metadata.area_id'
+        }
+      }
+    },
+    {
+      $group: {
+        _id: {
+          uuid: '$parentCrags.metadata.area_id',
+          name: '$parentCrags.area_name',
+          polygon: '$parentCrags.metadata.polygon'
+        },
+        childAreaList: {
+          $push: {
+            leftRightIndex: '$metadata.leftRightIndex',
+            uuid: '$metadata.area_id',
+            name: '$area_name'
+          }
+        }
+      }
+    },
+    {
+      $project: {
+        _id: 0,
+        uuid: '$_id.uuid',
+        name: '$_id.name',
+        polygon: '$_id.polygon',
+        childAreaList: 1
+      }
+    }
+  ])
+
+  const features: Array<Feature<Polygon, {
+    name: string
+    id: string
+  }>> = []
+
+  for await (const doc of rs) {
+    const polygonFeature = feature(doc.polygon, {
+      type: 'crag-group',
+      name: doc.name,
+      id: doc.uuid.toUUID().toString(),
+      children: doc.childAreaList.map(({ uuid, name, leftRightIndex }) => (
+        { id: uuid.toUUID().toString(), name, lr: leftRightIndex }))
+    })
+    features.push(polygonFeature)
+  }
+
+  stream.write(JSON.stringify(featureCollection(features)) + os.EOL)
+  stream.close()
+}
+
+async function onDBConnected (): Promise<void> {
+  logger.info('Start exporting crag data as Geojson')
+  await exportLeafCrags()
+  await exportCragGroups()
+  await gracefulExit()
+}
+
+void connectDB(onDBConnected)