diff --git a/.vscode/launch.json b/.vscode/launch.json index 7631bae6..388786e2 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -4,6 +4,19 @@ // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 "version": "0.2.0", "configurations": [ + { + "type": "node", + "request": "launch", + "name": "Crag Geojson", + "program": "${workspaceFolder}/src/db/export/CragGeojson/index.ts", + "preLaunchTask": "tsc: build - tsconfig.json", + "outFiles": [ + "${workspaceFolder}/build/**/*.js" + ], + "skipFiles": [ + "/**" + ], + }, { "type": "node", "request": "launch", @@ -48,14 +61,21 @@ "type": "node", "request": "launch", "name": "Launch API Server (serve-dev)", - "skipFiles": ["/**"], + "skipFiles": [ + "/**" + ], "program": "${workspaceFolder}/src/main.ts", "preLaunchTask": "tsc: build - tsconfig.json", - "outFiles": ["${workspaceFolder}/build/**/*.js"], + "outFiles": [ + "${workspaceFolder}/build/**/*.js" + ], "runtimeExecutable": "yarn", - "runtimeArgs": ["run", "serve-dev"], + "runtimeArgs": [ + "run", + "serve-dev" + ], "console": "integratedTerminal" - }, + }, { "name": "Debug Jest Tests", "type": "node", diff --git a/export-crag-data.sh b/export-crag-data.sh new file mode 100755 index 00000000..53799988 --- /dev/null +++ b/export-crag-data.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +if [ -z ${GITHUB_ACCESS_TOKEN} ] +then + echo "GITHUB_ACCESS_TOKEN not defined." + exit 1 +fi + +echo "cloning openbeta-export repository" +git clone --depth 1 --branch production https://ob-bot-user:${GITHUB_ACCESS_TOKEN}@github.com/OpenBeta/openbeta-export || exit 1 +git config user.name "db-export-bot" +git config user.email "db-export-bot@noreply" +cd .. + +echo "start exporting CRAG data..." +yarn export-crags + +echo "... finished export. Committing data..." + +git add -A +git commit -am "export crag data" +git push origin production diff --git a/package.json b/package.json index 4880fcee..cec74ab1 100644 --- a/package.json +++ b/package.json @@ -31,8 +31,9 @@ "@turf/area": "^6.5.0", "@turf/bbox": "^6.5.0", "@turf/bbox-polygon": "^6.5.0", - "@turf/centroid": "^6.5.0", "@turf/circle": "^6.5.0", + "@turf/convex": "^6.5.0", + "@turf/helpers": "^6.5.0", "@types/uuid": "^8.3.3", "apollo-datasource-mongodb": "^0.5.4", "apollo-server": "^3.9.0", @@ -83,7 +84,8 @@ "export:json:full": "yarn build && node build/db/export/json/index.js", "export-prod": "./export.sh", "prepare": "husky install", - "import-users": "tsc ; node build/db/utils/jobs/migration/CreateUsersCollection.js" + "import-users": "tsc ; node build/db/utils/jobs/migration/CreateUsersCollection.js", + "export-crags": "tsc ; node build/db/utils/jobs/CragGeojson/index.js" }, "standard": { "plugins": [ @@ -103,4 +105,4 @@ "engines": { "node": ">=16.14.0" } -} +} \ No newline at end of file diff --git a/src/db/AreaSchema.ts b/src/db/AreaSchema.ts index 1493c744..21735dbd 100644 --- a/src/db/AreaSchema.ts +++ b/src/db/AreaSchema.ts @@ -8,6 +8,20 @@ import { GradeContexts } from '../GradeUtils.js' const { Schema, connection } = mongoose +const polygonSchema = new mongoose.Schema({ + type: { + type: String, + enum: ['Polygon'], + required: true + }, + coordinates: { + type: [[[Number]]], // Array of arrays of arrays of numbers + required: true + } +}, { + _id: false +}) + const ChangeRecordMetadata = new Schema({ user: { type: 'object', @@ -32,6 +46,7 @@ const MetadataSchema = new Schema({ type: PointSchema, index: '2dsphere' }, + polygon: polygonSchema, bbox: [{ type: Number, required: true }], leftRightIndex: { type: Number, required: false }, ext_id: { type: String, required: false, index: true }, @@ -121,6 +136,7 @@ AreaSchema.index({ _deleting: 1 }, { expireAfterSeconds: 0 }) AreaSchema.index({ 'metadata.leftRightIndex': 1 }, { + name: 'leftRightIndex', partialFilterExpression: { 'metadata.leftRightIndex': { $gt: -1 @@ -128,6 +144,10 @@ AreaSchema.index({ } }) +AreaSchema.index({ + children: 1 +}) + export const createAreaModel = (name: string = 'areas'): mongoose.Model => { return connection.model(name, AreaSchema) } diff --git a/src/db/AreaTypes.ts b/src/db/AreaTypes.ts index 36fc5c15..ff478293 100644 --- a/src/db/AreaTypes.ts +++ b/src/db/AreaTypes.ts @@ -1,7 +1,7 @@ import mongoose from 'mongoose' import { MUUID } from 'uuid-mongodb' -import { BBox, Point } from '@turf/helpers' +import { BBox, Point, Polygon } from '@turf/helpers' import { ClimbType } from './ClimbTypes.js' import { ChangeRecordMetadataType } from './ChangeLogType.js' import { GradeContexts } from '../GradeUtils.js' @@ -118,8 +118,7 @@ export interface IAreaMetadata { */ isBoulder?: boolean /** - * Areas may be very large, and this point may represent the centroid of the area's bounds - * or a spec point chosen by users. + * Location of a wall or a boulder aka leaf node. Use `bbox` or `polygon` non-leaf areas. * */ lnglat: Point /** @@ -143,6 +142,11 @@ export interface IAreaMetadata { * GQL layer use these values for querying and identification of areas. */ area_id: MUUID + + /** + * A polygon (created by convex hull) containing all child areas. + */ + polygon?: Polygon } export interface IAreaContent { /** longform to mediumform description of this area. diff --git a/src/db/index.ts b/src/db/index.ts index 58bbe87b..25656e8a 100644 --- a/src/db/index.ts +++ b/src/db/index.ts @@ -55,7 +55,7 @@ export const connectDB = async (onConnected: () => any = defaultFn): Promise { + const model = getAreaModel() + + const stream = createWriteStream('crags.geojson', { encoding: 'utf-8' }) + + const features: Array> = [] + + for await (const doc of model.find({ 'metadata.leaf': true }).lean()) { + const { metadata, area_name: areaName, pathTokens, ancestors } = doc + + const ancestorArray = ancestors.split(',') + const pointFeature = point(doc.metadata.lnglat.coordinates, { + id: metadata.area_id.toUUID().toString(), + name: areaName, + type: 'crag', + parent: { + id: ancestorArray[ancestorArray.length - 2], + name: pathTokens[doc.pathTokens.length - 2] + } + }) + features.push(pointFeature) + } + stream.write(JSON.stringify(featureCollection(features)) + os.EOL) + stream.close() +} + +/** + * Export crag groups as Geojson. Crag groups are immediate parent of leaf areas (crags/boulders). + */ +async function exportCragGroups (): Promise { + const model = getAreaModel() + const stream = createWriteStream('crag-groups.geojson', { encoding: 'utf-8' }) + + interface CragGroup { + uuid: MUUID + name: string + polygon: Polygon + childAreaList: Array<{ + name: string + uuid: MUUID + leftRightIndex: number + }> + } + + const rs: CragGroup[] = await model.aggregate([ + { $match: { 'metadata.leaf': true } }, + { + $lookup: { + from: 'areas', + localField: '_id', + foreignField: 'children', + as: 'parentCrags' + } + }, + { + $match: { + $and: [ + { parentCrags: { $type: 'array', $ne: [] } } + ] + } + }, + { + $unwind: '$parentCrags' + }, + { + $addFields: { + parentCrags: { + childId: '$metadata.area_id' + } + } + }, + { + $group: { + _id: { + uuid: '$parentCrags.metadata.area_id', + name: '$parentCrags.area_name', + polygon: '$parentCrags.metadata.polygon' + }, + childAreaList: { + $push: { + leftRightIndex: '$metadata.leftRightIndex', + uuid: '$metadata.area_id', + name: '$area_name' + } + } + } + }, + { + $project: { + _id: 0, + uuid: '$_id.uuid', + name: '$_id.name', + polygon: '$_id.polygon', + childAreaList: 1 + } + } + ]) + + const features: Array> = [] + + for await (const doc of rs) { + const polygonFeature = feature(doc.polygon, { + type: 'crag-group', + name: doc.name, + id: doc.uuid.toUUID().toString(), + children: doc.childAreaList.map(({ uuid, name, leftRightIndex }) => ( + { id: uuid.toUUID().toString(), name, lr: leftRightIndex })) + }) + features.push(polygonFeature) + } + + stream.write(JSON.stringify(featureCollection(features)) + os.EOL) + stream.close() +} + +async function onDBConnected (): Promise { + logger.info('Start exporting crag data as Geojson') + await exportLeafCrags() + await exportCragGroups() + await gracefulExit() +} + +void connectDB(onDBConnected) diff --git a/src/db/utils/jobs/TreeUpdater.ts b/src/db/utils/jobs/TreeUpdater.ts index f393b349..5f41ade0 100644 --- a/src/db/utils/jobs/TreeUpdater.ts +++ b/src/db/utils/jobs/TreeUpdater.ts @@ -1,6 +1,7 @@ import mongoose from 'mongoose' -import { feature, geometry, featureCollection, Feature, BBox, Point } from '@turf/helpers' -import centroid from '@turf/centroid' +import { featureCollection, BBox, Point, Polygon } from '@turf/helpers' +import bbox2Polygon from '@turf/bbox-polygon' +import convexHull from '@turf/convex' import pLimit from 'p-limit' import { getAreaModel } from '../../AreaSchema.js' @@ -61,6 +62,7 @@ interface ResultType { bbox: BBox lnglat: Point aggregate: AggregateType + polygon?: Polygon } async function postOrderVisit (node: AreaMongoType): Promise { @@ -109,26 +111,14 @@ const leafReducer = (node: AreaType): ResultType => { } /** - * Calculate a center from multiple areas - * @param array of areas - * @returns new center (Point) + * Calculate convex hull polyon contain all child areas */ -const calculateNewCenterFromNodes = (nodes: ResultType[]): Point => { - // Convert area array to Geojson Feature array - const arrayOfFeatures = nodes.reduce((acc, curr) => { - if (curr.lnglat.coordinates[0] !== 0 && curr.lnglat.coordinates[1] !== 0) { - // non-default coordinates --> geojson feature - acc.push(feature(curr.lnglat)) - } - return acc - }, []) +const calculatePolygonFromChildren = (nodes: ResultType[]): Polygon | undefined => { + const childAsPolygons = nodes.map(node => bbox2Polygon(node.bbox)) + const fc = featureCollection(childAsPolygons) + const polygonFeature = convexHull(fc) - if (arrayOfFeatures.length > 0) { - // - convert array of features to a feature collection - // - calculate centroid - return centroid(featureCollection(arrayOfFeatures)).geometry - } - return geometry('Point', [0, 0]) as Point + return polygonFeature?.geometry } /** @@ -145,6 +135,7 @@ const nodesReducer = async (result: ResultType[], parent: AreaMongoType): Promis type: 'Point', coordinates: [0, 0] }, + polygon: undefined, density: 0, aggregate: { byGrade: [], @@ -167,21 +158,21 @@ const nodesReducer = async (result: ResultType[], parent: AreaMongoType): Promis bbox, lnglat, // we'll calculate a new center point later density: -1, + polygon: undefined, aggregate: mergeAggregates(acc.aggregate, aggregate) } }, initial) - z.lnglat = calculateNewCenterFromNodes(result) + z.polygon = calculatePolygonFromChildren(result) z.density = areaDensity(z.bbox, z.totalClimbs) - const { totalClimbs, bbox, density, aggregate, lnglat } = z - if (parent.metadata.lnglat.coordinates[0] === 0 && parent.metadata.lnglat.coordinates[1] === 0) { - parent.metadata.lnglat = lnglat - } + const { totalClimbs, bbox, density, aggregate } = z + parent.totalClimbs = totalClimbs parent.metadata.bbox = bbox parent.density = density parent.aggregate = aggregate + parent.metadata.polygon = z.polygon await parent.save() return z } diff --git a/src/graphql/resolvers.ts b/src/graphql/resolvers.ts index c3c81347..f9796a6a 100644 --- a/src/graphql/resolvers.ts +++ b/src/graphql/resolvers.ts @@ -254,7 +254,8 @@ const resolvers = { areaId, lng, lat, - mp_id: metadata.ext_id ?? '' + mp_id: metadata.ext_id ?? '', + polygon: metadata?.polygon?.coordinates[0] ?? null }) }, diff --git a/src/graphql/schema/Area.gql b/src/graphql/schema/Area.gql index ae13ae03..6118ef45 100644 --- a/src/graphql/schema/Area.gql +++ b/src/graphql/schema/Area.gql @@ -101,6 +101,9 @@ type AreaMetadata { mp_id: String! area_id: ID! areaId: ID! + + "Array of the polygon vertices (convex hull) containing child areas." + polygon: [[Float]] } """ diff --git a/yarn.lock b/yarn.lock index ad2ed3d8..665295ed 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1686,7 +1686,7 @@ "@turf/bbox-polygon@^6.5.0": version "6.5.0" - resolved "https://registry.npmjs.org/@turf/bbox-polygon/-/bbox-polygon-6.5.0.tgz" + resolved "https://registry.yarnpkg.com/@turf/bbox-polygon/-/bbox-polygon-6.5.0.tgz#f18128b012eedfa860a521d8f2b3779cc0801032" integrity sha512-+/r0NyL1lOG3zKZmmf6L8ommU07HliP4dgYToMoTxqzsWzyLjaj/OzgQ8rBmv703WJX+aS6yCmLuIhYqyufyuw== dependencies: "@turf/helpers" "^6.5.0" @@ -1699,14 +1699,6 @@ "@turf/helpers" "^6.5.0" "@turf/meta" "^6.5.0" -"@turf/centroid@^6.5.0": - version "6.5.0" - resolved "https://registry.npmjs.org/@turf/centroid/-/centroid-6.5.0.tgz" - integrity sha512-MwE1oq5E3isewPprEClbfU5pXljIK/GUOMbn22UM3IFPDJX0KeoyLNwghszkdmFp/qMGL/M13MMWvU+GNLXP/A== - dependencies: - "@turf/helpers" "^6.5.0" - "@turf/meta" "^6.5.0" - "@turf/circle@^6.5.0": version "6.5.0" resolved "https://registry.npmjs.org/@turf/circle/-/circle-6.5.0.tgz" @@ -1715,6 +1707,15 @@ "@turf/destination" "^6.5.0" "@turf/helpers" "^6.5.0" +"@turf/convex@^6.5.0": + version "6.5.0" + resolved "https://registry.yarnpkg.com/@turf/convex/-/convex-6.5.0.tgz#a7613e0d3795e2f5b9ce79a39271e86f54a3d354" + integrity sha512-x7ZwC5z7PJB0SBwNh7JCeCNx7Iu+QSrH7fYgK0RhhNop13TqUlvHMirMLRgf2db1DqUetrAO2qHJeIuasquUWg== + dependencies: + "@turf/helpers" "^6.5.0" + "@turf/meta" "^6.5.0" + concaveman "*" + "@turf/destination@^6.5.0": version "6.5.0" resolved "https://registry.npmjs.org/@turf/destination/-/destination-6.5.0.tgz" @@ -1725,7 +1726,7 @@ "@turf/helpers@^6.5.0": version "6.5.0" - resolved "https://registry.npmjs.org/@turf/helpers/-/helpers-6.5.0.tgz" + resolved "https://registry.yarnpkg.com/@turf/helpers/-/helpers-6.5.0.tgz#f79af094bd6b8ce7ed2bd3e089a8493ee6cae82e" integrity sha512-VbI1dV5bLFzohYYdgqwikdMVpe7pJ9X3E+dlr425wa2/sMJqYDhTO++ec38/pcPvPE6oD9WEEeU3Xu3gza+VPw== "@turf/invariant@^6.5.0": @@ -2934,6 +2935,16 @@ concat-map@0.0.1: resolved "https://registry.yarnpkg.com/concat-map/-/concat-map-0.0.1.tgz#d8a96bd77fd68df7793a73036a3ba0d5405d477b" integrity sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg== +concaveman@*: + version "1.2.1" + resolved "https://registry.yarnpkg.com/concaveman/-/concaveman-1.2.1.tgz#47d20b4521125c15fabf453653c2696d9ee41e0b" + integrity sha512-PwZYKaM/ckQSa8peP5JpVr7IMJ4Nn/MHIaWUjP4be+KoZ7Botgs8seAZGpmaOM+UZXawcdYRao/px9ycrCihHw== + dependencies: + point-in-polygon "^1.1.0" + rbush "^3.0.1" + robust-predicates "^2.0.4" + tinyqueue "^2.0.3" + constant-case@^1.1.0: version "1.1.2" resolved "https://registry.yarnpkg.com/constant-case/-/constant-case-1.1.2.tgz#8ec2ca5ba343e00aa38dbf4e200fd5ac907efd63" @@ -6250,6 +6261,11 @@ pkg-dir@^4.1.0, pkg-dir@^4.2.0: dependencies: find-up "^4.0.0" +point-in-polygon@^1.1.0: + version "1.1.0" + resolved "https://registry.yarnpkg.com/point-in-polygon/-/point-in-polygon-1.1.0.tgz#b0af2616c01bdee341cbf2894df643387ca03357" + integrity sha512-3ojrFwjnnw8Q9242TzgXuTD+eKiutbzyslcq1ydfu82Db2y+Ogbmyrkpv0Hgj31qwT3lbS9+QAAO/pIQM35XRw== + postcss@^8.3.11: version "8.4.18" resolved "https://registry.yarnpkg.com/postcss/-/postcss-8.4.18.tgz#6d50046ea7d3d66a85e0e782074e7203bc7fbca2" @@ -6403,6 +6419,11 @@ quick-format-unescaped@^4.0.3: resolved "https://registry.npmjs.org/quick-format-unescaped/-/quick-format-unescaped-4.0.4.tgz" integrity sha512-tYC1Q1hgyRuHgloV/YXs2w15unPVh8qfu/qCTfhTYamaw7fyhumKa2yGpdSo87vY32rIclj+4fWYQXUMs9EHvg== +quickselect@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/quickselect/-/quickselect-2.0.0.tgz#f19680a486a5eefb581303e023e98faaf25dd018" + integrity sha512-RKJ22hX8mHe3Y6wH/N3wCM6BWtjaxIyyUIkpHOvfFnxdI4yD4tBXEBKSbriGujF6jnSVkJrffuo6vxACiSSxIw== + range-parser@~1.2.1: version "1.2.1" resolved "https://registry.npmjs.org/range-parser/-/range-parser-1.2.1.tgz" @@ -6418,6 +6439,13 @@ raw-body@2.5.1: iconv-lite "0.4.24" unpipe "1.0.0" +rbush@^3.0.1: + version "3.0.1" + resolved "https://registry.yarnpkg.com/rbush/-/rbush-3.0.1.tgz#5fafa8a79b3b9afdfe5008403a720cc1de882ecf" + integrity sha512-XRaVO0YecOpEuIvbhbpTrZgoiI6xBlz6hnlr6EHhd+0x9ase6EmeN+hdwwUaJvLcsFFQ8iWVF1GAK1yB0BWi0w== + dependencies: + quickselect "^2.0.0" + rc@^1.2.7: version "1.2.8" resolved "https://registry.yarnpkg.com/rc/-/rc-1.2.8.tgz#cd924bf5200a075b83c188cd6b9e211b7fc0d3ed" @@ -6572,6 +6600,11 @@ rimraf@^3.0.2: dependencies: glob "^7.1.3" +robust-predicates@^2.0.4: + version "2.0.4" + resolved "https://registry.yarnpkg.com/robust-predicates/-/robust-predicates-2.0.4.tgz#0a2367a93abd99676d075981707f29cfb402248b" + integrity sha512-l4NwboJM74Ilm4VKfbAtFeGq7aEjWL+5kVFcmgFA2MrdnQWx9iE/tUGvxY5HyMI7o/WpSIUFLbC5fbeaHgSCYg== + run-parallel@^1.1.9: version "1.2.0" resolved "https://registry.npmjs.org/run-parallel/-/run-parallel-1.2.0.tgz" @@ -7145,6 +7178,11 @@ tiny-case@^1.0.3: resolved "https://registry.yarnpkg.com/tiny-case/-/tiny-case-1.0.3.tgz#d980d66bc72b5d5a9ca86fb7c9ffdb9c898ddd03" integrity sha512-Eet/eeMhkO6TX8mnUteS9zgPbUMQa4I6Kkp5ORiBD5476/m+PIRiumP5tmh5ioJpH7k51Kehawy2UDfsnxxY8Q== +tinyqueue@^2.0.3: + version "2.0.3" + resolved "https://registry.yarnpkg.com/tinyqueue/-/tinyqueue-2.0.3.tgz#64d8492ebf39e7801d7bd34062e29b45b2035f08" + integrity sha512-ppJZNDuKGgxzkHihX8v9v9G5f+18gzaTfrukGrq6ueg0lmH4nqVnA2IPG0AEH3jKEk2GRJCUhDoqpoiw3PHLBA== + title-case@^1.1.0: version "1.1.2" resolved "https://registry.yarnpkg.com/title-case/-/title-case-1.1.2.tgz#fae4a6ae546bfa22d083a0eea910a40d12ed4f5a"