From 4e75175cd6924415c030433bd5630306565f465e Mon Sep 17 00:00:00 2001 From: viet nguyen <3805254+vnugent@users.noreply.github.com> Date: Wed, 29 May 2024 15:56:51 +0200 Subject: [PATCH 1/3] refactor(maptiles): unifiy crag and boundary props --- .env | 2 +- .vscode/launch.json | 4 +- package.json | 2 +- scripts/upload-tiles.sh | 19 +- src/db/AreaTypes.ts | 6 + src/db/MediaObjectSchema.ts | 8 +- src/db/import/usa/AreaTransformer.ts | 1 + src/db/utils/jobs/MapTiles/exportCmd.ts | 224 ++++++++++++++++-------- src/graphql/resolvers.ts | 2 +- src/model/MediaDataSource.ts | 19 +- src/model/MutableAreaDataSource.ts | 1 + yarn.lock | 2 +- 12 files changed, 205 insertions(+), 85 deletions(-) diff --git a/.env b/.env index 667857ca..7cbbaa5e 100644 --- a/.env +++ b/.env @@ -18,4 +18,4 @@ TYPESENSE_API_KEY_RW=define_me AUTH0_DOMAIN=https://dev-fmjy7n5n.us.auth0.com AUTH0_KID=uciP2tJdJ4BKWoz73Fmln -MAPTILES_WORKING_DIR=./maptiles \ No newline at end of file +MAPTILES_WORKING_DIR=./maptiles diff --git a/.vscode/launch.json b/.vscode/launch.json index f242b393..3d769e4e 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -7,8 +7,8 @@ { "type": "node", "request": "launch", - "name": "Crag Geojson", - "program": "${workspaceFolder}/src/db/export/CragGeojson/index.ts", + "name": "Generate map tiles", + "program": "${workspaceFolder}/src/db/utils/jobs/MapTiles/exportCmd.ts", "preLaunchTask": "tsc: build - tsconfig.json", "outFiles": [ "${workspaceFolder}/build/**/*.js" diff --git a/package.json b/package.json index fdd1ee28..0b9729e4 100644 --- a/package.json +++ b/package.json @@ -57,7 +57,7 @@ "jsonwebtoken": "^8.5.1", "jwks-rsa": "^2.1.4", "mongoose": "6.4.7", - "mongoose-lean-virtuals": "0.9.1", + "mongoose-lean-virtuals": "^0.9.1", "node-fetch": "2", "p-limit": "^4.0.0", "pino": "^8.2.0", diff --git a/scripts/upload-tiles.sh b/scripts/upload-tiles.sh index 5cf997f7..e62f22d1 100755 --- a/scripts/upload-tiles.sh +++ b/scripts/upload-tiles.sh @@ -14,15 +14,28 @@ set +a S3_DEST=':s3,provider=Cloudflare,no_check_bucket=true,env_auth=true,acl=private:maptiles' echo "------ Generating crags tiles file ------" -tippecanoe --force -o ${MAPTILES_WORKING_DIR}/crags.pmtiles -l crags -n "Crags" -zg ${MAPTILES_WORKING_DIR}/crags.*.geojson +tippecanoe --force -o ${MAPTILES_WORKING_DIR}/crags.pmtiles \ + -l crags -n "Crags" \ + --coalesce-densest-as-needed \ + -z11 ${MAPTILES_WORKING_DIR}/crags.*.geojson echo "**Uploading to remote storage" rclone copy ${MAPTILES_WORKING_DIR}/crags.pmtiles ${S3_DEST} echo "------ Generating crag group tiles file ------" -tippecanoe --force -o ${MAPTILES_WORKING_DIR}/crag-groups.pmtiles -l crag-groups -n "Crag groups" -zg ${MAPTILES_WORKING_DIR}/crag-groups.geojson +tippecanoe --force -o ${MAPTILES_WORKING_DIR}/areas.pmtiles \ + -l areas -n "Areas" \ + --drop-densest-as-needed \ + -z8 ${MAPTILES_WORKING_DIR}/areas.geojson echo "**Uploading to remote storage" -rclone copy ${MAPTILES_WORKING_DIR}/crag-groups.pmtiles ${S3_DEST} +rclone copy ${MAPTILES_WORKING_DIR}/areas.pmtiles ${S3_DEST} +echo "------ Generating organizations tiles file ------" +tippecanoe --force -o ${MAPTILES_WORKING_DIR}/organizations.pmtiles \ + -l organizations -n "Organizations" \ + -zg ${MAPTILES_WORKING_DIR}/organizations.geojson + +echo "**Uploading to remote storage" +rclone copy ${MAPTILES_WORKING_DIR}/areas.pmtiles ${S3_DEST} exit $? diff --git a/src/db/AreaTypes.ts b/src/db/AreaTypes.ts index d230882a..7dad6116 100644 --- a/src/db/AreaTypes.ts +++ b/src/db/AreaTypes.ts @@ -37,6 +37,12 @@ export type AreaType = IAreaProps & { */ export interface IAreaProps extends AuthorMetadata { _id: mongoose.Types.ObjectId + + /** + * Area unique id (in UUID format). Same as `metadata.area_id`. + */ + uuid: MUUID + /** * ShortCodes are short, globally uniqe codes that identify significant climbing areas **/ diff --git a/src/db/MediaObjectSchema.ts b/src/db/MediaObjectSchema.ts index e1ef494f..690b24aa 100644 --- a/src/db/MediaObjectSchema.ts +++ b/src/db/MediaObjectSchema.ts @@ -1,7 +1,7 @@ import mongoose from 'mongoose' - import { MediaObject, EntityTag } from './MediaObjectTypes.js' import { PointSchema } from './ClimbSchema.js' +import { MUUID } from 'uuid-mongodb' const { Schema } = mongoose @@ -9,8 +9,12 @@ const UUID_TYPE = { type: 'object', value: { type: 'Buffer' } } +const muuidTransform = (v: MUUID): string => { + return v.toUUID().toString() +} + const EntitySchema = new Schema({ - targetId: { ...UUID_TYPE, index: true, transform: (v: any) => v.toUUID().toString() }, + targetId: { ...UUID_TYPE, index: true, transform: muuidTransform }, climbName: { type: Schema.Types.String }, areaName: { type: Schema.Types.String, required: true }, type: { type: Schema.Types.Number, required: true }, diff --git a/src/db/import/usa/AreaTransformer.ts b/src/db/import/usa/AreaTransformer.ts index aa5e38c7..055b85ed 100644 --- a/src/db/import/usa/AreaTransformer.ts +++ b/src/db/import/usa/AreaTransformer.ts @@ -75,6 +75,7 @@ export const makeDBArea = (node: AreaNode): AreaType => { } return { _id, + uuid, shortCode: '', area_name: areaName, children: Array.from(children), diff --git a/src/db/utils/jobs/MapTiles/exportCmd.ts b/src/db/utils/jobs/MapTiles/exportCmd.ts index 278faab8..b9ad39a6 100644 --- a/src/db/utils/jobs/MapTiles/exportCmd.ts +++ b/src/db/utils/jobs/MapTiles/exportCmd.ts @@ -1,3 +1,4 @@ +import muuid from 'uuid-mongodb' import { WriteStream, createWriteStream, existsSync, mkdirSync } from 'node:fs' import { point, @@ -7,15 +8,18 @@ import { Point, Polygon } from '@turf/helpers' +import convexHull from '@turf/convex' import os from 'node:os' -import { MUUID } from 'uuid-mongodb' import { connectDB, gracefulExit, getAreaModel, - getClimbModel + getClimbModel, + getOrganizationModel } from '../../../index.js' +import { AggregateType } from '../../../AreaTypes.js' +import { OrganizationType } from '../../../OrganizationTypes.js' import { logger } from '../../../../logger.js' import { ClimbType } from '../../../ClimbTypes.js' import MutableMediaDataSource from '../../../../model/MutableMediaDataSource.js' @@ -55,7 +59,8 @@ async function exportLeafCrags (): Promise { ancestors, content, gradeContext, - climbs + climbs, + totalClimbs } = doc const ancestorArray = ancestors.split(',') @@ -76,6 +81,7 @@ async function exportLeafCrags (): Promise { discipline: type, grade: grades })), + totalClimbs, ancestors: ancestorArray, pathTokens, gradeContext @@ -108,101 +114,102 @@ async function exportLeafCrags (): Promise { } /** - * Export crag groups as Geojson. Crag groups are immediate parent of leaf areas (crags/boulders). + * Export areas as Geojson. areas are immediate parent of leaf areas (crags/boulders). */ -async function exportCragGroups (): Promise { - logger.info('Exporting crag groups') - const stream = createWriteStream(`${workingDir}/crag-groups.geojson`, { encoding: 'utf-8' }) +async function exportAreas (): Promise { + logger.info('Exporting areas') + const stream = createWriteStream(`${workingDir}/areas.geojson`, { encoding: 'utf-8' }) const model = getAreaModel() - interface CragGroup { - uuid: MUUID - name: string - polygon: Polygon - childAreaList: Array<{ - name: string - uuid: MUUID + interface SimpleArea { + id: string + areaName: string + pathTokens: string[] + ancestors: string[] + metadata: { + isDestination: boolean + polygon: Polygon leftRightIndex: number - }> + } + media: [] + children: any[] + totalClimbs: number + aggregate: AggregateType + } + + const childAreaProjection = { + _id: 0, + id: { $last: { $split: ['$ancestors', ','] } }, + areaName: '$area_name', + totalClimbs: 1, + aggregate: 1 } - const rs: CragGroup[] = await model.aggregate([ - { $match: { 'metadata.leaf': true } }, + const rs: SimpleArea[] = await model.aggregate([ + { $match: { 'metadata.leaf': false } }, { $lookup: { from: 'areas', - localField: '_id', - foreignField: 'children', - as: 'parentCrags' - } - }, - { - $match: { - $and: [{ parentCrags: { $type: 'array', $ne: [] } }] - } - }, - { - $unwind: '$parentCrags' - }, - { - $addFields: { - parentCrags: { - childId: '$metadata.area_id' - } - } - }, - { - $group: { - _id: { - uuid: '$parentCrags.metadata.area_id', - name: '$parentCrags.area_name', - polygon: '$parentCrags.metadata.polygon' - }, - childAreaList: { - $push: { - leftRightIndex: '$metadata.leftRightIndex', - uuid: '$metadata.area_id', - name: '$area_name' - } - } + localField: 'children', + foreignField: '_id', + as: 'childAreas', + pipeline: [{ + $project: childAreaProjection + }, { + $sort: { 'metadata.leftRightIndex': 1 } + }] } }, { $project: { _id: 0, - uuid: '$_id.uuid', - name: '$_id.name', - polygon: '$_id.polygon', - childAreaList: 1 + id: { $last: { $split: ['$ancestors', ','] } }, + areaName: '$area_name', + content: 1, + metadata: { + isDestination: 1, + polygon: 1, + leftRightIndex: 1 + }, + pathTokens: 1, + ancestors: { $split: ['$ancestors', ','] }, + children: '$childAreas', + totalClimbs: 1, + aggregate: 1 } } ]) const features: Array< Feature< - Polygon, - { - name: string - } + Polygon > > = [] for await (const doc of rs) { const polygonFeature = feature( - doc.polygon, + doc.metadata.polygon, { - type: 'crag-group', - id: doc.uuid.toUUID().toString(), - name: doc.name, - children: doc.childAreaList.map(({ uuid, name, leftRightIndex }) => ({ - id: uuid.toUUID().toString(), - name, - lr: leftRightIndex - })) + type: 'areas', + ...doc, + media: await MutableMediaDataSource.getInstance().findMediaByAreaId(muuid.from(doc.id), { + width: 1, + height: 1, + mediaUrl: 1, + format: 1, + _id: 0, + 'entityTags.targetId': 1, + 'entityTags.ancestors': 1, + 'entityTags.climbName': 1, + 'entityTags.areaName': 1, + 'entityTags.type': 1 + }, + true), + metadata: doc.metadata }, { - id: doc.uuid.toUUID().toString() + id: doc.id } ) features.push(polygonFeature) @@ -212,6 +219,82 @@ async function exportCragGroups (): Promise { stream.close() } +/** + * Export Local Climbing Orgs as Geojson (work in progress) + */ +async function exportLCOs (): Promise { + logger.info('Exporting Local Climbing Orgs') + const stream = createWriteStream(`${workingDir}/organizations.geojson`, { encoding: 'utf-8' }) + const model = getOrganizationModel() + + const orgProjection = { + _change: 0, + _id: 0, + __v: 0 + } + + const areaProjection = { + name: '$area_name', + pathTokens: 1, + ancestors: 1, + uuid: '$metadata.area_id', + polygon: '$metadata.polygon' + } + + const rs = await model.aggregate([{ + $lookup: { + from: 'areas', + localField: 'associatedAreaIds', + foreignField: 'metadata.area_id', + as: 'associatedAreas', + pipeline: [{ + $project: areaProjection + }] + } + }, { + $lookup: { + from: 'areas', + localField: 'excludedAreaIds', + foreignField: 'metadata.area_id', + as: 'excludedAreas', + pipeline: [{ + $project: areaProjection + }] + } + }, { + $project: orgProjection + }]) + + const features: Array< + Feature< + Polygon, + { + id: string + name: string + } + > + > = [] + + // for each organization + for await (const org of rs) { + const members = org.associatedAreas.map((area: any) => feature(area.polygon)) + const holes = org.excludedAreas.map((area: any) => + feature(area.polygon) + ) + const boundary = convexHull(featureCollection(members.concat(holes))) + if (boundary != null) { + features.push( + feature(boundary.geometry, { + id: org.orgId.toUUID().toString(), + name: org.displayName + }) + ) + } + } + stream.write(JSON.stringify(featureCollection(features)) + os.EOL) + stream.close() +} + /** * Create working directory if it does not exist */ @@ -228,8 +311,9 @@ function prepareWorkingDir (): void { async function onDBConnected (): Promise { logger.info('Start exporting crag data as Geojson') prepareWorkingDir() + // await exportLCOs() await exportLeafCrags() - await exportCragGroups() + await exportAreas() await gracefulExit() } diff --git a/src/graphql/resolvers.ts b/src/graphql/resolvers.ts index 3eab464e..472e3b77 100644 --- a/src/graphql/resolvers.ts +++ b/src/graphql/resolvers.ts @@ -263,7 +263,7 @@ const resolvers = { media: async (node: any, args: any, { dataSources }: Context) => { const { media } = dataSources - return await media.findMediaByAreaId(node.metadata.area_id, node.ancestors) + return await media.findMediaByAreaId(node.metadata.area_id, null) }, authorMetadata: getAuthorMetadataFromBaseNode, diff --git a/src/model/MediaDataSource.ts b/src/model/MediaDataSource.ts index f768bee6..658a1ca4 100644 --- a/src/model/MediaDataSource.ts +++ b/src/model/MediaDataSource.ts @@ -269,10 +269,21 @@ export default class MediaDataSource extends MongoDataSource { * @param ancestors * @returns `UserMediaWithTags` array */ - async findMediaByAreaId (areaId: MUUID, ancestors: string): Promise { - return await this.mediaObjectModel.find({ - 'entityTags.ancestors': { $regex: areaId.toUUID().toString() } - }) + async findMediaByAreaId (areaId: MUUID, projection: any, shouldConvertUuidToString = false): Promise { + const transformFn = (doc: MediaObject): any => { + if (doc == null) return + // @ts-expect-error + doc.entityTags = doc.entityTags?.map(tag => ({ ...tag, targetId: tag.targetId.toUUID().toString() })) + return doc + } + const rs = await this.mediaObjectModel + .find({ + 'entityTags.ancestors': { $regex: areaId.toUUID().toString() } + }, projection) + .lean({ + transform: shouldConvertUuidToString ? transformFn : null + }) + return rs ?? [] } /** diff --git a/src/model/MutableAreaDataSource.ts b/src/model/MutableAreaDataSource.ts index 208671fb..41e68329 100644 --- a/src/model/MutableAreaDataSource.ts +++ b/src/model/MutableAreaDataSource.ts @@ -620,6 +620,7 @@ export const newAreaHelper = (areaName: string, parentAncestors: string, parentP const ancestors = parentAncestors + ',' + uuid.toUUID().toString() return { _id, + uuid, shortCode: '', area_name: areaName, children: [], diff --git a/yarn.lock b/yarn.lock index 88f4256b..59975a00 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5855,7 +5855,7 @@ mongodb@^4.13.0: "@aws-sdk/credential-providers" "^3.186.0" saslprep "^1.0.3" -mongoose-lean-virtuals@0.9.1: +mongoose-lean-virtuals@^0.9.1: version "0.9.1" resolved "https://registry.yarnpkg.com/mongoose-lean-virtuals/-/mongoose-lean-virtuals-0.9.1.tgz#7e5e8b7471237606d1a0d785f1b7240f86547d85" integrity sha512-jx4rhXuaQPam/lwef3z/FfYHlKdbFkDr9Qb7JEMeoa7y4pOuyJ83RkcNL25HRaoi4Bt71zKmV1cuJdv243t9aA== From e57aab2c7fcdbaa6e0b62558d2f1181666bff350 Mon Sep 17 00:00:00 2001 From: viet nguyen <3805254+vnugent@users.noreply.github.com> Date: Wed, 29 May 2024 16:47:32 +0200 Subject: [PATCH 2/3] remove unused upload --- scripts/upload-tiles.sh | 7 ------- 1 file changed, 7 deletions(-) diff --git a/scripts/upload-tiles.sh b/scripts/upload-tiles.sh index e62f22d1..f5208053 100755 --- a/scripts/upload-tiles.sh +++ b/scripts/upload-tiles.sh @@ -31,11 +31,4 @@ tippecanoe --force -o ${MAPTILES_WORKING_DIR}/areas.pmtiles \ echo "**Uploading to remote storage" rclone copy ${MAPTILES_WORKING_DIR}/areas.pmtiles ${S3_DEST} -echo "------ Generating organizations tiles file ------" -tippecanoe --force -o ${MAPTILES_WORKING_DIR}/organizations.pmtiles \ - -l organizations -n "Organizations" \ - -zg ${MAPTILES_WORKING_DIR}/organizations.geojson - -echo "**Uploading to remote storage" -rclone copy ${MAPTILES_WORKING_DIR}/areas.pmtiles ${S3_DEST} exit $? From 4550137d489eec729fd8aaf94fc43b3d937fdc3b Mon Sep 17 00:00:00 2001 From: viet nguyen <3805254+vnugent@users.noreply.github.com> Date: Wed, 29 May 2024 17:36:32 +0200 Subject: [PATCH 3/3] fix linting error --- src/db/utils/jobs/MapTiles/exportCmd.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/db/utils/jobs/MapTiles/exportCmd.ts b/src/db/utils/jobs/MapTiles/exportCmd.ts index b9ad39a6..6af22bff 100644 --- a/src/db/utils/jobs/MapTiles/exportCmd.ts +++ b/src/db/utils/jobs/MapTiles/exportCmd.ts @@ -19,7 +19,6 @@ import { getOrganizationModel } from '../../../index.js' import { AggregateType } from '../../../AreaTypes.js' -import { OrganizationType } from '../../../OrganizationTypes.js' import { logger } from '../../../../logger.js' import { ClimbType } from '../../../ClimbTypes.js' import MutableMediaDataSource from '../../../../model/MutableMediaDataSource.js' @@ -311,7 +310,7 @@ function prepareWorkingDir (): void { async function onDBConnected (): Promise { logger.info('Start exporting crag data as Geojson') prepareWorkingDir() - // await exportLCOs() + await exportLCOs() await exportLeafCrags() await exportAreas() await gracefulExit()