Skip to content

Commit

Permalink
fix: keep all areas' stats and geo data up-to-date
Browse files Browse the repository at this point in the history
  • Loading branch information
viet nguyen committed Jan 27, 2024
1 parent 6c0a1a4 commit f62725b
Show file tree
Hide file tree
Showing 4 changed files with 79 additions and 31 deletions.
2 changes: 2 additions & 0 deletions src/db/AreaTypes.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,8 @@ import { GradeContexts } from '../GradeUtils.js'
import { ExperimentalAuthorType } from './UserTypes.js'
import { AuthorMetadata } from '../types.js'

export type AreaDocumnent = mongoose.Document<unknown, any, AreaType> & AreaType

/**
* Areas are a grouping mechanism in the OpenBeta data model that allow
* the organization of climbs into geospatial and hierarchical structures.
Expand Down
22 changes: 11 additions & 11 deletions src/db/utils/jobs/TreeUpdaters/updateAllAreas.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,7 @@ export const updateAllAreas = async (): Promise<void> => {
}
}

export interface StatsAccumulator {
export interface StatsSummary {
density: number
totalClimbs: number
bbox?: BBox
Expand All @@ -67,7 +67,7 @@ export interface StatsAccumulator {
polygon?: Polygon
}

async function postOrderVisit (node: AreaMongoType): Promise<StatsAccumulator> {
async function postOrderVisit (node: AreaMongoType): Promise<StatsSummary> {
if (node.metadata.leaf || node.children.length === 0) {
return leafReducer((node.toObject() as AreaType))
}
Expand All @@ -92,7 +92,7 @@ async function postOrderVisit (node: AreaMongoType): Promise<StatsAccumulator> {
* @param node leaf area/crag
* @returns aggregate type
*/
export const leafReducer = (node: AreaType): StatsAccumulator => {
export const leafReducer = (node: AreaType): StatsSummary => {
return {
totalClimbs: node.totalClimbs,
bbox: node.metadata.bbox,
Expand All @@ -116,7 +116,7 @@ export const leafReducer = (node: AreaType): StatsAccumulator => {
/**
* Calculate convex hull polyon contain all child areas
*/
const calculatePolygonFromChildren = (nodes: StatsAccumulator[]): Feature<Polygon> | null => {
const calculatePolygonFromChildren = (nodes: StatsSummary[]): Feature<Polygon> | null => {
const childAsPolygons = nodes.reduce<Array<Feature<Polygon>>>((acc, curr) => {
if (curr.bbox != null) {
acc.push(bbox2Polygon(curr.bbox))
Expand All @@ -135,12 +135,12 @@ interface OPTIONS {

/**
* Calculate stats from a list of nodes
* @param result nodes
* @param childResults nodes
* @param parent parent node to save stats to
* @returns Calculated stats
*/
export const nodesReducer = async (result: StatsAccumulator[], parent: AreaMongoType, options?: OPTIONS): Promise<StatsAccumulator> => {
const initial: StatsAccumulator = {
export const nodesReducer = async (childResults: StatsSummary[], parent: AreaMongoType, options?: OPTIONS): Promise<StatsSummary> => {
const initial: StatsSummary = {
totalClimbs: 0,
bbox: undefined,
lnglat: undefined,
Expand All @@ -158,14 +158,14 @@ export const nodesReducer = async (result: StatsAccumulator[], parent: AreaMongo
}
}
}
let nodeSummary: StatsAccumulator = initial
if (result.length === 0) {
let nodeSummary: StatsSummary = initial
if (childResults.length === 0) {
const { totalClimbs, aggregate, density } = initial
parent.totalClimbs = totalClimbs
parent.density = density
parent.aggregate = aggregate
} else {
nodeSummary = result.reduce((acc, curr) => {
nodeSummary = childResults.reduce((acc, curr) => {
const { totalClimbs, aggregate, lnglat, bbox } = curr
return {
totalClimbs: acc.totalClimbs + totalClimbs,
Expand All @@ -177,7 +177,7 @@ export const nodesReducer = async (result: StatsAccumulator[], parent: AreaMongo
}
}, initial)

const polygon = calculatePolygonFromChildren(result)
const polygon = calculatePolygonFromChildren(childResults)
nodeSummary.polygon = polygon?.geometry
nodeSummary.bbox = bboxFromGeojson(polygon)
nodeSummary.density = areaDensity(nodeSummary.bbox, nodeSummary.totalClimbs)
Expand Down
37 changes: 20 additions & 17 deletions src/model/MutableAreaDataSource.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ import isoCountries from 'i18n-iso-countries'
import enJson from 'i18n-iso-countries/langs/en.json' assert { type: 'json' }
import bbox2Polygon from '@turf/bbox-polygon'

import { AreaType, AreaEditableFieldsType, OperationType, UpdateSortingOrderType } from '../db/AreaTypes.js'
import { AreaType, AreaDocumnent, AreaEditableFieldsType, OperationType, UpdateSortingOrderType } from '../db/AreaTypes.js'
import AreaDataSource from './AreaDataSource.js'
import { createRootNode } from '../db/import/usa/AreaTree.js'
import { makeDBArea } from '../db/import/usa/AreaTransformer.js'
Expand All @@ -20,13 +20,11 @@ import { GradeContexts } from '../GradeUtils.js'
import { sanitizeStrict } from '../utils/sanitize.js'
import { ExperimentalAuthorType } from '../db/UserTypes.js'
import { createInstance as createExperimentalUserDataSource } from '../model/ExperimentalUserDataSource.js'
import { StatsAccumulator, leafReducer, nodesReducer } from '../db/utils/jobs/TreeUpdaters/updateAllAreas.js'
import { StatsSummary, leafReducer, nodesReducer } from '../db/utils/jobs/TreeUpdaters/updateAllAreas.js'
import { bboxFrom } from '../geo-utils.js'

isoCountries.registerLocale(enJson)

type AreaDocumnent = mongoose.Document<unknown, any, AreaType> & AreaType

export default class MutableAreaDataSource extends AreaDataSource {
experimentalUserDataSource = createExperimentalUserDataSource()

Expand Down Expand Up @@ -485,14 +483,18 @@ export default class MutableAreaDataSource extends AreaDataSource {
}

/**
* Update area stats and geo data for a given leaf node and its ancestors
* Update area stats and geo data for a given leaf node and its ancestors.
* @param session
* @param changeRecord
* @param area
*/
async updateStatsAndGeoDataForSinglePath (session: ClientSession, changeRecord: ChangeRecordMetadataType, area: AreaDocumnent): Promise<void> {
const visitorFn = async (session: ClientSession, changeRecord: ChangeRecordMetadataType, area: AreaDocumnent, accumulator: StatsAccumulator): Promise<void> => {
/**
* Update function. For each node, recalculate stats and recursively update its acenstors until we reach the country node.
*/
const updateFn = async (session: ClientSession, changeRecord: ChangeRecordMetadataType, area: AreaDocumnent, childSummary: StatsSummary): Promise<void> => {
if (area.pathTokens.length <= 1) {
// we're at the root country node
return
}

Expand All @@ -506,26 +508,27 @@ export default class MutableAreaDataSource extends AreaDataSource {
.session(session)
.orFail()

logger.info(`###Updating stats for ${parentArea.area_name}`)
logger.info(` ##prev Area ${area._id} ${area.area_name}`)

const acc: StatsAccumulator[] = []
const acc: StatsSummary[] = []
/**
* Collect existing stats from all children. For affected node, use the stats from previous calculation.
*/
for (const childArea of parentArea.children) {
logger.info(` - ${childArea._id} ${childArea.area_name}`)

if (childArea._id.equals(area._id)) {
acc.push(accumulator)
acc.push(childSummary)
} else {
acc.push(leafReducer(childArea.toObject()))
}
}

const current = await nodesReducer(acc, parentArea as any as AreaDocumnent, { session, changeRecord })

await visitorFn(session, changeRecord, parentArea as any as AreaDocumnent, current)
await updateFn(session, changeRecord, parentArea as any as AreaDocumnent, current)
}
const accumulator = leafReducer(area.toObject())
await visitorFn(session, changeRecord, area, accumulator)

/**
* Begin calculation
*/
const leafStats = leafReducer(area.toObject())
await updateFn(session, changeRecord, area, leafStats)
}

static instance: MutableAreaDataSource
Expand Down
49 changes: 46 additions & 3 deletions src/model/MutableClimbDataSource.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,14 +2,18 @@ import muid, { MUUID } from 'uuid-mongodb'
import { UserInputError } from 'apollo-server'
import { ClientSession } from 'mongoose'

import { ClimbChangeDocType, ClimbChangeInputType, ClimbEditOperationType, IPitch } from '../db/ClimbTypes.js'
import { AreaDocumnent } from '../db/AreaTypes.js'
import { ClimbType, ClimbChangeDocType, ClimbChangeInputType, ClimbEditOperationType, IPitch } from '../db/ClimbTypes.js'
import ClimbDataSource from './ClimbDataSource.js'
import { createInstance as createExperimentalUserDataSource } from './ExperimentalUserDataSource.js'
import { sanitizeDisciplines, gradeContextToGradeScales, createGradeObject } from '../GradeUtils.js'
import { getClimbModel } from '../db/ClimbSchema.js'
import { ChangeRecordMetadataType } from '../db/ChangeLogType.js'
import { changelogDataSource } from './ChangeLogDataSource.js'
import { sanitize, sanitizeStrict } from '../utils/sanitize.js'
import MutableAreaDataSource from './MutableAreaDataSource.js'
import { aggregateCragStats } from '../db/utils/Aggregate.js'
import { getAreaModel } from '../db/AreaSchema.js'

export default class MutableClimbDataSource extends ClimbDataSource {
experimentalUserDataSource = createExperimentalUserDataSource()
Expand Down Expand Up @@ -219,8 +223,11 @@ export default class MutableClimbDataSource extends ClimbDataSource {
if (idList.length > 0) {
parent.set({ climbs: parent.climbs.concat(idList) })
}

await parent.save()

await updateStats(parent, session, _change)

if (idStrList.length === newClimbIds.length) {
return idStrList
}
Expand Down Expand Up @@ -266,11 +273,22 @@ export default class MutableClimbDataSource extends ClimbDataSource {
// see https://jira.mongodb.org/browse/NODE-2014
await session.withTransaction(
async (session) => {
const changeset = await changelogDataSource.create(session, userId, ClimbEditOperationType.deleteClimb)
const _change: ChangeRecordMetadataType = {
user: userId,
historyId: changeset._id,
operation: ClimbEditOperationType.deleteClimb,
seq: 0
}
// Remove climb IDs from parent.climbs[]
await this.areaModel.updateOne(
{ 'metadata.area_id': parentId },
{
$pullAll: { climbs: idList }
$pullAll: { climbs: idList },
$set: {
_change,
updatedBy: userId
}
},
{ session })

Expand All @@ -284,14 +302,16 @@ export default class MutableClimbDataSource extends ClimbDataSource {
[{
$set: {
_deleting: new Date(),
updatedBy: userId
updatedBy: userId,
_change
}
}],
{
upserted: false,
session
}).lean()
ret = rs.modifiedCount
await updateStats(parentId, session, _change)
})
return ret
}
Expand All @@ -307,3 +327,26 @@ export default class MutableClimbDataSource extends ClimbDataSource {
return MutableClimbDataSource.instance
}
}

/**
* Update stats for an area and its ancestors
* @param areaIdOrAreaCursor
* @param session
* @param changeRecord
*/
const updateStats = async (areaIdOrAreaCursor: MUUID | AreaDocumnent, session: ClientSession, changeRecord: ChangeRecordMetadataType): Promise<void> => {
let area: AreaDocumnent
if ((areaIdOrAreaCursor as AreaDocumnent).totalClimbs != null) {
area = areaIdOrAreaCursor as AreaDocumnent
} else {
area = await getAreaModel().findOne({ 'metadata.area_id': areaIdOrAreaCursor as MUUID }).session(session).orFail()
}

await area.populate<{ climbs: ClimbType[] }>({ path: 'climbs', model: getClimbModel() })
area.set({
totalClimbs: area.climbs.length,
aggregate: aggregateCragStats(area.toObject())
})
await area.save()
await MutableAreaDataSource.getInstance().updateStatsAndGeoDataForSinglePath(session, changeRecord, area)
}

0 comments on commit f62725b

Please sign in to comment.