From 511fa315c1aa15ada5e2530bfb6a2679ab9b4649 Mon Sep 17 00:00:00 2001 From: Xun Li Date: Wed, 20 Sep 2023 15:04:21 -0700 Subject: [PATCH 1/3] init --- src/constants/src/layers.ts | 3 +- src/layers/package.json | 1 + src/layers/src/geojson-layer/geojson-layer.ts | 32 ++- src/layers/src/geojson-layer/geojson-utils.ts | 73 ++++- src/layers/src/geojson-layer/polylabel.ts | 261 ++++++++++++++++++ src/reducers/src/vis-state-updaters.ts | 5 + src/table/src/kepler-table.ts | 1 + src/utils/src/filter-utils.ts | 33 ++- 8 files changed, 396 insertions(+), 13 deletions(-) create mode 100644 src/layers/src/geojson-layer/polylabel.ts diff --git a/src/constants/src/layers.ts b/src/constants/src/layers.ts index aed26df265..836e5a280f 100644 --- a/src/constants/src/layers.ts +++ b/src/constants/src/layers.ts @@ -503,5 +503,6 @@ export const EDITOR_AVAILABLE_LAYERS: string[] = [ LAYER_TYPES.hexagon, LAYER_TYPES.arc, LAYER_TYPES.line, - LAYER_TYPES.hexagonId + LAYER_TYPES.hexagonId, + LAYER_TYPES.geojson ]; diff --git a/src/layers/package.json b/src/layers/package.json index d399ad5ef6..2c0c5607c0 100644 --- a/src/layers/package.json +++ b/src/layers/package.json @@ -51,6 +51,7 @@ "@nebula.gl/layers": "1.0.2-alpha.1", "@nebula.gl/edit-modes": "1.0.2-alpha.1", "@turf/bbox": "^6.0.1", + "@turf/boolean-within": "^6.0.1", "@turf/helpers": "^6.1.4", "@types/geojson": "^7946.0.7", "@types/keymirror": "^0.1.1", diff --git a/src/layers/src/geojson-layer/geojson-layer.ts b/src/layers/src/geojson-layer/geojson-layer.ts index 6a03a5c1db..0521f65c9d 100644 --- a/src/layers/src/geojson-layer/geojson-layer.ts +++ b/src/layers/src/geojson-layer/geojson-layer.ts @@ -20,7 +20,9 @@ import uniq from 'lodash.uniq'; import {DATA_TYPES} from 'type-analyzer'; - +import {Feature, Polygon} from 'geojson'; +import booleanWithin from '@turf/boolean-within'; +import {point as turfPoint} from '@turf/helpers'; import Layer, { colorMaker, LayerBaseConfig, @@ -37,7 +39,8 @@ import { getGeojsonDataMaps, getGeojsonBounds, getGeojsonFeatureTypes, - GeojsonDataMaps + GeojsonDataMaps, + getGeojsonMeanCenters } from './geojson-utils'; import GeojsonLayerIcon from './geojson-layer-icon'; import { @@ -58,6 +61,7 @@ import { } from '@kepler.gl/types'; import {KeplerTable} from '@kepler.gl/table'; import {DataContainerInterface} from '@kepler.gl/utils'; +import { RowDataContainer } from 'src/utils/src/row-data-container'; const SUPPORTED_ANALYZER_TYPES = { [DATA_TYPES.GEOMETRY]: true, @@ -184,10 +188,12 @@ export default class GeoJsonLayer extends Layer { declare visConfigSettings: GeoJsonVisConfigSettings; declare meta: GeoJsonLayerMeta; dataToFeature: GeojsonDataMaps; + centroids: number[][]; constructor(props) { super(props); + this.centroids = []; this.dataToFeature = []; this.registerVisConfig(geojsonVisConfigs); this.getPositionAccessor = (dataContainer: DataContainerInterface) => @@ -359,9 +365,31 @@ export default class GeoJsonLayer extends Layer { }; } + getCentroids(): number[][] { + if (this.centroids.length === 0) { + this.centroids = getGeojsonMeanCenters(this.dataToFeature); + } + return this.centroids; + } + + isInPolygon(data: RowDataContainer, index: number, polygon: Feature): Boolean { + if (this.centroids.length === 0 || !this.centroids[index]) { + return false; + } + const isReactangle = polygon.properties?.shape === 'Rectangle'; + const point = this.centroids[index]; + // check if point is in polygon + if (isReactangle && polygon.properties?.bbox) { + const [minX, minY, maxX, maxY] = polygon.properties?.bbox; + return point[0] >= minX && point[0] <= maxX && point[1] >= minY && point[1] <= maxY; + } + return booleanWithin(turfPoint(point), polygon); + } + updateLayerMeta(dataContainer) { const getFeature = this.getPositionAccessor(dataContainer); this.dataToFeature = getGeojsonDataMaps(dataContainer, getFeature); + this.centroids = this.getCentroids(); // get bounds from features const bounds = getGeojsonBounds(this.dataToFeature); diff --git a/src/layers/src/geojson-layer/geojson-utils.ts b/src/layers/src/geojson-layer/geojson-utils.ts index 916d014bba..8170d306d2 100644 --- a/src/layers/src/geojson-layer/geojson-utils.ts +++ b/src/layers/src/geojson-layer/geojson-utils.ts @@ -24,7 +24,7 @@ import {parseSync} from '@loaders.gl/core'; import {WKBLoader, WKTLoader} from '@loaders.gl/wkt'; import {binaryToGeometry} from '@loaders.gl/gis'; -import {Feature, BBox} from 'geojson'; +import {Feature, BBox, Geometry} from 'geojson'; import {getSampleData, parseGeometryFromArrow, RawArrowFeature} from '@kepler.gl/utils'; export type GetFeature = (d: any) => Feature; @@ -119,6 +119,77 @@ export function getGeojsonDataMaps(dataContainer: any, getFeature: GetFeature): return dataToFeature; } +/** + * Get mean centers from geometries + * @param geometries geometries to get center from + * @returns + */ +function getMeanCenterFromGeometries(geometries: Geometry[]): number[] { + let centerX = 0; + let centerY = 0; + let numPoints = 0; + // iterate through geometries and get center + for (let i = 0; i < geometries.length; i++) { + const geometry = geometries[i]; + if (geometry.type === 'Point') { + return geometry.coordinates; + } else if (geometry.type === 'MultiPoint' || geometry.type === 'LineString') { + numPoints = geometry.coordinates.length; + for (let j = 0; j < numPoints; j++) { + const point = geometry.coordinates[j]; + centerX += point[0]; + centerY += point[1]; + } + } else if (geometry.type === 'MultiLineString' || geometry.type === 'Polygon') { + const numParts = geometry.coordinates.length; + for (let j = 0; j < numParts; j++) { + const part = geometry.coordinates[j]; + numPoints += part.length; + for (let k = 0; k < part.length; k++) { + const point = part[k]; + centerX += point[0]; + centerY += point[1]; + } + } + } else if (geometry.type === 'MultiPolygon') { + const numPolygons = geometry.coordinates.length; + for (let j = 0; j < numPolygons; j++) { + const numParts = geometry.coordinates[j].length; + for (let k = 0; k < numParts; k++) { + const part = geometry.coordinates[j][k]; + numPoints += part.length; + for (let l = 0; l < part.length; l++) { + const point = part[l]; + centerX += point[0]; + centerY += point[1]; + } + } + } + } + } + return numPoints > 0 ? [centerX / numPoints, centerY / numPoints] : []; +} + + +/** + * Get mean centroids of a geojson + * @param {GeojsonDataMaps} dataToFeature + * @returns {number[][]} [[lng, lat], ...] + */ +export function getGeojsonMeanCenters(dataToFeature: GeojsonDataMaps): number[][] { + const meanCenters: number[][] = []; + for (let i = 0; i < dataToFeature.length; i++) { + const feature = dataToFeature[i]; + if (feature) { + const geometries = feature.geometry.type === 'GeometryCollection' ? + feature.geometry.geometries : [feature.geometry]; + const center = getMeanCenterFromGeometries(geometries); + meanCenters.push(center); + } + } + return meanCenters; +} + /** * Parse geojson from string * @param {String} geoString diff --git a/src/layers/src/geojson-layer/polylabel.ts b/src/layers/src/geojson-layer/polylabel.ts new file mode 100644 index 0000000000..03c9a8c5ac --- /dev/null +++ b/src/layers/src/geojson-layer/polylabel.ts @@ -0,0 +1,261 @@ +// disable ts lint on this file +// @ts-nocheck + +// TinyQueue is a small footprint binary heap priority queue +class TinyQueue { + data: never[]; + length: any; + compare: (a: any, b: any) => 0 | 1 | -1; + constructor(data = [], compare = (a, b) => (a < b ? -1 : a > b ? 1 : 0)) { + this.data = data; + this.length = this.data.length; + this.compare = compare; + if (this.length > 0) { + for (let i = (this.length >> 1) - 1; i >= 0; i--) { + this._down(i); + } + } + } + + push(item) { + this.data.push(item); + this._up(this.length++); + } + + pop() { + if (this.length === 0) { + return undefined; + } + const top = this.data[0]; + const bottom = this.data.pop(); + if (--this.length > 0) { + this.data[0] = bottom; + this._down(0); + } + return top; + } + + peek() { + return this.data[0]; + } + + _up(pos) { + const {data, compare} = this; + const item = data[pos]; + + while (pos > 0) { + const parent = (pos - 1) >> 1; + const current = data[parent]; + if (compare(item, current) >= 0) { + break; + } + data[pos] = current; + pos = parent; + } + + data[pos] = item; + } + + _down(pos) { + const {data, compare} = this; + const halfLength = this.length >> 1; + const item = data[pos]; + + while (pos < halfLength) { + let bestChild = (pos << 1) + 1; // initially it is the left child + const right = bestChild + 1; + if (right < this.length && compare(data[right], data[bestChild]) < 0) { + bestChild = right; + } + if (compare(data[bestChild], item) >= 0) { + break; + } + data[pos] = data[bestChild]; + pos = bestChild; + } + data[pos] = item; + } +} + +function arrayDepth(v) { + return Array.isArray(v) ? 1 + Math.max(0, ...v.map(arrayDepth)) : 0; +} + +// Squared distance from a point to a segment +function getSegDistSq(px, py, a, b) { + let [x, y] = [a[0], a[1]]; + let [dx, dy] = [b[0] - x, b[1] - y]; + + if (dx !== 0 || dy !== 0) { + const t = ((px - x) * dx + (py - y) * dy) / (dx * dx + dy * dy); + if (t > 1) { + x = b[0]; + y = b[1]; + } else if (t > 0) { + x += dx * t; + y += dy * t; + } + } + dx = px - x; + dy = py - y; + return dx * dx + dy * dy; +} + +// Signed distance from a point to a polygon boundary (-ve if outside): +function pointToPolygonDist(x, y, polygon) { + let inside = false; + let minDistSq = Infinity; + + for (let k = 0; k < polygon.length; k++) { + const ring = polygon[k]; + + for (let i = 0, len = ring.length, j = len - 1; i < len; j = i++) { + const [a, b] = [ring[i], ring[j]]; + if (a[1] > y !== b[1] > y && x < ((b[0] - a[0]) * (y - a[1])) / (b[1] - a[1]) + a[0]) { + inside = !inside; + } + minDistSq = Math.min(minDistSq, getSegDistSq(x, y, a, b)); + } + } + return minDistSq === 0 ? 0 : (inside ? 1 : -1) * Math.sqrt(minDistSq); +} + +class Cell { + x: number; + y: number; + h: number; + d: number; + max: number; + + constructor(x: number, y: number, h: number, polygon: number[][] | number[][][] | number[][][][]) { + this.x = x; // cell center x + this.y = y; // cell center y + this.h = h; // half the cell size + this.d = pointToPolygonDist(x, y, polygon); // distance from cell center to polygon + this.max = this.d + this.h * Math.SQRT2; // max distance to polygon within a cell + } +} + +// Cell containing the polygon centroid +function getCentroidCell(polygon) { + let [area, x, y] = [0, 0, 0]; + const points = polygon[0]; + + for (let i = 0, len = points.length, j = len - 1; i < len; j = i++) { + const [a, b] = [points[i], points[j]]; + const f = a[0] * b[1] - b[0] * a[1]; + x += (a[0] + b[0]) * f; + y += (a[1] + b[1]) * f; + area += f * 3; + } + if (area === 0) { + return new Cell(points[0][0], points[0][1], 0, polygon); + } + return new Cell(x / area, y / area, 0, polygon); +} + +export function polylabel(poly, precision = 1, debug = false) { + let polygon = poly; + const depth = arrayDepth(poly); + + if (depth < 2) { + // No polygon provided. + return null; + } else if (depth == 2) { + // Single polygon. + polygon = [poly]; + } else { + // List of polygons. + let bestpoint = polylabel(polygon[0], precision, debug); + for (let p = 1; p < polygon.length; p++) { + const thispoint = polylabel(polygon[p], precision, debug); + if (thispoint[2] > bestpoint[2]) { + bestpoint = thispoint; + } + } + return bestpoint; + } + + // find the bounding box of the outer ring + let minX, minY, maxX, maxY; + for (let i = 0; i < polygon[0].length; i++) { + const p = polygon[0][i]; + if (!i || p[0] < minX) { + minX = p[0]; + } + if (!i || p[1] < minY) { + minY = p[1]; + } + if (!i || p[0] > maxX) { + maxX = p[0]; + } + if (!i || p[1] > maxY) { + maxY = p[1]; + } + } + + const [width, height] = [maxX - minX, maxY - minY]; + const cellSize = Math.min(width, height); + let h = cellSize / 2; + + if (cellSize === 0) { + const degeneratePoleOfInaccessibility = [minX, minY]; + degeneratePoleOfInaccessibility.distance = 0; + return degeneratePoleOfInaccessibility; + } + + // Priority queue of cells in order of their "potential" (max distance to polygon) + const cellQueue = new TinyQueue(undefined, (a, b) => b.max - a.max); + + // Cover polygon with initial cells + for (let x = minX; x < maxX; x += cellSize) { + for (let y = minY; y < maxY; y += cellSize) { + cellQueue.push(new Cell(x + h, y + h, h, polygon)); + } + } + + // Take centroid as the first best guess + let bestCell = getCentroidCell(polygon); + + // Special case for rectangular polygons + const bboxCell = new Cell(minX + width / 2, minY + height / 2, 0, polygon); + if (bboxCell.d > bestCell.d) { + bestCell = bboxCell; + } + + let numProbes = cellQueue.length; + + while (cellQueue.length) { + // Pick the most promising cell from the queue + const cell = cellQueue.pop(); + + // Update the best cell if we found a better one + if (cell.d > bestCell.d) { + bestCell = cell; + if (debug) { + console.log('found best %d after %d probes', Math.round(1e4 * cell.d) / 1e4, numProbes); + } + } + + // Do not drill down further if there's no chance of a better solution + if (cell.max - bestCell.d <= precision) { + continue; + } + // Split the cell into four cells + h = cell.h / 2; + cellQueue.push(new Cell(cell.x - h, cell.y - h, h, polygon)); + cellQueue.push(new Cell(cell.x + h, cell.y - h, h, polygon)); + cellQueue.push(new Cell(cell.x - h, cell.y + h, h, polygon)); + cellQueue.push(new Cell(cell.x + h, cell.y + h, h, polygon)); + numProbes += 4; + } + + if (debug) { + console.log('num probes: ' + numProbes); + console.log('best distance: ' + bestCell.d); + console.log('best point: [' + bestCell.x + ', ' + bestCell.y + ']'); + console.log(polygon); + } + + return [bestCell.x, bestCell.y, bestCell.d]; +} diff --git a/src/reducers/src/vis-state-updaters.ts b/src/reducers/src/vis-state-updaters.ts index d87c66cf1a..42f75964e2 100644 --- a/src/reducers/src/vis-state-updaters.ts +++ b/src/reducers/src/vis-state-updaters.ts @@ -18,6 +18,7 @@ // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN // THE SOFTWARE. +import bbox from '@turf/bbox'; import {console as Console} from 'global/window'; import {disableStackCapturing, withTask} from 'react-palm/tasks'; import cloneDeep from 'lodash.clonedeep'; @@ -2317,6 +2318,8 @@ export function setFeaturesUpdater( // if feature is part of a filter const filterId = feature && getFilterIdInFeature(feature); if (filterId && feature) { + // update bbox in feature.properties if GeoJson feature + if (feature.properties) feature.properties.bbox = bbox(feature); const featureValue = featureToFilterValue(feature, filterId); const filterIdx = state.filters.findIndex(fil => fil.id === filterId); // @ts-ignore @@ -2338,6 +2341,8 @@ export const setSelectedFeatureUpdater = ( state: VisState, {feature, selectionContext}: VisStateActions.SetSelectedFeatureUpdaterAction ): VisState => { + // update bbox in feature.properties if GeoJson feature + if (feature && feature.properties) feature.properties.bbox = bbox(feature); return { ...state, editor: { diff --git a/src/table/src/kepler-table.ts b/src/table/src/kepler-table.ts index f0083f6603..09b694598d 100644 --- a/src/table/src/kepler-table.ts +++ b/src/table/src/kepler-table.ts @@ -329,6 +329,7 @@ class KeplerTable { }; }, {}); + // TODO apply index to narrow down the filter range filterResult = filterDataByFilterTypes( {dynamicDomainFilters, cpuFilters, filterFuncs}, dataContainer diff --git a/src/utils/src/filter-utils.ts b/src/utils/src/filter-utils.ts index 5d10341a8f..993aa664e5 100644 --- a/src/utils/src/filter-utils.ts +++ b/src/utils/src/filter-utils.ts @@ -454,6 +454,14 @@ export const getPolygonFilterFunctor = (layer, filter, dataContainer) => { const pos = getCentroid({id}); return pos.every(Number.isFinite) && isInPolygon(pos, filter.value); }; + case LAYER_TYPES.geojson: + return data => { + if (layer.isInPolygon) { + return layer.isInPolygon(data, data.index, filter.value); + } + // show all geometries if can't apply filter + return true; + }; default: return () => true; } @@ -545,17 +553,24 @@ export function filterDataByFilterTypes( const filterFuncCaller = (filter: Filter) => filterFuncs[filter.id](filterContext); const numRows = dataContainer.numRows(); - for (let i = 0; i < numRows; ++i) { - filterContext.index = i; + if (dynamicDomainFilters) { + for (let i = 0; i < numRows; ++i) { + filterContext.index = i; - const matchForDomain = dynamicDomainFilters && dynamicDomainFilters.every(filterFuncCaller); - if (matchForDomain) { - filteredIndexForDomain.push(filterContext.index); + const matchForDomain = dynamicDomainFilters && dynamicDomainFilters.every(filterFuncCaller); + if (matchForDomain) { + filteredIndexForDomain.push(filterContext.index); + } } - - const matchForRender = cpuFilters && cpuFilters.every(filterFuncCaller); - if (matchForRender) { - filteredIndex.push(filterContext.index); + } + // TODO: with a index, we should be able to avoid iterate through all data + if (cpuFilters) { + for (let i = 0; i < numRows; ++i) { + filterContext.index = i; + const matchForRender = cpuFilters && cpuFilters.every(filterFuncCaller); + if (matchForRender) { + filteredIndex.push(filterContext.index); + } } } From 7af4f241eea3033a9608dae73ca436a501132309 Mon Sep 17 00:00:00 2001 From: Xun Li Date: Thu, 21 Sep 2023 15:25:16 -0700 Subject: [PATCH 2/3] wip --- src/layers/package.json | 5 ++- src/layers/src/geojson-layer/geojson-layer.ts | 18 ++++++++- src/layers/src/geojson-layer/geojson-utils.ts | 6 +++ src/table/src/kepler-table.ts | 39 +++++++++++++++---- src/utils/package.json | 2 +- src/utils/src/filter-utils.ts | 8 +++- src/utils/src/indexed-data-container.ts | 2 +- 7 files changed, 66 insertions(+), 14 deletions(-) diff --git a/src/layers/package.json b/src/layers/package.json index 2c0c5607c0..e8dd5fb1ef 100644 --- a/src/layers/package.json +++ b/src/layers/package.json @@ -48,10 +48,12 @@ "@loaders.gl/wkt": "^3.0.9", "@luma.gl/constants": "^8.5.10", "@mapbox/geojson-normalize": "0.0.1", - "@nebula.gl/layers": "1.0.2-alpha.1", "@nebula.gl/edit-modes": "1.0.2-alpha.1", + "@nebula.gl/layers": "1.0.2-alpha.1", "@turf/bbox": "^6.0.1", "@turf/boolean-within": "^6.0.1", + "@turf/center": "^6.1.4", + "@turf/center-of-mass": "^6.1.4", "@turf/helpers": "^6.1.4", "@types/geojson": "^7946.0.7", "@types/keymirror": "^0.1.1", @@ -59,6 +61,7 @@ "@types/lodash.uniq": "^4.5.7", "@types/styled-components": "^5.1.25", "d3-shape": "^1.2.0", + "flatbush": "^4.2.0", "global": "^4.3.0", "keymirror": "^0.1.1", "lodash.memoize": "^4.1.2", diff --git a/src/layers/src/geojson-layer/geojson-layer.ts b/src/layers/src/geojson-layer/geojson-layer.ts index 0521f65c9d..cd17b794ae 100644 --- a/src/layers/src/geojson-layer/geojson-layer.ts +++ b/src/layers/src/geojson-layer/geojson-layer.ts @@ -18,6 +18,7 @@ // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN // THE SOFTWARE. +import Flatbush from 'flatbush'; import uniq from 'lodash.uniq'; import {DATA_TYPES} from 'type-analyzer'; import {Feature, Polygon} from 'geojson'; @@ -189,6 +190,7 @@ export default class GeoJsonLayer extends Layer { declare meta: GeoJsonLayerMeta; dataToFeature: GeojsonDataMaps; centroids: number[][]; + static index = new Map(); constructor(props) { super(props); @@ -365,9 +367,22 @@ export default class GeoJsonLayer extends Layer { }; } + getSpatialIndex() { + if (!GeoJsonLayer.index.get(this.id) && this.centroids.length > 0) { + const index = new Flatbush(this.centroids.length); + this.centroids.forEach(c => index?.add(c[0], c[1], c[0], c[1])); + index.finish(); + GeoJsonLayer.index.set(this.id, index); + } + return GeoJsonLayer.index.get(this.id); + } + getCentroids(): number[][] { if (this.centroids.length === 0) { this.centroids = getGeojsonMeanCenters(this.dataToFeature); + console.time('build spatial index'); + this.getSpatialIndex(); + console.timeEnd('build spatial index'); } return this.centroids; } @@ -376,6 +391,8 @@ export default class GeoJsonLayer extends Layer { if (this.centroids.length === 0 || !this.centroids[index]) { return false; } + // check if index is in existed spatial index query + const isReactangle = polygon.properties?.shape === 'Rectangle'; const point = this.centroids[index]; // check if point is in polygon @@ -390,7 +407,6 @@ export default class GeoJsonLayer extends Layer { const getFeature = this.getPositionAccessor(dataContainer); this.dataToFeature = getGeojsonDataMaps(dataContainer, getFeature); this.centroids = this.getCentroids(); - // get bounds from features const bounds = getGeojsonBounds(this.dataToFeature); // if any of the feature has properties.radius set to be true diff --git a/src/layers/src/geojson-layer/geojson-utils.ts b/src/layers/src/geojson-layer/geojson-utils.ts index 8170d306d2..27213a3f97 100644 --- a/src/layers/src/geojson-layer/geojson-utils.ts +++ b/src/layers/src/geojson-layer/geojson-utils.ts @@ -20,6 +20,8 @@ import normalize from '@mapbox/geojson-normalize'; import bbox from '@turf/bbox'; +// import center from '@turf/center'; +// import centerOfMass from '@turf/center-of-mass'; import {parseSync} from '@loaders.gl/core'; import {WKBLoader, WKTLoader} from '@loaders.gl/wkt'; import {binaryToGeometry} from '@loaders.gl/gis'; @@ -177,16 +179,20 @@ function getMeanCenterFromGeometries(geometries: Geometry[]): number[] { * @returns {number[][]} [[lng, lat], ...] */ export function getGeojsonMeanCenters(dataToFeature: GeojsonDataMaps): number[][] { + console.time('getGeojsonMeanCenters'); const meanCenters: number[][] = []; for (let i = 0; i < dataToFeature.length; i++) { const feature = dataToFeature[i]; if (feature) { + // TODO: use line interpolate to get center of line for LineString + // meanCenters.push(centerOfMass(feature).geometry.coordinates); const geometries = feature.geometry.type === 'GeometryCollection' ? feature.geometry.geometries : [feature.geometry]; const center = getMeanCenterFromGeometries(geometries); meanCenters.push(center); } } + console.timeEnd('getGeojsonMeanCenters'); return meanCenters; } diff --git a/src/table/src/kepler-table.ts b/src/table/src/kepler-table.ts index 09b694598d..b169b0160e 100644 --- a/src/table/src/kepler-table.ts +++ b/src/table/src/kepler-table.ts @@ -20,13 +20,14 @@ import {console as Console} from 'global/console'; import {ascending, descending} from 'd3-array'; - +import {intersection} from 'lodash'; import { TRIP_POINT_FIELDS, SORT_ORDER, ALL_FIELD_TYPES, ALTITUDE_FIELDS, - SCALE_TYPES + SCALE_TYPES, + FILTER_TYPES } from '@kepler.gl/constants'; import { RGBColor, @@ -41,7 +42,7 @@ import { import {getGpuFilterProps, getDatasetFieldIndexForFilter} from './gpu-filter-utils'; -import {Layer} from '@kepler.gl/layers'; +import {Layer, KeplerGlLayers} from '@kepler.gl/layers'; import { generateHashId, getSortingFunction, @@ -59,7 +60,8 @@ import { getLogDomain, getOrdinalDomain, getQuantileDomain, - DataContainerInterface + DataContainerInterface, + createIndexedDataContainer } from '@kepler.gl/utils'; export type GpuFilter = { @@ -315,30 +317,51 @@ class KeplerTable { const shouldCalIndex = Boolean(this.changedFilters.cpu); let filterResult: FilterResult = {}; + console.time('Filter'); if (shouldCalDomain || shouldCalIndex) { const dynamicDomainFilters = shouldCalDomain ? filterRecord.dynamicDomain : null; const cpuFilters = shouldCalIndex ? filterRecord.cpu : null; + // use layer index to build IndexedDataContainer to narrow down the data + console.time('filterDataByFilterTypes'); + const preFilteredIndexes: number[][] = []; + for (const filter of filters) { + if (filter.type === FILTER_TYPES.polygon) { + const filteredLayers = layers.filter(l => l.config.dataId === this.id); + // iterator over filteredLayers, and use it's spatial index to filter data + filteredLayers?.forEach(layer => { + const index = KeplerGlLayers.GeojsonLayer.index.get(layer.id); + const [minX, minY, maxX, maxY] = filter.value.properties.bbox; + const foundIndexes = index?.search(minX, minY, maxX, maxY) || []; + preFilteredIndexes.push(foundIndexes); + }); + } + } + console.timeEnd('filterDataByFilterTypes'); + const filteredDataContainer = + preFilteredIndexes.length > 0 + ? createIndexedDataContainer(dataContainer, intersection(preFilteredIndexes.flat())) + : dataContainer; + const filterFuncs = filters.reduce((acc, filter) => { const fieldIndex = getDatasetFieldIndexForFilter(this.id, filter); const field = fieldIndex !== -1 ? fields[fieldIndex] : null; - return { ...acc, - [filter.id]: getFilterFunction(field, this.id, filter, layers, dataContainer) + [filter.id]: getFilterFunction(field, this.id, filter, layers, filteredDataContainer) }; }, {}); - // TODO apply index to narrow down the filter range filterResult = filterDataByFilterTypes( {dynamicDomainFilters, cpuFilters, filterFuncs}, - dataContainer + filteredDataContainer ); } this.filteredIndex = filterResult.filteredIndex || this.filteredIndex; this.filteredIndexForDomain = filterResult.filteredIndexForDomain || this.filteredIndexForDomain; + console.timeEnd('Filter'); return this; } diff --git a/src/utils/package.json b/src/utils/package.json index 90e9c6ed0f..ee270e8ce2 100644 --- a/src/utils/package.json +++ b/src/utils/package.json @@ -32,8 +32,8 @@ "dependencies": { "@kepler.gl/constants": "3.0.0-alpha.0", "@kepler.gl/types": "3.0.0-alpha.0", - "@loaders.gl/wkt": "3.0.9", "@loaders.gl/gis": "3.0.9", + "@loaders.gl/wkt": "3.0.9", "@luma.gl/constants": "^8.5.10", "@luma.gl/core": "^8.5.10", "@mapbox/geo-viewport": "^0.4.1", diff --git a/src/utils/src/filter-utils.ts b/src/utils/src/filter-utils.ts index 993aa664e5..f3c2ac94ca 100644 --- a/src/utils/src/filter-utils.ts +++ b/src/utils/src/filter-utils.ts @@ -456,6 +456,9 @@ export const getPolygonFilterFunctor = (layer, filter, dataContainer) => { }; case LAYER_TYPES.geojson: return data => { + if (filter.value.properties?.shape === 'Rectangle') { + return true; + } if (layer.isInPolygon) { return layer.isInPolygon(data, data.index, filter.value); } @@ -553,9 +556,10 @@ export function filterDataByFilterTypes( const filterFuncCaller = (filter: Filter) => filterFuncs[filter.id](filterContext); const numRows = dataContainer.numRows(); + const plainIndexes = dataContainer.getPlainIndex(); if (dynamicDomainFilters) { for (let i = 0; i < numRows; ++i) { - filterContext.index = i; + filterContext.index = plainIndexes[i]; const matchForDomain = dynamicDomainFilters && dynamicDomainFilters.every(filterFuncCaller); if (matchForDomain) { @@ -566,7 +570,7 @@ export function filterDataByFilterTypes( // TODO: with a index, we should be able to avoid iterate through all data if (cpuFilters) { for (let i = 0; i < numRows; ++i) { - filterContext.index = i; + filterContext.index = plainIndexes[i]; const matchForRender = cpuFilters && cpuFilters.every(filterFuncCaller); if (matchForRender) { filteredIndex.push(filterContext.index); diff --git a/src/utils/src/indexed-data-container.ts b/src/utils/src/indexed-data-container.ts index d2a6f09e6f..96a75bf112 100644 --- a/src/utils/src/indexed-data-container.ts +++ b/src/utils/src/indexed-data-container.ts @@ -110,7 +110,7 @@ export class IndexedDataContainer implements DataContainerInterface { } getPlainIndex(): number[] { - return this._indices.map((_, i) => i); + return this._indices; } flattenData(): any[][] { From 14d1452a1f3a323c87aa92eba453db635e6280e8 Mon Sep 17 00:00:00 2001 From: Xun Li Date: Wed, 27 Sep 2023 10:50:10 -0700 Subject: [PATCH 3/3] wip --- src/layers/src/geojson-layer/geojson-layer.ts | 85 ++++++++++++++----- src/layers/src/geojson-layer/geojson-utils.ts | 9 +- src/reducers/src/vis-state-updaters.ts | 5 ++ src/table/src/gpu-filter-utils.ts | 3 + src/table/src/kepler-table.ts | 29 +------ src/types/reducers.d.ts | 1 + src/utils/src/filter-utils.ts | 69 +++++++++------ 7 files changed, 123 insertions(+), 78 deletions(-) diff --git a/src/layers/src/geojson-layer/geojson-layer.ts b/src/layers/src/geojson-layer/geojson-layer.ts index cd17b794ae..bf13102afe 100644 --- a/src/layers/src/geojson-layer/geojson-layer.ts +++ b/src/layers/src/geojson-layer/geojson-layer.ts @@ -49,7 +49,8 @@ import { HIGHLIGH_COLOR_3D, CHANNEL_SCALES, ColorRange, - LAYER_VIS_CONFIGS + LAYER_VIS_CONFIGS, + FILTER_TYPES } from '@kepler.gl/constants'; import { VisConfigNumber, @@ -58,11 +59,12 @@ import { VisConfigRange, VisConfigBoolean, Merge, - RGBColor + RGBColor, + PolygonFilter } from '@kepler.gl/types'; import {KeplerTable} from '@kepler.gl/table'; import {DataContainerInterface} from '@kepler.gl/utils'; -import { RowDataContainer } from 'src/utils/src/row-data-container'; +import {RowDataContainer} from 'src/utils/src/row-data-container'; const SUPPORTED_ANALYZER_TYPES = { [DATA_TYPES.GEOMETRY]: true, @@ -184,17 +186,24 @@ export const defaultElevation = 500; export const defaultLineWidth = 1; export const defaultRadius = 1; +export type SpatialIndexProps = { + index: Flatbush; + search: (filter: PolygonFilter, layer: GeoJsonLayer) => number[]; +}; + export default class GeoJsonLayer extends Layer { declare config: GeoJsonLayerConfig; declare visConfigSettings: GeoJsonVisConfigSettings; declare meta: GeoJsonLayerMeta; dataToFeature: GeojsonDataMaps; centroids: number[][]; - static index = new Map(); + static spatialIndex = new Map(); + queryIndexes: Map; constructor(props) { super(props); + this.queryIndexes = new Map(); this.centroids = []; this.dataToFeature = []; this.registerVisConfig(geojsonVisConfigs); @@ -368,13 +377,42 @@ export default class GeoJsonLayer extends Layer { } getSpatialIndex() { - if (!GeoJsonLayer.index.get(this.id) && this.centroids.length > 0) { + if (!GeoJsonLayer.spatialIndex.get(this.id) && this.centroids.length > 0) { + console.time('create spatial index'); const index = new Flatbush(this.centroids.length); this.centroids.forEach(c => index?.add(c[0], c[1], c[0], c[1])); index.finish(); - GeoJsonLayer.index.set(this.id, index); + GeoJsonLayer.spatialIndex.set(this.id, { + index, + search: (filter: PolygonFilter, layer: GeoJsonLayer): number[] => { + console.time('search'); + const [minX, minY, maxX, maxY] = filter.value.properties.bbox; + const foundIndexes = index?.search(minX, minY, maxX, maxY) || []; + layer.queryIndexes.clear(); + if (filter.value.properties?.shape === 'Rectangle') { + foundIndexes.forEach(i => layer.queryIndexes.set(i, true)); + } else { + // use turf.js to check if point is in polygon + foundIndexes.forEach(i => { + const point = layer.centroids[i]; + if (booleanWithin(turfPoint(point), filter.value)) { + layer.queryIndexes.set(i, true); + } + }); + } + layer.queryIndexes.forEach((v, k) => { + const feat = layer.dataToFeature[k]; + if (feat?.properties) { + feat.properties.selected = 1; + } + }); + console.timeEnd('search'); + return foundIndexes; + } + }); + console.timeEnd('create spatial index'); } - return GeoJsonLayer.index.get(this.id); + return GeoJsonLayer.spatialIndex.get(this.id); } getCentroids(): number[][] { @@ -387,21 +425,24 @@ export default class GeoJsonLayer extends Layer { return this.centroids; } - isInPolygon(data: RowDataContainer, index: number, polygon: Feature): Boolean { - if (this.centroids.length === 0 || !this.centroids[index]) { - return false; - } - // check if index is in existed spatial index query - - const isReactangle = polygon.properties?.shape === 'Rectangle'; - const point = this.centroids[index]; - // check if point is in polygon - if (isReactangle && polygon.properties?.bbox) { - const [minX, minY, maxX, maxY] = polygon.properties?.bbox; - return point[0] >= minX && point[0] <= maxX && point[1] >= minY && point[1] <= maxY; - } - return booleanWithin(turfPoint(point), polygon); - } + // isInPolygon(data: RowDataContainer, index: number, polygon: Feature): Boolean { + // if (this.centroids.length === 0 || !this.centroids[index]) { + // return false; + // } + // const isReactangle = polygon.properties?.shape === 'Rectangle'; + // const point = this.centroids[index]; + // // check if point is in polygon using spatialIndex + // if (isReactangle && GeoJsonLayer.spatialIndex.get(this.id)) { + // const found = this.queryIndexes.get(index); + // return found || false; + // } + // // without spatialIndex, use turf.js + // if (isReactangle && polygon.properties?.bbox) { + // const [minX, minY, maxX, maxY] = polygon.properties?.bbox; + // return point[0] >= minX && point[0] <= maxX && point[1] >= minY && point[1] <= maxY; + // } + // return booleanWithin(turfPoint(point), polygon); + // } updateLayerMeta(dataContainer) { const getFeature = this.getPositionAccessor(dataContainer); diff --git a/src/layers/src/geojson-layer/geojson-utils.ts b/src/layers/src/geojson-layer/geojson-utils.ts index 27213a3f97..b74ed5099d 100644 --- a/src/layers/src/geojson-layer/geojson-utils.ts +++ b/src/layers/src/geojson-layer/geojson-utils.ts @@ -20,8 +20,8 @@ import normalize from '@mapbox/geojson-normalize'; import bbox from '@turf/bbox'; -// import center from '@turf/center'; -// import centerOfMass from '@turf/center-of-mass'; +import center from '@turf/center'; +import centerOfMass from '@turf/center-of-mass'; import {parseSync} from '@loaders.gl/core'; import {WKBLoader, WKTLoader} from '@loaders.gl/wkt'; import {binaryToGeometry} from '@loaders.gl/gis'; @@ -188,8 +188,9 @@ export function getGeojsonMeanCenters(dataToFeature: GeojsonDataMaps): number[][ // meanCenters.push(centerOfMass(feature).geometry.coordinates); const geometries = feature.geometry.type === 'GeometryCollection' ? feature.geometry.geometries : [feature.geometry]; - const center = getMeanCenterFromGeometries(geometries); - meanCenters.push(center); + // const cent = getMeanCenterFromGeometries(geometries); + const cent = centerOfMass(feature).geometry.coordinates; + meanCenters.push(cent); } } console.timeEnd('getGeojsonMeanCenters'); diff --git a/src/reducers/src/vis-state-updaters.ts b/src/reducers/src/vis-state-updaters.ts index 42f75964e2..14aee2a973 100644 --- a/src/reducers/src/vis-state-updaters.ts +++ b/src/reducers/src/vis-state-updaters.ts @@ -858,6 +858,11 @@ export function setFilterUpdater( dataId: newDataIds }; + // if (newFilter.gpu) { + // newFilter = setFilterGpuMode(newFilter, state.filters); + // newFilter = assignGpuChannel(newFilter, state.filters); + // } + break; default: break; diff --git a/src/table/src/gpu-filter-utils.ts b/src/table/src/gpu-filter-utils.ts index e212a9ed64..d4d630fd1c 100644 --- a/src/table/src/gpu-filter-utils.ts +++ b/src/table/src/gpu-filter-utils.ts @@ -183,6 +183,9 @@ const getFilterValueAccessor = ( if (!filter) { return 0; } + // if (filter.type === FILTER_TYPES.polygon) { + // return getData(dc, d, -1, filter); + // } const fieldIndex = getDatasetFieldIndexForFilter(dataId, filter); const field = fields[fieldIndex]; diff --git a/src/table/src/kepler-table.ts b/src/table/src/kepler-table.ts index b169b0160e..0fc6b884ce 100644 --- a/src/table/src/kepler-table.ts +++ b/src/table/src/kepler-table.ts @@ -20,7 +20,6 @@ import {console as Console} from 'global/console'; import {ascending, descending} from 'd3-array'; -import {intersection} from 'lodash'; import { TRIP_POINT_FIELDS, SORT_ORDER, @@ -60,8 +59,7 @@ import { getLogDomain, getOrdinalDomain, getQuantileDomain, - DataContainerInterface, - createIndexedDataContainer + DataContainerInterface } from '@kepler.gl/utils'; export type GpuFilter = { @@ -322,39 +320,18 @@ class KeplerTable { const dynamicDomainFilters = shouldCalDomain ? filterRecord.dynamicDomain : null; const cpuFilters = shouldCalIndex ? filterRecord.cpu : null; - // use layer index to build IndexedDataContainer to narrow down the data - console.time('filterDataByFilterTypes'); - const preFilteredIndexes: number[][] = []; - for (const filter of filters) { - if (filter.type === FILTER_TYPES.polygon) { - const filteredLayers = layers.filter(l => l.config.dataId === this.id); - // iterator over filteredLayers, and use it's spatial index to filter data - filteredLayers?.forEach(layer => { - const index = KeplerGlLayers.GeojsonLayer.index.get(layer.id); - const [minX, minY, maxX, maxY] = filter.value.properties.bbox; - const foundIndexes = index?.search(minX, minY, maxX, maxY) || []; - preFilteredIndexes.push(foundIndexes); - }); - } - } - console.timeEnd('filterDataByFilterTypes'); - const filteredDataContainer = - preFilteredIndexes.length > 0 - ? createIndexedDataContainer(dataContainer, intersection(preFilteredIndexes.flat())) - : dataContainer; - const filterFuncs = filters.reduce((acc, filter) => { const fieldIndex = getDatasetFieldIndexForFilter(this.id, filter); const field = fieldIndex !== -1 ? fields[fieldIndex] : null; return { ...acc, - [filter.id]: getFilterFunction(field, this.id, filter, layers, filteredDataContainer) + [filter.id]: getFilterFunction(field, this.id, filter, layers, dataContainer) }; }, {}); filterResult = filterDataByFilterTypes( {dynamicDomainFilters, cpuFilters, filterFuncs}, - filteredDataContainer + opt?.filteredDataContainer ? opt.filteredDataContainer : dataContainer ); } diff --git a/src/types/reducers.d.ts b/src/types/reducers.d.ts index aa4ee2ef7f..066b1c7c35 100644 --- a/src/types/reducers.d.ts +++ b/src/types/reducers.d.ts @@ -468,6 +468,7 @@ export type FilterDatasetOpt = { cpuOnly?: boolean; // ignore filter for domain calculation ignoreDomain?: boolean; + filteredDataContainer?: any; }; /* DUPLICATES OF FILTER TYPES ABOVE, REMOVE ONCE TYPES ABOVE ARE FIXED */ diff --git a/src/utils/src/filter-utils.ts b/src/utils/src/filter-utils.ts index f3c2ac94ca..cb285cc89a 100644 --- a/src/utils/src/filter-utils.ts +++ b/src/utils/src/filter-utils.ts @@ -36,6 +36,7 @@ import { FILTER_VIEW_TYPES } from '@kepler.gl/constants'; import {VisState} from '@kepler.gl/schemas'; +import {KeplerGlLayers} from '@kepler.gl/layers'; import * as ScaleUtils from './data-scale-utils'; import {h3IsValid} from 'h3-js'; @@ -63,6 +64,7 @@ import {DataContainerInterface} from './data-container-interface'; import {generateHashId, set, toArray} from './utils'; import {notNullorUndefined, timeToUnixMilli, unique} from './data-utils'; import {getCentroid} from './h3-utils'; +import { IndexedDataContainer } from './indexed-data-container'; export const durationSecond = 1000; export const durationMinute = durationSecond * 60; @@ -456,13 +458,7 @@ export const getPolygonFilterFunctor = (layer, filter, dataContainer) => { }; case LAYER_TYPES.geojson: return data => { - if (filter.value.properties?.shape === 'Rectangle') { - return true; - } - if (layer.isInPolygon) { - return layer.isInPolygon(data, data.index, filter.value); - } - // show all geometries if can't apply filter + // data are already filtered by spatial index return true; }; default: @@ -557,24 +553,17 @@ export function filterDataByFilterTypes( const numRows = dataContainer.numRows(); const plainIndexes = dataContainer.getPlainIndex(); - if (dynamicDomainFilters) { - for (let i = 0; i < numRows; ++i) { - filterContext.index = plainIndexes[i]; + for (let i = 0; i < numRows; ++i) { + filterContext.index = plainIndexes[i]; - const matchForDomain = dynamicDomainFilters && dynamicDomainFilters.every(filterFuncCaller); - if (matchForDomain) { - filteredIndexForDomain.push(filterContext.index); - } + const matchForDomain = dynamicDomainFilters && dynamicDomainFilters.every(filterFuncCaller); + if (matchForDomain) { + filteredIndexForDomain.push(filterContext.index); } - } - // TODO: with a index, we should be able to avoid iterate through all data - if (cpuFilters) { - for (let i = 0; i < numRows; ++i) { - filterContext.index = plainIndexes[i]; - const matchForRender = cpuFilters && cpuFilters.every(filterFuncCaller); - if (matchForRender) { - filteredIndex.push(filterContext.index); - } + + const matchForRender = cpuFilters && cpuFilters.every(filterFuncCaller); + if (matchForRender) { + filteredIndex.push(filterContext.index); } } @@ -968,7 +957,7 @@ export function getDefaultFilterPlotType(filter: Filter): string | null { */ export function applyFiltersToDatasets< K extends KeplerTableModel, - L extends {config: {dataId: string | null}} + L extends {config: {dataId: string | null}, id: string} >( datasetIds: string[], datasets: {[id: string]: K}, @@ -981,9 +970,36 @@ export function applyFiltersToDatasets< const appliedFilters = filters.filter(d => shouldApplyFilter(d, dataId)); const table = datasets[dataId]; + // use layer index to build IndexedDataContainer to narrow down the data for filtering + console.time('filterDataByFilterTypes'); + const preFilteredIndexes: number[][] = []; + for (const filter of filters) { + if (filter.type === FILTER_TYPES.polygon) { + // iterator over filteredLayers, and use it's spatial index to filter data + layersToFilter?.forEach(layer => { + const index = KeplerGlLayers.GeojsonLayer.spatialIndex.get(layer.id); + if (index) { + const foundIndexes = index.search(filter, layer) || []; + preFilteredIndexes.push(foundIndexes); + } + }); + } + } + const filteredDataContainerIndexes = {}; + // iterator preFilterIndexes + for (let i = 0; i < preFilteredIndexes.length; i++) { + const indexes = preFilteredIndexes[i]; + for (let j = 0; j < indexes.length; j++) { + const index = indexes[j]; + filteredDataContainerIndexes[index] = index; + } + } + const filteredDataContainer = Object.keys(filteredDataContainerIndexes).length > 0 ? new IndexedDataContainer(table.dataContainer, Object.values(filteredDataContainerIndexes)) : null; + console.timeEnd('filterDataByFilterTypes'); + return { ...acc, - [dataId]: table.filterTable(appliedFilters, layersToFilter, {}) + [dataId]: table.filterTable(appliedFilters, layersToFilter, filteredDataContainer ? {filteredDataContainer} : {}) }; }, datasets); } @@ -1127,7 +1143,8 @@ export function generatePolygonFilter< type: FILTER_TYPES.polygon, name, layerId, - value: featureToFilterValue(feature, filter.id, {isVisible: true}) + value: featureToFilterValue(feature, filter.id, {isVisible: true}), + gpu: false }; }