From 14d1452a1f3a323c87aa92eba453db635e6280e8 Mon Sep 17 00:00:00 2001 From: Xun Li Date: Wed, 27 Sep 2023 10:50:10 -0700 Subject: [PATCH 1/3] wip --- src/layers/src/geojson-layer/geojson-layer.ts | 85 ++++++++++++++----- src/layers/src/geojson-layer/geojson-utils.ts | 9 +- src/reducers/src/vis-state-updaters.ts | 5 ++ src/table/src/gpu-filter-utils.ts | 3 + src/table/src/kepler-table.ts | 29 +------ src/types/reducers.d.ts | 1 + src/utils/src/filter-utils.ts | 69 +++++++++------ 7 files changed, 123 insertions(+), 78 deletions(-) diff --git a/src/layers/src/geojson-layer/geojson-layer.ts b/src/layers/src/geojson-layer/geojson-layer.ts index cd17b794ae..bf13102afe 100644 --- a/src/layers/src/geojson-layer/geojson-layer.ts +++ b/src/layers/src/geojson-layer/geojson-layer.ts @@ -49,7 +49,8 @@ import { HIGHLIGH_COLOR_3D, CHANNEL_SCALES, ColorRange, - LAYER_VIS_CONFIGS + LAYER_VIS_CONFIGS, + FILTER_TYPES } from '@kepler.gl/constants'; import { VisConfigNumber, @@ -58,11 +59,12 @@ import { VisConfigRange, VisConfigBoolean, Merge, - RGBColor + RGBColor, + PolygonFilter } from '@kepler.gl/types'; import {KeplerTable} from '@kepler.gl/table'; import {DataContainerInterface} from '@kepler.gl/utils'; -import { RowDataContainer } from 'src/utils/src/row-data-container'; +import {RowDataContainer} from 'src/utils/src/row-data-container'; const SUPPORTED_ANALYZER_TYPES = { [DATA_TYPES.GEOMETRY]: true, @@ -184,17 +186,24 @@ export const defaultElevation = 500; export const defaultLineWidth = 1; export const defaultRadius = 1; +export type SpatialIndexProps = { + index: Flatbush; + search: (filter: PolygonFilter, layer: GeoJsonLayer) => number[]; +}; + export default class GeoJsonLayer extends Layer { declare config: GeoJsonLayerConfig; declare visConfigSettings: GeoJsonVisConfigSettings; declare meta: GeoJsonLayerMeta; dataToFeature: GeojsonDataMaps; centroids: number[][]; - static index = new Map(); + static spatialIndex = new Map(); + queryIndexes: Map; constructor(props) { super(props); + this.queryIndexes = new Map(); this.centroids = []; this.dataToFeature = []; this.registerVisConfig(geojsonVisConfigs); @@ -368,13 +377,42 @@ export default class GeoJsonLayer extends Layer { } getSpatialIndex() { - if (!GeoJsonLayer.index.get(this.id) && this.centroids.length > 0) { + if (!GeoJsonLayer.spatialIndex.get(this.id) && this.centroids.length > 0) { + console.time('create spatial index'); const index = new Flatbush(this.centroids.length); this.centroids.forEach(c => index?.add(c[0], c[1], c[0], c[1])); index.finish(); - GeoJsonLayer.index.set(this.id, index); + GeoJsonLayer.spatialIndex.set(this.id, { + index, + search: (filter: PolygonFilter, layer: GeoJsonLayer): number[] => { + console.time('search'); + const [minX, minY, maxX, maxY] = filter.value.properties.bbox; + const foundIndexes = index?.search(minX, minY, maxX, maxY) || []; + layer.queryIndexes.clear(); + if (filter.value.properties?.shape === 'Rectangle') { + foundIndexes.forEach(i => layer.queryIndexes.set(i, true)); + } else { + // use turf.js to check if point is in polygon + foundIndexes.forEach(i => { + const point = layer.centroids[i]; + if (booleanWithin(turfPoint(point), filter.value)) { + layer.queryIndexes.set(i, true); + } + }); + } + layer.queryIndexes.forEach((v, k) => { + const feat = layer.dataToFeature[k]; + if (feat?.properties) { + feat.properties.selected = 1; + } + }); + console.timeEnd('search'); + return foundIndexes; + } + }); + console.timeEnd('create spatial index'); } - return GeoJsonLayer.index.get(this.id); + return GeoJsonLayer.spatialIndex.get(this.id); } getCentroids(): number[][] { @@ -387,21 +425,24 @@ export default class GeoJsonLayer extends Layer { return this.centroids; } - isInPolygon(data: RowDataContainer, index: number, polygon: Feature): Boolean { - if (this.centroids.length === 0 || !this.centroids[index]) { - return false; - } - // check if index is in existed spatial index query - - const isReactangle = polygon.properties?.shape === 'Rectangle'; - const point = this.centroids[index]; - // check if point is in polygon - if (isReactangle && polygon.properties?.bbox) { - const [minX, minY, maxX, maxY] = polygon.properties?.bbox; - return point[0] >= minX && point[0] <= maxX && point[1] >= minY && point[1] <= maxY; - } - return booleanWithin(turfPoint(point), polygon); - } + // isInPolygon(data: RowDataContainer, index: number, polygon: Feature): Boolean { + // if (this.centroids.length === 0 || !this.centroids[index]) { + // return false; + // } + // const isReactangle = polygon.properties?.shape === 'Rectangle'; + // const point = this.centroids[index]; + // // check if point is in polygon using spatialIndex + // if (isReactangle && GeoJsonLayer.spatialIndex.get(this.id)) { + // const found = this.queryIndexes.get(index); + // return found || false; + // } + // // without spatialIndex, use turf.js + // if (isReactangle && polygon.properties?.bbox) { + // const [minX, minY, maxX, maxY] = polygon.properties?.bbox; + // return point[0] >= minX && point[0] <= maxX && point[1] >= minY && point[1] <= maxY; + // } + // return booleanWithin(turfPoint(point), polygon); + // } updateLayerMeta(dataContainer) { const getFeature = this.getPositionAccessor(dataContainer); diff --git a/src/layers/src/geojson-layer/geojson-utils.ts b/src/layers/src/geojson-layer/geojson-utils.ts index 27213a3f97..b74ed5099d 100644 --- a/src/layers/src/geojson-layer/geojson-utils.ts +++ b/src/layers/src/geojson-layer/geojson-utils.ts @@ -20,8 +20,8 @@ import normalize from '@mapbox/geojson-normalize'; import bbox from '@turf/bbox'; -// import center from '@turf/center'; -// import centerOfMass from '@turf/center-of-mass'; +import center from '@turf/center'; +import centerOfMass from '@turf/center-of-mass'; import {parseSync} from '@loaders.gl/core'; import {WKBLoader, WKTLoader} from '@loaders.gl/wkt'; import {binaryToGeometry} from '@loaders.gl/gis'; @@ -188,8 +188,9 @@ export function getGeojsonMeanCenters(dataToFeature: GeojsonDataMaps): number[][ // meanCenters.push(centerOfMass(feature).geometry.coordinates); const geometries = feature.geometry.type === 'GeometryCollection' ? feature.geometry.geometries : [feature.geometry]; - const center = getMeanCenterFromGeometries(geometries); - meanCenters.push(center); + // const cent = getMeanCenterFromGeometries(geometries); + const cent = centerOfMass(feature).geometry.coordinates; + meanCenters.push(cent); } } console.timeEnd('getGeojsonMeanCenters'); diff --git a/src/reducers/src/vis-state-updaters.ts b/src/reducers/src/vis-state-updaters.ts index 42f75964e2..14aee2a973 100644 --- a/src/reducers/src/vis-state-updaters.ts +++ b/src/reducers/src/vis-state-updaters.ts @@ -858,6 +858,11 @@ export function setFilterUpdater( dataId: newDataIds }; + // if (newFilter.gpu) { + // newFilter = setFilterGpuMode(newFilter, state.filters); + // newFilter = assignGpuChannel(newFilter, state.filters); + // } + break; default: break; diff --git a/src/table/src/gpu-filter-utils.ts b/src/table/src/gpu-filter-utils.ts index e212a9ed64..d4d630fd1c 100644 --- a/src/table/src/gpu-filter-utils.ts +++ b/src/table/src/gpu-filter-utils.ts @@ -183,6 +183,9 @@ const getFilterValueAccessor = ( if (!filter) { return 0; } + // if (filter.type === FILTER_TYPES.polygon) { + // return getData(dc, d, -1, filter); + // } const fieldIndex = getDatasetFieldIndexForFilter(dataId, filter); const field = fields[fieldIndex]; diff --git a/src/table/src/kepler-table.ts b/src/table/src/kepler-table.ts index b169b0160e..0fc6b884ce 100644 --- a/src/table/src/kepler-table.ts +++ b/src/table/src/kepler-table.ts @@ -20,7 +20,6 @@ import {console as Console} from 'global/console'; import {ascending, descending} from 'd3-array'; -import {intersection} from 'lodash'; import { TRIP_POINT_FIELDS, SORT_ORDER, @@ -60,8 +59,7 @@ import { getLogDomain, getOrdinalDomain, getQuantileDomain, - DataContainerInterface, - createIndexedDataContainer + DataContainerInterface } from '@kepler.gl/utils'; export type GpuFilter = { @@ -322,39 +320,18 @@ class KeplerTable { const dynamicDomainFilters = shouldCalDomain ? filterRecord.dynamicDomain : null; const cpuFilters = shouldCalIndex ? filterRecord.cpu : null; - // use layer index to build IndexedDataContainer to narrow down the data - console.time('filterDataByFilterTypes'); - const preFilteredIndexes: number[][] = []; - for (const filter of filters) { - if (filter.type === FILTER_TYPES.polygon) { - const filteredLayers = layers.filter(l => l.config.dataId === this.id); - // iterator over filteredLayers, and use it's spatial index to filter data - filteredLayers?.forEach(layer => { - const index = KeplerGlLayers.GeojsonLayer.index.get(layer.id); - const [minX, minY, maxX, maxY] = filter.value.properties.bbox; - const foundIndexes = index?.search(minX, minY, maxX, maxY) || []; - preFilteredIndexes.push(foundIndexes); - }); - } - } - console.timeEnd('filterDataByFilterTypes'); - const filteredDataContainer = - preFilteredIndexes.length > 0 - ? createIndexedDataContainer(dataContainer, intersection(preFilteredIndexes.flat())) - : dataContainer; - const filterFuncs = filters.reduce((acc, filter) => { const fieldIndex = getDatasetFieldIndexForFilter(this.id, filter); const field = fieldIndex !== -1 ? fields[fieldIndex] : null; return { ...acc, - [filter.id]: getFilterFunction(field, this.id, filter, layers, filteredDataContainer) + [filter.id]: getFilterFunction(field, this.id, filter, layers, dataContainer) }; }, {}); filterResult = filterDataByFilterTypes( {dynamicDomainFilters, cpuFilters, filterFuncs}, - filteredDataContainer + opt?.filteredDataContainer ? opt.filteredDataContainer : dataContainer ); } diff --git a/src/types/reducers.d.ts b/src/types/reducers.d.ts index aa4ee2ef7f..066b1c7c35 100644 --- a/src/types/reducers.d.ts +++ b/src/types/reducers.d.ts @@ -468,6 +468,7 @@ export type FilterDatasetOpt = { cpuOnly?: boolean; // ignore filter for domain calculation ignoreDomain?: boolean; + filteredDataContainer?: any; }; /* DUPLICATES OF FILTER TYPES ABOVE, REMOVE ONCE TYPES ABOVE ARE FIXED */ diff --git a/src/utils/src/filter-utils.ts b/src/utils/src/filter-utils.ts index f3c2ac94ca..cb285cc89a 100644 --- a/src/utils/src/filter-utils.ts +++ b/src/utils/src/filter-utils.ts @@ -36,6 +36,7 @@ import { FILTER_VIEW_TYPES } from '@kepler.gl/constants'; import {VisState} from '@kepler.gl/schemas'; +import {KeplerGlLayers} from '@kepler.gl/layers'; import * as ScaleUtils from './data-scale-utils'; import {h3IsValid} from 'h3-js'; @@ -63,6 +64,7 @@ import {DataContainerInterface} from './data-container-interface'; import {generateHashId, set, toArray} from './utils'; import {notNullorUndefined, timeToUnixMilli, unique} from './data-utils'; import {getCentroid} from './h3-utils'; +import { IndexedDataContainer } from './indexed-data-container'; export const durationSecond = 1000; export const durationMinute = durationSecond * 60; @@ -456,13 +458,7 @@ export const getPolygonFilterFunctor = (layer, filter, dataContainer) => { }; case LAYER_TYPES.geojson: return data => { - if (filter.value.properties?.shape === 'Rectangle') { - return true; - } - if (layer.isInPolygon) { - return layer.isInPolygon(data, data.index, filter.value); - } - // show all geometries if can't apply filter + // data are already filtered by spatial index return true; }; default: @@ -557,24 +553,17 @@ export function filterDataByFilterTypes( const numRows = dataContainer.numRows(); const plainIndexes = dataContainer.getPlainIndex(); - if (dynamicDomainFilters) { - for (let i = 0; i < numRows; ++i) { - filterContext.index = plainIndexes[i]; + for (let i = 0; i < numRows; ++i) { + filterContext.index = plainIndexes[i]; - const matchForDomain = dynamicDomainFilters && dynamicDomainFilters.every(filterFuncCaller); - if (matchForDomain) { - filteredIndexForDomain.push(filterContext.index); - } + const matchForDomain = dynamicDomainFilters && dynamicDomainFilters.every(filterFuncCaller); + if (matchForDomain) { + filteredIndexForDomain.push(filterContext.index); } - } - // TODO: with a index, we should be able to avoid iterate through all data - if (cpuFilters) { - for (let i = 0; i < numRows; ++i) { - filterContext.index = plainIndexes[i]; - const matchForRender = cpuFilters && cpuFilters.every(filterFuncCaller); - if (matchForRender) { - filteredIndex.push(filterContext.index); - } + + const matchForRender = cpuFilters && cpuFilters.every(filterFuncCaller); + if (matchForRender) { + filteredIndex.push(filterContext.index); } } @@ -968,7 +957,7 @@ export function getDefaultFilterPlotType(filter: Filter): string | null { */ export function applyFiltersToDatasets< K extends KeplerTableModel, - L extends {config: {dataId: string | null}} + L extends {config: {dataId: string | null}, id: string} >( datasetIds: string[], datasets: {[id: string]: K}, @@ -981,9 +970,36 @@ export function applyFiltersToDatasets< const appliedFilters = filters.filter(d => shouldApplyFilter(d, dataId)); const table = datasets[dataId]; + // use layer index to build IndexedDataContainer to narrow down the data for filtering + console.time('filterDataByFilterTypes'); + const preFilteredIndexes: number[][] = []; + for (const filter of filters) { + if (filter.type === FILTER_TYPES.polygon) { + // iterator over filteredLayers, and use it's spatial index to filter data + layersToFilter?.forEach(layer => { + const index = KeplerGlLayers.GeojsonLayer.spatialIndex.get(layer.id); + if (index) { + const foundIndexes = index.search(filter, layer) || []; + preFilteredIndexes.push(foundIndexes); + } + }); + } + } + const filteredDataContainerIndexes = {}; + // iterator preFilterIndexes + for (let i = 0; i < preFilteredIndexes.length; i++) { + const indexes = preFilteredIndexes[i]; + for (let j = 0; j < indexes.length; j++) { + const index = indexes[j]; + filteredDataContainerIndexes[index] = index; + } + } + const filteredDataContainer = Object.keys(filteredDataContainerIndexes).length > 0 ? new IndexedDataContainer(table.dataContainer, Object.values(filteredDataContainerIndexes)) : null; + console.timeEnd('filterDataByFilterTypes'); + return { ...acc, - [dataId]: table.filterTable(appliedFilters, layersToFilter, {}) + [dataId]: table.filterTable(appliedFilters, layersToFilter, filteredDataContainer ? {filteredDataContainer} : {}) }; }, datasets); } @@ -1127,7 +1143,8 @@ export function generatePolygonFilter< type: FILTER_TYPES.polygon, name, layerId, - value: featureToFilterValue(feature, filter.id, {isVisible: true}) + value: featureToFilterValue(feature, filter.id, {isVisible: true}), + gpu: false }; } From a56dd8b1c609a5db5544c915085a7ce5d241814a Mon Sep 17 00:00:00 2001 From: Xun Li Date: Thu, 28 Sep 2023 10:39:21 -0700 Subject: [PATCH 2/3] wip --- .../src/geojson-layer/brush-geojson-layer.ts | 44 +++++++++++++++++++ .../src/geojson-layer/brush-shader-module.ts | 42 ++++++++++++++++++ src/deckgl-layers/src/index.ts | 1 + src/layers/src/base-layer.ts | 5 ++- src/layers/src/geojson-layer/geojson-layer.ts | 28 +++++++----- src/table/src/gpu-filter-utils.ts | 22 +++++++--- src/utils/src/filter-utils.ts | 2 +- 7 files changed, 123 insertions(+), 21 deletions(-) create mode 100644 src/deckgl-layers/src/geojson-layer/brush-geojson-layer.ts create mode 100644 src/deckgl-layers/src/geojson-layer/brush-shader-module.ts diff --git a/src/deckgl-layers/src/geojson-layer/brush-geojson-layer.ts b/src/deckgl-layers/src/geojson-layer/brush-geojson-layer.ts new file mode 100644 index 0000000000..985ead9617 --- /dev/null +++ b/src/deckgl-layers/src/geojson-layer/brush-geojson-layer.ts @@ -0,0 +1,44 @@ +import {Layer, LayerExtension } from '@deck.gl/core'; +import {LayerContext} from '@deck.gl/core/lib/layer'; + +import shaderModule from './brush-shader-module'; + +const defaultProps = { +}; + +export type BrushGeoJsonExtensionProps = { +}; + +// Write an extension to brush geojson layer using the drawn polygon: +// an instanced attribute 'instanceHighlighted' is added to the layer to indicate whether the feature is highlighted +// the shader module is modified to discard the feature if instanceHighlighted is 0 +// the accessor getHighlighted is used to get the value of instanceHighlighted based on the search result in GeoJsonlayer +// From a test, deck: Updated attributes for 7314969 instances in azfyr45-polygons-fill in 162ms +export default class BrushGeoJsonExtension extends LayerExtension { + static defaultProps = defaultProps; + static extensionName = 'BrushGeoJsonExtension'; + + getShaders(extension: any) { + return { + modules: [shaderModule], + defines: {} + }; + } + + initializeState(this: Layer, context: LayerContext, extension: this) { + const attributeManager = this.getAttributeManager(); + if (attributeManager) { + attributeManager.addInstanced({ + instanceHighlighted: { + size: 1, + accessor: 'getHighlighted' + } + }); + } + } + updateState({ + props, + oldProps + }) { + } +} diff --git a/src/deckgl-layers/src/geojson-layer/brush-shader-module.ts b/src/deckgl-layers/src/geojson-layer/brush-shader-module.ts new file mode 100644 index 0000000000..9939bd3168 --- /dev/null +++ b/src/deckgl-layers/src/geojson-layer/brush-shader-module.ts @@ -0,0 +1,42 @@ +import {project} from '@deck.gl/core'; + +import type {BrushGeoJsonExtensionProps} from './brush-geojson-layer'; + +const vs = ``; + +const fs = ``; + +const inject = { + 'vs:#decl': ` + attribute float instanceHighlighted; + varying float vHighlighted; + `, + 'vs:#main-end': ` + if (instanceHighlighted == 0.) { + gl_Position = vec4(0.); + } + vHighlighted = instanceHighlighted; + `, + 'fs:#decl': ` + varying float vHighlighted; + `, + 'fs:DECKGL_FILTER_COLOR': ` + if (vHighlighted == 0.) { + discard; + } + ` +}; + +export default { + name: 'brush-geojson', + dependencies: [project], + vs, + fs, + inject, + getUniforms: (opts?: BrushGeoJsonExtensionProps): Record => { + if (!opts) { + return {}; + } + return {}; + } +} diff --git a/src/deckgl-layers/src/index.ts b/src/deckgl-layers/src/index.ts index 459c283ede..1c76cadeaf 100644 --- a/src/deckgl-layers/src/index.ts +++ b/src/deckgl-layers/src/index.ts @@ -9,6 +9,7 @@ export {default as EnhancedGridLayer} from './grid-layer/enhanced-cpu-grid-layer export {default as EnhancedHexagonLayer} from './hexagon-layer/enhanced-hexagon-layer'; export {default as EnhancedLineLayer} from './line-layer/line-layer'; export {default as SvgIconLayer} from './svg-icon-layer/svg-icon-layer'; +export {default as BrushGeoJsonExtension} from './geojson-layer/brush-geojson-layer'; export * from './layer-utils/shader-utils'; diff --git a/src/layers/src/base-layer.ts b/src/layers/src/base-layer.ts index e435ff7641..5290288ca9 100644 --- a/src/layers/src/base-layer.ts +++ b/src/layers/src/base-layer.ts @@ -24,6 +24,7 @@ import keymirror from 'keymirror'; import {DataFilterExtension} from '@deck.gl/extensions'; import {COORDINATE_SYSTEM} from '@deck.gl/core'; import {TextLayer} from '@deck.gl/layers'; +import {BrushGeoJsonExtension} from '@kepler.gl/deckgl-layers'; import DefaultLayerIcon from './default-layer-icon'; import {diffUpdateTriggers} from './layer-update'; @@ -203,7 +204,7 @@ export const LAYER_ID_LENGTH = 6; const MAX_SAMPLE_SIZE = 5000; const defaultDomain: [number, number] = [0, 1]; const dataFilterExtension = new DataFilterExtension({filterSize: MAX_GPU_FILTERS}); - +const brushGeoJsonExtension = new BrushGeoJsonExtension(); const defaultDataAccessor = dc => d => d; const defaultGetFieldValue = (field, d) => field.valueAccessor(d); @@ -1309,7 +1310,7 @@ class Layer { opacity: this.config.visConfig.opacity, highlightColor: this.config.highlightColor, // data filtering - extensions: [dataFilterExtension], + extensions: [dataFilterExtension, brushGeoJsonExtension], filterRange: gpuFilter ? gpuFilter.filterRange : undefined, // layer should be visible and if splitMap, shown in to one of panel diff --git a/src/layers/src/geojson-layer/geojson-layer.ts b/src/layers/src/geojson-layer/geojson-layer.ts index bf13102afe..7a7b3544ba 100644 --- a/src/layers/src/geojson-layer/geojson-layer.ts +++ b/src/layers/src/geojson-layer/geojson-layer.ts @@ -21,7 +21,6 @@ import Flatbush from 'flatbush'; import uniq from 'lodash.uniq'; import {DATA_TYPES} from 'type-analyzer'; -import {Feature, Polygon} from 'geojson'; import booleanWithin from '@turf/boolean-within'; import {point as turfPoint} from '@turf/helpers'; import Layer, { @@ -64,7 +63,6 @@ import { } from '@kepler.gl/types'; import {KeplerTable} from '@kepler.gl/table'; import {DataContainerInterface} from '@kepler.gl/utils'; -import {RowDataContainer} from 'src/utils/src/row-data-container'; const SUPPORTED_ANALYZER_TYPES = { [DATA_TYPES.GEOMETRY]: true, @@ -199,10 +197,12 @@ export default class GeoJsonLayer extends Layer { centroids: number[][]; static spatialIndex = new Map(); queryIndexes: Map; + queryBounds: number[]; constructor(props) { super(props); + this.queryBounds = []; this.queryIndexes = new Map(); this.centroids = []; this.dataToFeature = []; @@ -366,19 +366,24 @@ export default class GeoJsonLayer extends Layer { const dataAccessor = dc => d => ({index: d.properties.index}); const accessors = this.getAttributeAccessors({dataAccessor, dataContainer}); + const queryResultAccessor = d => { + // return (this?.queryBounds && this?.queryBounds?.length === 0) || d.properties.selected ? 1 : 0; + return Math.round(Math.random()) + }; + return { data, getFilterValue: gpuFilter.filterValueAccessor(dataContainer)( indexAccessor, customFilterValueAccessor ), + getHighlighted: queryResultAccessor, ...accessors }; } getSpatialIndex() { if (!GeoJsonLayer.spatialIndex.get(this.id) && this.centroids.length > 0) { - console.time('create spatial index'); const index = new Flatbush(this.centroids.length); this.centroids.forEach(c => index?.add(c[0], c[1], c[0], c[1])); index.finish(); @@ -387,6 +392,7 @@ export default class GeoJsonLayer extends Layer { search: (filter: PolygonFilter, layer: GeoJsonLayer): number[] => { console.time('search'); const [minX, minY, maxX, maxY] = filter.value.properties.bbox; + layer.queryBounds = [minX, minY, maxX, maxY]; const foundIndexes = index?.search(minX, minY, maxX, maxY) || []; layer.queryIndexes.clear(); if (filter.value.properties?.shape === 'Rectangle') { @@ -400,17 +406,16 @@ export default class GeoJsonLayer extends Layer { } }); } - layer.queryIndexes.forEach((v, k) => { - const feat = layer.dataToFeature[k]; - if (feat?.properties) { - feat.properties.selected = 1; - } - }); + // layer.queryIndexes.forEach((v, k) => { + // const feat = layer.dataToFeature[k]; + // if (feat?.properties) { + // feat.properties.selected = 1; + // } + // }); console.timeEnd('search'); return foundIndexes; } }); - console.timeEnd('create spatial index'); } return GeoJsonLayer.spatialIndex.get(this.id); } @@ -503,7 +508,8 @@ export default class GeoJsonLayer extends Layer { const updateTriggers = { ...this.getVisualChannelUpdateTriggers(), - getFilterValue: gpuFilter.filterValueUpdateTriggers + getFilterValue: gpuFilter.filterValueUpdateTriggers, + getHighlighted: this.queryBounds }; const defaultLayerProps = this.getDefaultDeckLayerProps(opts); diff --git a/src/table/src/gpu-filter-utils.ts b/src/table/src/gpu-filter-utils.ts index d4d630fd1c..85341269b2 100644 --- a/src/table/src/gpu-filter-utils.ts +++ b/src/table/src/gpu-filter-utils.ts @@ -163,7 +163,7 @@ const defaultGetIndex = d => d.index; * @param fieldIndex Column index in the data container. * @returns */ -const defaultGetData = (dc: DataContainerInterface, d: any, fieldIndex: number) => { +const defaultGetData = (dc: DataContainerInterface, d: any, fieldIndex: number, filter?: Filter) => { return dc.valueAt(d.index, fieldIndex); }; @@ -183,9 +183,9 @@ const getFilterValueAccessor = ( if (!filter) { return 0; } - // if (filter.type === FILTER_TYPES.polygon) { - // return getData(dc, d, -1, filter); - // } + if (filter.type === FILTER_TYPES.polygon) { + return getData(dc, d, -1, filter); + } const fieldIndex = getDatasetFieldIndexForFilter(dataId, filter); const field = fields[fieldIndex]; @@ -218,10 +218,18 @@ export function getGpuFilterProps(filters: Filter[], dataId: string, fields: Fie f.gpuChannel[f.dataId.indexOf(dataId)] === i ); - filterRange[i][0] = filter ? filter.value[0] - filter.domain?.[0] : 0; - filterRange[i][1] = filter ? filter.value[1] - filter.domain?.[0] : 0; + if (filter?.type === FILTER_TYPES.polygon) { + filterRange[i][0] = 1; + // get a number that is larger than 1 and changed when bbox changes + filterRange[i][1] = 1 + Math.abs(filter.value.properties.bbox.reduce((accu, d) => accu + d, 0)); + console.log(filterRange) + triggers[`gpuFilter_${i}`] = filter.id; + } else { + filterRange[i][0] = filter ? filter.value[0] - filter.domain?.[0] : 0; + filterRange[i][1] = filter ? filter.value[1] - filter.domain?.[0] : 0; + triggers[`gpuFilter_${i}`] = filter ? filter.name[filter.dataId.indexOf(dataId)] : null; + } - triggers[`gpuFilter_${i}`] = filter ? filter.name[filter.dataId.indexOf(dataId)] : null; channels.push(filter); } diff --git a/src/utils/src/filter-utils.ts b/src/utils/src/filter-utils.ts index cb285cc89a..666c54a9d1 100644 --- a/src/utils/src/filter-utils.ts +++ b/src/utils/src/filter-utils.ts @@ -1144,7 +1144,7 @@ export function generatePolygonFilter< name, layerId, value: featureToFilterValue(feature, filter.id, {isVisible: true}), - gpu: false + gpu: true }; } From 107ec6289938f035fb10e5f8060d60c4746958f4 Mon Sep 17 00:00:00 2001 From: Xun Li Date: Fri, 29 Sep 2023 10:34:35 -0700 Subject: [PATCH 3/3] impl gpu brush with react --- .../src/geojson-layer/brush-geojson-layer.ts | 27 +++++-- .../src/geojson-layer/brush-shader-module.ts | 79 ++++++++++++++++--- src/layers/src/base-layer.ts | 6 +- src/layers/src/geojson-layer/geojson-layer.ts | 37 ++++++--- src/table/src/gpu-filter-utils.ts | 6 +- 5 files changed, 119 insertions(+), 36 deletions(-) diff --git a/src/deckgl-layers/src/geojson-layer/brush-geojson-layer.ts b/src/deckgl-layers/src/geojson-layer/brush-geojson-layer.ts index 985ead9617..89cfa9590b 100644 --- a/src/deckgl-layers/src/geojson-layer/brush-geojson-layer.ts +++ b/src/deckgl-layers/src/geojson-layer/brush-geojson-layer.ts @@ -1,19 +1,28 @@ import {Layer, LayerExtension } from '@deck.gl/core'; import {LayerContext} from '@deck.gl/core/lib/layer'; +import GL from '@luma.gl/constants'; import shaderModule from './brush-shader-module'; const defaultProps = { + getCenter: {type: 'accessor', value: [0, 0]}, + enableBrushing: false, + brushRectangle: [0, 0, 0, 0], + brushPolygon: [] }; export type BrushGeoJsonExtensionProps = { + getCenter?: () => [number, number]; + enableBrushing?: boolean; + brushRectangle?: [number, number, number, number]; + brushPolygon?: number[]; }; // Write an extension to brush geojson layer using the drawn polygon: // an instanced attribute 'instanceHighlighted' is added to the layer to indicate whether the feature is highlighted // the shader module is modified to discard the feature if instanceHighlighted is 0 // the accessor getHighlighted is used to get the value of instanceHighlighted based on the search result in GeoJsonlayer -// From a test, deck: Updated attributes for 7314969 instances in azfyr45-polygons-fill in 162ms +// From a test, gl deck: Updated attributes for 7314969 instances in azfyr45-polygons-fill in 162ms export default class BrushGeoJsonExtension extends LayerExtension { static defaultProps = defaultProps; static extensionName = 'BrushGeoJsonExtension'; @@ -28,10 +37,18 @@ export default class BrushGeoJsonExtension extends LayerExtension { initializeState(this: Layer, context: LayerContext, extension: this) { const attributeManager = this.getAttributeManager(); if (attributeManager) { - attributeManager.addInstanced({ - instanceHighlighted: { - size: 1, - accessor: 'getHighlighted' + attributeManager.add({ + center: { + size: 2, + accessor: 'getCenter', + shaderAttributes: { + center: { + divisor: 0 + }, + instanceCenter: { + divisor: 1 + } + }, } }); } diff --git a/src/deckgl-layers/src/geojson-layer/brush-shader-module.ts b/src/deckgl-layers/src/geojson-layer/brush-shader-module.ts index 9939bd3168..6c809b4ca8 100644 --- a/src/deckgl-layers/src/geojson-layer/brush-shader-module.ts +++ b/src/deckgl-layers/src/geojson-layer/brush-shader-module.ts @@ -2,26 +2,68 @@ import {project} from '@deck.gl/core'; import type {BrushGeoJsonExtensionProps} from './brush-geojson-layer'; -const vs = ``; +const vs = ` + #ifdef NON_INSTANCED_MODEL + #define BRUSH_GEOJSON_ATTRIB center + #else + #define BRUSH_GEOJSON_ATTRIB instanceCenter + #endif + + attribute vec2 BRUSH_GEOJSON_ATTRIB; + uniform vec4 brush_rectangle; + uniform vec2 brush_polygon[516]; + uniform int brush_polygon_length; + uniform bool brushing_enabled; + + float center_in_polygon(vec2 point, vec2 poly[516]) { + float inside = 0.; + float x = point.x, y = point.y; + // for (int i = 0, j = brush_polygon_length - 1; i < brush_polygon_length; j = i++) { + // float xi = poly[i].x; + // float yi = poly[i].y; + // float xj = poly[j].x; + // float yj = poly[j].y; + // if ((yi > y) != (yj > y)) && (x < (xj - xi) * (y - yi) / (yj - yi) + xi)) { + // inside = 1. - inside; + // } + // } + return inside; + } + + float center_in_rectangle(vec2 point, vec4 rectangle) { + if (point.x >= rectangle.x && point.x <= rectangle.z && point.y >= rectangle.y && point.y <= rectangle.w) { + return 1.; + } + return 0.; + } +`; const fs = ``; const inject = { 'vs:#decl': ` - attribute float instanceHighlighted; - varying float vHighlighted; + varying float is_visible; `, 'vs:#main-end': ` - if (instanceHighlighted == 0.) { - gl_Position = vec4(0.); + is_visible = 0.; + if (brushing_enabled) { + is_visible = center_in_rectangle(BRUSH_GEOJSON_ATTRIB, brush_rectangle); + // // if (brush_polygon_length > 0 && is_visible == 1.) { + // // is_visible = center_in_polygon(BRUSH_GEOJSON_ATTRIB, brush_polygon); + // // } + // // position the current vertex out of screen + // if (is_visible == 0.) { + // gl_Position = vec4(0.); + // } } - vHighlighted = instanceHighlighted; `, 'fs:#decl': ` - varying float vHighlighted; + varying float is_visible; + uniform bool brushing_enabled; `, 'fs:DECKGL_FILTER_COLOR': ` - if (vHighlighted == 0.) { + // abandon the fragments if brush_enabled and it is not highlighted + if (brushing_enabled && is_visible == 0.) { discard; } ` @@ -30,13 +72,24 @@ const inject = { export default { name: 'brush-geojson', dependencies: [project], - vs, - fs, - inject, + vs: vs, + fs: fs, + inject: inject, getUniforms: (opts?: BrushGeoJsonExtensionProps): Record => { - if (!opts) { + if (!opts || !('extensions' in opts)) { return {}; } - return {}; + const { + enableBrushing = false, + brushRectangle = [0, 0, 0, 0], + brushPolygon = [] + } = opts; + + return { + brushing_enabled: enableBrushing, + brush_rectangle: brushRectangle, + brush_polygon: brushPolygon, + brush_polygon_length: brushPolygon ? brushPolygon.length : 0 + }; } } diff --git a/src/layers/src/base-layer.ts b/src/layers/src/base-layer.ts index 5290288ca9..52b6a3debc 100644 --- a/src/layers/src/base-layer.ts +++ b/src/layers/src/base-layer.ts @@ -24,7 +24,6 @@ import keymirror from 'keymirror'; import {DataFilterExtension} from '@deck.gl/extensions'; import {COORDINATE_SYSTEM} from '@deck.gl/core'; import {TextLayer} from '@deck.gl/layers'; -import {BrushGeoJsonExtension} from '@kepler.gl/deckgl-layers'; import DefaultLayerIcon from './default-layer-icon'; import {diffUpdateTriggers} from './layer-update'; @@ -203,8 +202,7 @@ export const LAYER_ID_LENGTH = 6; const MAX_SAMPLE_SIZE = 5000; const defaultDomain: [number, number] = [0, 1]; -const dataFilterExtension = new DataFilterExtension({filterSize: MAX_GPU_FILTERS}); -const brushGeoJsonExtension = new BrushGeoJsonExtension(); +const dataFilterExtension = new DataFilterExtension({ filterSize: MAX_GPU_FILTERS }); const defaultDataAccessor = dc => d => d; const defaultGetFieldValue = (field, d) => field.valueAccessor(d); @@ -1310,7 +1308,7 @@ class Layer { opacity: this.config.visConfig.opacity, highlightColor: this.config.highlightColor, // data filtering - extensions: [dataFilterExtension, brushGeoJsonExtension], + extensions: [dataFilterExtension], filterRange: gpuFilter ? gpuFilter.filterRange : undefined, // layer should be visible and if splitMap, shown in to one of panel diff --git a/src/layers/src/geojson-layer/geojson-layer.ts b/src/layers/src/geojson-layer/geojson-layer.ts index 7a7b3544ba..b2d6bf5a8a 100644 --- a/src/layers/src/geojson-layer/geojson-layer.ts +++ b/src/layers/src/geojson-layer/geojson-layer.ts @@ -63,6 +63,8 @@ import { } from '@kepler.gl/types'; import {KeplerTable} from '@kepler.gl/table'; import {DataContainerInterface} from '@kepler.gl/utils'; +import {BrushGeoJsonExtension} from '@kepler.gl/deckgl-layers'; + const SUPPORTED_ANALYZER_TYPES = { [DATA_TYPES.GEOMETRY]: true, @@ -347,8 +349,14 @@ export default class GeoJsonLayer extends Layer { return null; } - calculateDataAttribute({dataContainer, filteredIndex}, getPosition) { - return filteredIndex.map(i => this.dataToFeature[i]).filter(d => d); + calculateDataAttribute({ dataContainer, filteredIndex }, getPosition) { + return filteredIndex.map(i => { + const feat = this.dataToFeature[i]; + if (feat && feat.properties) { + feat.properties.centroid = this.centroids[i]; + } + return feat; + }).filter(d => d); } formatLayerData(datasets, oldLayerData) { @@ -358,7 +366,10 @@ export default class GeoJsonLayer extends Layer { const {gpuFilter, dataContainer} = datasets[this.config.dataId]; const {data} = this.updateData(datasets, oldLayerData); - const customFilterValueAccessor = (dc, d, fieldIndex) => { + const customFilterValueAccessor = (dc, d, fieldIndex, filter) => { + if (filter && filter.type === FILTER_TYPES.polygon) { + return d.properties.centroid[0]; + } return dc.valueAt(d.properties.index, fieldIndex); }; const indexAccessor = f => f.properties.index; @@ -367,8 +378,7 @@ export default class GeoJsonLayer extends Layer { const accessors = this.getAttributeAccessors({dataAccessor, dataContainer}); const queryResultAccessor = d => { - // return (this?.queryBounds && this?.queryBounds?.length === 0) || d.properties.selected ? 1 : 0; - return Math.round(Math.random()) + return d.properties.centroid; }; return { @@ -377,7 +387,10 @@ export default class GeoJsonLayer extends Layer { indexAccessor, customFilterValueAccessor ), - getHighlighted: queryResultAccessor, + enableBrushing: this.queryBounds.length > 0, + brushRectangle: this.queryBounds.length > 0 ? this.queryBounds : [0, 0, 0, 0], + brushPolygon: [], + getCenter: queryResultAccessor, ...accessors }; } @@ -423,9 +436,13 @@ export default class GeoJsonLayer extends Layer { getCentroids(): number[][] { if (this.centroids.length === 0) { this.centroids = getGeojsonMeanCenters(this.dataToFeature); - console.time('build spatial index'); + for (let i = 0; i < this.centroids.length; i++) { + const feat = this.dataToFeature[i]; + if (feat && feat.properties) { + feat.properties.centroid = this.centroids[i]; + } + } this.getSpatialIndex(); - console.timeEnd('build spatial index'); } return this.centroids; } @@ -508,8 +525,7 @@ export default class GeoJsonLayer extends Layer { const updateTriggers = { ...this.getVisualChannelUpdateTriggers(), - getFilterValue: gpuFilter.filterValueUpdateTriggers, - getHighlighted: this.queryBounds + getFilterValue: gpuFilter.filterValueUpdateTriggers }; const defaultLayerProps = this.getDefaultDeckLayerProps(opts); @@ -537,6 +553,7 @@ export default class GeoJsonLayer extends Layer { capRounded: true, jointRounded: true, updateTriggers, + extensions: [...defaultLayerProps.extensions, new BrushGeoJsonExtension()], _subLayerProps: { ...(featureTypes?.polygon ? {'polygons-stroke': opaOverwrite} : {}), ...(featureTypes?.line ? {linestrings: opaOverwrite} : {}), diff --git a/src/table/src/gpu-filter-utils.ts b/src/table/src/gpu-filter-utils.ts index 85341269b2..f5b570fb7e 100644 --- a/src/table/src/gpu-filter-utils.ts +++ b/src/table/src/gpu-filter-utils.ts @@ -219,10 +219,8 @@ export function getGpuFilterProps(filters: Filter[], dataId: string, fields: Fie ); if (filter?.type === FILTER_TYPES.polygon) { - filterRange[i][0] = 1; - // get a number that is larger than 1 and changed when bbox changes - filterRange[i][1] = 1 + Math.abs(filter.value.properties.bbox.reduce((accu, d) => accu + d, 0)); - console.log(filterRange) + filterRange[i][0] = filter.value.properties.bbox[0]; + filterRange[i][1] = filter.value.properties.bbox[2]; triggers[`gpuFilter_${i}`] = filter.id; } else { filterRange[i][0] = filter ? filter.value[0] - filter.domain?.[0] : 0;