Skip to content

Commit

Permalink
wip
Browse files Browse the repository at this point in the history
  • Loading branch information
@lixun910
lixun910 committed Sep 27, 2023
1 parent 7af4f24 commit 14d1452
Show file tree
Hide file tree
Showing 7 changed files with 123 additions and 78 deletions.
85 changes: 63 additions & 22 deletions src/layers/src/geojson-layer/geojson-layer.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,8 @@ import {
HIGHLIGH_COLOR_3D,
CHANNEL_SCALES,
ColorRange,
LAYER_VIS_CONFIGS
LAYER_VIS_CONFIGS,
FILTER_TYPES
} from '@kepler.gl/constants';
import {
VisConfigNumber,
Expand All @@ -58,11 +59,12 @@ import {
VisConfigRange,
VisConfigBoolean,
Merge,
RGBColor
RGBColor,
PolygonFilter
} from '@kepler.gl/types';
import {KeplerTable} from '@kepler.gl/table';
import {DataContainerInterface} from '@kepler.gl/utils';
import { RowDataContainer } from 'src/utils/src/row-data-container';
import {RowDataContainer} from 'src/utils/src/row-data-container';

const SUPPORTED_ANALYZER_TYPES = {
[DATA_TYPES.GEOMETRY]: true,
Expand Down Expand Up @@ -184,17 +186,24 @@ export const defaultElevation = 500;
export const defaultLineWidth = 1;
export const defaultRadius = 1;

export type SpatialIndexProps = {
index: Flatbush;
search: (filter: PolygonFilter, layer: GeoJsonLayer) => number[];
};

export default class GeoJsonLayer extends Layer {
declare config: GeoJsonLayerConfig;
declare visConfigSettings: GeoJsonVisConfigSettings;
declare meta: GeoJsonLayerMeta;
dataToFeature: GeojsonDataMaps;
centroids: number[][];
static index = new Map<string, Flatbush>();
static spatialIndex = new Map<string, SpatialIndexProps>();
queryIndexes: Map<number, boolean>;

constructor(props) {
super(props);

this.queryIndexes = new Map();
this.centroids = [];
this.dataToFeature = [];
this.registerVisConfig(geojsonVisConfigs);
Expand Down Expand Up @@ -368,13 +377,42 @@ export default class GeoJsonLayer extends Layer {
}

getSpatialIndex() {
if (!GeoJsonLayer.index.get(this.id) && this.centroids.length > 0) {
if (!GeoJsonLayer.spatialIndex.get(this.id) && this.centroids.length > 0) {
console.time('create spatial index');
const index = new Flatbush(this.centroids.length);
this.centroids.forEach(c => index?.add(c[0], c[1], c[0], c[1]));
index.finish();
GeoJsonLayer.index.set(this.id, index);
GeoJsonLayer.spatialIndex.set(this.id, {
index,
search: (filter: PolygonFilter, layer: GeoJsonLayer): number[] => {
console.time('search');
const [minX, minY, maxX, maxY] = filter.value.properties.bbox;
const foundIndexes = index?.search(minX, minY, maxX, maxY) || [];
layer.queryIndexes.clear();
if (filter.value.properties?.shape === 'Rectangle') {
foundIndexes.forEach(i => layer.queryIndexes.set(i, true));
} else {
// use turf.js to check if point is in polygon
foundIndexes.forEach(i => {
const point = layer.centroids[i];
if (booleanWithin(turfPoint(point), filter.value)) {
layer.queryIndexes.set(i, true);
}
});
}
layer.queryIndexes.forEach((v, k) => {
const feat = layer.dataToFeature[k];
if (feat?.properties) {
feat.properties.selected = 1;
}
});
console.timeEnd('search');
return foundIndexes;
}
});
console.timeEnd('create spatial index');
}
return GeoJsonLayer.index.get(this.id);
return GeoJsonLayer.spatialIndex.get(this.id);
}

getCentroids(): number[][] {
Expand All @@ -387,21 +425,24 @@ export default class GeoJsonLayer extends Layer {
return this.centroids;
}

isInPolygon(data: RowDataContainer, index: number, polygon: Feature<Polygon>): Boolean {
if (this.centroids.length === 0 || !this.centroids[index]) {
return false;
}
// check if index is in existed spatial index query

const isReactangle = polygon.properties?.shape === 'Rectangle';
const point = this.centroids[index];
// check if point is in polygon
if (isReactangle && polygon.properties?.bbox) {
const [minX, minY, maxX, maxY] = polygon.properties?.bbox;
return point[0] >= minX && point[0] <= maxX && point[1] >= minY && point[1] <= maxY;
}
return booleanWithin(turfPoint(point), polygon);
}
// isInPolygon(data: RowDataContainer, index: number, polygon: Feature<Polygon>): Boolean {
// if (this.centroids.length === 0 || !this.centroids[index]) {
// return false;
// }
// const isReactangle = polygon.properties?.shape === 'Rectangle';
// const point = this.centroids[index];
// // check if point is in polygon using spatialIndex
// if (isReactangle && GeoJsonLayer.spatialIndex.get(this.id)) {
// const found = this.queryIndexes.get(index);
// return found || false;
// }
// // without spatialIndex, use turf.js
// if (isReactangle && polygon.properties?.bbox) {
// const [minX, minY, maxX, maxY] = polygon.properties?.bbox;
// return point[0] >= minX && point[0] <= maxX && point[1] >= minY && point[1] <= maxY;
// }
// return booleanWithin(turfPoint(point), polygon);
// }

updateLayerMeta(dataContainer) {
const getFeature = this.getPositionAccessor(dataContainer);
Expand Down
9 changes: 5 additions & 4 deletions src/layers/src/geojson-layer/geojson-utils.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,8 +20,8 @@

import normalize from '@mapbox/geojson-normalize';
import bbox from '@turf/bbox';
// import center from '@turf/center';
// import centerOfMass from '@turf/center-of-mass';
import center from '@turf/center';
import centerOfMass from '@turf/center-of-mass';
import {parseSync} from '@loaders.gl/core';
import {WKBLoader, WKTLoader} from '@loaders.gl/wkt';
import {binaryToGeometry} from '@loaders.gl/gis';
Expand Down Expand Up @@ -188,8 +188,9 @@ export function getGeojsonMeanCenters(dataToFeature: GeojsonDataMaps): number[][
// meanCenters.push(centerOfMass(feature).geometry.coordinates);
const geometries = feature.geometry.type === 'GeometryCollection' ?
feature.geometry.geometries : [feature.geometry];
const center = getMeanCenterFromGeometries(geometries);
meanCenters.push(center);
// const cent = getMeanCenterFromGeometries(geometries);
const cent = centerOfMass(feature).geometry.coordinates;
meanCenters.push(cent);
}
}
console.timeEnd('getGeojsonMeanCenters');
Expand Down
5 changes: 5 additions & 0 deletions src/reducers/src/vis-state-updaters.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -858,6 +858,11 @@ export function setFilterUpdater(
dataId: newDataIds
};

// if (newFilter.gpu) {
// newFilter = setFilterGpuMode(newFilter, state.filters);
// newFilter = assignGpuChannel(newFilter, state.filters);
// }

break;
default:
break;
Expand Down
3 changes: 3 additions & 0 deletions src/table/src/gpu-filter-utils.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -183,6 +183,9 @@ const getFilterValueAccessor = (
if (!filter) {
return 0;
}
// if (filter.type === FILTER_TYPES.polygon) {
// return getData(dc, d, -1, filter);
// }
const fieldIndex = getDatasetFieldIndexForFilter(dataId, filter);
const field = fields[fieldIndex];

Expand Down
29 changes: 3 additions & 26 deletions src/table/src/kepler-table.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,6 @@

import {console as Console} from 'global/console';
import {ascending, descending} from 'd3-array';
import {intersection} from 'lodash';
import {
TRIP_POINT_FIELDS,
SORT_ORDER,
Expand Down Expand Up @@ -60,8 +59,7 @@ import {
getLogDomain,
getOrdinalDomain,
getQuantileDomain,
DataContainerInterface,
createIndexedDataContainer
DataContainerInterface
} from '@kepler.gl/utils';

export type GpuFilter = {
Expand Down Expand Up @@ -322,39 +320,18 @@ class KeplerTable {
const dynamicDomainFilters = shouldCalDomain ? filterRecord.dynamicDomain : null;
const cpuFilters = shouldCalIndex ? filterRecord.cpu : null;

// use layer index to build IndexedDataContainer to narrow down the data
console.time('filterDataByFilterTypes');
const preFilteredIndexes: number[][] = [];
for (const filter of filters) {
if (filter.type === FILTER_TYPES.polygon) {
const filteredLayers = layers.filter(l => l.config.dataId === this.id);
// iterator over filteredLayers, and use it's spatial index to filter data
filteredLayers?.forEach(layer => {
const index = KeplerGlLayers.GeojsonLayer.index.get(layer.id);
const [minX, minY, maxX, maxY] = filter.value.properties.bbox;
const foundIndexes = index?.search(minX, minY, maxX, maxY) || [];
preFilteredIndexes.push(foundIndexes);
});
}
}
console.timeEnd('filterDataByFilterTypes');
const filteredDataContainer =
preFilteredIndexes.length > 0
? createIndexedDataContainer(dataContainer, intersection(preFilteredIndexes.flat()))
: dataContainer;

const filterFuncs = filters.reduce((acc, filter) => {
const fieldIndex = getDatasetFieldIndexForFilter(this.id, filter);
const field = fieldIndex !== -1 ? fields[fieldIndex] : null;
return {
...acc,
[filter.id]: getFilterFunction(field, this.id, filter, layers, filteredDataContainer)
[filter.id]: getFilterFunction(field, this.id, filter, layers, dataContainer)
};
}, {});

filterResult = filterDataByFilterTypes(
{dynamicDomainFilters, cpuFilters, filterFuncs},
filteredDataContainer
opt?.filteredDataContainer ? opt.filteredDataContainer : dataContainer
);
}

Expand Down
1 change: 1 addition & 0 deletions src/types/reducers.d.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -468,6 +468,7 @@ export type FilterDatasetOpt = {
cpuOnly?: boolean;
// ignore filter for domain calculation
ignoreDomain?: boolean;
filteredDataContainer?: any;
};

/* DUPLICATES OF FILTER TYPES ABOVE, REMOVE ONCE TYPES ABOVE ARE FIXED */
Expand Down
69 changes: 43 additions & 26 deletions src/utils/src/filter-utils.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@ import {
FILTER_VIEW_TYPES
} from '@kepler.gl/constants';
import {VisState} from '@kepler.gl/schemas';
import {KeplerGlLayers} from '@kepler.gl/layers';
import * as ScaleUtils from './data-scale-utils';
import {h3IsValid} from 'h3-js';

Expand Down Expand Up @@ -63,6 +64,7 @@ import {DataContainerInterface} from './data-container-interface';
import {generateHashId, set, toArray} from './utils';
import {notNullorUndefined, timeToUnixMilli, unique} from './data-utils';
import {getCentroid} from './h3-utils';
import { IndexedDataContainer } from './indexed-data-container';

export const durationSecond = 1000;
export const durationMinute = durationSecond * 60;
Expand Down Expand Up @@ -456,13 +458,7 @@ export const getPolygonFilterFunctor = (layer, filter, dataContainer) => {
};
case LAYER_TYPES.geojson:
return data => {
if (filter.value.properties?.shape === 'Rectangle') {
return true;
}
if (layer.isInPolygon) {
return layer.isInPolygon(data, data.index, filter.value);
}
// show all geometries if can't apply filter
// data are already filtered by spatial index
return true;
};
default:
Expand Down Expand Up @@ -557,24 +553,17 @@ export function filterDataByFilterTypes(

const numRows = dataContainer.numRows();
const plainIndexes = dataContainer.getPlainIndex();
if (dynamicDomainFilters) {
for (let i = 0; i < numRows; ++i) {
filterContext.index = plainIndexes[i];
for (let i = 0; i < numRows; ++i) {
filterContext.index = plainIndexes[i];

const matchForDomain = dynamicDomainFilters && dynamicDomainFilters.every(filterFuncCaller);
if (matchForDomain) {
filteredIndexForDomain.push(filterContext.index);
}
const matchForDomain = dynamicDomainFilters && dynamicDomainFilters.every(filterFuncCaller);
if (matchForDomain) {
filteredIndexForDomain.push(filterContext.index);
}
}
// TODO: with a index, we should be able to avoid iterate through all data
if (cpuFilters) {
for (let i = 0; i < numRows; ++i) {
filterContext.index = plainIndexes[i];
const matchForRender = cpuFilters && cpuFilters.every(filterFuncCaller);
if (matchForRender) {
filteredIndex.push(filterContext.index);
}

const matchForRender = cpuFilters && cpuFilters.every(filterFuncCaller);
if (matchForRender) {
filteredIndex.push(filterContext.index);
}
}

Expand Down Expand Up @@ -968,7 +957,7 @@ export function getDefaultFilterPlotType(filter: Filter): string | null {
*/
export function applyFiltersToDatasets<
K extends KeplerTableModel<K, L>,
L extends {config: {dataId: string | null}}
L extends {config: {dataId: string | null}, id: string}
>(
datasetIds: string[],
datasets: {[id: string]: K},
Expand All @@ -981,9 +970,36 @@ export function applyFiltersToDatasets<
const appliedFilters = filters.filter(d => shouldApplyFilter(d, dataId));
const table = datasets[dataId];

// use layer index to build IndexedDataContainer to narrow down the data for filtering
console.time('filterDataByFilterTypes');
const preFilteredIndexes: number[][] = [];
for (const filter of filters) {
if (filter.type === FILTER_TYPES.polygon) {
// iterator over filteredLayers, and use it's spatial index to filter data
layersToFilter?.forEach(layer => {
const index = KeplerGlLayers.GeojsonLayer.spatialIndex.get(layer.id);
if (index) {
const foundIndexes = index.search(filter, layer) || [];
preFilteredIndexes.push(foundIndexes);
}
});
}
}
const filteredDataContainerIndexes = {};
// iterator preFilterIndexes
for (let i = 0; i < preFilteredIndexes.length; i++) {
const indexes = preFilteredIndexes[i];
for (let j = 0; j < indexes.length; j++) {
const index = indexes[j];
filteredDataContainerIndexes[index] = index;
}
}
const filteredDataContainer = Object.keys(filteredDataContainerIndexes).length > 0 ? new IndexedDataContainer(table.dataContainer, Object.values(filteredDataContainerIndexes)) : null;
console.timeEnd('filterDataByFilterTypes');

return {
...acc,
[dataId]: table.filterTable(appliedFilters, layersToFilter, {})
[dataId]: table.filterTable(appliedFilters, layersToFilter, filteredDataContainer ? {filteredDataContainer} : {})
};
}, datasets);
}
Expand Down Expand Up @@ -1127,7 +1143,8 @@ export function generatePolygonFilter<
type: FILTER_TYPES.polygon,
name,
layerId,
value: featureToFilterValue(feature, filter.id, {isVisible: true})
value: featureToFilterValue(feature, filter.id, {isVisible: true}),
gpu: false
};
}

Expand Down

0 comments on commit 14d1452

Please sign in to comment.