From 4e42ad7f8a3c657c041ac6307824aefeb0fa844a Mon Sep 17 00:00:00 2001 From: Xun Li Date: Wed, 18 Oct 2023 00:05:23 -0700 Subject: [PATCH] add hover on arrowlayer --- src/components/src/map-container.tsx | 4 +- src/constants/src/default-settings.ts | 3 +- src/layers/src/arrow-layer/arrow-layer.ts | 116 ++++---- src/layers/src/base-layer.ts | 1 + src/layers/src/geojson-layer/geojson-layer.ts | 19 +- src/processors/src/data-processor.ts | 2 +- src/table/src/kepler-table.ts | 6 +- src/utils/src/arrow-utils.ts | 248 +++++++++++++++++- 8 files changed, 319 insertions(+), 80 deletions(-) diff --git a/src/components/src/map-container.tsx b/src/components/src/map-container.tsx index d411d867cf..6baaf34aeb 100644 --- a/src/components/src/map-container.tsx +++ b/src/components/src/map-container.tsx @@ -738,7 +738,7 @@ export default function MapContainerFactory( mapboxApiAccessToken, mapboxApiUrl, layersForDeck, - editorInfo: false + editorInfo: primaryMap ? { editor, editorMenuActive, @@ -931,7 +931,7 @@ export default function MapContainerFactory( // TODO this should be part of onLayerHover arguments, investigate // @ts-ignore (does not fail with local yarn-test) data.mapIndex = index; - + console.log(data); this.props.visStateActions.onLayerHover(data); }, DEBOUNCE_MOUSE_MOVE_PROPAGATE); diff --git a/src/constants/src/default-settings.ts b/src/constants/src/default-settings.ts index 2d9023e22d..0f16143255 100644 --- a/src/constants/src/default-settings.ts +++ b/src/constants/src/default-settings.ts @@ -430,7 +430,8 @@ export const ALL_FIELD_TYPES = keyMirror({ timestamp: null, point: null, array: null, - object: null + object: null, + geoarrow: null }); // Data Table diff --git a/src/layers/src/arrow-layer/arrow-layer.ts b/src/layers/src/arrow-layer/arrow-layer.ts index abe5b6a129..4f892069bc 100644 --- a/src/layers/src/arrow-layer/arrow-layer.ts +++ b/src/layers/src/arrow-layer/arrow-layer.ts @@ -18,71 +18,47 @@ // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN // THE SOFTWARE. -import uniq from 'lodash.uniq'; -import {DATA_TYPES} from 'type-analyzer'; -import {Feature} from 'geojson'; - import {BinaryFeatures} from '@loaders.gl/schema'; import {GeoJsonLayer as DeckGLGeoJsonLayer} from '@deck.gl/layers'; -import { - GEOJSON_FIELDS, - HIGHLIGH_COLOR_3D, - CHANNEL_SCALES, - ColorRange, - LAYER_VIS_CONFIGS -} from '@kepler.gl/constants'; -import { - VisConfigNumber, - VisConfigColorSelect, - VisConfigColorRange, - VisConfigRange, - VisConfigBoolean, - Merge, - RGBColor -} from '@kepler.gl/types'; +import {HIGHLIGH_COLOR_3D} from '@kepler.gl/constants'; import {KeplerTable} from '@kepler.gl/table'; -import { DataContainerInterface, getBinaryGeometriesFromGeoArrowPolygon } from '@kepler.gl/utils'; - -import Layer, { - colorMaker, - LayerBaseConfig, - LayerBaseConfigPartial, - LayerColorConfig, - LayerColumn, - LayerHeightConfig, - LayerRadiusConfig, - LayerSizeConfig, - LayerStrokeColorConfig -} from '../base-layer'; import { - getGeojsonDataMaps, - getGeojsonBounds, - getGeojsonFeatureTypes, - GeojsonDataMaps -} from '../geojson-layer/geojson-utils'; -import GeoJsonLayer, {SUPPORTED_ANALYZER_TYPES} from '../geojson-layer/geojson-layer'; + DataContainerInterface, + getBinaryGeometriesFromGeoArrowPolygon, + parseGeometryFromArrow +} from '@kepler.gl/utils'; +import {Merge} from '@kepler.gl/types'; + +import {LayerColumn, LayerBaseConfig} from '../base-layer'; +import GeoJsonLayer, { + SUPPORTED_ANALYZER_TYPES, + GeoJsonLayerVisConfig, + GeoJsonLayerVisualChannelConfig +} from '../geojson-layer/geojson-layer'; export default class ArrowLayer extends GeoJsonLayer { - dataToFeature: BinaryFeatures[]; + binaryFeatures: BinaryFeatures[]; + dataContainer: DataContainerInterface | null; // constructor constructor(props) { super(props); - this.dataToFeature = []; + this.dataContainer = null; + this.binaryFeatures = []; } - static findDefaultLayerProps({label, metadata, fields = []}: KeplerTable) { - const geojsonColumns = fields - .filter(f => f.type === 'geojson' && SUPPORTED_ANALYZER_TYPES[f.analyzerType]) + static findDefaultLayerProps({label, fields = []}: KeplerTable) { + const geoarrowColumns = fields + .filter(f => f.type === 'geoarrow' && SUPPORTED_ANALYZER_TYPES[f.analyzerType]) .map(f => f.name); const defaultColumns = { - geojson: uniq([...GEOJSON_FIELDS.geojson, ...geojsonColumns]) + geojson: geoarrowColumns }; const foundColumns = this.findDefaultColumnField(defaultColumns, fields); - if (!foundColumns || !foundColumns.length || metadata.format !== 'arrow') { + if (!foundColumns || !foundColumns.length) { return {props: []}; } @@ -106,13 +82,19 @@ export default class ArrowLayer extends GeoJsonLayer { return 'GeoArrow'; } + get requiredLayerColumns() { + return ['geojson']; + } + calculateDataAttribute({dataContainer, filteredIndex}, getPosition) { // TODO: filter arrow table using predicate // filteredIndex.map(i => this.dataToFeature[i]).filter(d => d); - return this.dataToFeature; + this.dataContainer = dataContainer; + return this.binaryFeatures; } updateLayerMeta(dataContainer: DataContainerInterface) { + this.dataContainer = dataContainer; const {geojson} = this.config.columns; const geoColumn = dataContainer.getColumn(geojson.fieldIdx); @@ -120,12 +102,49 @@ export default class ArrowLayer extends GeoJsonLayer { const {binaryGeometries, bounds, featureTypes} = getBinaryGeometriesFromGeoArrowPolygon( geoColumn ); - this.dataToFeature = binaryGeometries; + this.binaryFeatures = binaryGeometries; const fixedRadius = false; this.updateMeta({bounds, fixedRadius, featureTypes}); } + isLayerHovered(objectInfo): boolean { + // there could be multiple deck.gl layers created from multiple chunks in arrow table + // the objectInfo.layer id should be `${this.id}-${i}` + if (objectInfo?.picked) { + const deckLayerId = objectInfo?.layer?.props?.id; + console.log(deckLayerId); + return deckLayerId.startsWith(this.id); + } + return false; + } + + hasHoveredObject(objectInfo) { + // hover object returns the index of the object in the data array + if (this.isLayerHovered(objectInfo) && objectInfo.index >= 0 && this.dataContainer) { + console.time('getHoverData'); + const {geojson} = this.config.columns; + const col = this.dataContainer.getColumn(geojson.fieldIdx); + const rawGeometry = col.get(objectInfo.index); + const hoveredFeature = parseGeometryFromArrow({ + encoding: col.metadata?.get('ARROW:extension:name'), + data: rawGeometry + }); + console.timeEnd('getHoverData'); + return hoveredFeature; + } + return null; + } + + getHoverData(object, dataContainer) { + // index of dataContainer is saved to feature.properties + const index = object; + if (index >= 0) { + return dataContainer.row(index); + } + return null; + } + renderLayer(opts) { const {data, gpuFilter, objectHovered, mapState, interactionConfig} = opts; @@ -155,6 +174,7 @@ export default class ArrowLayer extends GeoJsonLayer { const pickable = interactionConfig.tooltip.enabled; const hoveredObject = this.hasHoveredObject(objectHovered); + console.log(hoveredObject); const deckLayers = data.data.map((d, i) => { return new DeckGLGeoJsonLayer({ diff --git a/src/layers/src/base-layer.ts b/src/layers/src/base-layer.ts index 2541e98018..654bfd1ddf 100644 --- a/src/layers/src/base-layer.ts +++ b/src/layers/src/base-layer.ts @@ -1256,6 +1256,7 @@ class Layer { } hasHoveredObject(objectInfo) { + console.log('objectInfo.object', objectInfo?.object); return this.isLayerHovered(objectInfo) && objectInfo.object ? objectInfo.object : null; } diff --git a/src/layers/src/geojson-layer/geojson-layer.ts b/src/layers/src/geojson-layer/geojson-layer.ts index c387b902dd..97514fb96b 100644 --- a/src/layers/src/geojson-layer/geojson-layer.ts +++ b/src/layers/src/geojson-layer/geojson-layer.ts @@ -152,7 +152,7 @@ export type GeoJsonLayerVisConfig = { wireframe: boolean; }; -type GeoJsonLayerVisualChannelConfig = LayerColorConfig & +export type GeoJsonLayerVisualChannelConfig = LayerColorConfig & LayerStrokeColorConfig & LayerSizeConfig & LayerHeightConfig & @@ -168,7 +168,7 @@ export type GeoJsonLayerMeta = { fixedRadius?: boolean; }; -export const geoJsonRequiredColumns: ['geojson'] = ['geojson']; +export const geoJsonRequiredColumns = ['geojson']; export const featureAccessor = ({geojson}: GeoJsonLayerColumnsConfig) => ( dc: DataContainerInterface ) => d => dc.valueAt(d.index, geojson.fieldIdx); @@ -276,7 +276,7 @@ export default class GeoJsonLayer extends Layer { }; } - static findDefaultLayerProps({label, metadata, fields = []}: KeplerTable) { + static findDefaultLayerProps({label, fields = []}: KeplerTable) { const geojsonColumns = fields .filter(f => f.type === 'geojson' && SUPPORTED_ANALYZER_TYPES[f.analyzerType]) .map(f => f.name); @@ -286,7 +286,7 @@ export default class GeoJsonLayer extends Layer { }; const foundColumns = this.findDefaultColumnField(defaultColumns, fields); - if (!foundColumns || !foundColumns.length || metadata.format === 'arrow') { + if (!foundColumns || !foundColumns.length) { return {props: []}; } @@ -398,17 +398,6 @@ export default class GeoJsonLayer extends Layer { return this; } - hasLayerData(layerData) { - if (!layerData) { - return false; - } - return Boolean( - layerData.data && - (layerData.data.length || - ('points' in layerData.data && 'polygons' in layerData.data && 'lines' in layerData.data)) - ); - } - renderLayer(opts) { const {data, gpuFilter, objectHovered, mapState, interactionConfig} = opts; diff --git a/src/processors/src/data-processor.ts b/src/processors/src/data-processor.ts index bd8098d7ee..c8eef471bb 100644 --- a/src/processors/src/data-processor.ts +++ b/src/processors/src/data-processor.ts @@ -437,7 +437,7 @@ export function processArrowTable(arrowTable: ApacheArrowTable): ProcessorResult displayName: field.name, format: '', fieldIdx: index, - type: isGeometryColumn ? ALL_FIELD_TYPES.geojson : arrowDataTypeToFieldType(field.type), + type: isGeometryColumn ? ALL_FIELD_TYPES.geoarrow: arrowDataTypeToFieldType(field.type), analyzerType: isGeometryColumn ? AnalyzerDATA_TYPES.GEOMETRY : arrowDataTypeToAnalyzerDataType(field.type), diff --git a/src/table/src/kepler-table.ts b/src/table/src/kepler-table.ts index 99708b268a..7f04a400f8 100644 --- a/src/table/src/kepler-table.ts +++ b/src/table/src/kepler-table.ts @@ -149,8 +149,10 @@ class KeplerTable { // } const isArrow = info?.format === 'arrow'; - const dataContainer = createDataContainer( - isArrow ? data.rawData._children : data.rows, { + const dataContainerData = isArrow + ? [...Array(data.rawData?.numCols || 0).keys()].map(i => data.rawData.getColumnAt(i)) + : data.rows + const dataContainer = createDataContainer(dataContainerData, { // @ts-expect-error fields: data.fields, inputDataFormat: isArrow ? DataForm.COLS_ARRAY : DataForm.ROWS_ARRAY diff --git a/src/utils/src/arrow-utils.ts b/src/utils/src/arrow-utils.ts index bbd5d5a34f..761e30cdca 100644 --- a/src/utils/src/arrow-utils.ts +++ b/src/utils/src/arrow-utils.ts @@ -18,8 +18,27 @@ // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN // THE SOFTWARE. -import {ListVector, Float64Vector, Column as ArrowColumn} from 'apache-arrow'; +import {ListVector, FloatVector, Float64Vector, Column as ArrowColumn} from 'apache-arrow'; +import normalize from '@mapbox/geojson-normalize'; import {BinaryFeatures} from '@loaders.gl/schema'; +import {parseSync} from '@loaders.gl/core'; +import {WKBLoader, WKTLoader} from '@loaders.gl/wkt'; +import {binaryToGeometry} from '@loaders.gl/gis'; +import { + Feature, + MultiPolygon, + Position, + Polygon, + MultiPoint, + Point, + MultiLineString, + LineString +} from 'geojson'; + +type RawArrowFeature = { + encoding?: string; + data: any; +}; export enum GEOARROW_ENCODINGS { MULTI_POLYGON = 'geoarrow.multipolygon', @@ -27,7 +46,9 @@ export enum GEOARROW_ENCODINGS { MULTI_LINESTRING = 'geoarrow.multilinestring', LINESTRING = 'geoarrow.linestring', MULTI_POINT = 'geoarrow.multipoint', - POINT = 'geoarrow.point' + POINT = 'geoarrow.point', + WKB = 'geoarrow.wkb', + WKT = 'geoarrow.wkt' } /** @@ -180,18 +201,19 @@ export function getBinaryGeometriesFromGeoArrowPolygon( geoEncoding ); - const numOfFeatures = flatCoordinateArray.length / nDim; - const featureIds = new Uint32Array(numOfFeatures); - for (let i = 0; i < geomOffset.length; i++) { - for (let j = geomOffset[i]; j < geomOffset[i + 1]; j++) { + const numOfVertices = flatCoordinateArray.length / nDim; + const featureIds = new Uint32Array(numOfVertices); + for (let i = 0; i < geometries.length - 1; i++) { + const startIdx = geomOffset[geometries.valueOffsets[i]]; + const endIdx = geomOffset[geometries.valueOffsets[i + 1]]; + for (let j = startIdx; j < endIdx; j++) { featureIds[j] = i; } } - const globalFeatureIds = new Uint32Array(numOfFeatures); - for (let i = 0; i < geomOffset.length; i++) { - for (let j = geomOffset[i]; j < geomOffset[i + 1]; j++) { - globalFeatureIds[j] = i + globalFeatureIdOffset; - } + + const globalFeatureIds = new Uint32Array(numOfVertices); + for (let i = 0; i < numOfVertices; i++) { + globalFeatureIds[i] = featureIds[i] + globalFeatureIdOffset; } globalFeatureIdOffset += geometries.length; @@ -231,3 +253,207 @@ export function getBinaryGeometriesFromGeoArrowPolygon( return {binaryGeometries, bounds, featureTypes}; } + +/** + * convert Arrow MultiPolygon to geojson Feature + */ +function arrowMultiPolygonToFeature(arrowMultiPolygon: ListVector): MultiPolygon { + const multiPolygon: Position[][][] = []; + for (let m = 0; m < arrowMultiPolygon.length; m++) { + const arrowPolygon = arrowMultiPolygon.get(m); + const polygon: Position[][] = []; + for (let i = 0; arrowPolygon && i < arrowPolygon?.length; i++) { + const arrowRing = arrowPolygon?.get(i); + const ring: Position[] = []; + for (let j = 0; arrowRing && j < arrowRing.length; j++) { + const arrowCoord = arrowRing.get(j); + const coord: Position = Array.from(arrowCoord); + ring.push(coord); + } + polygon.push(ring); + } + multiPolygon.push(polygon); + } + const geometry: MultiPolygon = { + type: 'MultiPolygon', + coordinates: multiPolygon + }; + return geometry; +} + +/** + * convert Arrow Polygon to geojson Feature + */ +function arrowPolygonToFeature(arrowPolygon: ListVector): Polygon { + const polygon: Position[][] = []; + for (let i = 0; arrowPolygon && i < arrowPolygon.length; i++) { + const arrowRing = arrowPolygon.get(i); + const ring: Position[] = []; + for (let j = 0; arrowRing && j < arrowRing.length; j++) { + const arrowCoord = arrowRing.get(j); + const coords: Position = Array.from(arrowCoord); + ring.push(coords); + } + polygon.push(ring); + } + const geometry: Polygon = { + type: 'Polygon', + coordinates: polygon + }; + return geometry; +} + +/** + * convert Arrow MultiPoint to geojson MultiPoint + */ +function arrowMultiPointToFeature(arrowMultiPoint: ListVector): MultiPoint { + const multiPoint: Position[] = []; + for (let i = 0; arrowMultiPoint && i < arrowMultiPoint.length; i++) { + const arrowPoint = arrowMultiPoint.get(i); + if (arrowPoint) { + const coord: Position = Array.from(arrowPoint); + multiPoint.push(coord); + } + } + const geometry: MultiPoint = { + type: 'MultiPoint', + coordinates: multiPoint + }; + return geometry; +} + +/** + * convert Arrow Point to geojson Point + */ +function arrowPointToFeature(arrowPoint: FloatVector): Point { + const point: Position = Array.from(arrowPoint.values); + const geometry: Point = { + type: 'Point', + coordinates: point + }; + return geometry; +} + +/** + * convert Arrow MultiLineString to geojson MultiLineString + */ +function arrowMultiLineStringToFeature(arrowMultiLineString: ListVector): MultiLineString { + const multiLineString: Position[][] = []; + for (let i = 0; arrowMultiLineString && i < arrowMultiLineString.length; i++) { + const arrowLineString = arrowMultiLineString.get(i); + const lineString: Position[] = []; + for (let j = 0; arrowLineString && j < arrowLineString.length; j++) { + const arrowCoord = arrowLineString.get(j); + if (arrowCoord) { + const coords: Position = Array.from(arrowCoord); + lineString.push(coords); + } + } + multiLineString.push(lineString); + } + const geometry: MultiLineString = { + type: 'MultiLineString', + coordinates: multiLineString + }; + return geometry; +} + +/** + * convert Arrow LineString to geojson LineString + */ +function arrowLineStringToFeature(arrowLineString: ListVector): LineString { + const lineString: Position[] = []; + for (let i = 0; arrowLineString && i < arrowLineString.length; i++) { + const arrowCoord = arrowLineString.get(i); + if (arrowCoord) { + const coords: Position = Array.from(arrowCoord); + lineString.push(coords); + } + } + const geometry: LineString = { + type: 'LineString', + coordinates: lineString + }; + return geometry; +} + +/** + * convert Arrow wkb to geojson Geometry + */ +function arrowWkbToFeature(arrowWkb: Uint8Array): Feature | null { + const binaryGeo = parseSync(arrowWkb, WKBLoader); + const geometry = binaryToGeometry(binaryGeo); + const normalized = normalize(geometry); + + if (!normalized || !Array.isArray(normalized.features)) { + // fail to normalize geojson + return null; + } + + return normalized.features[0]; +} + +/** + * convert Arrow wkt to geojson Geometry + */ +function arrowWktToFeature(arrowWkt: string): Feature | null { + const geometry = parseSync(arrowWkt, WKTLoader); + const normalized = normalize(geometry); + + if (!normalized || !Array.isArray(normalized.features)) { + // fail to normalize geojson + return null; + } + + return normalized.features[0]; +} + +/** + * parse geometry from arrow data that is returned from processArrowData() + * + * @param rawData the raw geometry data returned from processArrowData, which is an object with two properties: encoding and data + * @see processArrowData + * @returns + */ +export function parseGeometryFromArrow(rawData: RawArrowFeature): Feature | null { + const encoding = rawData.encoding?.toLowerCase(); + const data = rawData.data; + if (!encoding || !data) return null; + + let geometry; + + switch (encoding) { + case GEOARROW_ENCODINGS.MULTI_POLYGON: + geometry = arrowMultiPolygonToFeature(data); + break; + case GEOARROW_ENCODINGS.POLYGON: + geometry = arrowPolygonToFeature(data); + break; + case GEOARROW_ENCODINGS.MULTI_POINT: + geometry = arrowMultiPointToFeature(data); + break; + case GEOARROW_ENCODINGS.POINT: + geometry = arrowPointToFeature(data); + break; + case GEOARROW_ENCODINGS.MULTI_LINESTRING: + geometry = arrowMultiLineStringToFeature(data); + break; + case GEOARROW_ENCODINGS.LINESTRING: + geometry = arrowLineStringToFeature(data); + break; + case GEOARROW_ENCODINGS.WKB: { + return arrowWkbToFeature(data); + } + case GEOARROW_ENCODINGS.WKT: { + return arrowWktToFeature(data); + } + default: { + throw Error('GeoArrow encoding not supported'); + } + } + return { + type: 'Feature', + geometry, + properties: {} + }; +}