diff --git a/examples/webpack.config.local.js b/examples/webpack.config.local.js index ba6b4912da..055ea766bf 100644 --- a/examples/webpack.config.local.js +++ b/examples/webpack.config.local.js @@ -256,7 +256,12 @@ function addBabelSettings(env, config, exampleDir) { ...config.module, rules: [ ...config.module.rules.filter(r => r.loader !== 'babel-loader'), - makeBabelRule(env, exampleDir) + makeBabelRule(env, exampleDir), + { + test: /\.mjs$/, + include: /node_modules/, + type: 'javascript/auto' + } ] } }; diff --git a/jest.setup.js b/jest.setup.js index c3ddadd8af..3025e41f66 100644 --- a/jest.setup.js +++ b/jest.setup.js @@ -31,3 +31,14 @@ jest.mock('@kepler.gl/utils', () => ({ hasPortableWidth: jest.fn(), hasMobileWidth: jest.fn() })); + +// TextEncoder / TextDecoder APIs are used by arrow, but are not provided by +// jsdom, all node versions supported provide these via the util module +if ( + typeof globalThis.TextEncoder === "undefined" || + typeof globalThis.TextDecoder === "undefined" +) { + const utils = require("util"); + globalThis.TextEncoder = utils.TextEncoder; + globalThis.TextDecoder = utils.TextDecoder; +} diff --git a/src/actions/src/action-types.ts b/src/actions/src/action-types.ts index 71db454a89..2ae1748a2c 100644 --- a/src/actions/src/action-types.ts +++ b/src/actions/src/action-types.ts @@ -104,6 +104,7 @@ export const ActionTypes = { LOAD_FILES: `${ACTION_PREFIX}LOAD_FILES`, LOAD_NEXT_FILE: `${ACTION_PREFIX}LOAD_NEXT_FILE`, LOAD_FILE_STEP_SUCCESS: `${ACTION_PREFIX}LOAD_FILE_STEP_SUCCESS`, + LOAD_BATCH_DATA_SUCCESS: `${ACTION_PREFIX}LOAD_BATCH_DATA_SUCCESS`, LOAD_FILES_ERR: `${ACTION_PREFIX}LOAD_FILES_ERR`, LOAD_FILES_SUCCESS: `${ACTION_PREFIX}LOAD_FILES_SUCCESS`, LAYER_COLOR_UI_CHANGE: `${ACTION_PREFIX}LAYER_COLOR_UI_CHANGE`, diff --git a/src/actions/src/vis-state-actions.ts b/src/actions/src/vis-state-actions.ts index 3480890bed..31fb94dfc1 100644 --- a/src/actions/src/vis-state-actions.ts +++ b/src/actions/src/vis-state-actions.ts @@ -1240,6 +1240,20 @@ export function loadFileStepSuccess({ }; } +export function loadBatchDataSuccess({ + fileName, + fileCache +}: { + fileName: string; + fileCache: FileCacheItem[]; +}): Merge { + return { + type: ActionTypes.LOAD_BATCH_DATA_SUCCESS, + fileName, + fileCache + }; +} + export type LoadFilesErrUpdaterAction = { fileName: string; error: any; diff --git a/src/components/package.json b/src/components/package.json index a9bfe192ac..b21a7dc230 100644 --- a/src/components/package.json +++ b/src/components/package.json @@ -67,8 +67,8 @@ "@types/lodash.throttle": "^4.1.7", "@types/lodash.uniq": "^4.5.7", "@types/lodash.uniqby": "^4.7.7", - "@types/react-copy-to-clipboard": "^5.0.2", "@types/react": "^18.0.28", + "@types/react-copy-to-clipboard": "^5.0.2", "@types/react-dom": "^18.0.11", "@types/react-lifecycles-compat": "^3.0.1", "@types/react-map-gl": "^6.1.3", diff --git a/src/components/src/map-container.tsx b/src/components/src/map-container.tsx index 52017b6496..4726c4297d 100644 --- a/src/components/src/map-container.tsx +++ b/src/components/src/map-container.tsx @@ -819,7 +819,6 @@ export default function MapContainerFactory( return null; } } - return (
number; +}; + +// TODO: this should be in deck.gl extensions +// A simple extension to filter arrow layer based on the result of CPU filteredIndex, +// so we can avoid filtering on the raw Arrow table and recreating geometry attributes. +// Specifically, an attribute `filtered` is added to the layer to indicate whether the feature has been Filtered +// the shader module is modified to discard the feature if filtered value is 0 +// the accessor getFiltered is used to get the value of `filtered` based on eht value `filteredIndex` in Arrowlayer +export default class FilterArrowExtension extends LayerExtension { + static defaultProps = defaultProps; + static extensionName = 'FilterArrowExtension'; + + getShaders(extension: any) { + return { + modules: [shaderModule], + defines: {} + }; + } + + initializeState(this: Layer, context: LayerContext, extension: this) { + const attributeManager = this.getAttributeManager(); + if (attributeManager) { + attributeManager.add({ + filtered: { + size: 1, + type: GL.FLOAT, + accessor: 'getFiltered', + shaderAttributes: { + filtered: { + divisor: 0 + }, + instanceFiltered: { + divisor: 1 + } + } + } + }); + } + } +} diff --git a/src/deckgl-layers/src/geojson-layer/filter-shader-module.ts b/src/deckgl-layers/src/geojson-layer/filter-shader-module.ts new file mode 100644 index 0000000000..2270e180fc --- /dev/null +++ b/src/deckgl-layers/src/geojson-layer/filter-shader-module.ts @@ -0,0 +1,39 @@ +import {project} from '@deck.gl/core'; + +const vs = ` + #ifdef NON_INSTANCED_MODEL + #define FILTER_ARROW_ATTRIB filtered + #else + #define FILTER_ARROW_ATTRIB instanceFiltered + #endif + attribute float FILTER_ARROW_ATTRIB; +`; + +const fs = ``; + +const inject = { + 'vs:#decl': ` + varying float is_filtered; + `, + 'vs:#main-end': ` + is_filtered = FILTER_ARROW_ATTRIB; + `, + 'fs:#decl': ` + varying float is_filtered; + `, + 'fs:DECKGL_FILTER_COLOR': ` + // abandon the fragments if it is not filtered + if (is_filtered == 0.) { + discard; + } + ` +}; + +export default { + name: 'filter-arrow', + dependencies: [project], + vs: vs, + fs: fs, + inject: inject, + getUniforms: () => {} +}; diff --git a/src/layers/src/base-layer.ts b/src/layers/src/base-layer.ts index d1c2fe0a32..6fd8640229 100644 --- a/src/layers/src/base-layer.ts +++ b/src/layers/src/base-layer.ts @@ -1104,7 +1104,12 @@ class Layer { const dataUpdateTriggers = this.getDataUpdateTriggers(layerDataset); const triggerChanged = this.getChangedTriggers(dataUpdateTriggers); - if (triggerChanged && triggerChanged.getMeta) { + // NOTES: + // 1) add checker `!oldLayerData`: oldLayerData is undefined means this is the first time layer is rendered + // the updateLayerMeta has already been called in setInitialLayerConfig + // 2) add checker `triggerChanged.getData`: when loading data batches for increamental rendering, + // triggerChanged.getData will be true for updateLayerMeta + if (triggerChanged && (triggerChanged.getMeta || triggerChanged.getData) && Boolean(oldLayerData)) { this.updateLayerMeta(dataContainer, getPosition); } diff --git a/src/layers/src/geojson-layer/geojson-layer.ts b/src/layers/src/geojson-layer/geojson-layer.ts index f307258b0a..1a6d2363e7 100644 --- a/src/layers/src/geojson-layer/geojson-layer.ts +++ b/src/layers/src/geojson-layer/geojson-layer.ts @@ -63,7 +63,7 @@ import {KeplerTable} from '@kepler.gl/table'; import {DataContainerInterface, ArrowDataContainer} from '@kepler.gl/utils'; import {FilterArrowExtension} from '@kepler.gl/deckgl-layers'; -const SUPPORTED_ANALYZER_TYPES = { +export const SUPPORTED_ANALYZER_TYPES = { [DATA_TYPES.GEOMETRY]: true, [DATA_TYPES.GEOMETRY_FROM_STRING]: true, [DATA_TYPES.PAIR_GEOMETRY_FROM_STRING]: true @@ -157,7 +157,7 @@ export type GeoJsonLayerVisConfig = { wireframe: boolean; }; -type GeoJsonLayerVisualChannelConfig = LayerColorConfig & +export type GeoJsonLayerVisualChannelConfig = LayerColorConfig & LayerStrokeColorConfig & LayerSizeConfig & LayerHeightConfig & @@ -222,11 +222,11 @@ export default class GeoJsonLayer extends Layer { get type() { return GeoJsonLayer.type; } - static get type(): 'geojson' { + static get type() { return 'geojson'; } - get name(): 'Polygon' { + get name() { return 'Polygon'; } @@ -419,7 +419,7 @@ export default class GeoJsonLayer extends Layer { const getGeoColumn = geoColumnAccessor(this.config.columns); const getGeoField = geoFieldAccessor(this.config.columns); - if (this.dataToFeature.length === 0) { + // if (this.dataToFeature.length === 0) { const updateLayerMetaFunc = dataContainer instanceof ArrowDataContainer ? getGeojsonLayerMetaFromArrow @@ -431,9 +431,13 @@ export default class GeoJsonLayer extends Layer { getGeoField }); - this.dataToFeature = dataToFeature; + // append new data from binaryGeometries to this.binaryFeatures + for (let i = this.dataToFeature.length; i < dataToFeature.length; ++i) { + this.dataToFeature.push(dataToFeature[i]); + } + // this.dataToFeature = dataToFeature; this.updateMeta({bounds, fixedRadius, featureTypes}); - } + // } } setInitialLayerConfig({dataContainer}) { @@ -497,7 +501,7 @@ export default class GeoJsonLayer extends Layer { const updateTriggers = { ...this.getVisualChannelUpdateTriggers(), getFilterValue: gpuFilter.filterValueUpdateTriggers, - getFiltered: this.filteredIndexTrigger + // getFiltered: this.filteredIndexTrigger }; const defaultLayerProps = this.getDefaultDeckLayerProps(opts); @@ -531,7 +535,7 @@ export default class GeoJsonLayer extends Layer { capRounded: true, jointRounded: true, updateTriggers, - extensions: [...defaultLayerProps.extensions, new FilterArrowExtension()], + // extensions: [...defaultLayerProps.extensions, new FilterArrowExtension()], _subLayerProps: { ...(featureTypes?.polygon ? {'polygons-stroke': opaOverwrite} : {}), ...(featureTypes?.line ? {linestrings: opaOverwrite} : {}), diff --git a/src/layers/src/geojson-layer/geojson-utils.ts b/src/layers/src/geojson-layer/geojson-utils.ts index 72e8082721..e965ecde52 100644 --- a/src/layers/src/geojson-layer/geojson-utils.ts +++ b/src/layers/src/geojson-layer/geojson-utils.ts @@ -43,7 +43,6 @@ export enum FeatureTypes { } /* eslint-enable */ - export function parseGeoJsonRawFeature(rawFeature: unknown): Feature | null { if (typeof rawFeature === 'object') { // Support GeoJson feature as object diff --git a/src/localization/src/translations/en.ts b/src/localization/src/translations/en.ts index 5af2058556..0b5420889f 100644 --- a/src/localization/src/translations/en.ts +++ b/src/localization/src/translations/en.ts @@ -120,6 +120,7 @@ export default { hexbin: 'hexbin', polygon: 'polygon', geojson: 'geojson', + geoarrow: 'geoarrow', cluster: 'cluster', icon: 'icon', heatmap: 'heatmap', diff --git a/src/processors/src/file-handler.ts b/src/processors/src/file-handler.ts index 425afc1997..10275ab33d 100644 --- a/src/processors/src/file-handler.ts +++ b/src/processors/src/file-handler.ts @@ -29,7 +29,7 @@ import { processKeplerglJSON, processRowObject } from './data-processor'; -import {generateHashId, isPlainObject} from '@kepler.gl/utils'; +import {generateHashId, generateHashIdFromString, isPlainObject} from '@kepler.gl/utils'; import {DATASET_FORMATS} from '@kepler.gl/constants'; import {Loader} from '@loaders.gl/loader-utils'; import {FileCacheItem, ValidKeplerGlMap} from './types'; @@ -61,6 +61,18 @@ const JSON_LOADER_OPTIONS = { ] }; +/** + * ProcessFileDataContent + * @typedef {Object} ProcessFileDataContent + * @property {Object} data - parsed data + * @property {string} fileName - file name + * @property {number} [length] - number of rows + * @property {Object} [progress] - progress of file processing + * @property {number} [progress.rowCount] - number of rows processed + * @property {number} [progress.rowCountInBatch] - number of rows in current batch + * @property {number} [progress.percent] - percent of file processed + * @property {Map} [metadata] - metadata e.g. for arrow data, metadata could be the schema.fields + */ export type ProcessFileDataContent = { data: unknown; fileName: string; @@ -215,10 +227,13 @@ export function processFileData({ fileCache: FileCacheItem[]; }): Promise { return new Promise((resolve, reject) => { - let {data} = content; + let {fileName, data} = content; let format: string | undefined; let processor: Function | undefined; + // generate unique id with length of 4 using fileName string + const id = generateHashIdFromString(fileName); + if (isArrowData(data)) { format = DATASET_FORMATS.arrow; processor = processArrowTable; @@ -241,6 +256,7 @@ export function processFileData({ { data: result, info: { + id, label: content.fileName, format } diff --git a/src/reducers/src/vis-state-updaters.ts b/src/reducers/src/vis-state-updaters.ts index 166f9ace33..a7ae41db9b 100644 --- a/src/reducers/src/vis-state-updaters.ts +++ b/src/reducers/src/vis-state-updaters.ts @@ -29,7 +29,12 @@ import isEqual from 'lodash.isequal'; import copy from 'copy-to-clipboard'; import deepmerge from 'deepmerge'; // Tasks -import {LOAD_FILE_TASK, UNWRAP_TASK, PROCESS_FILE_DATA, DELAY_TASK} from '@kepler.gl/tasks'; +import { + LOAD_FILE_TASK, + UNWRAP_TASK, + PROCESS_FILE_DATA, + DELAY_TASK +} from '@kepler.gl/tasks'; // Actions import { applyLayerConfig, @@ -40,6 +45,7 @@ import { loadFilesErr, loadFilesSuccess, loadFileStepSuccess, + loadBatchDataSuccess, loadNextFile, nextFileBatch, ReceiveMapConfigPayload, @@ -2181,7 +2187,23 @@ export function loadFileStepSuccessUpdater( return withTask( stateWithCache, - DELAY_TASK(200).map(filesToLoad.length ? loadNextFile : () => onFinish(fileCache)) + DELAY_TASK(0).map(filesToLoad.length ? loadNextFile : () => onFinish(fileCache)) + ); +} + +export function loadBatchDataSuccessUpdater( + state: VisState, + action: VisStateActions.LoadFileStepSuccessAction +): VisState { + if (!state.fileLoading) { + return state; + } + const {fileCache} = action; + const {onFinish} = state.fileLoading; + + return withTask( + state, + DELAY_TASK(0).map(() => onFinish(fileCache)) ); } @@ -2249,7 +2271,6 @@ export function processFileContentUpdater( action: VisStateActions.ProcessFileContentUpdaterAction ): VisState { const {content, fileCache} = action.payload; - const stateWithProgress = updateFileLoadingProgressUpdater(state, { fileName: content.fileName, progress: {percent: 1, message: 'processing...'} @@ -2287,12 +2308,27 @@ export const nextFileBatchUpdater = ( payload: {gen, fileName, progress, accumulated, onFinish} }: VisStateActions.NextFileBatchUpdaterAction ): VisState => { - const stateWithProgress = updateFileLoadingProgressUpdater(state, { + let stateWithProgress = updateFileLoadingProgressUpdater(state, { fileName, progress: parseProgress(state.fileLoadingProgress[fileName], progress) }); - return withTask( - stateWithProgress, + if (accumulated && accumulated.data?.length > 0) { + console.log(accumulated); + const payload = { + content: accumulated, + fileCache: [] + }; + stateWithProgress = processFileContentUpdater(stateWithProgress, {payload}); + } + return withTask(stateWithProgress, [ + ...(accumulated && accumulated.data?.length > 0 + ? [ + PROCESS_FILE_DATA({content: accumulated, fileCache: []}).bimap( + result => loadBatchDataSuccess({fileName, fileCache: result}), + err => loadFilesErr(fileName, err) + ) + ] + : []), UNWRAP_TASK(gen.next()).bimap( ({value, done}) => { return done @@ -2307,7 +2343,7 @@ export const nextFileBatchUpdater = ( }, err => loadFilesErr(fileName, err) ) - ); + ]); }; /** @@ -2334,7 +2370,7 @@ export const loadFilesErrUpdater = ( // kick off next file or finish return withTask( nextState, - DELAY_TASK(200).map(filesToLoad.length ? loadNextFile : () => onFinish(fileCache)) + DELAY_TASK(0).map(filesToLoad.length ? loadNextFile : () => onFinish(fileCache)) ); }; diff --git a/src/table/src/dataset-utils.ts b/src/table/src/dataset-utils.ts index 289ea2a88a..9fcd7c2f4e 100644 --- a/src/table/src/dataset-utils.ts +++ b/src/table/src/dataset-utils.ts @@ -68,6 +68,17 @@ export function createNewDataEntry( return {}; } + // check if dataset already exists + if (info && info.id && datasets[info.id]) { + // get keplerTable from datasets + const keplerTable = datasets[info.id]; + // update the data in keplerTable + keplerTable.update(validatedData); + return { + [keplerTable.id]: keplerTable + }; + } + info = info || {}; const color = info.color || getNewDatasetColor(datasets); diff --git a/src/table/src/kepler-table.ts b/src/table/src/kepler-table.ts index acf5b16ee8..a245db3dc3 100644 --- a/src/table/src/kepler-table.ts +++ b/src/table/src/kepler-table.ts @@ -202,6 +202,19 @@ class KeplerTable { this.disableDataOperation = disableDataOperation; } + /** + * update table with new data + * @param data - new data e.g. the arrow data with new batches loaded + */ + update(data: ProtoDataset['data']) { + const dataContainerData = data.cols ? data.cols : data.rows; + + this.dataContainer.update?.(dataContainerData); + this.allIndexes = this.dataContainer.getPlainIndex(); + this.filteredIndex = this.allIndexes; + this.filteredIndexForDomain = this.allIndexes; + } + get length() { return this.dataContainer.numRows(); } diff --git a/src/utils/src/arrow-data-container.ts b/src/utils/src/arrow-data-container.ts index c838128c88..d372846458 100644 --- a/src/utils/src/arrow-data-container.ts +++ b/src/utils/src/arrow-data-container.ts @@ -64,6 +64,14 @@ export class ArrowDataContainer implements DataContainerInterface { // this._colData = data.cols.map(c => c.toArray()); } + update(updateData: arrow.Vector[]) { + this._cols = updateData; + this._numColumns = this._cols.length; + this._numRows = this._cols[0].length; + // cache column data to make valueAt() faster + // this._colData = this._cols.map(c => c.toArray()); + } + numRows(): number { return this._numRows; } diff --git a/src/utils/src/data-container-interface.ts b/src/utils/src/data-container-interface.ts index cef686d1e1..2ac06ffbdb 100644 --- a/src/utils/src/data-container-interface.ts +++ b/src/utils/src/data-container-interface.ts @@ -61,6 +61,12 @@ export interface DataContainerInterface { */ column(columnIndex: number): Generator; + /** + * Updates the data container with new data. + * @param updateData updated data, e.g. for arrow data container, it's an array of arrow columns; for row data container, it's an array of rows. + */ + update?(updateData: any[]): void; + /** * Returns the column object at the specified index. * @param columnIndex Column index. @@ -75,6 +81,13 @@ export interface DataContainerInterface { */ getField?(columnIndex: number): Field; + /** + * Returns the field object at the specified index. + * @param columnIndex Column index. + * @returns The field object at the specified index. + */ + getField?(columnIndex: number): any; + /** * Returns contents of the data container as a two-dimensional array. * @returns Data. diff --git a/src/utils/src/utils.ts b/src/utils/src/utils.ts index eaa980296c..3441e3e510 100644 --- a/src/utils/src/utils.ts +++ b/src/utils/src/utils.ts @@ -20,6 +20,28 @@ import window from 'global/window'; +/** + * + * @param str + * @returns + */ +export function generateHashIdFromString(str: string): string { + // generate hash string based on string + let hash = 0; + let i; + let chr; + let len; + if (str.length === 0) return hash.toString(); + for (i = 0, len = str.length; i < len; i++) { + chr = str.charCodeAt(i); + // eslint-disable-next-line no-bitwise + hash = (hash << 5) - hash + chr; + // eslint-disable-next-line no-bitwise + hash |= 0; // Convert to 32bit integer + } + return hash.toString(36); +} + /** * Generate a hash string based on number of character * @param {number} count