[Test] Progressive rendering for big dataset

examples/webpack.config.local.js

@@ -256,7 +256,12 @@ function addBabelSettings(env, config, exampleDir) {
       ...config.module,
       rules: [
         ...config.module.rules.filter(r => r.loader !== 'babel-loader'),
-        makeBabelRule(env, exampleDir)
+        makeBabelRule(env, exampleDir),
+        {
+          test: /\.mjs$/,
+          include: /node_modules/,
+          type: 'javascript/auto'
+        }
       ]
     }
   };

jest.setup.js

@@ -31,3 +31,14 @@ jest.mock('@kepler.gl/utils', () => ({
   hasPortableWidth: jest.fn(),
   hasMobileWidth: jest.fn()
 }));
+
+// TextEncoder / TextDecoder APIs are used by arrow, but are not provided by
+// jsdom, all node versions supported provide these via the util module
+if (
+  typeof globalThis.TextEncoder === "undefined" ||
+  typeof globalThis.TextDecoder === "undefined"
+) {
+  const utils = require("util");
+  globalThis.TextEncoder = utils.TextEncoder;
+  globalThis.TextDecoder = utils.TextDecoder;
+}

src/actions/src/action-types.ts

@@ -104,6 +104,7 @@ export const ActionTypes = {
   LOAD_FILES: `${ACTION_PREFIX}LOAD_FILES`,
   LOAD_NEXT_FILE: `${ACTION_PREFIX}LOAD_NEXT_FILE`,
   LOAD_FILE_STEP_SUCCESS: `${ACTION_PREFIX}LOAD_FILE_STEP_SUCCESS`,
+  LOAD_BATCH_DATA_SUCCESS: `${ACTION_PREFIX}LOAD_BATCH_DATA_SUCCESS`,
   LOAD_FILES_ERR: `${ACTION_PREFIX}LOAD_FILES_ERR`,
   LOAD_FILES_SUCCESS: `${ACTION_PREFIX}LOAD_FILES_SUCCESS`,
   LAYER_COLOR_UI_CHANGE: `${ACTION_PREFIX}LAYER_COLOR_UI_CHANGE`,

src/actions/src/vis-state-actions.ts

@@ -1240,6 +1240,20 @@ export function loadFileStepSuccess({
   };
 }
 
+export function loadBatchDataSuccess({
+  fileName,
+  fileCache
+}: {
+  fileName: string;
+  fileCache: FileCacheItem[];
+}): Merge<LoadFileStepSuccessAction, {type: typeof ActionTypes.LOAD_BATCH_DATA_SUCCESS}> {
+  return {
+    type: ActionTypes.LOAD_BATCH_DATA_SUCCESS,
+    fileName,
+    fileCache
+  };
+}
+
 export type LoadFilesErrUpdaterAction = {
   fileName: string;
   error: any;

src/components/package.json

@@ -67,8 +67,8 @@
     "@types/lodash.throttle": "^4.1.7",
     "@types/lodash.uniq": "^4.5.7",
     "@types/lodash.uniqby": "^4.7.7",
-    "@types/react-copy-to-clipboard": "^5.0.2",
     "@types/react": "^18.0.28",
+    "@types/react-copy-to-clipboard": "^5.0.2",
     "@types/react-dom": "^18.0.11",
     "@types/react-lifecycles-compat": "^3.0.1",
     "@types/react-map-gl": "^6.1.3",

src/components/src/map-container.tsx

@@ -819,7 +819,6 @@ export default function MapContainerFactory(
           return null;
         }
       }
-
       return (
         <div
           {...(isInteractive

src/constants/src/default-settings.ts

@@ -497,6 +497,8 @@ const GREEN2 = '74, 165, 150';
 const RED = '237, 88, 106';
 const ORANGE = '231, 110, 58';
 
+export const ARROW_GEO_METADATA_KEY = 'geo';
+
 export const FIELD_TYPE_DISPLAY = {
   [ALL_FIELD_TYPES.boolean]: {
     label: 'bool',

src/constants/src/layers.ts

@@ -502,6 +502,7 @@ export const LAYER_TYPES = keyMirror({
   grid: null,
   hexagon: null,
   geojson: null,
+  geoarrow: null,
   cluster: null,
   icon: null,
   heatmap: null,

src/deckgl-layers/src/geojson-layer/filter-arrow-layer.ts

@@ -0,0 +1,54 @@
+import {Layer, LayerExtension} from '@deck.gl/core';
+import {LayerContext} from '@deck.gl/core/lib/layer';
+import GL from '@luma.gl/constants';
+
+import shaderModule from './filter-shader-module';
+
+const VALUE_FILTERED = 1;
+
+const defaultProps = {
+  getFiltered: {type: 'accessor', value: VALUE_FILTERED}
+};
+
+export type FilterArrowExtensionProps = {
+  getFiltered?: () => number;
+};
+
+// TODO: this should be in deck.gl extensions
+// A simple extension to filter arrow layer based on the result of CPU filteredIndex,
+// so we can avoid filtering on the raw Arrow table and recreating geometry attributes.
+// Specifically, an attribute `filtered` is added to the layer to indicate whether the feature has been Filtered
+// the shader module is modified to discard the feature if filtered value is 0
+// the accessor getFiltered is used to get the value of `filtered` based on eht value `filteredIndex` in Arrowlayer
+export default class FilterArrowExtension extends LayerExtension {
+  static defaultProps = defaultProps;
+  static extensionName = 'FilterArrowExtension';
+
+  getShaders(extension: any) {
+    return {
+      modules: [shaderModule],
+      defines: {}
+    };
+  }
+
+  initializeState(this: Layer<FilterArrowExtensionProps>, context: LayerContext, extension: this) {
+    const attributeManager = this.getAttributeManager();
+    if (attributeManager) {
+      attributeManager.add({
+        filtered: {
+          size: 1,
+          type: GL.FLOAT,
+          accessor: 'getFiltered',
+          shaderAttributes: {
+            filtered: {
+              divisor: 0
+            },
+            instanceFiltered: {
+              divisor: 1
+            }
+          }
+        }
+      });
+    }
+  }
+}

src/deckgl-layers/src/geojson-layer/filter-shader-module.ts

@@ -0,0 +1,39 @@
+import {project} from '@deck.gl/core';
+
+const vs = `
+  #ifdef NON_INSTANCED_MODEL
+    #define FILTER_ARROW_ATTRIB filtered
+  #else
+    #define FILTER_ARROW_ATTRIB instanceFiltered
+  #endif
+  attribute float FILTER_ARROW_ATTRIB;
+`;
+
+const fs = ``;
+
+const inject = {
+  'vs:#decl': `
+    varying float is_filtered;
+  `,
+  'vs:#main-end': `
+    is_filtered = FILTER_ARROW_ATTRIB;
+  `,
+  'fs:#decl': `
+    varying float is_filtered;
+  `,
+  'fs:DECKGL_FILTER_COLOR': `
+    // abandon the fragments if  it is not filtered
+    if (is_filtered == 0.) {
+      discard;
+    }
+  `
+};
+
+export default {
+  name: 'filter-arrow',
+  dependencies: [project],
+  vs: vs,
+  fs: fs,
+  inject: inject,
+  getUniforms: () => {}
+};

src/layers/src/base-layer.ts

@@ -1104,7 +1104,12 @@ class Layer {
     const dataUpdateTriggers = this.getDataUpdateTriggers(layerDataset);
     const triggerChanged = this.getChangedTriggers(dataUpdateTriggers);
 
-    if (triggerChanged && triggerChanged.getMeta) {
+    // NOTES:
+    // 1) add checker `!oldLayerData`: oldLayerData is undefined means this is the first time layer is rendered
+    // the updateLayerMeta has already been called in setInitialLayerConfig
+    // 2) add checker `triggerChanged.getData`: when loading data batches for increamental rendering,
+    // triggerChanged.getData will be true for updateLayerMeta
+    if (triggerChanged && (triggerChanged.getMeta || triggerChanged.getData) && Boolean(oldLayerData)) {
       this.updateLayerMeta(dataContainer, getPosition);
     }
 

src/layers/src/geojson-layer/geojson-layer.ts

@@ -63,7 +63,7 @@ import {KeplerTable} from '@kepler.gl/table';
 import {DataContainerInterface, ArrowDataContainer} from '@kepler.gl/utils';
 import {FilterArrowExtension} from '@kepler.gl/deckgl-layers';
 
-const SUPPORTED_ANALYZER_TYPES = {
+export const SUPPORTED_ANALYZER_TYPES = {
   [DATA_TYPES.GEOMETRY]: true,
   [DATA_TYPES.GEOMETRY_FROM_STRING]: true,
   [DATA_TYPES.PAIR_GEOMETRY_FROM_STRING]: true
@@ -157,7 +157,7 @@ export type GeoJsonLayerVisConfig = {
   wireframe: boolean;
 };
 
-type GeoJsonLayerVisualChannelConfig = LayerColorConfig &
+export type GeoJsonLayerVisualChannelConfig = LayerColorConfig &
   LayerStrokeColorConfig &
   LayerSizeConfig &
   LayerHeightConfig &
@@ -222,11 +222,11 @@ export default class GeoJsonLayer extends Layer {
   get type() {
     return GeoJsonLayer.type;
   }
-  static get type(): 'geojson' {
+  static get type() {
     return 'geojson';
   }
 
-  get name(): 'Polygon' {
+  get name() {
     return 'Polygon';
   }
 
@@ -419,7 +419,7 @@ export default class GeoJsonLayer extends Layer {
     const getGeoColumn = geoColumnAccessor(this.config.columns);
     const getGeoField = geoFieldAccessor(this.config.columns);
 
-    if (this.dataToFeature.length === 0) {
+    // if (this.dataToFeature.length === 0) {
       const updateLayerMetaFunc =
         dataContainer instanceof ArrowDataContainer
           ? getGeojsonLayerMetaFromArrow
@@ -431,9 +431,13 @@ export default class GeoJsonLayer extends Layer {
         getGeoField
       });
 
-      this.dataToFeature = dataToFeature;
+      // append new data from binaryGeometries to this.binaryFeatures
+      for (let i = this.dataToFeature.length; i < dataToFeature.length; ++i) {
+        this.dataToFeature.push(dataToFeature[i]);
+      }
+      // this.dataToFeature = dataToFeature;
       this.updateMeta({bounds, fixedRadius, featureTypes});
-    }
+    // }
   }
 
   setInitialLayerConfig({dataContainer}) {
@@ -497,7 +501,7 @@ export default class GeoJsonLayer extends Layer {
     const updateTriggers = {
       ...this.getVisualChannelUpdateTriggers(),
       getFilterValue: gpuFilter.filterValueUpdateTriggers,
-      getFiltered: this.filteredIndexTrigger
+      // getFiltered: this.filteredIndexTrigger
     };
 
     const defaultLayerProps = this.getDefaultDeckLayerProps(opts);
@@ -531,7 +535,7 @@ export default class GeoJsonLayer extends Layer {
         capRounded: true,
         jointRounded: true,
         updateTriggers,
-        extensions: [...defaultLayerProps.extensions, new FilterArrowExtension()],
+        // extensions: [...defaultLayerProps.extensions, new FilterArrowExtension()],
         _subLayerProps: {
           ...(featureTypes?.polygon ? {'polygons-stroke': opaOverwrite} : {}),
           ...(featureTypes?.line ? {linestrings: opaOverwrite} : {}),

src/layers/src/geojson-layer/geojson-utils.ts

@@ -43,7 +43,6 @@ export enum FeatureTypes {
 }
 
 /* eslint-enable */
-
 export function parseGeoJsonRawFeature(rawFeature: unknown): Feature | null {
   if (typeof rawFeature === 'object') {
     // Support GeoJson feature as object

src/localization/src/translations/en.ts

@@ -120,6 +120,7 @@ export default {
       hexbin: 'hexbin',
       polygon: 'polygon',
       geojson: 'geojson',
+      geoarrow: 'geoarrow',
       cluster: 'cluster',
       icon: 'icon',
       heatmap: 'heatmap',

src/processors/src/file-handler.ts

@@ -29,7 +29,7 @@ import {
   processKeplerglJSON,
   processRowObject
 } from './data-processor';
-import {generateHashId, isPlainObject} from '@kepler.gl/utils';
+import {generateHashId, generateHashIdFromString, isPlainObject} from '@kepler.gl/utils';
 import {DATASET_FORMATS} from '@kepler.gl/constants';
 import {Loader} from '@loaders.gl/loader-utils';
 import {FileCacheItem, ValidKeplerGlMap} from './types';
@@ -61,6 +61,18 @@ const JSON_LOADER_OPTIONS = {
   ]
 };
 
+/**
+ * ProcessFileDataContent
+ * @typedef {Object} ProcessFileDataContent
+ * @property {Object} data - parsed data
+ * @property {string} fileName - file name
+ * @property {number} [length] - number of rows
+ * @property {Object} [progress] - progress of file processing
+ * @property {number} [progress.rowCount] - number of rows processed
+ * @property {number} [progress.rowCountInBatch] - number of rows in current batch
+ * @property {number} [progress.percent] - percent of file processed
+ * @property {Map<string, string>} [metadata] - metadata e.g. for arrow data, metadata could be the schema.fields
+ */
 export type ProcessFileDataContent = {
   data: unknown;
   fileName: string;
@@ -215,10 +227,13 @@ export function processFileData({
   fileCache: FileCacheItem[];
 }): Promise<FileCacheItem[]> {
   return new Promise((resolve, reject) => {
-    let {data} = content;
+    let {fileName, data} = content;
     let format: string | undefined;
     let processor: Function | undefined;
 
+    // generate unique id with length of 4 using fileName string
+    const id = generateHashIdFromString(fileName);
+
     if (isArrowData(data)) {
       format = DATASET_FORMATS.arrow;
       processor = processArrowTable;
@@ -241,6 +256,7 @@ export function processFileData({
         {
           data: result,
           info: {
+            id,
             label: content.fileName,
             format
           }