init

src/actions/src/action-types.ts

@@ -104,6 +104,7 @@ export const ActionTypes = {
   LOAD_FILES: `${ACTION_PREFIX}LOAD_FILES`,
   LOAD_NEXT_FILE: `${ACTION_PREFIX}LOAD_NEXT_FILE`,
   LOAD_FILE_STEP_SUCCESS: `${ACTION_PREFIX}LOAD_FILE_STEP_SUCCESS`,
+  LOAD_BATCH_DATA_SUCCESS: `${ACTION_PREFIX}LOAD_BATCH_DATA_SUCCESS`,
   LOAD_FILES_ERR: `${ACTION_PREFIX}LOAD_FILES_ERR`,
   LOAD_FILES_SUCCESS: `${ACTION_PREFIX}LOAD_FILES_SUCCESS`,
   LAYER_COLOR_UI_CHANGE: `${ACTION_PREFIX}LAYER_COLOR_UI_CHANGE`,

src/actions/src/vis-state-actions.ts

@@ -1241,6 +1241,20 @@ export function loadFileStepSuccess({
   };
 }
 
+export function loadBatchDataSuccess({
+  fileName,
+  fileCache
+}: {
+  fileName: string;
+  fileCache: FileCacheItem[];
+}): Merge<LoadFileStepSuccessAction, {type: typeof ActionTypes.LOAD_BATCH_DATA_SUCCESS}> {
+  return {
+    type: ActionTypes.LOAD_BATCH_DATA_SUCCESS,
+    fileName,
+    fileCache
+  };
+}
+
 export type LoadFilesErrUpdaterAction = {
   fileName: string;
   error: any;

src/components/src/map-container.tsx

@@ -819,7 +819,7 @@ export default function MapContainerFactory(
           return null;
         }
       }
-
+      console.log(allDeckGlProps);
       return (
         <div
           {...(isInteractive

src/layers/src/arrow-layer/arrow-layer.ts

@@ -58,7 +58,7 @@ function updateBoundsFromGeoArrowSamples(
 }
 
 export default class ArrowLayer extends GeoJsonLayer {
-  binaryFeatures: BinaryFeatures[];
+  binaryFeatures: BinaryFeatures[]; // TODO: reuse dataToFeature?
   dataContainer: DataContainerInterface | null;
   filteredIndex: Uint8ClampedArray | null;
   filteredIndexTrigger: number[];
@@ -124,7 +124,7 @@ export default class ArrowLayer extends GeoJsonLayer {
       this.filteredIndex[filteredIndex[i]] = 1;
     }
 
-    this.filteredIndexTrigger = filteredIndex;
+    // this.filteredIndexTrigger = filteredIndex;
     this.dataContainer = dataContainer;
     return this.binaryFeatures;
   }
@@ -183,7 +183,11 @@ export default class ArrowLayer extends GeoJsonLayer {
       geoColumn,
       encoding
     );
-    this.binaryFeatures = binaryGeometries;
+
+    // append new data from binaryGeometries to this.binaryFeatures
+    for (let i = this.binaryFeatures.length; i < binaryGeometries.length; ++i) {
+      this.binaryFeatures.push(binaryGeometries[i]);
+    }
 
     // TODO: this should be removed once fix was applied in loaders.gl
     let bounds : [number, number, number, number] = [Infinity, Infinity, -Infinity, -Infinity]
@@ -261,7 +265,7 @@ export default class ArrowLayer extends GeoJsonLayer {
   }
 
   renderLayer(opts) {
-    const {data, gpuFilter, objectHovered, mapState, interactionConfig} = opts;
+    const {data: dataProps, gpuFilter, objectHovered, mapState, interactionConfig} = opts;
 
     const {fixedRadius, featureTypes} = this.meta;
     const radiusScale = this.getRadiusScaleByZoom(mapState, fixedRadius);
@@ -290,12 +294,13 @@ export default class ArrowLayer extends GeoJsonLayer {
 
     const pickable = interactionConfig.tooltip.enabled;
     const hoveredObject = this.hasHoveredObject(objectHovered);
+    const {data, ...props} = dataProps;
 
-    const deckLayers = data.data.map((d, i) => {
+    const deckLayers = data.map((d, i) => {
       return new DeckGLGeoJsonLayer({
         ...defaultLayerProps,
         ...layerProps,
-        ...data,
+        ...props,
         data: d,
         id: `${this.id}-${i}`,
         pickable,
@@ -310,7 +315,7 @@ export default class ArrowLayer extends GeoJsonLayer {
         capRounded: true,
         jointRounded: true,
         updateTriggers,
-        extensions: [...defaultLayerProps.extensions, new FilterArrowExtension()],
+        // extensions: [...defaultLayerProps.extensions, new FilterArrowExtension()],
         _subLayerProps: {
           ...(featureTypes?.polygon ? {'polygons-stroke': opaOverwrite} : {}),
           ...(featureTypes?.line ? {linestrings: opaOverwrite} : {}),
@@ -335,9 +340,9 @@ export default class ArrowLayer extends GeoJsonLayer {
               visible: defaultLayerProps.visible,
               wrapLongitude: false,
               data: [hoveredObject],
-              getLineWidth: data.getLineWidth,
-              getPointRadius: data.getPointRadius,
-              getElevation: data.getElevation,
+              getLineWidth: dataProps.getLineWidth,
+              getPointRadius: dataProps.getPointRadius,
+              getElevation: dataProps.getElevation,
               getLineColor: this.config.highlightColor,
               getFillColor: this.config.highlightColor,
               // always draw outline

src/layers/src/base-layer.ts

@@ -1104,7 +1104,12 @@ class Layer {
     const dataUpdateTriggers = this.getDataUpdateTriggers(layerDataset);
     const triggerChanged = this.getChangedTriggers(dataUpdateTriggers);
 
-    if (triggerChanged && triggerChanged.getMeta) {
+    // NOTES:
+    // 1) add checker `!oldLayerData`: oldLayerData is undefined means this is the first time layer is rendered
+    // the updateLayerMeta has already been called in setInitialLayerConfig
+    // 2) add checker `triggerChanged.getData`: when loading data batches for increamental rendering,
+    // triggerChanged.getData will be true for updateLayerMeta
+    if (triggerChanged && (triggerChanged.getMeta || triggerChanged.getData) && Boolean(oldLayerData)) {
       this.updateLayerMeta(dataContainer, getPosition);
     }
 

src/processors/src/file-handler.ts

@@ -29,7 +29,7 @@ import {
   processKeplerglJSON,
   processRowObject
 } from './data-processor';
-import {generateHashId, isPlainObject} from '@kepler.gl/utils';
+import {generateHashId, generateHashIdFromString, isPlainObject} from '@kepler.gl/utils';
 import {DATASET_FORMATS} from '@kepler.gl/constants';
 import {Loader} from '@loaders.gl/loader-utils';
 import {FileCacheItem, ValidKeplerGlMap} from './types';
@@ -225,10 +225,13 @@ export function processFileData({
   fileCache: FileCacheItem[];
 }): Promise<FileCacheItem[]> {
   return new Promise((resolve, reject) => {
-    let {data} = content;
+    let {fileName, data} = content;
     let format: string | undefined;
     let processor: Function | undefined;
 
+    // generate unique id with length of 4 using fileName string
+    const id = generateHashIdFromString(fileName);
+
     if (isArrowData(data)) {
       format = DATASET_FORMATS.arrow;
       processor = processArrowTable;
@@ -251,6 +254,7 @@ export function processFileData({
         {
           data: result,
           info: {
+            id,
             label: content.fileName,
             format
           }

src/reducers/src/vis-state-updaters.ts

@@ -29,7 +29,12 @@ import isEqual from 'lodash.isequal';
 import copy from 'copy-to-clipboard';
 import deepmerge from 'deepmerge';
 // Tasks
-import {LOAD_FILE_TASK, UNWRAP_TASK, PROCESS_FILE_DATA, DELAY_TASK} from '@kepler.gl/tasks';
+import {
+  LOAD_FILE_TASK,
+  UNWRAP_TASK,
+  PROCESS_FILE_DATA,
+  DELAY_TASK
+} from '@kepler.gl/tasks';
 // Actions
 import {
   applyLayerConfig,
@@ -40,6 +45,7 @@ import {
   loadFilesErr,
   loadFilesSuccess,
   loadFileStepSuccess,
+  loadBatchDataSuccess,
   loadNextFile,
   nextFileBatch,
   ReceiveMapConfigPayload,
@@ -2167,7 +2173,23 @@ export function loadFileStepSuccessUpdater(
 
   return withTask(
     stateWithCache,
-    DELAY_TASK(200).map(filesToLoad.length ? loadNextFile : () => onFinish(fileCache))
+    DELAY_TASK(0).map(filesToLoad.length ? loadNextFile : () => onFinish(fileCache))
+  );
+}
+
+export function loadBatchDataSuccessUpdater(
+  state: VisState,
+  action: VisStateActions.LoadFileStepSuccessAction
+): VisState {
+  if (!state.fileLoading) {
+    return state;
+  }
+  const {fileCache} = action;
+  const {onFinish} = state.fileLoading;
+
+  return withTask(
+    state,
+    DELAY_TASK(0).map(() => onFinish(fileCache))
   );
 }
 
@@ -2235,7 +2257,6 @@ export function processFileContentUpdater(
   action: VisStateActions.ProcessFileContentUpdaterAction
 ): VisState {
   const {content, fileCache} = action.payload;
-
   const stateWithProgress = updateFileLoadingProgressUpdater(state, {
     fileName: content.fileName,
     progress: {percent: 1, message: 'processing...'}
@@ -2273,12 +2294,36 @@ export const nextFileBatchUpdater = (
     payload: {gen, fileName, progress, accumulated, onFinish}
   }: VisStateActions.NextFileBatchUpdaterAction
 ): VisState => {
-  const stateWithProgress = updateFileLoadingProgressUpdater(state, {
+  let stateWithProgress = updateFileLoadingProgressUpdater(state, {
     fileName,
     progress: parseProgress(state.fileLoadingProgress[fileName], progress)
   });
-  return withTask(
+  console.log(
+    'nextFileBatchUpdater',
     stateWithProgress,
+    gen,
+    fileName,
+    progress,
+    accumulated,
+    onFinish
+  );
+  if (accumulated && accumulated.data?.length > 0) {
+    console.log(accumulated);
+    const payload = {
+      content: accumulated,
+      fileCache: []
+    };
+    stateWithProgress = processFileContentUpdater(stateWithProgress, {payload});
+  }
+  return withTask(stateWithProgress, [
+    ...(accumulated && accumulated.data?.length > 0
+      ? [
+          PROCESS_FILE_DATA({content: accumulated, fileCache: []}).bimap(
+            result => loadBatchDataSuccess({fileName, fileCache: result}),
+            err => loadFilesErr(fileName, err)
+          )
+        ]
+      : []),
     UNWRAP_TASK(gen.next()).bimap(
       ({value, done}) => {
         return done
@@ -2293,7 +2338,7 @@ export const nextFileBatchUpdater = (
       },
       err => loadFilesErr(fileName, err)
     )
-  );
+  ]);
 };
 
 /**
@@ -2320,7 +2365,7 @@ export const loadFilesErrUpdater = (
   // kick off next file or finish
   return withTask(
     nextState,
-    DELAY_TASK(200).map(filesToLoad.length ? loadNextFile : () => onFinish(fileCache))
+    DELAY_TASK(0).map(filesToLoad.length ? loadNextFile : () => onFinish(fileCache))
   );
 };
 

src/table/src/dataset-utils.ts

@@ -68,6 +68,17 @@ export function createNewDataEntry(
     return {};
   }
 
+  // check if dataset already exists
+  if (info && info.id && datasets[info.id]) {
+    // get keplerTable from datasets
+    const keplerTable = datasets[info.id];
+    // update the data in keplerTable
+    keplerTable.update(validatedData);
+    return {
+      [keplerTable.id]: keplerTable
+    };
+  }
+
   info = info || {};
   const color = info.color || getNewDatasetColor(datasets);
 

src/table/src/kepler-table.ts

@@ -202,6 +202,19 @@ class KeplerTable {
     this.disableDataOperation = disableDataOperation;
   }
 
+  /**
+   * update table with new data
+   * @param data - new data e.g. the arrow data with new batches loaded
+   */
+  update(data: ProtoDataset['data']) {
+    const dataContainerData = data.cols ? data.cols : data.rows;
+
+    this.dataContainer.update?.(dataContainerData);
+    this.allIndexes = this.dataContainer.getPlainIndex();
+    this.filteredIndex = this.allIndexes;
+    this.filteredIndexForDomain = this.allIndexes;
+  }
+
   get length() {
     return this.dataContainer.numRows();
   }

src/utils/src/arrow-data-container.ts

@@ -58,7 +58,7 @@ export class ArrowDataContainer implements DataContainerInterface {
   _numColumns: number;
   _numRows: number;
   _fields: Field[];
-  _colData: any[][];
+  // _colData: any[][];
 
   constructor(data: ArrowDataContainerInput) {
     if (!data.cols) {
@@ -74,7 +74,15 @@ export class ArrowDataContainer implements DataContainerInterface {
     this._numRows = data.cols[0].length;
     this._fields = data.fields || [];
     // cache column data to make valueAt() faster
-    this._colData = data.cols.map(c => c.toArray());
+    // this._colData = data.cols.map(c => c.toArray());
+  }
+
+  update(updateData: ArrowColumn<any>[]) {
+    this._cols = updateData;
+    this._numColumns = this._cols.length;
+    this._numRows = this._cols[0].length;
+    // cache column data to make valueAt() faster
+    // this._colData = this._cols.map(c => c.toArray());
   }
 
   numRows(): number {
@@ -86,7 +94,9 @@ export class ArrowDataContainer implements DataContainerInterface {
   }
 
   valueAt(rowIndex: number, columnIndex: number): any {
-    return this._colData[columnIndex][rowIndex];
+    // use cahced column data to make valueAt() faster
+    // return this._colData[columnIndex][rowIndex];
+    return this._cols[columnIndex].get(rowIndex);
   }
 
   row(rowIndex: number, sharedRow?: SharedRowOptions): DataRow {
@@ -100,7 +110,9 @@ export class ArrowDataContainer implements DataContainerInterface {
   }
 
   rowAsArray(rowIndex: number): any[] {
-    return this._colData.map(col => col[rowIndex]);
+    // use cached column data
+    // return this._colData.map(col => col[rowIndex]);
+    return this._cols.map(col => col.get(rowIndex));
   }
 
   rows(sharedRow: SharedRowOptions) {

src/utils/src/data-container-interface.ts

@@ -60,14 +60,19 @@ export interface DataContainerInterface {
    */
   column(columnIndex: number): Generator<any, void, any>;
 
+  /**
+   * Updates the data container with new data.
+   * @param updateData updated data, e.g. for arrow data container, it's an array of arrow columns; for row data container, it's an array of rows.
+   */
+  update?(updateData: any[]): void;
+
   /**
    * Returns the column object at the specified index.
    * @param columnIndex Column index.
    * @returns The column object at the specified index.
    */
   getColumn?(columnIndex: number): any;
 
-
   /**
    * Returns the field object at the specified index.
    * @param columnIndex Column index.