Skip to content

Commit

Permalink
test with loaders.gl 4.1.0-alpha2
Browse files Browse the repository at this point in the history 
Signed-off-by: Xun Li <[email protected]>
  • Loading branch information
@lixun910
lixun910 committed Dec 6, 2023
1 parent acaf968 commit f91824d
Show file tree
Hide file tree
Showing 13 changed files with 218 additions and 198 deletions.
20 changes: 20 additions & 0 deletions examples/demo-app/src/cloud-providers/foursquare/foursquare-icon.js
View file
Original file line number Diff line number Diff line change
@@ -1,3 +1,23 @@
// Copyright (c) 2023 Uber Technologies, Inc.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

import React, {Component} from 'react';
import {Icons} from '@kepler.gl/components';
import PropTypes from 'prop-types';
Expand Down
20 changes: 20 additions & 0 deletions examples/demo-app/src/cloud-providers/foursquare/foursquare-provider.js
View file
Original file line number Diff line number Diff line change
@@ -1,3 +1,23 @@
// Copyright (c) 2023 Uber Technologies, Inc.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

import FSQIcon from './foursquare-icon';
import {Provider, KEPLER_FORMAT} from '@kepler.gl/cloud-providers';
import {Auth0Client} from '@auth0/auth0-spa-js';
Expand Down
16 changes: 8 additions & 8 deletions package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -114,7 +114,7 @@
"@hubble.gl/core": "1.2.0-alpha.6",
"@hubble.gl/react": "1.2.0-alpha.6",
"@kepler.gl/components": "3.0.0-alpha.1",
"@loaders.gl/polyfills": "^4.0.3",
"@loaders.gl/polyfills": "^4.1.0-alpha.2",
"@types/mapbox__geo-viewport": "^0.4.1",
"@typescript-eslint/parser": "^5.27.0",
"eslint-config-developit": "^1.2.0",
Expand Down Expand Up @@ -142,7 +142,7 @@
"@babel/traverse": "^7.12.1",
"@cfaester/enzyme-adapter-react-18": "^0.7.0",
"@deck.gl/test-utils": "^8.9.27",
"@loaders.gl/polyfills": "^4.0.3",
"@loaders.gl/polyfills": "^4.1.0-alpha.2",
"@luma.gl/test-utils": "^8.5.20",
"@nebula.gl/layers": "1.0.2-alpha.1",
"@probe.gl/env": "^3.5.0",
Expand Down Expand Up @@ -216,12 +216,12 @@
"webpack-stats-plugin": "^0.2.1"
},
"resolutions": {
"@loaders.gl/core": "^4.0.3",
"@loaders.gl/csv": "^4.0.3",
"@loaders.gl/gltf": "^4.0.3",
"@loaders.gl/json": "^4.0.3",
"@loaders.gl/loader-utils": "^4.0.3",
"@loaders.gl/polyfills": "^4.0.3",
"@loaders.gl/core": "^4.1.0-alpha.2",
"@loaders.gl/csv": "^4.1.0-alpha.2",
"@loaders.gl/gltf": "^4.1.0-alpha.2",
"@loaders.gl/json": "^4.1.0-alpha.2",
"@loaders.gl/loader-utils": "^4.1.0-alpha.2",
"@loaders.gl/polyfills": "^4.1.0-alpha.2",
"@luma.gl/constants": "8.5.20",
"@luma.gl/core": "8.5.20",
"@luma.gl/experimental": "8.5.20",
Expand Down
10 changes: 5 additions & 5 deletions src/layers/package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,11 +42,11 @@
"@kepler.gl/table": "3.0.0-alpha.1",
"@kepler.gl/types": "3.0.0-alpha.1",
"@kepler.gl/utils": "3.0.0-alpha.1",
"@loaders.gl/arrow": "^4.0.3",
"@loaders.gl/core": "^4.0.3",
"@loaders.gl/gis": "^4.0.3",
"@loaders.gl/gltf": "^4.0.3",
"@loaders.gl/wkt": "^4.0.3",
"@loaders.gl/arrow": "^4.1.0-alpha.2",
"@loaders.gl/core": "^4.1.0-alpha.2",
"@loaders.gl/gis": "^4.1.0-alpha.2",
"@loaders.gl/gltf": "^4.1.0-alpha.2",
"@loaders.gl/wkt": "^4.1.0-alpha.2",
"@luma.gl/constants": "^8.5.20",
"@mapbox/geojson-normalize": "0.0.1",
"@nebula.gl/edit-modes": "1.0.2-alpha.1",
Expand Down
32 changes: 10 additions & 22 deletions src/layers/src/geojson-layer/geojson-layer.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -411,19 +411,6 @@ export default class GeoJsonLayer extends Layer {
};
}

isInPolygon(data: DataContainerInterface, index: number, polygon: Feature<Polygon>): Boolean {
if (this.centroids.length === 0 || !this.centroids[index]) {
return false;
}
const isReactangle = polygon.properties?.shape === 'Rectangle';
const point = this.centroids[index];
// without spatialIndex, use turf.js
if (isReactangle && polygon.properties?.bbox) {
const [minX, minY, maxX, maxY] = polygon.properties?.bbox;
return point[0] >= minX && point[0] <= maxX && point[1] >= minY && point[1] <= maxY;
}
return booleanWithin(turfPoint(point), polygon);
}
updateLayerMeta(dataContainer) {
// check datasource is arrow format if dataContainer is arrow data container
this.dataContainer = dataContainer;
Expand All @@ -435,18 +422,21 @@ export default class GeoJsonLayer extends Layer {
if (dataContainer instanceof ArrowDataContainer) {
const {geojson} = this.config.columns;
if (this.dataToFeature.length < dataContainer.numChunks()) {
const {binaryGeometries, bounds, featureTypes, meanCenters} =
const {binaryGeometries, bounds, featureTypes} =
this.dataToFeature.length === 0
? getGeojsonLayerMetaFromArrow({dataContainer, getGeoColumn, getGeoField, chunkIndex: 0})
? getGeojsonLayerMetaFromArrow({
dataContainer,
getGeoColumn,
getGeoField,
chunkIndex: 0
})
: dataContainer.getBinaryData(geojson.fieldIdx);

if (meanCenters) this.centroids = meanCenters;
if (this.dataToFeature.length === 0) {
dataContainer.updateBinaryData(geojson.fieldIdx, 0, {
binaryGeometries,
bounds,
featureTypes,
meanCenters
featureTypes
});
// not update bounds for every batch, to avoid interrupt user interacts with map while loading the map incrementally
// since there is no feature.properties.radius, we set fixedRadius to false
Expand All @@ -457,14 +447,12 @@ export default class GeoJsonLayer extends Layer {
}
} else {
if (this.dataToFeature.length === 0) {
const {dataToFeature, bounds, fixedRadius, featureTypes, centroids} = getGeojsonLayerMeta({
const {dataToFeature, bounds, fixedRadius, featureTypes} = getGeojsonLayerMeta({
dataContainer,
getFeature
});

if (centroids) this.centroids = centroids;
this.dataToFeature = dataToFeature;
this.updateMeta({ bounds, fixedRadius, featureTypes });
this.updateMeta({bounds, fixedRadius, featureTypes});
}
}
}
Expand Down
20 changes: 9 additions & 11 deletions src/layers/src/layer-utils.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -62,29 +62,29 @@ export function getGeojsonLayerMetaFromArrow({
}: {
dataContainer: DataContainerInterface;
getGeoColumn: (dataContainer: DataContainerInterface) => unknown;
getGeoField: (dataContainer: DataContainerInterface) => Field | null;
getGeoField: (dataContainer: DataContainerInterface) => Field | null;
chunkIndex?: number;
}): BinaryDataFromGeoArrow & {meanCenters?: number[][]} {
}): BinaryDataFromGeoArrow & {meanCenters?: number[][]} {
const geoColumn = getGeoColumn(dataContainer) as arrow.Vector;
const arrowField = getGeoField(dataContainer);

const encoding = arrowField?.metadata?.get('ARROW:extension:name');
const options: BinaryGeometriesFromArrowOptions = {
...(chunkIndex !== undefined && chunkIndex >= 0 ? {chunkIndex} : {}),
triangulate: true,
meanCenter: true
calculateMeanCenters: false
};
// create binary data from arrow data for GeoJsonLayer
const {binaryGeometries, featureTypes, bounds} = getBinaryGeometriesFromArrow(
geoColumn,
encoding
encoding,
options
);

return {
binaryGeometries,
featureTypes,
bounds,
meanCenters
bounds
};
}

Expand Down Expand Up @@ -114,10 +114,7 @@ export function getHoveredObjectFromArrow(
const field = fieldAccessor(dataContainer);
const encoding = field?.metadata?.get('ARROW:extension:name');

const hoveredFeature = parseGeometryFromArrow({
encoding,
data: rawGeometry
});
const hoveredFeature = parseGeometryFromArrow(rawGeometry, encoding);

const properties = dataContainer.rowAsArray(objectInfo.index).reduce((prev, cur, i) => {
const fieldName = dataContainer?.getField?.(i).name;
Expand All @@ -129,7 +126,8 @@ export function getHoveredObjectFromArrow(

return hoveredFeature
? {
...hoveredFeature,
type: 'Feature',
geometry: hoveredFeature,
properties: {
...properties,
index: objectInfo.index
Expand Down
10 changes: 5 additions & 5 deletions src/processors/package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,11 +35,11 @@
"@kepler.gl/schemas": "3.0.0-alpha.1",
"@kepler.gl/types": "3.0.0-alpha.1",
"@kepler.gl/utils": "3.0.0-alpha.1",
"@loaders.gl/arrow": "^4.0.3",
"@loaders.gl/core": "^4.0.3",
"@loaders.gl/csv": "^4.0.3",
"@loaders.gl/json": "^4.0.3",
"@loaders.gl/loader-utils": "^4.0.3",
"@loaders.gl/arrow": "^4.1.0-alpha.2",
"@loaders.gl/core": "^4.1.0-alpha.2",
"@loaders.gl/csv": "^4.1.0-alpha.2",
"@loaders.gl/json": "^4.1.0-alpha.2",
"@loaders.gl/loader-utils": "^4.1.0-alpha.2",
"@mapbox/geojson-normalize": "0.0.1",
"@nebula.gl/edit-modes": "1.0.2-alpha.1",
"@turf/helpers": "^6.1.4",
Expand Down
13 changes: 5 additions & 8 deletions src/processors/src/file-handler.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ import * as arrow from 'apache-arrow';
import {parseInBatches} from '@loaders.gl/core';
import {JSONLoader, _JSONPath} from '@loaders.gl/json';
import {CSVLoader} from '@loaders.gl/csv';
import {ArrowLoader, hardClone, parseGeoArrowOnWorker} from '@loaders.gl/arrow';
import {ArrowLoader, hardClone, parseGeoArrowOnWorker, ParseGeoArrowInput} from '@loaders.gl/arrow';
import {
processArrowTable,
processGeojson,
Expand Down Expand Up @@ -319,6 +319,7 @@ export async function applyProgressiveLoadGeoArrow(

// chunkIndex is the index of last batch has been loaded in dataContainer
const chunkIndex = dataContainer.numChunks() - 1;
const chunkOffset = geoColumn?.data[0].length * chunkIndex;
const geometryChunk = geoColumn?.data[chunkIndex];
const chunkCopy = hardClone(geometryChunk, true);

Expand All @@ -336,21 +337,17 @@ export async function applyProgressiveLoadGeoArrow(
dictionary: chunkCopy.dictionary
};

const sourceData = {
const sourceData: ParseGeoArrowInput = {
operation: 'parse-geoarrow',
chunkData,
chunkIndex,
chunkOffset,
geometryEncoding: encoding,
calculateMeanCenters: true,
triangle: true
};

const parsedGeoArrowData = await parseGeoArrowOnWorker(
sourceData,
{
_workerType: 'test'
}
);
const parsedGeoArrowData = await parseGeoArrowOnWorker(sourceData);

// dataContainer.updateBinaryData(
// geojson.fieldIdx,
Expand Down
2 changes: 1 addition & 1 deletion src/reducers/package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@
"@kepler.gl/tasks": "3.0.0-alpha.1",
"@kepler.gl/types": "3.0.0-alpha.1",
"@kepler.gl/utils": "3.0.0-alpha.1",
"@loaders.gl/loader-utils": "^4.0.3",
"@loaders.gl/loader-utils": "^4.1.0-alpha.2",
"@types/lodash.clonedeep": "^4.5.7",
"@types/lodash.flattendeep": "^4.4.7",
"@types/lodash.get": "^4.4.6",
Expand Down
64 changes: 32 additions & 32 deletions src/reducers/src/vis-state-updaters.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2240,12 +2240,11 @@ export function makeLoadFileTask(file, fileCache, loaders: Loader[] = [], loadOp
);
}


export function progressiveLoadCompletedUpdater(
state: VisState,
action: VisStateActions.ProgressiveLoadCompletedUpdaterAction
): VisState {
const {parsedData, inputData} = action.payload;
const {parsedData} = action.payload;

// for each layer id in parsedBinaryDataForLayers
// find the layer in state.layers
Expand Down Expand Up @@ -2273,11 +2272,13 @@ export function progressiveLoadCompletedUpdater(
if (dataContainer instanceof ArrowDataContainer) {
const {fieldIndex, parsedGeoArrowData} = parsedData[layerId];
keplerTable.filteredIndex = dataContainer.getFilteredIndex(fieldIndex);
dataContainer.updateBinaryData(
fieldIndex,
parsedGeoArrowData.chunkIndex,
parsedGeoArrowData.binaryDataFromGeoArrow
);
if (parsedGeoArrowData.binaryDataFromGeoArrow) {
dataContainer.updateBinaryData(
fieldIndex,
parsedGeoArrowData.chunkIndex,
parsedGeoArrowData.binaryDataFromGeoArrow
);
}
}

const newLayer = layer;
Expand Down Expand Up @@ -2366,31 +2367,30 @@ export const nextFileBatchUpdater = (
progress: parseProgress(state.fileLoadingProgress[fileName], progress)
});

return withTask(
stateWithProgress, [
...(fileName.endsWith('arrow') && accumulated && accumulated.data?.length > 0
? [
PROCESS_FILE_DATA({content: accumulated, fileCache: []}).bimap(
result => loadBatchDataSuccess({fileName, fileCache: result}),
err => loadFilesErr(fileName, err)
)
]
: []),
UNWRAP_TASK(gen.next()).bimap(
({value, done}) => {
return done
? onFinish(accumulated)
: nextFileBatch({
gen,
fileName,
progress: value.progress,
accumulated: value,
onFinish
});
},
err => loadFilesErr(fileName, err)
)
]);
return withTask(stateWithProgress, [
...(fileName.endsWith('arrow') && accumulated && accumulated.data?.length > 0
? [
PROCESS_FILE_DATA({content: accumulated, fileCache: []}).bimap(
result => loadBatchDataSuccess({fileName, fileCache: result}),
err => loadFilesErr(fileName, err)
)
]
: []),
UNWRAP_TASK(gen.next()).bimap(
({value, done}) => {
return done
? onFinish(accumulated)
: nextFileBatch({
gen,
fileName,
progress: value.progress,
accumulated: value,
onFinish
});
},
err => loadFilesErr(fileName, err)
)
]);
};

/**
Expand Down
2 changes: 1 addition & 1 deletion src/schemas/package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@
"@kepler.gl/table": "3.0.0-alpha.1",
"@kepler.gl/types": "3.0.0-alpha.1",
"@kepler.gl/utils": "3.0.0-alpha.1",
"@loaders.gl/loader-utils": "^4.0.3",
"@loaders.gl/loader-utils": "^4.1.0-alpha.2",
"@types/keymirror": "^0.1.1",
"@types/lodash.clonedeep": "^4.5.7",
"@types/lodash.pick": "^4.4.6",
Expand Down
Loading

0 comments on commit f91824d

Please sign in to comment.