Skip to content

Commit

Permalink
add arrow wkb/wkt
Browse files Browse the repository at this point in the history
  • Loading branch information
@lixun910
lixun910 committed Sep 18, 2023
1 parent 672ef02 commit 2be9c74
Show file tree
Hide file tree
Showing 5 changed files with 22,386 additions and 16,089 deletions.
67 changes: 32 additions & 35 deletions src/layers/src/geojson-layer/geojson-utils.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

import {ListVector, FloatVector, BinaryVector, Utf8Vector} from 'apache-arrow';
import {ListVector, FloatVector} from 'apache-arrow';
import Console from 'global/console';
import normalize from '@mapbox/geojson-normalize';
import bbox from '@turf/bbox';
Expand Down Expand Up @@ -65,7 +65,7 @@ type RawArrowFeature = {

export function parseGeoJsonRawFeature(rawFeature: {} | Feature | RawArrowFeature): Feature | null {
if (rawFeature && typeof rawFeature === 'object') {
if ('encoding' in rawFeature && rawFeature.encoding?.startsWith('geoarrow')) {
if ('encoding' in rawFeature && rawFeature.encoding) {
// Support GeoArrow data
return parseGeometryFromArrow(rawFeature);
}
Expand Down Expand Up @@ -172,17 +172,6 @@ export function parseGeometryFromString(geoString: string): Feature | null {
}
}

// try parse as wkb using loaders.gl WKBLoader
if (!parsedGeo) {
try {
const buffer = Buffer.from(geoString, 'hex');
const binaryGeo = parseSync(buffer, WKBLoader);
parsedGeo = binaryToGeometry(binaryGeo);
} catch (e) {
return null;
}
}

if (!parsedGeo) {
return null;
}
Expand Down Expand Up @@ -247,6 +236,17 @@ export function getGeojsonFeatureTypes(allFeatures: GeojsonDataMaps): FeatureTyp
return featureTypes;
}

export enum GEOARROW_ENCODINGS {
MULTI_POLYGON = 'geoarrow.multipolygon',
POLYGON = 'geoarrow.polygon',
MULTI_LINESTRING = 'geoarrow.multilinestring',
LINESTRING = 'geoarrow.linestring',
MULTI_POINT = 'geoarrow.multipoint',
POINT = 'geoarrow.point',
WKB = 'wkb',
WKT = 'wkt'
}

/**
* convert Arrow MultiPolygon to geojson Feature
*/
Expand All @@ -268,7 +268,7 @@ function arrowMultiPolygonToFeature(arrowMultiPolygon: ListVector): MultiPolygon
multiPolygon.push(polygon);
}
const geometry: MultiPolygon = {
type: 'MultiPolygon',
type: FeatureTypes.MultiPolygon,
coordinates: multiPolygon
};
return geometry;
Expand All @@ -290,7 +290,7 @@ function arrowPolygonToFeature(arrowPolygon: ListVector): Polygon {
polygon.push(ring);
}
const geometry: Polygon = {
type: 'Polygon',
type: FeatureTypes.Polygon,
coordinates: polygon
};
return geometry;
Expand All @@ -309,7 +309,7 @@ function arrowMultiPointToFeature(arrowMultiPoint: ListVector): MultiPoint {
}
}
const geometry: MultiPoint = {
type: 'MultiPoint',
type: FeatureTypes.MultiPoint,
coordinates: multiPoint
};
return geometry;
Expand All @@ -321,7 +321,7 @@ function arrowMultiPointToFeature(arrowMultiPoint: ListVector): MultiPoint {
function arrowPointToFeature(arrowPoint: FloatVector): Point {
const point: Position = Array.from(arrowPoint.values);
const geometry: Point = {
type: 'Point',
type: FeatureTypes.Point,
coordinates: point
};
return geometry;
Expand All @@ -345,7 +345,7 @@ function arrowMultiLineStringToFeature(arrowMultiLineString: ListVector): MultiL
multiLineString.push(lineString);
}
const geometry: MultiLineString = {
type: 'MultiLineString',
type: FeatureTypes.MultiLineString,
coordinates: multiLineString
};
return geometry;
Expand All @@ -364,7 +364,7 @@ function arrowLineStringToFeature(arrowLineString: ListVector): LineString {
}
}
const geometry: LineString = {
type: 'LineString',
type: FeatureTypes.LineString,
coordinates: lineString
};
return geometry;
Expand All @@ -373,8 +373,8 @@ function arrowLineStringToFeature(arrowLineString: ListVector): LineString {
/**
* convert Arrow wkb to geojson Geometry
*/
function arrowWkbToFeature(arrowWkb: BinaryVector): Feature | null {
const binaryGeo = parseSync(arrowWkb.values, WKBLoader);
function arrowWkbToFeature(arrowWkb: Uint8Array): Feature | null {
const binaryGeo = parseSync(arrowWkb, WKBLoader);
const geometry = binaryToGeometry(binaryGeo);
const normalized = normalize(geometry);

Expand All @@ -389,8 +389,8 @@ function arrowWkbToFeature(arrowWkb: BinaryVector): Feature | null {
/**
* convert Arrow wkt to geojson Geometry
*/
function arrowWktToFeature(arrowWkt: Utf8Vector): Feature | null {
const geometry = parseSync(arrowWkt.get(0) || '', WKTLoader);
function arrowWktToFeature(arrowWkt: string): Feature | null {
const geometry = parseSync(arrowWkt, WKTLoader);
const normalized = normalize(geometry);

if (!normalized || !Array.isArray(normalized.features)) {
Expand All @@ -409,41 +409,38 @@ function arrowWktToFeature(arrowWkt: Utf8Vector): Feature | null {
* @returns
*/
export function parseGeometryFromArrow(rawData: RawArrowFeature): Feature | null {
const encoding = rawData.encoding;
const encoding = rawData.encoding?.toLowerCase();
const data = rawData.data;
if (!encoding || !data) return null;

let geometry;

switch (encoding) {
case 'geoarrow.multipolygon':
case GEOARROW_ENCODINGS.MULTI_POLYGON:
geometry = arrowMultiPolygonToFeature(data);
break;
case 'geoarrow.polygon':
case GEOARROW_ENCODINGS.POLYGON:
geometry = arrowPolygonToFeature(data);
break;
case 'geoarrow.multipoint':
case GEOARROW_ENCODINGS.MULTI_POINT:
geometry = arrowMultiPointToFeature(data);
break;
case 'geoarrow.point':
case GEOARROW_ENCODINGS.POINT:
geometry = arrowPointToFeature(data);
break;
case 'geoarrow.multilinestring':
case GEOARROW_ENCODINGS.MULTI_LINESTRING:
geometry = arrowMultiLineStringToFeature(data);
break;
case 'geoarrow.linestring':
case GEOARROW_ENCODINGS.LINESTRING:
geometry = arrowLineStringToFeature(data);
break;
case 'geoarrow.wkb': {
// convert to wkb
case GEOARROW_ENCODINGS.WKB: {
return arrowWkbToFeature(data);
}
case 'geoarrow.wkt': {
// convert to wkt
case GEOARROW_ENCODINGS.WKT: {
return arrowWktToFeature(data);
}
default: {
// encoding is not supported, skip
Console.error('GeoArrow encoding not supported');
return null;
}
Expand Down
12 changes: 3 additions & 9 deletions src/processors/src/data-processor.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,8 @@
// THE SOFTWARE.

import {Table as ApacheArrowTable, Field as ArrowField, ListVector} from 'apache-arrow';
import {csvParseRows} from 'd3-dsv';
import { csvParseRows } from 'd3-dsv';
import {Console} from 'global/console';
import {DATA_TYPES as AnalyzerDATA_TYPES} from 'type-analyzer';
import normalize from '@mapbox/geojson-normalize';
import {ALL_FIELD_TYPES, DATASET_FORMATS, GUIDES_FILE_FORMAT_DOC, ARROW_GEO_METADATA_KEY} from '@kepler.gl/constants';
Expand All @@ -37,7 +38,6 @@ import {
} from '@kepler.gl/utils';
import {KeplerGlSchema, ParsedDataset, SavedMap, LoadedMap} from '@kepler.gl/schemas';
import {Feature} from '@nebula.gl/edit-modes';
import {ProcessFileDataContent} from './file-handler';

// if any of these value occurs in csv, parse it to null;
// const CSV_NULLS = ['', 'null', 'NULL', 'Null', 'NaN', '/N'];
Expand Down Expand Up @@ -393,12 +393,6 @@ export function processKeplerglDataset(
return Array.isArray(rawData) ? results : results[0];
}

export function processArrowColumnarData(content: ProcessFileDataContent): ProcessorResult | null {
// const { progress, metadata, fileName, length, data, ...columnarData } = content;
// const table = ApacheArrowTable.new(columnarData);
return null;
}

/**
* Parse a arrow table with geometry columns and return a dataset
*
Expand All @@ -419,7 +413,7 @@ export function processArrowTable(arrowTable: ApacheArrowTable): ProcessorResult
// check if schema_version in geoMeta equals to '0.1.0'
const SCHEMA_VERSION = '0.1.0';
if (geoMeta.schema_version !== SCHEMA_VERSION) {
console.error('Schema version not supported');
Console.error('Apache Arrow schema version not supported');
return null;
}
// get all geometry columns
Expand Down
16 changes: 1 addition & 15 deletions src/types/layers.d.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -57,20 +57,6 @@ export type LayerWeightConfig = {
weightField: VisualChannelField;
};

export enum GEOARROW_ENCODINGS {
MULTI_POLYGON = 'geoarrow.multipolygon',
POLYGON = 'geoarrow.polygon',
MULTI_LINESTRING = 'geoarrow.multilinestring',
LINESTRING = 'geoarrow.linestring',
MULTI_POINT = 'geoarrow.multipoint',
POINT = 'geoarrow.point'
}

export type GeoArrowFieldMetaData = {
encoding: GEOARROW_ENCODINGS;
crs: string;
};

export type Field = {
analyzerType: string;
id?: string;
Expand All @@ -81,7 +67,7 @@ export type Field = {
fieldIdx: number;
valueAccessor(v: {index: number}): any;
filterProps?: any;
metadata?: object | GeoArrowFieldMetaData;
metadata?: object;
displayFormat?: string;
};

Expand Down
Loading

0 comments on commit 2be9c74

Please sign in to comment.