[Feat] Support WKB geometry column in CSV

src/components/src/side-panel/add-by-dataset-button.tsx

@@ -25,8 +25,7 @@ import {Datasets} from '@kepler.gl/table';
 
 import Tippy from '@tippyjs/react';
 import {Add} from '../common/icons';
-import {Button} from '../common/styled-components';
-import {DatasetSquare} from '../..';
+import {Button, DatasetSquare} from '../common/styled-components';
 import Typeahead from '../common/item-selector/typeahead';
 import Accessor from '../common/item-selector/accessor';
 import {useIntl} from 'react-intl';

src/layers/package.json

@@ -43,7 +43,9 @@
     "@kepler.gl/types": "3.0.0-alpha.0",
     "@kepler.gl/utils": "3.0.0-alpha.0",
     "@loaders.gl/core": "^3.0.9",
+    "@loaders.gl/gis": "^3.0.9",
     "@loaders.gl/gltf": "^3.0.9",
+    "@loaders.gl/wkt": "^3.0.9",
     "@luma.gl/constants": "^8.5.10",
     "@mapbox/geojson-normalize": "0.0.1",
     "@nebula.gl/layers": "1.0.2-alpha.1",

src/layers/src/geojson-layer/geojson-utils.ts

@@ -21,6 +21,9 @@
 import wktParser from 'wellknown';
 import normalize from '@mapbox/geojson-normalize';
 import bbox from '@turf/bbox';
+import {parseSync} from '@loaders.gl/core';
+import {WKBLoader} from '@loaders.gl/wkt';
+import {binaryToGeometry} from '@loaders.gl/gis';
 
 import {Feature, BBox} from 'geojson';
 import {getSampleData} from '@kepler.gl/utils';
@@ -138,6 +141,17 @@ export function parseGeometryFromString(geoString: string): Feature | null {
     }
   }
 
+  // try parse as wkb using loaders.gl WKBLoader
+  if (!parsedGeo) {
+    try {
+      const buffer = Buffer.from(geoString, 'hex');
+      const binaryGeo = parseSync(buffer, WKBLoader);
+      parsedGeo = binaryToGeometry(binaryGeo);
+    } catch (e) {
+      return null;
+    }
+  }
+
   if (!parsedGeo) {
     return null;
   }

src/processors/src/data-processor.ts

@@ -105,7 +105,7 @@ export const PARSE_FIELD_VALUE_FROM_STRING = {
  *  options: {centerMap: true, readOnly: true}
  * }));
  */
-export function processCsvData(rawData: unknown[][], header?: string[]): ProcessorResult {
+export function processCsvData(rawData: unknown[][] | string, header?: string[]): ProcessorResult {
   let rows: unknown[][] | undefined;
   let headerRow: string[] | undefined;
 

src/utils/src/dataset-utils.ts

@@ -388,6 +388,27 @@ export function getSampleForTypeAnalyze({
   return sample;
 }
 
+/**
+ * Check if string is a valid Well-known binary (WKB) in HEX format
+ * https://en.wikipedia.org/wiki/Well-known_text_representation_of_geometry
+ *
+ * @param str input string
+ * @returns true if string is a valid WKB in HEX format
+ */
+export function isHexWkb(str: string | null): boolean {
+  if (!str) return false;
+  // check if the length of the string is even and is at least 10 characters long
+  if (str.length < 10 || str.length % 2 !== 0) {
+    return false;
+  }
+  // check if first two characters are 00 or 01
+  if (!str.startsWith('00') && !str.startsWith('01')) {
+    return false;
+  }
+  // check if the rest of the string is a valid hex
+  return /^[0-9a-fA-F]+$/.test(str.slice(2));
+}
+
 /**
  * Analyze field types from data in `string` format, e.g. uploaded csv.
  * Assign `type`, `fieldIdx` and `format` (timestamp only) to each field
@@ -446,7 +467,13 @@ export function getFieldsFromData(data: RowData, fieldOrder: string[]): Field[]
     const name = fieldByIndex[index];
 
     const fieldMeta = metadata.find(m => m.key === field);
-    const {type, format} = fieldMeta || {};
+    let type = fieldMeta.type;
+    const format = fieldMeta.format;
+
+    // check if string is hex wkb
+    if (type === AnalyzerDATA_TYPES.STRING) {
+      type = data.some(d => isHexWkb(d[name])) ? AnalyzerDATA_TYPES.GEOMETRY : type;
+    }
 
     return {
       name,

test/node/utils/data-processor-test.js

@@ -72,7 +72,10 @@ test('Processor -> getFieldsFromData', t => {
       value: '4',
       surge: '1.2',
       isTrip: 'true',
-      zeroOnes: '0'
+      zeroOnes: '0',
+      geojson: '{"type":"Point","coordinates":[-122.4194155,37.7749295]}',
+      wkt: 'POINT (-122.4194155 37.7749295)',
+      wkb: '0101000020E6100000E17A14AE47D25EC0F6F3F6F2F7F94040'
     },
     {
       time: '2016-09-17 00:30:08',
@@ -81,7 +84,10 @@ test('Processor -> getFieldsFromData', t => {
       value: '3',
       surge: null,
       isTrip: 'false',
-      zeroOnes: '1'
+      zeroOnes: '1',
+      geojson: '{"type":"Polygon","coordinates":[[[-122.4194155,37.7749295],[-122.4194155,37.7749295],[-122.4194155,37.7749295]]]}',
+      wkt: 'POLYGON ((-122.4194155 37.7749295, -122.4194155 37.7749295, -122.4194155 37.7749295))',
+      wkb: '0103000020E61000000100000005000000E17A14AE47D25EC0F6F3F6F2F7F940400000000E17A14AE47D25EC0F6F3F6F2F7F940400000000E17A14AE47D25EC0F6F3F6F2F7F94040'
     },
     {
       time: null,
@@ -90,7 +96,10 @@ test('Processor -> getFieldsFromData', t => {
       value: '2',
       surge: '1.3',
       isTrip: null,
-      zeroOnes: '1'
+      zeroOnes: '1',
+      geojson: '{"type":"LineString","coordinates":[[-122.4194155,37.7749295],[-122.4194155,37.7749295]]}',
+      wkt: 'LINESTRING (-122.4194155 37.7749295, -122.4194155 37.7749295)',
+      wkb: '0102000020E610000002000000E17A14AE47D25EC0F6F3F6F2F7F94040E17A14AE47D25EC0F6F3F6F2F7F94040'
     },
     {
       time: null,
@@ -99,7 +108,10 @@ test('Processor -> getFieldsFromData', t => {
       value: '0',
       surge: '1.4',
       isTrip: null,
-      zeroOnes: '0'
+      zeroOnes: '0',
+      geojson: '{"type":"MultiPoint","coordinates":[[-122.4194155,37.7749295],[-122.4194155,37.7749295]]}',
+      wkt: 'MULTIPOINT (-122.4194155 37.7749295, -122.4194155 37.7749295)',
+      wkb: '0104000020E6100000020000000101000000E17A14AE47D25EC0F6F3F6F2F7F94040101000000E17A14AE47D25EC0F6F3F6F2F7F94040'
     }
   ];
 
@@ -112,7 +124,10 @@ test('Processor -> getFieldsFromData', t => {
     'integer',
     'real',
     'boolean',
-    'integer'
+    'integer',
+    'geojson',
+    'geojson',
+    'geojson'
   ];
 
   fields.forEach((f, i) =>

test/node/utils/dataset-utils-test.js

@@ -79,3 +79,30 @@ test('datasetUtils.findDefaultColorField', t => {
   }
   t.end();
 });
+
+test('datasetUtils.isHexWkb', t => {
+  t.notOk(isHexWkb(''), 'empty string is not a valid hex wkb');
+
+  t.notOk(isHexWkb(null), 'null is not a valid hex wkb');
+
+  const countyFIPS = '06075';
+  t.notOk(isHexWkb(countyFIPS), 'FIPS code should not be a valid hex wkb');
+
+  const h3Code = '8a2a1072b59ffff';
+  t.notOk(isHexWkb(h3Code), 'H3 code should not be a valid hex wkb');
+
+  const randomHexStr = '8a2a1072b59ffff';
+  t.notOk(isHexWkb(randomHexStr), 'A random hex string should not be a valid hex wkb');
+
+  const validWkt = '0101000000000000000000f03f0000000000000040';
+  t.ok(isHexWkb(validWkt), 'A valid hex wkb should be valid');
+
+  const validEWkt = '0101000020e6100000000000000000f03f0000000000000040';
+  t.ok(isHexWkb(validEWkt), 'A valid hex ewkb should be valid');
+
+  const validWktNDR = '00000000013ff0000000000000400000000000000040';
+  t.ok(isHexWkb(validWktNDR), 'A valid hex wkb in NDR should be valid');
+
+  const validEWktNDR = '0020000001000013ff0000000000400000000000000040';
+  t.ok(isHexWkb(validEWktNDR), 'A valid hex ewkb in NDR should be valid');
+});

tsconfig.json

@@ -21,7 +21,7 @@
     "strict": true,
     "resolveJsonModule": true,
     "isolatedModules": true,
-    "baseUrl": "./src",
+    "baseUrl": ".",
     "paths": {
       "*": ["*"],
       // Map all modules to their source