diff --git a/src/components/src/side-panel/add-by-dataset-button.tsx b/src/components/src/side-panel/add-by-dataset-button.tsx index 9bc09b1c4a..bf339861d3 100644 --- a/src/components/src/side-panel/add-by-dataset-button.tsx +++ b/src/components/src/side-panel/add-by-dataset-button.tsx @@ -25,8 +25,7 @@ import {Datasets} from '@kepler.gl/table'; import Tippy from '@tippyjs/react'; import {Add} from '../common/icons'; -import {Button} from '../common/styled-components'; -import {DatasetSquare} from '../..'; +import {Button, DatasetSquare} from '../common/styled-components'; import Typeahead from '../common/item-selector/typeahead'; import Accessor from '../common/item-selector/accessor'; import {useIntl} from 'react-intl'; diff --git a/src/layers/package.json b/src/layers/package.json index 48ac2a9630..63a66843a5 100644 --- a/src/layers/package.json +++ b/src/layers/package.json @@ -43,7 +43,9 @@ "@kepler.gl/types": "3.0.0-alpha.0", "@kepler.gl/utils": "3.0.0-alpha.0", "@loaders.gl/core": "^3.0.9", + "@loaders.gl/gis": "^3.0.9", "@loaders.gl/gltf": "^3.0.9", + "@loaders.gl/wkt": "^3.0.9", "@luma.gl/constants": "^8.5.10", "@mapbox/geojson-normalize": "0.0.1", "@nebula.gl/layers": "1.0.2-alpha.1", diff --git a/src/layers/src/geojson-layer/geojson-utils.ts b/src/layers/src/geojson-layer/geojson-utils.ts index fcecb50156..a03a5174e5 100644 --- a/src/layers/src/geojson-layer/geojson-utils.ts +++ b/src/layers/src/geojson-layer/geojson-utils.ts @@ -21,6 +21,9 @@ import wktParser from 'wellknown'; import normalize from '@mapbox/geojson-normalize'; import bbox from '@turf/bbox'; +import {parseSync} from '@loaders.gl/core'; +import {WKBLoader} from '@loaders.gl/wkt'; +import {binaryToGeometry} from '@loaders.gl/gis'; import {Feature, BBox} from 'geojson'; import {getSampleData} from '@kepler.gl/utils'; @@ -138,6 +141,17 @@ export function parseGeometryFromString(geoString: string): Feature | null { } } + // try parse as wkb using loaders.gl WKBLoader + if (!parsedGeo) { + try { + const buffer = Buffer.from(geoString, 'hex'); + const binaryGeo = parseSync(buffer, WKBLoader); + parsedGeo = binaryToGeometry(binaryGeo); + } catch (e) { + return null; + } + } + if (!parsedGeo) { return null; } diff --git a/src/processors/src/data-processor.ts b/src/processors/src/data-processor.ts index 3348834123..e5d7609c33 100644 --- a/src/processors/src/data-processor.ts +++ b/src/processors/src/data-processor.ts @@ -105,7 +105,7 @@ export const PARSE_FIELD_VALUE_FROM_STRING = { * options: {centerMap: true, readOnly: true} * })); */ -export function processCsvData(rawData: unknown[][], header?: string[]): ProcessorResult { +export function processCsvData(rawData: unknown[][] | string, header?: string[]): ProcessorResult { let rows: unknown[][] | undefined; let headerRow: string[] | undefined; diff --git a/src/utils/src/dataset-utils.ts b/src/utils/src/dataset-utils.ts index 596ac32d7c..53e8d94810 100644 --- a/src/utils/src/dataset-utils.ts +++ b/src/utils/src/dataset-utils.ts @@ -388,6 +388,27 @@ export function getSampleForTypeAnalyze({ return sample; } +/** + * Check if string is a valid Well-known binary (WKB) in HEX format + * https://en.wikipedia.org/wiki/Well-known_text_representation_of_geometry + * + * @param str input string + * @returns true if string is a valid WKB in HEX format + */ +export function isHexWkb(str: string | null): boolean { + if (!str) return false; + // check if the length of the string is even and is at least 10 characters long + if (str.length < 10 || str.length % 2 !== 0) { + return false; + } + // check if first two characters are 00 or 01 + if (!str.startsWith('00') && !str.startsWith('01')) { + return false; + } + // check if the rest of the string is a valid hex + return /^[0-9a-fA-F]+$/.test(str.slice(2)); +} + /** * Analyze field types from data in `string` format, e.g. uploaded csv. * Assign `type`, `fieldIdx` and `format` (timestamp only) to each field @@ -446,7 +467,13 @@ export function getFieldsFromData(data: RowData, fieldOrder: string[]): Field[] const name = fieldByIndex[index]; const fieldMeta = metadata.find(m => m.key === field); - const {type, format} = fieldMeta || {}; + let type = fieldMeta.type; + const format = fieldMeta.format; + + // check if string is hex wkb + if (type === AnalyzerDATA_TYPES.STRING) { + type = data.some(d => isHexWkb(d[name])) ? AnalyzerDATA_TYPES.GEOMETRY : type; + } return { name, diff --git a/test/node/utils/data-processor-test.js b/test/node/utils/data-processor-test.js index 0c843910a0..dca2e149d7 100644 --- a/test/node/utils/data-processor-test.js +++ b/test/node/utils/data-processor-test.js @@ -72,7 +72,10 @@ test('Processor -> getFieldsFromData', t => { value: '4', surge: '1.2', isTrip: 'true', - zeroOnes: '0' + zeroOnes: '0', + geojson: '{"type":"Point","coordinates":[-122.4194155,37.7749295]}', + wkt: 'POINT (-122.4194155 37.7749295)', + wkb: '0101000020E6100000E17A14AE47D25EC0F6F3F6F2F7F94040' }, { time: '2016-09-17 00:30:08', @@ -81,7 +84,10 @@ test('Processor -> getFieldsFromData', t => { value: '3', surge: null, isTrip: 'false', - zeroOnes: '1' + zeroOnes: '1', + geojson: '{"type":"Polygon","coordinates":[[[-122.4194155,37.7749295],[-122.4194155,37.7749295],[-122.4194155,37.7749295]]]}', + wkt: 'POLYGON ((-122.4194155 37.7749295, -122.4194155 37.7749295, -122.4194155 37.7749295))', + wkb: '0103000020E61000000100000005000000E17A14AE47D25EC0F6F3F6F2F7F940400000000E17A14AE47D25EC0F6F3F6F2F7F940400000000E17A14AE47D25EC0F6F3F6F2F7F94040' }, { time: null, @@ -90,7 +96,10 @@ test('Processor -> getFieldsFromData', t => { value: '2', surge: '1.3', isTrip: null, - zeroOnes: '1' + zeroOnes: '1', + geojson: '{"type":"LineString","coordinates":[[-122.4194155,37.7749295],[-122.4194155,37.7749295]]}', + wkt: 'LINESTRING (-122.4194155 37.7749295, -122.4194155 37.7749295)', + wkb: '0102000020E610000002000000E17A14AE47D25EC0F6F3F6F2F7F94040E17A14AE47D25EC0F6F3F6F2F7F94040' }, { time: null, @@ -99,7 +108,10 @@ test('Processor -> getFieldsFromData', t => { value: '0', surge: '1.4', isTrip: null, - zeroOnes: '0' + zeroOnes: '0', + geojson: '{"type":"MultiPoint","coordinates":[[-122.4194155,37.7749295],[-122.4194155,37.7749295]]}', + wkt: 'MULTIPOINT (-122.4194155 37.7749295, -122.4194155 37.7749295)', + wkb: '0104000020E6100000020000000101000000E17A14AE47D25EC0F6F3F6F2F7F94040101000000E17A14AE47D25EC0F6F3F6F2F7F94040' } ]; @@ -112,7 +124,10 @@ test('Processor -> getFieldsFromData', t => { 'integer', 'real', 'boolean', - 'integer' + 'integer', + 'geojson', + 'geojson', + 'geojson' ]; fields.forEach((f, i) => diff --git a/test/node/utils/dataset-utils-test.js b/test/node/utils/dataset-utils-test.js index d1ba17836d..15278e191c 100644 --- a/test/node/utils/dataset-utils-test.js +++ b/test/node/utils/dataset-utils-test.js @@ -79,3 +79,30 @@ test('datasetUtils.findDefaultColorField', t => { } t.end(); }); + +test('datasetUtils.isHexWkb', t => { + t.notOk(isHexWkb(''), 'empty string is not a valid hex wkb'); + + t.notOk(isHexWkb(null), 'null is not a valid hex wkb'); + + const countyFIPS = '06075'; + t.notOk(isHexWkb(countyFIPS), 'FIPS code should not be a valid hex wkb'); + + const h3Code = '8a2a1072b59ffff'; + t.notOk(isHexWkb(h3Code), 'H3 code should not be a valid hex wkb'); + + const randomHexStr = '8a2a1072b59ffff'; + t.notOk(isHexWkb(randomHexStr), 'A random hex string should not be a valid hex wkb'); + + const validWkt = '0101000000000000000000f03f0000000000000040'; + t.ok(isHexWkb(validWkt), 'A valid hex wkb should be valid'); + + const validEWkt = '0101000020e6100000000000000000f03f0000000000000040'; + t.ok(isHexWkb(validEWkt), 'A valid hex ewkb should be valid'); + + const validWktNDR = '00000000013ff0000000000000400000000000000040'; + t.ok(isHexWkb(validWktNDR), 'A valid hex wkb in NDR should be valid'); + + const validEWktNDR = '0020000001000013ff0000000000400000000000000040'; + t.ok(isHexWkb(validEWktNDR), 'A valid hex ewkb in NDR should be valid'); +}); diff --git a/tsconfig.json b/tsconfig.json index deb467a201..91a7b87c47 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -21,7 +21,7 @@ "strict": true, "resolveJsonModule": true, "isolatedModules": true, - "baseUrl": "./src", + "baseUrl": ".", "paths": { "*": ["*"], // Map all modules to their source