From c40aa5529cf7fe10cfe4a2f4732fcb4405da0ff5 Mon Sep 17 00:00:00 2001 From: Axel Bocciarelli Date: Wed, 4 Sep 2024 15:07:54 +0200 Subject: [PATCH] Merge signed and unsigned integer types --- packages/app/src/metadata-viewer/utils.ts | 5 +++- .../app/src/providers/h5grove/utils.test.ts | 7 ++--- packages/app/src/providers/h5grove/utils.ts | 6 ++-- packages/app/src/providers/hsds/utils.test.ts | 5 ++-- packages/app/src/providers/hsds/utils.ts | 7 +---- packages/app/src/providers/mock/mock-file.ts | 29 ++++++++++--------- packages/app/src/providers/utils.ts | 22 +++++--------- packages/app/src/vis-packs/core/utils.ts | 7 +++-- .../src/vis-packs/core/visualizations.test.ts | 7 ++--- packages/h5wasm/src/utils.ts | 5 +--- packages/h5wasm/src/worker-utils.ts | 8 ++--- packages/shared/src/guards.ts | 15 +++++++--- packages/shared/src/hdf5-models.ts | 13 +++++++-- packages/shared/src/hdf5-utils.ts | 26 +++++------------ 14 files changed, 76 insertions(+), 86 deletions(-) diff --git a/packages/app/src/metadata-viewer/utils.ts b/packages/app/src/metadata-viewer/utils.ts index 329d8c867..ff7e5e0dd 100644 --- a/packages/app/src/metadata-viewer/utils.ts +++ b/packages/app/src/metadata-viewer/utils.ts @@ -1,5 +1,6 @@ import { isH5WebComplex, + isIntegerType, isNumericType, isScalarShape, } from '@h5web/shared/guards'; @@ -29,7 +30,9 @@ export function renderShape(shape: Shape): string { export function renderType(type: DType): string { if (isNumericType(type)) { const { endianness, size } = type; - return `${type.class}, ${size}-bit${endianness ? `, ${endianness}` : ''}`; + const signStr = isIntegerType(type) && !type.signed ? ' (unsigned)' : ''; + const endiannessStr = endianness ? `, ${endianness}` : ''; + return `${type.class}${signStr}, ${size}-bit${endiannessStr}`; } if (type.class === DTypeClass.String) { diff --git a/packages/app/src/providers/h5grove/utils.test.ts b/packages/app/src/providers/h5grove/utils.test.ts index 3b657781c..4ecd9354d 100644 --- a/packages/app/src/providers/h5grove/utils.test.ts +++ b/packages/app/src/providers/h5grove/utils.test.ts @@ -12,7 +12,6 @@ import { referenceType, strType, timeType, - uintType, unknownType, } from '@h5web/shared/hdf5-utils'; import { describe, expect, it } from 'vitest'; @@ -23,10 +22,10 @@ import { parseDType } from './utils'; describe('parseDType', () => { it('should convert integer types', () => { expect(parseDType({ class: 0, size: 1, order: 0, sign: 1 })).toStrictEqual( - intType(8, H5T_ORDER.LE), + intType(true, 8, H5T_ORDER.LE), ); expect(parseDType({ class: 0, size: 8, order: 1, sign: 0 })).toStrictEqual( - uintType(64, H5T_ORDER.BE), + intType(false, 64, H5T_ORDER.BE), ); }); @@ -83,7 +82,7 @@ describe('parseDType', () => { base: { class: 0, size: 4, order: 0, sign: 0 }, members: { FOO: 41, BAR: 42 }, }), - ).toStrictEqual(enumType(uintType(), { FOO: 41, BAR: 42 })); + ).toStrictEqual(enumType(intType(false), { FOO: 41, BAR: 42 })); expect( parseDType({ diff --git a/packages/app/src/providers/h5grove/utils.ts b/packages/app/src/providers/h5grove/utils.ts index 94e87e016..4ac37e631 100644 --- a/packages/app/src/providers/h5grove/utils.ts +++ b/packages/app/src/providers/h5grove/utils.ts @@ -1,5 +1,5 @@ import { isNumericType } from '@h5web/shared/guards'; -import { H5T_CLASS } from '@h5web/shared/h5t'; +import { H5T_CLASS, H5T_SIGN } from '@h5web/shared/h5t'; import type { Attribute, ChildEntity, @@ -15,7 +15,7 @@ import { compoundOrCplxType, enumOrBoolType, floatType, - intOrUintType, + intType, opaqueType, referenceType, strType, @@ -149,7 +149,7 @@ export function parseDType(type: H5GroveType): DType { const { class: h5tClass, size } = type; if (h5tClass === H5T_CLASS.INTEGER) { - return intOrUintType(type.sign, size * 8, type.order); + return intType(type.sign === H5T_SIGN.SIGN_2, size * 8, type.order); } if (h5tClass === H5T_CLASS.FLOAT) { diff --git a/packages/app/src/providers/hsds/utils.test.ts b/packages/app/src/providers/hsds/utils.test.ts index 418091850..1794e8d16 100644 --- a/packages/app/src/providers/hsds/utils.test.ts +++ b/packages/app/src/providers/hsds/utils.test.ts @@ -8,7 +8,6 @@ import { floatType, intType, strType, - uintType, unknownType, } from '@h5web/shared/hdf5-utils'; import { describe, expect, it } from 'vitest'; @@ -29,12 +28,12 @@ interface TestType { const leInt = { hsds: { class: 'H5T_INTEGER', base: 'H5T_STD_I8LE' }, - hdf5: intType(8, H5T_ORDER.LE), + hdf5: intType(true, 8, H5T_ORDER.LE), } satisfies TestType; const beUint = { hsds: { class: 'H5T_INTEGER', base: 'H5T_STD_U64BE' }, - hdf5: uintType(64, H5T_ORDER.BE), + hdf5: intType(false, 64, H5T_ORDER.BE), } satisfies TestType; const leFloat = { diff --git a/packages/app/src/providers/hsds/utils.ts b/packages/app/src/providers/hsds/utils.ts index cc1fce295..ea954ff63 100644 --- a/packages/app/src/providers/hsds/utils.ts +++ b/packages/app/src/providers/hsds/utils.ts @@ -23,7 +23,6 @@ import { floatType, intType, strType, - uintType, unknownType, } from '@h5web/shared/hdf5-utils'; @@ -88,11 +87,7 @@ function convertHsdsNumericType(hsdsType: HsdsNumericType): NumericType { return floatType(size, h5tOrder); } - if (sign === 'U') { - return uintType(size, h5tOrder); - } - - return intType(size, h5tOrder); + return intType(sign === 'I', size, h5tOrder); } function convertHsdsCompoundType( diff --git a/packages/app/src/providers/mock/mock-file.ts b/packages/app/src/providers/mock/mock-file.ts index c6862940d..b7689da23 100644 --- a/packages/app/src/providers/mock/mock-file.ts +++ b/packages/app/src/providers/mock/mock-file.ts @@ -12,7 +12,6 @@ import { opaqueType, printableCompoundType, strType, - uintType, unknownType, } from '@h5web/shared/hdf5-utils'; import { @@ -65,13 +64,13 @@ export function makeMockFile(): GroupWithChildren { scalar('scalar_compound', ['foo', 2], { type: compoundType({ str: strType(H5T_CSET.ASCII, H5T_STR.NULLPAD, 3), - int: intType(8), + int: intType(true, 8), }), attributes: [ scalarAttr('attr', ['foo', 2], { type: compoundType({ str: strType(H5T_CSET.UTF8), - int: intType(8), + int: intType(true, 8), }), }), ], @@ -89,10 +88,10 @@ export function makeMockFile(): GroupWithChildren { ], }), scalar('scalar_enum', 2, { - type: enumType(uintType(8), ENUM_MAPPING), + type: enumType(intType(false, 8), ENUM_MAPPING), attributes: [ scalarAttr('attr', 2, { - type: enumType(uintType(8), ENUM_MAPPING), + type: enumType(intType(false, 8), ENUM_MAPPING), }), ], }), @@ -119,7 +118,9 @@ export function makeMockFile(): GroupWithChildren { }), }), array('oneD_bool'), - array('oneD_enum', { type: enumType(uintType(8), ENUM_MAPPING) }), + array('oneD_enum', { + type: enumType(intType(false, 8), ENUM_MAPPING), + }), array('twoD'), array('twoD_cplx'), array('twoD_compound', { @@ -132,7 +133,9 @@ export function makeMockFile(): GroupWithChildren { }), }), array('twoD_bool'), - array('twoD_enum', { type: enumType(uintType(8), ENUM_MAPPING) }), + array('twoD_enum', { + type: enumType(intType(false, 8), ENUM_MAPPING), + }), array('threeD'), array('threeD_bool'), array('threeD_cplx'), @@ -140,13 +143,13 @@ export function makeMockFile(): GroupWithChildren { array('fourD'), ]), group('typed_arrays', [ - array('uint8', { type: uintType(8) }), - array('int16', { type: intType(16) }), + array('uint8', { type: intType(false, 8) }), + array('int16', { type: intType(true, 16) }), array('float32', { type: floatType(32) }), array('float64', { type: floatType(64) }), - withImageAttr(array('uint8_rgb', { type: uintType(8) })), - withImageAttr(array('int8_rgb', { type: intType(8) })), - withImageAttr(array('int32_rgb', { type: intType(32) })), + withImageAttr(array('uint8_rgb', { type: intType(false, 8) })), + withImageAttr(array('int8_rgb', { type: intType(true, 8) })), + withImageAttr(array('int32_rgb', { type: intType(true, 32) })), withImageAttr(array('float32_rgb', { type: floatType(32) })), ]), nxGroup('nexus_entry', 'NXentry', { @@ -277,7 +280,7 @@ export function makeMockFile(): GroupWithChildren { nxData('bool', { signal: array('twoD_bool') }), nxData('enum', { signal: array('twoD_enum', { - type: enumType(uintType(8), ENUM_MAPPING), + type: enumType(intType(false, 8), ENUM_MAPPING), }), }), ], diff --git a/packages/app/src/providers/utils.ts b/packages/app/src/providers/utils.ts index 262d43e90..8cadda0cf 100644 --- a/packages/app/src/providers/utils.ts +++ b/packages/app/src/providers/utils.ts @@ -1,11 +1,10 @@ -import { isEnumType, isNumericType } from '@h5web/shared/guards'; +import { isEnumType, isFloatType, isIntegerType } from '@h5web/shared/guards'; import type { ArrayShape, Dataset, DType, ScalarShape, } from '@h5web/shared/hdf5-models'; -import { DTypeClass } from '@h5web/shared/hdf5-models'; import type { OnProgress } from '@h5web/shared/react-suspense-fetch'; import type { AxiosProgressEvent } from 'axios'; @@ -18,15 +17,8 @@ export function typedArrayFromDType(dtype: DType) { return typedArrayFromDType(dtype.base); } - if (!isNumericType(dtype)) { - return undefined; - } - - /* Adapted from https://github.com/ludwigschubert/js-numpy-parser/blob/v1.2.3/src/main.js#L116 */ - const { class: dtypeClass, size } = dtype; - - if (dtypeClass === DTypeClass.Integer) { - switch (size) { + if (isIntegerType(dtype) && dtype.signed) { + switch (dtype.size) { case 8: return Int8Array; case 16: @@ -38,8 +30,8 @@ export function typedArrayFromDType(dtype: DType) { } } - if (dtypeClass === DTypeClass.Unsigned) { - switch (size) { + if (isIntegerType(dtype) && !dtype.signed) { + switch (dtype.size) { case 8: return Uint8Array; case 16: @@ -51,8 +43,8 @@ export function typedArrayFromDType(dtype: DType) { } } - if (dtypeClass === DTypeClass.Float) { - switch (size) { + if (isFloatType(dtype)) { + switch (dtype.size) { case 16: // No support for 16-bit floating values in JS return undefined; case 32: diff --git a/packages/app/src/vis-packs/core/utils.ts b/packages/app/src/vis-packs/core/utils.ts index caf4d6ba3..263d76d85 100644 --- a/packages/app/src/vis-packs/core/utils.ts +++ b/packages/app/src/vis-packs/core/utils.ts @@ -1,4 +1,4 @@ -import { isNumericType } from '@h5web/shared/guards'; +import { isIntegerType, isNumericType } from '@h5web/shared/guards'; import type { ArrayValue, NumericLikeType } from '@h5web/shared/hdf5-models'; import { DTypeClass } from '@h5web/shared/hdf5-models'; import type { Axis, Domain, NumArray } from '@h5web/shared/vis-models'; @@ -100,10 +100,11 @@ const TYPE_STRINGS: Record = { [DTypeClass.Bool]: 'bool', [DTypeClass.Enum]: 'enum', [DTypeClass.Integer]: 'int', - [DTypeClass.Unsigned]: 'uint', [DTypeClass.Float]: 'float', }; export function formatNumLikeType(type: NumericLikeType): string { - return `${TYPE_STRINGS[type.class]}${isNumericType(type) ? type.size : ''}`; + const unsignedPrefix = isIntegerType(type) && !type.signed ? 'u' : ''; + const sizeSuffix = isNumericType(type) ? type.size : ''; + return `${unsignedPrefix}${TYPE_STRINGS[type.class]}${sizeSuffix}`; } diff --git a/packages/app/src/vis-packs/core/visualizations.test.ts b/packages/app/src/vis-packs/core/visualizations.test.ts index 31e005628..b9e35a44b 100644 --- a/packages/app/src/vis-packs/core/visualizations.test.ts +++ b/packages/app/src/vis-packs/core/visualizations.test.ts @@ -6,7 +6,6 @@ import { floatType, intType, strType, - uintType, } from '@h5web/shared/hdf5-utils'; import { assertMockAttribute, @@ -31,19 +30,19 @@ const mockStore = { } as AttrValuesStore; const scalarInt = dataset('int', intType(), []); -const scalarUint = dataset('uint', uintType(), []); +const scalarUint = dataset('uint', intType(false, 32), []); const scalarFloat = dataset('float', floatType(), []); const scalarStr = dataset('float', strType(), []); const scalarBool = dataset('bool', boolType(), []); const scalarCplx = dataset('cplx', cplxType(floatType()), []); const scalarCompound = dataset('comp', compoundType({ int: intType() }), []); const oneDInt = dataset('int_1d', intType(), [5]); -const oneDUint = dataset('uint_1d', uintType(), [5]); +const oneDUint = dataset('uint_1d', intType(false, 32), [5]); const oneDBool = dataset('bool_1d', boolType(), [3]); const oneDCplx = dataset('cplx_1d', cplxType(floatType()), [10]); const oneDCompound = dataset('comp_1d', compoundType({ int: intType() }), [5]); const twoDInt = dataset('int_2d', intType(), [5, 3]); -const twoDUint = dataset('uint_2d', uintType(), [5, 3]); +const twoDUint = dataset('uint_2d', intType(false, 32), [5, 3]); const twoDBool = dataset('bool_2d', boolType(), [3, 2]); const twoDCplx = dataset('cplx_2d', cplxType(floatType()), [2, 2]); const twoDStr = dataset('str_2d', strType(), [5, 3]); diff --git a/packages/h5wasm/src/utils.ts b/packages/h5wasm/src/utils.ts index d60138d13..c96e00134 100644 --- a/packages/h5wasm/src/utils.ts +++ b/packages/h5wasm/src/utils.ts @@ -30,10 +30,7 @@ export function hasBigInts(type: DType): boolean { return Object.values(type.fields).some(hasBigInts); } - return ( - (type.class === DTypeClass.Integer || type.class === DTypeClass.Unsigned) && - type.size === 64 - ); + return type.class === DTypeClass.Integer && type.size === 64; } export function sanitizeBigInts(value: unknown): unknown { diff --git a/packages/h5wasm/src/worker-utils.ts b/packages/h5wasm/src/worker-utils.ts index 196954336..3bea2600f 100644 --- a/packages/h5wasm/src/worker-utils.ts +++ b/packages/h5wasm/src/worker-utils.ts @@ -4,7 +4,7 @@ import { isNumericType, } from '@h5web/shared/guards'; import type { H5T_STR } from '@h5web/shared/h5t'; -import { H5T_CLASS, H5T_ORDER, H5T_SIGN } from '@h5web/shared/h5t'; +import { H5T_CLASS, H5T_ORDER } from '@h5web/shared/h5t'; import type { Attribute, ChildEntity, @@ -22,7 +22,7 @@ import { enumOrBoolType, floatType, getNameFromPath, - intOrUintType, + intType, opaqueType, referenceType, strType, @@ -213,8 +213,8 @@ function parseDType(metadata: Metadata): DType { if (h5tClass === H5T_CLASS.INTEGER) { const { signed, littleEndian } = metadata; - return intOrUintType( - signed ? H5T_SIGN.SIGN_2 : H5T_SIGN.NONE, + return intType( + signed, size * 8, littleEndian ? H5T_ORDER.LE : H5T_ORDER.BE, ); diff --git a/packages/shared/src/guards.ts b/packages/shared/src/guards.ts index abc5cdc32..5507bd733 100644 --- a/packages/shared/src/guards.ts +++ b/packages/shared/src/guards.ts @@ -11,9 +11,11 @@ import type { DType, Entity, EnumType, + FloatType, Group, GroupWithChildren, H5WebComplex, + IntegerType, NumericLikeType, NumericType, Primitive, @@ -34,7 +36,6 @@ import type { import { AXIS_SCALE_TYPES, COLOR_SCALE_TYPES, getValues } from './vis-utils'; const PRINTABLE_DTYPES = new Set([ - DTypeClass.Unsigned, DTypeClass.Integer, DTypeClass.Float, DTypeClass.String, @@ -294,10 +295,16 @@ export function assertStringType( } } +export function isIntegerType(type: DType): type is IntegerType { + return type.class === DTypeClass.Integer; +} + +export function isFloatType(type: DType): type is FloatType { + return type.class === DTypeClass.Float; +} + export function isNumericType(type: DType): type is NumericType { - return [DTypeClass.Integer, DTypeClass.Unsigned, DTypeClass.Float].includes( - type.class, - ); + return isIntegerType(type) || isFloatType(type); } export function hasNumericType( diff --git a/packages/shared/src/hdf5-models.ts b/packages/shared/src/hdf5-models.ts index 307f95b47..cba669bbf 100644 --- a/packages/shared/src/hdf5-models.ts +++ b/packages/shared/src/hdf5-models.ts @@ -102,7 +102,6 @@ export type ScalarShape = []; export enum DTypeClass { Bool = 'Boolean', Integer = 'Integer', - Unsigned = 'Integer (unsigned)', Float = 'Float', Complex = 'Complex', String = 'String', @@ -121,6 +120,7 @@ export type Endianness = (typeof H5T_TO_ENDIANNESS)[H5T_ORDER]; export type CharSet = (typeof H5T_TO_CHAR_SET)[H5T_CSET]; export type StrPad = (typeof H5T_TO_STR_PAD)[H5T_STR]; +export type NumericType = IntegerType | FloatType; export type NumericLikeType = NumericType | BooleanType | EnumType; export type PrintableType = StringType | NumericLikeType | ComplexType; @@ -138,8 +138,15 @@ export interface BooleanType { class: DTypeClass.Bool; } -export interface NumericType { - class: DTypeClass.Integer | DTypeClass.Unsigned | DTypeClass.Float; +export interface IntegerType { + class: DTypeClass.Integer; + signed: boolean; + size: number; + endianness: Endianness | undefined; +} + +export interface FloatType { + class: DTypeClass.Float; size: number; endianness: Endianness | undefined; } diff --git a/packages/shared/src/hdf5-utils.ts b/packages/shared/src/hdf5-utils.ts index d4dae41ae..634ff7ff5 100644 --- a/packages/shared/src/hdf5-utils.ts +++ b/packages/shared/src/hdf5-utils.ts @@ -2,7 +2,6 @@ import { isNumericType } from './guards'; import { H5T_CSET, H5T_ORDER, - H5T_SIGN, H5T_STR, H5T_TO_CHAR_SET, H5T_TO_ENDIANNESS, @@ -19,6 +18,7 @@ import type { EnumType, GroupWithChildren, H5WebComplex, + IntegerType, NumericType, OpaqueType, PrintableType, @@ -52,31 +52,19 @@ export function getNameFromPath(path: string) { /* ----------------- */ /* ----- TYPES ----- */ -export function intType(size = 32, h5tOrder = H5T_ORDER.LE): NumericType { +export function intType( + signed = true, + size = 32, + h5tOrder = H5T_ORDER.LE, +): IntegerType { return { class: DTypeClass.Integer, + signed, endianness: H5T_TO_ENDIANNESS[h5tOrder], size, }; } -export function uintType(size = 32, h5tOrder = H5T_ORDER.LE): NumericType { - return { - class: DTypeClass.Unsigned, - endianness: H5T_TO_ENDIANNESS[h5tOrder], - size, - }; -} - -export function intOrUintType( - h5tSign: H5T_SIGN, - size = 32, - h5tOrder = H5T_ORDER.LE, -) { - const func = h5tSign === H5T_SIGN.SIGN_2 ? intType : uintType; - return func(size, h5tOrder); -} - export function floatType(size = 32, h5tOrder = H5T_ORDER.LE): NumericType { return { class: DTypeClass.Float,