Merge pull request #1662 from silx-kit/virtual-sources

Load compression plugins for virtual datasets with h5wasm
silx-kit · Jun 7, 2024 · ead05b2 · ead05b2
2 parents 5adb7de + 055dc78
commit ead05b2
Show file tree

Hide file tree

Showing 3 changed files with 40 additions and 13 deletions.
diff --git a/packages/h5wasm/src/h5wasm-api.ts b/packages/h5wasm/src/h5wasm-api.ts
@@ -1,12 +1,11 @@
 import type { ExportFormat, ExportURL, ValuesStoreParams } from '@h5web/app';
 import { DataProviderApi } from '@h5web/app';
-import { isDefined } from '@h5web/shared/guards';
+import { assertDataset, isDefined } from '@h5web/shared/guards';
 import type {
   ArrayShape,
   AttributeValues,
   Dataset,
   Entity,
-  Filter,
   ProvidedEntity,
   Value,
 } from '@h5web/shared/hdf5-models';
@@ -42,7 +41,7 @@ export class H5WasmApi extends DataProviderApi {
     const { dataset, selection } = params;
     const fileId = await this.fileId;
 
-    await this.processFilters(dataset.filters);
+    await this.processFilters(dataset);
 
     try {
       const value = await this.remote.getValue(fileId, dataset.path, selection);
@@ -94,8 +93,21 @@ export class H5WasmApi extends DataProviderApi {
     return this.remote.closeFile(await this.fileId);
   }
 
-  private async processFilters(filters: Filter[] = []): Promise<void> {
-    const pluginsToLoad = filters
+  private async processFilters(dataset: Dataset): Promise<void> {
+    const fileId = await this.fileId;
+
+    // Retrieve filters of any local virtual source datasets
+    const localSources = await Promise.all(
+      (dataset.virtualSources?.filter(({ file }) => file === '.') || []).map(
+        ({ path }) => this.remote.getEntity(fileId, path),
+      ),
+    );
+    const allFilters = [dataset, ...localSources].flatMap((source) => {
+      assertDataset(source);
+      return source.filters || [];
+    });
+
+    const pluginsToLoad = allFilters
       .map(({ id }) => PLUGINS_BY_FILTER_ID[id])
       .filter(isDefined);
 

diff --git a/packages/h5wasm/src/worker-utils.ts b/packages/h5wasm/src/worker-utils.ts
@@ -11,6 +11,7 @@ import type {
   DType,
   Group,
   ProvidedEntity,
+  VirtualSource,
 } from '@h5web/shared/hdf5-models';
 import { EntityKind } from '@h5web/shared/hdf5-models';
 import {
@@ -130,15 +131,16 @@ export function parseEntity(
 
   if (kind === h5wasm.H5G_DATASET) {
     const metadata = h5wasm.get_dataset_metadata(fileId, path);
-    const { chunks, maxshape, shape, ...rawType } = metadata; // keep `rawType` concise
+    const { chunks, maxshape, shape, virtual_sources, ...rawType } = metadata; // keep `rawType` concise
 
     return {
       ...baseEntity,
       kind: EntityKind.Dataset,
-      shape: metadata.shape,
+      shape,
       type: parseDType(metadata),
-      chunks: metadata.chunks ?? undefined,
+      chunks: chunks ?? undefined,
       filters: h5wasm.get_dataset_filters(fileId, path),
+      virtualSources: parseVirtualSources(metadata),
       attributes: parseAttributes(h5wasm, fileId, path),
       rawType,
     };
@@ -287,6 +289,13 @@ function parseDType(metadata: Metadata): DType {
   return unknownType();
 }
 
+function parseVirtualSources(metadata: Metadata): VirtualSource[] | undefined {
+  return metadata.virtual_sources?.map(({ file_name, dset_name }) => ({
+    file: file_name,
+    path: dset_name,
+  }));
+}
+
 export function readSelectedValue(
   h5wDataset: H5WasmDataset,
   selection: string | undefined,

diff --git a/packages/shared/src/hdf5-models.ts b/packages/shared/src/hdf5-models.ts
@@ -51,6 +51,7 @@ export interface Dataset<S extends Shape = Shape, T extends DType = DType>
   rawType?: unknown;
   chunks?: number[];
   filters?: Filter[];
+  virtualSources?: VirtualSource[];
 }
 
 export interface Datatype<T = DType> extends Entity {
@@ -76,6 +77,16 @@ export interface Attribute<S extends Shape = Shape, T extends DType = DType> {
   type: T;
 }
 
+export interface Filter {
+  id: number;
+  name: string;
+}
+
+export interface VirtualSource {
+  file: string;
+  path: string;
+}
+
 export type NumArrayDataset = Dataset<ArrayShape, NumericType>;
 
 /* ----------------- */
@@ -227,8 +238,3 @@ export type AttributeValues = Record<string, unknown>;
 
 export type H5WebComplex = [real: number, imag: number];
 export type ComplexArray = (ComplexArray | H5WebComplex)[];
-
-export interface Filter {
-  id: number;
-  name: string;
-}