Enh/read parquet files

deno.json

@@ -32,6 +32,7 @@
     "@cliffy/command": "jsr:@effigies/[email protected]",
     "@cliffy/table": "jsr:@effigies/[email protected]",
     "@hed/validator": "npm:[email protected]",
+    "@hyparquet": "npm:[email protected]",
     "@ignore": "npm:[email protected]",
     "@libs/xml": "jsr:@libs/[email protected]",
     "@mango/nifti": "npm:@bids/[email protected]",

src/files/browser.ts

@@ -29,6 +29,10 @@ export class BIDSFileBrowser implements BIDSFile {
     return this.#file.size
   }
 
+  get size(): number {
+    return this.#file.size
+  }
+
   get stream(): ReadableStream<Uint8Array> {
     return this.#file.stream()
   }
@@ -44,6 +48,12 @@ export class BIDSFileBrowser implements BIDSFile {
   async readBytes(size: number, offset = 0): Promise<Uint8Array<ArrayBuffer>> {
     return new Uint8Array(await this.#file.slice(offset, size).arrayBuffer())
   }
+
+  async slice(start: number, end: number): Promise<ArrayBuffer> {
+    return (await this.readBytes(end - start, start)).buffer
+  }
+
+  /**
 }
 
 /**

src/files/deno.ts

@@ -45,6 +45,10 @@ export class BIDSFileDeno implements BIDSFile {
     return this.#fileInfo ? this.#fileInfo.size : -1
   }
 
+  get byteLength(): number {
+    return this.#fileInfo ? this.#fileInfo.size : -1
+  }
+
   get stream(): ReadableStream<Uint8Array> {
     const handle = this.#openHandle()
     return handle.readable
@@ -87,6 +91,9 @@ export class BIDSFileDeno implements BIDSFile {
     return buf.subarray(0, nbytes)
   }
 
+  async slice(start: number, end: number): Promise<ArrayBuffer> {
+    return (await this.readBytes(end - start, start)).buffer
+  }
   /**
    * Return a Deno file handle
    */

src/files/parquet.test.ts

@@ -0,0 +1,18 @@
+import { assertEquals } from '@std/assert'
+import { FileIgnoreRules } from './ignore.ts'
+import { BIDSFileDeno } from './deno.ts'
+
+import { loadParquet } from './parquet.ts'
+
+Deno.test('Test loading parquet file', async (t) => {
+  const ignore = new FileIgnoreRules([])
+  await t.step('Load participants.parquet', async () => {
+    const path = 'participants.parquet'
+    const root = './tests/data/'
+    const file = new BIDSFileDeno(root, path, ignore)
+    const participantsMap = await loadParquet(file)
+    const keys = Object.keys(participantsMap)
+    assertEquals(keys.length, 3)
+    keys.map(key => assertEquals(participantsMap.get(key)?.length, 16))
+  })
+})

src/files/parquet.ts

@@ -0,0 +1,15 @@
+import { asyncBufferFromFile, parquetRead, ParquetReadOptions, ColumnData } from '@hyparquet'
+import type { BIDSFile } from '../types/filetree.ts'
+import { ColumnsMap } from '../types/columns.ts'
+import { createUTF8Stream } from './streams.ts'
+
+export async function loadParquet(file: BIDSFile, maxRows: number = -1): Promise<ColumnsMap> {
+  let columnsMap = new ColumnsMap()
+  const readOpts: ParquetReadOptions = {
+    file: file,
+    // @ts-expect-error
+    onChunk: (data) => columnsMap[data.columnName] = data.columnData.map(entry => String(entry)), 
+  }
+  await parquetRead(readOpts)
+  return columnsMap
+}

src/types/filetree.ts

@@ -15,6 +15,10 @@ export interface BIDSFile {
   text: () => Promise<string>
   // Read a range of bytes
   readBytes: (size: number, offset?: number) => Promise<Uint8Array<ArrayBuffer>>
+  // Alternative to readBytes used for ArrayBuffer compatibility
+  slice: (start: number, end: number) => Promise<ArrayBuffer>
+  // Alternative to size used for ArrayBuffer compatibility
+  byteLength: number
   // Access the parent directory
   parent: FileTree
   // File has been viewed