Merge branch 'main' into feat/add-keys-to-assoc-coordsys

Author: effigies

changelog.d/20250916_171340_markiewicz_bep020.md

@@ -0,0 +1,47 @@
+<!--
+A new scriv changelog fragment.
+
+Uncomment the section that is right (remove the HTML comment wrapper).
+For top level release notes, leave all the headers commented out.
+-->
+
+### Added
+
+- Support for `associations.physio` and `associations.events.sidecar`.
+
+<!--
+### Changed
+
+- A bullet item for the Changed category.
+
+-->
+<!--
+### Fixed
+
+- A bullet item for the Fixed category.
+
+-->
+<!--
+### Deprecated
+
+- A bullet item for the Deprecated category.
+
+-->
+<!--
+### Removed
+
+- A bullet item for the Removed category.
+
+-->
+<!--
+### Security
+
+- A bullet item for the Security category.
+
+-->
+<!--
+### Infrastructure
+
+- A bullet item for the Infrastructure category.
+
+-->

src/files/inheritance.ts

@@ -1,5 +1,6 @@
 import type { BIDSFile, FileTree } from '../types/filetree.ts'
 import { readEntities } from '../schema/entities.ts'
+import { loadJSON } from './json.ts'
 
 type Ret<T> = T extends [string, ...string[]] ? (BIDSFile | BIDSFile[]) : BIDSFile
 
@@ -77,3 +78,20 @@ export function* walkBack<T extends string[]>(
     fileTree = fileTree.parent
   }
 }
+
+export async function readSidecars(
+  source: BIDSFile,
+): Promise<Map<string, Record<string, unknown>>> {
+  const ret: Map<string, Record<string, unknown>> = new Map()
+  for (const file of walkBack(source)) {
+    try {
+      ret.set(file.path, await loadJSON(file))
+    } catch (e: any) {
+      // Expect JSON parsing errors to be handled when the file is loaded directly
+      if (!e?.code) {
+        throw e
+      }
+    }
+  }
+  return ret
+}

src/schema/associations.test.ts

@@ -0,0 +1,32 @@
+import { assertEquals, assertObjectMatch } from '@std/assert'
+import type { BIDSFile, FileTree } from '../types/filetree.ts'
+import { loadSchema } from '../setup/loadSchema.ts'
+import { pathsToTree } from '../files/filetree.test.ts'
+import { nullReadBytes } from '../tests/nullReadBytes.ts'
+import { rootFileTree } from './fixtures.test.ts'
+import { BIDSContext } from './context.ts'
+import { buildAssociations } from './associations.ts'
+
+Deno.test('Test association loading', async (t) => {
+  const schema = await loadSchema()
+  await t.step('Load associations for events.tsv', async () => {
+    const eventsFile = rootFileTree.get(
+      'sub-01/ses-01/func/sub-01_ses-01_task-movie_physio.tsv.gz',
+    ) as BIDSFile
+    const context = new BIDSContext(eventsFile, undefined, rootFileTree)
+    context.dataset.schema = schema
+    const associations = await buildAssociations(context)
+    assertObjectMatch(associations, {
+      events: {
+        sidecar: {
+          StimulusPresentation: {
+            ScreenDistance: 1.8,
+            ScreenOrigin: ['top', 'left'],
+            ScreenResolution: [1920, 1080],
+            ScreenSize: [0.472, 0.265],
+          },
+        },
+      },
+    })
+  })
+})

src/schema/associations.ts

@@ -6,7 +6,7 @@ import type { BIDSContext } from './context.ts'
 import { loadJSON } from '../files/json.ts'
 import { loadTSV } from '../files/tsv.ts'
 import { parseBvalBvec } from '../files/dwi.ts'
-import { walkBack } from '../files/inheritance.ts'
+import { readSidecars, walkBack } from '../files/inheritance.ts'
 import { evalCheck } from './applyRules.ts'
 import { expressionFunctions } from './expressionLanguage.ts'
 import { readEntities } from './entities.ts'
@@ -15,11 +15,20 @@ import { readText } from '../files/access.ts'
 
 type LoadFunction = (file: BIDSFile, options: any) => Promise<any>
 type MultiLoadFunction = (files: BIDSFile[], options: any) => Promise<any>
+interface WithSidecar {
+  sidecar: Record<string, unknown>
+}
 
 function defaultAssociation(file: BIDSFile, _options: any): Promise<{ path: string }> {
   return Promise.resolve({ path: file.path })
 }
 
+async function constructSidecar(file: BIDSFile): Promise<Record<string, unknown>> {
+  const sidecars = await readSidecars(file)
+  // Note ordering here gives precedence to the more specific sidecar
+  return sidecars.values().reduce((acc, json) => ({ ...json, ...acc }), {})
+}
+
 /**
  * This object describes lookup functions for files associated to data files in a bids dataset.
  * For any given data file we iterate over the associations defined schema.meta.associations.
@@ -29,14 +38,15 @@ function defaultAssociation(file: BIDSFile, _options: any): Promise<{ path: stri
  * Many associations only consist of a path; this object is for more complex associations.
  */
 const associationLookup: Record<string, LoadFunction> = {
-  events: async (file: BIDSFile, options: { maxRows: number }): Promise<Events> => {
+  events: async (file: BIDSFile, options: { maxRows: number }): Promise<Events & WithSidecar> => {
     const columns = await loadTSV(file, options.maxRows)
       .catch((e) => {
         return new Map()
       })
     return {
       path: file.path,
       onset: columns.get('onset') || [],
+      sidecar: await constructSidecar(file),
     }
   },
   aslcontext: async (
@@ -90,6 +100,12 @@ const associationLookup: Record<string, LoadFunction> = {
       sampling_frequency: columns.get('sampling_frequency'),
     }
   },
+  physio: async (file: BIDSFile, options: any): Promise<{path: string} & WithSidecar> => {
+    return {
+      path: file.path,
+      sidecar: await constructSidecar(file),
+    }
+  },
 }
 const multiAssociationLookup: Record<string, MultiLoadFunction> = {
   coordsystems: async (
@@ -116,18 +132,12 @@ export async function buildAssociations(
   const associations: Associations = {}
 
   const schema: MetaSchema = context.dataset.schema as MetaSchema
-  // Augment rule type with an entities field that should be present in BIDS 1.10.1+
-  type ruleType = MetaSchema['meta']['associations'][keyof MetaSchema['meta']['associations']]
-  type AugmentedRuleType = ruleType & {
-    target: ruleType['target'] & { entities?: string[] }
-  }
 
   Object.assign(context, expressionFunctions)
   // @ts-expect-error
   context.exists.bind(context)
 
-  for (const key of Object.keys(schema.meta.associations)) {
-    const rule = schema.meta.associations[key] as AugmentedRuleType
+  for (const [key, rule] of Object.entries(schema.meta.associations)) {
     if (!rule.selectors!.every((x) => evalCheck(x, context))) {
       continue
     }

src/schema/context.ts

@@ -17,7 +17,7 @@ import { ColumnsMap } from '../types/columns.ts'
 import { readEntities } from './entities.ts'
 import { findDatatype } from './datatypes.ts'
 import { DatasetIssues } from '../issues/datasetIssues.ts'
-import { walkBack } from '../files/inheritance.ts'
+import { readSidecars } from '../files/inheritance.ts'
 import { parseGzip } from '../files/gzip.ts'
 import { loadTSV, loadTSVGZ } from '../files/tsv.ts'
 import { parseTIFF } from '../files/tiff.ts'
@@ -201,29 +201,18 @@ export class BIDSContext implements Context {
     if (this.extension === '.json') {
       return
     }
-    let sidecars: BIDSFile[] = []
+    let sidecars: Map<string, Record<string, unknown>>
     try {
-      sidecars = [...walkBack(this.file)]
-    } catch (error) {
-      if (
-        error && typeof error === 'object' && 'code' in error &&
-        error.code === 'MULTIPLE_INHERITABLE_FILES'
-      ) {
-        // @ts-expect-error
+      sidecars = await readSidecars(this.file)
+    } catch (error: any) {
+      if (error?.code) {
         this.dataset.issues.add(error)
+        return
       } else {
         throw error
       }
     }
-    for (const file of sidecars) {
-      const json = await loadJSON(file).catch((error): Record<string, unknown> => {
-        if (error.key) {
-          this.dataset.issues.add({ code: error.key, location: file.path })
-          return {}
-        } else {
-          throw error
-        }
-      })
+    for (const [path, json] of sidecars.entries()) {
       const overrides = Object.keys(this.sidecar).filter((x) => Object.hasOwn(json, x))
       for (const key of overrides) {
         if (json[key] !== this.sidecar[key]) {
@@ -232,14 +221,14 @@ export class BIDSContext implements Context {
             code: 'SIDECAR_FIELD_OVERRIDE',
             subCode: key,
             location: overrideLocation,
-            issueMessage: `Sidecar key defined in ${file.path} overrides previous value (${
+            issueMessage: `Sidecar key defined in ${path} overrides previous value (${
               json[key]
             }) from ${overrideLocation}`,
           })
         }
       }
       this.sidecar = { ...json, ...this.sidecar }
-      Object.keys(json).map((x) => this.sidecarKeyOrigin[x] ??= file.path)
+      Object.keys(json).map((x) => this.sidecarKeyOrigin[x] ??= path)
     }
     // Hack: round RepetitionTime to 3 decimal places; schema should add rounding function
     if (typeof this.sidecar.RepetitionTime === 'number') {

src/schema/fixtures.test.ts

@@ -18,9 +18,15 @@ function readBytes(json: string) {
 export const rootFileTree = pathsToTree([
   '/dataset_description.json',
   '/T1w.json',
+  '/task-movie_events.json',
+  '/task-movie_physio.json',
   '/sub-01/ses-01_T1w.json',
   '/sub-01/ses-01/anat/sub-01_ses-01_T1w.nii.gz',
   '/sub-01/ses-01/anat/sub-01_ses-01_T1w.json',
+  '/sub-01/ses-01/func/sub-01_ses-01_task-movie_bold.nii.gz',
+  '/sub-01/ses-01/func/sub-01_ses-01_task-movie_bold.nii.gz',
+  '/sub-01/ses-01/func/sub-01_ses-01_task-movie_events.tsv',
+  '/sub-01/ses-01/func/sub-01_ses-01_task-movie_physio.tsv.gz',
   ...[...Array(10).keys()].map((i) => `/stimuli/stimfile${i}.png`),
 ])
 
@@ -35,5 +41,24 @@ const anatFileTree = subjectFileTree.directories[0].directories[0] as FileTree
 
 export const dataFile = anatFileTree.get('sub-01_ses-01_T1w.nii.gz') as BIDSFile
 const anatJSONFile = anatFileTree.get('sub-01_ses-01_T1w.json') as BIDSFile
-anatJSONFile.readBytes = (size: number) =>
-  Promise.resolve(new TextEncoder().encode(anatJson) as Uint8Array<ArrayBuffer>)
+anatJSONFile.readBytes = readBytes(anatJson)
+
+const eventsSidecar = rootFileTree.get('task-movie_events.json') as BIDSFile
+eventsSidecar.readBytes = readBytes(
+  JSON.stringify({
+    StimulusPresentation: {
+      ScreenDistance: 1.8,
+      ScreenOrigin: ['top', 'left'],
+      ScreenResolution: [1920, 1080],
+      ScreenSize: [0.472, 0.265],
+    },
+  }),
+)
+const physioSidecar = rootFileTree.get('task-movie_physio.json') as BIDSFile
+physioSidecar.readBytes = readBytes(
+  JSON.stringify({
+    SamplingFrequency: 100,
+    StartTime: 0,
+    PhysioType: 'eyetrack',
+  }),
+)