From 00cd70381d74a126dc5e526aad7ab89c3823e186 Mon Sep 17 00:00:00 2001 From: "Christopher J. Markiewicz" Date: Fri, 12 Sep 2025 12:03:24 -0400 Subject: [PATCH 1/8] rf: Separate sidecar reading from accounting/error reporting --- src/files/inheritance.ts | 18 ++++++++++++++++++ src/schema/context.ts | 29 +++++++++-------------------- 2 files changed, 27 insertions(+), 20 deletions(-) diff --git a/src/files/inheritance.ts b/src/files/inheritance.ts index 5cf4f656..fbc4f38d 100644 --- a/src/files/inheritance.ts +++ b/src/files/inheritance.ts @@ -1,5 +1,6 @@ import type { BIDSFile, FileTree } from '../types/filetree.ts' import { readEntities } from '../schema/entities.ts' +import { loadJSON } from './json.ts' type Ret = T extends [string, ...string[]] ? (BIDSFile | BIDSFile[]) : BIDSFile @@ -77,3 +78,20 @@ export function* walkBack( fileTree = fileTree.parent } } + +export async function readSidecars( + source: BIDSFile, +): Promise>> { + const ret: Map> = new Map() + for (const file of walkBack(source)) { + try { + ret.set(file.path, await loadJSON(file)) + } catch (e: any) { + // Expect JSON parsing errors to be handled when the file is loaded directly + if (!e?.code) { + throw e + } + } + } + return ret +} diff --git a/src/schema/context.ts b/src/schema/context.ts index 92f9a51c..e9e58ba9 100644 --- a/src/schema/context.ts +++ b/src/schema/context.ts @@ -16,7 +16,7 @@ import { FileTree } from '../types/filetree.ts' import { ColumnsMap } from '../types/columns.ts' import { readEntities } from './entities.ts' import { DatasetIssues } from '../issues/datasetIssues.ts' -import { walkBack } from '../files/inheritance.ts' +import { readSidecars } from '../files/inheritance.ts' import { parseGzip } from '../files/gzip.ts' import { loadTSV, loadTSVGZ } from '../files/tsv.ts' import { parseTIFF } from '../files/tiff.ts' @@ -194,29 +194,18 @@ export class BIDSContext implements Context { if (this.extension === '.json') { return } - let sidecars: BIDSFile[] = [] + let sidecars: Map> try { - sidecars = [...walkBack(this.file)] - } catch (error) { - if ( - error && typeof error === 'object' && 'code' in error && - error.code === 'MULTIPLE_INHERITABLE_FILES' - ) { - // @ts-expect-error + sidecars = await readSidecars(this.file) + } catch (error: any) { + if (error?.code) { this.dataset.issues.add(error) + return } else { throw error } } - for (const file of sidecars) { - const json = await loadJSON(file).catch((error): Record => { - if (error.key) { - this.dataset.issues.add({ code: error.key, location: file.path }) - return {} - } else { - throw error - } - }) + for (const [path, json] of sidecars.entries()) { const overrides = Object.keys(this.sidecar).filter((x) => Object.hasOwn(json, x)) for (const key of overrides) { if (json[key] !== this.sidecar[key]) { @@ -225,14 +214,14 @@ export class BIDSContext implements Context { code: 'SIDECAR_FIELD_OVERRIDE', subCode: key, location: overrideLocation, - issueMessage: `Sidecar key defined in ${file.path} overrides previous value (${ + issueMessage: `Sidecar key defined in ${path} overrides previous value (${ json[key] }) from ${overrideLocation}`, }) } } this.sidecar = { ...json, ...this.sidecar } - Object.keys(json).map((x) => this.sidecarKeyOrigin[x] ??= file.path) + Object.keys(json).map((x) => this.sidecarKeyOrigin[x] ??= path) } // Hack: round RepetitionTime to 3 decimal places; schema should add rounding function if (typeof this.sidecar.RepetitionTime === 'number') { From 55c5591274c893ad0d4dee4ae447ed10d597a668 Mon Sep 17 00:00:00 2001 From: "Christopher J. Markiewicz" Date: Mon, 15 Sep 2025 11:17:21 -0400 Subject: [PATCH 2/8] feat: Add associations.events.sidecar --- src/schema/associations.ts | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/src/schema/associations.ts b/src/schema/associations.ts index 6849f6d8..507d6f47 100644 --- a/src/schema/associations.ts +++ b/src/schema/associations.ts @@ -5,16 +5,26 @@ import type { BIDSFile } from '../types/filetree.ts' import type { BIDSContext } from './context.ts' import { loadTSV } from '../files/tsv.ts' import { parseBvalBvec } from '../files/dwi.ts' -import { walkBack } from '../files/inheritance.ts' +import { readSidecars, walkBack } from '../files/inheritance.ts' import { evalCheck } from './applyRules.ts' import { expressionFunctions } from './expressionLanguage.ts' import { readText } from '../files/access.ts' +interface WithSidecar { + sidecar: Record +} + function defaultAssociation(file: BIDSFile, _options: any): Promise<{ path: string }> { return Promise.resolve({ path: file.path }) } +async function constructSidecar(file: BIDSFile): Promise> { + const sidecars = await readSidecars(file) + // Note ordering here gives precedence to the more specific sidecar + return sidecars.values().reduce((acc, json) => ({ ...json, ...acc }), {}) +} + /** * This object describes lookup functions for files associated to data files in a bids dataset. * For any given data file we iterate over the associations defined schema.meta.associations. @@ -24,7 +34,7 @@ function defaultAssociation(file: BIDSFile, _options: any): Promise<{ path: stri * Many associations only consist of a path; this object is for more complex associations. */ const associationLookup = { - events: async (file: BIDSFile, options: { maxRows: number }): Promise => { + events: async (file: BIDSFile, options: { maxRows: number }): Promise => { const columns = await loadTSV(file, options.maxRows) .catch((e) => { return new Map() @@ -32,6 +42,7 @@ const associationLookup = { return { path: file.path, onset: columns.get('onset') || [], + sidecar: await constructSidecar(file), } }, aslcontext: async ( From 99e98a5248811916ba41c81ec3854ae6666ec6d7 Mon Sep 17 00:00:00 2001 From: "Christopher J. Markiewicz" Date: Mon, 15 Sep 2025 11:21:01 -0400 Subject: [PATCH 3/8] test: Verify that events.sidecar gets loaded for physio.tsv.gz --- src/schema/associations.test.ts | 32 ++++++++++++++++++++++++++++++++ src/schema/fixtures.test.ts | 29 +++++++++++++++++++++++++++-- 2 files changed, 59 insertions(+), 2 deletions(-) create mode 100644 src/schema/associations.test.ts diff --git a/src/schema/associations.test.ts b/src/schema/associations.test.ts new file mode 100644 index 00000000..5d9661a0 --- /dev/null +++ b/src/schema/associations.test.ts @@ -0,0 +1,32 @@ +import { assertEquals, assertObjectMatch } from '@std/assert' +import type { BIDSFile, FileTree } from '../types/filetree.ts' +import { loadSchema } from '../setup/loadSchema.ts' +import { pathsToTree } from '../files/filetree.ts' +import { nullReadBytes } from '../tests/nullReadBytes.ts' +import { rootFileTree } from './fixtures.test.ts' +import { BIDSContext } from './context.ts' +import { buildAssociations } from './associations.ts' + +Deno.test('Test association loading', async (t) => { + const schema = await loadSchema() + await t.step('Load associations for events.tsv', async () => { + const eventsFile = rootFileTree.get( + 'sub-01/ses-01/func/sub-01_ses-01_task-movie_physio.tsv.gz', + ) as BIDSFile + const context = new BIDSContext(eventsFile, undefined, rootFileTree) + context.dataset.schema = schema + const associations = await buildAssociations(context) + assertObjectMatch(associations, { + events: { + sidecar: { + StimulusPresentation: { + ScreenDistance: 1.8, + ScreenOrigin: ['top', 'left'], + ScreenResolution: [1920, 1080], + ScreenSize: [0.472, 0.265], + }, + }, + }, + }) + }) +}) diff --git a/src/schema/fixtures.test.ts b/src/schema/fixtures.test.ts index a236dd66..12c44800 100644 --- a/src/schema/fixtures.test.ts +++ b/src/schema/fixtures.test.ts @@ -18,9 +18,15 @@ function readBytes(json: string) { export const rootFileTree = pathsToTree([ '/dataset_description.json', '/T1w.json', + '/task-movie_events.json', + '/task-movie_physio.json', '/sub-01/ses-01_T1w.json', '/sub-01/ses-01/anat/sub-01_ses-01_T1w.nii.gz', '/sub-01/ses-01/anat/sub-01_ses-01_T1w.json', + '/sub-01/ses-01/func/sub-01_ses-01_task-movie_bold.nii.gz', + '/sub-01/ses-01/func/sub-01_ses-01_task-movie_bold.nii.gz', + '/sub-01/ses-01/func/sub-01_ses-01_task-movie_events.tsv', + '/sub-01/ses-01/func/sub-01_ses-01_task-movie_physio.tsv.gz', ...[...Array(10).keys()].map((i) => `/stimuli/stimfile${i}.png`), ]) @@ -35,5 +41,24 @@ const anatFileTree = subjectFileTree.directories[0].directories[0] as FileTree export const dataFile = anatFileTree.get('sub-01_ses-01_T1w.nii.gz') as BIDSFile const anatJSONFile = anatFileTree.get('sub-01_ses-01_T1w.json') as BIDSFile -anatJSONFile.readBytes = (size: number) => - Promise.resolve(new TextEncoder().encode(anatJson) as Uint8Array) +anatJSONFile.readBytes = readBytes(anatJson) + +const eventsSidecar = rootFileTree.get('task-movie_events.json') as BIDSFile +eventsSidecar.readBytes = readBytes( + JSON.stringify({ + StimulusPresentation: { + ScreenDistance: 1.8, + ScreenOrigin: ['top', 'left'], + ScreenResolution: [1920, 1080], + ScreenSize: [0.472, 0.265], + }, + }), +) +const physioSidecar = rootFileTree.get('task-movie_physio.json') as BIDSFile +physioSidecar.readBytes = readBytes( + JSON.stringify({ + SamplingFrequency: 100, + StartTime: 0, + PhysioType: 'eyetrack', + }), +) From 6500e75fd9a1579be5414b0a28d5bbd0bf888392 Mon Sep 17 00:00:00 2001 From: "Christopher J. Markiewicz" Date: Mon, 15 Sep 2025 11:22:36 -0400 Subject: [PATCH 4/8] type: Remove outdated type casts --- src/schema/associations.ts | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/src/schema/associations.ts b/src/schema/associations.ts index 507d6f47..6ceb4e32 100644 --- a/src/schema/associations.ts +++ b/src/schema/associations.ts @@ -104,18 +104,12 @@ export async function buildAssociations( const associations: Associations = {} const schema: MetaSchema = context.dataset.schema as MetaSchema - // Augment rule type with an entities field that should be present in BIDS 1.10.1+ - type ruleType = MetaSchema['meta']['associations'][keyof MetaSchema['meta']['associations']] - type AugmentedRuleType = ruleType & { - target: ruleType['target'] & { entities?: string[] } - } Object.assign(context, expressionFunctions) // @ts-expect-error context.exists.bind(context) - for (const key of Object.keys(schema.meta.associations)) { - const rule = schema.meta.associations[key] as AugmentedRuleType + for (const [key, rule] of Object.entries(schema.meta.associations)) { if (!rule.selectors!.every((x) => evalCheck(x, context))) { continue } From b93fe1a781398d69c85d3c7f50d58ef35513f04c Mon Sep 17 00:00:00 2001 From: "Christopher J. Markiewicz" Date: Tue, 16 Sep 2025 16:44:35 -0400 Subject: [PATCH 5/8] chore: Track BEP020 examples --- tests/data/bids-examples | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/data/bids-examples b/tests/data/bids-examples index 7db0b97c..163e25f6 160000 --- a/tests/data/bids-examples +++ b/tests/data/bids-examples @@ -1 +1 @@ -Subproject commit 7db0b97ce1aa8cdbad8f66e7b6fed8ce1c40462c +Subproject commit 163e25f670d4e0512b63c8754daffa848a8240e9 From 227c1682250e44012733f2f25a583f41ecce0efb Mon Sep 17 00:00:00 2001 From: "Christopher J. Markiewicz" Date: Tue, 16 Sep 2025 17:04:04 -0400 Subject: [PATCH 6/8] fix: Restore physio sidecar loader --- src/schema/associations.ts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/schema/associations.ts b/src/schema/associations.ts index 6ceb4e32..3c014415 100644 --- a/src/schema/associations.ts +++ b/src/schema/associations.ts @@ -96,6 +96,12 @@ const associationLookup = { sampling_frequency: columns.get('sampling_frequency'), } }, + physio: async (file: BIDSFile, options: any): Promise<{path: string} & WithSidecar> => { + return { + path: file.path, + sidecar: await constructSidecar(file), + } + }, } export async function buildAssociations( From 52ae234e835b05f307cffb8ab9d3d42f83d5e7e7 Mon Sep 17 00:00:00 2001 From: "Christopher J. Markiewicz" Date: Tue, 16 Sep 2025 17:07:11 -0400 Subject: [PATCH 7/8] chore: Track BEP020 schema --- .github/workflows/deno_tests.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/deno_tests.yml b/.github/workflows/deno_tests.yml index 5ef541db..9595992b 100644 --- a/.github/workflows/deno_tests.yml +++ b/.github/workflows/deno_tests.yml @@ -52,11 +52,14 @@ jobs: strategy: matrix: os: [ubuntu-latest, macos-latest, windows-latest] - allow-net: [true, false] + # Resume disabled network tests when draft schema is not needed + allow-net: [true] #, false] fail-fast: false defaults: run: shell: bash + env: + BIDS_SCHEMA: https://bids-specification--1128.org.readthedocs.build/en/1128/schema.json steps: - uses: actions/checkout@v4 From 2592e30f9b136699147cb251d5aab5cdc133a7a6 Mon Sep 17 00:00:00 2001 From: "Christopher J. Markiewicz" Date: Tue, 16 Sep 2025 17:14:51 -0400 Subject: [PATCH 8/8] doc: Add changelog entry --- .../20250916_171340_markiewicz_bep020.md | 47 +++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100644 changelog.d/20250916_171340_markiewicz_bep020.md diff --git a/changelog.d/20250916_171340_markiewicz_bep020.md b/changelog.d/20250916_171340_markiewicz_bep020.md new file mode 100644 index 00000000..0d2ad2c1 --- /dev/null +++ b/changelog.d/20250916_171340_markiewicz_bep020.md @@ -0,0 +1,47 @@ + + +### Added + +- Support for `associations.physio` and `associations.events.sidecar`. + + + + + + +