[ENH] improve performance and testing (#618)

* update timing

* try speed up test by not indexing dependencies

* run on latest and oldest matlab

* [DATALAD] Recorded changes

* [DATALAD] Recorded changes

* cancel previous runs

* speed up some tests or functions

* run slow and fast tests in parallel

* windows env var fix

* try using globals

* fix

* add more slow test

* add doc

+bids/+internal/list_all_trial_types.m

@@ -54,6 +54,8 @@
 
   trial_type_list = {};
 
+  BIDS = bids.layout(BIDS, 'index_dependencies', false);
+
   event_files = bids.query(BIDS, 'data', ...
                            'suffix', 'events', ...
                            'extension', '.tsv', ...

+bids/+util/create_sessions_tsv.m

@@ -49,7 +49,8 @@
   %%
   output_filenames = {};
 
-  layout = bids.layout(layout_or_path, 'use_schema', use_schema);
+  layout = bids.layout(layout_or_path, 'use_schema', use_schema, ...
+                       'index_dependencies', false);
 
   sessions_list = bids.query(layout, 'sessions');
   if isempty(sessions_list) && use_schema

+bids/File.m

@@ -83,6 +83,7 @@
   % Modify metadata
   %
   % .. code-block:: matlab
+  %
   %   % Adding new value
   %   f = f.metadata_add('NewField', 'new value');
   %   f.metadata()

+bids/diagnostic.m

@@ -78,7 +78,8 @@
   parse(args, varargin{:});
 
   %%
-  BIDS = bids.layout(args.Results.BIDS, 'use_schema', args.Results.use_schema);
+  BIDS = bids.layout(args.Results.BIDS, 'use_schema', args.Results.use_schema, ...
+                     'index_dependencies', false);
 
   output_path = args.Results.output_path;
 

+bids/layout.m

@@ -26,8 +26,8 @@
   %                           any ``derivatives`` folder in the BIDS dataset.
   % :type  index_derivatives: logical
   %
-  % :param index_dependencies: if ``true`` this will index the dependencies
-  %                            (json files, associated TSV files for each file...)
+  % :param index_dependencies: if ``true`` this will index the explicit dependencies
+  %                            (with "IntendedFor" in json files)
   % :type  index_dependencies: logical
   %
   % :param filter: Can be used to index only a subset of the dataset.

.github/workflows/run_examples.yml

@@ -1,6 +1,11 @@
 ---
 name: tests_examples
 
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
 on:
   push:
     branches:

.github/workflows/run_tests_matlab.yml

@@ -1,6 +1,10 @@
 ---
 name: 'matlab: tests'
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
 on:
   push:
     branches:
@@ -19,8 +23,9 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        version: [R2021a, R2021b, R2022a, R2022b]
+        version: [R2021a, R2023a]
         os: [ubuntu-latest, macos-latest, windows-latest]
+        test: [slow, fast]
 
 
     runs-on: ${{matrix.os}}
@@ -57,7 +62,15 @@ jobs:
         git clone https://github.com/MOxUnit/MOxUnit.git --depth 1
         git clone https://github.com/MOcov/MOcov.git --depth 1
 
-    - name: Run commands
+    - name: Run slow tests
+      if: matrix.test == 'slow'
+      uses: matlab-actions/[email protected]
+      with:
+        command: run MOxUnit/MOxUnit/moxunit_set_path(); addpath(fullfile(pwd, 'MOcov', 'MOcov')); global SLOW; SLOW=true; addpath(getenv('GITHUB_WORKSPACE'));
+          success = run_tests(); assert(success);
+
+    - name: Run fast tests
+      if: matrix.test == 'fast'
       uses: matlab-actions/[email protected]
       with:
         command: run MOxUnit/MOxUnit/moxunit_set_path(); addpath(fullfile(pwd, 'MOcov', 'MOcov')); addpath(getenv('GITHUB_WORKSPACE')); success = run_tests();

.github/workflows/run_tests_octave.yml

@@ -1,6 +1,10 @@
 ---
 name: 'octave: tests'
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
 on:
   push:
     branches:
@@ -17,6 +21,11 @@ env:
 jobs:
   test:
 
+    strategy:
+      fail-fast: false
+      matrix:
+        test: [slow, fast]
+
     runs-on: ubuntu-22.04
 
     steps:
@@ -66,6 +75,13 @@ jobs:
         mkoctfile --mex jsonread.c jsmn.c -DJSMN_PARENT_LINKS
         octave $OCTFLAGS --eval "addpath(fullfile(pwd)); savepath();"
 
-    - name: Run unit tests Octave
+    - name: Run fast tests Octave
+      if: matrix.test == 'fast'
+      run: |
+        octave $OCTFLAGS --eval "success = run_tests(); assert(success);"
+
+    - name: Run slow tests Octave
+      if: matrix.test == 'slow'
       run: |
+        export SLOW=true
         octave $OCTFLAGS --eval "success = run_tests(); assert(success);"

.github/workflows/update_schema.yml

@@ -21,6 +21,10 @@ name: update schema
 # │ │ │ │ │
 # * * * * *
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
 on:
   push:
     branches:

.github/workflows/validate.yml

@@ -1,6 +1,10 @@
 ---
 name: validate
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
 on:
   push:
     branches: ['*']

docs/source/index.rst

@@ -24,6 +24,7 @@ To see how to install BIDS-Matlab, please check
    stats_model
    transformers
    schema
+   performance
    dev_doc
 
 

docs/source/performance.rst

@@ -0,0 +1,21 @@
+Performance
+***********
+
+bids-matlab's performance may vary
+depending on which options you choose for indexing with :func:`bids.layout`.
+
+Relying on the BIDS schema (``'use_schema', true``) to only include files that comply with it
+may slow down performance a bit.
+
+Index dependencies (``'index_dependencies', true``) to detect files
+with explicit dependencies between them
+(for example when a fieldmap mentions that is intended for a specific bold run)
+is much slower and you may consider setting this to ``false``
+when if you know that you have no such dependencies in your dataset.
+
+
+.. _fig_benchmark:
+.. figure::  images/timing.png
+   :align:   center
+
+   layout indexing performance depending on chosen options

run_tests.m

@@ -3,6 +3,8 @@
 
   % (C) Copyright 2021 BIDS-MATLAB developers
 
+  tic;
+
   fprintf('\nRunning tests\n');
 
   if nargin < 1
@@ -16,7 +18,12 @@
   addpath(fullfile(pwd, 'tests', 'utils'));
 
   folderToCover = fullfile(pwd, '+bids');
+
   testFolder = fullfile(pwd, 'tests');
+  if run_slow_test_only
+    fprintf('Running only slow tests\n');
+    testFolder = fullfile(pwd, 'tests', 'tests_slow');
+  end
 
   if with_coverage
     success = moxunit_runtests(testFolder, ...
@@ -35,4 +42,6 @@
 
   end
 
+  toc;
+
 end

tests/test_report.m

@@ -14,7 +14,7 @@ function test_report_asl()
 
   BIDS = fullfile(cfg.pth_bids_example, datasets);
 
-  BIDS = bids.layout(BIDS, 'use_schema', true);
+  BIDS = bids.layout(BIDS, 'use_schema', true, 'index_dependencies', false);
 
   filter.modality = 'perf';
 
@@ -46,7 +46,7 @@ function test_report_basic()
   for i = 1:numel(datasets)
 
     BIDS = fullfile(cfg.pth_bids_example, datasets{i});
-    BIDS = bids.layout(BIDS, 'use_schema', true);
+    BIDS = bids.layout(BIDS, 'use_schema', true, 'index_dependencies', false);
 
     for j = 1:numel(modalities)
 
@@ -81,7 +81,7 @@ function test_report_pet()
 
   BIDS = fullfile(cfg.pth_bids_example, datasets);
 
-  BIDS = bids.layout(BIDS, 'use_schema', true);
+  BIDS = bids.layout(BIDS, 'use_schema', true, 'index_dependencies', false);
 
   filter.modality = 'pet';
 
@@ -120,7 +120,7 @@ function test_report_moae_data()
                               'verbose', true, ...
                               'delete_previous', false);
 
-  BIDS = bids.layout(pth, 'use_schema', true);
+  BIDS = bids.layout(pth, 'use_schema', true, 'index_dependencies', false);
 
   report = bids.report(BIDS, ...
                        'output_path', cfg.output_path, ...