feat: algorithm validators (#123)

Author: c-dilks

.github/workflows/ci.yml

@@ -47,10 +47,6 @@ defaults:
 env:
   hipo_version: f40da676bbd1745398e9fdf233ff213ff98798f1
   num_events: 1000
-  # sanitizer options; NOTE: suppression path assumes the build directory is a subdirectory of the top-level repo directory
-  UBSAN_OPTIONS: halt_on_error=1:abort_on_error=1:print_summary=1:print_stacktrace=1:suppressions=../iguana_src/.github/UBSAN.supp
-  ASAN_OPTIONS: halt_on_error=1:abort_on_error=1:print_summary=1
-
 
 jobs:
 
@@ -364,9 +360,10 @@ jobs:
         run: meson setup build-iguana iguana_src --wipe
       - name: meson configure
         run: |
-          meson configure                           \
-            -Dtest_data_file=$(pwd)/test_data.hipo  \
-            -Dtest_num_events=${{ env.num_events }} \
+          meson configure                               \
+            -Dtest_data_file=$(pwd)/test_data.hipo      \
+            -Dtest_num_events=${{ env.num_events }}     \
+            -Dtest_output_dir=$(pwd)/validation_results \
             build-iguana
           case "${{ matrix.mode }}" in
             coverage)
@@ -417,6 +414,12 @@ jobs:
           name: coverage-report
           retention-days: 7
           path: coverage-report
+      - uses: actions/upload-artifact@v4
+        if: ${{ matrix.mode == 'coverage' }} # select one job-matrix element, since we only need one copy of this artifact
+        with:
+          name: _validation_results
+          retention-days: 7
+          path: validation_results
 
   # run examples
   # - a bit redundant, with `meson test` from `test_iguana`, but this ensures
@@ -522,7 +525,7 @@ jobs:
     env:
       CC: gcc
       CXX: g++
-      build_id: cpp-gcc-release
+      build_id: cpp-gcc-release-noROOT # FIXME: ROOT libs sporadically can't be found on some runners; for now use `noROOT` build
     steps:
       ### dependencies and test data
       - uses: actions/checkout@v4

.gitignore

@@ -9,3 +9,5 @@
 
 # data files
 *.hipo
+*.root
+*.png

README.md

@@ -6,6 +6,7 @@
 1. [Design Notes](doc/design.md)
 1. [API documentation](https://jeffersonlab.github.io/iguana/doxygen)
 1. [Developing a new Algorithm](src/iguana/algorithms/example/README.md)
+1. [Algorithm Tests and Validators](doc/testing.md)
 1. [Repository Maintenance](doc/maintenance.md)
 
 ## Status

bind/python/iguana-example-00-basic.py

@@ -13,7 +13,12 @@
 numEvents = int(sys.argv[2]) if len(sys.argv)>2 else 3
 
 reader = hipo.reader(inFile)
-banks  = reader.getBanks(["REC::Particle", "RUN::config"]);
+banks  = reader.getBanks([
+    "RUN::config",
+    "REC::Particle",
+    "REC::Calorimeter",
+    "REC::Track",
+    "REC::Scintillator"])
 
 seq = iguana.AlgorithmSequence('pyiguana')
 seq.Add('clas12::EventBuilderFilter')
@@ -33,8 +38,8 @@ def prettyPrint(message, bank):
 seq.Start(banks)
 while(reader.next(banks) and (numEvents==0 or iEvent < numEvents)):
     iEvent += 1
-    prettyPrint("BEFORE", banks[0])
+    prettyPrint("BEFORE", banks[1])
     seq.Run(banks)
-    prettyPrint("AFTER", banks[0])
+    prettyPrint("AFTER", banks[1])
 
 seq.Stop()

bind/python/iguana-example-01-bank-rows.py

@@ -42,7 +42,9 @@
         pid = particleBank.getInt('pid', row)
         if(algo_eventbuilder_filter.Filter(pid)):
 
-            sector = 1 # FIXME: get the sector number
+            sector = 1 # FIXME: get the sector number. The algorithm `clas12::SectorFinder` can do this, however
+                       # it requires reading full `hipo::bank` objects, whereas this example is meant to demonstrate
+                       # `iguana` usage operating _only_ on bank row elements
 
             px, py, pz, = algo_momentum_correction.Transform(
                     particleBank.getFloat("px", row),

doc/testing.md

@@ -0,0 +1,39 @@
+# Testing and Validating Algorithms
+
+## Running a Single Test
+
+The installed executable `iguana-test` is used for running single tests. Run
+it with no arguments for a usage guide. For example:
+```bash
+iguana/bin/iguana-test validator -f data.hipo -n 0 -a clas12::MomentumCorrectionValidator -o validation_plots
+```
+will run the validator `MomentumCorrectionValidator` and write its output to `./validation_plots/`, assuming:
+- your installation `prefix` is `./iguana`
+- your data are found in `./data.hipo`
+
+## Running All of the Tests
+
+While in your build directory, you may run
+```bash
+meson test
+```
+to run all available tests. However, depending on your build options, not all of them may run. For example,
+since the `validator` tests need input data, you need to set the build option `test_data_file` to a sample input
+file, otherwise tests which need input data will not run.
+
+For further usage of `meson test`, for example how to run a single test or control verbosity, run
+```bash
+meson test --help
+```
+> [!TIP]
+> - if you are testing on a _large_ data set, you may need to increase the timeout with the `-t` option
+>   and parallelize with the `-j` option
+> - if a test is failing, use `--print-errorlogs` to see the `stderr` stream and add `--no-stdsplit` if you need to see the `stdout` stream;
+>   if the streams' outputs are mixed up or not appearing, try prepending `stdbuf` as:
+>   ```bash
+>   stdbuf -o0 meson test [OPTIONS]...
+>   ```
+
+> [!NOTE]
+> All of the available tests run on the Continuous Integration (CI), so if you are developing and you submit a pull
+> requests, you may rely on the CI logs to check if the tests were successful

examples/iguana-example-00-basic.cc

@@ -20,9 +20,13 @@ int main(int argc, char** argv)
   hipo::reader reader(inFileName);
 
   // set banks
-  hipo::banklist banks = reader.getBanks({"REC::Particle", "RUN::config"});
-  enum banks_enum { b_particle,
-                    b_config }; // TODO: users shouldn't have to do this
+  hipo::banklist banks = reader.getBanks({"RUN::config",
+                                          "REC::Particle",
+                                          "REC::Calorimeter",
+                                          "REC::Track",
+                                          "REC::Scintillator"});
+  enum banks_enum { b_config,
+                    b_particle }; // TODO: users shouldn't have to do this
 
   // iguana algorithm sequence
   iguana::AlgorithmSequence seq;

examples/iguana-example-01-bank-rows.cc

@@ -49,7 +49,9 @@ int main(int argc, char** argv)
       auto pid = particleBank.getInt("pid", row);
       if(algo_eventbuilder_filter.Filter(pid)) {
 
-        int sector = 1; // FIXME: get the sector number
+        int sector = 1; // FIXME: get the sector number. The algorithm `clas12::SectorFinder` can do this, however
+                        // it requires reading full `hipo::bank` objects, whereas this example is meant to demonstrate
+                        // `iguana` usage operating _only_ on bank row elements
 
         // if accepted PID, correct its momentum
         auto [px, py, pz] = algo_momentum_correction.Transform(

examples/meson.build

@@ -27,6 +27,7 @@ foreach example, info : example_sources
     link_args:           ROOT_dep_link_args,
     install:             true,
     install_rpath:       ':'.join(project_exe_rpath),
+    build_rpath:         ROOT_dep_rpath,
   )
   if fs.is_file(get_option('test_data_file'))
     test(
@@ -36,6 +37,7 @@ foreach example, info : example_sources
         'test_args',
         [ get_option('test_data_file'), get_option('test_num_events') ]
       ),
+      env: project_test_env,
     )
   endif
 endforeach

meson.build

@@ -94,14 +94,25 @@ message('Dependency include dirs = [', ', '.join(dep_inc_paths), ']')
 # handle ROOT
 ROOT_dep_inc_dirs  = []
 ROOT_dep_link_args = []
+ROOT_dep_link_args_for_validators = []
+ROOT_dep_rpath = ''
 if ROOT_dep.found()
   ROOT_dep_inc_dirs += include_directories(run_command('root-config', '--incdir', check: true).stdout().strip())
+  ROOT_libdir = run_command('root-config', '--libdir', check: true).stdout().strip()
   ROOT_dep_link_args += [
-    '-L' + run_command('root-config', '--libdir',   check: true).stdout().strip(),
     # ROOT libraries that we need (safer than `root-config --libs`)
+    '-L' + ROOT_libdir,
     '-lCore',
     '-lGenVector',
   ]
+  ROOT_dep_link_args_for_validators = [
+    # additional ROOT libraries for validators (namely, graphics libraries)
+    '-L' + ROOT_libdir,
+    '-lRIO',
+    '-lHist',
+    '-lGpad',
+  ]
+  ROOT_dep_rpath = ROOT_libdir
 endif
 
 # general project vars
@@ -110,11 +121,32 @@ project_inc       = include_directories('src')
 project_libs      = []
 project_deps      = declare_dependency(dependencies: [ fmt_dep, yamlcpp_dep, hipo_dep ] ) # do NOT include ROOT here
 project_etc       = get_option('sysconfdir') / meson.project_name()
+project_test_env  = environment()
 project_pkg_vars  = [
   'dep_pkgconfigdirs=' + ':'.join(get_option('pkg_config_path')),
   'dep_libdirs='       + ':'.join(dep_lib_paths),
 ]
 
+# sanitizer settings
+project_test_env.set(
+  'UBSAN_OPTIONS',
+  'halt_on_error=1',
+  'abort_on_error=1',
+  'print_summary=1',
+  'print_stacktrace=1',
+  'suppressions=' + meson.project_source_root() / 'meson' / 'ubsan.supp',
+)
+project_test_env.set(
+  'ASAN_OPTIONS',
+  'halt_on_error=1',
+  'abort_on_error=1',
+  'print_summary=1',
+)
+project_test_env.set(
+  'LSAN_OPTIONS',
+  'suppressions=' + meson.project_source_root() / 'meson' / 'lsan.supp',
+)
+
 # executables' rpath
 project_exe_rpath = []
 if host_machine.system() != 'darwin'