Loaders emit per-test metadata, compact test_id, raw_units (#508)

HISTORY.md

@@ -2,6 +2,22 @@
 
 ## [Unreleased]
 
+* Loaders emit per-test metadata (#508, v2 Phase 2, themes V2-05/06/08):
+  `from_raw` now routes loader-parsed metadata (tester `test_ID`,
+  `channel_index`, `creator`, `schedule_file_name`) into `meta_test_dependent`
+  (so it persists and surfaces in `Data.tests`; the orphan attributes remain
+  set for backward compatibility), stamps the compact `test_id` grouping key
+  (0) onto raw (tester ids stay as provenance — note: this overwrites Arbin's
+  per-row tester Test_ID values in the raw column), and records load
+  provenance (`uuid` — new per load until #510 persists it, `source_kind`,
+  `source_type`, `source_uri`, `raw_file_names`, `loaded_datetime`) that the
+  derived `TestMeta` record now carries. Config-driven loaders stamp
+  instrument `raw_units` on the returned `Data` (shared
+  `internal_settings.merge_raw_units` helper). Arbin `Global_Table.Comments`
+  now maps to `meta_common.comment`; the full vendor-column mapping (incl.
+  deliberately dropped columns) is documented in the arbin_res module
+  docstring. Loader goldens regenerated accordingly.
+
 * Campaign merge (#507, v2 Phase 1-2, themes V2-03/V2-07): `CellpyCell.merge`
   is rewritten (old signature was dead code) — `merge(cells, mode="campaign")`
   folds different tests into one multi-test object: distinct compact `test_id`

cellpy/parameters/internal_settings.py

@@ -652,6 +652,22 @@ def get_default_raw_units(*args, **kwargs) -> CellpyUnits:
     )
 
 
+def merge_raw_units(loader_units) -> CellpyUnits:
+    """Overlay instrument-declared raw units onto the defaults (issue #508).
+
+    Single source for the merge used both by loaders (stamping
+    ``data.raw_units`` at load time) and by ``CellpyCell._set_raw_units``.
+    Unknown unit labels are logged and skipped.
+    """
+    raw_units = get_default_raw_units()
+    for key in loader_units or {}:
+        if key in raw_units:
+            raw_units[key] = loader_units[key]
+        else:
+            logging.debug(f"Got unconventional raw-unit label: {key}")
+    return raw_units
+
+
 def get_default_raw_limits() -> CellpyLimits:
     """Returns an augmented dictionary with units as default for raw data"""
     return CellpyLimits()

cellpy/readers/cellreader.py

@@ -23,6 +23,7 @@
 import sys
 import time
 import datetime
+import uuid
 import warnings
 from pathlib import Path
 from typing import Union, Sequence, List, Optional, Iterable, Any
@@ -57,6 +58,7 @@
     headers_step_table,
     headers_summary,
     get_default_raw_units,
+    merge_raw_units,
     get_default_output_units,
     CELLPY_FILE_VERSION,
     MINIMUM_CELLPY_FILE_VERSION,
@@ -101,6 +103,11 @@
 #     return Q(value, *args, **kwargs)
 
 
+# Instruments that read from a database rather than a file (used for the
+# provenance ``source_kind`` and for cellpy.get's db-path detection).
+DB_READER_INSTRUMENTS = ("arbin_sql", "arbin_sql_7")
+
+
 class CellpyCell:
     """Main class for working and storing data.
 
@@ -519,6 +526,7 @@ def vacant(cls, cell=None):
             new_cell.data.meta_test_dependent = cell.data.meta_test_dependent
             new_cell.data._extra_tests = dict(cell.data._extra_tests)
             new_cell.data._active_test_id = cell.data._active_test_id
+            new_cell.data._provenance = dict(cell.data._provenance)
 
             new_cell.data.raw_data_files = cell.data.raw_data_files
             new_cell.data.raw_data_files_length = cell.data.raw_data_files_length
@@ -748,14 +756,29 @@ def register_instrument_readers(self):
         #     self.instrument_factory.register_builder(instrument_id, instrument)
 
     def _set_raw_units(self):
-        raw_units = get_default_raw_units()
-        new_raw_units = self.loader_class.get_raw_units()
-        for key in new_raw_units:
-            if key in raw_units:
-                raw_units[key] = new_raw_units[key]
-            else:
-                logging.debug(f"Got unconventional raw-unit label: {key}")
-        return raw_units
+        return merge_raw_units(self.loader_class.get_raw_units())
+
+    @staticmethod
+    def _route_loader_meta_to_boxes(data):
+        """Route loader-set orphan attributes into the meta boxes (issue #508).
+
+        Loaders historically parked parsed metadata as plain attributes on
+        ``Data`` (never serialized, invisible to ``Data.tests``). Copy them
+        into ``meta_test_dependent`` so they persist and surface in the
+        derived ``TestMeta`` record. The orphan attributes stay set for
+        backward compatibility. ``test_name`` has no home in the metadata
+        model and remains orphan-only.
+        """
+        box = data.meta_test_dependent
+        for attr in ("test_ID", "channel_index", "creator", "schedule_file_name"):
+            # normalize absent-ish values: raw-file backends differ by
+            # platform (Windows ODBC yields '' where Linux mdbtools yields
+            # NaN) - both mean "not provided"
+            value = test_meta._unwrap(getattr(data, attr, None))
+            if isinstance(value, str) and not value.strip():
+                value = None
+            if value is not None:
+                setattr(box, attr, value)
 
     def _set_instrument(self, instrument, **kwargs):
         self.loader_class = self.instrument_factory.create(instrument, **kwargs)
@@ -1423,6 +1446,24 @@ def from_raw(
 
         data.raw_units = self._set_raw_units()
 
+        # issue #508 (V2-05/06): finalize per-test metadata on the loaded object.
+        # - route loader-parsed orphan attributes into the meta boxes
+        # - stamp the compact test_id grouping key onto raw (0 for a single,
+        #   unmerged test; tester-assigned ids remain as provenance in
+        #   meta_test_dependent.test_ID) - must run after the loader's own
+        #   intra-file merging (e.g. arbin_res) and the continuation folds
+        # - record load provenance for the derived TestMeta record
+        self._route_loader_meta_to_boxes(data)
+        data.raw[self.headers_normal.test_id_txt] = data.active_test_id
+        data._provenance = {
+            "uuid": str(uuid.uuid4()),
+            "source_kind": "db" if self.tester in DB_READER_INSTRUMENTS else "file",
+            "source_type": self.tester,
+            "source_uri": str(data.loaded_from),
+            "raw_file_names": [f.name for f in data.raw_data_files],
+            "loaded_datetime": datetime.datetime.now().isoformat(),
+        }
+
         self.data = data
         self._invent_a_cell_name(self.file_names)  # TODO (v1.0.0): fix me
         self.last_uploaded_from = "raw"
@@ -5464,7 +5505,7 @@ def get(
 
     from cellpy import log
 
-    db_readers = ["arbin_sql", "arbin_sql_7"]
+    db_readers = list(DB_READER_INSTRUMENTS)
     instruments_with_colliding_file_suffix = ["arbin_sql_h5"]
 
     step_kwargs = step_kwargs or {}

cellpy/readers/data_structures.py

@@ -528,6 +528,11 @@ def __init__(self, **kwargs):
         # property (issue #506).
         self.meta_test_dependent = CellpyMetaIndividualTest()
         self._extra_tests: Dict[int, TestMeta] = {}
+        # Load provenance for the active test (issue #508): keys restricted to
+        # the core-only TestMeta fields (uuid, source_*, raw_file_names,
+        # loaded_datetime). Filled by CellpyCell.from_raw; merged into the
+        # derived TestMeta record; NOT persisted in cellpy-file v8 (#510).
+        self._provenance: Dict[str, Any] = {}
         # Compact per-test grouping key of the active test (0 = single,
         # unmerged; matches the engine's test_id convention). Note: the legacy
         # ``meta_test_dependent.test_ID`` is the *tester-assigned* id (e.g.

cellpy/readers/instruments/arbin_res.py

@@ -1,4 +1,31 @@
-"""arbin res-type data files"""
+"""arbin res-type data files.
+
+Vendor metadata mapping (issue #508, V2-08 — "no silent drops"): the Arbin
+``Global_Table`` columns and what cellpy does with them:
+
+===============================  ==============================================
+Global_Table column              destination
+===============================  ==============================================
+Channel_Index                    ``meta_test_dependent.channel_index``
+Creator                          ``meta_test_dependent.creator``
+Item_ID                          ``meta_test_dependent.test_ID`` (tester id;
+                                 provenance — the compact grouping key is
+                                 ``Data.active_test_id``)
+Schedule_File_Name               ``meta_test_dependent.schedule_file_name``
+Start_DateTime                   ``meta_common.start_datetime``
+Test_Name                        ``Data.test_name`` (orphan attribute; no home
+                                 in the metadata model yet)
+Comments                         ``meta_common.comment`` (when non-empty and
+                                 the box still holds its default)
+Test_ID                          raw ``test_id`` column during intra-file
+                                 multi-test stitching (``_merge``); overwritten
+                                 with the compact key after load
+Applications_Path,               deliberately dropped (tester housekeeping,
+Channel_Number, Channel_Type,    no metadata value)
+DAQ_Index, Log_* flags,
+Mapped_Aux_* columns
+===============================  ==============================================
+"""
 import cellpy.config as config
 
 import logging
@@ -1093,6 +1120,23 @@ def _init_data(self, file_name, global_data_df, test_no=None):
             self.arbin_headers_global.test_name_txt
         ].values[0]
 
+        # vendor metadata mining (issue #508, V2-08): Comments -> comment,
+        # only when non-empty and the box still holds its default.
+        try:
+            comments = selected_global_data_df[
+                self.arbin_headers_global.comments_txt
+            ].values[0]
+        except (KeyError, IndexError):
+            comments = None
+        # empty values differ by platform (Windows ODBC: '', Linux mdbtools:
+        # NaN) - only map a real, non-empty string
+        if (
+            isinstance(comments, str)
+            and comments.strip()
+            and data.meta_common.comment in (None, "")
+        ):
+            data.meta_common.comment = comments
+
         data.raw_data_files.append(self.fid)
         return data
 

cellpy/readers/instruments/base.py

@@ -18,7 +18,7 @@
 from cellpy.exceptions import WrongFileVersion
 import cellpy.internals.connections
 import cellpy.readers.data_structures as core
-from cellpy.parameters.internal_settings import headers_normal
+from cellpy.parameters.internal_settings import headers_normal, merge_raw_units
 from cellpy.readers.instruments.configurations import (
     ModelParameters,
     register_configuration_from_module,
@@ -602,6 +602,10 @@ def loader(self, name: Union[str, pathlib.Path], **kwargs: str) -> core.Data:
 
         data.raw = data_df
         data.raw_data_files_length.append(len(data_df))
+        # stamp instrument units by value so a Data obtained directly from the
+        # loader carries correct raw_units (issue #508); CellpyCell.from_raw
+        # re-applies the same merge (idempotent).
+        data.raw_units = merge_raw_units(self.get_raw_units())
         data.summary = (
             pd.DataFrame()
         )  # creating an empty frame - loading summary is not implemented

cellpy/readers/test_meta.py

@@ -90,6 +90,14 @@ def build_active_test_meta(data) -> TestMeta:
     cell, test = meta_mapping.legacy_meta_to_core(common, individual)
     test.cell = cell
     test.test_id = data.active_test_id
+    # Load provenance (issue #508): the core-only TestMeta fields have no
+    # legacy home, so they come from Data._provenance (filled at load time,
+    # not persisted in cellpy-file v8 - see #510).
+    provenance = getattr(data, "_provenance", None) or {}
+    for key in _CORE_ONLY_TEST_FIELDS - {"cell", "comment"}:
+        value = provenance.get(key)
+        if value is not None:
+            setattr(test, key, value)
     return test
 
 

tests/data/goldens/loader_arbin_res/meta.json

@@ -31,8 +31,8 @@
     "tot_mass": 1.0
   },
   "meta_test_dependent": {
-    "channel_index": null,
-    "creator": null,
+    "channel_index": 3,
+    "creator": "hfa",
     "cycle_mode": "anode",
     "test_ID": null,
     "test_type": null,

tests/data/goldens/loader_custom/metrics.json

@@ -1,6 +1,6 @@
 {
   "instrument": "custom",
-  "n_columns": 10,
+  "n_columns": 11,
   "n_rows": 2444,
   "source": "testdata/data/custom_data_001.csv",
   "suite": "loader_custom"

tests/data/goldens/loader_maccor_txt/metrics.json

@@ -1,6 +1,6 @@
 {
   "instrument": "maccor_txt",
-  "n_columns": 62,
+  "n_columns": 63,
   "n_rows": 6704,
   "source": "testdata/data/maccor_001.txt",
   "suite": "loader_maccor_txt"

tests/data/goldens/loader_neware_txt/metrics.json

@@ -1,6 +1,6 @@
 {
   "instrument": "neware_txt",
-  "n_columns": 26,
+  "n_columns": 27,
   "n_rows": 9065,
   "source": "testdata/data/neware_uio.csv",
   "suite": "loader_neware_txt"

tests/data/goldens/loader_pec_csv/meta.json

@@ -44,7 +44,7 @@
     "channel_index": null,
     "creator": null,
     "cycle_mode": "anode",
-    "test_ID": null,
+    "test_ID": "187",
     "test_type": null,
     "voltage_lim_high": 1.0,
     "voltage_lim_low": 0.0