batch plotting multi v0

.gitignore

@@ -126,10 +126,5 @@ settings.json
 /examples/cellpy batch utility/data/cellpyfiles/*
 /examples/cellpy batch utility/out/*
 /docs/jupyter_execute/
-/cellpy/utils/data/20160805_test001_45_cc.h5
-/cellpy/utils/data/20160805_test001_45_cc_01.res
-/cellpy/utils/data/20180418_sf033_4_cc.h5
-/cellpy/utils/data/20231115_rate_cc.h5
-/cellpy/utils/data/aux_multi_x.res
-/cellpy/utils/data/pec.csv
+/cellpy/utils/data/*
 /import.cvs

appveyor.yml

@@ -7,6 +7,7 @@ environment:
     - PYTHON_VERSION: 3.10
     - PYTHON_VERSION: 3.11
     - PYTHON_VERSION: 3.12
+    - PYTHON_VERSION: 3.13
 
 init:
   - "ECHO %PYTHON_VERSION% %MINICONDA% %PYTHON_ARC%"

cellpy/_version.py

@@ -1 +1 @@
-__version__ = "1.0.3a1"
+__version__ = "1.0.3a3"

cellpy/cli.py

@@ -633,7 +633,7 @@ def _check_import_pyodbc():
         elif ODBC == "pypyodbc":
             click.echo(" you stated that you prefer the pypyodbc loader")
             try:
-                import pypyodbc as dbloader
+                import pypyodbc as dbloader # type: ignore
             except ImportError:
                 click.echo(" Failed! Could not import it.")
                 click.echo(" try 'pip install pypyodbc'")
@@ -1271,7 +1271,7 @@ def _run_journals(folder_name, debug, silent, raw, cellpyfile, minimal):
 
 def _run_project(our_new_project, **kwargs):
     try:
-        import papermill as pm
+        import papermill as pm # type: ignore
     except ImportError:
         click.echo(
             "[cellpy]: You need to install papermill for automatically execute the notebooks."
@@ -1738,20 +1738,22 @@ def _new(
             else:
                 selected_project_dir = None
                 click.echo(f"Select another directory instead")
-
+    CREATE_NEW_DIR = "Create new project..."
     if not selected_project_dir:
         project_dirs = [
             d.name
             for d in directory.iterdir()
             if d.is_dir() and not d.name.startswith(".")
         ]
-        project_dirs.insert(0, "[create new dir]")
+        print(f"project_dirs: {project_dirs}")
+        project_dirs.insert(0, CREATE_NEW_DIR)
+        print(f"project_dirs: {project_dirs}")
 
         project_dir = cookiecutter.prompt.read_user_choice(
             "project folder", project_dirs
         )
 
-        if project_dir == "[create new dir]":
+        if project_dir == CREATE_NEW_DIR:
             default_name = "cellpy_project"
             temp_default_name = default_name
             for j in range(999):
@@ -1814,7 +1816,7 @@ def _new(
         click.echo("WARNING - experimental feature - use at your own risk")
         input("Press Enter to continue...")
         try:
-            import papermill as pm
+            import papermill as pm # type: ignore
         except ImportError:
             click.echo(
                 "[cellpy]: You need to install papermill for automatically execute the notebooks."

cellpy/internals/otherpath.py

@@ -143,6 +143,7 @@ def __init__(self, *args, **kwargs):
             path_string = "."
         else:
             path_string = self.__original
+
         self._original = self.__original
         self._check_external(path_string)
         # pathlib.PurePath and Path for Python 3.12 seems to have an __init__ method
@@ -151,7 +152,6 @@ def __init__(self, *args, **kwargs):
         # does not have a self._raw_path attribute).
         # Instead of running e.g. super().__init__(self._raw_other_path) we do this
         # instead (which is what the __init__ method does in Python 3.12):
-        self._raw_path = self._raw_other_path
         super().__init__(self._raw_other_path, *args)
         self.__doc__ += f"\nOriginal documentation:\n\n{self._pathlib_doc}"
 
@@ -1211,7 +1211,7 @@ def _glob_with_fabric(
 
 
 def _new_other_path_version():
-    python_version = sys.version_info
+    python_version = (sys.version_info.major, sys.version_info.minor)
     return python_version >= NEW_OTHER_PATH_VERSION
 
 

cellpy/readers/dbreader.py

@@ -88,7 +88,6 @@ def __init__(
         self.headers = self.db_sheet_cols.headers
 
         if db_frame is not None:
-            print("Using frame instead of file")
             self.table = db_frame.copy()
         else:
             self.skiprows, self.nrows = self._find_out_what_rows_to_skip()
@@ -98,6 +97,7 @@ def __init__(
 
         if batch:
             self.selected_batch = self.select_batch(batch, batch_col_name=batch_col_name)
+
         logging.debug("got table")
 
     def __str__(self):
@@ -144,6 +144,7 @@ def select_batch(
     def _select_batch(self, batch, batch_col_name=None, case_sensitive=True, drop=True, clean=False):
         if not batch_col_name:
             batch_col_name = self.db_sheet_cols.batch
+
         logging.debug("selecting batch - %s" % batch)
         sheet = self.table
         identity = self.db_sheet_cols.id
@@ -164,7 +165,8 @@ def _select_batch(self, batch, batch_col_name=None, case_sensitive=True, drop=Tr
                 sheet.drop_duplicates(inplace=True)
                 sheet.dropna(inplace=True)
                 return sheet.values.astype(int)
-        return sheet.loc[:, identity].values.astype(int)
+        out = sheet.loc[:, identity].values.astype(int)
+        return out
 
     def from_batch(
         self,

cellpy/readers/instruments/biologics_mpr.py

@@ -66,7 +66,7 @@ def _read_modules(fileobj):
     module_magic = fileobj.read(len(b"MODULE"))
     logging.debug(f"-")
     hdr_bytes = fileobj.read(hdr_dtype.itemsize)
-    hdr = np.fromstring(hdr_bytes, dtype=hdr_dtype, count=1)
+    hdr = np.frombuffer(hdr_bytes, dtype=hdr_dtype, count=1)
     hdr_dict = dict(((n, hdr[n][0]) for n in hdr_dtype.names))
     hdr_dict["offset"] = fileobj.tell()
     hdr_dict["data"] = fileobj.read(hdr_dict["length"])
@@ -346,16 +346,16 @@ def _load_mpr_data(self, filename, bad_steps):
             raise IOError("No data module!")
 
         data_version = data_module["version"]
-        n_data_points = np.fromstring(data_module["data"][:4], dtype="<u4")
+        n_data_points = np.frombuffer(data_module["data"][:4], dtype="<u4")
         n_data_points = n_data_points[0]
-        n_columns = np.fromstring(data_module["data"][4:5], dtype="u1")
+        n_columns = np.frombuffer(data_module["data"][4:5], dtype="u1")
         n_columns = n_columns[0]
 
         logging.debug(f"data (points, cols): {n_data_points}, {n_columns}")
 
         if data_version == 0:
             logging.debug("data version 0")
-            column_types = np.fromstring(
+            column_types = np.frombuffer(
                 data_module["data"][5:], dtype="u1", count=n_columns
             )
 
@@ -364,7 +364,7 @@ def _load_mpr_data(self, filename, bad_steps):
 
         elif data_version == 2:
             logging.debug("data version 2")
-            column_types = np.fromstring(
+            column_types = np.frombuffer(
                 data_module["data"][5:], dtype="<u2", count=n_columns
             )
             main_data = data_module["data"][405:]
@@ -397,7 +397,7 @@ def _load_mpr_data(self, filename, bad_steps):
                 f"WARNING! You have defined {p} bytes, but it seems it should be [{len(main_data) / n_data_points}]"
             )
         bulk = main_data
-        bulk_data = np.fromstring(bulk, dtype=dtype)
+        bulk_data = np.frombuffer(bulk, dtype=dtype)
         mpr_data = pd.DataFrame(bulk_data)
 
         # ------------- log  -----------------------------------

cellpy/utils/batch_tools/batch_journals.py

@@ -201,18 +201,25 @@ def from_db(self, project=None, name=None, batch_col=None, dbreader_kwargs=None,
             dbreader_kwargs = {}
 
         logging.debug(f"batch_name, batch_col, dbreader_kwargs: {name}, {batch_col}, {dbreader_kwargs}")
-
         if self.db_reader is not None:
             if isinstance(self.db_reader, dbreader.Reader):  # Simple excel-db
                 id_keys = self.db_reader.select_batch(name, batch_col, **dbreader_kwargs)
+
+                # Check for duplicates in id_keys
+                if len(id_keys) != len(set(id_keys)):
+                    duplicates = [x for x in id_keys if id_keys.count(x) > 1]
+                    unique_duplicates = list(set(duplicates))
+                    logging.warning(f"Found duplicate id_keys: {unique_duplicates}")
+                else:
+                    logging.debug("No duplicates found in id_keys")
                 logging.debug(f"id_keys: {id_keys}")
                 self.pages = self.engine(self.db_reader, id_keys, **kwargs)
             else:
                 logging.debug("creating journal pages using advanced reader methods (not simple excel-db)")
                 self.pages = self.engine(self.db_reader, batch_name=name, **kwargs)
 
             if self.pages.empty:
-                logging.critical(f"EMPTY JOURNAL: are you sure you have provided correct input to batch?")
+                logging.critical("EMPTY JOURNAL: are you sure you have provided correct input to batch?")
                 logging.critical(f"name: {name}")
                 logging.critical(f"project: {self.project}")
                 logging.critical(f"batch_col: {batch_col}")

cellpy/utils/batch_tools/engines.py

@@ -192,30 +192,29 @@ def simple_db_engine(
         logging.debug("No reader provided. Creating one myself.")
 
     if cell_ids is None:
+        logging.debug("cell_ids is None")
         pages_dict = reader.from_batch(
             batch_name=batch_name,
             include_key=include_key,
             include_individual_arguments=include_individual_arguments,
         )
+        logging.debug("pages_dict: {pages_dict}")
 
     else:
+        logging.debug("cell_ids is not None")
         pages_dict = dict()
         # TODO: rename this to "cell" or "cell_id" or something similar:
         pages_dict[hdr_journal["filename"]] = _query(reader.get_cell_name, cell_ids)
         if include_key:
-
             pages_dict[hdr_journal["id_key"]] = cell_ids
-
         if include_individual_arguments:
             pages_dict[hdr_journal["argument"]] = _query(reader.get_args, cell_ids)
-
         pages_dict[hdr_journal["mass"]] = _query(reader.get_mass, cell_ids)
         pages_dict[hdr_journal["total_mass"]] = _query(reader.get_total_mass, cell_ids)
         try:
             pages_dict[hdr_journal["nom_cap_specifics"]] = _query(reader.get_nom_cap_specifics, cell_ids)
         except Exception as e:
             pages_dict[hdr_journal["nom_cap_specifics"]] = "gravimetric"
-
         try:
             # updated 06.01.2025: some old db files returns None for file_name_indicator
             _file_name_indicator = _query(reader.get_file_name_indicator, cell_ids)
@@ -227,18 +226,28 @@ def simple_db_engine(
                 hdr_journal["filename"]
             ]  # TODO: use of "filename"!
 
-        pages_dict[hdr_journal["loading"]] = _query(reader.get_loading, cell_ids)
-        pages_dict[hdr_journal["nom_cap"]] = _query(reader.get_nom_cap, cell_ids)
-        pages_dict[hdr_journal["area"]] = _query(reader.get_area, cell_ids)
-        pages_dict[hdr_journal["experiment"]] = _query(reader.get_experiment_type, cell_ids)
-        pages_dict[hdr_journal["fixed"]] = _query(reader.inspect_hd5f_fixed, cell_ids)
-        pages_dict[hdr_journal["label"]] = _query(reader.get_label, cell_ids)
-        pages_dict[hdr_journal["cell_type"]] = _query(reader.get_cell_type, cell_ids)
-        pages_dict[hdr_journal["instrument"]] = _query(reader.get_instrument, cell_ids)
+        journal_fields = [
+            ("loading", reader.get_loading),
+            ("nom_cap", reader.get_nom_cap),
+            ("area", reader.get_area),
+            ("experiment", reader.get_experiment_type),
+            ("fixed", reader.inspect_hd5f_fixed),
+            ("label", reader.get_label),
+            ("cell_type", reader.get_cell_type),
+            ("instrument", reader.get_instrument),
+            ("comment", reader.get_comment),
+            ("group", reader.get_group),
+        ]
+
+        for field_name, reader_method in journal_fields:
+            try:
+                pages_dict[hdr_journal[field_name]] = _query(reader_method, cell_ids)
+            except Exception as e:
+                logging.debug(f"Error in getting {field_name}: {e}")
+
         pages_dict[hdr_journal["raw_file_names"]] = []
         pages_dict[hdr_journal["cellpy_file_name"]] = []
-        pages_dict[hdr_journal["comment"]] = _query(reader.get_comment, cell_ids)
-        pages_dict[hdr_journal["group"]] = _query(reader.get_group, cell_ids)
+
         if additional_column_names is not None:
             for k in additional_column_names:
                 try:
@@ -250,16 +259,19 @@ def simple_db_engine(
         del reader
 
     for key in list(pages_dict.keys()):
-        logging.debug("%s: %s" % (key, str(pages_dict[key])))
+        logging.debug(f"[length: {len(pages_dict[key]):04d}] {key}: {str(pages_dict[key])}")
 
     _groups = pages_dict[hdr_journal["group"]]
     groups = helper.fix_groups(_groups)
     pages_dict[hdr_journal["group"]] = groups
     my_timer_start = time.time()
+    logging.debug("finding files")
     pages_dict = helper.find_files(pages_dict, file_list=file_list, pre_path=pre_path, **kwargs)
+    logging.debug("files found")
+    logging.debug(f"pages_dict: {pages_dict}")
     my_timer_end = time.time()
     if (my_timer_end - my_timer_start) > 5.0:
-        logging.critical(
+        logging.debug(
             "The function _find_files was very slow. "
             "Save your journal so you don't have to run it again! "
             "You can load it again using the from_journal(journal_name) method."
@@ -294,15 +306,30 @@ def simple_db_engine(
         # TODO: check if drop=False works [#index]
         pages.set_index(hdr_journal["filename"], inplace=True)  # edit this to allow for
         # non-numeric index-names (for tab completion and python-box)
+    _check_pages_frame(pages)
     return pages
 
 
+def _check_pages_frame(pages):
+    logging.debug(f"pages.columns: {pages.columns}")
+    logging.debug(f"pages.index: {pages.index}")
+    logging.debug(f"pages.index.unique(): {pages.index.unique()}")
+    logging.debug(f"pages.dtypes: {pages.dtypes}")
+    duplicates = pages.index.duplicated()
+    if duplicates.any():
+        logging.critical(f"Oh no! Found {duplicates.sum()} duplicate cell names in your db - this is not allowed!")
+        logging.critical(f"Duplicate cell names: {pages.index[duplicates].tolist()}")
+    else:
+        logging.debug("No duplicate indices found")
+    logging.debug(f"pages.shape: {pages.shape}")
+
+
 def _report_suspected_duplicate_id(e, what="do it", on=None):
     logging.warning(f"could not {what}")
     logging.warning(f"{on}")
     logging.warning("maybe you have a corrupted db?")
     logging.warning(
         "typically happens if the cell_id is not unique (several rows or records in "
-        "your db has the same cell_id or key)"
+        "your db has the same cell_id or key) or if you have non-unique cell names"
     )
     logging.warning(e)