From b31b240886641dc20bbc192e64ae0115cf6b2a4c Mon Sep 17 00:00:00 2001 From: jepegit Date: Mon, 9 Jun 2025 11:45:02 +0200 Subject: [PATCH 01/37] fix bug in OtherPathsNew --- cellpy/internals/otherpath.py | 4 ++-- requirements_dev.txt | 40 +++++------------------------------ 2 files changed, 7 insertions(+), 37 deletions(-) diff --git a/cellpy/internals/otherpath.py b/cellpy/internals/otherpath.py index 2e36b7e5..47faf5b1 100644 --- a/cellpy/internals/otherpath.py +++ b/cellpy/internals/otherpath.py @@ -143,6 +143,7 @@ def __init__(self, *args, **kwargs): path_string = "." else: path_string = self.__original + self._original = self.__original self._check_external(path_string) # pathlib.PurePath and Path for Python 3.12 seems to have an __init__ method @@ -151,7 +152,6 @@ def __init__(self, *args, **kwargs): # does not have a self._raw_path attribute). # Instead of running e.g. super().__init__(self._raw_other_path) we do this # instead (which is what the __init__ method does in Python 3.12): - self._raw_path = self._raw_other_path super().__init__(self._raw_other_path, *args) self.__doc__ += f"\nOriginal documentation:\n\n{self._pathlib_doc}" @@ -1211,7 +1211,7 @@ def _glob_with_fabric( def _new_other_path_version(): - python_version = sys.version_info + python_version = (sys.version_info.major, sys.version_info.minor) return python_version >= NEW_OTHER_PATH_VERSION diff --git a/requirements_dev.txt b/requirements_dev.txt index a6ab38fc..04ecf451 100644 --- a/requirements_dev.txt +++ b/requirements_dev.txt @@ -1,45 +1,15 @@ -PyGithub>=1.43 -Sphinx>=1.5.5 +PyGithub +Sphinx black build bumpver cookiecutter -coverage>=4.3.4 -cryptography>=1.8.1 -dateparser -decorator -fabric -flake8>=3.3.0 +coverage +flake8 invoke ipykernel -kaleido==0.1.* -lmfit -matplotlib>=1.5.3 nox -numpy>=2.0 -openpyxl -pandas pandoc -pint -pip>=9.0.1 -plotly pytest-benchmark pytest-timeout -pytest>=3.0.7 -python-box -python-dotenv -requests -requests -rich -ruamel.yaml -scipy -seaborn -sqlalchemy-access -sqlalchemy>=2.0.0 -tables -tqdm -twine>=1.9.1 -watchdog>=0.8.3 -wheel>=0.29.0 -xlrd -charset_normalizer +pytest From 0aa3ca8d409c3c12d80d3eb3b6013745cec218a8 Mon Sep 17 00:00:00 2001 From: jepegit Date: Mon, 9 Jun 2025 14:26:41 +0200 Subject: [PATCH 02/37] update to fromstring in biologics --- cellpy/readers/instruments/biologics_mpr.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/cellpy/readers/instruments/biologics_mpr.py b/cellpy/readers/instruments/biologics_mpr.py index f2f5b86b..0b70d6f3 100644 --- a/cellpy/readers/instruments/biologics_mpr.py +++ b/cellpy/readers/instruments/biologics_mpr.py @@ -66,7 +66,7 @@ def _read_modules(fileobj): module_magic = fileobj.read(len(b"MODULE")) logging.debug(f"-") hdr_bytes = fileobj.read(hdr_dtype.itemsize) - hdr = np.fromstring(hdr_bytes, dtype=hdr_dtype, count=1) + hdr = np.frombuffer(hdr_bytes, dtype=hdr_dtype, count=1) hdr_dict = dict(((n, hdr[n][0]) for n in hdr_dtype.names)) hdr_dict["offset"] = fileobj.tell() hdr_dict["data"] = fileobj.read(hdr_dict["length"]) @@ -346,16 +346,16 @@ def _load_mpr_data(self, filename, bad_steps): raise IOError("No data module!") data_version = data_module["version"] - n_data_points = np.fromstring(data_module["data"][:4], dtype=" Date: Mon, 9 Jun 2025 15:08:43 +0200 Subject: [PATCH 03/37] update .gitignore --- .gitignore | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/.gitignore b/.gitignore index 26672aec..5165f6b6 100644 --- a/.gitignore +++ b/.gitignore @@ -126,10 +126,5 @@ settings.json /examples/cellpy batch utility/data/cellpyfiles/* /examples/cellpy batch utility/out/* /docs/jupyter_execute/ -/cellpy/utils/data/20160805_test001_45_cc.h5 -/cellpy/utils/data/20160805_test001_45_cc_01.res -/cellpy/utils/data/20180418_sf033_4_cc.h5 -/cellpy/utils/data/20231115_rate_cc.h5 -/cellpy/utils/data/aux_multi_x.res -/cellpy/utils/data/pec.csv +/cellpy/utils/data/* /import.cvs From 5684d63832ebd7f62724372845a63f25e71c863b Mon Sep 17 00:00:00 2001 From: jepegit Date: Mon, 9 Jun 2025 19:08:59 +0200 Subject: [PATCH 04/37] enforce numpy 2 for actions --- appveyor.yml | 1 + github_actions_environment.yml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/appveyor.yml b/appveyor.yml index 8a0f2b15..bfd2b2ca 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -7,6 +7,7 @@ environment: - PYTHON_VERSION: 3.10 - PYTHON_VERSION: 3.11 - PYTHON_VERSION: 3.12 + - PYTHON_VERSION: 3.13 init: - "ECHO %PYTHON_VERSION% %MINICONDA% %PYTHON_ARC%" diff --git a/github_actions_environment.yml b/github_actions_environment.yml index edfe4629..abf233a1 100644 --- a/github_actions_environment.yml +++ b/github_actions_environment.yml @@ -15,7 +15,7 @@ dependencies: - lmfit - matplotlib - nbmake - - numpy + - numpy >= 2.0.0 - openpyxl - pandas - pint From 6e40e07b9b69e7c0f1c022d02215975eaff362d3 Mon Sep 17 00:00:00 2001 From: jepegit Date: Mon, 9 Jun 2025 19:10:57 +0200 Subject: [PATCH 05/37] numpy >= 2 --- github_actions_environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/github_actions_environment.yml b/github_actions_environment.yml index abf233a1..644643e3 100644 --- a/github_actions_environment.yml +++ b/github_actions_environment.yml @@ -15,7 +15,7 @@ dependencies: - lmfit - matplotlib - nbmake - - numpy >= 2.0.0 + - numpy >= 2 - openpyxl - pandas - pint From 7eb4eb2143feb4a1f85e1e046760b7b951f66525 Mon Sep 17 00:00:00 2001 From: jepegit Date: Mon, 9 Jun 2025 19:13:27 +0200 Subject: [PATCH 06/37] version spec without spaces --- github_actions_environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/github_actions_environment.yml b/github_actions_environment.yml index 644643e3..960ad652 100644 --- a/github_actions_environment.yml +++ b/github_actions_environment.yml @@ -15,7 +15,7 @@ dependencies: - lmfit - matplotlib - nbmake - - numpy >= 2 + - numpy>=2 - openpyxl - pandas - pint From 927a34478fa93960eb4720dbe3dce1fad4148c74 Mon Sep 17 00:00:00 2001 From: jepegit Date: Thu, 12 Jun 2025 16:37:53 +0200 Subject: [PATCH 07/37] summary plot now with fullcell standard --- cellpy/utils/plotutils.py | 215 ++++++++++++++++++++++++++++---------- local/notes.md | 12 +++ 2 files changed, 169 insertions(+), 58 deletions(-) diff --git a/cellpy/utils/plotutils.py b/cellpy/utils/plotutils.py index b308ee47..857fff17 100644 --- a/cellpy/utils/plotutils.py +++ b/cellpy/utils/plotutils.py @@ -11,6 +11,7 @@ import os import pickle as pkl import sys +from typing import Any, Callable import warnings from io import StringIO from pathlib import Path @@ -599,7 +600,41 @@ def create_col_info(c): hdr.coulombic_efficiency, ], ) - return x_columns, y_cols + + x_transformations = dict( + ) + + def _normalize_col(x: np.ndarray, normalization_factor: float = 1.0, normalization_type: str = "max", normalization_scaler: float = 1.0) -> np.ndarray: + if normalization_type == "divide": + return (x / normalization_factor) * normalization_scaler + elif normalization_type == "multiply": + return (x * normalization_factor) * normalization_scaler + elif normalization_type == "area": + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + area = np.trapzoid(x, dx=1) + return (x / area / normalization_factor) * normalization_scaler + elif normalization_type == "max": + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + x_max = x.max() + return (x / x_max / normalization_factor) * normalization_scaler + else: + raise ValueError(f"Invalid normalization type: {normalization_type}") + + y_transformations: dict[str, dict[str, Callable]] = dict( + fullcell_standard_gravimetric={ + hdr.cumulated_discharge_capacity_loss + "_gravimetric": _normalize_col, + }, + fullcell_standard_areal={ + hdr.cumulated_discharge_capacity_loss + "_areal": _normalize_col, + }, + fullcell_standard_absolute={ + hdr.cumulated_discharge_capacity_loss + "_absolute": _normalize_col, + }, + ) + + return x_columns, y_cols, x_transformations, y_transformations def create_label_dict(c): @@ -682,26 +717,26 @@ def summary_plot( seaborn_style: str = "dark", formation_cycles=3, show_formation=True, + x_axis_domain_formation_fraction=0.2, column_separator=0.01, reset_losses=True, link_capacity_scales=False, + fullcell_standard_normalization_type="divide", + fullcell_standard_normalization_factor=None, + fullcell_standard_normalization_scaler=1.0, **kwargs, ): """Create a summary plot. - Args: - c: cellpy object x: x-axis column (default: 'cycle_index') y: y-axis column or column set. Currently, the following predefined sets exists: - - - "voltages", "capacities_gravimetric", "capacities_areal", "capacities", - "capacities_gravimetric_split_constant_voltage", "capacities_areal_split_constant_voltage", - "capacities_gravimetric_coulombic_efficiency", "capacities_areal_coulombic_efficiency", - "capacities_absolute_coulombic_efficiency", - "fullcell_standard_gravimetric", "fullcell_standard_areal", "fullcell_standard_absolute", - + "voltages", "capacities_gravimetric", "capacities_areal", "capacities", + "capacities_gravimetric_split_constant_voltage", "capacities_areal_split_constant_voltage", + "capacities_gravimetric_coulombic_efficiency", "capacities_areal_coulombic_efficiency", + "capacities_absolute_coulombic_efficiency", + "fullcell_standard_gravimetric", "fullcell_standard_areal", "fullcell_standard_absolute", height: height of the plot (for plotly) width: width of the plot (for plotly) markers: use markers @@ -721,9 +756,20 @@ def summary_plot( seaborn_style: name of the seaborn style to use formation_cycles: number of formation cycles to show show_formation: show formation cycles + x_axis_domain_formation_fraction: fraction of the x-axis domain for the formation cycles (default: 0.2) column_separator: separation between columns when splitting the plot (only for plotly) reset_losses: reset the losses to the first cycle (only for fullcell_standard plots) link_capacity_scales: link the capacity scales (only for fullcell_standard plots) + fullcell_standard_normalization_type: normalization type for the fullcell standard plots (Cumulated loss) (divide, multiply, area, max, False) + if normalization_type is max, the normalization factor is set to the maximum value of the capacity column if not provided + if normalization_type is divide, the normalization is done by dividing by the normalization factor and then multiplying by the scaler + if normalization_type is multiply, the normalization is done by multiplying by the normalization factor and then multiplying by the scaler + if normalization_type is area, the normalization is done by dividing by the area and then multiplying by the scaler + if normalization_type is False, no normalization is done + fullcell_standard_normalization_factor: normalization factor for the fullcell standard plots + fullcell_standard_normalization_scaler: scaler for the fullcell standard plots + plotly_[update trace parameter]: additional parameters for the plotly traces + (e.g. use plotly_marker_size=10 for updating the marker_size to 10) **kwargs: includes additional parameters for the plotting backend (not properly documented yet). Returns: @@ -763,11 +809,11 @@ def summary_plot( if formation_cycles < 1: show_formation = False - x_cols, y_cols = create_col_info(c) + x_cols, y_cols, x_trans, y_trans = create_col_info(c) x_axis_labels, y_axis_label = create_label_dict(c) - def _auto_range(fig, axis_name_1, axis_name_2): + def _auto_range(fig: Any, axis_name_1: str, axis_name_2: str) -> list: min_y = np.inf max_y = -np.inf full_axis_name_1 = axis_name_1.replace("y", "yaxis") @@ -781,14 +827,16 @@ def _auto_range(fig, axis_name_1, axis_name_2): _range_2 = [np.inf, -np.inf] _range = [min(_range_1[0], _range_2[0]), max(_range_1[1], _range_2[1])] - for t in deepcopy(fig.data): + for i,t in enumerate(deepcopy(fig.data)): if t.yaxis in [axis_name_1, axis_name_2]: y = deepcopy(t.y) try: + y = np.array(y, dtype=float) min_y = np.ma.masked_invalid(y).min() max_y = np.ma.masked_invalid(y).max() except Exception as e: - warnings.warn(f"Could not calculate min and max for y-axis: {e}") + warnings.warn(f"Could not calculate min and max for y-axis (data set {i}): {e}") + _range = [min(_range[0], min_y), max(_range[1], max_y)] _range = [0.95 * _range[0], 1.05 * _range[1]] return _range @@ -807,6 +855,11 @@ def _auto_range(fig, axis_name_1, axis_name_2): width=width, ) + additional_kwargs_plotly_update_traces = dict() + for k in list(kwargs.keys()): + if k.startswith("plotly_"): + additional_kwargs_plotly_update_traces[k.replace("plotly_", "")] = kwargs.pop(k) + additional_kwargs_seaborn = dict() # standard fullcell plot (ce, discharge cap, loss, cv-only) if y.startswith("fullcell_standard_"): @@ -836,7 +889,6 @@ def _auto_range(fig, axis_name_1, axis_name_2): s.loc[s["variable"].str.contains(r"_cv$"), row] = 3 # cv data additional_kwargs_plotly["facet_row"] = row - if reset_losses: # Get the first value for each cumulated loss variable first_values = s[s["variable"].str.contains(r"cumulated.*loss")].groupby("variable")["value"].transform("first") @@ -844,6 +896,30 @@ def _auto_range(fig, axis_name_1, axis_name_2): mask = s["variable"].str.contains(r"cumulated.*loss") s.loc[mask, "value"] = s.loc[mask, "value"] - first_values + if fullcell_standard_normalization_type: + # set normalization factor + if fullcell_standard_normalization_factor is None: + if fullcell_standard_normalization_type == "max": + fullcell_standard_normalization_factor = s[s[row] == 1].max().value + fullcell_standard_normalization_type = "divide" + elif fullcell_standard_normalization_type == "area": + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + area = np.trapezoid(s[s[row] == 1].value, dx=1) + fullcell_standard_normalization_factor = area + fullcell_standard_normalization_type = "divide" + else: + fullcell_standard_normalization_factor = 1.0 + + trans_kwargs = dict( + normalization_factor=fullcell_standard_normalization_factor, + normalization_type=fullcell_standard_normalization_type, + normalization_scaler=fullcell_standard_normalization_scaler, + ) + + for col, trans in y_trans.get(y, {}).items(): + s.loc[s["variable"] == col, "value"] = trans(s.loc[s["variable"] == col, "value"].values, **trans_kwargs) + # filter on constant voltage vs constant current # Remark! absoulte capacities are not implemented yet. elif y.endswith("_split_constant_voltage"): @@ -922,14 +998,13 @@ def _auto_range(fig, axis_name_1, axis_name_2): }, **kwargs, ) - if y.startswith("fullcell_standard"): - print("Warning: Experimental feature - fullcell standard plot") - print(" - missing: does not work yet for show_formation=False") + + fig.update_traces(**additional_kwargs_plotly_update_traces) if show_formation: formation_header = 'Formation' - x_axis_domain_formation = [0.0, 0.2 - column_separator / 2] - x_axis_domain_rest = [0.2 + column_separator / 2, 0.95] + x_axis_domain_formation = [0.0, x_axis_domain_formation_fraction - column_separator / 2] + x_axis_domain_rest = [x_axis_domain_formation_fraction + column_separator / 2, 0.95] max_cycle_formation = s.loc[formation_cycle_selector, x].max() min_cycle_rest = s.loc[~formation_cycle_selector, x].min() if x == _hdr_summary.normalized_cycle_index: @@ -1028,9 +1103,13 @@ def _auto_range(fig, axis_name_1, axis_name_2): fig.update_yaxes(autorange=False) fig.update_layout(xaxis_domain=x_axis_domain_formation, scene_domain_x=x_axis_domain_formation) - range_1 = _auto_range(fig, "y", "y2") - range_2 = _auto_range(fig, "y3", "y4") - range_3 = _auto_range(fig, "y5", "y6") + if y.startswith("fullcell_standard_") and fullcell_standard_normalization_type: + range_2 = [0.0, fullcell_standard_normalization_scaler] + else: + range_2 = _auto_range(fig, "y3", "y4") + + range_1 = _auto_range(fig, "y", "y2") + range_3 = _auto_range(fig, "y5", "y6") range_4 = _auto_range(fig, "y7", "y8") fig.update_layout( @@ -1053,6 +1132,52 @@ def _auto_range(fig, axis_name_1, axis_name_2): annotations = [_plotly_label_dict(formation_header, 0.08, 1.0)] + 7 * [PLOTLY_BLANK_LABEL] fig.layout["annotations"] = annotations + if y.startswith("fullcell_standard_"): + space = 0.02 + ce_domain_start, ce_domain_end = 0.9, 1.0 + capacity_domain_start, capacity_domain_end = 0.6, 0.9 - space + loss_domain_start, loss_domain_end = 0.3, 0.6 - space + cv_domain_start, cv_domain_end = 0.0, 0.3 - space + + # Format y-axis labels with HTML for proper alignment + capacity_unit = _get_capacity_unit(c, mode=y.split("_")[-1]) + ce_label = "Coulombic
Efficiency (%)" + capacity_label = f"Capacity
({capacity_unit})" + if fullcell_standard_normalization_type: + _norm_label = f"[{fullcell_standard_normalization_scaler:.1f}/{fullcell_standard_normalization_factor:.1f} {capacity_unit}]" + loss_label = f"Cumulated
Loss (norm.)
{_norm_label}" + + else: + loss_label = f"Cumulated
Loss ({capacity_unit})" + cv_label = f"CV Capacity
({capacity_unit})" + + fig.update_layout( + yaxis8={"domain": [ce_domain_start, ce_domain_end]}, + yaxis7={"title": dict(text=ce_label), "domain": [ce_domain_start, ce_domain_end]}, + yaxis6={"domain": [capacity_domain_start, capacity_domain_end]}, + yaxis5={"title": dict(text=capacity_label), "domain": [capacity_domain_start, capacity_domain_end]}, + yaxis4={"domain": [loss_domain_start, loss_domain_end]}, + yaxis3={"title": dict(text=loss_label), "domain": [loss_domain_start, loss_domain_end]}, + yaxis2={"domain": [cv_domain_start, cv_domain_end]}, + yaxis1={"title": dict(text=cv_label), "domain": [cv_domain_start, cv_domain_end]}, + ) + if show_formation: + fig.update_layout( + xaxis1={"title": dict(text="")}, + ) + if x_axis_domain_formation_fraction < 0.1: + fig.update_layout( + xaxis1={"showticklabels": False}, + ) + + if link_capacity_scales: + fig.update_layout( + yaxis={"matches": "y2"}, + yaxis2={"matches": "y3"}, + yaxis3={"matches": "y4"}, + yaxis4={"matches": "y5"}, + yaxis5={"matches": "y6"}, + ) else: raise NotImplementedError("Not implemented for more than four rows") else: @@ -1061,43 +1186,17 @@ def _auto_range(fig, axis_name_1, axis_name_2): yaxis=dict(domain=[0.0, 0.65]), yaxis2={"title": dict(text="Coulombic Efficiency"), "domain": [0.7, 1.0]}, ) - - if y.startswith("fullcell_standard_"): - # CE - space = 0.02 - ce_domain_start, ce_domain_end = 0.9, 1.0 - capacity_domain_start, capacity_domain_end = 0.6, 0.9 - space - loss_domain_start, loss_domain_end = 0.3, 0.6 - space - cv_domain_start, cv_domain_end = 0.0, 0.3 - space - - # Format y-axis labels with HTML for proper alignment - capacity_unit = _get_capacity_unit(c, mode=y.split("_")[-1]) - ce_label = "Coulombic
Efficiency (%)" - capacity_label = f"Capacity
({capacity_unit})" - loss_label = f"Cumulated
Loss ({capacity_unit})" - cv_label = f"CV Capacity
({capacity_unit})" - - fig.update_layout( - yaxis8={"domain": [ce_domain_start, ce_domain_end]}, - yaxis7={"title": dict(text=ce_label), "domain": [ce_domain_start, ce_domain_end]}, - yaxis6={"domain": [capacity_domain_start, capacity_domain_end]}, - yaxis5={"title": dict(text=capacity_label), "domain": [capacity_domain_start, capacity_domain_end]}, - yaxis4={"domain": [loss_domain_start, loss_domain_end]}, - yaxis3={"title": dict(text=loss_label), "domain": [loss_domain_start, loss_domain_end]}, - yaxis2={"domain": [cv_domain_start, cv_domain_end]}, - yaxis1={"title": dict(text=cv_label), "domain": [cv_domain_start, cv_domain_end]}, - ) - - - if link_capacity_scales: + if y.startswith("fullcell_standard_"): + capacity_unit = _get_capacity_unit(c, mode=y.split("_")[-1]) fig.update_layout( - yaxis={"matches": "y2"}, - yaxis2={"matches": "y3"}, - yaxis3={"matches": "y4"}, - yaxis4={"matches": "y5"}, - yaxis5={"matches": "y6"}, + yaxis1={"title": dict(text="CV Capacity")}, + yaxis2={"title": dict(text="Cumulated
Loss")}, + yaxis3={"title": dict(text=f"Capacity
({capacity_unit})")}, + yaxis4={"title": dict(text="Coulombic
Efficiency")}, ) - + + + if x_range is not None: if not show_formation: fig.update_layout(xaxis=dict(range=x_range)) diff --git a/local/notes.md b/local/notes.md index 4f80781b..11b41681 100644 --- a/local/notes.md +++ b/local/notes.md @@ -45,3 +45,15 @@ def test_from_raw_external(cellpy_data_instance, parameters): ### TODO warnings -> test_neware.py::test_get_neware_from_h5: cellpy/cellpy/readers/cellreader.py:1828: UserWarning: no fid_table - you should update your cellpy-file + +### TODO Error +-> File c:\scripting\cellpy\.venv\Lib\site-packages\plotly\io\_renderers.py:425, in show(fig, renderer, validate, **kwargs) + 420 raise ValueError( + 421 "Mime type rendering requires ipython but it is not installed" + 422 ) + 424 if not nbformat or Version(nbformat.__version__) < Version("4.2.0"): +--> 425 raise ValueError( + 426 "Mime type rendering requires nbformat>=4.2.0 but it is not installed" + 427 ) + 429 display_jupyter_version_warnings() + 431 ipython_display.display(bundle, raw=True) From dcd89ef29bc37ef62d5633507b70aa23f37a14ed Mon Sep 17 00:00:00 2001 From: jepegit Date: Thu, 12 Jun 2025 16:51:08 +0200 Subject: [PATCH 08/37] small bugfix --- cellpy/utils/plotutils.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/cellpy/utils/plotutils.py b/cellpy/utils/plotutils.py index 857fff17..0960f61b 100644 --- a/cellpy/utils/plotutils.py +++ b/cellpy/utils/plotutils.py @@ -187,6 +187,12 @@ def save_image_files(figure, name="my_figure", scale=3.0, dpi=300, backend="plot def _make_plotly_template(name="axis"): + if not plotly_available: + print("Plotly not available") + return None + import plotly.graph_objects as go + import plotly.io as pio + tick_label_width = 6 title_font_size = 22 title_font_family = "Arial" @@ -1511,6 +1517,8 @@ def _calculate_seaborn_plot_properties(number_of_rows, number_of_cols): print("Warning: Experimental feature - fullcell standard plot") print(" - missing: individual scaling for coulombic efficiency") print(" - missing: different heights for specific rows") + print(" - missing: individual scaling for cumulated loss if normalization is used") + print(" - missing: appropriate y title for cumulated loss if normalization is used") capacity_unit = _get_capacity_unit(c, mode=y.split("_")[-1]) ce_label = "Coulombic Efficiency (%)" From 5982824fdb85ea1fffd41819e4036223e8628c0f Mon Sep 17 00:00:00 2001 From: jepegit Date: Sun, 15 Jun 2025 13:53:01 +0200 Subject: [PATCH 09/37] fix summary plot standard --- cellpy/utils/plotutils.py | 128 ++++++++++++++++++++++++++++++-------- 1 file changed, 103 insertions(+), 25 deletions(-) diff --git a/cellpy/utils/plotutils.py b/cellpy/utils/plotutils.py index 0960f61b..e764c4d3 100644 --- a/cellpy/utils/plotutils.py +++ b/cellpy/utils/plotutils.py @@ -711,6 +711,7 @@ def summary_plot( x_range: list = None, y_range: list = None, ce_range: list = None, + norm_range: list = None, split: bool = True, auto_convert_legend_labels: bool = True, interactive: bool = True, @@ -750,6 +751,7 @@ def summary_plot( x_range: limits for x-axis y_range: limits for y-axis ce_range: limits for coulombic efficiency (if present) + norm_range: limits for normalized capacity (if present) split: split the plot auto_convert_legend_labels: convert the legend labels to a nicer format. interactive: use interactive plotting (plotly) @@ -795,6 +797,7 @@ def summary_plot( show_y_labels_on_right_pane = kwargs.pop("show_y_labels_on_right_pane", False) number_of_rows = 1 + max_val_normalized_col = 0.0 if interactive and not plotly_available: warnings.warn("plotly not available, and it is currently the only supported interactive backend") @@ -820,6 +823,7 @@ def summary_plot( def _auto_range(fig: Any, axis_name_1: str, axis_name_2: str) -> list: + # only works for plotly min_y = np.inf max_y = -np.inf full_axis_name_1 = axis_name_1.replace("y", "yaxis") @@ -848,7 +852,6 @@ def _auto_range(fig: Any, axis_name_1: str, axis_name_2: str) -> list: return _range - # ------------------- main -------------------------------------------- y_header = "value" color = "variable" row = "row" @@ -867,7 +870,9 @@ def _auto_range(fig: Any, axis_name_1: str, axis_name_2: str) -> list: additional_kwargs_plotly_update_traces[k.replace("plotly_", "")] = kwargs.pop(k) additional_kwargs_seaborn = dict() - # standard fullcell plot (ce, discharge cap, loss, cv-only) + + # ------------------- collecting data ----------------------------------------- + if y.startswith("fullcell_standard_"): if additional_kwargs_plotly.get("height") is None: additional_kwargs_plotly["height"] = 800 @@ -877,6 +882,8 @@ def _auto_range(fig: Any, axis_name_1: str, axis_name_2: str) -> list: if summary.index.name == x: summary = summary.reset_index(drop=False) + # Remark! Possible code duplication with the 'partition_summary_cv_steps' used in + # the 'if y.endswith("_split_constant_voltage")' block: summary_only_cv = c.make_summary(selector_type="only-cv", create_copy=True).data.summary if summary_only_cv.index.name == x: summary_only_cv = summary_only_cv.reset_index(drop=False) @@ -902,18 +909,21 @@ def _auto_range(fig: Any, axis_name_1: str, axis_name_2: str) -> list: mask = s["variable"].str.contains(r"cumulated.*loss") s.loc[mask, "value"] = s.loc[mask, "value"] - first_values - if fullcell_standard_normalization_type: - # set normalization factor + if fullcell_standard_normalization_type is not False: + if fullcell_standard_normalization_factor is None: + if fullcell_standard_normalization_type == "max": fullcell_standard_normalization_factor = s[s[row] == 1].max().value fullcell_standard_normalization_type = "divide" + elif fullcell_standard_normalization_type == "area": with warnings.catch_warnings(): warnings.simplefilter("ignore") area = np.trapezoid(s[s[row] == 1].value, dx=1) fullcell_standard_normalization_factor = area fullcell_standard_normalization_type = "divide" + else: fullcell_standard_normalization_factor = 1.0 @@ -925,9 +935,11 @@ def _auto_range(fig: Any, axis_name_1: str, axis_name_2: str) -> list: for col, trans in y_trans.get(y, {}).items(): s.loc[s["variable"] == col, "value"] = trans(s.loc[s["variable"] == col, "value"].values, **trans_kwargs) + max_val_normalized_col = s.loc[s["variable"] == col, "value"].max() # filter on constant voltage vs constant current # Remark! absoulte capacities are not implemented yet. + # Remark! uses the 'partition_summary_cv_steps' function - consider using that also for the fullcell standard plot to avoid code duplication elif y.endswith("_split_constant_voltage"): cap_type = "capacities_gravimetric" if y.startswith("capacities_gravimetric") else "capacities_areal" column_set = y_cols[cap_type] @@ -986,7 +998,7 @@ def _auto_range(fig: Any, axis_name_1: str, axis_name_2: str) -> list: if interactive: import plotly.express as px - from plotly.subplots import make_subplots + # from plotly.subplots import make_subplots set_plotly_template(plotly_template) @@ -1033,10 +1045,10 @@ def _auto_range(fig: Any, axis_name_1: str, axis_name_2: str) -> list: domain=x_axis_domain_rest, matches=None, ), - yaxis2=dict(matches="y", showticklabels=show_y_labels_on_right_pane), ) annotations = [{"text": formation_header, "x": 0.08, "y": 1.02, "showarrow": False}, PLOTLY_BLANK_LABEL] fig.update_layout(annotations=annotations) + fig.update_layout(yaxis2=dict(matches="y", showticklabels=show_y_labels_on_right_pane),) elif number_of_rows == 2: fig.update_yaxes(matches="y") @@ -1109,15 +1121,20 @@ def _auto_range(fig: Any, axis_name_1: str, axis_name_2: str) -> list: fig.update_yaxes(autorange=False) fig.update_layout(xaxis_domain=x_axis_domain_formation, scene_domain_x=x_axis_domain_formation) - if y.startswith("fullcell_standard_") and fullcell_standard_normalization_type: - range_2 = [0.0, fullcell_standard_normalization_scaler] + range_1 = _auto_range(fig, "y", "y2") + + if y.startswith("fullcell_standard_") and fullcell_standard_normalization_type is not False: + range_2 = [0.0, max(max_val_normalized_col, fullcell_standard_normalization_scaler)] + range_2 = norm_range or range_2 else: range_2 = _auto_range(fig, "y3", "y4") - range_1 = _auto_range(fig, "y", "y2") range_3 = _auto_range(fig, "y5", "y6") range_4 = _auto_range(fig, "y7", "y8") + if y.startswith("fullcell_standard_"): + range_4 = eff_lim or range_4 + fig.update_layout( xaxis2=dict(range=x_axis_range_rest, domain=x_axis_domain_rest, matches=None), xaxis3=dict(range=x_axis_range_formation, domain=x_axis_domain_formation, matches="x"), @@ -1187,12 +1204,23 @@ def _auto_range(fig: Any, axis_name_1: str, axis_name_2: str) -> list: else: raise NotImplementedError("Not implemented for more than four rows") else: + # TODO: refactor so that we do not have specify this: if y.endswith("_efficiency"): fig.update_layout( yaxis=dict(domain=[0.0, 0.65]), yaxis2={"title": dict(text="Coulombic Efficiency"), "domain": [0.7, 1.0]}, ) if y.startswith("fullcell_standard_"): + if fullcell_standard_normalization_type is not False: + range_2 = [0.0, max(max_val_normalized_col, fullcell_standard_normalization_scaler)] + range_2 = norm_range or range_2 + fig.update_layout(yaxis2=dict(range=range_2)) + if eff_lim is not None: + # update yaxis2 + fig.update_layout(yaxis4=dict(range=eff_lim)) + + fig.update_layout(yaxis2=dict()) + capacity_unit = _get_capacity_unit(c, mode=y.split("_")[-1]) fig.update_layout( yaxis1={"title": dict(text="CV Capacity")}, @@ -1200,9 +1228,33 @@ def _auto_range(fig: Any, axis_name_1: str, axis_name_2: str) -> list: yaxis3={"title": dict(text=f"Capacity
({capacity_unit})")}, yaxis4={"title": dict(text="Coulombic
Efficiency")}, ) - - - + fig.layout["annotations"] = 4 * [PLOTLY_BLANK_LABEL] + + space = 0.02 + ce_domain_start, ce_domain_end = 0.9, 1.0 + capacity_domain_start, capacity_domain_end = 0.6, 0.9 - space + loss_domain_start, loss_domain_end = 0.3, 0.6 - space + cv_domain_start, cv_domain_end = 0.0, 0.3 - space + + # Format y-axis labels with HTML for proper alignment + capacity_unit = _get_capacity_unit(c, mode=y.split("_")[-1]) + ce_label = "Coulombic
Efficiency (%)" + capacity_label = f"Capacity
({capacity_unit})" + if fullcell_standard_normalization_type: + _norm_label = f"[{fullcell_standard_normalization_scaler:.1f}/{fullcell_standard_normalization_factor:.1f} {capacity_unit}]" + loss_label = f"Cumulated
Loss (norm.)
{_norm_label}" + + else: + loss_label = f"Cumulated
Loss ({capacity_unit})" + cv_label = f"CV Capacity
({capacity_unit})" + + fig.update_layout( + yaxis4={"title": dict(text=ce_label), "domain": [ce_domain_start, ce_domain_end]}, + yaxis3={"title": dict(text=capacity_label), "domain": [capacity_domain_start, capacity_domain_end]}, + yaxis2={"title": dict(text=loss_label), "domain": [loss_domain_start, loss_domain_end]}, + yaxis1={"title": dict(text=cv_label), "domain": [cv_domain_start, cv_domain_end]}, + ) + if x_range is not None: if not show_formation: fig.update_layout(xaxis=dict(range=x_range)) @@ -1372,6 +1424,7 @@ def _calculate_seaborn_plot_properties(number_of_rows, number_of_cols): y_range = y_range or [min_value - 0.05 * abs(min_value), max_value + 0.05 * abs(max_value)] _efficiency_label = r"Efficiency (%)" + if is_efficiency_plot: facet_kws["sharey"] = False gridspec_kws["height_ratios"] = [1, 4] @@ -1515,17 +1568,31 @@ def _calculate_seaborn_plot_properties(number_of_rows, number_of_cols): elif is_fullcell_standard_plot: print("Warning: Experimental feature - fullcell standard plot") - print(" - missing: individual scaling for coulombic efficiency") - print(" - missing: different heights for specific rows") - print(" - missing: individual scaling for cumulated loss if normalization is used") - print(" - missing: appropriate y title for cumulated loss if normalization is used") + + seaborn_plot_height = 2.0 # hardcoded for now + seaborn_plot_aspect = 1.5 if show_formation else 3.0 # hardcoded for now capacity_unit = _get_capacity_unit(c, mode=y.split("_")[-1]) - ce_label = "Coulombic Efficiency (%)" - capacity_label = f"Capacity ({capacity_unit})" - loss_label = f"Cumulated Loss ({capacity_unit})" - cv_label = f"CV Capacity ({capacity_unit})" + ce_label = "Coulombic\nEfficiency (%)" + capacity_label = f"Capacity\n({capacity_unit})" + + loss_label = f"Cumulated Loss\n({capacity_unit})" + if fullcell_standard_normalization_type: + _norm_label = f"[{fullcell_standard_normalization_scaler:.1f}/{fullcell_standard_normalization_factor:.1f} {capacity_unit}]" + loss_label = f"Cumulated\nLoss (norm.)\n{_norm_label}" + else: + loss_label = f"Cumulated\nLoss\n({capacity_unit})" + + cv_label = f"CV Capacity\n({capacity_unit})" + + facet_kws["sharey"] = False + gridspec_kws["height_ratios"] = [1, 3, 3, 3] + number_of_rows = 4 + + if fullcell_standard_normalization_type is not False: + cum_loss_info_range = norm_range or [0.0, max(max_val_normalized_col, fullcell_standard_normalization_scaler)] + cv_info = dict( title="", xlim=x_range, @@ -1538,7 +1605,7 @@ def _calculate_seaborn_plot_properties(number_of_rows, number_of_cols): cum_loss_info = dict( title="", xlim=x_range, - ylim=y_range, + ylim=cum_loss_info_range, row=2, col="standard", yticks=False, @@ -1556,7 +1623,7 @@ def _calculate_seaborn_plot_properties(number_of_rows, number_of_cols): ce_info = dict( title="", xlim=x_range, - ylim=y_range, + ylim=eff_lim, row=0, col="standard", yticks=False, @@ -1578,7 +1645,6 @@ def _calculate_seaborn_plot_properties(number_of_rows, number_of_cols): info_dicts.append(capacity_info) info_dicts.append(ce_info) - if show_formation: cv_info_formation = dict( ylabel=cv_label, @@ -1594,7 +1660,7 @@ def _calculate_seaborn_plot_properties(number_of_rows, number_of_cols): ylabel=loss_label, title="", xlim=xlim_formation, - ylim=y_range, + ylim=cum_loss_info_range, row=2, col="formation", yticks=True, @@ -1614,7 +1680,7 @@ def _calculate_seaborn_plot_properties(number_of_rows, number_of_cols): ylabel=ce_label, title="", xlim=xlim_formation, - ylim=y_range, + ylim=eff_lim, row=0, col="formation", yticks=True, @@ -1624,6 +1690,17 @@ def _calculate_seaborn_plot_properties(number_of_rows, number_of_cols): info_dicts.append(loss_info_formation) info_dicts.append(cap_info_formation) info_dicts.append(ce_info_formation) + + if verbose: + print(f"{y_header=}") + print(f"{color=}") + print(f"{seaborn_plot_height=}") + print(f"{seaborn_plot_aspect=}") + print(f"{markers=}") + print(f"{additional_kwargs_seaborn=}") + print(f"{facet_kws=}") + print(f"{kwargs=}") + print(f"{info_dicts=}") else: if is_multi_row: @@ -1685,6 +1762,7 @@ def _calculate_seaborn_plot_properties(number_of_rows, number_of_cols): facet_kws["gridspec_kws"] = gridspec_kws + sns_fig = sns.relplot( data=s, x=x, From 78d3c3fceff98621c6aa3b77d13f07a1438be2b8 Mon Sep 17 00:00:00 2001 From: jepegit Date: Sun, 15 Jun 2025 22:44:32 +0200 Subject: [PATCH 10/37] summaryplot standard seaborn tweaks --- cellpy/utils/plotutils.py | 90 +++++++++++++++++++++++++-------------- 1 file changed, 58 insertions(+), 32 deletions(-) diff --git a/cellpy/utils/plotutils.py b/cellpy/utils/plotutils.py index e764c4d3..525be03d 100644 --- a/cellpy/utils/plotutils.py +++ b/cellpy/utils/plotutils.py @@ -712,6 +712,7 @@ def summary_plot( y_range: list = None, ce_range: list = None, norm_range: list = None, + cv_share_range: list = None, split: bool = True, auto_convert_legend_labels: bool = True, interactive: bool = True, @@ -724,6 +725,7 @@ def summary_plot( seaborn_style: str = "dark", formation_cycles=3, show_formation=True, + show_legend=True, x_axis_domain_formation_fraction=0.2, column_separator=0.01, reset_losses=True, @@ -752,6 +754,7 @@ def summary_plot( y_range: limits for y-axis ce_range: limits for coulombic efficiency (if present) norm_range: limits for normalized capacity (if present) + cv_share_range: limits for cv share (if present) split: split the plot auto_convert_legend_labels: convert the legend labels to a nicer format. interactive: use interactive plotting (plotly) @@ -764,6 +767,7 @@ def summary_plot( seaborn_style: name of the seaborn style to use formation_cycles: number of formation cycles to show show_formation: show formation cycles + show_legend: show the legend x_axis_domain_formation_fraction: fraction of the x-axis domain for the formation cycles (default: 0.2) column_separator: separation between columns when splitting the plot (only for plotly) reset_losses: reset the losses to the first cycle (only for fullcell_standard plots) @@ -795,6 +799,11 @@ def summary_plot( smart_link = kwargs.pop("smart_link", True) show_y_labels_on_right_pane = kwargs.pop("show_y_labels_on_right_pane", False) + seaborn_facecolor = kwargs.pop("seaborn_facecolor", "#EAEAF2") + seaborn_edgecolor = kwargs.pop("seaborn_edgecolor", "black") + seaborn_style_dict_default = {"axes.facecolor": seaborn_facecolor, "axes.edgecolor": seaborn_edgecolor} + seaborn_style_dict = kwargs.pop("seaborn_style_dict", seaborn_style_dict_default) + seaborn_marker_size = kwargs.pop("seaborn_marker_size", 7) number_of_rows = 1 max_val_normalized_col = 0.0 @@ -1005,6 +1014,8 @@ def _auto_range(fig: Any, axis_name_1: str, axis_name_2: str) -> list: if show_formation: additional_kwargs_plotly["facet_col"] = col_id + + fig = px.line( s, x=x, @@ -1018,6 +1029,11 @@ def _auto_range(fig: Any, axis_name_1: str, axis_name_2: str) -> list: ) fig.update_traces(**additional_kwargs_plotly_update_traces) + if not show_legend: + fig.update_layout(showlegend=False) + + if y_range is not None: + fig.update_layout(yaxis=dict(range=y_range)) if show_formation: formation_header = 'Formation' @@ -1134,6 +1150,7 @@ def _auto_range(fig: Any, axis_name_1: str, axis_name_2: str) -> list: if y.startswith("fullcell_standard_"): range_4 = eff_lim or range_4 + range_1 = cv_share_range or range_1 fig.update_layout( xaxis2=dict(range=x_axis_range_rest, domain=x_axis_domain_rest, matches=None), @@ -1218,8 +1235,8 @@ def _auto_range(fig: Any, axis_name_1: str, axis_name_2: str) -> list: if eff_lim is not None: # update yaxis2 fig.update_layout(yaxis4=dict(range=eff_lim)) - - fig.update_layout(yaxis2=dict()) + if cv_share_range is not None: + fig.update_layout(yaxis=dict(range=cv_share_range)) capacity_unit = _get_capacity_unit(c, mode=y.split("_")[-1]) fig.update_layout( @@ -1262,9 +1279,6 @@ def _auto_range(fig: Any, axis_name_1: str, axis_name_2: str) -> list: # The x_range is handled a bit differently when showing formation cycles # This is done within if show_formation block - if y_range is not None: - fig.update_layout(yaxis=dict(range=y_range)) - if split: if show_formation: if not share_y and not smart_link: @@ -1278,7 +1292,7 @@ def _auto_range(fig: Any, axis_name_1: str, axis_name_2: str) -> list: else: fig.update_layout(xaxis_rangeslider_visible=True) - if auto_convert_legend_labels: + if auto_convert_legend_labels and show_legend: for trace in fig.data: name = trace.name name = name.replace("_", " ").title() @@ -1348,11 +1362,6 @@ def _clean_up_axis(fig, info_dicts=None, row_id="row", col_id="cycle_type"): if yticks is False: a.set_yticks([]) - seaborn_facecolor = kwargs.pop("seaborn_facecolor", "#EAEAF2") - seaborn_edgecolor = kwargs.pop("seaborn_edgecolor", "black") - seaborn_style_dict_default = {"axes.facecolor": seaborn_facecolor, "axes.edgecolor": seaborn_edgecolor} - seaborn_style_dict = kwargs.pop("seaborn_style_dict", seaborn_style_dict_default) - sns.set_style(seaborn_style, seaborn_style_dict) sns.set_palette(seaborn_palette) sns.set_context(kwargs.pop("seaborn_context", "notebook")) @@ -1377,25 +1386,39 @@ def _clean_up_axis(fig, info_dicts=None, row_id="row", col_id="cycle_type"): additional_kwargs_seaborn["row"] = row number_of_rows = s[row].nunique() - def _calculate_seaborn_plot_properties(number_of_rows, number_of_cols): + def _calculate_seaborn_plot_properties(number_of_rows, number_of_cols, plot_type="default"): ## Maybe implement some proper calculations later... # _default_seaborn_plot_height = 2.4 + 0.4 * number_of_rows # _default_seaborn_plot_aspect = 1.0 + 2.0 / number_of_rows - - _selector = { - (1, 1): (4.0, 2.05), - (1, 2): (4.0, 1.0), - (2, 1): (2.8, 2.8), - (2, 2): (2.8, 1.4), - (3, 1): (3.0, 2.7), - (3, 2): (3.0, 1.35), - } + # seaborn_plot_height = 2.0 # hardcoded for now + # seaborn_plot_aspect = 1.5 if show_formation else 3.0 # hardcoded for now + + if plot_type == "fullcell_standard": + _selector = { + (4, 1): (2.0, 4.0), + (4, 2): (2.0, 2.0), + } + else: + _selector = { + (1, 1): (4.0, 2.05), + (1, 2): (4.0, 1.0), + (2, 1): (2.8, 2.8), + (2, 2): (2.8, 1.4), + (3, 1): (3.0, 2.7), + (3, 2): (3.0, 1.35), + (4, 1): (3.0, 2.7), + (4, 2): (3.0, 1.35), + } return _selector.get((number_of_rows, number_of_cols), (4.0, 1.8)) + + if y.startswith("fullcell_standard_"): + plot_type = "fullcell_standard" + else: + plot_type = "default" _default_seaborn_plot_height, _default_seaborn_plot_aspect = _calculate_seaborn_plot_properties( - number_of_rows, number_of_cols + number_of_rows, number_of_cols, plot_type=plot_type ) - seaborn_plot_height = kwargs.pop("seaborn_plot_height", _default_seaborn_plot_height) seaborn_plot_aspect = kwargs.pop("seaborn_plot_aspect", _default_seaborn_plot_aspect) @@ -1505,9 +1528,9 @@ def _calculate_seaborn_plot_properties(number_of_rows, number_of_cols): elif is_split_constant_voltage_plot: if is_multi_row: - y_range_cv = kwargs.pop("y_range_cv", y_range) + cv_share_range = cv_share_range or y_range for r, _x, _y_range in zip( - ["all", "without CV", "with CV"], [False, False, None], [y_range, y_range, y_range_cv] + ["all", "without CV", "with CV"], [False, False, None], [y_range, y_range, cv_share_range] ): _d = dict( ylabel=y_label, @@ -1569,9 +1592,6 @@ def _calculate_seaborn_plot_properties(number_of_rows, number_of_cols): elif is_fullcell_standard_plot: print("Warning: Experimental feature - fullcell standard plot") - seaborn_plot_height = 2.0 # hardcoded for now - seaborn_plot_aspect = 1.5 if show_formation else 3.0 # hardcoded for now - capacity_unit = _get_capacity_unit(c, mode=y.split("_")[-1]) ce_label = "Coulombic\nEfficiency (%)" capacity_label = f"Capacity\n({capacity_unit})" @@ -1596,7 +1616,7 @@ def _calculate_seaborn_plot_properties(number_of_rows, number_of_cols): cv_info = dict( title="", xlim=x_range, - ylim=y_range, + ylim=cv_share_range or y_range, row=3, col="standard", yticks=False, @@ -1650,7 +1670,7 @@ def _calculate_seaborn_plot_properties(number_of_rows, number_of_cols): ylabel=cv_label, title="", xlim=xlim_formation, - ylim=y_range, + ylim=cv_share_range or y_range, row=3, col="formation", yticks=True, @@ -1762,7 +1782,6 @@ def _calculate_seaborn_plot_properties(number_of_rows, number_of_cols): facet_kws["gridspec_kws"] = gridspec_kws - sns_fig = sns.relplot( data=s, x=x, @@ -1772,6 +1791,7 @@ def _calculate_seaborn_plot_properties(number_of_rows, number_of_cols): aspect=seaborn_plot_aspect, kind="line", marker="o" if markers else None, + legend=show_legend, **additional_kwargs_seaborn, facet_kws=facet_kws, **kwargs, @@ -1779,7 +1799,7 @@ def _calculate_seaborn_plot_properties(number_of_rows, number_of_cols): sns_fig.set_axis_labels(x_label, y_label) - if auto_convert_legend_labels: + if auto_convert_legend_labels and show_legend: legend = sns_fig.legend if legend is not None: for le in legend.get_texts(): @@ -1791,6 +1811,12 @@ def _calculate_seaborn_plot_properties(number_of_rows, number_of_cols): le.set_text(name) sns_fig.legend.set_title(None) + if markers: + for ax in sns_fig.axes.flat: + lines = ax.get_lines() + for line in lines: + line.set_markersize(seaborn_marker_size) + fig = sns_fig.figure _clean_up_axis(fig, info_dicts=info_dicts, row_id=row_id, col_id=col_id) fig.align_ylabels() From fbde03a577a764785b06eba61bdacbd0a4742b36 Mon Sep 17 00:00:00 2001 From: jepegit Date: Tue, 17 Jun 2025 13:36:08 +0200 Subject: [PATCH 11/37] line hooks in summaryplot --- cellpy/utils/plotutils.py | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/cellpy/utils/plotutils.py b/cellpy/utils/plotutils.py index 525be03d..35bcd51e 100644 --- a/cellpy/utils/plotutils.py +++ b/cellpy/utils/plotutils.py @@ -733,6 +733,7 @@ def summary_plot( fullcell_standard_normalization_type="divide", fullcell_standard_normalization_factor=None, fullcell_standard_normalization_scaler=1.0, + seaborn_line_hooks: list[tuple[str, list, dict]] = None, **kwargs, ): """Create a summary plot. @@ -780,8 +781,11 @@ def summary_plot( if normalization_type is False, no normalization is done fullcell_standard_normalization_factor: normalization factor for the fullcell standard plots fullcell_standard_normalization_scaler: scaler for the fullcell standard plots - plotly_[update trace parameter]: additional parameters for the plotly traces + plotly_[update trace parameter]: additional parameters for the plotly traces (e.g. use plotly_marker_size=10 for updating the marker_size to 10) + seaborn_[update line parameter]: additional parameters for the seaborn lines (not many options available yet) + (e.g. use seaborn_marker_size=10 for updating the marker_size to 10) + seaborn_line_hooks: list of functions to hook into the seaborn lines (e.g. to update the marker_size) **kwargs: includes additional parameters for the plotting backend (not properly documented yet). Returns: @@ -793,6 +797,14 @@ def summary_plot( If you want to modify the non-interactive (matplotlib) plot, you can get the axes from the returned figure by ``axes = figure.get_axes()``. + Example: + >> axes = figure.get_axes() + >> ylabel = axes[0].get_ylabel() + >> if "Coulombic" in ylabel: + >> axes[0].set_ylabel("C.E. (%)") + >> else: + >> print(f"This is not the coulombic efficiency axis: {ylabel=}") + """ from copy import deepcopy @@ -1817,6 +1829,14 @@ def _calculate_seaborn_plot_properties(number_of_rows, number_of_cols, plot_type for line in lines: line.set_markersize(seaborn_marker_size) + if seaborn_line_hooks: + for ax in sns_fig.axes.flat: + lines = ax.get_lines() + for line in lines: + for hook, args, kwargs in seaborn_line_hooks: + if hasattr(line, hook): + getattr(line, hook)(*args, **kwargs) + fig = sns_fig.figure _clean_up_axis(fig, info_dicts=info_dicts, row_id=row_id, col_id=col_id) fig.align_ylabels() From 81994b091f9c5078870000fa9b2069b7c4b5930e Mon Sep 17 00:00:00 2001 From: jepegit Date: Tue, 17 Jun 2025 13:36:22 +0200 Subject: [PATCH 12/37] bump version 1.0.3a1 -> 1.0.3a2 --- cellpy/_version.py | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cellpy/_version.py b/cellpy/_version.py index 21958b35..99f36a8e 100644 --- a/cellpy/_version.py +++ b/cellpy/_version.py @@ -1 +1 @@ -__version__ = "1.0.3a1" +__version__ = "1.0.3a2" diff --git a/pyproject.toml b/pyproject.toml index 00a96c90..634cc729 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,7 +15,7 @@ markers = [ line-length = 120 [tool.bumpver] -current_version = "1.0.3a1" +current_version = "1.0.3a2" version_pattern = "MAJOR.MINOR.PATCH[PYTAG][NUM]" commit_message = "bump version {old_version} -> {new_version}" commit = true From 4f2a5d889c2f23fa06dc6315976e3f186f68ec0b Mon Sep 17 00:00:00 2001 From: jepegit Date: Tue, 17 Jun 2025 14:25:30 +0200 Subject: [PATCH 13/37] start implement partitioning on cv for summary collector --- cellpy/utils/helpers.py | 62 ++++++++++++++++++++--------------------- 1 file changed, 31 insertions(+), 31 deletions(-) diff --git a/cellpy/utils/helpers.py b/cellpy/utils/helpers.py index 099faa09..d60e62b6 100644 --- a/cellpy/utils/helpers.py +++ b/cellpy/utils/helpers.py @@ -818,16 +818,20 @@ def concatenate_summaries( def add_cv_step_columns(columns: list) -> list: """Add columns for CV steps. """ - raise NotImplementedError("This function is not implemented yet") + new_columns = [] + for col in columns: + if "_capacity" in col: + new_columns.extend([col, col + "_cv", col + "_non_cv"]) + else: + new_columns.append(col) + return new_columns -def partition_summary_based_on_cv_steps( +def _partition_summary_based_on_cv_steps( c, - x: str, column_set: list, - split: bool = False, - var_name: str = "variable", - value_name: str = "value", + x: str = None, + ): """Partition the summary data into CV and non-CV steps. @@ -842,17 +846,22 @@ def partition_summary_based_on_cv_steps( Returns: ``pandas.DataFrame`` (melted with columns x, var_name, value_name, and optionally "row" if split is True) """ - raise NotImplementedError("This function is not implemented yet") import pandas as pd - summary = c.data.summary + if not x: + x = hdr_summary["cycle_index"] + + # in case the column set already contains cv cols: + column_set = [col for col in column_set if not "_cv" in col] + + summary = c.data.summary.copy() summary_no_cv = c.make_summary(selector_type="non-cv", create_copy=True).data.summary summary_only_cv = c.make_summary(selector_type="only-cv", create_copy=True).data.summary if x != summary.index.name: - summary.set_index(x, inplace=True) - summary_no_cv.set_index(x, inplace=True) - summary_only_cv.set_index(x, inplace=True) + summary.set_index(x, inplace=True, drop=True) + summary_no_cv.set_index(x, inplace=True, drop=True) + summary_only_cv.set_index(x, inplace=True, drop=True) summary = summary[column_set] @@ -862,23 +871,7 @@ def partition_summary_based_on_cv_steps( summary_only_cv = summary_only_cv[column_set] summary_only_cv.columns = [col + "_cv" for col in summary_only_cv.columns] - if split: - id_vars = [x, "row"] - summary_no_cv["row"] = "without CV" - summary_only_cv["row"] = "with CV" - summary["row"] = "all" - else: - id_vars = x - - summary_no_cv = summary_no_cv.reset_index() - summary_only_cv = summary_only_cv.reset_index() - summary = summary.reset_index() - summary_no_cv = summary_no_cv.melt(id_vars, var_name=var_name, value_name=value_name) - summary_only_cv = summary_only_cv.melt(id_vars, var_name=var_name, value_name=value_name) - summary = summary.melt(id_vars, var_name=var_name, value_name=value_name) - - s = pd.concat([summary, summary_no_cv, summary_only_cv], axis=0) - s = s.reset_index(drop=True) + s = pd.concat([summary, summary_no_cv, summary_only_cv], axis=1) return s @@ -1069,10 +1062,12 @@ def concat_summaries( c = add_normalized_capacity(c, norm_cycles=normalize_capacity_on, scale=scale_by) - if partition_by_cv: - print("partitioning by cv_step") - s = partition_summary_based_on_cv_steps(c) if rate is not None: + # TODO: update this so that it works with partitioned data + if partition_by_cv: + print("partitioning by cv_step is not possible with rate selection") + print("skipping rate selection") + continue s = select_summary_based_on_rate( c, rate=rate, @@ -1082,6 +1077,11 @@ def concat_summaries( inverse=inverse, inverted=inverted, ) + elif partition_by_cv: + print("partitioning by cv_step") + s = _partition_summary_based_on_cv_steps(c, column_set=output_columns) + print("partition done") + print(f"{s.columns=}") else: s = c.data.summary From 1f939250d3fae86fa7dc89f656b167f8d4c9ee5b Mon Sep 17 00:00:00 2001 From: jepegit Date: Tue, 17 Jun 2025 14:29:25 +0200 Subject: [PATCH 14/37] bug identified --- cellpy/utils/helpers.py | 1 + 1 file changed, 1 insertion(+) diff --git a/cellpy/utils/helpers.py b/cellpy/utils/helpers.py index d60e62b6..fdc2ad3d 100644 --- a/cellpy/utils/helpers.py +++ b/cellpy/utils/helpers.py @@ -833,6 +833,7 @@ def _partition_summary_based_on_cv_steps( x: str = None, ): + # TODO: seems to be a bug here - probably due to concatinating (missing cycles) """Partition the summary data into CV and non-CV steps. Args: From ebf8af153e5ac3a69f19ac48b889dc313ce12bf0 Mon Sep 17 00:00:00 2001 From: jepegit Date: Sat, 21 Jun 2025 11:56:44 +0200 Subject: [PATCH 15/37] fix summaryplot with simple capacity retention for fullcell standard --- cellpy/utils/plotutils.py | 203 +++++++++++++++++++++++++++++++++----- 1 file changed, 179 insertions(+), 24 deletions(-) diff --git a/cellpy/utils/plotutils.py b/cellpy/utils/plotutils.py index 35bcd51e..d2b8b36b 100644 --- a/cellpy/utils/plotutils.py +++ b/cellpy/utils/plotutils.py @@ -17,6 +17,7 @@ from pathlib import Path import matplotlib.pyplot as plt +import pandas as pd from cellpy.parameters.internal_settings import ( get_headers_journal, @@ -587,24 +588,48 @@ def create_col_info(c): capacities_areal_coulombic_efficiency=_capacities_areal + [hdr.coulombic_efficiency], capacities_absolute_coulombic_efficiency=_capacities_absolute + [hdr.coulombic_efficiency], - fullcell_standard_gravimetric=[ + fullcell_standard_cumloss_gravimetric=[ hdr.charge_capacity+"_gravimetric" + "_cv", hdr.cumulated_discharge_capacity_loss + "_gravimetric", hdr.discharge_capacity+"_gravimetric", hdr.coulombic_efficiency, ], - fullcell_standard_areal=[ + fullcell_standard_cumloss_areal=[ hdr.charge_capacity+"_areal" + "_cv", hdr.cumulated_discharge_capacity_loss + "_areal", hdr.discharge_capacity+"_areal", hdr.coulombic_efficiency, ], - fullcell_standard_absolute=[ + fullcell_standard_cumloss_absolute=[ hdr.charge_capacity+"_absolute" + "_cv", hdr.cumulated_discharge_capacity_loss + "_absolute", hdr.discharge_capacity+"_absolute", hdr.coulombic_efficiency, ], + fullcell_standard_gravimetric=[ + hdr.charge_capacity+"_gravimetric" + "_cv", + hdr.discharge_capacity + "_gravimetric", + "mod_01_"+hdr.discharge_capacity+"_gravimetric", + hdr.coulombic_efficiency, + ], + fullcell_standard_areal=[ + hdr.charge_capacity+"_areal" + "_cv", + hdr.discharge_capacity + "_areal", + "mod_01_"+hdr.discharge_capacity+"_areal", + hdr.coulombic_efficiency, + ], + fullcell_standard_absolute=[ + hdr.charge_capacity+"_absolute" + "_cv", + hdr.discharge_capacity + "_absolute", + "mod_01_"+hdr.discharge_capacity+"_absolute", + hdr.coulombic_efficiency, + ], + fullcell_standard_dev=[ + hdr.charge_capacity+"_gravimetric" + "_cv", + hdr.discharge_capacity + "_gravimetric", + hdr.coulombic_efficiency, + "mod_01_"+hdr.discharge_capacity+"_gravimetric", + ], ) x_transformations = dict( @@ -613,6 +638,8 @@ def create_col_info(c): def _normalize_col(x: np.ndarray, normalization_factor: float = 1.0, normalization_type: str = "max", normalization_scaler: float = 1.0) -> np.ndarray: if normalization_type == "divide": return (x / normalization_factor) * normalization_scaler + elif normalization_type == "shift-divide": + return ((normalization_factor - x) / normalization_factor) * normalization_scaler elif normalization_type == "multiply": return (x * normalization_factor) * normalization_scaler elif normalization_type == "area": @@ -627,16 +654,43 @@ def _normalize_col(x: np.ndarray, normalization_factor: float = 1.0, normalizati return (x / x_max / normalization_factor) * normalization_scaler else: raise ValueError(f"Invalid normalization type: {normalization_type}") - - y_transformations: dict[str, dict[str, Callable]] = dict( + + # transformation info on the form: column_name: {(row_number, new_column_name): transformation_function} + y_transformations: dict[str, dict[tuple[int, str], dict[str, Callable]]] = dict( + fullcell_standard_cumloss_gravimetric={ + hdr.cumulated_discharge_capacity_loss + "_gravimetric": { + (2, hdr.cumulated_discharge_capacity_loss + "_gravimetric"): _normalize_col + }, + }, + fullcell_standard_cumloss_areal={ + hdr.cumulated_discharge_capacity_loss + "_areal": { + (2, hdr.cumulated_discharge_capacity_loss + "_areal"): _normalize_col + }, + }, + fullcell_standard_cumloss_absolute={ + hdr.cumulated_discharge_capacity_loss + "_absolute": { + (2, hdr.cumulated_discharge_capacity_loss + "_absolute"): _normalize_col + }, + }, fullcell_standard_gravimetric={ - hdr.cumulated_discharge_capacity_loss + "_gravimetric": _normalize_col, + "mod_01_"+hdr.discharge_capacity+"_gravimetric": { + (2, hdr.discharge_capacity + "_retention" + "_gravimetric"): _normalize_col + }, }, fullcell_standard_areal={ - hdr.cumulated_discharge_capacity_loss + "_areal": _normalize_col, + "mod_01_"+hdr.discharge_capacity+"_areal": { + (2, hdr.discharge_capacity + "_retention" + "_areal"): _normalize_col + }, }, fullcell_standard_absolute={ - hdr.cumulated_discharge_capacity_loss + "_absolute": _normalize_col, + "mod_01_"+hdr.discharge_capacity+"_absolute": { + (2, hdr.discharge_capacity + "_retention" + "_absolute"): _normalize_col + }, + }, + fullcell_standard_dev={ + "mod_01_"+hdr.discharge_capacity+"_gravimetric": { + (2, hdr.discharge_capacity + "_retention" + "_gravimetric"): _normalize_col + }, }, ) @@ -730,7 +784,7 @@ def summary_plot( column_separator=0.01, reset_losses=True, link_capacity_scales=False, - fullcell_standard_normalization_type="divide", + fullcell_standard_normalization_type="on-max", fullcell_standard_normalization_factor=None, fullcell_standard_normalization_scaler=1.0, seaborn_line_hooks: list[tuple[str, list, dict]] = None, @@ -808,6 +862,14 @@ def summary_plot( """ from copy import deepcopy + import re + + dev_mode = kwargs.pop("dev_mode", False) + if dev_mode: + print("DEV: dev_mode") + fullcell_standard_normalization_type = "shift-divide" + fullcell_standard_normalization_factor = 120.0 + fullcell_standard_normalization_scaler = 100.0 smart_link = kwargs.pop("smart_link", True) show_y_labels_on_right_pane = kwargs.pop("show_y_labels_on_right_pane", False) @@ -913,37 +975,53 @@ def _auto_range(fig: Any, axis_name_1: str, axis_name_2: str) -> list: s = s.reset_index(drop=True) s = s.melt(x) - s = s.loc[s.variable.isin(column_set)] - + s = s.loc[s.variable.isin(column_set)] # using strickt naming convention for "duplicated" columns ('mod__' so it will not be picked up here) + if dev_mode: + print(f"{column_set=} -> {s.variable.unique()}") number_of_rows = 4 s[row] = 1 # default row for capacity # Set row numbers using regex patterns s.loc[s["variable"].str.contains(r"_efficiency$"), row] = 0 # coulombic efficiency - s.loc[s["variable"].str.contains(r"cumulated.*loss"), row] = 2 # cumulated loss + s.loc[s["variable"].str.contains(r"cumulated.*loss"), row] = 2 # cumulated loss [will be removed?] + s.loc[s["variable"].str.startswith(r"mod_01_"), row] = 2 # capacity retention s.loc[s["variable"].str.contains(r"_cv$"), row] = 3 # cv data additional_kwargs_plotly["facet_row"] = row if reset_losses: + if dev_mode: + print("DEV: reset_losses") # Get the first value for each cumulated loss variable first_values = s[s["variable"].str.contains(r"cumulated.*loss")].groupby("variable")["value"].transform("first") # Shift all values by subtracting the first value mask = s["variable"].str.contains(r"cumulated.*loss") s.loc[mask, "value"] = s.loc[mask, "value"] - first_values + + # old normalization code: if fullcell_standard_normalization_type is not False: + if dev_mode: + print(f"{fullcell_standard_normalization_type=}") + print(f"{fullcell_standard_normalization_factor=}") + print(f"{fullcell_standard_normalization_scaler=}") + print("\nTransformations:") + print(f"{y_trans.get(y, {}).items()=}") if fullcell_standard_normalization_factor is None: - if fullcell_standard_normalization_type == "max": + if fullcell_standard_normalization_type == "on-max": fullcell_standard_normalization_factor = s[s[row] == 1].max().value - fullcell_standard_normalization_type = "divide" + fullcell_standard_normalization_type = "shift-divide" + + elif fullcell_standard_normalization_type == "max": + fullcell_standard_normalization_factor = s[s[row] == 1].max().value + fullcell_standard_normalization_type = "shift-divide" elif fullcell_standard_normalization_type == "area": with warnings.catch_warnings(): warnings.simplefilter("ignore") area = np.trapezoid(s[s[row] == 1].value, dx=1) fullcell_standard_normalization_factor = area - fullcell_standard_normalization_type = "divide" + fullcell_standard_normalization_type = "shift-divide" else: fullcell_standard_normalization_factor = 1.0 @@ -954,9 +1032,58 @@ def _auto_range(fig: Any, axis_name_1: str, axis_name_2: str) -> list: normalization_scaler=fullcell_standard_normalization_scaler, ) - for col, trans in y_trans.get(y, {}).items(): - s.loc[s["variable"] == col, "value"] = trans(s.loc[s["variable"] == col, "value"].values, **trans_kwargs) - max_val_normalized_col = s.loc[s["variable"] == col, "value"].max() + if dev_mode: + print(f"{trans_kwargs=}") + print(f"{y_trans.get(y, {}).items()=}") + + # transform the data + max_row_val = s[row].max() + for col, trans_dict in y_trans.get(y, {}).items(): + + for (new_row_val, new_col), trans in trans_dict.items(): + + if new_col in s["variable"].values: + # transforming on existing column (not using the new_row_val) + s.loc[s["variable"] == col, "value"] = trans(s.loc[s["variable"] == col, "value"].values, **trans_kwargs) + else: + # creating new column (using the new_row_val) + old_col = col + if new_row_val is not None: + row_val = new_row_val + else: + row_val = s.loc[s["variable"] == col, row] + if not row_val.empty: + row_val = row_val.values[0] + else: + max_row_val += 1 + row_val = max_row_val + if dev_mode: + print(f"{row_val=}") + + if old_col.startswith("mod_"): + old_col = re.sub(r'^mod_\d{2}_', '', old_col) + new_col_frame_section = s.loc[s["variable"] == old_col].copy() + new_col_frame_section["variable"] = new_col + new_col_frame_section["row"] = row_val + if dev_mode: + print(f"{old_col=} -> {new_col_frame_section.head()=}") + transformed_values = trans(new_col_frame_section["value"].values, **trans_kwargs) + new_col_frame_section["value"] = transformed_values + s = pd.concat([s, new_col_frame_section], ignore_index=True) + s = s.reset_index(drop=True) + s = s.sort_values(by=["row", "variable"]) + + if dev_mode: + print(f"{new_col=} -> {s.loc[s['variable'] == new_col, 'value'].values[:5]}") + + max_val_normalized_col = s.loc[s["variable"] == new_col, "value"].max() + + if dev_mode: + print("----------------------------------------------") + print(f"{s.variable.unique()=}") + print(f"{s.variable.value_counts()=}") + print(f"{column_set=}") + print(f"{s=}") # filter on constant voltage vs constant current # Remark! absoulte capacities are not implemented yet. @@ -1014,7 +1141,7 @@ def _auto_range(fig: Any, axis_name_1: str, axis_name_2: str) -> list: s[col_id] = "standard" s.loc[formation_cycle_selector, col_id] = "formation" - if verbose: + if verbose or dev_mode: _report_summary_plot_info(c, x, y, x_label, x_axis_labels, x_cols, y_label, y_axis_label, y_cols) if interactive: @@ -1026,7 +1153,12 @@ def _auto_range(fig: Any, axis_name_1: str, axis_name_2: str) -> list: if show_formation: additional_kwargs_plotly["facet_col"] = col_id - + if dev_mode: + print(f"{y_header=}") + print(f"{y_label=}") + print(f"{additional_kwargs_plotly=}") + print(f"{x_label=}") + print(f"{kwargs=}") fig = px.line( s, @@ -1039,6 +1171,16 @@ def _auto_range(fig: Any, axis_name_1: str, axis_name_2: str) -> list: }, **kwargs, ) + if dev_mode: + print("------------giving up here---------------------------") + # Error in row value in the data + print(f"{s=}") + print(f"{s.variable.unique()=}") + print(f"{s.variable.value_counts()=}") + print(f"{fig.data=}") + print(f"{fig.layout=}") + print("-----------------------------------------------------") + # return fig, s fig.update_traces(**additional_kwargs_plotly_update_traces) if not show_legend: @@ -1145,6 +1287,9 @@ def _auto_range(fig: Any, axis_name_1: str, axis_name_2: str) -> list: fig.layout["annotations"] = annotations elif number_of_rows == 4: + if dev_mode: + print("DEV: number_of_rows == 4") + print(f"{fig.data=}") fig.update_yaxes(matches="y") fig.update_yaxes(autorange=False) fig.update_layout(xaxis_domain=x_axis_domain_formation, scene_domain_x=x_axis_domain_formation) @@ -1157,7 +1302,13 @@ def _auto_range(fig: Any, axis_name_1: str, axis_name_2: str) -> list: else: range_2 = _auto_range(fig, "y3", "y4") - range_3 = _auto_range(fig, "y5", "y6") + range_3 = _auto_range(fig, "y5", "y6") + + if dev_mode: + print(f"{range_1=}") + print(f"{range_2=}") + print(f"{range_3=}") + range_4 = _auto_range(fig, "y7", "y8") if y.startswith("fullcell_standard_"): @@ -1192,15 +1343,19 @@ def _auto_range(fig: Any, axis_name_1: str, axis_name_2: str) -> list: cv_domain_start, cv_domain_end = 0.0, 0.3 - space # Format y-axis labels with HTML for proper alignment - capacity_unit = _get_capacity_unit(c, mode=y.split("_")[-1]) + mode = y.split("_")[-1] + if dev_mode: + mode = "gravimetric" + capacity_unit = _get_capacity_unit(c, mode=mode) + ce_label = "Coulombic
Efficiency (%)" capacity_label = f"Capacity
({capacity_unit})" if fullcell_standard_normalization_type: _norm_label = f"[{fullcell_standard_normalization_scaler:.1f}/{fullcell_standard_normalization_factor:.1f} {capacity_unit}]" - loss_label = f"Cumulated
Loss (norm.)
{_norm_label}" + loss_label = f"Capacity
Retention (norm.)
{_norm_label}" else: - loss_label = f"Cumulated
Loss ({capacity_unit})" + loss_label = f"Capacity
Retention ({capacity_unit})" cv_label = f"CV Capacity
({capacity_unit})" fig.update_layout( From ec91d632683d4c4fefd7fc747245100980d001ea Mon Sep 17 00:00:00 2001 From: jepegit Date: Sat, 21 Jun 2025 20:14:40 +0200 Subject: [PATCH 16/37] more tweaks on summaryplot - seems to work OK now --- cellpy/utils/plotutils.py | 214 +++++++++++++++----------------------- 1 file changed, 82 insertions(+), 132 deletions(-) diff --git a/cellpy/utils/plotutils.py b/cellpy/utils/plotutils.py index d2b8b36b..b389f1f8 100644 --- a/cellpy/utils/plotutils.py +++ b/cellpy/utils/plotutils.py @@ -11,7 +11,7 @@ import os import pickle as pkl import sys -from typing import Any, Callable +from typing import Any, Callable, Optional import warnings from io import StringIO from pathlib import Path @@ -727,9 +727,11 @@ def create_label_dict(c): "voltages": f"Voltage ({c.cellpy_units.voltage})", "capacities_gravimetric": _cap_gravimetric_label, "capacities_areal": _cap_areal_label, + "capacities_absolute": _cap_absolute_label, "capacities": _cap_label, "capacities_gravimetric_split_constant_voltage": _cap_gravimetric_label, "capacities_areal_split_constant_voltage": _cap_areal_label, + "capacities_absolute_split_constant_voltage": _cap_absolute_label, "capacities_gravimetric_coulombic_efficiency": _cap_gravimetric_label, "capacities_areal_coulombic_efficiency": _cap_areal_label, "capacities_absolute_coulombic_efficiency": _cap_absolute_label, @@ -756,17 +758,17 @@ def _get_capacity_unit(c, mode="gravimetric", seperator="/"): # TODO: add support for standard 4-pane plot def summary_plot( c, - x: str = None, - y: str = "capacities_gravimetric_coulombic_efficiency", - height: int = None, + x: Optional[str] = None, + y: str = "capacities_gravimetric_coulombic_efficiency", # Consider setting default to 'fullcell_standard_gravimetric' + height: Optional[int] = None, width: int = 900, markers: bool = True, - title=None, - x_range: list = None, - y_range: list = None, - ce_range: list = None, - norm_range: list = None, - cv_share_range: list = None, + title: Optional[str] = None, + x_range: Optional[list] = None, + y_range: Optional[list] = None, + ce_range: Optional[list] = None, + norm_range: Optional[list] = None, + cv_share_range: Optional[list] = None, split: bool = True, auto_convert_legend_labels: bool = True, interactive: bool = True, @@ -774,22 +776,22 @@ def summary_plot( rangeslider: bool = False, return_data: bool = False, verbose: bool = False, - plotly_template: str = None, + plotly_template: Optional[str] = None, seaborn_palette: str = "deep", seaborn_style: str = "dark", - formation_cycles=3, - show_formation=True, - show_legend=True, - x_axis_domain_formation_fraction=0.2, - column_separator=0.01, - reset_losses=True, - link_capacity_scales=False, - fullcell_standard_normalization_type="on-max", - fullcell_standard_normalization_factor=None, - fullcell_standard_normalization_scaler=1.0, - seaborn_line_hooks: list[tuple[str, list, dict]] = None, + formation_cycles: int = 3, + show_formation: bool = True, + show_legend: bool = True, + x_axis_domain_formation_fraction: float = 0.2, + column_separator: float = 0.01, + reset_losses: bool = True, + link_capacity_scales: bool = False, + fullcell_standard_normalization_type: str = "on-max", + fullcell_standard_normalization_factor: Optional[float] = None, + fullcell_standard_normalization_scaler: float = 1.0, + seaborn_line_hooks: Optional[list[tuple[str, list, dict]]] = None, **kwargs, -): +) -> Any: """Create a summary plot. Args: @@ -827,8 +829,10 @@ def summary_plot( column_separator: separation between columns when splitting the plot (only for plotly) reset_losses: reset the losses to the first cycle (only for fullcell_standard plots) link_capacity_scales: link the capacity scales (only for fullcell_standard plots) - fullcell_standard_normalization_type: normalization type for the fullcell standard plots (Cumulated loss) (divide, multiply, area, max, False) + fullcell_standard_normalization_type: normalization type for the fullcell standard plots (capacity retention) (divide, multiply, area, max, on-max, False) + if normalization_type is on-max, the normalization factor is set to the maximum value of the capacity column if not provided if normalization_type is max, the normalization factor is set to the maximum value of the capacity column if not provided + if normalization_type is shift-divide, the normalization is done by shifting the data by the normalization factor and then dividing by the normalization factor if normalization_type is divide, the normalization is done by dividing by the normalization factor and then multiplying by the scaler if normalization_type is multiply, the normalization is done by multiplying by the normalization factor and then multiplying by the scaler if normalization_type is area, the normalization is done by dividing by the area and then multiplying by the scaler @@ -867,9 +871,6 @@ def summary_plot( dev_mode = kwargs.pop("dev_mode", False) if dev_mode: print("DEV: dev_mode") - fullcell_standard_normalization_type = "shift-divide" - fullcell_standard_normalization_factor = 120.0 - fullcell_standard_normalization_scaler = 100.0 smart_link = kwargs.pop("smart_link", True) show_y_labels_on_right_pane = kwargs.pop("show_y_labels_on_right_pane", False) @@ -879,6 +880,13 @@ def summary_plot( seaborn_style_dict = kwargs.pop("seaborn_style_dict", seaborn_style_dict_default) seaborn_marker_size = kwargs.pop("seaborn_marker_size", 7) + # only used for fullcell_standard plots in interactive mode for now + plotly_row_ratios = kwargs.pop("fullcell_standard_row_height_ratios", [0.3, 0.6, 0.9]) + plotly_row_space = kwargs.pop("fullcell_standard_row_space", 0.02) + # fullcell_standard does not respect the split parameter + if y.startswith("fullcell_standard_") and not split: + logging.debug("fullcell_standard does not respect the split parameter") + number_of_rows = 1 max_val_normalized_col = 0.0 @@ -976,8 +984,7 @@ def _auto_range(fig: Any, axis_name_1: str, axis_name_2: str) -> list: s = s.reset_index(drop=True) s = s.melt(x) s = s.loc[s.variable.isin(column_set)] # using strickt naming convention for "duplicated" columns ('mod__' so it will not be picked up here) - if dev_mode: - print(f"{column_set=} -> {s.variable.unique()}") + number_of_rows = 4 s[row] = 1 # default row for capacity # Set row numbers using regex patterns @@ -988,8 +995,6 @@ def _auto_range(fig: Any, axis_name_1: str, axis_name_2: str) -> list: additional_kwargs_plotly["facet_row"] = row if reset_losses: - if dev_mode: - print("DEV: reset_losses") # Get the first value for each cumulated loss variable first_values = s[s["variable"].str.contains(r"cumulated.*loss")].groupby("variable")["value"].transform("first") # Shift all values by subtracting the first value @@ -997,34 +1002,34 @@ def _auto_range(fig: Any, axis_name_1: str, axis_name_2: str) -> list: s.loc[mask, "value"] = s.loc[mask, "value"] - first_values - # old normalization code: if fullcell_standard_normalization_type is not False: - if dev_mode: - print(f"{fullcell_standard_normalization_type=}") - print(f"{fullcell_standard_normalization_factor=}") - print(f"{fullcell_standard_normalization_scaler=}") - print("\nTransformations:") - print(f"{y_trans.get(y, {}).items()=}") if fullcell_standard_normalization_factor is None: - if fullcell_standard_normalization_type == "on-max": + # need a special case for the cumloss plots + if y.startswith("fullcell_standard_cumloss_"): + print("only allowing for 'divide' for cumloss plots") fullcell_standard_normalization_factor = s[s[row] == 1].max().value - fullcell_standard_normalization_type = "shift-divide" + fullcell_standard_normalization_type = "divide" - elif fullcell_standard_normalization_type == "max": - fullcell_standard_normalization_factor = s[s[row] == 1].max().value - fullcell_standard_normalization_type = "shift-divide" + else: + if fullcell_standard_normalization_type == "on-max": + fullcell_standard_normalization_factor = s[s[row] == 1].max().value + fullcell_standard_normalization_type = "shift-divide" - elif fullcell_standard_normalization_type == "area": - with warnings.catch_warnings(): - warnings.simplefilter("ignore") - area = np.trapezoid(s[s[row] == 1].value, dx=1) - fullcell_standard_normalization_factor = area - fullcell_standard_normalization_type = "shift-divide" + elif fullcell_standard_normalization_type == "max": + fullcell_standard_normalization_factor = s[s[row] == 1].max().value + fullcell_standard_normalization_type = "shift-divide" - else: - fullcell_standard_normalization_factor = 1.0 + elif fullcell_standard_normalization_type == "area": + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + area = np.trapezoid(s[s[row] == 1].value, dx=1) + fullcell_standard_normalization_factor = area + fullcell_standard_normalization_type = "shift-divide" + + else: + fullcell_standard_normalization_factor = 1.0 trans_kwargs = dict( normalization_factor=fullcell_standard_normalization_factor, @@ -1032,9 +1037,6 @@ def _auto_range(fig: Any, axis_name_1: str, axis_name_2: str) -> list: normalization_scaler=fullcell_standard_normalization_scaler, ) - if dev_mode: - print(f"{trans_kwargs=}") - print(f"{y_trans.get(y, {}).items()=}") # transform the data max_row_val = s[row].max() @@ -1057,34 +1059,20 @@ def _auto_range(fig: Any, axis_name_1: str, axis_name_2: str) -> list: else: max_row_val += 1 row_val = max_row_val - if dev_mode: - print(f"{row_val=}") if old_col.startswith("mod_"): old_col = re.sub(r'^mod_\d{2}_', '', old_col) new_col_frame_section = s.loc[s["variable"] == old_col].copy() new_col_frame_section["variable"] = new_col new_col_frame_section["row"] = row_val - if dev_mode: - print(f"{old_col=} -> {new_col_frame_section.head()=}") transformed_values = trans(new_col_frame_section["value"].values, **trans_kwargs) new_col_frame_section["value"] = transformed_values s = pd.concat([s, new_col_frame_section], ignore_index=True) s = s.reset_index(drop=True) s = s.sort_values(by=["row", "variable"]) - if dev_mode: - print(f"{new_col=} -> {s.loc[s['variable'] == new_col, 'value'].values[:5]}") - max_val_normalized_col = s.loc[s["variable"] == new_col, "value"].max() - if dev_mode: - print("----------------------------------------------") - print(f"{s.variable.unique()=}") - print(f"{s.variable.value_counts()=}") - print(f"{column_set=}") - print(f"{s=}") - # filter on constant voltage vs constant current # Remark! absoulte capacities are not implemented yet. # Remark! uses the 'partition_summary_cv_steps' function - consider using that also for the fullcell standard plot to avoid code duplication @@ -1153,13 +1141,6 @@ def _auto_range(fig: Any, axis_name_1: str, axis_name_2: str) -> list: if show_formation: additional_kwargs_plotly["facet_col"] = col_id - if dev_mode: - print(f"{y_header=}") - print(f"{y_label=}") - print(f"{additional_kwargs_plotly=}") - print(f"{x_label=}") - print(f"{kwargs=}") - fig = px.line( s, x=x, @@ -1171,16 +1152,6 @@ def _auto_range(fig: Any, axis_name_1: str, axis_name_2: str) -> list: }, **kwargs, ) - if dev_mode: - print("------------giving up here---------------------------") - # Error in row value in the data - print(f"{s=}") - print(f"{s.variable.unique()=}") - print(f"{s.variable.value_counts()=}") - print(f"{fig.data=}") - print(f"{fig.layout=}") - print("-----------------------------------------------------") - # return fig, s fig.update_traces(**additional_kwargs_plotly_update_traces) if not show_legend: @@ -1287,9 +1258,6 @@ def _auto_range(fig: Any, axis_name_1: str, axis_name_2: str) -> list: fig.layout["annotations"] = annotations elif number_of_rows == 4: - if dev_mode: - print("DEV: number_of_rows == 4") - print(f"{fig.data=}") fig.update_yaxes(matches="y") fig.update_yaxes(autorange=False) fig.update_layout(xaxis_domain=x_axis_domain_formation, scene_domain_x=x_axis_domain_formation) @@ -1303,12 +1271,6 @@ def _auto_range(fig: Any, axis_name_1: str, axis_name_2: str) -> list: range_2 = _auto_range(fig, "y3", "y4") range_3 = _auto_range(fig, "y5", "y6") - - if dev_mode: - print(f"{range_1=}") - print(f"{range_2=}") - print(f"{range_3=}") - range_4 = _auto_range(fig, "y7", "y8") if y.startswith("fullcell_standard_"): @@ -1336,16 +1298,13 @@ def _auto_range(fig: Any, axis_name_1: str, axis_name_2: str) -> list: fig.layout["annotations"] = annotations if y.startswith("fullcell_standard_"): - space = 0.02 - ce_domain_start, ce_domain_end = 0.9, 1.0 - capacity_domain_start, capacity_domain_end = 0.6, 0.9 - space - loss_domain_start, loss_domain_end = 0.3, 0.6 - space - cv_domain_start, cv_domain_end = 0.0, 0.3 - space + ce_domain_start, ce_domain_end = plotly_row_ratios[2], 1.0 + capacity_domain_start, capacity_domain_end = plotly_row_ratios[1], plotly_row_ratios[2] - plotly_row_space + loss_domain_start, loss_domain_end = plotly_row_ratios[0], plotly_row_ratios[1] - plotly_row_space + cv_domain_start, cv_domain_end = 0.0, plotly_row_ratios[0] - plotly_row_space # Format y-axis labels with HTML for proper alignment mode = y.split("_")[-1] - if dev_mode: - mode = "gravimetric" capacity_unit = _get_capacity_unit(c, mode=mode) ce_label = "Coulombic
Efficiency (%)" @@ -1395,30 +1354,20 @@ def _auto_range(fig: Any, axis_name_1: str, axis_name_2: str) -> list: yaxis2={"title": dict(text="Coulombic Efficiency"), "domain": [0.7, 1.0]}, ) if y.startswith("fullcell_standard_"): + range_1 = eff_lim or _auto_range(fig, "y4", "y4") + range_2 = y_range or _auto_range(fig, "y3", "y3") + range_3 = _auto_range(fig, "y2", "y2") if fullcell_standard_normalization_type is not False: - range_2 = [0.0, max(max_val_normalized_col, fullcell_standard_normalization_scaler)] - range_2 = norm_range or range_2 - fig.update_layout(yaxis2=dict(range=range_2)) - if eff_lim is not None: - # update yaxis2 - fig.update_layout(yaxis4=dict(range=eff_lim)) - if cv_share_range is not None: - fig.update_layout(yaxis=dict(range=cv_share_range)) - - capacity_unit = _get_capacity_unit(c, mode=y.split("_")[-1]) - fig.update_layout( - yaxis1={"title": dict(text="CV Capacity")}, - yaxis2={"title": dict(text="Cumulated
Loss")}, - yaxis3={"title": dict(text=f"Capacity
({capacity_unit})")}, - yaxis4={"title": dict(text="Coulombic
Efficiency")}, - ) + range_3 = [0.0, max(max_val_normalized_col, fullcell_standard_normalization_scaler)] + range_3 = norm_range or range_3 + + range_4 = cv_share_range or _auto_range(fig, "y", "y") fig.layout["annotations"] = 4 * [PLOTLY_BLANK_LABEL] - space = 0.02 - ce_domain_start, ce_domain_end = 0.9, 1.0 - capacity_domain_start, capacity_domain_end = 0.6, 0.9 - space - loss_domain_start, loss_domain_end = 0.3, 0.6 - space - cv_domain_start, cv_domain_end = 0.0, 0.3 - space + ce_domain_start, ce_domain_end = plotly_row_ratios[2], 1.0 + capacity_domain_start, capacity_domain_end = plotly_row_ratios[1], plotly_row_ratios[2] - plotly_row_space + loss_domain_start, loss_domain_end = plotly_row_ratios[0], plotly_row_ratios[1] - plotly_row_space + cv_domain_start, cv_domain_end = 0.0, plotly_row_ratios[0] - plotly_row_space # Format y-axis labels with HTML for proper alignment capacity_unit = _get_capacity_unit(c, mode=y.split("_")[-1]) @@ -1426,17 +1375,17 @@ def _auto_range(fig: Any, axis_name_1: str, axis_name_2: str) -> list: capacity_label = f"Capacity
({capacity_unit})" if fullcell_standard_normalization_type: _norm_label = f"[{fullcell_standard_normalization_scaler:.1f}/{fullcell_standard_normalization_factor:.1f} {capacity_unit}]" - loss_label = f"Cumulated
Loss (norm.)
{_norm_label}" + loss_label = f"Capacity
Retention (norm.)
{_norm_label}" else: - loss_label = f"Cumulated
Loss ({capacity_unit})" + loss_label = f"Capacity
Retention ({capacity_unit})" cv_label = f"CV Capacity
({capacity_unit})" fig.update_layout( - yaxis4={"title": dict(text=ce_label), "domain": [ce_domain_start, ce_domain_end]}, - yaxis3={"title": dict(text=capacity_label), "domain": [capacity_domain_start, capacity_domain_end]}, - yaxis2={"title": dict(text=loss_label), "domain": [loss_domain_start, loss_domain_end]}, - yaxis1={"title": dict(text=cv_label), "domain": [cv_domain_start, cv_domain_end]}, + yaxis4={"title": dict(text=ce_label), "domain": [ce_domain_start, ce_domain_end], "matches": None, "range": range_1}, + yaxis3={"title": dict(text=capacity_label), "domain": [capacity_domain_start, capacity_domain_end], "matches": None, "range": range_2}, + yaxis2={"title": dict(text=loss_label), "domain": [loss_domain_start, loss_domain_end], "matches": None, "range": range_3}, + yaxis={"title": dict(text=cv_label), "domain": [cv_domain_start, cv_domain_end], "matches": None, "range": range_4}, ) if x_range is not None: @@ -1757,18 +1706,18 @@ def _calculate_seaborn_plot_properties(number_of_rows, number_of_cols, plot_type info_dicts.append(_d) elif is_fullcell_standard_plot: - print("Warning: Experimental feature - fullcell standard plot") + capacity_unit = _get_capacity_unit(c, mode=y.split("_")[-1]) ce_label = "Coulombic\nEfficiency (%)" capacity_label = f"Capacity\n({capacity_unit})" - loss_label = f"Cumulated Loss\n({capacity_unit})" + loss_label = f"Capacity\nRetention\n({capacity_unit})" if fullcell_standard_normalization_type: _norm_label = f"[{fullcell_standard_normalization_scaler:.1f}/{fullcell_standard_normalization_factor:.1f} {capacity_unit}]" - loss_label = f"Cumulated\nLoss (norm.)\n{_norm_label}" + loss_label = f"Capacity\nRetention (norm.)\n{_norm_label}" else: - loss_label = f"Cumulated\nLoss\n({capacity_unit})" + loss_label = f"Capacity\nRetention\n({capacity_unit})" cv_label = f"CV Capacity\n({capacity_unit})" @@ -1947,6 +1896,7 @@ def _calculate_seaborn_plot_properties(number_of_rows, number_of_cols, plot_type ) info_dicts.append(_d) + facet_kws["gridspec_kws"] = gridspec_kws sns_fig = sns.relplot( From 7f31015793f77c9f50292cbb1dcd3781673de080 Mon Sep 17 00:00:00 2001 From: jepegit Date: Sat, 21 Jun 2025 21:42:00 +0200 Subject: [PATCH 17/37] now with notebook_docstring_printer --- cellpy/utils/plotutils.py | 106 ++++++++++++++++++++++++++++---------- 1 file changed, 80 insertions(+), 26 deletions(-) diff --git a/cellpy/utils/plotutils.py b/cellpy/utils/plotutils.py index b389f1f8..dd6f5faf 100644 --- a/cellpy/utils/plotutils.py +++ b/cellpy/utils/plotutils.py @@ -51,6 +51,50 @@ PLOTLY_BASE_TEMPLATE = "plotly" IMAGE_TO_FILE_TIMEOUT = 30 +def notebook_docstring_printer(func, default_show_docstring=False): + """ + Decorator that prints the function's docstring when called from a notebook environment. + + This decorator checks if the function is being called from a Jupyter notebook + or IPython environment and prints the function's docstring if it is. + + Args: + func: The function to decorate + + Returns: + The decorated function + """ + + def wrapper(*args, **kwargs): + # Check if we're in a notebook environment + show_docstring = kwargs.pop("show_docstring", default_show_docstring) + if show_docstring: + try: + # Check for IPython/Jupyter environment + import IPython + ipython = IPython.get_ipython() + if ipython is not None and hasattr(ipython, 'kernel'): + # We're in a notebook environment + if func.__doc__: + print(f"{func.__name__} docstring:") + print("-" * (len(func.__name__) + 12)) + print(func.__doc__) + print("-" * (len(func.__name__) + 12)) + else: + print(f"No docstring found for {func.__name__}") + except (ImportError, AttributeError): + # Not in a notebook environment, continue silently + pass + + # Call the original function + return func(*args, **kwargs) + + # Preserve the original function's metadata + wrapper.__name__ = func.__name__ + wrapper.__doc__ = func.__doc__ + wrapper.__module__ = func.__module__ + + return wrapper # from collectors - tools for loading and saving plots: def load_figure(filename, backend=None): @@ -134,7 +178,8 @@ def _image_exporter_plotly(figure, filename, timeout=IMAGE_TO_FILE_TIMEOUT, **kw print(f" - saved image file: {filename}") -def save_image_files(figure, name="my_figure", scale=3.0, dpi=300, backend="plotly", formats: list = None): +@notebook_docstring_printer +def save_image_files(figure: Any, name: str = "my_figure", scale: float = 3.0, dpi: int = 300, backend: str = "plotly", formats: Optional[list] = None): """Save to image files (png, svg, json/pickle). Notes: @@ -536,18 +581,42 @@ def create_colormarkerlist(groups, sub_groups, symbol_label="all", color_style_l return _color_list, _symbol_list -def create_col_info(c): +def create_col_info(c: Any) -> tuple[tuple, dict, dict, dict]: """Create column information for summary plots. + This function is called by summary_plot together with create_label_dict. The two functions need to be updated together. + Not optimal. So feel free to refactor it. + Args: c: cellpy object Returns: - x_columns (tuple), y_cols (dict) + x_columns (tuple), y_cols (dict), x_transformations (dict), y_transformations (dict) """ + + def _normalize_col(x: np.ndarray, normalization_factor: float = 1.0, normalization_type: str = "max", normalization_scaler: float = 1.0) -> np.ndarray: + # a bit random collection of normalization types... + + if normalization_type == "divide": + return (x / normalization_factor) * normalization_scaler + elif normalization_type == "shift-divide": + return ((normalization_factor - x) / normalization_factor) * normalization_scaler + elif normalization_type == "multiply": + return (x * normalization_factor) * normalization_scaler + elif normalization_type == "area": + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + area = np.trapzoid(x, dx=1) + return (x / area / normalization_factor) * normalization_scaler + elif normalization_type == "max": + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + x_max = x.max() + return (x / x_max / normalization_factor) * normalization_scaler + else: + raise ValueError(f"Invalid normalization type: {normalization_type}") - # TODO: add support for more column sets and individual columns hdr = c.headers_summary _cap_cols = [hdr.charge_capacity_raw, hdr.discharge_capacity_raw] _capacities_gravimetric = [col + "_gravimetric" for col in _cap_cols] @@ -635,25 +704,6 @@ def create_col_info(c): x_transformations = dict( ) - def _normalize_col(x: np.ndarray, normalization_factor: float = 1.0, normalization_type: str = "max", normalization_scaler: float = 1.0) -> np.ndarray: - if normalization_type == "divide": - return (x / normalization_factor) * normalization_scaler - elif normalization_type == "shift-divide": - return ((normalization_factor - x) / normalization_factor) * normalization_scaler - elif normalization_type == "multiply": - return (x * normalization_factor) * normalization_scaler - elif normalization_type == "area": - with warnings.catch_warnings(): - warnings.simplefilter("ignore") - area = np.trapzoid(x, dx=1) - return (x / area / normalization_factor) * normalization_scaler - elif normalization_type == "max": - with warnings.catch_warnings(): - warnings.simplefilter("ignore") - x_max = x.max() - return (x / x_max / normalization_factor) * normalization_scaler - else: - raise ValueError(f"Invalid normalization type: {normalization_type}") # transformation info on the form: column_name: {(row_number, new_column_name): transformation_function} y_transformations: dict[str, dict[tuple[int, str], dict[str, Callable]]] = dict( @@ -700,6 +750,9 @@ def _normalize_col(x: np.ndarray, normalization_factor: float = 1.0, normalizati def create_label_dict(c): """Create label dictionary for summary plots. + This function is called by summary_plot together with create_col_info. The two functions need to be updated together. + Not optimal. So feel free to refactor it. + Args: c: cellpy object @@ -722,7 +775,6 @@ def create_label_dict(c): _cap_absolute_label = f"Capacity ({c.cellpy_units.charge})" _cap_label = f"Capacity ({c.data.raw_units.charge})" - # TODO: probably need to add something for cumulated discharge capacity loss also... y_axis_label = { "voltages": f"Voltage ({c.cellpy_units.voltage})", "capacities_gravimetric": _cap_gravimetric_label, @@ -752,10 +804,9 @@ def _get_capacity_unit(c, mode="gravimetric", seperator="/"): return specific_selector.get(mode, "-") -# TODO: add formation cycles handling for seaborn # TODO: consistent parameter names (e.g. y_range vs ylim) between summary_plot, plot_cycles, raw_plot, cycle_info_plot and batchutils # TODO: consistent function names (raw_plot vs plot_raw etc) -# TODO: add support for standard 4-pane plot +@notebook_docstring_printer def summary_plot( c, x: Optional[str] = None, @@ -3021,6 +3072,9 @@ def _check_summary_plotter_plotly(): fig.show() + + + if __name__ == "__main__": # _check_plotter_plotly() # _check_plotter_matplotlib() From fa9e6290c6d87bbc58a04f5f7db22647c55cbb2c Mon Sep 17 00:00:00 2001 From: jepegit Date: Sat, 21 Jun 2025 21:57:21 +0200 Subject: [PATCH 18/37] clean up a bit --- cellpy/utils/plotutils.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/cellpy/utils/plotutils.py b/cellpy/utils/plotutils.py index dd6f5faf..119ac8a7 100644 --- a/cellpy/utils/plotutils.py +++ b/cellpy/utils/plotutils.py @@ -849,7 +849,7 @@ def summary_plot( c: cellpy object x: x-axis column (default: 'cycle_index') y: y-axis column or column set. Currently, the following predefined sets exists: - "voltages", "capacities_gravimetric", "capacities_areal", "capacities", + "voltages", "capacities_gravimetric", "capacities_areal", "capacities_absolute", "capacities_gravimetric_split_constant_voltage", "capacities_areal_split_constant_voltage", "capacities_gravimetric_coulombic_efficiency", "capacities_areal_coulombic_efficiency", "capacities_absolute_coulombic_efficiency", @@ -880,12 +880,16 @@ def summary_plot( column_separator: separation between columns when splitting the plot (only for plotly) reset_losses: reset the losses to the first cycle (only for fullcell_standard plots) link_capacity_scales: link the capacity scales (only for fullcell_standard plots) - fullcell_standard_normalization_type: normalization type for the fullcell standard plots (capacity retention) (divide, multiply, area, max, on-max, False) + fullcell_standard_normalization_type: normalization type for the fullcell standard plots (capacity retention) + (divide, multiply, area, max, on-max, False) if normalization_type is on-max, the normalization factor is set to the maximum value of the capacity column if not provided if normalization_type is max, the normalization factor is set to the maximum value of the capacity column if not provided - if normalization_type is shift-divide, the normalization is done by shifting the data by the normalization factor and then dividing by the normalization factor - if normalization_type is divide, the normalization is done by dividing by the normalization factor and then multiplying by the scaler - if normalization_type is multiply, the normalization is done by multiplying by the normalization factor and then multiplying by the scaler + if normalization_type is shift-divide, the normalization is done by shifting the data by the normalization factor and + then dividing by the normalization factor + if normalization_type is divide, the normalization is done by dividing by the normalization factor and + then multiplying by the scaler + if normalization_type is multiply, the normalization is done by multiplying by the normalization factor + and then multiplying by the scaler if normalization_type is area, the normalization is done by dividing by the area and then multiplying by the scaler if normalization_type is False, no normalization is done fullcell_standard_normalization_factor: normalization factor for the fullcell standard plots @@ -904,9 +908,8 @@ def summary_plot( Hint: If you want to modify the non-interactive (matplotlib) plot, you can get the axes from the - returned figure by ``axes = figure.get_axes()``. + returned figure by ``axes = figure.get_axes()``: - Example: >> axes = figure.get_axes() >> ylabel = axes[0].get_ylabel() >> if "Coulombic" in ylabel: @@ -1125,7 +1128,6 @@ def _auto_range(fig: Any, axis_name_1: str, axis_name_2: str) -> list: max_val_normalized_col = s.loc[s["variable"] == new_col, "value"].max() # filter on constant voltage vs constant current - # Remark! absoulte capacities are not implemented yet. # Remark! uses the 'partition_summary_cv_steps' function - consider using that also for the fullcell standard plot to avoid code duplication elif y.endswith("_split_constant_voltage"): cap_type = "capacities_gravimetric" if y.startswith("capacities_gravimetric") else "capacities_areal" @@ -1326,6 +1328,7 @@ def _auto_range(fig: Any, axis_name_1: str, axis_name_2: str) -> list: if y.startswith("fullcell_standard_"): range_4 = eff_lim or range_4 + range_3 = y_range or range_3 range_1 = cv_share_range or range_1 fig.update_layout( From 40784d53b1d0854912afecc3c73b006636749c2a Mon Sep 17 00:00:00 2001 From: jepegit Date: Sun, 22 Jun 2025 13:13:04 +0200 Subject: [PATCH 19/37] concat_summaries update - allow for hooks and cv-share --- cellpy/utils/helpers.py | 79 +++++++++++++++++++++++++++++------------ 1 file changed, 57 insertions(+), 22 deletions(-) diff --git a/cellpy/utils/helpers.py b/cellpy/utils/helpers.py index fdc2ad3d..94fd1d8a 100644 --- a/cellpy/utils/helpers.py +++ b/cellpy/utils/helpers.py @@ -1,6 +1,7 @@ import logging import os import pathlib +from typing import Optional import warnings from copy import deepcopy @@ -96,7 +97,7 @@ def _make_average( if col == hdr_norm_cycle and skip_st_dev_for_equivalent_cycle_index: if number_of_cols > 1: normalized_cycle_index_frame = ( - new_frame[col].agg(["mean"], axis=1).rename(columns={"mean": "equivalent_cycle"}) + new_frame[col].agg(["mean"], skipna=True, axis=1).rename(columns={"mean": "equivalent_cycle"}) ) else: normalized_cycle_index_frame = new_frame[col].copy() @@ -106,7 +107,7 @@ def _make_average( new_col_name_std = "std" if number_of_cols > 1: - avg_frame = new_frame[col].agg(["mean", "std"], axis=1) + avg_frame = new_frame[col].agg(["mean", "std"], skipna=True, axis=1) else: avg_frame = pd.DataFrame(data=new_frame[col].values, columns=[new_col_name_mean]) avg_frame[new_col_name_std] = not_a_number @@ -829,32 +830,24 @@ def add_cv_step_columns(columns: list) -> list: def _partition_summary_based_on_cv_steps( c, - column_set: list, + column_set: Optional[list] = None, x: str = None, - ): - # TODO: seems to be a bug here - probably due to concatinating (missing cycles) """Partition the summary data into CV and non-CV steps. Args: c: cellpy object - x: x-axis column name column_set: names of columns to include - split: add additional column that can be used to split the data when plotting. - var_name: name of the variable column after melting - value_name: name of the value column after melting + x: x-axis column name (default is "cycle_index") Returns: - ``pandas.DataFrame`` (melted with columns x, var_name, value_name, and optionally "row" if split is True) + ``pandas.DataFrame`` """ import pandas as pd if not x: x = hdr_summary["cycle_index"] - # in case the column set already contains cv cols: - column_set = [col for col in column_set if not "_cv" in col] - summary = c.data.summary.copy() summary_no_cv = c.make_summary(selector_type="non-cv", create_copy=True).data.summary @@ -864,6 +857,17 @@ def _partition_summary_based_on_cv_steps( summary_no_cv.set_index(x, inplace=True, drop=True) summary_only_cv.set_index(x, inplace=True, drop=True) + + + if column_set is None: + column_set = summary.columns.tolist() + else: + # allow for non-existing columns in the dataframe: + column_set = [col for col in column_set if col in summary.columns] + + # in case the column set already contains cv cols: + column_set = [col for col in column_set if not "_cv" in col] + summary = summary[column_set] summary_no_cv = summary_no_cv[column_set] @@ -876,6 +880,7 @@ def _partition_summary_based_on_cv_steps( return s + def concat_summaries( b: Batch, max_cycle=None, @@ -900,6 +905,11 @@ def concat_summaries( only_selected=False, experimental_feature_cell_selector=None, partition_by_cv=False, + replace_inf_with_nan=True, + individual_summary_hooks=None, + concatenated_summary_hooks=None, + *args, + **kwargs, ) -> pd.DataFrame: """Merge all summaries in a batch into a gigantic summary data frame. @@ -932,12 +942,20 @@ def concat_summaries( recalc_step_table_kwargs (dict): keyword arguments to be used when recalculating the step table. If not given, it will not recalculate the step table. only_selected (bool): only use the selected cells. + experimental_feature_cell_selector (list): list of cell names to select. + partition_by_cv (bool): if True, partition the data by cv_step. + replace_inf_with_nan (bool): if True, replace inf with nan in the concatenated summary data. + individual_summary_hooks (list): list of functions to be applied to the individual summary data. + concatenated_summary_hooks (list): list of functions to be applied to the concatenated summary data. + + *args,**kwargs: additional arguments to be passed to the hooks. Returns: ``pandas.DataFrame`` """ if key_index_bounds is None: + # TODO: consider changing this to [1, -1] key_index_bounds = [1, -2] cell_names_nest = [] @@ -1023,6 +1041,8 @@ def concat_summaries( output_columns = add_cv_step_columns(output_columns) for gno, cell_names in zip(group_nest, cell_names_nest): + # NOTE: to allow for hooks to add columns, all functions that operates in this loop + # must allow for non-existing columns in the dataframe! frames_sub = [] keys_sub = [] for cell_id in cell_names: @@ -1066,9 +1086,8 @@ def concat_summaries( if rate is not None: # TODO: update this so that it works with partitioned data if partition_by_cv: - print("partitioning by cv_step is not possible with rate selection") - print("skipping rate selection") - continue + print("partitioning by cv_step is experimental for rate selection") + s = select_summary_based_on_rate( c, rate=rate, @@ -1077,16 +1096,21 @@ def concat_summaries( rate_column=rate_column, inverse=inverse, inverted=inverted, + partition_by_cv=partition_by_cv, ) elif partition_by_cv: - print("partitioning by cv_step") s = _partition_summary_based_on_cv_steps(c, column_set=output_columns) - print("partition done") - print(f"{s.columns=}") else: s = c.data.summary + # ADD ADDITIONAL PROCESSING HERE (e.g. copy-and-normalization) + if individual_summary_hooks is not None: + logging.info("Experimental feature: applying individual summary hooks") + for hook in individual_summary_hooks: + logging.info(f" -applying {hook.__name__} to {cell_id}") + s, output_columns = hook(s, columns=output_columns, *args, **kwargs) + if columns is not None: s = s.loc[:, output_columns].copy() @@ -1118,8 +1142,10 @@ def concat_summaries( logging.info("Got several columns with same test-name") logging.info("Renaming.") keys = fix_group_names(keys) + if replace_inf_with_nan: + frames = [frame.replace([np.inf, -np.inf], np.nan) for frame in frames] - return collect_frames(frames, group_it, hdr_norm_cycle, keys, normalize_cycles) + return collect_frames(frames, group_it, hdr_norm_cycle, keys, normalize_cycles, concatenated_summary_hooks) else: logging.info("Empty - nothing to concatenate!") return pd.DataFrame() @@ -1196,7 +1222,7 @@ def fix_group_names(keys): return keys -def collect_frames(frames, group_it: bool, hdr_norm_cycle: str, keys: list, normalize_cycles: bool): +def collect_frames(frames, group_it: bool, hdr_norm_cycle: str, keys: list, normalize_cycles: bool, hooks: list = None): """Helper function for concat_summaries.""" cycle_header = "cycle" normalized_cycle_header = "equivalent_cycle" @@ -1213,6 +1239,10 @@ def collect_frames(frames, group_it: bool, hdr_norm_cycle: str, keys: list, norm if normalize_cycles: cdf = cdf.rename(columns={hdr_norm_cycle: normalized_cycle_header}) + if hooks is not None: + for hook in hooks: + cdf = hook(cdf) + return cdf @@ -1232,6 +1262,7 @@ def select_summary_based_on_rate( inverse=False, inverted=False, fix_index=True, + partition_by_cv=False, ): """Select only cycles charged or discharged with a given rate. @@ -1272,7 +1303,11 @@ def select_summary_based_on_rate( cycle_number_header = hdr_summary["cycle_index"] step_table = cell.data.steps - summary = cell.data.summary + + if partition_by_cv: + summary = _partition_summary_based_on_cv_steps(cell.data.summary) + else: + summary = cell.data.summary if summary.index.name != cycle_number_header: warnings.warn(f"{cycle_number_header} not set as index\n" f"Current index :: {summary.index}\n") From 29cfd22ed6880633e1a030d3bf8e3a60857dd275 Mon Sep 17 00:00:00 2001 From: jepegit Date: Sun, 22 Jun 2025 20:15:08 +0200 Subject: [PATCH 20/37] improve concat_summaries (allow for other averaging methods and for filtering out too big or too small values) --- cellpy/utils/helpers.py | 54 ++++++++++++++++++++++++++++------------- 1 file changed, 37 insertions(+), 17 deletions(-) diff --git a/cellpy/utils/helpers.py b/cellpy/utils/helpers.py index 94fd1d8a..3d75f2f0 100644 --- a/cellpy/utils/helpers.py +++ b/cellpy/utils/helpers.py @@ -82,6 +82,7 @@ def _make_average( frames, columns=None, skip_st_dev_for_equivalent_cycle_index=True, + average_method="mean", ): hdr_norm_cycle = hdr_summary["normalized_cycle_index"] not_a_number = np.nan @@ -97,17 +98,17 @@ def _make_average( if col == hdr_norm_cycle and skip_st_dev_for_equivalent_cycle_index: if number_of_cols > 1: normalized_cycle_index_frame = ( - new_frame[col].agg(["mean"], skipna=True, axis=1).rename(columns={"mean": "equivalent_cycle"}) + new_frame[col].agg([average_method], skipna=True, axis=1).rename(columns={average_method: "equivalent_cycle"}) ) else: normalized_cycle_index_frame = new_frame[col].copy() else: - new_col_name_mean = "mean" + new_col_name_mean = average_method new_col_name_std = "std" if number_of_cols > 1: - avg_frame = new_frame[col].agg(["mean", "std"], skipna=True, axis=1) + avg_frame = new_frame[col].agg([average_method, "std"], skipna=True, axis=1) else: avg_frame = pd.DataFrame(data=new_frame[col].values, columns=[new_col_name_mean]) avg_frame[new_col_name_std] = not_a_number @@ -120,12 +121,14 @@ def _make_average( final_frame = pd.concat(new_frames, axis=0) cols = final_frame.columns.to_list() new_cols = [] - for n in ["variable", "mean", "std"]: + for n in ["variable", average_method, "std"]: if n in cols: new_cols.append(n) cols.remove(n) cols.extend(new_cols) final_frame = final_frame.reindex(columns=cols) + # rename the mean column to "mean" for backward compatibility: + final_frame = final_frame.rename(columns={average_method: "mean"}) return final_frame @@ -908,6 +911,10 @@ def concat_summaries( replace_inf_with_nan=True, individual_summary_hooks=None, concatenated_summary_hooks=None, + average_method="mean", + replace_extremes_with_nan=True, + low_limit=-10e5, + high_limit=10e5, *args, **kwargs, ) -> pd.DataFrame: @@ -946,8 +953,14 @@ def concat_summaries( partition_by_cv (bool): if True, partition the data by cv_step. replace_inf_with_nan (bool): if True, replace inf with nan in the concatenated summary data. individual_summary_hooks (list): list of functions to be applied to the individual summary data. - concatenated_summary_hooks (list): list of functions to be applied to the concatenated summary data. - + concatenated_summary_hooks (list): list of functions to be applied to the concatenated summary data + (passed to the collect_frames function). + average_method (str): method to be used when averaging the summary data. Remark that for backward compatibility, + the column name will be "mean" regardless of the actual method used. + replace_extremes_with_nan (bool): if True, replace values outside the range [low_limit, high_limit] with nan + in the concatenated summary data if they are grouped. + low_limit (float): lower limit for replacing extremes with nan if replace_extremes_with_nan is True. + high_limit (float): upper limit for replacing extremes with nan if replace_extremes_with_nan is True. *args,**kwargs: additional arguments to be passed to the hooks. Returns: @@ -1124,9 +1137,10 @@ def concat_summaries( keys_sub.append(cell_id) if group_it: + # TODO: update this to allow for more advanced naming of groups cell_id = create_group_names(custom_group_labels, gno, key_index_bounds, keys_sub, pages) try: - s = _make_average(frames_sub, output_columns) + s = _make_average(frames_sub, output_columns, average_method=average_method) except ValueError as e: print("could not make average!") print(e) @@ -1142,9 +1156,18 @@ def concat_summaries( logging.info("Got several columns with same test-name") logging.info("Renaming.") keys = fix_group_names(keys) + if replace_inf_with_nan: + # a lot of plotting tools do not like inf values, so we replace them with nan frames = [frame.replace([np.inf, -np.inf], np.nan) for frame in frames] + if replace_extremes_with_nan and group_it: + # averaging sometimes gives extreme values, so we replace them with nan + logging.debug(f"Replacing extremes with nan: {low_limit} < mean < {high_limit}") + for frame in frames: + frame.loc[frame["mean"] < low_limit, "mean"] = np.nan + frame.loc[frame["mean"] > high_limit, "mean"] = np.nan + return collect_frames(frames, group_it, hdr_norm_cycle, keys, normalize_cycles, concatenated_summary_hooks) else: logging.info("Empty - nothing to concatenate!") @@ -1154,6 +1177,11 @@ def concat_summaries( def create_group_names(custom_group_labels, gno, key_index_bounds, keys_sub, pages): """Helper function for concat_summaries. + The prioritisation of methods for creating the group name is as follows: + 1. custom_group_labels (if given) + 2. group_label in pages (if given) + 3. key_index_bounds and keys_sub (if no other option is available) + Args: custom_group_labels (dict): dictionary of custom labels (key must be the group number). gno (int): group number. @@ -1165,17 +1193,8 @@ def create_group_names(custom_group_labels, gno, key_index_bounds, keys_sub, pag """ - # TODO: improve this one - cell_id = None - - # print("----------------------------------------------------------------------") - # print(f"custom_group_labels: {custom_group_labels}") - # print(f"gno: {gno}") - # print(f"key_index_bounds: {key_index_bounds}") - # print(f"keys_sub: {keys_sub}") - # print("----------------------------------------------------------------------") - + if custom_group_labels is not None: if isinstance(custom_group_labels, dict): if gno in custom_group_labels: @@ -1199,6 +1218,7 @@ def create_group_names(custom_group_labels, gno, key_index_bounds, keys_sub, pag return cell_id if cell_id is None: + # nothing else worked (or were chosen) - falling back to using key_index_bounds splitter = "_" cell_id = list( set([splitter.join(k.split(splitter)[key_index_bounds[0] : key_index_bounds[1]]) for k in keys_sub]) From 32fc6c212d79187122d34062300ad6e48f57d42e Mon Sep 17 00:00:00 2001 From: jepegit Date: Sun, 22 Jun 2025 21:38:38 +0200 Subject: [PATCH 21/37] add posibility to drop columns and also to filter low and high for non-grouped --- cellpy/utils/helpers.py | 47 +++++++++++++++++++++++++++++++---------- 1 file changed, 36 insertions(+), 11 deletions(-) diff --git a/cellpy/utils/helpers.py b/cellpy/utils/helpers.py index 3d75f2f0..f202d396 100644 --- a/cellpy/utils/helpers.py +++ b/cellpy/utils/helpers.py @@ -911,6 +911,7 @@ def concat_summaries( replace_inf_with_nan=True, individual_summary_hooks=None, concatenated_summary_hooks=None, + drop_columns=None, average_method="mean", replace_extremes_with_nan=True, low_limit=-10e5, @@ -951,14 +952,15 @@ def concat_summaries( only_selected (bool): only use the selected cells. experimental_feature_cell_selector (list): list of cell names to select. partition_by_cv (bool): if True, partition the data by cv_step. - replace_inf_with_nan (bool): if True, replace inf with nan in the concatenated summary data. + replace_inf_with_nan (bool): if True, replace inf with nan in the summary data. individual_summary_hooks (list): list of functions to be applied to the individual summary data. concatenated_summary_hooks (list): list of functions to be applied to the concatenated summary data (passed to the collect_frames function). + drop_columns (list): list of columns to drop before concatenation. average_method (str): method to be used when averaging the summary data. Remark that for backward compatibility, the column name will be "mean" regardless of the actual method used. replace_extremes_with_nan (bool): if True, replace values outside the range [low_limit, high_limit] with nan - in the concatenated summary data if they are grouped. + in the summary data. low_limit (float): lower limit for replacing extremes with nan if replace_extremes_with_nan is True. high_limit (float): upper limit for replacing extremes with nan if replace_extremes_with_nan is True. *args,**kwargs: additional arguments to be passed to the hooks. @@ -1117,15 +1119,21 @@ def concat_summaries( else: s = c.data.summary - # ADD ADDITIONAL PROCESSING HERE (e.g. copy-and-normalization) if individual_summary_hooks is not None: logging.info("Experimental feature: applying individual summary hooks") for hook in individual_summary_hooks: logging.info(f" -applying {hook.__name__} to {cell_id}") - s, output_columns = hook(s, columns=output_columns, *args, **kwargs) + s, output_columns = hook(s, columns=output_columns.copy(), *args, **kwargs) if columns is not None: + # Filter out columns that don't exist in the dataframe to avoid KeyError + output_columns = [col for col in output_columns if col in s.columns] s = s.loc[:, output_columns].copy() + if drop_columns: + logging.debug(f"Dropping columns: {drop_columns}") + logging.debug(f"Columns in s before dropping: {s.columns}") + s = s.drop(columns=drop_columns, errors="ignore") + logging.debug(f"Columns in s after dropping: {s.columns}") # add group and subgroup if not group_it: @@ -1136,10 +1144,15 @@ def concat_summaries( frames_sub.append(s) keys_sub.append(cell_id) + + if group_it: # TODO: update this to allow for more advanced naming of groups cell_id = create_group_names(custom_group_labels, gno, key_index_bounds, keys_sub, pages) try: + # if we used drop_columns, we need to remove them from the output_columns + if drop_columns: + output_columns = [col for col in output_columns if col not in drop_columns] s = _make_average(frames_sub, output_columns, average_method=average_method) except ValueError as e: print("could not make average!") @@ -1156,17 +1169,29 @@ def concat_summaries( logging.info("Got several columns with same test-name") logging.info("Renaming.") keys = fix_group_names(keys) + + if replace_inf_with_nan: # a lot of plotting tools do not like inf values, so we replace them with nan frames = [frame.replace([np.inf, -np.inf], np.nan) for frame in frames] - if replace_extremes_with_nan and group_it: - # averaging sometimes gives extreme values, so we replace them with nan - logging.debug(f"Replacing extremes with nan: {low_limit} < mean < {high_limit}") - for frame in frames: - frame.loc[frame["mean"] < low_limit, "mean"] = np.nan - frame.loc[frame["mean"] > high_limit, "mean"] = np.nan + if replace_extremes_with_nan: + if group_it: + # averaging sometimes gives extreme values, so we replace them with nan + logging.debug(f"Replacing extremes with nan: {low_limit} < mean < {high_limit}") + for frame in frames: + frame.loc[frame["mean"] < low_limit, "mean"] = np.nan + frame.loc[frame["mean"] > high_limit, "mean"] = np.nan + else: + logging.debug(f"Replacing extremes with nan: {low_limit} < column < {high_limit}") + for frame in frames: + # these frames can have multiple of columns that we dont now the name of so we need to iterate over them + # and check if they are floats. + for col in frame.columns: + if pd.api.types.is_float_dtype(frame[col]): + frame.loc[frame[col] < low_limit, col] = np.nan + frame.loc[frame[col] > high_limit, col] = np.nan return collect_frames(frames, group_it, hdr_norm_cycle, keys, normalize_cycles, concatenated_summary_hooks) else: @@ -1194,7 +1219,7 @@ def create_group_names(custom_group_labels, gno, key_index_bounds, keys_sub, pag """ cell_id = None - + if custom_group_labels is not None: if isinstance(custom_group_labels, dict): if gno in custom_group_labels: From 04956b71d28a7648bafad12125f63a8d257b0617 Mon Sep 17 00:00:00 2001 From: jepegit Date: Sun, 22 Jun 2025 22:51:34 +0200 Subject: [PATCH 22/37] start looking at plotting --- cellpy/utils/collectors.py | 69 ++++++++++++++++++++++++++++++++++---- 1 file changed, 62 insertions(+), 7 deletions(-) diff --git a/cellpy/utils/collectors.py b/cellpy/utils/collectors.py index d702ab72..7cd5d9d9 100644 --- a/cellpy/utils/collectors.py +++ b/cellpy/utils/collectors.py @@ -1986,6 +1986,7 @@ def sequence_plotter( if y_label_mapper: annotations = fig.layout.annotations if annotations: + # TODO: THIS IS WHERE I NEED TO FIX THE LABELS try: # might consider a more robust method here - currently # it assumes that the mapper is a list with same order @@ -2387,9 +2388,14 @@ def summary_plotter(collected_curves, cycles_to_plot=None, backend="plotly", **k u_sub = units["cellpy_units"].specific_gravimetric elif v.endswith("_volumetric"): u_sub = units["cellpy_units"].specific_volumetric + u_top = None if "_capacity" in v: u_top = units["cellpy_units"].charge + if "_norm" in v: + u_top = "normalized" + if v == "coulombic_efficiency": + u_top = "%" # creating label: u = u_top or "Value" @@ -2399,12 +2405,15 @@ def summary_plotter(collected_curves, cycles_to_plot=None, backend="plotly", **k u = f"{u}/{u_sub}" v = v[:-1] v = " ".join(v).title() + if v.endswith("Cv"): + v = v.replace("Cv", "CV") label_mapper[y].append(f"{v} ({u})") # TODO: need to refactor and fix how the classes are created so that leftover kwargs are not sent to the backend # (for example if another collector is used and registers a kwarg without popping it) _ = kwargs.pop("method", None) # also set in BatchCyclesCollector + height_fractions = kwargs.pop("height_fractions", []) fig = _cycles_plotter( collected_curves, @@ -2427,13 +2436,59 @@ def summary_plotter(collected_curves, cycles_to_plot=None, backend="plotly", **k if backend == "plotly": # TODO: implement having different heights of the subplots - height_fractions = kwargs.pop("height_fractions", []) - if len(height_fractions) < 0: - # this was suggested by CoPilot (not sure if it works): - for i, h in enumerate(height_fractions): - fig.update_yaxes(row=i + 1, matches=None, showticklabels=True, fixedrange=True) - fig.update_layout(height=fig.layout.height + h) - fig.update_yaxes(matches=None, showticklabels=True) + + if len(height_fractions) > 0: + # Determine number of rows in the original figure + print("THIS IS EXPERIMENTAL") + number_of_rows = len([key for key in fig.layout if key.startswith('yaxis')]) + if number_of_rows == 0: + number_of_rows = 1 # Default to 1 if no y-axes found + + # Only proceed if height_fractions matches the number of rows + if len(height_fractions) != number_of_rows: + print(f"Warning: height_fractions length ({len(height_fractions)}) does not match number of rows ({number_of_rows}). Ignoring height_fractions.") + else: + # Update subplot heights using make_subplots parameters + from plotly.subplots import make_subplots + + # Get current figure data and layout properties + current_data = fig.data + current_layout = fig.layout + + # Create new figure with custom row heights + new_fig = make_subplots( + rows=number_of_rows, + cols=1, + shared_xaxes=True, + row_heights=height_fractions, + vertical_spacing=0.02, + subplot_titles=[ann.text for ann in current_layout.annotations] if current_layout.annotations else None + ) + + new_height_fractions = {} + for key in new_fig.layout: + if key.startswith('yaxis'): + new_height_fractions[key] = new_fig.layout[key].domain + + + # Add traces from original figure + for trace in current_data: + new_fig.add_trace(trace) + + # Update layout properties from original figure (including theme) + new_fig.update_layout(current_layout) + for key in new_height_fractions: + new_fig.layout[key].domain = new_height_fractions[key] + + fig = new_fig + # Preserve x-axis linking and only show labels on bottom row with small gaps + fig.update_xaxes(matches='x') + fig.update_yaxes(matches=None, showticklabels=True) + + # Only show x-axis labels on the bottom subplot + for i in range(1, len(height_fractions)): + fig.update_xaxes(showticklabels=False, row=i, col=1) + return fig if backend == "seaborn": print("using seaborn (experimental feature)") From 9ce3afcb792b27bdbeaaaaa566ec929ccef27540 Mon Sep 17 00:00:00 2001 From: jepegit Date: Mon, 23 Jun 2025 22:00:30 +0200 Subject: [PATCH 23/37] better plotting --- cellpy/utils/collectors.py | 154 ++++++++++++++++++++++++++++--------- 1 file changed, 119 insertions(+), 35 deletions(-) diff --git a/cellpy/utils/collectors.py b/cellpy/utils/collectors.py index 7cd5d9d9..bb832fe8 100644 --- a/cellpy/utils/collectors.py +++ b/cellpy/utils/collectors.py @@ -1571,13 +1571,77 @@ def legend_replacer(trace, df, group_legends=True): hovertemplate=f"{cell_label}
{trace.hovertemplate}", ) +def _plotly_y_label_cleaner(y_label_mapper, split_at=20): + """Clean up the y-label mapper for plotly. + + The y-label mapper is a dictionary that maps the variable name to the y-label. The y-labels are + expected to be in the form of "Variable Name (unit)". If the y-label is too long, it is + split into multiple lines. + This is done to avoid the y-labels from being too long and wrapping around. + + Discharge Capacity Retention Gravimetric Norm (%) should become: + Discharge Capacity
Retention Gravimetric Norm
(%) + + Args: + y_label_mapper (dict): the y-label mapper. + + Returns: + dict: the cleaned up y-label mapper. + + """ -def spread_plot(curves, plotly_arguments, **kwargs): - """Create a spread plot (error-bands instead of error-bars).""" + new_y_label_mapper = {} + for k, v in y_label_mapper.items(): + if len(v) > split_at: + # First split on " (" pattern + v = "
(".join(v.split(" (")) + + # Then check if any resulting line is still too long and split on spaces + lines = v.split("
") + final_lines = [] + for line in lines: + if len(line) > split_at and " " in line: + # Split long lines on spaces + words = line.split(" ") + current_line = "" + for word in words: + if len(current_line + " " + word) > split_at and current_line: + final_lines.append(current_line) + current_line = word + else: + if current_line: + current_line += " " + word + else: + current_line = word + if current_line: + final_lines.append(current_line) + else: + final_lines.append(line) + v = "
".join(final_lines) + new_y_label_mapper[k] = v + return new_y_label_mapper +def spread_plot(curves, plotly_arguments=None, y_label_mapper=None, **kwargs): + """Create a spread plot (error-bands instead of error-bars). + + This is an experimental feature that is not yet fully tested. It uses make_subplots to create the figure, + and then adds the traces one by one. This methodology will eventually replace the use of plotly.express + for all the summary plots. + + """ from plotly.subplots import make_subplots + + + + if y_label_mapper is None: + y_label_mapper = {} + else: + y_label_mapper = _plotly_y_label_cleaner(y_label_mapper) + selected_variables = curves["variable"].unique() number_of_rows = len(selected_variables) + # TODO: change this (only temporary fix to allow height fractions to be set by spread_plot) + height_fractions = kwargs.get("height_fractions_spread", [1/number_of_rows]*number_of_rows) colors = plotly.colors.qualitative.Plotly opacity = 0.2 @@ -1597,13 +1661,18 @@ def spread_plot(curves, plotly_arguments, **kwargs): fig = make_subplots( rows=number_of_rows, cols=1, - shared_xaxes=True, - vertical_spacing=0.05, + start_cell=plotly_arguments.get("plotly_start_cell", "top-left"), + shared_xaxes=plotly_arguments.get("plotly_shared_xaxes", True), + row_heights=height_fractions, + vertical_spacing=plotly_arguments.get("plotly_vertical_spacing", 0.01), ) + y_labels = {} for i, (cell, data) in enumerate(g): color = color_list[i % len(color_list)] for row_number, variable in enumerate(selected_variables): + y_label = y_label_mapper.get(variable, variable) + y_labels[row_number] = y_label if row_number == 0: show_legend = True else: @@ -1657,15 +1726,18 @@ def spread_plot(curves, plotly_arguments, **kwargs): row=row_number + 1, col=1, ) + for row_number, y_label in y_labels.items(): + fig.update_yaxes(title_text=y_label, row=row_number + 1, col=1) fig.update_layout(legend_tracegroupgap=0) + # fig.update_layout(hovermode="x") if labels := plotly_arguments.get("labels"): - fig.update_xaxes(title=labels.get("cycle", None)) + fig.update_xaxes(title=labels.get("cycle", None), row=number_of_rows) # Hack to remove the x-axis title that appears on the top of the plot: - if number_of_rows > 1: - fig.update_layout(xaxis_title=None) + # if number_of_rows > 1: + # fig.update_layout(xaxis_title=None) if hover_mode := kwargs.pop("hovermode", None): fig.update_layout(hovermode=hover_mode) @@ -1865,7 +1937,6 @@ def sequence_plotter( seaborn_arguments["style"] = z elif method == "summary": - logging.info("sequence-plotter - summary") if cycles is not None: curves = collected_curves.loc[collected_curves.cycle.isin(cycles), :] else: @@ -1880,6 +1951,8 @@ def sequence_plotter( plotly_arguments["color"] = z seaborn_arguments["hue"] = z + + # ----------------- individual plotting calls ----------------------------- # TODO: move as much as possible up to the parsing of arguments # (i.e. prepare for future refactoring) @@ -1960,14 +2033,22 @@ def sequence_plotter( elif method == "summary": if spread: logging.critical("using spread is an experimental feature and might not work as expected") - fig = spread_plot(curves, plotly_arguments, **kwargs) + fig = spread_plot(curves, plotly_arguments=plotly_arguments, y_label_mapper=y_label_mapper, **kwargs) else: + # remove all kwargs that are only intended for spread_plot + _ = kwargs.pop("height_fractions_spread", None) + _ = plotly_arguments.pop("plotly_start_cell", None) + _ = plotly_arguments.pop("plotly_shared_xaxes", None) + _ = plotly_arguments.pop("plotly_vertical_spacing", None) + _ = kwargs.pop("plotly_start_cell", None) + _ = kwargs.pop("plotly_shared_xaxes", None) + _ = kwargs.pop("plotly_vertical_spacing", None) + fig = px.line( curves, **plotly_arguments, **kwargs, ) - if group_cells: # all cells in same group has same color try: fig.for_each_trace( @@ -1983,33 +2064,29 @@ def sequence_plotter( print("failed") print(e) - if y_label_mapper: + if y_label_mapper and not spread: + y_label_mapper = _plotly_y_label_cleaner(y_label_mapper) annotations = fig.layout.annotations if annotations: - # TODO: THIS IS WHERE I NEED TO FIX THE LABELS try: - # might consider a more robust method here - currently - # it assumes that the mapper is a list with same order - # and length as number of rows - for i, (a, la) in enumerate(zip(annotations, y_label_mapper)): + for i in range(len(annotations)): row = i + 1 - fig.for_each_yaxis( - functools.partial(y_axis_replacer, label=la), - row=row, - ) + for k, v in y_label_mapper.items(): + if annotations[i].text.endswith(k): + fig.for_each_yaxis( + functools.partial(y_axis_replacer, label=v), + row=row, + ) + break + fig.update_annotations(text="") + except Exception as e: print("failed") print(e) else: try: - if spread: - for i, la in enumerate(y_label_mapper): - row = i + 1 - fig.for_each_yaxis(functools.partial(y_axis_replacer, label=la), row=row) - - else: - fig.for_each_yaxis( + fig.for_each_yaxis( functools.partial(y_axis_replacer, label=y_label_mapper[0]), ) except Exception as e: @@ -2376,8 +2453,14 @@ def summary_plotter(collected_curves, cycles_to_plot=None, backend="plotly", **k label_mapper = { f"{y}": None, } + # order the variables by a given order: + order_variables = kwargs.pop("order_variables", None) + if order_variables: + collected_curves[g] = collected_curves[g].astype(pd.CategoricalDtype(categories=order_variables, ordered=True)) + collected_curves = collected_curves.sort_values(by=[g, z, x]) + if units: - label_mapper[y] = [] + label_mapper[y] = {} variables = list(collected_curves[g].unique()) for v in variables: # extract units: @@ -2399,15 +2482,15 @@ def summary_plotter(collected_curves, cycles_to_plot=None, backend="plotly", **k # creating label: u = u_top or "Value" - v = v.split("_") + v2 = v.split("_") if u_sub: u_sub = u_sub.replace("**", "") u = f"{u}/{u_sub}" - v = v[:-1] - v = " ".join(v).title() - if v.endswith("Cv"): - v = v.replace("Cv", "CV") - label_mapper[y].append(f"{v} ({u})") + v2 = v2[:-1] + v2 = " ".join(v2).title() + if v2.endswith("Cv"): + v2 = v2.replace("Cv", "CV") + label_mapper[y][v] = f"{v2} ({u})" # TODO: need to refactor and fix how the classes are created so that leftover kwargs are not sent to the backend # (for example if another collector is used and registers a kwarg without popping it) @@ -2459,8 +2542,9 @@ def summary_plotter(collected_curves, cycles_to_plot=None, backend="plotly", **k new_fig = make_subplots( rows=number_of_rows, cols=1, + # start_cell="bottom-left", shared_xaxes=True, - row_heights=height_fractions, + row_heights=height_fractions[::-1], vertical_spacing=0.02, subplot_titles=[ann.text for ann in current_layout.annotations] if current_layout.annotations else None ) From 13e49d3c91a5e17a560ff0f07dd6e7a6d900e186 Mon Sep 17 00:00:00 2001 From: jepegit Date: Tue, 24 Jun 2025 14:29:53 +0200 Subject: [PATCH 24/37] create helper function (collectors.standard_gravimetric_collector) --- cellpy/utils/collectors.py | 66 ++++++++++++++++++++++++++++++++------ 1 file changed, 57 insertions(+), 9 deletions(-) diff --git a/cellpy/utils/collectors.py b/cellpy/utils/collectors.py index bb832fe8..72264bbe 100644 --- a/cellpy/utils/collectors.py +++ b/cellpy/utils/collectors.py @@ -793,6 +793,47 @@ def _output_path(self, serial_number=None): f = d / n return f +def standard_gravimetric_collector(b, **kwargs): + """Create a standard gravimetric collector. + + This is a temporary hack to allow for making standard plot no. 1. + """ + + def _copy_and_normalize(df, columns, norm_factor=120.0, *args, **kwargs): + # modify the dataframe: + df = df.assign( + discharge_capacity_retention_gravimetric_norm=lambda x: 100*(norm_factor - x.discharge_capacity_gravimetric) / norm_factor, + ) + # modify the output columns: + if "discharge_capacity_retention_gravimetric_norm" not in columns: + columns.append("discharge_capacity_retention_gravimetric_norm") + return df, columns + + group_it = kwargs.pop("group_it", True) + interactive = kwargs.pop("interactive", True) + + if not interactive: + raise NotImplementedError("Only interactive mode is implemented for standard_gravimetric_collector") + backend = kwargs.pop("backend", "plotly") + if backend != "plotly": + raise NotImplementedError("Only plotly backend is implemented for standard_gravimetric_collector") + columns=["charge_capacity_gravimetric", "discharge_capacity_gravimetric", "coulombic_efficiency"] + data_collector_arguments=dict( + partition_by_cv=True, + individual_summary_hooks=[_copy_and_normalize], + drop_columns=["charge_capacity_gravimetric", "charge_capacity_gravimetric_non_cv", "discharge_capacity_gravimetric_cv", "discharge_capacity_gravimetric_non_cv"], + average_method="mean", + key_index_bounds=[0, 4], + ) + plotter_arguments=dict( + spread=group_it, + markers=True, + height_fractions_spread=[0.1, 0.3, 0.3, 0.3], + order_variables=["coulombic_efficiency", "discharge_capacity_gravimetric", "discharge_capacity_retention_gravimetric_norm", "charge_capacity_gravimetric_cv"], + ) + data_collector_arguments.update(kwargs.pop("data_collector_arguments", {})) + plotter_arguments.update(kwargs.pop("plotter_arguments", {})) + return BatchSummaryCollector(b, group_it=group_it, interactive=interactive, backend=backend, columns=columns, data_collector_arguments=data_collector_arguments, plotter_arguments=plotter_arguments, **kwargs) class BatchSummaryCollector(BatchCollector): # Three main levels of arguments to the plotter and collector funcs is available: @@ -1620,6 +1661,7 @@ def _plotly_y_label_cleaner(y_label_mapper, split_at=20): v = "
".join(final_lines) new_y_label_mapper[k] = v return new_y_label_mapper + def spread_plot(curves, plotly_arguments=None, y_label_mapper=None, **kwargs): """Create a spread plot (error-bands instead of error-bars). @@ -1631,8 +1673,6 @@ def spread_plot(curves, plotly_arguments=None, y_label_mapper=None, **kwargs): from plotly.subplots import make_subplots - - if y_label_mapper is None: y_label_mapper = {} else: @@ -2463,15 +2503,16 @@ def summary_plotter(collected_curves, cycles_to_plot=None, backend="plotly", **k label_mapper[y] = {} variables = list(collected_curves[g].unique()) for v in variables: - # extract units: + + # unit label u_sub = None - if v.endswith("_areal"): + if v.endswith("_areal") or v.endswith("_areal_cv"): u_sub = units["cellpy_units"].specific_areal - elif v.endswith("_gravimetric"): + elif v.endswith("_gravimetric") or v.endswith("_gravimetric_cv"): u_sub = units["cellpy_units"].specific_gravimetric - elif v.endswith("_volumetric"): + elif v.endswith("_volumetric") or v.endswith("_volumetric_cv"): u_sub = units["cellpy_units"].specific_volumetric - + u_top = None if "_capacity" in v: u_top = units["cellpy_units"].charge @@ -2480,16 +2521,23 @@ def summary_plotter(collected_curves, cycles_to_plot=None, backend="plotly", **k if v == "coulombic_efficiency": u_top = "%" - # creating label: u = u_top or "Value" + + # variable label v2 = v.split("_") if u_sub: u_sub = u_sub.replace("**", "") u = f"{u}/{u_sub}" - v2 = v2[:-1] + if v2[-1] == "cv": + v2 = v2[:-2] + v2.append("cv") + else: + v2 = v2[:-1] v2 = " ".join(v2).title() + if v2.endswith("Cv"): v2 = v2.replace("Cv", "CV") + label_mapper[y][v] = f"{v2} ({u})" # TODO: need to refactor and fix how the classes are created so that leftover kwargs are not sent to the backend From a51f4bcf5784748ac9509744ea513d8e9d4eb03e Mon Sep 17 00:00:00 2001 From: jepegit Date: Tue, 24 Jun 2025 16:50:06 +0200 Subject: [PATCH 25/37] clean up a bit --- cellpy/utils/collectors.py | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/cellpy/utils/collectors.py b/cellpy/utils/collectors.py index 72264bbe..f59373ac 100644 --- a/cellpy/utils/collectors.py +++ b/cellpy/utils/collectors.py @@ -793,12 +793,21 @@ def _output_path(self, serial_number=None): f = d / n return f -def standard_gravimetric_collector(b, **kwargs): +def standard_gravimetric_collector(b, norm_factor=120.0, **kwargs): """Create a standard gravimetric collector. This is a temporary hack to allow for making standard plot no. 1. + + Args: + norm_factor (float): the factor to normalize the discharge capacity retention by. + group_it (bool): if True, group the cells by the key_index_bounds. + data_collector_arguments (dict): the data collector arguments. + plotter_arguments (dict): the plotter arguments. + **kwargs: additional arguments sent to the collector. """ + from functools import partial + def _copy_and_normalize(df, columns, norm_factor=120.0, *args, **kwargs): # modify the dataframe: df = df.assign( @@ -811,7 +820,10 @@ def _copy_and_normalize(df, columns, norm_factor=120.0, *args, **kwargs): group_it = kwargs.pop("group_it", True) interactive = kwargs.pop("interactive", True) - + + _copy_and_normalize_partial = partial(_copy_and_normalize, norm_factor=norm_factor) + _copy_and_normalize_partial.__name__ = "copy_and_normalize" + if not interactive: raise NotImplementedError("Only interactive mode is implemented for standard_gravimetric_collector") backend = kwargs.pop("backend", "plotly") @@ -820,7 +832,7 @@ def _copy_and_normalize(df, columns, norm_factor=120.0, *args, **kwargs): columns=["charge_capacity_gravimetric", "discharge_capacity_gravimetric", "coulombic_efficiency"] data_collector_arguments=dict( partition_by_cv=True, - individual_summary_hooks=[_copy_and_normalize], + individual_summary_hooks=[_copy_and_normalize_partial], drop_columns=["charge_capacity_gravimetric", "charge_capacity_gravimetric_non_cv", "discharge_capacity_gravimetric_cv", "discharge_capacity_gravimetric_non_cv"], average_method="mean", key_index_bounds=[0, 4], From 0eff01c4297c1a34b75292637c7c0259b8b8d19e Mon Sep 17 00:00:00 2001 From: jepegit Date: Wed, 25 Jun 2025 15:29:19 +0200 Subject: [PATCH 26/37] bug hunting - session 1 --- cellpy/readers/dbreader.py | 6 ++- cellpy/utils/batch_tools/batch_journals.py | 9 +++- cellpy/utils/batch_tools/engines.py | 61 ++++++++++++++++------ cellpy/utils/helpers.py | 3 ++ 4 files changed, 59 insertions(+), 20 deletions(-) diff --git a/cellpy/readers/dbreader.py b/cellpy/readers/dbreader.py index f6ec87ca..3f15d2a5 100644 --- a/cellpy/readers/dbreader.py +++ b/cellpy/readers/dbreader.py @@ -88,7 +88,6 @@ def __init__( self.headers = self.db_sheet_cols.headers if db_frame is not None: - print("Using frame instead of file") self.table = db_frame.copy() else: self.skiprows, self.nrows = self._find_out_what_rows_to_skip() @@ -98,6 +97,7 @@ def __init__( if batch: self.selected_batch = self.select_batch(batch, batch_col_name=batch_col_name) + logging.debug("got table") def __str__(self): @@ -144,6 +144,7 @@ def select_batch( def _select_batch(self, batch, batch_col_name=None, case_sensitive=True, drop=True, clean=False): if not batch_col_name: batch_col_name = self.db_sheet_cols.batch + logging.debug("selecting batch - %s" % batch) sheet = self.table identity = self.db_sheet_cols.id @@ -164,7 +165,8 @@ def _select_batch(self, batch, batch_col_name=None, case_sensitive=True, drop=Tr sheet.drop_duplicates(inplace=True) sheet.dropna(inplace=True) return sheet.values.astype(int) - return sheet.loc[:, identity].values.astype(int) + out = sheet.loc[:, identity].values.astype(int) + return out def from_batch( self, diff --git a/cellpy/utils/batch_tools/batch_journals.py b/cellpy/utils/batch_tools/batch_journals.py index f3872d53..6937383d 100644 --- a/cellpy/utils/batch_tools/batch_journals.py +++ b/cellpy/utils/batch_tools/batch_journals.py @@ -201,10 +201,17 @@ def from_db(self, project=None, name=None, batch_col=None, dbreader_kwargs=None, dbreader_kwargs = {} logging.debug(f"batch_name, batch_col, dbreader_kwargs: {name}, {batch_col}, {dbreader_kwargs}") - if self.db_reader is not None: if isinstance(self.db_reader, dbreader.Reader): # Simple excel-db id_keys = self.db_reader.select_batch(name, batch_col, **dbreader_kwargs) + + # Check for duplicates in id_keys + if len(id_keys) != len(set(id_keys)): + duplicates = [x for x in id_keys if id_keys.count(x) > 1] + unique_duplicates = list(set(duplicates)) + logging.warning(f"Found duplicate id_keys: {unique_duplicates}") + else: + logging.debug("No duplicates found in id_keys") logging.debug(f"id_keys: {id_keys}") self.pages = self.engine(self.db_reader, id_keys, **kwargs) else: diff --git a/cellpy/utils/batch_tools/engines.py b/cellpy/utils/batch_tools/engines.py index 80ccbe7b..db9b13fa 100644 --- a/cellpy/utils/batch_tools/engines.py +++ b/cellpy/utils/batch_tools/engines.py @@ -192,30 +192,29 @@ def simple_db_engine( logging.debug("No reader provided. Creating one myself.") if cell_ids is None: + logging.debug("cell_ids is None") pages_dict = reader.from_batch( batch_name=batch_name, include_key=include_key, include_individual_arguments=include_individual_arguments, ) + logging.debug("pages_dict: {pages_dict}") else: + logging.debug("cell_ids is not None") pages_dict = dict() # TODO: rename this to "cell" or "cell_id" or something similar: pages_dict[hdr_journal["filename"]] = _query(reader.get_cell_name, cell_ids) if include_key: - pages_dict[hdr_journal["id_key"]] = cell_ids - if include_individual_arguments: pages_dict[hdr_journal["argument"]] = _query(reader.get_args, cell_ids) - pages_dict[hdr_journal["mass"]] = _query(reader.get_mass, cell_ids) pages_dict[hdr_journal["total_mass"]] = _query(reader.get_total_mass, cell_ids) try: pages_dict[hdr_journal["nom_cap_specifics"]] = _query(reader.get_nom_cap_specifics, cell_ids) except Exception as e: pages_dict[hdr_journal["nom_cap_specifics"]] = "gravimetric" - try: # updated 06.01.2025: some old db files returns None for file_name_indicator _file_name_indicator = _query(reader.get_file_name_indicator, cell_ids) @@ -227,18 +226,28 @@ def simple_db_engine( hdr_journal["filename"] ] # TODO: use of "filename"! - pages_dict[hdr_journal["loading"]] = _query(reader.get_loading, cell_ids) - pages_dict[hdr_journal["nom_cap"]] = _query(reader.get_nom_cap, cell_ids) - pages_dict[hdr_journal["area"]] = _query(reader.get_area, cell_ids) - pages_dict[hdr_journal["experiment"]] = _query(reader.get_experiment_type, cell_ids) - pages_dict[hdr_journal["fixed"]] = _query(reader.inspect_hd5f_fixed, cell_ids) - pages_dict[hdr_journal["label"]] = _query(reader.get_label, cell_ids) - pages_dict[hdr_journal["cell_type"]] = _query(reader.get_cell_type, cell_ids) - pages_dict[hdr_journal["instrument"]] = _query(reader.get_instrument, cell_ids) + journal_fields = [ + ("loading", reader.get_loading), + ("nom_cap", reader.get_nom_cap), + ("area", reader.get_area), + ("experiment", reader.get_experiment_type), + ("fixed", reader.inspect_hd5f_fixed), + ("label", reader.get_label), + ("cell_type", reader.get_cell_type), + ("instrument", reader.get_instrument), + ("comment", reader.get_comment), + ("group", reader.get_group), + ] + + for field_name, reader_method in journal_fields: + try: + pages_dict[hdr_journal[field_name]] = _query(reader_method, cell_ids) + except Exception as e: + logging.debug(f"Error in getting {field_name}: {e}") + pages_dict[hdr_journal["raw_file_names"]] = [] pages_dict[hdr_journal["cellpy_file_name"]] = [] - pages_dict[hdr_journal["comment"]] = _query(reader.get_comment, cell_ids) - pages_dict[hdr_journal["group"]] = _query(reader.get_group, cell_ids) + if additional_column_names is not None: for k in additional_column_names: try: @@ -250,16 +259,19 @@ def simple_db_engine( del reader for key in list(pages_dict.keys()): - logging.debug("%s: %s" % (key, str(pages_dict[key]))) + logging.debug(f"[length: {len(pages_dict[key]):04d}] {key}: {str(pages_dict[key])}") _groups = pages_dict[hdr_journal["group"]] groups = helper.fix_groups(_groups) pages_dict[hdr_journal["group"]] = groups my_timer_start = time.time() + logging.debug("finding files") pages_dict = helper.find_files(pages_dict, file_list=file_list, pre_path=pre_path, **kwargs) + logging.debug("files found") + logging.debug(f"pages_dict: {pages_dict}") my_timer_end = time.time() if (my_timer_end - my_timer_start) > 5.0: - logging.critical( + logging.debug( "The function _find_files was very slow. " "Save your journal so you don't have to run it again! " "You can load it again using the from_journal(journal_name) method." @@ -294,15 +306,30 @@ def simple_db_engine( # TODO: check if drop=False works [#index] pages.set_index(hdr_journal["filename"], inplace=True) # edit this to allow for # non-numeric index-names (for tab completion and python-box) + _check_pages_frame(pages) return pages +def _check_pages_frame(pages): + logging.debug(f"pages.columns: {pages.columns}") + logging.debug(f"pages.index: {pages.index}") + logging.debug(f"pages.index.unique(): {pages.index.unique()}") + logging.debug(f"pages.dtypes: {pages.dtypes}") + duplicates = pages.index.duplicated() + if duplicates.any(): + logging.critical(f"Oh no! Found {duplicates.sum()} duplicate cell names in your db - this is not allowed!") + logging.critical(f"Duplicate cell names: {pages.index[duplicates].tolist()}") + else: + logging.debug("No duplicate indices found") + logging.debug(f"pages.shape: {pages.shape}") + + def _report_suspected_duplicate_id(e, what="do it", on=None): logging.warning(f"could not {what}") logging.warning(f"{on}") logging.warning("maybe you have a corrupted db?") logging.warning( "typically happens if the cell_id is not unique (several rows or records in " - "your db has the same cell_id or key)" + "your db has the same cell_id or key) or if you have non-unique cell names" ) logging.warning(e) diff --git a/cellpy/utils/helpers.py b/cellpy/utils/helpers.py index f202d396..a2871e7d 100644 --- a/cellpy/utils/helpers.py +++ b/cellpy/utils/helpers.py @@ -108,6 +108,9 @@ def _make_average( new_col_name_std = "std" if number_of_cols > 1: + # sqr = _ensure_numeric((avg - values) ** 2) + # TODO: Fix this - RuntimeWarning: invalid value encountered in subtract + # Could consider using np.nanmean(new_frame[col]) instead of np.mean(new_frame[col])? avg_frame = new_frame[col].agg([average_method, "std"], skipna=True, axis=1) else: avg_frame = pd.DataFrame(data=new_frame[col].values, columns=[new_col_name_mean]) From 0e0654905ef6165f99393d379952a4ce3661e29d Mon Sep 17 00:00:00 2001 From: jepegit Date: Thu, 26 Jun 2025 12:31:30 +0200 Subject: [PATCH 27/37] fix bug in summary collector (concat summaries) that mutated list of selected columns --- cellpy/utils/helpers.py | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/cellpy/utils/helpers.py b/cellpy/utils/helpers.py index a2871e7d..7863ad95 100644 --- a/cellpy/utils/helpers.py +++ b/cellpy/utils/helpers.py @@ -111,6 +111,10 @@ def _make_average( # sqr = _ensure_numeric((avg - values) ** 2) # TODO: Fix this - RuntimeWarning: invalid value encountered in subtract # Could consider using np.nanmean(new_frame[col]) instead of np.mean(new_frame[col])? + + # Replace inf with nan + new_frame[col] = new_frame[col].replace([np.inf, -np.inf], np.nan) + avg_frame = new_frame[col].agg([average_method, "std"], skipna=True, axis=1) else: avg_frame = pd.DataFrame(data=new_frame[col].values, columns=[new_col_name_mean]) @@ -1064,6 +1068,7 @@ def concat_summaries( frames_sub = [] keys_sub = [] for cell_id in cell_names: + output_columns_current_cell = output_columns.copy() logging.debug(f"Processing [{cell_id}]") group = pages.loc[cell_id, "group"] sub_group = pages.loc[cell_id, "sub_group"] @@ -1102,7 +1107,6 @@ def concat_summaries( c = add_normalized_capacity(c, norm_cycles=normalize_capacity_on, scale=scale_by) if rate is not None: - # TODO: update this so that it works with partitioned data if partition_by_cv: print("partitioning by cv_step is experimental for rate selection") @@ -1117,7 +1121,7 @@ def concat_summaries( partition_by_cv=partition_by_cv, ) elif partition_by_cv: - s = _partition_summary_based_on_cv_steps(c, column_set=output_columns) + s = _partition_summary_based_on_cv_steps(c, column_set=output_columns_current_cell) else: s = c.data.summary @@ -1126,11 +1130,15 @@ def concat_summaries( logging.info("Experimental feature: applying individual summary hooks") for hook in individual_summary_hooks: logging.info(f" -applying {hook.__name__} to {cell_id}") - s, output_columns = hook(s, columns=output_columns.copy(), *args, **kwargs) + s, output_columns_current_cell = hook(s, columns=output_columns_current_cell.copy(), *args, **kwargs) + output_columns = output_columns_current_cell.copy() + if columns is not None: - # Filter out columns that don't exist in the dataframe to avoid KeyError - output_columns = [col for col in output_columns if col in s.columns] + # Fill columns that don't exist in the dataframe with nan + for col in output_columns: + if col not in s.columns: + s[col] = np.nan s = s.loc[:, output_columns].copy() if drop_columns: logging.debug(f"Dropping columns: {drop_columns}") @@ -1138,6 +1146,7 @@ def concat_summaries( s = s.drop(columns=drop_columns, errors="ignore") logging.debug(f"Columns in s after dropping: {s.columns}") + # add group and subgroup if not group_it: s = s.assign(group=group, sub_group=sub_group, group_label=group_label, label=label) @@ -1155,8 +1164,10 @@ def concat_summaries( try: # if we used drop_columns, we need to remove them from the output_columns if drop_columns: - output_columns = [col for col in output_columns if col not in drop_columns] - s = _make_average(frames_sub, output_columns, average_method=average_method) + output_columns_current_group = [col for col in output_columns if col not in drop_columns] + else: + output_columns_current_group = output_columns.copy() + s = _make_average(frames_sub, output_columns_current_group, average_method=average_method) except ValueError as e: print("could not make average!") print(e) From 5305c544efeb0b5afe699164674ef43e7ae4efca Mon Sep 17 00:00:00 2001 From: jepegit Date: Fri, 27 Jun 2025 19:29:27 +0200 Subject: [PATCH 28/37] fix new standard (norm cv share etc) --- cellpy/utils/collectors.py | 118 +++++++++++++++++++++++++++---------- cellpy/utils/helpers.py | 9 ++- 2 files changed, 93 insertions(+), 34 deletions(-) diff --git a/cellpy/utils/collectors.py b/cellpy/utils/collectors.py index f59373ac..fb2a6020 100644 --- a/cellpy/utils/collectors.py +++ b/cellpy/utils/collectors.py @@ -8,7 +8,7 @@ from pprint import pprint from pathlib import Path import textwrap -from typing import Any +from typing import Any, Union import time from itertools import count from multiprocessing import Process @@ -131,10 +131,15 @@ def generate_output_path(name, directory, serial_number=None): f = d / name return f - -def _image_exporter_plotly(figure, filename, timeout=IMAGE_TO_FILE_TIMEOUT, **kwargs): +def incremental_image_exporter_plotly(self, filename, timeout=IMAGE_TO_FILE_TIMEOUT, **kwargs): + use_subprocess = kwargs.pop("use_subprocess", True) + + if not use_subprocess: + print(f"saving image to {filename}") + self.figure.write_image(filename, **kwargs) + return p = Process( - target=figure.write_image, + target=self.figure.write_image, args=(filename,), name="save_plotly_image_to_file", kwargs=kwargs, @@ -146,6 +151,11 @@ def _image_exporter_plotly(figure, filename, timeout=IMAGE_TO_FILE_TIMEOUT, **kw print(f"Oops, {p} timeouts! Could not save {filename}") if p.exitcode == 0: print(f" - saved image file: {filename}") + else: + print(f"Oops, {p} failed with exitcode: {p.exitcode}") + print("Could it be that you have not installed the required packages?") + print("Try to install kaleido:") + print("pip install kaleido") def save_plotly_figure(figure, name=None, directory=".", serial_number=None): @@ -171,8 +181,8 @@ def save_plotly_figure(figure, name=None, directory=".", serial_number=None): filename_svg = filename_pre.with_suffix(".svg") filename_json = filename_pre.with_suffix(".json") - _image_exporter_plotly(figure, filename_png, scale=3.0) - _image_exporter_plotly(figure, filename_svg) + incremental_image_exporter_plotly(figure, filename_png, scale=3.0) + incremental_image_exporter_plotly(figure, filename_svg) figure.write_json(filename_json) print(f" - saved plotly json file: {filename_json}") @@ -706,10 +716,22 @@ def to_hdf5(self, serial_number=None): filename = filename.with_suffix(".h5") data = self.data + # Convert categorical columns to strings to avoid HDF5 issues + for col in data.columns: + if pd.api.types.is_categorical_dtype(data[col]): + logging.info(f"converting categorical column {col} to string") + data[col] = data[col].astype(str) + data.to_hdf(filename, key=HDF_KEY, mode="w") print(f" - saved hdf5 file: {filename}") def _image_exporter_plotly(self, filename, timeout=IMAGE_TO_FILE_TIMEOUT, **kwargs): + use_subprocess = kwargs.pop("use_subprocess", True) + + if not use_subprocess: + print(f"saving image to {filename}") + self.figure.write_image(filename, **kwargs) + return p = Process( target=self.figure.write_image, args=(filename,), @@ -723,6 +745,13 @@ def _image_exporter_plotly(self, filename, timeout=IMAGE_TO_FILE_TIMEOUT, **kwar print(f"Oops, {p} timeouts! Could not save {filename}") if p.exitcode == 0: print(f" - saved image file: {filename}") + else: + print(f"Oops, {p} failed with exitcode: {p.exitcode}") + print("Could it be that you have not installed the required packages?") + print("Try to install kaleido:") + print("pip install kaleido") + + def to_image_files(self, serial_number=None): """Save to image files (png, svg, json). @@ -767,19 +796,29 @@ def to_image_files(self, serial_number=None): print(f"TODO: implement saving {filename_svg}") print(f"TODO: implement saving {filename_json}") - def save(self, serial_number=None): + def save(self, serial_number=None, save_hdf5=True, save_image_files=True, to_csv_kwargs:Union[dict, None] = None): """Save to csv, hdf5 and image files. Args: serial_number (int): serial number to append to the filename. + save_hdf5 (bool): save to hdf5 file. + save_image_files (bool): save to image files. + to_csv_kwargs (dict): keyword arguments sent to the csv writer. """ - self.to_csv(serial_number=serial_number) - self.to_hdf5(serial_number=serial_number) + if to_csv_kwargs is None: + to_csv_kwargs = {} + self.to_csv(serial_number=serial_number, **to_csv_kwargs) + if save_hdf5: + try: + self.to_hdf5(serial_number=serial_number) + except Exception as e: + print(f"Error saving hdf5 file: {e}") if self._figure_valid(): - self.to_image_files(serial_number=serial_number) + if save_image_files: + self.to_image_files(serial_number=serial_number) def _output_path(self, serial_number=None): d = Path(self.figure_directory) @@ -798,6 +837,11 @@ def standard_gravimetric_collector(b, norm_factor=120.0, **kwargs): This is a temporary hack to allow for making standard plot no. 1. + Coulombic Efficiency + Discharge Capacity (gravimetric) + Discharge Capacity Retention (Normalized) + CV share Charge Capacity (Normalized) + Args: norm_factor (float): the factor to normalize the discharge capacity retention by. group_it (bool): if True, group the cells by the key_index_bounds. @@ -808,21 +852,26 @@ def standard_gravimetric_collector(b, norm_factor=120.0, **kwargs): from functools import partial - def _copy_and_normalize(df, columns, norm_factor=120.0, *args, **kwargs): + def _copy_and_normalize(df, columns, col="discharge_capacity_gravimetric",norm_factor=120.0, *args, **kwargs): # modify the dataframe: - df = df.assign( - discharge_capacity_retention_gravimetric_norm=lambda x: 100*(norm_factor - x.discharge_capacity_gravimetric) / norm_factor, - ) + _kwargs = { + f"{col}_norm": lambda x: 100*(x[col]) / norm_factor, + } + df = df.assign(**_kwargs) + # modify the output columns: - if "discharge_capacity_retention_gravimetric_norm" not in columns: - columns.append("discharge_capacity_retention_gravimetric_norm") + if f"{col}_norm" not in columns: + columns.append(f"{col}_norm") return df, columns group_it = kwargs.pop("group_it", True) interactive = kwargs.pop("interactive", True) - _copy_and_normalize_partial = partial(_copy_and_normalize, norm_factor=norm_factor) - _copy_and_normalize_partial.__name__ = "copy_and_normalize" + _copy_and_normalize_discharge_capacity_gravimetric_partial = partial(_copy_and_normalize, norm_factor=norm_factor) + _copy_and_normalize_discharge_capacity_gravimetric_partial.__name__ = "copy_and_normalize_discharge_capacity_gravimetric" + + _copy_and_normalize_charge_capacity_gravimetric_cv_partial = partial(_copy_and_normalize, norm_factor=norm_factor, col="charge_capacity_gravimetric_cv") + _copy_and_normalize_charge_capacity_gravimetric_cv_partial.__name__ = "copy_and_normalize_charge_capacity_gravimetric_cv" if not interactive: raise NotImplementedError("Only interactive mode is implemented for standard_gravimetric_collector") @@ -832,8 +881,8 @@ def _copy_and_normalize(df, columns, norm_factor=120.0, *args, **kwargs): columns=["charge_capacity_gravimetric", "discharge_capacity_gravimetric", "coulombic_efficiency"] data_collector_arguments=dict( partition_by_cv=True, - individual_summary_hooks=[_copy_and_normalize_partial], - drop_columns=["charge_capacity_gravimetric", "charge_capacity_gravimetric_non_cv", "discharge_capacity_gravimetric_cv", "discharge_capacity_gravimetric_non_cv"], + individual_summary_hooks=[_copy_and_normalize_discharge_capacity_gravimetric_partial, _copy_and_normalize_charge_capacity_gravimetric_cv_partial], + drop_columns=["charge_capacity_gravimetric_cv","charge_capacity_gravimetric", "charge_capacity_gravimetric_non_cv", "discharge_capacity_gravimetric_cv", "discharge_capacity_gravimetric_non_cv"], average_method="mean", key_index_bounds=[0, 4], ) @@ -841,7 +890,7 @@ def _copy_and_normalize(df, columns, norm_factor=120.0, *args, **kwargs): spread=group_it, markers=True, height_fractions_spread=[0.1, 0.3, 0.3, 0.3], - order_variables=["coulombic_efficiency", "discharge_capacity_gravimetric", "discharge_capacity_retention_gravimetric_norm", "charge_capacity_gravimetric_cv"], + order_variables=["coulombic_efficiency", "discharge_capacity_gravimetric", "discharge_capacity_gravimetric_norm", "charge_capacity_gravimetric_cv_norm"], ) data_collector_arguments.update(kwargs.pop("data_collector_arguments", {})) plotter_arguments.update(kwargs.pop("plotter_arguments", {})) @@ -1593,7 +1642,14 @@ def _hist_eq(trace): def y_axis_replacer(ax, label): """Replace y-axis label in matplotlib plots.""" - ax.update(title_text=label) + if isinstance(label, dict): + + _label = label.get(ax.title.text, None) + if _label is None: + _label = list(label.values())[0] + ax.update(title_text=_label) + else: + ax.update(title_text=label) return ax @@ -2067,8 +2123,7 @@ def sequence_plotter( if markers is not True: fig.for_each_trace(remove_markers) except Exception as e: - print("failed") - print(e) + print(f"sequence_plotter - fig_pr_cycle - failed {e} [{z}]") elif method == "film": fig = px.density_heatmap(curves, **plotly_arguments, **kwargs) @@ -2084,7 +2139,7 @@ def sequence_plotter( elif method == "summary": if spread: - logging.critical("using spread is an experimental feature and might not work as expected") + logging.info("using spread is an experimental feature and might not work as expected") fig = spread_plot(curves, plotly_arguments=plotly_arguments, y_label_mapper=y_label_mapper, **kwargs) else: # remove all kwargs that are only intended for spread_plot @@ -2113,8 +2168,7 @@ def sequence_plotter( if markers is not True: fig.for_each_trace(remove_markers) except Exception as e: - print("failed") - print(e) + print(f"sequence_plotter - summary - failed {e} [{group}]") if y_label_mapper and not spread: y_label_mapper = _plotly_y_label_cleaner(y_label_mapper) @@ -2134,16 +2188,16 @@ def sequence_plotter( fig.update_annotations(text="") except Exception as e: - print("failed") - print(e) + print(f"sequence_plotter - summary - y-label mapper failed {e} [{group}]") else: try: fig.for_each_yaxis( - functools.partial(y_axis_replacer, label=y_label_mapper[0]), + functools.partial(y_axis_replacer, label=y_label_mapper), ) except Exception as e: - print("failed") - print(e) + print(f"sequence_plotter - summary - y-label mapper - no annotations - failed {e} [{group}]") + print(f"y_label_mapper: {y_label_mapper}") + print(f"annotations: {annotations}") else: print(f"method '{method}' is not supported by plotly") diff --git a/cellpy/utils/helpers.py b/cellpy/utils/helpers.py index 7863ad95..10d09719 100644 --- a/cellpy/utils/helpers.py +++ b/cellpy/utils/helpers.py @@ -970,12 +970,15 @@ def concat_summaries( in the summary data. low_limit (float): lower limit for replacing extremes with nan if replace_extremes_with_nan is True. high_limit (float): upper limit for replacing extremes with nan if replace_extremes_with_nan is True. + remove_last (bool): if True, remove the last cycle from the summary data. *args,**kwargs: additional arguments to be passed to the hooks. Returns: ``pandas.DataFrame`` """ + remove_last = kwargs.pop("remove_last", False) + if key_index_bounds is None: # TODO: consider changing this to [1, -1] key_index_bounds = [1, -2] @@ -1126,6 +1129,10 @@ def concat_summaries( else: s = c.data.summary + if remove_last: + s = s.iloc[:-1] + + if individual_summary_hooks is not None: logging.info("Experimental feature: applying individual summary hooks") for hook in individual_summary_hooks: @@ -1156,8 +1163,6 @@ def concat_summaries( frames_sub.append(s) keys_sub.append(cell_id) - - if group_it: # TODO: update this to allow for more advanced naming of groups cell_id = create_group_names(custom_group_labels, gno, key_index_bounds, keys_sub, pages) From dfc8e15eec3cfa77a92cc2e56f099fbc830eac37 Mon Sep 17 00:00:00 2001 From: jepegit Date: Fri, 27 Jun 2025 23:29:14 +0200 Subject: [PATCH 29/37] fix another bug --- cellpy/utils/collectors.py | 29 ++++++++++++++++++++++------- 1 file changed, 22 insertions(+), 7 deletions(-) diff --git a/cellpy/utils/collectors.py b/cellpy/utils/collectors.py index fb2a6020..de3a498e 100644 --- a/cellpy/utils/collectors.py +++ b/cellpy/utils/collectors.py @@ -2156,7 +2156,9 @@ def sequence_plotter( **plotly_arguments, **kwargs, ) + if group_cells: # all cells in same group has same color + try: fig.for_each_trace( functools.partial( @@ -2177,13 +2179,23 @@ def sequence_plotter( try: for i in range(len(annotations)): row = i + 1 - for k, v in y_label_mapper.items(): - if annotations[i].text.endswith(k): + if annotations[i].text.startswith("variable="): + variable = annotations[i].text.split("=")[1] + if variable in y_label_mapper: + v = y_label_mapper[variable] fig.for_each_yaxis( functools.partial(y_axis_replacer, label=v), row=row, ) - break + else: + for k, v in y_label_mapper.items(): + + if annotations[i].text.endswith(k): + fig.for_each_yaxis( + functools.partial(y_axis_replacer, label=v), + row=row, + ) + break fig.update_annotations(text="") @@ -2501,6 +2513,9 @@ def summary_plotter(collected_curves, cycles_to_plot=None, backend="plotly", **k 2) mixed long and wide format where the variables are own columns. """ + # start_cell is used to determine the starting cell for the subplots (plotly) + start_cell = kwargs.pop("start_cell", "bottom-left") + col_headers = collected_curves.columns.to_list() # need to manually update this if new columns are added to collected_curves that should not be plotted: @@ -2656,7 +2671,7 @@ def summary_plotter(collected_curves, cycles_to_plot=None, backend="plotly", **k new_fig = make_subplots( rows=number_of_rows, cols=1, - # start_cell="bottom-left", + start_cell=start_cell, shared_xaxes=True, row_heights=height_fractions[::-1], vertical_spacing=0.02, @@ -2683,9 +2698,9 @@ def summary_plotter(collected_curves, cycles_to_plot=None, backend="plotly", **k fig.update_xaxes(matches='x') fig.update_yaxes(matches=None, showticklabels=True) - # Only show x-axis labels on the bottom subplot - for i in range(1, len(height_fractions)): - fig.update_xaxes(showticklabels=False, row=i, col=1) + # Only show x-axis labels on the bottom subplot (not needed anymore?) + # for i in range(1, len(height_fractions)): + # fig.update_xaxes(showticklabels=False, row=i, col=1) return fig if backend == "seaborn": From 82a8af7f77a441734e82c2274b6ef0a088ecf7f1 Mon Sep 17 00:00:00 2001 From: jepegit Date: Fri, 27 Jun 2025 23:29:27 +0200 Subject: [PATCH 30/37] bump version 1.0.3a2 -> 1.0.3a3 --- cellpy/_version.py | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cellpy/_version.py b/cellpy/_version.py index 99f36a8e..d5701129 100644 --- a/cellpy/_version.py +++ b/cellpy/_version.py @@ -1 +1 @@ -__version__ = "1.0.3a2" +__version__ = "1.0.3a3" diff --git a/pyproject.toml b/pyproject.toml index 634cc729..72e02339 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,7 +15,7 @@ markers = [ line-length = 120 [tool.bumpver] -current_version = "1.0.3a2" +current_version = "1.0.3a3" version_pattern = "MAJOR.MINOR.PATCH[PYTAG][NUM]" commit_message = "bump version {old_version} -> {new_version}" commit = true From 5fc7f1fe239b544da48241f93799f5e6bfb098f4 Mon Sep 17 00:00:00 2001 From: jepegit Date: Sat, 28 Jun 2025 13:59:09 +0200 Subject: [PATCH 31/37] fix ica --- cellpy/utils/ica.py | 79 ++++++++++++++++++++++++++++----------------- 1 file changed, 49 insertions(+), 30 deletions(-) diff --git a/cellpy/utils/ica.py b/cellpy/utils/ica.py index 19c57ff9..3df757f0 100644 --- a/cellpy/utils/ica.py +++ b/cellpy/utils/ica.py @@ -484,36 +484,55 @@ def dqdv_cycle(cycle_df, splitter=True, label_direction=False, **kwargs): c_first = cycle_df.loc[cycle_df["direction"] == -1] c_last = cycle_df.loc[cycle_df["direction"] == 1] - converter = Converter(**kwargs) - - converter.set_data(c_first["capacity"], c_first["voltage"]) - converter.inspect_data() - converter.pre_process_data() - converter.increment_data() - converter.post_process_data() - voltage_first = converter.voltage_processed - incremental_capacity_first = converter.incremental_capacity - - if splitter: - voltage_first = np.append(voltage_first, np.nan) - incremental_capacity_first = np.append(incremental_capacity_first, np.nan) - - converter = Converter(**kwargs) - - converter.set_data(c_last["capacity"], c_last["voltage"]) - converter.inspect_data() - converter.pre_process_data() - converter.increment_data() + # first try: - converter.post_process_data() - except ValueError: - logging.debug("ValueError - trying again with different settings") - converter.post_smoothing = False - print(converter) - converter.post_process_data() - voltage_last = converter.voltage_processed[::-1] - incremental_capacity_last = converter.incremental_capacity[::-1] - + converter = Converter(**kwargs) + converter.set_data(c_first["capacity"], c_first["voltage"]) + converter.inspect_data() + converter.pre_process_data() + converter.increment_data() + try: + converter.post_process_data() + except ValueError: + logging.debug("ValueError - trying again with different settings") + converter.post_smoothing = False + print(converter) + converter.post_process_data() + voltage_first = converter.voltage_processed + incremental_capacity_first = converter.incremental_capacity + + if splitter: + voltage_first = np.append(voltage_first, np.nan) + incremental_capacity_first = np.append(incremental_capacity_first, np.nan) + except Exception as e: + print(f"Error in dqdv_cycle - first") + print(f"error-message: '{e}'") + voltage_first = np.array([]) + incremental_capacity_first = np.array([]) + + # last + try: + converter = Converter(**kwargs) + converter.set_data(c_last["capacity"], c_last["voltage"]) + converter.inspect_data() + converter.pre_process_data() + converter.increment_data() + try: + converter.post_process_data() + except ValueError: + logging.debug("ValueError - trying again with different settings") + converter.post_smoothing = False + print(converter) + converter.post_process_data() + voltage_last = converter.voltage_processed[::-1] + incremental_capacity_last = converter.incremental_capacity[::-1] + except Exception as e: + print(f"Error in dqdv_cycle - last") + print(f"error-message: '{e}'") + voltage_last = np.array([]) + incremental_capacity_last = np.array([]) + + # combine voltage = np.concatenate((voltage_first, voltage_last)) incremental_capacity = np.concatenate( (incremental_capacity_first, incremental_capacity_last) @@ -613,7 +632,6 @@ def dqdv_cycles(cycles_df, not_merged=False, label_direction=False, **kwargs): ) _d = {"voltage": v, "dq": dq} _cols = ["voltage", "dq"] - _ica_df = pd.DataFrame(_d) if not not_merged: _cols.insert(0, "cycle") @@ -894,6 +912,7 @@ def _dqdv_combinded_frame(cell, tidy=True, label_direction=False, **kwargs): insert_nan=False, number_of_points=number_of_points, ) + ica_df = dqdv_cycles( cycles, not_merged=not tidy, label_direction=label_direction, **kwargs ) From 7c856cc30d0fb0ec587afae7225102723c1543a9 Mon Sep 17 00:00:00 2001 From: jepegit Date: Sun, 29 Jun 2025 10:50:58 +0200 Subject: [PATCH 32/37] new label for create new projectdir in cellpy new --- cellpy/cli.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/cellpy/cli.py b/cellpy/cli.py index dfe66bcf..49b586d2 100644 --- a/cellpy/cli.py +++ b/cellpy/cli.py @@ -633,7 +633,7 @@ def _check_import_pyodbc(): elif ODBC == "pypyodbc": click.echo(" you stated that you prefer the pypyodbc loader") try: - import pypyodbc as dbloader + import pypyodbc as dbloader # type: ignore except ImportError: click.echo(" Failed! Could not import it.") click.echo(" try 'pip install pypyodbc'") @@ -1271,7 +1271,7 @@ def _run_journals(folder_name, debug, silent, raw, cellpyfile, minimal): def _run_project(our_new_project, **kwargs): try: - import papermill as pm + import papermill as pm # type: ignore except ImportError: click.echo( "[cellpy]: You need to install papermill for automatically execute the notebooks." @@ -1738,20 +1738,22 @@ def _new( else: selected_project_dir = None click.echo(f"Select another directory instead") - + CREATE_NEW_DIR = "Create new project..." if not selected_project_dir: project_dirs = [ d.name for d in directory.iterdir() if d.is_dir() and not d.name.startswith(".") ] - project_dirs.insert(0, "[create new dir]") + print(f"project_dirs: {project_dirs}") + project_dirs.insert(0, CREATE_NEW_DIR) + print(f"project_dirs: {project_dirs}") project_dir = cookiecutter.prompt.read_user_choice( "project folder", project_dirs ) - if project_dir == "[create new dir]": + if project_dir == CREATE_NEW_DIR: default_name = "cellpy_project" temp_default_name = default_name for j in range(999): @@ -1814,7 +1816,7 @@ def _new( click.echo("WARNING - experimental feature - use at your own risk") input("Press Enter to continue...") try: - import papermill as pm + import papermill as pm # type: ignore except ImportError: click.echo( "[cellpy]: You need to install papermill for automatically execute the notebooks." From c36819eb26788833855c5b588d74a61e0ccda173 Mon Sep 17 00:00:00 2001 From: jepegit Date: Tue, 1 Jul 2025 11:59:02 +0200 Subject: [PATCH 33/37] allow additional arguments to plotly save images --- cellpy/utils/collectors.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/cellpy/utils/collectors.py b/cellpy/utils/collectors.py index de3a498e..ea806c32 100644 --- a/cellpy/utils/collectors.py +++ b/cellpy/utils/collectors.py @@ -753,7 +753,7 @@ def _image_exporter_plotly(self, filename, timeout=IMAGE_TO_FILE_TIMEOUT, **kwar - def to_image_files(self, serial_number=None): + def to_image_files(self, serial_number=None, **kwargs): """Save to image files (png, svg, json). Notes: @@ -776,8 +776,8 @@ def to_image_files(self, serial_number=None): filename_pickle = filename_pre.with_suffix(".pickle") if self.backend == "plotly": - self._image_exporter_plotly(filename_png, scale=3.0) - self._image_exporter_plotly(filename_svg) + self._image_exporter_plotly(filename_png, scale=3.0, **kwargs) + self._image_exporter_plotly(filename_svg, **kwargs) self.figure.write_json(filename_json) print(f" - saved plotly json file: {filename_json}") elif self.backend == "seaborn": @@ -796,7 +796,7 @@ def to_image_files(self, serial_number=None): print(f"TODO: implement saving {filename_svg}") print(f"TODO: implement saving {filename_json}") - def save(self, serial_number=None, save_hdf5=True, save_image_files=True, to_csv_kwargs:Union[dict, None] = None): + def save(self, serial_number=None, save_hdf5=True, save_image_files=True, to_csv_kwargs:Union[dict, None] = None, to_image_files_kwargs:Union[dict, None] = None): """Save to csv, hdf5 and image files. Args: @@ -818,7 +818,7 @@ def save(self, serial_number=None, save_hdf5=True, save_image_files=True, to_csv if self._figure_valid(): if save_image_files: - self.to_image_files(serial_number=serial_number) + self.to_image_files(serial_number=serial_number, **to_image_files_kwargs) def _output_path(self, serial_number=None): d = Path(self.figure_directory) From bf89ea88e0c5fabdbaee93f9013d917e3a1d3b34 Mon Sep 17 00:00:00 2001 From: jepegit Date: Wed, 12 Nov 2025 15:32:54 +0100 Subject: [PATCH 34/37] small fix --- cellpy/utils/batch_tools/batch_journals.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cellpy/utils/batch_tools/batch_journals.py b/cellpy/utils/batch_tools/batch_journals.py index 6937383d..e1ca3b3f 100644 --- a/cellpy/utils/batch_tools/batch_journals.py +++ b/cellpy/utils/batch_tools/batch_journals.py @@ -219,7 +219,7 @@ def from_db(self, project=None, name=None, batch_col=None, dbreader_kwargs=None, self.pages = self.engine(self.db_reader, batch_name=name, **kwargs) if self.pages.empty: - logging.critical(f"EMPTY JOURNAL: are you sure you have provided correct input to batch?") + logging.critical("EMPTY JOURNAL: are you sure you have provided correct input to batch?") logging.critical(f"name: {name}") logging.critical(f"project: {self.project}") logging.critical(f"batch_col: {batch_col}") From 585b7db267b3d9308d88f2048f712ce04cee358d Mon Sep 17 00:00:00 2001 From: jepegit Date: Wed, 12 Nov 2025 15:46:28 +0100 Subject: [PATCH 35/37] fix two bugs --- cellpy/utils/collectors.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/cellpy/utils/collectors.py b/cellpy/utils/collectors.py index ea806c32..47de400a 100644 --- a/cellpy/utils/collectors.py +++ b/cellpy/utils/collectors.py @@ -131,15 +131,15 @@ def generate_output_path(name, directory, serial_number=None): f = d / name return f -def incremental_image_exporter_plotly(self, filename, timeout=IMAGE_TO_FILE_TIMEOUT, **kwargs): +def incremental_image_exporter_plotly(figure, filename, timeout=IMAGE_TO_FILE_TIMEOUT, **kwargs): use_subprocess = kwargs.pop("use_subprocess", True) if not use_subprocess: print(f"saving image to {filename}") - self.figure.write_image(filename, **kwargs) + figure.write_image(filename, **kwargs) return p = Process( - target=self.figure.write_image, + target=figure.write_image, args=(filename,), name="save_plotly_image_to_file", kwargs=kwargs, @@ -818,6 +818,8 @@ def save(self, serial_number=None, save_hdf5=True, save_image_files=True, to_csv if self._figure_valid(): if save_image_files: + if to_image_files_kwargs is None: + to_image_files_kwargs = {} self.to_image_files(serial_number=serial_number, **to_image_files_kwargs) def _output_path(self, serial_number=None): From 2356c1f3c2b769ba2285ce3e09a2d19aa1290503 Mon Sep 17 00:00:00 2001 From: jepegit Date: Wed, 12 Nov 2025 16:12:55 +0100 Subject: [PATCH 36/37] bug fixes --- cellpy/cli.py | 4 +--- cellpy/utils/helpers.py | 6 +++--- cellpy/utils/plotutils.py | 2 ++ 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/cellpy/cli.py b/cellpy/cli.py index 49b586d2..caed43fa 100644 --- a/cellpy/cli.py +++ b/cellpy/cli.py @@ -1737,7 +1737,7 @@ def _new( else: selected_project_dir = None - click.echo(f"Select another directory instead") + click.echo("Select another directory instead") CREATE_NEW_DIR = "Create new project..." if not selected_project_dir: project_dirs = [ @@ -1745,9 +1745,7 @@ def _new( for d in directory.iterdir() if d.is_dir() and not d.name.startswith(".") ] - print(f"project_dirs: {project_dirs}") project_dirs.insert(0, CREATE_NEW_DIR) - print(f"project_dirs: {project_dirs}") project_dir = cookiecutter.prompt.read_user_choice( "project folder", project_dirs diff --git a/cellpy/utils/helpers.py b/cellpy/utils/helpers.py index 10d09719..79de3f2a 100644 --- a/cellpy/utils/helpers.py +++ b/cellpy/utils/helpers.py @@ -1369,7 +1369,7 @@ def select_summary_based_on_rate( step_table = cell.data.steps if partition_by_cv: - summary = _partition_summary_based_on_cv_steps(cell.data.summary) + summary = _partition_summary_based_on_cv_steps(cell) else: summary = cell.data.summary @@ -1380,7 +1380,7 @@ def select_summary_based_on_rate( summary.set_index(cycle_number_header, drop=True, inplace=True) else: print(f"{cycle_number_header} not set as index!") - print(f"Please, set the cycle index header as index before proceeding!") + print("Please, set the cycle index header as index before proceeding!") return summary if on: @@ -1437,7 +1437,7 @@ def add_normalized_capacity(cell, norm_cycles=None, individual_normalization=Fal try: norm_val_charge = cell.data.summary.loc[norm_cycles, col_name_charge].mean() except KeyError as e: - print(f"Oh no! Are you sure these cycle indexes exist?") + print("Oh no! Are you sure these cycle indexes exist?") print(f" norm_cycles: {norm_cycles}") print(f" cycle indexes: {list(cell.data.summary.index)}") raise KeyError from e diff --git a/cellpy/utils/plotutils.py b/cellpy/utils/plotutils.py index 119ac8a7..a7f5692d 100644 --- a/cellpy/utils/plotutils.py +++ b/cellpy/utils/plotutils.py @@ -1782,6 +1782,8 @@ def _calculate_seaborn_plot_properties(number_of_rows, number_of_cols, plot_type if fullcell_standard_normalization_type is not False: cum_loss_info_range = norm_range or [0.0, max(max_val_normalized_col, fullcell_standard_normalization_scaler)] + else: + cum_loss_info_range = norm_range or y_range cv_info = dict( title="", From c3310a4ff9c1bc3ea774d73180407f5b2eafcf49 Mon Sep 17 00:00:00 2001 From: jepegit Date: Wed, 12 Nov 2025 16:18:04 +0100 Subject: [PATCH 37/37] somehow strange fix of ica tests --- tests/test_ica.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_ica.py b/tests/test_ica.py index 4ba714d5..baacd2ab 100644 --- a/tests/test_ica.py +++ b/tests/test_ica.py @@ -179,7 +179,7 @@ def test_dqdv_multi_cycles_tidy(dataset): assert "voltage" in df_ica.columns assert "cycle" in df_ica.columns assert "dq" in df_ica.columns - assert df_ica.size == 26379 + assert df_ica.size == 26667 def test_dqdv_multi_cycles_wide(dataset): @@ -189,7 +189,7 @@ def test_dqdv_multi_cycles_wide(dataset): assert cycles_available.issuperset(cycles_processed) assert "voltage" in df_ica.columns.get_level_values(1) assert "dq" in df_ica.columns.get_level_values(1) - assert df_ica.size == 37536 + assert df_ica.size == 39744 # TODO - aulv: this test should be un-commented when hist-method