Various fixes

src/expipe_plugin_cinpla/scripts/data_processing.py → src/expipe_plugin_cinpla/data_loader.py

@@ -7,7 +7,7 @@
 import spikeinterface.extractors as se
 
 from pynwb import NWBHDF5IO
-from .utils import _get_data_path
+from .scripts.utils import _get_data_path
 
 
 def get_data_path(action):
@@ -260,22 +260,23 @@ def load_spiketrains(data_path, channel_group=None, lim=None):
             unit_id for unit_index, unit_id in enumerate(sorting.unit_ids) if groups[unit_index] == channel_group
         ]
     sptr = []
-    # build neo pbjects
+    # build neo objects
     for unit in unit_ids:
-        times = sorting.get_unit_spike_train(unit, return_times=True) * pq.s
+        spike_times = sorting.get_unit_spike_train(unit, return_times=True) * pq.s
         if lim is None:
             times = recording.get_times() * pq.s
             t_start = times[0]
             t_stop = times[-1]
         else:
             t_start = pq.Quantity(lim[0], "s")
             t_stop = pq.Quantity(lim[1], "s")
-        mask = (times >= t_start) & (times <= t_stop)
-        times = times[mask]
+        mask = (spike_times >= t_start) & (spike_times <= t_stop)
+        spike_times = spike_times[mask]
 
         st = neo.SpikeTrain(
-            times=times, t_start=t_start, t_stop=t_stop, sampling_rate=sorting.sampling_frequency * pq.Hz
+            times=spike_times, t_start=t_start, t_stop=t_stop, sampling_rate=sorting.sampling_frequency * pq.Hz
         )
+        st.annotations.update({"name": unit})
         for p in sorting.get_property_keys():
             st.annotations.update({p: sorting.get_unit_property(unit, p)})
         sptr.append(st)
@@ -313,7 +314,7 @@ def load_unit_annotations(data_path, channel_group=None):
         ]
 
     for unit in unit_ids:
-        annotations = {}
+        annotations = {"name": unit}
         for p in sorting.get_property_keys():
             annotations.update({p: sorting.get_unit_property(unit, p)})
         units.append(annotations)

src/expipe_plugin_cinpla/nwbutils/interfaces/openephystrackinginterface.py

@@ -78,14 +78,15 @@ def add_to_nwbfile(
                             rising = rising[:-1]
 
                         if len(rising) == len(falling):
-                            nwbfile.add_trial_column(
-                                name="channel",
-                                description="Open Ephys channel",
-                            )
-                            nwbfile.add_trial_column(
-                                name="processor",
-                                description="Open Ephys processor that recorded the event",
-                            )
+                            if nwbfile.trials is None:
+                                nwbfile.add_trial_column(
+                                    name="channel",
+                                    description="Open Ephys channel",
+                                )
+                                nwbfile.add_trial_column(
+                                    name="processor",
+                                    description="Open Ephys processor that recorded the event",
+                                )
                             start_times = times[rising].rescale("s").magnitude
                             stop_times = times[falling].rescale("s").magnitude
                             for start, stop in zip(start_times, stop_times):

src/expipe_plugin_cinpla/scripts/convert_old_project.py

@@ -128,6 +128,20 @@ def convert_old_project(
             delimiter = "*" * len(process_msg)
             print(f"\n{delimiter}\n{process_msg}\n{delimiter}\n")
             old_action = old_actions[action_id]
+
+            old_action_folder = old_project.path / "actions" / action_id
+            new_action_folder = new_project.path / "actions" / action_id
+            old_data_folder = old_action_folder / "data"
+            new_data_folder = new_action_folder / "data"
+            # main.exdir
+            old_exdir_folder = old_data_folder / "main.exdir"
+
+            if exist_ok and not new_action_folder.is_dir():
+                # Copy action that previously failed
+                print(f">>> Re-copying action {action_id} to new project\n")
+                shutil.copytree(
+                    old_action_folder, new_action_folder, ignore=shutil.ignore_patterns("main.exdir", ".git")
+                )
             new_action = new_project.actions[action_id]
 
             # replace file in attributes.yaml
@@ -136,18 +150,12 @@ def convert_old_project(
             attributes_str = attributes_str.replace("main.exdir", "main.nwb")
             attributes_file.write_text(attributes_str)
 
-            old_data_folder = old_project.path / "actions" / action_id / "data"
-            new_data_folder = new_project.path / "actions" / action_id / "data"
-
-            # main.exdir
-            old_exdir_folder = old_data_folder / "main.exdir"
-
             # find open-ephys folder
             acquisition_folder = old_exdir_folder / "acquisition"
             openephys_folders = [p for p in acquisition_folder.iterdir() if p.is_dir()]
             if len(openephys_folders) != 1:
                 print(f"Found {len(openephys_folders)} openephys folders in {acquisition_folder}!")
-                continue
+                raise ValueError("Expected to find exactly one openephys folder")
             openephys_path = openephys_folders[0]
             # here we assume the following action name: {entity_id}-{date}-{session}
             entity_id = action_id.split("-")[0]

src/expipe_plugin_cinpla/scripts/curation.py

@@ -138,6 +138,10 @@ def apply_curation(self, sorter, curated_sorting):
         else:
             recording = self.load_processed_recording(sorter)
 
+            # remove excess spikes
+            print("Removing excess spikes from curated sorting")
+            curated_sorting = sc.remove_excess_spikes(curated_sorting, recording=recording)
+
             # if not sort by group, extract dense and estimate group
             if "group" not in curated_sorting.get_property_keys():
                 compute_and_set_unit_groups(curated_sorting, recording)

src/expipe_plugin_cinpla/scripts/utils.py

@@ -1,6 +1,6 @@
 import sys
 import shutil
-from datetime import datetime
+from datetime import datetime, timedelta
 from pathlib import Path, PureWindowsPath
 import numpy as np
 
@@ -47,7 +47,7 @@ def query_yes_no(question, default="yes", answer=None):
 
 
 def deltadate(adjustdate, regdate):
-    delta = regdate - adjustdate if regdate > adjustdate else datetime.timedelta.max
+    delta = regdate - adjustdate if regdate > adjustdate else timedelta.max
     return delta
 
 
@@ -176,19 +176,18 @@ def _make_data_path(action, overwrite, suffix=".nwb"):
 
 
 def _get_data_path(action):
-    if "main" not in action.data:
-        return
     try:
+        if "main" not in action.data:
+            return
         data_path = action.data_path("main")
+        if not data_path.is_dir():
+            action_path = action._backend.path
+            project_path = action_path.parent.parent
+            # data_path = action.data['main']
+            data_path = project_path / str(Path(PureWindowsPath(action.data["main"])))
+        return data_path
     except:
-        data_path = Path("None")
-        pass
-    if not data_path.is_dir():
-        action_path = action._backend.path
-        project_path = action_path.parent.parent
-        # data_path = action.data['main']
-        data_path = project_path / str(Path(PureWindowsPath(action.data["main"])))
-    return data_path
+        return
 
 
 def register_templates(action, templates, overwrite=False):