From 120b8a175bf6203321d6f602dece70648def0ed3 Mon Sep 17 00:00:00 2001 From: John Date: Thu, 29 Aug 2024 15:07:58 -0500 Subject: [PATCH 01/11] allows manually marked bad channels to be excluded from reference / interpolated, without being removed (which had been causing an error) --- pyprep/reference.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/pyprep/reference.py b/pyprep/reference.py index 8fea7cc..4c40e5a 100644 --- a/pyprep/reference.py +++ b/pyprep/reference.py @@ -83,7 +83,7 @@ def __init__( raw.load_data() self.raw = raw.copy() self.ch_names = self.raw.ch_names - self.raw.pick_types(eeg=True, eog=False, meg=False) + self.raw.pick_types(eeg=True, eog=False, meg=False, exclude = []) self.ch_names_eeg = self.raw.ch_names self.EEG = self.raw.get_data() self.reference_channels = params["ref_chs"] @@ -97,6 +97,7 @@ def __init__( self.random_state = check_random_state(random_state) self._extra_info = {} self.matlab_strict = matlab_strict + self.bads_manual = raw.info["bads"] def perform_reference(self, max_iterations=4): """Estimate the true signal mean and interpolate bad channels. @@ -149,6 +150,7 @@ def perform_reference(self, max_iterations=4): self.bad_before_interpolation = noisy_detector.get_bads(verbose=True) self.EEG_before_interpolation = self.EEG.copy() self.noisy_channels_before_interpolation = noisy_detector.get_bads(as_dict=True) + self.noisy_channels_before_interpolation['bad_manual'] = self.bads_manual self._extra_info["interpolated"] = noisy_detector._extra_info bad_channels = _union(self.bad_before_interpolation, self.unusable_channels) @@ -223,7 +225,7 @@ def robust_reference(self, max_iterations=4): # Determine channels to use/exclude from initial reference estimation self.unusable_channels = _union( noisy_detector.bad_by_nan + noisy_detector.bad_by_flat, - noisy_detector.bad_by_SNR, + noisy_detector.bad_by_SNR + self.bads_manual, ) reference_channels = _set_diff(self.reference_channels, self.unusable_channels) @@ -237,6 +239,7 @@ def robust_reference(self, max_iterations=4): "bad_by_SNR": [], "bad_by_dropout": [], "bad_by_ransac": [], + "bad_manual": self.bads_manual, "bad_all": [], } @@ -282,7 +285,8 @@ def robust_reference(self, max_iterations=4): noisy[bad_type] = _union(noisy[bad_type], noisy_new[bad_type]) if bad_type not in ignore: bad_chans.update(noisy[bad_type]) - noisy["bad_all"] = list(bad_chans) + noisy["bad_manual"] = self.bads_manual + noisy["bad_all"] = list(bad_chans) + self.bads_manual logger.info(f"Bad channels: {noisy}") if ( From 1ce3e54b33936f8e75a035a452bcfdfbff55b106 Mon Sep 17 00:00:00 2001 From: John Veillette Date: Thu, 29 Aug 2024 15:15:50 -0500 Subject: [PATCH 02/11] adds new contributor to CITATION.cff --- CITATION.cff | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CITATION.cff b/CITATION.cff index dce4ee7..4e3e838 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -51,6 +51,9 @@ authors: - given-names: Ayush family-names: Agarwal affiliation: 'Techno India University, India' + - given-names: John + family-names: Veillette + affiliation: 'Department of Psychology, University of Chicago, Chicago, IL, USA' type: software repository-code: 'https://github.com/sappelhoff/pyprep' license: MIT From 6650396136eb26aabca033ea9301c52455780b30 Mon Sep 17 00:00:00 2001 From: John Veillette Date: Thu, 29 Aug 2024 15:18:06 -0500 Subject: [PATCH 03/11] adds new contributor ORCID --- CITATION.cff | 1 + 1 file changed, 1 insertion(+) diff --git a/CITATION.cff b/CITATION.cff index 4e3e838..1da3c61 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -54,6 +54,7 @@ authors: - given-names: John family-names: Veillette affiliation: 'Department of Psychology, University of Chicago, Chicago, IL, USA' + orcid: 'https://orcid.org/0000-0002-0332-4372' type: software repository-code: 'https://github.com/sappelhoff/pyprep' license: MIT From d94fec878175e8030c7e608d58b2d42fa4b8ac32 Mon Sep 17 00:00:00 2001 From: John Veillette Date: Thu, 29 Aug 2024 15:24:38 -0500 Subject: [PATCH 04/11] Update authors.rst --- docs/authors.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/authors.rst b/docs/authors.rst index 13779d0..0ddb8f6 100644 --- a/docs/authors.rst +++ b/docs/authors.rst @@ -10,3 +10,4 @@ .. _Stefan Appelhoff: https://stefanappelhoff.com/ .. _Victor Xiang: https://github.com/Nick3151 .. _Yorguin Mantilla: https://github.com/yjmantilla +.. _John Veillette: https://github.com/john-veillette From c1bd96f0868711493f2a3548d8e9568af3124a57 Mon Sep 17 00:00:00 2001 From: John Veillette Date: Thu, 29 Aug 2024 15:28:05 -0500 Subject: [PATCH 05/11] Update changelog.rst --- docs/changelog.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 331724c..d389553 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -31,7 +31,7 @@ Changelog Bug ~~~ -- nothing yet +- :class:`~pyprep.Reference` now keeps and interpolates channels channels manually marked as bad before PREP, by `John Veillette`_ (:gh:`146`) Code health ~~~~~~~~~~~ From f4a72ee35bd73103114a7910d2dbe456566b6a4c Mon Sep 17 00:00:00 2001 From: John Date: Thu, 29 Aug 2024 15:46:45 -0500 Subject: [PATCH 06/11] fixes some formatting issues flagged by CI --- pyprep/reference.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyprep/reference.py b/pyprep/reference.py index 4c40e5a..ddf80b8 100644 --- a/pyprep/reference.py +++ b/pyprep/reference.py @@ -83,7 +83,7 @@ def __init__( raw.load_data() self.raw = raw.copy() self.ch_names = self.raw.ch_names - self.raw.pick_types(eeg=True, eog=False, meg=False, exclude = []) + self.raw.pick_types(eeg=True, eog=False, meg=False, exclude=[]) self.ch_names_eeg = self.raw.ch_names self.EEG = self.raw.get_data() self.reference_channels = params["ref_chs"] @@ -150,7 +150,7 @@ def perform_reference(self, max_iterations=4): self.bad_before_interpolation = noisy_detector.get_bads(verbose=True) self.EEG_before_interpolation = self.EEG.copy() self.noisy_channels_before_interpolation = noisy_detector.get_bads(as_dict=True) - self.noisy_channels_before_interpolation['bad_manual'] = self.bads_manual + self.noisy_channels_before_interpolation["bad_manual"] = self.bads_manual self._extra_info["interpolated"] = noisy_detector._extra_info bad_channels = _union(self.bad_before_interpolation, self.unusable_channels) From 85d1a2dfa6b02964e355dec82b7f79a8ba546ef8 Mon Sep 17 00:00:00 2001 From: John Date: Mon, 16 Sep 2024 16:35:39 -0500 Subject: [PATCH 07/11] adds manual bad channels to find_noisy_channels --- pyprep/find_noisy_channels.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pyprep/find_noisy_channels.py b/pyprep/find_noisy_channels.py index 65431d9..74d844c 100644 --- a/pyprep/find_noisy_channels.py +++ b/pyprep/find_noisy_channels.py @@ -59,6 +59,7 @@ def __init__(self, raw, do_detrend=True, random_state=None, matlab_strict=False) raw.load_data() self.raw_mne = raw.copy() + self.bad_by_manual = raw.info["bads"] self.raw_mne.pick_types(eeg=True) self.sample_rate = raw.info["sfreq"] if do_detrend: @@ -162,6 +163,7 @@ def get_bads(self, verbose=False, as_dict=False): "bad_by_SNR": self.bad_by_SNR, "bad_by_dropout": self.bad_by_dropout, "bad_by_ransac": self.bad_by_ransac, + "bad_by_manual": self.bad_by_manual } all_bads = set() From b7ec26ed10dbf9d52d78a557a540e2779fdf9792 Mon Sep 17 00:00:00 2001 From: John Date: Mon, 16 Sep 2024 16:36:25 -0500 Subject: [PATCH 08/11] changes name of manual bads to match find_noisy_channels (which requires a specific format to work properly) --- pyprep/reference.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pyprep/reference.py b/pyprep/reference.py index ddf80b8..c754673 100644 --- a/pyprep/reference.py +++ b/pyprep/reference.py @@ -150,7 +150,7 @@ def perform_reference(self, max_iterations=4): self.bad_before_interpolation = noisy_detector.get_bads(verbose=True) self.EEG_before_interpolation = self.EEG.copy() self.noisy_channels_before_interpolation = noisy_detector.get_bads(as_dict=True) - self.noisy_channels_before_interpolation["bad_manual"] = self.bads_manual + self.noisy_channels_before_interpolation["bad_by_manual"] = self.bads_manual self._extra_info["interpolated"] = noisy_detector._extra_info bad_channels = _union(self.bad_before_interpolation, self.unusable_channels) @@ -239,7 +239,7 @@ def robust_reference(self, max_iterations=4): "bad_by_SNR": [], "bad_by_dropout": [], "bad_by_ransac": [], - "bad_manual": self.bads_manual, + "bad_by_manual": self.bads_manual, "bad_all": [], } @@ -285,7 +285,7 @@ def robust_reference(self, max_iterations=4): noisy[bad_type] = _union(noisy[bad_type], noisy_new[bad_type]) if bad_type not in ignore: bad_chans.update(noisy[bad_type]) - noisy["bad_manual"] = self.bads_manual + noisy["bad_by_manual"] = self.bads_manual noisy["bad_all"] = list(bad_chans) + self.bads_manual logger.info(f"Bad channels: {noisy}") From c43e967275bcc049dc252c87f074646a61213a9a Mon Sep 17 00:00:00 2001 From: John Date: Mon, 16 Sep 2024 16:40:08 -0500 Subject: [PATCH 09/11] adds test coverage --- tests/test_find_noisy_channels.py | 13 +++++++++++++ tests/test_reference.py | 1 + 2 files changed, 14 insertions(+) diff --git a/tests/test_find_noisy_channels.py b/tests/test_find_noisy_channels.py index be2e6c1..b323936 100644 --- a/tests/test_find_noisy_channels.py +++ b/tests/test_find_noisy_channels.py @@ -73,6 +73,19 @@ def test_bad_by_nan(raw_tmp): nd.find_bad_by_nan_flat() assert nd.bad_by_nan == [raw_tmp.ch_names[nan_idx]] +def test_bad_by_manual(raw_tmp): + """Test the detection of channels marked bad a priori.""" + # Insert a NaN value into a random channel + n_chans = raw_tmp.get_data().shape[0] + nan_idx = int(rng.integers(0, n_chans, 1)[0]) + raw_tmp._data[nan_idx, 3] = np.nan + raw_tmp.info["bads"] = [raw.ch_names[0]] + + # Test record of a priori bad channels on NoisyChannels init + nd = NoisyChannels(raw_tmp, do_detrend=False) + assert nd.bad_by_manual == [raw.ch_names[0]] + assert raw.ch_names[0] in nd.get_bads(as_dict = False) + def test_bad_by_flat(raw_tmp): """Test the detection of channels with flat or very weak signals.""" diff --git a/tests/test_reference.py b/tests/test_reference.py index 5f01ed1..44ab3d7 100644 --- a/tests/test_reference.py +++ b/tests/test_reference.py @@ -23,6 +23,7 @@ def test_basic_input(raw, montage): reference = Reference(raw_tmp, params, ransac=False) reference.perform_reference() assert isinstance(reference.noisy_channels, dict) + assert isinstance(reference.noisy_channels["bad_by_manual"], list) assert isinstance(reference.noisy_channels_original, dict) assert isinstance(reference.bad_before_interpolation, list) assert isinstance(reference.reference_signal, np.ndarray) From b957cea2617942e1244ea09998d88729457e4972 Mon Sep 17 00:00:00 2001 From: John Date: Mon, 16 Sep 2024 16:44:10 -0500 Subject: [PATCH 10/11] fixes bug and formatting in test --- tests/test_find_noisy_channels.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/test_find_noisy_channels.py b/tests/test_find_noisy_channels.py index b323936..76ad737 100644 --- a/tests/test_find_noisy_channels.py +++ b/tests/test_find_noisy_channels.py @@ -79,12 +79,13 @@ def test_bad_by_manual(raw_tmp): n_chans = raw_tmp.get_data().shape[0] nan_idx = int(rng.integers(0, n_chans, 1)[0]) raw_tmp._data[nan_idx, 3] = np.nan - raw_tmp.info["bads"] = [raw.ch_names[0]] + raw_tmp.info["bads"] = [raw_tmp.ch_names[0]] # Test record of a priori bad channels on NoisyChannels init nd = NoisyChannels(raw_tmp, do_detrend=False) - assert nd.bad_by_manual == [raw.ch_names[0]] - assert raw.ch_names[0] in nd.get_bads(as_dict = False) + assert nd.bad_by_manual == [raw_tmp.ch_names[0]] + assert raw_tmp.ch_names[0] in nd.get_bads(as_dict=False) + raw_tmp.info["bads"] = [] def test_bad_by_flat(raw_tmp): From f465aa1977196657365ab49cf20d314a18473a4e Mon Sep 17 00:00:00 2001 From: John Date: Mon, 16 Sep 2024 16:50:00 -0500 Subject: [PATCH 11/11] format --- pyprep/find_noisy_channels.py | 2 +- tests/test_find_noisy_channels.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyprep/find_noisy_channels.py b/pyprep/find_noisy_channels.py index 74d844c..44b1a34 100644 --- a/pyprep/find_noisy_channels.py +++ b/pyprep/find_noisy_channels.py @@ -163,7 +163,7 @@ def get_bads(self, verbose=False, as_dict=False): "bad_by_SNR": self.bad_by_SNR, "bad_by_dropout": self.bad_by_dropout, "bad_by_ransac": self.bad_by_ransac, - "bad_by_manual": self.bad_by_manual + "bad_by_manual": self.bad_by_manual, } all_bads = set() diff --git a/tests/test_find_noisy_channels.py b/tests/test_find_noisy_channels.py index 76ad737..594fbc5 100644 --- a/tests/test_find_noisy_channels.py +++ b/tests/test_find_noisy_channels.py @@ -73,9 +73,9 @@ def test_bad_by_nan(raw_tmp): nd.find_bad_by_nan_flat() assert nd.bad_by_nan == [raw_tmp.ch_names[nan_idx]] + def test_bad_by_manual(raw_tmp): """Test the detection of channels marked bad a priori.""" - # Insert a NaN value into a random channel n_chans = raw_tmp.get_data().shape[0] nan_idx = int(rng.integers(0, n_chans, 1)[0]) raw_tmp._data[nan_idx, 3] = np.nan