Merge branch 'main' into cleanup

.circleci/config.yml

@@ -50,7 +50,7 @@ jobs:
             pip install --upgrade --progress-bar off pip
             pip install --upgrade --progress-bar off "autoreject @ https://api.github.com/repos/autoreject/autoreject/zipball/master" "mne[hdf5] @ git+https://github.com/mne-tools/mne-python@main" "mne-bids[full] @ https://api.github.com/repos/mne-tools/mne-bids/zipball/main" numba
             pip install -ve .[tests]
-            pip install "PyQt6!=6.6.1" "PyQt6-Qt6!=6.6.1,!=6.6.2,!=6.6.3"
+            pip install "PyQt6!=6.6.1" "PyQt6-Qt6!=6.6.1,!=6.6.2,!=6.6.3,!=6.7.0"
       - run:
           name: Check Qt
           command: |
@@ -170,15 +170,15 @@ jobs:
           at: ~/
       - restore_cache:
           keys:
-            - data-cache-ds000247-2
+            - data-cache-ds000247-3
       - bash_env
       - run:
           name: Get ds000247
           command: |
             $DOWNLOAD_DATA ds000247
       - codecov/upload
       - save_cache:
-          key: data-cache-ds000247-2
+          key: data-cache-ds000247-3
           paths:
             - ~/mne_data/ds000247
 
@@ -475,7 +475,7 @@ jobs:
       - bash_env
       - restore_cache:
           keys:
-            - data-cache-ds000247-2
+            - data-cache-ds000247-3
       - run:
           name: test ds000247
           command: $RUN_TESTS ds000247

docs/source/v1.9.md.inc

@@ -9,6 +9,8 @@
   [`epochs_metadata_tmin`][mne_bids_pipeline._config.epochs_metadata_tmin] and
   [`epochs_metadata_tmax`][mne_bids_pipeline._config.epochs_metadata_tmax]. (#873 by @hoechenberger)
 - If you requested processing of non-existing subjects, we will now provide a more helpful error message. (#928 by @hoechenberger)
+- We improved the logging output for automnated epochs rejection and cleaning via ICA and SSP. (#936, #937 by @hoechenberger)
+- ECG and EOG signals created during ICA artifact detection are now saved to disk. (#938 by @hoechenberger)
 
 ### :warning: Behavior changes
 
@@ -22,6 +24,7 @@
 - When using automated bad channel detection, now indicate the generated `*_bads.tsv` files whether a channel
   had previously already been marked as bad in the dataset. Resulting entries in the TSV file may now look like:
   `"pre-existing (before MNE-BIDS-pipeline was run) & auto-noisy"` (previously: only `"auto-noisy"`). (#930 by @hoechenberger)
+- The `ica_ctps_ecg_threshold` has been renamed to [`ica_ecg_threshold`][mne_bids_pipeline._config.ica_ecg_threshold]. (#935 by @hoechenberger)
 
 ### :package: Requirements
 
@@ -44,6 +47,7 @@
 - Fixed a compatibility bug with joblib 1.4.0. (#899 by @larsoner)
 - Fixed how "original" raw data is included in the report. Previously, bad channels, subject, and experimenter name would not
   be displayed correctly. (#930 by @hoechenberger)
+- In the report's table of contents, don't put the run numbers in quotation marks. (#933 by @hoechenberger)
 
 ### :medical_symbol: Code health and infrastructure
 

mne_bids_pipeline/_config.py

@@ -1362,9 +1362,10 @@
 `1` or `None` to not perform any decimation.
 """
 
-ica_ctps_ecg_threshold: float = 0.1
+ica_ecg_threshold: float = 0.1
 """
-The threshold parameter passed to `find_bads_ecg` method.
+The cross-trial phase statistics (CTPS) threshold parameter used for detecting
+ECG-related ICs.
 """
 
 ica_eog_threshold: float = 3.0

mne_bids_pipeline/_config_import.py

@@ -415,6 +415,9 @@ def _pydantic_validate(
     "N_JOBS": dict(
         new_name="n_jobs",
     ),
+    "ica_ctps_ecg_threshold": dict(
+        new_name="ica_ecg_threshold",
+    ),
 }
 
 

mne_bids_pipeline/_report.py

@@ -854,7 +854,7 @@ def _add_raw(
     extra_html: str | None = None,
 ):
     if bids_path_in.run is not None:
-        title += f", run {repr(bids_path_in.run)}"
+        title += f", run {bids_path_in.run}"
     elif bids_path_in.task in ("noise", "rest"):
         title += f", {bids_path_in.task}"
     plot_raw_psd = (

mne_bids_pipeline/steps/preprocessing/_06a1_fit_ica.py

@@ -177,9 +177,14 @@ def run_ica(
         ar.fit(epochs)
         ar_reject_log = ar.get_reject_log(epochs)
         epochs = epochs[~ar_reject_log.bad_epochs]
+
+        n_epochs_before_reject = len(epochs)
+        n_epochs_rejected = ar_reject_log.bad_epochs.sum()
+        n_epochs_after_reject = n_epochs_before_reject - n_epochs_rejected
+
         ar_n_interpolate_ = ar.n_interpolate_
         msg = (
-            f"autoreject marked {ar_reject_log.bad_epochs.sum()} epochs as bad "
+            f"autoreject marked {n_epochs_rejected} epochs as bad "
             f"(cross-validated n_interpolate limit: {ar_n_interpolate_})"
         )
         logger.info(**gen_log_kwargs(message=msg))
@@ -193,11 +198,33 @@ def run_ica(
             ch_types=cfg.ch_types,
             param="ica_reject",
         )
-        msg = f"Using PTP rejection thresholds: {ica_reject}"
-        logger.info(**gen_log_kwargs(message=msg))
+        n_epochs_before_reject = len(epochs)
         epochs.drop_bad(reject=ica_reject)
+        n_epochs_after_reject = len(epochs)
+        n_epochs_rejected = n_epochs_before_reject - n_epochs_after_reject
+
+        msg = (
+            f"Removed {n_epochs_rejected} of {n_epochs_before_reject} epochs via PTP "
+            f"rejection thresholds: {ica_reject}"
+        )
+        logger.info(**gen_log_kwargs(message=msg))
         ar = None
-    msg = "Saving ICA epochs to disk."
+
+    if 0 < n_epochs_after_reject < 0.5 * n_epochs_before_reject:
+        msg = (
+            "More than 50% of all epochs rejected. Please check the "
+            "rejection thresholds."
+        )
+        logger.warning(**gen_log_kwargs(message=msg))
+    elif n_epochs_after_reject == 0:
+        rejection_type = (
+            cfg.ica_reject if cfg.ica_reject == "autoreject_local" else "PTP-based"
+        )
+        raise RuntimeError(
+            f"No epochs remaining after {rejection_type} rejection. Cannot continue."
+        )
+
+    msg = f"Saving {n_epochs_after_reject} ICA epochs to disk."
     logger.info(**gen_log_kwargs(message=msg))
     epochs.save(
         out_files["epochs"],

mne_bids_pipeline/steps/preprocessing/_06a2_find_ica_artifacts.py

@@ -60,14 +60,12 @@ def detect_bad_components(
         inds, scores = ica.find_bads_ecg(
             epochs,
             method="ctps",
-            threshold=cfg.ica_ctps_ecg_threshold,
+            threshold=cfg.ica_ecg_threshold,
             ch_name=ch_names,
         )
 
     if not inds:
-        adjust_setting = (
-            "ica_eog_threshold" if which == "eog" else "ica_ctps_ecg_threshold"
-        )
+        adjust_setting = f"ica_{which}_threshold"
         warn = (
             f"No {artifact}-related ICs detected, this is highly "
             f"suspicious. A manual check is suggested. You may wish to "
@@ -77,7 +75,7 @@ def detect_bad_components(
     else:
         msg = (
             f"Detected {len(inds)} {artifact}-related ICs in "
-            f"{len(epochs)} {artifact} epochs."
+            f"{len(epochs)} {artifact} epochs: {', '.join([str(i) for i in inds])}"
         )
         logger.info(**gen_log_kwargs(message=msg))
 
@@ -131,6 +129,9 @@ def find_ica_artifacts(
     bids_basename = raw_fnames[0].copy().update(processing=None, split=None, run=None)
     out_files = dict()
     out_files["ica"] = bids_basename.copy().update(processing="ica", suffix="ica")
+    out_files["ecg"] = bids_basename.copy().update(processing="ica+ecg", suffix="ave")
+    out_files["eog"] = bids_basename.copy().update(processing="ica+eog", suffix="ave")
+
     # DO NOT add this to out_files["ica"] because we expect it to be modified by users.
     # If the modify it and it's in out_files, caching will detect the hash change and
     # consider *this step* a cache miss, and it will run again, overwriting the user's
@@ -290,6 +291,18 @@ def find_ica_artifacts(
     ecg_scores = None if len(ecg_scores) == 0 else ecg_scores
     eog_scores = None if len(eog_scores) == 0 else eog_scores
 
+    # Save ECG and EOG evokeds to disk.
+    for artifact_name, artifact_evoked in zip(("ecg", "eog"), (ecg_evoked, eog_evoked)):
+        if artifact_evoked:
+            msg = f"Saving {artifact_name.upper()} artifact: {out_files[artifact_name]}"
+            logger.info(**gen_log_kwargs(message=msg))
+            artifact_evoked.save(out_files[artifact_name], overwrite=True)
+        else:
+            # Don't track the non-existent output file
+            del out_files[artifact_name]
+
+    del artifact_name, artifact_evoked
+
     title = "ICA: components"
     with _open_report(
         cfg=cfg,
@@ -333,7 +346,7 @@ def get_config(
         ica_l_freq=config.ica_l_freq,
         ica_reject=config.ica_reject,
         ica_eog_threshold=config.ica_eog_threshold,
-        ica_ctps_ecg_threshold=config.ica_ctps_ecg_threshold,
+        ica_ecg_threshold=config.ica_ecg_threshold,
         autoreject_n_interpolate=config.autoreject_n_interpolate,
         random_state=config.random_state,
         ch_types=config.ch_types,

mne_bids_pipeline/steps/preprocessing/_08a_apply_ica.py

@@ -138,11 +138,14 @@ def apply_ica_epochs(
     epochs = mne.read_epochs(in_files.pop("epochs"), preload=True)
 
     # Now actually reject the components.
-    msg = f'Rejecting ICs: {", ".join([str(ic) for ic in ica.exclude])}'
+    msg = (
+        f'Rejecting ICs with the following indices: '
+        f'{", ".join([str(i) for i in ica.exclude])}'
+    )
     logger.info(**gen_log_kwargs(message=msg))
     epochs_cleaned = ica.apply(epochs.copy())  # Copy b/c works in-place!
 
-    msg = "Saving reconstructed epochs after ICA."
+    msg = f"Saving {len(epochs)} reconstructed epochs after ICA."
     logger.info(**gen_log_kwargs(message=msg))
     epochs_cleaned.save(
         out_files["epochs"],

mne_bids_pipeline/steps/preprocessing/_08b_apply_ssp.py

@@ -58,6 +58,10 @@ def apply_ssp_epochs(
     epochs = mne.read_epochs(in_files.pop("epochs"), preload=True)
     projs = mne.read_proj(in_files.pop("proj"))
     epochs_cleaned = epochs.copy().add_proj(projs).apply_proj()
+
+    msg = f"Saving {len(epochs_cleaned)} reconstructed epochs after SSP."
+    logger.info(**gen_log_kwargs(message=msg))
+
     epochs_cleaned.save(
         out_files["epochs"],
         overwrite=True,

mne_bids_pipeline/steps/preprocessing/_09_ptp_reject.py

@@ -142,14 +142,18 @@ def drop_ptp(
                     logger.info(**gen_log_kwargs(message=msg))
                     reject[ch_type] = threshold
 
-        msg = f"Using PTP rejection thresholds: {reject}"
-        logger.info(**gen_log_kwargs(message=msg))
-
         n_epochs_before_reject = len(epochs)
         epochs.reject_tmin = cfg.reject_tmin
         epochs.reject_tmax = cfg.reject_tmax
         epochs.drop_bad(reject=reject)
         n_epochs_after_reject = len(epochs)
+        n_epochs_rejected = n_epochs_before_reject - n_epochs_after_reject
+
+        msg = (
+            f"Removed {n_epochs_rejected} of {n_epochs_before_reject} epochs via PTP "
+            f"rejection thresholds: {reject}"
+        )
+        logger.info(**gen_log_kwargs(message=msg))
 
     if 0 < n_epochs_after_reject < 0.5 * n_epochs_before_reject:
         msg = (
@@ -167,7 +171,7 @@ def drop_ptp(
             f"No epochs remaining after {rejection_type} rejection. Cannot continue."
         )
 
-    msg = "Saving cleaned, baseline-corrected epochs …"
+    msg = f"Saving {n_epochs_after_reject} cleaned, baseline-corrected epochs …"
 
     epochs.apply_baseline(cfg.baseline)
     epochs.save(

mne_bids_pipeline/tests/conftest.py

@@ -70,6 +70,7 @@ def pytest_configure(config):
     ignore:nperseg .* is greater.*:UserWarning
     # NumPy 2.0
     ignore:__array_wrap__ must accept context.*:DeprecationWarning
+    ignore:__array__ implementation doesn't accept.*:DeprecationWarning
     """
     for warning_line in warning_lines.split("\n"):
         warning_line = warning_line.strip()