ENH: ICA/PSD-fixes, TimedMessageBox, PreCommit, Ruff (#48)

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

.github/workflows/run_tests.yml

@@ -6,22 +6,6 @@ on:
       - main
 
 jobs:
-  black:
-    name: Check code style with black
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-      - name: Set up Python
-        uses: actions/setup-python@v4
-        with:
-          python-version: 3.11
-      - name: Install dependencies
-        run: |
-          python -m pip install --upgrade pip
-          python -m pip install black
-      - name: Check code style
-        run: black --check .
-
   pytest:
     strategy:
       fail-fast: false
@@ -56,14 +40,20 @@ jobs:
           python -m pip install --upgrade pip
           python -m pip install -ve .
           python -m pip install -r requirements_dev.txt
+      - name: Get testing version
+        run: |
+          mkdir -p mne/datasets
+          curl --output-dir mne/datasets/ --remote-name https://raw.githubusercontent.com/mne-tools/mne-python/main/mne/datasets/config.py
+          curl --remote-name https://raw.githubusercontent.com/mne-tools/mne-python/main/tools/get_testing_version.sh
+          bash ./get_testing_version.sh
       - uses: actions/cache@v3
         with:
-          key: ${{ runner.os }}-sample-data
+          key: ${{ env.TESTING_VERSION }}
           path: ~/mne_data
         name: 'Cache testing data'
       - run: python -c 'import mne; print(mne.datasets.testing.data_path(verbose=True))'
         name: 'Download sample data'
       - name: Show system information
         run: mne sys_info
       - run: pytest
-        name: Run Tests
+        name: Run Tests

.pre-commit-config.yaml

@@ -0,0 +1,25 @@
+repos:
+  # Pre-commit hooks
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.4.0
+    hooks:
+    - id: check-json
+    - id: check-toml
+    - id: end-of-file-fixer
+    - id: fix-encoding-pragma
+    - id: requirements-txt-fixer
+    - id: trailing-whitespace
+
+  # Black
+  - repo: https://github.com/psf/black
+    rev: 23.7.0
+    hooks:
+      - id: black
+        args: [ --quiet ]
+
+  # Ruff
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.0.286
+    hooks:
+      - id: ruff
+        args: [--fix, --exit-non-zero-on-fix]

LICENSE.txt

@@ -26,4 +26,4 @@ DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
 SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
 CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
 OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

doc/Makefile

@@ -22,4 +22,4 @@ help:
 view:
 	@python -c "import webbrowser; webbrowser.open_new_tab('file://$(PWD)/build/html/index.html')"
 
-show: view
+show: view

doc/source/conf.py

@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 # Configuration file for the Sphinx documentation builder.
 #
 # This file only contains a selection of the most common options. For a full

doc/source/parameter_widgets.rst

@@ -20,5 +20,3 @@ Parameter Widgets
    SliderGui
    StringGui
    TupleGui
-
-

mne_pipeline_hd/_version.py

@@ -1 +1,2 @@
+# -*- coding: utf-8 -*-
 __version__ = "0.3.3a0.dev0"

mne_pipeline_hd/development/dev_utils.py

@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 """
 Authors: Martin Schulz <[email protected]>
 License: BSD 3-Clause

mne_pipeline_hd/development/development_considerations.md

@@ -47,4 +47,4 @@ setting should be device/OS-dependent:
    e.g. `img_format` or `show_plots`).
 2. QSettings(), which is stored by Qt on an OS-depending location and which may
    differ between devices/OS. Settings which dependent on the device/OS should
-   go here (e.g. `n_jobs` or `use_cuda`)
+   go here (e.g. `n_jobs` or `use_cuda`)

mne_pipeline_hd/extra/default_settings.json

@@ -31,4 +31,4 @@
     "app_font_size": 10,
     "app_style": "auto"
   }
-}
+}

mne_pipeline_hd/extra/functions.csv

@@ -5,12 +5,14 @@ add_erm_ssp;Empty-Room SSP;MEEG;Compute;Preprocessing;True;False;;operations;bas
 eeg_reference_raw;Set EEG Reference;MEEG;Compute;Preprocessing;False;False;;operations;basic;meeg,ref_channels
 find_events;Find events;MEEG;Compute;events;False;False;;operations;basic;meeg,stim_channels,min_duration,shortest_event,adjust_timeline_by_msec
 find_6ch_binary_events;Find events HD;MEEG;Compute;events;False;False;;operations;basic;meeg,min_duration,shortest_event,adjust_timeline_by_msec
-run_ica;Run ICA;MEEG;Compute;Preprocessing;False;False;;operations;basic;meeg,ica_method,ica_fitto,n_components,ica_noise_cov,ica_remove_proj,ica_reject,ica_autoreject,ch_types,ch_names,reject_by_annotation,ica_eog,eog_channel,ica_ecg,ecg_channel
-apply_ica;Apply ICA;MEEG;Compute;Preprocessing;False;False;;operations;basic;meeg,ica_apply_target,n_pca_components
 epoch_raw;Get Epochs;MEEG;Compute;events;False;False;;operations;basic;meeg,ch_types,ch_names,t_epoch,baseline,apply_proj,reject,flat,reject_by_annotation,bad_interpolation,use_autoreject,consensus_percs,n_interpolates,overwrite_ar,decim,n_jobs
 estimate_noise_covariance;Noise-Covariance;MEEG;Compute;Preprocessing;False;False;;operations;basic;meeg,baseline,n_jobs,noise_cov_mode,noise_cov_method
+run_ica;Run ICA;MEEG;Compute;Preprocessing;False;False;;operations;basic;meeg,ica_method,ica_fitto,n_components,ica_noise_cov,ica_remove_proj,ica_reject,ica_autoreject,overwrite_ar,ch_types,ch_names,reject_by_annotation,ica_eog,eog_channel,ica_ecg,ecg_channel
+apply_ica;Apply ICA;MEEG;Compute;Preprocessing;False;False;;operations;basic;meeg,ica_apply_target,n_pca_components
 get_evokeds;Get Evokeds;MEEG;Compute;events;False;False;;operations;basic;meeg
 interpolate_bads;Interpolate Bads;MEEG;Compute;Preprocessing;False;False;;operations;basic;meeg,bad_interpolation
+compute_psd_raw;Compute PSD (Raw);MEEG;Compute;Time-Frequency;False;False;;operations;basic;meeg,psd_method,n_jobs
+compute_psd_epochs;Compute PSD (Epochs);MEEG;Compute;Time-Frequency;False;False;;operations;basic;meeg,psd_method,n_jobs
 tfr;Time-Frequency;MEEG;Compute;Time-Frequency;False;False;;operations;basic;meeg,tfr_freqs,tfr_n_cycles,tfr_average,tfr_use_fft,tfr_baseline,tfr_baseline_mode,tfr_method,multitaper_bandwidth,stockwell_width,n_jobs
 apply_watershed;;FSMRI;Compute;MRI-Preprocessing;False;False;;operations;basic;fsmri
 prepare_bem;;FSMRI;Compute;MRI-Preprocessing;False;False;;operations;basic;fsmri,bem_spacing,bem_conductivity
@@ -41,10 +43,10 @@ plot_sensors;;MEEG;Plot;Forward;True;False;;plot;basic;meeg,plot_sensors_kind,ch
 plot_raw;;MEEG;Plot;Raw;True;False;;plot;basic;meeg,show_plots
 plot_filtered;;MEEG;Plot;Raw;True;False;;plot;basic;meeg,show_plots
 plot_events;;MEEG;Plot;events;True;False;;plot;basic;meeg,show_plots
-plot_power_spectra;;MEEG;Plot;Time-Frequency;True;False;;plot;basic;meeg,show_plots,n_jobs
-plot_power_spectra_topo;;MEEG;Plot;Time-Frequency;True;False;;plot;basic;meeg,show_plots,n_jobs
-plot_power_spectra_epochs;;MEEG;Plot;Time-Frequency;True;False;;plot;basic;meeg,show_plots,n_jobs
-plot_power_spectra_epochs_topo;;MEEG;Plot;Time-Frequency;True;False;;plot;basic;meeg,show_plots,n_jobs
+plot_power_spectra;;MEEG;Plot;Time-Frequency;True;False;;plot;basic;meeg,show_plots
+plot_power_spectra_topomap;;MEEG;Plot;Time-Frequency;True;False;;plot;basic;meeg,psd_topomap_bands,show_plots
+plot_power_spectra_epochs;;MEEG;Plot;Time-Frequency;True;False;;plot;basic;meeg,show_plots
+plot_power_spectra_epochs_topomap;;MEEG;Plot;Time-Frequency;True;False;;plot;basic;meeg,psd_topomap_bands,show_plots
 plot_tfr;;MEEG;Plot;Time-Frequency;True;False;;plot;basic;meeg,show_plots
 plot_epochs;;MEEG;Plot;Epochs;True;False;;plot;basic;meeg,show_plots
 plot_epochs_image;;MEEG;Plot;Epochs;True;False;;plot;basic;meeg,show_plots
@@ -77,6 +79,6 @@ plot_grand_avg_connect;;Group;Plot;Grand-Average;True;False;;plot;basic;group,co
 plot_ica_components;Plot ICA-Components;MEEG;Plot;ICA;True;False;;plot;basic;meeg,show_plots
 plot_ica_sources;Plot ICA-Sources;MEEG;Plot;ICA;True;False;;plot;basic;meeg,ica_source_data,show_plots
 plot_ica_overlay;Plot ICA-Overlay;MEEG;Plot;ICA;True;False;;plot;basic;meeg,ica_overlay_data,show_plots
-plot_ica_properties;Plot ICA-Properties;MEEG;Plot;ICA;True;False;;plot;basic;meeg,show_plots
+plot_ica_properties;Plot ICA-Properties;MEEG;Plot;ICA;True;False;;plot;basic;meeg,ica_fitto,show_plots
 plot_ica_scores;Plot ICA-Scores;MEEG;Plot;ICA;True;False;;plot;basic;meeg,show_plots
 print_info;Print Info;MEEG;Plot;Raw;False;False;;operations;basic;meeg

mne_pipeline_hd/extra/license.txt

@@ -26,4 +26,4 @@ DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
 SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
 CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
 OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

mne_pipeline_hd/extra/parameters.csv

@@ -33,7 +33,7 @@ reject;;epochs;{'mag':3000e-15, 'grad':3000e-13, 'eeg':100e-6, 'eog':200e-6};;Ch
 flat;;epochs;{'mag': 1e-15, 'grad':1e-13, 'eeg': 1e-6};;Chose flat-thresholds;DictGui;{'none_select': True}
 decim;;epochs;1;;Downsampling-Factor for epochs;IntGui;
 ica_method;ICA-Method;ICA;fastica;;The method for calculating ICA;ComboGui;{'options': ['fastica', 'infomax', 'picard']}
-ica_fitto;Fit ICA to:;ICA;'raw_filtered';;The data to fit the ICA to;ComboGui;{'options': ['raw', 'raw_filtered', 'epochs']}
+ica_fitto;Fit ICA to:;ICA;'raw_filtered';;The data to fit the ICA to;ComboGui;{'options': {'raw': 'Raw (unfiltered)', 'raw_filtered': 'Raw (filtered)', 'epochs': 'Epochs'}}
 ica_apply_target;Apply ICA to:;ICA;raw_filtered;;The target object (Raw, Epochs or Evoked) to apply ICA and remove the selected components from;ComboGui;{'options': {'raw_filtered': 'Raw (filtered)', 'epochs': 'Epochs', 'evoked': 'Evoked'}}
 n_components;;ICA;25;;The number of components for ICA;IntGui;
 max_pca_components;;ICA;None;;;IntGui;{'none_select': True}
@@ -88,12 +88,14 @@ ecd_orientations;;Inverse;{};;;DictGui;
 morph_to;;Grand-Average;fsaverage;;name of the freesurfer subject to be morphed to;StringGui;
 ica_source_data;;ICA;raw_filtered;;Which data to plot in sources-plot from ICA;ComboGui;{'options': {'raw': 'Raw (unfiltered)', 'raw_filtered': 'Raw (filtered)', 'epochs': 'Epochs', 'epochs_eog': 'Epochs (EOG)', 'epochs_ecg': 'Epochs (ECG)', 'evoked': 'Evoked', 'evoked_eog': 'Evoked (EOG)', 'evoked_ecg': 'Evoked (ECG)'}}
 ica_overlay_data;;ICA;evoked;;Which data to plot in overlay-plot from ICA;ComboGui;{'options': {'raw': 'Raw (unfiltered)', 'raw_filtered': 'Raw (filtered)', 'evoked': 'Evoked', 'evoked_eog': 'Evoked (EOG)', 'evoked_ecg': 'Evoked (ECG)'}}
-plot_sensors_kind;;Plot;'topomap';;The kind of plot for plot_sensors;ComboGui;{'options':['topomap', '3d', 'select']}
+plot_sensors_kind;;Plot;'topomap';;The kind of plot for plot_sensors;ComboGui;{'options': ['topomap', '3d', 'select']}
 erm_ssp_duration;;Preprocessing;1;s;The time-chunk to use for ssp;IntGui;
 erm_n_grad;;Preprocessing;2;;The number of projections for Gradiometer;IntGui;
 erm_n_mag;;Preprocessing;2;;The number of projections for Magnetometer;IntGui;
 erm_n_eeg;;Preprocessing;0;;The number of projections for EEG;IntGui;
 ga_interpolate_bads;;Grand-Average;True;;If to interpolate bad channels for the Grand-Average;BoolGui;
 ga_drop_bads;;Grand-Average;True;;If to drop bad channels for the Grand-Average;BoolGui;
 connectivity_vmin;;Connectivity;None;;Minimum value for colormap;FloatGui;{'step': 0.01, 'none_select':True}
-connectivity_vmax;;Connectivity;None;;Maximum value for colormap;FloatGui;{'step': 0.01, 'none_select':True}
+connectivity_vmax;;Connectivity;None;;Maximum value for colormap;FloatGui;{'step': 0.01, 'none_select':True}
+psd_method;;Time-Frequency;welch;;The method for spectral estimation;ComboGui;{'options': ['welch', 'multitaper']}
+psd_topomap_bands;;Time-Frequency;None;;The frequency bands for the topomap-plot;DictGui;{'none_select': True}

mne_pipeline_hd/functions/operations.py

@@ -18,7 +18,6 @@
 from itertools import combinations
 from os import environ
 from os.path import isdir, isfile, join
-from pathlib import Path
 
 import autoreject as ar
 import mne
@@ -557,7 +556,7 @@ def epoch_raw(
         if use_autoreject == "Threshold":
             reject = meeg.load_json("autoreject_threshold")
             if reject is None or overwrite_ar:
-                reject = ar.get_rejection_threshold(epochs)
+                reject = ar.get_rejection_threshold(epochs, random_state=8)
                 meeg.save_json("autoreject_threshold", reject)
             print(
                 f"Dropping bad epochs with autoreject"
@@ -595,6 +594,7 @@ def run_ica(
     ica_remove_proj,
     ica_reject,
     ica_autoreject,
+    overwrite_ar,
     ch_types,
     ch_names,
     reject_by_annotation,
@@ -604,16 +604,9 @@ def run_ica(
     ecg_channel,
     **kwargs,
 ):
-    if ica_fitto == "epochs":
-        data = meeg.load_epochs()
-        # Bad-Channels and Channel-Types are already picked in epoch_raw
-    else:
-        if ica_fitto == "raw":
-            data = meeg.load_raw()
-
-        else:
-            data = meeg.load_filtered()
-
+    data = meeg.load(ica_fitto)
+    # Bad-Channels and Channel-Types are already picked in epochs
+    if ica_fitto != "epochs":
         data.pick(ch_types, exclude="bads")
         if len(ch_names) > 0 and ch_names != "all":
             data.pick_channels(ch_names)
@@ -645,12 +638,12 @@ def run_ica(
         simulated_epochs = mne.Epochs(
             data, simulated_events, baseline=None, tmin=0, tmax=1, proj=False
         )
-        reject = ar.get_rejection_threshold(simulated_epochs)
+        reject = ar.get_rejection_threshold(simulated_epochs, random_state=8)
         print(f"Autoreject Rejection-Threshold: {reject}")
     elif ica_autoreject and ica_fitto == "epochs":
         reject = meeg.load_json("autoreject_threshold")
-        if not reject:
-            reject = ar.get_rejection_threshold(data)
+        if reject is None or overwrite_ar:
+            reject = ar.get_rejection_threshold(data, random_state=8)
             meeg.save_json("autoreject_threshold", reject)
     else:
         reject = ica_reject
@@ -711,6 +704,11 @@ def run_ica(
             meeg.save_eog_epochs(eog_epochs)
             meeg.save_json("eog_indices", eog_indices)
             meeg.save_json("eog_scores", eog_scores)
+    else:
+        # Remove old eog_epochs, eog_indices and eog_scores if new ICA is calculated
+        meeg.remove_path("eog_epochs")
+        meeg.remove_json("eog_indices")
+        meeg.remove_json("eog_scores")
 
     if ica_ecg:
         create_ecg_kwargs = check_kwargs(kwargs, mne.preprocessing.create_ecg_epochs)
@@ -748,6 +746,11 @@ def run_ica(
             meeg.save_ecg_epochs(ecg_epochs)
             meeg.save_json("ecg_indices", ecg_indices)
             meeg.save_json("ecg_scores", ecg_scores)
+    else:
+        # Remove old ecg_epochs, ecg_indices and ecg_scores if new ICA is calculated
+        meeg.remove_path("ecg_epochs")
+        meeg.remove_json("ecg_indices")
+        meeg.remove_json("ecg_scores")
 
     meeg.save_ica(ica)
     # Add components to ica_exclude-dictionary
@@ -837,6 +840,22 @@ def grand_avg_evokeds(group, ga_interpolate_bads, ga_drop_bads):
     group.save_ga_evokeds(ga_evokeds)
 
 
+def compute_psd_raw(meeg, psd_method, n_jobs, **kwargs):
+    raw = meeg.load_filtered()
+    psd_raw = raw.compute_psd(
+        method=psd_method, fmax=raw.info["lowpass"], n_jobs=n_jobs, **kwargs
+    )
+    meeg.save_psd_raw(psd_raw)
+
+
+def compute_psd_epochs(meeg, psd_method, n_jobs, **kwargs):
+    epochs = meeg.load_epochs()
+    psd_epochs = epochs.compute_psd(
+        method=psd_method, fmax=epochs.info["lowpass"], n_jobs=n_jobs, **kwargs
+    )
+    meeg.save_psd_epochs(psd_epochs)
+
+
 def tfr(
     meeg,
     tfr_freqs,