From 9f8f0c67d541e295b17c7064c560a6dce97da803 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Erik=20Bj=C3=A4reholt?= Date: Fri, 25 Mar 2022 11:35:55 +0100 Subject: [PATCH 01/20] ci: fix broken CI @JohnGriffiths please be more careful next time, heh: https://github.com/NeuroTechX/eeg-notebooks/commit/3efcb158f5cb2111072320a67046315789c3101b --- .github/workflows/test.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index c678b7a51..c5c3cee13 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -61,11 +61,10 @@ jobs: pip install . - name: Run eegnb install test shell: bash - pip install -U psychtoolbox # JG_ADD run: | if [ "$RUNNER_OS" == "Linux" ]; then Xvfb :0 -screen 0 1024x768x24 -ac +extension GLX +render -noreset &> xvfb.log & - / + pip install -U psychtoolbox # JG_ADD export DISPLAY=:0 fi eegnb --help From 405dfb2f80d7c1940e5e8ba1c9b8ed623bc6526c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Erik=20Bj=C3=A4reholt?= Date: Fri, 25 Mar 2022 11:41:20 +0100 Subject: [PATCH 02/20] build(deps): removed version locks for doc deps To work around https://github.com/sphinx-doc/sphinx/issues/10291 --- requirements.txt | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/requirements.txt b/requirements.txt index ea8be01ca..36daafec0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -45,14 +45,14 @@ nbval types-requests # Docs requirements -sphinx==3.1.1 -sphinx-gallery==0.8.1 -sphinx_rtd_theme==0.5.0 -sphinx-tabs==1.3.0 -sphinx-copybutton==0.3.1 -sphinxcontrib-httpdomain==1.7.0 -numpydoc==1.1.0 -recommonmark==0.6.0 -versioneer==0.19 -rst2pdf==0.98 -docutils==0.17 +sphinx +sphinx-gallery +sphinx_rtd_theme +sphinx-tabs +sphinx-copybutton +sphinxcontrib-httpdomain +numpydoc +recommonmark +versioneer +rst2pdf +docutils From 9d8b0a81c5b12a2d01a12d84c108693555bd013c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Erik=20Bj=C3=A4reholt?= Date: Fri, 25 Mar 2022 11:46:03 +0100 Subject: [PATCH 03/20] build(deps): added pygame, to resolve PsychoPy 'no audio device' error on import --- requirements.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/requirements.txt b/requirements.txt index 36daafec0..1aed7e133 100644 --- a/requirements.txt +++ b/requirements.txt @@ -18,6 +18,9 @@ h5py>=3.1.0 pytest-shutil pyo>=1.0.3; platform_system == "Linux" +# This was needed due to https://discourse.psychopy.org/t/missing-sound-libraries-for-psychopy3-standalone-release-3-2-3-for-win32/9162 +pygame + # This might try to build from source on linux (since there are no wheels for Linux on PyPI) # You can pass `--find-links=https://extras.wxpython.org/wxPython4/extras/linux/gtk3/ubuntu-20.04/` your `pip install` to use the prebuilt wheels at the link. wxPython>=4.0 ; platform_system == "Linux" From af4323df6c61d38e7e1d1fb4e5ea417ea0b535aa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Erik=20Bj=C3=A4reholt?= Date: Fri, 25 Mar 2022 11:51:29 +0100 Subject: [PATCH 04/20] docs: fixed docs config --- doc/conf.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/conf.py b/doc/conf.py index cae9a5dcc..d4503b43d 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -263,7 +263,7 @@ def setup(app): 'backreferences_dir': 'generated', # Where to drop linking files between examples & API 'doc_module': ('eeg-notebooks'), 'reference_url': {'eeg-notebooks': None}, - 'remove_conffig_comments': True} + 'remove_config_comments': True} """ sphinx_gallery_conf = { @@ -284,7 +284,7 @@ def setup(app): 'backreferences_dir': 'generated', # Where to drop linking files between examples & API 'doc_module': ('eeg-notebooks',), 'reference_url': {'eeg-notebooksS': None}, - 'remove_conffig_comments': True, + 'remove_config_comments': True, } """ From 00be98212456e1acad93a549a4559b7a76491140 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Erik=20Bj=C3=A4reholt?= Date: Fri, 25 Mar 2022 12:09:26 +0100 Subject: [PATCH 05/20] ci: added installation of libportaudio2 on Ubuntu --- .github/workflows/test.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index c5c3cee13..b057d1c44 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -44,7 +44,8 @@ jobs: # xvfb is a dependency to create a virtual display # libgtk-3-dev is a requirement for wxPython # freeglut3-dev is a requirement for a wxPython dependency - sudo apt-get -y install xvfb libgtk-3-dev freeglut3-dev + # libportaudio2 might be required to import psychopy on Ubuntu + sudo apt-get -y install xvfb libgtk-3-dev freeglut3-dev libportaudio2 - name: Install Linux dependencies if: "startsWith(runner.os, 'Linux')" run: | From fc49c843afbee8492f231bd8c3b1c2cee126dbbb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Erik=20Bj=C3=A4reholt?= Date: Fri, 25 Mar 2022 12:10:25 +0100 Subject: [PATCH 06/20] ci: added experimental macOS + py-3.9 run --- .github/workflows/test.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index b057d1c44..3cf9b0f01 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -17,7 +17,9 @@ jobs: python_version: [3.7] include: - os: ubuntu-latest - python_version: 3.8 + python_version: 3.9 + - os: macOS-latest + python_version: 3.9 # Experimental: Python 3.9 # See issue: https://github.com/NeuroTechX/eeg-notebooks/issues/50 #- os: ubuntu-18.04 From 81105c336ddb5255b08a2b2d9436189efa6d0d25 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Erik=20Bj=C3=A4reholt?= Date: Fri, 25 Mar 2022 12:12:50 +0100 Subject: [PATCH 07/20] ci: removed unnecessary explicit install of psychtoolbox --- .github/workflows/test.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 3cf9b0f01..a46357610 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -67,7 +67,6 @@ jobs: run: | if [ "$RUNNER_OS" == "Linux" ]; then Xvfb :0 -screen 0 1024x768x24 -ac +extension GLX +render -noreset &> xvfb.log & - pip install -U psychtoolbox # JG_ADD export DISPLAY=:0 fi eegnb --help From d0a793b14c90217ac485779ede64fcb2410508cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Erik=20Bj=C3=A4reholt?= Date: Fri, 25 Mar 2022 12:16:32 +0100 Subject: [PATCH 08/20] build(deps): updated psychopy to 2022.1.1 (to fix issues in CI) --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 1aed7e133..f2f3bb050 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ # Main repo requirements -psychopy==2020.2.3 +psychopy==2022.1.1 psychtoolbox scikit-learn>=0.23.2 pandas>=1.1.4 From 19be96f633a3649581e2d973a2d62f85743be8dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Erik=20Bj=C3=A4reholt?= Date: Fri, 25 Mar 2022 12:24:08 +0100 Subject: [PATCH 09/20] ci: improved CI config, added libpulse-dev --- .github/workflows/test.yml | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index a46357610..9e624ca1f 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -46,21 +46,20 @@ jobs: # xvfb is a dependency to create a virtual display # libgtk-3-dev is a requirement for wxPython # freeglut3-dev is a requirement for a wxPython dependency - # libportaudio2 might be required to import psychopy on Ubuntu - sudo apt-get -y install xvfb libgtk-3-dev freeglut3-dev libportaudio2 + # portaudio19-dev *might* be required to import psychopy on Ubuntu + # libpulse-dev required to build sphinx stuff, for some strange reason + sudo apt-get -y install xvfb libgtk-3-dev freeglut3-dev portaudio19-dev libpulse-dev + - name: Upgrade pip + run: | + python -m pip install --upgrade pip wheel - name: Install Linux dependencies if: "startsWith(runner.os, 'Linux')" run: | - python -m pip install --upgrade pip wheel - # Install wxPython wheels since they are distribution-specific and therefore not on PyPI # See: https://wxpython.org/pages/downloads/index.html pip install -U -f https://extras.wxpython.org/wxPython4/extras/linux/gtk3/ubuntu-18.04 wxPython - - pip install . - - name: Install MacOS/Windows dependencies + - name: Install dependencies run: | - python -m pip install --upgrade pip wheel pip install . - name: Run eegnb install test shell: bash From 2e0f55e03f5df2c811e369cbe6ed1168737a87da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Erik=20Bj=C3=A4reholt?= Date: Fri, 25 Mar 2022 12:41:10 +0100 Subject: [PATCH 10/20] ci: bumped Python to 3.8, added libsdl2-dev --- .github/workflows/test.yml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 9e624ca1f..b3f378f14 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -14,12 +14,12 @@ jobs: fail-fast: false matrix: os: [ubuntu-latest, windows-latest, macOS-latest] - python_version: [3.7] + python_version: [3.8] include: - os: ubuntu-latest python_version: 3.9 - - os: macOS-latest - python_version: 3.9 + #- os: macOS-latest + # python_version: 3.9 # Experimental: Python 3.9 # See issue: https://github.com/NeuroTechX/eeg-notebooks/issues/50 #- os: ubuntu-18.04 @@ -48,7 +48,8 @@ jobs: # freeglut3-dev is a requirement for a wxPython dependency # portaudio19-dev *might* be required to import psychopy on Ubuntu # libpulse-dev required to build sphinx stuff, for some strange reason - sudo apt-get -y install xvfb libgtk-3-dev freeglut3-dev portaudio19-dev libpulse-dev + # libsdl2-dev required by psychopy + sudo apt-get -y install xvfb libgtk-3-dev freeglut3-dev portaudio19-dev libpulse-dev libsdl2-dev - name: Upgrade pip run: | python -m pip install --upgrade pip wheel From d678aeddde0076dc4e235e0559e6c233a31bff1d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Erik=20Bj=C3=A4reholt?= Date: Fri, 25 Mar 2022 12:43:19 +0100 Subject: [PATCH 11/20] build(deps): loosened requirements for seaborn, removed unneeded requirements --- requirements.txt | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/requirements.txt b/requirements.txt index f2f3bb050..2436971cd 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,34 +1,27 @@ # Main repo requirements psychopy==2022.1.1 -psychtoolbox scikit-learn>=0.23.2 pandas>=1.1.4 numpy>=1.19.4 mne>=0.20.8 -seaborn==0.9.0 +seaborn>=0.11.0 pyriemann>=0.2.7 jupyter muselsl>=2.0.2 brainflow>=4.8.2 gdown matplotlib>=3.3.3 -pysocks==1.7.1 -pyserial==3.5 +pysocks>=1.7.1 +pyserial>=3.5 h5py>=3.1.0 pytest-shutil pyo>=1.0.3; platform_system == "Linux" -# This was needed due to https://discourse.psychopy.org/t/missing-sound-libraries-for-psychopy3-standalone-release-3-2-3-for-win32/9162 -pygame - # This might try to build from source on linux (since there are no wheels for Linux on PyPI) # You can pass `--find-links=https://extras.wxpython.org/wxPython4/extras/linux/gtk3/ubuntu-20.04/` your `pip install` to use the prebuilt wheels at the link. wxPython>=4.0 ; platform_system == "Linux" click -# can be removed once minimum version is Python 3.7 -dataclasses; python_version == '3.6' - # pywinhook needs some special treatment since there are only wheels on PyPI for Python 3.7-3.8, and building requires special tools (swig, VS C++ tools) # See issue: https://github.com/NeuroTechX/eeg-notebooks/issues/29 pywinhook>=1.6.0 ; platform_system == "Windows" and (python_version == "3.7" or python_version == "3.8") From a5c93898c9ee54726de0e011892a36693d0df720 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Erik=20Bj=C3=A4reholt?= Date: Fri, 25 Mar 2022 12:53:41 +0100 Subject: [PATCH 12/20] ci: set job names --- .github/workflows/test.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index b3f378f14..554491ea6 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -8,7 +8,7 @@ on: jobs: test: - name: ${{ matrix.os }}, py-${{ matrix.python_version }} + name: test (${{ matrix.os }}, py-${{ matrix.python_version }}) runs-on: ${{ matrix.os }} strategy: fail-fast: false @@ -81,6 +81,7 @@ jobs: pytest typecheck: + name: typecheck (${{ matrix.os }}, py-${{ matrix.python_version }}) runs-on: ${{ matrix.os }} strategy: fail-fast: false From 0c32c9e34182cea4b5c5960841cf19fbd94ff46d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Erik=20Bj=C3=A4reholt?= Date: Fri, 25 Mar 2022 12:57:41 +0100 Subject: [PATCH 13/20] ci: more fixes --- .github/workflows/docs.yml | 8 +++++++- .github/workflows/test.yml | 5 +++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index d208acd84..af4b64112 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -8,7 +8,7 @@ on: jobs: build: - runs-on: ubuntu-18.04 + runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - name: Set up Python @@ -17,6 +17,12 @@ jobs: python-version: 3.8 - name: Install dependencies run: | + # update archive links + sudo apt update + + # See test.yml for description of the following deps + sudo apt-get -y install xvfb libgtk-3-dev freeglut3-dev portaudio19-dev libpulse-dev pulseaudio libsdl2-dev + python -m pip install --upgrade pip wheel # Install wxPython wheels since they are distribution-specific and therefore not on PyPI diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 554491ea6..c178e99f1 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -47,9 +47,10 @@ jobs: # libgtk-3-dev is a requirement for wxPython # freeglut3-dev is a requirement for a wxPython dependency # portaudio19-dev *might* be required to import psychopy on Ubuntu - # libpulse-dev required to build sphinx stuff, for some strange reason + # pulseaudio *might* be required to actually run the tests (on PsychoPy import) + # libpulse-dev required to build pocketsphinx (speech recognition dependency of psychopy) # libsdl2-dev required by psychopy - sudo apt-get -y install xvfb libgtk-3-dev freeglut3-dev portaudio19-dev libpulse-dev libsdl2-dev + sudo apt-get -y install xvfb libgtk-3-dev freeglut3-dev portaudio19-dev libpulse-dev pulseaudio libsdl2-dev - name: Upgrade pip run: | python -m pip install --upgrade pip wheel From 89e80e46ecf32824b5d29fc9c59f774c8e0673d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Erik=20Bj=C3=A4reholt?= Date: Fri, 25 Mar 2022 13:02:30 +0100 Subject: [PATCH 14/20] ci: fix install of deps for typechecking job --- .github/workflows/test.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index c178e99f1..69f32d9b8 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -101,10 +101,8 @@ jobs: run: | # update archive links sudo apt-get update - # xvfb is a dependency to create a virtual display - # libgtk-3-dev is a requirement for wxPython - # freeglut3-dev is a requirement for a wxPython dependency - sudo apt-get -y install xvfb libgtk-3-dev freeglut3-dev + # See comment above (in the test job) for details about these dependencies + sudo apt-get -y install xvfb libgtk-3-dev freeglut3-dev portaudio19-dev libpulse-dev pulseaudio libsdl2-dev - name: Install Linux dependencies if: "startsWith(runner.os, 'Linux')" run: | From f67bde6d0909821d1db3946562047d473370c2c5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Erik=20Bj=C3=A4reholt?= Date: Fri, 25 Mar 2022 13:53:15 +0100 Subject: [PATCH 15/20] fix: fixed seaborn compatibility --- eegnb/analysis/utils.py | 41 ++++- eegnb/analysis/utils_old.py | 294 ------------------------------------ eegnb/cli/__main__.py | 8 +- eegnb/datasets/datasets.py | 18 ++- 4 files changed, 48 insertions(+), 313 deletions(-) delete mode 100755 eegnb/analysis/utils_old.py diff --git a/eegnb/analysis/utils.py b/eegnb/analysis/utils.py index bd67baa9a..59818b9ca 100644 --- a/eegnb/analysis/utils.py +++ b/eegnb/analysis/utils.py @@ -2,11 +2,11 @@ from copy import deepcopy import math import logging +import sys from collections import OrderedDict from glob import glob -from typing import Union, List, Dict +from typing import Union, List from time import sleep, time -from numpy.core.fromnumeric import std import pandas as pd import numpy as np @@ -16,13 +16,13 @@ from mne.channels import make_standard_montage from mne.filter import create_filter from matplotlib import pyplot as plt +from scipy import stats from scipy.signal import lfilter, lfilter_zi from eegnb import _get_recording_dir from eegnb.devices.eeg import EEG from eegnb.devices.utils import EEG_INDICES, SAMPLE_FREQS - # this should probably not be done here sns.set_context("talk") @@ -32,6 +32,29 @@ logger = logging.getLogger(__name__) +def _bootstrap(data, n_boot: int, ci: float): + """From: https://stackoverflow.com/a/47582329/965332""" + boot_dist = [] + for i in range(int(n_boot)): + resampler = np.random.randint(0, data.shape[0], data.shape[0]) + sample = data.take(resampler, axis=0) + boot_dist.append(np.mean(sample, axis=0)) + b = np.array(boot_dist) + s1 = np.apply_along_axis(stats.scoreatpercentile, 0, b, 50 - ci / 2) + s2 = np.apply_along_axis(stats.scoreatpercentile, 0, b, 50 + ci / 2) + return (s1, s2) + + +def _tsplotboot(ax, data, n_boot: int, ci: float, **kw): + """From: https://stackoverflow.com/a/47582329/965332""" + x = np.arange(data.shape[1]) + est = np.mean(data, axis=0) + cis = _bootstrap(data, n_boot, ci) + ax.fill_between(x, cis[0], cis[1], alpha=0.2, **kw) + ax.plot(x, est, **kw) + ax.margins(x=0) + + def load_csv_as_raw( fnames: List[str], sfreq: float, @@ -247,15 +270,19 @@ def plot_conditions( plot_axes.append(axes[axis_x, axis_y]) axes = plot_axes + print("\n\n\n\n\n\n") + for ch in range(channel_count): for cond, color in zip(conditions.values(), palette): - sns.tsplot( - X[y.isin(cond), ch], - time=times, + y_cond = y.isin(cond) + # make X[y_cond, ch] one-dimensional + X_cond = X[y_cond, ch] + _tsplotboot( + ax=axes[ch], + data=X_cond, color=color, n_boot=n_boot, ci=ci, - ax=axes[ch], ) if diff_waveform: diff --git a/eegnb/analysis/utils_old.py b/eegnb/analysis/utils_old.py deleted file mode 100755 index 16f01ffe5..000000000 --- a/eegnb/analysis/utils_old.py +++ /dev/null @@ -1,294 +0,0 @@ -# -*- coding: utf-8 -*- - -from glob import glob -import os -from collections import OrderedDict - -from mne import create_info, concatenate_raws -from mne.io import RawArray -from mne.channels import make_standard_montage -import pandas as pd -import numpy as np -import seaborn as sns -from matplotlib import pyplot as plt - - -sns.set_context("talk") -sns.set_style("white") - - -def load_muse_csv_as_raw( - filename, - sfreq=256.0, - ch_ind=[0, 1, 2, 3], - stim_ind=5, - replace_ch_names=None, - verbose=1, -): - """Load CSV files into a Raw object. - - Args: - filename (str or list): path or paths to CSV files to load - - Keyword Args: - subject_nb (int or str): subject number. If 'all', load all - subjects. - session_nb (int or str): session number. If 'all', load all - sessions. - sfreq (float): EEG sampling frequency - ch_ind (list): indices of the EEG channels to keep - stim_ind (int): index of the stim channel - replace_ch_names (dict or None): dictionary containing a mapping to - rename channels. Useful when an external electrode was used. - - Returns: - (mne.io.array.array.RawArray): loaded EEG - """ - n_channel = len(ch_ind) - - raw = [] - print(filename) - for fname in filename: - print(fname) - # read the file - data = pd.read_csv(fname, index_col=0) - - # name of each channels - ch_names = list(data.columns)[0:n_channel] + ["Stim"] - - if replace_ch_names is not None: - ch_names = [ - c if c not in replace_ch_names.keys() else replace_ch_names[c] - for c in ch_names - ] - - # type of each channels - ch_types = ["eeg"] * n_channel + ["stim"] - montage = make_standard_montage("standard_1005") - - # get data and exclude Aux channel - data = data.values[:, ch_ind + [stim_ind]].T - - # convert in Volts (from uVolts) - data[:-1] *= 1e-6 - - # create MNE object - info = create_info( - ch_names=ch_names, - ch_types=ch_types, - sfreq=sfreq, - montage=montage, - verbose=verbose, - ) - raw.append(RawArray(data=data, info=info, verbose=verbose)) - - # concatenate all raw objects - print("raw is") - print(raw) - raws = concatenate_raws(raw, verbose=verbose) - - return raws - - -def load_data( - data_dir, - site="eegnb_examples", - experiment="visual_n170", - device="muse2016", - subject_nb=1, - session_nb=1, - sfreq=256.0, - ch_ind=[0, 1, 2, 3], - stim_ind=5, - replace_ch_names=None, - verbose=1, -): - - """Load CSV files from the /data directory into a Raw object. - - Args: - data_dir (str): directory inside /data that contains the - CSV files to load, e.g., 'auditory/P300' - - Keyword Args: - subject_nb (int or str): subject number. If 'all', load all - subjects. - session_nb (int or str): session number. If 'all', load all - sessions. - sfreq (float): EEG sampling frequency - ch_ind (list): indices of the EEG channels to keep - stim_ind (int): index of the stim channel - replace_ch_names (dict or None): dictionary containing a mapping to - rename channels. Useful when an external electrode was used. - - Returns: - (mne.io.array.array.RawArray): loaded EEG - """ - if subject_nb == "all": - subject_nb = "*" - if session_nb == "all": - session_nb = "*" - - subject_nb_str = "%04.f" % subject_nb - session_nb_str = "%03.f" % session_nb - subsess = "subject{}/session{}/*.csv".format(subject_nb_str, session_nb_str) - data_path = os.path.join(data_dir, experiment, site, device, subsess) - fnames = glob(data_path) - - return load_muse_csv_as_raw( - fnames, - sfreq=sfreq, - ch_ind=ch_ind, - stim_ind=stim_ind, - replace_ch_names=replace_ch_names, - verbose=verbose, - ) - - -def plot_conditions( - epochs, - conditions=OrderedDict(), - ci=97.5, - n_boot=1000, - title="", - palette=None, - ylim=(-6, 6), - diff_waveform=(1, 2), -): - """Plot ERP conditions. - - Args: - epochs (mne.epochs): EEG epochs - - Keyword Args: - conditions (OrderedDict): dictionary that contains the names of the - conditions to plot as keys, and the list of corresponding marker - numbers as value. E.g., - - conditions = {'Non-target': [0, 1], - 'Target': [2, 3, 4]} - - ci (float): confidence interval in range [0, 100] - n_boot (int): number of bootstrap samples - title (str): title of the figure - palette (list): color palette to use for conditions - ylim (tuple): (ymin, ymax) - diff_waveform (tuple or None): tuple of ints indicating which - conditions to subtract for producing the difference waveform. - If None, do not plot a difference waveform - - Returns: - (matplotlib.figure.Figure): figure object - (list of matplotlib.axes._subplots.AxesSubplot): list of axes - """ - if isinstance(conditions, dict): - conditions = OrderedDict(conditions) - - if palette is None: - palette = sns.color_palette("hls", len(conditions) + 1) - - X = epochs.get_data() * 1e6 - times = epochs.times - y = pd.Series(epochs.events[:, -1]) - - fig, axes = plt.subplots(2, 2, figsize=[12, 6], sharex=True, sharey=True) - axes = [axes[1, 0], axes[0, 0], axes[0, 1], axes[1, 1]] - - for ch in range(4): - for cond, color in zip(conditions.values(), palette): - sns.tsplot( - X[y.isin(cond), ch], - time=times, - color=color, - n_boot=n_boot, - ci=ci, - ax=axes[ch], - ) - - if diff_waveform: - diff = np.nanmean(X[y == diff_waveform[1], ch], axis=0) - np.nanmean( - X[y == diff_waveform[0], ch], axis=0 - ) - axes[ch].plot(times, diff, color="k", lw=1) - - axes[ch].set_title(epochs.ch_names[ch]) - axes[ch].set_ylim(ylim) - axes[ch].axvline( - x=0, ymin=ylim[0], ymax=ylim[1], color="k", lw=1, label="_nolegend_" - ) - - axes[0].set_xlabel("Time (s)") - axes[0].set_ylabel("Amplitude (uV)") - axes[-1].set_xlabel("Time (s)") - axes[1].set_ylabel("Amplitude (uV)") - - if diff_waveform: - legend = ["{} - {}".format(diff_waveform[1], diff_waveform[0])] + list( - conditions.keys() - ) - else: - legend = conditions.keys() - axes[-1].legend(legend, loc="lower right") - sns.despine() - plt.tight_layout() - - if title: - fig.suptitle(title, fontsize=20) - - return fig, axes - - -def plot_highlight_regions( - x, y, hue, hue_thresh=0, xlabel="", ylabel="", legend_str=() -): - """Plot a line with highlighted regions based on additional value. - - Plot a line and highlight ranges of x for which an additional value - is lower than a threshold. For example, the additional value might be - pvalues, and the threshold might be 0.05. - - Args: - x (array_like): x coordinates - y (array_like): y values of same shape as `x` - - Keyword Args: - hue (array_like): values to be plotted as hue based on `hue_thresh`. - Must be of the same shape as `x` and `y`. - hue_thresh (float): threshold to be applied to `hue`. Regions for which - `hue` is lower than `hue_thresh` will be highlighted. - xlabel (str): x-axis label - ylabel (str): y-axis label - legend_str (tuple): legend for the line and the highlighted regions - - Returns: - (matplotlib.figure.Figure): figure object - (list of matplotlib.axes._subplots.AxesSubplot): list of axes - """ - fig, axes = plt.subplots(1, 1, figsize=(10, 5), sharey=True) - - axes.plot(x, y, lw=2, c="k") - plt.xlabel(xlabel) - plt.ylabel(ylabel) - - kk = 0 - a = [] - while kk < len(hue): - if hue[kk] < hue_thresh: - b = kk - kk += 1 - while kk < len(hue): - if hue[kk] > hue_thresh: - break - else: - kk += 1 - a.append([b, kk - 1]) - else: - kk += 1 - - st = (x[1] - x[0]) / 2.0 - for p in a: - axes.axvspan(x[p[0]] - st, x[p[1]] + st, facecolor="g", alpha=0.5) - plt.legend(legend_str) - sns.despine() - - return fig, axes diff --git a/eegnb/cli/__main__.py b/eegnb/cli/__main__.py index 6e57fbcef..e19aca4c9 100644 --- a/eegnb/cli/__main__.py +++ b/eegnb/cli/__main__.py @@ -1,9 +1,7 @@ -from eegnb import DATA_DIR import click -from time import sleep -from os import path import os -import shutil + +from eegnb import DATA_DIR from eegnb.datasets.datasets import zip_data_folders from .introprompt import intro_prompt @@ -34,7 +32,7 @@ def runexp( recdur: float = None, outfname: str = None, prompt: bool = False, - dosigqualcheck = True, + dosigqualcheck=True, ): """ Run experiment. diff --git a/eegnb/datasets/datasets.py b/eegnb/datasets/datasets.py index 3fc7ae876..98d16f758 100644 --- a/eegnb/datasets/datasets.py +++ b/eegnb/datasets/datasets.py @@ -1,11 +1,17 @@ -import os,sys,glob,shutil,numpy as np, pandas as pd -import requests, zipfile,gdown +import os +import glob +import shutil +import zipfile + +import requests +import gdown + from datetime import datetime from eegnb import DATA_DIR # eegnb example data sites. do not select these when zipping recordings -eegnb_sites = ['eegnb_examples', 'grifflab_dev', 'jadinlab_home'] +eegnb_sites = ["eegnb_examples", "grifflab_dev", "jadinlab_home"] def fetch_dataset( @@ -64,7 +70,7 @@ def fetch_dataset( } # If no non-default top-level data path specified, use default - if data_dir == None: + if data_dir is None: data_dir = DATA_DIR # check parameter entries @@ -160,9 +166,7 @@ def fetch_dataset( return fnames - -def zip_data_folders(experiment: str, - site: str="local"): +def zip_data_folders(experiment: str, site: str = "local"): """ Run data zipping From ea3d3e4882e55f46a1f44156a900f6fed0af64fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Erik=20Bj=C3=A4reholt?= Date: Fri, 25 Mar 2022 13:53:53 +0100 Subject: [PATCH 16/20] build: refactored common build/testing/CI commands into Makefile --- .github/workflows/docs.yml | 14 +++---------- .github/workflows/test.yml | 43 +++++++++++--------------------------- Makefile | 29 +++++++++++++++++++++++++ 3 files changed, 44 insertions(+), 42 deletions(-) create mode 100644 Makefile diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index af4b64112..e0bf54141 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -17,22 +17,14 @@ jobs: python-version: 3.8 - name: Install dependencies run: | - # update archive links - sudo apt update - - # See test.yml for description of the following deps - sudo apt-get -y install xvfb libgtk-3-dev freeglut3-dev portaudio19-dev libpulse-dev pulseaudio libsdl2-dev - + make install-deps-apt python -m pip install --upgrade pip wheel - # Install wxPython wheels since they are distribution-specific and therefore not on PyPI - # See: https://wxpython.org/pages/downloads/index.html - pip install -U -f https://extras.wxpython.org/wxPython4/extras/linux/gtk3/ubuntu-18.04 wxPython - + make install-deps-wxpython pip install . - name: Build docs run: | - cd doc && make html + make docs - name: Deploy Docs uses: peaceiris/actions-gh-pages@v3 if: github.ref == 'refs/heads/master' diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 69f32d9b8..f30ab53c3 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -41,28 +41,17 @@ jobs: - name: Install APT dependencies if: "startsWith(runner.os, 'Linux')" run: | - # update archive links - sudo apt-get update - # xvfb is a dependency to create a virtual display - # libgtk-3-dev is a requirement for wxPython - # freeglut3-dev is a requirement for a wxPython dependency - # portaudio19-dev *might* be required to import psychopy on Ubuntu - # pulseaudio *might* be required to actually run the tests (on PsychoPy import) - # libpulse-dev required to build pocketsphinx (speech recognition dependency of psychopy) - # libsdl2-dev required by psychopy - sudo apt-get -y install xvfb libgtk-3-dev freeglut3-dev portaudio19-dev libpulse-dev pulseaudio libsdl2-dev + make install-deps-apt - name: Upgrade pip run: | python -m pip install --upgrade pip wheel - name: Install Linux dependencies if: "startsWith(runner.os, 'Linux')" run: | - # Install wxPython wheels since they are distribution-specific and therefore not on PyPI - # See: https://wxpython.org/pages/downloads/index.html - pip install -U -f https://extras.wxpython.org/wxPython4/extras/linux/gtk3/ubuntu-18.04 wxPython + make install-deps-wxpython - name: Install dependencies run: | - pip install . + make build - name: Run eegnb install test shell: bash run: | @@ -79,7 +68,7 @@ jobs: Xvfb :0 -screen 0 1024x768x24 -ac +extension GLX +render -noreset &> xvfb.log & export DISPLAY=:0 fi - pytest + make test typecheck: name: typecheck (${{ matrix.os }}, py-${{ matrix.python_version }}) @@ -99,25 +88,17 @@ jobs: - name: Install APT dependencies if: "startsWith(runner.os, 'Linux')" run: | - # update archive links - sudo apt-get update - # See comment above (in the test job) for details about these dependencies - sudo apt-get -y install xvfb libgtk-3-dev freeglut3-dev portaudio19-dev libpulse-dev pulseaudio libsdl2-dev + make install-deps-apt + - name: Upgrade pip + run: | + python -m pip install --upgrade pip wheel - name: Install Linux dependencies if: "startsWith(runner.os, 'Linux')" run: | - python -m pip install --upgrade pip wheel - - # Install wxPython wheels since they are distribution-specific and therefore not on PyPI - # See: https://wxpython.org/pages/downloads/index.html - pip install -U -f https://extras.wxpython.org/wxPython4/extras/linux/gtk3/ubuntu-18.04 wxPython - - pip install . - - name: Install MacOS/Windows dependencies + make install-deps-wxpython + - name: Install dependencies run: | - python -m pip install --upgrade pip wheel - pip install . + make build - name: Typecheck run: | - # Exclude visual_cueing due to errors - python -m mypy --exclude 'examples/visual_cueing' + make typecheck diff --git a/Makefile b/Makefile new file mode 100644 index 000000000..ae218c41a --- /dev/null +++ b/Makefile @@ -0,0 +1,29 @@ +build: + pip install . + +test: + pytest + +typecheck: + # Exclude visual_cueing due to errors + python -m mypy --exclude 'examples/visual_cueing' + +docs: + cd doc && make html + +install-deps-apt: + sudo apt-get update # update archive links + + # xvfb is a dependency to create a virtual display + # libgtk-3-dev is a requirement for wxPython + # freeglut3-dev is a requirement for a wxPython dependency + # portaudio19-dev *might* be required to import psychopy on Ubuntu + # pulseaudio *might* be required to actually run the tests (on PsychoPy import) + # libpulse-dev required to build pocketsphinx (speech recognition dependency of psychopy) + # libsdl2-dev required by psychopy + sudo apt-get -y install xvfb libgtk-3-dev freeglut3-dev portaudio19-dev libpulse-dev pulseaudio libsdl2-dev + +install-deps-wxpython: +# Install wxPython wheels since they are distribution-specific and therefore not on PyPI +# See: https://wxpython.org/pages/downloads/index.html + pip install -U -f https://extras.wxpython.org/wxPython4/extras/linux/gtk3/ubuntu-18.04 wxPython From bfad1341aaa0e0ed14d8210b918160d59bf65df2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Erik=20Bj=C3=A4reholt?= Date: Fri, 25 Mar 2022 14:23:14 +0100 Subject: [PATCH 17/20] fix: fixed notebook loading data with incorrect device_name --- examples/visual_n170/01r__n170_viz.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/visual_n170/01r__n170_viz.py b/examples/visual_n170/01r__n170_viz.py index 9aa48f7d4..53ba630e6 100644 --- a/examples/visual_n170/01r__n170_viz.py +++ b/examples/visual_n170/01r__n170_viz.py @@ -59,7 +59,7 @@ subject = 1 session = 1 raw = load_data(subject,session, - experiment='visual-N170', site='eegnb_examples', device_name='muse2016_bfn', + experiment='visual-N170', site='eegnb_examples', device_name='muse2016', data_dir = eegnb_data_path) ################################################################################################### From 11824731cd821839614bf016c574a31da2501486 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Erik=20Bj=C3=A4reholt?= Date: Fri, 25 Mar 2022 14:35:09 +0100 Subject: [PATCH 18/20] fix: fixed incorrect indentation in Makefile --- Makefile | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index ae218c41a..4175cb02f 100644 --- a/Makefile +++ b/Makefile @@ -11,6 +11,9 @@ typecheck: docs: cd doc && make html +clean: + cd doc && make clean + install-deps-apt: sudo apt-get update # update archive links @@ -24,6 +27,6 @@ install-deps-apt: sudo apt-get -y install xvfb libgtk-3-dev freeglut3-dev portaudio19-dev libpulse-dev pulseaudio libsdl2-dev install-deps-wxpython: -# Install wxPython wheels since they are distribution-specific and therefore not on PyPI -# See: https://wxpython.org/pages/downloads/index.html - pip install -U -f https://extras.wxpython.org/wxPython4/extras/linux/gtk3/ubuntu-18.04 wxPython + # Install wxPython wheels since they are distribution-specific and therefore not on PyPI + # See: https://wxpython.org/pages/downloads/index.html + pip install -U -f https://extras.wxpython.org/wxPython4/extras/linux/gtk3/ubuntu-18.04 wxPython From 139879439bef6f2deabfdce545af87c24fe38e4e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Erik=20Bj=C3=A4reholt?= Date: Fri, 25 Mar 2022 14:48:50 +0100 Subject: [PATCH 19/20] fix: fixed plot_conditions --- eegnb/analysis/utils.py | 30 +++++++++++++++++------------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/eegnb/analysis/utils.py b/eegnb/analysis/utils.py index 59818b9ca..603ed14cd 100644 --- a/eegnb/analysis/utils.py +++ b/eegnb/analysis/utils.py @@ -45,13 +45,17 @@ def _bootstrap(data, n_boot: int, ci: float): return (s1, s2) -def _tsplotboot(ax, data, n_boot: int, ci: float, **kw): +def _tsplotboot(ax, data, time: list, n_boot: int, ci: float, color): """From: https://stackoverflow.com/a/47582329/965332""" - x = np.arange(data.shape[1]) + # Time forms the xaxis of the plot + if time is None: + x = np.arange(data.shape[1]) + else: + x = np.asarray(time) est = np.mean(data, axis=0) cis = _bootstrap(data, n_boot, ci) - ax.fill_between(x, cis[0], cis[1], alpha=0.2, **kw) - ax.plot(x, est, **kw) + ax.fill_between(x, cis[0], cis[1], alpha=0.2, color=color) + ax.plot(x, est, color=color) ax.margins(x=0) @@ -175,7 +179,9 @@ def load_data( site = "*" data_path = ( - _get_recording_dir(device_name, experiment, subject_str, session_str, site, data_dir) + _get_recording_dir( + device_name, experiment, subject_str, session_str, site, data_dir + ) / "*.csv" ) fnames = glob(str(data_path)) @@ -216,7 +222,8 @@ def plot_conditions( ylim=(-6, 6), diff_waveform=(1, 2), channel_count=4, - channel_order=None): + channel_order=None, +): """Plot ERP conditions. Args: epochs (mne.epochs): EEG epochs @@ -242,10 +249,9 @@ def plot_conditions( """ if channel_order: - channel_order = np.array(channel_order) + channel_order = np.array(channel_order) else: - channel_order = np.array(range(channel_count)) - + channel_order = np.array(range(channel_count)) if isinstance(conditions, dict): conditions = OrderedDict(conditions) @@ -255,7 +261,7 @@ def plot_conditions( X = epochs.get_data() * 1e6 - X = X[:,channel_order] + X = X[:, channel_order] times = epochs.times y = pd.Series(epochs.events[:, -1]) @@ -270,16 +276,14 @@ def plot_conditions( plot_axes.append(axes[axis_x, axis_y]) axes = plot_axes - print("\n\n\n\n\n\n") - for ch in range(channel_count): for cond, color in zip(conditions.values(), palette): y_cond = y.isin(cond) - # make X[y_cond, ch] one-dimensional X_cond = X[y_cond, ch] _tsplotboot( ax=axes[ch], data=X_cond, + time=times, color=color, n_boot=n_boot, ci=ci, From 430bafb79d661246f2acbcbefac01bdb70f56c31 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Erik=20Bj=C3=A4reholt?= Date: Fri, 25 Mar 2022 14:53:40 +0100 Subject: [PATCH 20/20] fix: throw exception for unhandled case --- eegnb/datasets/datasets.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/eegnb/datasets/datasets.py b/eegnb/datasets/datasets.py index 98d16f758..21add031c 100644 --- a/eegnb/datasets/datasets.py +++ b/eegnb/datasets/datasets.py @@ -91,12 +91,9 @@ def fetch_dataset( destination = os.path.join(data_dir, "downloaded_data.zip") if download_method == "gdown": - URL = "https://drive.google.com/uc?id=" + gdrive_locs[experiment] gdown.download(URL, destination, quiet=False) - elif download_method == "requests": - URL = "https://docs.google.com/uc?export=download" session = requests.Session() @@ -120,6 +117,8 @@ def fetch_dataset( for chunk in response.iter_content(CHUNK_SIZE): if chunk: f.write(chunk) + else: + raise ValueError("download_method not supported") # unzip the file with zipfile.ZipFile(destination, "r") as zip_ref: