process_three_and_five_echo_datasets.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Process five-echo flashing checkerboard dataset for software demo"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "import os\n",
    "import os.path as op\n",
    "from glob import glob\n",
    "import webbrowser\n",
    "\n",
    "from tedana.workflows import tedana_workflow"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Download 5 echo data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current\n",
      "                                 Dload  Upload   Total   Spent    Left  Speed\n",
      "100   459  100   459    0     0   2004      0 --:--:-- --:--:-- --:--:--  1995\n",
      "100 68.5M  100 68.5M    0     0  7838k      0  0:00:08  0:00:08 --:--:-- 8779k\n",
      "CPU times: user 155 ms, sys: 71.4 ms, total: 226 ms\n",
      "Wall time: 9.45 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "dset_dir5 = 'five-echo-dataset/'\n",
    "wd = os.getcwd()\n",
    "\n",
    "if not op.isdir(dset_dir5):\n",
    "    os.mkdir(dset_dir5)\n",
    "\n",
    "!curl -L -o five_echo_NIH.tar.xz https://osf.io/ea5v3/download\n",
    "!tar xf five_echo_NIH.tar.xz -C five-echo-dataset\n",
    "os.remove('five_echo_NIH.tar.xz')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Run workflow on 5 echo data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "Using output directory: /Users/handwerkerd/code/tedana_community/ohbm-2021-multiecho/five-echo-dataset\n",
      "Loading input data: ['five-echo-dataset/p06.SBJ01_S09_Task11_e1.sm.nii.gz', 'five-echo-dataset/p06.SBJ01_S09_Task11_e2.sm.nii.gz', 'five-echo-dataset/p06.SBJ01_S09_Task11_e3.sm.nii.gz', 'five-echo-dataset/p06.SBJ01_S09_Task11_e4.sm.nii.gz', 'five-echo-dataset/p06.SBJ01_S09_Task11_e5.sm.nii.gz']\n",
      "Computing EPI mask from first echo\n",
      "310 voxels in user-defined mask do not have good signal. Removing voxels from mask.\n",
      "Computing T2* map\n",
      "Optimally combining data with voxel-wise T2* estimates\n",
      "Writing optimally combined data set: /Users/handwerkerd/code/tedana_community/ohbm-2021-multiecho/five-echo-dataset/desc-optcom_bold.nii.gz\n",
      "Computing PCA of optimally combined multi-echo data\n",
      "Performing dimensionality reduction based on GIFT (https://trendscenter.org/software/gift/) and Li, Y. O., Adali, T., & Calhoun, V. D. (2007). Estimating the number of independent components for functional magnetic resonance imaging data. Human Brain Mapping, 28(11), 1251–1266. https://doi.org/10.1002/hbm.20359\n",
      "Performing SVD on original data...\n",
      "SVD done on original data\n",
      "Estimating the subsampling depth for effective i.i.d samples...\n",
      "Generating subsampled i.i.d. data...\n",
      "Performing SVD on subsampled i.i.d. data...\n",
      "SVD done on subsampled i.i.d. data\n",
      "Effective number of i.i.d. samples 2997\n",
      "Perform eigen spectrum adjustment ...\n",
      "Estimating the dimensionality ...\n",
      "Estimated number of components is 33\n",
      "Calculating weight maps\n",
      "Calculating parameter estimate maps for optimally combined data\n",
      "Calculating z-statistic maps\n",
      "Calculating F-statistic maps\n",
      "Thresholding z-statistic maps\n",
      "Calculating T2* F-statistic maps\n",
      "Calculating S0 F-statistic maps\n",
      "Counting significant voxels in T2* F-statistic maps\n",
      "Counting significant voxels in S0 F-statistic maps\n",
      "Thresholding optimal combination beta maps to match T2* F-statistic maps\n",
      "Thresholding optimal combination beta maps to match S0 F-statistic maps\n",
      "Calculating kappa and rho\n",
      "Calculating variance explained\n",
      "Calculating normalized variance explained\n",
      "Calculating DSI between thresholded T2* F-statistic and optimal combination beta maps\n",
      "Calculating DSI between thresholded S0 F-statistic and optimal combination beta maps\n",
      "Calculating signal-noise t-statistics\n",
      "Counting significant noise voxels from z-statistic maps\n",
      "Calculating decision table score\n",
      "Selected 33 components with mdl dimensionality detection\n",
      "ICA with random seed 42 converged in 61 iterations\n",
      "Making second component selection guess from ICA results\n",
      "Calculating weight maps\n",
      "Calculating parameter estimate maps for optimally combined data\n",
      "Calculating z-statistic maps\n",
      "Calculating F-statistic maps\n",
      "Thresholding z-statistic maps\n",
      "Calculating T2* F-statistic maps\n",
      "Calculating S0 F-statistic maps\n",
      "Counting significant voxels in T2* F-statistic maps\n",
      "Counting significant voxels in S0 F-statistic maps\n",
      "Thresholding optimal combination beta maps to match T2* F-statistic maps\n",
      "Thresholding optimal combination beta maps to match S0 F-statistic maps\n",
      "Calculating kappa and rho\n",
      "Calculating variance explained\n",
      "Calculating normalized variance explained\n",
      "Calculating DSI between thresholded T2* F-statistic and optimal combination beta maps\n",
      "Calculating DSI between thresholded S0 F-statistic and optimal combination beta maps\n",
      "Calculating signal-noise t-statistics\n",
      "Counting significant noise voxels from z-statistic maps\n",
      "Calculating decision table score\n",
      "Performing ICA component selection with Kundu decision tree v2.5\n",
      "NumExpr defaulting to 8 threads.\n",
      "/Users/handwerkerd/code/tedana_community/master/tedana/tedana/selection/_utils.py:87: RuntimeWarning: invalid value encountered in true_divide\n",
      "  b_hat = np.reshape(b / np.sqrt((b ** 2).sum()), (2, 1))\n",
      "Variance explained by ICA decomposition: 95.06%\n",
      "Writing high-Kappa time series: /Users/handwerkerd/code/tedana_community/ohbm-2021-multiecho/five-echo-dataset/desc-optcomAccepted_bold.nii.gz\n",
      "Writing low-Kappa time series: /Users/handwerkerd/code/tedana_community/ohbm-2021-multiecho/five-echo-dataset/desc-optcomRejected_bold.nii.gz\n",
      "Writing denoised time series: /Users/handwerkerd/code/tedana_community/ohbm-2021-multiecho/five-echo-dataset/desc-optcomDenoised_bold.nii.gz\n",
      "Writing full ICA coefficient feature set: /Users/handwerkerd/code/tedana_community/ohbm-2021-multiecho/five-echo-dataset/desc-ICA_components.nii.gz\n",
      "Writing denoised ICA coefficient feature set: /Users/handwerkerd/code/tedana_community/ohbm-2021-multiecho/five-echo-dataset/desc-ICAAccepted_components.nii.gz\n",
      "Writing Z-normalized spatial component maps: /Users/handwerkerd/code/tedana_community/ohbm-2021-multiecho/five-echo-dataset/desc-ICAAccepted_stat-z_components.nii.gz\n",
      "Workflow completed\n",
      "Making figures folder with static component maps and timecourse plots.\n",
      "Generating dynamic report\n",
      "CPU times: user 1min, sys: 3.83 s, total: 1min 4s\n",
      "Wall time: 44.3 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "files = sorted(glob(op.join(dset_dir5, 'p06*.nii.gz')))\n",
    "tes = [15.4, 29.7, 44.0, 58.3, 72.6]\n",
    "tedana_workflow(files, tes, out_dir=dset_dir5, tedort=False, fixed_seed=42)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Open report for 5 echo data in web browser"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "/Users/handwerkerd/code/tedana_community/ohbm-2021-multiecho/five-echo-dataset/tedana_report.html\n"
     ]
    },
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "metadata": {},
     "execution_count": 4
    }
   ],
   "source": [
    "new = 2 # open in a new tab, if possible\n",
    "\n",
    "url = str(os.path.abspath(dset_dir5 + 'tedana_report.html'))\n",
    "print(url)\n",
    "webbrowser.open(\"file://\" + url,new=new)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Download 3 echo data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current\n",
      "                                 Dload  Upload   Total   Spent    Left  Speed\n",
      "100   459  100   459    0     0   2306      0 --:--:-- --:--:-- --:--:--  2306\n",
      "100  125M  100  125M    0     0  4527k      0  0:00:28  0:00:28 --:--:-- 5769k\n",
      "CPU times: user 562 ms, sys: 222 ms, total: 783 ms\n",
      "Wall time: 28.5 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "dset_dir3 = 'three-echo-dataset/'\n",
    "wd = os.getcwd()\n",
    "\n",
    "if not op.isdir(dset_dir3):\n",
    "    os.mkdir(dset_dir3)\n",
    "\n",
    "!curl -L -o three-echo-dataset/three_echo_Cornell_zcat.nii.gz https://osf.io/jy8ta/download"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Run workflow on 3 echo data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "Using output directory: /Users/handwerkerd/code/tedana_community/ohbm-2021-multiecho/three-echo-dataset\n",
      "Loading input data: ['t', 'h', 'r', 'e', 'e', '-', 'e', 'c', 'h', 'o', '-', 'd', 'a', 't', 'a', 's', 'e', 't', '/', 't', 'h', 'r', 'e', 'e', '_', 'e', 'c', 'h', 'o', '_', 'C', 'o', 'r', 'n', 'e', 'l', 'l', '_', 'z', 'c', 'a', 't', '.', 'n', 'i', 'i', '.', 'g', 'z']\n",
      "Computing EPI mask from first echo\n",
      "5589 voxels in user-defined mask do not have good signal. Removing voxels from mask.\n",
      "Computing T2* map\n",
      "Optimally combining data with voxel-wise T2* estimates\n",
      "Writing optimally combined data set: /Users/handwerkerd/code/tedana_community/ohbm-2021-multiecho/three-echo-dataset/desc-optcom_bold.nii.gz\n",
      "Computing PCA of optimally combined multi-echo data\n",
      "Performing dimensionality reduction based on GIFT (https://trendscenter.org/software/gift/) and Li, Y. O., Adali, T., & Calhoun, V. D. (2007). Estimating the number of independent components for functional magnetic resonance imaging data. Human Brain Mapping, 28(11), 1251–1266. https://doi.org/10.1002/hbm.20359\n",
      "Performing SVD on original data...\n",
      "SVD done on original data\n",
      "Estimating the subsampling depth for effective i.i.d samples...\n",
      "Generating subsampled i.i.d. data...\n",
      "Performing SVD on subsampled i.i.d. data...\n",
      "SVD done on subsampled i.i.d. data\n",
      "Effective number of i.i.d. samples 4731\n",
      "Perform eigen spectrum adjustment ...\n",
      "Estimating the dimensionality ...\n",
      "Estimated number of components is 51\n",
      "Calculating weight maps\n",
      "Calculating parameter estimate maps for optimally combined data\n",
      "Calculating z-statistic maps\n",
      "Calculating F-statistic maps\n",
      "Thresholding z-statistic maps\n",
      "Calculating T2* F-statistic maps\n",
      "Calculating S0 F-statistic maps\n",
      "Counting significant voxels in T2* F-statistic maps\n",
      "Counting significant voxels in S0 F-statistic maps\n",
      "Thresholding optimal combination beta maps to match T2* F-statistic maps\n",
      "Thresholding optimal combination beta maps to match S0 F-statistic maps\n",
      "Calculating kappa and rho\n",
      "Calculating variance explained\n",
      "Calculating normalized variance explained\n",
      "Calculating DSI between thresholded T2* F-statistic and optimal combination beta maps\n",
      "Calculating DSI between thresholded S0 F-statistic and optimal combination beta maps\n",
      "Calculating signal-noise t-statistics\n",
      "Counting significant noise voxels from z-statistic maps\n",
      "Calculating decision table score\n",
      "Selected 51 components with mdl dimensionality detection\n",
      "ICA with random seed 42 converged in 67 iterations\n",
      "Making second component selection guess from ICA results\n",
      "Calculating weight maps\n",
      "Calculating parameter estimate maps for optimally combined data\n",
      "Calculating z-statistic maps\n",
      "Calculating F-statistic maps\n",
      "Thresholding z-statistic maps\n",
      "Calculating T2* F-statistic maps\n",
      "Calculating S0 F-statistic maps\n",
      "Counting significant voxels in T2* F-statistic maps\n",
      "Counting significant voxels in S0 F-statistic maps\n",
      "Thresholding optimal combination beta maps to match T2* F-statistic maps\n",
      "Thresholding optimal combination beta maps to match S0 F-statistic maps\n",
      "Calculating kappa and rho\n",
      "Calculating variance explained\n",
      "Calculating normalized variance explained\n",
      "Calculating DSI between thresholded T2* F-statistic and optimal combination beta maps\n",
      "Calculating DSI between thresholded S0 F-statistic and optimal combination beta maps\n",
      "Calculating signal-noise t-statistics\n",
      "Counting significant noise voxels from z-statistic maps\n",
      "Calculating decision table score\n",
      "Performing ICA component selection with Kundu decision tree v2.5\n",
      "Variance explained by ICA decomposition: 94.85%\n",
      "Writing high-Kappa time series: /Users/handwerkerd/code/tedana_community/ohbm-2021-multiecho/three-echo-dataset/desc-optcomAccepted_bold.nii.gz\n",
      "Writing low-Kappa time series: /Users/handwerkerd/code/tedana_community/ohbm-2021-multiecho/three-echo-dataset/desc-optcomRejected_bold.nii.gz\n",
      "Writing denoised time series: /Users/handwerkerd/code/tedana_community/ohbm-2021-multiecho/three-echo-dataset/desc-optcomDenoised_bold.nii.gz\n",
      "Writing full ICA coefficient feature set: /Users/handwerkerd/code/tedana_community/ohbm-2021-multiecho/three-echo-dataset/desc-ICA_components.nii.gz\n",
      "Writing denoised ICA coefficient feature set: /Users/handwerkerd/code/tedana_community/ohbm-2021-multiecho/three-echo-dataset/desc-ICAAccepted_components.nii.gz\n",
      "Writing Z-normalized spatial component maps: /Users/handwerkerd/code/tedana_community/ohbm-2021-multiecho/three-echo-dataset/desc-ICAAccepted_stat-z_components.nii.gz\n",
      "Workflow completed\n",
      "Making figures folder with static component maps and timecourse plots.\n",
      "Generating dynamic report\n",
      "CPU times: user 2min 20s, sys: 6.97 s, total: 2min 27s\n",
      "Wall time: 1min 38s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "files = op.join(dset_dir3, 'three_echo_Cornell_zcat.nii.gz') # sorted(glob(op.join(dset_dir3, '*.nii.gz')))\n",
    "tes = [14.5, 38.5, 62.5]\n",
    "tedana_workflow(files, tes, out_dir=dset_dir3, tedort=False, fixed_seed=42)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Open report for 3 echo data in web browser"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "/Users/handwerkerd/code/tedana_community/ohbm-2021-multiecho/three-echo-dataset/tedana_report.html\n"
     ]
    },
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "metadata": {},
     "execution_count": 7
    }
   ],
   "source": [
    "new = 2 # open in a new tab, if possible\n",
    "\n",
    "url = str(os.path.abspath(dset_dir3 + 'tedana_report.html'))\n",
    "print(url)\n",
    "webbrowser.open(\"file://\" + url,new=new)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "name": "python3",
   "display_name": "Python 3.8.8 64-bit ('base': conda)"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  },
  "interpreter": {
   "hash": "dca0ade3e726a953b501b15e8e990130d2b7799f14cfd9f4271676035ebe5511"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}