main_get_data.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "download_directory = 'https://mhealth.md2k.org/images/datasets/mORALdataset09072020.zip'\n",
    "destination_directory = './data/'\n",
    "import requests, zipfile, io\n",
    "r = requests.get(download_directory)\n",
    "z = zipfile.ZipFile(io.BytesIO(r.content))\n",
    "z.extractall(destination_directory)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "zipfile_directory = './data/mORAL_dataset_for_python_upload_09072020/data.zip'\n",
    "destination_directory = './data/mORAL_dataset_for_python_upload_09072020/raw_data/'\n",
    "with zipfile.ZipFile(zipfile_directory) as zf:\n",
    "        zf.extractall(destination_directory)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from datetime import datetime\n",
    "from joblib import Parallel,delayed\n",
    "import pickle\n",
    "sensor_data_directory = './data/mORAL_dataset_for_python_upload_09072020/raw_data/data/sensor_data/'\n",
    "all_participants_directory = [os.path.join(sensor_data_directory,a) for a in os.listdir(sensor_data_directory) if len(a)<=4]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "./data/mORAL_dataset_for_python_upload_09072020/raw_data/data/sensor_data/9a6b/d2017-04-28\n",
      "./data/mORAL_dataset_for_python_upload_09072020/raw_data/data/sensor_data/93a2/d2018-06-22\n",
      "./data/mORAL_dataset_for_python_upload_09072020/raw_data/data/sensor_data/896d/d2018-05-08\n",
      "./data/mORAL_dataset_for_python_upload_09072020/raw_data/data/sensor_data/896d/d2018-05-05\n",
      "./data/mORAL_dataset_for_python_upload_09072020/raw_data/data/sensor_data/896d/d2018-05-03\n",
      "./data/mORAL_dataset_for_python_upload_09072020/raw_data/data/sensor_data/896d/d2018-05-06\n",
      "./data/mORAL_dataset_for_python_upload_09072020/raw_data/data/sensor_data/896d/d2018-05-04\n",
      "./data/mORAL_dataset_for_python_upload_09072020/raw_data/data/sensor_data/896d/d2018-05-02\n",
      "./data/mORAL_dataset_for_python_upload_09072020/raw_data/data/sensor_data/896d/d2018-05-09\n",
      "./data/mORAL_dataset_for_python_upload_09072020/raw_data/data/sensor_data/89f4/d2018-12-28\n",
      "./data/mORAL_dataset_for_python_upload_09072020/raw_data/data/sensor_data/89f4/d2018-12-09\n",
      "./data/mORAL_dataset_for_python_upload_09072020/raw_data/data/sensor_data/89f4/d2018-12-08\n",
      "./data/mORAL_dataset_for_python_upload_09072020/raw_data/data/sensor_data/89f4/d2018-12-20\n",
      "./data/mORAL_dataset_for_python_upload_09072020/raw_data/data/sensor_data/aa22/d2017-04-27\n",
      "./data/mORAL_dataset_for_python_upload_09072020/raw_data/data/sensor_data/aa22/d2017-04-28\n",
      "./data/mORAL_dataset_for_python_upload_09072020/raw_data/data/sensor_data/aa22/d2017-04-30\n",
      "./data/mORAL_dataset_for_python_upload_09072020/raw_data/data/sensor_data/aa22/d2017-04-26\n",
      "./data/mORAL_dataset_for_python_upload_09072020/raw_data/data/sensor_data/aa22/d2017-04-25\n",
      "./data/mORAL_dataset_for_python_upload_09072020/raw_data/data/sensor_data/aa22/d2017-04-29\n",
      "./data/mORAL_dataset_for_python_upload_09072020/raw_data/data/sensor_data/b0e8/d2018-10-12\n",
      "./data/mORAL_dataset_for_python_upload_09072020/raw_data/data/sensor_data/b0e8/d2018-10-15\n",
      "./data/mORAL_dataset_for_python_upload_09072020/raw_data/data/sensor_data/b0e8/d2018-10-13\n",
      "./data/mORAL_dataset_for_python_upload_09072020/raw_data/data/sensor_data/b0e8/d2018-10-17\n",
      "./data/mORAL_dataset_for_python_upload_09072020/raw_data/data/sensor_data/b0e8/d2018-09-22\n",
      "./data/mORAL_dataset_for_python_upload_09072020/raw_data/data/sensor_data/b0e8/d2018-09-25\n",
      "./data/mORAL_dataset_for_python_upload_09072020/raw_data/data/sensor_data/b0e8/d2018-09-30\n",
      "./data/mORAL_dataset_for_python_upload_09072020/raw_data/data/sensor_data/b0e8/d2018-10-16\n",
      "./data/mORAL_dataset_for_python_upload_09072020/raw_data/data/sensor_data/b0e8/d2018-10-18\n",
      "./data/mORAL_dataset_for_python_upload_09072020/raw_data/data/sensor_data/b0e8/d2018-09-23\n",
      "./data/mORAL_dataset_for_python_upload_09072020/raw_data/data/sensor_data/b0e8/d2018-10-14\n",
      "./data/mORAL_dataset_for_python_upload_09072020/raw_data/data/sensor_data/b0e8/d2018-10-19\n",
      "./data/mORAL_dataset_for_python_upload_09072020/raw_data/data/sensor_data/b0e8/d2018-09-24\n",
      "./data/mORAL_dataset_for_python_upload_09072020/raw_data/data/sensor_data/b0e8/d2018-09-26\n",
      "./data/mORAL_dataset_for_python_upload_09072020/raw_data/data/sensor_data/b0e8/d2018-09-21\n",
      "./data/mORAL_dataset_for_python_upload_09072020/raw_data/data/sensor_data/a764/d2018-08-23\n",
      "./data/mORAL_dataset_for_python_upload_09072020/raw_data/data/sensor_data/a764/d2018-08-19\n",
      "./data/mORAL_dataset_for_python_upload_09072020/raw_data/data/sensor_data/a764/d2018-08-20\n",
      "./data/mORAL_dataset_for_python_upload_09072020/raw_data/data/sensor_data/a764/d2018-08-22\n",
      "./data/mORAL_dataset_for_python_upload_09072020/raw_data/data/sensor_data/a764/d2018-08-21\n",
      "./data/mORAL_dataset_for_python_upload_09072020/raw_data/data/sensor_data/9eee/d2018-12-18\n",
      "./data/mORAL_dataset_for_python_upload_09072020/raw_data/data/sensor_data/9eee/d2018-12-02\n",
      "./data/mORAL_dataset_for_python_upload_09072020/raw_data/data/sensor_data/b67b/d2017-05-21\n",
      "./data/mORAL_dataset_for_python_upload_09072020/raw_data/data/sensor_data/b67b/d2017-05-22\n",
      "./data/mORAL_dataset_for_python_upload_09072020/raw_data/data/sensor_data/a64e/d2017-09-24\n",
      "./data/mORAL_dataset_for_python_upload_09072020/raw_data/data/sensor_data/a64e/d2017-09-23\n",
      "./data/mORAL_dataset_for_python_upload_09072020/raw_data/data/sensor_data/a64e/d2017-10-10\n",
      "./data/mORAL_dataset_for_python_upload_09072020/raw_data/data/sensor_data/a64e/d2017-09-30\n",
      "./data/mORAL_dataset_for_python_upload_09072020/raw_data/data/sensor_data/820c/d2017-03-22\n"
     ]
    }
   ],
   "source": [
    "def groupby_window_size(df):\n",
    "    if df.shape[0]<100:\n",
    "        return pd.DataFrame([],columns=['user','timestamp','starttime','endtime','data'])\n",
    "    df = df.sort_values('timestamp').reset_index(drop=True)\n",
    "    df['magnitude'] = df.apply(lambda a:np.linalg.norm([a['x'],a['y'],a['z']]),axis=1)\n",
    "    if df['magnitude'].std()<=not_wearing_standard_deviation:\n",
    "        return pd.DataFrame([],columns=['user','timestamp','starttime','endtime','data'])\n",
    "    user = df['user'].values[0]\n",
    "    timestamp = df['timestamp'].values[0]\n",
    "    starttime = df['time'].min()\n",
    "    endtime = df['time'].max()\n",
    "    data = df[['time','x','y','z']].values\n",
    "    return pd.DataFrame([[user,timestamp,starttime,endtime,data]],columns=['user','timestamp','starttime','endtime','data'])\n",
    "\n",
    "wrist = 'right'\n",
    "window_size = 20\n",
    "not_wearing_standard_deviation = 0.001174897554939529\n",
    "all_data_participant = []\n",
    "for participant_directory in all_participants_directory:\n",
    "    for day in os.listdir(participant_directory):\n",
    "        day_directory = os.path.join(participant_directory,day)\n",
    "        try:\n",
    "            right_aclx = pd.read_csv(os.path.join(day_directory,'{}-wrist-accelx.csv'.format(wrist)),names=['time','x'])\n",
    "            right_acly = pd.read_csv(os.path.join(day_directory,'{}-wrist-accely.csv'.format(wrist)),names=['time','y'])\n",
    "            right_aclz = pd.read_csv(os.path.join(day_directory,'{}-wrist-accelz.csv'.format(wrist)),names=['time','z'])\n",
    "        except FileNotFoundError:\n",
    "            print(day_directory)\n",
    "            continue\n",
    "        data = right_aclz.merge(right_acly,on='time', how='inner').merge(right_aclx,on='time', how='inner')\n",
    "        user = participant_directory.split('/')[-1]\n",
    "        data['user'] = [user]*data.shape[0]\n",
    "        data['timestamp'] = data['time'].apply(lambda a:datetime.utcfromtimestamp(a/1000))\n",
    "        grouped_data = pd.concat(Parallel(n_jobs=-1,verbose=0)(delayed(groupby_window_size)(df) for i,df in data.groupby(pd.Grouper(key='timestamp',freq='{}S'.format(window_size)),as_index=False)))\n",
    "        all_data_participant.append(grouped_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_data = pd.concat(all_data_participant)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [],
   "source": [
    "pickle.dump(all_data,open('./data/mORAL_dataset_for_python_upload_09072020/processed_data/all_data_{}_wrist.p'.format(wrist),'wb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "test1",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5 (default, Sep  4 2020, 07:30:14) \n[GCC 7.3.0]"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "51b244ab9aca612e739a0539ae1af887c58db9e180d786deb0ab1761def69c1f"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}