data_concatenation.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import os\n",
    "import glob\n",
    "from datetime import datetime"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Assuming this notebook was started from nn_likelihoods directory\n",
    "basedir = os.getcwd()\n",
    "files = os.listdir(basedir + '/data_storage')\n",
    "\n",
    "# code that stores names of files of interest in a list\n",
    "# mutable to interest of specific task\n",
    "data_files_of_interest = [s for s in files if 'data_3500000' in s and not 'test' in  s and not 'train' in s]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "# make full file paths out of each data file of interest\n",
    "cnt = 0\n",
    "file_str_add = basedir + '/data_storage/'\n",
    "for file in data_files_of_interest:\n",
    "    data_files_of_interest[cnt] = file_str_add + file \n",
    "    cnt += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in files\n",
    "data_list = []\n",
    "cnt = 0\n",
    "for file in data_files_of_interest:\n",
    "    data_list.append(pd.read_csv(file)[['v', 'a', 'w', 'rt', 'choice']])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Concatenate\n",
    "new_data = pd.concat(data_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Write back to file\n",
    "\n",
    "# Getting some info used in file naming convention\n",
    "n_samples = new_data.shape[0]\n",
    "f_signature = '_from_simulation_mix_'\n",
    "cur_time = datetime.now().strftime('%m_%d_%y_%H_%M_%S')\n",
    "\n",
    "# Write to file\n",
    "new_data.to_csv(file_str_add + 'data_' + str(n_samples) + f_signature + cur_time + '.csv')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}