Ensemble.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Ensemble Learning\n",
    "\n",
    "A very well-studied field, the type of ensemble learning we'll be doing here is called Stacking (I think). We take the models that we've learned before, and we train another model (the meta-model) on the outputs of those models. If we have 3 models, and each model has 6 outputs, the meta-model has 18 inputs and 6 outputs. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The autoreload extension is already loaded. To reload it, use:\n",
      "  %reload_ext autoreload\n"
     ]
    }
   ],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "\n",
    "import os\n",
    "os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"0\"\n",
    "\n",
    "# Tensorflow\n",
    "import tensorflow as tf\n",
    "from tensorflow.python.tools import freeze_graph\n",
    "from tensorflow.python.tools import optimize_for_inference_lib\n",
    "\n",
    "# Keras\n",
    "import keras\n",
    "from keras import regularizers\n",
    "from keras.models import Sequential\n",
    "from keras.layers import (Activation, Dense, Dropout, Flatten, Conv2D, Conv1D, \n",
    "                          MaxPooling2D, GlobalAveragePooling2D, MaxPooling1D, Lambda, LSTM)\n",
    "from keras.layers.normalization import BatchNormalization\n",
    "from keras.callbacks import Callback, ReduceLROnPlateau, ModelCheckpoint\n",
    "from keras.utils import to_categorical, multi_gpu_model\n",
    "import keras.backend as K\n",
    "\n",
    "import librosa\n",
    "import multiprocessing as mp\n",
    "import numpy as np\n",
    "import scipy.io.wavfile\n",
    "from scipy.fftpack import dct\n",
    "from sklearn.model_selection import train_test_split\n",
    "from tqdm import tqdm_notebook as tqdm\n",
    "import time\n",
    "from pprint import pprint\n",
    "import uuid\n",
    "import glob\n",
    "import math"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "L6_RAW_DATA_DIR = \"data/\"\n",
    "MFCC_PROCESSED_DATA_DIR = \"mfcc-processed-data/\"\n",
    "AMP_PROCESSED_DATA_DIR = \"amp-processed-data/\"\n",
    "ENSEMBLE_PROCESSED_DATA_DIR = \"ensemble-processed-data/\"\n",
    "MODEL_NAME = 'EdgeAnalytics_AudioSet'\n",
    "AUDIO_LENGTH = 44100\n",
    "\n",
    "batch_size = 128\n",
    "epochs = 20\n",
    "channel = 1\n",
    "verbose = 1\n",
    "l6_classes = 6\n",
    "nmfcc = 128\n",
    "nmels = 128\n",
    "AUDIO_LENGTH = 44100\n",
    "hop_length = 512\n",
    "n_fft = 1024\n",
    "nframe = int(math.ceil(AUDIO_LENGTH / hop_length))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_labels(path):\n",
    "    labels = [i for i in sorted(os.listdir(path)) if i[0] != \".\"]\n",
    "    label_indices = np.arange(0, len(labels))\n",
    "    return labels, label_indices, to_categorical(label_indices)\n",
    "\n",
    "def wav2mfcc(file_path):\n",
    "    mfcc_vectors = []\n",
    "    audio_buf,_ = librosa.load(file_path, mono=True, sr=44100)\n",
    "    audio_buf = (audio_buf - np.mean(audio_buf)) / np.std(audio_buf)\n",
    "    \n",
    "    remaining_buf = audio_buf.copy()\n",
    "    while remaining_buf.shape[0] > AUDIO_LENGTH:\n",
    "        # Add the first AUDIO_LENGTH of the buffer as a new vector to train on\n",
    "        new_buf = remaining_buf[ : AUDIO_LENGTH ]\n",
    "        mfcc = librosa.feature.mfcc(new_buf, sr=44100,S=None, n_mfcc=nmfcc, n_fft=n_fft, hop_length=hop_length, n_mels=nmels)\n",
    "#         print(mfcc.shape)\n",
    "        mfcc_vectors.append(mfcc)\n",
    "\n",
    "        # Shrink the buffer by 1/2 * AUDIO_LENGTH\n",
    "        remaining_buf = remaining_buf[ int(AUDIO_LENGTH / 2) : ]\n",
    "#         print(\"Length of remaining buf after shrink: {}\".format(len(remaining_buf)))\n",
    "\n",
    "    # Whatever is left, pad and stick in the training data\n",
    "    remaining_buf = np.concatenate((remaining_buf, np.zeros(shape=(AUDIO_LENGTH - len(remaining_buf)))))\n",
    "    mfcc = librosa.feature.mfcc(remaining_buf,sr=44100,S=None, n_mfcc=nmfcc, n_fft=n_fft, hop_length=hop_length,n_mels=nmels)\n",
    "    mfcc_vectors.append(mfcc)\n",
    "    \n",
    "    return mfcc_vectors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def label_to_amplitude_vecs(args) -> None:\n",
    "    label, input_path, output_path, tqdm_position = args\n",
    "\n",
    "    # Get all audio files for this label\n",
    "    wavfiles = [os.path.join(input_path, label, wavfile) for wavfile in os.listdir(os.path.join(input_path, label))]\n",
    "\n",
    "    # tqdm is amazing, so print all the things this way\n",
    "    print(\" \", end=\"\", flush=True)\n",
    "    twavs = tqdm(wavfiles, position=tqdm_position)\n",
    "    \n",
    "    vectors = []\n",
    "    for i, wavfile in enumerate(twavs):\n",
    "        # Load the audio file; this also works for .flac files\n",
    "        audio_buf, _ = librosa.load(wavfile, mono=True, sr=44100)\n",
    "        audio_buf = audio_buf.reshape(-1, 1)\n",
    "        audio_buf = (audio_buf - np.mean(audio_buf)) / np.std(audio_buf)            \n",
    "        remaining_buf = audio_buf.copy()\n",
    "        while remaining_buf.shape[0] > AUDIO_LENGTH:\n",
    "            # Add the first AUDIO_LENGTH of the buffer as a new vector to train on\n",
    "            new_buf = remaining_buf[ : AUDIO_LENGTH ]\n",
    "            vectors.append(new_buf)\n",
    "            \n",
    "            # Shrink the buffer by 1/2 * AUDIO_LENGTH\n",
    "            remaining_buf = remaining_buf[ int(AUDIO_LENGTH / 2) : ]\n",
    "#             print(\"Length of remaining buf after shrink: {}\".format(len(remaining_buf)))\n",
    "\n",
    "            \n",
    "        # Whatever is left, pad and stick in the training data\n",
    "        remaining_buf = np.concatenate((remaining_buf, np.zeros(shape=(AUDIO_LENGTH - len(remaining_buf), 1))))\n",
    "        vectors.append(remaining_buf)\n",
    "        \n",
    "#         print(\"After wavefile {}: {}\".format(i, len(vectors)))\n",
    "        \n",
    "        # Update tqdm\n",
    "        twavs.set_description(\"Label - '{}'\".format(label))\n",
    "        twavs.refresh()\n",
    "    np_vectors = np.array(vectors)\n",
    "    np.save(os.path.join(output_path, label + '.npy'), np_vectors)\n",
    "    return np_vectors.shape\n",
    "\n",
    "def label_to_mfcc_vecs(args) -> None:\n",
    "    label, input_path, output_path, tqdm_position = args\n",
    "    \n",
    "    vectors = []\n",
    "\n",
    "    wavfiles = [os.path.join(input_path, label, wavfile) for wavfile in os.listdir(os.path.join(input_path, label))]\n",
    "    \n",
    "    # tqdm is amazing, so print all the things this way\n",
    "    print(\" \", end=\"\", flush=True)\n",
    "    twavs = tqdm(wavfiles, position=tqdm_position)\n",
    "    for i, wavfile in enumerate(twavs):\n",
    "        vectors_for_file = wav2mfcc(wavfile)\n",
    "        for v in vectors_for_file:\n",
    "            vectors.append(v)\n",
    "#         print(\"After wavefile {}: {}\".format(i, len(vectors)))\n",
    "        \n",
    "        # Update tqdm\n",
    "        twavs.set_description(\"Label - '{}'\".format(label))\n",
    "        twavs.refresh()\n",
    "#     np.delete(vectors, 0)  # deletes first zero entry    \n",
    "    np_vectors = np.array(vectors)\n",
    "    np.save(os.path.join(output_path, label + '.npy'), np_vectors)\n",
    "    return np_vectors.shape   \n",
    "    \n",
    "def process_data_amplitude(input_path, output_path):    \n",
    "    labels, _, _ = get_labels(input_path)\n",
    "    pool = mp.Pool()\n",
    "    result = pool.map(label_to_amplitude_vecs, \n",
    "                     [(label, input_path, output_path, tqdm_position) \n",
    "                          for tqdm_position, label in enumerate(labels)])\n",
    "    pool.close()\n",
    "    return result\n",
    "\n",
    "def process_data_mfcc(input_path, output_path):    \n",
    "    labels, _, _ = get_labels(input_path)\n",
    "    pool = mp.Pool()\n",
    "    result = pool.map(label_to_mfcc_vecs, \n",
    "                     [(label, input_path, output_path, tqdm_position) \n",
    "                          for tqdm_position, label in enumerate(labels)])\n",
    "    pool.close()\n",
    "    return result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2cb216d3e99d43f589a1e6723012c54f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=202), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "(4255, 44100, 1)\n",
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1566efacf7ee4562b35da7a1d0796e69",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=286), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(2930, 44100, 1)\n",
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "79bb0d948a2d4d56931332f36ef334d9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=169), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(911, 44100, 1)\n",
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ec0e5201eaa64966b29320e778f4c5fb",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=146), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(2204, 44100, 1)\n",
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "93366472d4fc4ad98baf572aaaaabba5",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=407), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(3043, 44100, 1)\n",
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f87295c49b254e6cbfcac854069949c1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=398), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "labels, _, _ = get_labels(L6_RAW_DATA_DIR)\n",
    "for tqdm_position, label in enumerate(labels):\n",
    "    shape = label_to_amplitude_vecs((label, L6_RAW_DATA_DIR, AMP_PROCESSED_DATA_DIR, tqdm_position))\n",
    "    print(shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "labels, _, _ = get_labels(L6_RAW_DATA_DIR)\n",
    "for tqdm_position, label in enumerate(labels):\n",
    "    shape = label_to_mfcc_vecs((label, L6_RAW_DATA_DIR, MFCC_PROCESSED_DATA_DIR, tqdm_position))\n",
    "    print(shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# We load the amplitude model and create its training data\n",
    "process_data_amplitude(L6_RAW_DATA_DIR, AMP_PROCESSED_DATA_DIR)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# We load the MFCC model and create its training data\n",
    "process_data_mfcc(L6_RAW_DATA_DIR, MFCC_PROCESSED_DATA_DIR)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "mfcc_model = keras.models.load_model(\"out/mfcc-l6-acc94.hdf5\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "amp_model = keras.models.load_model(\"out/amplitude-l6-acc91.hdf5\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def vecs_to_predictions(model, vecs):\n",
    "    print(vecs.shape)\n",
    "    preds = model.predict(vecs)\n",
    "    print(preds.shape)\n",
    "    return preds\n",
    "\n",
    "def load_processed_data(labels, processed_data_path):    \n",
    "    # Getting first arrays\n",
    "    X = np.load(os.path.join(processed_data_path, labels[0] + '.npy'))\n",
    "    y = np.zeros(X.shape[0])\n",
    "\n",
    "    # Append all of the dataset into one single array, same goes for y\n",
    "    for i, label in tqdm(enumerate(labels[1:])):\n",
    "        \n",
    "        x = np.load(os.path.join(processed_data_path, label + '.npy'))\n",
    "        print(os.path.join(processed_data_path, label + '.npy'))\n",
    "        print (X.shape)\n",
    "        print (x.shape)\n",
    "        X = np.vstack((X, x))\n",
    "        y = np.append(y, np.full(x.shape[0], fill_value= (i + 1)))\n",
    "\n",
    "    assert X.shape[0] == len(y)\n",
    "\n",
    "    return X, y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "amp_predictions = np.load(os.path.join(ENSEMBLE_PROCESSED_DATA_DIR, \"amplitude-predictions.npy\"))\n",
    "mfcc_predictions = np.load(os.path.join(ENSEMBLE_PROCESSED_DATA_DIR, \"mfcc-predictions.npy\"))\n",
    "merged_data = np.load(os.path.join(ENSEMBLE_PROCESSED_DATA_DIR, \"merged-predictions.npy\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Amplitude data\n",
    "all_labels, _, _ = get_labels(L6_RAW_DATA_DIR)\n",
    "amp_inputs, amp_labels = load_processed_data(all_labels, AMP_PROCESSED_DATA_DIR)\n",
    "\n",
    "\n",
    "# MFCC data\n",
    "all_labels, _, _ = get_labels(L6_RAW_DATA_DIR)\n",
    "mfcc_inputs, mfcc_labels = load_processed_data(all_labels, MFCC_PROCESSED_DATA_DIR)\n",
    "# MFCCs require reshaping\n",
    "mfcc_inputs = mfcc_inputs.reshape(mfcc_inputs.shape[0], nmfcc, nframe, channel)\n",
    "mfcc_labels = to_categorical(mfcc_labels)\n",
    "\n",
    "\n",
    "# Get predictions\n",
    "amp_predictions = vecs_to_predictions(amp_model, amp_inputs)\n",
    "mfcc_predictions = vecs_to_predictions(mfcc_model, mfcc_inputs)\n",
    "\n",
    "\n",
    "# Convert predictions to input features to ensemble\n",
    "merged_data = np.array([ np.array([ *i, *j ]) for i, j in zip(amp_predictions, mfcc_predictions) ])\n",
    "\n",
    "\n",
    "# Save \n",
    "np.save(os.path.join(ENSEMBLE_PROCESSED_DATA_DIR, \"amplitude-predictions.npy\"), amp_predictions)\n",
    "np.save(os.path.join(ENSEMBLE_PROCESSED_DATA_DIR, \"mfcc-predictions.npy\"), mfcc_predictions)\n",
    "np.save(os.path.join(ENSEMBLE_PROCESSED_DATA_DIR, \"merged-predictions.npy\"), merged_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Train test split\n",
    "X_train, X_test, y_train, y_test = train_test_split(merged_data, mfcc_labels, test_size=0.50, random_state=42, shuffle=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_ensemble():\n",
    "    m = Sequential()\n",
    "    m.add(Dense(50, activation='relu', input_shape=[12]))\n",
    "    m.add(Dropout(0.25))\n",
    "    m.add(Dense(50, activation='relu'))\n",
    "    m.add(Dropout(0.25))\n",
    "    m.add(Dense(50, activation='relu'))\n",
    "    m.add(Dropout(0.25))\n",
    "    m.add(Dense(6, activation='softmax', name='label'))\n",
    "    return m\n",
    "\n",
    "def train(model, X_train, y_train_hot, X_test, y_test_hot):\n",
    "    model.compile(optimizer='adam',\n",
    "                  loss='categorical_crossentropy',\n",
    "                  metrics=['accuracy'])\n",
    "    print(model.summary())\n",
    "    \n",
    "    reduce_lr = ReduceLROnPlateau(monitor='val_acc', factor=0.5, patience=10, min_lr=0.00001, verbose=1)\n",
    "    mcp_save = ModelCheckpoint('out/ensemble.{epoch:02d}-{val_acc:.2f}.hdf5', save_best_only=True, monitor='val_acc', mode='max')\n",
    "    model.fit(X_train, \n",
    "              y_train_hot, \n",
    "              batch_size=batch_size, \n",
    "              epochs=epochs, \n",
    "              verbose=verbose, \n",
    "              validation_data=(X_test, y_test_hot),\n",
    "              callbacks=[reduce_lr, mcp_save],\n",
    "              shuffle=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "keras.backend.clear_session()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "epochs = 100\n",
    "model = get_ensemble()\n",
    "train(model, X_train, y_train, X_test, y_test)\n",
    "model.save(\"out/ensemble.hdf5\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "converter = tf.lite.TFLiteConverter.from_keras_model_file('out/ensemble.25-0.97.hdf5')\n",
    "tfmodel = converter.convert()\n",
    "open (\"out/ensemble-l6-acc97.tflite\" , \"wb\") .write(tfmodel)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "'Python Interactive'",
   "language": "python",
   "name": "982f43d3-7dc3-4fa7-93af-960f1a6e1e9b"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}