07_physics-inspiredAI/Rotated-Mnist.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This notebook is an extension of Asad Khan's notebook from an earlier session: 02_deepLearning/02_Mnist.ipynb. \n",
    "\n",
    "__In this notebook we explore the following topics:__\n",
    "\n",
    "- Translational invariance in CNNs: e.g. small shifts in an image do not affect the output\n",
    "- How to make a CNN robust to image rotations"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Section 1: Repeat the initial steps from the deep learning session"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Specifically, we're keeping a bunch of the content from 02_deepLearning/02_Mnist.ipynb to set up the MNIST data and train a model."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "# Mnist"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "import os\n",
    "os.environ[\"CUDA_VISIBLE_DEVICES\"]=\"0\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "from numpy.random import seed\n",
    "seed(1)\n",
    "import tensorflow as tf\n",
    "tf.random.set_seed(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "import tensorflow.keras as keras\n",
    "import numpy as np\n",
    "keras.__version__"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true,
    "hidden": true
   },
   "source": [
    "## Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%env HTTP_PROXY=http://theta-proxy.tmi.alcf.anl.gov:3128\n",
    "%env HTTPS_PROXY=http://theta-proxy.tmi.alcf.anl.gov:3128"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "from tensorflow.keras.datasets import mnist\n",
    "from tensorflow.keras.utils import to_categorical\n",
    "\n",
    "(train_images, train_labels), (test_images, test_labels) = mnist.load_data()\n",
    "\n",
    "train_images = train_images.reshape((60000, 28, 28, 1))\n",
    "train_images = train_images.astype('float32') / 255\n",
    "\n",
    "test_images = test_images.reshape((10000, 28, 28, 1))\n",
    "test_images = test_images.astype('float32') / 255\n",
    "\n",
    "train_labels = to_categorical(train_labels)\n",
    "test_labels = to_categorical(test_labels)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true,
    "hidden": true
   },
   "source": [
    "### Visualize a sample of Training Data "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "    \n",
    "# obtain one batch of training images\n",
    "images, labels = train_images[:20], train_labels[:20]\n",
    "\n",
    "\n",
    "# plot the images in the batch, along with the corresponding labels\n",
    "fig = plt.figure(figsize=(25, 4))\n",
    "for idx in np.arange(20):\n",
    "    ax = fig.add_subplot(2, int(20/2), idx+1, xticks=[], yticks=[])\n",
    "    ax.imshow(np.squeeze(images[idx]), cmap='gray')\n",
    "    # print out the correct label for each image\n",
    "    ax.set_title(str(np.argmax(labels[idx])))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true,
    "hidden": true
   },
   "source": [
    "### View an Image in More Detail "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "img = np.squeeze(train_images[1])\n",
    "\n",
    "fig = plt.figure(figsize = (12,12)) \n",
    "ax = fig.add_subplot(111)\n",
    "ax.imshow(img, cmap='gray')\n",
    "width, height = img.shape\n",
    "thresh = img.max()/2.5\n",
    "for x in range(width):\n",
    "    for y in range(height):\n",
    "        val = round(img[x][y],2) if img[x][y] !=0 else 0\n",
    "        ax.annotate(str(val), xy=(y,x),\n",
    "                    horizontalalignment='center',\n",
    "                    verticalalignment='center',\n",
    "                    color='white' if img[x][y]<thresh else 'black')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true,
    "hidden": true
   },
   "source": [
    "## Define the Network Architecture"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "from tensorflow.keras import layers\n",
    "from tensorflow.keras import models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def define_cnn():\n",
    "    model = models.Sequential()\n",
    "    model.add(layers.Conv2D(32, (3, 3), activation='relu', input_shape=(28, 28, 1)))\n",
    "    model.add(layers.MaxPooling2D((2, 2)))\n",
    "    model.add(layers.Conv2D(64, (3, 3), activation='relu'))\n",
    "    model.add(layers.MaxPooling2D((2, 2)))\n",
    "    model.add(layers.Conv2D(64, (3, 3), activation='relu'))\n",
    "    model.add(layers.Flatten())\n",
    "    model.add(layers.Dense(64, activation='relu'))\n",
    "    model.add(layers.Dense(10, activation='softmax'))\n",
    "    \n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "model = define_cnn()\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true,
    "hidden": true
   },
   "source": [
    "## Compile and Train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "from tensorflow.keras import optimizers\n",
    "\n",
    "model.compile(optimizer=optimizers.RMSprop(learning_rate=1e-4),\n",
    "              loss='categorical_crossentropy',\n",
    "              metrics=['accuracy'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "model.fit(train_images, train_labels, epochs=5, batch_size=64)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true,
    "hidden": true
   },
   "source": [
    "## Test the Trained Network"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "hidden": true
   },
   "source": [
    "Finally, we test our trained model on previously unseen test data and evaluate it's performance. Testing on unseen data is a good way to check that our model generalizes well. It may also be useful to be granular in this analysis and take a look at how this model performs on each class as well as looking at its overall loss and accuracy."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "test_loss, test_acc = model.evaluate(test_images, test_labels, verbose=0)\n",
    "print('Test Accuracy: {:.4f} and Test Loss: {:.4f}'.format(test_acc, test_loss))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "def per_class_accuracy(model, test_images, test_labels):\n",
    "    # initialize lists to monitor test accuracy\n",
    "    class_correct = list(0. for i in range(10))\n",
    "    class_total = list(0. for i in range(10))\n",
    "\n",
    "    pred = model.predict(test_images, verbose=0)\n",
    "    pred = np.argmax(pred, axis=-1)\n",
    "    labels = np.argmax(test_labels, axis=-1)\n",
    "\n",
    "    for i in range(len(pred)):\n",
    "        correct = pred[i] == labels[i]\n",
    "        class_correct[labels[i]] += correct\n",
    "        class_total[labels[i]] += 1\n",
    "\n",
    "\n",
    "    for i in range(10):\n",
    "        if class_total[i] > 0:\n",
    "            print('Test Accuracy of %5s: %2d%% (%2d/%2d)' % (\n",
    "                str(i), 100 * class_correct[i] / class_total[i],\n",
    "                np.sum(class_correct[i]), np.sum(class_total[i])))\n",
    "        else:\n",
    "            print('Test Accuracy of %5s: N/A (no training examples)' % (classes[i]))\n",
    "\n",
    "    print('\\nTest Accuracy (Overall): %2d%% (%2d/%2d)' % (\n",
    "        100. * np.sum(class_correct) / np.sum(class_total),\n",
    "        np.sum(class_correct), np.sum(class_total)))  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "per_class_accuracy(model, test_images, test_labels)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "hidden": true
   },
   "source": [
    "Lets see predictions on individual test images. Since `model.predict` expects a batch of images, i.e, a 4-d tensor, we expand the first dimension of the test image before feeding into model."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "index = 0\n",
    "pred = model.predict(np.expand_dims(test_images[index], axis=0)); pred = np.argmax(pred)\n",
    "plt.imshow(np.squeeze(test_images[index]), cmap='gray')\n",
    "plt.title('Prediction: {}'.format(pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "#Define a python generator [More on Generators in cifar-10 notebook]\n",
    "import itertools\n",
    "\n",
    "def generator(images, labels, batch_size):\n",
    "    iterable_1 = iter(images)\n",
    "    iterable_2 = iter(labels)\n",
    "    while True:\n",
    "        l1 = tuple(itertools.islice(iterable_1, 0, batch_size))\n",
    "        l2 = tuple(itertools.islice(iterable_2, 0, batch_size))\n",
    "        if l1 and l2:\n",
    "            yield np.vstack(np.expand_dims(l1, axis=0)), np.vstack(np.expand_dims(l2, axis=0))\n",
    "        else:\n",
    "            break\n",
    "            \n",
    "test_generator = generator(test_images, test_labels, 20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "def plot_predictions(test_generator, model):\n",
    "    images, labels = next(test_generator)\n",
    "    labels = np.argmax(labels, axis=-1)\n",
    "\n",
    "    # get sample outputs\n",
    "    preds = model.predict(images)\n",
    "    # convert output probabilities to predicted class\n",
    "    preds = np.argmax(preds, axis=-1)\n",
    "\n",
    "    # plot the images in the batch, along with predicted and true labels\n",
    "    fig = plt.figure(figsize=(25, 4))\n",
    "    for idx in np.arange(20):\n",
    "        ax = fig.add_subplot(2, int(20/2), idx+1, xticks=[], yticks=[])\n",
    "        ax.imshow(np.squeeze(images[idx]), cmap='gray')\n",
    "        ax.set_title(\"{} ({})\".format(str(preds[idx]), str(labels[idx])),\n",
    "                     color=(\"green\" if preds[idx]==labels[idx] else \"red\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_predictions(test_generator, model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_incorrect_predictions(model, test_images, test_labels):\n",
    "    # A sample of Incorrect Predictions\n",
    "    preds = np.argmax(model.predict(test_images), axis=-1)\n",
    "    labels = np.argmax(test_labels, axis=-1)\n",
    "    incorrect_idxs = np.where( preds != labels )[0]\n",
    "\n",
    "    # plot the images in the batch, along with predicted and true labels\n",
    "    fig = plt.figure(figsize=(25, 4))\n",
    "    for i in np.arange(20):\n",
    "        idx = incorrect_idxs[i]\n",
    "        ax = fig.add_subplot(2, int(20/2), i+1, xticks=[], yticks=[])\n",
    "        ax.imshow(np.squeeze(test_images[idx]), cmap='gray')\n",
    "        ax.set_title(\"{} ({})\".format(str(preds[idx]), str(labels[idx])),\n",
    "                     color=(\"green\" if preds[idx]==labels[idx] else \"red\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_incorrect_predictions(model, test_images, test_labels)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Part 2: Translational invariance"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "People sometimes say that CNNs are invariant to translations, meaning that you can shift the pixels and get the same answer. This depends on the specific network and task but is roughly true. "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "To be more specific, we can reference the definitions in the Sections 9.2 and 9.3 of the book *Deep Learning* by Goodfellow, et al., freely available [here](https://www.deeplearningbook.org/)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The parameter sharing in a convolutional layer makes the layer *equivariant* to translation: for example if you shift the image by 2 to the left and then apply the convolutional layer, the output is also shifted by 2 to the left.\n",
    "\n",
    "In math: if *g* is a shifting function and the convolutional layer is called *f*, then f(g(x)) = g(f(x))."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Pooling layers are *invariant* to small translations: if you shift an image by 2 to the left and then apply the pooling layer, the output does not change."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In math: if g is a shifting function and the pooling layer is called f, then f(g(x)) = f(x). "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<img src=\"http://personal.ie.cuhk.edu.hk/~ccloy/project_target_code/images/fig3.png\" style=\"width: 600px;\"/>\n",
    "\n",
    "[src: [Deep Convolutional-Shepard Interpolation Neural Networks for Image Classification Tasks](https://link.springer.com/chapter/10.1007%2F978-3-319-93000-8_21)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from scipy import ndimage"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def compare_pair(original_image, transformed_image, model):\n",
    "    pred1 = model.predict(np.expand_dims(original_image, axis=0)); pred1 = np.argmax(pred1)\n",
    "    pred2 = model.predict(np.expand_dims(transformed_image, axis=0)); pred2 = np.argmax(pred2)\n",
    "\n",
    "    fig = plt.figure(figsize=(17, 8))\n",
    "    plt.subplot(1,2,1)\n",
    "    plt.imshow(np.squeeze(original_image), cmap='gray')\n",
    "    plt.title('Prediction: {} (Original Image)'.format(pred1))\n",
    "    plt.subplot(1,2,2)\n",
    "    plt.imshow(np.squeeze(transformed_image), cmap='gray')\n",
    "    plt.title('Prediction: {} (Transformed Image)'.format(pred2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def compare_with_shifting(index, right_shift, down_shift, model):\n",
    "    original_image = test_images[index]\n",
    "    shifted_image = ndimage.shift(original_image, shift=np.array([down_shift, right_shift, 0]), cval=0)\n",
    "    compare_pair(original_image, shifted_image, model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# example that works (at least with my trained model)\n",
    "compare_with_shifting(index=0, right_shift=-5, down_shift=-2, model=model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# small translation okay on this example (at least with my trained model)\n",
    "compare_with_shifting(index=1, right_shift=-1, down_shift=-1, model=model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# but it's more fragile to bigger translations (at least with my trained model)\n",
    "compare_with_shifting(index=1, right_shift=-5, down_shift=-2, model=model)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Here's an example paper that aims to address this:\n",
    "Zhang, \"Making Convolutional Networks Shift-Invariant Again\" ICML 2019 ([paper](http://proceedings.mlr.press/v97/zhang19a/zhang19a.pdf), [code](https://github.com/adobe/antialiased-cnns))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Part 3: Rotational invariance"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Is a CNN robust to rotated images? There's no reason to expect it to be."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def compare_with_rotating(index, rotation, model):\n",
    "    original_image = test_images[index]\n",
    "    rotated_image = ndimage.rotate(original_image, angle=rotation, cval=0)\n",
    "    compare_pair(original_image, rotated_image, model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "compare_with_rotating(index=1, rotation=90, model=model)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "There are many recent papers on making versions of CNNs that are rotationally equivariant or invariant. Some examples are: \n",
    "* Cohen and Welling, \"Group Equivariant Convolutional Networks\" ICML 2016 ([paper](http://proceedings.mlr.press/v48/cohenc16.pdf), [code](https://github.com/tscohen/GrouPy))\n",
    "* Worrall et al., \"Harmonic Networks: Deep Translation and Rotation Equivariance\" CVPR 2017 ([paper](https://openaccess.thecvf.com/content_cvpr_2017/papers/Worrall_Harmonic_Networks_Deep_CVPR_2017_paper.pdf), [code](https://github.com/danielewworrall/harmonicConvolutions))\n",
    "* Weiler, Hamprecht, and Storath \"Learning Steerable Filters for Rotation Equivariant CNNs\" CVPR 2018 ([paper](https://openaccess.thecvf.com/content_cvpr_2018/papers/Weiler_Learning_Steerable_Filters_CVPR_2018_paper.pdf))\n",
    "* Weiler and Cesa \"General E(2)-Equivariant Steerable CNNs\" NeurIPS 2019 ([paper](https://papers.nips.cc/paper/2019/file/45d6637b718d0f24a237069fe41b0db4-Paper.pdf), [code](https://github.com/QUVA-Lab/e2cnn))\n",
    "* Romero, et al. \"Attentive Group Equivariant Convolutional Networks\" ICML 2020 ([paper](http://proceedings.mlr.press/v119/romero20a/romero20a.pdf), [code](https://github.com/dwromero/att_gconvs))\n",
    "* Wang, et al. \"Incorporating Symmetry into Deep Dynamics Models for Improved Generalization\" ICLR 2021 ([paper](https://openreview.net/pdf?id=wta_8Hx2KD), [code](https://github.com/Rose-STL-Lab/Equivariant-Net))\n",
    "\n",
    "However, for the purposes of this tutorial we will explore a simpler approach: augmenting the dataset with rotations."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def augment_data(images, labels):\n",
    "    num_images = images.shape[0]\n",
    "    augmented_images = np.concatenate((images, images), axis=0)\n",
    "    augmented_labels = np.concatenate((labels, labels), axis=0)\n",
    "    rotation_options = [90, 180, 270]\n",
    "    for j in np.arange(num_images):\n",
    "        rotation = rotation_options[np.random.randint(0,3)]\n",
    "        augmented_images[j+num_images, :, :, :] = ndimage.rotate(images[j,:,:,:], angle=rotation, cval=0)\n",
    "    shuffled_indices = np.random.permutation(augmented_images.shape[0])\n",
    "    augmented_images = augmented_images[shuffled_indices, :, :, :]\n",
    "    augmented_labels = augmented_labels[shuffled_indices, :]\n",
    "    return augmented_images, augmented_labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_images_aug, train_labels_aug = augment_data(train_images, train_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_images_aug, test_labels_aug = augment_data(test_images, test_labels)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Note that this may be a tricky problem, even for humans, because, for example, an upside-down 6 can look like a 9. However, Rotated MNIST is a benchmark in many of the above papers. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# obtain one batch of training images\n",
    "images, labels = train_images_aug[:20], train_labels_aug[:20]\n",
    "\n",
    "\n",
    "# plot the images in the batch, along with the corresponding labels\n",
    "fig = plt.figure(figsize=(25, 4))\n",
    "for idx in np.arange(20):\n",
    "    ax = fig.add_subplot(2, int(20/2), idx+1, xticks=[], yticks=[])\n",
    "    ax.imshow(np.squeeze(images[idx]), cmap='gray')\n",
    "    # print out the correct label for each image\n",
    "    ax.set_title(str(np.argmax(labels[idx])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Let's use the same CNN architecture but retrain on this augmented dataset.\n",
    "model2 = define_cnn()\n",
    "model2.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model2.compile(optimizer=optimizers.RMSprop(learning_rate=1e-4),\n",
    "              loss='categorical_crossentropy',\n",
    "              metrics=['accuracy'])\n",
    "model2.fit(train_images_aug, train_labels_aug, epochs=5, batch_size=64)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Testing the new network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_loss, test_acc = model2.evaluate(test_images_aug, test_labels_aug, verbose=0)\n",
    "print('Test Accuracy: {:.4f} and Test Loss: {:.4f}'.format(test_acc, test_loss))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "per_class_accuracy(model2, test_images_aug, test_labels_aug)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_generator2 = generator(test_images_aug, test_labels_aug, 20)\n",
    "plot_predictions(test_generator2, model2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_incorrect_predictions(model2, test_images_aug, test_labels_aug)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "conda/2021-11-30",
   "language": "python",
   "name": "conda-2021-11-30"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}