rsa_on_fashionmnist.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "rsa_on_fashionmnist.ipynb",
      "version": "0.3.2",
      "provenance": [],
      "collapsed_sections": [],
      "toc_visible": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "3Yktr3lCr8w6",
        "colab_type": "text"
      },
      "source": [
        "# Introduction\n",
        "\n",
        "This script performs RSA on FashionMNIST models. The loop in which the models are trained is commented out as the trained models are provided on github. They need to be saved in the directory specified here. Ensure that the notebook is able to save files in the working directory."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zgPYJLMsSqMw",
        "colab_type": "code",
        "outputId": "79705096-2c0f-4855-ba3f-20edb370bda7",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "from google.colab import drive \n",
        "drive.mount('/content/gdrive', force_remount=True)\n",
        "working_directory       = 'path here/' \n",
        "fashionmnist_saves_conv = 'directory name here'  \n",
        "fashionmnist_saves_mlp  = 'directory name here'"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Mounted at /content/gdrive\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ih5bXSTQPMzp",
        "colab_type": "text"
      },
      "source": [
        "# Model training and computing Performances"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "gP677tVQ2oPa",
        "colab_type": "text"
      },
      "source": [
        "Data is split into 1) train 2) test and 3) valid. Models are trained on train, accuracies estimated on test. Valid is used to perform RSA on."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YysK5ZBnS3Nh",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import torchvision\n",
        "import torch\n",
        "import numpy as np\n",
        "import scipy.stats\n",
        "import sklearn.manifold\n",
        "import matplotlib.pyplot as plt\n",
        "from matplotlib.patches import Rectangle\n",
        "import seaborn\n",
        "import pandas as pd\n",
        "import sklearn\n",
        "import os\n",
        "import pickle\n",
        "import time\n",
        "from scipy.spatial import distance\n",
        "from sklearn.metrics.pairwise import manhattan_distances\n",
        "import torch.nn as nn\n",
        "from copy import deepcopy\n",
        "\n",
        "\n",
        "transforms = torchvision.transforms.Compose([\n",
        "                           torchvision.transforms.ToTensor(),\n",
        "                       ])\n",
        "\n",
        "FashionMNIST_train = torchvision.datasets.FashionMNIST(root='./data',train=True, transform=transforms, download=True)\n",
        "FashionMNIST_train, FashionMNIST_valid = sklearn.model_selection.train_test_split(FashionMNIST_train, test_size=0.1, random_state=1)\n",
        "FashionMNIST_test = torchvision.datasets.FashionMNIST(root='./data',train=False, transform= transforms, download=True)\n",
        "\n",
        "dataloader_train = torch.utils.data.DataLoader(FashionMNIST_train, batch_size = 200, shuffle = True)\n",
        "dataloader_test = torch.utils.data.DataLoader(FashionMNIST_test, batch_size = 10000, shuffle = False)\n",
        "\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "NqqcZ1BC6CSg",
        "colab_type": "text"
      },
      "source": [
        "## Model class definitions\n",
        "Model classes are defined with hidden layer sizes as variables. "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Pkyw3L6kS7W0",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class feedforward(torch.nn.Module):\n",
        "    \n",
        "    def __init__(self, D_in, Hidden, Hidden2, Hidden3, D_out):\n",
        "  \n",
        "        super(feedforward, self).__init__()\n",
        "        self.linear1 = torch.nn.Linear(D_in, Hidden)\n",
        "        self.linear2 = torch.nn.Linear(Hidden, Hidden2)\n",
        "        self.linear3 = torch.nn.Linear(Hidden2, Hidden3)\n",
        "        self.linear4 = torch.nn.Linear(Hidden3, D_out)\n",
        "        self.sigmoid = torch.nn.Sigmoid()\n",
        "\n",
        "\n",
        "    def forward(self, x):\n",
        "      \n",
        "        l1_activ = self.sigmoid(self.linear1(x))\n",
        "        l2_activ = self.sigmoid(self.linear2(l1_activ))\n",
        "        l3_activ = self.sigmoid(self.linear3(l2_activ))\n",
        "        y_pred =   self.linear4(l3_activ)\n",
        "        \n",
        "        return y_pred, l3_activ, l2_activ, l1_activ\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "class ConvNet(torch.nn.Module):\n",
        "    \n",
        "    def __init__(self, h, j):\n",
        "        \n",
        "        super(ConvNet, self).__init__()\n",
        "        \n",
        "        self.layer1 = nn.Sequential(nn.Conv2d(1, h * 2, kernel_size=5, stride=1, padding=2),\n",
        "                                    nn.MaxPool2d(kernel_size=2, stride=2),\n",
        "                                    nn.ReLU(),\n",
        "                                    nn.BatchNorm2d(h * 2))\n",
        "     \n",
        "        self.fc1 = nn.Sequential(nn.Linear(14 * 14 * h * 2, j * 16),\n",
        "                                 nn.ReLU(),\n",
        "                                 nn.BatchNorm1d(j * 16),\n",
        "                                 nn.Dropout(p = 0.5))\n",
        "       \n",
        "        self.fc2 = nn.Sequential(nn.Linear(j * 16, j * 8),\n",
        "                                 nn.Sigmoid(),\n",
        "                                 nn.BatchNorm1d(j * 8),\n",
        "                                 nn.Dropout(p = 0.5))                           \n",
        "      \n",
        "        self.fc3 = nn.Linear(j*8, 10)\n",
        "\n",
        "        \n",
        "    def forward(self, x):\n",
        "        \n",
        "        conv1 = self.layer1(x)\n",
        "        reshaped = conv1.reshape(conv1.size(0), -1)\n",
        "        fc1 = self.fc1(reshaped)\n",
        "        fc2 = self.fc2(fc1)                             \n",
        "        out = self.fc3(fc2)\n",
        "        return [out, fc2, fc1, reshaped, conv1]\n",
        "            \n",
        "criterion = torch.nn.CrossEntropyLoss()\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "yd3mfm2o50Wy",
        "colab_type": "text"
      },
      "source": [
        "## Training CNNs in a loop\n",
        "This loop is used to generate CNNs. Every model has a different number of filters/units in each hidden layer. Every model is saved in a seperate file in the same folder. Accuracy is reported but not saved. It is computed again when loading the models, to ensure there is no bug."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TJZRtPifTG37",
        "colab_type": "code",
        "outputId": "d9025f8c-ed6c-407e-ae92-dcf694f426ec",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 55
        }
      },
      "source": [
        "'''\n",
        "model_index = 0\n",
        "\n",
        "for h in range(2,15):\n",
        "    for j in range(2,16):\n",
        "          \n",
        "        model = ConvNet(h, j).cuda()\n",
        "        criterion = torch.nn.CrossEntropyLoss()\n",
        "        optimizer = torch.optim.Adam(model.parameters(), lr= 0.01)\n",
        "\n",
        "\n",
        "        for epoch in range(15):\n",
        "            for (inputs, labels) in (dataloader_train):\n",
        "    \n",
        "                model.train()\n",
        "                inputs, labels = inputs.cuda(), labels.cuda()\n",
        "                outputs = model(inputs)[0]\n",
        "                loss = criterion(outputs, labels)\n",
        "                optimizer.zero_grad()\n",
        "                loss.backward()\n",
        "                optimizer.step()\n",
        "                \n",
        "        with torch.no_grad():\n",
        "            for (inputs, labels) in (dataloader_test):\n",
        "            \n",
        "                model.eval()\n",
        "                inputs, labels = inputs.cuda(), labels.cuda()\n",
        "                out = model(inputs)[0]\n",
        "                total = labels.size(0)\n",
        "                correct = ((torch.max(out,1)[1]) == labels).sum()\n",
        "                acc =   correct.float() / total\n",
        "                print(acc)\n",
        "        \n",
        "            state = {'layers': [h,j],\n",
        "                     'epoch': epoch, 'state_dict': model.state_dict()}\n",
        "            try:\n",
        "              \n",
        "                 os.makedirs(f'{working_directory}/{fashionmnist_saves_conv/')\n",
        "                \n",
        "            except FileExistsError:\n",
        "              \n",
        "                pass\n",
        "              \n",
        "            torch.save(state, f'{working_directory}/{fashionmnist_saves_conv/{model_index}.tar')\n",
        "            model_index += 1\n",
        "            \n",
        "'''\n"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "\"\\nmodel_index = 0\\n\\nfor h in range(2,15):\\n    for j in range(2,16):\\n          \\n        model = ConvNet(h, j).cuda()\\n        criterion = torch.nn.CrossEntropyLoss()\\n        optimizer = torch.optim.Adam(model.parameters(), lr= 0.01)\\n\\n\\n        for epoch in range(15):\\n            for (inputs, labels) in (dataloader_train):\\n    \\n                model.train()\\n                inputs, labels = inputs.cuda(), labels.cuda()\\n                outputs = model(inputs)[0]\\n                loss = criterion(outputs, labels)\\n                optimizer.zero_grad()\\n                loss.backward()\\n                optimizer.step()\\n                \\n        with torch.no_grad():\\n            for (inputs, labels) in (dataloader_test):\\n            \\n                model.eval()\\n                inputs, labels = inputs.cuda(), labels.cuda()\\n                out = model(inputs)[0]\\n                total = labels.size(0)\\n                correct = ((torch.max(out,1)[1]) == labels).sum()\\n                acc =   correct.float() / total\\n                print(acc)\\n        \\n            state = {'layers': [h,j],\\n                     'epoch': epoch, 'state_dict': model.state_dict()}\\n            try:\\n              \\n                 os.makedirs(f'{working_directory}/{fashionmnist_saves_conv/')\\n                \\n            except FileExistsError:\\n              \\n                pass\\n              \\n            torch.save(state, f'{working_directory}/{fashionmnist_saves_conv/{model_index}.tar')\\n            model_index += 1\\n            \\n\""
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 4
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "XY9PX5jD62jL",
        "colab_type": "text"
      },
      "source": [
        "## Compute CNN accuracies\n",
        "Loads the CNNs and computes accuracies, saves accuracies as a list."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3fQh_QV8F0Kl",
        "colab_type": "code",
        "outputId": "3aafe851-90a7-4526-92c1-b11283911df9",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "source": [
        "acc_list = []\n",
        "conv_saves = f'{working_directory}/{fashionmnist_saves_conv}/'\n",
        "for save in os.listdir(conv_saves):\n",
        "    \n",
        "    model_dict = torch.load(f'{working_directory}/{fashionmnist_saves_conv}/{save}')\n",
        "    layers = model_dict['layers']\n",
        "    model = ConvNet(*layers).cuda()\n",
        "    model.load_state_dict(model_dict['state_dict'])\n",
        "\n",
        "  \n",
        "    with torch.no_grad():\n",
        "         for i, (inputs, labels) in enumerate(dataloader_test):\n",
        "        \n",
        "            inputs, labels = (inputs, labels)\n",
        "            inputs, labels = inputs.cuda(), labels.long().cuda()\n",
        "            out = model(inputs)[0]\n",
        "            total = labels.size(0)\n",
        "            correct = ((torch.max(out,1)[1]) == labels).sum()\n",
        "            acc =   correct.float() / total\n",
        "            \n",
        "    print(f'Test accuracy of the network: {acc}')\n",
        "    acc_list.append([acc.cpu().numpy(),save])\n",
        "    \n",
        "with open(f'{working_directory}/accuracy_list_fashionmnist_cnn.pkl', 'wb') as fp:\n",
        "   \n",
        "    pickle.dump(acc_list, fp)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Test accuracy of the network: 0.8107999563217163\n",
            "Test accuracy of the network: 0.8517000079154968\n",
            "Test accuracy of the network: 0.8671000003814697\n",
            "Test accuracy of the network: 0.8725000023841858\n",
            "Test accuracy of the network: 0.8823999762535095\n",
            "Test accuracy of the network: 0.8789999485015869\n",
            "Test accuracy of the network: 0.883899986743927\n",
            "Test accuracy of the network: 0.8834999799728394\n",
            "Test accuracy of the network: 0.8795999884605408\n",
            "Test accuracy of the network: 0.8817999958992004\n",
            "Test accuracy of the network: 0.8860999941825867\n",
            "Test accuracy of the network: 0.887999951839447\n",
            "Test accuracy of the network: 0.8880999684333801\n",
            "Test accuracy of the network: 0.8818999528884888\n",
            "Test accuracy of the network: 0.8305000066757202\n",
            "Test accuracy of the network: 0.8637999892234802\n",
            "Test accuracy of the network: 0.8736000061035156\n",
            "Test accuracy of the network: 0.8860999941825867\n",
            "Test accuracy of the network: 0.8831999897956848\n",
            "Test accuracy of the network: 0.8849999904632568\n",
            "Test accuracy of the network: 0.8840000033378601\n",
            "Test accuracy of the network: 0.8845999836921692\n",
            "Test accuracy of the network: 0.8898999691009521\n",
            "Test accuracy of the network: 0.8906999826431274\n",
            "Test accuracy of the network: 0.8871999979019165\n",
            "Test accuracy of the network: 0.8902999758720398\n",
            "Test accuracy of the network: 0.8896999955177307\n",
            "Test accuracy of the network: 0.8922999501228333\n",
            "Test accuracy of the network: 0.824400007724762\n",
            "Test accuracy of the network: 0.8720999956130981\n",
            "Test accuracy of the network: 0.8827999830245972\n",
            "Test accuracy of the network: 0.8845999836921692\n",
            "Test accuracy of the network: 0.8876999616622925\n",
            "Test accuracy of the network: 0.8905999660491943\n",
            "Test accuracy of the network: 0.8876000046730042\n",
            "Test accuracy of the network: 0.8939999938011169\n",
            "Test accuracy of the network: 0.8894000053405762\n",
            "Test accuracy of the network: 0.8955000042915344\n",
            "Test accuracy of the network: 0.8902999758720398\n",
            "Test accuracy of the network: 0.8938999772071838\n",
            "Test accuracy of the network: 0.8920999765396118\n",
            "Test accuracy of the network: 0.8903999924659729\n",
            "Test accuracy of the network: 0.8317999839782715\n",
            "Test accuracy of the network: 0.8702999949455261\n",
            "Test accuracy of the network: 0.8819999694824219\n",
            "Test accuracy of the network: 0.8901000022888184\n",
            "Test accuracy of the network: 0.8887999653816223\n",
            "Test accuracy of the network: 0.8916999697685242\n",
            "Test accuracy of the network: 0.8923999667167664\n",
            "Test accuracy of the network: 0.8894999623298645\n",
            "Test accuracy of the network: 0.8922999501228333\n",
            "Test accuracy of the network: 0.8892999887466431\n",
            "Test accuracy of the network: 0.8981999754905701\n",
            "Test accuracy of the network: 0.8930000066757202\n",
            "Test accuracy of the network: 0.8985999822616577\n",
            "Test accuracy of the network: 0.8949999809265137\n",
            "Test accuracy of the network: 0.8278999924659729\n",
            "Test accuracy of the network: 0.8748999834060669\n",
            "Test accuracy of the network: 0.8837999701499939\n",
            "Test accuracy of the network: 0.8908999562263489\n",
            "Test accuracy of the network: 0.8894000053405762\n",
            "Test accuracy of the network: 0.8913999795913696\n",
            "Test accuracy of the network: 0.8953999876976013\n",
            "Test accuracy of the network: 0.897599995136261\n",
            "Test accuracy of the network: 0.8970999717712402\n",
            "Test accuracy of the network: 0.8923999667167664\n",
            "Test accuracy of the network: 0.8949999809265137\n",
            "Test accuracy of the network: 0.8930000066757202\n",
            "Test accuracy of the network: 0.8955999612808228\n",
            "Test accuracy of the network: 0.8873999714851379\n",
            "Test accuracy of the network: 0.8353999853134155\n",
            "Test accuracy of the network: 0.8761000037193298\n",
            "Test accuracy of the network: 0.8855999708175659\n",
            "Test accuracy of the network: 0.88919997215271\n",
            "Test accuracy of the network: 0.8952999711036682\n",
            "Test accuracy of the network: 0.8991999626159668\n",
            "Test accuracy of the network: 0.8937999606132507\n",
            "Test accuracy of the network: 0.8991999626159668\n",
            "Test accuracy of the network: 0.8978999853134155\n",
            "Test accuracy of the network: 0.8996999859809875\n",
            "Test accuracy of the network: 0.8985999822616577\n",
            "Test accuracy of the network: 0.8949999809265137\n",
            "Test accuracy of the network: 0.8940999507904053\n",
            "Test accuracy of the network: 0.8988999724388123\n",
            "Test accuracy of the network: 0.8446999788284302\n",
            "Test accuracy of the network: 0.8746999502182007\n",
            "Test accuracy of the network: 0.8812999725341797\n",
            "Test accuracy of the network: 0.8871999979019165\n",
            "Test accuracy of the network: 0.8933999538421631\n",
            "Test accuracy of the network: 0.8962000012397766\n",
            "Test accuracy of the network: 0.8947999477386475\n",
            "Test accuracy of the network: 0.8996999859809875\n",
            "Test accuracy of the network: 0.8991999626159668\n",
            "Test accuracy of the network: 0.8992999792098999\n",
            "Test accuracy of the network: 0.8976999521255493\n",
            "Test accuracy of the network: 0.8977999687194824\n",
            "Test accuracy of the network: 0.9019999504089355\n",
            "Test accuracy of the network: 0.9031999707221985\n",
            "Test accuracy of the network: 0.8378999829292297\n",
            "Test accuracy of the network: 0.8737999796867371\n",
            "Test accuracy of the network: 0.8869999647140503\n",
            "Test accuracy of the network: 0.8885999917984009\n",
            "Test accuracy of the network: 0.8937999606132507\n",
            "Test accuracy of the network: 0.8876999616622925\n",
            "Test accuracy of the network: 0.8991999626159668\n",
            "Test accuracy of the network: 0.895799994468689\n",
            "Test accuracy of the network: 0.8958999514579773\n",
            "Test accuracy of the network: 0.8995999693870544\n",
            "Test accuracy of the network: 0.8981999754905701\n",
            "Test accuracy of the network: 0.896399974822998\n",
            "Test accuracy of the network: 0.9023999571800232\n",
            "Test accuracy of the network: 0.8999999761581421\n",
            "Test accuracy of the network: 0.8380999565124512\n",
            "Test accuracy of the network: 0.877299964427948\n",
            "Test accuracy of the network: 0.8912999629974365\n",
            "Test accuracy of the network: 0.8939999938011169\n",
            "Test accuracy of the network: 0.8955000042915344\n",
            "Test accuracy of the network: 0.9013999700546265\n",
            "Test accuracy of the network: 0.8977999687194824\n",
            "Test accuracy of the network: 0.899899959564209\n",
            "Test accuracy of the network: 0.8971999883651733\n",
            "Test accuracy of the network: 0.899899959564209\n",
            "Test accuracy of the network: 0.9000999927520752\n",
            "Test accuracy of the network: 0.9000999927520752\n",
            "Test accuracy of the network: 0.8989999890327454\n",
            "Test accuracy of the network: 0.8980000019073486\n",
            "Test accuracy of the network: 0.8351999521255493\n",
            "Test accuracy of the network: 0.8779000043869019\n",
            "Test accuracy of the network: 0.8899999856948853\n",
            "Test accuracy of the network: 0.8947999477386475\n",
            "Test accuracy of the network: 0.8950999975204468\n",
            "Test accuracy of the network: 0.9003999829292297\n",
            "Test accuracy of the network: 0.8994999527931213\n",
            "Test accuracy of the network: 0.8980000019073486\n",
            "Test accuracy of the network: 0.899899959564209\n",
            "Test accuracy of the network: 0.8962999582290649\n",
            "Test accuracy of the network: 0.8958999514579773\n",
            "Test accuracy of the network: 0.9023000001907349\n",
            "Test accuracy of the network: 0.8851000070571899\n",
            "Test accuracy of the network: 0.8969999551773071\n",
            "Test accuracy of the network: 0.8393999934196472\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "QJ-76CQD8jWl",
        "colab_type": "text"
      },
      "source": [
        "## Train MLPs in a loop\n",
        "Train the MLPs in a loop (similar to CNNs). Save a a dictionary including all necessary information to reload them."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1kNHtU_t4W6r",
        "colab_type": "code",
        "outputId": "c7927d0d-fa52-40a1-85bd-e021b25a35e0",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 55
        }
      },
      "source": [
        "'''\n",
        "model_index = 0\n",
        "for i in range(8):\n",
        "    for j in range(8):\n",
        "        for h in range(8):\n",
        "            \n",
        "            layers = [28*28, i*2 +j*2 + h*2 +3, j*2 + h*2 +3, h*2 +3 ,10]\n",
        "            model   = feedforward(*layers).cuda()\n",
        "            optimizer = torch.optim.Adam(model.parameters(), lr= 0.001)\n",
        "            for epoch in range(15):\n",
        "                for h, (inputs, labels) in enumerate(dataloader_train):\n",
        "        \n",
        "                    inputs, labels = (inputs, labels)\n",
        "                    inputs, labels = torch.autograd.Variable(inputs).view((-1, 28*28)).float().cuda(), torch.autograd.Variable(labels).long().cuda()\n",
        "                    out = model(inputs)[0]\n",
        "                    loss = criterion(out, labels)\n",
        "                    optimizer.zero_grad()\n",
        "                    loss.backward()\n",
        "                    optimizer.step()\n",
        "               \n",
        "            state = {'layers': layers,\n",
        "                     'epoch': epoch,'state_dict': model.state_dict(), 'optimizer': optimizer.state_dict()}\n",
        "            try:\n",
        "              \n",
        "                os.makedirs(f'{working_directory}/{fashionmnist_saves_mlp}')\n",
        "            except FileExistsError:\n",
        "              \n",
        "                pass\n",
        "            \n",
        "            torch.save(state, f'{working_directory}/{fashionmnist_saves_mlp}/{model_index}.tar')\n",
        "            print(model_index)\n",
        "            model_index += 1\n",
        "            \n",
        "'''\n"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "\"\\nmodel_index = 0\\nfor i in range(8):\\n    for j in range(8):\\n        for h in range(8):\\n            \\n            layers = [28*28, i*2 +j*2 + h*2 +3, j*2 + h*2 +3, h*2 +3 ,10]\\n            model   = feedforward(*layers).cuda()\\n            optimizer = torch.optim.Adam(model.parameters(), lr= 0.001)\\n            for epoch in range(15):\\n                for h, (inputs, labels) in enumerate(dataloader_train):\\n        \\n                    inputs, labels = (inputs, labels)\\n                    inputs, labels = torch.autograd.Variable(inputs).view((-1, 28*28)).float().cuda(), torch.autograd.Variable(labels).long().cuda()\\n                    out = model(inputs)[0]\\n                    loss = criterion(out, labels)\\n                    optimizer.zero_grad()\\n                    loss.backward()\\n                    optimizer.step()\\n               \\n            state = {'layers': layers,\\n                     'epoch': epoch,'state_dict': model.state_dict(), 'optimizer': optimizer.state_dict()}\\n            try:\\n              \\n                os.makedirs(f'{working_directory}/{fashionmnist_saves_mlp}')\\n            except FileExistsError:\\n              \\n                pass\\n            \\n            torch.save(state, f'{working_directory}/{fashionmnist_saves_mlp}/{model_index}.tar')\\n            print(model_index)\\n            model_index += 1\\n            \\n\""
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 6
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "8qWmucSE8uzO",
        "colab_type": "text"
      },
      "source": [
        "## Compute MLP accuracies \n",
        "Loads the saved MLP models and computes accuracies. Saves accuracies in a list."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "J8s4ty1hGTD5",
        "colab_type": "code",
        "outputId": "2a2353b7-e228-45b4-c445-279a86c4304d",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "source": [
        "mlp_saves = f'{working_directory}/{fashionmnist_saves_mlp}'\n",
        "\n",
        "acc_list = []\n",
        "for save in os.listdir(mlp_saves):\n",
        "    \n",
        "    model_dict = torch.load(f'{working_directory}/{fashionmnist_saves_mlp}/{save}')\n",
        "    layers = model_dict['layers']\n",
        "    model = feedforward(*layers).cuda()\n",
        "    model.eval()\n",
        "    model.load_state_dict(model_dict['state_dict'])\n",
        "\n",
        "  \n",
        "    with torch.no_grad():\n",
        "         for i, (inputs, labels) in enumerate(dataloader_test):    \n",
        "        \n",
        "            inputs, labels = (inputs, labels)\n",
        "            inputs, labels = torch.autograd.Variable(inputs).view((-1, 28*28)).float().cuda(), torch.autograd.Variable(labels).long().cuda()\n",
        "            out = model(inputs)[0]\n",
        "            total = labels.size(0)\n",
        "            correct = ((torch.max(out,1)[1]) == labels).sum()\n",
        "            acc =   correct.float() / total\n",
        "            \n",
        "    print(f'Test accuracy of the network: {acc}')\n",
        "    acc_list.append([acc.cpu().numpy(),save])\n",
        "    \n",
        "with open(f'{working_directory}/accuracy_list_fashionmnist_mlp.pkl', 'wb') as fp:\n",
        "    \n",
        "    pickle.dump(acc_list, fp)\n",
        "  "
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Test accuracy of the network: 0.5507000088691711\n",
            "Test accuracy of the network: 0.675599992275238\n",
            "Test accuracy of the network: 0.8018999695777893\n",
            "Test accuracy of the network: 0.7699999809265137\n",
            "Test accuracy of the network: 0.6991999745368958\n",
            "Test accuracy of the network: 0.5741999745368958\n",
            "Test accuracy of the network: 0.6505999565124512\n",
            "Test accuracy of the network: 0.7443000078201294\n",
            "Test accuracy of the network: 0.7166999578475952\n",
            "Test accuracy of the network: 0.4892999827861786\n",
            "Test accuracy of the network: 0.7339000105857849\n",
            "Test accuracy of the network: 0.5722999572753906\n",
            "Test accuracy of the network: 0.4869999885559082\n",
            "Test accuracy of the network: 0.8082000017166138\n",
            "Test accuracy of the network: 0.7788999676704407\n",
            "Test accuracy of the network: 0.8014000058174133\n",
            "Test accuracy of the network: 0.6636999845504761\n",
            "Test accuracy of the network: 0.7128999829292297\n",
            "Test accuracy of the network: 0.6534000039100647\n",
            "Test accuracy of the network: 0.8265999555587769\n",
            "Test accuracy of the network: 0.804099977016449\n",
            "Test accuracy of the network: 0.6995999813079834\n",
            "Test accuracy of the network: 0.7462999820709229\n",
            "Test accuracy of the network: 0.838699996471405\n",
            "Test accuracy of the network: 0.7870999574661255\n",
            "Test accuracy of the network: 0.6581999659538269\n",
            "Test accuracy of the network: 0.8393999934196472\n",
            "Test accuracy of the network: 0.7985000014305115\n",
            "Test accuracy of the network: 0.821899950504303\n",
            "Test accuracy of the network: 0.8438999652862549\n",
            "Test accuracy of the network: 0.6013000011444092\n",
            "Test accuracy of the network: 0.8410999774932861\n",
            "Test accuracy of the network: 0.786799967288971\n",
            "Test accuracy of the network: 0.5271999835968018\n",
            "Test accuracy of the network: 0.8508999943733215\n",
            "Test accuracy of the network: 0.802899956703186\n",
            "Test accuracy of the network: 0.7569999694824219\n",
            "Test accuracy of the network: 0.8440999984741211\n",
            "Test accuracy of the network: 0.8460999727249146\n",
            "Test accuracy of the network: 0.820099949836731\n",
            "Test accuracy of the network: 0.8057999610900879\n",
            "Test accuracy of the network: 0.8509999513626099\n",
            "Test accuracy of the network: 0.8522999882698059\n",
            "Test accuracy of the network: 0.8147000074386597\n",
            "Test accuracy of the network: 0.8226000070571899\n",
            "Test accuracy of the network: 0.8190999627113342\n",
            "Test accuracy of the network: 0.8334999680519104\n",
            "Test accuracy of the network: 0.8489999771118164\n",
            "Test accuracy of the network: 0.8230999708175659\n",
            "Test accuracy of the network: 0.8407999873161316\n",
            "Test accuracy of the network: 0.8466999530792236\n",
            "Test accuracy of the network: 0.8399999737739563\n",
            "Test accuracy of the network: 0.7710999846458435\n",
            "Test accuracy of the network: 0.8303999900817871\n",
            "Test accuracy of the network: 0.7752999663352966\n",
            "Test accuracy of the network: 0.7339000105857849\n",
            "Test accuracy of the network: 0.5385000109672546\n",
            "Test accuracy of the network: 0.7874000072479248\n",
            "Test accuracy of the network: 0.7658999562263489\n",
            "Test accuracy of the network: 0.848800003528595\n",
            "Test accuracy of the network: 0.8513000011444092\n",
            "Test accuracy of the network: 0.8201999664306641\n",
            "Test accuracy of the network: 0.8371999859809875\n",
            "Test accuracy of the network: 0.8537999987602234\n",
            "Test accuracy of the network: 0.6592999696731567\n",
            "Test accuracy of the network: 0.6837999820709229\n",
            "Test accuracy of the network: 0.8262999653816223\n",
            "Test accuracy of the network: 0.7568999528884888\n",
            "Test accuracy of the network: 0.8062999844551086\n",
            "Test accuracy of the network: 0.6751999855041504\n",
            "Test accuracy of the network: 0.7870999574661255\n",
            "Test accuracy of the network: 0.7889999747276306\n",
            "Test accuracy of the network: 0.7760999798774719\n",
            "Test accuracy of the network: 0.7964999675750732\n",
            "Test accuracy of the network: 0.8294000029563904\n",
            "Test accuracy of the network: 0.8017999529838562\n",
            "Test accuracy of the network: 0.6462000012397766\n",
            "Test accuracy of the network: 0.7206999659538269\n",
            "Test accuracy of the network: 0.7283999919891357\n",
            "Test accuracy of the network: 0.6776999831199646\n",
            "Test accuracy of the network: 0.8064000010490417\n",
            "Test accuracy of the network: 0.6383999586105347\n",
            "Test accuracy of the network: 0.6380999684333801\n",
            "Test accuracy of the network: 0.8283999562263489\n",
            "Test accuracy of the network: 0.7426999807357788\n",
            "Test accuracy of the network: 0.49959999322891235\n",
            "Test accuracy of the network: 0.8144999742507935\n",
            "Test accuracy of the network: 0.8468999862670898\n",
            "Test accuracy of the network: 0.7529000043869019\n",
            "Test accuracy of the network: 0.8345999717712402\n",
            "Test accuracy of the network: 0.8112999796867371\n",
            "Test accuracy of the network: 0.7906999588012695\n",
            "Test accuracy of the network: 0.8216999769210815\n",
            "Test accuracy of the network: 0.6553999781608582\n",
            "Test accuracy of the network: 0.7382000088691711\n",
            "Test accuracy of the network: 0.8542999625205994\n",
            "Test accuracy of the network: 0.5734999775886536\n",
            "Test accuracy of the network: 0.8479999899864197\n",
            "Test accuracy of the network: 0.8082000017166138\n",
            "Test accuracy of the network: 0.7724999785423279\n",
            "Test accuracy of the network: 0.6331999897956848\n",
            "Test accuracy of the network: 0.8319999575614929\n",
            "Test accuracy of the network: 0.8377999663352966\n",
            "Test accuracy of the network: 0.8580999970436096\n",
            "Test accuracy of the network: 0.8447999954223633\n",
            "Test accuracy of the network: 0.7526999711990356\n",
            "Test accuracy of the network: 0.8048999905586243\n",
            "Test accuracy of the network: 0.8029999732971191\n",
            "Test accuracy of the network: 0.8535999655723572\n",
            "Test accuracy of the network: 0.8563999533653259\n",
            "Test accuracy of the network: 0.8535999655723572\n",
            "Test accuracy of the network: 0.8589999675750732\n",
            "Test accuracy of the network: 0.7534999847412109\n",
            "Test accuracy of the network: 0.722599983215332\n",
            "Test accuracy of the network: 0.7691999673843384\n",
            "Test accuracy of the network: 0.8175999522209167\n",
            "Test accuracy of the network: 0.760699987411499\n",
            "Test accuracy of the network: 0.8204999566078186\n",
            "Test accuracy of the network: 0.8342999815940857\n",
            "Test accuracy of the network: 0.8544999957084656\n",
            "Test accuracy of the network: 0.7124999761581421\n",
            "Test accuracy of the network: 0.8613999485969543\n",
            "Test accuracy of the network: 0.8489999771118164\n",
            "Test accuracy of the network: 0.6207999587059021\n",
            "Test accuracy of the network: 0.8211999535560608\n",
            "Test accuracy of the network: 0.858299970626831\n",
            "Test accuracy of the network: 0.8432999849319458\n",
            "Test accuracy of the network: 0.8458999991416931\n",
            "Test accuracy of the network: 0.6926999688148499\n",
            "Test accuracy of the network: 0.6965999603271484\n",
            "Test accuracy of the network: 0.7626000046730042\n",
            "Test accuracy of the network: 0.7937999963760376\n",
            "Test accuracy of the network: 0.7581999897956848\n",
            "Test accuracy of the network: 0.8459999561309814\n",
            "Test accuracy of the network: 0.7342000007629395\n",
            "Test accuracy of the network: 0.7732999920845032\n",
            "Test accuracy of the network: 0.4878999888896942\n",
            "Test accuracy of the network: 0.773099958896637\n",
            "Test accuracy of the network: 0.6017999649047852\n",
            "Test accuracy of the network: 0.7990999817848206\n",
            "Test accuracy of the network: 0.8316999673843384\n",
            "Test accuracy of the network: 0.8112999796867371\n",
            "Test accuracy of the network: 0.6548999547958374\n",
            "Test accuracy of the network: 0.8459999561309814\n",
            "Test accuracy of the network: 0.7878999710083008\n",
            "Test accuracy of the network: 0.7107999920845032\n",
            "Test accuracy of the network: 0.6326000094413757\n",
            "Test accuracy of the network: 0.7109000086784363\n",
            "Test accuracy of the network: 0.8447999954223633\n",
            "Test accuracy of the network: 0.7479999661445618\n",
            "Test accuracy of the network: 0.8464999794960022\n",
            "Test accuracy of the network: 0.8509999513626099\n",
            "Test accuracy of the network: 0.7411999702453613\n",
            "Test accuracy of the network: 0.7164999842643738\n",
            "Test accuracy of the network: 0.799299955368042\n",
            "Test accuracy of the network: 0.856499969959259\n",
            "Test accuracy of the network: 0.8465999960899353\n",
            "Test accuracy of the network: 0.8519999980926514\n",
            "Test accuracy of the network: 0.8579999804496765\n",
            "Test accuracy of the network: 0.8556999564170837\n",
            "Test accuracy of the network: 0.7285999655723572\n",
            "Test accuracy of the network: 0.7531999945640564\n",
            "Test accuracy of the network: 0.819599986076355\n",
            "Test accuracy of the network: 0.7723000049591064\n",
            "Test accuracy of the network: 0.7434999942779541\n",
            "Test accuracy of the network: 0.8524999618530273\n",
            "Test accuracy of the network: 0.8448999524116516\n",
            "Test accuracy of the network: 0.793999969959259\n",
            "Test accuracy of the network: 0.5388999581336975\n",
            "Test accuracy of the network: 0.8374999761581421\n",
            "Test accuracy of the network: 0.8267999887466431\n",
            "Test accuracy of the network: 0.7286999821662903\n",
            "Test accuracy of the network: 0.837399959564209\n",
            "Test accuracy of the network: 0.8460999727249146\n",
            "Test accuracy of the network: 0.6771000027656555\n",
            "Test accuracy of the network: 0.851099967956543\n",
            "Test accuracy of the network: 0.7547000050544739\n",
            "Test accuracy of the network: 0.8089999556541443\n",
            "Test accuracy of the network: 0.6682999730110168\n",
            "Test accuracy of the network: 0.8549000024795532\n",
            "Test accuracy of the network: 0.8478999733924866\n",
            "Test accuracy of the network: 0.7723999619483948\n",
            "Test accuracy of the network: 0.8227999806404114\n",
            "Test accuracy of the network: 0.85999995470047\n",
            "Test accuracy of the network: 0.762999951839447\n",
            "Test accuracy of the network: 0.8610000014305115\n",
            "Test accuracy of the network: 0.7447999715805054\n",
            "Test accuracy of the network: 0.8328999876976013\n",
            "Test accuracy of the network: 0.8651999831199646\n",
            "Test accuracy of the network: 0.8287000060081482\n",
            "Test accuracy of the network: 0.8406999707221985\n",
            "Test accuracy of the network: 0.8542999625205994\n",
            "Test accuracy of the network: 0.7960999608039856\n",
            "Test accuracy of the network: 0.531499981880188\n",
            "Test accuracy of the network: 0.7670999765396118\n",
            "Test accuracy of the network: 0.7782999873161316\n",
            "Test accuracy of the network: 0.8113999962806702\n",
            "Test accuracy of the network: 0.8411999940872192\n",
            "Test accuracy of the network: 0.8330999612808228\n",
            "Test accuracy of the network: 0.8486999869346619\n",
            "Test accuracy of the network: 0.4000999927520752\n",
            "Test accuracy of the network: 0.8229999542236328\n",
            "Test accuracy of the network: 0.7954999804496765\n",
            "Test accuracy of the network: 0.8215999603271484\n",
            "Test accuracy of the network: 0.8561999797821045\n",
            "Test accuracy of the network: 0.7220999598503113\n",
            "Test accuracy of the network: 0.847000002861023\n",
            "Test accuracy of the network: 0.8496999740600586\n",
            "Test accuracy of the network: 0.7191999554634094\n",
            "Test accuracy of the network: 0.693399965763092\n",
            "Test accuracy of the network: 0.6222999691963196\n",
            "Test accuracy of the network: 0.6340999603271484\n",
            "Test accuracy of the network: 0.7319999933242798\n",
            "Test accuracy of the network: 0.8039000034332275\n",
            "Test accuracy of the network: 0.8611999750137329\n",
            "Test accuracy of the network: 0.8603999614715576\n",
            "Test accuracy of the network: 0.7590000033378601\n",
            "Test accuracy of the network: 0.7125999927520752\n",
            "Test accuracy of the network: 0.553600013256073\n",
            "Test accuracy of the network: 0.8075000047683716\n",
            "Test accuracy of the network: 0.8188999891281128\n",
            "Test accuracy of the network: 0.589199960231781\n",
            "Test accuracy of the network: 0.8499000072479248\n",
            "Test accuracy of the network: 0.850600004196167\n",
            "Test accuracy of the network: 0.5940999984741211\n",
            "Test accuracy of the network: 0.7106999754905701\n",
            "Test accuracy of the network: 0.7639999985694885\n",
            "Test accuracy of the network: 0.7523999810218811\n",
            "Test accuracy of the network: 0.8587999939918518\n",
            "Test accuracy of the network: 0.8280999660491943\n",
            "Test accuracy of the network: 0.858199954032898\n",
            "Test accuracy of the network: 0.838699996471405\n",
            "Test accuracy of the network: 0.7049999833106995\n",
            "Test accuracy of the network: 0.7622999548912048\n",
            "Test accuracy of the network: 0.8551999926567078\n",
            "Test accuracy of the network: 0.7497000098228455\n",
            "Test accuracy of the network: 0.8578999638557434\n",
            "Test accuracy of the network: 0.8597999811172485\n",
            "Test accuracy of the network: 0.8392999768257141\n",
            "Test accuracy of the network: 0.8333999514579773\n",
            "Test accuracy of the network: 0.7240999937057495\n",
            "Test accuracy of the network: 0.7368999719619751\n",
            "Test accuracy of the network: 0.6994999647140503\n",
            "Test accuracy of the network: 0.852400004863739\n",
            "Test accuracy of the network: 0.820099949836731\n",
            "Test accuracy of the network: 0.8578999638557434\n",
            "Test accuracy of the network: 0.8118000030517578\n",
            "Test accuracy of the network: 0.8452000021934509\n",
            "Test accuracy of the network: 0.7476999759674072\n",
            "Test accuracy of the network: 0.7335999608039856\n",
            "Test accuracy of the network: 0.8047999739646912\n",
            "Test accuracy of the network: 0.8711999654769897\n",
            "Test accuracy of the network: 0.8555999994277954\n",
            "Test accuracy of the network: 0.8298999667167664\n",
            "Test accuracy of the network: 0.8583999872207642\n",
            "Test accuracy of the network: 0.849399983882904\n",
            "Test accuracy of the network: 0.5210999846458435\n",
            "Test accuracy of the network: 0.7389000058174133\n",
            "Test accuracy of the network: 0.8070999979972839\n",
            "Test accuracy of the network: 0.51910001039505\n",
            "Test accuracy of the network: 0.852899968624115\n",
            "Test accuracy of the network: 0.7813999652862549\n",
            "Test accuracy of the network: 0.8518999814987183\n",
            "Test accuracy of the network: 0.8557999730110168\n",
            "Test accuracy of the network: 0.7998999953269958\n",
            "Test accuracy of the network: 0.8223999738693237\n",
            "Test accuracy of the network: 0.7949999570846558\n",
            "Test accuracy of the network: 0.7351999878883362\n",
            "Test accuracy of the network: 0.8526999950408936\n",
            "Test accuracy of the network: 0.8499000072479248\n",
            "Test accuracy of the network: 0.8583999872207642\n",
            "Test accuracy of the network: 0.6651999950408936\n",
            "Test accuracy of the network: 0.676800012588501\n",
            "Test accuracy of the network: 0.8513999581336975\n",
            "Test accuracy of the network: 0.8181999921798706\n",
            "Test accuracy of the network: 0.724399983882904\n",
            "Test accuracy of the network: 0.8450999855995178\n",
            "Test accuracy of the network: 0.8226000070571899\n",
            "Test accuracy of the network: 0.8501999974250793\n",
            "Test accuracy of the network: 0.8552999496459961\n",
            "Test accuracy of the network: 0.8202999830245972\n",
            "Test accuracy of the network: 0.7953999638557434\n",
            "Test accuracy of the network: 0.8158999681472778\n",
            "Test accuracy of the network: 0.861799955368042\n",
            "Test accuracy of the network: 0.6176999807357788\n",
            "Test accuracy of the network: 0.8586999773979187\n",
            "Test accuracy of the network: 0.8353999853134155\n",
            "Test accuracy of the network: 0.8604999780654907\n",
            "Test accuracy of the network: 0.7502999901771545\n",
            "Test accuracy of the network: 0.8545999526977539\n",
            "Test accuracy of the network: 0.6825999617576599\n",
            "Test accuracy of the network: 0.835099995136261\n",
            "Test accuracy of the network: 0.8639000058174133\n",
            "Test accuracy of the network: 0.8556999564170837\n",
            "Test accuracy of the network: 0.8623999953269958\n",
            "Test accuracy of the network: 0.8565999865531921\n",
            "Test accuracy of the network: 0.7461999654769897\n",
            "Test accuracy of the network: 0.7963999509811401\n",
            "Test accuracy of the network: 0.6370999813079834\n",
            "Test accuracy of the network: 0.8156999945640564\n",
            "Test accuracy of the network: 0.7723999619483948\n",
            "Test accuracy of the network: 0.8648999929428101\n",
            "Test accuracy of the network: 0.8639999628067017\n",
            "Test accuracy of the network: 0.8560999631881714\n",
            "Test accuracy of the network: 0.7375999689102173\n",
            "Test accuracy of the network: 0.8291999697685242\n",
            "Test accuracy of the network: 0.6250999569892883\n",
            "Test accuracy of the network: 0.8592000007629395\n",
            "Test accuracy of the network: 0.8596999645233154\n",
            "Test accuracy of the network: 0.8499999642372131\n",
            "Test accuracy of the network: 0.854200005531311\n",
            "Test accuracy of the network: 0.769599974155426\n",
            "Test accuracy of the network: 0.818399965763092\n",
            "Test accuracy of the network: 0.7684999704360962\n",
            "Test accuracy of the network: 0.8447999954223633\n",
            "Test accuracy of the network: 0.8240000009536743\n",
            "Test accuracy of the network: 0.8623999953269958\n",
            "Test accuracy of the network: 0.8613999485969543\n",
            "Test accuracy of the network: 0.8606999516487122\n",
            "Test accuracy of the network: 0.8596000075340271\n",
            "Test accuracy of the network: 0.7720999717712402\n",
            "Test accuracy of the network: 0.6064000129699707\n",
            "Test accuracy of the network: 0.8244999647140503\n",
            "Test accuracy of the network: 0.8514999747276306\n",
            "Test accuracy of the network: 0.8601999878883362\n",
            "Test accuracy of the network: 0.8578999638557434\n",
            "Test accuracy of the network: 0.848800003528595\n",
            "Test accuracy of the network: 0.8478999733924866\n",
            "Test accuracy of the network: 0.8145999908447266\n",
            "Test accuracy of the network: 0.6782000064849854\n",
            "Test accuracy of the network: 0.5927000045776367\n",
            "Test accuracy of the network: 0.8351999521255493\n",
            "Test accuracy of the network: 0.6161999702453613\n",
            "Test accuracy of the network: 0.6782000064849854\n",
            "Test accuracy of the network: 0.8100000023841858\n",
            "Test accuracy of the network: 0.7999999523162842\n",
            "Test accuracy of the network: 0.7134000062942505\n",
            "Test accuracy of the network: 0.8206999897956848\n",
            "Test accuracy of the network: 0.694599986076355\n",
            "Test accuracy of the network: 0.8345999717712402\n",
            "Test accuracy of the network: 0.8241999745368958\n",
            "Test accuracy of the network: 0.8567000031471252\n",
            "Test accuracy of the network: 0.854200005531311\n",
            "Test accuracy of the network: 0.8531999588012695\n",
            "Test accuracy of the network: 0.6811000108718872\n",
            "Test accuracy of the network: 0.7989999651908875\n",
            "Test accuracy of the network: 0.535099983215332\n",
            "Test accuracy of the network: 0.8593999743461609\n",
            "Test accuracy of the network: 0.8589999675750732\n",
            "Test accuracy of the network: 0.8517999649047852\n",
            "Test accuracy of the network: 0.8608999848365784\n",
            "Test accuracy of the network: 0.8400999903678894\n",
            "Test accuracy of the network: 0.7393999695777893\n",
            "Test accuracy of the network: 0.8533999919891357\n",
            "Test accuracy of the network: 0.8486999869346619\n",
            "Test accuracy of the network: 0.5985999703407288\n",
            "Test accuracy of the network: 0.8402000069618225\n",
            "Test accuracy of the network: 0.8589999675750732\n",
            "Test accuracy of the network: 0.8285999894142151\n",
            "Test accuracy of the network: 0.8024999499320984\n",
            "Test accuracy of the network: 0.802899956703186\n",
            "Test accuracy of the network: 0.6541000008583069\n",
            "Test accuracy of the network: 0.8180999755859375\n",
            "Test accuracy of the network: 0.6807999610900879\n",
            "Test accuracy of the network: 0.8678999543190002\n",
            "Test accuracy of the network: 0.8621000051498413\n",
            "Test accuracy of the network: 0.8392999768257141\n",
            "Test accuracy of the network: 0.6825999617576599\n",
            "Test accuracy of the network: 0.715999960899353\n",
            "Test accuracy of the network: 0.764799952507019\n",
            "Test accuracy of the network: 0.863599956035614\n",
            "Test accuracy of the network: 0.8669999837875366\n",
            "Test accuracy of the network: 0.8616999983787537\n",
            "Test accuracy of the network: 0.8639000058174133\n",
            "Test accuracy of the network: 0.8569999933242798\n",
            "Test accuracy of the network: 0.8625999689102173\n",
            "Test accuracy of the network: 0.3962000012397766\n",
            "Test accuracy of the network: 0.7781999707221985\n",
            "Test accuracy of the network: 0.8680999875068665\n",
            "Test accuracy of the network: 0.7710999846458435\n",
            "Test accuracy of the network: 0.770799994468689\n",
            "Test accuracy of the network: 0.8491999506950378\n",
            "Test accuracy of the network: 0.868399977684021\n",
            "Test accuracy of the network: 0.8689000010490417\n",
            "Test accuracy of the network: 0.5756999850273132\n",
            "Test accuracy of the network: 0.8181999921798706\n",
            "Test accuracy of the network: 0.7799999713897705\n",
            "Test accuracy of the network: 0.5360000133514404\n",
            "Test accuracy of the network: 0.7738999724388123\n",
            "Test accuracy of the network: 0.847000002861023\n",
            "Test accuracy of the network: 0.8574000000953674\n",
            "Test accuracy of the network: 0.85589998960495\n",
            "Test accuracy of the network: 0.792199969291687\n",
            "Test accuracy of the network: 0.8155999779701233\n",
            "Test accuracy of the network: 0.7026000022888184\n",
            "Test accuracy of the network: 0.8592999577522278\n",
            "Test accuracy of the network: 0.8227999806404114\n",
            "Test accuracy of the network: 0.8619999885559082\n",
            "Test accuracy of the network: 0.8572999835014343\n",
            "Test accuracy of the network: 0.8317999839782715\n",
            "Test accuracy of the network: 0.43209999799728394\n",
            "Test accuracy of the network: 0.6879000067710876\n",
            "Test accuracy of the network: 0.5906000137329102\n",
            "Test accuracy of the network: 0.6814999580383301\n",
            "Test accuracy of the network: 0.8587999939918518\n",
            "Test accuracy of the network: 0.854699969291687\n",
            "Test accuracy of the network: 0.8004999756813049\n",
            "Test accuracy of the network: 0.8227999806404114\n",
            "Test accuracy of the network: 0.7655999660491943\n",
            "Test accuracy of the network: 0.6298999786376953\n",
            "Test accuracy of the network: 0.8578999638557434\n",
            "Test accuracy of the network: 0.8343999981880188\n",
            "Test accuracy of the network: 0.861799955368042\n",
            "Test accuracy of the network: 0.776199996471405\n",
            "Test accuracy of the network: 0.8596999645233154\n",
            "Test accuracy of the network: 0.854200005531311\n",
            "Test accuracy of the network: 0.8248999714851379\n",
            "Test accuracy of the network: 0.6567999720573425\n",
            "Test accuracy of the network: 0.786899983882904\n",
            "Test accuracy of the network: 0.7699999809265137\n",
            "Test accuracy of the network: 0.7511000037193298\n",
            "Test accuracy of the network: 0.8588999509811401\n",
            "Test accuracy of the network: 0.861799955368042\n",
            "Test accuracy of the network: 0.8556999564170837\n",
            "Test accuracy of the network: 0.8303999900817871\n",
            "Test accuracy of the network: 0.8287999629974365\n",
            "Test accuracy of the network: 0.8567000031471252\n",
            "Test accuracy of the network: 0.8618999719619751\n",
            "Test accuracy of the network: 0.7361999750137329\n",
            "Test accuracy of the network: 0.8563999533653259\n",
            "Test accuracy of the network: 0.865399956703186\n",
            "Test accuracy of the network: 0.8608999848365784\n",
            "Test accuracy of the network: 0.7024999856948853\n",
            "Test accuracy of the network: 0.8331999778747559\n",
            "Test accuracy of the network: 0.8312000036239624\n",
            "Test accuracy of the network: 0.8521999716758728\n",
            "Test accuracy of the network: 0.8276999592781067\n",
            "Test accuracy of the network: 0.8639000058174133\n",
            "Test accuracy of the network: 0.8689000010490417\n",
            "Test accuracy of the network: 0.8618999719619751\n",
            "Test accuracy of the network: 0.499099999666214\n",
            "Test accuracy of the network: 0.8543999791145325\n",
            "Test accuracy of the network: 0.8490999937057495\n",
            "Test accuracy of the network: 0.8113999962806702\n",
            "Test accuracy of the network: 0.8679999709129333\n",
            "Test accuracy of the network: 0.8600999712944031\n",
            "Test accuracy of the network: 0.8565999865531921\n",
            "Test accuracy of the network: 0.8299999833106995\n",
            "Test accuracy of the network: 0.7928999662399292\n",
            "Test accuracy of the network: 0.6747999787330627\n",
            "Test accuracy of the network: 0.8308999538421631\n",
            "Test accuracy of the network: 0.8495999574661255\n",
            "Test accuracy of the network: 0.8453999757766724\n",
            "Test accuracy of the network: 0.8521999716758728\n",
            "Test accuracy of the network: 0.8136999607086182\n",
            "Test accuracy of the network: 0.8180999755859375\n",
            "Test accuracy of the network: 0.7755999565124512\n",
            "Test accuracy of the network: 0.8529999852180481\n",
            "Test accuracy of the network: 0.7874999642372131\n",
            "Test accuracy of the network: 0.8495999574661255\n",
            "Test accuracy of the network: 0.8562999963760376\n",
            "Test accuracy of the network: 0.8519999980926514\n",
            "Test accuracy of the network: 0.8154000043869019\n",
            "Test accuracy of the network: 0.8464999794960022\n",
            "Test accuracy of the network: 0.8111000061035156\n",
            "Test accuracy of the network: 0.7340999841690063\n",
            "Test accuracy of the network: 0.6861000061035156\n",
            "Test accuracy of the network: 0.8583999872207642\n",
            "Test accuracy of the network: 0.8520999550819397\n",
            "Test accuracy of the network: 0.8438000082969666\n",
            "Test accuracy of the network: 0.8590999841690063\n",
            "Test accuracy of the network: 0.8664999604225159\n",
            "Test accuracy of the network: 0.8438000082969666\n",
            "Test accuracy of the network: 0.8623999953269958\n",
            "Test accuracy of the network: 0.7935999631881714\n",
            "Test accuracy of the network: 0.7687999606132507\n",
            "Test accuracy of the network: 0.847000002861023\n",
            "Test accuracy of the network: 0.8632999658584595\n",
            "Test accuracy of the network: 0.8679999709129333\n",
            "Test accuracy of the network: 0.8679999709129333\n",
            "Test accuracy of the network: 0.7802000045776367\n",
            "Test accuracy of the network: 0.7256999611854553\n",
            "Test accuracy of the network: 0.667199969291687\n",
            "Test accuracy of the network: 0.8511999845504761\n",
            "Test accuracy of the network: 0.8604999780654907\n",
            "Test accuracy of the network: 0.8325999975204468\n",
            "Test accuracy of the network: 0.8514999747276306\n",
            "Test accuracy of the network: 0.8637999892234802\n",
            "Test accuracy of the network: 0.7919999957084656\n",
            "Test accuracy of the network: 0.8598999977111816\n",
            "Test accuracy of the network: 0.8319999575614929\n",
            "Test accuracy of the network: 0.8086999654769897\n",
            "Test accuracy of the network: 0.8376999497413635\n",
            "Test accuracy of the network: 0.8650999665260315\n",
            "Test accuracy of the network: 0.8664999604225159\n",
            "Test accuracy of the network: 0.8660999536514282\n",
            "Test accuracy of the network: 0.5507999658584595\n",
            "Test accuracy of the network: 0.8565999865531921\n",
            "Test accuracy of the network: 0.858299970626831\n",
            "Test accuracy of the network: 0.7667999863624573\n",
            "Test accuracy of the network: 0.8693999648094177\n",
            "Test accuracy of the network: 0.861299991607666\n",
            "Test accuracy of the network: 0.8629999756813049\n",
            "Test accuracy of the network: 0.8501999974250793\n",
            "Test accuracy of the network: 0.8305000066757202\n",
            "Test accuracy of the network: 0.8544999957084656\n",
            "Test accuracy of the network: 0.7753999829292297\n",
            "Test accuracy of the network: 0.8561999797821045\n",
            "Test accuracy of the network: 0.85999995470047\n",
            "Test accuracy of the network: 0.8562999963760376\n",
            "Test accuracy of the network: 0.866599977016449\n",
            "Test accuracy of the network: 0.8567999601364136\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "xUEu0tceQCuy",
        "colab_type": "text"
      },
      "source": [
        "# RSA"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "235SLzWCLc_g",
        "colab_type": "text"
      },
      "source": [
        "## A function to compute Input RDMs & confusion matrices\n",
        "This function calculates \n",
        "\n",
        "1.   Activations for a number num_samples of sample images from FashionMNIST for every model in the save files and for the desired layer.\n",
        "2.   The Input RDM (=correlation Matrix) for every model\n",
        "3.   The confusion matrices of every model (but does not as a default).\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QDmdSqVBwJ7u",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "mlp_saves = f'{working_directory}/{fashionmnist_saves_mlp}/'\n",
        "conv_saves = f'{working_directory}/{fashionmnist_saves_conv}/'\n",
        "\n",
        "\n",
        "def calculate_activations_correlations(files, path, layer, num_samples=1000, show_progress=True, activations_dict = {}, corr_distances_dict = {}, \n",
        "                                       confusion_dict = {}, calc_confusion = False):\n",
        "  \n",
        "  dataloader_valid = torch.utils.data.DataLoader(FashionMNIST_valid, batch_size = num_samples, shuffle = False)\n",
        "  inputs, labels  = next(iter(dataloader_valid))\n",
        "  index_sorted =  np.argsort(labels.numpy())\n",
        "  inputs, labels = inputs[index_sorted, :], labels[index_sorted]\n",
        "  inputs_copy, labels_copy = deepcopy(inputs), deepcopy(labels)\n",
        "  n = len(labels)\n",
        "\n",
        "  def calculate_confusion(model, inputs, labels):\n",
        "      \n",
        "      confusion_matrix = np.zeros((10,10))\n",
        "      with torch.no_grad():\n",
        "          \n",
        "          model.eval()\n",
        "          inputs, labels = inputs.cuda(), labels.cuda()\n",
        "          out = model(inputs)[0]\n",
        "        \n",
        "          for true_label in range(10):\n",
        "              \n",
        "              true_index = (labels==true_label) # in case not all labels are in the sample\n",
        "                \n",
        "              if(true_index.sum() != 0) :\n",
        "                \n",
        "                   (torch.max(out,1)[1])\n",
        "                  \n",
        "                   for output in torch.max(out[true_index],1)[1]:\n",
        "                       for estimate in range(10):\n",
        "                        \n",
        "                           confusion_matrix[true_label, estimate] += (output == estimate).sum()\n",
        "              else:\n",
        "                \n",
        "                  pass\n",
        "    \n",
        "      return(confusion_matrix)\n",
        "\n",
        "  \n",
        "\n",
        "\n",
        "  def activations_list(model):\n",
        "    \n",
        "      activations = []\n",
        "      with torch.no_grad():\n",
        "          model.eval()\n",
        "          for i in range(n):\n",
        "            \n",
        "              if model_type == ConvNet:\n",
        "                \n",
        "                  single_input = inputs[i].cuda().view(1,1,28,28)\n",
        "              else:\n",
        "                \n",
        "                  single_input = inputs[i].cuda()\n",
        "                  \n",
        "              activations.append(model(single_input)[layer].cpu().numpy().squeeze())\n",
        "      return(activations)\n",
        "\n",
        "  def input_rdm(list_of_activations):\n",
        "      \n",
        "      correlation = np.zeros((n,n))\n",
        "      for i in range(n):\n",
        "          for j in range(i, n):\n",
        "            \n",
        "              correlation[i,j] = correlation[j, i] = 1 - scipy.stats.pearsonr(list_of_activations[i], list_of_activations[j])[0]\n",
        "      return(correlation)\n",
        "\n",
        "  for i in range(len(files,)):\n",
        "      if show_progress:\n",
        "          \n",
        "          print(i)\n",
        "          \n",
        "      inputs, labels = inputs_copy, labels_copy\n",
        "      model_name = files[i]\n",
        "      if path[i]==0:\n",
        "        \n",
        "          path_to_saves = mlp_saves\n",
        "          model_type = feedforward\n",
        "          inputs, labels = torch.autograd.Variable(inputs).view((-1, 28*28)).float(), torch.autograd.Variable(labels).long()\n",
        "      \n",
        "      else: \n",
        "        \n",
        "          path_to_saves = conv_saves\n",
        "          model_type = ConvNet\n",
        "         \n",
        "     \n",
        "      model_dict = torch.load(f'{path_to_saves}{model_name}')\n",
        "      layers = model_dict['layers']\n",
        "      model = model_type(*layers).cuda()\n",
        "      model.load_state_dict(model_dict['state_dict'])\n",
        "      activations_dict[i] = activations_list(model)\n",
        "      corr_distances_dict[i] = input_rdm(activations_dict[i])\n",
        "      if calc_confusion:\n",
        "        \n",
        "          confusion_dict[i]= calculate_confusion(model, inputs, labels)\n",
        "          \n",
        "      \n",
        "  return corr_distances_dict,  confusion_dict\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "AICiuBArM6mw",
        "colab_type": "text"
      },
      "source": [
        "## Get dataframe of model accuracies & locations\n",
        "\n",
        "Here, the accuracy list from MLPs and CNNs are arranged into one dataframe, which contains there accuracies, location in their model type's folder and their model type. Then, the dataframe is ordered by accuracies. This allows obtaining a Model RDM with models sorted by accuracy."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ISWMOeP0wbRo",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "with open (f'{working_directory}/accuracy_list_fashionmnist_mlp.pkl', 'rb') as fp:\n",
        "    \n",
        "    acc_list_mlp = pickle.load(fp)\n",
        "    \n",
        "with open (f'{working_directory}/accuracy_list_fashionmnist_cnn.pkl', 'rb') as fp:\n",
        "    \n",
        "    acc_list_convnet = pickle.load(fp)\n",
        "   \n",
        "acc_df_mlp = pd.DataFrame(acc_list_mlp, columns = ['acc', 'loc'])\n",
        "acc_df_mlp['model_type'] = np.zeros(len(acc_df_mlp))\n",
        "acc_df_convnet = pd.DataFrame(acc_list_convnet, columns = ['acc', 'loc'])\n",
        "acc_df_convnet['model_type'] = np.ones(len(acc_df_convnet))\n",
        "acc_df =  pd.concat([acc_df_mlp,acc_df_convnet], axis=0, ignore_index = True) \n",
        "acc_df['index'] = acc_df.index\n",
        "acc_df = acc_df.sort_values(by = 'acc', ascending = True)\n",
        "\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "FO2cxL0DOpTB",
        "colab_type": "text"
      },
      "source": [
        "## Calculate (or load) Input Rdms and confusion matrices\n",
        "Run the function if_calculate_activations_correlations to compute Input RDMs and confusion matrices on 400 samples taken from the FashionMNIST validation data for the penultimate layer. Save every returned dictionary using pickle."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "w-cu73ewwq9o",
        "colab_type": "code",
        "outputId": "f4a3870c-ea89-44d6-a463-4c1a521d8396",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 53
        }
      },
      "source": [
        "calculate_activations_correlations_should_be_run = True\n",
        "files = acc_df['loc']\n",
        "path  = acc_df['model_type']\n",
        "if calculate_activations_correlations_should_be_run:\n",
        " \n",
        "    input_rdm_dict_fashionmnist, confusion_dict_fashionmnist = calculate_activations_correlations(list(files), list(path), num_samples= 400, calc_confusion = True, layer = 2)\n",
        "\n",
        "    \n",
        "    with open(f'{working_directory}/input_rdm_dict_fashionmnist.pkl', 'wb') as fp:\n",
        "     \n",
        "        pickle.dump(input_rdm_dict_fashionmnist, fp)\n",
        "  \n",
        "    with open(f'{working_directory}/confusion_dict_fashionmnist.pkl', 'wb') as fp:\n",
        "    \n",
        "        pickle.dump(confusion_dict_fashionmnist, fp)    \n",
        "\n",
        "else:\n",
        "  \n",
        "    with open (f'{working_directory}/input_rdm_dict_fashionmnist.pkl', 'rb') as fp:\n",
        "   \n",
        "        input_rdm_dict_fashionmnist = pickle.load(fp)\n"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "0\n",
            "1\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "xP4JSAzcoR7O",
        "colab_type": "text"
      },
      "source": [
        "## Function for Model RDM\n",
        "Functions do:\n",
        "\n",
        "1.   define the rank-correlation distance between two upper triangular matrices of Input RDMs\n",
        "2.   calculate the distance between two Input RDMs for a given distance function\n",
        "3.   calculate the Model RDM (depends on 2 (necessery) and 1 (if rank correlation distance needed)\n",
        "4.   visualize the Model RDM (shows every 15th axis tick)\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bEkz58oewxgf",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def rank_corr_dist(triu_model_1, triu_model_2):\n",
        "  return 1 - scipy.stats.spearmanr(triu_model_1.flatten(), triu_model_2.flatten())[0]\n",
        "\n",
        "def dist_between_input_rdms(dist_fun, corr_matrix_1, corr_matrix_2):\n",
        "  triu_model_1 = corr_matrix_1[np.triu_indices(corr_matrix_1.shape[0], k = 1)]\n",
        "  triu_model_2 = corr_matrix_2[np.triu_indices(corr_matrix_2.shape[0], k = 1)]\n",
        "  \n",
        "  return dist_fun(triu_model_1, triu_model_2)\n",
        "\n",
        "def calc_rdm(dist_fun, corr_distances_dict,  files=files):\n",
        "  n = len(files)\n",
        "  rdm = np.zeros((n,n))\n",
        "  for i in range(n):\n",
        "      for j in range(i, n):\n",
        "          rdm[i,j] = rdm[j, i] = dist_between_input_rdms(dist_fun,\n",
        "                                                            corr_distances_dict[i], \n",
        "                                                            corr_distances_dict[j])\n",
        "  return rdm\n",
        "\n",
        "def visualise_rdm(rdm, acc = acc_df['acc'] ):\n",
        "  plt.figure(figsize=(14,14))\n",
        "  ax = seaborn.heatmap(rdm, xticklabels = np.around(acc.astype('float') * 100), yticklabels = np.around(acc.astype('float') * 100), cmap='rainbow')\n",
        "  n = 15  \n",
        "  [l.set_visible(False) for (i,l) in enumerate(ax.xaxis.get_ticklabels()) if i % n != 0]\n",
        "  [l.set_visible(False) for (i,l) in enumerate(ax.yaxis.get_ticklabels()) if i % n != 0]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "SPhO1FJOqM97",
        "colab_type": "text"
      },
      "source": [
        "## Calculate euclidean distance Model rdm\n",
        "Calculates the Model RDM for the given dictionary (dictionary can contain Input RDMs for any layer, right now it is the penultimate layer)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KYjwUdyvw_VA",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "rdm = calc_rdm(distance.euclidean, corr_distances_dict_fashionmnist)\n",
        "visualise_rdm(rdm)"
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}