PUCRS_Aprendizado_de_Máquina_T2.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "gpuType": "V100",
      "machine_shape": "hm",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/debemdeboas/pucrs-aprendizado-de-maquina-t2/blob/master/PUCRS_Aprendizado_de_M%C3%A1quina_T2.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import random\n",
        "\n",
        "import matplotlib.pyplot as plt\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "import requests\n",
        "import seaborn as sns\n",
        "import torch\n",
        "import torch.nn as nn\n",
        "import torch.nn.functional as F\n",
        "import torch.optim as optim\n",
        "import torchvision\n",
        "import tensorflow as tf\n",
        "from PIL import Image\n",
        "from torch.utils.data import DataLoader\n",
        "from torch.utils.data.dataset import Dataset\n",
        "from torchvision import datasets\n",
        "from torchvision import transforms as T\n",
        "from torchvision.transforms import functional as TF\n",
        "from tqdm import tqdm\n",
        "from enum import IntEnum\n",
        "\n",
        "random.seed(18103725)\n",
        "torch.manual_seed(18103725)"
      ],
      "metadata": {
        "id": "yKqD4YWivM_g",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "a596920a-a9fe-4dab-80b8-5194ec15dc3b"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<torch._C.Generator at 0x7d3c1eeda250>"
            ]
          },
          "metadata": {},
          "execution_count": 2
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Download and extract our dataset.\n",
        "\n",
        "This will download a tarred file and extract it into `dist`. Then, we're renaming it to `animes/`.\n",
        "This directory contains the following files:\n",
        "- `animes.csv`, the CSV containing anime IDs, URLs, titles, genres, and poster path\n",
        "- `animes.pkl`, serialized (pickled) list of `Anime` instances. This isn't used by this notebook\n",
        "- `images/`, a directory that contains all of our anime posters as `images/<mal_id>.jpg` files\n"
      ],
      "metadata": {
        "id": "-Sz2tJ-38Ypr"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "b9ZTAtF86a_E"
      },
      "outputs": [],
      "source": [
        "ds = requests.get(\"https://public-s3.debem.dev/anime_dataset.tar.xz\", allow_redirects=True)\n",
        "\n",
        "with open(\"anime_dataset.tar.xz\", \"wb\") as f:\n",
        "    f.write(ds.content)\n",
        "\n",
        "!tar xf anime_dataset.tar.xz"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df = pd.read_csv(\"animes.csv\")\n",
        "df = df.dropna()\n",
        "df[\"genres\"] = df[\"genres\"].str.split('|')\n",
        "\n",
        "df[\"img\"] = df[\"img_path\"].map(lambda x: Image.open(x).convert(\"RGB\"))"
      ],
      "metadata": {
        "id": "DP85KZamJ4Xf"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "all_genres_to_idx = dict()\n",
        "all_genres_amnt = dict()\n",
        "all_genres = list()\n",
        "for gl in df.genres:\n",
        "    for g in gl:\n",
        "        if g not in all_genres_to_idx:\n",
        "            all_genres.append(g)\n",
        "            all_genres_to_idx[g] = len(all_genres) - 1\n",
        "            all_genres_amnt[g] = 0\n",
        "        all_genres_amnt[g] += 1"
      ],
      "metadata": {
        "id": "yLBqs5DFEC1e"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "class AccIdx(IntEnum):\n",
        "    TP = 0\n",
        "    FP = 1\n",
        "    TN = 2\n",
        "    FN = 3\n",
        "\n",
        "class PosterMultiLabelDataset(Dataset):\n",
        "    def __init__(self, df: pd.DataFrame, transform=None, *args, **kwargs):\n",
        "        self.df = df\n",
        "        if transform:\n",
        "            self.transform = transform\n",
        "        else:\n",
        "            self.transform = T.Compose([\n",
        "                T.RandomHorizontalFlip(p=0.5),\n",
        "                T.ColorJitter(brightness=0.3, contrast=0.3, saturation=0.3, hue=0),\n",
        "                T.Resize((256, 256)),\n",
        "                T.RandomResizedCrop(224),\n",
        "                T.ToTensor(),\n",
        "                T.Normalize(mean=[0.5, 0.5, 0.5],\n",
        "                             std=[0.5, 0.5, 0.5]),\n",
        "                ])\n",
        "\n",
        "    def __len__(self) -> int:\n",
        "        return len(self.df)\n",
        "\n",
        "    def __getitem__(self, idx):\n",
        "        return self.transform(self.df.loc[idx].img), torch.Tensor([1 if g in self.df.loc[idx].genres else 0 for g in all_genres])"
      ],
      "metadata": {
        "id": "F9X1yBFZslO5"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Now let's get some transfer learning done.\n",
        "\n",
        "We'll use a pre-trained convolutional network to analyze the posters to define which genres a given anime belongs to.\n",
        "Each anime can belong to any number of genres.\n"
      ],
      "metadata": {
        "id": "0Z5QkRZL8enF"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')\n",
        "\n",
        "def validation(model, loader, criterion):\n",
        "    model.eval()\n",
        "    val_loss = 0\n",
        "    with torch.no_grad():\n",
        "        for (images, labels) in loader:\n",
        "            images = images.to(device)\n",
        "            labels = labels.to(device)\n",
        "            outputs = model(images)\n",
        "            loss = criterion(outputs,labels)\n",
        "            val_loss +=loss\n",
        "    return val_loss/len(loader)\n",
        "\n",
        "\n",
        "def train(model, trainloader, testloader, optimizer, criterion, epochs):\n",
        "    for epoch in range(epochs):\n",
        "        model.train()\n",
        "        running_loss = 0\n",
        "        for i, (images, labels) in tqdm(enumerate(trainloader)):\n",
        "            images = images.to(device)\n",
        "            labels = labels.to(device)\n",
        "            model.zero_grad()\n",
        "            outputs = model(images)\n",
        "            loss = criterion(outputs, labels)\n",
        "            loss.backward()\n",
        "            optimizer.step()\n",
        "            running_loss += loss.item()\n",
        "        val_loss = validation(model, testloader, criterion)\n",
        "        print(f'Epoch: {epoch+1} | Loss: {running_loss/len(trainloader)} | Val Loss: {val_loss}')\n",
        "\n",
        "\n",
        "def accuracy(model, loader, positive_threshold = 0):\n",
        "    model.eval()\n",
        "    correct = 0\n",
        "    total = 0\n",
        "    # Create a numpy matrix of $len(all_genres) x 4$ à la confusion matrix\n",
        "    totals = np.zeros((len(all_genres), 4))\n",
        "    with torch.no_grad():\n",
        "        for (img, lbl) in loader:\n",
        "            img = img.to(device)\n",
        "            lbl = lbl.to(device)\n",
        "            output = model(img)\n",
        "            tp = (output > positive_threshold) & (lbl > 0)\n",
        "            fp = (output > positive_threshold) & (lbl == 0)\n",
        "            tn = (output < positive_threshold) & (lbl == 0)\n",
        "            fn = (output < positive_threshold) & (lbl > 0)\n",
        "            stacked = torch.stack((tp, fp, tn, fn), dim=2)\n",
        "            totals += torch.sum(stacked, dim=0).cpu().numpy()\n",
        "    return totals\n"
      ],
      "metadata": {
        "id": "kXodITD1GmMa"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# from torchvision.models import resnet50, ResNet50_Weights\n",
        "# from torchvision.models import regnet_y_16gf, RegNet_Y_16GF_Weights\n",
        "from torchvision.models import regnet_x_3_2gf, RegNet_X_3_2GF_Weights\n",
        "\n",
        "torch.backends.cudnn.benchmark = True # speed up training by using the inbuilt cudnn auto-tuner\n",
        "\n",
        "if True:\n",
        "    # model = resnet50(weights=ResNet50_Weights.DEFAULT)\n",
        "    # model.fc = nn.Linear(2048, len(all_genres))\n",
        "\n",
        "    model = regnet_x_3_2gf(weights=RegNet_X_3_2GF_Weights.DEFAULT)\n",
        "    model.fc = nn.Linear(1008, len(all_genres))\n",
        "\n",
        "    for name, params in model.named_parameters():\n",
        "        if name not in ('fc.weight', 'fc.bias'): #final block\n",
        "            params.requires_grad = False\n",
        "else:\n",
        "    # download pre-trained version from our bucket:\n",
        "    model_f = requests.get(\"https://public-s3.debem.dev/model.pt\", allow_redirects=True)\n",
        "    with open('model.pt', 'wb') as f:\n",
        "        f.write(model_f.content)\n",
        "    model = torch.load('model.pt')\n",
        "\n",
        "model.to(device)\n",
        "criterion = nn.BCEWithLogitsLoss()\n",
        "optimizer = optim.AdamW(model.parameters(), lr=0.01)\n",
        "epochs = 6"
      ],
      "metadata": {
        "id": "KmEkiYjHyntS",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "d49c9d8b-5d6f-40da-8bf4-229aeaab7acf"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Downloading: \"https://download.pytorch.org/models/regnet_x_3_2gf-7071aa85.pth\" to /root/.cache/torch/hub/checkpoints/regnet_x_3_2gf-7071aa85.pth\n",
            "100%|██████████| 58.8M/58.8M [00:04<00:00, 14.2MB/s]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# dados de treino = 70%, validação = 15%, teste = 15%\n",
        "trainData = df.sample(frac=0.7)\n",
        "trainDataLeftover = df.drop(trainData.index)\n",
        "validationData = trainDataLeftover.sample(frac=0.5)\n",
        "testData = trainDataLeftover.drop(validationData.index)\n",
        "\n",
        "testData = testData.reset_index()\n",
        "validationData = validationData.reset_index()\n",
        "trainData = trainData.reset_index()"
      ],
      "metadata": {
        "id": "mTxzmlSGxE1s"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "trainloader = DataLoader(PosterMultiLabelDataset(df=trainData), batch_size=124, shuffle=True, pin_memory=True, num_workers=8)\n",
        "testloader = DataLoader(PosterMultiLabelDataset(df=testData), batch_size=124, shuffle=False, pin_memory=True, num_workers=8)"
      ],
      "metadata": {
        "id": "4llD2deT4dWI"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "train(model, trainloader, testloader, optimizer, criterion, epochs)"
      ],
      "metadata": {
        "id": "2MBwVmJmSTS-",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "ad8a4c10-5f99-4db7-864e-b4c79b317956"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "138it [01:10,  1.96it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch: 1 | Loss: 0.14123738702872526 | Val Loss: 0.12419117242097855\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "138it [01:05,  2.10it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch: 2 | Loss: 0.11617765235512154 | Val Loss: 0.12410176545381546\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "138it [01:04,  2.12it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch: 3 | Loss: 0.11378749319608661 | Val Loss: 0.11906985193490982\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "138it [01:03,  2.16it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch: 4 | Loss: 0.11417094294143759 | Val Loss: 0.12972134351730347\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "138it [01:04,  2.15it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch: 5 | Loss: 0.11441177162139313 | Val Loss: 0.13087742030620575\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "138it [01:05,  2.12it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch: 6 | Loss: 0.1136257181726936 | Val Loss: 0.12185613811016083\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def predict(model, data: pd.Series, transform):\n",
        "    img = Image.open(data[\"img_path\"])\n",
        "    img = transform(img)\n",
        "    prediction = model(torch.unsqueeze(img, 0).to(device))\n",
        "    print(prediction)\n",
        "    return prediction"
      ],
      "metadata": {
        "id": "cWk9gbFPQOoZ"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "torch.save(model, 'model.pt')"
      ],
      "metadata": {
        "id": "H9GPqLKEo0xn"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "transforms_val = T.Compose([\n",
        "    T.Resize((224, 224)),\n",
        "    T.ToTensor(),\n",
        "    T.Normalize(mean=[0.5, 0.5, 0.5],\n",
        "                std=[0.5, 0.5, 0.5]),\n",
        "    ])\n",
        "\n",
        "valloader = DataLoader(\n",
        "    PosterMultiLabelDataset(\n",
        "        df=validationData,\n",
        "        transform=transforms_val),\n",
        "    batch_size=50,\n",
        "    shuffle=False)\n",
        "acc = accuracy(model, valloader, positive_threshold=1)"
      ],
      "metadata": {
        "id": "S6vBCpuqAhvw"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def get_acc_info(acc):\n",
        "    total_pos = np.sum(acc[:2])\n",
        "    total_neg = np.sum(acc[2:4])\n",
        "    with np.errstate(divide='ignore', invalid='ignore'):\n",
        "        return (\n",
        "            np.nan_to_num(np.divide(np.sum(acc[AccIdx.TP]), total_pos)),\n",
        "            np.nan_to_num(np.divide(np.sum(acc[AccIdx.FP]), total_pos)),\n",
        "            np.nan_to_num(np.divide(np.sum(acc[AccIdx.TN]), total_neg)),\n",
        "            np.nan_to_num(np.divide(np.sum(acc[AccIdx.FN]), total_neg)),\n",
        "        )\n",
        "\n",
        "def get_acc(tp, fp, tn, fn):\n",
        "    with np.errstate(divide='ignore', invalid='ignore'):\n",
        "        return np.nan_to_num((tn + tp) / (tn + fp + tp + fn))\n",
        "\n",
        "def get_f1_score(tp, fp, tn, fn):\n",
        "    with np.errstate(divide='ignore', invalid='ignore'):\n",
        "        precision = tp / (tp + fp)\n",
        "        recall = tp / (tp + fn)\n",
        "        f1 = (2 * (precision * recall)) / (precision + recall)\n",
        "        return map(np.nan_to_num, (precision, recall, f1))"
      ],
      "metadata": {
        "id": "Lxme3ysvmr8T"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "per_cat_results = dict()\n",
        "per_cat_resuts_csv = dict()\n",
        "for i, category in enumerate(all_genres):\n",
        "    tp, fp, tn, fn = get_acc_info(acc[i])\n",
        "    curr_acc = get_acc(tp, fp, tn, fn)\n",
        "    precision, recall, f1 = get_f1_score(tp, fp, tn, fn)\n",
        "    per_cat_results[category] = (\n",
        "        '+' + f' {category} '.center(31, '-') + '+\\n'\n",
        "        '|' + f' {all_genres_amnt[category]} instances'.center(31) + '|\\n'\n",
        "        f'| True positive:  {(tp*100):12g}% |\\n'\n",
        "        f'| False positive: {(fp*100):12g}% |\\n'\n",
        "        f'| True negative:  {(tn*100):12g}% |\\n'\n",
        "        f'| False negative: {(fn*100):12g}% |\\n'\n",
        "        f'| Accuracy:       {(curr_acc*100):12g}% |\\n'\n",
        "        f'| Precision:      {(precision*100):12g}% |\\n'\n",
        "        f'| Recall:         {(recall*100):12g}% |\\n'\n",
        "        f'| F1 Score:       {(f1*100):12g}% |\\n'\n",
        "        f'+{\"\":-^31}+\\n'\n",
        "    )\n",
        "    per_cat_resuts_csv[category] = {l[0]: l[1] for l in (map(lambda s: list(map(str.strip, s[1:-1].split(':'))), per_cat_results[category].split('\\n')[2:-2]))}\n",
        "    per_cat_resuts_csv[category]['instances'] = all_genres_amnt[category]\n"
      ],
      "metadata": {
        "id": "oLYB7wCbnLld"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "for _, v in sorted(per_cat_results.items(), key=lambda kv: kv[0]):\n",
        "    print(v)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "0yxxE5a67Kke",
        "outputId": "e094d9c7-57a9-4a59-9b4c-15ae610fafa3"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+------------ Action -----------+\n",
            "|         4954 instances        |\n",
            "| True positive:       68.8889% |\n",
            "| False positive:      31.1111% |\n",
            "| True negative:       80.4227% |\n",
            "| False negative:      19.5773% |\n",
            "| Accuracy:            74.6558% |\n",
            "| Precision:           68.8889% |\n",
            "| Recall:              77.8703% |\n",
            "| F1 Score:            73.1048% |\n",
            "+-------------------------------+\n",
            "\n",
            "+---------- Adult Cast ---------+\n",
            "|          536 instances        |\n",
            "| True positive:             0% |\n",
            "| False positive:            0% |\n",
            "| True negative:       97.9951% |\n",
            "| False negative:      2.00494% |\n",
            "| Accuracy:            97.9951% |\n",
            "| Precision:                 0% |\n",
            "| Recall:                    0% |\n",
            "| F1 Score:                  0% |\n",
            "+-------------------------------+\n",
            "\n",
            "+---------- Adventure ----------+\n",
            "|         3954 instances        |\n",
            "| True positive:            50% |\n",
            "| False positive:           50% |\n",
            "| True negative:       83.4204% |\n",
            "| False negative:      16.5796% |\n",
            "| Accuracy:            66.7102% |\n",
            "| Precision:                50% |\n",
            "| Recall:              75.0981% |\n",
            "| F1 Score:            60.0314% |\n",
            "+-------------------------------+\n",
            "\n",
            "+------- Anthropomorphic -------+\n",
            "|          908 instances        |\n",
            "| True positive:             0% |\n",
            "| False positive:          100% |\n",
            "| True negative:       96.4806% |\n",
            "| False negative:      3.51938% |\n",
            "| Accuracy:            48.2403% |\n",
            "| Precision:                 0% |\n",
            "| Recall:                    0% |\n",
            "| F1 Score:                  0% |\n",
            "+-------------------------------+\n",
            "\n",
            "+--------- Avant Garde ---------+\n",
            "|          841 instances        |\n",
            "| True positive:            50% |\n",
            "| False positive:           50% |\n",
            "| True negative:       96.2902% |\n",
            "| False negative:      3.70981% |\n",
            "| Accuracy:            73.1451% |\n",
            "| Precision:                50% |\n",
            "| Recall:              93.0929% |\n",
            "| F1 Score:            65.0577% |\n",
            "+-------------------------------+\n",
            "\n",
            "+-------- Award Winning --------+\n",
            "|          242 instances        |\n",
            "| True positive:             0% |\n",
            "| False positive:            0% |\n",
            "| True negative:       98.9289% |\n",
            "| False negative:      1.07113% |\n",
            "| Accuracy:            98.9289% |\n",
            "| Precision:                 0% |\n",
            "| Recall:                    0% |\n",
            "| F1 Score:                  0% |\n",
            "+-------------------------------+\n",
            "\n",
            "+---------- Boys Love ----------+\n",
            "|          160 instances        |\n",
            "| True positive:             0% |\n",
            "| False positive:            0% |\n",
            "| True negative:       99.3408% |\n",
            "| False negative:      0.65916% |\n",
            "| Accuracy:            99.3408% |\n",
            "| Precision:                 0% |\n",
            "| Recall:                    0% |\n",
            "| F1 Score:                  0% |\n",
            "+-------------------------------+\n",
            "\n",
            "+------------ CGDCT ------------+\n",
            "|          233 instances        |\n",
            "| True positive:             0% |\n",
            "| False positive:            0% |\n",
            "| True negative:       99.2584% |\n",
            "| False negative:     0.741555% |\n",
            "| Accuracy:            99.2584% |\n",
            "| Precision:                 0% |\n",
            "| Recall:                    0% |\n",
            "| F1 Score:                  0% |\n",
            "+-------------------------------+\n",
            "\n",
            "+---------- Childcare ----------+\n",
            "|          67 instances         |\n",
            "| True positive:             0% |\n",
            "| False positive:            0% |\n",
            "| True negative:       99.7803% |\n",
            "| False negative:      0.21972% |\n",
            "| Accuracy:            99.7803% |\n",
            "| Precision:                 0% |\n",
            "| Recall:                    0% |\n",
            "| F1 Score:                  0% |\n",
            "+-------------------------------+\n",
            "\n",
            "+-------- Combat Sports --------+\n",
            "|          91 instances         |\n",
            "| True positive:             0% |\n",
            "| False positive:            0% |\n",
            "| True negative:        99.643% |\n",
            "| False negative:     0.357045% |\n",
            "| Accuracy:             99.643% |\n",
            "| Precision:                 0% |\n",
            "| Recall:                    0% |\n",
            "| F1 Score:                  0% |\n",
            "+-------------------------------+\n",
            "\n",
            "+------------ Comedy -----------+\n",
            "|         7134 instances        |\n",
            "| True positive:       73.0769% |\n",
            "| False positive:      26.9231% |\n",
            "| True negative:       70.3734% |\n",
            "| False negative:      29.6266% |\n",
            "| Accuracy:            71.7252% |\n",
            "| Precision:           73.0769% |\n",
            "| Recall:              71.1533% |\n",
            "| F1 Score:            72.1023% |\n",
            "+-------------------------------+\n",
            "\n",
            "+-------- Crossdressing --------+\n",
            "|          46 instances         |\n",
            "| True positive:             0% |\n",
            "| False positive:            0% |\n",
            "| True negative:       99.8077% |\n",
            "| False negative:     0.192255% |\n",
            "| Accuracy:            99.8077% |\n",
            "| Precision:                 0% |\n",
            "| Recall:                    0% |\n",
            "| F1 Score:                  0% |\n",
            "+-------------------------------+\n",
            "\n",
            "+--------- Delinquents ---------+\n",
            "|          63 instances         |\n",
            "| True positive:             0% |\n",
            "| False positive:            0% |\n",
            "| True negative:       99.8077% |\n",
            "| False negative:     0.192255% |\n",
            "| Accuracy:            99.8077% |\n",
            "| Precision:                 0% |\n",
            "| Recall:                    0% |\n",
            "| F1 Score:                  0% |\n",
            "+-------------------------------+\n",
            "\n",
            "+---------- Detective ----------+\n",
            "|          296 instances        |\n",
            "| True positive:             0% |\n",
            "| False positive:            0% |\n",
            "| True negative:       98.9563% |\n",
            "| False negative:      1.04367% |\n",
            "| Accuracy:            98.9563% |\n",
            "| Precision:                 0% |\n",
            "| Recall:                    0% |\n",
            "| F1 Score:                  0% |\n",
            "+-------------------------------+\n",
            "\n",
            "+------------ Drama ------------+\n",
            "|         2871 instances        |\n",
            "| True positive:            50% |\n",
            "| False positive:           50% |\n",
            "| True negative:       88.7208% |\n",
            "| False negative:      11.2792% |\n",
            "| Accuracy:            69.3604% |\n",
            "| Precision:                50% |\n",
            "| Recall:              81.5937% |\n",
            "| F1 Score:            62.0043% |\n",
            "+-------------------------------+\n",
            "\n",
            "+------------ Ecchi ------------+\n",
            "|          791 instances        |\n",
            "| True positive:             0% |\n",
            "| False positive:            0% |\n",
            "| True negative:       96.4845% |\n",
            "| False negative:      3.51552% |\n",
            "| Accuracy:            96.4845% |\n",
            "| Precision:                 0% |\n",
            "| Recall:                    0% |\n",
            "| F1 Score:                  0% |\n",
            "+-------------------------------+\n",
            "\n",
            "+--------- Educational ---------+\n",
            "|          269 instances        |\n",
            "| True positive:             0% |\n",
            "| False positive:            0% |\n",
            "| True negative:       98.9563% |\n",
            "| False negative:      1.04367% |\n",
            "| Accuracy:            98.9563% |\n",
            "| Precision:                 0% |\n",
            "| Recall:                    0% |\n",
            "| F1 Score:                  0% |\n",
            "+-------------------------------+\n",
            "\n",
            "+----------- Erotica -----------+\n",
            "|          71 instances         |\n",
            "| True positive:             0% |\n",
            "| False positive:            0% |\n",
            "| True negative:       99.6704% |\n",
            "| False negative:      0.32958% |\n",
            "| Accuracy:            99.6704% |\n",
            "| Precision:                 0% |\n",
            "| Recall:                    0% |\n",
            "| F1 Score:                  0% |\n",
            "+-------------------------------+\n",
            "\n",
            "+----------- Fantasy -----------+\n",
            "|         5473 instances        |\n",
            "| True positive:       48.0712% |\n",
            "| False positive:      51.9288% |\n",
            "| True negative:       79.4189% |\n",
            "| False negative:      20.5811% |\n",
            "| Accuracy:            63.7451% |\n",
            "| Precision:           48.0712% |\n",
            "| Recall:              70.0212% |\n",
            "| F1 Score:            57.0063% |\n",
            "+-------------------------------+\n",
            "\n",
            "+---------- Gag Humor ----------+\n",
            "|          241 instances        |\n",
            "| True positive:             0% |\n",
            "| False positive:            0% |\n",
            "| True negative:       99.2859% |\n",
            "| False negative:      0.71409% |\n",
            "| Accuracy:            99.2859% |\n",
            "| Precision:                 0% |\n",
            "| Recall:                    0% |\n",
            "| F1 Score:                  0% |\n",
            "+-------------------------------+\n",
            "\n",
            "+---------- Girls Love ---------+\n",
            "|          94 instances         |\n",
            "| True positive:             0% |\n",
            "| False positive:            0% |\n",
            "| True negative:       99.5056% |\n",
            "| False negative:      0.49437% |\n",
            "| Accuracy:            99.5056% |\n",
            "| Precision:                 0% |\n",
            "| Recall:                    0% |\n",
            "| F1 Score:                  0% |\n",
            "+-------------------------------+\n",
            "\n",
            "+------------- Gore ------------+\n",
            "|          241 instances        |\n",
            "| True positive:             0% |\n",
            "| False positive:            0% |\n",
            "| True negative:       99.0937% |\n",
            "| False negative:     0.906344% |\n",
            "| Accuracy:            99.0937% |\n",
            "| Precision:                 0% |\n",
            "| Recall:                    0% |\n",
            "| F1 Score:                  0% |\n",
            "+-------------------------------+\n",
            "\n",
            "+----------- Gourmet -----------+\n",
            "|          161 instances        |\n",
            "| True positive:             0% |\n",
            "| False positive:            0% |\n",
            "| True negative:       99.4232% |\n",
            "| False negative:     0.576765% |\n",
            "| Accuracy:            99.4232% |\n",
            "| Precision:                 0% |\n",
            "| Recall:                    0% |\n",
            "| F1 Score:                  0% |\n",
            "+-------------------------------+\n",
            "\n",
            "+------------ Harem ------------+\n",
            "|          415 instances        |\n",
            "| True positive:             0% |\n",
            "| False positive:            0% |\n",
            "| True negative:       98.2697% |\n",
            "| False negative:      1.73029% |\n",
            "| Accuracy:            98.2697% |\n",
            "| Precision:                 0% |\n",
            "| Recall:                    0% |\n",
            "| F1 Score:                  0% |\n",
            "+-------------------------------+\n",
            "\n",
            "+------- High Stakes Game ------+\n",
            "|          45 instances         |\n",
            "| True positive:             0% |\n",
            "| False positive:            0% |\n",
            "| True negative:       99.8901% |\n",
            "| False negative:      0.10986% |\n",
            "| Accuracy:            99.8901% |\n",
            "| Precision:                 0% |\n",
            "| Recall:                    0% |\n",
            "| F1 Score:                  0% |\n",
            "+-------------------------------+\n",
            "\n",
            "+---------- Historical ---------+\n",
            "|         1501 instances        |\n",
            "| True positive:       66.6667% |\n",
            "| False positive:      33.3333% |\n",
            "| True negative:       94.3925% |\n",
            "| False negative:      5.60748% |\n",
            "| Accuracy:            80.5296% |\n",
            "| Precision:           66.6667% |\n",
            "| Recall:              92.2414% |\n",
            "| F1 Score:             77.396% |\n",
            "+-------------------------------+\n",
            "\n",
            "+------------ Horror -----------+\n",
            "|          511 instances        |\n",
            "| True positive:             0% |\n",
            "| False positive:            0% |\n",
            "| True negative:       97.9676% |\n",
            "| False negative:      2.03241% |\n",
            "| Accuracy:            97.9676% |\n",
            "| Precision:                 0% |\n",
            "| Recall:                    0% |\n",
            "| F1 Score:                  0% |\n",
            "+-------------------------------+\n",
            "\n",
            "+-------- Idols (Female) -------+\n",
            "|          300 instances        |\n",
            "| True positive:             0% |\n",
            "| False positive:            0% |\n",
            "| True negative:       98.5993% |\n",
            "| False negative:      1.40071% |\n",
            "| Accuracy:            98.5993% |\n",
            "| Precision:                 0% |\n",
            "| Recall:                    0% |\n",
            "| F1 Score:                  0% |\n",
            "+-------------------------------+\n",
            "\n",
            "+--------- Idols (Male) --------+\n",
            "|          173 instances        |\n",
            "| True positive:             0% |\n",
            "| False positive:            0% |\n",
            "| True negative:       99.1761% |\n",
            "| False negative:     0.823949% |\n",
            "| Accuracy:            99.1761% |\n",
            "| Precision:                 0% |\n",
            "| Recall:                    0% |\n",
            "| F1 Score:                  0% |\n",
            "+-------------------------------+\n",
            "\n",
            "+------------ Isekai -----------+\n",
            "|          336 instances        |\n",
            "| True positive:             0% |\n",
            "| False positive:          100% |\n",
            "| True negative:        98.681% |\n",
            "| False negative:      1.31904% |\n",
            "| Accuracy:            49.3405% |\n",
            "| Precision:                 0% |\n",
            "| Recall:                    0% |\n",
            "| F1 Score:                  0% |\n",
            "+-------------------------------+\n",
            "\n",
            "+---------- Iyashikei ----------+\n",
            "|          170 instances        |\n",
            "| True positive:             0% |\n",
            "| False positive:            0% |\n",
            "| True negative:       99.4507% |\n",
            "| False negative:       0.5493% |\n",
            "| Accuracy:            99.4507% |\n",
            "| Precision:                 0% |\n",
            "| Recall:                    0% |\n",
            "| F1 Score:                  0% |\n",
            "+-------------------------------+\n",
            "\n",
            "+------------ Josei ------------+\n",
            "|          138 instances        |\n",
            "| True positive:             0% |\n",
            "| False positive:          100% |\n",
            "| True negative:       99.4505% |\n",
            "| False negative:     0.549451% |\n",
            "| Accuracy:            49.7253% |\n",
            "| Precision:                 0% |\n",
            "| Recall:                    0% |\n",
            "| F1 Score:                  0% |\n",
            "+-------------------------------+\n",
            "\n",
            "+------------- Kids ------------+\n",
            "|         6265 instances        |\n",
            "| True positive:        74.141% |\n",
            "| False positive:       25.859% |\n",
            "| True negative:       83.3225% |\n",
            "| False negative:      16.6775% |\n",
            "| Accuracy:            78.7318% |\n",
            "| Precision:            74.141% |\n",
            "| Recall:              81.6365% |\n",
            "| F1 Score:            77.7084% |\n",
            "+-------------------------------+\n",
            "\n",
            "+--------- Love Polygon --------+\n",
            "|          88 instances         |\n",
            "| True positive:             0% |\n",
            "| False positive:            0% |\n",
            "| True negative:        99.588% |\n",
            "| False negative:     0.411975% |\n",
            "| Accuracy:             99.588% |\n",
            "| Precision:                 0% |\n",
            "| Recall:                    0% |\n",
            "| F1 Score:                  0% |\n",
            "+-------------------------------+\n",
            "\n",
            "+------ Magical Sex Shift ------+\n",
            "|          29 instances         |\n",
            "| True positive:             0% |\n",
            "| False positive:            0% |\n",
            "| True negative:       99.8627% |\n",
            "| False negative:     0.137325% |\n",
            "| Accuracy:            99.8627% |\n",
            "| Precision:                 0% |\n",
            "| Recall:                    0% |\n",
            "| F1 Score:                  0% |\n",
            "+-------------------------------+\n",
            "\n",
            "+--------- Mahou Shoujo --------+\n",
            "|          328 instances        |\n",
            "| True positive:            60% |\n",
            "| False positive:           40% |\n",
            "| True negative:       98.4873% |\n",
            "| False negative:      1.51265% |\n",
            "| Accuracy:            79.2437% |\n",
            "| Precision:                60% |\n",
            "| Recall:              97.5409% |\n",
            "| F1 Score:            74.2976% |\n",
            "+-------------------------------+\n",
            "\n",
            "+--------- Martial Arts --------+\n",
            "|          595 instances        |\n",
            "| True positive:             0% |\n",
            "| False positive:            0% |\n",
            "| True negative:       97.7204% |\n",
            "| False negative:      2.27959% |\n",
            "| Accuracy:            97.7204% |\n",
            "| Precision:                 0% |\n",
            "| Recall:                    0% |\n",
            "| F1 Score:                  0% |\n",
            "+-------------------------------+\n",
            "\n",
            "+------------ Mecha ------------+\n",
            "|         1272 instances        |\n",
            "| True positive:           100% |\n",
            "| False positive:            0% |\n",
            "| True negative:        94.912% |\n",
            "| False negative:      5.08801% |\n",
            "| Accuracy:             97.456% |\n",
            "| Precision:               100% |\n",
            "| Recall:              95.1583% |\n",
            "| F1 Score:            97.5191% |\n",
            "+-------------------------------+\n",
            "\n",
            "+----------- Medical -----------+\n",
            "|          44 instances         |\n",
            "| True positive:             0% |\n",
            "| False positive:            0% |\n",
            "| True negative:       99.7803% |\n",
            "| False negative:      0.21972% |\n",
            "| Accuracy:            99.7803% |\n",
            "| Precision:                 0% |\n",
            "| Recall:                    0% |\n",
            "| F1 Score:                  0% |\n",
            "+-------------------------------+\n",
            "\n",
            "+----------- Military ----------+\n",
            "|          701 instances        |\n",
            "| True positive:             0% |\n",
            "| False positive:            0% |\n",
            "| True negative:        97.226% |\n",
            "| False negative:      2.77396% |\n",
            "| Accuracy:             97.226% |\n",
            "| Precision:                 0% |\n",
            "| Recall:                    0% |\n",
            "| F1 Score:                  0% |\n",
            "+-------------------------------+\n",
            "\n",
            "+------------ Music ------------+\n",
            "|         4072 instances        |\n",
            "| True positive:       74.0741% |\n",
            "| False positive:      25.9259% |\n",
            "| True negative:       83.6193% |\n",
            "| False negative:      16.3807% |\n",
            "| Accuracy:            78.8467% |\n",
            "| Precision:           74.0741% |\n",
            "| Recall:              81.8907% |\n",
            "| F1 Score:            77.7865% |\n",
            "+-------------------------------+\n",
            "\n",
            "+----------- Mystery -----------+\n",
            "|          880 instances        |\n",
            "| True positive:             0% |\n",
            "| False positive:            0% |\n",
            "| True negative:       96.4021% |\n",
            "| False negative:      3.59791% |\n",
            "| Accuracy:            96.4021% |\n",
            "| Precision:                 0% |\n",
            "| Recall:                    0% |\n",
            "| F1 Score:                  0% |\n",
            "+-------------------------------+\n",
            "\n",
            "+---------- Mythology ----------+\n",
            "|          569 instances        |\n",
            "| True positive:             0% |\n",
            "| False positive:            0% |\n",
            "| True negative:       97.7753% |\n",
            "| False negative:      2.22466% |\n",
            "| Accuracy:            97.7753% |\n",
            "| Precision:                 0% |\n",
            "| Recall:                    0% |\n",
            "| F1 Score:                  0% |\n",
            "+-------------------------------+\n",
            "\n",
            "+------- Organized Crime -------+\n",
            "|          82 instances         |\n",
            "| True positive:             0% |\n",
            "| False positive:            0% |\n",
            "| True negative:        99.643% |\n",
            "| False negative:     0.357045% |\n",
            "| Accuracy:             99.643% |\n",
            "| Precision:                 0% |\n",
            "| Recall:                    0% |\n",
            "| F1 Score:                  0% |\n",
            "+-------------------------------+\n",
            "\n",
            "+-------- Otaku Culture --------+\n",
            "|          92 instances         |\n",
            "| True positive:             0% |\n",
            "| False positive:            0% |\n",
            "| True negative:        99.643% |\n",
            "| False negative:     0.357045% |\n",
            "| Accuracy:             99.643% |\n",
            "| Precision:                 0% |\n",
            "| Recall:                    0% |\n",
            "| F1 Score:                  0% |\n",
            "+-------------------------------+\n",
            "\n",
            "+------------ Parody -----------+\n",
            "|          738 instances        |\n",
            "| True positive:             0% |\n",
            "| False positive:          100% |\n",
            "| True negative:       96.9505% |\n",
            "| False negative:      3.04945% |\n",
            "| Accuracy:            48.4753% |\n",
            "| Precision:                 0% |\n",
            "| Recall:                    0% |\n",
            "| F1 Score:                  0% |\n",
            "+-------------------------------+\n",
            "\n",
            "+------- Performing Arts -------+\n",
            "|          121 instances        |\n",
            "| True positive:             0% |\n",
            "| False positive:            0% |\n",
            "| True negative:       99.5056% |\n",
            "| False negative:      0.49437% |\n",
            "| Accuracy:            99.5056% |\n",
            "| Precision:                 0% |\n",
            "| Recall:                    0% |\n",
            "| F1 Score:                  0% |\n",
            "+-------------------------------+\n",
            "\n",
            "+------------- Pets ------------+\n",
            "|          102 instances        |\n",
            "| True positive:             0% |\n",
            "| False positive:          100% |\n",
            "| True negative:       99.7253% |\n",
            "| False negative:     0.274725% |\n",
            "| Accuracy:            49.8626% |\n",
            "| Precision:                 0% |\n",
            "| Recall:                    0% |\n",
            "| F1 Score:                  0% |\n",
            "+-------------------------------+\n",
            "\n",
            "+-------- Psychological --------+\n",
            "|          421 instances        |\n",
            "| True positive:             0% |\n",
            "| False positive:            0% |\n",
            "| True negative:       98.4345% |\n",
            "| False negative:       1.5655% |\n",
            "| Accuracy:            98.4345% |\n",
            "| Precision:                 0% |\n",
            "| Recall:                    0% |\n",
            "| F1 Score:                  0% |\n",
            "+-------------------------------+\n",
            "\n",
            "+------------ Racing -----------+\n",
            "|          245 instances        |\n",
            "| True positive:             0% |\n",
            "| False positive:            0% |\n",
            "| True negative:       99.0937% |\n",
            "| False negative:     0.906344% |\n",
            "| Accuracy:            99.0937% |\n",
            "| Precision:                 0% |\n",
            "| Recall:                    0% |\n",
            "| F1 Score:                  0% |\n",
            "+-------------------------------+\n",
            "\n",
            "+-------- Reincarnation --------+\n",
            "|          130 instances        |\n",
            "| True positive:             0% |\n",
            "| False positive:            0% |\n",
            "| True negative:       99.5331% |\n",
            "| False negative:     0.466905% |\n",
            "| Accuracy:            99.5331% |\n",
            "| Precision:                 0% |\n",
            "| Recall:                    0% |\n",
            "| F1 Score:                  0% |\n",
            "+-------------------------------+\n",
            "\n",
            "+-------- Reverse Harem --------+\n",
            "|          76 instances         |\n",
            "| True positive:             0% |\n",
            "| False positive:            0% |\n",
            "| True negative:       99.8352% |\n",
            "| False negative:      0.16479% |\n",
            "| Accuracy:            99.8352% |\n",
            "| Precision:                 0% |\n",
            "| Recall:                    0% |\n",
            "| F1 Score:                  0% |\n",
            "+-------------------------------+\n",
            "\n",
            "+----------- Romance -----------+\n",
            "|         1997 instances        |\n",
            "| True positive:       46.1538% |\n",
            "| False positive:      53.8462% |\n",
            "| True negative:        92.171% |\n",
            "| False negative:      7.82898% |\n",
            "| Accuracy:            69.1624% |\n",
            "| Precision:           46.1538% |\n",
            "| Recall:              85.4973% |\n",
            "| F1 Score:            59.9467% |\n",
            "+-------------------------------+\n",
            "\n",
            "+------- Romantic Subtext ------+\n",
            "|          53 instances         |\n",
            "| True positive:             0% |\n",
            "| False positive:            0% |\n",
            "| True negative:       99.8352% |\n",
            "| False negative:      0.16479% |\n",
            "| Accuracy:            99.8352% |\n",
            "| Precision:                 0% |\n",
            "| Recall:                    0% |\n",
            "| F1 Score:                  0% |\n",
            "+-------------------------------+\n",
            "\n",
            "+----------- Samurai -----------+\n",
            "|          232 instances        |\n",
            "| True positive:             0% |\n",
            "| False positive:            0% |\n",
            "| True negative:       99.0387% |\n",
            "| False negative:     0.961274% |\n",
            "| Accuracy:            99.0387% |\n",
            "| Precision:                 0% |\n",
            "| Recall:                    0% |\n",
            "| F1 Score:                  0% |\n",
            "+-------------------------------+\n",
            "\n",
            "+------------ School -----------+\n",
            "|         1937 instances        |\n",
            "| True positive:            80% |\n",
            "| False positive:           20% |\n",
            "| True negative:       92.2442% |\n",
            "| False negative:      7.75578% |\n",
            "| Accuracy:            86.1221% |\n",
            "| Precision:                80% |\n",
            "| Recall:              91.1621% |\n",
            "| F1 Score:            85.2171% |\n",
            "+-------------------------------+\n",
            "\n",
            "+------------ Sci-Fi -----------+\n",
            "|         3159 instances        |\n",
            "| True positive:             0% |\n",
            "| False positive:            0% |\n",
            "| True negative:        86.652% |\n",
            "| False negative:       13.348% |\n",
            "| Accuracy:             86.652% |\n",
            "| Precision:                 0% |\n",
            "| Recall:                    0% |\n",
            "| F1 Score:                  0% |\n",
            "+-------------------------------+\n",
            "\n",
            "+------------ Seinen -----------+\n",
            "|          990 instances        |\n",
            "| True positive:             0% |\n",
            "| False positive:            0% |\n",
            "| True negative:       96.5943% |\n",
            "| False negative:      3.40566% |\n",
            "| Accuracy:            96.5943% |\n",
            "| Precision:                 0% |\n",
            "| Recall:                    0% |\n",
            "| F1 Score:                  0% |\n",
            "+-------------------------------+\n",
            "\n",
            "+------------ Shoujo -----------+\n",
            "|          480 instances        |\n",
            "| True positive:            25% |\n",
            "| False positive:           75% |\n",
            "| True negative:       97.9884% |\n",
            "| False negative:      2.01157% |\n",
            "| Accuracy:            61.4942% |\n",
            "| Precision:                25% |\n",
            "| Recall:              92.5529% |\n",
            "| F1 Score:            39.3665% |\n",
            "+-------------------------------+\n",
            "\n",
            "+----------- Shounen -----------+\n",
            "|         1921 instances        |\n",
            "| True positive:          62.5% |\n",
            "| False positive:         37.5% |\n",
            "| True negative:        92.926% |\n",
            "| False negative:      7.07404% |\n",
            "| Accuracy:             77.713% |\n",
            "| Precision:              62.5% |\n",
            "| Recall:              89.8324% |\n",
            "| F1 Score:            73.7141% |\n",
            "+-------------------------------+\n",
            "\n",
            "+----------- Showbiz -----------+\n",
            "|          35 instances         |\n",
            "| True positive:             0% |\n",
            "| False positive:            0% |\n",
            "| True negative:       99.8627% |\n",
            "| False negative:     0.137325% |\n",
            "| Accuracy:            99.8627% |\n",
            "| Precision:                 0% |\n",
            "| Recall:                    0% |\n",
            "| F1 Score:                  0% |\n",
            "+-------------------------------+\n",
            "\n",
            "+-------- Slice of Life --------+\n",
            "|         1647 instances        |\n",
            "| True positive:             0% |\n",
            "| False positive:            0% |\n",
            "| True negative:       93.3809% |\n",
            "| False negative:      6.61906% |\n",
            "| Accuracy:            93.3809% |\n",
            "| Precision:                 0% |\n",
            "| Recall:                    0% |\n",
            "| F1 Score:                  0% |\n",
            "+-------------------------------+\n",
            "\n",
            "+------------ Space ------------+\n",
            "|          619 instances        |\n",
            "| True positive:             0% |\n",
            "| False positive:            0% |\n",
            "| True negative:       97.5556% |\n",
            "| False negative:      2.44438% |\n",
            "| Accuracy:            97.5556% |\n",
            "| Precision:                 0% |\n",
            "| Recall:                    0% |\n",
            "| F1 Score:                  0% |\n",
            "+-------------------------------+\n",
            "\n",
            "+------------ Sports -----------+\n",
            "|          770 instances        |\n",
            "| True positive:           100% |\n",
            "| False positive:            0% |\n",
            "| True negative:       97.1688% |\n",
            "| False negative:      2.83123% |\n",
            "| Accuracy:            98.5844% |\n",
            "| Precision:               100% |\n",
            "| Recall:              97.2467% |\n",
            "| F1 Score:            98.6041% |\n",
            "+-------------------------------+\n",
            "\n",
            "+-------- Strategy Game --------+\n",
            "|          323 instances        |\n",
            "| True positive:             0% |\n",
            "| False positive:            0% |\n",
            "| True negative:        98.819% |\n",
            "| False negative:      1.18099% |\n",
            "| Accuracy:             98.819% |\n",
            "| Precision:                 0% |\n",
            "| Recall:                    0% |\n",
            "| F1 Score:                  0% |\n",
            "+-------------------------------+\n",
            "\n",
            "+--------- Super Power ---------+\n",
            "|          693 instances        |\n",
            "| True positive:             0% |\n",
            "| False positive:            0% |\n",
            "| True negative:       97.3634% |\n",
            "| False negative:      2.63664% |\n",
            "| Accuracy:            97.3634% |\n",
            "| Precision:                 0% |\n",
            "| Recall:                    0% |\n",
            "| F1 Score:                  0% |\n",
            "+-------------------------------+\n",
            "\n",
            "+--------- Supernatural --------+\n",
            "|         1421 instances        |\n",
            "| True positive:            50% |\n",
            "| False positive:           50% |\n",
            "| True negative:       94.2841% |\n",
            "| False negative:      5.71586% |\n",
            "| Accuracy:            72.1421% |\n",
            "| Precision:                50% |\n",
            "| Recall:              89.7411% |\n",
            "| F1 Score:            64.2195% |\n",
            "+-------------------------------+\n",
            "\n",
            "+----------- Survival ----------+\n",
            "|          91 instances         |\n",
            "| True positive:             0% |\n",
            "| False positive:            0% |\n",
            "| True negative:       99.6979% |\n",
            "| False negative:     0.302115% |\n",
            "| Accuracy:            99.6979% |\n",
            "| Precision:                 0% |\n",
            "| Recall:                    0% |\n",
            "| F1 Score:                  0% |\n",
            "+-------------------------------+\n",
            "\n",
            "+----------- Suspense ----------+\n",
            "|          261 instances        |\n",
            "| True positive:             0% |\n",
            "| False positive:            0% |\n",
            "| True negative:       99.0113% |\n",
            "| False negative:     0.988739% |\n",
            "| Accuracy:            99.0113% |\n",
            "| Precision:                 0% |\n",
            "| Recall:                    0% |\n",
            "| F1 Score:                  0% |\n",
            "+-------------------------------+\n",
            "\n",
            "+--------- Team Sports ---------+\n",
            "|          307 instances        |\n",
            "| True positive:           100% |\n",
            "| False positive:            0% |\n",
            "| True negative:       98.7634% |\n",
            "| False negative:       1.2366% |\n",
            "| Accuracy:            99.3817% |\n",
            "| Precision:               100% |\n",
            "| Recall:              98.7785% |\n",
            "| F1 Score:            99.3855% |\n",
            "+-------------------------------+\n",
            "\n",
            "+--------- Time Travel ---------+\n",
            "|          142 instances        |\n",
            "| True positive:             0% |\n",
            "| False positive:            0% |\n",
            "| True negative:       99.2035% |\n",
            "| False negative:     0.796484% |\n",
            "| Accuracy:            99.2035% |\n",
            "| Precision:                 0% |\n",
            "| Recall:                    0% |\n",
            "| F1 Score:                  0% |\n",
            "+-------------------------------+\n",
            "\n",
            "+----------- Vampire -----------+\n",
            "|          161 instances        |\n",
            "| True positive:             0% |\n",
            "| False positive:            0% |\n",
            "| True negative:       99.1761% |\n",
            "| False negative:     0.823949% |\n",
            "| Accuracy:            99.1761% |\n",
            "| Precision:                 0% |\n",
            "| Recall:                    0% |\n",
            "| F1 Score:                  0% |\n",
            "+-------------------------------+\n",
            "\n",
            "+---------- Video Game ---------+\n",
            "|          150 instances        |\n",
            "| True positive:             0% |\n",
            "| False positive:            0% |\n",
            "| True negative:       99.5331% |\n",
            "| False negative:     0.466905% |\n",
            "| Accuracy:            99.5331% |\n",
            "| Precision:                 0% |\n",
            "| Recall:                    0% |\n",
            "| F1 Score:                  0% |\n",
            "+-------------------------------+\n",
            "\n",
            "+--------- Visual Arts ---------+\n",
            "|          90 instances         |\n",
            "| True positive:             0% |\n",
            "| False positive:            0% |\n",
            "| True negative:       99.6704% |\n",
            "| False negative:      0.32958% |\n",
            "| Accuracy:            99.6704% |\n",
            "| Precision:                 0% |\n",
            "| Recall:                    0% |\n",
            "| F1 Score:                  0% |\n",
            "+-------------------------------+\n",
            "\n",
            "+---------- Workplace ----------+\n",
            "|          180 instances        |\n",
            "| True positive:             0% |\n",
            "| False positive:            0% |\n",
            "| True negative:       99.2584% |\n",
            "| False negative:     0.741555% |\n",
            "| Accuracy:            99.2584% |\n",
            "| Precision:                 0% |\n",
            "| Recall:                    0% |\n",
            "| F1 Score:                  0% |\n",
            "+-------------------------------+\n",
            "\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'+--------- Team Sports ---------+\\n|          307 instances        |\\n| True positive:           100% |\\n| False positive:            0% |\\n| True negative:       98.7634% |\\n| False negative:       1.2366% |\\n| Accuracy:            99.3817% |\\n| Precision:               100% |\\n| Recall:              98.7785% |\\n| F1 Score:            99.3855% |\\n+-------------------------------+\\n'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 19
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "total_pos = np.sum(acc[:, :2])\n",
        "total_neg = np.sum(acc[:, 2:4])\n",
        "tp = np.sum(acc[:, AccIdx.TP]) / total_pos\n",
        "fp = np.sum(acc[:, AccIdx.FP]) / total_pos\n",
        "tn = np.sum(acc[:, AccIdx.TN]) / total_neg\n",
        "fn = np.sum(acc[:, AccIdx.FN]) / total_neg\n",
        "\n",
        "curr_acc = get_acc(tp, fp, tn, fn)\n",
        "precision, recall, f1 = get_f1_score(tp, fp, tn, fn)\n",
        "print('+' + f' totals '.center(31, '-') + '+')\n",
        "print('|' + f' {len(validationData)} instances'.center(31) + '|')\n",
        "print(f'| True positive:  {(tp*100):12g}% |')\n",
        "print(f'| False positive: {(fp*100):12g}% |')\n",
        "print(f'| True negative:  {(tn*100):12g}% |')\n",
        "print(f'| False negative: {(fn*100):12g}% |')\n",
        "print(f'| Accuracy:       {(curr_acc*100):12g}% |')\n",
        "print(f'| Precision:      {(precision*100):12g}% |')\n",
        "print(f'| Recall:         {(recall*100):12g}% |')\n",
        "print(f'| F1 Score:       {(f1*100):12g}% |')\n",
        "print(f'+{\"\":-^31}+')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "eUD8nuaMM9go",
        "outputId": "1e6e384b-8ee5-44bb-ae35-b8138fbac054"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+------------ totals -----------+\n",
            "|         3641 instances        |\n",
            "| True positive:        63.495% |\n",
            "| False positive:       36.505% |\n",
            "| True negative:       96.5648% |\n",
            "| False negative:      3.43515% |\n",
            "| Accuracy:            80.0299% |\n",
            "| Precision:            63.495% |\n",
            "| Recall:              94.8676% |\n",
            "| F1 Score:            76.0737% |\n",
            "+-------------------------------+\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "pd.DataFrame.from_dict(per_cat_resuts_csv, orient='index').to_csv('per_cat_res.csv')"
      ],
      "metadata": {
        "id": "9q1fB-Em8WrC"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}