SO66103912_NB_priors.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "SO66103912-NB_priors.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "authorship_tag": "ABX9TyPwxXsvvJJsAWpzYSL7FVjD",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/bmreiniger/datascience.stackexchange/blob/master/SO66103912_NB_priors.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "26GOaFxnRsSM"
      },
      "source": [
        "from sklearn.naive_bayes import BernoulliNB, CategoricalNB, ComplementNB\r\n",
        "from sklearn.datasets import fetch_20newsgroups\r\n",
        "from sklearn.metrics import brier_score_loss\r\n",
        "from sklearn.feature_extraction.text import CountVectorizer\r\n",
        "import pandas as pd\r\n",
        "\r\n",
        "X, y = fetch_20newsgroups(return_X_y=True)\r\n",
        "\r\n",
        "X = CountVectorizer(binary=True).fit_transform(X)\r\n",
        "y = y<5"
      ],
      "execution_count": 13,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "DWN11KwK-Ge3",
        "outputId": "5e0796a8-5b06-41af-8b17-7c8e3f51f3ee"
      },
      "source": [
        "y_mean = y.mean()\r\n",
        "y_mean"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.24951387661304578"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 2
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "qkZqk03iR68R",
        "outputId": "7c8a374e-603c-496b-9857-ee528768babe"
      },
      "source": [
        "preds = {}\r\n",
        "for other_prob in [0.1, y_mean, 0.5, 1-y_mean, 2]:\r\n",
        "    model = BernoulliNB(class_prior=[1-y_mean, other_prob])\r\n",
        "    model.fit(X, y)\r\n",
        "    print(model.class_log_prior_)\r\n",
        "    print(brier_score_loss(y, model.predict_proba(X)[:, 1]))\r\n",
        "    preds[other_prob] = model.predict_proba(X)\r\n"
      ],
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[-0.28703412 -2.30258509]\n",
            "0.055614231239509426\n",
            "[-0.28703412 -1.38824075]\n",
            "0.05285587841729709\n",
            "[-0.28703412 -0.69314718]\n",
            "0.05094316624448597\n",
            "[-0.28703412 -0.28703412]\n",
            "0.049901117563745274\n",
            "[-0.28703412  0.69314718]\n",
            "0.04769378858170929\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "kvqLo9LaSAD8",
        "outputId": "824d201c-9f4c-4085-a6a2-bc5507b904ee"
      },
      "source": [
        "preds[0.1], preds[y_mean]"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(array([[1.00000000e+00, 8.88797640e-17],\n",
              "        [3.84402315e-02, 9.61559769e-01],\n",
              "        [9.99930218e-01, 6.97816055e-05],\n",
              "        ...,\n",
              "        [4.43707462e-01, 5.56292538e-01],\n",
              "        [9.54444306e-05, 9.99904556e-01],\n",
              "        [1.00000000e+00, 1.33579256e-13]]),\n",
              " array([[1.00000000e+00, 2.21767345e-16],\n",
              "        [1.57692821e-02, 9.84230718e-01],\n",
              "        [9.99825903e-01, 1.74096625e-04],\n",
              "        ...,\n",
              "        [2.42233496e-01, 7.57766504e-01],\n",
              "        [3.82543412e-05, 9.99961746e-01],\n",
              "        [1.00000000e+00, 3.33298780e-13]]))"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 4
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "AFEdHbVWVNel",
        "outputId": "7cf64e30-dd21-4246-a8ff-ba49a5f31c36"
      },
      "source": [
        "model = BernoulliNB(class_prior=[(1-y_mean)/10, y_mean/10])\r\n",
        "model.fit(X, y)\r\n",
        "print(model.class_log_prior_)\r\n",
        "print(brier_score_loss(y, model.predict_proba(X)[:, 1]))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[-2.58961921 -3.69082584]\n",
            "0.05285587841729718\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "KFBCIJp7XK81",
        "outputId": "7816400d-599b-4fd7-e60e-c7cc80d8a4de"
      },
      "source": [
        "model._joint_log_likelihood(X)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([[-399.88626177, -435.93116461],\n",
              "       [-476.000472  , -471.86667554],\n",
              "       [-764.94857873, -773.60430472],\n",
              "       ...,\n",
              "       [-411.61884458, -410.47837141],\n",
              "       [-558.23542283, -548.06420758],\n",
              "       [-410.91301252, -439.64274959]])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 6
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "vOgLChthy2yh",
        "outputId": "2182b1cb-6d1c-445c-a505-caa0560b0b9e"
      },
      "source": [
        "from scipy.special import logsumexp\r\n",
        "jll = model._joint_log_likelihood(X)\r\n",
        "log_prob_x = logsumexp(jll, axis=1)\r\n",
        "print(log_prob_x)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[-399.88626177 -471.8507806  -764.94840462 ... -410.20099143 -548.06416932\n",
            " -410.91301252]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "AZYys6jaYQXF",
        "outputId": "18a3dff1-b128-4373-b1d2-7f99f7c25b96"
      },
      "source": [
        "model = BernoulliNB(class_prior=[1-y_mean, y_mean])\r\n",
        "model.fit(X, y)\r\n",
        "print(model.class_log_prior_)\r\n",
        "model._joint_log_likelihood(X)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[-0.28703412 -1.38824075]\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([[-397.58367668, -433.62857952],\n",
              "       [-473.69788691, -469.56409045],\n",
              "       [-762.64599364, -771.30171963],\n",
              "       ...,\n",
              "       [-409.31625949, -408.17578632],\n",
              "       [-555.93283774, -545.76162248],\n",
              "       [-408.61042742, -437.34016449]])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PIMScMQEYbxF"
      },
      "source": [
        "import numpy as np\r\n",
        "sc_models = [\r\n",
        "    BernoulliNB(class_prior=[(1-y_mean)/10, y_mean/10]),\r\n",
        "    BernoulliNB(class_prior=[1-y_mean, y_mean]),\r\n",
        "    BernoulliNB(),\r\n",
        "]\r\n",
        "\r\n",
        "preds = []\r\n",
        "for model in sc_models:\r\n",
        "    model.fit(X, y)\r\n",
        "    preds.append(model.predict_proba(X)[:, 1])\r\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "1psxfH8J2LYu",
        "outputId": "985fb25f-de65-4d2c-e641-ea4c3f87e66d"
      },
      "source": [
        "np.allclose(preds[0], preds[1]), np.allclose(preds[0], preds[2])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(True, True)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 12
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2dzwAxja2PCU"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}