RL_gender_bias.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "toc_visible": true,
      "authorship_tag": "ABX9TyNvlBTTqVcFSySx1x0Zu6Ek",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/mojcakloos/Kloos_Mojca_RL_gender_bias/blob/src/RL_gender_bias.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Install dependencies, import modules, load data"
      ],
      "metadata": {
        "id": "YeEnNC8DtrS5"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "#install dependencies\n",
        "!pip install tensorflow\n",
        "!pip install transformers\n",
        "!pip install keras-rl2\n",
        "!pip install pytorch_pretrained_bert"
      ],
      "metadata": {
        "id": "7r-YpiuNtyGb"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#import libraries\n",
        "import torch\n",
        "from torch.utils.data import TensorDataset, DataLoader, RandomSampler, SequentialSampler\n",
        "from tensorflow.keras.preprocessing.sequence import pad_sequences\n",
        "from sklearn.model_selection import train_test_split\n",
        "from transformers import BertTokenizer, BertConfig, BertModel\n",
        "from pytorch_pretrained_bert.optimization import BertAdam\n",
        "from transformers import  BertForSequenceClassification\n",
        "from tqdm import tqdm, trange\n",
        "import matplotlib.pyplot as plt\n",
        "from transformers import BertTokenizer\n",
        "from sklearn import preprocessing\n",
        "from nltk import word_tokenize\n",
        "\n",
        "import pandas as pd\n",
        "import json\n",
        "import pickle\n",
        "import math\n",
        "from collections import defaultdict, Counter\n",
        "from scipy.stats import pearsonr\n",
        "import keras\n",
        "import tensorflow as tf\n",
        "import numpy as np\n",
        "import ast\n",
        "import matplotlib.pyplot as plt\n",
        "\n",
        "from keras.models import Model, Sequential\n",
        "from keras.layers import Input, Dense, Conv2D, MaxPooling2D, Flatten, Activation, Embedding\n",
        "from keras.optimizers import Adam, SGD\n",
        "from keras.layers import LSTM\n",
        "import gym\n",
        "from gym import error, spaces\n",
        "from gym import utils\n",
        "from gym.utils import seeding\n",
        "from sklearn.metrics import classification_report, confusion_matrix, accuracy_score\n",
        "\n",
        "from PIL import Image\n",
        "import keras.backend as K\n",
        "from keras.optimizers import Adam\n",
        "from tensorflow.python.keras.backend import set_session\n",
        "from rl.agents.dqn import DQNAgent\n",
        "from rl.policy import LinearAnnealedPolicy, BoltzmannQPolicy, EpsGreedyQPolicy\n",
        "from rl.memory import SequentialMemory\n",
        "from rl.core import Processor\n",
        "\n",
        "\n",
        "# specify GPU device\n",
        "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
        "n_gpu = torch.cuda.device_count()\n",
        "torch.cuda.get_device_name(0)"
      ],
      "metadata": {
        "id": "vIEbmLtOt2QR"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def load_data():\n",
        "\"\"\"\n",
        "Read in dataset, tokenize and pad biographies, transform to input ids for embedding layer\n",
        "save input ids to file\n",
        "encode labels\n",
        "\n",
        "\"\"\"\n",
        "  config = [255710, 194] #number of biographies, maxlength of sequence\n",
        "  train_data = pd.read_pickle(\"/content/drive/MyDrive/thesis/data/train.pickle\")\n",
        "  dev_data = pd.read_pickle(\"/content/drive/MyDrive/thesis/data/dev.pickle\")\n",
        "  test_data = pd.read_pickle(\"/content/drive/MyDrive/thesis/data/test.pickle\")\n",
        "  with open(\"/content/drive/MyDrive/thesis/data/gender_distribution.txt\", \"r\") as infile:\n",
        "      reader = infile.read()\n",
        "      distribution_dict = json.loads(reader)\n",
        "\n",
        "  #set model and tokenizer: Both BERT pretrained\n",
        "  model = BertModel.from_pretrained('bert-base-uncased',output_hidden_states=True)\n",
        "  tokenizer = BertTokenizer.from_pretrained('bert-base-uncased', do_lower_case=True)\n",
        "  le = preprocessing.LabelEncoder()\n",
        "  MAX_LEN = 194\n",
        "\n",
        "  #training set\n",
        "  text_labels = []\n",
        "  bios = []\n",
        "  train_gender_list = []\n",
        "  for bio in train_data:\n",
        "    bios.append(bio[\"hard_text\"])\n",
        "    text_labels.append(bio[\"p\"])\n",
        "    if bio[\"g\"] == \"m\":\n",
        "      train_gender_list.append(0)\n",
        "    else:\n",
        "      train_gender_list.append(1)\n",
        "\n",
        "  le.fit(text_labels)\n",
        "  train_labels = le.transform(text_labels)\n",
        "\n",
        "  sentences = []\n",
        "  for bio in bios:\n",
        "    sentence = \"[CLS] \" + bio + \" [SEP]\"\n",
        "    sentences.append(sentence)\n",
        "\n",
        "\n",
        "  tokenized_texts = []\n",
        "  for i, sentence in enumerate(sentences):\n",
        "    tokenized_texts.append(tokenizer.tokenize(sentence))\n",
        "\n",
        "\n",
        "  # Pad our input tokens -> training dat\n",
        "  train_input_ids = pad_sequences([tokenizer.convert_tokens_to_ids(txt) for txt in tokenized_texts],\n",
        "                                maxlen=MAX_LEN, dtype=\"long\", truncating=\"post\", padding=\"post\")\n",
        "  #   # # Use the BERT tokenizer to convert the tokens to their index numbers in the BERT vocabulary\n",
        "  train_input_ids = [tokenizer.convert_tokens_to_ids(x) for x in tokenized_texts]\n",
        "  train_input_ids = pad_sequences(train_input_ids, maxlen=MAX_LEN, dtype=\"long\", truncating=\"post\", padding=\"post\")\n",
        "\n",
        "  #save input ids to file\n",
        "  np.save('/content/drive/MyDrive/thesis/data/train_input_ids.npy', np.array(train_input_ids, dtype=object), allow_pickle=True)\n",
        "\n",
        "   #development set\n",
        "  dev_text_labels = []\n",
        "  dev_bios = []\n",
        "  dev_gender_list = []\n",
        "  for bio in dev_data:\n",
        "    dev_bios.append(bio[\"hard_text\"])\n",
        "    dev_text_labels.append(bio[\"p\"])\n",
        "    if bio[\"g\"] == \"m\":\n",
        "      dev_gender_list.append(0)\n",
        "    else:\n",
        "      dev_gender_list.append(1)\n",
        "  # le = preprocessing.LabelEncoder()\n",
        "  le.fit(dev_text_labels)\n",
        "  dev_labels = le.transform(dev_text_labels)\n",
        "\n",
        "  dev_sentences = []\n",
        "  for bio in dev_bios:\n",
        "    sentence = \"[CLS] \" + bio + \" [SEP]\"\n",
        "    dev_sentences.append(sentence)\n",
        "\n",
        "  dev_tokenized_texts = []\n",
        "  for i, sentence in enumerate(dev_sentences):\n",
        "    dev_tokenized_texts.append(tokenizer.tokenize(sentence))\n",
        "  # #pad dev data\n",
        "  dev_input_ids = pad_sequences([tokenizer.convert_tokens_to_ids(txt) for txt in dev_tokenized_texts],\n",
        "                            maxlen=MAX_LEN, dtype=\"long\", truncating=\"post\", padding=\"post\")\n",
        "  dev_input_ids = [tokenizer.convert_tokens_to_ids(x) for x in dev_tokenized_texts]\n",
        "  dev_input_ids = pad_sequences(dev_input_ids, maxlen=MAX_LEN, dtype=\"long\", truncating=\"post\", padding=\"post\")\n",
        "\n",
        "  #save to file\n",
        "  np.save('/content/drive/MyDrive/thesis/data/dev_input_ids.npy', np.array(dev_input_ids, dtype=object), allow_pickle=True)\n",
        "\n",
        "\n",
        "  #test set\n",
        "  test_bios = []\n",
        "  test_text_labels = []\n",
        "  test_gender_list = []\n",
        "  for bio in test_data:\n",
        "    test_bios.append(bio[\"hard_text\"])\n",
        "    test_text_labels.append(bio[\"p\"])\n",
        "    if bio[\"g\"] == \"m\":\n",
        "      test_gender_list.append(0)\n",
        "    else:\n",
        "      test_gender_list.append(1)\n",
        "\n",
        "  le = preprocessing.LabelEncoder()\n",
        "  le.fit(test_text_labels)\n",
        "  test_labels = le.transform(test_text_labels)\n",
        "\n",
        "  test_sentences = []\n",
        "  for bio in test_bios:\n",
        "    sentence = \"[CLS] \" + bio + \" [SEP]\"\n",
        "    test_sentences.append(sentence)\n",
        "\n",
        "  test_tokenized_texts = []\n",
        "  for i, sentence in enumerate(test_sentences):\n",
        "    test_tokenized_texts.append(tokenizer.tokenize(sentence))\n",
        "\n",
        "#pad test data\n",
        "  test_input_ids = pad_sequences([tokenizer.convert_tokens_to_ids(txt) for txt in test_tokenized_texts],\n",
        "                            maxlen=MAX_LEN, dtype=\"long\", truncating=\"post\", padding=\"post\")\n",
        "  test_input_ids = [tokenizer.convert_tokens_to_ids(x) for x in test_tokenized_texts]\n",
        "  test_input_ids = pad_sequences(test_input_ids, maxlen=MAX_LEN, dtype=\"long\", truncating=\"post\", padding=\"post\")\n",
        "#save to file\n",
        "  np.save('/content/drive/MyDrive/thesis/data/test_input_ids.npy', np.array(test_input_ids, dtype=object), allow_pickle=True)"
      ],
      "metadata": {
        "id": "-Jv5VKDPt3LN"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# if running for the first time, uncomment the next line to preprocess all data\n",
        "# load_data()"
      ],
      "metadata": {
        "id": "_fIX4TyOufnu"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#if you have saved input ids, load all inputs from file\n",
        "train = pd.read_pickle(\"/content/drive/MyDrive/thesis/data/train.pickle\")\n",
        "dev = pd.read_pickle(\"/content/drive/MyDrive/thesis/data/dev.pickle\")\n",
        "test = pd.read_pickle(\"/content/drive/MyDrive/thesis/data/test.pickle\")\n",
        "all_data = train+dev+test\n",
        "\n",
        "x_train = np.load('/content/drive/MyDrive/thesis/data/train_input_ids.npy', allow_pickle =True)\n",
        "x_dev =np.load('/content/drive/MyDrive/thesis/data/dev_input_ids.npy', allow_pickle =True)\n",
        "x_test =np.load('/content/drive/MyDrive/thesis/data/test_input_ids.npy', allow_pickle =True)\n",
        "\n",
        "with open ('/content/drive/MyDrive/thesis/data/train_labels', 'rb') as fp:\n",
        "    y_train = pickle.load(fp)\n",
        "with open ('/content/drive/MyDrive/thesis/data/dev_labels', 'rb') as fp:\n",
        "    y_dev = pickle.load(fp)\n",
        "with open ('/content/drive/MyDrive/thesis/data/test_labels', 'rb') as fp:\n",
        "    y_test = pickle.load(fp)\n",
        "with open ('/content/drive/MyDrive/thesis/data/gender_list', 'rb') as fp:\n",
        "    gender_list = pickle.load(fp)\n",
        "# with open ('/content/drive/MyDrive/thesis/data/dev_gender_list', 'rb') as fp:\n",
        "#     dev_gender_list = pickle.load(fp)\n",
        "with open ('/content/drive/MyDrive/thesis/data/test_gender_list', 'rb') as fp:\n",
        "    test_gender_list = pickle.load(fp)\n",
        "with open(\"/content/drive/MyDrive/thesis/data/gender_distribution.txt\", \"r\") as infile:\n",
        "    reader = infile.read()\n",
        "    distribution_dict = json.loads(reader)"
      ],
      "metadata": {
        "id": "ssmIcQDwuqA9"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## code for evaluating models"
      ],
      "metadata": {
        "id": "rOA7Ljw9w2pI"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "#code inspired by https://github.com/tue-alga/debias-mean-projection/tree/main/notebooks\n",
        "\n",
        "def load_dictionary(path):\n",
        "\n",
        "    with open(path, \"r\", encoding = \"utf-8\") as f:\n",
        "\n",
        "        lines = f.readlines()\n",
        "\n",
        "    k2v, v2k = {}, {}\n",
        "    for line in lines:\n",
        "\n",
        "        k,v = line.strip().split(\"\\t\")\n",
        "        v = int(v)\n",
        "        k2v[k] = v\n",
        "        v2k[v] = k\n",
        "\n",
        "    return k2v, v2k\n",
        "\n",
        "def count_profs_and_gender(data):\n",
        "  # count number of individuals with a specific gender in every occupation\n",
        "  # returns dictionary\n",
        "    counter = defaultdict(Counter)\n",
        "    for entry in data:\n",
        "        gender, prof = entry[\"g\"], entry[\"p\"]\n",
        "        counter[prof][gender] += 1\n",
        "\n",
        "    return counter\n",
        "\n",
        "def get_prof2fem(all_data):\n",
        "\n",
        "  counter = count_profs_and_gender(train+dev+test)\n",
        "  f,m = 0., 0.\n",
        "  prof2fem = dict()\n",
        "\n",
        "  for k, values in counter.items():\n",
        "      f += values['f']\n",
        "      m += values['m']\n",
        "      prof2fem[k] = values['f']/(values['f'] + values['m'])\n",
        "  return(prof2fem)"
      ],
      "metadata": {
        "id": "SrEOITEauv-1"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "p2i, i2p = load_dictionary('/content/drive/MyDrive/thesis/data/profession2index.txt')\n",
        "prof2fem = get_prof2fem(test)"
      ],
      "metadata": {
        "id": "o5pkqAGluyTP"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#code for evaluation metrics, modified from https://github.com/tue-alga/debias-mean-projection/tree/main/notebooks\n",
        "\n",
        "def get_tpr(y_pred, y_true, p2i, i2p, test_gender_list):\n",
        "    scores = defaultdict(Counter)\n",
        "    prof_count_total = defaultdict(Counter)\n",
        "\n",
        "    for y_hat, y, g in zip(y_pred, y_true, test_gender_list):\n",
        "      if int(y) == y_hat:\n",
        "        scores[(i2p[y])][g] +=1\n",
        "      prof_count_total[(i2p[y])][g] +=1\n",
        "\n",
        "    tpr_dict = defaultdict(dict)\n",
        "    tpr_change = dict()\n",
        "    tpr_ratio = []\n",
        "\n",
        "    for profession, scores_dict in scores.items():\n",
        "\n",
        "        good_m, good_f = scores_dict[0], scores_dict[1]\n",
        "        prof_total_f = prof_count_total[profession][1]\n",
        "        prof_total_m = prof_count_total[profession][0]\n",
        "        tpr_m = (good_m) / prof_total_m\n",
        "        tpr_f = (good_f) / prof_total_f\n",
        "\n",
        "        tpr_dict[profession][0] = tpr_m\n",
        "        tpr_dict[profession][1] = tpr_f\n",
        "        tpr_ratio.append(0)\n",
        "        tpr_change[profession] = tpr_f - tpr_m\n",
        "    return tpr_dict, tpr_change, np.mean(np.abs(tpr_ratio))\n",
        "\n",
        "def similarity_vs_tpr(tpr_dict, title, measure, prof2fem):\n",
        "\n",
        "    professions = list(tpr_dict.keys())\n",
        "\n",
        "    tpr_lst = [tpr_dict[p] for p in professions]\n",
        "    sim_lst = [prof2fem[p] for p in professions]\n",
        "\n",
        "\n",
        "    plt.plot(sim_lst, tpr_lst, marker = \"o\", linestyle = \"none\")\n",
        "    plt.xlabel(\"% women\", fontsize = 13)\n",
        "    plt.ylabel(r'$GAP_{female,y}^{TPR}$', fontsize = 13)\n",
        "    for p in professions:\n",
        "        x,y = prof2fem[p], tpr_dict[p]\n",
        "        plt.annotate(p , (x,y), size = 7, color = \"red\")\n",
        "\n",
        "    plt.ylim(-1,1)\n",
        "    z = np.polyfit(sim_lst, tpr_lst, 1)\n",
        "    p = np.poly1d(z)\n",
        "    plt.plot(sim_lst,p(sim_lst),\"b--\")\n",
        "    print(\"Correlation: {}; p-value: {}\".format(*pearsonr(sim_lst, tpr_lst)))\n",
        "\n",
        "#MK: function that calculates aggregated TPR scores\n",
        "\n",
        "def calc_tpr_gap(tpr_gaps):\n",
        "    n = len(tpr_gaps) # number of occupations\n",
        "    sum_of_squares = sum([gap**2 for gap in tpr_gaps.values()]) # sum of squares of TPR-GAPs\n",
        "    rms = math.sqrt(sum_of_squares/n) # root mean square error\n",
        "    return rms"
      ],
      "metadata": {
        "id": "VG6eyxw8v6sT"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Vanilla classifier without RL"
      ],
      "metadata": {
        "id": "b4s7Wwo4u2ne"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def get_model(input_shape,output):\n",
        "    top_words, max_words = input_shape\n",
        "    model = Sequential()\n",
        "    model.add(Embedding(top_words, 128, input_length=max_words))\n",
        "    model.add(Flatten())\n",
        "    model.add(Dense(250))\n",
        "    model.add(Activation('relu'))\n",
        "    model.add(Dense(output))\n",
        "    model.add(Activation(\"softmax\"))\n",
        "    return model"
      ],
      "metadata": {
        "id": "7QgYH9IZu1Eg"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#define shape of data [number of biographies, max len of biography]\n",
        "in_shape = [255710, 194]\n",
        "num_classes = 28\n",
        "#set seed\n",
        "np.random.seed(50)\n",
        "model = get_model(in_shape, num_classes)\n",
        "optimizer = keras.optimizers.Adam(learning_rate = 0.0001)\n",
        "model.compile(loss=\"sparse_categorical_crossentropy\",optimizer=optimizer, metrics=[\"accuracy\"])"
      ],
      "metadata": {
        "id": "SxdCuI-au8T6"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#train and save the model\n",
        "model.fit(x_train, y_train, epochs = 3, batch_size = 64)\n",
        "model.save('/content/drive/MyDrive/thesis/bl1_5')"
      ],
      "metadata": {
        "id": "csDoV7v_vAOg"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#load pretrained model\n",
        "pretrained_bl = tf.keras.models.load_model('/content/drive/MyDrive/thesis/bl1_1')"
      ],
      "metadata": {
        "id": "3hQRWjVCvQOM"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#predict on test set\n",
        "y_pred = model.predict(x_test)\n",
        "y_pred=np.argmax(y_pred,axis=1)\n",
        "\n",
        "#write predictions to file\n",
        "with open('/content/drive/MyDrive/thesis/results/bl1_pred_1.txt', \"w\") as outfile:\n",
        "  for item in y_pred:\n",
        "    outfile.write(str(item)+\"\\n\")"
      ],
      "metadata": {
        "id": "36UtXiisvleN"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## evaluate"
      ],
      "metadata": {
        "id": "7ql8EGfNvyl2"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "#load predictions\n",
        "pred_bl1 = []\n",
        "with open('/content/drive/MyDrive/thesis/results/bl1_pred_1.txt', \"r\") as infile:\n",
        "  predictions = infile.read()\n",
        "  preds_tolist = predictions.split(\"\\n\")\n",
        "  for pred in preds_tolist:\n",
        "    if pred != \"\":\n",
        "      pred_bl1.append(int(pred))\n",
        "\n",
        "tpr_scores, tpr_gaps, tpr_mean_ratio = get_tpr(pred_bl1, y_test, p2i, i2p, test_gender_list)\n",
        "\n",
        "print(\"Overall TPR Gap:\"+str(calc_tpr_gap(pred_bl1, y_test)))\n",
        "\n",
        "print(\"correlation\")\n",
        "similarity_vs_tpr(tpr_scores, \"TPR scores BL 1\", \"TPR\", prof2fem)\n",
        "\n",
        "print(\"classification report\")\n",
        "print(classification_report(pred_bl1, y_test))\n"
      ],
      "metadata": {
        "id": "QhmJ4VMwvti7"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "#RL with reward for correct classification\n"
      ],
      "metadata": {
        "id": "wYNak-dMxa23"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "#code taken from https://github.com/linenus/DRL-For-imbalanced-Classification\n",
        "#train and save model\n",
        "class ClassifyEnv(gym.Env):\n",
        "    def __init__(self, mode, x_train, y_train, gender_list, distribution_dict):  # mode means training or testing\n",
        "        self.mode = mode\n",
        "        self.Env_data = x_train\n",
        "        self.Answer = y_train\n",
        "        self.id = np.arange(x_train.shape[0])\n",
        "        self.game_len = self.Env_data.shape[0]\n",
        "        self.num_classes = len(set(self.Answer))\n",
        "        self.action_space = spaces.Discrete(self.num_classes)\n",
        "        self.step_ind = 0\n",
        "        self.y_pred = []\n",
        "\n",
        "    def seed(self, seed=None):\n",
        "        self.np_random, seed = seeding.np_random(14)\n",
        "        # print(f\"SEED:{seed}\")\n",
        "        return [seed]\n",
        "\n",
        "    def step(self, a):\n",
        "        self.y_pred.append(a)\n",
        "        y_true_cur = []\n",
        "        info = {}\n",
        "        terminal = False\n",
        "        # MK:give positive reward for correct classification, negative for wrong prediction\n",
        "        if a == int(self.Answer[self.id[self.step_ind]]):\n",
        "          reward = 1\n",
        "        else:\n",
        "          reward = -1\n",
        "\n",
        "        self.step_ind += 1\n",
        "\n",
        "        # #update the info dict\n",
        "        if self.step_ind == self.game_len - 1:\n",
        "            terminal = True\n",
        "        return self.Env_data[self.id[self.step_ind]], reward, terminal, info\n",
        "\n",
        "    def My_metrics(self, y_pre, y_true):\n",
        "        print(classification_report(y_true, y_pre))\n",
        "        return(accuracy_score(y_true, y_pre))\n",
        "\n",
        "\n",
        "    def reset(self):\n",
        "        if self.mode == 'train':\n",
        "            np.random.shuffle(self.id)\n",
        "        self.step_ind = 0\n",
        "        self.y_pred = []\n",
        "        return self.Env_data[self.id[self.step_ind]]\n",
        "\n",
        "num_training_steps = 1022840\n",
        "num_classes = len(set(y_test))\n",
        "mode = 'train'\n",
        "env = ClassifyEnv(mode,x_train, y_train, gender_list, distribution_dict)\n",
        "nb_actions = num_classes\n",
        "training_steps = num_training_steps\n",
        "in_shape = [255710, 100]\n",
        "model = get_rl_model(in_shape, num_classes)\n",
        "\n",
        "\n",
        "\n",
        "INPUT_SHAPE = in_shape\n",
        "\n",
        "\n",
        "class ClassifyProcessor(Processor):\n",
        "    def process_observation(self, observation):\n",
        "        return observation\n",
        "\n",
        "    def process_state_batch(self, batch):\n",
        "        return batch.reshape((-1, INPUT_SHAPE[1]))\n",
        "\n",
        "    def process_reward(self, reward):\n",
        "        return reward\n",
        "\n",
        "\n",
        "memory = SequentialMemory(limit=100000, window_length=1)\n",
        "processor = ClassifyProcessor()\n",
        "policy = LinearAnnealedPolicy(EpsGreedyQPolicy(), attr='eps', value_max=1., value_min=.1, value_test=.05,\n",
        "                              nb_steps=100000)\n",
        "dqn = DQNAgent(model=model, nb_actions=nb_actions, policy=policy, memory=memory,\n",
        "              processor=processor, nb_steps_warmup=50000, gamma=0.5, target_model_update=10000,\n",
        "              train_interval=4, delta_clip=1.)\n",
        "dqn.compile(Adam(learning_rate=.0001), metrics=['accuracy'])\n",
        "\n",
        "dqn.fit(env, nb_steps=training_steps, log_interval=10000, nb_max_episode_steps =255709)\n",
        "\n",
        "#save model to file\n",
        "model.save('/content/drive/MyDrive/thesis/baseline2/bl2_1')"
      ],
      "metadata": {
        "id": "7_oDEYrpxd_R"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#load pretrained model\n",
        "bl2_1 = tf.keras.models.load_model('/content/drive/MyDrive/thesis/baseline2/bl2_1')\n",
        "\n",
        "\n",
        "dqn_bl2_1= DQNAgent(model=bl2_1, nb_actions=nb_actions, policy=policy, memory=memory,\n",
        "               processor=processor, nb_steps_warmup=50000, gamma=0.5, target_model_update=10000,\n",
        "               train_interval=4, delta_clip=1.)\n",
        "pred_bl2 = []\n",
        "#predict on test set\n",
        "for i,bio in enumerate(x_test):\n",
        "  pred = dqn_bl2_1.forward(bio)\n",
        "  pred_bl2.append(pred)\n",
        "\n",
        "#write predictions to file\n",
        "with open('/content/drive/MyDrive/thesis/results/bl2_pred_1.txt', \"w\") as outfile:\n",
        "  for item in pred_bl2:\n",
        "    outfile.write(str(item)+\"\\n\")"
      ],
      "metadata": {
        "id": "I7dzstdNx7iQ"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## evaluate"
      ],
      "metadata": {
        "id": "p0wjZuE40med"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "#load predictions from file\n",
        "pred_bl2 = []\n",
        "with open('/content/drive/MyDrive/thesis/results/bl2_pred_1.txt', \"r\") as infile:\n",
        "  predictions = infile.read()\n",
        "  preds_tolist = predictions.split(\"\\n\")\n",
        "  for pred in preds_tolist:\n",
        "    if pred != \"\":\n",
        "      pred_bl2.append(int(pred))\n",
        "\n",
        "tpr_scores, tpr_gaps, tpr_mean_ratio = get_tpr(pred_bl2, y_test, p2i, i2p, test_gender_list)\n",
        "\n",
        "print(\"Overall TPR Gap:\"+str(calc_tpr_gap(pred_bl2, y_test)))\n",
        "\n",
        "print(\"correlation\")\n",
        "similarity_vs_tpr(tpr_scores, \"TPR scores BL 2\", \"TPR\", prof2fem)\n",
        "\n",
        "print(\"classification report\")\n",
        "print(classification_report(pred_bl2, y_test))"
      ],
      "metadata": {
        "id": "SAtTiISs0l7Y"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Additional experiment: RL with reward for correct classification + scrubbed input"
      ],
      "metadata": {
        "id": "EXm9HTlP2kOw"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def load_data_nogender():\n",
        "\n",
        "  config = [255710, 194] #number of biographies, maxlength of sequence\n",
        "  train_data = pd.read_pickle(\"/content/drive/MyDrive/thesis/data/train.pickle\")\n",
        "  # dev_data = pd.read_pickle(\"/content/drive/MyDrive/thesis/data/dev.pickle\")\n",
        "  test_data = pd.read_pickle(\"/content/drive/MyDrive/thesis/data/test.pickle\")\n",
        "  # with open(\"/content/drive/MyDrive/thesis/data/gender_distribution.txt\", \"r\") as infile:\n",
        "  #     reader = infile.read()\n",
        "  #     distribution_dict = json.loads(reader)\n",
        "\n",
        "  #set tokenizer: Both BERT pretrained\n",
        "  tokenizer = BertTokenizer.from_pretrained('bert-base-uncased', do_lower_case=True)\n",
        "  le = preprocessing.LabelEncoder()\n",
        "  MAX_LEN = 194\n",
        "\n",
        "  #training set\n",
        "  text_labels = []\n",
        "  bios = []\n",
        "  train_gender_list = []\n",
        "  for bio in train_data:\n",
        "    bios.append(bio[\"text_without_gender\"])\n",
        "    text_labels.append(bio[\"p\"])\n",
        "    if bio[\"g\"] == \"m\":\n",
        "      train_gender_list.append(0)\n",
        "    else:\n",
        "      train_gender_list.append(1)\n",
        "\n",
        "  le.fit(text_labels)\n",
        "  train_labels = le.transform(text_labels)\n",
        "\n",
        "  sentences = []\n",
        "  for bio in bios:\n",
        "    sentence = \"[CLS] \" + bio + \" [SEP]\"\n",
        "    sentences.append(sentence)\n",
        "\n",
        "\n",
        "  tokenized_texts = []\n",
        "  for i, sentence in enumerate(sentences):\n",
        "    tokenized_texts.append(tokenizer.tokenize(sentence))\n",
        "\n",
        "\n",
        "  # Pad our input tokens -> training dat\n",
        "  train_input_ids = pad_sequences([tokenizer.convert_tokens_to_ids(txt) for txt in tokenized_texts],\n",
        "                                maxlen=MAX_LEN, dtype=\"long\", truncating=\"post\", padding=\"post\")\n",
        "  #   # # Use the BERT tokenizer to convert the tokens to their index numbers in the BERT vocabulary\n",
        "  train_input_ids = [tokenizer.convert_tokens_to_ids(x) for x in tokenized_texts]\n",
        "  train_input_ids = pad_sequences(train_input_ids, maxlen=MAX_LEN, dtype=\"long\", truncating=\"post\", padding=\"post\")\n",
        "  tokens_tensor = torch.tensor([train_input_ids])\n",
        "\n",
        "  np.save('/content/drive/MyDrive/thesis/data/train_inputs_nogender.npy', np.array(train_input_ids, dtype=object), allow_pickle=True)\n",
        "\n",
        "\n",
        "\n",
        "#   #test set\n",
        "  test_bios = []\n",
        "  test_text_labels = []\n",
        "  test_gender_list = []\n",
        "  for bio in test_data:\n",
        "    test_bios.append(bio[\"text_without_gender\"])\n",
        "    test_text_labels.append(bio[\"p\"])\n",
        "    if bio[\"g\"] == \"m\":\n",
        "      test_gender_list.append(0)\n",
        "    else:\n",
        "      test_gender_list.append(1)\n",
        "\n",
        "  le = preprocessing.LabelEncoder()\n",
        "  le.fit(test_text_labels)\n",
        "  test_labels = le.transform(test_text_labels)\n",
        "\n",
        "  test_sentences = []\n",
        "  for bio in test_bios:\n",
        "    sentence = \"[CLS] \" + bio + \" [SEP]\"\n",
        "    test_sentences.append(sentence)\n",
        "\n",
        "  test_tokenized_texts = []\n",
        "  for i, sentence in enumerate(test_sentences):\n",
        "    test_tokenized_texts.append(tokenizer.tokenize(sentence))\n",
        "\n",
        "#pad test data\n",
        "  test_input_ids = pad_sequences([tokenizer.convert_tokens_to_ids(txt) for txt in test_tokenized_texts],\n",
        "                            maxlen=MAX_LEN, dtype=\"long\", truncating=\"post\", padding=\"post\")\n",
        "  test_input_ids = [tokenizer.convert_tokens_to_ids(x) for x in test_tokenized_texts]\n",
        "  test_input_ids = pad_sequences(test_input_ids, maxlen=MAX_LEN, dtype=\"long\", truncating=\"post\", padding=\"post\")\n",
        "\n",
        "  np.save('/content/drive/MyDrive/thesis/data/test_inputs_nogender.npy', np.array(test_input_ids, dtype=object), allow_pickle=True)"
      ],
      "metadata": {
        "id": "LS7ddZfw2o68"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "load_data_nogender()"
      ],
      "metadata": {
        "id": "ji3imymK21hs"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "x_train_nogender = np.load('/content/drive/MyDrive/thesis/data/train_inputs_nogender.npy', allow_pickle =True)\n",
        "x_test_nogender =np.load('/content/drive/MyDrive/thesis/data/test_inputs_nogender.npy', allow_pickle =True)"
      ],
      "metadata": {
        "id": "cD2HtYwA26Am"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "class ClassifyEnv(gym.Env):\n",
        "    def __init__(self, mode, x_train, y_train, gender_list, distribution_dict):  # mode means training or testing\n",
        "        self.mode = mode\n",
        "        self.Env_data = x_train\n",
        "        self.Answer = y_train\n",
        "        self.id = np.arange(x_train.shape[0])\n",
        "        self.game_len = self.Env_data.shape[0]\n",
        "        self.num_classes = len(set(self.Answer))\n",
        "        self.action_space = spaces.Discrete(self.num_classes)\n",
        "        self.step_ind = 0\n",
        "        self.y_pred = []\n",
        "\n",
        "    def seed(self, seed=None):\n",
        "        self.np_random, seed = seeding.np_random(14)\n",
        "        # print(f\"SEED:{seed}\")\n",
        "        return [seed]\n",
        "\n",
        "    def step(self, a):\n",
        "        self.y_pred.append(a)\n",
        "        y_true_cur = []\n",
        "        info = {}\n",
        "        terminal = False\n",
        "        #give positive reward for correct classification, negative for wrong prediction\n",
        "        if a == int(self.Answer[self.id[self.step_ind]]):\n",
        "          reward = 1\n",
        "        else:\n",
        "          reward = -1\n",
        "          # if self.mode == 'train':\n",
        "          #           terminal = True\n",
        "\n",
        "\n",
        "\n",
        "        self.step_ind += 1\n",
        "        # #update the info dict\n",
        "        if self.step_ind == self.game_len - 1:\n",
        "        #     y_true_cur = self.Answer[self.id]\n",
        "        #     info['gmean'], info['fmeasure'] = self.My_metrics(np.array(self.y_pred),\n",
        "        #                                                       np.array(y_true_cur[:self.step_ind]))\n",
        "            terminal = True\n",
        "        return self.Env_data[self.id[self.step_ind]], reward, terminal, info\n",
        "\n",
        "    def My_metrics(self, y_pre, y_true):\n",
        "        print(classification_report(y_true, y_pre))\n",
        "        return(accuracy_score(y_true, y_pre))\n",
        "\n",
        "    # return: (states, observations)\n",
        "    def reset(self):\n",
        "        if self.mode == 'train':\n",
        "            np.random.shuffle(self.id)\n",
        "        self.step_ind = 0\n",
        "        self.y_pred = []\n",
        "        return self.Env_data[self.id[self.step_ind]]\n",
        "\n",
        "num_training_steps = 1022840\n",
        "num_classes = len(set(y_test))\n",
        "mode = 'train'\n",
        "env = ClassifyEnv(mode,x_train_nogender, y_train, gender_list, distribution_dict)\n",
        "nb_actions = num_classes\n",
        "training_steps = num_training_steps\n",
        "# np.random.seed(14)\n",
        "in_shape = [255710, 194]\n",
        "model = get_rl_model(in_shape, num_classes)\n",
        "\n",
        "\n",
        "\n",
        "INPUT_SHAPE = in_shape\n",
        "\n",
        "\n",
        "class ClassifyProcessor(Processor):\n",
        "    def process_observation(self, observation):\n",
        "        return observation\n",
        "\n",
        "    def process_state_batch(self, batch):\n",
        "        return batch.reshape((-1, INPUT_SHAPE[1]))\n",
        "\n",
        "    def process_reward(self, reward):\n",
        "        return reward\n",
        "\n",
        "\n",
        "memory = SequentialMemory(limit=100000, window_length=1)\n",
        "processor = ClassifyProcessor()\n",
        "policy = LinearAnnealedPolicy(EpsGreedyQPolicy(), attr='eps', value_max=1., value_min=.1, value_test=.05,\n",
        "                              nb_steps=100000)\n",
        "dqn = DQNAgent(model=model, nb_actions=nb_actions, policy=policy, memory=memory,\n",
        "              processor=processor, nb_steps_warmup=50000, gamma=0.5, target_model_update=10000,\n",
        "              train_interval=4, delta_clip=1.)\n",
        "dqn.compile(Adam(learning_rate=.0001), metrics=['accuracy'])\n",
        "\n",
        "dqn.fit(env, nb_steps=training_steps, log_interval=10000, nb_max_episode_steps =255709)\n",
        "model.save(f'/content/drive/MyDrive/thesis/baseline2/bl2_nogender')"
      ],
      "metadata": {
        "id": "fZ5EWAys2-bz"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "bl2_nogender = tf.keras.models.load_model('/content/drive/MyDrive/thesis/baseline2/bl2_nogender')\n",
        "dqn_bl2_nogender= DQNAgent(model=bl2_nogender, nb_actions=nb_actions, policy=policy, memory=memory,\n",
        "               processor=processor, nb_steps_warmup=50000, gamma=0.5, target_model_update=10000,\n",
        "               train_interval=4, delta_clip=1.)\n",
        "\n",
        "pred_m2 = []\n",
        "for i,bio in enumerate(x_test_nogender):\n",
        "  pred = dqn_bl2_nogender.forward(bio)\n",
        "  pred_m2.append(pred)\n",
        "\n",
        "with open('/content/drive/MyDrive/thesis/results/bl2_pred_nogender.txt', \"w\") as outfile:\n",
        "  for item in pred_m2:\n",
        "    outfile.write(str(item)+\"\\n\")"
      ],
      "metadata": {
        "id": "E8OR94h53Be2"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## evaluate additional experiment"
      ],
      "metadata": {
        "id": "xpzAlleV3WJL"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "#load predictions\n",
        "pred_bl2_nogender = []\n",
        "with open('/content/drive/MyDrive/thesis/results/bl2_pred_nogender.txt', \"r\") as infile:\n",
        "  predictions = infile.read()\n",
        "  preds_tolist = predictions.split(\"\\n\")\n",
        "  for pred in preds_tolist:\n",
        "    if pred != \"\":\n",
        "      pred_bl2_nogender.append(int(pred))\n",
        "\n",
        "tpr_scores, tpr_gaps, tpr_mean_ratio = get_tpr(pred_bl2_nogender, y_test, p2i, i2p, test_gender_list)\n",
        "\n",
        "print(\"Overall TPR Gap:\"+str(calc_tpr_gap(pred_bl2_nogender, y_test)))\n",
        "\n",
        "print(\"correlation\")\n",
        "similarity_vs_tpr(tpr_scores, \"TPR scores BL 2\", \"TPR\", prof2fem)\n",
        "\n",
        "print(\"classification report\")\n",
        "print(classification_report(pred_bl2_nogender, y_test))"
      ],
      "metadata": {
        "id": "L2d82WrB3VhV"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# RL with minority-sensitive classification"
      ],
      "metadata": {
        "id": "Nb_CyohP1kbj"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "class ClassifyEnv(gym.Env):\n",
        "    def __init__(self, mode, x_train, y_train, gender_list, distribution_dict):  # mode means training or testing\n",
        "        self.mode = mode\n",
        "        self.Env_data = x_train\n",
        "        self.Answer = y_train\n",
        "        self.id = np.arange(x_train.shape[0])\n",
        "        self.game_len = self.Env_data.shape[0]\n",
        "        self.num_classes = len(set(self.Answer))\n",
        "        self.action_space = spaces.Discrete(self.num_classes)\n",
        "        self.step_ind = 0\n",
        "        self.y_pred = []\n",
        "\n",
        "    def seed(self, seed=None):\n",
        "        self.np_random, seed = seeding.np_random(50)\n",
        "        return [seed]\n",
        "\n",
        "    def step(self, a):\n",
        "        self.y_pred.append(a)\n",
        "        y_true_cur = []\n",
        "        info = {}\n",
        "        terminal = False\n",
        "\n",
        "\n",
        "        if a == int(self.Answer[self.id[self.step_ind]]): #if prediction is correct\n",
        "          for k,v in distribution_dict.items():\n",
        "            if a ==int(k):\n",
        "              if int(gender_list[self.id[self.step_ind]]) == v[\"minority\"]: #if the gender of the current bio is the minority gender\n",
        "                # reward = 1\n",
        "                reward = 1-v[\"perc_minority\"] #reward is equal to 1 minus the percentage of the minority group\n",
        "              else:\n",
        "                reward = 0.5 #if the gender is in the majority group, the reward is 0.5\n",
        "\n",
        "        else: #if the prediction is incorrect\n",
        "          for k,v in distribution_dict.items():\n",
        "            if a == int(k):\n",
        "              if int(gender_list[self.id[self.step_ind]]) == v[\"minority\"]: #if the gender of the current bio is minority gender\n",
        "                reward = -(1-v[\"perc_minority\"]) #agent gets higher punishment\n",
        "                # reward = -1\n",
        "                if self.mode == \"train\":\n",
        "                  terminal = True #stop the episode\n",
        "              else:\n",
        "                reward = -0.5 #agent gets punishment only for wrong prediciton\n",
        "\n",
        "        self.step_ind += 1\n",
        "        #update the info dict\n",
        "        if self.step_ind == self.game_len - 1:\n",
        "            # y_true_cur = self.Answer[self.id]\n",
        "            # info['gmean'], info['fmeasure'] = self.My_metrics(np.array(self.y_pred),\n",
        "            #                                                   np.array(y_true_cur[:self.step_ind]))\n",
        "            terminal = True\n",
        "        return self.Env_data[self.id[self.step_ind]], reward, terminal, info\n",
        "\n",
        "    def My_metrics(self, y_pre, y_true):\n",
        "        print(classification_report(y_true, y_pre))\n",
        "        return(accuracy_score(y_true, y_pre))\n",
        "\n",
        "    # return: (states, observations)\n",
        "    def reset(self):\n",
        "        if self.mode == 'train':\n",
        "            np.random.shuffle(self.id)\n",
        "        self.step_ind = 0\n",
        "        self.y_pred = []\n",
        "        return self.Env_data[self.id[self.step_ind]]"
      ],
      "metadata": {
        "id": "8EO4M_O61o4_"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "num_training_steps = 1022840\n",
        "\n",
        "num_classes = len(set(y_test))\n",
        "mode = 'test'\n",
        "env = ClassifyEnv(mode,x_train, y_train, gender_list, distribution_dict)\n",
        "nb_actions = num_classes\n",
        "training_steps = num_training_steps\n",
        "\n",
        "in_shape = [255710, 194]\n",
        "\n",
        "\n",
        "INPUT_SHAPE = in_shape\n",
        "\n",
        "\n",
        "class ClassifyProcessor(Processor):\n",
        "    def process_observation(self, observation):\n",
        "        return observation\n",
        "\n",
        "    def process_state_batch(self, batch):\n",
        "        return batch.reshape((-1, INPUT_SHAPE[1]))\n",
        "\n",
        "    def process_reward(self, reward):\n",
        "        return reward\n",
        "\n",
        "\n",
        "memory = SequentialMemory(limit=100000, window_length=1)\n",
        "processor = ClassifyProcessor()\n",
        "policy = LinearAnnealedPolicy(EpsGreedyQPolicy(), attr='eps', value_max=1., value_min=.1, value_test=.05,\n",
        "                              nb_steps=100000)\n",
        "dqn = DQNAgent(model=model, nb_actions=nb_actions, policy=policy, memory=memory,\n",
        "               processor=processor, nb_steps_warmup=50000, gamma=0.5, target_model_update=10000,\n",
        "               train_interval=4, delta_clip=1.)\n",
        "dqn.compile(Adam(learning_rate=.0001), metrics=['accuracy'])\n",
        "\n",
        "dqn.fit(env, nb_steps=training_steps, log_interval=10000)\n",
        "\n",
        "#save model to file\n",
        "model.save('/content/drive/MyDrive/thesis/main_rl_1')"
      ],
      "metadata": {
        "id": "-c-HSydV1s3d"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "pred_main_1 = []\n",
        "for bio in x_test:\n",
        "  pred = dqn.forward(bio)\n",
        "  pred_main_1.append(pred)\n",
        "\n",
        "#write predictions to file\n",
        "with open('/content/drive/MyDrive/thesis/results/main_pred_1.txt', \"w\") as outfile:\n",
        "  for item in pred_main_1:\n",
        "    outfile.write(str(item)+\"\\n\")"
      ],
      "metadata": {
        "id": "g52cLeK618I_"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#load predictions\n",
        "pred_main = []\n",
        "with open('/content/drive/MyDrive/thesis/results/main_pred_1.txt', \"r\") as infile:\n",
        "  predictions = infile.read()\n",
        "  preds_tolist = predictions.split(\"\\n\")\n",
        "  for pred in preds_tolist:\n",
        "    if pred != \"\":\n",
        "      pred_main.append(int(pred))\n",
        "\n",
        "tpr_scores, tpr_gaps, tpr_mean_ratio = get_tpr(pred_main, y_test, p2i, i2p, test_gender_list)\n",
        "\n",
        "print(\"Overall TPR Gap:\"+str(calc_tpr_gap(pred_main, y_test)))\n",
        "\n",
        "print(\"correlation\")\n",
        "similarity_vs_tpr(tpr_scores, \"TPR scores Main 1\", \"TPR\", prof2fem)\n",
        "\n",
        "print(\"classification report\")\n",
        "print(classification_report(pred_main, y_test))"
      ],
      "metadata": {
        "id": "QFZZa7Ub2Mcl"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}