03classifier_training_data_creation.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print('kernel is live')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import os\n",
    "import json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "pd.options.display.max_colwidth = 400"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_path = '/data_volume/antivax/'\n",
    "tweets = pd.read_parquet(data_path+'all_geotagged_tweets.parquet') #nonpublic data\n",
    "df = tweets[tweets['retweeted_tweet_id']==-1].drop_duplicates(subset=['text']).reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 6357813 entries, 0 to 17464208\n",
      "Data columns (total 8 columns):\n",
      "tweet_id              int64\n",
      "text                  object\n",
      "tweet_time            datetime64[ns]\n",
      "user_id               int64\n",
      "lang                  object\n",
      "retweeted_tweet_id    int64\n",
      "FIPS                  int64\n",
      "is_antivax            int64\n",
      "dtypes: datetime64[ns](1), int64(5), object(2)\n",
      "memory usage: 436.6+ MB\n"
     ]
    }
   ],
   "source": [
    "df.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "50000"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df2 = df.sample(50000).drop_duplicates(subset='text').reset_index()[['text']]\n",
    "len(df2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "df3=df2.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "df2 = df3[45000:50000] # select 5000 tweets to manually label"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "# dump to jsonl file\n",
    "json_str = ''\n",
    "for t in df2.to_dict('records'):\n",
    "    json_str+=json.dumps(t)+'\\n'\n",
    "json_str = json_str[:-1]\n",
    "\n",
    "with open('/data_volume/antivax/random_sample_5000j.jsonl', 'w+') as f:\n",
    "    f.write(json_str)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# use a keyword search to look for tweets that are more likely to be antivax related, to get a more balanced training dataset. \n",
    "# these are NOT used for test data. test data is a pure random sample of all_geotagged_tweets.parquet, also labeled using Prodigy.\n",
    "antivax_keywords = [\n",
    "                    'experiment', \n",
    "                    'palsy','plandemic','survival rate', 'unprov', 'against humanity', \n",
    "                    'not take', 'msm', 'hoax', 'rush', 'poison', 'injur', 'untest', \n",
    "                    'chemical', 'not a vaccine', 'fertil','miscar','autism', 'not get',\n",
    "                    \"health defense\",'nuremberg',\n",
    "                    'guinea','depopulat', 'censor','bioweapon', '\"vaccine\"',\"'vaccine'\",\n",
    "                    'gene', 'vaer'\n",
    "]\n",
    "\n",
    "df3 = df2[df2['text'].apply(lambda x: any([(k in x.lower()) for k in antivax_keywords]))].reset_index(drop=True)\n",
    "print(len(df3))\n",
    "\n",
    "# dump to jsonl file\n",
    "json_str = ''\n",
    "for t in df3.sample(frac=1,replace=False).to_dict('records'):\n",
    "    json_str+=json.dumps(t)+'\\n'\n",
    "json_str = json_str[:-1]\n",
    "\n",
    "with open('/data_volume/antivax/contain_antivax_keyword.jsonl', 'w+') as f:\n",
    "    f.write(json_str)\n",
    "    \n",
    "df3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Conduct manual labeling using Prodigy\n",
    "\n",
    "# label using:\n",
    "# python3 -m prodigy textcat.manual antivax /data_volume/antivax/tweets_to_label.jsonl --label Antivax\n",
    "\n",
    "#then:\n",
    "# python3 -m prodigy db-out antivax >> /data_volume/home/jmbollen/antivax_paper/prodigy_labels.jsonl\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Clean up the Prodigy output to a usable csv\n",
    "\n",
    "# load jsonl\n",
    "labels = list()\n",
    "with open('/data_volume/home/jmbollen/antivax_paper/prodigy_labels.jsonl','r') as f:\n",
    "    for line in f.readlines():\n",
    "        labels.append(json.loads(line))\n",
    "# labels = labels[:2400]\n",
    "with open('/data_volume/antivax/random_train_labeled.jsonl','r') as f:\n",
    "    for line in f.readlines():\n",
    "        labels.append(json.loads(line))\n",
    "        \n",
    "#convert to df\n",
    "labels = pd.DataFrame(labels)\n",
    "labels = labels[['text','answer']]\n",
    "labels = labels[labels['answer']!='ignore'].reset_index(drop=True) #drop ignored tweets\n",
    "labels['answer'] = labels['answer'].map({'accept':True,'reject':False})\n",
    "labels = labels.rename(columns ={'answer':'is_antivax'})\n",
    "labels = labels.drop_duplicates().dropna().reset_index(drop=True)\n",
    "labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "labels['is_antivax'].sum()/len(labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "labels.to_csv('./labeled_data/antivax_labels.csv',index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# rebalance labeled data by word frequency\n",
    "\n",
    "some training data was selected by keywords associated with antivax content. this biased the training data.  this rebalancing attempts to correct for that (and does so quite successfully, judging by the improvement in classifier results on the unbiased test data). \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# try to correct for oversampling of certain words introduced by the keyword search above.\n",
    "\n",
    "print('live')\n",
    "\n",
    "import pandas as pd\n",
    "import itertools\n",
    "from sklearn.feature_extraction.text import ENGLISH_STOP_WORDS as stopwords\n",
    "\n",
    "df = pd.read_csv('./labeled_data/antivax_labels.csv')\n",
    "df2 = pd.read_parquet('/data_volume/antivax/'+'all_geotagged_tweets.parquet')\n",
    "\n",
    "df2 = df2.sample(50000)\n",
    "\n",
    "punct = \"\"\"!\"#$%&'()*+, -./:;<=>?@[\\]^_`{|}~\"\"\"\n",
    "def flatten_list_of_lists(l):\n",
    "    return list(itertools.chain(*l))\n",
    "\n",
    "def find_oversampled_words(df, baseline_words):\n",
    "    df['words'] = df['text'].apply(lambda x: [i.strip(punct) for i in x.lower().split() if (i not in stopwords and len(i)>2)])\n",
    "    train_words = pd.Series(flatten_list_of_lists(list(df['words']))).value_counts()\n",
    "    train_words = train_words[train_words>5]\n",
    "    \n",
    "    words = list(set(train_words.index).intersection(set(baseline_words.index)))\n",
    "    baseline_words = baseline_words[words]\n",
    "    baseline_words = (baseline_words/baseline_words.sum()).sort_values(ascending=False)\n",
    "    train_words = train_words[words]\n",
    "    train_words = (train_words/train_words.sum()).sort_values(ascending=False)\n",
    "    \n",
    "    dif = (train_words-baseline_words)/baseline_words**(1/2)\n",
    "    oversampled_words = set(dif[dif>0.02].index).difference(set([\"i'm\", \"doesn't\", \"san\",\"antonio\"]))\n",
    "\n",
    "    return oversampled_words\n",
    "\n",
    "df2['words'] = df2['text'].apply(lambda x: [i.strip(punct) for i in x.lower().split() if (i not in stopwords and len(i)>2)])\n",
    "baseline_words = pd.Series(flatten_list_of_lists(list(df2['words']))).value_counts()\n",
    "baseline_words = baseline_words[baseline_words>5]\n",
    "\n",
    "oversampled_words = find_oversampled_words(df, baseline_words)\n",
    "\n",
    "while len(oversampled_words)>0:\n",
    "    df['contains_oversampled'] = df['words'].apply(lambda words: any([(word in oversampled_words) for word in words]))\n",
    "    df = df.drop(df[df['contains_oversampled']].sample(1).index)\n",
    "    oversampled_words = find_oversampled_words(df, baseline_words)\n",
    "\n",
    "df.to_parquet('./labeled_data/antivax_labels_rebalanced.parquet')\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}