zero-shot.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "source": [
    "## ZeroShot learning for SIB-200 dataset by generating text\n",
    "\n",
    "from transformers import BloomForCausalLM\n",
    "from transformers import BloomTokenizerFast\n",
    "import os\n",
    "import pandas as pd\n",
    "import torch\n",
    "import warnings\n",
    "from transformers import XGLMTokenizer, XGLMForCausalLM\n",
    "from sklearn.metrics import f1_score\n",
    "import torch.nn.functional as F\n",
    "\n",
    "\n",
    "model_address = \"bigscience/bloomz-560m\"\n",
    "\n",
    "# Filter out FutureWarning messages\n",
    "warnings.simplefilter(action='ignore', category=FutureWarning)\n",
    "\n",
    "# Assuming data directory contains multiple subdirectories with test.tsv files\n",
    "data_directory = \"sib-200/data/annotated\"\n",
    "\n",
    "if model_address.startswith(\"facebook\"):\n",
    "    # Load ZeroShot learning model and tokenizer\n",
    "    model = XGLMForCausalLM.from_pretrained(model_address)\n",
    "    tokenizer = XGLMTokenizer.from_pretrained(model_address)\n",
    "if model_address.startswith(\"bigscience\"):\n",
    "    # Load ZeroShot learning model and tokenizer\n",
    "    model = BloomForCausalLM.from_pretrained(model_address)\n",
    "    tokenizer = BloomTokenizerFast.from_pretrained(model_address)\n",
    "\n",
    "# Move model to GPU if available\n",
    "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "model.to(device)\n",
    "\n",
    "# List of labels to use for ZeroShot learning\n",
    "list_of_labels = [\"science\", \"travel\", \"politics\", \"sports\", \"health\", \"entertainment\", \"geography\"]\n",
    "\n",
    "# Function to predict category given text\n",
    "def predict_category(text):\n",
    "    l = \"\\n- \".join(list_of_labels)\n",
    "    prompt = f\"SENTENCE:\\n {text} \\n Is this SENTENCE {', '.join(list_of_labels)}? \\nOPTIONS:\\n-{l}\\n-ANSWER: \"\n",
    "    inputs = tokenizer(prompt, return_tensors=\"pt\").to(device)\n",
    "    tokens = tokenizer.encode(prompt, add_special_tokens=True)\n",
    "    result_length = len(tokens) + 10\n",
    "    generated_text = tokenizer.decode(model.generate(inputs[\"input_ids\"],\n",
    "                                                     max_length=result_length,\n",
    "                                                     num_beams=4,\n",
    "                                                     no_repeat_ngram_size=2,\n",
    "                                                     early_stopping=True\n",
    "                                                     )[0])\n",
    "\n",
    "    del inputs\n",
    "\n",
    "    found_labels = 0\n",
    "    found_label = \"\"\n",
    "\n",
    "    for label in list_of_labels:\n",
    "        if label in generated_text[generated_text.find('ANSWER')+3:].lower():\n",
    "            found_labels += 1\n",
    "            found_label = label\n",
    "\n",
    "    if found_labels == 1:\n",
    "        return found_label, generated_text\n",
    "    else:\n",
    "        return \"N/A\", generated_text\n",
    "\n",
    "# Function to predict category given text\n",
    "def predict_category2(prompt, alternatives):\n",
    "    alt_tokens = tokenizer.encode(\" \" + \" \".join(alternatives), add_special_tokens=False)[:-1] # Remove the last token because \"geography\" is 2 tokens!\n",
    "    prompt = prompt + '\\n'\n",
    "    inputs = tokenizer(prompt, return_tensors=\"pt\").to(device)\n",
    "    tokens = tokenizer.encode(prompt, add_special_tokens=True)\n",
    "    result_length = len(tokens) + 10\n",
    "    outputs = model.generate(inputs[\"input_ids\"],\n",
    "                             max_length=result_length,\n",
    "                             output_scores=True,\n",
    "                             return_dict_in_generate=True\n",
    "                             )\n",
    "    scores = outputs.scores[0][0][alt_tokens]\n",
    "    found_label = alternatives[torch.argmax(scores)]\n",
    "    confidence = F.softmax(scores, dim=0)[torch.argmax(scores)].item()\n",
    "    return found_label\n",
    "\n",
    "# Output directory for saving DataFrames\n",
    "output_directory = \"zero-shot/sib-200/\" + model_address[model_address.find('/')+1:] + \"/generate\"\n",
    "\n",
    "# Create the output directory if it doesn't exist\n",
    "os.makedirs(output_directory, exist_ok=True)\n",
    "\n",
    "# Iterate through subdirectories in the data directory\n",
    "for language in os.listdir(data_directory)[:10]:\n",
    "# for language in ['sat_Olck', 'shn_Mymr']:\n",
    "    # Initialize an empty DataFrame to store results\n",
    "    results_df = pd.DataFrame(columns=['text', 'actual_category', 'predicted_category', 'generated_text'])\n",
    "\n",
    "    subdir = os.path.join(data_directory, language)\n",
    "    for file in os.listdir(subdir):\n",
    "        # Check if the file is a test.tsv file\n",
    "        if file.endswith(\"test.tsv\"):\n",
    "            file_path = os.path.join(subdir, file)\n",
    "\n",
    "            # Read the test.tsv file into a DataFrame\n",
    "            df = pd.read_csv(file_path, sep='\\t')\n",
    "\n",
    "            # Iterate through rows and compare predicted category with actual category\n",
    "            for index, row in df.iterrows():\n",
    "                text = row['text']\n",
    "                actual_category = row['category']\n",
    "                \n",
    "                with torch.no_grad():  # Disable gradient calculation\n",
    "                    # Predict category using your ZeroShot learning model\n",
    "                    predicted_category, generated_text = predict_category(text)\n",
    "                    top_next_word = predict_category2(text, list_of_labels)\n",
    "\n",
    "                # Append the results to the DataFrame\n",
    "                results_df = results_df.append({'text': text,\n",
    "                                                'actual_category': actual_category,\n",
    "                                                'predicted_category': 'science/technology' if predicted_category == 'science' else predicted_category,\n",
    "                                                'generated_text': generated_text,\n",
    "                                                'next_token': top_next_word}, ignore_index=True)\n",
    "                torch.cuda.empty_cache()\n",
    "            # Save the results DataFrame to a CSV file in the output directory\n",
    "            # results_file_path = os.path.join(output_directory, f'{language}.csv')\n",
    "            # results_df.to_csv(results_file_path, index=False)\n",
    "            # \n",
    "            # print(f\"Results saved to {results_file_path}\")\n",
    "    print(f\"F1 score of {language} by all sequence is {f1_score(results_df['actual_category'], results_df['predicted_category'], average='macro')}\")\n",
    "    print(f\"F1 score of {language} by next token is {f1_score(results_df['actual_category'], results_df['next_token'], average='macro')}\")"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2024-02-10T17:23:57.331374840Z",
     "start_time": "2024-02-10T17:15:37.264093785Z"
    }
   },
   "id": "89a087fa2a408ae6",
   "execution_count": 4,
   "outputs": []
  },
  {
   "cell_type": "code",
   "source": [
    "# ZeroShot learning for SIB-200 dataset by using top logprobs\n",
    "import os\n",
    "import pandas as pd\n",
    "import torch\n",
    "import torch.nn.functional as F\n",
    "from transformers import XGLMTokenizer, XGLMForCausalLM\n",
    "from transformers import BloomForCausalLM, BloomTokenizerFast\n",
    "import warnings\n",
    "from sklearn.metrics import f1_score\n",
    "\n",
    "model_address = \"bigscience/bloom-560m\"\n",
    "# Filter out FutureWarning messages\n",
    "warnings.simplefilter(action='ignore', category=FutureWarning)\n",
    "\n",
    "# Assuming data directory contains multiple subdirectories with test.tsv files\n",
    "data_directory = \"sib-200/data/annotated\"\n",
    "\n",
    "if model_address.startswith(\"facebook\"):\n",
    "    # Load ZeroShot learning model and tokenizer\n",
    "    model = XGLMForCausalLM.from_pretrained(model_address)\n",
    "    tokenizer = XGLMTokenizer.from_pretrained(model_address)\n",
    "if model_address.startswith(\"bigscience\"):\n",
    "    # Load ZeroShot learning model and tokenizer\n",
    "    model = BloomForCausalLM.from_pretrained(model_address)\n",
    "    tokenizer = BloomTokenizerFast.from_pretrained(model_address)\n",
    "\n",
    "# Move model to GPU if available\n",
    "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "model.to(device)\n",
    "\n",
    "def get_logprobs(prompt):\n",
    "    inputs = tokenizer(prompt, return_tensors=\"pt\").to(device)\n",
    "    input_ids, output_ids = inputs[\"input_ids\"], inputs[\"input_ids\"][:, 1:]\n",
    "    outputs = model(**inputs, labels=input_ids)\n",
    "    logits = outputs.logits\n",
    "    logprobs = torch.gather(F.log_softmax(logits, dim=2), 2, output_ids.unsqueeze(2))\n",
    "    return logprobs\n",
    "\n",
    "\n",
    "def xglm_prediction(prompt, alternatives):\n",
    "    lprobs = [get_logprobs(prompt + \"\\n \" + alt).sum() for alt in alternatives]\n",
    "    return alternatives[lprobs.index(max(lprobs))]\n",
    "\n",
    "# Function to predict category given text\n",
    "def predict_category(prompt, alternatives):\n",
    "    alt_tokens = tokenizer.encode(\" \" + \" \".join(alternatives), add_special_tokens=False)[:-1] # Remove the last token because \"geography\" is 2 tokens!\n",
    "    prompt = prompt + '\\n'\n",
    "    inputs = tokenizer(prompt, return_tensors=\"pt\").to(device)\n",
    "    tokens = tokenizer.encode(prompt, add_special_tokens=True)\n",
    "    result_length = len(tokens) + 10\n",
    "    outputs = model.generate(inputs[\"input_ids\"],\n",
    "                             max_length=result_length,\n",
    "                             output_scores=True,\n",
    "                             return_dict_in_generate=True\n",
    "                             )\n",
    "    scores = outputs.scores[0][0][alt_tokens]\n",
    "    found_label = alternatives[torch.argmax(scores)]\n",
    "    confidence = F.softmax(scores, dim=0)[torch.argmax(scores)].item()\n",
    "    return found_label\n",
    "\n",
    "alternatives = [\"science\", \"travel\", \"politics\", \"sports\", \"health\", \"entertainment\", \"geography\"]\n",
    "\n",
    "# Output directory for saving DataFrames\n",
    "output_directory = \"zero-shot/sib-200/\" + model_address[model_address.find('/')+1:] + \"/top_logprobs\"\n",
    "\n",
    "# Create the output directory if it doesn't exist\n",
    "os.makedirs(output_directory, exist_ok=True)\n",
    "\n",
    "# Iterate through subdirectories in the data directory\n",
    "for language in os.listdir(data_directory)[:10]:\n",
    "    # Initialize an empty DataFrame to store results\n",
    "    results_df = pd.DataFrame(columns=['text', 'actual_category', 'predicted_category'])\n",
    "\n",
    "    subdir = os.path.join(data_directory, language)\n",
    "    for file in os.listdir(subdir):\n",
    "        # Check if the file is a test.tsv file\n",
    "        if file.endswith(\"test.tsv\"):\n",
    "            file_path = os.path.join(subdir, file)\n",
    "\n",
    "            # Read the test.tsv file into a DataFrame\n",
    "            df = pd.read_csv(file_path, sep='\\t')\n",
    "\n",
    "            # Iterate through rows and compare predicted category with actual category\n",
    "            for index, row in df.iterrows():\n",
    "                text = row['text']\n",
    "                actual_category = row['category']\n",
    "\n",
    "                # Predict category using your ZeroShot learning model\n",
    "                with torch.no_grad():  # Disable gradient calculation\n",
    "                    predicted_category = xglm_prediction(text, alternatives)\n",
    "                    top_next_word = predict_category(text, alternatives)\n",
    "\n",
    "                # Append the results to the DataFrame\n",
    "                results_df = results_df.append({'text': text,\n",
    "                                                'actual_category': actual_category,\n",
    "                                                'predicted_category': 'science/technology' if predicted_category == 'science' else predicted_category,\n",
    "                                                'next_token': top_next_word}, ignore_index=True)\n",
    "                torch.cuda.empty_cache()\n",
    "                del text, actual_category, predicted_category\n",
    "            # Save the results DataFrame to a CSV file in the output directory\n",
    "            # results_file_path = os.path.join(output_directory, f'{language}.csv')\n",
    "            # results_df.to_csv(results_file_path, index=False)\n",
    "            # \n",
    "            # print(f\"Results saved to {results_file_path}\")\n",
    "    print(f\"F1 score of {language} by all sequence is {f1_score(results_df['actual_category'], results_df['predicted_category'], average='macro')}\")\n",
    "    print(f\"F1 score of {language} by next token is {f1_score(results_df['actual_category'], results_df['next_token'], average='macro')}\")"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2024-02-10T17:38:28.106484302Z",
     "start_time": "2024-02-10T17:23:57.337812074Z"
    }
   },
   "id": "66b5fce3c8077555",
   "execution_count": 5,
   "outputs": []
  },
  {
   "cell_type": "code",
   "source": [
    "# ZeroShot learning for SIB-200 dataset by using top logprobs\n",
    "import os\n",
    "import pandas as pd\n",
    "import torch\n",
    "import torch.nn.functional as F\n",
    "from transformers import XGLMTokenizer, XGLMForCausalLM\n",
    "from transformers import BloomForCausalLM, BloomTokenizerFast\n",
    "import warnings\n",
    "from sklearn.metrics import f1_score\n",
    "\n",
    "model_address = \"facebook/xglm-564M\"\n",
    "# Filter out FutureWarning messages\n",
    "warnings.simplefilter(action='ignore', category=FutureWarning)\n",
    "\n",
    "# Assuming data directory contains multiple subdirectories with test.tsv files\n",
    "data_directory = \"sib-200/data/annotated\"\n",
    "\n",
    "if model_address.startswith(\"facebook\"):\n",
    "    # Load ZeroShot learning model and tokenizer\n",
    "    model = XGLMForCausalLM.from_pretrained(model_address)\n",
    "    tokenizer = XGLMTokenizer.from_pretrained(model_address)\n",
    "if model_address.startswith(\"bigscience\"):\n",
    "    # Load ZeroShot learning model and tokenizer\n",
    "    model = BloomForCausalLM.from_pretrained(model_address)\n",
    "    tokenizer = BloomTokenizerFast.from_pretrained(model_address)\n",
    "\n",
    "# Move model to GPU if available\n",
    "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "model.to(device)\n",
    "\n",
    "def get_logprobs(prompt):\n",
    "    inputs = tokenizer(prompt, return_tensors=\"pt\").to(device)\n",
    "    input_ids, output_ids = inputs[\"input_ids\"], inputs[\"input_ids\"][:, 1:]\n",
    "    outputs = model(**inputs, labels=input_ids)\n",
    "    logits = outputs.logits\n",
    "    logprobs = torch.gather(F.log_softmax(logits, dim=2), 2, output_ids.unsqueeze(2))\n",
    "    return logprobs\n",
    "\n",
    "\n",
    "def xglm_prediction(prompt, alternatives):\n",
    "    lprobs = [get_logprobs(prompt + \"\\n \" + alt).sum() for alt in alternatives]\n",
    "    return alternatives[lprobs.index(max(lprobs))]\n",
    "\n",
    "# Function to predict category given text\n",
    "def predict_category(prompt, alternatives):\n",
    "    alt_tokens = tokenizer.encode(\" \" + \" \".join(alternatives), add_special_tokens=False)[:-1] # Remove the last token because \"geography\" is 2 tokens!\n",
    "    prompt = prompt + '\\n'\n",
    "    inputs = tokenizer(prompt, return_tensors=\"pt\").to(device)\n",
    "    tokens = tokenizer.encode(prompt, add_special_tokens=True)\n",
    "    result_length = len(tokens) + 10\n",
    "    outputs = model.generate(inputs[\"input_ids\"],\n",
    "                             max_length=result_length,\n",
    "                             output_scores=True,\n",
    "                             return_dict_in_generate=True\n",
    "                             )\n",
    "    scores = outputs.scores[0][0][alt_tokens]\n",
    "    found_label = alternatives[torch.argmax(scores)]\n",
    "    confidence = F.softmax(scores, dim=0)[torch.argmax(scores)].item()\n",
    "    return found_label\n",
    "\n",
    "alternatives = [\"science\", \"travel\", \"politics\", \"sports\", \"health\", \"entertainment\", \"geography\"]\n",
    "\n",
    "# Output directory for saving DataFrames\n",
    "output_directory = \"zero-shot/sib-200/\" + model_address[model_address.find('/')+1:] + \"/top_logprobs\"\n",
    "\n",
    "# Create the output directory if it doesn't exist\n",
    "os.makedirs(output_directory, exist_ok=True)\n",
    "\n",
    "# Iterate through subdirectories in the data directory\n",
    "for language in os.listdir(data_directory)[:10]:\n",
    "    # Initialize an empty DataFrame to store results\n",
    "    results_df = pd.DataFrame(columns=['text', 'actual_category', 'predicted_category'])\n",
    "\n",
    "    subdir = os.path.join(data_directory, language)\n",
    "    for file in os.listdir(subdir):\n",
    "        # Check if the file is a test.tsv file\n",
    "        if file.endswith(\"test.tsv\"):\n",
    "            file_path = os.path.join(subdir, file)\n",
    "\n",
    "            # Read the test.tsv file into a DataFrame\n",
    "            df = pd.read_csv(file_path, sep='\\t')\n",
    "\n",
    "            # Iterate through rows and compare predicted category with actual category\n",
    "            for index, row in df.iterrows():\n",
    "                text = row['text']\n",
    "                actual_category = row['category']\n",
    "\n",
    "                # Predict category using your ZeroShot learning model\n",
    "                with torch.no_grad():  # Disable gradient calculation\n",
    "                    predicted_category = xglm_prediction(text, alternatives)\n",
    "                    top_next_word = predict_category(text, alternatives)\n",
    "\n",
    "                # Append the results to the DataFrame\n",
    "                results_df = results_df.append({'text': text,\n",
    "                                                'actual_category': actual_category,\n",
    "                                                'predicted_category': 'science/technology' if predicted_category == 'science' else predicted_category,\n",
    "                                                'next_token': top_next_word}, ignore_index=True)\n",
    "                torch.cuda.empty_cache()\n",
    "                del text, actual_category, predicted_category\n",
    "            # Save the results DataFrame to a CSV file in the output directory\n",
    "            # results_file_path = os.path.join(output_directory, f'{language}.csv')\n",
    "            # results_df.to_csv(results_file_path, index=False)\n",
    "            # \n",
    "            # print(f\"Results saved to {results_file_path}\")\n",
    "    print(f\"F1 score of {language} by all sequence is {f1_score(results_df['actual_category'], results_df['predicted_category'], average='macro')}\")\n",
    "    print(f\"F1 score of {language} by next token is {f1_score(results_df['actual_category'], results_df['next_token'], average='macro')}\")"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2024-02-10T17:47:40.162226235Z",
     "start_time": "2024-02-10T17:38:28.106411562Z"
    }
   },
   "id": "4811b0df5fa29bbb",
   "execution_count": 6,
   "outputs": []
  },
  {
   "cell_type": "code",
   "source": [
    "import pandas as pd\n",
    "from sklearn.metrics import f1_score\n",
    "\n",
    "# Read the Excel file into a DataFrame\n",
    "df = pd.read_excel(\"SIB-200 languages - ACL.xlsx\")\n",
    "\n",
    "# Calculte F1 score for each language and add it to the DataFrame\n",
    "def calculate_f1(language, model, method):\n",
    "    # Read the CSV file into a DataFrame\n",
    "    results_df = pd.read_csv(f\"zero-shot/sib-200/{model}/{method}/{language}.csv\")\n",
    "\n",
    "    # Replace NaN values with 'N/A'\n",
    "    results_df = results_df.fillna('N/A')\n",
    "\n",
    "    # Calculate the F1 score\n",
    "    f1 = f1_score(results_df['actual_category'], results_df['predicted_category'], average='macro')\n",
    "\n",
    "    # Add the F1 score to the DataFrame\n",
    "    df.loc[df['Folder Name'] == language, f'F1 {model} {method}'] = f1\n",
    "\n",
    "# Iterate through languages and calculate F1 score for each language\n",
    "for language in df['Folder Name']:\n",
    "    calculate_f1(language, \"xglm-564M\", \"top_logprobs\")\n",
    "    calculate_f1(language, \"xglm-1.7B\", \"top_logprobs\")\n",
    "    calculate_f1(language, \"xglm-2.9B\", \"top_logprobs\")\n",
    "    calculate_f1(language, \"xglm-7.5B\", \"top_logprobs\")\n",
    "    calculate_f1(language, \"bloom-560M\", \"top_logprobs\")\n",
    "    calculate_f1(language, \"bloom-1b1\", \"top_logprobs\")\n",
    "    calculate_f1(language, \"bloom-1b7\", \"top_logprobs\")\n",
    "    calculate_f1(language, \"bloom-3b\", \"top_logprobs\")\n",
    "    calculate_f1(language, \"bloom-7b1\", \"top_logprobs\")\n",
    "    calculate_f1(language, \"bloomz-560M\", \"generate\")\n",
    "    calculate_f1(language, \"bloomz-1b1\", \"generate\")\n",
    "    calculate_f1(language, \"bloomz-1b7\", \"generate\")\n",
    "    calculate_f1(language, \"bloomz-3b\", \"generate\")\n",
    "    calculate_f1(language, \"bloomz-7b1\", \"generate\")\n",
    "\n",
    "# Save the updated DataFrame to the Excel file\n",
    "df.to_excel(\"SIB-200 languages - ACL.xlsx\", index=False)"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2024-01-28T17:44:24.276373630Z",
     "start_time": "2024-01-28T17:44:13.556646468Z"
    }
   },
   "id": "7a2b195881c57d00",
   "execution_count": 2,
   "outputs": []
  },
  {
   "cell_type": "code",
   "source": [],
   "metadata": {
    "collapsed": false
   },
   "id": "5b21d6cf483e3847",
   "execution_count": null,
   "outputs": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}