evaluate_sensitivity_core_features.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "os.environ['NOTEBOOK_MODE'] = '1'\n",
    "\n",
    "import pandas as pd\n",
    "\n",
    "from collections import defaultdict\n",
    "from utils import *\n",
    "from inspection_utils import *\n",
    "\n",
    "from dataset_paths import IMAGENET_PATH, SALIENT_IMAGENET_PATH"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "noise_stds_list = [0.25]\n",
    "model_name_list = ['resnet18', 'resnet34', 'resnet50', 'vgg19_bn', 'inception_v3', 'mnasnet_100', \n",
    "                   'wide_resnet50_2', 'efficientnet_b0', 'efficientnet_b4', 'tf_efficientnet_b7',\n",
    "                   'clip_vit_b16', 'clip_vit_b32']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "csv_path = 'mturk_results/discover_spurious_features.csv'\n",
    "mturk_results_discover = MTurk_Results(csv_path)\n",
    "\n",
    "core_features_dict = mturk_results_discover.core_features_dict\n",
    "spurious_features_dict = mturk_results_discover.spurious_features_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "batch_size = 32\n",
    "num_decimal = 3\n",
    "\n",
    "failure_analysis_instance = NoiseSensitivityAnalysis(IMAGENET_PATH,\n",
    "                                                     SALIENT_IMAGENET_PATH, \n",
    "                                                     batch_size=batch_size)\n",
    "\n",
    "class_df = pd.read_csv('class_metadata.csv')\n",
    "class_indices = class_df['class_index'].tolist()\n",
    "synsets = class_df['synset'].tolist()\n",
    "class_names = dict(zip(class_indices, synsets))\n",
    "\n",
    "inspection_results_root = 'evaluation_results'\n",
    "os.makedirs(inspection_results_root, exist_ok=True)\n",
    "inspection_results_path = os.path.join(inspection_results_root, 'noise_in_core_features')\n",
    "os.makedirs(inspection_results_path, exist_ok=True)\n",
    "\n",
    "for model_name in model_name_list:\n",
    "    inspection_model, _ = load_inspection_model(model_name)\n",
    "    print_with_stars(\" Model name: {:s} \".format(model_name))\n",
    "    model_results_path = os.path.join(inspection_results_path, model_name)\n",
    "    os.makedirs(model_results_path, exist_ok=True)\n",
    "\n",
    "    \n",
    "    for noise_std in noise_stds_list:\n",
    "        inspection_results_dict = defaultdict(list)\n",
    "        for class_index in range(1000):\n",
    "            full_class_name = class_names[class_index]\n",
    "            \n",
    "            core_features = core_features_dict[class_index]\n",
    "            if len(core_features) > 0:\n",
    "                acc_tuple = failure_analysis_instance.compute_noisy_acc(inspection_model, \n",
    "                                                                        class_index, core_features, \n",
    "                                                                        noise_std=noise_std)\n",
    "                clean_acc, spurious_acc = acc_tuple\n",
    "\n",
    "                inspection_results_dict['class_index'].append(class_index)\n",
    "                inspection_results_dict['class_name'].append(full_class_name)\n",
    "\n",
    "                inspection_results_dict['clean_accuracy'].append(\n",
    "                    round(100 * clean_acc, num_decimal))\n",
    "                inspection_results_dict['spurious_accuracy'].append(\n",
    "                    round(100 * spurious_acc, num_decimal))\n",
    "\n",
    "                inspection_results_df = pd.DataFrame.from_dict(inspection_results_dict)\n",
    "                curr_results_path = os.path.join(model_results_path, 'std_' + str(noise_std) + '.csv')\n",
    "                inspection_results_df.to_csv(curr_results_path, index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}