plot_print_results_precision_oncology.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Notebook to plot and print tables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# imports\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from tqdm.notebook import tqdm\n",
    "import os\n",
    "import json\n",
    "import scipy\n",
    "\n",
    "#!pip install pycombat\n",
    "import sys\n",
    "from pycombat import Combat\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import utils as utils\n",
    "import argparse\n",
    "import socket\n",
    "from sklearn.preprocessing import MinMaxScaler\n",
    "from sklearn.model_selection import GroupKFold\n",
    "import tensorflow as tf\n",
    "import models as models\n",
    "import evaluation as evaluation\n",
    "import traceback\n",
    "import paccmann_model as paccmann_model\n",
    "import joblib\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "\n",
    "%pylab inline\n",
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "\n",
    "print(socket.gethostname())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Params"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "save_dir = 'results/'\n",
    "save_prefix = ''\n",
    "source = 'gdsc'\n",
    "targets = ['ccle','pancreas','xenografts','beat_aml']\n",
    "target_number_mapping = {'ccle':[0,10,50,100,500,1000,5000,10000],\n",
    "                         'pancreas':[0,10,50,100,500,10000],\n",
    "                         'beat_aml':[0,10,50,100,500,1000,5000,10000,20000],\n",
    "                         'xenografts':[0,10,50,10000]}\n",
    "\n",
    "flag_normalize_descriptors = True\n",
    "train_mode = 'precision_oncology'\n",
    "use_netpop = 'ensemble'\n",
    "\n",
    "investigate_list = ['pearson_lab']#'pearson_inhib',]#'pearson','pearson_inhib','spearman','spearman_inhib']\n",
    "metric_mapping_dict = {'pearson':'Pearson correlation',\n",
    "                      'pearson_inhib':'Per-inhibitor pearson correlation',\n",
    "                      'spearman':'Spearman correlation',\n",
    "                      'spearman_inhib':'Per-inhibitor spearman correlation',\n",
    "                      'pearson_lab': 'Per-specimen pearson correlation'}\n",
    "\n",
    "model_mapping_dict = {'nn_baseline_scratch' :'Conv NN [scratch]',\n",
    "                      'nn_baseline_pretrain':'Conv NN [pre-train]',\n",
    "                      'nn_paccmann_scratch' : 'PaccMann [scratch]',\n",
    "                      'nn_paccmann_pretrain' :'PaccMann [pre-train]',\n",
    "                      'rf':'Random Forest',\n",
    "                      'tDNN_scratch': 'tDNN [scratch]',\n",
    "                      'tDNN_pretrain': 'tDNN [pre-train]'}\n",
    "\n",
    "\n",
    "model_names = ['nn_baseline_scratch', 'nn_baseline_pretrain', \n",
    "               'nn_paccmann_scratch', 'nn_paccmann_pretrain',\n",
    "               'tDNN_scratch', 'tDNN_pretrain',\n",
    "              # 'rf',\n",
    "              ]\n",
    "\n",
    "investigate_list_small = ['pearson_lab']#['pearson_inhib','pearson_lab']\n",
    "\n",
    "# plot params\n",
    "pval = 0.05 #p-value for ttest\n",
    "decs = 3\n",
    "invers_scale_list = [False]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "for target in targets:\n",
    "    use_samples_list = target_number_mapping[target]\n",
    "    list_use_samples = []\n",
    "    all_result_dicts = []\n",
    "    for use_samples in use_samples_list:\n",
    "        cur_load_path = save_dir +  save_prefix + source + '_' + target + '_' + str(use_samples) +\\\n",
    "                '_' + str(flag_normalize_descriptors) + '_' + str(train_mode) + '_' + str(use_netpop) + '.joblib'\n",
    "        print('cur_load_path: ' + cur_load_path)\n",
    "        if not os.path.exists(cur_load_path):\n",
    "            print('does not exist')\n",
    "            continue\n",
    "        cur_tmp_dict = joblib.load(cur_load_path)\n",
    "        all_result_dicts.append(cur_tmp_dict)\n",
    "        list_use_samples.append(use_samples)\n",
    "    \n",
    "    print('source: ' + str(source))\n",
    "    print('target: ' + str(target))\n",
    "    result_table_dict = dict()\n",
    "    keys = np.arange(len(all_result_dicts))\n",
    "    for j in tqdm(np.arange(len(keys))):\n",
    "        cur_key = list_use_samples[j]\n",
    "        if cur_key not in result_table_dict:\n",
    "            result_table_dict[cur_key] = dict()\n",
    "        cur_num_train_results = all_result_dicts[j]\n",
    "        model_keys = model_mapping_dict.keys()\n",
    "        for model_key in model_keys:\n",
    "            if model_key not in result_table_dict[cur_key]:\n",
    "                result_table_dict[cur_key][model_key] = dict()\n",
    "            if model_key in cur_num_train_results:\n",
    "                cur_model_results = cur_num_train_results[model_key]\n",
    "                min_max_scaler = None\n",
    "                if 'min_max_scaler' in cur_num_train_results:\n",
    "                    min_max_scaler = cur_num_train_results['min_max_scaler']\n",
    "                tmp_res_dict = evaluation.get_metric_dict(cur_model_results['pred_complete'], cur_model_results['gt_complete'],\n",
    "                                            cur_model_results['inhib_data_complete'],cur_model_results['lab_data_complete'],\n",
    "                                            min_max_scaler, invers_scale_list = invers_scale_list,\n",
    "                                            flag_calc_per_lab = True,\n",
    "                                            flag_calc_per_inhib = False)\n",
    "\n",
    "                tmp_res_keys = tmp_res_dict.keys()\n",
    "                for key in tmp_res_keys:\n",
    "                    result_table_dict[cur_key][model_key][key] = tmp_res_dict[key]\n",
    "            \n",
    "    \"\"\"\n",
    "    print('################################################')\n",
    "    print('#')\n",
    "    print('# PRINT TABLE')\n",
    "    print('#')\n",
    "    print('################################################')\n",
    "    for investigate in investigate_list:\n",
    "        num_trains = result_table_dict.keys()\n",
    "        for num_train in num_trains:\n",
    "            cur_str = metric_mapping_dict[investigate]\n",
    "            if num_train is None:\n",
    "                display_number = 'all'\n",
    "            else:\n",
    "                display_number = str(num_train)\n",
    "            cur_str += ' & ' + display_number\n",
    "            cur_results = result_table_dict[num_train]        \n",
    "            result_lists = []\n",
    "            result_means = []\n",
    "            result_strings = []\n",
    "            for model in model_names:\n",
    "                if model not in cur_results:\n",
    "                    continue\n",
    "                model_res = cur_results[model]\n",
    "                cur_list = model_res[investigate + '_list']\n",
    "                result_lists.append(cur_list)\n",
    "                result_means.append(np.nanmean(cur_list))\n",
    "                result_string = str(np.round(np.nanmean(cur_list),decimals = decs))\n",
    "                if 'pretrain' in model and model.replace('pretrain','scratch') in model_names:\n",
    "                    pretrain_list = cur_list\n",
    "                    scratch_list = cur_results[model.replace('pretrain','scratch')][investigate + '_list']\n",
    "                    if np.mean(pretrain_list) > np.mean(scratch_list):\n",
    "                        # perform statistical test to compare pretrain with scratch version\n",
    "                        stats,pvalue = scipy.stats.ttest_ind(pretrain_list,scratch_list)\n",
    "\n",
    "                        if pvalue < pval:\n",
    "                            result_string += '*'\n",
    "                result_strings.append(result_string)\n",
    "            value_arg_sort = np.argsort(result_means)[::-1]\n",
    "            # perform statistical test best with second best model\n",
    "            stats,pvalue = scipy.stats.ttest_ind(result_lists[value_arg_sort[0]],result_lists[value_arg_sort[1]])\n",
    "\n",
    "            if pvalue < pval:\n",
    "                result_strings[value_arg_sort[0]] += '$\\dagger$'\n",
    "            result_strings[value_arg_sort[0]] = '\\\\textbf{' + result_strings[value_arg_sort[0]] + '}'\n",
    "            number_string = ' & '.join(result_strings)\n",
    "            cur_str += ' &' + number_string + '\\\\\\\\'\n",
    "            print(cur_str)\n",
    "    \"\"\"\n",
    "\n",
    "\n",
    "    print('################################################')\n",
    "    print('#')\n",
    "    print('# PRINT TABLE ONLY PEARSON')\n",
    "    print('#')\n",
    "    print('################################################')    \n",
    "\n",
    "    for investigate in investigate_list_small:\n",
    "        print('investigate: ' + str(investigate))\n",
    "        num_trains = result_table_dict.keys()\n",
    "        for num_train in num_trains:\n",
    "            cur_str = ''#metric_mapping_dict[investigate]\n",
    "            if num_train is None:\n",
    "                display_number = 'all'\n",
    "            else:\n",
    "                display_number = str(num_train)\n",
    "            cur_str += display_number #' & ' + display_number\n",
    "            cur_results = result_table_dict[num_train]        \n",
    "            result_lists = []\n",
    "            result_means = []\n",
    "            restult_std_errors = []\n",
    "            result_strings = []\n",
    "            for model in model_names:\n",
    "                if model not in cur_results:\n",
    "                    continue\n",
    "                model_res = cur_results[model]\n",
    "                if len(model_res) > 0:\n",
    "                    cur_list = model_res[investigate + '_list']\n",
    "                    result_lists.append(cur_list)\n",
    "                    result_means.append(np.mean(cur_list))\n",
    "                    restult_std_errors.append(np.std(cur_list) / np.sqrt(len(cur_list)))\n",
    "                    result_string = str(np.round(np.mean(cur_list),decimals = decs)) +\\\n",
    "                                ' $\\pm$ ' + str(np.round(np.std(cur_list) / np.sqrt(len(cur_list)),decimals = decs))\n",
    "                    if 'pretrain' in model and model.replace('pretrain','scratch') in model_names:\n",
    "                        pretrain_list = cur_list\n",
    "                        try:\n",
    "                            scratch_list = cur_results[model.replace('pretrain','scratch')][investigate + '_list']\n",
    "                            if np.mean(pretrain_list) > np.mean(scratch_list):\n",
    "                                # perform statistical test to compare pretrain with scratch version\n",
    "                                stats,pvalue = scipy.stats.ttest_ind(pretrain_list,scratch_list)\n",
    "\n",
    "                                if pvalue < pval:\n",
    "                                    result_string += '*'\n",
    "                        except:\n",
    "                            pass\n",
    "                else:\n",
    "                    result_string = '--'\n",
    "                result_strings.append(result_string)\n",
    "            value_arg_sort = np.argsort(result_means)[::-1]\n",
    "            # perform statistical test best with second best model\n",
    "            stats,pvalue = scipy.stats.ttest_ind(result_lists[value_arg_sort[0]],result_lists[value_arg_sort[1]])\n",
    "\n",
    "            if pvalue < pval:\n",
    "                result_strings[value_arg_sort[0]] += '$\\dagger$'\n",
    "            result_strings[value_arg_sort[0]] = '\\\\textbf{' + result_strings[value_arg_sort[0]] + '}'\n",
    "            number_string = ' & '.join(result_strings)\n",
    "            cur_str += ' &' + number_string + '\\\\\\\\'\n",
    "            print(cur_str)\n",
    "    \n",
    "    \"\"\"\n",
    "    print('################################################')\n",
    "    print('#')\n",
    "    print('# PRINT TABLE ONLY MSE')\n",
    "    print('#')\n",
    "    print('################################################')    \n",
    "\n",
    "    mse_list = ['MSE','MSE_lab','MSE_inhib']\n",
    "    for investigate in mse_list:\n",
    "        print('investigate: ' + str(investigate))\n",
    "        num_trains = result_table_dict.keys()\n",
    "        for num_train in num_trains:\n",
    "            cur_str = ''#metric_mapping_dict[investigate]\n",
    "            if num_train is None:\n",
    "                display_number = 'all'\n",
    "            else:\n",
    "                display_number = str(num_train)\n",
    "            cur_str += display_number #' & ' + display_number\n",
    "            cur_results = result_table_dict[num_train]        \n",
    "            result_lists = []\n",
    "            result_means = []\n",
    "            restult_std_errors = []\n",
    "            result_strings = []\n",
    "            for model in model_names:\n",
    "                if model not in cur_results:\n",
    "                    continue\n",
    "                model_res = cur_results[model]\n",
    "                cur_list = model_res[investigate + '_list']\n",
    "                result_lists.append(cur_list)\n",
    "                result_means.append(np.mean(cur_list))\n",
    "                restult_std_errors.append(np.std(cur_list) / np.sqrt(len(cur_list)))\n",
    "                result_string = str(np.round(np.mean(cur_list),decimals = decs)) +\\\n",
    "                            ' $\\pm$ ' + str(np.round(np.std(cur_list) / np.sqrt(len(cur_list)),decimals = decs))\n",
    "                if 'pretrain' in model and model.replace('pretrain','scratch') in model_names:\n",
    "                    pretrain_list = cur_list\n",
    "                    scratch_list = cur_results[model.replace('pretrain','scratch')][investigate + '_list']\n",
    "                    if np.mean(pretrain_list) > np.mean(scratch_list):\n",
    "                        # perform statistical test to compare pretrain with scratch version\n",
    "                        stats,pvalue = scipy.stats.ttest_ind(pretrain_list,scratch_list)\n",
    "\n",
    "                        if pvalue < pval:\n",
    "                            result_string += '*'\n",
    "                result_strings.append(result_string)\n",
    "            value_arg_sort = np.argsort(result_means)[::-1]\n",
    "            # perform statistical test best with second best model\n",
    "            stats,pvalue = scipy.stats.ttest_ind(result_lists[value_arg_sort[0]],result_lists[value_arg_sort[1]])\n",
    "\n",
    "            if pvalue < pval:\n",
    "                result_strings[value_arg_sort[0]] += '$\\dagger$'\n",
    "            result_strings[value_arg_sort[0]] = '\\\\textbf{' + result_strings[value_arg_sort[0]] + '}'\n",
    "            number_string = ' & '.join(result_strings)\n",
    "            cur_str += ' &' + number_string + '\\\\\\\\'\n",
    "            print(cur_str)\n",
    "    \"\"\"\n",
    "    \n",
    "    print('################################################')\n",
    "    print('#')\n",
    "    print('# PLOT FIGURE')\n",
    "    print('#')\n",
    "    print('################################################')\n",
    "    \n",
    "    use_model_names = ['nn_baseline_scratch',                 \n",
    "                 'nn_paccmann_scratch',\n",
    "                 'tDNN_scratch',\n",
    "                 'nn_baseline_pretrain',\n",
    "                 'nn_paccmann_pretrain',                \n",
    "                 'tDNN_pretrain']\n",
    "    for investigate in investigate_list:\n",
    "        fig_filename = 'plots/' + train_mode + '_'+ target + '_' + investigate + '.pdf'\n",
    "        for model_name in use_model_names:\n",
    "            if model_name not in result_table_dict[list(num_trains)[0]]:\n",
    "                continue\n",
    "            display_name = model_mapping_dict[model_name]\n",
    "            mean_vals = []\n",
    "            std_err_vals = []\n",
    "            key_vals = []\n",
    "            for num_train in num_trains:\n",
    "                cur_vals = result_table_dict[num_train][model_name][investigate + '_list']\n",
    "                mean_vals.append(np.mean(cur_vals))\n",
    "                std_err_vals.append(np.std(cur_vals) / np.sqrt(len(cur_vals)))\n",
    "                display_key = num_train\n",
    "                if display_key == list(num_trains)[-1]:\n",
    "                    display_key = 'all'\n",
    "                key_vals.append(display_key)\n",
    "            plt.errorbar(np.arange(len(num_trains)),mean_vals,std_err_vals,label=display_name)\n",
    "\n",
    "        plt.xticks(np.arange(len(num_trains)),key_vals)\n",
    "        plt.xlabel('Number of used examples for target data set')\n",
    "        plt.ylabel(metric_mapping_dict[investigate]) \n",
    "        plt.legend(ncol=2,loc=4)\n",
    "        plt.savefig(fig_filename)\n",
    "        plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_models = ['tDNN_pretrain','nn_baseline_pretrain',\n",
    "              'nn_paccmann_pretrain','rf']\n",
    "\n",
    "for plot_model in plot_models:\n",
    "    gt_complete = all_result_dicts[-1][plot_model]['gt_complete']\n",
    "    pred_complete = all_result_dicts[-1][plot_model]['pred_complete']\n",
    "    gt_completes = []\n",
    "    pred_completes = []\n",
    "    for i in range(len(gt_complete)):\n",
    "        gt_completes += list(gt_complete[i])\n",
    "        pred_completes += list(pred_complete[i].flatten())\n",
    "    pearson = scipy.stats.pearsonr(pred_completes,gt_completes)[0]\n",
    "    plt.scatter(gt_completes,pred_completes,label=model_mapping_dict[plot_model] + ' [' +\\\n",
    "                str(np.round(pearson,decimals=3))+ ']',alpha=0.5)\n",
    "    plt.legend()\n",
    "    plt.xlabel('GT')\n",
    "    plt.ylabel('Prediction')\n",
    "    plt.show()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}