3.build-hetnets/process_xCell.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Process xCell cell-type genesets\n",
    "\n",
    "The genesets were extracted from additional file 3 in [Aran et al. 2017](https://doi.org/10.1186/s13059-017-1349-1 \"xCell: digitally portraying the tissue cellular heterogeneity landscape\"). The data consist of 489 gene signatures. Many of these signatures represent redundant cell-types with different evidence."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "from datetime import datetime\n",
    "import csv\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(489, 201)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th># of genes</th>\n",
       "      <th>Unnamed: 2</th>\n",
       "      <th>Unnamed: 3</th>\n",
       "      <th>Unnamed: 4</th>\n",
       "      <th>Unnamed: 5</th>\n",
       "      <th>Unnamed: 6</th>\n",
       "      <th>Unnamed: 7</th>\n",
       "      <th>Unnamed: 8</th>\n",
       "      <th>Unnamed: 9</th>\n",
       "      <th>Unnamed: 10</th>\n",
       "      <th>...</th>\n",
       "      <th>Unnamed: 192</th>\n",
       "      <th>Unnamed: 193</th>\n",
       "      <th>Unnamed: 194</th>\n",
       "      <th>Unnamed: 195</th>\n",
       "      <th>Unnamed: 196</th>\n",
       "      <th>Unnamed: 197</th>\n",
       "      <th>Unnamed: 198</th>\n",
       "      <th>Unnamed: 199</th>\n",
       "      <th>Unnamed: 200</th>\n",
       "      <th>Unnamed: 201</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Celltype_Source_ID</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>aDC_HPCA_1</th>\n",
       "      <td>8</td>\n",
       "      <td>C1QA</td>\n",
       "      <td>C1QB</td>\n",
       "      <td>CD80</td>\n",
       "      <td>IL12B</td>\n",
       "      <td>CCL13</td>\n",
       "      <td>CCL17</td>\n",
       "      <td>CCL19</td>\n",
       "      <td>CCL22</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>aDC_HPCA_2</th>\n",
       "      <td>10</td>\n",
       "      <td>C1QA</td>\n",
       "      <td>C1QB</td>\n",
       "      <td>CD80</td>\n",
       "      <td>FPR3</td>\n",
       "      <td>HLA-DQA1</td>\n",
       "      <td>IL12B</td>\n",
       "      <td>CCL13</td>\n",
       "      <td>CCL17</td>\n",
       "      <td>CCL19</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>aDC_HPCA_3</th>\n",
       "      <td>10</td>\n",
       "      <td>C1QA</td>\n",
       "      <td>C1QB</td>\n",
       "      <td>CD80</td>\n",
       "      <td>FPR3</td>\n",
       "      <td>HLA-DQA1</td>\n",
       "      <td>IL12B</td>\n",
       "      <td>CCL13</td>\n",
       "      <td>CCL17</td>\n",
       "      <td>CCL19</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>aDC_IRIS_1</th>\n",
       "      <td>19</td>\n",
       "      <td>CD80</td>\n",
       "      <td>IL3RA</td>\n",
       "      <td>IL12B</td>\n",
       "      <td>CXCL9</td>\n",
       "      <td>PTGIR</td>\n",
       "      <td>CCL8</td>\n",
       "      <td>CCL13</td>\n",
       "      <td>CCL17</td>\n",
       "      <td>CCL19</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>aDC_IRIS_2</th>\n",
       "      <td>173</td>\n",
       "      <td>ACHE</td>\n",
       "      <td>ADPRH</td>\n",
       "      <td>ALOX15B</td>\n",
       "      <td>ANXA5</td>\n",
       "      <td>XIAP</td>\n",
       "      <td>ARF3</td>\n",
       "      <td>RHOG</td>\n",
       "      <td>ATP1B3</td>\n",
       "      <td>BLVRA</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 201 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                    # of genes Unnamed: 2 Unnamed: 3 Unnamed: 4 Unnamed: 5  \\\n",
       "Celltype_Source_ID                                                           \n",
       "aDC_HPCA_1                   8       C1QA       C1QB       CD80      IL12B   \n",
       "aDC_HPCA_2                  10       C1QA       C1QB       CD80       FPR3   \n",
       "aDC_HPCA_3                  10       C1QA       C1QB       CD80       FPR3   \n",
       "aDC_IRIS_1                  19       CD80      IL3RA      IL12B      CXCL9   \n",
       "aDC_IRIS_2                 173       ACHE      ADPRH    ALOX15B      ANXA5   \n",
       "\n",
       "                   Unnamed: 6 Unnamed: 7 Unnamed: 8 Unnamed: 9 Unnamed: 10  \\\n",
       "Celltype_Source_ID                                                           \n",
       "aDC_HPCA_1              CCL13      CCL17      CCL19      CCL22         NaN   \n",
       "aDC_HPCA_2           HLA-DQA1      IL12B      CCL13      CCL17       CCL19   \n",
       "aDC_HPCA_3           HLA-DQA1      IL12B      CCL13      CCL17       CCL19   \n",
       "aDC_IRIS_1              PTGIR       CCL8      CCL13      CCL17       CCL19   \n",
       "aDC_IRIS_2               XIAP       ARF3       RHOG     ATP1B3       BLVRA   \n",
       "\n",
       "                       ...      Unnamed: 192 Unnamed: 193 Unnamed: 194  \\\n",
       "Celltype_Source_ID     ...                                               \n",
       "aDC_HPCA_1             ...               NaN          NaN          NaN   \n",
       "aDC_HPCA_2             ...               NaN          NaN          NaN   \n",
       "aDC_HPCA_3             ...               NaN          NaN          NaN   \n",
       "aDC_IRIS_1             ...               NaN          NaN          NaN   \n",
       "aDC_IRIS_2             ...               NaN          NaN          NaN   \n",
       "\n",
       "                   Unnamed: 195 Unnamed: 196 Unnamed: 197 Unnamed: 198  \\\n",
       "Celltype_Source_ID                                                       \n",
       "aDC_HPCA_1                  NaN          NaN          NaN          NaN   \n",
       "aDC_HPCA_2                  NaN          NaN          NaN          NaN   \n",
       "aDC_HPCA_3                  NaN          NaN          NaN          NaN   \n",
       "aDC_IRIS_1                  NaN          NaN          NaN          NaN   \n",
       "aDC_IRIS_2                  NaN          NaN          NaN          NaN   \n",
       "\n",
       "                   Unnamed: 199 Unnamed: 200 Unnamed: 201  \n",
       "Celltype_Source_ID                                         \n",
       "aDC_HPCA_1                  NaN          NaN          NaN  \n",
       "aDC_HPCA_2                  NaN          NaN          NaN  \n",
       "aDC_HPCA_3                  NaN          NaN          NaN  \n",
       "aDC_IRIS_1                  NaN          NaN          NaN  \n",
       "aDC_IRIS_2                  NaN          NaN          NaN  \n",
       "\n",
       "[5 rows x 201 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Load Additional File 3\n",
    "xcell_file = os.path.join('data', '13059_2017_1349_MOESM3_ESM.xlsx')\n",
    "xcell_df = pd.read_excel(xcell_file, index_col=0)\n",
    "print(xcell_df.shape)\n",
    "xcell_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "CD8+ T-cells          15\n",
       "NK cells              15\n",
       "Monocytes             15\n",
       "DC                    12\n",
       "Endothelial cells     12\n",
       "Neutrophils           12\n",
       "MPP                   12\n",
       "CD4+ naive T-cells    12\n",
       "Erythrocytes          12\n",
       "Macrophages           12\n",
       "dtype: int64"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Distribution of number of cell types\n",
    "cell_types = pd.Series([x.split('_')[0] for x in xcell_df.index])\n",
    "cell_types.value_counts().head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD8CAYAAABn919SAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAADblJREFUeJzt3X+M5HV9x/HXq1wbj1vlR44bz4O4ashVwirCxNiamFkp9SqEo3+YSNAcgWb9Q9CYM3pq0po0mkuU2iYaGyL0LtGyMUQCAapczm5JEzTuUmCPXvEInnjneScBLi6S6Lbv/rFf0r1ltzPz/X53vjNvno/ksjuz39nv+zM/npn77s6OI0IAgNH3B00PAACoB0EHgCQIOgAkQdABIAmCDgBJEHQASIKgA0ASBB0AkiDoAJDEhkHubPPmzTE+Pj7IXfbtpZde0qZNm5oeo7Is65BYyzDKsg5pNNYyNzf3XERc0G27gQZ9fHxcs7Ozg9xl32ZmZtTpdJoeo7Is65BYyzDKsg5pNNZi++e9bMchFwBIgqADQBIEHQCSIOgAkARBB4AkCDoAJEHQASAJgg4ASRB0AEhioK8URX/G9zxQ+rK7JxZ1Y8nLH917den9jqoq13U/VrtdXovXN9YHz9ABIAmCDgBJEHQASIKgA0ASBB0AkiDoAJAEQQeAJAg6ACRB0AEgCYIOAEkQdABIgqADQBIEHQCSIOgAkETXoNu+yPa/2j5s+0nbnyzOP9/2AdtHio/nrf+4AIC19PIMfVHS7oh4u6T3SPq47Usk7ZF0MCIulnSwOA0AaEjXoEfEiYh4tPj8N5IOS9omaaek/cVm+yVdt15DAgC66+sYuu1xSe+S9GNJrYg4IS1FX9KWuocDAPTOEdHbhvaYpH+T9KWI+J7tFyPi3GVffyEiXnUc3faUpClJarVaV0xPT9cz+TpZWFjQ2NhY02NIkuaPny592dZG6eTL5S47se2c0vtdD4O4Tapc1/1Y7XYZtuu7F8P0OKlqFNYyOTk5FxHtbtv1FHTbfyjpfkk/iIi/K857SlInIk7Y3ippJiK2/3/fp91ux+zsbE8LaMrMzIw6nU7TY0iq/p6it82Xe8vYYXuPy0HcJoN8T9GVt8uwXd+9GKbHSVWjsBbbPQW9l99ysaQ7JB1+JeaF+yTtKj7fJeneMoMCAOrRy1O490r6qKR5248V531e0l5J37V9s6RnJX1ofUYEAPSia9Aj4t8leY0vX1nvOACAsnilKAAkQdABIAmCDgBJEHQASIKgA0ASBB0AkiDoAJAEQQeAJAg6ACRB0AEgCYIOAEkQdABIgqADQBIEHQCSIOgAkARBB4AkCDoAJEHQASAJgg4ASRB0AEiCoANAEgQdAJIg6ACQBEEHgCQIOgAkQdABIAmCDgBJEHQASIKgA0ASBB0AkiDoAJAEQQeAJAg6ACRB0AEgCYIOAEkQdABIgqADQBIEHQCS6Bp023faPmX70LLzvmj7uO3Hin8fXN8xAQDd9PIMfZ+kHauc/7WIuKz492C9YwEA+tU16BHxsKTnBzALAKCCKsfQb7H9RHFI5rzaJgIAlOKI6L6RPS7p/oi4tDjdkvScpJD0t5K2RsRNa1x2StKUJLVarSump6drGXy9LCwsaGxsrOkxJEnzx0+Xvmxro3Ty5XKXndh2Tun9rodB3CZVrut+rHa7DNv13YthepxUNQprmZycnIuIdrftSgW916+t1G63Y3Z2tuv+mjQzM6NOp9P0GJKk8T0PlL7s7olF3Ta/odRlj+69uvR+18MgbpMq13U/Vrtdhu367sUwPU6qGoW12O4p6KUOudjeuuzkX0o6tNa2AIDB6PoUzvZdkjqSNts+JulvJHVsX6alQy5HJX1sHWcEAPSga9Aj4vpVzr5jHWYBAFTAK0UBIAmCDgBJEHQASIKgA0ASBB0AkiDoAJAEQQeAJAg6ACRB0AEgCYIOAEkQdABIgqADQBIEHQCSIOgAkARBB4AkCDoAJEHQASAJgg4ASRB0AEiCoANAEgQdAJIg6ACQBEEHgCQIOgAkQdABIAmCDgBJEHQASIKgA0ASBB0AkiDoAJAEQQeAJAg6ACRB0AEgCYIOAEkQdABIgqADQBIEHQCSIOgAkETXoNu+0/Yp24eWnXe+7QO2jxQfz1vfMQEA3fTyDH2fpB0rztsj6WBEXCzpYHEaANCgrkGPiIclPb/i7J2S9hef75d0Xc1zAQD6VPYYeisiTkhS8XFLfSMBAMpwRHTfyB6XdH9EXFqcfjEizl329RciYtXj6LanJE1JUqvVumJ6errUoPPHT5e6XL9aG6WTL5953sS2cway75WqrHm1dfSqqfWuZWFhQWNjY+u6j9fi/auKQdwmgzIKa5mcnJyLiHa37TaU/P4nbW+NiBO2t0o6tdaGEXG7pNslqd1uR6fTKbXDG/c8UOpy/do9sajb5s+8Wo7e0BnIvleqsubV1tGrpta7lpmZGZW93/TqtXj/qmIQt8mgZFpL2UMu90naVXy+S9K99YwDACirl19bvEvSI5K22z5m+2ZJeyVdZfuIpKuK0wCABnX9P3lEXL/Gl66seRYAQAW8UhQAkiDoAJAEQQeAJAg6ACRB0AEgCYIOAEkQdABIgqADQBIEHQCSIOgAkARBB4AkCDoAJEHQASAJgg4ASRB0AEiCoANAEgQdAJIg6ACQBEEHgCQIOgAkQdABIAmCDgBJEHQASGJD0wMAaMb4ngdKX3b3xKJurHD5o3uvLn1ZrI1n6ACQBEEHgCQIOgAkQdABIAmCDgBJEHQASIKgA0ASBB0AkiDoAJAEQQeAJAg6ACRB0AEgCYIOAEkQdABIotKfz7V9VNJvJP23pMWIaNcxFACgf3X8PfTJiHiuhu8DAKiAQy4AkETVoIekh2zP2Z6qYyAAQDmOiPIXtt8UEb+0vUXSAUm3RsTDK7aZkjQlSa1W64rp6elS+5o/frr0nP1obZROvnzmeRPbzhnIvleqsubV1tGrpta7loWFBY2Nja3rPrh/9afK/UsarvvYIO5fVU1OTs718jPKSkE/4xvZX5S0EBFfXWubdrsds7Ozpb5/lfc/7MfuiUXdNn/mjxaaev/Dqu/5uHIdvRq293ucmZlRp9NZ131w/+pPlfuXNFz3sUHcv6qy3VPQSx9ysb3J9utf+VzSn0s6VPb7AQCqqfJbLi1J99h+5fv8c0R8v5apAAB9Kx30iHhG0jtrnAUAUAG/tggASRB0AEiCoANAEgQdAJIg6ACQBEEHgCQIOgAkQdABIAmCDgBJEHQASIKgA0ASBB0AkiDoAJAEQQeAJKr8PXQAGCmrvUvT7olF3TiAd6waxLs08QwdAJIg6ACQBEEHgCQIOgAkQdABIAmCDgBJEHQASIKgA0ASBB0AkiDoAJAEQQeAJAg6ACRB0AEgCYIOAEkQdABIgqADQBIEHQCSIOgAkARBB4AkCDoAJEHQASAJgg4ASRB0AEiiUtBt77D9lO2nbe+paygAQP9KB932WZK+IekvJF0i6Xrbl9Q1GACgP1Weob9b0tMR8UxE/E7StKSd9YwFAOhXlaBvk/SLZaePFecBABrgiCh3QftDkj4QEX9VnP6opHdHxK0rtpuSNFWc3C7pqfLjDsRmSc81PUQNsqxDYi3DKMs6pNFYy5sj4oJuG22osINjki5advpCSb9cuVFE3C7p9gr7GSjbsxHRbnqOqrKsQ2ItwyjLOqRca6lyyOUnki62/RbbfyTpw5Luq2csAEC/Sj9Dj4hF27dI+oGksyTdGRFP1jYZAKAvVQ65KCIelPRgTbMMi5E5PNRFlnVIrGUYZVmHlGgtpX8oCgAYLrz0HwCSIOjL2D7L9n/Yvr/pWaqwfa7tu23/l+3Dtv+k6ZnKsP0p20/aPmT7Ltuva3qmfti+0/Yp24eWnXe+7QO2jxQfz2tyxl6ssY6vFPevJ2zfY/vcJmfs1WprWfa1T9sO25ubmK0OBP1Mn5R0uOkhavAPkr4fEX8s6Z0awTXZ3ibpE5LaEXGpln7w/uFmp+rbPkk7Vpy3R9LBiLhY0sHi9LDbp1ev44CkSyPiHZJ+Kulzgx6qpH169Vpk+yJJV0l6dtAD1YmgF2xfKOlqSd9qepYqbL9B0vsk3SFJEfG7iHix2alK2yBpo+0Nks7WKq9zGGYR8bCk51ecvVPS/uLz/ZKuG+hQJay2joh4KCIWi5M/0tLrUIbeGreJJH1N0mckjfQPFQn6//l7Ld2g/9P0IBW9VdKvJf1TcfjoW7Y3NT1UvyLiuKSvaukZ0wlJpyPioWanqkUrIk5IUvFxS8Pz1OEmSf/S9BBl2b5W0vGIeLzpWaoi6JJsXyPpVETMNT1LDTZIulzSNyPiXZJe0mj8t/4MxbHlnZLeIulNkjbZ/kizU2El21+QtCjpO03PUobtsyV9QdJfNz1LHQj6kvdKutb2US391cj32/52syOVdkzSsYj4cXH6bi0FftT8maSfRcSvI+L3kr4n6U8bnqkOJ21vlaTi46mG5ynN9i5J10i6IUb395/fpqUnDY8Xj/8LJT1q+42NTlUSQZcUEZ+LiAsjYlxLP3j7YUSM5LPBiPiVpF/Y3l6cdaWk/2xwpLKelfQe22fbtpbWMXI/3F3FfZJ2FZ/vknRvg7OUZnuHpM9KujYiftv0PGVFxHxEbImI8eLxf0zS5cXjaOQQ9JxulfQd209IukzSlxuep2/F/zDulvSopHkt3VdH6hV9tu+S9Iik7baP2b5Z0l5JV9k+oqXfqtjb5Iy9WGMdX5f0ekkHbD9m+x8bHbJHa6wlDV4pCgBJ8AwdAJIg6ACQBEEHgCQIOgAkQdABIAmCDgBJEHQASIKgA0AS/wv/UJD1WfKJ2AAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Distribution of number of cell types\n",
    "cell_types.value_counts().hist(bins=10);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load curated gene names from versioned resource \n",
    "commit = '721204091a96e55de6dcad165d6d8265e67e2a48'\n",
    "url = 'https://raw.githubusercontent.com/cognoma/genes/{}/data/genes.tsv'.format(commit)\n",
    "gene_df = pd.read_table(url)\n",
    "\n",
    "# Only consider protein-coding genes\n",
    "gene_df = (\n",
    "    gene_df.query(\"gene_type == 'protein-coding'\")\n",
    ")\n",
    "\n",
    "symbol_to_entrez = dict(zip(gene_df.symbol,\n",
    "                            gene_df.entrez_gene_id))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Add alternative symbols to entrez mapping dictionary\n",
    "gene_df = gene_df.dropna(axis='rows', subset=['synonyms'])\n",
    "gene_df.synonyms = gene_df.synonyms.str.split('|')\n",
    "\n",
    "all_syn = (\n",
    "    gene_df.apply(lambda x: pd.Series(x.synonyms), axis=1)\n",
    "    .stack()\n",
    "    .reset_index(level=1, drop=True)\n",
    ")\n",
    "\n",
    "# Name the synonym series and join with rest of genes\n",
    "all_syn.name = 'all_synonyms'\n",
    "gene_with_syn_df = gene_df.join(all_syn)\n",
    "\n",
    "# Remove rows that have redundant symbols in all_synonyms\n",
    "gene_with_syn_df = (\n",
    "    gene_with_syn_df\n",
    "    \n",
    "    # Drop synonyms that are duplicated - can't be sure of mapping\n",
    "    .drop_duplicates(['all_synonyms'], keep=False)\n",
    "\n",
    "    # Drop rows in which the symbol appears in the list of synonyms\n",
    "    .query('symbol not in all_synonyms')\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create a synonym to entrez mapping and add to dictionary\n",
    "synonym_to_entrez = dict(zip(gene_with_syn_df.all_synonyms,\n",
    "                             gene_with_syn_df.entrez_gene_id))\n",
    "\n",
    "symbol_to_entrez.update(synonym_to_entrez)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x7f4331cfc320>"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYAAAAD8CAYAAAB+UHOxAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAD5lJREFUeJzt3W2MXGd5h/HrJuZN2ZYkJFlFtttNi1UlyGqIVkkkqmpL2sQkVZ1KRDKKwEGp3A+mAslScfiSFohkJCAIFZDcxsIgSoh4aSwSiVohI9oPJMFAExIL2QWXbGPFok4CC4Jq4e6HeUwHZ3dn9nXGe18/yZpznvOcM8+5dbL/OWfOnERmIkmq52XDHoAkaTgMAEkqygCQpKIMAEkqygCQpKIMAEkqygCQpKIMAEkqygCQpKI2DHsAC7n44ovzkksu4fzzzx/2UEbaT3/6U2u0AOuzMOvT37lWoyNHjvwoMy/p12+kA2BiYoIPfehDTE1NDXsoI63T6VijBVifhVmf/s61GkXEfw3Sz0tAklSUASBJRRkAklSUASBJRRkAklSUASBJRRkAklSUASBJRRkAklTUSP8SeLkm9j44UL8T+25e5ZFI0ujxDECSijIAJKmodX0JaFBeKpJUkWcAklSUASBJRRkAklSUASBJRRkAklSUASBJRRkAklSUASBJRRkAklSUASBJRRkAklSUASBJRRkAklSUASBJRRkAklSUASBJRRkAklSUASBJRRkAklSUASBJRRkAklSUASBJRQ0cABFxXkR8OyK+0uYvj4hHI+JYRHw+Il7R2l/Z5o+35RM927iztX8vIm5c6Z2RJA1uMWcA7wKO9sx/ELgnM7cAzwN3tPY7gOcz83XAPa0fEXElsAN4PbAN+EREnLe84UuSlmqgAIiITcDNwD+1+QDeBHyhdTkI3NKmt7d52vLrW//twH2Z+YvM/AFwHLhmJXZCkrR4g54BfBT4W+BXbf61wAuZOdvmp4GNbXoj8AxAW/5i6//r9jnWkSStsQ39OkTEnwOnMvNIREydaZ6ja/ZZttA6ve+3C9gFMD4+zszMDJ1Op98w57Rn62z/Touw1HGstuXUqALrszDr0996rVHfAADeCPxFRNwEvAr4bbpnBBdExIb2KX8T8GzrPw1sBqYjYgPwGuB0T/sZvev8WmbuB/YDTE5O5tjYGFNTU0vYNbh974NLWm8+J25b2jhWW6fTWXKNKrA+C7M+/a3XGvW9BJSZd2bmpsycoPsl7tcy8zbgEeAtrdtO4IE2fajN05Z/LTOzte9odwldDmwBHluxPZEkLcogZwDzeQ9wX0R8APg2cG9rvxf4TEQcp/vJfwdAZj4VEfcDTwOzwO7M/OUy3l+StAyLCoDM7ACdNv195riLJzN/Dtw6z/p3A3cvdpCjYmLAS0on9t28yiORpOXzl8CSVJQBIElFGQCSVJQBIElFGQCSVJQBIElFGQCSVJQBIElFGQCSVJQBIElFGQCSVJQBIElFGQCSVJQBIElFGQCSVJQBIElFGQCSVJQBIElFGQCSVJQBIElFGQCSVJQBIElFGQCSVJQBIElFGQCSVJQBIElFGQCSVJQBIElFGQCSVJQBIElFGQCSVJQBIElFGQCSVJQBIElFGQCSVJQBIElF9Q2AiHhVRDwWEf8REU9FxN+39ssj4tGIOBYRn4+IV7T2V7b54235RM+27mzt34uIG1drpyRJ/Q1yBvAL4E2Z+YfAVcC2iLgO+CBwT2ZuAZ4H7mj97wCez8zXAfe0fkTElcAO4PXANuATEXHeSu6MJGlwfQMgu2ba7MvbvwTeBHyhtR8EbmnT29s8bfn1ERGt/b7M/EVm/gA4DlyzInshSVq0DYN0ap/UjwCvAz4O/CfwQmbOti7TwMY2vRF4BiAzZyPiReC1rf0bPZvtXaf3vXYBuwDGx8eZmZmh0+ksbq+aPVtn+3daBUsd71Itp0YVWJ+FWZ/+1muNBgqAzPwlcFVEXAB8Gbhirm7tNeZZNl/72e+1H9gPMDk5mWNjY0xNTQ0yzJe4fe+DS1pvuU7cNrWm79fpdJZcowqsz8KsT3/rtUaLugsoM18AOsB1wAURcSZANgHPtulpYDNAW/4a4HRv+xzrSJLW2CB3AV3SPvkTEa8G/hQ4CjwCvKV12wk80KYPtXna8q9lZrb2He0uocuBLcBjK7UjkqTFGeQS0GXAwfY9wMuA+zPzKxHxNHBfRHwA+DZwb+t/L/CZiDhO95P/DoDMfCoi7geeBmaB3e3SkiRpCPoGQGY+AbxhjvbvM8ddPJn5c+DWebZ1N3D34ocpSVpp/hJYkooyACSpKANAkooyACSpKANAkooyACSpKANAkooyACSpKANAkooa6GmgWh0TAz6t9MS+m1d5JJIq8gxAkooyACSpKANAkooyACSpKANAkooyACSpKANAkooyACSpKANAkooyACSpKANAkooyACSpKANAkooyACSpKANAkooyACSpKANAkooyACSpKANAkooyACSpKANAkooyACSpKANAkooyACSpKANAkorqGwARsTkiHomIoxHxVES8q7VfFBGHI+JYe72wtUdEfCwijkfEExFxdc+2drb+xyJi5+rtliSpn0HOAGaBPZl5BXAdsDsirgT2Ag9n5hbg4TYP8GZgS/u3C/gkdAMDuAu4FrgGuOtMaEiS1l7fAMjMk5n5rTb9E+AosBHYDhxs3Q4Ct7Tp7cCns+sbwAURcRlwI3A4M09n5vPAYWDbiu6NJGlgGxbTOSImgDcAjwLjmXkSuiEREZe2bhuBZ3pWm25t87Wf/R676J45MD4+zszMDJ1OZzHD/LU9W2eXtN5yDTreQcfXb3vLqVEF1mdh1qe/9VqjgQMgIsaALwLvzswfR8S8XedoywXaf7Mhcz+wH2BycjLHxsaYmpoadJi/4fa9Dy5pveU6cdvUQP0GHV+/7XU6nSXXqALrszDr0996rdFAdwFFxMvp/vH/bGZ+qTU/1y7t0F5PtfZpYHPP6puAZxdolyQNwSB3AQVwL3A0Mz/Ss+gQcOZOnp3AAz3tb293A10HvNguFX0VuCEiLmxf/t7Q2iRJQzDIJaA3Am8DnoyI77S29wL7gPsj4g7gh8CtbdlDwE3AceBnwDsAMvN0RLwfeLz1e19mnl6RvZAkLVrfAMjMf2fu6/cA18/RP4Hd82zrAHBgMQOUJK0OfwksSUUZAJJUlAEgSUUZAJJU1KJ+CazhmOjzg7E9W2e5fe+DnNh38xqNSNJ64BmAJBVlAEhSUQaAJBVlAEhSUQaAJBVlAEhSUQaAJBVlAEhSUQaAJBVlAEhSUQaAJBVlAEhSUQaAJBVlAEhSUQaAJBVlAEhSUQaAJBVlAEhSUQaAJBVlAEhSUQaAJBVlAEhSUQaAJBVlAEhSUQaAJBVlAEhSUQaAJBVlAEhSUQaAJBVlAEhSUX0DICIORMSpiPhuT9tFEXE4Io611wtbe0TExyLieEQ8ERFX96yzs/U/FhE7V2d3JEmD2jBAn08B/wB8uqdtL/BwZu6LiL1t/j3Am4Et7d+1wCeBayPiIuAuYBJI4EhEHMrM51dqR7Q4E3sfHKjfiX03r/JIJA1L3zOAzPw6cPqs5u3AwTZ9ELilp/3T2fUN4IKIuAy4ETicmafbH/3DwLaV2AFJ0tIs9TuA8cw8CdBeL23tG4FnevpNt7b52iVJQzLIJaDFiDnacoH2l24gYhewC2B8fJyZmRk6nc6SBrNn6+yS1luuQce7UuMbf3V3W4up06DvvdTaj5LlHEMVWJ/+1muNlhoAz0XEZZl5sl3iOdXap4HNPf02Ac+29qmz2jtzbTgz9wP7ASYnJ3NsbIypqam5uvZ1+4DXuVfaidumBuq3UuPbs3WWDz+5YeD3Xcx7L2abo6rT6Sz5GKrA+vS3Xmu01EtAh4Azd/LsBB7oaX97uxvoOuDFdonoq8ANEXFhu2PohtYmSRqSvmcAEfE5up/eL46Iabp38+wD7o+IO4AfAre27g8BNwHHgZ8B7wDIzNMR8X7g8dbvfZl59hfLkqQ11DcAMvOt8yy6fo6+CeyeZzsHgAOLGp0kadX4S2BJKsoAkKSiDABJKsoAkKSiDABJKsoAkKSiDABJKsoAkKSiDABJKsoAkKSiDABJKsoAkKSiDABJKmql/49gWmf8n8dL65dnAJJUlAEgSUUZAJJUlAEgSUUZAJJUlAEgSUUZAJJUlAEgSUUZAJJUlAEgSUUZAJJUlM8C0pry2ULS6PAMQJKKMgAkqSgDQJKKMgAkqSgDQJKK8i4gjSTvFpJWn2cAklSUASBJRXkJSOe0QS4V7dk6y9TqD0U656z5GUBEbIuI70XE8YjYu9bvL0nqWtMzgIg4D/g48GfANPB4RBzKzKfXchzSQvwCWlWs9SWga4Djmfl9gIi4D9gOGAA65xgUOtetdQBsBJ7pmZ8Grl3jMUgjadBAGZTBo34iM9fuzSJuBW7MzL9q828DrsnMv+npswvY1Wb/APgf4EdrNshz08VYo4VYn4VZn/7OtRr9bmZe0q/TWp8BTAObe+Y3Ac/2dsjM/cD+M/MR8c3MnFyb4Z2brNHCrM/CrE9/67VGa30X0OPAloi4PCJeAewADq3xGCRJrPEZQGbORsQ7ga8C5wEHMvOptRyDJKlrzX8IlpkPAQ8tYpX9/buUZ40WZn0WZn36W5c1WtMvgSVJo8NnAUlSUSMdAD42YmERcSIinoyI70TEN4c9nlEQEQci4lREfLen7aKIOBwRx9rrhcMc4zDNU5+/i4j/bsfRdyLipmGOcZgiYnNEPBIRRyPiqYh4V2tfl8fQyAZAz2Mj3gxcCbw1Iq4c7qhG0p9k5lXr8Ra1JfoUsO2str3Aw5m5BXi4zVf1KV5aH4B72nF0VfuerqpZYE9mXgFcB+xuf3fW5TE0sgFAz2MjMvN/gTOPjZDmlZlfB06f1bwdONimDwK3rOmgRsg89VGTmScz81tt+ifAUbpPMFiXx9AoB8Bcj43YOKSxjKoE/jUijrRfUGtu45l5Err/gQOXDnk8o+idEfFEu0S0Li5vLFdETABvAB5lnR5DoxwAMUebtyz9pjdm5tV0L5Ptjog/HvaAdE76JPD7wFXASeDDwx3O8EXEGPBF4N2Z+eNhj2e1jHIA9H1sRHWZ+Wx7PQV8me5lM73UcxFxGUB7PTXk8YyUzHwuM3+Zmb8C/pHix1FEvJzuH//PZuaXWvO6PIZGOQB8bMQCIuL8iPitM9PADcB3F16rrEPAzja9E3hgiGMZOWf+sDV/SeHjKCICuBc4mpkf6Vm0Lo+hkf4hWLsd7aP8/2Mj7h7ykEZGRPwe3U/90P1F9z9bH4iIzwFTdJ/e+BxwF/AvwP3A7wA/BG7NzJJfhM5Tnym6l38SOAH89Znr3dVExB8B/wY8CfyqNb+X7vcA6+4YGukAkCStnlG+BCRJWkUGgCQVZQBIUlEGgCQVZQBIUlEGgCQVZQBIUlEGgCQV9X+f9HMoGXc/pAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# How many entrez genes match to duplicate symbols\n",
    "pd.DataFrame.from_dict(symbol_to_entrez, orient='index').loc[:, 0].value_counts().hist(bins=30)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load gene updater\n",
    "url = 'https://raw.githubusercontent.com/cognoma/genes/{}/data/updater.tsv'.format(commit)\n",
    "updater_df = pd.read_table(url)\n",
    "old_to_new_entrez = dict(zip(updater_df.old_entrez_gene_id,\n",
    "                             updater_df.new_entrez_gene_id))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "geneset_url = 'https://doi.org/10.1186/s13059-017-1349-1'\n",
    "xcell_gmt_file = os.path.join('data', 'xcell_all_entrez.gmt')\n",
    "\n",
    "with open(xcell_gmt_file, 'w') as csvfile:\n",
    "    xcell_writer = csv.writer(csvfile, delimiter='\\t')\n",
    "    for row in xcell_df.iterrows():\n",
    "        geneset_name = row[0].replace(' ', '-')\n",
    "        genes = row[1][2:].dropna().tolist()\n",
    "\n",
    "        # There are data issues of input datetime types\n",
    "        # Ignore these genes for now\n",
    "        # See https://doi.org/10.1186/s13059-016-1044-7\n",
    "        genes = [x for x in genes if not type(x) == datetime.date]\n",
    "\n",
    "        # Convert genes to entrez IDs (considering alternate mappings)\n",
    "        genes = [symbol_to_entrez[x] for x in genes if x in symbol_to_entrez]\n",
    "\n",
    "        # Remove empty cells in the cell file\n",
    "        #genes = [x for x in genes if x != '']\n",
    "        row_to_write = [geneset_name, geneset_url] + list(set(genes))\n",
    "        xcell_writer.writerow(row_to_write)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:interpret-compression]",
   "language": "python",
   "name": "conda-env-interpret-compression-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}