Data.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import seaborn as sns\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import functions\n",
    "\n",
    "\n",
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(17270, 5)"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#We use eukaryota only\n",
    "\n",
    "#Pick non sp from signalp5 training \n",
    "\n",
    "signalP = pd.read_csv('data/signalp5/train_set.fasta', \\\n",
    "                       sep='>', lineterminator='>', header=None)\n",
    "signalP['Entry'] = signalP[0].apply(lambda x: x.split('|')[0])\n",
    "signalP['Kingdom'] = signalP[0].apply(lambda x: x.split('|')[1])\n",
    "signalP['Label'] = signalP[0].apply(lambda x: x.split('|')[2])\n",
    "euk_signalp = signalP[signalP['Kingdom'].apply(lambda x: 'EUKARYA' in x)]\n",
    "signalP['Accession'] = signalP[0].apply(lambda x: x.split('\\n')[0])\n",
    "\n",
    "\n",
    "#Select Eukarya\n",
    "# signalP['Protein'] = signalP[0].apply(lambda x: x.split('\\n')[1])\n",
    "euk_signalp = signalP[signalP['Kingdom'] == 'EUKARYA'].copy()\n",
    "euk_signalp.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "NO_SP    14656\n",
       "SP        2614\n",
       "Name: Label, dtype: int64"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "euk_signalp.Label.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "export = euk_signalp[['Entry']]\n",
    "export.to_csv('results/export_to_uniprot.fa', sep='\\n', index=None, header=None)\n",
    "##Uniprot id mapping to get sequences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Volumes/userdata/student_users/bikashkumarbhandari/anaconda3/lib/python3.7/site-packages/IPython/core/interactiveshell.py:3063: DtypeWarning: Columns (16,17) have mixed types.Specify dtype option on import or set low_memory=False.\n",
      "  interactivity=interactivity, compiler=compiler, result=result)\n"
     ]
    }
   ],
   "source": [
    "#sequences\n",
    "seqs = functions.read_seqs('results/signalp5/uniprot-yourlist M202011108471C63D39733769F8E060B506551E12040258H.fasta.gz')\n",
    "#annotations\n",
    "annots = pd.read_csv('results/signalp5/uniprot-yourlist M202011108471C63D39733769F8E060B506551E12040258H.tab.gz', sep='\\t')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(17267, 39)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_seqs = pd.merge(seqs, annots, on='Entry')\n",
    "all_seqs.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Entry</th>\n",
       "      <th>All</th>\n",
       "      <th>Protein</th>\n",
       "      <th>Entry name</th>\n",
       "      <th>Status</th>\n",
       "      <th>Protein names</th>\n",
       "      <th>Gene names</th>\n",
       "      <th>Organism</th>\n",
       "      <th>Length</th>\n",
       "      <th>Signal peptide</th>\n",
       "      <th>...</th>\n",
       "      <th>Taxonomic lineage (SUPERFAMILY)</th>\n",
       "      <th>Taxonomic lineage (SUPERKINGDOM)</th>\n",
       "      <th>Cross-reference (Pfam)</th>\n",
       "      <th>Biotechnological use</th>\n",
       "      <th>Motif</th>\n",
       "      <th>Domain [FT]</th>\n",
       "      <th>Cross-reference (InterPro)</th>\n",
       "      <th>Cross-reference (PROSITE)</th>\n",
       "      <th>Cleavage</th>\n",
       "      <th>Accession</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Q8TF40</td>\n",
       "      <td>sp|Q8TF40|FNIP1_HUMAN Folliculin-interacting p...</td>\n",
       "      <td>MAPTLFQKLFSKRTGLGAPGRDARDPDCGFSWPLPEFDPSQIRLIV...</td>\n",
       "      <td>FNIP1_HUMAN</td>\n",
       "      <td>reviewed</td>\n",
       "      <td>Folliculin-interacting protein 1</td>\n",
       "      <td>FNIP1 KIAA1961</td>\n",
       "      <td>Homo sapiens (Human)</td>\n",
       "      <td>1166</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>Hominoidea (apes)</td>\n",
       "      <td>Eukaryota</td>\n",
       "      <td>PF14638;PF14637;PF14636;</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>DOMAIN 37..478;  /note=\"uDENN FNIP1/2-type\";  ...</td>\n",
       "      <td>IPR037545;IPR028086;IPR026156;IPR028085;IPR028...</td>\n",
       "      <td>PS51836;</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Q8TF40|EUKARYA|NO_SP|0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Q1ENB6</td>\n",
       "      <td>sp|Q1ENB6|FOLC_ARATH Folate synthesis bifuncti...</td>\n",
       "      <td>MDFTSLETTTFEEVVIALGSNVGNRMNNFKEALRLMKDYGISVTRH...</td>\n",
       "      <td>FOLC_ARATH</td>\n",
       "      <td>reviewed</td>\n",
       "      <td>Folate synthesis bifunctional protein [Include...</td>\n",
       "      <td>CytHPPK/DHPS At1g69190 F23O10.22 F4N2.15</td>\n",
       "      <td>Arabidopsis thaliana (Mouse-ear cress)</td>\n",
       "      <td>484</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Eukaryota</td>\n",
       "      <td>PF01288;PF00809;</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>DOMAIN 202..470;  /note=\"Pterin-binding\";  /ev...</td>\n",
       "      <td>IPR006390;IPR011005;IPR000550;IPR035907;IPR000...</td>\n",
       "      <td>PS00792;PS00793;PS00794;PS50972;</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Q1ENB6|EUKARYA|NO_SP|1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>P36001</td>\n",
       "      <td>sp|P36001|FOLC_YEAST Probable folylpolyglutama...</td>\n",
       "      <td>MDDISGRQTLPRINRLLEHVGNPQDSLSILHIAGTNGKETVSKFLT...</td>\n",
       "      <td>FOLC_YEAST</td>\n",
       "      <td>reviewed</td>\n",
       "      <td>Probable folylpolyglutamate synthase (EC 6.3.2...</td>\n",
       "      <td>RMA1 YKL132C</td>\n",
       "      <td>Saccharomyces cerevisiae (strain ATCC 204508 /...</td>\n",
       "      <td>430</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Eukaryota</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>IPR001645;IPR018109;IPR036565;IPR036615;</td>\n",
       "      <td>PS01011;PS01012;</td>\n",
       "      <td>NaN</td>\n",
       "      <td>P36001|EUKARYA|NO_SP|1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>P55317</td>\n",
       "      <td>sp|P55317|FOXA1_HUMAN Hepatocyte nuclear facto...</td>\n",
       "      <td>MLGTVKMEGHETSDWNSYYADTQEAYSSVPVSNMNSGLGSMNSMNT...</td>\n",
       "      <td>FOXA1_HUMAN</td>\n",
       "      <td>reviewed</td>\n",
       "      <td>Hepatocyte nuclear factor 3-alpha (HNF-3-alpha...</td>\n",
       "      <td>FOXA1 HNF3A TCF3A</td>\n",
       "      <td>Homo sapiens (Human)</td>\n",
       "      <td>472</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>Hominoidea (apes)</td>\n",
       "      <td>Eukaryota</td>\n",
       "      <td>PF00250;PF08430;PF09354;</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>IPR013638;IPR001766;IPR018533;IPR018122;IPR030...</td>\n",
       "      <td>PS00657;PS00658;PS50039;</td>\n",
       "      <td>NaN</td>\n",
       "      <td>P55317|EUKARYA|NO_SP|4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>P35583</td>\n",
       "      <td>sp|P35583|FOXA2_MOUSE Hepatocyte nuclear facto...</td>\n",
       "      <td>MLGAVKMEGHEPSDWSSYYAEPEGYSSVSNMNAGLGMNGMNTYMSM...</td>\n",
       "      <td>FOXA2_MOUSE</td>\n",
       "      <td>reviewed</td>\n",
       "      <td>Hepatocyte nuclear factor 3-beta (HNF-3-beta) ...</td>\n",
       "      <td>Foxa2 Hnf3b Tcf-3b Tcf3b</td>\n",
       "      <td>Mus musculus (Mouse)</td>\n",
       "      <td>459</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Eukaryota</td>\n",
       "      <td>PF00250;PF08430;PF09354;</td>\n",
       "      <td>NaN</td>\n",
       "      <td>MOTIF 106..113;  /note=\"Nuclear localization s...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>IPR013638;IPR001766;IPR018533;IPR018122;IPR030...</td>\n",
       "      <td>PS00657;PS00658;PS50039;</td>\n",
       "      <td>NaN</td>\n",
       "      <td>P35583|EUKARYA|NO_SP|4</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 40 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "    Entry                                                All  \\\n",
       "0  Q8TF40  sp|Q8TF40|FNIP1_HUMAN Folliculin-interacting p...   \n",
       "1  Q1ENB6  sp|Q1ENB6|FOLC_ARATH Folate synthesis bifuncti...   \n",
       "2  P36001  sp|P36001|FOLC_YEAST Probable folylpolyglutama...   \n",
       "3  P55317  sp|P55317|FOXA1_HUMAN Hepatocyte nuclear facto...   \n",
       "4  P35583  sp|P35583|FOXA2_MOUSE Hepatocyte nuclear facto...   \n",
       "\n",
       "                                             Protein   Entry name    Status  \\\n",
       "0  MAPTLFQKLFSKRTGLGAPGRDARDPDCGFSWPLPEFDPSQIRLIV...  FNIP1_HUMAN  reviewed   \n",
       "1  MDFTSLETTTFEEVVIALGSNVGNRMNNFKEALRLMKDYGISVTRH...   FOLC_ARATH  reviewed   \n",
       "2  MDDISGRQTLPRINRLLEHVGNPQDSLSILHIAGTNGKETVSKFLT...   FOLC_YEAST  reviewed   \n",
       "3  MLGTVKMEGHETSDWNSYYADTQEAYSSVPVSNMNSGLGSMNSMNT...  FOXA1_HUMAN  reviewed   \n",
       "4  MLGAVKMEGHEPSDWSSYYAEPEGYSSVSNMNAGLGMNGMNTYMSM...  FOXA2_MOUSE  reviewed   \n",
       "\n",
       "                                       Protein names  \\\n",
       "0                   Folliculin-interacting protein 1   \n",
       "1  Folate synthesis bifunctional protein [Include...   \n",
       "2  Probable folylpolyglutamate synthase (EC 6.3.2...   \n",
       "3  Hepatocyte nuclear factor 3-alpha (HNF-3-alpha...   \n",
       "4  Hepatocyte nuclear factor 3-beta (HNF-3-beta) ...   \n",
       "\n",
       "                                 Gene names  \\\n",
       "0                            FNIP1 KIAA1961   \n",
       "1  CytHPPK/DHPS At1g69190 F23O10.22 F4N2.15   \n",
       "2                              RMA1 YKL132C   \n",
       "3                         FOXA1 HNF3A TCF3A   \n",
       "4                  Foxa2 Hnf3b Tcf-3b Tcf3b   \n",
       "\n",
       "                                            Organism  Length Signal peptide  \\\n",
       "0                               Homo sapiens (Human)    1166            NaN   \n",
       "1             Arabidopsis thaliana (Mouse-ear cress)     484            NaN   \n",
       "2  Saccharomyces cerevisiae (strain ATCC 204508 /...     430            NaN   \n",
       "3                               Homo sapiens (Human)     472            NaN   \n",
       "4                               Mus musculus (Mouse)     459            NaN   \n",
       "\n",
       "   ...  Taxonomic lineage (SUPERFAMILY) Taxonomic lineage (SUPERKINGDOM)  \\\n",
       "0  ...                Hominoidea (apes)                        Eukaryota   \n",
       "1  ...                              NaN                        Eukaryota   \n",
       "2  ...                              NaN                        Eukaryota   \n",
       "3  ...                Hominoidea (apes)                        Eukaryota   \n",
       "4  ...                              NaN                        Eukaryota   \n",
       "\n",
       "     Cross-reference (Pfam) Biotechnological use  \\\n",
       "0  PF14638;PF14637;PF14636;                  NaN   \n",
       "1          PF01288;PF00809;                  NaN   \n",
       "2                       NaN                  NaN   \n",
       "3  PF00250;PF08430;PF09354;                  NaN   \n",
       "4  PF00250;PF08430;PF09354;                  NaN   \n",
       "\n",
       "                                               Motif  \\\n",
       "0                                                NaN   \n",
       "1                                                NaN   \n",
       "2                                                NaN   \n",
       "3                                                NaN   \n",
       "4  MOTIF 106..113;  /note=\"Nuclear localization s...   \n",
       "\n",
       "                                         Domain [FT]  \\\n",
       "0  DOMAIN 37..478;  /note=\"uDENN FNIP1/2-type\";  ...   \n",
       "1  DOMAIN 202..470;  /note=\"Pterin-binding\";  /ev...   \n",
       "2                                                NaN   \n",
       "3                                                NaN   \n",
       "4                                                NaN   \n",
       "\n",
       "                          Cross-reference (InterPro)  \\\n",
       "0  IPR037545;IPR028086;IPR026156;IPR028085;IPR028...   \n",
       "1  IPR006390;IPR011005;IPR000550;IPR035907;IPR000...   \n",
       "2           IPR001645;IPR018109;IPR036565;IPR036615;   \n",
       "3  IPR013638;IPR001766;IPR018533;IPR018122;IPR030...   \n",
       "4  IPR013638;IPR001766;IPR018533;IPR018122;IPR030...   \n",
       "\n",
       "          Cross-reference (PROSITE) Cleavage               Accession  \n",
       "0                          PS51836;      NaN  Q8TF40|EUKARYA|NO_SP|0  \n",
       "1  PS00792;PS00793;PS00794;PS50972;      NaN  Q1ENB6|EUKARYA|NO_SP|1  \n",
       "2                  PS01011;PS01012;      NaN  P36001|EUKARYA|NO_SP|1  \n",
       "3          PS00657;PS00658;PS50039;      NaN  P55317|EUKARYA|NO_SP|4  \n",
       "4          PS00657;PS00658;PS50039;      NaN  P35583|EUKARYA|NO_SP|4  \n",
       "\n",
       "[5 rows x 40 columns]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def cleavage(x):\n",
    "    try:\n",
    "        return int(x.split('..')[1].split(';')[0])\n",
    "    except Exception:\n",
    "        if str(x) != 'nan':\n",
    "            print(x)\n",
    "        return np.nan\n",
    "    \n",
    "all_seqs['Cleavage'] = all_seqs['Signal peptide'].apply(cleavage)\n",
    "all_seqs = all_seqs[np.concatenate([all_seqs.columns[:3], all_seqs.columns[4:]])]\n",
    "\n",
    "all_seqs = all_seqs.merge(euk_signalp[['Entry', 'Accession']], on='Entry')\n",
    "all_seqs.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def chopping(a, p):\n",
    "    '''\n",
    "    Chops sequences at p position left and right of cleavage site a\n",
    "    '''\n",
    "    seq, cleavage = a.split('#')\n",
    "    try:\n",
    "        cleavage = int(float(cleavage))\n",
    "    except Exception:\n",
    "        print(cleavage)\n",
    "    try:\n",
    "        start = cleavage-p\n",
    "    except Exception:\n",
    "        print(cleavage, p)\n",
    "    spaces = 0\n",
    "    if start < 0:\n",
    "        spaces = -1 * start\n",
    "        start = 0\n",
    "    try:\n",
    "        s = seq[start:cleavage+p+1]\n",
    "    except Exception:\n",
    "        print(a, cleavage)\n",
    "    return '_'* spaces + s"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "set()"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sp_ = all_seqs[~all_seqs['Signal peptide'].isna()]\n",
    "# s = sp_[sp_['Signal peptide'].apply(lambda x: '0000269' in str(x))].copy()\n",
    "s = sp_.copy()\n",
    "s['Label'] = 1\n",
    "set(sp_.Entry) - set(s.Entry)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "41    2607\n",
       "39       1\n",
       "35       1\n",
       "Name: Aligned, dtype: int64"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Align signal peptides\n",
    "\n",
    "s['Temp'] = s['Protein'] + '#' + s['Cleavage'].astype(str)\n",
    "\n",
    "\n",
    "s['Aligned'] = s['Temp'].apply(lambda x: chopping(x, 20))\n",
    "s['Aligned'].apply(len).value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['Entry', 'All', 'Protein', 'Entry name', 'Status', 'Protein names',\n",
       "       'Gene names', 'Organism', 'Length', 'Signal peptide',\n",
       "       'Taxonomic lineage IDs', 'Date of creation', 'Keywords',\n",
       "       'Taxonomic lineage (all)', 'Taxonomic lineage (FAMILY)',\n",
       "       'Gene ontology (biological process)', 'Transmembrane',\n",
       "       'Allergenic properties', 'Toxic dose', 'Pharmaceutical use',\n",
       "       'Involvement in disease', 'Domain [CC]', 'Gene ontology (GO)',\n",
       "       'Gene ontology (molecular function)', 'Protein families',\n",
       "       'Taxonomic lineage (CLASS)', 'Taxonomic lineage (COHORT)',\n",
       "       'Taxonomic lineage (KINGDOM)', 'Taxonomic lineage (ORDER)',\n",
       "       'Taxonomic lineage (SUPERCLASS)', 'Taxonomic lineage (SUPERFAMILY)',\n",
       "       'Taxonomic lineage (SUPERKINGDOM)', 'Cross-reference (Pfam)',\n",
       "       'Biotechnological use', 'Motif', 'Domain [FT]',\n",
       "       'Cross-reference (InterPro)', 'Cross-reference (PROSITE)', 'Cleavage',\n",
       "       'Accession'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_seqs.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "14655"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# P10184 is not SP\n",
    "\n",
    "ns = all_seqs[all_seqs.Entry.apply(lambda x: x not in sp_.Entry.values)].copy()\n",
    "ns['Label'] = 0\n",
    "ns['Aligned'] = ns['Protein'].str[1:42]\n",
    "ns.shape[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "columns = ['Entry', 'All', 'Protein', 'Entry name', 'Status', 'Protein names',\n",
    "       'Gene names', 'Organism', 'Length', 'Signal peptide',\n",
    "       'Taxonomic lineage IDs', 'Date of creation', 'Keywords',\n",
    "       'Taxonomic lineage (all)', 'Taxonomic lineage (FAMILY)',\n",
    "       'Gene ontology (biological process)', 'Transmembrane',\n",
    "       'Allergenic properties', 'Toxic dose', 'Pharmaceutical use',\n",
    "       'Involvement in disease', 'Domain [CC]', 'Gene ontology (GO)',\n",
    "       'Gene ontology (molecular function)', 'Protein families',\n",
    "       'Taxonomic lineage (CLASS)', 'Taxonomic lineage (COHORT)',\n",
    "       'Taxonomic lineage (KINGDOM)', 'Taxonomic lineage (ORDER)',\n",
    "       'Taxonomic lineage (SUPERCLASS)', 'Taxonomic lineage (SUPERFAMILY)',\n",
    "       'Taxonomic lineage (SUPERKINGDOM)', 'Cross-reference (Pfam)',\n",
    "       'Biotechnological use', 'Motif', 'Domain [FT]',\n",
    "       'Cross-reference (InterPro)', 'Cross-reference (PROSITE)', 'Cleavage',\n",
    "       'Accession', 'Label', 'Aligned']\n",
    "\n",
    "final_df = pd.concat([s[columns], ns[columns]]).reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    14655\n",
       "1     2609\n",
       "Name: Label, dtype: int64"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "final_df.Label.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "final_df.to_pickle('results/signalp5_train_all.pkl.gz')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "for_website = pd.read_pickle('results/signalp5_train_all.pkl.gz')\n",
    "for_website.to_csv('results/SignalPeptide_Data.csv.gz', columns=['Accession', 'Protein', 'Label'],  index=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0        MARVAWGLLWLLLGSAGAQYEKYSFRGFPPEDLMPLAAAYGHALEQ...\n",
       "1        MKAVITLLFLACILVVTYGDLICGTNYCKDHPCTSPIARASCRSPA...\n",
       "2        MKIILTLSIFLICFLQLGQSVIDPSQNEVMSDLLFNLYGYDKSLDP...\n",
       "3        MKHLLTLALCFSSINAVAVTVPHKAVGTGIPEGSLQFLSLRASAPI...\n",
       "4        MLFKSLSKLATAAAFFAGVATADDVPAIEVVGNKFFYSNNGSQFYI...\n",
       "                               ...                        \n",
       "17259    MDTLRSISSELVRSSQLRWTLFSVIANMSLTLSIWIGVYNMEVNSE...\n",
       "17260    MQGPAGNASRGLPGGPPSTVASGAGRCESGALMHSFGIFLQGLLGV...\n",
       "17261    MDSPEKLEKNNLVGTNKSRLGVCGWILFFLSFLLMLVTFPISVWMC...\n",
       "17262    MEELVGLREGSSGKPVTLQELWGPCPRIRRGIRGGLEWLKERLFRA...\n",
       "17263    MEPHVLGAGLYWLLLPCTLLAASLLRFNALSLVYLLFLLLLPWLPG...\n",
       "Name: Protein, Length: 17264, dtype: object"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "for_website.Protein"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# chopping(all_seqs[all_seqs.Label == 1].loc[13612]['Protein'] + '#' + str(all_seqs[all_seqs.Label == 1].loc[13612]['Cleavage']), 10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "17264"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "14655 + 2609"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Benchmark set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "benchmark = pd.read_csv('data/signalp5/benchmark_set.fasta', \\\n",
    "                       sep='>', lineterminator='>', header=None)\n",
    "benchmark['Entry'] = benchmark[0].apply(lambda x: x.split('|')[0])\n",
    "bm = final_df[final_df.Entry.apply(lambda x: x in benchmark.Entry.values)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "bm = bm.reset_index(drop=True)\n",
    "bm.to_pickle('results/signalp5_benchmark.pkl.gz')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "export = bm[['Entry', 'Protein']].copy()\n",
    "export['Entry'] = '>' + export['Entry']\n",
    "export.to_csv('results/benchmark.fa', index=False, header=None, sep='\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    7246\n",
       "1     211\n",
       "Name: Label, dtype: int64"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bm.Label.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "for_website = pd.read_pickle('results/signalp5_benchmark.pkl.gz')\n",
    "for_website.to_csv('results/SignalPeptide_Benchmark.csv.gz', columns=['Accession', 'Protein', 'Label'],  index=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}