preprocessing.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "2c63e5e8",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from openbabel import openbabel as ob\n",
    "from openbabel import pybel as pb\n",
    "from molvs import standardize_smiles"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "86e3c42f",
   "metadata": {},
   "outputs": [],
   "source": [
    "train = pd.read_csv(\"Task/train.csv\", index_col=0)\n",
    "train[\"Smiles\"].replace(\"se\", \"Se\", inplace=True, regex=True)\n",
    "test = pd.read_csv(\"Task/test.csv\", index_col=0)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1d71427f",
   "metadata": {},
   "source": [
    "Possible preprocessing steps:\n",
    "<ul>\n",
    "    <li>Standardization</li>\n",
    "    <li>Desalting</li>\n",
    "    <li>Neutralization</li>\n",
    "    <li>Tautomer normalization</li>\n",
    "    <li>Removing of duplicates</li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "b2861b07",
   "metadata": {},
   "outputs": [],
   "source": [
    "# they are all on a large distance from the core\n",
    "# provided for the reference\n",
    "ions = [\".[Na+]\", \".[Ca+2]\", \".[Br-]\", \".[Cl-]\", \".[Zn+2]\", \".[K+]\",\n",
    "        \".[I-]\", \".[Li+]\", \".[Li]\", \".[Sr+2]\", \".[Ag+]\", \".[Mg+2]\"]\n",
    "\n",
    "remove_atoms = [\n",
    "    \"Ag\", # there is no pure Ag except for ionic form\n",
    "    \"Al\", # 1 big structure in train, False, no structures in test\n",
    "    \"As\", # 2 structures in train with [As], all False, 3 structures in test.                     *\n",
    "    \"B\",  # 13 structures in train, all False, 6 structures in test                               *\n",
    "    \"Ca\", # there is no pure Ca except for ionic form\n",
    "    \"K\",  # there is no pure K except for ionic form\n",
    "    \"Li\", # there is no pure Li except for ionic form\n",
    "    \"Mg\", # there is no pure Mg except for ionic form\n",
    "    \"Na\", # there is no pure Na except for ionic form\n",
    "    \"Se\", # 2 structures in train with [Se], all False, no structures in test\n",
    "          # 1 structure in train with [se], True\n",
    "    \"Si\", # 2 structures in train with [Si] (one garbage), all False, no structures in test\n",
    "    \"Sr\", # there is no pure Sr except for ionic form\n",
    "    \"Zn\", # 2 structures in train with [Zn], all False, 1 structure in test                       *\n",
    "]\n",
    "\n",
    "other_atoms = [\"Br\", \"C\", \"Cl\", \"F\", \"H\", \"I\", \"N\", \"O\", \"P\", \"S\"]\n",
    "\n",
    "# we remove shorter (or equal) fragments with possible exceptions\n",
    "\n",
    "# special cases\n",
    "\n",
    "special_train = [\n",
    "    1172, # garbage ?\n",
    "    2569, # [I-][K+]\n",
    "    \n",
    "]\n",
    "special_test = [\n",
    "    1101, # salt is larger than the core\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "2497a3f7",
   "metadata": {},
   "outputs": [],
   "source": [
    "def remove_salt(smile):\n",
    "    tokens = smile.split(\".\")\n",
    "    max_token = \"\"\n",
    "    for token in tokens:\n",
    "        if len(token) > len(max_token):\n",
    "            max_token = token\n",
    "    return max_token"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "bd5bbd97",
   "metadata": {},
   "outputs": [],
   "source": [
    "train[\"Smiles\"] = train[\"Smiles\"].apply(remove_salt)\n",
    "test[\"Smiles\"] = test[\"Smiles\"].apply(remove_salt)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "916ce0e9",
   "metadata": {},
   "outputs": [],
   "source": [
    "atoms = remove_atoms[:]\n",
    "atoms[atoms.index(\"B\")] = \"B[^r]\"\n",
    "train = train.drop(\n",
    "    train[train[\"Smiles\"].str.contains(\"|\".join(atoms))].index\n",
    ").reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "ba1a53da",
   "metadata": {},
   "outputs": [],
   "source": [
    "# we need to neutralize structures, espicially those, \n",
    "# from which we've removed the salts\n",
    "# https://baoilleach.blogspot.com/2019/12/no-charge-simple-approach-to.html\n",
    "\n",
    "pat_v3 = ob.OBSmartsPattern()\n",
    "pat_v3.Init(\"[+1!h0!$([*]~[-1,-2,-3,-4]),-1!$([*]~[+1,+2,+3,+4])]\")\n",
    "\n",
    "def neutralize(mol):\n",
    "    \"\"\"Neutralize charges of +1 or -1\"\"\"\n",
    "    mol = mol.OBMol\n",
    "    pat_v3.Match(mol)\n",
    "    for match in pat_v3.GetMapList():\n",
    "        atom = mol.GetAtom(match[0])\n",
    "        chg = atom.GetFormalCharge()\n",
    "        hcount = atom.GetImplicitHCount()\n",
    "        atom.SetFormalCharge(0)\n",
    "        atom.SetImplicitHCount(hcount - chg)\n",
    "    return pb.Molecule(mol)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "62b66009",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Nc1nc(O)c2c(n1)NCC(CNc1ccc(C(=O)N[C@@H](CCC(=O)[O-])C(=O)[O-])cc1)N2C=O'"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train[\"Smiles\"][12]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "8ac37f08",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/svg+xml": [
       "<svg xmlns=\"http://www.w3.org/2000/svg\" font-family=\"sans-serif\" height=\"100\" stroke=\"rgb(0,0,0)\" stroke-linecap=\"round\" stroke-width=\"2\" viewBox=\"0 0 391.769 500\" width=\"100\" x=\"0\" y=\"0\">\n",
       "<line opacity=\"1.0\" stroke=\"rgb(0,0,0)\" stroke-width=\"2.0\" x1=\"340.5\" x2=\"317.1\" y1=\"326.5\" y2=\"340.0\" />\n",
       "<line opacity=\"1.0\" stroke=\"rgb(0,0,0)\" stroke-width=\"2.0\" x1=\"282.5\" x2=\"282.5\" y1=\"400.0\" y2=\"427.0\" />\n",
       "<line opacity=\"1.0\" stroke=\"rgb(0,0,0)\" stroke-width=\"2.0\" x1=\"178.6\" x2=\"143.9\" y1=\"380.0\" y2=\"400.0\" />\n",
       "<line opacity=\"1.0\" stroke=\"rgb(0,0,0)\" stroke-width=\"2.0\" x1=\"143.9\" x2=\"120.5\" y1=\"400.0\" y2=\"386.5\" />\n",
       "<line opacity=\"1.0\" stroke=\"rgb(0,0,0)\" stroke-width=\"2.0\" x1=\"109.3\" x2=\"109.3\" y1=\"367.0\" y2=\"340.0\" />\n",
       "<line opacity=\"1.0\" stroke=\"rgb(0,0,0)\" stroke-width=\"2.0\" x1=\"109.3\" x2=\"109.3\" y1=\"260.0\" y2=\"220.0\" />\n",
       "<line opacity=\"1.0\" stroke=\"rgb(0,0,0)\" stroke-width=\"2.0\" x1=\"107.8\" x2=\"84.4\" y1=\"222.6\" y2=\"209.1\" />\n",
       "<line opacity=\"1.0\" stroke=\"rgb(0,0,0)\" stroke-width=\"2.0\" x1=\"110.8\" x2=\"87.4\" y1=\"217.4\" y2=\"203.9\" />\n",
       "<line opacity=\"1.0\" stroke=\"rgb(0,0,0)\" stroke-width=\"2.0\" x1=\"109.3\" x2=\"132.7\" y1=\"220.0\" y2=\"206.5\" />\n",
       "<line opacity=\"1.0\" stroke=\"rgb(0,0,0)\" stroke-width=\"2.0\" x1=\"143.9\" x2=\"143.9\" y1=\"187.0\" y2=\"160.0\" />\n",
       "<polygon fill=\"rgb(0,0,0)\" points=\"143.923048 160.000000 107.282032 143.464102 111.282032 136.535898 \" stroke=\"rgb(0,0,0)\" stroke-width=\"2.000000\" />\n",
       "<line opacity=\"1.0\" stroke=\"rgb(0,0,0)\" stroke-width=\"2.0\" x1=\"109.3\" x2=\"109.3\" y1=\"140.0\" y2=\"100.0\" />\n",
       "<line opacity=\"1.0\" stroke=\"rgb(0,0,0)\" stroke-width=\"2.0\" x1=\"109.3\" x2=\"74.6\" y1=\"100.0\" y2=\"80.0\" />\n",
       "<line opacity=\"1.0\" stroke=\"rgb(0,0,0)\" stroke-width=\"2.0\" x1=\"76.1\" x2=\"52.8\" y1=\"82.6\" y2=\"96.1\" />\n",
       "<line opacity=\"1.0\" stroke=\"rgb(0,0,0)\" stroke-width=\"2.0\" x1=\"73.1\" x2=\"49.8\" y1=\"77.4\" y2=\"90.9\" />\n",
       "<line opacity=\"1.0\" stroke=\"rgb(0,0,0)\" stroke-width=\"2.0\" x1=\"74.6\" x2=\"74.6\" y1=\"80.0\" y2=\"53.0\" />\n",
       "<line opacity=\"1.0\" stroke=\"rgb(0,0,0)\" stroke-width=\"2.0\" x1=\"143.9\" x2=\"178.6\" y1=\"160.0\" y2=\"140.0\" />\n",
       "<line opacity=\"1.0\" stroke=\"rgb(0,0,0)\" stroke-width=\"2.0\" x1=\"175.6\" x2=\"175.6\" y1=\"140.0\" y2=\"113.0\" />\n",
       "<line opacity=\"1.0\" stroke=\"rgb(0,0,0)\" stroke-width=\"2.0\" x1=\"181.6\" x2=\"181.6\" y1=\"140.0\" y2=\"113.0\" />\n",
       "<line opacity=\"1.0\" stroke=\"rgb(0,0,0)\" stroke-width=\"2.0\" x1=\"178.6\" x2=\"201.9\" y1=\"140.0\" y2=\"153.5\" />\n",
       "<line opacity=\"1.0\" stroke=\"rgb(0,0,0)\" stroke-width=\"2.0\" x1=\"213.2\" x2=\"213.2\" y1=\"413.0\" y2=\"440.0\" />\n",
       "<line opacity=\"1.0\" stroke=\"rgb(0,0,0)\" stroke-width=\"2.0\" x1=\"214.7\" x2=\"238.1\" y1=\"437.4\" y2=\"450.9\" />\n",
       "<line opacity=\"1.0\" stroke=\"rgb(0,0,0)\" stroke-width=\"2.0\" x1=\"211.7\" x2=\"235.1\" y1=\"442.6\" y2=\"456.1\" />\n",
       "<line opacity=\"1.0\" stroke=\"rgb(0,0,0)\" stroke-width=\"2.0\" x1=\"317.1\" x2=\"317.1\" y1=\"367.0\" y2=\"340.0\" />\n",
       "<line opacity=\"1.0\" stroke=\"rgb(0,0,0)\" stroke-width=\"2.0\" x1=\"317.1\" x2=\"293.9\" y1=\"340.0\" y2=\"326.6\" />\n",
       "<line opacity=\"1.0\" stroke=\"rgb(0,0,0)\" stroke-width=\"2.0\" x1=\"308.3\" x2=\"292.1\" y1=\"343.2\" y2=\"333.8\" />\n",
       "<line opacity=\"1.0\" stroke=\"rgb(0,0,0)\" stroke-width=\"2.0\" x1=\"271.2\" x2=\"247.8\" y1=\"326.5\" y2=\"340.0\" />\n",
       "<line opacity=\"1.0\" stroke=\"rgb(0,0,0)\" stroke-width=\"2.0\" x1=\"247.8\" x2=\"247.8\" y1=\"340.0\" y2=\"380.0\" />\n",
       "<line opacity=\"1.0\" stroke=\"rgb(0,0,0)\" stroke-width=\"2.0\" x1=\"255.0\" x2=\"255.0\" y1=\"346.0\" y2=\"374.0\" />\n",
       "<line opacity=\"1.0\" stroke=\"rgb(0,0,0)\" stroke-width=\"2.0\" x1=\"247.8\" x2=\"282.5\" y1=\"380.0\" y2=\"400.0\" />\n",
       "<line opacity=\"1.0\" stroke=\"rgb(0,0,0)\" stroke-width=\"2.0\" x1=\"282.5\" x2=\"305.7\" y1=\"400.0\" y2=\"386.6\" />\n",
       "<line opacity=\"1.0\" stroke=\"rgb(0,0,0)\" stroke-width=\"2.0\" x1=\"284.1\" x2=\"300.4\" y1=\"390.8\" y2=\"381.4\" />\n",
       "<line opacity=\"1.0\" stroke=\"rgb(0,0,0)\" stroke-width=\"2.0\" x1=\"143.9\" x2=\"109.3\" y1=\"320.0\" y2=\"340.0\" />\n",
       "<line opacity=\"1.0\" stroke=\"rgb(0,0,0)\" stroke-width=\"2.0\" x1=\"109.3\" x2=\"74.6\" y1=\"340.0\" y2=\"320.0\" />\n",
       "<line opacity=\"1.0\" stroke=\"rgb(0,0,0)\" stroke-width=\"2.0\" x1=\"107.7\" x2=\"83.4\" y1=\"330.8\" y2=\"316.8\" />\n",
       "<line opacity=\"1.0\" stroke=\"rgb(0,0,0)\" stroke-width=\"2.0\" x1=\"74.6\" x2=\"74.6\" y1=\"320.0\" y2=\"280.0\" />\n",
       "<line opacity=\"1.0\" stroke=\"rgb(0,0,0)\" stroke-width=\"2.0\" x1=\"74.6\" x2=\"109.3\" y1=\"280.0\" y2=\"260.0\" />\n",
       "<line opacity=\"1.0\" stroke=\"rgb(0,0,0)\" stroke-width=\"2.0\" x1=\"83.4\" x2=\"107.7\" y1=\"283.2\" y2=\"269.2\" />\n",
       "<line opacity=\"1.0\" stroke=\"rgb(0,0,0)\" stroke-width=\"2.0\" x1=\"109.3\" x2=\"143.9\" y1=\"260.0\" y2=\"280.0\" />\n",
       "<line opacity=\"1.0\" stroke=\"rgb(0,0,0)\" stroke-width=\"2.0\" x1=\"143.9\" x2=\"143.9\" y1=\"280.0\" y2=\"320.0\" />\n",
       "<line opacity=\"1.0\" stroke=\"rgb(0,0,0)\" stroke-width=\"2.0\" x1=\"136.7\" x2=\"136.7\" y1=\"286.0\" y2=\"314.0\" />\n",
       "<line opacity=\"1.0\" stroke=\"rgb(0,0,0)\" stroke-width=\"2.0\" x1=\"178.6\" x2=\"201.9\" y1=\"380.0\" y2=\"393.5\" />\n",
       "<line opacity=\"1.0\" stroke=\"rgb(0,0,0)\" stroke-width=\"2.0\" x1=\"224.5\" x2=\"247.8\" y1=\"393.5\" y2=\"380.0\" />\n",
       "<line opacity=\"1.0\" stroke=\"rgb(0,0,0)\" stroke-width=\"2.0\" x1=\"247.8\" x2=\"224.5\" y1=\"340.0\" y2=\"326.5\" />\n",
       "<line opacity=\"1.0\" stroke=\"rgb(0,0,0)\" stroke-width=\"2.0\" x1=\"201.9\" x2=\"178.6\" y1=\"326.5\" y2=\"340.0\" />\n",
       "<line opacity=\"1.0\" stroke=\"rgb(0,0,0)\" stroke-width=\"2.0\" x1=\"178.6\" x2=\"178.6\" y1=\"340.0\" y2=\"380.0\" />\n",
       "<text fill=\"rgb(12,12,255)\" font-size=\"16\" font-weight=\"bold\" stroke-width=\"0\" x=\"347.769145\" y=\"328.000000\">NH</text>\n",
       "<text fill=\"rgb(12,12,255)\" font-size=\"13\" font-weight=\"bold\" stroke-width=\"0\" x=\"371.769145\" y=\"331.680000\">2</text>\n",
       "<text fill=\"rgb(12,12,255)\" font-size=\"16\" font-weight=\"bold\" stroke-width=\"0\" x=\"311.128129\" y=\"388.000000\">N</text>\n",
       "<text fill=\"rgb(255,12,12)\" font-size=\"16\" font-weight=\"bold\" stroke-width=\"0\" x=\"276.487113\" y=\"448.000000\">OH</text>\n",
       "<text fill=\"rgb(12,12,255)\" font-size=\"16\" font-weight=\"bold\" stroke-width=\"0\" x=\"276.487113\" y=\"328.000000\">N</text>\n",
       "<text fill=\"rgb(12,12,255)\" font-size=\"16\" font-weight=\"bold\" stroke-width=\"0\" x=\"207.205081\" y=\"328.000000\">N</text>\n",
       "<text fill=\"rgb(12,12,255)\" font-size=\"16\" font-weight=\"bold\" stroke-width=\"0\" x=\"207.205081\" y=\"312.000000\">H</text>\n",
       "<text fill=\"rgb(12,12,255)\" font-size=\"16\" font-weight=\"bold\" stroke-width=\"0\" x=\"91.282032\" y=\"388.000000\">HN</text>\n",
       "<text fill=\"rgb(255,12,12)\" font-size=\"16\" font-weight=\"bold\" stroke-width=\"0\" x=\"68.641016\" y=\"208.000000\">O</text>\n",
       "<text fill=\"rgb(12,12,255)\" font-size=\"16\" font-weight=\"bold\" stroke-width=\"0\" x=\"137.923048\" y=\"208.000000\">NH</text>\n",
       "<text fill=\"rgb(255,12,12)\" font-size=\"16\" font-weight=\"bold\" stroke-width=\"0\" x=\"34.000000\" y=\"108.000000\">O</text>\n",
       "<text fill=\"rgb(255,12,12)\" font-size=\"16\" font-weight=\"bold\" stroke-width=\"0\" x=\"79.441016\" y=\"25.888000\">_</text>\n",
       "<text fill=\"rgb(255,12,12)\" font-size=\"16\" font-weight=\"bold\" stroke-width=\"0\" x=\"68.641016\" y=\"48.000000\">O</text>\n",
       "<text fill=\"rgb(255,12,12)\" font-size=\"16\" font-weight=\"bold\" stroke-width=\"0\" x=\"172.564065\" y=\"108.000000\">O</text>\n",
       "<text fill=\"rgb(255,12,12)\" font-size=\"16\" font-weight=\"bold\" stroke-width=\"0\" x=\"218.005081\" y=\"145.888000\">_</text>\n",
       "<text fill=\"rgb(255,12,12)\" font-size=\"16\" font-weight=\"bold\" stroke-width=\"0\" x=\"207.205081\" y=\"168.000000\">O</text>\n",
       "<text fill=\"rgb(12,12,255)\" font-size=\"16\" font-weight=\"bold\" stroke-width=\"0\" x=\"207.205081\" y=\"408.000000\">N</text>\n",
       "<text fill=\"rgb(255,12,12)\" font-size=\"16\" font-weight=\"bold\" stroke-width=\"0\" x=\"241.846097\" y=\"468.000000\">O</text>\n",
       "</svg>\n"
      ],
      "text/plain": [
       "<openbabel.pybel.Molecule at 0x7fbdc98e3490>"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pb.readstring('smi', train[\"Smiles\"][12])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "37a0fa5d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/svg+xml": [
       "<svg xmlns=\"http://www.w3.org/2000/svg\" font-family=\"sans-serif\" height=\"100\" stroke=\"rgb(0,0,0)\" stroke-linecap=\"round\" stroke-width=\"2\" viewBox=\"0 0 391.769 500\" width=\"100\" x=\"0\" y=\"0\">\n",
       "<line opacity=\"1.0\" stroke=\"rgb(0,0,0)\" stroke-width=\"2.0\" x1=\"340.5\" x2=\"317.1\" y1=\"326.5\" y2=\"340.0\" />\n",
       "<line opacity=\"1.0\" stroke=\"rgb(0,0,0)\" stroke-width=\"2.0\" x1=\"282.5\" x2=\"282.5\" y1=\"400.0\" y2=\"427.0\" />\n",
       "<line opacity=\"1.0\" stroke=\"rgb(0,0,0)\" stroke-width=\"2.0\" x1=\"178.6\" x2=\"143.9\" y1=\"380.0\" y2=\"400.0\" />\n",
       "<line opacity=\"1.0\" stroke=\"rgb(0,0,0)\" stroke-width=\"2.0\" x1=\"143.9\" x2=\"120.5\" y1=\"400.0\" y2=\"386.5\" />\n",
       "<line opacity=\"1.0\" stroke=\"rgb(0,0,0)\" stroke-width=\"2.0\" x1=\"109.3\" x2=\"109.3\" y1=\"367.0\" y2=\"340.0\" />\n",
       "<line opacity=\"1.0\" stroke=\"rgb(0,0,0)\" stroke-width=\"2.0\" x1=\"109.3\" x2=\"109.3\" y1=\"260.0\" y2=\"220.0\" />\n",
       "<line opacity=\"1.0\" stroke=\"rgb(0,0,0)\" stroke-width=\"2.0\" x1=\"107.8\" x2=\"84.4\" y1=\"222.6\" y2=\"209.1\" />\n",
       "<line opacity=\"1.0\" stroke=\"rgb(0,0,0)\" stroke-width=\"2.0\" x1=\"110.8\" x2=\"87.4\" y1=\"217.4\" y2=\"203.9\" />\n",
       "<line opacity=\"1.0\" stroke=\"rgb(0,0,0)\" stroke-width=\"2.0\" x1=\"109.3\" x2=\"132.7\" y1=\"220.0\" y2=\"206.5\" />\n",
       "<line opacity=\"1.0\" stroke=\"rgb(0,0,0)\" stroke-width=\"2.0\" x1=\"143.9\" x2=\"143.9\" y1=\"187.0\" y2=\"160.0\" />\n",
       "<polygon fill=\"rgb(0,0,0)\" points=\"143.923048 160.000000 107.282032 143.464102 111.282032 136.535898 \" stroke=\"rgb(0,0,0)\" stroke-width=\"2.000000\" />\n",
       "<line opacity=\"1.0\" stroke=\"rgb(0,0,0)\" stroke-width=\"2.0\" x1=\"109.3\" x2=\"109.3\" y1=\"140.0\" y2=\"100.0\" />\n",
       "<line opacity=\"1.0\" stroke=\"rgb(0,0,0)\" stroke-width=\"2.0\" x1=\"109.3\" x2=\"74.6\" y1=\"100.0\" y2=\"80.0\" />\n",
       "<line opacity=\"1.0\" stroke=\"rgb(0,0,0)\" stroke-width=\"2.0\" x1=\"76.1\" x2=\"52.8\" y1=\"82.6\" y2=\"96.1\" />\n",
       "<line opacity=\"1.0\" stroke=\"rgb(0,0,0)\" stroke-width=\"2.0\" x1=\"73.1\" x2=\"49.8\" y1=\"77.4\" y2=\"90.9\" />\n",
       "<line opacity=\"1.0\" stroke=\"rgb(0,0,0)\" stroke-width=\"2.0\" x1=\"74.6\" x2=\"74.6\" y1=\"80.0\" y2=\"53.0\" />\n",
       "<line opacity=\"1.0\" stroke=\"rgb(0,0,0)\" stroke-width=\"2.0\" x1=\"143.9\" x2=\"178.6\" y1=\"160.0\" y2=\"140.0\" />\n",
       "<line opacity=\"1.0\" stroke=\"rgb(0,0,0)\" stroke-width=\"2.0\" x1=\"175.6\" x2=\"175.6\" y1=\"140.0\" y2=\"113.0\" />\n",
       "<line opacity=\"1.0\" stroke=\"rgb(0,0,0)\" stroke-width=\"2.0\" x1=\"181.6\" x2=\"181.6\" y1=\"140.0\" y2=\"113.0\" />\n",
       "<line opacity=\"1.0\" stroke=\"rgb(0,0,0)\" stroke-width=\"2.0\" x1=\"178.6\" x2=\"201.9\" y1=\"140.0\" y2=\"153.5\" />\n",
       "<line opacity=\"1.0\" stroke=\"rgb(0,0,0)\" stroke-width=\"2.0\" x1=\"213.2\" x2=\"213.2\" y1=\"413.0\" y2=\"440.0\" />\n",
       "<line opacity=\"1.0\" stroke=\"rgb(0,0,0)\" stroke-width=\"2.0\" x1=\"214.7\" x2=\"238.1\" y1=\"437.4\" y2=\"450.9\" />\n",
       "<line opacity=\"1.0\" stroke=\"rgb(0,0,0)\" stroke-width=\"2.0\" x1=\"211.7\" x2=\"235.1\" y1=\"442.6\" y2=\"456.1\" />\n",
       "<line opacity=\"1.0\" stroke=\"rgb(0,0,0)\" stroke-width=\"2.0\" x1=\"317.1\" x2=\"317.1\" y1=\"367.0\" y2=\"340.0\" />\n",
       "<line opacity=\"1.0\" stroke=\"rgb(0,0,0)\" stroke-width=\"2.0\" x1=\"317.1\" x2=\"293.9\" y1=\"340.0\" y2=\"326.6\" />\n",
       "<line opacity=\"1.0\" stroke=\"rgb(0,0,0)\" stroke-width=\"2.0\" x1=\"308.3\" x2=\"292.1\" y1=\"343.2\" y2=\"333.8\" />\n",
       "<line opacity=\"1.0\" stroke=\"rgb(0,0,0)\" stroke-width=\"2.0\" x1=\"271.2\" x2=\"247.8\" y1=\"326.5\" y2=\"340.0\" />\n",
       "<line opacity=\"1.0\" stroke=\"rgb(0,0,0)\" stroke-width=\"2.0\" x1=\"247.8\" x2=\"247.8\" y1=\"340.0\" y2=\"380.0\" />\n",
       "<line opacity=\"1.0\" stroke=\"rgb(0,0,0)\" stroke-width=\"2.0\" x1=\"255.0\" x2=\"255.0\" y1=\"346.0\" y2=\"374.0\" />\n",
       "<line opacity=\"1.0\" stroke=\"rgb(0,0,0)\" stroke-width=\"2.0\" x1=\"247.8\" x2=\"282.5\" y1=\"380.0\" y2=\"400.0\" />\n",
       "<line opacity=\"1.0\" stroke=\"rgb(0,0,0)\" stroke-width=\"2.0\" x1=\"282.5\" x2=\"305.7\" y1=\"400.0\" y2=\"386.6\" />\n",
       "<line opacity=\"1.0\" stroke=\"rgb(0,0,0)\" stroke-width=\"2.0\" x1=\"284.1\" x2=\"300.4\" y1=\"390.8\" y2=\"381.4\" />\n",
       "<line opacity=\"1.0\" stroke=\"rgb(0,0,0)\" stroke-width=\"2.0\" x1=\"143.9\" x2=\"109.3\" y1=\"320.0\" y2=\"340.0\" />\n",
       "<line opacity=\"1.0\" stroke=\"rgb(0,0,0)\" stroke-width=\"2.0\" x1=\"109.3\" x2=\"74.6\" y1=\"340.0\" y2=\"320.0\" />\n",
       "<line opacity=\"1.0\" stroke=\"rgb(0,0,0)\" stroke-width=\"2.0\" x1=\"107.7\" x2=\"83.4\" y1=\"330.8\" y2=\"316.8\" />\n",
       "<line opacity=\"1.0\" stroke=\"rgb(0,0,0)\" stroke-width=\"2.0\" x1=\"74.6\" x2=\"74.6\" y1=\"320.0\" y2=\"280.0\" />\n",
       "<line opacity=\"1.0\" stroke=\"rgb(0,0,0)\" stroke-width=\"2.0\" x1=\"74.6\" x2=\"109.3\" y1=\"280.0\" y2=\"260.0\" />\n",
       "<line opacity=\"1.0\" stroke=\"rgb(0,0,0)\" stroke-width=\"2.0\" x1=\"83.4\" x2=\"107.7\" y1=\"283.2\" y2=\"269.2\" />\n",
       "<line opacity=\"1.0\" stroke=\"rgb(0,0,0)\" stroke-width=\"2.0\" x1=\"109.3\" x2=\"143.9\" y1=\"260.0\" y2=\"280.0\" />\n",
       "<line opacity=\"1.0\" stroke=\"rgb(0,0,0)\" stroke-width=\"2.0\" x1=\"143.9\" x2=\"143.9\" y1=\"280.0\" y2=\"320.0\" />\n",
       "<line opacity=\"1.0\" stroke=\"rgb(0,0,0)\" stroke-width=\"2.0\" x1=\"136.7\" x2=\"136.7\" y1=\"286.0\" y2=\"314.0\" />\n",
       "<line opacity=\"1.0\" stroke=\"rgb(0,0,0)\" stroke-width=\"2.0\" x1=\"178.6\" x2=\"201.9\" y1=\"380.0\" y2=\"393.5\" />\n",
       "<line opacity=\"1.0\" stroke=\"rgb(0,0,0)\" stroke-width=\"2.0\" x1=\"224.5\" x2=\"247.8\" y1=\"393.5\" y2=\"380.0\" />\n",
       "<line opacity=\"1.0\" stroke=\"rgb(0,0,0)\" stroke-width=\"2.0\" x1=\"247.8\" x2=\"224.5\" y1=\"340.0\" y2=\"326.5\" />\n",
       "<line opacity=\"1.0\" stroke=\"rgb(0,0,0)\" stroke-width=\"2.0\" x1=\"201.9\" x2=\"178.6\" y1=\"326.5\" y2=\"340.0\" />\n",
       "<line opacity=\"1.0\" stroke=\"rgb(0,0,0)\" stroke-width=\"2.0\" x1=\"178.6\" x2=\"178.6\" y1=\"340.0\" y2=\"380.0\" />\n",
       "<text fill=\"rgb(12,12,255)\" font-size=\"16\" font-weight=\"bold\" stroke-width=\"0\" x=\"347.769145\" y=\"328.000000\">NH</text>\n",
       "<text fill=\"rgb(12,12,255)\" font-size=\"13\" font-weight=\"bold\" stroke-width=\"0\" x=\"371.769145\" y=\"331.680000\">2</text>\n",
       "<text fill=\"rgb(12,12,255)\" font-size=\"16\" font-weight=\"bold\" stroke-width=\"0\" x=\"311.128129\" y=\"388.000000\">N</text>\n",
       "<text fill=\"rgb(255,12,12)\" font-size=\"16\" font-weight=\"bold\" stroke-width=\"0\" x=\"276.487113\" y=\"448.000000\">OH</text>\n",
       "<text fill=\"rgb(12,12,255)\" font-size=\"16\" font-weight=\"bold\" stroke-width=\"0\" x=\"276.487113\" y=\"328.000000\">N</text>\n",
       "<text fill=\"rgb(12,12,255)\" font-size=\"16\" font-weight=\"bold\" stroke-width=\"0\" x=\"207.205081\" y=\"328.000000\">N</text>\n",
       "<text fill=\"rgb(12,12,255)\" font-size=\"16\" font-weight=\"bold\" stroke-width=\"0\" x=\"207.205081\" y=\"312.000000\">H</text>\n",
       "<text fill=\"rgb(12,12,255)\" font-size=\"16\" font-weight=\"bold\" stroke-width=\"0\" x=\"91.282032\" y=\"388.000000\">HN</text>\n",
       "<text fill=\"rgb(255,12,12)\" font-size=\"16\" font-weight=\"bold\" stroke-width=\"0\" x=\"68.641016\" y=\"208.000000\">O</text>\n",
       "<text fill=\"rgb(12,12,255)\" font-size=\"16\" font-weight=\"bold\" stroke-width=\"0\" x=\"137.923048\" y=\"208.000000\">NH</text>\n",
       "<text fill=\"rgb(255,12,12)\" font-size=\"16\" font-weight=\"bold\" stroke-width=\"0\" x=\"34.000000\" y=\"108.000000\">O</text>\n",
       "<text fill=\"rgb(255,12,12)\" font-size=\"16\" font-weight=\"bold\" stroke-width=\"0\" x=\"68.641016\" y=\"48.000000\">OH</text>\n",
       "<text fill=\"rgb(255,12,12)\" font-size=\"16\" font-weight=\"bold\" stroke-width=\"0\" x=\"172.564065\" y=\"108.000000\">O</text>\n",
       "<text fill=\"rgb(255,12,12)\" font-size=\"16\" font-weight=\"bold\" stroke-width=\"0\" x=\"207.205081\" y=\"168.000000\">OH</text>\n",
       "<text fill=\"rgb(12,12,255)\" font-size=\"16\" font-weight=\"bold\" stroke-width=\"0\" x=\"207.205081\" y=\"408.000000\">N</text>\n",
       "<text fill=\"rgb(255,12,12)\" font-size=\"16\" font-weight=\"bold\" stroke-width=\"0\" x=\"241.846097\" y=\"468.000000\">O</text>\n",
       "</svg>\n"
      ],
      "text/plain": [
       "<openbabel.pybel.Molecule at 0x7fbdc96e0210>"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "neutralize(pb.readstring('smi', train[\"Smiles\"][12]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "8ee61f24",
   "metadata": {},
   "outputs": [],
   "source": [
    "train[\"Smiles\"] = train[\"Smiles\"].apply(lambda x: neutralize(pb.readstring(\"smi\", x)).write().strip())\n",
    "test[\"Smiles\"] = test[\"Smiles\"].apply(lambda x: neutralize(pb.readstring(\"smi\", x)).write().strip())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "d8f14fae",
   "metadata": {},
   "outputs": [],
   "source": [
    "# no effect on duplicates\n",
    "train[\"Smiles\"] = train[\"Smiles\"].apply(standardize_smiles)\n",
    "test[\"Smiles\"] = test[\"Smiles\"].apply(standardize_smiles)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "5d76a792",
   "metadata": {},
   "outputs": [],
   "source": [
    "train = train.drop_duplicates(subset=['Smiles']).reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "1327320b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Smiles</th>\n",
       "      <th>Active</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>COc1ccc2[nH]cc(CCN)c2c1</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>CCCN1CCC[C@H](c2cccc(O)c2)C1</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>O=C(NO)c1cnc(N2CCN(S(=O)(=O)c3ccc4ccccc4c3)CC2...</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Nc1cccc(CNC(=O)c2ccc(Oc3ccc(OCc4cccc(F)c4)cc3)...</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Fc1ccccc1CNCc1ccc(-c2ccnc3[nH]ccc23)cc1</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5354</th>\n",
       "      <td>COc1cc(N2CCN(C)CC2)ccc1Nc1ncc2c(n1)N(C1CCC1)C(...</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5355</th>\n",
       "      <td>O=C(Oc1ccc([N+](=O)[O-])cc1)N1CCC(C(O)(c2ccc3c...</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5356</th>\n",
       "      <td>Nc1nonc1/C(=N/O)Nc1ccc(F)c(Br)c1</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5357</th>\n",
       "      <td>Oc1cccc2cccnc12</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5358</th>\n",
       "      <td>OC(c1ccc(-c2ccc(CN3CCN(Cc4ccncc4)CC3)cc2)c(F)c...</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5359 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                 Smiles  Active\n",
       "0                               COc1ccc2[nH]cc(CCN)c2c1   False\n",
       "1                          CCCN1CCC[C@H](c2cccc(O)c2)C1   False\n",
       "2     O=C(NO)c1cnc(N2CCN(S(=O)(=O)c3ccc4ccccc4c3)CC2...   False\n",
       "3     Nc1cccc(CNC(=O)c2ccc(Oc3ccc(OCc4cccc(F)c4)cc3)...   False\n",
       "4               Fc1ccccc1CNCc1ccc(-c2ccnc3[nH]ccc23)cc1   False\n",
       "...                                                 ...     ...\n",
       "5354  COc1cc(N2CCN(C)CC2)ccc1Nc1ncc2c(n1)N(C1CCC1)C(...   False\n",
       "5355  O=C(Oc1ccc([N+](=O)[O-])cc1)N1CCC(C(O)(c2ccc3c...   False\n",
       "5356                   Nc1nonc1/C(=N/O)Nc1ccc(F)c(Br)c1   False\n",
       "5357                                    Oc1cccc2cccnc12   False\n",
       "5358  OC(c1ccc(-c2ccc(CN3CCN(Cc4ccncc4)CC3)cc2)c(F)c...   False\n",
       "\n",
       "[5359 rows x 2 columns]"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "18ee5758",
   "metadata": {},
   "outputs": [],
   "source": [
    "diff_df = pd.merge(test, train, how='left', on=\"Smiles\", indicator='Exists')\n",
    "diff_df = diff_df[diff_df[\"Exists\"]==\"both\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "efe2cb70",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Smiles</th>\n",
       "      <th>Active</th>\n",
       "      <th>Exists</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>NC(CO)C(=O)NNCc1ccc(O)c(O)c1O</td>\n",
       "      <td>True</td>\n",
       "      <td>both</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>COc1cccc2c1C(=O)c1c(O)c3c(c(O)c1C2=O)C[C@@](O)...</td>\n",
       "      <td>False</td>\n",
       "      <td>both</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41</th>\n",
       "      <td>COc1ccnc(C[S+]([O-])c2nc3cc(OC(F)F)ccc3[nH]2)c1OC</td>\n",
       "      <td>True</td>\n",
       "      <td>both</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>54</th>\n",
       "      <td>CC(C)Nc1cccnc1N1CCN(C(=O)c2cc3cc(NS(C)(=O)=O)c...</td>\n",
       "      <td>False</td>\n",
       "      <td>both</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>60</th>\n",
       "      <td>CN(C)C(=O)Oc1cc(OC(=O)N(C)C)cc(C(O)CNC(C)(C)C)c1</td>\n",
       "      <td>False</td>\n",
       "      <td>both</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1544</th>\n",
       "      <td>CCCCCCOc1nsnc1C1=CCCN(C)C1</td>\n",
       "      <td>False</td>\n",
       "      <td>both</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1547</th>\n",
       "      <td>NC[C@@H]1O[C@H](O[C@@H]2[C@@H](CO)O[C@@H](O[C@...</td>\n",
       "      <td>False</td>\n",
       "      <td>both</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1583</th>\n",
       "      <td>N/C(=N\\C(=O)c1nc(Cl)c(N)nc1N)NCc1ccccc1</td>\n",
       "      <td>False</td>\n",
       "      <td>both</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1593</th>\n",
       "      <td>Oc1cccc2cccnc12</td>\n",
       "      <td>False</td>\n",
       "      <td>both</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1607</th>\n",
       "      <td>CN=C(N)NC(=O)Nc1c(C)cccc1C</td>\n",
       "      <td>False</td>\n",
       "      <td>both</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>100 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                 Smiles Active Exists\n",
       "2                         NC(CO)C(=O)NNCc1ccc(O)c(O)c1O   True   both\n",
       "27    COc1cccc2c1C(=O)c1c(O)c3c(c(O)c1C2=O)C[C@@](O)...  False   both\n",
       "41    COc1ccnc(C[S+]([O-])c2nc3cc(OC(F)F)ccc3[nH]2)c1OC   True   both\n",
       "54    CC(C)Nc1cccnc1N1CCN(C(=O)c2cc3cc(NS(C)(=O)=O)c...  False   both\n",
       "60     CN(C)C(=O)Oc1cc(OC(=O)N(C)C)cc(C(O)CNC(C)(C)C)c1  False   both\n",
       "...                                                 ...    ...    ...\n",
       "1544                         CCCCCCOc1nsnc1C1=CCCN(C)C1  False   both\n",
       "1547  NC[C@@H]1O[C@H](O[C@@H]2[C@@H](CO)O[C@@H](O[C@...  False   both\n",
       "1583            N/C(=N\\C(=O)c1nc(Cl)c(N)nc1N)NCc1ccccc1  False   both\n",
       "1593                                    Oc1cccc2cccnc12  False   both\n",
       "1607                         CN=C(N)NC(=O)Nc1c(C)cccc1C  False   both\n",
       "\n",
       "[100 rows x 3 columns]"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "diff_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "faac40d8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Smiles</th>\n",
       "      <th>Active</th>\n",
       "      <th>Exists</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>NC(CO)C(=O)NNCc1ccc(O)c(O)c1O</td>\n",
       "      <td>True</td>\n",
       "      <td>both</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41</th>\n",
       "      <td>COc1ccnc(C[S+]([O-])c2nc3cc(OC(F)F)ccc3[nH]2)c1OC</td>\n",
       "      <td>True</td>\n",
       "      <td>both</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>453</th>\n",
       "      <td>CCN(CC)C(=S)S</td>\n",
       "      <td>True</td>\n",
       "      <td>both</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1173</th>\n",
       "      <td>Nc1ccc2cc3ccc(N)cc3nc2c1</td>\n",
       "      <td>True</td>\n",
       "      <td>both</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                 Smiles Active Exists\n",
       "2                         NC(CO)C(=O)NNCc1ccc(O)c(O)c1O   True   both\n",
       "41    COc1ccnc(C[S+]([O-])c2nc3cc(OC(F)F)ccc3[nH]2)c1OC   True   both\n",
       "453                                       CCN(CC)C(=S)S   True   both\n",
       "1173                           Nc1ccc2cc3ccc(N)cc3nc2c1   True   both"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "diff_df[diff_df[\"Active\"] == True]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "4b0ed60b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2, 41, 453, 1173]\n"
     ]
    }
   ],
   "source": [
    "# indices in test which are True\n",
    "print(list(diff_df[diff_df[\"Active\"] == True].index))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "c0318fe4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Smiles</th>\n",
       "      <th>Active</th>\n",
       "      <th>Exists</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>COc1cccc2c1C(=O)c1c(O)c3c(c(O)c1C2=O)C[C@@](O)...</td>\n",
       "      <td>False</td>\n",
       "      <td>both</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>54</th>\n",
       "      <td>CC(C)Nc1cccnc1N1CCN(C(=O)c2cc3cc(NS(C)(=O)=O)c...</td>\n",
       "      <td>False</td>\n",
       "      <td>both</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>60</th>\n",
       "      <td>CN(C)C(=O)Oc1cc(OC(=O)N(C)C)cc(C(O)CNC(C)(C)C)c1</td>\n",
       "      <td>False</td>\n",
       "      <td>both</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>167</th>\n",
       "      <td>Cn1c(=O)cc(N2CCC[C@@H](N)C2)n(Cc2ccccc2C#N)c1=O</td>\n",
       "      <td>False</td>\n",
       "      <td>both</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>177</th>\n",
       "      <td>CN(C)CC(c1ccc(O)cc1)C1(O)CCCCC1</td>\n",
       "      <td>False</td>\n",
       "      <td>both</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1544</th>\n",
       "      <td>CCCCCCOc1nsnc1C1=CCCN(C)C1</td>\n",
       "      <td>False</td>\n",
       "      <td>both</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1547</th>\n",
       "      <td>NC[C@@H]1O[C@H](O[C@@H]2[C@@H](CO)O[C@@H](O[C@...</td>\n",
       "      <td>False</td>\n",
       "      <td>both</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1583</th>\n",
       "      <td>N/C(=N\\C(=O)c1nc(Cl)c(N)nc1N)NCc1ccccc1</td>\n",
       "      <td>False</td>\n",
       "      <td>both</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1593</th>\n",
       "      <td>Oc1cccc2cccnc12</td>\n",
       "      <td>False</td>\n",
       "      <td>both</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1607</th>\n",
       "      <td>CN=C(N)NC(=O)Nc1c(C)cccc1C</td>\n",
       "      <td>False</td>\n",
       "      <td>both</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>96 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                 Smiles Active Exists\n",
       "27    COc1cccc2c1C(=O)c1c(O)c3c(c(O)c1C2=O)C[C@@](O)...  False   both\n",
       "54    CC(C)Nc1cccnc1N1CCN(C(=O)c2cc3cc(NS(C)(=O)=O)c...  False   both\n",
       "60     CN(C)C(=O)Oc1cc(OC(=O)N(C)C)cc(C(O)CNC(C)(C)C)c1  False   both\n",
       "167     Cn1c(=O)cc(N2CCC[C@@H](N)C2)n(Cc2ccccc2C#N)c1=O  False   both\n",
       "177                     CN(C)CC(c1ccc(O)cc1)C1(O)CCCCC1  False   both\n",
       "...                                                 ...    ...    ...\n",
       "1544                         CCCCCCOc1nsnc1C1=CCCN(C)C1  False   both\n",
       "1547  NC[C@@H]1O[C@H](O[C@@H]2[C@@H](CO)O[C@@H](O[C@...  False   both\n",
       "1583            N/C(=N\\C(=O)c1nc(Cl)c(N)nc1N)NCc1ccccc1  False   both\n",
       "1593                                    Oc1cccc2cccnc12  False   both\n",
       "1607                         CN=C(N)NC(=O)Nc1c(C)cccc1C  False   both\n",
       "\n",
       "[96 rows x 3 columns]"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "diff_df[diff_df[\"Active\"] == False]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "e1fb8ee5",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[27, 54, 60, 167, 177, 187, 213, 220, 239, 255, 256, 260, 272, 296, 358, 366, 372, 386, 389, 402, 427, 447, 455, 487, 489, 492, 497, 521, 529, 531, 539, 542, 545, 564, 572, 590, 654, 662, 695, 698, 703, 740, 754, 779, 793, 805, 824, 832, 852, 853, 871, 888, 922, 951, 954, 961, 966, 974, 979, 990, 992, 1047, 1053, 1057, 1082, 1099, 1101, 1114, 1135, 1154, 1156, 1159, 1191, 1192, 1209, 1230, 1233, 1242, 1249, 1355, 1358, 1367, 1389, 1396, 1411, 1425, 1495, 1501, 1522, 1523, 1542, 1544, 1547, 1583, 1593, 1607]\n"
     ]
    }
   ],
   "source": [
    "# indices in test which are False\n",
    "print(list(diff_df[diff_df[\"Active\"] == False].index))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "589bc94f",
   "metadata": {},
   "outputs": [],
   "source": [
    "train.to_csv(\"Clean/train.csv\")\n",
    "test.to_csv(\"Clean/test.csv\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}