my_devel.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "from os.path import join\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from scipy import sparse\n",
    "import pickle\n",
    "import torch\n",
    "import torch.optim as optim\n",
    "from torch.utils.data import Dataset\n",
    "from utils import *\n",
    "from tqdm import tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "class TextAndNearestNeighborsDataset(Dataset):\n",
    "\n",
    "    def __init__(self, dataset_name, data_dir, subset='train'):\n",
    "        \"\"\"\n",
    "        Args:\n",
    "            data_dir (string): Directory for loading and saving train, test, and cv dataframes.\n",
    "            subset (string): Specify subset of the datasets. The choices are: train, test, cv.\n",
    "        \"\"\"\n",
    "        self.dataset_name = dataset_name\n",
    "        self.data_dir = os.path.join(data_dir, dataset_name)\n",
    "        self.subset = subset\n",
    "        fn = '{}.{}.pkl'.format(dataset_name, subset)\n",
    "        self.df = self.load_df(self.data_dir, fn)\n",
    "        self.docid2index = {docid: index for index, docid in enumerate(list(self.df.index))}\n",
    "        \n",
    "        if dataset_name in ['reuters', 'rcv1', 'tmc']:\n",
    "            self.single_label = False\n",
    "        else:\n",
    "            self.single_label = True\n",
    "\n",
    "    def load_df(self, data_dir, df_file):\n",
    "        df_file = os.path.join(data_dir, df_file)\n",
    "        return pd.read_pickle(df_file)\n",
    "        \n",
    "    def __len__(self):\n",
    "        return len(self.df)\n",
    "\n",
    "    def __getitem__(self, idx):\n",
    "        doc_id = self.df.iloc[idx].name\n",
    "        doc_bow = self.df.iloc[idx].bow\n",
    "        doc_bow = torch.from_numpy(doc_bow.toarray().squeeze().astype(np.float32))\n",
    "        \n",
    "        label = self.df.iloc[idx].label\n",
    "        label = torch.from_numpy(label.toarray().squeeze().astype(np.float32))\n",
    "                \n",
    "        neighbors = torch.LongTensor(self.df.iloc[idx].neighbors)\n",
    "        return (doc_id, doc_bow, label, neighbors)\n",
    "    \n",
    "    def num_classes(self):\n",
    "        return self.df.iloc[0].label.shape[1]\n",
    "    \n",
    "    def num_features(self):\n",
    "        return self.df.iloc[0].bow.shape[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset_name = 'reuters'\n",
    "\n",
    "train_set = TextAndNearestNeighborsDataset('reuters', 'dataset/clean', 'train')\n",
    "train_loader = torch.utils.data.DataLoader(dataset=train_set, batch_size=100, shuffle=False)\n",
    "test_set = TextAndNearestNeighborsDataset('reuters', 'dataset/clean', 'test')\n",
    "test_loader = torch.utils.data.DataLoader(dataset=test_set, batch_size=100, shuffle=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Walk type: BFS with maximum nodes of: 50\n"
     ]
    }
   ],
   "source": [
    "def BFS_walk(df, start_node_id, num_steps, max_branch_factor=50):\n",
    "    if isinstance(start_node_id, list):\n",
    "        queue = list(start_node_id)\n",
    "    else:\n",
    "        queue = [start_node_id]\n",
    "        \n",
    "    visited_nodes = set()\n",
    "    curr_step = 0\n",
    "    while len(queue) > 0:         \n",
    "        curr_node_id = queue.pop(0)\n",
    "        while curr_node_id in visited_nodes:\n",
    "            if len(queue) <= 0:\n",
    "                #if not isinstance(start_node_id, list):\n",
    "                #    visited_nodes.remove(start_node_id)\n",
    "                return list(visited_nodes)\n",
    "            curr_node_id = queue.pop(0)\n",
    "        \n",
    "        nn_list = list(train_set.df.loc[curr_node_id].neighbors[:max_branch_factor])\n",
    "        #np.random.shuffle(nn_list)\n",
    "        queue += nn_list\n",
    "        visited_nodes.add(curr_node_id)\n",
    "        curr_step += 1\n",
    "        if curr_step > num_steps:\n",
    "            break\n",
    "    \n",
    "    #if not isinstance(start_node_id, list):\n",
    "    #    visited_nodes.remove(start_node_id)\n",
    "    return list(visited_nodes)    \n",
    "\n",
    "walk_type, max_nodes = 'BFS', 50\n",
    "max_nodes = int(max_nodes)\n",
    "print(\"Walk type: {} with maximum nodes of: {}\".format(walk_type, max_nodes))\n",
    "\n",
    "if walk_type == 'BFS':\n",
    "    neighbor_sample_func = BFS_walk\n",
    "elif walk_type == 'DFS':\n",
    "    neighbor_sample_func = DFS_walk\n",
    "elif walk_type == 'Random':\n",
    "    neighbor_sample_func = Random_walk\n",
    "else:\n",
    "    neighbor_sample_func = None\n",
    "    print(\"The model will only takes the immediate neighbors.\")\n",
    "    #assert(False), \"unknown walk type (has to be one of the following: BFS, DFS, Random)\"\n",
    "\n",
    "def get_neighbors(ids, df, max_nodes, batch_size, traversal_func):\n",
    "    cols = []\n",
    "    rows = []\n",
    "    for idx, node_id in enumerate(ids):\n",
    "        nn_indices = traversal_func(df, node_id.item(), max_nodes)\n",
    "        col = [train_set.docid2index[v] for v in nn_indices]\n",
    "        rows += [idx] * len(col)\n",
    "        cols += col\n",
    "    data = [1] * len(cols)\n",
    "    connections = sparse.csr_matrix((data, (rows, cols)), shape=(batch_size, len(df)))\n",
    "    return torch.from_numpy(connections.toarray()).type(torch.FloatTensor)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "os.environ[\"CUDA_VISIBLE_DEVICES\"]=\"1\"\n",
    "device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_dim = train_set.num_classes()\n",
    "num_bits = 8\n",
    "num_features = train_set[0][1].size(0)\n",
    "num_nodes = len(train_set)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "EdgeReg(\n",
       "  (encoder): Sequential(\n",
       "    (0): Linear(in_features=10000, out_features=1000, bias=True)\n",
       "    (1): ReLU(inplace)\n",
       "    (2): Linear(in_features=1000, out_features=1000, bias=True)\n",
       "    (3): ReLU(inplace)\n",
       "    (4): Dropout(p=0.1)\n",
       "  )\n",
       "  (h_to_mu): Linear(in_features=1000, out_features=8, bias=True)\n",
       "  (h_to_logvar): Sequential(\n",
       "    (0): Linear(in_features=1000, out_features=8, bias=True)\n",
       "    (1): Sigmoid()\n",
       "  )\n",
       "  (decoder): Sequential(\n",
       "    (0): Linear(in_features=8, out_features=10000, bias=True)\n",
       "    (1): LogSoftmax()\n",
       "  )\n",
       "  (nn_decoder): Sequential(\n",
       "    (0): Linear(in_features=8, out_features=7763, bias=True)\n",
       "    (1): LogSoftmax()\n",
       "  )\n",
       ")"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from model.EdgeReg import *\n",
    "\n",
    "model = EdgeReg(dataset_name, num_features, num_nodes, num_bits, dropoutProb=0.1, device=device)\n",
    "model.to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████| 78/78 [01:29<00:00,  1.14s/it]\n",
      "  0%|                      | 0/78 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "precision at 100: 0.6618\n",
      "node2hash epoch:1 loss:542.3105 Best Precision:(1)0.662\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████| 78/78 [01:21<00:00,  1.05s/it]\n",
      "  0%|                      | 0/78 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "precision at 100: 0.7203\n",
      "node2hash epoch:2 loss:405.1187 Best Precision:(2)0.720\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████| 78/78 [01:19<00:00,  1.02s/it]\n",
      "  0%|                      | 0/78 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "precision at 100: 0.7145\n",
      "node2hash epoch:3 loss:382.8038 Best Precision:(2)0.720\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████| 78/78 [01:16<00:00,  1.02it/s]\n",
      "  0%|                      | 0/78 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "precision at 100: 0.6988\n",
      "node2hash epoch:4 loss:375.1534 Best Precision:(2)0.720\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████| 78/78 [01:15<00:00,  1.03it/s]\n",
      "  0%|                      | 0/78 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "precision at 100: 0.6872\n",
      "node2hash epoch:5 loss:371.6333 Best Precision:(2)0.720\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████| 78/78 [01:16<00:00,  1.02it/s]\n",
      "  0%|                      | 0/78 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "precision at 100: 0.6904\n",
      "node2hash epoch:6 loss:369.6670 Best Precision:(2)0.720\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████| 78/78 [01:11<00:00,  1.09it/s]\n",
      "  0%|                      | 0/78 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "precision at 100: 0.7112\n",
      "node2hash epoch:7 loss:368.0412 Best Precision:(2)0.720\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████| 78/78 [01:15<00:00,  1.03it/s]\n",
      "  0%|                      | 0/78 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "precision at 100: 0.6926\n",
      "node2hash epoch:8 loss:367.9449 Best Precision:(2)0.720\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████| 78/78 [01:14<00:00,  1.04it/s]\n",
      "  0%|                      | 0/78 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "precision at 100: 0.6991\n",
      "node2hash epoch:9 loss:367.3656 Best Precision:(2)0.720\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████| 78/78 [01:15<00:00,  1.03it/s]\n",
      "  0%|                      | 0/78 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "precision at 100: 0.6955\n",
      "node2hash epoch:10 loss:367.2956 Best Precision:(2)0.720\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████| 78/78 [01:19<00:00,  1.02s/it]\n",
      "  0%|                      | 0/78 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "precision at 100: 0.6745\n",
      "node2hash epoch:11 loss:367.5660 Best Precision:(2)0.720\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████| 78/78 [01:19<00:00,  1.02s/it]\n",
      "  0%|                      | 0/78 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "precision at 100: 0.6831\n",
      "node2hash epoch:12 loss:367.8835 Best Precision:(2)0.720\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████| 78/78 [01:13<00:00,  1.06it/s]\n",
      "  0%|                      | 0/78 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "precision at 100: 0.6809\n",
      "node2hash epoch:13 loss:368.5842 Best Precision:(2)0.720\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████| 78/78 [01:13<00:00,  1.06it/s]\n",
      "  0%|                      | 0/78 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "precision at 100: 0.6893\n",
      "node2hash epoch:14 loss:368.9242 Best Precision:(2)0.720\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████| 78/78 [01:11<00:00,  1.09it/s]\n",
      "  0%|                      | 0/78 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "precision at 100: 0.6784\n",
      "node2hash epoch:15 loss:370.0969 Best Precision:(2)0.720\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████| 78/78 [01:18<00:00,  1.01s/it]\n",
      "  0%|                      | 0/78 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "precision at 100: 0.6844\n",
      "node2hash epoch:16 loss:369.5918 Best Precision:(2)0.720\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████| 78/78 [01:14<00:00,  1.04it/s]\n",
      "  0%|                      | 0/78 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "precision at 100: 0.6820\n",
      "node2hash epoch:17 loss:369.3615 Best Precision:(2)0.720\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████| 78/78 [01:23<00:00,  1.07s/it]\n",
      "  0%|                      | 0/78 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "precision at 100: 0.6721\n",
      "node2hash epoch:18 loss:369.8176 Best Precision:(2)0.720\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████| 78/78 [01:25<00:00,  1.10s/it]\n",
      "  0%|                      | 0/78 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "precision at 100: 0.6868\n",
      "node2hash epoch:19 loss:370.2511 Best Precision:(2)0.720\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████| 78/78 [01:27<00:00,  1.12s/it]\n",
      "                          "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "precision at 100: 0.6871\n",
      "node2hash epoch:20 loss:370.2210 Best Precision:(2)0.720\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": []
    }
   ],
   "source": [
    "optimizer = optim.Adam(model.parameters(), lr=0.01)\n",
    "kl_weight = 0.\n",
    "kl_step = 1 / 5000.\n",
    "\n",
    "best_precision = 0\n",
    "best_precision_epoch = 0\n",
    "\n",
    "edge_weight = 10.\n",
    "edge_step = 1 / 1000.\n",
    "\n",
    "for epoch in range(20):\n",
    "    avg_loss = []\n",
    "    for ids, xb, yb, nb in tqdm(train_loader, ncols=50):\n",
    "        xb = xb.to(device)\n",
    "        yb = yb.to(device)\n",
    "\n",
    "        nb = get_neighbors(ids, train_set.df, max_nodes, xb.size(0), neighbor_sample_func)\n",
    "        nb = nb.to(device)\n",
    "\n",
    "        logprob_w, logprob_nn, mu, logvar = model(xb)\n",
    "        kl_loss = EdgeReg.calculate_KL_loss(mu, logvar)\n",
    "        reconstr_loss = EdgeReg.compute_reconstr_loss(logprob_w, xb)\n",
    "        nn_reconstr_loss = EdgeReg.compute_edge_reconstr_loss(logprob_nn, nb)\n",
    "\n",
    "        loss = reconstr_loss + edge_weight * nn_reconstr_loss + kl_weight * kl_loss\n",
    "        \n",
    "        optimizer.zero_grad()\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "\n",
    "        kl_weight = min(kl_weight + kl_step, 1.)\n",
    "        edge_weight = min(edge_weight + edge_step, 1.)\n",
    "        avg_loss.append(loss.item())\n",
    "        \n",
    "    with torch.no_grad():\n",
    "        train_b, test_b, train_y, test_y = model.get_binary_code(train_loader, test_loader)\n",
    "        retrieved_indices = retrieve_topk(test_b.to(device), train_b.to(device), topK=100)\n",
    "        prec = compute_precision_at_k(retrieved_indices, test_y.to(device), train_y.to(device), topK=100)\n",
    "        print(\"precision at 100: {:.4f}\".format(prec.item()))\n",
    "\n",
    "        if prec.item() > best_precision:\n",
    "            best_precision = prec.item()\n",
    "            best_precision_epoch = epoch + 1\n",
    "\n",
    "        print('{} epoch:{} loss:{:.4f} Best Precision:({}){:.3f}'.format(model.get_name(), epoch+1, np.mean(avg_loss), best_precision_epoch, best_precision))\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Research2018",
   "language": "python",
   "name": "research"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}