GraphSage-sample-eigen-ft-products_gpu5.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using backend: pytorch\n"
     ]
    }
   ],
   "source": [
    "import dgl\n",
    "import time\n",
    "import torch as th\n",
    "import numpy as np\n",
    "import torch.nn.functional as F\n",
    "\n",
    "from load_data import load_ogb\n",
    "from model import SAGE, GAT, get_eigen\n",
    "from train_eval import train_ft_embed"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This notebook shows the performance of GraphSage on the OGBN-products graph with trainable embeddings on the nodes. We use scaled eigenvectors to initialize the node embeddings. The logic here is that if eigenvectors can serve as positional node embeddings, fine-tuning the node embeddings should give us even better performance. In the experiment below, we tune the node embeddings with a much smaller learning rate than the GNN model."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:root:The OGB package is out of date. Your version is 1.3.1, while the latest version is 1.3.2.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "load ogbn-products\n",
      "finish loading ogbn-products\n",
      "finish constructing ogbn-products\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:6: UserWarning: Casting complex values to real discards the imaginary part (Triggered internally at  /pytorch/aten/src/ATen/native/Copy.cpp:219.)\n",
      "  \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "embedding is in  cuda:5\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/numpy/core/fromnumeric.py:3373: RuntimeWarning: Mean of empty slice.\n",
      "  out=out, **kwargs)\n",
      "/usr/local/lib/python3.6/dist-packages/numpy/core/_methods.py:170: RuntimeWarning: invalid value encountered in double_scalars\n",
      "  ret = ret.dtype.type(ret / rcount)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 00000 | Step 00000 | Loss 3.8511 | Train Acc 0.0176 | Speed (samples/sec) nan | GPU 4989.3 MB\n",
      "Epoch 00000 | Step 00020 | Loss 1.8255 | Train Acc 0.5273 | Speed (samples/sec) 346.1535 | GPU 5001.3 MB\n",
      "Epoch 00000 | Step 00040 | Loss 0.8426 | Train Acc 0.8076 | Speed (samples/sec) 362.6535 | GPU 5005.0 MB\n",
      "Epoch 00000 | Step 00060 | Loss 0.6259 | Train Acc 0.8604 | Speed (samples/sec) 368.2415 | GPU 5005.0 MB\n",
      "Epoch 00000 | Step 00080 | Loss 0.5090 | Train Acc 0.8809 | Speed (samples/sec) 363.2799 | GPU 5005.0 MB\n",
      "Epoch 00000 | Step 00100 | Loss 0.4553 | Train Acc 0.8984 | Speed (samples/sec) 360.6319 | GPU 5005.5 MB\n",
      "Epoch 00000 | Step 00120 | Loss 0.3845 | Train Acc 0.8984 | Speed (samples/sec) 361.3783 | GPU 5010.6 MB\n",
      "Epoch 00000 | Step 00140 | Loss 0.4392 | Train Acc 0.8945 | Speed (samples/sec) 362.8149 | GPU 5010.6 MB\n",
      "Epoch 00000 | Step 00160 | Loss 0.3487 | Train Acc 0.9102 | Speed (samples/sec) 360.3278 | GPU 5010.6 MB\n",
      "Epoch 00000 | Step 00180 | Loss 0.3094 | Train Acc 0.9219 | Speed (samples/sec) 357.6228 | GPU 5010.6 MB\n",
      "Epoch Time(s): 556.7734\n",
      "Epoch 00001 | Step 00000 | Loss 0.2601 | Train Acc 0.9355 | Speed (samples/sec) 354.4830 | GPU 5010.6 MB\n",
      "Epoch 00001 | Step 00020 | Loss 0.2470 | Train Acc 0.9297 | Speed (samples/sec) 353.8008 | GPU 5012.3 MB\n",
      "Epoch 00001 | Step 00040 | Loss 0.2285 | Train Acc 0.9375 | Speed (samples/sec) 352.8139 | GPU 5012.3 MB\n",
      "Epoch 00001 | Step 00060 | Loss 0.2057 | Train Acc 0.9434 | Speed (samples/sec) 350.1079 | GPU 5012.3 MB\n",
      "Epoch 00001 | Step 00080 | Loss 0.2524 | Train Acc 0.9326 | Speed (samples/sec) 347.6590 | GPU 5012.3 MB\n",
      "Epoch 00001 | Step 00100 | Loss 0.2047 | Train Acc 0.9453 | Speed (samples/sec) 347.8332 | GPU 5012.3 MB\n",
      "Epoch 00001 | Step 00120 | Loss 0.2293 | Train Acc 0.9463 | Speed (samples/sec) 348.0498 | GPU 5012.3 MB\n",
      "Epoch 00001 | Step 00140 | Loss 0.2253 | Train Acc 0.9385 | Speed (samples/sec) 347.1675 | GPU 5012.3 MB\n",
      "Epoch 00001 | Step 00160 | Loss 0.2390 | Train Acc 0.9414 | Speed (samples/sec) 346.9579 | GPU 5012.3 MB\n",
      "Epoch 00001 | Step 00180 | Loss 0.2818 | Train Acc 0.9248 | Speed (samples/sec) 347.8940 | GPU 5012.3 MB\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/245 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch Time(s): 576.2032\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 245/245 [00:40<00:00,  6.05it/s]\n",
      "100%|██████████| 245/245 [01:12<00:00,  3.37it/s]\n",
      "100%|██████████| 245/245 [01:15<00:00,  3.24it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val acc 0.9164, Test Acc: 0.7900\n",
      "Best val acc 0.9164, best test acc: 0.7900\n",
      "Epoch 00002 | Step 00000 | Loss 0.1192 | Train Acc 0.9688 | Speed (samples/sec) 347.9365 | GPU 5012.3 MB\n",
      "Epoch 00002 | Step 00020 | Loss 0.1126 | Train Acc 0.9619 | Speed (samples/sec) 347.9582 | GPU 5012.3 MB\n",
      "Epoch 00002 | Step 00040 | Loss 0.1717 | Train Acc 0.9492 | Speed (samples/sec) 348.0539 | GPU 5012.3 MB\n",
      "Epoch 00002 | Step 00060 | Loss 0.1063 | Train Acc 0.9668 | Speed (samples/sec) 348.1312 | GPU 5012.3 MB\n",
      "Epoch 00002 | Step 00080 | Loss 0.0851 | Train Acc 0.9746 | Speed (samples/sec) 349.0316 | GPU 5012.3 MB\n",
      "Epoch 00002 | Step 00100 | Loss 0.1102 | Train Acc 0.9707 | Speed (samples/sec) 350.2460 | GPU 5012.3 MB\n",
      "Epoch 00002 | Step 00120 | Loss 0.1041 | Train Acc 0.9736 | Speed (samples/sec) 350.3929 | GPU 5012.3 MB\n",
      "Epoch 00002 | Step 00140 | Loss 0.0791 | Train Acc 0.9805 | Speed (samples/sec) 350.6753 | GPU 5013.3 MB\n",
      "Epoch 00002 | Step 00160 | Loss 0.1163 | Train Acc 0.9707 | Speed (samples/sec) 351.7326 | GPU 5013.3 MB\n",
      "Epoch 00002 | Step 00180 | Loss 0.0919 | Train Acc 0.9766 | Speed (samples/sec) 353.4482 | GPU 5013.3 MB\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/245 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch Time(s): 540.1638\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 245/245 [00:38<00:00,  6.33it/s]\n",
      "100%|██████████| 245/245 [01:14<00:00,  3.30it/s]\n",
      "100%|██████████| 245/245 [01:11<00:00,  3.41it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val acc 0.9080, Test Acc: 0.7552\n",
      "Epoch 00003 | Step 00000 | Loss 0.0421 | Train Acc 0.9893 | Speed (samples/sec) 353.5396 | GPU 5013.3 MB\n",
      "Epoch 00003 | Step 00020 | Loss 0.0308 | Train Acc 0.9932 | Speed (samples/sec) 354.6715 | GPU 5013.3 MB\n",
      "Epoch 00003 | Step 00040 | Loss 0.0384 | Train Acc 0.9893 | Speed (samples/sec) 355.6797 | GPU 5013.3 MB\n",
      "Epoch 00003 | Step 00060 | Loss 0.0458 | Train Acc 0.9863 | Speed (samples/sec) 356.6837 | GPU 5013.3 MB\n",
      "Epoch 00003 | Step 00080 | Loss 0.0220 | Train Acc 0.9941 | Speed (samples/sec) 357.8489 | GPU 5013.3 MB\n",
      "Epoch 00003 | Step 00100 | Loss 0.0368 | Train Acc 0.9941 | Speed (samples/sec) 359.1034 | GPU 5013.3 MB\n",
      "Epoch 00003 | Step 00120 | Loss 0.0607 | Train Acc 0.9854 | Speed (samples/sec) 359.5108 | GPU 5013.3 MB\n",
      "Epoch 00003 | Step 00140 | Loss 0.0215 | Train Acc 0.9951 | Speed (samples/sec) 359.9368 | GPU 5013.3 MB\n",
      "Epoch 00003 | Step 00160 | Loss 0.0452 | Train Acc 0.9873 | Speed (samples/sec) 360.2676 | GPU 5013.3 MB\n",
      "Epoch 00003 | Step 00180 | Loss 0.0339 | Train Acc 0.9883 | Speed (samples/sec) 361.2053 | GPU 5013.3 MB\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/245 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch Time(s): 510.2466\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 245/245 [00:36<00:00,  6.67it/s]\n",
      "100%|██████████| 245/245 [01:15<00:00,  3.24it/s]\n",
      "100%|██████████| 245/245 [01:15<00:00,  3.25it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val acc 0.9044, Test Acc: 0.7688\n",
      "Epoch 00004 | Step 00000 | Loss 0.0168 | Train Acc 0.9951 | Speed (samples/sec) 361.3733 | GPU 5013.3 MB\n",
      "Epoch 00004 | Step 00020 | Loss 0.0132 | Train Acc 0.9971 | Speed (samples/sec) 361.4291 | GPU 5013.3 MB\n",
      "Epoch 00004 | Step 00040 | Loss 0.0089 | Train Acc 0.9980 | Speed (samples/sec) 360.9142 | GPU 5013.3 MB\n",
      "Epoch 00004 | Step 00060 | Loss 0.0333 | Train Acc 0.9922 | Speed (samples/sec) 360.4740 | GPU 5013.3 MB\n",
      "Epoch 00004 | Step 00080 | Loss 0.0146 | Train Acc 0.9951 | Speed (samples/sec) 359.8317 | GPU 5013.3 MB\n",
      "Epoch 00004 | Step 00100 | Loss 0.0159 | Train Acc 0.9971 | Speed (samples/sec) 359.6620 | GPU 5013.3 MB\n",
      "Epoch 00004 | Step 00120 | Loss 0.0285 | Train Acc 0.9912 | Speed (samples/sec) 359.4451 | GPU 5013.3 MB\n",
      "Epoch 00004 | Step 00140 | Loss 0.0091 | Train Acc 0.9971 | Speed (samples/sec) 359.0218 | GPU 5013.3 MB\n",
      "Epoch 00004 | Step 00160 | Loss 0.0244 | Train Acc 0.9902 | Speed (samples/sec) 358.8574 | GPU 5013.3 MB\n",
      "Epoch 00004 | Step 00180 | Loss 0.0182 | Train Acc 0.9912 | Speed (samples/sec) 358.6728 | GPU 5013.3 MB\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/245 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch Time(s): 567.3406\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 245/245 [00:34<00:00,  7.01it/s]\n",
      "100%|██████████| 245/245 [01:19<00:00,  3.07it/s]\n",
      "100%|██████████| 245/245 [01:20<00:00,  3.05it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val acc 0.9021, Test Acc: 0.7717\n",
      "Epoch 00005 | Step 00000 | Loss 0.0078 | Train Acc 0.9971 | Speed (samples/sec) 358.2240 | GPU 5013.3 MB\n",
      "Epoch 00005 | Step 00020 | Loss 0.0033 | Train Acc 1.0000 | Speed (samples/sec) 358.0394 | GPU 5013.3 MB\n",
      "Epoch 00005 | Step 00040 | Loss 0.0130 | Train Acc 0.9971 | Speed (samples/sec) 357.9825 | GPU 5013.3 MB\n",
      "Epoch 00005 | Step 00060 | Loss 0.0058 | Train Acc 0.9980 | Speed (samples/sec) 357.7237 | GPU 5013.3 MB\n",
      "Epoch 00005 | Step 00080 | Loss 0.0043 | Train Acc 1.0000 | Speed (samples/sec) 357.5339 | GPU 5013.3 MB\n",
      "Epoch 00005 | Step 00100 | Loss 0.0105 | Train Acc 0.9961 | Speed (samples/sec) 357.4081 | GPU 5013.3 MB\n",
      "Epoch 00005 | Step 00120 | Loss 0.0082 | Train Acc 0.9980 | Speed (samples/sec) 357.5009 | GPU 5013.3 MB\n",
      "Epoch 00005 | Step 00140 | Loss 0.0093 | Train Acc 0.9980 | Speed (samples/sec) 357.4048 | GPU 5013.3 MB\n",
      "Epoch 00005 | Step 00160 | Loss 0.0073 | Train Acc 0.9990 | Speed (samples/sec) 356.9130 | GPU 5013.3 MB\n",
      "Epoch 00005 | Step 00180 | Loss 0.0062 | Train Acc 0.9990 | Speed (samples/sec) 356.8545 | GPU 5013.3 MB\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/245 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch Time(s): 562.5267\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 245/245 [00:33<00:00,  7.27it/s]\n",
      "100%|██████████| 245/245 [01:14<00:00,  3.30it/s]\n",
      "100%|██████████| 245/245 [01:15<00:00,  3.25it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val acc 0.8987, Test Acc: 0.7567\n",
      "Epoch 00006 | Step 00000 | Loss 0.0038 | Train Acc 0.9990 | Speed (samples/sec) 356.7552 | GPU 5013.3 MB\n",
      "Epoch 00006 | Step 00020 | Loss 0.0040 | Train Acc 0.9990 | Speed (samples/sec) 356.9354 | GPU 5013.3 MB\n",
      "Epoch 00006 | Step 00040 | Loss 0.0136 | Train Acc 0.9980 | Speed (samples/sec) 357.4940 | GPU 5013.3 MB\n",
      "Epoch 00006 | Step 00060 | Loss 0.0081 | Train Acc 0.9971 | Speed (samples/sec) 357.7558 | GPU 5013.3 MB\n",
      "Epoch 00006 | Step 00080 | Loss 0.0081 | Train Acc 0.9971 | Speed (samples/sec) 357.7583 | GPU 5013.3 MB\n",
      "Epoch 00006 | Step 00100 | Loss 0.0030 | Train Acc 0.9990 | Speed (samples/sec) 357.9478 | GPU 5013.3 MB\n",
      "Epoch 00006 | Step 00120 | Loss 0.0083 | Train Acc 0.9980 | Speed (samples/sec) 358.1884 | GPU 5013.3 MB\n",
      "Epoch 00006 | Step 00140 | Loss 0.0082 | Train Acc 0.9980 | Speed (samples/sec) 358.6465 | GPU 5013.3 MB\n",
      "Epoch 00006 | Step 00160 | Loss 0.0072 | Train Acc 0.9961 | Speed (samples/sec) 358.6945 | GPU 5013.3 MB\n",
      "Epoch 00006 | Step 00180 | Loss 0.0040 | Train Acc 0.9990 | Speed (samples/sec) 358.6322 | GPU 5013.3 MB\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/245 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch Time(s): 533.2568\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 245/245 [00:38<00:00,  6.39it/s]\n",
      "100%|██████████| 245/245 [01:13<00:00,  3.32it/s]\n",
      "100%|██████████| 245/245 [01:15<00:00,  3.26it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val acc 0.9006, Test Acc: 0.7647\n",
      "Epoch 00007 | Step 00000 | Loss 0.0073 | Train Acc 0.9980 | Speed (samples/sec) 358.3912 | GPU 5013.3 MB\n",
      "Epoch 00007 | Step 00020 | Loss 0.0051 | Train Acc 0.9980 | Speed (samples/sec) 358.4976 | GPU 5013.3 MB\n",
      "Epoch 00007 | Step 00040 | Loss 0.0034 | Train Acc 0.9990 | Speed (samples/sec) 358.6210 | GPU 5013.3 MB\n",
      "Epoch 00007 | Step 00060 | Loss 0.0084 | Train Acc 0.9980 | Speed (samples/sec) 358.9384 | GPU 5013.3 MB\n",
      "Epoch 00007 | Step 00080 | Loss 0.0058 | Train Acc 1.0000 | Speed (samples/sec) 359.0890 | GPU 5013.3 MB\n",
      "Epoch 00007 | Step 00100 | Loss 0.0034 | Train Acc 0.9990 | Speed (samples/sec) 359.0965 | GPU 5013.3 MB\n",
      "Epoch 00007 | Step 00120 | Loss 0.0022 | Train Acc 1.0000 | Speed (samples/sec) 359.0909 | GPU 5013.3 MB\n",
      "Epoch 00007 | Step 00140 | Loss 0.0069 | Train Acc 0.9990 | Speed (samples/sec) 359.6250 | GPU 5013.3 MB\n",
      "Epoch 00007 | Step 00160 | Loss 0.0026 | Train Acc 0.9990 | Speed (samples/sec) 359.9371 | GPU 5013.3 MB\n",
      "Epoch 00007 | Step 00180 | Loss 0.0055 | Train Acc 0.9980 | Speed (samples/sec) 360.0478 | GPU 5013.3 MB\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/245 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch Time(s): 530.1490\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 245/245 [00:36<00:00,  6.69it/s]\n",
      "100%|██████████| 245/245 [01:16<00:00,  3.21it/s]\n",
      "100%|██████████| 245/245 [01:15<00:00,  3.26it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val acc 0.8999, Test Acc: 0.7664\n",
      "Epoch 00008 | Step 00000 | Loss 0.0107 | Train Acc 0.9980 | Speed (samples/sec) 359.8908 | GPU 5013.3 MB\n",
      "Epoch 00008 | Step 00020 | Loss 0.0044 | Train Acc 0.9980 | Speed (samples/sec) 360.0612 | GPU 5013.3 MB\n",
      "Epoch 00008 | Step 00040 | Loss 0.0019 | Train Acc 1.0000 | Speed (samples/sec) 360.0221 | GPU 5013.3 MB\n",
      "Epoch 00008 | Step 00060 | Loss 0.0033 | Train Acc 0.9990 | Speed (samples/sec) 359.8979 | GPU 5013.3 MB\n",
      "Epoch 00008 | Step 00080 | Loss 0.0013 | Train Acc 1.0000 | Speed (samples/sec) 359.8201 | GPU 5013.3 MB\n",
      "Epoch 00008 | Step 00100 | Loss 0.0127 | Train Acc 0.9951 | Speed (samples/sec) 359.9454 | GPU 5013.3 MB\n",
      "Epoch 00008 | Step 00120 | Loss 0.0041 | Train Acc 0.9980 | Speed (samples/sec) 360.0663 | GPU 5013.3 MB\n",
      "Epoch 00008 | Step 00140 | Loss 0.0068 | Train Acc 0.9980 | Speed (samples/sec) 360.1757 | GPU 5013.3 MB\n",
      "Epoch 00008 | Step 00160 | Loss 0.0024 | Train Acc 1.0000 | Speed (samples/sec) 360.1647 | GPU 5013.3 MB\n",
      "Epoch 00008 | Step 00180 | Loss 0.0082 | Train Acc 0.9980 | Speed (samples/sec) 360.2784 | GPU 5013.3 MB\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/245 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch Time(s): 541.6928\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 245/245 [00:35<00:00,  6.99it/s]\n",
      "100%|██████████| 245/245 [01:15<00:00,  3.23it/s]\n",
      "100%|██████████| 245/245 [01:15<00:00,  3.26it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val acc 0.8987, Test Acc: 0.7569\n",
      "Epoch 00009 | Step 00000 | Loss 0.0054 | Train Acc 0.9971 | Speed (samples/sec) 360.1624 | GPU 5013.3 MB\n",
      "Epoch 00009 | Step 00020 | Loss 0.0021 | Train Acc 0.9990 | Speed (samples/sec) 360.3930 | GPU 5013.3 MB\n",
      "Epoch 00009 | Step 00040 | Loss 0.0040 | Train Acc 0.9980 | Speed (samples/sec) 360.5107 | GPU 5013.3 MB\n",
      "Epoch 00009 | Step 00060 | Loss 0.0126 | Train Acc 0.9971 | Speed (samples/sec) 360.3915 | GPU 5013.3 MB\n",
      "Epoch 00009 | Step 00080 | Loss 0.0024 | Train Acc 1.0000 | Speed (samples/sec) 360.2824 | GPU 5013.3 MB\n",
      "Epoch 00009 | Step 00100 | Loss 0.0156 | Train Acc 0.9971 | Speed (samples/sec) 360.3982 | GPU 5013.3 MB\n",
      "Epoch 00009 | Step 00120 | Loss 0.0031 | Train Acc 0.9980 | Speed (samples/sec) 360.5102 | GPU 5013.3 MB\n",
      "Epoch 00009 | Step 00140 | Loss 0.0016 | Train Acc 1.0000 | Speed (samples/sec) 360.5485 | GPU 5013.3 MB\n",
      "Epoch 00009 | Step 00160 | Loss 0.0011 | Train Acc 1.0000 | Speed (samples/sec) 360.4929 | GPU 5013.3 MB\n",
      "Epoch 00009 | Step 00180 | Loss 0.0006 | Train Acc 1.0000 | Speed (samples/sec) 360.3749 | GPU 5013.3 MB\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/245 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch Time(s): 542.0847\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 245/245 [00:35<00:00,  6.98it/s]\n",
      "100%|██████████| 245/245 [01:16<00:00,  3.21it/s]\n",
      "100%|██████████| 245/245 [01:17<00:00,  3.17it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val acc 0.9008, Test Acc: 0.7635\n",
      "Epoch 00010 | Step 00000 | Loss 0.0048 | Train Acc 0.9990 | Speed (samples/sec) 360.4109 | GPU 5013.3 MB\n",
      "Epoch 00010 | Step 00020 | Loss 0.0016 | Train Acc 1.0000 | Speed (samples/sec) 360.4918 | GPU 5013.3 MB\n",
      "Epoch 00010 | Step 00040 | Loss 0.0009 | Train Acc 1.0000 | Speed (samples/sec) 360.6252 | GPU 5013.3 MB\n",
      "Epoch 00010 | Step 00060 | Loss 0.0037 | Train Acc 0.9990 | Speed (samples/sec) 360.6785 | GPU 5013.3 MB\n",
      "Epoch 00010 | Step 00080 | Loss 0.0016 | Train Acc 0.9990 | Speed (samples/sec) 360.7485 | GPU 5013.3 MB\n",
      "Epoch 00010 | Step 00100 | Loss 0.0043 | Train Acc 0.9990 | Speed (samples/sec) 360.7370 | GPU 5013.3 MB\n",
      "Epoch 00010 | Step 00120 | Loss 0.0031 | Train Acc 0.9990 | Speed (samples/sec) 360.9547 | GPU 5013.3 MB\n",
      "Epoch 00010 | Step 00140 | Loss 0.0012 | Train Acc 1.0000 | Speed (samples/sec) 361.1640 | GPU 5013.3 MB\n",
      "Epoch 00010 | Step 00160 | Loss 0.0082 | Train Acc 0.9980 | Speed (samples/sec) 361.0637 | GPU 5013.3 MB\n",
      "Epoch 00010 | Step 00180 | Loss 0.0099 | Train Acc 0.9990 | Speed (samples/sec) 361.0232 | GPU 5013.3 MB\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/245 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch Time(s): 538.1211\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 245/245 [00:34<00:00,  7.07it/s]\n",
      "100%|██████████| 245/245 [01:14<00:00,  3.27it/s]\n",
      "100%|██████████| 245/245 [01:18<00:00,  3.13it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val acc 0.8983, Test Acc: 0.7362\n",
      "Epoch 00011 | Step 00000 | Loss 0.0028 | Train Acc 0.9990 | Speed (samples/sec) 360.8512 | GPU 5013.3 MB\n",
      "Epoch 00011 | Step 00020 | Loss 0.0023 | Train Acc 0.9990 | Speed (samples/sec) 360.8445 | GPU 5013.3 MB\n",
      "Epoch 00011 | Step 00040 | Loss 0.0014 | Train Acc 0.9990 | Speed (samples/sec) 360.9036 | GPU 5013.3 MB\n",
      "Epoch 00011 | Step 00060 | Loss 0.0066 | Train Acc 0.9990 | Speed (samples/sec) 360.8542 | GPU 5013.3 MB\n",
      "Epoch 00011 | Step 00080 | Loss 0.0039 | Train Acc 0.9990 | Speed (samples/sec) 360.7223 | GPU 5013.3 MB\n",
      "Epoch 00011 | Step 00100 | Loss 0.0008 | Train Acc 1.0000 | Speed (samples/sec) 360.6175 | GPU 5013.3 MB\n",
      "Epoch 00011 | Step 00120 | Loss 0.0014 | Train Acc 1.0000 | Speed (samples/sec) 360.6743 | GPU 5013.3 MB\n",
      "Epoch 00011 | Step 00140 | Loss 0.0034 | Train Acc 0.9990 | Speed (samples/sec) 360.7711 | GPU 5013.3 MB\n",
      "Epoch 00011 | Step 00160 | Loss 0.0005 | Train Acc 1.0000 | Speed (samples/sec) 360.7325 | GPU 5013.3 MB\n",
      "Epoch 00011 | Step 00180 | Loss 0.0006 | Train Acc 1.0000 | Speed (samples/sec) 360.5747 | GPU 5013.3 MB\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/245 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch Time(s): 553.3708\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 245/245 [00:39<00:00,  6.17it/s]\n",
      "100%|██████████| 245/245 [01:14<00:00,  3.27it/s]\n",
      "100%|██████████| 245/245 [01:23<00:00,  2.95it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val acc 0.8997, Test Acc: 0.7655\n",
      "Epoch 00012 | Step 00000 | Loss 0.0020 | Train Acc 0.9990 | Speed (samples/sec) 360.3581 | GPU 5013.3 MB\n",
      "Epoch 00012 | Step 00020 | Loss 0.0016 | Train Acc 0.9990 | Speed (samples/sec) 360.1319 | GPU 5013.3 MB\n",
      "Epoch 00012 | Step 00040 | Loss 0.0028 | Train Acc 0.9990 | Speed (samples/sec) 360.1789 | GPU 5013.3 MB\n",
      "Epoch 00012 | Step 00060 | Loss 0.0015 | Train Acc 1.0000 | Speed (samples/sec) 360.1484 | GPU 5013.3 MB\n",
      "Epoch 00012 | Step 00080 | Loss 0.0028 | Train Acc 0.9990 | Speed (samples/sec) 359.9871 | GPU 5013.3 MB\n",
      "Epoch 00012 | Step 00100 | Loss 0.0046 | Train Acc 0.9980 | Speed (samples/sec) 359.8901 | GPU 5013.3 MB\n",
      "Epoch 00012 | Step 00120 | Loss 0.0021 | Train Acc 0.9990 | Speed (samples/sec) 359.9420 | GPU 5013.3 MB\n",
      "Epoch 00012 | Step 00140 | Loss 0.0027 | Train Acc 0.9990 | Speed (samples/sec) 360.0158 | GPU 5013.3 MB\n",
      "Epoch 00012 | Step 00160 | Loss 0.0003 | Train Acc 1.0000 | Speed (samples/sec) 359.9108 | GPU 5013.3 MB\n",
      "Epoch 00012 | Step 00180 | Loss 0.0053 | Train Acc 0.9990 | Speed (samples/sec) 359.7899 | GPU 5013.3 MB\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/245 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch Time(s): 560.0984\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 245/245 [00:37<00:00,  6.52it/s]\n",
      "100%|██████████| 245/245 [01:15<00:00,  3.26it/s]\n",
      "100%|██████████| 245/245 [01:22<00:00,  2.96it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val acc 0.9013, Test Acc: 0.7634\n",
      "Epoch 00013 | Step 00000 | Loss 0.0075 | Train Acc 0.9980 | Speed (samples/sec) 359.6206 | GPU 5013.3 MB\n",
      "Epoch 00013 | Step 00020 | Loss 0.0032 | Train Acc 0.9990 | Speed (samples/sec) 359.4371 | GPU 5013.3 MB\n",
      "Epoch 00013 | Step 00040 | Loss 0.0032 | Train Acc 0.9990 | Speed (samples/sec) 359.3739 | GPU 5013.3 MB\n",
      "Epoch 00013 | Step 00060 | Loss 0.0065 | Train Acc 0.9980 | Speed (samples/sec) 359.4153 | GPU 5013.3 MB\n",
      "Epoch 00013 | Step 00080 | Loss 0.0009 | Train Acc 1.0000 | Speed (samples/sec) 359.5068 | GPU 5013.3 MB\n",
      "Epoch 00013 | Step 00100 | Loss 0.0052 | Train Acc 0.9980 | Speed (samples/sec) 359.7761 | GPU 5013.3 MB\n",
      "Epoch 00013 | Step 00120 | Loss 0.0024 | Train Acc 0.9990 | Speed (samples/sec) 359.8849 | GPU 5013.3 MB\n",
      "Epoch 00013 | Step 00140 | Loss 0.0076 | Train Acc 0.9980 | Speed (samples/sec) 359.9647 | GPU 5013.3 MB\n",
      "Epoch 00013 | Step 00160 | Loss 0.0067 | Train Acc 0.9990 | Speed (samples/sec) 359.9853 | GPU 5013.3 MB\n",
      "Epoch 00013 | Step 00180 | Loss 0.0036 | Train Acc 0.9990 | Speed (samples/sec) 360.1096 | GPU 5013.3 MB\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/245 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch Time(s): 536.4811\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 245/245 [00:36<00:00,  6.71it/s]\n",
      "100%|██████████| 245/245 [01:11<00:00,  3.42it/s]\n",
      "100%|██████████| 245/245 [01:14<00:00,  3.29it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val acc 0.9002, Test Acc: 0.7488\n",
      "Epoch 00014 | Step 00000 | Loss 0.0014 | Train Acc 1.0000 | Speed (samples/sec) 360.1282 | GPU 5013.3 MB\n",
      "Epoch 00014 | Step 00020 | Loss 0.0006 | Train Acc 1.0000 | Speed (samples/sec) 360.2593 | GPU 5013.3 MB\n",
      "Epoch 00014 | Step 00040 | Loss 0.0051 | Train Acc 0.9980 | Speed (samples/sec) 360.4722 | GPU 5013.3 MB\n",
      "Epoch 00014 | Step 00060 | Loss 0.0093 | Train Acc 0.9980 | Speed (samples/sec) 360.6602 | GPU 5013.3 MB\n",
      "Epoch 00014 | Step 00080 | Loss 0.0008 | Train Acc 1.0000 | Speed (samples/sec) 360.9164 | GPU 5013.3 MB\n",
      "Epoch 00014 | Step 00100 | Loss 0.0029 | Train Acc 0.9990 | Speed (samples/sec) 361.1169 | GPU 5013.3 MB\n",
      "Epoch 00014 | Step 00120 | Loss 0.0113 | Train Acc 0.9971 | Speed (samples/sec) 361.2970 | GPU 5013.3 MB\n",
      "Epoch 00014 | Step 00140 | Loss 0.0023 | Train Acc 0.9990 | Speed (samples/sec) 361.5538 | GPU 5013.3 MB\n",
      "Epoch 00014 | Step 00160 | Loss 0.0065 | Train Acc 0.9980 | Speed (samples/sec) 361.7125 | GPU 5013.3 MB\n",
      "Epoch 00014 | Step 00180 | Loss 0.0035 | Train Acc 0.9990 | Speed (samples/sec) 361.9156 | GPU 5013.3 MB\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/245 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch Time(s): 506.0893\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 245/245 [00:33<00:00,  7.25it/s]\n",
      "100%|██████████| 245/245 [01:12<00:00,  3.38it/s]\n",
      "100%|██████████| 245/245 [01:11<00:00,  3.43it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val acc 0.8975, Test Acc: 0.7573\n",
      "Epoch 00015 | Step 00000 | Loss 0.0048 | Train Acc 0.9980 | Speed (samples/sec) 361.9859 | GPU 5013.3 MB\n",
      "Epoch 00015 | Step 00020 | Loss 0.0007 | Train Acc 1.0000 | Speed (samples/sec) 362.1369 | GPU 5013.3 MB\n",
      "Epoch 00015 | Step 00040 | Loss 0.0018 | Train Acc 0.9990 | Speed (samples/sec) 362.2490 | GPU 5013.3 MB\n",
      "Epoch 00015 | Step 00060 | Loss 0.0011 | Train Acc 1.0000 | Speed (samples/sec) 362.4063 | GPU 5013.3 MB\n",
      "Epoch 00015 | Step 00080 | Loss 0.0024 | Train Acc 1.0000 | Speed (samples/sec) 362.6659 | GPU 5013.3 MB\n",
      "Epoch 00015 | Step 00100 | Loss 0.0003 | Train Acc 1.0000 | Speed (samples/sec) 362.8731 | GPU 5013.3 MB\n",
      "Epoch 00015 | Step 00120 | Loss 0.0035 | Train Acc 0.9990 | Speed (samples/sec) 363.1494 | GPU 5013.3 MB\n",
      "Epoch 00015 | Step 00140 | Loss 0.0035 | Train Acc 0.9990 | Speed (samples/sec) 363.4049 | GPU 5013.3 MB\n",
      "Epoch 00015 | Step 00160 | Loss 0.0027 | Train Acc 0.9990 | Speed (samples/sec) 363.6633 | GPU 5013.3 MB\n",
      "Epoch 00015 | Step 00180 | Loss 0.0055 | Train Acc 0.9971 | Speed (samples/sec) 363.8518 | GPU 5013.3 MB\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/245 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch Time(s): 500.3953\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 245/245 [00:35<00:00,  7.00it/s]\n",
      "100%|██████████| 245/245 [01:10<00:00,  3.49it/s]\n",
      "100%|██████████| 245/245 [01:12<00:00,  3.39it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val acc 0.9002, Test Acc: 0.7670\n",
      "Epoch 00016 | Step 00000 | Loss 0.0014 | Train Acc 1.0000 | Speed (samples/sec) 363.8618 | GPU 5013.3 MB\n",
      "Epoch 00016 | Step 00020 | Loss 0.0071 | Train Acc 0.9990 | Speed (samples/sec) 364.0343 | GPU 5013.3 MB\n",
      "Epoch 00016 | Step 00040 | Loss 0.0010 | Train Acc 1.0000 | Speed (samples/sec) 364.2515 | GPU 5013.3 MB\n",
      "Epoch 00016 | Step 00060 | Loss 0.0051 | Train Acc 0.9990 | Speed (samples/sec) 364.4199 | GPU 5013.3 MB\n",
      "Epoch 00016 | Step 00080 | Loss 0.0242 | Train Acc 0.9980 | Speed (samples/sec) 364.6852 | GPU 5013.3 MB\n",
      "Epoch 00016 | Step 00100 | Loss 0.0105 | Train Acc 0.9980 | Speed (samples/sec) 364.8470 | GPU 5013.3 MB\n",
      "Epoch 00016 | Step 00120 | Loss 0.0046 | Train Acc 0.9990 | Speed (samples/sec) 364.9968 | GPU 5013.3 MB\n",
      "Epoch 00016 | Step 00140 | Loss 0.0017 | Train Acc 1.0000 | Speed (samples/sec) 365.1360 | GPU 5013.3 MB\n",
      "Epoch 00016 | Step 00160 | Loss 0.0007 | Train Acc 1.0000 | Speed (samples/sec) 365.4257 | GPU 5013.3 MB\n",
      "Epoch 00016 | Step 00180 | Loss 0.0005 | Train Acc 1.0000 | Speed (samples/sec) 365.6225 | GPU 5013.3 MB\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/245 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch Time(s): 498.6882\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 245/245 [00:32<00:00,  7.51it/s]\n",
      "100%|██████████| 245/245 [01:12<00:00,  3.39it/s]\n",
      "100%|██████████| 245/245 [01:11<00:00,  3.41it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val acc 0.9009, Test Acc: 0.7669\n",
      "Epoch 00017 | Step 00000 | Loss 0.0016 | Train Acc 1.0000 | Speed (samples/sec) 365.6206 | GPU 5013.3 MB\n",
      "Epoch 00017 | Step 00020 | Loss 0.0074 | Train Acc 0.9980 | Speed (samples/sec) 365.7878 | GPU 5013.3 MB\n",
      "Epoch 00017 | Step 00040 | Loss 0.0031 | Train Acc 0.9990 | Speed (samples/sec) 365.9213 | GPU 5013.3 MB\n",
      "Epoch 00017 | Step 00060 | Loss 0.0030 | Train Acc 0.9980 | Speed (samples/sec) 366.0731 | GPU 5013.3 MB\n",
      "Epoch 00017 | Step 00080 | Loss 0.0013 | Train Acc 0.9990 | Speed (samples/sec) 366.2331 | GPU 5013.3 MB\n",
      "Epoch 00017 | Step 00100 | Loss 0.0031 | Train Acc 0.9990 | Speed (samples/sec) 366.3872 | GPU 5013.3 MB\n",
      "Epoch 00017 | Step 00120 | Loss 0.0007 | Train Acc 1.0000 | Speed (samples/sec) 366.5469 | GPU 5013.3 MB\n",
      "Epoch 00017 | Step 00140 | Loss 0.0081 | Train Acc 0.9971 | Speed (samples/sec) 366.6684 | GPU 5013.3 MB\n",
      "Epoch 00017 | Step 00160 | Loss 0.0046 | Train Acc 0.9980 | Speed (samples/sec) 366.8156 | GPU 5013.3 MB\n",
      "Epoch 00017 | Step 00180 | Loss 0.0104 | Train Acc 0.9971 | Speed (samples/sec) 367.0082 | GPU 5013.3 MB\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/245 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch Time(s): 502.6479\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 245/245 [00:32<00:00,  7.45it/s]\n",
      "100%|██████████| 245/245 [01:10<00:00,  3.46it/s]\n",
      "100%|██████████| 245/245 [01:07<00:00,  3.63it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val acc 0.8985, Test Acc: 0.7631\n",
      "Epoch 00018 | Step 00000 | Loss 0.0175 | Train Acc 0.9990 | Speed (samples/sec) 367.0152 | GPU 5013.3 MB\n",
      "Epoch 00018 | Step 00020 | Loss 0.0009 | Train Acc 1.0000 | Speed (samples/sec) 367.3499 | GPU 5013.3 MB\n",
      "Epoch 00018 | Step 00040 | Loss 0.0043 | Train Acc 0.9980 | Speed (samples/sec) 367.7151 | GPU 5013.3 MB\n",
      "Epoch 00018 | Step 00060 | Loss 0.0053 | Train Acc 0.9980 | Speed (samples/sec) 367.9839 | GPU 5013.3 MB\n",
      "Epoch 00018 | Step 00080 | Loss 0.0129 | Train Acc 0.9980 | Speed (samples/sec) 368.2057 | GPU 5013.3 MB\n",
      "Epoch 00018 | Step 00100 | Loss 0.0011 | Train Acc 1.0000 | Speed (samples/sec) 368.4838 | GPU 5013.3 MB\n",
      "Epoch 00018 | Step 00120 | Loss 0.0060 | Train Acc 0.9980 | Speed (samples/sec) 368.6893 | GPU 5013.3 MB\n",
      "Epoch 00018 | Step 00140 | Loss 0.0009 | Train Acc 1.0000 | Speed (samples/sec) 368.9403 | GPU 5013.3 MB\n",
      "Epoch 00018 | Step 00160 | Loss 0.0108 | Train Acc 0.9980 | Speed (samples/sec) 369.1728 | GPU 5013.3 MB\n",
      "Epoch 00018 | Step 00180 | Loss 0.0134 | Train Acc 0.9971 | Speed (samples/sec) 369.3667 | GPU 5013.3 MB\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/245 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch Time(s): 475.8372\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 245/245 [00:30<00:00,  7.91it/s]\n",
      "100%|██████████| 245/245 [01:07<00:00,  3.62it/s]\n",
      "100%|██████████| 245/245 [01:08<00:00,  3.56it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val acc 0.8999, Test Acc: 0.7602\n",
      "Epoch 00019 | Step 00000 | Loss 0.0011 | Train Acc 1.0000 | Speed (samples/sec) 369.4070 | GPU 5013.3 MB\n",
      "Epoch 00019 | Step 00020 | Loss 0.0171 | Train Acc 0.9961 | Speed (samples/sec) 369.6732 | GPU 5013.3 MB\n",
      "Epoch 00019 | Step 00040 | Loss 0.0045 | Train Acc 0.9980 | Speed (samples/sec) 369.9219 | GPU 5013.3 MB\n",
      "Epoch 00019 | Step 00060 | Loss 0.0022 | Train Acc 0.9990 | Speed (samples/sec) 370.1807 | GPU 5013.3 MB\n",
      "Epoch 00019 | Step 00080 | Loss 0.0019 | Train Acc 0.9990 | Speed (samples/sec) 370.3640 | GPU 5013.3 MB\n",
      "Epoch 00019 | Step 00100 | Loss 0.0020 | Train Acc 1.0000 | Speed (samples/sec) 370.5634 | GPU 5013.3 MB\n",
      "Epoch 00019 | Step 00120 | Loss 0.0037 | Train Acc 0.9980 | Speed (samples/sec) 370.8572 | GPU 5013.3 MB\n",
      "Epoch 00019 | Step 00140 | Loss 0.0017 | Train Acc 0.9990 | Speed (samples/sec) 371.0589 | GPU 5013.3 MB\n",
      "Epoch 00019 | Step 00160 | Loss 0.0068 | Train Acc 0.9990 | Speed (samples/sec) 371.2727 | GPU 5013.3 MB\n",
      "Epoch 00019 | Step 00180 | Loss 0.0034 | Train Acc 0.9990 | Speed (samples/sec) 371.5016 | GPU 5013.3 MB\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/245 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch Time(s): 475.8958\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 245/245 [00:32<00:00,  7.49it/s]\n",
      "100%|██████████| 245/245 [01:09<00:00,  3.53it/s]\n",
      "100%|██████████| 245/245 [01:09<00:00,  3.52it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val acc 0.8997, Test Acc: 0.7566\n",
      "Avg epoch time: 670.0859654108683\n",
      "Best val acc 0.9164, best test acc: 0.7900\n"
     ]
    }
   ],
   "source": [
    "g, num_labels = load_ogb('ogbn-products')\n",
    "g.ndata.pop('features')\n",
    "\n",
    "in_feats = 128\n",
    "eigen_vals, eigen_vecs = get_eigen(g, in_feats, 'ogbn-products')\n",
    "nfeat = th.tensor(eigen_vecs * np.sqrt(eigen_vals).reshape((1, len(eigen_vals))), dtype=th.float32)\n",
    "labels = g.ndata.pop('labels')\n",
    "\n",
    "hyperparams = {\n",
    "    'batch_size': 1024,\n",
    "    'num_workers': 0,\n",
    "    'num_hidden': 256,\n",
    "    'num_layers': 3,\n",
    "    'dropout': 0.5,\n",
    "    'lr': 0.003,\n",
    "    'sparse_lr': 0.0003,\n",
    "    'num_epochs': 20,\n",
    "    'fanouts': [15,10,5],\n",
    "    'eval_batch_size': 10000\n",
    "}\n",
    "device = th.device('cuda:5')\n",
    "\n",
    "# Pack data\n",
    "data = g, nfeat, labels\n",
    "# Define model\n",
    "model = SAGE(in_feats, hyperparams['num_hidden'], num_labels,\n",
    "             hyperparams['num_layers'], F.relu, hyperparams['dropout'])\n",
    "train_ft_embed(model, data, hyperparams, device, 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}