GraphSage-sample-eigen-products_gpu1.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using backend: pytorch\n"
     ]
    }
   ],
   "source": [
    "import dgl\n",
    "import time\n",
    "import torch as th\n",
    "import numpy as np\n",
    "import torch.nn.functional as F\n",
    "\n",
    "from load_data import load_ogb\n",
    "from model import SAGE, GAT, get_eigen\n",
    "from train_eval import train_sample"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This notebook shows the performance of GraphSage on the OGBN-products graph with scaled eigenvectors as positional embeddings for nodes. The positional node embeddings are fixed in this model."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "load ogbn-products\n",
      "finish loading ogbn-products\n",
      "finish constructing ogbn-products\n",
      "Epoch 00000 | Step 00000 | Loss 3.8544 | Train Acc 0.0176 | Speed (samples/sec) nan | GPU 780.6 MB\n",
      "Epoch 00000 | Step 00020 | Loss 1.9016 | Train Acc 0.5156 | Speed (samples/sec) 6019.9496 | GPU 788.2 MB\n",
      "Epoch 00000 | Step 00040 | Loss 1.2882 | Train Acc 0.6602 | Speed (samples/sec) 5974.9644 | GPU 789.3 MB\n",
      "Epoch 00000 | Step 00060 | Loss 1.1621 | Train Acc 0.6973 | Speed (samples/sec) 5980.1294 | GPU 790.0 MB\n",
      "Epoch 00000 | Step 00080 | Loss 1.0794 | Train Acc 0.7061 | Speed (samples/sec) 5997.3360 | GPU 790.0 MB\n",
      "Epoch 00000 | Step 00100 | Loss 0.9954 | Train Acc 0.7412 | Speed (samples/sec) 5979.2930 | GPU 790.0 MB\n",
      "Epoch 00000 | Step 00120 | Loss 0.9324 | Train Acc 0.7305 | Speed (samples/sec) 6042.2025 | GPU 790.0 MB\n",
      "Epoch 00000 | Step 00140 | Loss 0.8243 | Train Acc 0.7783 | Speed (samples/sec) 6063.7062 | GPU 790.6 MB\n",
      "Epoch 00000 | Step 00160 | Loss 0.8547 | Train Acc 0.7705 | Speed (samples/sec) 6107.2708 | GPU 790.6 MB\n",
      "Epoch 00000 | Step 00180 | Loss 0.8318 | Train Acc 0.7822 | Speed (samples/sec) 6124.2000 | GPU 790.6 MB\n",
      "Epoch Time(s): 36.0263\n",
      "Epoch 00001 | Step 00000 | Loss 0.8166 | Train Acc 0.8008 | Speed (samples/sec) 6071.3024 | GPU 790.6 MB\n",
      "Epoch 00001 | Step 00020 | Loss 0.7727 | Train Acc 0.8096 | Speed (samples/sec) 6072.8980 | GPU 790.6 MB\n",
      "Epoch 00001 | Step 00040 | Loss 0.7625 | Train Acc 0.7949 | Speed (samples/sec) 6086.6647 | GPU 790.6 MB\n",
      "Epoch 00001 | Step 00060 | Loss 0.8352 | Train Acc 0.7812 | Speed (samples/sec) 6101.0394 | GPU 790.6 MB\n",
      "Epoch 00001 | Step 00080 | Loss 0.7308 | Train Acc 0.8027 | Speed (samples/sec) 6115.1131 | GPU 790.6 MB\n",
      "Epoch 00001 | Step 00100 | Loss 0.6951 | Train Acc 0.8193 | Speed (samples/sec) 6119.6203 | GPU 791.3 MB\n",
      "Epoch 00001 | Step 00120 | Loss 0.6232 | Train Acc 0.8203 | Speed (samples/sec) 6141.0125 | GPU 791.3 MB\n",
      "Epoch 00001 | Step 00140 | Loss 0.6649 | Train Acc 0.8184 | Speed (samples/sec) 6144.4071 | GPU 791.3 MB\n",
      "Epoch 00001 | Step 00160 | Loss 0.7051 | Train Acc 0.8232 | Speed (samples/sec) 6146.7547 | GPU 791.3 MB\n",
      "Epoch 00001 | Step 00180 | Loss 0.6085 | Train Acc 0.8252 | Speed (samples/sec) 6142.6992 | GPU 791.3 MB\n",
      "Epoch Time(s): 35.1325\n",
      "Epoch 00002 | Step 00000 | Loss 0.6213 | Train Acc 0.8193 | Speed (samples/sec) 6112.3033 | GPU 791.3 MB\n",
      "Epoch 00002 | Step 00020 | Loss 0.5945 | Train Acc 0.8369 | Speed (samples/sec) 6104.6164 | GPU 791.3 MB\n",
      "Epoch 00002 | Step 00040 | Loss 0.6341 | Train Acc 0.8203 | Speed (samples/sec) 6110.5578 | GPU 791.3 MB\n",
      "Epoch 00002 | Step 00060 | Loss 0.5966 | Train Acc 0.8379 | Speed (samples/sec) 6120.2716 | GPU 791.3 MB\n",
      "Epoch 00002 | Step 00080 | Loss 0.5936 | Train Acc 0.8330 | Speed (samples/sec) 6121.7432 | GPU 791.3 MB\n",
      "Epoch 00002 | Step 00100 | Loss 0.6646 | Train Acc 0.8262 | Speed (samples/sec) 6117.1141 | GPU 791.3 MB\n",
      "Epoch 00002 | Step 00120 | Loss 0.6772 | Train Acc 0.8125 | Speed (samples/sec) 6124.9561 | GPU 791.3 MB\n",
      "Epoch 00002 | Step 00140 | Loss 0.6871 | Train Acc 0.8125 | Speed (samples/sec) 6132.9295 | GPU 791.3 MB\n",
      "Epoch 00002 | Step 00160 | Loss 0.5557 | Train Acc 0.8398 | Speed (samples/sec) 6128.1892 | GPU 791.3 MB\n",
      "Epoch 00002 | Step 00180 | Loss 0.6574 | Train Acc 0.8076 | Speed (samples/sec) 6122.5861 | GPU 791.3 MB\n",
      "Epoch Time(s): 35.9534\n",
      "Epoch 00003 | Step 00000 | Loss 0.6906 | Train Acc 0.8047 | Speed (samples/sec) 6109.9276 | GPU 791.3 MB\n",
      "Epoch 00003 | Step 00020 | Loss 0.6128 | Train Acc 0.8262 | Speed (samples/sec) 6104.2820 | GPU 791.3 MB\n",
      "Epoch 00003 | Step 00040 | Loss 0.6065 | Train Acc 0.8389 | Speed (samples/sec) 6108.8406 | GPU 791.3 MB\n",
      "Epoch 00003 | Step 00060 | Loss 0.5410 | Train Acc 0.8486 | Speed (samples/sec) 6118.2228 | GPU 791.3 MB\n",
      "Epoch 00003 | Step 00080 | Loss 0.5767 | Train Acc 0.8408 | Speed (samples/sec) 6124.1327 | GPU 791.4 MB\n",
      "Epoch 00003 | Step 00100 | Loss 0.5865 | Train Acc 0.8330 | Speed (samples/sec) 6125.6926 | GPU 791.4 MB\n",
      "Epoch 00003 | Step 00120 | Loss 0.6623 | Train Acc 0.8086 | Speed (samples/sec) 6124.9188 | GPU 791.4 MB\n",
      "Epoch 00003 | Step 00140 | Loss 0.5797 | Train Acc 0.8369 | Speed (samples/sec) 6123.9315 | GPU 791.4 MB\n",
      "Epoch 00003 | Step 00160 | Loss 0.5722 | Train Acc 0.8320 | Speed (samples/sec) 6124.1633 | GPU 791.4 MB\n",
      "Epoch 00003 | Step 00180 | Loss 0.5667 | Train Acc 0.8477 | Speed (samples/sec) 6122.2347 | GPU 791.4 MB\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/245 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch Time(s): 35.8194\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 245/245 [00:38<00:00,  6.36it/s]\n",
      "100%|██████████| 245/245 [00:50<00:00,  4.88it/s]\n",
      "100%|██████████| 245/245 [00:46<00:00,  5.31it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val acc 0.8298, Test Acc: 0.6761\n",
      "Best val acc 0.8298, best test acc: 0.6761\n",
      "Epoch 00004 | Step 00000 | Loss 0.5744 | Train Acc 0.8408 | Speed (samples/sec) 6107.6874 | GPU 791.4 MB\n",
      "Epoch 00004 | Step 00020 | Loss 0.6007 | Train Acc 0.8359 | Speed (samples/sec) 6113.7388 | GPU 791.4 MB\n",
      "Epoch 00004 | Step 00040 | Loss 0.6034 | Train Acc 0.8301 | Speed (samples/sec) 6116.8245 | GPU 791.4 MB\n",
      "Epoch 00004 | Step 00060 | Loss 0.5607 | Train Acc 0.8398 | Speed (samples/sec) 6120.2550 | GPU 791.4 MB\n",
      "Epoch 00004 | Step 00080 | Loss 0.5402 | Train Acc 0.8418 | Speed (samples/sec) 6121.5181 | GPU 791.4 MB\n",
      "Epoch 00004 | Step 00100 | Loss 0.5812 | Train Acc 0.8564 | Speed (samples/sec) 6117.7308 | GPU 791.4 MB\n",
      "Epoch 00004 | Step 00120 | Loss 0.5674 | Train Acc 0.8477 | Speed (samples/sec) 6122.2888 | GPU 791.4 MB\n",
      "Epoch 00004 | Step 00140 | Loss 0.5463 | Train Acc 0.8398 | Speed (samples/sec) 6125.5965 | GPU 791.4 MB\n",
      "Epoch 00004 | Step 00160 | Loss 0.5821 | Train Acc 0.8350 | Speed (samples/sec) 6131.8325 | GPU 791.4 MB\n",
      "Epoch 00004 | Step 00180 | Loss 0.6265 | Train Acc 0.8184 | Speed (samples/sec) 6135.3550 | GPU 791.4 MB\n",
      "Epoch Time(s): 35.3120\n",
      "Epoch 00005 | Step 00000 | Loss 0.5659 | Train Acc 0.8447 | Speed (samples/sec) 6124.9288 | GPU 791.4 MB\n",
      "Epoch 00005 | Step 00020 | Loss 0.6165 | Train Acc 0.8242 | Speed (samples/sec) 6118.0788 | GPU 791.4 MB\n",
      "Epoch 00005 | Step 00040 | Loss 0.5623 | Train Acc 0.8369 | Speed (samples/sec) 6123.7658 | GPU 791.4 MB\n",
      "Epoch 00005 | Step 00060 | Loss 0.5096 | Train Acc 0.8633 | Speed (samples/sec) 6127.6569 | GPU 791.4 MB\n",
      "Epoch 00005 | Step 00080 | Loss 0.6092 | Train Acc 0.8311 | Speed (samples/sec) 6132.3659 | GPU 791.4 MB\n",
      "Epoch 00005 | Step 00100 | Loss 0.5085 | Train Acc 0.8545 | Speed (samples/sec) 6134.2445 | GPU 791.4 MB\n",
      "Epoch 00005 | Step 00120 | Loss 0.5748 | Train Acc 0.8350 | Speed (samples/sec) 6138.5892 | GPU 791.4 MB\n",
      "Epoch 00005 | Step 00140 | Loss 0.5148 | Train Acc 0.8584 | Speed (samples/sec) 6140.1151 | GPU 791.4 MB\n",
      "Epoch 00005 | Step 00160 | Loss 0.5504 | Train Acc 0.8516 | Speed (samples/sec) 6142.3569 | GPU 791.4 MB\n",
      "Epoch 00005 | Step 00180 | Loss 0.4804 | Train Acc 0.8525 | Speed (samples/sec) 6138.3613 | GPU 791.4 MB\n",
      "Epoch Time(s): 35.3329\n",
      "Epoch 00006 | Step 00000 | Loss 0.4792 | Train Acc 0.8623 | Speed (samples/sec) 6133.0725 | GPU 791.4 MB\n",
      "Epoch 00006 | Step 00020 | Loss 0.5465 | Train Acc 0.8359 | Speed (samples/sec) 6132.8678 | GPU 791.4 MB\n",
      "Epoch 00006 | Step 00040 | Loss 0.5671 | Train Acc 0.8623 | Speed (samples/sec) 6131.0690 | GPU 791.4 MB\n",
      "Epoch 00006 | Step 00060 | Loss 0.5105 | Train Acc 0.8535 | Speed (samples/sec) 6128.0503 | GPU 791.4 MB\n",
      "Epoch 00006 | Step 00080 | Loss 0.5056 | Train Acc 0.8604 | Speed (samples/sec) 6129.3279 | GPU 791.7 MB\n",
      "Epoch 00006 | Step 00100 | Loss 0.5350 | Train Acc 0.8428 | Speed (samples/sec) 6128.8416 | GPU 791.7 MB\n",
      "Epoch 00006 | Step 00120 | Loss 0.4522 | Train Acc 0.8779 | Speed (samples/sec) 6128.5511 | GPU 791.8 MB\n",
      "Epoch 00006 | Step 00140 | Loss 0.5049 | Train Acc 0.8643 | Speed (samples/sec) 6132.8714 | GPU 791.8 MB\n",
      "Epoch 00006 | Step 00160 | Loss 0.5615 | Train Acc 0.8398 | Speed (samples/sec) 6129.8238 | GPU 791.8 MB\n",
      "Epoch 00006 | Step 00180 | Loss 0.5165 | Train Acc 0.8408 | Speed (samples/sec) 6129.1431 | GPU 791.8 MB\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/245 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch Time(s): 36.4076\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 245/245 [00:39<00:00,  6.18it/s]\n",
      "100%|██████████| 245/245 [00:48<00:00,  5.02it/s]\n",
      "100%|██████████| 245/245 [00:47<00:00,  5.14it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val acc 0.7764, Test Acc: 0.6117\n",
      "Epoch 00007 | Step 00000 | Loss 0.7289 | Train Acc 0.7988 | Speed (samples/sec) 6121.4543 | GPU 791.8 MB\n",
      "Epoch 00007 | Step 00020 | Loss 0.5493 | Train Acc 0.8447 | Speed (samples/sec) 6126.2350 | GPU 791.8 MB\n",
      "Epoch 00007 | Step 00040 | Loss 0.5425 | Train Acc 0.8438 | Speed (samples/sec) 6130.9755 | GPU 791.8 MB\n",
      "Epoch 00007 | Step 00060 | Loss 0.5311 | Train Acc 0.8516 | Speed (samples/sec) 6131.9336 | GPU 791.8 MB\n",
      "Epoch 00007 | Step 00080 | Loss 0.6043 | Train Acc 0.8242 | Speed (samples/sec) 6132.7007 | GPU 791.8 MB\n",
      "Epoch 00007 | Step 00100 | Loss 0.4780 | Train Acc 0.8555 | Speed (samples/sec) 6138.1277 | GPU 791.8 MB\n",
      "Epoch 00007 | Step 00120 | Loss 0.5044 | Train Acc 0.8535 | Speed (samples/sec) 6143.2546 | GPU 791.8 MB\n",
      "Epoch 00007 | Step 00140 | Loss 0.4776 | Train Acc 0.8623 | Speed (samples/sec) 6146.5125 | GPU 791.8 MB\n",
      "Epoch 00007 | Step 00160 | Loss 0.4892 | Train Acc 0.8633 | Speed (samples/sec) 6149.7394 | GPU 791.8 MB\n",
      "Epoch 00007 | Step 00180 | Loss 0.5104 | Train Acc 0.8584 | Speed (samples/sec) 6157.0242 | GPU 791.8 MB\n",
      "Epoch Time(s): 34.2843\n",
      "Epoch 00008 | Step 00000 | Loss 0.5698 | Train Acc 0.8398 | Speed (samples/sec) 6152.8979 | GPU 791.8 MB\n",
      "Epoch 00008 | Step 00020 | Loss 0.5255 | Train Acc 0.8467 | Speed (samples/sec) 6156.3581 | GPU 791.8 MB\n",
      "Epoch 00008 | Step 00040 | Loss 0.4366 | Train Acc 0.8770 | Speed (samples/sec) 6161.3967 | GPU 791.8 MB\n",
      "Epoch 00008 | Step 00060 | Loss 0.4677 | Train Acc 0.8721 | Speed (samples/sec) 6166.3036 | GPU 791.8 MB\n",
      "Epoch 00008 | Step 00080 | Loss 0.5230 | Train Acc 0.8574 | Speed (samples/sec) 6171.2487 | GPU 791.8 MB\n",
      "Epoch 00008 | Step 00100 | Loss 0.4846 | Train Acc 0.8633 | Speed (samples/sec) 6170.3226 | GPU 791.8 MB\n",
      "Epoch 00008 | Step 00120 | Loss 0.4827 | Train Acc 0.8545 | Speed (samples/sec) 6171.0799 | GPU 791.8 MB\n",
      "Epoch 00008 | Step 00140 | Loss 0.5275 | Train Acc 0.8477 | Speed (samples/sec) 6175.7943 | GPU 791.8 MB\n",
      "Epoch 00008 | Step 00160 | Loss 0.4496 | Train Acc 0.8779 | Speed (samples/sec) 6174.9522 | GPU 791.8 MB\n",
      "Epoch 00008 | Step 00180 | Loss 0.4771 | Train Acc 0.8691 | Speed (samples/sec) 6173.1455 | GPU 791.8 MB\n",
      "Epoch Time(s): 34.5559\n",
      "Epoch 00009 | Step 00000 | Loss 0.5522 | Train Acc 0.8428 | Speed (samples/sec) 6167.7161 | GPU 791.8 MB\n",
      "Epoch 00009 | Step 00020 | Loss 0.5030 | Train Acc 0.8506 | Speed (samples/sec) 6169.5669 | GPU 791.8 MB\n",
      "Epoch 00009 | Step 00040 | Loss 0.5098 | Train Acc 0.8574 | Speed (samples/sec) 6167.5287 | GPU 791.8 MB\n",
      "Epoch 00009 | Step 00060 | Loss 0.4896 | Train Acc 0.8594 | Speed (samples/sec) 6165.2535 | GPU 791.8 MB\n",
      "Epoch 00009 | Step 00080 | Loss 0.5042 | Train Acc 0.8525 | Speed (samples/sec) 6163.6836 | GPU 791.8 MB\n",
      "Epoch 00009 | Step 00100 | Loss 0.4966 | Train Acc 0.8535 | Speed (samples/sec) 6161.6989 | GPU 791.8 MB\n",
      "Epoch 00009 | Step 00120 | Loss 0.5056 | Train Acc 0.8486 | Speed (samples/sec) 6160.9749 | GPU 791.8 MB\n",
      "Epoch 00009 | Step 00140 | Loss 0.5209 | Train Acc 0.8711 | Speed (samples/sec) 6161.4950 | GPU 791.8 MB\n",
      "Epoch 00009 | Step 00160 | Loss 0.4988 | Train Acc 0.8506 | Speed (samples/sec) 6156.6789 | GPU 791.8 MB\n",
      "Epoch 00009 | Step 00180 | Loss 0.5506 | Train Acc 0.8408 | Speed (samples/sec) 6155.3940 | GPU 791.8 MB\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/245 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch Time(s): 36.5858\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 245/245 [00:38<00:00,  6.31it/s]\n",
      "100%|██████████| 245/245 [00:47<00:00,  5.17it/s]\n",
      "100%|██████████| 245/245 [00:48<00:00,  5.04it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val acc 0.8594, Test Acc: 0.7056\n",
      "Best val acc 0.8594, best test acc: 0.7056\n",
      "Epoch 00010 | Step 00000 | Loss 0.5822 | Train Acc 0.8369 | Speed (samples/sec) 6148.4466 | GPU 791.8 MB\n",
      "Epoch 00010 | Step 00020 | Loss 0.5442 | Train Acc 0.8408 | Speed (samples/sec) 6149.0747 | GPU 791.8 MB\n",
      "Epoch 00010 | Step 00040 | Loss 0.5436 | Train Acc 0.8477 | Speed (samples/sec) 6149.5069 | GPU 791.8 MB\n",
      "Epoch 00010 | Step 00060 | Loss 0.4992 | Train Acc 0.8574 | Speed (samples/sec) 6150.0209 | GPU 791.8 MB\n",
      "Epoch 00010 | Step 00080 | Loss 0.5596 | Train Acc 0.8330 | Speed (samples/sec) 6150.4253 | GPU 791.8 MB\n",
      "Epoch 00010 | Step 00100 | Loss 0.5257 | Train Acc 0.8564 | Speed (samples/sec) 6150.8144 | GPU 791.8 MB\n",
      "Epoch 00010 | Step 00120 | Loss 0.5315 | Train Acc 0.8477 | Speed (samples/sec) 6151.5088 | GPU 791.8 MB\n",
      "Epoch 00010 | Step 00140 | Loss 0.5407 | Train Acc 0.8486 | Speed (samples/sec) 6153.1639 | GPU 791.8 MB\n",
      "Epoch 00010 | Step 00160 | Loss 0.5158 | Train Acc 0.8564 | Speed (samples/sec) 6153.2579 | GPU 791.8 MB\n",
      "Epoch 00010 | Step 00180 | Loss 0.5362 | Train Acc 0.8525 | Speed (samples/sec) 6154.5325 | GPU 791.8 MB\n",
      "Epoch Time(s): 35.4356\n",
      "Epoch 00011 | Step 00000 | Loss 0.4237 | Train Acc 0.8691 | Speed (samples/sec) 6149.3274 | GPU 791.8 MB\n",
      "Epoch 00011 | Step 00020 | Loss 0.4759 | Train Acc 0.8613 | Speed (samples/sec) 6148.7731 | GPU 791.8 MB\n",
      "Epoch 00011 | Step 00040 | Loss 0.5896 | Train Acc 0.8359 | Speed (samples/sec) 6149.1795 | GPU 791.8 MB\n",
      "Epoch 00011 | Step 00060 | Loss 0.4580 | Train Acc 0.8545 | Speed (samples/sec) 6147.1151 | GPU 791.8 MB\n",
      "Epoch 00011 | Step 00080 | Loss 0.5100 | Train Acc 0.8613 | Speed (samples/sec) 6148.1511 | GPU 791.8 MB\n",
      "Epoch 00011 | Step 00100 | Loss 0.4608 | Train Acc 0.8740 | Speed (samples/sec) 6149.6742 | GPU 791.8 MB\n",
      "Epoch 00011 | Step 00120 | Loss 0.4922 | Train Acc 0.8652 | Speed (samples/sec) 6149.8541 | GPU 791.8 MB\n",
      "Epoch 00011 | Step 00140 | Loss 0.5112 | Train Acc 0.8662 | Speed (samples/sec) 6150.4056 | GPU 791.8 MB\n",
      "Epoch 00011 | Step 00160 | Loss 0.4570 | Train Acc 0.8662 | Speed (samples/sec) 6151.2942 | GPU 791.8 MB\n",
      "Epoch 00011 | Step 00180 | Loss 0.4902 | Train Acc 0.8662 | Speed (samples/sec) 6150.4850 | GPU 791.8 MB\n",
      "Epoch Time(s): 35.5958\n",
      "Epoch 00012 | Step 00000 | Loss 0.6750 | Train Acc 0.8193 | Speed (samples/sec) 6147.0294 | GPU 791.8 MB\n",
      "Epoch 00012 | Step 00020 | Loss 0.6193 | Train Acc 0.8145 | Speed (samples/sec) 6145.8394 | GPU 791.8 MB\n",
      "Epoch 00012 | Step 00040 | Loss 0.4673 | Train Acc 0.8711 | Speed (samples/sec) 6144.4412 | GPU 791.8 MB\n",
      "Epoch 00012 | Step 00060 | Loss 0.4810 | Train Acc 0.8584 | Speed (samples/sec) 6145.3691 | GPU 791.8 MB\n",
      "Epoch 00012 | Step 00080 | Loss 0.4492 | Train Acc 0.8691 | Speed (samples/sec) 6146.9020 | GPU 791.8 MB\n",
      "Epoch 00012 | Step 00100 | Loss 0.4665 | Train Acc 0.8574 | Speed (samples/sec) 6147.3299 | GPU 791.8 MB\n",
      "Epoch 00012 | Step 00120 | Loss 0.4963 | Train Acc 0.8633 | Speed (samples/sec) 6149.1032 | GPU 791.8 MB\n",
      "Epoch 00012 | Step 00140 | Loss 0.4690 | Train Acc 0.8564 | Speed (samples/sec) 6151.9603 | GPU 791.8 MB\n",
      "Epoch 00012 | Step 00160 | Loss 0.4405 | Train Acc 0.8789 | Speed (samples/sec) 6151.0537 | GPU 791.8 MB\n",
      "Epoch 00012 | Step 00180 | Loss 0.5025 | Train Acc 0.8623 | Speed (samples/sec) 6150.2135 | GPU 791.8 MB\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/245 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch Time(s): 35.5599\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 245/245 [00:39<00:00,  6.23it/s]\n",
      "100%|██████████| 245/245 [00:49<00:00,  4.95it/s]\n",
      "100%|██████████| 245/245 [00:48<00:00,  5.03it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val acc 0.7914, Test Acc: 0.6285\n",
      "Epoch 00013 | Step 00000 | Loss 0.7152 | Train Acc 0.8027 | Speed (samples/sec) 6145.7797 | GPU 791.8 MB\n",
      "Epoch 00013 | Step 00020 | Loss 0.4954 | Train Acc 0.8574 | Speed (samples/sec) 6145.4460 | GPU 791.8 MB\n",
      "Epoch 00013 | Step 00040 | Loss 0.5039 | Train Acc 0.8564 | Speed (samples/sec) 6144.7758 | GPU 791.8 MB\n",
      "Epoch 00013 | Step 00060 | Loss 0.4361 | Train Acc 0.8643 | Speed (samples/sec) 6145.1595 | GPU 791.8 MB\n",
      "Epoch 00013 | Step 00080 | Loss 0.4801 | Train Acc 0.8662 | Speed (samples/sec) 6146.2020 | GPU 791.8 MB\n",
      "Epoch 00013 | Step 00100 | Loss 0.4733 | Train Acc 0.8662 | Speed (samples/sec) 6147.2955 | GPU 791.8 MB\n",
      "Epoch 00013 | Step 00120 | Loss 0.4503 | Train Acc 0.8604 | Speed (samples/sec) 6148.9212 | GPU 791.8 MB\n",
      "Epoch 00013 | Step 00140 | Loss 0.4936 | Train Acc 0.8604 | Speed (samples/sec) 6147.3651 | GPU 791.8 MB\n",
      "Epoch 00013 | Step 00160 | Loss 0.4691 | Train Acc 0.8564 | Speed (samples/sec) 6148.3299 | GPU 791.8 MB\n",
      "Epoch 00013 | Step 00180 | Loss 0.4455 | Train Acc 0.8633 | Speed (samples/sec) 6149.7532 | GPU 791.8 MB\n",
      "Epoch Time(s): 35.4736\n",
      "Epoch 00014 | Step 00000 | Loss 0.6481 | Train Acc 0.8145 | Speed (samples/sec) 6146.0314 | GPU 791.8 MB\n",
      "Epoch 00014 | Step 00020 | Loss 0.4773 | Train Acc 0.8604 | Speed (samples/sec) 6146.2733 | GPU 791.8 MB\n",
      "Epoch 00014 | Step 00040 | Loss 0.4508 | Train Acc 0.8662 | Speed (samples/sec) 6146.8882 | GPU 791.8 MB\n",
      "Epoch 00014 | Step 00060 | Loss 0.4455 | Train Acc 0.8691 | Speed (samples/sec) 6146.8891 | GPU 791.8 MB\n",
      "Epoch 00014 | Step 00080 | Loss 0.5262 | Train Acc 0.8496 | Speed (samples/sec) 6146.5684 | GPU 791.8 MB\n",
      "Epoch 00014 | Step 00100 | Loss 0.5481 | Train Acc 0.8398 | Speed (samples/sec) 6147.0000 | GPU 791.8 MB\n",
      "Epoch 00014 | Step 00120 | Loss 0.4829 | Train Acc 0.8701 | Speed (samples/sec) 6147.9870 | GPU 791.8 MB\n",
      "Epoch 00014 | Step 00140 | Loss 0.5206 | Train Acc 0.8555 | Speed (samples/sec) 6148.5155 | GPU 791.8 MB\n",
      "Epoch 00014 | Step 00160 | Loss 0.4616 | Train Acc 0.8818 | Speed (samples/sec) 6149.6734 | GPU 791.8 MB\n",
      "Epoch 00014 | Step 00180 | Loss 0.4640 | Train Acc 0.8760 | Speed (samples/sec) 6150.1337 | GPU 791.8 MB\n",
      "Epoch Time(s): 35.5315\n",
      "Epoch 00015 | Step 00000 | Loss 0.5357 | Train Acc 0.8535 | Speed (samples/sec) 6146.0912 | GPU 791.8 MB\n",
      "Epoch 00015 | Step 00020 | Loss 0.4643 | Train Acc 0.8643 | Speed (samples/sec) 6146.2143 | GPU 791.8 MB\n",
      "Epoch 00015 | Step 00040 | Loss 0.4241 | Train Acc 0.8770 | Speed (samples/sec) 6146.4409 | GPU 791.8 MB\n",
      "Epoch 00015 | Step 00060 | Loss 0.4294 | Train Acc 0.8887 | Speed (samples/sec) 6148.0721 | GPU 791.8 MB\n",
      "Epoch 00015 | Step 00080 | Loss 0.4282 | Train Acc 0.8691 | Speed (samples/sec) 6149.1827 | GPU 791.8 MB\n",
      "Epoch 00015 | Step 00100 | Loss 0.4899 | Train Acc 0.8604 | Speed (samples/sec) 6150.6759 | GPU 791.8 MB\n",
      "Epoch 00015 | Step 00120 | Loss 0.3799 | Train Acc 0.8867 | Speed (samples/sec) 6152.9821 | GPU 791.8 MB\n",
      "Epoch 00015 | Step 00140 | Loss 0.4579 | Train Acc 0.8701 | Speed (samples/sec) 6155.4894 | GPU 791.8 MB\n",
      "Epoch 00015 | Step 00160 | Loss 0.4984 | Train Acc 0.8477 | Speed (samples/sec) 6157.8367 | GPU 791.8 MB\n",
      "Epoch 00015 | Step 00180 | Loss 0.5050 | Train Acc 0.8604 | Speed (samples/sec) 6157.3347 | GPU 791.8 MB\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/245 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch Time(s): 34.8824\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 245/245 [00:38<00:00,  6.40it/s]\n",
      "100%|██████████| 245/245 [00:48<00:00,  5.01it/s]\n",
      "100%|██████████| 245/245 [00:48<00:00,  5.03it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val acc 0.8752, Test Acc: 0.7439\n",
      "Best val acc 0.8752, best test acc: 0.7439\n",
      "Epoch 00016 | Step 00000 | Loss 0.4461 | Train Acc 0.8730 | Speed (samples/sec) 6155.6904 | GPU 791.8 MB\n",
      "Epoch 00016 | Step 00020 | Loss 0.4201 | Train Acc 0.8916 | Speed (samples/sec) 6154.8901 | GPU 791.8 MB\n",
      "Epoch 00016 | Step 00040 | Loss 0.4060 | Train Acc 0.8867 | Speed (samples/sec) 6153.1934 | GPU 791.8 MB\n",
      "Epoch 00016 | Step 00060 | Loss 0.4487 | Train Acc 0.8701 | Speed (samples/sec) 6153.4593 | GPU 791.8 MB\n",
      "Epoch 00016 | Step 00080 | Loss 0.4492 | Train Acc 0.8633 | Speed (samples/sec) 6154.1611 | GPU 791.8 MB\n",
      "Epoch 00016 | Step 00100 | Loss 0.4696 | Train Acc 0.8613 | Speed (samples/sec) 6153.1460 | GPU 791.8 MB\n",
      "Epoch 00016 | Step 00120 | Loss 0.4631 | Train Acc 0.8486 | Speed (samples/sec) 6151.8756 | GPU 791.8 MB\n",
      "Epoch 00016 | Step 00140 | Loss 0.4084 | Train Acc 0.8799 | Speed (samples/sec) 6152.3095 | GPU 791.8 MB\n",
      "Epoch 00016 | Step 00160 | Loss 0.4665 | Train Acc 0.8623 | Speed (samples/sec) 6152.5715 | GPU 791.8 MB\n",
      "Epoch 00016 | Step 00180 | Loss 0.4849 | Train Acc 0.8711 | Speed (samples/sec) 6153.7184 | GPU 791.8 MB\n",
      "Epoch Time(s): 35.7805\n",
      "Epoch 00017 | Step 00000 | Loss 0.5049 | Train Acc 0.8496 | Speed (samples/sec) 6151.1433 | GPU 791.8 MB\n",
      "Epoch 00017 | Step 00020 | Loss 0.4650 | Train Acc 0.8623 | Speed (samples/sec) 6149.2599 | GPU 791.8 MB\n",
      "Epoch 00017 | Step 00040 | Loss 0.4824 | Train Acc 0.8652 | Speed (samples/sec) 6148.1929 | GPU 791.8 MB\n",
      "Epoch 00017 | Step 00060 | Loss 0.3862 | Train Acc 0.8887 | Speed (samples/sec) 6148.0528 | GPU 791.8 MB\n",
      "Epoch 00017 | Step 00080 | Loss 0.4616 | Train Acc 0.8584 | Speed (samples/sec) 6147.7363 | GPU 791.8 MB\n",
      "Epoch 00017 | Step 00100 | Loss 0.4322 | Train Acc 0.8701 | Speed (samples/sec) 6149.2302 | GPU 791.8 MB\n",
      "Epoch 00017 | Step 00120 | Loss 0.4517 | Train Acc 0.8721 | Speed (samples/sec) 6149.0396 | GPU 791.8 MB\n",
      "Epoch 00017 | Step 00140 | Loss 0.4376 | Train Acc 0.8652 | Speed (samples/sec) 6149.8117 | GPU 791.8 MB\n",
      "Epoch 00017 | Step 00160 | Loss 0.4975 | Train Acc 0.8545 | Speed (samples/sec) 6150.5900 | GPU 791.8 MB\n",
      "Epoch 00017 | Step 00180 | Loss 0.4897 | Train Acc 0.8613 | Speed (samples/sec) 6152.1518 | GPU 791.8 MB\n",
      "Epoch Time(s): 35.4888\n",
      "Epoch 00018 | Step 00000 | Loss 0.6337 | Train Acc 0.8340 | Speed (samples/sec) 6148.8600 | GPU 791.8 MB\n",
      "Epoch 00018 | Step 00020 | Loss 0.4795 | Train Acc 0.8613 | Speed (samples/sec) 6147.7592 | GPU 791.8 MB\n",
      "Epoch 00018 | Step 00040 | Loss 0.4520 | Train Acc 0.8662 | Speed (samples/sec) 6147.9607 | GPU 791.8 MB\n",
      "Epoch 00018 | Step 00060 | Loss 0.4257 | Train Acc 0.8818 | Speed (samples/sec) 6146.1186 | GPU 791.8 MB\n",
      "Epoch 00018 | Step 00080 | Loss 0.4542 | Train Acc 0.8730 | Speed (samples/sec) 6144.8242 | GPU 791.8 MB\n",
      "Epoch 00018 | Step 00100 | Loss 0.3745 | Train Acc 0.8916 | Speed (samples/sec) 6145.0476 | GPU 791.8 MB\n",
      "Epoch 00018 | Step 00120 | Loss 0.5136 | Train Acc 0.8555 | Speed (samples/sec) 6144.5852 | GPU 791.8 MB\n",
      "Epoch 00018 | Step 00140 | Loss 0.4855 | Train Acc 0.8633 | Speed (samples/sec) 6143.7253 | GPU 791.8 MB\n",
      "Epoch 00018 | Step 00160 | Loss 0.4641 | Train Acc 0.8770 | Speed (samples/sec) 6143.0519 | GPU 791.8 MB\n",
      "Epoch 00018 | Step 00180 | Loss 0.4856 | Train Acc 0.8535 | Speed (samples/sec) 6141.6526 | GPU 791.8 MB\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/245 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch Time(s): 37.1903\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 245/245 [00:38<00:00,  6.34it/s]\n",
      "100%|██████████| 245/245 [00:52<00:00,  4.70it/s]\n",
      "100%|██████████| 245/245 [00:52<00:00,  4.66it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val acc 0.8741, Test Acc: 0.7163\n",
      "Epoch 00019 | Step 00000 | Loss 0.3862 | Train Acc 0.8828 | Speed (samples/sec) 6137.5992 | GPU 791.8 MB\n",
      "Epoch 00019 | Step 00020 | Loss 0.4564 | Train Acc 0.8691 | Speed (samples/sec) 6133.6706 | GPU 791.8 MB\n",
      "Epoch 00019 | Step 00040 | Loss 0.5154 | Train Acc 0.8613 | Speed (samples/sec) 6132.9414 | GPU 791.8 MB\n",
      "Epoch 00019 | Step 00060 | Loss 0.4601 | Train Acc 0.8701 | Speed (samples/sec) 6131.6125 | GPU 791.8 MB\n",
      "Epoch 00019 | Step 00080 | Loss 0.4061 | Train Acc 0.8906 | Speed (samples/sec) 6129.7509 | GPU 791.8 MB\n",
      "Epoch 00019 | Step 00100 | Loss 0.4178 | Train Acc 0.8691 | Speed (samples/sec) 6127.4091 | GPU 791.8 MB\n",
      "Epoch 00019 | Step 00120 | Loss 0.4388 | Train Acc 0.8848 | Speed (samples/sec) 6125.6230 | GPU 791.8 MB\n",
      "Epoch 00019 | Step 00140 | Loss 0.4589 | Train Acc 0.8809 | Speed (samples/sec) 6123.6024 | GPU 791.8 MB\n",
      "Epoch 00019 | Step 00160 | Loss 0.4922 | Train Acc 0.8633 | Speed (samples/sec) 6120.7244 | GPU 791.8 MB\n",
      "Epoch 00019 | Step 00180 | Loss 0.4156 | Train Acc 0.8711 | Speed (samples/sec) 6118.8047 | GPU 791.8 MB\n",
      "Epoch Time(s): 38.7569\n",
      "Epoch 00020 | Step 00000 | Loss 0.4772 | Train Acc 0.8613 | Speed (samples/sec) 6114.9764 | GPU 791.8 MB\n",
      "Epoch 00020 | Step 00020 | Loss 0.4414 | Train Acc 0.8604 | Speed (samples/sec) 6113.5035 | GPU 791.8 MB\n",
      "Epoch 00020 | Step 00040 | Loss 0.4700 | Train Acc 0.8672 | Speed (samples/sec) 6110.7568 | GPU 791.8 MB\n",
      "Epoch 00020 | Step 00060 | Loss 0.4502 | Train Acc 0.8701 | Speed (samples/sec) 6108.4891 | GPU 791.8 MB\n",
      "Epoch 00020 | Step 00080 | Loss 0.5566 | Train Acc 0.8545 | Speed (samples/sec) 6107.8074 | GPU 791.8 MB\n",
      "Epoch 00020 | Step 00100 | Loss 0.4061 | Train Acc 0.8857 | Speed (samples/sec) 6105.7441 | GPU 791.8 MB\n",
      "Epoch 00020 | Step 00120 | Loss 0.4283 | Train Acc 0.8779 | Speed (samples/sec) 6102.5607 | GPU 791.8 MB\n",
      "Epoch 00020 | Step 00140 | Loss 0.4985 | Train Acc 0.8457 | Speed (samples/sec) 6100.7825 | GPU 791.8 MB\n",
      "Epoch 00020 | Step 00160 | Loss 0.4441 | Train Acc 0.8691 | Speed (samples/sec) 6098.3727 | GPU 791.8 MB\n",
      "Epoch 00020 | Step 00180 | Loss 0.3836 | Train Acc 0.8984 | Speed (samples/sec) 6096.4895 | GPU 791.8 MB\n",
      "Epoch Time(s): 38.9134\n",
      "Epoch 00021 | Step 00000 | Loss 0.9176 | Train Acc 0.7627 | Speed (samples/sec) 6093.1004 | GPU 791.8 MB\n",
      "Epoch 00021 | Step 00020 | Loss 0.4819 | Train Acc 0.8486 | Speed (samples/sec) 6090.7844 | GPU 791.8 MB\n",
      "Epoch 00021 | Step 00040 | Loss 0.4673 | Train Acc 0.8623 | Speed (samples/sec) 6089.2306 | GPU 791.8 MB\n",
      "Epoch 00021 | Step 00060 | Loss 0.4799 | Train Acc 0.8633 | Speed (samples/sec) 6087.1198 | GPU 791.8 MB\n",
      "Epoch 00021 | Step 00080 | Loss 0.4570 | Train Acc 0.8701 | Speed (samples/sec) 6085.8279 | GPU 791.8 MB\n",
      "Epoch 00021 | Step 00100 | Loss 0.4158 | Train Acc 0.8867 | Speed (samples/sec) 6083.8663 | GPU 791.8 MB\n",
      "Epoch 00021 | Step 00120 | Loss 0.4406 | Train Acc 0.8779 | Speed (samples/sec) 6081.3589 | GPU 791.8 MB\n",
      "Epoch 00021 | Step 00140 | Loss 0.4058 | Train Acc 0.8896 | Speed (samples/sec) 6078.8298 | GPU 791.8 MB\n",
      "Epoch 00021 | Step 00160 | Loss 0.4528 | Train Acc 0.8555 | Speed (samples/sec) 6077.0322 | GPU 791.8 MB\n",
      "Epoch 00021 | Step 00180 | Loss 0.3903 | Train Acc 0.8799 | Speed (samples/sec) 6076.5414 | GPU 791.8 MB\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/245 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch Time(s): 39.1095\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 245/245 [00:42<00:00,  5.75it/s]\n",
      "100%|██████████| 245/245 [00:53<00:00,  4.59it/s]\n",
      "100%|██████████| 245/245 [00:53<00:00,  4.56it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val acc 0.8773, Test Acc: 0.7180\n",
      "Best val acc 0.8773, best test acc: 0.7180\n",
      "Epoch 00022 | Step 00000 | Loss 0.4637 | Train Acc 0.8721 | Speed (samples/sec) 6073.2675 | GPU 791.8 MB\n",
      "Epoch 00022 | Step 00020 | Loss 0.4183 | Train Acc 0.8906 | Speed (samples/sec) 6070.6106 | GPU 791.8 MB\n",
      "Epoch 00022 | Step 00040 | Loss 0.4503 | Train Acc 0.8730 | Speed (samples/sec) 6070.1145 | GPU 791.8 MB\n",
      "Epoch 00022 | Step 00060 | Loss 0.4669 | Train Acc 0.8770 | Speed (samples/sec) 6069.0622 | GPU 791.8 MB\n",
      "Epoch 00022 | Step 00080 | Loss 0.4466 | Train Acc 0.8691 | Speed (samples/sec) 6066.9555 | GPU 791.8 MB\n",
      "Epoch 00022 | Step 00100 | Loss 0.5028 | Train Acc 0.8555 | Speed (samples/sec) 6066.3180 | GPU 791.8 MB\n",
      "Epoch 00022 | Step 00120 | Loss 0.4907 | Train Acc 0.8535 | Speed (samples/sec) 6065.0176 | GPU 791.8 MB\n",
      "Epoch 00022 | Step 00140 | Loss 0.4761 | Train Acc 0.8691 | Speed (samples/sec) 6063.7724 | GPU 791.8 MB\n",
      "Epoch 00022 | Step 00160 | Loss 0.4106 | Train Acc 0.8838 | Speed (samples/sec) 6061.1698 | GPU 791.8 MB\n",
      "Epoch 00022 | Step 00180 | Loss 0.3760 | Train Acc 0.8760 | Speed (samples/sec) 6059.7002 | GPU 791.8 MB\n",
      "Epoch Time(s): 38.8652\n",
      "Epoch 00023 | Step 00000 | Loss 0.7780 | Train Acc 0.7891 | Speed (samples/sec) 6056.4279 | GPU 791.8 MB\n",
      "Epoch 00023 | Step 00020 | Loss 0.4465 | Train Acc 0.8652 | Speed (samples/sec) 6054.8851 | GPU 791.8 MB\n",
      "Epoch 00023 | Step 00040 | Loss 0.4885 | Train Acc 0.8496 | Speed (samples/sec) 6054.3676 | GPU 791.8 MB\n",
      "Epoch 00023 | Step 00060 | Loss 0.4316 | Train Acc 0.8623 | Speed (samples/sec) 6051.8794 | GPU 791.8 MB\n",
      "Epoch 00023 | Step 00080 | Loss 0.4318 | Train Acc 0.8691 | Speed (samples/sec) 6051.8164 | GPU 792.3 MB\n",
      "Epoch 00023 | Step 00100 | Loss 0.4643 | Train Acc 0.8633 | Speed (samples/sec) 6048.9120 | GPU 792.3 MB\n",
      "Epoch 00023 | Step 00120 | Loss 0.4022 | Train Acc 0.8760 | Speed (samples/sec) 6045.7622 | GPU 792.3 MB\n",
      "Epoch 00023 | Step 00140 | Loss 0.5110 | Train Acc 0.8662 | Speed (samples/sec) 6042.7848 | GPU 792.3 MB\n",
      "Epoch 00023 | Step 00160 | Loss 0.4667 | Train Acc 0.8594 | Speed (samples/sec) 6041.2277 | GPU 792.3 MB\n",
      "Epoch 00023 | Step 00180 | Loss 0.4406 | Train Acc 0.8672 | Speed (samples/sec) 6038.1620 | GPU 792.3 MB\n",
      "Epoch Time(s): 39.6442\n",
      "Epoch 00024 | Step 00000 | Loss 0.4777 | Train Acc 0.8594 | Speed (samples/sec) 6035.1691 | GPU 792.3 MB\n",
      "Epoch 00024 | Step 00020 | Loss 0.5014 | Train Acc 0.8516 | Speed (samples/sec) 6033.2200 | GPU 792.3 MB\n",
      "Epoch 00024 | Step 00040 | Loss 0.4570 | Train Acc 0.8623 | Speed (samples/sec) 6031.0288 | GPU 792.3 MB\n",
      "Epoch 00024 | Step 00060 | Loss 0.4174 | Train Acc 0.8643 | Speed (samples/sec) 6029.2572 | GPU 792.3 MB\n",
      "Epoch 00024 | Step 00080 | Loss 0.4086 | Train Acc 0.8770 | Speed (samples/sec) 6026.3693 | GPU 792.3 MB\n",
      "Epoch 00024 | Step 00100 | Loss 0.4952 | Train Acc 0.8584 | Speed (samples/sec) 6024.8130 | GPU 792.3 MB\n",
      "Epoch 00024 | Step 00120 | Loss 0.4942 | Train Acc 0.8633 | Speed (samples/sec) 6022.7180 | GPU 792.3 MB\n",
      "Epoch 00024 | Step 00140 | Loss 0.4529 | Train Acc 0.8711 | Speed (samples/sec) 6020.5647 | GPU 792.3 MB\n",
      "Epoch 00024 | Step 00160 | Loss 0.4257 | Train Acc 0.8799 | Speed (samples/sec) 6019.0234 | GPU 792.3 MB\n",
      "Epoch 00024 | Step 00180 | Loss 0.4514 | Train Acc 0.8574 | Speed (samples/sec) 6017.0651 | GPU 792.3 MB\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/245 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch Time(s): 40.3105\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 245/245 [00:43<00:00,  5.62it/s]\n",
      "100%|██████████| 245/245 [00:52<00:00,  4.68it/s]\n",
      "100%|██████████| 245/245 [00:50<00:00,  4.85it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val acc 0.8809, Test Acc: 0.7252\n",
      "Best val acc 0.8809, best test acc: 0.7252\n",
      "Epoch 00025 | Step 00000 | Loss 0.3849 | Train Acc 0.8867 | Speed (samples/sec) 6012.8511 | GPU 792.3 MB\n",
      "Epoch 00025 | Step 00020 | Loss 0.4681 | Train Acc 0.8623 | Speed (samples/sec) 6010.4121 | GPU 792.3 MB\n",
      "Epoch 00025 | Step 00040 | Loss 0.4191 | Train Acc 0.8721 | Speed (samples/sec) 6009.2713 | GPU 792.3 MB\n",
      "Epoch 00025 | Step 00060 | Loss 0.3854 | Train Acc 0.8887 | Speed (samples/sec) 6008.1685 | GPU 792.3 MB\n",
      "Epoch 00025 | Step 00080 | Loss 0.4004 | Train Acc 0.9004 | Speed (samples/sec) 6007.0615 | GPU 792.3 MB\n",
      "Epoch 00025 | Step 00100 | Loss 0.4645 | Train Acc 0.8721 | Speed (samples/sec) 6006.3592 | GPU 792.3 MB\n",
      "Epoch 00025 | Step 00120 | Loss 0.4344 | Train Acc 0.8750 | Speed (samples/sec) 6006.3770 | GPU 792.3 MB\n",
      "Epoch 00025 | Step 00140 | Loss 0.4009 | Train Acc 0.8789 | Speed (samples/sec) 6005.8891 | GPU 792.3 MB\n",
      "Epoch 00025 | Step 00160 | Loss 0.5018 | Train Acc 0.8643 | Speed (samples/sec) 6005.1088 | GPU 792.3 MB\n",
      "Epoch 00025 | Step 00180 | Loss 0.4417 | Train Acc 0.8857 | Speed (samples/sec) 6004.1923 | GPU 792.3 MB\n",
      "Epoch Time(s): 38.2650\n",
      "Epoch 00026 | Step 00000 | Loss 0.5123 | Train Acc 0.8486 | Speed (samples/sec) 6002.0854 | GPU 792.3 MB\n",
      "Epoch 00026 | Step 00020 | Loss 0.4192 | Train Acc 0.8789 | Speed (samples/sec) 6000.7083 | GPU 792.3 MB\n",
      "Epoch 00026 | Step 00040 | Loss 0.4666 | Train Acc 0.8691 | Speed (samples/sec) 6000.1559 | GPU 792.3 MB\n",
      "Epoch 00026 | Step 00060 | Loss 0.4426 | Train Acc 0.8711 | Speed (samples/sec) 5999.0935 | GPU 792.3 MB\n",
      "Epoch 00026 | Step 00080 | Loss 0.4447 | Train Acc 0.8662 | Speed (samples/sec) 5998.9103 | GPU 792.3 MB\n",
      "Epoch 00026 | Step 00100 | Loss 0.4920 | Train Acc 0.8643 | Speed (samples/sec) 5998.2493 | GPU 792.3 MB\n",
      "Epoch 00026 | Step 00120 | Loss 0.4254 | Train Acc 0.8857 | Speed (samples/sec) 5998.1562 | GPU 792.3 MB\n",
      "Epoch 00026 | Step 00140 | Loss 0.3606 | Train Acc 0.8906 | Speed (samples/sec) 5997.6907 | GPU 792.3 MB\n",
      "Epoch 00026 | Step 00160 | Loss 0.4372 | Train Acc 0.8711 | Speed (samples/sec) 5997.2137 | GPU 792.3 MB\n",
      "Epoch 00026 | Step 00180 | Loss 0.3878 | Train Acc 0.8779 | Speed (samples/sec) 5995.2151 | GPU 792.3 MB\n",
      "Epoch Time(s): 37.8621\n",
      "Epoch 00027 | Step 00000 | Loss 0.4429 | Train Acc 0.8574 | Speed (samples/sec) 5992.6165 | GPU 792.3 MB\n",
      "Epoch 00027 | Step 00020 | Loss 0.4587 | Train Acc 0.8721 | Speed (samples/sec) 5991.3339 | GPU 792.3 MB\n",
      "Epoch 00027 | Step 00040 | Loss 0.5071 | Train Acc 0.8652 | Speed (samples/sec) 5990.8184 | GPU 792.3 MB\n",
      "Epoch 00027 | Step 00060 | Loss 0.4430 | Train Acc 0.8721 | Speed (samples/sec) 5989.9879 | GPU 792.3 MB\n",
      "Epoch 00027 | Step 00080 | Loss 0.4931 | Train Acc 0.8740 | Speed (samples/sec) 5989.2015 | GPU 792.3 MB\n",
      "Epoch 00027 | Step 00100 | Loss 0.4002 | Train Acc 0.8828 | Speed (samples/sec) 5988.7372 | GPU 792.3 MB\n",
      "Epoch 00027 | Step 00120 | Loss 0.3962 | Train Acc 0.8838 | Speed (samples/sec) 5987.8919 | GPU 792.3 MB\n",
      "Epoch 00027 | Step 00140 | Loss 0.4397 | Train Acc 0.8799 | Speed (samples/sec) 5987.4460 | GPU 792.3 MB\n",
      "Epoch 00027 | Step 00160 | Loss 0.4212 | Train Acc 0.8789 | Speed (samples/sec) 5985.9651 | GPU 792.3 MB\n",
      "Epoch 00027 | Step 00180 | Loss 0.4356 | Train Acc 0.8750 | Speed (samples/sec) 5984.7506 | GPU 792.3 MB\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/245 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch Time(s): 38.4514\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 245/245 [00:43<00:00,  5.58it/s]\n",
      "100%|██████████| 245/245 [00:55<00:00,  4.42it/s]\n",
      "100%|██████████| 245/245 [00:55<00:00,  4.41it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val acc 0.8739, Test Acc: 0.7090\n",
      "Epoch 00028 | Step 00000 | Loss 0.4328 | Train Acc 0.8730 | Speed (samples/sec) 5983.1011 | GPU 792.3 MB\n",
      "Epoch 00028 | Step 00020 | Loss 0.4862 | Train Acc 0.8672 | Speed (samples/sec) 5982.4179 | GPU 792.3 MB\n",
      "Epoch 00028 | Step 00040 | Loss 0.4186 | Train Acc 0.8838 | Speed (samples/sec) 5980.7022 | GPU 792.3 MB\n",
      "Epoch 00028 | Step 00060 | Loss 0.3829 | Train Acc 0.8760 | Speed (samples/sec) 5978.6426 | GPU 792.3 MB\n",
      "Epoch 00028 | Step 00080 | Loss 0.4207 | Train Acc 0.8848 | Speed (samples/sec) 5977.4320 | GPU 792.3 MB\n",
      "Epoch 00028 | Step 00100 | Loss 0.4200 | Train Acc 0.8809 | Speed (samples/sec) 5976.2690 | GPU 792.3 MB\n",
      "Epoch 00028 | Step 00120 | Loss 0.4467 | Train Acc 0.8799 | Speed (samples/sec) 5975.3417 | GPU 792.3 MB\n",
      "Epoch 00028 | Step 00140 | Loss 0.3276 | Train Acc 0.9082 | Speed (samples/sec) 5974.6011 | GPU 792.3 MB\n",
      "Epoch 00028 | Step 00160 | Loss 0.4136 | Train Acc 0.8779 | Speed (samples/sec) 5973.5651 | GPU 792.3 MB\n",
      "Epoch 00028 | Step 00180 | Loss 0.4753 | Train Acc 0.8701 | Speed (samples/sec) 5972.5489 | GPU 792.3 MB\n",
      "Epoch Time(s): 39.0538\n",
      "Epoch 00029 | Step 00000 | Loss 0.4257 | Train Acc 0.8760 | Speed (samples/sec) 5970.0331 | GPU 792.3 MB\n",
      "Epoch 00029 | Step 00020 | Loss 0.4169 | Train Acc 0.8760 | Speed (samples/sec) 5968.6339 | GPU 792.3 MB\n",
      "Epoch 00029 | Step 00040 | Loss 0.3910 | Train Acc 0.8916 | Speed (samples/sec) 5967.9732 | GPU 792.3 MB\n",
      "Epoch 00029 | Step 00060 | Loss 0.4415 | Train Acc 0.8711 | Speed (samples/sec) 5967.0441 | GPU 792.3 MB\n",
      "Epoch 00029 | Step 00080 | Loss 0.4627 | Train Acc 0.8623 | Speed (samples/sec) 5966.1869 | GPU 792.3 MB\n",
      "Epoch 00029 | Step 00100 | Loss 0.3885 | Train Acc 0.8906 | Speed (samples/sec) 5965.6395 | GPU 792.3 MB\n",
      "Epoch 00029 | Step 00120 | Loss 0.3905 | Train Acc 0.8750 | Speed (samples/sec) 5964.9222 | GPU 792.3 MB\n",
      "Epoch 00029 | Step 00140 | Loss 0.4740 | Train Acc 0.8740 | Speed (samples/sec) 5964.2998 | GPU 792.3 MB\n",
      "Epoch 00029 | Step 00160 | Loss 0.3874 | Train Acc 0.8945 | Speed (samples/sec) 5963.5223 | GPU 792.3 MB\n",
      "Epoch 00029 | Step 00180 | Loss 0.4315 | Train Acc 0.8682 | Speed (samples/sec) 5963.5045 | GPU 792.3 MB\n",
      "Epoch Time(s): 38.2514\n",
      "Epoch 00030 | Step 00000 | Loss 0.3559 | Train Acc 0.8906 | Speed (samples/sec) 5961.3451 | GPU 792.3 MB\n",
      "Epoch 00030 | Step 00020 | Loss 0.4801 | Train Acc 0.8672 | Speed (samples/sec) 5961.3468 | GPU 792.3 MB\n",
      "Epoch 00030 | Step 00040 | Loss 0.3823 | Train Acc 0.8906 | Speed (samples/sec) 5960.9557 | GPU 792.3 MB\n",
      "Epoch 00030 | Step 00060 | Loss 0.4429 | Train Acc 0.8750 | Speed (samples/sec) 5960.8639 | GPU 792.3 MB\n",
      "Epoch 00030 | Step 00080 | Loss 0.4512 | Train Acc 0.8701 | Speed (samples/sec) 5960.5108 | GPU 792.3 MB\n",
      "Epoch 00030 | Step 00100 | Loss 0.4605 | Train Acc 0.8730 | Speed (samples/sec) 5959.6034 | GPU 792.3 MB\n",
      "Epoch 00030 | Step 00120 | Loss 0.4098 | Train Acc 0.8887 | Speed (samples/sec) 5958.5368 | GPU 792.3 MB\n",
      "Epoch 00030 | Step 00140 | Loss 0.4318 | Train Acc 0.8711 | Speed (samples/sec) 5957.9092 | GPU 792.3 MB\n",
      "Epoch 00030 | Step 00160 | Loss 0.4365 | Train Acc 0.8760 | Speed (samples/sec) 5956.9484 | GPU 792.3 MB\n",
      "Epoch 00030 | Step 00180 | Loss 0.3973 | Train Acc 0.8750 | Speed (samples/sec) 5956.4996 | GPU 792.3 MB\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/245 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch Time(s): 38.0709\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 245/245 [00:42<00:00,  5.75it/s]\n",
      "100%|██████████| 245/245 [00:54<00:00,  4.50it/s]\n",
      "100%|██████████| 245/245 [00:50<00:00,  4.81it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val acc 0.8773, Test Acc: 0.7175\n",
      "Epoch 00031 | Step 00000 | Loss 0.3961 | Train Acc 0.8828 | Speed (samples/sec) 5955.1060 | GPU 792.3 MB\n",
      "Epoch 00031 | Step 00020 | Loss 0.3705 | Train Acc 0.8945 | Speed (samples/sec) 5954.2725 | GPU 792.3 MB\n",
      "Epoch 00031 | Step 00040 | Loss 0.3941 | Train Acc 0.8760 | Speed (samples/sec) 5953.7363 | GPU 792.3 MB\n",
      "Epoch 00031 | Step 00060 | Loss 0.3927 | Train Acc 0.9004 | Speed (samples/sec) 5953.1380 | GPU 792.3 MB\n",
      "Epoch 00031 | Step 00080 | Loss 0.4155 | Train Acc 0.8740 | Speed (samples/sec) 5952.5978 | GPU 792.3 MB\n",
      "Epoch 00031 | Step 00100 | Loss 0.4178 | Train Acc 0.8828 | Speed (samples/sec) 5952.4549 | GPU 792.3 MB\n",
      "Epoch 00031 | Step 00120 | Loss 0.4006 | Train Acc 0.8838 | Speed (samples/sec) 5951.0390 | GPU 792.3 MB\n",
      "Epoch 00031 | Step 00140 | Loss 0.4589 | Train Acc 0.8613 | Speed (samples/sec) 5950.0411 | GPU 792.3 MB\n",
      "Epoch 00031 | Step 00160 | Loss 0.4895 | Train Acc 0.8633 | Speed (samples/sec) 5949.8745 | GPU 792.3 MB\n",
      "Epoch 00031 | Step 00180 | Loss 0.4315 | Train Acc 0.8760 | Speed (samples/sec) 5949.1617 | GPU 792.3 MB\n",
      "Epoch Time(s): 38.8438\n",
      "Epoch 00032 | Step 00000 | Loss 0.4147 | Train Acc 0.8887 | Speed (samples/sec) 5947.2579 | GPU 792.3 MB\n",
      "Epoch 00032 | Step 00020 | Loss 0.4077 | Train Acc 0.8877 | Speed (samples/sec) 5945.9955 | GPU 792.3 MB\n",
      "Epoch 00032 | Step 00040 | Loss 0.4810 | Train Acc 0.8574 | Speed (samples/sec) 5945.0985 | GPU 792.3 MB\n",
      "Epoch 00032 | Step 00060 | Loss 0.4102 | Train Acc 0.8867 | Speed (samples/sec) 5945.0606 | GPU 792.3 MB\n",
      "Epoch 00032 | Step 00080 | Loss 0.4237 | Train Acc 0.8652 | Speed (samples/sec) 5944.3176 | GPU 792.3 MB\n",
      "Epoch 00032 | Step 00100 | Loss 0.3558 | Train Acc 0.9014 | Speed (samples/sec) 5943.5030 | GPU 792.3 MB\n",
      "Epoch 00032 | Step 00120 | Loss 0.3747 | Train Acc 0.8848 | Speed (samples/sec) 5943.0591 | GPU 792.3 MB\n",
      "Epoch 00032 | Step 00140 | Loss 0.3785 | Train Acc 0.8936 | Speed (samples/sec) 5942.0140 | GPU 792.3 MB\n",
      "Epoch 00032 | Step 00160 | Loss 0.4481 | Train Acc 0.8740 | Speed (samples/sec) 5941.3362 | GPU 792.3 MB\n",
      "Epoch 00032 | Step 00180 | Loss 0.4645 | Train Acc 0.8799 | Speed (samples/sec) 5940.6676 | GPU 792.3 MB\n",
      "Epoch Time(s): 38.7375\n",
      "Epoch 00033 | Step 00000 | Loss 0.5304 | Train Acc 0.8486 | Speed (samples/sec) 5938.2808 | GPU 792.3 MB\n",
      "Epoch 00033 | Step 00020 | Loss 0.4189 | Train Acc 0.8848 | Speed (samples/sec) 5937.8261 | GPU 792.3 MB\n",
      "Epoch 00033 | Step 00040 | Loss 0.4323 | Train Acc 0.8740 | Speed (samples/sec) 5937.4081 | GPU 792.3 MB\n",
      "Epoch 00033 | Step 00060 | Loss 0.3974 | Train Acc 0.9043 | Speed (samples/sec) 5938.2389 | GPU 792.3 MB\n",
      "Epoch 00033 | Step 00080 | Loss 0.4777 | Train Acc 0.8652 | Speed (samples/sec) 5938.3419 | GPU 792.3 MB\n",
      "Epoch 00033 | Step 00100 | Loss 0.4269 | Train Acc 0.8799 | Speed (samples/sec) 5938.5669 | GPU 792.3 MB\n",
      "Epoch 00033 | Step 00120 | Loss 0.4208 | Train Acc 0.8867 | Speed (samples/sec) 5938.8997 | GPU 792.3 MB\n",
      "Epoch 00033 | Step 00140 | Loss 0.4152 | Train Acc 0.8770 | Speed (samples/sec) 5939.2454 | GPU 792.3 MB\n",
      "Epoch 00033 | Step 00160 | Loss 0.4536 | Train Acc 0.8740 | Speed (samples/sec) 5939.4478 | GPU 792.3 MB\n",
      "Epoch 00033 | Step 00180 | Loss 0.4086 | Train Acc 0.8828 | Speed (samples/sec) 5939.2784 | GPU 792.3 MB\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/245 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch Time(s): 37.0802\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 245/245 [00:39<00:00,  6.25it/s]\n",
      "100%|██████████| 245/245 [00:49<00:00,  4.99it/s]\n",
      "100%|██████████| 245/245 [00:49<00:00,  4.93it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val acc 0.8357, Test Acc: 0.6852\n",
      "Epoch 00034 | Step 00000 | Loss 0.5607 | Train Acc 0.8408 | Speed (samples/sec) 5937.9568 | GPU 792.3 MB\n",
      "Epoch 00034 | Step 00020 | Loss 0.4663 | Train Acc 0.8682 | Speed (samples/sec) 5938.0981 | GPU 792.3 MB\n",
      "Epoch 00034 | Step 00040 | Loss 0.4115 | Train Acc 0.8828 | Speed (samples/sec) 5938.0037 | GPU 792.3 MB\n",
      "Epoch 00034 | Step 00060 | Loss 0.4033 | Train Acc 0.8838 | Speed (samples/sec) 5938.3521 | GPU 792.3 MB\n",
      "Epoch 00034 | Step 00080 | Loss 0.3906 | Train Acc 0.8828 | Speed (samples/sec) 5938.2788 | GPU 792.3 MB\n",
      "Epoch 00034 | Step 00100 | Loss 0.4143 | Train Acc 0.8809 | Speed (samples/sec) 5938.0419 | GPU 792.3 MB\n",
      "Epoch 00034 | Step 00120 | Loss 0.4320 | Train Acc 0.8750 | Speed (samples/sec) 5937.1887 | GPU 792.3 MB\n",
      "Epoch 00034 | Step 00140 | Loss 0.4616 | Train Acc 0.8564 | Speed (samples/sec) 5937.6832 | GPU 792.3 MB\n",
      "Epoch 00034 | Step 00160 | Loss 0.4324 | Train Acc 0.8682 | Speed (samples/sec) 5937.8791 | GPU 792.3 MB\n",
      "Epoch 00034 | Step 00180 | Loss 0.3740 | Train Acc 0.8896 | Speed (samples/sec) 5937.7028 | GPU 792.3 MB\n",
      "Epoch Time(s): 37.5555\n",
      "Epoch 00035 | Step 00000 | Loss 0.4049 | Train Acc 0.8936 | Speed (samples/sec) 5935.5654 | GPU 792.3 MB\n",
      "Epoch 00035 | Step 00020 | Loss 0.4008 | Train Acc 0.8877 | Speed (samples/sec) 5936.1169 | GPU 792.3 MB\n",
      "Epoch 00035 | Step 00040 | Loss 0.4258 | Train Acc 0.8857 | Speed (samples/sec) 5937.5520 | GPU 792.3 MB\n",
      "Epoch 00035 | Step 00060 | Loss 0.4253 | Train Acc 0.8701 | Speed (samples/sec) 5938.0524 | GPU 792.3 MB\n",
      "Epoch 00035 | Step 00080 | Loss 0.4546 | Train Acc 0.8760 | Speed (samples/sec) 5938.5418 | GPU 792.3 MB\n",
      "Epoch 00035 | Step 00100 | Loss 0.4096 | Train Acc 0.8779 | Speed (samples/sec) 5939.2174 | GPU 792.3 MB\n",
      "Epoch 00035 | Step 00120 | Loss 0.3890 | Train Acc 0.8906 | Speed (samples/sec) 5939.2820 | GPU 792.3 MB\n",
      "Epoch 00035 | Step 00140 | Loss 0.4008 | Train Acc 0.8916 | Speed (samples/sec) 5939.7188 | GPU 792.3 MB\n",
      "Epoch 00035 | Step 00160 | Loss 0.4040 | Train Acc 0.8809 | Speed (samples/sec) 5940.1787 | GPU 792.3 MB\n",
      "Epoch 00035 | Step 00180 | Loss 0.4569 | Train Acc 0.8643 | Speed (samples/sec) 5940.0598 | GPU 792.3 MB\n",
      "Epoch Time(s): 36.3899\n",
      "Epoch 00036 | Step 00000 | Loss 0.6870 | Train Acc 0.7930 | Speed (samples/sec) 5937.8021 | GPU 792.3 MB\n",
      "Epoch 00036 | Step 00020 | Loss 0.4720 | Train Acc 0.8701 | Speed (samples/sec) 5938.0818 | GPU 792.3 MB\n",
      "Epoch 00036 | Step 00040 | Loss 0.4659 | Train Acc 0.8623 | Speed (samples/sec) 5938.4799 | GPU 792.3 MB\n",
      "Epoch 00036 | Step 00060 | Loss 0.3779 | Train Acc 0.8906 | Speed (samples/sec) 5938.3263 | GPU 792.3 MB\n",
      "Epoch 00036 | Step 00080 | Loss 0.4493 | Train Acc 0.8633 | Speed (samples/sec) 5937.7330 | GPU 792.3 MB\n",
      "Epoch 00036 | Step 00100 | Loss 0.4120 | Train Acc 0.8916 | Speed (samples/sec) 5937.4241 | GPU 792.3 MB\n",
      "Epoch 00036 | Step 00120 | Loss 0.4541 | Train Acc 0.8633 | Speed (samples/sec) 5937.0666 | GPU 792.3 MB\n",
      "Epoch 00036 | Step 00140 | Loss 0.4398 | Train Acc 0.8760 | Speed (samples/sec) 5936.6130 | GPU 792.3 MB\n",
      "Epoch 00036 | Step 00160 | Loss 0.4032 | Train Acc 0.8926 | Speed (samples/sec) 5936.4586 | GPU 792.3 MB\n",
      "Epoch 00036 | Step 00180 | Loss 0.4080 | Train Acc 0.8740 | Speed (samples/sec) 5935.6739 | GPU 792.3 MB\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/245 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch Time(s): 37.9210\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 245/245 [00:41<00:00,  5.95it/s]\n",
      "100%|██████████| 245/245 [00:49<00:00,  4.93it/s]\n",
      "100%|██████████| 245/245 [00:51<00:00,  4.76it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val acc 0.8824, Test Acc: 0.7508\n",
      "Best val acc 0.8824, best test acc: 0.7508\n",
      "Epoch 00037 | Step 00000 | Loss 0.4295 | Train Acc 0.8848 | Speed (samples/sec) 5934.8470 | GPU 792.3 MB\n",
      "Epoch 00037 | Step 00020 | Loss 0.3696 | Train Acc 0.8906 | Speed (samples/sec) 5934.3321 | GPU 792.3 MB\n",
      "Epoch 00037 | Step 00040 | Loss 0.4447 | Train Acc 0.8877 | Speed (samples/sec) 5934.9164 | GPU 792.3 MB\n",
      "Epoch 00037 | Step 00060 | Loss 0.4573 | Train Acc 0.8750 | Speed (samples/sec) 5934.4157 | GPU 792.3 MB\n",
      "Epoch 00037 | Step 00080 | Loss 0.4351 | Train Acc 0.8760 | Speed (samples/sec) 5935.3200 | GPU 792.3 MB\n",
      "Epoch 00037 | Step 00100 | Loss 0.3766 | Train Acc 0.8984 | Speed (samples/sec) 5935.7015 | GPU 792.3 MB\n",
      "Epoch 00037 | Step 00120 | Loss 0.4377 | Train Acc 0.8770 | Speed (samples/sec) 5935.9417 | GPU 792.3 MB\n",
      "Epoch 00037 | Step 00140 | Loss 0.4047 | Train Acc 0.8789 | Speed (samples/sec) 5935.9255 | GPU 792.3 MB\n",
      "Epoch 00037 | Step 00160 | Loss 0.3741 | Train Acc 0.8945 | Speed (samples/sec) 5936.3011 | GPU 792.3 MB\n",
      "Epoch 00037 | Step 00180 | Loss 0.4593 | Train Acc 0.8662 | Speed (samples/sec) 5937.0268 | GPU 792.3 MB\n",
      "Epoch Time(s): 36.6346\n",
      "Epoch 00038 | Step 00000 | Loss 0.4027 | Train Acc 0.8838 | Speed (samples/sec) 5935.6051 | GPU 792.3 MB\n",
      "Epoch 00038 | Step 00020 | Loss 0.4162 | Train Acc 0.8633 | Speed (samples/sec) 5936.3101 | GPU 792.3 MB\n",
      "Epoch 00038 | Step 00040 | Loss 0.3939 | Train Acc 0.8809 | Speed (samples/sec) 5936.4975 | GPU 792.3 MB\n",
      "Epoch 00038 | Step 00060 | Loss 0.4515 | Train Acc 0.8799 | Speed (samples/sec) 5936.7706 | GPU 792.3 MB\n",
      "Epoch 00038 | Step 00080 | Loss 0.4393 | Train Acc 0.8809 | Speed (samples/sec) 5937.7467 | GPU 792.3 MB\n",
      "Epoch 00038 | Step 00100 | Loss 0.4262 | Train Acc 0.8799 | Speed (samples/sec) 5938.2481 | GPU 792.3 MB\n",
      "Epoch 00038 | Step 00120 | Loss 0.3849 | Train Acc 0.8887 | Speed (samples/sec) 5938.1911 | GPU 792.3 MB\n",
      "Epoch 00038 | Step 00140 | Loss 0.4424 | Train Acc 0.8652 | Speed (samples/sec) 5938.3148 | GPU 792.3 MB\n",
      "Epoch 00038 | Step 00160 | Loss 0.3862 | Train Acc 0.8877 | Speed (samples/sec) 5938.5319 | GPU 792.3 MB\n",
      "Epoch 00038 | Step 00180 | Loss 0.4018 | Train Acc 0.8896 | Speed (samples/sec) 5937.7161 | GPU 792.3 MB\n",
      "Epoch Time(s): 36.8815\n",
      "Epoch 00039 | Step 00000 | Loss 0.4570 | Train Acc 0.8652 | Speed (samples/sec) 5936.0692 | GPU 792.3 MB\n",
      "Epoch 00039 | Step 00020 | Loss 0.4724 | Train Acc 0.8633 | Speed (samples/sec) 5935.8410 | GPU 792.3 MB\n",
      "Epoch 00039 | Step 00040 | Loss 0.4052 | Train Acc 0.8750 | Speed (samples/sec) 5935.2504 | GPU 792.3 MB\n",
      "Epoch 00039 | Step 00060 | Loss 0.4089 | Train Acc 0.8779 | Speed (samples/sec) 5935.0134 | GPU 792.3 MB\n",
      "Epoch 00039 | Step 00080 | Loss 0.4122 | Train Acc 0.8770 | Speed (samples/sec) 5934.7935 | GPU 792.3 MB\n",
      "Epoch 00039 | Step 00100 | Loss 0.4056 | Train Acc 0.8799 | Speed (samples/sec) 5934.7369 | GPU 792.3 MB\n",
      "Epoch 00039 | Step 00120 | Loss 0.4542 | Train Acc 0.8789 | Speed (samples/sec) 5933.9101 | GPU 792.3 MB\n",
      "Epoch 00039 | Step 00140 | Loss 0.4173 | Train Acc 0.8857 | Speed (samples/sec) 5933.8350 | GPU 792.3 MB\n",
      "Epoch 00039 | Step 00160 | Loss 0.3624 | Train Acc 0.9004 | Speed (samples/sec) 5933.2527 | GPU 792.3 MB\n",
      "Epoch 00039 | Step 00180 | Loss 0.4276 | Train Acc 0.8672 | Speed (samples/sec) 5932.6306 | GPU 792.3 MB\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/245 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch Time(s): 38.3240\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 245/245 [00:41<00:00,  5.90it/s]\n",
      "100%|██████████| 245/245 [00:53<00:00,  4.62it/s]\n",
      "100%|██████████| 245/245 [00:50<00:00,  4.83it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val acc 0.8635, Test Acc: 0.7160\n",
      "Epoch 00040 | Step 00000 | Loss 0.4438 | Train Acc 0.8721 | Speed (samples/sec) 5931.1993 | GPU 792.3 MB\n",
      "Epoch 00040 | Step 00020 | Loss 0.4558 | Train Acc 0.8555 | Speed (samples/sec) 5931.5191 | GPU 792.3 MB\n",
      "Epoch 00040 | Step 00040 | Loss 0.3326 | Train Acc 0.9004 | Speed (samples/sec) 5931.2591 | GPU 792.3 MB\n",
      "Epoch 00040 | Step 00060 | Loss 0.4425 | Train Acc 0.8770 | Speed (samples/sec) 5930.6048 | GPU 792.3 MB\n",
      "Epoch 00040 | Step 00080 | Loss 0.3740 | Train Acc 0.8926 | Speed (samples/sec) 5930.7380 | GPU 792.3 MB\n",
      "Epoch 00040 | Step 00100 | Loss 0.4401 | Train Acc 0.8652 | Speed (samples/sec) 5930.3695 | GPU 792.3 MB\n",
      "Epoch 00040 | Step 00120 | Loss 0.3803 | Train Acc 0.8955 | Speed (samples/sec) 5930.8324 | GPU 792.3 MB\n",
      "Epoch 00040 | Step 00140 | Loss 0.4279 | Train Acc 0.8848 | Speed (samples/sec) 5930.4701 | GPU 792.3 MB\n",
      "Epoch 00040 | Step 00160 | Loss 0.3567 | Train Acc 0.8955 | Speed (samples/sec) 5930.3318 | GPU 792.3 MB\n",
      "Epoch 00040 | Step 00180 | Loss 0.4030 | Train Acc 0.8896 | Speed (samples/sec) 5930.1885 | GPU 792.3 MB\n",
      "Epoch Time(s): 37.9145\n",
      "Epoch 00041 | Step 00000 | Loss 0.3761 | Train Acc 0.8770 | Speed (samples/sec) 5928.4863 | GPU 792.3 MB\n",
      "Epoch 00041 | Step 00020 | Loss 0.4058 | Train Acc 0.8789 | Speed (samples/sec) 5928.9037 | GPU 792.3 MB\n",
      "Epoch 00041 | Step 00040 | Loss 0.3856 | Train Acc 0.8994 | Speed (samples/sec) 5929.2281 | GPU 792.3 MB\n",
      "Epoch 00041 | Step 00060 | Loss 0.4661 | Train Acc 0.8633 | Speed (samples/sec) 5929.1577 | GPU 792.3 MB\n",
      "Epoch 00041 | Step 00080 | Loss 0.4058 | Train Acc 0.8701 | Speed (samples/sec) 5929.4771 | GPU 792.3 MB\n",
      "Epoch 00041 | Step 00100 | Loss 0.3528 | Train Acc 0.8955 | Speed (samples/sec) 5929.7574 | GPU 792.3 MB\n",
      "Epoch 00041 | Step 00120 | Loss 0.4352 | Train Acc 0.8809 | Speed (samples/sec) 5929.4639 | GPU 792.3 MB\n",
      "Epoch 00041 | Step 00140 | Loss 0.4135 | Train Acc 0.8779 | Speed (samples/sec) 5929.6681 | GPU 792.3 MB\n",
      "Epoch 00041 | Step 00160 | Loss 0.4079 | Train Acc 0.8848 | Speed (samples/sec) 5929.9432 | GPU 792.3 MB\n",
      "Epoch 00041 | Step 00180 | Loss 0.4232 | Train Acc 0.8779 | Speed (samples/sec) 5930.4574 | GPU 792.3 MB\n",
      "Epoch Time(s): 36.6491\n",
      "Epoch 00042 | Step 00000 | Loss 0.4263 | Train Acc 0.8799 | Speed (samples/sec) 5929.0562 | GPU 792.3 MB\n",
      "Epoch 00042 | Step 00020 | Loss 0.4754 | Train Acc 0.8613 | Speed (samples/sec) 5929.1279 | GPU 792.3 MB\n",
      "Epoch 00042 | Step 00040 | Loss 0.3944 | Train Acc 0.8857 | Speed (samples/sec) 5929.4333 | GPU 792.3 MB\n",
      "Epoch 00042 | Step 00060 | Loss 0.4268 | Train Acc 0.8711 | Speed (samples/sec) 5929.6768 | GPU 792.3 MB\n",
      "Epoch 00042 | Step 00080 | Loss 0.4074 | Train Acc 0.8779 | Speed (samples/sec) 5930.0295 | GPU 792.3 MB\n",
      "Epoch 00042 | Step 00100 | Loss 0.4363 | Train Acc 0.8750 | Speed (samples/sec) 5929.9738 | GPU 792.3 MB\n",
      "Epoch 00042 | Step 00120 | Loss 0.3721 | Train Acc 0.8877 | Speed (samples/sec) 5930.1612 | GPU 792.3 MB\n",
      "Epoch 00042 | Step 00140 | Loss 0.4119 | Train Acc 0.8789 | Speed (samples/sec) 5930.5449 | GPU 792.3 MB\n",
      "Epoch 00042 | Step 00160 | Loss 0.4216 | Train Acc 0.8779 | Speed (samples/sec) 5930.9223 | GPU 792.3 MB\n",
      "Epoch 00042 | Step 00180 | Loss 0.3591 | Train Acc 0.8936 | Speed (samples/sec) 5931.1381 | GPU 792.3 MB\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/245 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch Time(s): 36.9172\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 245/245 [00:39<00:00,  6.18it/s]\n",
      "100%|██████████| 245/245 [00:51<00:00,  4.74it/s]\n",
      "100%|██████████| 245/245 [00:49<00:00,  4.93it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val acc 0.8858, Test Acc: 0.7279\n",
      "Best val acc 0.8858, best test acc: 0.7279\n",
      "Epoch 00043 | Step 00000 | Loss 0.3827 | Train Acc 0.8926 | Speed (samples/sec) 5929.9041 | GPU 792.3 MB\n",
      "Epoch 00043 | Step 00020 | Loss 0.4326 | Train Acc 0.8740 | Speed (samples/sec) 5930.0099 | GPU 792.3 MB\n",
      "Epoch 00043 | Step 00040 | Loss 0.4315 | Train Acc 0.8682 | Speed (samples/sec) 5930.6868 | GPU 792.3 MB\n",
      "Epoch 00043 | Step 00060 | Loss 0.4274 | Train Acc 0.8730 | Speed (samples/sec) 5931.0474 | GPU 792.3 MB\n",
      "Epoch 00043 | Step 00080 | Loss 0.4020 | Train Acc 0.8936 | Speed (samples/sec) 5931.7264 | GPU 792.3 MB\n",
      "Epoch 00043 | Step 00100 | Loss 0.4550 | Train Acc 0.8818 | Speed (samples/sec) 5931.4391 | GPU 792.3 MB\n",
      "Epoch 00043 | Step 00120 | Loss 0.4108 | Train Acc 0.8779 | Speed (samples/sec) 5931.8824 | GPU 792.3 MB\n",
      "Epoch 00043 | Step 00140 | Loss 0.3896 | Train Acc 0.8809 | Speed (samples/sec) 5932.1424 | GPU 792.3 MB\n",
      "Epoch 00043 | Step 00160 | Loss 0.4448 | Train Acc 0.8779 | Speed (samples/sec) 5932.4834 | GPU 792.3 MB\n",
      "Epoch 00043 | Step 00180 | Loss 0.3655 | Train Acc 0.8926 | Speed (samples/sec) 5933.1668 | GPU 792.3 MB\n",
      "Epoch Time(s): 36.2119\n",
      "Epoch 00044 | Step 00000 | Loss 0.8719 | Train Acc 0.7910 | Speed (samples/sec) 5931.7459 | GPU 792.3 MB\n",
      "Epoch 00044 | Step 00020 | Loss 0.4399 | Train Acc 0.8730 | Speed (samples/sec) 5932.2908 | GPU 792.3 MB\n",
      "Epoch 00044 | Step 00040 | Loss 0.4181 | Train Acc 0.8711 | Speed (samples/sec) 5932.6750 | GPU 792.3 MB\n",
      "Epoch 00044 | Step 00060 | Loss 0.4003 | Train Acc 0.8809 | Speed (samples/sec) 5933.0506 | GPU 792.3 MB\n",
      "Epoch 00044 | Step 00080 | Loss 0.4209 | Train Acc 0.8887 | Speed (samples/sec) 5933.3807 | GPU 792.3 MB\n",
      "Epoch 00044 | Step 00100 | Loss 0.3784 | Train Acc 0.8887 | Speed (samples/sec) 5934.1644 | GPU 792.3 MB\n",
      "Epoch 00044 | Step 00120 | Loss 0.4424 | Train Acc 0.8828 | Speed (samples/sec) 5934.2505 | GPU 792.3 MB\n",
      "Epoch 00044 | Step 00140 | Loss 0.3658 | Train Acc 0.8965 | Speed (samples/sec) 5934.0830 | GPU 792.3 MB\n",
      "Epoch 00044 | Step 00160 | Loss 0.4072 | Train Acc 0.8818 | Speed (samples/sec) 5934.5287 | GPU 792.3 MB\n",
      "Epoch 00044 | Step 00180 | Loss 0.3808 | Train Acc 0.8945 | Speed (samples/sec) 5935.3193 | GPU 792.3 MB\n",
      "Epoch Time(s): 36.1873\n",
      "Epoch 00045 | Step 00000 | Loss 0.4453 | Train Acc 0.8691 | Speed (samples/sec) 5934.0059 | GPU 792.3 MB\n",
      "Epoch 00045 | Step 00020 | Loss 0.4057 | Train Acc 0.8916 | Speed (samples/sec) 5934.0310 | GPU 792.3 MB\n",
      "Epoch 00045 | Step 00040 | Loss 0.3041 | Train Acc 0.9209 | Speed (samples/sec) 5934.6343 | GPU 792.3 MB\n",
      "Epoch 00045 | Step 00060 | Loss 0.4460 | Train Acc 0.8730 | Speed (samples/sec) 5934.9915 | GPU 792.3 MB\n",
      "Epoch 00045 | Step 00080 | Loss 0.4494 | Train Acc 0.8730 | Speed (samples/sec) 5935.3210 | GPU 792.3 MB\n",
      "Epoch 00045 | Step 00100 | Loss 0.3791 | Train Acc 0.8867 | Speed (samples/sec) 5935.4316 | GPU 792.3 MB\n",
      "Epoch 00045 | Step 00120 | Loss 0.3670 | Train Acc 0.8896 | Speed (samples/sec) 5935.4946 | GPU 792.3 MB\n",
      "Epoch 00045 | Step 00140 | Loss 0.3624 | Train Acc 0.8936 | Speed (samples/sec) 5935.6343 | GPU 792.3 MB\n",
      "Epoch 00045 | Step 00160 | Loss 0.4722 | Train Acc 0.8652 | Speed (samples/sec) 5935.6591 | GPU 792.3 MB\n",
      "Epoch 00045 | Step 00180 | Loss 0.4195 | Train Acc 0.8818 | Speed (samples/sec) 5936.1857 | GPU 792.3 MB\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/245 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch Time(s): 36.2868\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 245/245 [00:40<00:00,  6.12it/s]\n",
      "100%|██████████| 245/245 [00:51<00:00,  4.80it/s]\n",
      "100%|██████████| 245/245 [00:49<00:00,  4.99it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val acc 0.8847, Test Acc: 0.7111\n",
      "Epoch 00046 | Step 00000 | Loss 0.4711 | Train Acc 0.8691 | Speed (samples/sec) 5935.0547 | GPU 792.3 MB\n",
      "Epoch 00046 | Step 00020 | Loss 0.4282 | Train Acc 0.8828 | Speed (samples/sec) 5936.2986 | GPU 792.3 MB\n",
      "Epoch 00046 | Step 00040 | Loss 0.3499 | Train Acc 0.9004 | Speed (samples/sec) 5937.4623 | GPU 792.3 MB\n",
      "Epoch 00046 | Step 00060 | Loss 0.4556 | Train Acc 0.8760 | Speed (samples/sec) 5938.5340 | GPU 792.3 MB\n",
      "Epoch 00046 | Step 00080 | Loss 0.4067 | Train Acc 0.8809 | Speed (samples/sec) 5939.5502 | GPU 792.3 MB\n",
      "Epoch 00046 | Step 00100 | Loss 0.4251 | Train Acc 0.8760 | Speed (samples/sec) 5939.9162 | GPU 792.3 MB\n",
      "Epoch 00046 | Step 00120 | Loss 0.3999 | Train Acc 0.8838 | Speed (samples/sec) 5940.3261 | GPU 792.3 MB\n",
      "Epoch 00046 | Step 00140 | Loss 0.4475 | Train Acc 0.8789 | Speed (samples/sec) 5941.0026 | GPU 792.3 MB\n",
      "Epoch 00046 | Step 00160 | Loss 0.4180 | Train Acc 0.8867 | Speed (samples/sec) 5941.2900 | GPU 792.3 MB\n",
      "Epoch 00046 | Step 00180 | Loss 0.4309 | Train Acc 0.8730 | Speed (samples/sec) 5941.2685 | GPU 792.3 MB\n",
      "Epoch Time(s): 35.5460\n",
      "Epoch 00047 | Step 00000 | Loss 0.4364 | Train Acc 0.8760 | Speed (samples/sec) 5939.9353 | GPU 792.3 MB\n",
      "Epoch 00047 | Step 00020 | Loss 0.3834 | Train Acc 0.8818 | Speed (samples/sec) 5939.4586 | GPU 792.3 MB\n",
      "Epoch 00047 | Step 00040 | Loss 0.4218 | Train Acc 0.8770 | Speed (samples/sec) 5939.5130 | GPU 792.3 MB\n",
      "Epoch 00047 | Step 00060 | Loss 0.3989 | Train Acc 0.8828 | Speed (samples/sec) 5940.0051 | GPU 792.3 MB\n",
      "Epoch 00047 | Step 00080 | Loss 0.3732 | Train Acc 0.8838 | Speed (samples/sec) 5940.7283 | GPU 792.3 MB\n",
      "Epoch 00047 | Step 00100 | Loss 0.4362 | Train Acc 0.8730 | Speed (samples/sec) 5940.7645 | GPU 792.3 MB\n",
      "Epoch 00047 | Step 00120 | Loss 0.4951 | Train Acc 0.8613 | Speed (samples/sec) 5940.7148 | GPU 792.3 MB\n",
      "Epoch 00047 | Step 00140 | Loss 0.3844 | Train Acc 0.8887 | Speed (samples/sec) 5940.8964 | GPU 792.3 MB\n",
      "Epoch 00047 | Step 00160 | Loss 0.3677 | Train Acc 0.8945 | Speed (samples/sec) 5941.1965 | GPU 792.3 MB\n",
      "Epoch 00047 | Step 00180 | Loss 0.3983 | Train Acc 0.8848 | Speed (samples/sec) 5941.0670 | GPU 792.3 MB\n",
      "Epoch Time(s): 36.9596\n",
      "Epoch 00048 | Step 00000 | Loss 0.5722 | Train Acc 0.8398 | Speed (samples/sec) 5939.6283 | GPU 792.3 MB\n",
      "Epoch 00048 | Step 00020 | Loss 0.4016 | Train Acc 0.8906 | Speed (samples/sec) 5939.8966 | GPU 792.3 MB\n",
      "Epoch 00048 | Step 00040 | Loss 0.3710 | Train Acc 0.8838 | Speed (samples/sec) 5940.1879 | GPU 792.3 MB\n",
      "Epoch 00048 | Step 00060 | Loss 0.3505 | Train Acc 0.8955 | Speed (samples/sec) 5940.1791 | GPU 792.3 MB\n",
      "Epoch 00048 | Step 00080 | Loss 0.4015 | Train Acc 0.8789 | Speed (samples/sec) 5940.1659 | GPU 792.3 MB\n",
      "Epoch 00048 | Step 00100 | Loss 0.3424 | Train Acc 0.8887 | Speed (samples/sec) 5940.5001 | GPU 792.3 MB\n",
      "Epoch 00048 | Step 00120 | Loss 0.4364 | Train Acc 0.8770 | Speed (samples/sec) 5940.6348 | GPU 792.3 MB\n",
      "Epoch 00048 | Step 00140 | Loss 0.3651 | Train Acc 0.8896 | Speed (samples/sec) 5940.7368 | GPU 792.3 MB\n",
      "Epoch 00048 | Step 00160 | Loss 0.4575 | Train Acc 0.8682 | Speed (samples/sec) 5940.5124 | GPU 792.3 MB\n",
      "Epoch 00048 | Step 00180 | Loss 0.4024 | Train Acc 0.8857 | Speed (samples/sec) 5940.5673 | GPU 792.3 MB\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/245 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch Time(s): 36.7697\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 245/245 [00:39<00:00,  6.25it/s]\n",
      "100%|██████████| 245/245 [00:50<00:00,  4.85it/s]\n",
      "100%|██████████| 245/245 [00:51<00:00,  4.79it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val acc 0.8863, Test Acc: 0.7074\n",
      "Best val acc 0.8863, best test acc: 0.7074\n",
      "Epoch 00049 | Step 00000 | Loss 0.3805 | Train Acc 0.8906 | Speed (samples/sec) 5939.3450 | GPU 792.3 MB\n",
      "Epoch 00049 | Step 00020 | Loss 0.3424 | Train Acc 0.8965 | Speed (samples/sec) 5938.9181 | GPU 792.3 MB\n",
      "Epoch 00049 | Step 00040 | Loss 0.4212 | Train Acc 0.8760 | Speed (samples/sec) 5938.6383 | GPU 792.3 MB\n",
      "Epoch 00049 | Step 00060 | Loss 0.3989 | Train Acc 0.8809 | Speed (samples/sec) 5938.2831 | GPU 792.3 MB\n",
      "Epoch 00049 | Step 00080 | Loss 0.4668 | Train Acc 0.8555 | Speed (samples/sec) 5938.5465 | GPU 792.3 MB\n",
      "Epoch 00049 | Step 00100 | Loss 0.4271 | Train Acc 0.8809 | Speed (samples/sec) 5938.3405 | GPU 792.3 MB\n",
      "Epoch 00049 | Step 00120 | Loss 0.4044 | Train Acc 0.8838 | Speed (samples/sec) 5938.2036 | GPU 792.3 MB\n",
      "Epoch 00049 | Step 00140 | Loss 0.3922 | Train Acc 0.8809 | Speed (samples/sec) 5938.5685 | GPU 792.3 MB\n",
      "Epoch 00049 | Step 00160 | Loss 0.4372 | Train Acc 0.8770 | Speed (samples/sec) 5939.3187 | GPU 792.3 MB\n",
      "Epoch 00049 | Step 00180 | Loss 0.3668 | Train Acc 0.8887 | Speed (samples/sec) 5939.6881 | GPU 792.3 MB\n",
      "Epoch Time(s): 36.6456\n",
      "Avg epoch time: 40.22959917916192\n",
      "Best val acc 0.8863, best test acc: 0.7074\n"
     ]
    }
   ],
   "source": [
    "g, num_labels = load_ogb('ogbn-products')\n",
    "g.ndata.pop('features')\n",
    "\n",
    "in_feats = 128\n",
    "eigen_vals, eigen_vecs = get_eigen(g, in_feats, 'ogbn-products')\n",
    "train_g = val_g = test_g = g\n",
    "train_nfeat = val_nfeat = test_nfeat = th.tensor(eigen_vecs * np.sqrt(eigen_vals).reshape((1, len(eigen_vals))),\n",
    "                                                 dtype=th.float32)\n",
    "train_labels = val_labels = test_labels = g.ndata.pop('labels')\n",
    "\n",
    "hyperparams = {\n",
    "    'batch_size': 1024,\n",
    "    'num_workers': 4,\n",
    "    'num_hidden': 256,\n",
    "    'num_layers': 3,\n",
    "    'dropout': 0.5,\n",
    "    'lr': 0.003,\n",
    "    'num_epochs': 50,\n",
    "    'fanouts': [15,10,5],\n",
    "    'eval_batch_size': 10000\n",
    "}\n",
    "device = th.device('cuda:1')\n",
    "\n",
    "# Pack data\n",
    "data = train_g, val_g, test_g, train_nfeat, train_labels, \\\n",
    "           val_nfeat, val_labels, test_nfeat, test_labels\n",
    "# Define model\n",
    "in_feats = train_nfeat.shape[1]\n",
    "model = SAGE(in_feats, hyperparams['num_hidden'], num_labels,\n",
    "             hyperparams['num_layers'], F.relu, hyperparams['dropout'])\n",
    "train_sample(model, data, hyperparams, device, 3)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The experiment below is to show that we have to scale eigenvectors with square root of eigenvalues to achieve good performance. If we don't scale the eigenvectors, the performance is worse."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "load ogbn-products\n",
      "finish loading ogbn-products\n",
      "finish constructing ogbn-products\n",
      "Epoch 00000 | Step 00000 | Loss 3.8492 | Train Acc 0.0391 | Speed (samples/sec) nan | GPU 792.3 MB\n",
      "Epoch 00000 | Step 00020 | Loss 2.5491 | Train Acc 0.3066 | Speed (samples/sec) 5920.3537 | GPU 792.3 MB\n",
      "Epoch 00000 | Step 00040 | Loss 2.5437 | Train Acc 0.2900 | Speed (samples/sec) 6031.4227 | GPU 792.3 MB\n",
      "Epoch 00000 | Step 00060 | Loss 2.2843 | Train Acc 0.3301 | Speed (samples/sec) 5926.8878 | GPU 792.3 MB\n",
      "Epoch 00000 | Step 00080 | Loss 2.0064 | Train Acc 0.4482 | Speed (samples/sec) 5861.9924 | GPU 792.3 MB\n",
      "Epoch 00000 | Step 00100 | Loss 1.6505 | Train Acc 0.5488 | Speed (samples/sec) 5835.7202 | GPU 792.3 MB\n",
      "Epoch 00000 | Step 00120 | Loss 1.4899 | Train Acc 0.6006 | Speed (samples/sec) 5788.7462 | GPU 792.3 MB\n",
      "Epoch 00000 | Step 00140 | Loss 1.2912 | Train Acc 0.6475 | Speed (samples/sec) 5721.5846 | GPU 792.3 MB\n",
      "Epoch 00000 | Step 00160 | Loss 1.1949 | Train Acc 0.6738 | Speed (samples/sec) 5729.0147 | GPU 792.3 MB\n",
      "Epoch 00000 | Step 00180 | Loss 1.1041 | Train Acc 0.7002 | Speed (samples/sec) 5749.2999 | GPU 792.3 MB\n",
      "Epoch Time(s): 38.4508\n",
      "Epoch 00001 | Step 00000 | Loss 1.4943 | Train Acc 0.5938 | Speed (samples/sec) 5698.9616 | GPU 792.3 MB\n",
      "Epoch 00001 | Step 00020 | Loss 1.1631 | Train Acc 0.6670 | Speed (samples/sec) 5735.1250 | GPU 792.3 MB\n",
      "Epoch 00001 | Step 00040 | Loss 1.0834 | Train Acc 0.7061 | Speed (samples/sec) 5742.7016 | GPU 792.3 MB\n",
      "Epoch 00001 | Step 00060 | Loss 0.9661 | Train Acc 0.7383 | Speed (samples/sec) 5742.0009 | GPU 792.3 MB\n",
      "Epoch 00001 | Step 00080 | Loss 0.9419 | Train Acc 0.7344 | Speed (samples/sec) 5741.3174 | GPU 792.3 MB\n",
      "Epoch 00001 | Step 00100 | Loss 0.9467 | Train Acc 0.7432 | Speed (samples/sec) 5744.1136 | GPU 792.3 MB\n",
      "Epoch 00001 | Step 00120 | Loss 0.9486 | Train Acc 0.7373 | Speed (samples/sec) 5736.2887 | GPU 792.3 MB\n",
      "Epoch 00001 | Step 00140 | Loss 0.9985 | Train Acc 0.7295 | Speed (samples/sec) 5734.2278 | GPU 792.3 MB\n",
      "Epoch 00001 | Step 00160 | Loss 0.8415 | Train Acc 0.7598 | Speed (samples/sec) 5739.8945 | GPU 792.3 MB\n",
      "Epoch 00001 | Step 00180 | Loss 0.8845 | Train Acc 0.7578 | Speed (samples/sec) 5736.4238 | GPU 792.3 MB\n",
      "Epoch Time(s): 38.1032\n",
      "Epoch 00002 | Step 00000 | Loss 0.9325 | Train Acc 0.7393 | Speed (samples/sec) 5698.1483 | GPU 792.3 MB\n",
      "Epoch 00002 | Step 00020 | Loss 0.9433 | Train Acc 0.7295 | Speed (samples/sec) 5700.8925 | GPU 792.3 MB\n",
      "Epoch 00002 | Step 00040 | Loss 0.8420 | Train Acc 0.7705 | Speed (samples/sec) 5705.5948 | GPU 792.3 MB\n",
      "Epoch 00002 | Step 00060 | Loss 0.9401 | Train Acc 0.7598 | Speed (samples/sec) 5706.0989 | GPU 792.3 MB\n",
      "Epoch 00002 | Step 00080 | Loss 0.9451 | Train Acc 0.7490 | Speed (samples/sec) 5709.4594 | GPU 792.3 MB\n",
      "Epoch 00002 | Step 00100 | Loss 0.7769 | Train Acc 0.7969 | Speed (samples/sec) 5714.4410 | GPU 792.3 MB\n",
      "Epoch 00002 | Step 00120 | Loss 0.8327 | Train Acc 0.7666 | Speed (samples/sec) 5712.1131 | GPU 792.3 MB\n",
      "Epoch 00002 | Step 00140 | Loss 0.9077 | Train Acc 0.7617 | Speed (samples/sec) 5707.7458 | GPU 792.3 MB\n",
      "Epoch 00002 | Step 00160 | Loss 0.8379 | Train Acc 0.7803 | Speed (samples/sec) 5691.6543 | GPU 792.3 MB\n",
      "Epoch 00002 | Step 00180 | Loss 0.8673 | Train Acc 0.7803 | Speed (samples/sec) 5691.9785 | GPU 792.3 MB\n",
      "Epoch Time(s): 39.0715\n",
      "Epoch 00003 | Step 00000 | Loss 1.0451 | Train Acc 0.7051 | Speed (samples/sec) 5672.6712 | GPU 792.3 MB\n",
      "Epoch 00003 | Step 00020 | Loss 0.7858 | Train Acc 0.7861 | Speed (samples/sec) 5679.5152 | GPU 792.3 MB\n",
      "Epoch 00003 | Step 00040 | Loss 0.8334 | Train Acc 0.7686 | Speed (samples/sec) 5673.8425 | GPU 792.3 MB\n",
      "Epoch 00003 | Step 00060 | Loss 0.8259 | Train Acc 0.7646 | Speed (samples/sec) 5668.2340 | GPU 792.3 MB\n",
      "Epoch 00003 | Step 00080 | Loss 0.8332 | Train Acc 0.8096 | Speed (samples/sec) 5670.4396 | GPU 792.3 MB\n",
      "Epoch 00003 | Step 00100 | Loss 0.7606 | Train Acc 0.7988 | Speed (samples/sec) 5676.0407 | GPU 792.3 MB\n",
      "Epoch 00003 | Step 00120 | Loss 0.8024 | Train Acc 0.7832 | Speed (samples/sec) 5681.9444 | GPU 792.3 MB\n",
      "Epoch 00003 | Step 00140 | Loss 0.6990 | Train Acc 0.7998 | Speed (samples/sec) 5688.1269 | GPU 792.3 MB\n",
      "Epoch 00003 | Step 00160 | Loss 0.7800 | Train Acc 0.7881 | Speed (samples/sec) 5694.7141 | GPU 792.3 MB\n",
      "Epoch 00003 | Step 00180 | Loss 0.6777 | Train Acc 0.8135 | Speed (samples/sec) 5702.2420 | GPU 792.3 MB\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/245 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch Time(s): 38.2594\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 245/245 [00:41<00:00,  5.89it/s]\n",
      "100%|██████████| 245/245 [00:51<00:00,  4.79it/s]\n",
      "100%|██████████| 245/245 [00:51<00:00,  4.80it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val acc 0.7648, Test Acc: 0.5986\n",
      "Best val acc 0.7648, best test acc: 0.5986\n",
      "Epoch 00004 | Step 00000 | Loss 0.8630 | Train Acc 0.7627 | Speed (samples/sec) 5684.6560 | GPU 792.3 MB\n",
      "Epoch 00004 | Step 00020 | Loss 0.7219 | Train Acc 0.7988 | Speed (samples/sec) 5690.6060 | GPU 792.3 MB\n",
      "Epoch 00004 | Step 00040 | Loss 0.7355 | Train Acc 0.7988 | Speed (samples/sec) 5701.0012 | GPU 792.3 MB\n",
      "Epoch 00004 | Step 00060 | Loss 0.7399 | Train Acc 0.7920 | Speed (samples/sec) 5709.0480 | GPU 792.3 MB\n",
      "Epoch 00004 | Step 00080 | Loss 0.7502 | Train Acc 0.7783 | Speed (samples/sec) 5712.6029 | GPU 792.3 MB\n",
      "Epoch 00004 | Step 00100 | Loss 0.6806 | Train Acc 0.8066 | Speed (samples/sec) 5722.7550 | GPU 792.3 MB\n",
      "Epoch 00004 | Step 00120 | Loss 0.7247 | Train Acc 0.7959 | Speed (samples/sec) 5728.8752 | GPU 792.3 MB\n",
      "Epoch 00004 | Step 00140 | Loss 0.7185 | Train Acc 0.8135 | Speed (samples/sec) 5733.8097 | GPU 792.3 MB\n",
      "Epoch 00004 | Step 00160 | Loss 0.6016 | Train Acc 0.8457 | Speed (samples/sec) 5736.5886 | GPU 792.3 MB\n",
      "Epoch 00004 | Step 00180 | Loss 0.7202 | Train Acc 0.7910 | Speed (samples/sec) 5743.5266 | GPU 792.3 MB\n",
      "Epoch Time(s): 37.0868\n",
      "Epoch 00005 | Step 00000 | Loss 0.6855 | Train Acc 0.8105 | Speed (samples/sec) 5739.1809 | GPU 792.3 MB\n",
      "Epoch 00005 | Step 00020 | Loss 0.7335 | Train Acc 0.8066 | Speed (samples/sec) 5741.1368 | GPU 792.3 MB\n",
      "Epoch 00005 | Step 00040 | Loss 0.7199 | Train Acc 0.7920 | Speed (samples/sec) 5744.7101 | GPU 792.3 MB\n",
      "Epoch 00005 | Step 00060 | Loss 0.6782 | Train Acc 0.8262 | Speed (samples/sec) 5748.7652 | GPU 792.3 MB\n",
      "Epoch 00005 | Step 00080 | Loss 0.6873 | Train Acc 0.8076 | Speed (samples/sec) 5749.7250 | GPU 792.3 MB\n",
      "Epoch 00005 | Step 00100 | Loss 0.7408 | Train Acc 0.7988 | Speed (samples/sec) 5754.1785 | GPU 792.3 MB\n",
      "Epoch 00005 | Step 00120 | Loss 0.6309 | Train Acc 0.8154 | Speed (samples/sec) 5755.9237 | GPU 792.3 MB\n",
      "Epoch 00005 | Step 00140 | Loss 0.6951 | Train Acc 0.7930 | Speed (samples/sec) 5759.3356 | GPU 792.3 MB\n",
      "Epoch 00005 | Step 00160 | Loss 0.6829 | Train Acc 0.8115 | Speed (samples/sec) 5760.5838 | GPU 792.3 MB\n",
      "Epoch 00005 | Step 00180 | Loss 0.6508 | Train Acc 0.8184 | Speed (samples/sec) 5761.5918 | GPU 792.3 MB\n",
      "Epoch Time(s): 37.5244\n",
      "Epoch 00006 | Step 00000 | Loss 0.6939 | Train Acc 0.7871 | Speed (samples/sec) 5759.1174 | GPU 792.3 MB\n",
      "Epoch 00006 | Step 00020 | Loss 0.5537 | Train Acc 0.8398 | Speed (samples/sec) 5758.7260 | GPU 792.3 MB\n",
      "Epoch 00006 | Step 00040 | Loss 0.6794 | Train Acc 0.8115 | Speed (samples/sec) 5763.1909 | GPU 792.3 MB\n",
      "Epoch 00006 | Step 00060 | Loss 0.6825 | Train Acc 0.8037 | Speed (samples/sec) 5766.7492 | GPU 792.3 MB\n",
      "Epoch 00006 | Step 00080 | Loss 0.6820 | Train Acc 0.8252 | Speed (samples/sec) 5771.3833 | GPU 792.3 MB\n",
      "Epoch 00006 | Step 00100 | Loss 0.6776 | Train Acc 0.8252 | Speed (samples/sec) 5775.8931 | GPU 792.3 MB\n",
      "Epoch 00006 | Step 00120 | Loss 0.5545 | Train Acc 0.8359 | Speed (samples/sec) 5779.7870 | GPU 792.3 MB\n",
      "Epoch 00006 | Step 00140 | Loss 0.6954 | Train Acc 0.8086 | Speed (samples/sec) 5782.7970 | GPU 792.3 MB\n",
      "Epoch 00006 | Step 00160 | Loss 0.6597 | Train Acc 0.8125 | Speed (samples/sec) 5788.4021 | GPU 792.3 MB\n",
      "Epoch 00006 | Step 00180 | Loss 0.6659 | Train Acc 0.8281 | Speed (samples/sec) 5793.6689 | GPU 792.3 MB\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/245 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch Time(s): 36.6953\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 245/245 [00:40<00:00,  6.04it/s]\n",
      "100%|██████████| 245/245 [00:50<00:00,  4.86it/s]\n",
      "100%|██████████| 245/245 [00:49<00:00,  4.90it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val acc 0.8105, Test Acc: 0.6464\n",
      "Best val acc 0.8105, best test acc: 0.6464\n",
      "Epoch 00007 | Step 00000 | Loss 0.6884 | Train Acc 0.8174 | Speed (samples/sec) 5787.9332 | GPU 792.3 MB\n",
      "Epoch 00007 | Step 00020 | Loss 0.6162 | Train Acc 0.8359 | Speed (samples/sec) 5783.8225 | GPU 792.3 MB\n",
      "Epoch 00007 | Step 00040 | Loss 0.6884 | Train Acc 0.8105 | Speed (samples/sec) 5787.6885 | GPU 792.3 MB\n",
      "Epoch 00007 | Step 00060 | Loss 0.6162 | Train Acc 0.8252 | Speed (samples/sec) 5792.9322 | GPU 792.3 MB\n",
      "Epoch 00007 | Step 00080 | Loss 0.6767 | Train Acc 0.8086 | Speed (samples/sec) 5798.6604 | GPU 792.3 MB\n",
      "Epoch 00007 | Step 00100 | Loss 0.6376 | Train Acc 0.8164 | Speed (samples/sec) 5804.9585 | GPU 792.3 MB\n",
      "Epoch 00007 | Step 00120 | Loss 0.6511 | Train Acc 0.8125 | Speed (samples/sec) 5806.1263 | GPU 792.3 MB\n",
      "Epoch 00007 | Step 00140 | Loss 0.6869 | Train Acc 0.8066 | Speed (samples/sec) 5811.9025 | GPU 792.3 MB\n",
      "Epoch 00007 | Step 00160 | Loss 0.6490 | Train Acc 0.8252 | Speed (samples/sec) 5816.5335 | GPU 792.3 MB\n",
      "Epoch 00007 | Step 00180 | Loss 0.6431 | Train Acc 0.8184 | Speed (samples/sec) 5819.7837 | GPU 792.3 MB\n",
      "Epoch Time(s): 36.1032\n",
      "Epoch 00008 | Step 00000 | Loss 1.0588 | Train Acc 0.7285 | Speed (samples/sec) 5817.3546 | GPU 792.3 MB\n",
      "Epoch 00008 | Step 00020 | Loss 0.7228 | Train Acc 0.8018 | Speed (samples/sec) 5821.8400 | GPU 792.3 MB\n",
      "Epoch 00008 | Step 00040 | Loss 0.6869 | Train Acc 0.8096 | Speed (samples/sec) 5829.9756 | GPU 792.3 MB\n",
      "Epoch 00008 | Step 00060 | Loss 0.6202 | Train Acc 0.8340 | Speed (samples/sec) 5834.0446 | GPU 792.3 MB\n",
      "Epoch 00008 | Step 00080 | Loss 0.5987 | Train Acc 0.8320 | Speed (samples/sec) 5837.7966 | GPU 792.3 MB\n",
      "Epoch 00008 | Step 00100 | Loss 0.6777 | Train Acc 0.8203 | Speed (samples/sec) 5842.1455 | GPU 792.3 MB\n",
      "Epoch 00008 | Step 00120 | Loss 0.5806 | Train Acc 0.8457 | Speed (samples/sec) 5846.5543 | GPU 792.3 MB\n",
      "Epoch 00008 | Step 00140 | Loss 0.5589 | Train Acc 0.8447 | Speed (samples/sec) 5849.7208 | GPU 792.3 MB\n",
      "Epoch 00008 | Step 00160 | Loss 0.5844 | Train Acc 0.8428 | Speed (samples/sec) 5852.3797 | GPU 792.3 MB\n",
      "Epoch 00008 | Step 00180 | Loss 0.6563 | Train Acc 0.8086 | Speed (samples/sec) 5854.9933 | GPU 792.3 MB\n",
      "Epoch Time(s): 35.3191\n",
      "Epoch 00009 | Step 00000 | Loss 0.6165 | Train Acc 0.8330 | Speed (samples/sec) 5849.7558 | GPU 792.3 MB\n",
      "Epoch 00009 | Step 00020 | Loss 0.6632 | Train Acc 0.8145 | Speed (samples/sec) 5848.9936 | GPU 792.3 MB\n",
      "Epoch 00009 | Step 00040 | Loss 0.6524 | Train Acc 0.8213 | Speed (samples/sec) 5850.8338 | GPU 793.4 MB\n",
      "Epoch 00009 | Step 00060 | Loss 0.6219 | Train Acc 0.8252 | Speed (samples/sec) 5852.0014 | GPU 793.4 MB\n",
      "Epoch 00009 | Step 00080 | Loss 0.5540 | Train Acc 0.8447 | Speed (samples/sec) 5851.3971 | GPU 793.4 MB\n",
      "Epoch 00009 | Step 00100 | Loss 0.6246 | Train Acc 0.8281 | Speed (samples/sec) 5851.0712 | GPU 793.4 MB\n",
      "Epoch 00009 | Step 00120 | Loss 0.5953 | Train Acc 0.8418 | Speed (samples/sec) 5852.4902 | GPU 793.4 MB\n",
      "Epoch 00009 | Step 00140 | Loss 0.5931 | Train Acc 0.8311 | Speed (samples/sec) 5854.2046 | GPU 793.4 MB\n",
      "Epoch 00009 | Step 00160 | Loss 0.5898 | Train Acc 0.8457 | Speed (samples/sec) 5853.6310 | GPU 793.4 MB\n",
      "Epoch 00009 | Step 00180 | Loss 0.5882 | Train Acc 0.8252 | Speed (samples/sec) 5853.4141 | GPU 793.4 MB\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/245 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch Time(s): 37.3515\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 245/245 [00:40<00:00,  6.01it/s]\n",
      "100%|██████████| 245/245 [00:51<00:00,  4.76it/s]\n",
      "100%|██████████| 245/245 [00:50<00:00,  4.82it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val acc 0.8396, Test Acc: 0.6995\n",
      "Best val acc 0.8396, best test acc: 0.6995\n",
      "Epoch 00010 | Step 00000 | Loss 0.5972 | Train Acc 0.8330 | Speed (samples/sec) 5849.2357 | GPU 793.4 MB\n",
      "Epoch 00010 | Step 00020 | Loss 0.6139 | Train Acc 0.8301 | Speed (samples/sec) 5845.9355 | GPU 793.4 MB\n",
      "Epoch 00010 | Step 00040 | Loss 0.5536 | Train Acc 0.8369 | Speed (samples/sec) 5844.8828 | GPU 793.4 MB\n",
      "Epoch 00010 | Step 00060 | Loss 0.6526 | Train Acc 0.8115 | Speed (samples/sec) 5845.1208 | GPU 793.4 MB\n",
      "Epoch 00010 | Step 00080 | Loss 0.6168 | Train Acc 0.8174 | Speed (samples/sec) 5845.5274 | GPU 793.4 MB\n",
      "Epoch 00010 | Step 00100 | Loss 0.6014 | Train Acc 0.8223 | Speed (samples/sec) 5845.5442 | GPU 793.4 MB\n",
      "Epoch 00010 | Step 00120 | Loss 0.5646 | Train Acc 0.8398 | Speed (samples/sec) 5845.0011 | GPU 793.4 MB\n",
      "Epoch 00010 | Step 00140 | Loss 0.6784 | Train Acc 0.8105 | Speed (samples/sec) 5846.6006 | GPU 793.4 MB\n",
      "Epoch 00010 | Step 00160 | Loss 0.5653 | Train Acc 0.8311 | Speed (samples/sec) 5847.9712 | GPU 793.4 MB\n",
      "Epoch 00010 | Step 00180 | Loss 0.5993 | Train Acc 0.8350 | Speed (samples/sec) 5843.9861 | GPU 793.4 MB\n",
      "Epoch Time(s): 38.0281\n",
      "Epoch 00011 | Step 00000 | Loss 1.1700 | Train Acc 0.6279 | Speed (samples/sec) 5838.8845 | GPU 793.4 MB\n",
      "Epoch 00011 | Step 00020 | Loss 0.6988 | Train Acc 0.8086 | Speed (samples/sec) 5843.6370 | GPU 793.4 MB\n",
      "Epoch 00011 | Step 00040 | Loss 0.5543 | Train Acc 0.8438 | Speed (samples/sec) 5845.6530 | GPU 793.4 MB\n",
      "Epoch 00011 | Step 00060 | Loss 0.5955 | Train Acc 0.8174 | Speed (samples/sec) 5847.7813 | GPU 793.4 MB\n",
      "Epoch 00011 | Step 00080 | Loss 0.5974 | Train Acc 0.8359 | Speed (samples/sec) 5848.3682 | GPU 793.4 MB\n",
      "Epoch 00011 | Step 00100 | Loss 0.5521 | Train Acc 0.8447 | Speed (samples/sec) 5849.1437 | GPU 793.4 MB\n",
      "Epoch 00011 | Step 00120 | Loss 0.6147 | Train Acc 0.8311 | Speed (samples/sec) 5852.7556 | GPU 793.4 MB\n",
      "Epoch 00011 | Step 00140 | Loss 0.6207 | Train Acc 0.8281 | Speed (samples/sec) 5855.0958 | GPU 793.4 MB\n",
      "Epoch 00011 | Step 00160 | Loss 0.6380 | Train Acc 0.8135 | Speed (samples/sec) 5859.4951 | GPU 793.4 MB\n",
      "Epoch 00011 | Step 00180 | Loss 0.6269 | Train Acc 0.8301 | Speed (samples/sec) 5864.9397 | GPU 793.4 MB\n",
      "Epoch Time(s): 35.4337\n",
      "Epoch 00012 | Step 00000 | Loss 0.6793 | Train Acc 0.8105 | Speed (samples/sec) 5864.0142 | GPU 793.4 MB\n",
      "Epoch 00012 | Step 00020 | Loss 0.6434 | Train Acc 0.8164 | Speed (samples/sec) 5864.7602 | GPU 793.4 MB\n",
      "Epoch 00012 | Step 00040 | Loss 0.5276 | Train Acc 0.8350 | Speed (samples/sec) 5866.6303 | GPU 793.4 MB\n",
      "Epoch 00012 | Step 00060 | Loss 0.5641 | Train Acc 0.8379 | Speed (samples/sec) 5870.0136 | GPU 793.4 MB\n",
      "Epoch 00012 | Step 00080 | Loss 0.5994 | Train Acc 0.8350 | Speed (samples/sec) 5873.5421 | GPU 793.4 MB\n",
      "Epoch 00012 | Step 00100 | Loss 0.6501 | Train Acc 0.8096 | Speed (samples/sec) 5875.4338 | GPU 793.4 MB\n",
      "Epoch 00012 | Step 00120 | Loss 0.5868 | Train Acc 0.8447 | Speed (samples/sec) 5877.6271 | GPU 793.4 MB\n",
      "Epoch 00012 | Step 00140 | Loss 0.5232 | Train Acc 0.8467 | Speed (samples/sec) 5880.8313 | GPU 793.4 MB\n",
      "Epoch 00012 | Step 00160 | Loss 0.5683 | Train Acc 0.8320 | Speed (samples/sec) 5883.8385 | GPU 793.4 MB\n",
      "Epoch 00012 | Step 00180 | Loss 0.5776 | Train Acc 0.8379 | Speed (samples/sec) 5885.1257 | GPU 793.4 MB\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/245 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch Time(s): 35.6829\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 245/245 [00:40<00:00,  6.03it/s]\n",
      "100%|██████████| 245/245 [00:50<00:00,  4.83it/s]\n",
      "100%|██████████| 245/245 [00:49<00:00,  4.98it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val acc 0.8167, Test Acc: 0.6601\n",
      "Epoch 00013 | Step 00000 | Loss 0.6674 | Train Acc 0.8047 | Speed (samples/sec) 5880.2467 | GPU 793.4 MB\n",
      "Epoch 00013 | Step 00020 | Loss 0.5327 | Train Acc 0.8525 | Speed (samples/sec) 5880.9835 | GPU 793.4 MB\n",
      "Epoch 00013 | Step 00040 | Loss 0.5588 | Train Acc 0.8467 | Speed (samples/sec) 5881.3071 | GPU 793.4 MB\n",
      "Epoch 00013 | Step 00060 | Loss 0.5363 | Train Acc 0.8428 | Speed (samples/sec) 5881.4215 | GPU 793.4 MB\n",
      "Epoch 00013 | Step 00080 | Loss 0.5346 | Train Acc 0.8398 | Speed (samples/sec) 5881.0711 | GPU 793.4 MB\n",
      "Epoch 00013 | Step 00100 | Loss 0.4943 | Train Acc 0.8604 | Speed (samples/sec) 5882.1066 | GPU 793.4 MB\n",
      "Epoch 00013 | Step 00120 | Loss 0.5858 | Train Acc 0.8457 | Speed (samples/sec) 5882.2913 | GPU 793.4 MB\n",
      "Epoch 00013 | Step 00140 | Loss 0.5727 | Train Acc 0.8359 | Speed (samples/sec) 5881.9124 | GPU 793.4 MB\n",
      "Epoch 00013 | Step 00160 | Loss 0.5016 | Train Acc 0.8477 | Speed (samples/sec) 5881.2993 | GPU 793.4 MB\n",
      "Epoch 00013 | Step 00180 | Loss 0.5498 | Train Acc 0.8379 | Speed (samples/sec) 5880.7348 | GPU 793.4 MB\n",
      "Epoch Time(s): 37.2267\n",
      "Epoch 00014 | Step 00000 | Loss 1.1232 | Train Acc 0.7158 | Speed (samples/sec) 5875.3688 | GPU 793.4 MB\n",
      "Epoch 00014 | Step 00020 | Loss 0.6550 | Train Acc 0.8223 | Speed (samples/sec) 5875.2191 | GPU 793.4 MB\n",
      "Epoch 00014 | Step 00040 | Loss 0.5673 | Train Acc 0.8340 | Speed (samples/sec) 5877.0241 | GPU 793.4 MB\n",
      "Epoch 00014 | Step 00060 | Loss 0.5723 | Train Acc 0.8330 | Speed (samples/sec) 5877.7603 | GPU 793.4 MB\n",
      "Epoch 00014 | Step 00080 | Loss 0.5813 | Train Acc 0.8271 | Speed (samples/sec) 5878.5866 | GPU 793.4 MB\n",
      "Epoch 00014 | Step 00100 | Loss 0.5558 | Train Acc 0.8438 | Speed (samples/sec) 5878.2401 | GPU 793.4 MB\n",
      "Epoch 00014 | Step 00120 | Loss 0.5508 | Train Acc 0.8467 | Speed (samples/sec) 5876.7020 | GPU 793.4 MB\n",
      "Epoch 00014 | Step 00140 | Loss 0.5154 | Train Acc 0.8574 | Speed (samples/sec) 5876.4786 | GPU 793.4 MB\n",
      "Epoch 00014 | Step 00160 | Loss 0.5908 | Train Acc 0.8223 | Speed (samples/sec) 5877.7302 | GPU 793.4 MB\n",
      "Epoch 00014 | Step 00180 | Loss 0.5518 | Train Acc 0.8438 | Speed (samples/sec) 5876.1097 | GPU 793.4 MB\n",
      "Epoch Time(s): 37.1957\n",
      "Epoch 00015 | Step 00000 | Loss 0.9404 | Train Acc 0.7285 | Speed (samples/sec) 5874.5535 | GPU 793.4 MB\n",
      "Epoch 00015 | Step 00020 | Loss 0.6296 | Train Acc 0.8232 | Speed (samples/sec) 5873.1445 | GPU 793.4 MB\n",
      "Epoch 00015 | Step 00040 | Loss 0.6189 | Train Acc 0.8320 | Speed (samples/sec) 5875.1274 | GPU 793.4 MB\n",
      "Epoch 00015 | Step 00060 | Loss 0.5787 | Train Acc 0.8359 | Speed (samples/sec) 5875.3204 | GPU 793.4 MB\n",
      "Epoch 00015 | Step 00080 | Loss 0.5910 | Train Acc 0.8340 | Speed (samples/sec) 5874.3746 | GPU 793.4 MB\n",
      "Epoch 00015 | Step 00100 | Loss 0.5806 | Train Acc 0.8340 | Speed (samples/sec) 5875.4336 | GPU 793.4 MB\n",
      "Epoch 00015 | Step 00120 | Loss 0.5629 | Train Acc 0.8408 | Speed (samples/sec) 5876.0126 | GPU 793.4 MB\n",
      "Epoch 00015 | Step 00140 | Loss 0.5340 | Train Acc 0.8438 | Speed (samples/sec) 5876.0520 | GPU 793.4 MB\n",
      "Epoch 00015 | Step 00160 | Loss 0.5620 | Train Acc 0.8408 | Speed (samples/sec) 5876.9921 | GPU 793.4 MB\n",
      "Epoch 00015 | Step 00180 | Loss 0.5948 | Train Acc 0.8418 | Speed (samples/sec) 5878.6628 | GPU 793.4 MB\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/245 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch Time(s): 37.3120\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 245/245 [00:40<00:00,  6.05it/s]\n",
      "100%|██████████| 245/245 [00:51<00:00,  4.75it/s]\n",
      "100%|██████████| 245/245 [00:49<00:00,  4.92it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val acc 0.7441, Test Acc: 0.5948\n",
      "Epoch 00016 | Step 00000 | Loss 1.0339 | Train Acc 0.7402 | Speed (samples/sec) 5873.7165 | GPU 793.4 MB\n",
      "Epoch 00016 | Step 00020 | Loss 0.6036 | Train Acc 0.8311 | Speed (samples/sec) 5874.1890 | GPU 793.4 MB\n",
      "Epoch 00016 | Step 00040 | Loss 0.5047 | Train Acc 0.8555 | Speed (samples/sec) 5875.3425 | GPU 793.4 MB\n",
      "Epoch 00016 | Step 00060 | Loss 0.5483 | Train Acc 0.8496 | Speed (samples/sec) 5874.8266 | GPU 793.4 MB\n",
      "Epoch 00016 | Step 00080 | Loss 0.5554 | Train Acc 0.8359 | Speed (samples/sec) 5875.7813 | GPU 793.4 MB\n",
      "Epoch 00016 | Step 00100 | Loss 0.5795 | Train Acc 0.8242 | Speed (samples/sec) 5876.4488 | GPU 793.4 MB\n",
      "Epoch 00016 | Step 00120 | Loss 0.5951 | Train Acc 0.8369 | Speed (samples/sec) 5876.6048 | GPU 793.4 MB\n",
      "Epoch 00016 | Step 00140 | Loss 0.5812 | Train Acc 0.8359 | Speed (samples/sec) 5877.5858 | GPU 793.4 MB\n",
      "Epoch 00016 | Step 00160 | Loss 0.6305 | Train Acc 0.8301 | Speed (samples/sec) 5878.8014 | GPU 793.4 MB\n",
      "Epoch 00016 | Step 00180 | Loss 0.5584 | Train Acc 0.8535 | Speed (samples/sec) 5879.0626 | GPU 793.4 MB\n",
      "Epoch Time(s): 36.9011\n",
      "Epoch 00017 | Step 00000 | Loss 0.6985 | Train Acc 0.7959 | Speed (samples/sec) 5875.9860 | GPU 793.4 MB\n",
      "Epoch 00017 | Step 00020 | Loss 0.5988 | Train Acc 0.8262 | Speed (samples/sec) 5877.1962 | GPU 793.4 MB\n",
      "Epoch 00017 | Step 00040 | Loss 0.5207 | Train Acc 0.8545 | Speed (samples/sec) 5878.0953 | GPU 793.4 MB\n",
      "Epoch 00017 | Step 00060 | Loss 0.5937 | Train Acc 0.8281 | Speed (samples/sec) 5878.3284 | GPU 793.4 MB\n",
      "Epoch 00017 | Step 00080 | Loss 0.4569 | Train Acc 0.8643 | Speed (samples/sec) 5880.5702 | GPU 793.4 MB\n",
      "Epoch 00017 | Step 00100 | Loss 0.6063 | Train Acc 0.8145 | Speed (samples/sec) 5880.2318 | GPU 793.4 MB\n",
      "Epoch 00017 | Step 00120 | Loss 0.5679 | Train Acc 0.8408 | Speed (samples/sec) 5880.5194 | GPU 793.4 MB\n",
      "Epoch 00017 | Step 00140 | Loss 0.5428 | Train Acc 0.8379 | Speed (samples/sec) 5882.1411 | GPU 793.4 MB\n",
      "Epoch 00017 | Step 00160 | Loss 0.5734 | Train Acc 0.8369 | Speed (samples/sec) 5883.1340 | GPU 793.4 MB\n",
      "Epoch 00017 | Step 00180 | Loss 0.5210 | Train Acc 0.8574 | Speed (samples/sec) 5883.7145 | GPU 793.4 MB\n",
      "Epoch Time(s): 36.5682\n",
      "Epoch 00018 | Step 00000 | Loss 0.6935 | Train Acc 0.7871 | Speed (samples/sec) 5880.9302 | GPU 793.4 MB\n",
      "Epoch 00018 | Step 00020 | Loss 0.5082 | Train Acc 0.8555 | Speed (samples/sec) 5881.0735 | GPU 793.4 MB\n",
      "Epoch 00018 | Step 00040 | Loss 0.5735 | Train Acc 0.8398 | Speed (samples/sec) 5881.2679 | GPU 793.4 MB\n",
      "Epoch 00018 | Step 00060 | Loss 0.5334 | Train Acc 0.8506 | Speed (samples/sec) 5881.7381 | GPU 793.4 MB\n",
      "Epoch 00018 | Step 00080 | Loss 0.5345 | Train Acc 0.8516 | Speed (samples/sec) 5881.1426 | GPU 793.4 MB\n",
      "Epoch 00018 | Step 00100 | Loss 0.5336 | Train Acc 0.8643 | Speed (samples/sec) 5881.2375 | GPU 793.4 MB\n",
      "Epoch 00018 | Step 00120 | Loss 0.4896 | Train Acc 0.8506 | Speed (samples/sec) 5881.9313 | GPU 793.4 MB\n",
      "Epoch 00018 | Step 00140 | Loss 0.5652 | Train Acc 0.8359 | Speed (samples/sec) 5882.2718 | GPU 793.4 MB\n",
      "Epoch 00018 | Step 00160 | Loss 0.4958 | Train Acc 0.8652 | Speed (samples/sec) 5884.4370 | GPU 793.4 MB\n",
      "Epoch 00018 | Step 00180 | Loss 0.5661 | Train Acc 0.8389 | Speed (samples/sec) 5884.7754 | GPU 793.4 MB\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/245 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch Time(s): 37.0399\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 245/245 [00:41<00:00,  5.90it/s]\n",
      "100%|██████████| 245/245 [00:48<00:00,  5.06it/s]\n",
      "100%|██████████| 245/245 [00:50<00:00,  4.88it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val acc 0.8342, Test Acc: 0.6845\n",
      "Epoch 00019 | Step 00000 | Loss 0.5439 | Train Acc 0.8477 | Speed (samples/sec) 5883.0856 | GPU 793.4 MB\n",
      "Epoch 00019 | Step 00020 | Loss 0.5371 | Train Acc 0.8418 | Speed (samples/sec) 5883.8895 | GPU 793.4 MB\n",
      "Epoch 00019 | Step 00040 | Loss 0.5783 | Train Acc 0.8486 | Speed (samples/sec) 5883.8580 | GPU 793.4 MB\n",
      "Epoch 00019 | Step 00060 | Loss 0.5424 | Train Acc 0.8389 | Speed (samples/sec) 5884.3116 | GPU 793.4 MB\n",
      "Epoch 00019 | Step 00080 | Loss 0.4901 | Train Acc 0.8643 | Speed (samples/sec) 5885.0921 | GPU 793.4 MB\n",
      "Epoch 00019 | Step 00100 | Loss 0.5804 | Train Acc 0.8379 | Speed (samples/sec) 5885.3696 | GPU 793.4 MB\n",
      "Epoch 00019 | Step 00120 | Loss 0.5582 | Train Acc 0.8389 | Speed (samples/sec) 5886.5591 | GPU 793.4 MB\n",
      "Epoch 00019 | Step 00140 | Loss 0.5069 | Train Acc 0.8750 | Speed (samples/sec) 5888.0592 | GPU 793.4 MB\n",
      "Epoch 00019 | Step 00160 | Loss 0.5721 | Train Acc 0.8389 | Speed (samples/sec) 5888.0882 | GPU 793.4 MB\n",
      "Epoch 00019 | Step 00180 | Loss 0.5203 | Train Acc 0.8496 | Speed (samples/sec) 5888.2338 | GPU 793.4 MB\n",
      "Epoch Time(s): 36.5762\n",
      "Epoch 00020 | Step 00000 | Loss 0.7701 | Train Acc 0.7900 | Speed (samples/sec) 5885.8717 | GPU 793.4 MB\n",
      "Epoch 00020 | Step 00020 | Loss 0.5406 | Train Acc 0.8477 | Speed (samples/sec) 5887.5444 | GPU 793.4 MB\n",
      "Epoch 00020 | Step 00040 | Loss 0.5039 | Train Acc 0.8613 | Speed (samples/sec) 5889.5229 | GPU 793.4 MB\n",
      "Epoch 00020 | Step 00060 | Loss 0.5758 | Train Acc 0.8291 | Speed (samples/sec) 5890.8048 | GPU 793.4 MB\n",
      "Epoch 00020 | Step 00080 | Loss 0.5596 | Train Acc 0.8418 | Speed (samples/sec) 5891.4838 | GPU 793.4 MB\n",
      "Epoch 00020 | Step 00100 | Loss 0.5274 | Train Acc 0.8584 | Speed (samples/sec) 5892.3679 | GPU 793.4 MB\n",
      "Epoch 00020 | Step 00120 | Loss 0.5780 | Train Acc 0.8408 | Speed (samples/sec) 5891.8467 | GPU 793.4 MB\n",
      "Epoch 00020 | Step 00140 | Loss 0.5551 | Train Acc 0.8574 | Speed (samples/sec) 5892.3323 | GPU 793.4 MB\n",
      "Epoch 00020 | Step 00160 | Loss 0.4885 | Train Acc 0.8535 | Speed (samples/sec) 5892.2874 | GPU 793.4 MB\n",
      "Epoch 00020 | Step 00180 | Loss 0.5039 | Train Acc 0.8701 | Speed (samples/sec) 5892.3380 | GPU 793.4 MB\n",
      "Epoch Time(s): 36.3408\n",
      "Epoch 00021 | Step 00000 | Loss 0.7932 | Train Acc 0.7539 | Speed (samples/sec) 5890.0427 | GPU 793.4 MB\n",
      "Epoch 00021 | Step 00020 | Loss 0.6346 | Train Acc 0.8125 | Speed (samples/sec) 5889.7637 | GPU 793.4 MB\n",
      "Epoch 00021 | Step 00040 | Loss 0.5259 | Train Acc 0.8643 | Speed (samples/sec) 5890.1029 | GPU 793.4 MB\n",
      "Epoch 00021 | Step 00060 | Loss 0.4648 | Train Acc 0.8652 | Speed (samples/sec) 5890.3412 | GPU 793.4 MB\n",
      "Epoch 00021 | Step 00080 | Loss 0.5949 | Train Acc 0.8428 | Speed (samples/sec) 5890.6986 | GPU 793.4 MB\n",
      "Epoch 00021 | Step 00100 | Loss 0.5221 | Train Acc 0.8516 | Speed (samples/sec) 5890.5957 | GPU 793.4 MB\n",
      "Epoch 00021 | Step 00120 | Loss 0.5808 | Train Acc 0.8340 | Speed (samples/sec) 5891.0746 | GPU 793.4 MB\n",
      "Epoch 00021 | Step 00140 | Loss 0.5419 | Train Acc 0.8379 | Speed (samples/sec) 5891.0654 | GPU 793.4 MB\n",
      "Epoch 00021 | Step 00160 | Loss 0.5567 | Train Acc 0.8369 | Speed (samples/sec) 5891.7441 | GPU 793.4 MB\n",
      "Epoch 00021 | Step 00180 | Loss 0.5795 | Train Acc 0.8330 | Speed (samples/sec) 5892.2251 | GPU 793.4 MB\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/245 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch Time(s): 36.9199\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 245/245 [00:41<00:00,  5.93it/s]\n",
      "100%|██████████| 245/245 [00:51<00:00,  4.71it/s]\n",
      "100%|██████████| 245/245 [00:50<00:00,  4.90it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val acc 0.8182, Test Acc: 0.6575\n",
      "Epoch 00022 | Step 00000 | Loss 0.6775 | Train Acc 0.7988 | Speed (samples/sec) 5889.8396 | GPU 793.4 MB\n",
      "Epoch 00022 | Step 00020 | Loss 0.5580 | Train Acc 0.8408 | Speed (samples/sec) 5890.9054 | GPU 793.4 MB\n",
      "Epoch 00022 | Step 00040 | Loss 0.5825 | Train Acc 0.8271 | Speed (samples/sec) 5892.0171 | GPU 793.4 MB\n",
      "Epoch 00022 | Step 00060 | Loss 0.5173 | Train Acc 0.8623 | Speed (samples/sec) 5892.3774 | GPU 793.4 MB\n",
      "Epoch 00022 | Step 00080 | Loss 0.5369 | Train Acc 0.8535 | Speed (samples/sec) 5892.1058 | GPU 793.4 MB\n",
      "Epoch 00022 | Step 00100 | Loss 0.5091 | Train Acc 0.8496 | Speed (samples/sec) 5894.3851 | GPU 793.4 MB\n",
      "Epoch 00022 | Step 00120 | Loss 0.5113 | Train Acc 0.8486 | Speed (samples/sec) 5895.8497 | GPU 793.4 MB\n",
      "Epoch 00022 | Step 00140 | Loss 0.4908 | Train Acc 0.8525 | Speed (samples/sec) 5896.7390 | GPU 793.4 MB\n",
      "Epoch 00022 | Step 00160 | Loss 0.5023 | Train Acc 0.8506 | Speed (samples/sec) 5897.6785 | GPU 793.4 MB\n",
      "Epoch 00022 | Step 00180 | Loss 0.5693 | Train Acc 0.8516 | Speed (samples/sec) 5898.7133 | GPU 793.4 MB\n",
      "Epoch Time(s): 36.0618\n",
      "Epoch 00023 | Step 00000 | Loss 0.8915 | Train Acc 0.7871 | Speed (samples/sec) 5896.3931 | GPU 793.4 MB\n",
      "Epoch 00023 | Step 00020 | Loss 0.6421 | Train Acc 0.8281 | Speed (samples/sec) 5896.7141 | GPU 793.4 MB\n",
      "Epoch 00023 | Step 00040 | Loss 0.5240 | Train Acc 0.8428 | Speed (samples/sec) 5898.4006 | GPU 793.4 MB\n",
      "Epoch 00023 | Step 00060 | Loss 0.4911 | Train Acc 0.8584 | Speed (samples/sec) 5900.1139 | GPU 793.4 MB\n",
      "Epoch 00023 | Step 00080 | Loss 0.5155 | Train Acc 0.8447 | Speed (samples/sec) 5900.0656 | GPU 793.4 MB\n",
      "Epoch 00023 | Step 00100 | Loss 0.5461 | Train Acc 0.8496 | Speed (samples/sec) 5901.1158 | GPU 793.4 MB\n",
      "Epoch 00023 | Step 00120 | Loss 0.5168 | Train Acc 0.8535 | Speed (samples/sec) 5902.6353 | GPU 793.4 MB\n",
      "Epoch 00023 | Step 00140 | Loss 0.4710 | Train Acc 0.8574 | Speed (samples/sec) 5904.0675 | GPU 793.4 MB\n",
      "Epoch 00023 | Step 00160 | Loss 0.5238 | Train Acc 0.8555 | Speed (samples/sec) 5904.6046 | GPU 793.4 MB\n",
      "Epoch 00023 | Step 00180 | Loss 0.6461 | Train Acc 0.8330 | Speed (samples/sec) 5905.6273 | GPU 793.4 MB\n",
      "Epoch Time(s): 35.8384\n",
      "Epoch 00024 | Step 00000 | Loss 0.6163 | Train Acc 0.8252 | Speed (samples/sec) 5903.4147 | GPU 793.4 MB\n",
      "Epoch 00024 | Step 00020 | Loss 0.5790 | Train Acc 0.8398 | Speed (samples/sec) 5903.4061 | GPU 793.4 MB\n",
      "Epoch 00024 | Step 00040 | Loss 0.5495 | Train Acc 0.8477 | Speed (samples/sec) 5903.6063 | GPU 793.4 MB\n",
      "Epoch 00024 | Step 00060 | Loss 0.5681 | Train Acc 0.8398 | Speed (samples/sec) 5903.9257 | GPU 793.4 MB\n",
      "Epoch 00024 | Step 00080 | Loss 0.5104 | Train Acc 0.8447 | Speed (samples/sec) 5903.5253 | GPU 793.4 MB\n",
      "Epoch 00024 | Step 00100 | Loss 0.5231 | Train Acc 0.8428 | Speed (samples/sec) 5905.3143 | GPU 793.4 MB\n",
      "Epoch 00024 | Step 00120 | Loss 0.5740 | Train Acc 0.8330 | Speed (samples/sec) 5906.2736 | GPU 793.4 MB\n",
      "Epoch 00024 | Step 00140 | Loss 0.4732 | Train Acc 0.8564 | Speed (samples/sec) 5906.7346 | GPU 793.4 MB\n",
      "Epoch 00024 | Step 00160 | Loss 0.5335 | Train Acc 0.8535 | Speed (samples/sec) 5907.1585 | GPU 793.4 MB\n",
      "Epoch 00024 | Step 00180 | Loss 0.5837 | Train Acc 0.8418 | Speed (samples/sec) 5907.8556 | GPU 793.4 MB\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/245 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch Time(s): 36.8023\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 245/245 [00:40<00:00,  6.02it/s]\n",
      "100%|██████████| 245/245 [00:50<00:00,  4.85it/s]\n",
      "100%|██████████| 245/245 [00:49<00:00,  4.96it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val acc 0.8392, Test Acc: 0.6922\n",
      "Epoch 00025 | Step 00000 | Loss 0.5162 | Train Acc 0.8604 | Speed (samples/sec) 5905.3516 | GPU 793.4 MB\n",
      "Epoch 00025 | Step 00020 | Loss 0.5258 | Train Acc 0.8643 | Speed (samples/sec) 5905.5916 | GPU 793.4 MB\n",
      "Epoch 00025 | Step 00040 | Loss 0.5481 | Train Acc 0.8398 | Speed (samples/sec) 5905.8770 | GPU 793.4 MB\n",
      "Epoch 00025 | Step 00060 | Loss 0.5412 | Train Acc 0.8291 | Speed (samples/sec) 5905.6284 | GPU 793.4 MB\n",
      "Epoch 00025 | Step 00080 | Loss 0.5506 | Train Acc 0.8408 | Speed (samples/sec) 5906.5443 | GPU 793.4 MB\n",
      "Epoch 00025 | Step 00100 | Loss 0.4937 | Train Acc 0.8496 | Speed (samples/sec) 5907.2090 | GPU 793.4 MB\n",
      "Epoch 00025 | Step 00120 | Loss 0.4814 | Train Acc 0.8643 | Speed (samples/sec) 5907.5137 | GPU 793.4 MB\n",
      "Epoch 00025 | Step 00140 | Loss 0.4975 | Train Acc 0.8516 | Speed (samples/sec) 5907.3937 | GPU 793.4 MB\n",
      "Epoch 00025 | Step 00160 | Loss 0.4878 | Train Acc 0.8535 | Speed (samples/sec) 5907.5047 | GPU 793.4 MB\n",
      "Epoch 00025 | Step 00180 | Loss 0.4574 | Train Acc 0.8721 | Speed (samples/sec) 5907.7446 | GPU 793.4 MB\n",
      "Epoch Time(s): 36.7081\n",
      "Epoch 00026 | Step 00000 | Loss 0.7421 | Train Acc 0.7803 | Speed (samples/sec) 5906.0892 | GPU 793.4 MB\n",
      "Epoch 00026 | Step 00020 | Loss 0.5978 | Train Acc 0.8252 | Speed (samples/sec) 5906.2791 | GPU 793.4 MB\n",
      "Epoch 00026 | Step 00040 | Loss 0.5371 | Train Acc 0.8428 | Speed (samples/sec) 5905.3838 | GPU 793.4 MB\n",
      "Epoch 00026 | Step 00060 | Loss 0.4770 | Train Acc 0.8652 | Speed (samples/sec) 5905.9245 | GPU 793.4 MB\n",
      "Epoch 00026 | Step 00080 | Loss 0.5716 | Train Acc 0.8398 | Speed (samples/sec) 5905.6496 | GPU 793.4 MB\n",
      "Epoch 00026 | Step 00100 | Loss 0.5210 | Train Acc 0.8564 | Speed (samples/sec) 5904.5409 | GPU 793.4 MB\n",
      "Epoch 00026 | Step 00120 | Loss 0.5652 | Train Acc 0.8525 | Speed (samples/sec) 5904.5034 | GPU 793.4 MB\n",
      "Epoch 00026 | Step 00140 | Loss 0.5181 | Train Acc 0.8555 | Speed (samples/sec) 5904.7789 | GPU 793.4 MB\n",
      "Epoch 00026 | Step 00160 | Loss 0.5009 | Train Acc 0.8564 | Speed (samples/sec) 5903.9271 | GPU 793.4 MB\n",
      "Epoch 00026 | Step 00180 | Loss 0.5007 | Train Acc 0.8516 | Speed (samples/sec) 5903.5902 | GPU 793.4 MB\n",
      "Epoch Time(s): 37.5065\n",
      "Epoch 00027 | Step 00000 | Loss 0.5973 | Train Acc 0.8262 | Speed (samples/sec) 5901.7568 | GPU 793.4 MB\n",
      "Epoch 00027 | Step 00020 | Loss 0.5283 | Train Acc 0.8398 | Speed (samples/sec) 5901.9371 | GPU 793.4 MB\n",
      "Epoch 00027 | Step 00040 | Loss 0.5025 | Train Acc 0.8506 | Speed (samples/sec) 5902.2002 | GPU 793.4 MB\n",
      "Epoch 00027 | Step 00060 | Loss 0.5423 | Train Acc 0.8516 | Speed (samples/sec) 5902.9634 | GPU 793.4 MB\n",
      "Epoch 00027 | Step 00080 | Loss 0.5497 | Train Acc 0.8418 | Speed (samples/sec) 5903.8773 | GPU 793.4 MB\n",
      "Epoch 00027 | Step 00100 | Loss 0.5125 | Train Acc 0.8438 | Speed (samples/sec) 5903.7327 | GPU 793.4 MB\n",
      "Epoch 00027 | Step 00120 | Loss 0.5453 | Train Acc 0.8408 | Speed (samples/sec) 5903.5299 | GPU 793.4 MB\n",
      "Epoch 00027 | Step 00140 | Loss 0.4745 | Train Acc 0.8711 | Speed (samples/sec) 5903.4685 | GPU 793.4 MB\n",
      "Epoch 00027 | Step 00160 | Loss 0.5043 | Train Acc 0.8477 | Speed (samples/sec) 5903.3796 | GPU 793.4 MB\n",
      "Epoch 00027 | Step 00180 | Loss 0.5412 | Train Acc 0.8340 | Speed (samples/sec) 5904.0693 | GPU 793.4 MB\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/245 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch Time(s): 36.7262\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 245/245 [00:41<00:00,  5.94it/s]\n",
      "100%|██████████| 245/245 [00:50<00:00,  4.86it/s]\n",
      "100%|██████████| 245/245 [00:50<00:00,  4.85it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val acc 0.7752, Test Acc: 0.6237\n",
      "Epoch 00028 | Step 00000 | Loss 0.8237 | Train Acc 0.7744 | Speed (samples/sec) 5902.3555 | GPU 793.4 MB\n",
      "Epoch 00028 | Step 00020 | Loss 0.5735 | Train Acc 0.8467 | Speed (samples/sec) 5902.3605 | GPU 793.4 MB\n",
      "Epoch 00028 | Step 00040 | Loss 0.5083 | Train Acc 0.8535 | Speed (samples/sec) 5903.8161 | GPU 793.4 MB\n",
      "Epoch 00028 | Step 00060 | Loss 0.5350 | Train Acc 0.8418 | Speed (samples/sec) 5905.3371 | GPU 793.4 MB\n",
      "Epoch 00028 | Step 00080 | Loss 0.5107 | Train Acc 0.8594 | Speed (samples/sec) 5906.4831 | GPU 793.4 MB\n",
      "Epoch 00028 | Step 00100 | Loss 0.5378 | Train Acc 0.8477 | Speed (samples/sec) 5907.4068 | GPU 793.4 MB\n",
      "Epoch 00028 | Step 00120 | Loss 0.4742 | Train Acc 0.8701 | Speed (samples/sec) 5908.1726 | GPU 793.4 MB\n",
      "Epoch 00028 | Step 00140 | Loss 0.4938 | Train Acc 0.8643 | Speed (samples/sec) 5909.6665 | GPU 793.4 MB\n",
      "Epoch 00028 | Step 00160 | Loss 0.4892 | Train Acc 0.8623 | Speed (samples/sec) 5910.2121 | GPU 793.4 MB\n",
      "Epoch 00028 | Step 00180 | Loss 0.5103 | Train Acc 0.8555 | Speed (samples/sec) 5910.0412 | GPU 793.4 MB\n",
      "Epoch Time(s): 35.6424\n",
      "Epoch 00029 | Step 00000 | Loss 0.5582 | Train Acc 0.8545 | Speed (samples/sec) 5908.6268 | GPU 793.4 MB\n",
      "Epoch 00029 | Step 00020 | Loss 0.5510 | Train Acc 0.8389 | Speed (samples/sec) 5909.3092 | GPU 793.4 MB\n",
      "Epoch 00029 | Step 00040 | Loss 0.5634 | Train Acc 0.8311 | Speed (samples/sec) 5910.3622 | GPU 793.4 MB\n",
      "Epoch 00029 | Step 00060 | Loss 0.5415 | Train Acc 0.8506 | Speed (samples/sec) 5911.4257 | GPU 793.4 MB\n",
      "Epoch 00029 | Step 00080 | Loss 0.5370 | Train Acc 0.8447 | Speed (samples/sec) 5912.1690 | GPU 793.4 MB\n",
      "Epoch 00029 | Step 00100 | Loss 0.5006 | Train Acc 0.8643 | Speed (samples/sec) 5913.4724 | GPU 793.4 MB\n",
      "Epoch 00029 | Step 00120 | Loss 0.5351 | Train Acc 0.8477 | Speed (samples/sec) 5914.2981 | GPU 793.4 MB\n",
      "Epoch 00029 | Step 00140 | Loss 0.4660 | Train Acc 0.8623 | Speed (samples/sec) 5915.2180 | GPU 793.4 MB\n",
      "Epoch 00029 | Step 00160 | Loss 0.4842 | Train Acc 0.8721 | Speed (samples/sec) 5916.1668 | GPU 793.4 MB\n",
      "Epoch 00029 | Step 00180 | Loss 0.4261 | Train Acc 0.8652 | Speed (samples/sec) 5916.5601 | GPU 793.4 MB\n",
      "Epoch Time(s): 35.5213\n",
      "Epoch 00030 | Step 00000 | Loss 0.7513 | Train Acc 0.7773 | Speed (samples/sec) 5915.0357 | GPU 793.4 MB\n",
      "Epoch 00030 | Step 00020 | Loss 0.4799 | Train Acc 0.8525 | Speed (samples/sec) 5916.2207 | GPU 793.4 MB\n",
      "Epoch 00030 | Step 00040 | Loss 0.5284 | Train Acc 0.8525 | Speed (samples/sec) 5917.2084 | GPU 793.4 MB\n",
      "Epoch 00030 | Step 00060 | Loss 0.4989 | Train Acc 0.8574 | Speed (samples/sec) 5918.1407 | GPU 793.4 MB\n",
      "Epoch 00030 | Step 00080 | Loss 0.4608 | Train Acc 0.8643 | Speed (samples/sec) 5918.4123 | GPU 793.5 MB\n",
      "Epoch 00030 | Step 00100 | Loss 0.5280 | Train Acc 0.8545 | Speed (samples/sec) 5917.9091 | GPU 793.5 MB\n",
      "Epoch 00030 | Step 00120 | Loss 0.5026 | Train Acc 0.8555 | Speed (samples/sec) 5917.7518 | GPU 793.5 MB\n",
      "Epoch 00030 | Step 00140 | Loss 0.4991 | Train Acc 0.8584 | Speed (samples/sec) 5917.9147 | GPU 793.5 MB\n",
      "Epoch 00030 | Step 00160 | Loss 0.4864 | Train Acc 0.8584 | Speed (samples/sec) 5917.7590 | GPU 793.5 MB\n",
      "Epoch 00030 | Step 00180 | Loss 0.5355 | Train Acc 0.8457 | Speed (samples/sec) 5917.9038 | GPU 793.5 MB\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/245 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch Time(s): 36.5472\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 245/245 [00:41<00:00,  5.94it/s]\n",
      "100%|██████████| 245/245 [00:49<00:00,  4.91it/s]\n",
      "100%|██████████| 245/245 [00:50<00:00,  4.89it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val acc 0.8368, Test Acc: 0.6884\n",
      "Epoch 00031 | Step 00000 | Loss 0.5774 | Train Acc 0.8320 | Speed (samples/sec) 5915.6863 | GPU 793.5 MB\n",
      "Epoch 00031 | Step 00020 | Loss 0.5012 | Train Acc 0.8584 | Speed (samples/sec) 5915.7557 | GPU 793.5 MB\n",
      "Epoch 00031 | Step 00040 | Loss 0.4452 | Train Acc 0.8701 | Speed (samples/sec) 5914.9021 | GPU 793.5 MB\n",
      "Epoch 00031 | Step 00060 | Loss 0.5066 | Train Acc 0.8564 | Speed (samples/sec) 5914.3876 | GPU 793.5 MB\n",
      "Epoch 00031 | Step 00080 | Loss 0.5260 | Train Acc 0.8545 | Speed (samples/sec) 5914.9184 | GPU 793.5 MB\n",
      "Epoch 00031 | Step 00100 | Loss 0.4634 | Train Acc 0.8652 | Speed (samples/sec) 5915.1267 | GPU 793.5 MB\n",
      "Epoch 00031 | Step 00120 | Loss 0.5367 | Train Acc 0.8535 | Speed (samples/sec) 5915.0526 | GPU 793.5 MB\n",
      "Epoch 00031 | Step 00140 | Loss 0.5078 | Train Acc 0.8525 | Speed (samples/sec) 5915.1658 | GPU 793.5 MB\n",
      "Epoch 00031 | Step 00160 | Loss 0.4232 | Train Acc 0.8760 | Speed (samples/sec) 5915.6578 | GPU 793.5 MB\n",
      "Epoch 00031 | Step 00180 | Loss 0.5103 | Train Acc 0.8525 | Speed (samples/sec) 5915.7519 | GPU 793.5 MB\n",
      "Epoch Time(s): 37.3066\n",
      "Epoch 00032 | Step 00000 | Loss 1.2581 | Train Acc 0.7148 | Speed (samples/sec) 5914.1508 | GPU 793.5 MB\n",
      "Epoch 00032 | Step 00020 | Loss 0.5754 | Train Acc 0.8359 | Speed (samples/sec) 5913.9390 | GPU 793.5 MB\n",
      "Epoch 00032 | Step 00040 | Loss 0.5953 | Train Acc 0.8379 | Speed (samples/sec) 5913.8197 | GPU 793.5 MB\n",
      "Epoch 00032 | Step 00060 | Loss 0.4940 | Train Acc 0.8721 | Speed (samples/sec) 5914.3622 | GPU 793.5 MB\n",
      "Epoch 00032 | Step 00080 | Loss 0.5438 | Train Acc 0.8506 | Speed (samples/sec) 5914.9056 | GPU 793.5 MB\n",
      "Epoch 00032 | Step 00100 | Loss 0.5145 | Train Acc 0.8486 | Speed (samples/sec) 5915.0413 | GPU 793.5 MB\n",
      "Epoch 00032 | Step 00120 | Loss 0.5709 | Train Acc 0.8330 | Speed (samples/sec) 5915.3276 | GPU 793.5 MB\n",
      "Epoch 00032 | Step 00140 | Loss 0.4734 | Train Acc 0.8643 | Speed (samples/sec) 5915.6798 | GPU 793.5 MB\n",
      "Epoch 00032 | Step 00160 | Loss 0.5087 | Train Acc 0.8545 | Speed (samples/sec) 5915.5676 | GPU 793.5 MB\n",
      "Epoch 00032 | Step 00180 | Loss 0.4676 | Train Acc 0.8633 | Speed (samples/sec) 5914.7086 | GPU 793.5 MB\n",
      "Epoch Time(s): 37.0044\n",
      "Epoch 00033 | Step 00000 | Loss 0.6156 | Train Acc 0.8193 | Speed (samples/sec) 5913.2676 | GPU 793.5 MB\n",
      "Epoch 00033 | Step 00020 | Loss 0.5565 | Train Acc 0.8496 | Speed (samples/sec) 5914.2867 | GPU 793.5 MB\n",
      "Epoch 00033 | Step 00040 | Loss 0.4847 | Train Acc 0.8623 | Speed (samples/sec) 5914.1915 | GPU 793.5 MB\n",
      "Epoch 00033 | Step 00060 | Loss 0.5457 | Train Acc 0.8301 | Speed (samples/sec) 5914.4830 | GPU 793.5 MB\n",
      "Epoch 00033 | Step 00080 | Loss 0.5012 | Train Acc 0.8623 | Speed (samples/sec) 5914.6072 | GPU 793.5 MB\n",
      "Epoch 00033 | Step 00100 | Loss 0.5027 | Train Acc 0.8516 | Speed (samples/sec) 5914.9211 | GPU 793.5 MB\n",
      "Epoch 00033 | Step 00120 | Loss 0.4661 | Train Acc 0.8760 | Speed (samples/sec) 5915.4258 | GPU 793.5 MB\n",
      "Epoch 00033 | Step 00140 | Loss 0.4351 | Train Acc 0.8799 | Speed (samples/sec) 5915.6804 | GPU 793.5 MB\n",
      "Epoch 00033 | Step 00160 | Loss 0.4403 | Train Acc 0.8789 | Speed (samples/sec) 5915.7699 | GPU 793.5 MB\n",
      "Epoch 00033 | Step 00180 | Loss 0.5340 | Train Acc 0.8535 | Speed (samples/sec) 5915.7180 | GPU 793.5 MB\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/245 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch Time(s): 36.7671\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 245/245 [00:40<00:00,  6.01it/s]\n",
      "100%|██████████| 245/245 [00:50<00:00,  4.85it/s]\n",
      "100%|██████████| 245/245 [00:49<00:00,  4.98it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val acc 0.8498, Test Acc: 0.6983\n",
      "Best val acc 0.8498, best test acc: 0.6983\n",
      "Epoch 00034 | Step 00000 | Loss 0.5010 | Train Acc 0.8643 | Speed (samples/sec) 5914.2675 | GPU 793.5 MB\n",
      "Epoch 00034 | Step 00020 | Loss 0.5700 | Train Acc 0.8320 | Speed (samples/sec) 5914.8286 | GPU 793.5 MB\n",
      "Epoch 00034 | Step 00040 | Loss 0.4920 | Train Acc 0.8564 | Speed (samples/sec) 5915.3794 | GPU 793.5 MB\n",
      "Epoch 00034 | Step 00060 | Loss 0.5404 | Train Acc 0.8398 | Speed (samples/sec) 5915.4919 | GPU 793.5 MB\n",
      "Epoch 00034 | Step 00080 | Loss 0.5306 | Train Acc 0.8506 | Speed (samples/sec) 5915.3955 | GPU 793.5 MB\n",
      "Epoch 00034 | Step 00100 | Loss 0.4910 | Train Acc 0.8643 | Speed (samples/sec) 5915.5709 | GPU 793.5 MB\n",
      "Epoch 00034 | Step 00120 | Loss 0.4957 | Train Acc 0.8633 | Speed (samples/sec) 5916.1678 | GPU 793.5 MB\n",
      "Epoch 00034 | Step 00140 | Loss 0.4791 | Train Acc 0.8613 | Speed (samples/sec) 5916.0245 | GPU 793.5 MB\n",
      "Epoch 00034 | Step 00160 | Loss 0.5060 | Train Acc 0.8730 | Speed (samples/sec) 5915.7926 | GPU 793.5 MB\n",
      "Epoch 00034 | Step 00180 | Loss 0.4277 | Train Acc 0.8730 | Speed (samples/sec) 5916.0691 | GPU 793.5 MB\n",
      "Epoch Time(s): 36.7705\n",
      "Epoch 00035 | Step 00000 | Loss 1.8086 | Train Acc 0.6025 | Speed (samples/sec) 5914.5097 | GPU 793.5 MB\n",
      "Epoch 00035 | Step 00020 | Loss 0.6671 | Train Acc 0.8223 | Speed (samples/sec) 5915.4801 | GPU 793.5 MB\n",
      "Epoch 00035 | Step 00040 | Loss 0.5386 | Train Acc 0.8457 | Speed (samples/sec) 5915.7925 | GPU 793.5 MB\n",
      "Epoch 00035 | Step 00060 | Loss 0.5270 | Train Acc 0.8516 | Speed (samples/sec) 5916.0372 | GPU 793.5 MB\n",
      "Epoch 00035 | Step 00080 | Loss 0.5666 | Train Acc 0.8379 | Speed (samples/sec) 5916.1656 | GPU 793.5 MB\n",
      "Epoch 00035 | Step 00100 | Loss 0.4895 | Train Acc 0.8623 | Speed (samples/sec) 5916.1773 | GPU 793.5 MB\n",
      "Epoch 00035 | Step 00120 | Loss 0.4896 | Train Acc 0.8486 | Speed (samples/sec) 5917.2948 | GPU 793.5 MB\n",
      "Epoch 00035 | Step 00140 | Loss 0.5719 | Train Acc 0.8281 | Speed (samples/sec) 5917.5828 | GPU 793.5 MB\n",
      "Epoch 00035 | Step 00160 | Loss 0.6090 | Train Acc 0.8252 | Speed (samples/sec) 5918.0179 | GPU 793.5 MB\n",
      "Epoch 00035 | Step 00180 | Loss 0.5986 | Train Acc 0.8389 | Speed (samples/sec) 5918.2486 | GPU 793.5 MB\n",
      "Epoch Time(s): 36.0870\n",
      "Epoch 00036 | Step 00000 | Loss 0.4496 | Train Acc 0.8760 | Speed (samples/sec) 5916.5778 | GPU 793.5 MB\n",
      "Epoch 00036 | Step 00020 | Loss 0.4539 | Train Acc 0.8730 | Speed (samples/sec) 5916.3103 | GPU 793.5 MB\n",
      "Epoch 00036 | Step 00040 | Loss 0.4524 | Train Acc 0.8584 | Speed (samples/sec) 5916.5991 | GPU 793.5 MB\n",
      "Epoch 00036 | Step 00060 | Loss 0.4996 | Train Acc 0.8477 | Speed (samples/sec) 5916.6216 | GPU 793.5 MB\n",
      "Epoch 00036 | Step 00080 | Loss 0.5324 | Train Acc 0.8506 | Speed (samples/sec) 5917.0562 | GPU 793.5 MB\n",
      "Epoch 00036 | Step 00100 | Loss 0.5435 | Train Acc 0.8447 | Speed (samples/sec) 5917.1910 | GPU 793.5 MB\n",
      "Epoch 00036 | Step 00120 | Loss 0.4769 | Train Acc 0.8682 | Speed (samples/sec) 5916.9253 | GPU 793.5 MB\n",
      "Epoch 00036 | Step 00140 | Loss 0.4859 | Train Acc 0.8662 | Speed (samples/sec) 5916.8529 | GPU 793.5 MB\n",
      "Epoch 00036 | Step 00160 | Loss 0.5174 | Train Acc 0.8682 | Speed (samples/sec) 5917.4575 | GPU 793.5 MB\n",
      "Epoch 00036 | Step 00180 | Loss 0.5100 | Train Acc 0.8574 | Speed (samples/sec) 5916.9092 | GPU 793.5 MB\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/245 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch Time(s): 37.2463\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 245/245 [00:40<00:00,  6.12it/s]\n",
      "100%|██████████| 245/245 [00:49<00:00,  4.92it/s]\n",
      "100%|██████████| 245/245 [00:48<00:00,  5.07it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val acc 0.7949, Test Acc: 0.6338\n",
      "Epoch 00037 | Step 00000 | Loss 0.7432 | Train Acc 0.7959 | Speed (samples/sec) 5915.2410 | GPU 793.5 MB\n",
      "Epoch 00037 | Step 00020 | Loss 0.5821 | Train Acc 0.8320 | Speed (samples/sec) 5915.0670 | GPU 793.5 MB\n",
      "Epoch 00037 | Step 00040 | Loss 0.4984 | Train Acc 0.8486 | Speed (samples/sec) 5915.3179 | GPU 793.5 MB\n",
      "Epoch 00037 | Step 00060 | Loss 0.5291 | Train Acc 0.8594 | Speed (samples/sec) 5914.9378 | GPU 793.5 MB\n",
      "Epoch 00037 | Step 00080 | Loss 0.5022 | Train Acc 0.8643 | Speed (samples/sec) 5914.9526 | GPU 793.5 MB\n",
      "Epoch 00037 | Step 00100 | Loss 0.4726 | Train Acc 0.8623 | Speed (samples/sec) 5914.7465 | GPU 793.5 MB\n",
      "Epoch 00037 | Step 00120 | Loss 0.4861 | Train Acc 0.8584 | Speed (samples/sec) 5915.1767 | GPU 793.5 MB\n",
      "Epoch 00037 | Step 00140 | Loss 0.5972 | Train Acc 0.8330 | Speed (samples/sec) 5915.3261 | GPU 793.5 MB\n",
      "Epoch 00037 | Step 00160 | Loss 0.4734 | Train Acc 0.8525 | Speed (samples/sec) 5915.5258 | GPU 793.5 MB\n",
      "Epoch 00037 | Step 00180 | Loss 0.5138 | Train Acc 0.8545 | Speed (samples/sec) 5915.5517 | GPU 793.5 MB\n",
      "Epoch Time(s): 37.0644\n",
      "Epoch 00038 | Step 00000 | Loss 0.5417 | Train Acc 0.8398 | Speed (samples/sec) 5914.1013 | GPU 793.5 MB\n",
      "Epoch 00038 | Step 00020 | Loss 0.5405 | Train Acc 0.8574 | Speed (samples/sec) 5914.4573 | GPU 793.5 MB\n",
      "Epoch 00038 | Step 00040 | Loss 0.5266 | Train Acc 0.8447 | Speed (samples/sec) 5914.2978 | GPU 793.5 MB\n",
      "Epoch 00038 | Step 00060 | Loss 0.5379 | Train Acc 0.8369 | Speed (samples/sec) 5914.5076 | GPU 793.5 MB\n",
      "Epoch 00038 | Step 00080 | Loss 0.4921 | Train Acc 0.8574 | Speed (samples/sec) 5914.8000 | GPU 793.5 MB\n",
      "Epoch 00038 | Step 00100 | Loss 0.4996 | Train Acc 0.8564 | Speed (samples/sec) 5915.2370 | GPU 793.5 MB\n",
      "Epoch 00038 | Step 00120 | Loss 0.5003 | Train Acc 0.8604 | Speed (samples/sec) 5915.8346 | GPU 793.5 MB\n",
      "Epoch 00038 | Step 00140 | Loss 0.5085 | Train Acc 0.8574 | Speed (samples/sec) 5915.8806 | GPU 793.5 MB\n",
      "Epoch 00038 | Step 00160 | Loss 0.4944 | Train Acc 0.8506 | Speed (samples/sec) 5916.6102 | GPU 793.5 MB\n",
      "Epoch 00038 | Step 00180 | Loss 0.5478 | Train Acc 0.8457 | Speed (samples/sec) 5916.7575 | GPU 793.5 MB\n",
      "Epoch Time(s): 36.3357\n",
      "Epoch 00039 | Step 00000 | Loss 1.0100 | Train Acc 0.7568 | Speed (samples/sec) 5915.3403 | GPU 793.5 MB\n",
      "Epoch 00039 | Step 00020 | Loss 0.5345 | Train Acc 0.8564 | Speed (samples/sec) 5915.2459 | GPU 793.5 MB\n",
      "Epoch 00039 | Step 00040 | Loss 0.5029 | Train Acc 0.8574 | Speed (samples/sec) 5915.5847 | GPU 793.5 MB\n",
      "Epoch 00039 | Step 00060 | Loss 0.5461 | Train Acc 0.8389 | Speed (samples/sec) 5915.7929 | GPU 793.5 MB\n",
      "Epoch 00039 | Step 00080 | Loss 0.5610 | Train Acc 0.8545 | Speed (samples/sec) 5915.3639 | GPU 793.5 MB\n",
      "Epoch 00039 | Step 00100 | Loss 0.4715 | Train Acc 0.8682 | Speed (samples/sec) 5915.6755 | GPU 793.5 MB\n",
      "Epoch 00039 | Step 00120 | Loss 0.5317 | Train Acc 0.8428 | Speed (samples/sec) 5916.3149 | GPU 793.5 MB\n",
      "Epoch 00039 | Step 00140 | Loss 0.4922 | Train Acc 0.8652 | Speed (samples/sec) 5916.9251 | GPU 793.5 MB\n",
      "Epoch 00039 | Step 00160 | Loss 0.5298 | Train Acc 0.8477 | Speed (samples/sec) 5917.5628 | GPU 793.5 MB\n",
      "Epoch 00039 | Step 00180 | Loss 0.4351 | Train Acc 0.8691 | Speed (samples/sec) 5918.1785 | GPU 793.5 MB\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/245 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch Time(s): 36.3895\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 245/245 [00:41<00:00,  5.97it/s]\n",
      "100%|██████████| 245/245 [00:49<00:00,  4.91it/s]\n",
      "100%|██████████| 245/245 [00:51<00:00,  4.79it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val acc 0.8524, Test Acc: 0.7015\n",
      "Best val acc 0.8524, best test acc: 0.7015\n",
      "Epoch 00040 | Step 00000 | Loss 0.4976 | Train Acc 0.8457 | Speed (samples/sec) 5917.1799 | GPU 793.5 MB\n",
      "Epoch 00040 | Step 00020 | Loss 0.4588 | Train Acc 0.8672 | Speed (samples/sec) 5916.8199 | GPU 793.5 MB\n",
      "Epoch 00040 | Step 00040 | Loss 0.4963 | Train Acc 0.8584 | Speed (samples/sec) 5916.6979 | GPU 793.5 MB\n",
      "Epoch 00040 | Step 00060 | Loss 0.4786 | Train Acc 0.8604 | Speed (samples/sec) 5916.9112 | GPU 793.5 MB\n",
      "Epoch 00040 | Step 00080 | Loss 0.4949 | Train Acc 0.8506 | Speed (samples/sec) 5917.1283 | GPU 793.5 MB\n",
      "Epoch 00040 | Step 00100 | Loss 0.4967 | Train Acc 0.8516 | Speed (samples/sec) 5916.9296 | GPU 793.5 MB\n",
      "Epoch 00040 | Step 00120 | Loss 0.4997 | Train Acc 0.8613 | Speed (samples/sec) 5916.8854 | GPU 793.5 MB\n",
      "Epoch 00040 | Step 00140 | Loss 0.5527 | Train Acc 0.8398 | Speed (samples/sec) 5917.2307 | GPU 793.5 MB\n",
      "Epoch 00040 | Step 00160 | Loss 0.4512 | Train Acc 0.8672 | Speed (samples/sec) 5917.5044 | GPU 793.5 MB\n",
      "Epoch 00040 | Step 00180 | Loss 0.4403 | Train Acc 0.8740 | Speed (samples/sec) 5917.0353 | GPU 793.5 MB\n",
      "Epoch Time(s): 37.1434\n",
      "Epoch 00041 | Step 00000 | Loss 0.5611 | Train Acc 0.8418 | Speed (samples/sec) 5915.4619 | GPU 793.5 MB\n",
      "Epoch 00041 | Step 00020 | Loss 0.4756 | Train Acc 0.8643 | Speed (samples/sec) 5914.7762 | GPU 793.5 MB\n",
      "Epoch 00041 | Step 00040 | Loss 0.5260 | Train Acc 0.8467 | Speed (samples/sec) 5914.3480 | GPU 793.5 MB\n",
      "Epoch 00041 | Step 00060 | Loss 0.4768 | Train Acc 0.8535 | Speed (samples/sec) 5913.2347 | GPU 793.5 MB\n",
      "Epoch 00041 | Step 00080 | Loss 0.4991 | Train Acc 0.8643 | Speed (samples/sec) 5913.2392 | GPU 793.5 MB\n",
      "Epoch 00041 | Step 00100 | Loss 0.4217 | Train Acc 0.8809 | Speed (samples/sec) 5912.9010 | GPU 793.5 MB\n",
      "Epoch 00041 | Step 00120 | Loss 0.5168 | Train Acc 0.8623 | Speed (samples/sec) 5912.5568 | GPU 793.5 MB\n",
      "Epoch 00041 | Step 00140 | Loss 0.4844 | Train Acc 0.8682 | Speed (samples/sec) 5913.0575 | GPU 793.5 MB\n",
      "Epoch 00041 | Step 00160 | Loss 0.5359 | Train Acc 0.8457 | Speed (samples/sec) 5912.8511 | GPU 793.5 MB\n",
      "Epoch 00041 | Step 00180 | Loss 0.4981 | Train Acc 0.8438 | Speed (samples/sec) 5912.8649 | GPU 793.5 MB\n",
      "Epoch Time(s): 38.0710\n",
      "Epoch 00042 | Step 00000 | Loss 0.5067 | Train Acc 0.8408 | Speed (samples/sec) 5912.2886 | GPU 793.5 MB\n",
      "Epoch 00042 | Step 00020 | Loss 0.4896 | Train Acc 0.8604 | Speed (samples/sec) 5912.1263 | GPU 793.5 MB\n",
      "Epoch 00042 | Step 00040 | Loss 0.4615 | Train Acc 0.8633 | Speed (samples/sec) 5912.3768 | GPU 793.5 MB\n",
      "Epoch 00042 | Step 00060 | Loss 0.4302 | Train Acc 0.8730 | Speed (samples/sec) 5912.5952 | GPU 793.5 MB\n",
      "Epoch 00042 | Step 00080 | Loss 0.4412 | Train Acc 0.8760 | Speed (samples/sec) 5913.1138 | GPU 793.5 MB\n",
      "Epoch 00042 | Step 00100 | Loss 0.4839 | Train Acc 0.8613 | Speed (samples/sec) 5912.8941 | GPU 793.5 MB\n",
      "Epoch 00042 | Step 00120 | Loss 0.4980 | Train Acc 0.8516 | Speed (samples/sec) 5912.5395 | GPU 793.5 MB\n",
      "Epoch 00042 | Step 00140 | Loss 0.4574 | Train Acc 0.8652 | Speed (samples/sec) 5912.9392 | GPU 793.5 MB\n",
      "Epoch 00042 | Step 00160 | Loss 0.5057 | Train Acc 0.8516 | Speed (samples/sec) 5913.1486 | GPU 793.5 MB\n",
      "Epoch 00042 | Step 00180 | Loss 0.4606 | Train Acc 0.8721 | Speed (samples/sec) 5912.8812 | GPU 793.5 MB\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/245 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch Time(s): 37.0849\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 245/245 [00:41<00:00,  5.92it/s]\n",
      "100%|██████████| 245/245 [00:50<00:00,  4.88it/s]\n",
      "100%|██████████| 245/245 [00:50<00:00,  4.84it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val acc 0.8650, Test Acc: 0.7015\n",
      "Best val acc 0.8650, best test acc: 0.7015\n",
      "Epoch 00043 | Step 00000 | Loss 0.5175 | Train Acc 0.8633 | Speed (samples/sec) 5911.2870 | GPU 793.5 MB\n",
      "Epoch 00043 | Step 00020 | Loss 0.4789 | Train Acc 0.8613 | Speed (samples/sec) 5910.9740 | GPU 793.5 MB\n",
      "Epoch 00043 | Step 00040 | Loss 0.5095 | Train Acc 0.8613 | Speed (samples/sec) 5910.7056 | GPU 793.5 MB\n",
      "Epoch 00043 | Step 00060 | Loss 0.5084 | Train Acc 0.8574 | Speed (samples/sec) 5911.3015 | GPU 793.5 MB\n",
      "Epoch 00043 | Step 00080 | Loss 0.4849 | Train Acc 0.8604 | Speed (samples/sec) 5911.4334 | GPU 793.5 MB\n",
      "Epoch 00043 | Step 00100 | Loss 0.4813 | Train Acc 0.8604 | Speed (samples/sec) 5911.2419 | GPU 793.5 MB\n",
      "Epoch 00043 | Step 00120 | Loss 0.4902 | Train Acc 0.8672 | Speed (samples/sec) 5912.0491 | GPU 793.5 MB\n",
      "Epoch 00043 | Step 00140 | Loss 0.5131 | Train Acc 0.8633 | Speed (samples/sec) 5911.8833 | GPU 793.5 MB\n",
      "Epoch 00043 | Step 00160 | Loss 0.5291 | Train Acc 0.8467 | Speed (samples/sec) 5912.2858 | GPU 793.5 MB\n",
      "Epoch 00043 | Step 00180 | Loss 0.4849 | Train Acc 0.8623 | Speed (samples/sec) 5912.3758 | GPU 793.5 MB\n",
      "Epoch Time(s): 36.8874\n",
      "Epoch 00044 | Step 00000 | Loss 0.6008 | Train Acc 0.8291 | Speed (samples/sec) 5911.4261 | GPU 793.5 MB\n",
      "Epoch 00044 | Step 00020 | Loss 0.5439 | Train Acc 0.8525 | Speed (samples/sec) 5911.3257 | GPU 793.5 MB\n",
      "Epoch 00044 | Step 00040 | Loss 0.4952 | Train Acc 0.8467 | Speed (samples/sec) 5911.5549 | GPU 793.5 MB\n",
      "Epoch 00044 | Step 00060 | Loss 0.4447 | Train Acc 0.8721 | Speed (samples/sec) 5911.6431 | GPU 793.5 MB\n",
      "Epoch 00044 | Step 00080 | Loss 0.4826 | Train Acc 0.8643 | Speed (samples/sec) 5911.8419 | GPU 793.5 MB\n",
      "Epoch 00044 | Step 00100 | Loss 0.4781 | Train Acc 0.8623 | Speed (samples/sec) 5912.2006 | GPU 793.5 MB\n",
      "Epoch 00044 | Step 00120 | Loss 0.5178 | Train Acc 0.8477 | Speed (samples/sec) 5912.0569 | GPU 793.5 MB\n",
      "Epoch 00044 | Step 00140 | Loss 0.4410 | Train Acc 0.8750 | Speed (samples/sec) 5912.3106 | GPU 793.5 MB\n",
      "Epoch 00044 | Step 00160 | Loss 0.4981 | Train Acc 0.8496 | Speed (samples/sec) 5912.1593 | GPU 793.5 MB\n",
      "Epoch 00044 | Step 00180 | Loss 0.4780 | Train Acc 0.8555 | Speed (samples/sec) 5912.1329 | GPU 793.5 MB\n",
      "Epoch Time(s): 36.9066\n",
      "Epoch 00045 | Step 00000 | Loss 0.8646 | Train Acc 0.7480 | Speed (samples/sec) 5911.0788 | GPU 793.5 MB\n",
      "Epoch 00045 | Step 00020 | Loss 0.4772 | Train Acc 0.8799 | Speed (samples/sec) 5911.1508 | GPU 793.5 MB\n",
      "Epoch 00045 | Step 00040 | Loss 0.4694 | Train Acc 0.8682 | Speed (samples/sec) 5911.9532 | GPU 793.5 MB\n",
      "Epoch 00045 | Step 00060 | Loss 0.4638 | Train Acc 0.8623 | Speed (samples/sec) 5912.3990 | GPU 793.5 MB\n",
      "Epoch 00045 | Step 00080 | Loss 0.4628 | Train Acc 0.8711 | Speed (samples/sec) 5911.9931 | GPU 793.5 MB\n",
      "Epoch 00045 | Step 00100 | Loss 0.4888 | Train Acc 0.8594 | Speed (samples/sec) 5912.2603 | GPU 793.5 MB\n",
      "Epoch 00045 | Step 00120 | Loss 0.4896 | Train Acc 0.8750 | Speed (samples/sec) 5912.3670 | GPU 793.5 MB\n",
      "Epoch 00045 | Step 00140 | Loss 0.4754 | Train Acc 0.8623 | Speed (samples/sec) 5912.8571 | GPU 793.5 MB\n",
      "Epoch 00045 | Step 00160 | Loss 0.4266 | Train Acc 0.8809 | Speed (samples/sec) 5913.2073 | GPU 793.5 MB\n",
      "Epoch 00045 | Step 00180 | Loss 0.4671 | Train Acc 0.8643 | Speed (samples/sec) 5913.7698 | GPU 793.5 MB\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/245 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch Time(s): 36.4351\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 245/245 [00:41<00:00,  5.91it/s]\n",
      "100%|██████████| 245/245 [00:48<00:00,  5.03it/s]\n",
      "100%|██████████| 245/245 [00:48<00:00,  5.02it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val acc 0.8466, Test Acc: 0.6890\n",
      "Epoch 00046 | Step 00000 | Loss 0.4957 | Train Acc 0.8447 | Speed (samples/sec) 5912.9835 | GPU 793.5 MB\n",
      "Epoch 00046 | Step 00020 | Loss 0.4769 | Train Acc 0.8496 | Speed (samples/sec) 5913.1762 | GPU 793.5 MB\n",
      "Epoch 00046 | Step 00040 | Loss 0.4341 | Train Acc 0.8750 | Speed (samples/sec) 5912.8257 | GPU 793.5 MB\n",
      "Epoch 00046 | Step 00060 | Loss 0.4338 | Train Acc 0.8711 | Speed (samples/sec) 5912.8255 | GPU 793.5 MB\n",
      "Epoch 00046 | Step 00080 | Loss 0.4515 | Train Acc 0.8721 | Speed (samples/sec) 5913.1611 | GPU 793.5 MB\n",
      "Epoch 00046 | Step 00100 | Loss 0.4737 | Train Acc 0.8701 | Speed (samples/sec) 5913.5636 | GPU 793.5 MB\n",
      "Epoch 00046 | Step 00120 | Loss 0.5422 | Train Acc 0.8555 | Speed (samples/sec) 5913.8796 | GPU 793.5 MB\n",
      "Epoch 00046 | Step 00140 | Loss 0.4511 | Train Acc 0.8740 | Speed (samples/sec) 5914.0924 | GPU 793.5 MB\n",
      "Epoch 00046 | Step 00160 | Loss 0.4875 | Train Acc 0.8584 | Speed (samples/sec) 5914.1689 | GPU 793.5 MB\n",
      "Epoch 00046 | Step 00180 | Loss 0.5008 | Train Acc 0.8643 | Speed (samples/sec) 5914.2619 | GPU 793.5 MB\n",
      "Epoch Time(s): 36.7261\n",
      "Epoch 00047 | Step 00000 | Loss 0.5781 | Train Acc 0.8311 | Speed (samples/sec) 5913.3119 | GPU 793.5 MB\n",
      "Epoch 00047 | Step 00020 | Loss 0.4951 | Train Acc 0.8408 | Speed (samples/sec) 5913.3479 | GPU 793.5 MB\n",
      "Epoch 00047 | Step 00040 | Loss 0.5184 | Train Acc 0.8545 | Speed (samples/sec) 5913.3491 | GPU 793.5 MB\n",
      "Epoch 00047 | Step 00060 | Loss 0.4470 | Train Acc 0.8818 | Speed (samples/sec) 5913.7539 | GPU 793.5 MB\n",
      "Epoch 00047 | Step 00080 | Loss 0.5422 | Train Acc 0.8604 | Speed (samples/sec) 5913.6983 | GPU 793.5 MB\n",
      "Epoch 00047 | Step 00100 | Loss 0.4593 | Train Acc 0.8623 | Speed (samples/sec) 5913.7362 | GPU 793.5 MB\n",
      "Epoch 00047 | Step 00120 | Loss 0.4707 | Train Acc 0.8662 | Speed (samples/sec) 5914.3725 | GPU 793.5 MB\n",
      "Epoch 00047 | Step 00140 | Loss 0.5517 | Train Acc 0.8438 | Speed (samples/sec) 5914.3984 | GPU 793.5 MB\n",
      "Epoch 00047 | Step 00160 | Loss 0.5381 | Train Acc 0.8623 | Speed (samples/sec) 5914.9284 | GPU 793.5 MB\n",
      "Epoch 00047 | Step 00180 | Loss 0.4609 | Train Acc 0.8721 | Speed (samples/sec) 5915.2751 | GPU 793.5 MB\n",
      "Epoch Time(s): 36.4364\n",
      "Epoch 00048 | Step 00000 | Loss 0.5594 | Train Acc 0.8496 | Speed (samples/sec) 5914.1913 | GPU 793.5 MB\n",
      "Epoch 00048 | Step 00020 | Loss 0.4933 | Train Acc 0.8633 | Speed (samples/sec) 5914.1038 | GPU 793.5 MB\n",
      "Epoch 00048 | Step 00040 | Loss 0.5175 | Train Acc 0.8467 | Speed (samples/sec) 5913.7749 | GPU 793.5 MB\n",
      "Epoch 00048 | Step 00060 | Loss 0.4821 | Train Acc 0.8623 | Speed (samples/sec) 5914.5013 | GPU 793.5 MB\n",
      "Epoch 00048 | Step 00080 | Loss 0.4855 | Train Acc 0.8574 | Speed (samples/sec) 5914.2806 | GPU 793.5 MB\n",
      "Epoch 00048 | Step 00100 | Loss 0.5288 | Train Acc 0.8506 | Speed (samples/sec) 5913.8723 | GPU 793.5 MB\n",
      "Epoch 00048 | Step 00120 | Loss 0.4414 | Train Acc 0.8633 | Speed (samples/sec) 5914.4659 | GPU 793.5 MB\n",
      "Epoch 00048 | Step 00140 | Loss 0.5042 | Train Acc 0.8652 | Speed (samples/sec) 5914.5010 | GPU 793.5 MB\n",
      "Epoch 00048 | Step 00160 | Loss 0.4969 | Train Acc 0.8623 | Speed (samples/sec) 5914.7728 | GPU 793.5 MB\n",
      "Epoch 00048 | Step 00180 | Loss 0.5122 | Train Acc 0.8447 | Speed (samples/sec) 5914.7591 | GPU 793.5 MB\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/245 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch Time(s): 36.8791\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 245/245 [00:41<00:00,  5.97it/s]\n",
      "100%|██████████| 245/245 [00:52<00:00,  4.68it/s]\n",
      "100%|██████████| 245/245 [00:49<00:00,  4.99it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val acc 0.7696, Test Acc: 0.6354\n",
      "Epoch 00049 | Step 00000 | Loss 1.0017 | Train Acc 0.7432 | Speed (samples/sec) 5913.9363 | GPU 793.5 MB\n",
      "Epoch 00049 | Step 00020 | Loss 0.7232 | Train Acc 0.7900 | Speed (samples/sec) 5913.9449 | GPU 793.5 MB\n",
      "Epoch 00049 | Step 00040 | Loss 0.5584 | Train Acc 0.8379 | Speed (samples/sec) 5914.0973 | GPU 793.5 MB\n",
      "Epoch 00049 | Step 00060 | Loss 0.5969 | Train Acc 0.8447 | Speed (samples/sec) 5914.1447 | GPU 793.5 MB\n",
      "Epoch 00049 | Step 00080 | Loss 0.5389 | Train Acc 0.8516 | Speed (samples/sec) 5914.2318 | GPU 793.5 MB\n",
      "Epoch 00049 | Step 00100 | Loss 0.4937 | Train Acc 0.8633 | Speed (samples/sec) 5913.7983 | GPU 793.5 MB\n",
      "Epoch 00049 | Step 00120 | Loss 0.5446 | Train Acc 0.8447 | Speed (samples/sec) 5913.8996 | GPU 793.5 MB\n",
      "Epoch 00049 | Step 00140 | Loss 0.4961 | Train Acc 0.8525 | Speed (samples/sec) 5913.9018 | GPU 793.5 MB\n",
      "Epoch 00049 | Step 00160 | Loss 0.5539 | Train Acc 0.8467 | Speed (samples/sec) 5913.9655 | GPU 793.5 MB\n",
      "Epoch 00049 | Step 00180 | Loss 0.4675 | Train Acc 0.8633 | Speed (samples/sec) 5914.1272 | GPU 793.5 MB\n",
      "Epoch Time(s): 36.9865\n",
      "Avg epoch time: 40.10270723766751\n",
      "Best val acc 0.8650, best test acc: 0.7015\n"
     ]
    }
   ],
   "source": [
    "g, num_labels = load_ogb('ogbn-products')\n",
    "g.ndata.pop('features')\n",
    "\n",
    "in_feats = 128\n",
    "eigen_vals, eigen_vecs = get_eigen(g, in_feats, 'ogbn-products')\n",
    "train_g = val_g = test_g = g\n",
    "# Don't scale eigenvectors.\n",
    "train_nfeat = val_nfeat = test_nfeat = th.tensor(eigen_vecs, dtype=th.float32)\n",
    "train_labels = val_labels = test_labels = g.ndata.pop('labels')\n",
    "\n",
    "hyperparams = {\n",
    "    'batch_size': 1024,\n",
    "    'num_workers': 4,\n",
    "    'num_hidden': 256,\n",
    "    'num_layers': 3,\n",
    "    'dropout': 0.5,\n",
    "    'lr': 0.003,\n",
    "    'num_epochs': 50,\n",
    "    'fanouts': [15,10,5],\n",
    "    'eval_batch_size': 10000\n",
    "}\n",
    "device = th.device('cuda:1')\n",
    "\n",
    "# Pack data\n",
    "data = train_g, val_g, test_g, train_nfeat, train_labels, \\\n",
    "           val_nfeat, val_labels, test_nfeat, test_labels\n",
    "# Define model\n",
    "in_feats = train_nfeat.shape[1]\n",
    "model = SAGE(in_feats, hyperparams['num_hidden'], num_labels,\n",
    "             hyperparams['num_layers'], F.relu, hyperparams['dropout'])\n",
    "train_sample(model, data, hyperparams, device, 3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}