notebooks/issue-36.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from nats_bench import create\n",
    "from tqdm import tqdm\n",
    "import numpy as np\n",
    "\n",
    "def close_to(a, b, eps=1e-4):\n",
    "    if b != 0 and abs(a-b) / abs(b) > eps:\n",
    "        return False\n",
    "    if a != 0 and abs(a-b) / abs(a) > eps:\n",
    "        return False\n",
    "    return True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def check_flops_params(xapi):\n",
    "    print(f\"Check {xapi}\")\n",
    "    datasets = (\"cifar10-valid\", \"cifar10\", \"cifar100\", \"ImageNet16-120\")\n",
    "    counts = 0\n",
    "    for index in tqdm(range(len(xapi))):\n",
    "        for dataset in datasets:\n",
    "            info_12 = xapi.get_cost_info(index, dataset, hp=\"12\")\n",
    "            info_full = xapi.get_cost_info(index, dataset, hp=xapi.full_epochs_in_paper)\n",
    "            assert close_to(info_12['flops'], info_full['flops']), f\"The {index}-th \" \\\n",
    "            f\"architecture has issues on {dataset} \" \\\n",
    "            f\"-- {info_12['flops']} vs {info_full['flops']}.\"  # check the FLOPs\n",
    "            assert close_to(info_12['params'], info_full['params']), f\"The {index}-th \" \\\n",
    "            f\"architecture has issues on {dataset} \" \\\n",
    "            f\"-- {info_12['params']} vs {info_full['params']}.\"  # check the number of parameters\n",
    "            counts += 1\n",
    "    print(f\"Check {xapi} completed -- {counts} arch-dataset pairs.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create the API for size search space\n",
    "api = create(None, 'sss', fast_mode=True, verbose=False)\n",
    "print(f'There are {len(api)} architectures in the size search space -- {api}')\n",
    "check_flops_params(api)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/15625 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2022-01-20 01:03:30] Try to use the default NATS-Bench (topology) path from fast_mode=True and path=None.\n",
      "There are 15625 architectures in the topology search space -- NATStopology(0/15625 architectures, fast_mode=True, file=None)\n",
      "Check NATStopology(0/15625 architectures, fast_mode=True, file=None)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 15625/15625 [20:06<00:00, 12.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Check NATStopology(15625/15625 architectures, fast_mode=True, file=None) completed -- 62500 arch-dataset pairs.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "# Create the API for topology search space\n",
    "api = create(None, 'tss', fast_mode=True, verbose=False)\n",
    "print(f'There are {len(api)} architectures in the topology search space -- {api}')\n",
    "check_flops_params(api)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# # This code block is to figure out the real reason of issue#16\n",
    "# from xautodl.models import get_cell_based_tiny_net\n",
    "# from xautodl.utils import count_parameters_in_MB\n",
    "# from xautodl.utils import get_model_infos\n",
    "\n",
    "# api = create(None, 'tss', fast_mode=True, verbose=False)\n",
    "# print(api)\n",
    "\n",
    "# index, dataset = 296, \"cifar10\"\n",
    "# arch = \"|skip_connect~0|+|none~0|nor_conv_3x3~1|+|avg_pool_3x3~0|nor_conv_3x3~1|nor_conv_3x3~2|\"\n",
    "# index = api.query_index_by_arch(arch)\n",
    "\n",
    "\n",
    "# info_12 = api.get_cost_info(index, dataset, hp=\"12\")\n",
    "# info_full = api.get_cost_info(index, dataset, hp=api.full_epochs_in_paper)\n",
    "# print(info_12)\n",
    "# print(info_full)\n",
    "\n",
    "# config_12 = api.get_net_config(index, dataset)\n",
    "# print(config_12)\n",
    "# config_full = api.get_net_config(index, dataset)\n",
    "# print(config_full)\n",
    "\n",
    "# # create the network, which is the sub-class of torch.nn.Module\n",
    "# network = get_cell_based_tiny_net(config_full)\n",
    "\n",
    "# flop, param = get_model_infos(network, (1, 3, 32, 32))\n",
    "# print(f\"FLOPs={flop}, param={param}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# results = api.query_meta_info_by_index(index, hp=api.full_epochs_in_paper)\n",
    "# print(results.all_results.keys())\n",
    "# print(\"\")\n",
    "# print(results.dataset_seed[dataset])\n",
    "# print(results.get_compute_costs(dataset))\n",
    "# print(\"\")\n",
    "# print(results.all_results[(dataset, 777)].flop)\n",
    "# print(results.all_results[(dataset, 888)].flop)\n",
    "# print(results.all_results[(dataset, 999)].flop)\n",
    "# print(results.all_results[('cifar100', 777)])"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}