ensemble-pyramidnet.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "import torch.nn.functional as F\n",
    "from torchvision import datasets, transforms, models\n",
    "from torch.utils.data import DataLoader\n",
    "\n",
    "import numpy as np\n",
    "\n",
    "import argparse\n",
    "import logging\n",
    "from datetime import datetime\n",
    "\n",
    "import os\n",
    "import sys\n",
    "import random\n",
    "\n",
    "from enums import (\n",
    "    ModelEnum,\n",
    "    OptimizerEnum,\n",
    "    LRSchedulerEnum,\n",
    "    MixEnum\n",
    ")\n",
    "from preprocess.preprocess_data import DataPreProcessor\n",
    "from preprocess.preprocess_model import ModelPreProcessor\n",
    "from train import Trainer\n",
    "\n",
    "from change_randomseed import RANDOMSEED"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_arg_parse():\n",
    "    parser = argparse.ArgumentParser()\n",
    "    \n",
    "    # parser.add_argument('-t', '--test', type=int, help='테스트모드', default=0)\n",
    "    # parser.add_argument('-pm', '--model_path', type=str, help='모델 폴더 이름', default='')\n",
    "    # parser.add_argument('--amp', type=int, help='amp 옵션', default=0)\n",
    "    \n",
    "    # parser.add_argument('-rs', '--random_seed', type=int, help='학습 랜덤 시드. -1은 랜덤 시드를 고정하지 않음.', default=4943872)\n",
    "    # parser.add_argument('-lf', '--log_file', type=int, help='로그 파일 출력 여부. 0=false, 1=true', default=1)\n",
    "    # parser.add_argument('-po', '--port', type=int, default=2033)\n",
    "    \n",
    "    # parser.add_argument('-m', '--model', type=ModelEnum, help='학습 모델', choices=list(ModelEnum), default=ModelEnum.custom)\n",
    "    # parser.add_argument('-p', '--parallel', type=int, help='멀티 gpu 사용 여부. 0=false, 1=true', default=0)\n",
    "    # parser.add_argument('-op', '--optimizer', type=OptimizerEnum, help='옵티마이저', choices=list(OptimizerEnum), default=OptimizerEnum.sgd)\n",
    "    # parser.add_argument('-ls', '--lr_scheduler', type=LRSchedulerEnum, help='lr 스케쥴러', choices=list(LRSchedulerEnum), default=LRSchedulerEnum.custom_annealing)\n",
    "    \n",
    "    # parser.add_argument('-ds', '--split_ratio', type=float, help='train/validation 분할 비율', default=0.2)\n",
    "    # parser.add_argument('-am', '--augmentation_mode', type=int, help='data augmentation mode', default=0)\n",
    "    # parser.add_argument('-asp', '--augment_split', type=float, help='augmentation 분할 비율', default=0.5)\n",
    "    # parser.add_argument('-w', '--num_worker', type=int, help='train/validation 분할 비율', default=0)\n",
    "    # parser.add_argument('-b', '--batch_size', type=int, help='학습 배치사이즈', default=128)\n",
    "    # parser.add_argument('-mc', '--mix_step', type=int, help='mix 적용시 몇 step마다 적용할지. 0은 모든 step에 적용.', default=0)\n",
    "    # parser.add_argument('-mt', '--mix_method', type=MixEnum, help='mix 방법', choices=list(MixEnum), default=MixEnum.mixup)\n",
    "    \n",
    "    # parser.add_argument('-pd', '--p_depth', type=int, help='pyramnidnet depth', default=164)\n",
    "    # parser.add_argument('-pa', '--p_alpha', type=int, help='pyramnidnet alpha', default=84)\n",
    "    # parser.add_argument('-ps', '--p_shake', type=int, help='pyramnidnet shake', default=0)\n",
    "    \n",
    "    # parser.add_argument('-e', '--epoch', type=int, help='epoch', default=100)\n",
    "    # parser.add_argument('-mlr', '--max_learning_rate', type=float, help='optimizer/scheduler max learning rate 설정 (custom cos scheduler는 반대)', default=0.1)\n",
    "    # parser.add_argument('-milr', '--min_learning_rate', type=float, help='optimizer/scheduler min learning rate 설정 (custom cos scheduler는 반대)', default=1e-4)\n",
    "    # parser.add_argument('-wd', '--weight_decay', type=float, help='optimizer weight decay 설정', default=5e-4)\n",
    "    # parser.add_argument('-gc', '--gradient_clip', type=float, help='gradient clip 설정. -1은 비활성화', default=0.1)\n",
    "    # parser.add_argument('-lsm', '--label_smoothing', type=float, help='label smoothing 설정', default=0)\n",
    "    # parser.add_argument('-es', '--early_stopping', type=int, help='ealry stoppin epoch 지정. -1은 비활성화', default=-1)\n",
    "    # parser.add_argument('-ad', '--adaptive', type=int, help=\"adaptive SAM 사용 여부\", default=1)\n",
    "    # parser.add_argument('--rho', type=int, help=\"SAM rho 파라미터\", default=2.0)\n",
    "    # parser.add_argument('-cm', '--cos_max', type=int, help=\"cos annealing 주기\", default=50)\n",
    "    # parser.add_argument('-sm', '--step_milestone', nargs='+', type=int, help='step lr scheduler milestone', default=[50])\n",
    "    \n",
    "    args = parser.parse_args('')\n",
    "    # args.p_shake = args.p_shake == 1\n",
    "    \n",
    "    args.test = 0\n",
    "    args.model_path = 'pyramid'\n",
    "    args.amp = 0\n",
    "    args.random_seed = RANDOMSEED\n",
    "    args.log_file = 0\n",
    "    args.port = 2033\n",
    "    args.model = ModelEnum.pyramidnet_custom\n",
    "    args.parallel = 0\n",
    "    args.optimizer = OptimizerEnum.sam\n",
    "    args.lr_scheduler = LRSchedulerEnum.step_lr\n",
    "    args.split_ratio = 0.2\n",
    "    args.augmentation_mode = 0\n",
    "    args.augment_split = 0.5\n",
    "    args.num_worker = 6\n",
    "    args.batch_size = 64\n",
    "    args.mix_step = 0\n",
    "    args.mix_method = MixEnum.cutmix\n",
    "    args.p_depth = 163\n",
    "    args.p_alpha = 84\n",
    "    args.p_shake = False\n",
    "    args.epoch = 120\n",
    "    args.max_learning_rate = 0.1\n",
    "    args.min_learning_rate = 0.001\n",
    "    args.weight_decay = 0.0001\n",
    "    args.gradient_clip = -1.0\n",
    "    args.label_smoothing = 0.1\n",
    "    args.early_stopping = -1\n",
    "    args.adaptive = 1\n",
    "    args.rho = 2\n",
    "    args.cos_max = 50\n",
    "    args.step_milestone = [40, 70, 100]\n",
    "    args.cut_p = 0.5\n",
    "    args.nesterov = True\n",
    "    \n",
    "    return args"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def init_logger(args):\n",
    "    logger = logging.getLogger(\"main\")\n",
    "    logger.setLevel(logging.DEBUG)\n",
    "\n",
    "    formatter = logging.Formatter(\"%(asctime)s %(levelname)s:%(message)s\")\n",
    "    handler = logging.StreamHandler()\n",
    "    handler.setFormatter(formatter)\n",
    "    logger.addHandler(handler)\n",
    "    \n",
    "    if args.log_file == 1:\n",
    "        log_save_path = \"./log\"\n",
    "        if not os.path.exists(log_save_path):\n",
    "            os.makedirs(log_save_path)\n",
    "\n",
    "        datetime_now = datetime.now().strftime(\"%Y%m%d_%H%M%S\")\n",
    "        formatter_file = logging.Formatter(\"%(asctime)s %(levelname)s:%(message)s\")\n",
    "        handler_file = logging.FileHandler(os.path.join(log_save_path, f'{datetime_now}.log'))\n",
    "        handler_file.setLevel(logging.DEBUG)\n",
    "        handler_file.setFormatter(formatter_file)\n",
    "        logger.addHandler(handler_file)\n",
    "\n",
    "    def catch_exception(exc_type, exc_value, exc_traceback):\n",
    "        if issubclass(exc_type, KeyboardInterrupt):\n",
    "            sys.__excepthook__(exc_type, exc_value, exc_traceback)\n",
    "            return\n",
    "\n",
    "        logger = logging.getLogger(\"main\")\n",
    "        \n",
    "        logger.error(\n",
    "            \"Unexpected exception.\",\n",
    "            exc_info=(exc_type, exc_value, exc_traceback)\n",
    "        )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def fix_random(args):\n",
    "    logger = logging.getLogger(\"main\")\n",
    "    \n",
    "    if args.random_seed == -1:\n",
    "        logger.debug('random seed not fix')\n",
    "        return\n",
    "    \n",
    "    seed = args.random_seed\n",
    "    deterministic = True\n",
    "    random.seed(seed)\n",
    "    np.random.seed(seed)\n",
    "    torch.manual_seed(seed)\n",
    "    torch.cuda.manual_seed_all(seed)\n",
    "    if 1:\n",
    "        torch.backends.cudnn.deterministic = True\n",
    "        torch.backends.cudnn.benchmark = False\n",
    "    logger.debug('random seed fix')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def setup_for_distributed(is_master):\n",
    "    import builtins as __builtin__\n",
    "    builtin_print = __builtin__.print\n",
    "\n",
    "    def print(*args, **kwargs):\n",
    "        force = kwargs.pop('force', False)\n",
    "        if is_master or force:\n",
    "            builtin_print(*args, **kwargs)\n",
    "\n",
    "    __builtin__.print = print"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def init_distributed_training(rank, opts):\n",
    "    opts.rank = rank\n",
    "    opts.gpu = opts.rank % torch.cuda.device_count()\n",
    "    local_gpu_id = int(opts.gpu_ids[opts.rank])\n",
    "    torch.cuda.set_device(local_gpu_id)\n",
    "    \n",
    "    if opts.rank is not None:\n",
    "        print(\"Use GPU: {} for training\".format(local_gpu_id))\n",
    "\n",
    "    torch.distributed.init_process_group(backend='nccl',\n",
    "                            init_method='tcp://127.0.0.1:' + str(opts.port),\n",
    "                            world_size=opts.ngpus_per_node,\n",
    "                            rank=opts.rank)\n",
    "\n",
    "    torch.distributed.barrier()\n",
    "    setup_for_distributed(opts.rank == 0)\n",
    "    print('opts :',opts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def main(rank, args):\n",
    "    if args.parallel == 1:\n",
    "        init_distributed_training(rank, args)\n",
    "    \n",
    "    \n",
    "    logger = logging.getLogger('main')\n",
    "    logger.debug(f'args: {vars(args)}')\n",
    "    \n",
    "    logger.debug(f'init data preprocessing')\n",
    "    \n",
    "    data_prep = DataPreProcessor(args.parallel, args.augmentation_mode)\n",
    "    \n",
    "    data_prep.transform_data()\n",
    "    data_prep.split_data(args.split_ratio, args.augment_split)\n",
    "    data_prep.get_data_loader(args.batch_size, args.num_worker)\n",
    "    \n",
    "    logger.debug(f'init model')\n",
    "    if args.model is ModelEnum.custom:\n",
    "        # Custom 모델 추가\n",
    "        model_pre = ModelPreProcessor(args, None)\n",
    "    else:\n",
    "        model_pre = ModelPreProcessor(args)\n",
    "    \n",
    "    logger.debug(f'init trainer')\n",
    "    trainer = Trainer(args, model_pre, data_prep)\n",
    "    \n",
    "    if args.test == 1:\n",
    "        logger.debug(f'test start')\n",
    "        trainer.model_save_path = os.path.join('.', 'model_dir', args.model_path)\n",
    "        trainer.get_result()\n",
    "    else:\n",
    "        logger.debug(f'train start')\n",
    "        trainer.train()\n",
    "        trainer.get_result()\n",
    "        trainer.save_history()\n",
    "\n",
    "    logger.debug(f'finish process')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "args = get_arg_parse()\n",
    "init_logger(args)\n",
    "fix_random(args)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "main(0, args)"
   ]
  }
 ],
 "metadata": {
  "language_info": {
   "name": "python"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}