08_27_dstc_cnn_댓다.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "oZzs_nhybnAH"
   },
   "source": [
    "# Subtask 1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "DsnkXy1UxxDD"
   },
   "source": [
    "## Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "id": "WVnQDBPLjSZO",
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "import numpy as np \n",
    "import copy #to copy data\n",
    "import json #data가 json 형식으로 제공됨\n",
    "\n",
    "import torch\n",
    "from torch.utils.data import Dataset #store sample and label\n",
    "\n",
    "from transformers import * "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "id": "RpRzP9llzCaJ"
   },
   "outputs": [],
   "source": [
    "#데이터셋 확인\n",
    "data_path=\"hi/e_train.json\"\n",
    "dialog_data = json.load(open(data_path, 'r', encoding='utf-8')) #데이터 경로 지정받아서 불러오기"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "id": "YWcjJFbcPkX3"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'bos_token': '[BOS]',\n",
       " 'eos_token': '[EOS]',\n",
       " 'additional_special_tokens': ['[speaker1]', '[speaker2]', '[IMG]', '[TAG]'],\n",
       " 'pad_token': '[PAD]'}"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# special_tokens: 토큰화 과정에서 문장이 분할되지 않도록 spial token을 지정해줌\n",
    "SPECIAL_TOKENS = ['[BOS]', '[EOS]', '[speaker1]', '[speaker2]', '[IMG]', '[TAG]', '[PAD]']\n",
    "SPECIAL_TOKENS_DICT = {'bos_token':'[BOS]', 'eos_token':'[EOS]', 'additional_special_tokens':['[speaker1]', '[speaker2]', '[IMG]', '[TAG]'], 'pad_token':'[PAD]'}\n",
    "SPECIAL_TOKENS_DICT\n",
    "#BOS: Begin of Sentence\n",
    "#EOS: End Of Sentence"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "id": "97D7fHia4ocg"
   },
   "outputs": [],
   "source": [
    "#객체를 받아 토큰화시키는 함수 : str 일때, dictionary일때, 그 외일때 나눠서 \n",
    "def tokenize(obj, tokenizer):\n",
    "    if isinstance(obj, str): #obj가 srt 형식인지 아닌지\n",
    "        return tokenizer.convert_tokens_to_ids(tokenizer.tokenize(obj))\n",
    "    if isinstance(obj, dict):\n",
    "        return dict((n, tokenize(o, tokenizer)) for n,o in obj.items())\n",
    "    return list(tokenize(o, tokenizer) for o in obj) \n",
    "# return=> token의 id 또는 List of id"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "id": "uEvJUuXAxitd"
   },
   "outputs": [],
   "source": [
    "# history와 answer data로 분리\n",
    "def get_data(tokenizer, data_path, meme_feature_path):\n",
    "    dialog_data = json.load(open(data_path, 'r', encoding='utf-8')) #데이터 경로 지정받아서 불러오기\n",
    "    dialog_list = [] \n",
    "    for idx in dialog_data.keys(): #dialog_data.keys()=> 대화 각각 하나 하나 \n",
    "        dialog = dialog_data[idx] #몇번째 대화\n",
    "        history = [] \n",
    "        \n",
    "        for i in range(len(dialog)): \n",
    "            if 'txt' in dialog[i].keys(): #각 대화에서 Txt가 있다면 토큰화해서 저장\n",
    "                dialog[i]['txt'] = tokenize(dialog[i]['txt'], tokenizer) \n",
    "            if i == 0: \n",
    "                history.append(dialog[i]) \n",
    "                continue \n",
    "            pair = {'history': copy.deepcopy(history), 'answer': copy.deepcopy(dialog[i])}  #copy.deepcopy: 내부객체들까지 모두 새롭게 copy\n",
    "            #history: 주고받은 대화들, answer: 마지막에 한 발화 \n",
    "            #짝 지어서 새로 저장\n",
    "            dialog_list.append(pair) \n",
    "            history.append(dialog[i]) \n",
    "        # break \n",
    "    id2feature = json.load(open(meme_feature_path, 'r', encoding='utf-8'))  #meme의 ID\n",
    "    return dialog_list, id2feature \n",
    "#dialog_list: history와 answer로 나누어진 대화 데이터 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "id": "4o1dUTT3xqwu"
   },
   "outputs": [],
   "source": [
    "# build input type from data: 데이터로에서 input 유형 구축  \n",
    "def build_input_from_segments(history, tokenizer, id2feature, answer=None): \n",
    "    bos, eos, speaker1, speaker2, img, tag =tokenizer.convert_tokens_to_ids(SPECIAL_TOKENS[:-1]) \n",
    "    #SPECIAL_TOKENS[:-1] = ['[BOS]', '[EOS]', '[speaker1]', '[speaker2]', '[IMG]', '[TAG]']\n",
    "    #PAD는 벡터 길이를 맞춰주기 위함이므로 굳이 id로 변환해주지 않아도 됨\n",
    "\n",
    "\n",
    "    #history data는 현재 img_id와 emotion_id가 둘다 포함되어 있음: text와 meme이 함께 있는 상태 \n",
    "    #-> 이를 분리해서 따로 넣어주자\n",
    "    history_txt = [] #history 중 text 부분\n",
    "    history_img = [] #history 중 image 부분\n",
    "    labels = [] #bos eos 토큰 말고 추가적으로 나오는 토큰들의 value: IMG, TAG의 특정값ㅠ\n",
    "    token_type_ids = [] #토큰화 유형 아이디\n",
    "    \n",
    "    # to avoid out of length, cut the partial sequence\n",
    "    ans_len = 4 \n",
    "    \n",
    "    #대답이 존재하는데 그 대답에 text가 존재할 경우\n",
    "    if answer is not None and 'txt' in answer.keys():\n",
    "        ans_len += len(answer['txt']) #answer 중 텍스트 부분의 길이 누적합\n",
    "    \n",
    "    for i in range(len(history)-1, -1, -1): \n",
    "        # 길이 재기\n",
    "        cur_len = 4 #초기값 4로 설정해둠(current length)\n",
    "        if 'txt' in history[i].keys():\n",
    "            cur_len += len(history[i]['txt']) #history 중 텍스트 부분의 길이 누적합\n",
    "        if len(token_type_ids) + ans_len + cur_len > 500: \n",
    "            break #특정 길이를 넘어가면 멈추기\n",
    "            \n",
    "        #speaker id: 1,2만 존재하도록   \n",
    "        if history[i]['speaker_id'] == '[speaker1]': \n",
    "            speaker_id = speaker1 \n",
    "        else:\n",
    "            speaker_id = speaker2 \n",
    "         \n",
    "        #history에 img있는 경우: history img로 저장\n",
    "        if 'img_id' in history[i].keys(): \n",
    "            history_img = [id2feature[history[i]['img_id']]] + history_img\n",
    "            token_type_ids = [img] + token_type_ids \n",
    "            labels = [-100] + labels \n",
    "        \n",
    "        #history에 txt있는 경우: history text로 저장\n",
    "        if 'txt' in history[i].keys(): \n",
    "            content = [bos] + history[i]['txt'] + [eos] #문장과 시작과 끝 포함\n",
    "            history_txt = content + history_txt \n",
    "            token_type_ids = [speaker_id] * len(content) + token_type_ids \n",
    "            labels = [-100] * len(content) + labels \n",
    "        else: \n",
    "            content = [bos] + [eos] \n",
    "            history_txt = content + history_txt \n",
    "            token_type_ids = [speaker_id] * len(content) + token_type_ids \n",
    "            labels = [-100] * len(content) + labels \n",
    "    \n",
    "        history_txt = [speaker_id] + history_txt \n",
    "        token_type_ids = [speaker_id] + token_type_ids \n",
    "        labels = [-100] + labels \n",
    "    \n",
    "    #대답이 존재하는 경우 \n",
    "    if answer is not None: \n",
    "        #발화자 지정\n",
    "        if answer['speaker_id'] == '[speaker1]': \n",
    "            speaker_id = speaker1 \n",
    "        else:\n",
    "            speaker_id = speaker2 \n",
    "    \n",
    "        history_txt += [speaker_id] \n",
    "        token_type_ids += [speaker_id]\n",
    "        \n",
    "        if 'txt' in answer.keys(): \n",
    "            #첫시작하는 토큰 + 대답에서 text 부분 + 끝내는 토큰\n",
    "            content = [bos] + answer['txt'] + [eos] \n",
    "            history_txt += content \n",
    "            token_type_ids += [speaker_id] * len(content) \n",
    "            labels += content \n",
    "        else: \n",
    "            content = [bos] + [eos] \n",
    "            history_txt += content \n",
    "            token_type_ids += [speaker_id] * len(content) \n",
    "            labels += content \n",
    "    \n",
    "        labels += [-100, -100] \n",
    "        history_txt += [tag] \n",
    "        token_type_ids += [img] \n",
    "        \n",
    "        #meme_flag: 0 또는 1로 구성 \n",
    "        if 'img_id' in answer.keys(): \n",
    "            history_img += [id2feature[answer['img_id']]] \n",
    "            meme_flag = [1]\n",
    "        else:\n",
    "            history_img += [[0.0]*512] \n",
    "            meme_flag = [0] \n",
    "    return history_txt, history_img, token_type_ids, labels[1:], meme_flag #output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "id": "yJc7Z8GlxmMg"
   },
   "outputs": [],
   "source": [
    "class MODDataset(Dataset): \n",
    "    def __init__(self, dialogs, id2feature, tokenizer): \n",
    "        self.dialogs = dialogs \n",
    "        self.id2feature = id2feature \n",
    "        self.tokenizer = tokenizer \n",
    "        #필요한 변수 선언: dialogs, id2feature, tokenizer=> 대화(text), 밈, 토큰화하기 위한 tokenizer\n",
    "    \n",
    "    def __len__(self):\n",
    "        return len(self.dialogs) \n",
    "        #데이터셋의 샘플 개수 반환: len 함수 \n",
    "    \n",
    "    def __getitem__(self, index): \n",
    "      #__getitem__: 주어진 idxex 에 해당하는 샘플을 데이터셋에서 불러오고 반환해줌\n",
    "    \n",
    "      #get data 함수에서 history 와 answer로 나눴었음 \n",
    "        his = copy.deepcopy(self.dialogs[index]['history'])  #history data\n",
    "        ans = copy.deepcopy(self.dialogs[index]['answer'])  #answer data\n",
    "        # print(his)\n",
    "        # print(ans)\n",
    "        \n",
    "        #tensor 형태로 변환: 각 형태에 맞춰\n",
    "        history_txt, histroy_img, token_type_ids, labels, meme_flag = build_input_from_segments(his, self.tokenizer, self.id2feature, ans) \n",
    "        history_txt = torch.LongTensor(history_txt) #64 bit integer\n",
    "        histroy_img = torch.from_numpy(np.array(histroy_img)).float() \n",
    "        #numpy로 변경 후 float 형식으로 변경 \n",
    "        token_type_ids = torch.Tensor(token_type_ids).long()\n",
    "        labels = torch.Tensor(labels).long()\n",
    "        meme_flag = torch.Tensor(meme_flag).long() \n",
    "        #torch.Tensor:tensor 값 변경하더라도 numpy 변화 x\n",
    "        #torch.from_numpy:array의 dtype을 상속받고 tensor와 메모리 버퍼를 공유하기 때문에 tensor의 값이 변경되면 Numpy array값이 변경\n",
    "        return history_txt, histroy_img, token_type_ids, labels, meme_flag #Output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "id": "9ZyCXqQnxuqY",
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['[speaker1]', '[BOS]', 'H', 'aha', '.', 'Ġ.', 'Ġ.', 'ĠDon', \"'t\", 'Ġknow', 'Ġwho', 'Ġi', 'Ġam', '[EOS]', '[speaker2]', '[BOS]', 'O', 'uy', 'ang', '?', '[EOS]', '[TAG]']\n",
      "torch.Size([2, 512])\n",
      "['[speaker1]', '[speaker1]', '[speaker1]', '[speaker1]', '[speaker1]', '[speaker1]', '[speaker1]', '[speaker1]', '[speaker1]', '[speaker1]', '[speaker1]', '[speaker1]', '[speaker1]', '[speaker1]', '[IMG]', '[speaker2]', '[speaker2]', '[speaker2]', '[speaker2]', '[speaker2]', '[speaker2]', '[speaker2]', '[IMG]']\n",
      "[None, None, None, None, None, None, None, None, None, None, None, None, None, None, '[BOS]', 'O', 'uy', 'ang', '?', '[EOS]', None, None]\n"
     ]
    }
   ],
   "source": [
    "if __name__ == '__main__': \n",
    "    data_path = 'hi/e_train.json' \n",
    "    meme_feature_path = 'hi/id2feature.json'\n",
    "    tokenizer = GPT2Tokenizer.from_pretrained('gpt2', do_lower_case=True)\n",
    "    #공백도 취급: 공백여부에 따라도 다르게 토큰화됨\n",
    "    tokenizer.add_special_tokens(SPECIAL_TOKENS_DICT)\n",
    "    dialog_list, id2feature = get_data(tokenizer, data_path, meme_feature_path) \n",
    "    dataset = MODDataset(dialog_list, id2feature, tokenizer) \n",
    "    history_txt, history_img, token_type_ids, labels, meme_flag = dataset[0]\n",
    "    print(tokenizer.convert_ids_to_tokens(history_txt))\n",
    "    print(history_img.size())\n",
    "    print(tokenizer.convert_ids_to_tokens(token_type_ids))\n",
    "    print(tokenizer.convert_ids_to_tokens(labels))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(tensor([50259, 50257,    39, 12236,    13,   764,   764,  2094,   470,   760,\n",
       "           508,  1312,   716, 50258, 50260, 50257,    46,  4669,   648,    30,\n",
       "         50258, 50262]),\n",
       " tensor([[ 0.2078, -0.0766,  0.4716,  ...,  0.0949,  0.1990, -0.2000],\n",
       "         [ 0.0000,  0.0000,  0.0000,  ...,  0.0000,  0.0000,  0.0000]]),\n",
       " tensor([50259, 50259, 50259, 50259, 50259, 50259, 50259, 50259, 50259, 50259,\n",
       "         50259, 50259, 50259, 50259, 50261, 50260, 50260, 50260, 50260, 50260,\n",
       "         50260, 50260, 50261]),\n",
       " tensor([ -100,  -100,  -100,  -100,  -100,  -100,  -100,  -100,  -100,  -100,\n",
       "          -100,  -100,  -100,  -100, 50257,    46,  4669,   648,    30, 50258,\n",
       "          -100,  -100]),\n",
       " tensor([0]))"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "oz1rH_AGEszo"
   },
   "source": [
    "Construct a GPT-2 tokenizer. Based on byte-level Byte-Pair-Encoding.\n",
    "Byte Pair Encoding (BPE)\n",
    "gpt-2는 Byte Pair Encoding를 거친 토큰을 입력 단위로 사용합니다.\n",
    "\n",
    "BPE는 서브워드를 분리하는 알고리즘으로, 빈도수에 따라 문자를 병합하여 서브워드를 구성합니다. 단어를 문자(char) 단위로 쪼갠 뒤, 가장 빈도수가 높은 쌍을 하나로 통합하는 과정을 반복하여 토큰 딕셔너리를 만듭니다.\n",
    "\n",
    "앞으로 단어, 토큰이라고 불리는 것은 모두 BPE token을 의미합니다.\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "LGgeZ7LH-8kY"
   },
   "source": [
    "## Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "id": "8tfahfTQ-zg9"
   },
   "outputs": [],
   "source": [
    "import torch \n",
    "from torch import nn \n",
    "from torch.nn import CrossEntropyLoss, MSELoss\n",
    "import os \n",
    "from transformers import * "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "_QzVmoSOHu4-"
   },
   "source": [
    "class 형태의 모델은 항상 nn.Module 을 상속받아야 하며, super(모델명, self).__init__() 을 통해 nn.Module.__init__() 을 실행시키는 코드가 필요합니다.\n",
    "forward() 는 모델이 학습데이터를 입력받아서 forward propagation을 진행시키는 함수이고, 반드시 forward 라는 이름의 함수이어야 합니다.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {
    "id": "SKpLH8cPoyEH"
   },
   "outputs": [],
   "source": [
    "class MemeDialoGPT(GPT2PreTrainedModel): \n",
    "    def __init__(self, config): \n",
    "        #super: 자식클래스에서 부모클래스의 내용을 사용하고 싶은 경우, super().부모클래스내용, 여기서 부모클래스는 MODDataset\n",
    "        #MODDataset에서 지정한 변수들 그대로 사용\n",
    "        super(MemeDialoGPT, self).__init__(config) \n",
    "        self.transformer = GPT2Model(config) #GPT2 모델을 편의상 transformer로 정의 \n",
    "        \n",
    "        \n",
    "        ## for text hidden state\n",
    "        self.lm_head = nn.Linear(config.n_embd, config.vocab_size, bias=False) \n",
    "        # n_embd: Dimensionality of the embeddings and hidden states\n",
    "        # vocab_size:the number of different tokens that can be represented by the inputs_ids passed when calling GPT2Model or TFGPT2Model.\n",
    "\n",
    "        ## for image hidden state # E JARIE CNN GANUNG HAL DUT\n",
    "        #self.img_ff = nn.Linear(512, config.n_embd)  \n",
    "        #input: 512개의 픽셀 사용하는 이미지/ output: 임베딩 결과와 hidden state의 차원\n",
    "        #self.img_ff = nn.Conv2d(2, 512, kernel_size=1)\n",
    "        #self.img_ff = nn.Conv2d(2, 512, kernel_size=1)\n",
    "        self.img_ff = nn.Conv2d(512, config.n_embd, kernel_size=1)\n",
    "        #self.img_ff2 = nn.Linear()\n",
    "        #, kernel_size=(3, 3), stride=(2, 2), padding=(2, 2)\n",
    "        #self.img_inverse_ff = nn.Conv2d(config.n_embd, 512, kernel_size=1)\n",
    "        #self.img_inverse_ff = nn.Conv2d(512, 512, kernel_size=1)\n",
    "        #, kernel_size=(3, 3), stride=(2, 2), padding=(2, 2)\n",
    "        self.img_inverse_ff = nn.Linear(config.n_embd, 512)\n",
    "        #config.n_embd = 768\n",
    "        \n",
    "        #img_ff와 input, output 반대\n",
    "        #GPT2모델은 masked-self attention 활용하는 자기 회귀 모델임: inverse 함수 필요\n",
    "        \n",
    "        # predict the meme usage \n",
    "        #사용 여부에 따라 0,1\n",
    "        self.lm_flag = nn.Linear(config.n_embd, 2)\n",
    "\n",
    "    def tie_weights(self):  #Tie the weights between the input embeddings and the output embeddings: 각각의 임베딩 결과물 벡터에는 가중치가 부여되어 있는데 input의 w와 output의 w 묶음.  \n",
    "        self._tie_or_clone_weights(self.lm_head, self.transformer.wte) \n",
    "    \n",
    "    def forward(self, input_embeds, token_type_ids, labels=None, img_feature=None, meme_flag=None): \n",
    "      #모델이 학습데이터를 입력받아서 forward propagation을 진행시키는 함수\n",
    "      #input_embeds: input_construct 함수의 return 값\n",
    "        #GPT2 모델에 임베딩한 input들을 넣음\n",
    "        transformer_outputs = self.transformer(inputs_embeds=input_embeds, token_type_ids=token_type_ids) \n",
    "        \n",
    "        #output값 중 일부를 hidden state로 지정해서 정보를 저장하고 넘김\n",
    "        #Sequence of hidden-states at the output of the last layer of the model\n",
    "        #text랑 image가 담는 정보가 다를 수밖에 없으므로 따로 저장\n",
    "        hidden_states = transformer_outputs[0]\n",
    "        txt_hidden_states, img_hidden_states = hidden_states[:-1, :], hidden_states[-1, :].unsqueeze(0) \n",
    "        print(txt_hidden_states.size())\n",
    "        print(img_hidden_states.size())\n",
    "        lm_logits = self.lm_head(txt_hidden_states) #hidden state를 어휘에 대한 확률 분포로 변환\n",
    "        img_regs = self.img_inverse_ff(img_hidden_states) #픽셀마다 명암에 대한 밀도값 추정: 픽셀값 추정/ regression\n",
    "        print(lm_logits.size())\n",
    "        print(img_regs.size())\n",
    "        \n",
    "        outputs = (lm_logits,) + (img_regs, ) \n",
    "        \n",
    "        #섭테 1,2,3 모두 활용 가능 \n",
    "        if labels is not None:  #answer 존재\n",
    "            txt_loss_fct = CrossEntropyLoss(ignore_index=-100) \n",
    "            loss = txt_loss_fct(lm_logits, labels)  \n",
    "            #loss=CrossEntropyLoss(lm_logits, labels,ignore_index=-100) \n",
    "\n",
    "            #CrossEntropyLoss: 분류 문제처럼 확률값으로 나올때. meme이 있는 경우 쓸지 안쓸지 확률로 고려해야하기 때문에 사용해줌\n",
    "\n",
    "\n",
    "            if meme_flag is not None: \n",
    "              #meme_flag가 존재:  answer에 meme 있는지 여부에 따라 0,1 나눴었음\n",
    "                mf_logits = self.lm_flag(img_hidden_states)  #0,1 : binary classification\n",
    "                mf_loss_fct = CrossEntropyLoss()\n",
    "                mf_flag_loss = mf_loss_fct(mf_logits, meme_flag) \n",
    "#                 mf_flag_loss = CrossEntropyLoss(mf_logits, meme_flag) \n",
    "                loss += mf_flag_loss #위에 만든 loss에 더해짐\n",
    "                outputs = (mf_logits,) + outputs\n",
    "\n",
    "            if img_feature[0][0] != 0.:  #적잘한 Meme을 잘 썼는지 평가: 픽셀값 예측통해\n",
    "                img_loss_fct = MSELoss() \n",
    "                loss += img_loss_fct(img_regs, img_feature) \n",
    "#                 meme 없는 경우: \n",
    "#                 loss += MSELoss(img_regs, img_feature) \n",
    "                #img_feature: def img_feature_read\n",
    "            outputs = (loss,) + outputs \n",
    "        return outputs   "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "-1yaYUOAoh54"
   },
   "source": [
    "손실 함수란 신경망이 학습할 수 있도록 해주는 지표이다. 머신러닝 모델의 출력값과 사용자가 원하는 출력값의 차이, 즉 오차를 말한다. 이 손실 함수 값이 최소화되도록 하는 가중치와 편향을 찾는 것이 바로 학습이다. 일반적인 손실 함수로 평균 제곱 오차나 교차 엔트로피 오차를 사용"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "4h0CdV82_PC4"
   },
   "source": [
    "## train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {
    "id": "4r3unRo8LOiV",
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# #각각 다른 파일에 있을 경우에는 class를 다시 불러와줘야함\n",
    "# from model import MemeDialoGPT \n",
    "# from dataset import MODDataset, get_data \n",
    "# from utils import accuracy_compute, AverageMeter, meme_classify_accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "metadata": {
    "id": "PJVqY8wGuh8z"
   },
   "outputs": [],
   "source": [
    "SPECIAL_TOKENS = ['[BOS]', '[EOS]', '[speaker1]', '[speaker2]', '[IMG]', '[TAG]', '[PAD]']\n",
    "SPECIAL_TOKENS_DICT = {'bos_token':'[BOS]', 'eos_token':'[EOS]', 'additional_special_tokens':['[speaker1]', '[speaker2]', '[IMG]', '[TAG]'], 'pad_token':'[PAD]'}\n",
    "\n",
    "# data parameters\n",
    "train_data_path = 'hi/e_train.json'\n",
    "val_data_path = 'hi/e_validation.json' \n",
    "feature_path = 'hi/id2feature.json'\n",
    "\n",
    "\n",
    "# model parameters\n",
    "use_cuda = torch.cuda.is_available() \n",
    "device = torch.device('cuda' if use_cuda else 'cpu') \n",
    "model_path = 'hi'\n",
    "gpt_path = 'gpt2' \n",
    "ckpt_usage = False\n",
    "lr = 6e-5\n",
    "epochs = 1\n",
    "gradient_accumulation_steps = 1\n",
    "print_freq = 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "metadata": {
    "id": "gPTgQUuIuvKj"
   },
   "outputs": [],
   "source": [
    " # concatenate the input \n",
    "def input_construct(history_txt_embs, history_img_embs, token_type_ids, tokenizer): \n",
    "        ## in train fuction:이미지와 텍스트 따로 임베딩해줬음\n",
    "        #history_txt_embs = model.transformer.wte(history_txt) \n",
    "        #history_img_embs = model.img_ff(history_img) \n",
    "    bos, eos, speaker1, speaker2, img, tag = tokenizer.convert_tokens_to_ids(SPECIAL_TOKENS[:-1]) \n",
    "    emb_length = token_type_ids.size(-1) \n",
    "    emb_dim = history_txt_embs.size(-1)\n",
    "    img_num = history_img_embs.size(0) \n",
    "\n",
    "    #0으로만 이루어진 텐서하나 만들고 임베딩할 길이와 차원\n",
    "    input_embs = torch.zeros((emb_length, emb_dim)).to(device)\n",
    "\n",
    "    #임베딩 결과를 정리\n",
    "    #순서가 현재 meme\n",
    "    txt_idx = 0 \n",
    "    img_idx = 0 \n",
    "    left_idx  = 0 \n",
    "    right_idx = 0 \n",
    "    while right_idx < emb_length: \n",
    "        #if right_idx == emb_length-1 and token_type_ids[right_idx] == img: \n",
    "        #    break \n",
    "\n",
    "        #right index가 embeding length까지 1씩 추가되면서 반복문 돌던 중 meme이 나오면\n",
    "        #거기까지 text, 그 이후 meme\n",
    "        if right_idx < emb_length-1 and token_type_ids[right_idx] == img:\n",
    "            txt_length = right_idx - left_idx \n",
    "            input_embs[left_idx:right_idx, :] = history_txt_embs[txt_idx:txt_idx+txt_length, :] \n",
    "            txt_idx += txt_length \n",
    "            input_embs[right_idx,:] = history_img_embs[img_idx, :] \n",
    "            img_idx += 1\n",
    "            left_idx = right_idx + 1 \n",
    "        right_idx += 1\n",
    "    txt_length = right_idx - left_idx \n",
    "    if txt_length > 0: \n",
    "        input_embs[left_idx:right_idx, :] = history_txt_embs[txt_idx:, :]\n",
    "    # img_feature = history_img_embs[img_idx,:] \n",
    "    return input_embs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "metadata": {
    "id": "myzgwtWc_M8m"
   },
   "outputs": [],
   "source": [
    "#meme 이미지 불러옴\n",
    "def img_feature_read(feature_path): \n",
    "    with open(feature_path, 'r', encoding='utf-8') as f: \n",
    "        id2feature_dict = json.load(f) \n",
    "    img_features = [] \n",
    "    for id in id2feature_dict.keys(): #key: 이미지 번호\n",
    "        img_features.append(id2feature_dict[id]) \n",
    "        #제공해준 meme 모음 파일과 비교하면서 대화 속 meme list 저장\n",
    "    img_features = np.array(img_features) \n",
    "    img_features = torch.from_numpy(img_features).float().to(device)\n",
    "    #tensor 형태로 저장\n",
    "    print(img_features)\n",
    "    return img_features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "metadata": {
    "id": "RsdQliJ8u1QO"
   },
   "outputs": [],
   "source": [
    "def meme_retrieval_compute(cur_img_feature, target_img_feature, cat_img_features): \n",
    "    # (1, 512)\n",
    "    #현재 이미지 피쳐와 타겟 이미지 피쳐 간 거리 계산\n",
    "    cur_dist = torch.dist(cur_img_feature, target_img_feature, p=2)\n",
    "    # print(cat_img_features.size())\n",
    "    #307개의 이미지\n",
    "    cur_img_list = cur_img_feature.repeat(307,1) #img_regs\n",
    "    #오차의 제곱합의 루트: rmse\n",
    "    total_dist = torch.sqrt(torch.sum((cur_img_list - cat_img_features)**2, dim=1))\n",
    "    # print(total_dist) \n",
    "    sorted_total, _ = torch.sort(total_dist) \n",
    "    # print(sorted_total) \n",
    "    return torch.gt(sorted_total[90],cur_dist)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "metadata": {
    "id": "tGQ8ywE_uqEL"
   },
   "outputs": [],
   "source": [
    "def train(model, tokenizer, optimizer, dataset, epoch): \n",
    "    model.train() \n",
    "    cat_img_features = img_feature_read(feature_path) \n",
    "    avg_loss = AverageMeter() \n",
    "    avg_acc = AverageMeter() \n",
    "    iteration = 1\n",
    "    meme_correct_num = 0 \n",
    "    meme_total_num = 0\n",
    "\n",
    "    for instance in dataset: \n",
    "        history_txt, history_img, token_type_ids, labels, meme_flag = instance \n",
    "        history_txt, history_img, token_type_ids, labels, meme_flag = history_txt.to(device).squeeze(0), history_img.to(device).squeeze(0), \\\n",
    "                                                                        token_type_ids.to(device).squeeze(0), labels.to(device).squeeze(0), meme_flag.to(device).squeeze(0)   \n",
    "        history_txt_embs = model.transformer.wte(history_txt) \n",
    "        #print(history_txt_embs.size()\n",
    "        print(history_img.size())\n",
    "        #history_img1 = history_img.transpose(0, 1)\n",
    "        #print(history_img1.size())\n",
    "        history_img2 = history_img.unsqueeze(-1)\n",
    "        print(history_img2.size())\n",
    "        history_img3 = history_img2.unsqueeze(-1)\n",
    "        print(history_img3.size())\n",
    "        history_img_embs = model.img_ff(history_img3)\n",
    "        print(history_img_embs.size())\n",
    "        #history_img_embs = history_img_embs.view(history_img_embs.shape[0], -1)\n",
    "        history_img_embs = history_img_embs.view(2, -1)\n",
    "        print(history_img_embs.size()) \n",
    "        #print(token_type_ids) \n",
    "        #print(history_txt)\n",
    "        input_embs = input_construct(history_txt_embs, history_img_embs, token_type_ids, tokenizer) \n",
    "        input_embs = input_embs.to(device) \n",
    "        img_feature = history_img[-1, :].unsqueeze(0)\n",
    "        # print(input_embs.size()) \n",
    "        # print(img_feature.size()) \n",
    "        loss, mf_logits, lm_logits, cur_img_feature = model(input_embs, token_type_ids, labels, img_feature, meme_flag) \n",
    "        \n",
    "        loss.backward()\n",
    "        torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0) \n",
    "\n",
    "        if iteration % gradient_accumulation_steps == 0:\n",
    "            optimizer.step()\n",
    "            optimizer.zero_grad()\n",
    "            #meme이 있는 경우 적절하게 사용되었지만 파악함\n",
    "        if img_feature[0][0] != 0.: \n",
    "            if meme_retrieval_compute(cur_img_feature, img_feature, cat_img_features):\n",
    "                meme_correct_num += 1 \n",
    "            meme_total_num += 1 \n",
    "        acc = accuracy_compute(lm_logits, labels, 5)\n",
    "        #잘 분류되었는지 체크\n",
    "        #acc = meme_classify_accuracy(mf_logits, meme_flag).item()\n",
    "        avg_acc.update(acc)\n",
    "        \n",
    "        avg_loss.update(loss.item())\n",
    "        \n",
    "        # print status \n",
    "#        if iteration % print_freq == 0:\n",
    "#             print('Epoch:[{0}][{1}/{2}]\\t'\n",
    "#             'Loss {loss.val:.4f} ({loss.avg:.4f})\\t'\n",
    "#             'Classify Acc {acc.val:.3f} ({acc.avg:.3f})\\t'\n",
    "#             'Meme Acc {mac:.3f}'.format(epoch, iteration, len(dataset),loss=avg_loss, acc=avg_acc, mac=float(meme_correct_num/meme_total_num)))\n",
    "        if iteration % print_freq == 0:\n",
    "            print('Epoch:[{0}][{1}/{2}]\\t'\n",
    "            'Loss {loss.val:.4f} ({loss.avg:.4f})\\t'\n",
    "            'Classify Acc {acc.val:.3f} ({acc.avg:.3f})\\t'.format(epoch, iteration, len(dataset),loss=avg_loss, acc=avg_acc))\n",
    "        print(lm_logits)\n",
    "        print(labels)\n",
    "        print(acc)\n",
    "        print(lm_logits.size())\n",
    "        print(labels.size())\n",
    "        \n",
    "        \n",
    "        iteration += 1\n",
    "        break\n",
    "    return avg_loss.avg\n",
    "        # print(loss)\n",
    "        # break "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "metadata": {
    "id": "jAc9x1C9uyAI"
   },
   "outputs": [],
   "source": [
    "def validate(model, tokenizer, dataset, epoch): \n",
    "    \n",
    "    model.eval() \n",
    "    avg_loss = AverageMeter() \n",
    "    avg_acc = AverageMeter() \n",
    "    avg_bleu = AverageMeter() \n",
    "    iteration = 1 \n",
    "    cat_img_features = img_feature_read(feature_path) \n",
    "    meme_correct_num = 0 \n",
    "    meme_total_num = 0\n",
    "\n",
    "    with torch.no_grad(): \n",
    "        for instance in dataset: \n",
    "            history_txt, history_img, token_type_ids, labels, meme_flag = instance \n",
    "            history_txt, history_img, token_type_ids, labels, meme_flag  = history_txt.to(device).squeeze(0), history_img.to(device).squeeze(0), \\\n",
    "                                                                            token_type_ids.to(device).squeeze(0), labels.to(device).squeeze(0), meme_flag.to(device).squeeze(0) \n",
    "            history_txt_embs = model.transformer.wte(history_txt)\n",
    "            #history_img1 = history_img.transpose(0, 1)\n",
    "            #print(history_img1.size())\n",
    "            history_img2 = history_img.unsqueeze(-1)\n",
    "            print(history_img2.size())\n",
    "            history_img3 = history_img2.unsqueeze(-1)\n",
    "            print(history_img3.size())\n",
    "            history_img_embs = model.img_ff(history_img3)\n",
    "            print(history_img_embs.size())\n",
    "        #history_img_embs = history_img_embs.view(history_img_embs.shape[0], -1)\n",
    "            history_img_embs = history_img_embs.view(2, -1)\n",
    "            #print(history_img_embs.size())\n",
    "            #history_img_embs = model.img_ff2(history_img_embs)\n",
    "            #print(history_img_embs.size())\n",
    "            #history_img_embs = model.img_ff_lin\n",
    "            \n",
    "            \n",
    "            #history_img_embs = model.img_ff(history_img) \n",
    "            \n",
    "            input_embs = input_construct(history_txt_embs, history_img_embs, token_type_ids, tokenizer) \n",
    "            input_embs = input_embs.to(device) \n",
    "            if input_embs.size(-2) > 450:\n",
    "                continue\n",
    "            img_feature = history_img[-1, :].unsqueeze(0) \n",
    "            loss, mf_logits, lm_logits, cur_img_feature = model(input_embs, token_type_ids, labels, img_feature, meme_flag) \n",
    "            # compare cur_img_feature is among topk with img_feature \n",
    "            # print(cur_img_feature.size())   (1, 512) \n",
    "            if img_feature[0][0] != 0.: \n",
    "                if meme_retrieval_compute(cur_img_feature, img_feature, cat_img_features):\n",
    "                    meme_correct_num += 1 \n",
    "                meme_total_num += 1 \n",
    "            acc = accuracy_compute(lm_logits, labels, k=5) \n",
    "            #acc = meme_classify_accuracy(mf_logits, meme_flag).item()\n",
    "            avg_acc.update(acc) \n",
    "            avg_loss.update(loss.item()) \n",
    "            if iteration % print_freq == 0:\n",
    "                print('Epoch:[{0}][{1}/{2}]\\t'\n",
    "                      'Loss {loss.val:.4f} ({loss.avg:.4f})\\t'\n",
    "                      'Classify Acc {acc.val:.3f} ({acc.avg:.3f})\\t'.format(epoch, iteration, len(dataset),loss=avg_loss, acc=avg_acc))\n",
    "\n",
    "#                 print('Epoch:[{0}][{1}/{2}]\\t'\n",
    "#                 'Loss {loss.val:.4f} ({loss.avg:.4f})\\t'\n",
    "#                 'Acc {acc.val:.3f} ({acc.avg:.3f})\\t'\n",
    "#                 'Meme Acc {mac:.3f}'.format(epoch, iteration, len(dataset),loss=avg_loss, acc=avg_acc, mac=float(meme_correct_num/meme_total_num))) \n",
    "        \n",
    "            iteration += 1 \n",
    "            break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "metadata": {
    "id": "qyrtHRDCui3N"
   },
   "outputs": [],
   "source": [
    "#메인 함수 \n",
    "def main(): \n",
    "    \n",
    "    # model initialize  #모델  초기화\n",
    "    if ckpt_usage == True: \n",
    "        ckpt_path = 'gpt2' \n",
    "         \n",
    "\n",
    "    else:\n",
    "        tokenizer = GPT2Tokenizer.from_pretrained(gpt_path, do_lower_case=True) #gpt 모델 사용 \n",
    "        model = MemeDialoGPT.from_pretrained(gpt_path)\n",
    "        tokenizer.add_special_tokens(SPECIAL_TOKENS_DICT) \n",
    "        model.resize_token_embeddings(len(tokenizer))\n",
    "    model = model.to(device) \n",
    "    optimizer = AdamW(model.parameters(), lr=lr)\n",
    "\n",
    "    # data read : train, validation\n",
    "    #데이터 불러와서 로드까지!\n",
    "    train_dialogs, id2feature = get_data(tokenizer, train_data_path, feature_path) #여기서 왜 자꾸 오류? : 불러오는데 문제 있?\n",
    "    val_dialogs, _ = get_data(tokenizer, val_data_path, feature_path) \n",
    "    #print(len(train_dialogs))\n",
    "    train_dataset = MODDataset(train_dialogs, id2feature, tokenizer) \n",
    "    val_dataset = MODDataset(val_dialogs, id2feature, tokenizer) \n",
    "    # print(len(train_dataset))\n",
    "    train_loader = DataLoader(train_dataset, batch_size=1, num_workers=4, pin_memory=True)\n",
    "    val_loader = DataLoader(val_dataset, batch_size=1, num_workers=4, pin_memory=True) \n",
    "    \n",
    "    for epoch in range(epochs): \n",
    "        \n",
    "        # one epoch's training\n",
    "        val_loss = train(model=model, tokenizer=tokenizer, optimizer=optimizer, dataset=train_loader, epoch=epoch) \n",
    "        \n",
    "        # one epoch's validation \n",
    "        validate(model=model, tokenizer=tokenizer, dataset=val_loader, epoch=epoch)\n",
    "        \n",
    "        #break 트라이1\n",
    "        # save checkpoint \n",
    "        torch.save({'model':model.state_dict(), 'optimizer': optimizer.state_dict()},\\\n",
    "            '%s/epoch_%d_loss_%.3f'%(model_path, epoch, val_loss))\n",
    "        model.config.to_json_file(os.path.join(model_path, 'config.json'))\n",
    "        tokenizer.save_vocabulary(model_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "metadata": {
    "id": "qBo4umO_bnAU"
   },
   "outputs": [],
   "source": [
    "from transformers import GPT2Tokenizer, GPT2Model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "MHrU0s05_UJH"
   },
   "source": [
    "## utils"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "metadata": {
    "id": "BKIsgwldbnAV"
   },
   "outputs": [],
   "source": [
    "import torch \n",
    "\n",
    "# calculate the accuracy of response  \n",
    "def accuracy_compute(lm_logits, targets, k=5):\n",
    "    _, idx = torch.topk(lm_logits, k, 1)\n",
    "    correct = idx.eq(targets.view(-1,1).expand_as(idx))\n",
    "    correct_total = correct.view(-1).float().sum().item()\n",
    "    nums = targets.view(-1).detach().cpu().numpy()\n",
    "    length = 0\n",
    "    for num in nums:\n",
    "        if num != -100:\n",
    "            length += 1\n",
    "            \n",
    "    return correct_total / float(length)\n",
    "\n",
    "#지금 correct_total이 [x, y] 식으로 출력되고 length"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 137,
   "metadata": {
    "id": "a7F8-szebnAV"
   },
   "outputs": [],
   "source": [
    "def meme_classify_accuracy(mf_logits, meme_flag):\n",
    "    prediction = torch.argmax(mf_logits, 1) \n",
    "    return (prediction == meme_flag).sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 138,
   "metadata": {
    "id": "8Z3gJA80_T1_"
   },
   "outputs": [],
   "source": [
    "# class for evaluation metric \n",
    "class AverageMeter(object):\n",
    "    def __init__(self):\n",
    "        self.reset()\n",
    "    \n",
    "    def reset(self):\n",
    "        self.val = 0.0\n",
    "        self.avg = 0.0\n",
    "        self.sum = 0.0\n",
    "        self.count = 0\n",
    "    \n",
    "    def update(self, val, n=1):\n",
    "        self.val = val\n",
    "        self.sum += val * n\n",
    "        self.count += n\n",
    "        self.avg = self.sum / self.count"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 139,
   "metadata": {
    "id": "pNWwk3kiKMxL"
   },
   "outputs": [],
   "source": [
    "from torch.utils.data import Dataset, DataLoader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 140,
   "metadata": {
    "id": "dCVD34e-bnAV",
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of MemeDialoGPT were not initialized from the model checkpoint at gpt2 and are newly initialized: ['h.7.attn.masked_bias', 'h.4.attn.masked_bias', 'h.5.attn.masked_bias', 'h.1.attn.masked_bias', 'h.11.attn.masked_bias', 'img_inverse_ff.bias', 'lm_flag.bias', 'h.6.attn.masked_bias', 'h.9.attn.masked_bias', 'lm_head.weight', 'h.3.attn.masked_bias', 'img_inverse_ff.weight', 'h.8.attn.masked_bias', 'lm_flag.weight', 'img_ff.bias', 'h.10.attn.masked_bias', 'img_ff.weight', 'h.0.attn.masked_bias', 'h.2.attn.masked_bias']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[-0.2045, -0.3605,  0.1353,  ...,  0.2115, -0.2773, -0.4641],\n",
      "        [-0.0759, -0.0538,  0.1329,  ...,  0.6213, -0.1812, -0.1301],\n",
      "        [-0.1741, -0.1418,  0.1408,  ...,  0.5004, -0.1981, -0.4364],\n",
      "        ...,\n",
      "        [-0.1725, -0.2610, -0.2571,  ...,  0.7686, -0.2020, -0.2577],\n",
      "        [-0.0091, -0.2522, -0.0359,  ...,  0.3179, -0.1381, -0.0518],\n",
      "        [-0.3150, -0.2858,  0.2325,  ...,  0.2793,  0.1454, -0.2166]],\n",
      "       device='cuda:0')\n",
      "torch.Size([2, 512])\n",
      "torch.Size([2, 512, 1])\n",
      "torch.Size([2, 512, 1, 1])\n",
      "torch.Size([2, 768, 1, 1])\n",
      "torch.Size([2, 768])\n",
      "torch.Size([22, 768])\n",
      "torch.Size([1, 768])\n",
      "torch.Size([22, 50264])\n",
      "torch.Size([1, 512])\n",
      "Epoch:[0][1/859685]\tLoss 53.6130 (53.6130)\tClassify Acc 0.333 (0.333)\t\n",
      "tensor([[-1.2757e+01, -1.2444e+01, -1.4971e+01,  ...,  4.5236e-01,\n",
      "          1.2282e-02, -3.4585e-01],\n",
      "        [-9.3402e+01, -9.2599e+01, -1.0109e+02,  ...,  1.9279e+00,\n",
      "         -2.5267e-02, -1.1982e-01],\n",
      "        [-8.4746e+01, -8.4042e+01, -8.6261e+01,  ...,  1.6783e+00,\n",
      "          6.1961e-01, -1.2970e-01],\n",
      "        ...,\n",
      "        [-5.9068e+01, -6.1192e+01, -6.6594e+01,  ...,  5.9538e-01,\n",
      "          8.1910e-01,  3.3847e-01],\n",
      "        [-1.0898e+02, -1.0917e+02, -1.0979e+02,  ...,  1.6707e+00,\n",
      "         -2.4108e-01, -3.4387e-01],\n",
      "        [-3.6613e+01, -3.7393e+01, -4.1276e+01,  ...,  1.2474e-01,\n",
      "          5.9083e-01,  7.5211e-01]], device='cuda:0', grad_fn=<MmBackward>)\n",
      "tensor([ -100,  -100,  -100,  -100,  -100,  -100,  -100,  -100,  -100,  -100,\n",
      "         -100,  -100,  -100,  -100, 50257,    46,  4669,   648,    30, 50258,\n",
      "         -100,  -100], device='cuda:0')\n",
      "0.3333333333333333\n",
      "torch.Size([22, 50264])\n",
      "torch.Size([22])\n",
      "tensor([[-0.2045, -0.3605,  0.1353,  ...,  0.2115, -0.2773, -0.4641],\n",
      "        [-0.0759, -0.0538,  0.1329,  ...,  0.6213, -0.1812, -0.1301],\n",
      "        [-0.1741, -0.1418,  0.1408,  ...,  0.5004, -0.1981, -0.4364],\n",
      "        ...,\n",
      "        [-0.1725, -0.2610, -0.2571,  ...,  0.7686, -0.2020, -0.2577],\n",
      "        [-0.0091, -0.2522, -0.0359,  ...,  0.3179, -0.1381, -0.0518],\n",
      "        [-0.3150, -0.2858,  0.2325,  ...,  0.2793,  0.1454, -0.2166]],\n",
      "       device='cuda:0')\n",
      "torch.Size([2, 512, 1])\n",
      "torch.Size([2, 512, 1, 1])\n",
      "torch.Size([2, 768, 1, 1])\n",
      "torch.Size([26, 768])\n",
      "torch.Size([1, 768])\n",
      "torch.Size([26, 50264])\n",
      "torch.Size([1, 512])\n",
      "Epoch:[0][1/12666]\tLoss 95.1973 (95.1973)\tClassify Acc 0.200 (0.200)\t\n"
     ]
    }
   ],
   "source": [
    "#함수 시행\n",
    "if __name__ == '__main__': \n",
    "    main()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 141,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 위 칸에 출력되는게 train score고,\n",
    "# 아래칸에 출력되는게 validation score같음...\n",
    "\n",
    "# 근데 지금보면 데이터수가 859685 : 12666 으로 거의 68:1 인거 같음.\n",
    "# validation data갯수가 작네?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "accelerator": "TPU",
  "colab": {
   "collapsed_sections": [],
   "name": "Baseline_code_0806.ipynb",
   "private_outputs": true,
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}